mirror of
https://github.com/TeamNewPipe/NewPipeExtractor.git
synced 2025-04-27 23:40:36 +05:30
Merge pull request #1246 from Isira-Seneviratne/BufferedReader-lines
Use BufferedReader#lines()
This commit is contained in:
litetex
GitHub
commit
3a33cefbc0
@ -14,9 +14,12 @@ import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
@ -110,7 +113,7 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
|
||||
|
||||
// Return it only if it has items (it exits early if it's the wrong file
|
||||
// format), otherwise try the next file
|
||||
if (csvItems.size() > 0) {
|
||||
if (!csvItems.isEmpty()) {
|
||||
return csvItems;
|
||||
}
|
||||
} catch (final ExtractionException e) {
|
||||
@ -138,69 +141,25 @@ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
|
||||
// The first line is always a header
|
||||
// Header names are different based on the locale
|
||||
// Fortunately the data is always the same order no matter what locale
|
||||
|
||||
int currentLine = 0;
|
||||
String line = "";
|
||||
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(contentInputStream))) {
|
||||
final List<SubscriptionItem> subscriptionItems = new ArrayList<>();
|
||||
|
||||
// ignore header and skip first line
|
||||
currentLine = 1;
|
||||
line = br.readLine();
|
||||
|
||||
while ((line = br.readLine()) != null) {
|
||||
currentLine++;
|
||||
|
||||
// Exit early if we've read the first few lines and we haven't added any items
|
||||
// It's likely we're in the wrong file
|
||||
if (currentLine > 5 && subscriptionItems.size() == 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
// First comma
|
||||
final int i1 = line.indexOf(",");
|
||||
if (i1 == -1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Second comma
|
||||
final int i2 = line.indexOf(",", i1 + 1);
|
||||
if (i2 == -1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Third comma or line length
|
||||
int i3 = line.indexOf(",", i2 + 1);
|
||||
if (i3 == -1) {
|
||||
i3 = line.length();
|
||||
}
|
||||
|
||||
// Channel URL from second entry
|
||||
final String channelUrl = line
|
||||
.substring(i1 + 1, i2)
|
||||
.replace("http://", "https://");
|
||||
if (!channelUrl.startsWith(BASE_CHANNEL_URL)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Channel title from third entry
|
||||
final String channelTitle = line.substring(i2 + 1, i3);
|
||||
|
||||
final SubscriptionItem newItem
|
||||
= new SubscriptionItem(service.getServiceId(), channelUrl, channelTitle);
|
||||
subscriptionItems.add(newItem);
|
||||
}
|
||||
|
||||
return subscriptionItems;
|
||||
} catch (final IOException e) {
|
||||
if (line == null) {
|
||||
line = "<null>";
|
||||
} else if (line.length() > 10) {
|
||||
line = line.substring(0, 10) + "...";
|
||||
}
|
||||
throw new InvalidSourceException("Error reading CSV file on line = \"" + line
|
||||
+ "\", line number = " + currentLine, e);
|
||||
try (var reader = new BufferedReader(new InputStreamReader(contentInputStream))) {
|
||||
return reader.lines()
|
||||
.skip(1) // ignore header and skip first line
|
||||
.map(line -> line.split(","))
|
||||
.filter(values -> values.length >= 3)
|
||||
.map(values -> {
|
||||
// Channel URL from second entry
|
||||
final String channelUrl = values[1].replace("http://", "https://");
|
||||
return channelUrl.startsWith(BASE_CHANNEL_URL)
|
||||
? new SubscriptionItem(
|
||||
service.getServiceId(),
|
||||
channelUrl,
|
||||
values[2]) // Channel title from third entry
|
||||
: null;
|
||||
})
|
||||
.filter(Objects::nonNull)
|
||||
.collect(Collectors.toUnmodifiableList());
|
||||
} catch (final UncheckedIOException | IOException e) {
|
||||
throw new InvalidSourceException("Error reading CSV file", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -26,8 +26,7 @@ import java.util.List;
|
||||
/**
|
||||
* Test for {@link YoutubeSubscriptionExtractor}
|
||||
*/
|
||||
public class YoutubeSubscriptionExtractorTest {
|
||||
|
||||
class YoutubeSubscriptionExtractorTest {
|
||||
|
||||
private static YoutubeSubscriptionExtractor subscriptionExtractor;
|
||||
private static LinkHandlerFactory urlHandler;
|
||||
@ -41,7 +40,7 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFromInputStream() throws Exception {
|
||||
void testFromInputStream() throws Exception {
|
||||
final List<SubscriptionItem> subscriptionItems = subscriptionExtractor.fromInputStream(
|
||||
new FileInputStream(resolveTestResource("youtube_takeout_import_test.json")));
|
||||
assertEquals(7, subscriptionItems.size());
|
||||
@ -55,14 +54,14 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEmptySourceException() throws Exception {
|
||||
void testEmptySourceException() throws Exception {
|
||||
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
|
||||
new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)));
|
||||
assertTrue(items.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSubscriptionWithEmptyTitleInSource() throws Exception {
|
||||
void testSubscriptionWithEmptyTitleInSource() throws Exception {
|
||||
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]";
|
||||
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
|
||||
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
|
||||
@ -74,7 +73,7 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSubscriptionWithInvalidUrlInSource() throws Exception {
|
||||
void testSubscriptionWithInvalidUrlInSource() throws Exception {
|
||||
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," +
|
||||
"{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]";
|
||||
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
|
||||
@ -87,8 +86,8 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInvalidSourceException() {
|
||||
List<String> invalidList = Arrays.asList(
|
||||
void testInvalidSourceException() {
|
||||
final List<String> invalidList = Arrays.asList(
|
||||
"<xml><notvalid></notvalid></xml>",
|
||||
"<opml><notvalid></notvalid></opml>",
|
||||
"{\"a\":\"b\"}",
|
||||
@ -100,13 +99,14 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
"\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28",
|
||||
"gibberish");
|
||||
|
||||
for (String invalidContent : invalidList) {
|
||||
for (final String invalidContent : invalidList) {
|
||||
try {
|
||||
byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
|
||||
final byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
|
||||
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
|
||||
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
|
||||
} catch (final Exception e) {
|
||||
boolean correctType = e instanceof SubscriptionExtractor.InvalidSourceException;
|
||||
final boolean correctType =
|
||||
e instanceof SubscriptionExtractor.InvalidSourceException;
|
||||
if (!correctType) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
@ -117,7 +117,7 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
|
||||
private static void assertSubscriptionItems(final List<SubscriptionItem> subscriptionItems)
|
||||
throws Exception {
|
||||
assertTrue(subscriptionItems.size() > 0);
|
||||
assertTrue(!subscriptionItems.isEmpty());
|
||||
|
||||
for (final SubscriptionItem item : subscriptionItems) {
|
||||
assertNotNull(item.getName());
|
||||
@ -128,7 +128,7 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void fromZipInputStream() throws Exception {
|
||||
void fromZipInputStream() throws Exception {
|
||||
final List<String> zipPaths = Arrays.asList(
|
||||
"youtube_takeout_import_test_1.zip",
|
||||
"youtube_takeout_import_test_2.zip"
|
||||
@ -144,13 +144,13 @@ public class YoutubeSubscriptionExtractorTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void fromCsvInputStream() throws Exception {
|
||||
void fromCsvInputStream() throws Exception {
|
||||
final List<String> csvPaths = Arrays.asList(
|
||||
"youtube_takeout_import_test_1.csv",
|
||||
"youtube_takeout_import_test_2.csv"
|
||||
);
|
||||
|
||||
for (String path : csvPaths)
|
||||
for (final String path : csvPaths)
|
||||
{
|
||||
final File file = resolveTestResource(path);
|
||||
final FileInputStream fileInputStream = new FileInputStream(file);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user