PaperTowel/NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Introduce Page class

Browse Source
This commit is contained in:
wb9688
2020-04-15 14:09:46 +02:00
parent e3bfdba135
commit 4cc312086a
43 changed files with 477 additions and 429 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java
View File

@@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -86,8 +87,6 @@ public final class DefaultTests {
public static <T extends InfoItem> void assertNoMoreItems(ListExtractor<T> extractor) throws Exception {
final ListExtractor.InfoItemsPage<T> initialPage = extractor.getInitialPage();
assertFalse("More items available when it shouldn't", initialPage.hasNextPage());
final String nextPageUrl = initialPage.getNextPageUrl();
assertTrue("Next page is not empty or null", isNullOrEmpty(nextPageUrl));
}
public static void assertNoDuplicatedItems(StreamingService expectedService,
@@ -121,7 +120,7 @@ public final class DefaultTests {
public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestMoreItems(ListExtractor<T> extractor) throws Exception {
final ListExtractor.InfoItemsPage<T> initialPage = extractor.getInitialPage();
assertTrue("Doesn't have more items", initialPage.hasNextPage());
ListExtractor.InfoItemsPage<T> nextPage = extractor.getPage(initialPage.getNextPageUrl());
ListExtractor.InfoItemsPage<T> nextPage = extractor.getPage(initialPage.getNextPage());
final List<T> items = nextPage.getItems();
assertFalse("Next page is empty", items.isEmpty());
assertEmptyErrors("Next page have errors", nextPage.getErrors());
@@ -131,9 +130,9 @@ public final class DefaultTests {
}
public static void defaultTestGetPageInNewExtractor(ListExtractor<? extends InfoItem> extractor, ListExtractor<? extends InfoItem> newExtractor) throws Exception {
final String nextPageUrl = extractor.getInitialPage().getNextPageUrl();
final Page nextPage = extractor.getInitialPage().getNextPage();
final ListExtractor.InfoItemsPage<? extends InfoItem> page = newExtractor.getPage(nextPageUrl);
final ListExtractor.InfoItemsPage<? extends InfoItem> page = newExtractor.getPage(nextPage);
defaultTestListOfItems(extractor.getService(), page.getItems(), page.getErrors());
}
}

48
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java
View File

@@ -5,6 +5,7 @@ import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -14,7 +15,9 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.List;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
public class PeertubeCommentsExtractorTest {
@@ -31,37 +34,38 @@ public class PeertubeCommentsExtractorTest {
@Test
public void testGetComments() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
assertTrue(comments.getErrors().isEmpty());
boolean result = findInComments(comments, "@root A great documentary on a great guy.");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
comments = extractor.getPage(comments.getNextPage());
result = findInComments(comments, "@root A great documentary on a great guy.");
}
assertTrue(result);
}
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertTrue(commentsInfo.getErrors().isEmpty());
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertEquals("Comments", commentsInfo.getName());
boolean result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
String nextPage = commentsInfo.getNextPageUrl();
while (!Utils.isBlank(nextPage) && !result) {
final InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
Page nextPage = commentsInfo.getNextPage();
InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
while (moreItems.hasNextPage() && !result) {
moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "Loved it!!!");
nextPage = moreItems.getNextPageUrl();
nextPage = moreItems.getNextPage();
}
assertTrue(result);
}
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (final CommentsInfoItem c : comments.getItems()) {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
@@ -71,16 +75,16 @@ public class PeertubeCommentsExtractorTest {
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertEquals(-1, c.getLikeCount());
assertFalse(c.getLikeCount() != -1);
}
}
private boolean findInComments(final InfoItemsPage<CommentsInfoItem> comments, final String comment) {
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
return findInComments(comments.getItems(), comment);
}
private boolean findInComments(final List<CommentsInfoItem> comments, final String comment) {
for (final CommentsInfoItem c : comments) {
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
for (CommentsInfoItem c : comments) {
if (c.getCommentText().contains(comment)) {
return true;
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java
View File

@@ -51,7 +51,7 @@ public class PeertubeSearchExtractorTest {
extractor.fetchPage();
final InfoItemsPage<InfoItem> page1 = extractor.getInitialPage();
final InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPageUrl());
final InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPage());
assertNoDuplicatedItems(PeerTube, page1, page2);
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View File

@@ -268,7 +268,7 @@ public class SoundcloudPlaylistExtractorTest {
ListExtractor.InfoItemsPage<StreamInfoItem> currentPage = defaultTestMoreItems(extractor);
// Test for 2 more levels
for (int i = 0; i < 2; i++) {
currentPage = extractor.getPage(currentPage.getNextPageUrl());
currentPage = extractor.getPage(currentPage.getNextPage());
defaultTestListOfItems(SoundCloud, currentPage.getItems(), currentPage.getErrors());
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java
View File

@@ -119,7 +119,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
final InfoItemsPage<InfoItem> page1 = extractor.getInitialPage();
final InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPageUrl());
final InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPage());
assertNoDuplicatedItems(SoundCloud, page1, page2);
}

24
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View File

@@ -5,6 +5,7 @@ import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -22,7 +23,6 @@ import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeCommentsExtractorTest {
private static final String urlYT = "https://www.youtube.com/watch?v=D00Au7k3i6o";
private static final String urlInvidious = "https://invidio.us/watch?v=D00Au7k3i6o";
private static YoutubeCommentsExtractor extractorYT;
@@ -33,6 +33,7 @@ public class YoutubeCommentsExtractorTest {
NewPipe.init(DownloaderTestImpl.getInstance());
extractorYT = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(urlYT);
extractorYT.fetchPage();
extractorInvidious = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(urlInvidious);
}
@@ -44,12 +45,11 @@ public class YoutubeCommentsExtractorTest {
}
private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException {
boolean result;
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
result = findInComments(comments, "s1ck m3m3");
boolean result = findInComments(comments, "s1ck m3m3");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
comments = extractor.getPage(comments.getNextPage());
result = findInComments(comments, "s1ck m3m3");
}
@@ -63,16 +63,18 @@ public class YoutubeCommentsExtractorTest {
}
private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException {
boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo(url);
result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
/* String nextPage = commentsInfo.getNextPageUrl();
while (!Utils.isBlank(nextPage) && !result) {
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
assertEquals("Comments", commentsInfo.getName());
boolean result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
Page nextPage = commentsInfo.getNextPage();
InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
while (moreItems.hasNextPage() && !result) {
moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "s1ck m3m3");
nextPage = moreItems.getNextPageUrl();
}*/
nextPage = moreItems.getNextPage();
}
return result;
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@@ -209,7 +209,7 @@ public class YoutubePlaylistExtractorTest {
// test for 2 more levels
for (int i = 0; i < 2; i++) {
currentPage = extractor.getPage(currentPage.getNextPageUrl());
currentPage = extractor.getPage(currentPage.getNextPage());
defaultTestListOfItems(YouTube, currentPage.getItems(), currentPage.getErrors());
}
}

11
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java
View File

@@ -19,8 +19,9 @@ import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS;
public class YoutubeSearchExtractorTest {
public static class All extends DefaultSearchExtractorTest {
@@ -189,13 +190,11 @@ public class YoutubeSearchExtractorTest {
final ListExtractor.InfoItemsPage<InfoItem> initialPage = extractor().getInitialPage();
// YouTube actually gives us an empty next page, but after that, no more pages.
assertTrue(initialPage.hasNextPage());
final ListExtractor.InfoItemsPage<InfoItem> nextEmptyPage = extractor.getPage(initialPage.getNextPageUrl());
final ListExtractor.InfoItemsPage<InfoItem> nextEmptyPage = extractor.getPage(initialPage.getNextPage());
assertEquals(0, nextEmptyPage.getItems().size());
assertEmptyErrors("Empty page has errors", nextEmptyPage.getErrors());
assertFalse("More items available when it shouldn't", nextEmptyPage.hasNextPage());
final String nextPageUrl = nextEmptyPage.getNextPageUrl();
assertTrue("Next page is not empty or null", isNullOrEmpty(nextPageUrl));
}
}
@@ -207,7 +206,7 @@ public class YoutubeSearchExtractorTest {
extractor.fetchPage();
final ListExtractor.InfoItemsPage<InfoItem> page1 = extractor.getInitialPage();
final ListExtractor.InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPageUrl());
final ListExtractor.InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPage());
assertNoDuplicatedItems(YouTube, page1, page2);
}