PaperTowel/NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge TNP/dev into fynngodau/dev

Browse Source
This commit is contained in:
Fynn Godau
2020-08-02 16:23:53 +02:00
parent 96de834b67 5ac80624a4
commit c12ef3a02d
79 changed files with 1262 additions and 1331 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java
View File

@@ -4,123 +4,97 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Request;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.net.ssl.HttpsURLConnection;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.List;
import java.util.Map;
public class DownloaderTestImpl extends Downloader {
import okhttp3.OkHttpClient;
import okhttp3.RequestBody;
import okhttp3.ResponseBody;
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:68.0) Gecko/20100101 Firefox/68.0";
private static final String DEFAULT_HTTP_ACCEPT_LANGUAGE = "en";
public final class DownloaderTestImpl extends Downloader {
private static final String USER_AGENT
= "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:68.0) Gecko/20100101 Firefox/68.0";
private static DownloaderTestImpl instance;
private OkHttpClient client;
private static DownloaderTestImpl instance = null;
private DownloaderTestImpl(final OkHttpClient.Builder builder) {
this.client = builder.readTimeout(30, TimeUnit.SECONDS).build();
}
private DownloaderTestImpl() {
/**
* It's recommended to call exactly once in the entire lifetime of the application.
*
* @param builder if null, default builder will be used
* @return a new instance of {@link DownloaderTestImpl}
*/
public static DownloaderTestImpl init(@Nullable final OkHttpClient.Builder builder) {
instance = new DownloaderTestImpl(
builder != null ? builder : new OkHttpClient.Builder());
return instance;
}
public static DownloaderTestImpl getInstance() {
if (instance == null) {
synchronized (DownloaderTestImpl.class) {
if (instance == null) {
instance = new DownloaderTestImpl();
}
}
init(null);
}
return instance;
}
private void setDefaultHeaders(URLConnection connection) {
connection.setRequestProperty("User-Agent", USER_AGENT);
connection.setRequestProperty("Accept-Language", DEFAULT_HTTP_ACCEPT_LANGUAGE);
}
@Override
public Response execute(@Nonnull Request request) throws IOException, ReCaptchaException {
public Response execute(@Nonnull final Request request)
throws IOException, ReCaptchaException {
final String httpMethod = request.httpMethod();
final String url = request.url();
final Map<String, List<String>> headers = request.headers();
@Nullable final byte[] dataToSend = request.dataToSend();
@Nullable final Localization localization = request.localization();
final byte[] dataToSend = request.dataToSend();
final HttpsURLConnection connection = (HttpsURLConnection) new URL(url).openConnection();
RequestBody requestBody = null;
if (dataToSend != null) {
requestBody = RequestBody.create(null, dataToSend);
}
connection.setConnectTimeout(30 * 1000); // 30s
connection.setReadTimeout(30 * 1000); // 30s
connection.setRequestMethod(httpMethod);
setDefaultHeaders(connection);
final okhttp3.Request.Builder requestBuilder = new okhttp3.Request.Builder()
.method(httpMethod, requestBody).url(url)
.addHeader("User-Agent", USER_AGENT);
for (Map.Entry<String, List<String>> pair : headers.entrySet()) {
final String headerName = pair.getKey();
final List<String> headerValueList = pair.getValue();
if (headerValueList.size() > 1) {
connection.setRequestProperty(headerName, null);
requestBuilder.removeHeader(headerName);
for (String headerValue : headerValueList) {
connection.addRequestProperty(headerName, headerValue);
requestBuilder.addHeader(headerName, headerValue);
}
} else if (headerValueList.size() == 1) {
connection.setRequestProperty(headerName, headerValueList.get(0));
requestBuilder.header(headerName, headerValueList.get(0));
}
}
@Nullable OutputStream outputStream = null;
@Nullable InputStreamReader input = null;
try {
if (dataToSend != null && dataToSend.length > 0) {
connection.setDoOutput(true);
connection.setRequestProperty("Content-Length", dataToSend.length + "");
outputStream = connection.getOutputStream();
outputStream.write(dataToSend);
}
final okhttp3.Response response = client.newCall(requestBuilder.build()).execute();
final InputStream inputStream = connection.getInputStream();
final StringBuilder response = new StringBuilder();
if (response.code() == 429) {
response.close();
// Not passing any charset for decoding here... something to keep in mind.
input = new InputStreamReader(inputStream);
int readCount;
char[] buffer = new char[32 * 1024];
while ((readCount = input.read(buffer)) != -1) {
response.append(buffer, 0, readCount);
}
final int responseCode = connection.getResponseCode();
final String responseMessage = connection.getResponseMessage();
final Map<String, List<String>> responseHeaders = connection.getHeaderFields();
final String latestUrl = connection.getURL().toString();
return new Response(responseCode, responseMessage, responseHeaders, response.toString(), latestUrl);
} catch (Exception e) {
final int responseCode = connection.getResponseCode();
/*
* HTTP 429 == Too Many Request
* Receive from Youtube.com = ReCaptcha challenge request
* See : https://github.com/rg3/youtube-dl/issues/5138
*/
if (responseCode == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested", url);
} else if (responseCode != -1) {
final String latestUrl = connection.getURL().toString();
return new Response(responseCode, connection.getResponseMessage(), connection.getHeaderFields(), null, latestUrl);
}
throw new IOException("Error occurred while fetching the content", e);
} finally {
if (outputStream != null) outputStream.close();
if (input != null) input.close();
throw new ReCaptchaException("reCaptcha Challenge requested", url);
}
final ResponseBody body = response.body();
String responseBodyToReturn = null;
if (body != null) {
responseBodyToReturn = body.string();
}
final String latestUrl = response.request().url().toString();
return new Response(response.code(), response.message(), response.headers().toMultimap(),
responseBodyToReturn, latestUrl);
}
}

15
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java
View File

@@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -84,9 +85,8 @@ public final class DefaultTests {
}
public static <T extends InfoItem> void assertNoMoreItems(ListExtractor<T> extractor) throws Exception {
assertFalse("More items available when it shouldn't", extractor.hasNextPage());
final String nextPageUrl = extractor.getNextPageUrl();
assertTrue("Next page is not empty or null", isNullOrEmpty(nextPageUrl));
final ListExtractor.InfoItemsPage<T> initialPage = extractor.getInitialPage();
assertFalse("More items available when it shouldn't", initialPage.hasNextPage());
}
public static void assertNoDuplicatedItems(StreamingService expectedService,
@@ -118,8 +118,9 @@ public final class DefaultTests {
}
public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestMoreItems(ListExtractor<T> extractor) throws Exception {
assertTrue("Doesn't have more items", extractor.hasNextPage());
ListExtractor.InfoItemsPage<T> nextPage = extractor.getPage(extractor.getNextPageUrl());
final ListExtractor.InfoItemsPage<T> initialPage = extractor.getInitialPage();
assertTrue("Doesn't have more items", initialPage.hasNextPage());
ListExtractor.InfoItemsPage<T> nextPage = extractor.getPage(initialPage.getNextPage());
final List<T> items = nextPage.getItems();
assertFalse("Next page is empty", items.isEmpty());
assertEmptyErrors("Next page have errors", nextPage.getErrors());
@@ -129,9 +130,9 @@ public final class DefaultTests {
}
public static void defaultTestGetPageInNewExtractor(ListExtractor<? extends InfoItem> extractor, ListExtractor<? extends InfoItem> newExtractor) throws Exception {
final String nextPageUrl = extractor.getNextPageUrl();
final Page nextPage = extractor.getInitialPage().getNextPage();
final ListExtractor.InfoItemsPage<? extends InfoItem> page = newExtractor.getPage(nextPageUrl);
final ListExtractor.InfoItemsPage<? extends InfoItem> page = newExtractor.getPage(nextPage);
defaultTestListOfItems(extractor.getService(), page.getItems(), page.getErrors());
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java
View File

@@ -143,7 +143,7 @@ public class BandcampPlaylistExtractorTest {
@Test
public void getNextPageUrl() throws IOException, ExtractionException {
assertNull(extractor.getNextPageUrl());
assertNull(extractor.getPage(extractor.getInitialPage().getNextPage()));
}
}
}

12
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java
View File

@@ -5,11 +5,7 @@ package org.schabi.newpipe.extractor.services.bandcamp;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.search.SearchExtractor;
@@ -102,9 +98,11 @@ public class BandcampSearchExtractorTest {
// A query practically guaranteed to have the maximum amount of pages
SearchExtractor extractor = Bandcamp.getSearchExtractor("e");
assertEquals("https://bandcamp.com/search?q=e&page=2", extractor.getInitialPage().getNextPageUrl());
Page page2 = extractor.getInitialPage().getNextPage();
assertEquals("https://bandcamp.com/search?q=e&page=2", page2.getUrl());
assertEquals("https://bandcamp.com/search?q=e&page=3", extractor.getPage(extractor.getNextPageUrl()).getNextPageUrl());
Page page3 = extractor.getPage(page2).getNextPage();
assertEquals("https://bandcamp.com/search?q=e&page=3", page3.getUrl());
}
public static class DefaultTest extends DefaultSearchExtractorTest {

143
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java
View File

@@ -5,6 +5,7 @@ import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -14,78 +15,104 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.List;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
public class PeertubeCommentsExtractorTest {
public static class Default {
private static PeertubeCommentsExtractor extractor;
private static PeertubeCommentsExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
}
@Test
public void testGetComments() throws IOException, ExtractionException {
boolean result = false;
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
result = findInComments(comments, "@root A great documentary on a great guy.");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
result = findInComments(comments, "@root A great documentary on a great guy.");
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
}
assertTrue(result);
}
@Test
public void testGetComments() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
boolean result = findInComments(comments, "@root A great documentary on a great guy.");
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertEquals("Comments", commentsInfo.getName());
result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPage());
result = findInComments(comments, "@root A great documentary on a great guy.");
}
String nextPage = commentsInfo.getNextPageUrl();
while (!Utils.isBlank(nextPage) && !result) {
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "Loved it!!!");
nextPage = moreItems.getNextPageUrl();
assertTrue(result);
}
assertTrue(result);
}
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertEquals("Comments", commentsInfo.getName());
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
assertFalse(Utils.isBlank(c.getCommentId()));
assertFalse(Utils.isBlank(c.getCommentText()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() != -1);
boolean result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
Page nextPage = commentsInfo.getNextPage();
InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
while (moreItems.hasNextPage() && !result) {
moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "Loved it!!!");
nextPage = moreItems.getNextPage();
}
assertTrue(result);
}
}
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
return findInComments(comments.getItems(), comment);
}
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
for (CommentsInfoItem c : comments) {
if (c.getCommentText().contains(comment)) {
return true;
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
assertFalse(Utils.isBlank(c.getCommentId()));
assertFalse(Utils.isBlank(c.getCommentText()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() != -1);
}
}
return false;
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
return findInComments(comments.getItems(), comment);
}
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
for (CommentsInfoItem c : comments) {
if (c.getCommentText().contains(comment)) {
return true;
}
}
return false;
}
}
public static class DeletedComments {
private static PeertubeCommentsExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
}
@Test
public void testGetComments() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
assertTrue(comments.getErrors().isEmpty());
}
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
assertTrue(commentsInfo.getErrors().isEmpty());
}
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java
View File

@@ -161,7 +161,7 @@ public class PeertubeStreamExtractorDefaultTest {
@Test
public void testGetAgeLimit() throws ExtractionException, IOException {
assertEquals(0, extractor.getAgeLimit());
PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/0d501633-f2d9-4476-87c6-71f1c02402a4");
PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://nocensoring.net/videos/embed/dbd8e5e1-c527-49b6-b70c-89101dbb9c08");
ageLimit.fetchPage();
assertEquals(18, ageLimit.getAgeLimit());
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java
View File

@@ -26,6 +26,7 @@ public class PeertubeStreamLinkHandlerFactoryTest {
public void getId() throws Exception {
assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId());
assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId());
assertEquals("9c9de5e8-0a1e-484a-b099-e80766180a6d", linkHandler.fromUrl("https://framatube.org/videos/embed/9c9de5e8-0a1e-484a-b099-e80766180a6d").getId());
}
@@ -33,5 +34,6 @@ public class PeertubeStreamLinkHandlerFactoryTest {
public void testAcceptUrl() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60"));
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa"));
assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/embed/9c9de5e8-0a1e-484a-b099-e80766180a6d"));
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java
View File

@@ -51,7 +51,7 @@ public class PeertubeSearchExtractorTest {
extractor.fetchPage();
final InfoItemsPage<InfoItem> page1 = extractor.getInitialPage();
final InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPageUrl());
final InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPage());
assertNoDuplicatedItems(PeerTube, page1, page2);
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View File

@@ -268,7 +268,7 @@ public class SoundcloudPlaylistExtractorTest {
ListExtractor.InfoItemsPage<StreamInfoItem> currentPage = defaultTestMoreItems(extractor);
// Test for 2 more levels
for (int i = 0; i < 2; i++) {
currentPage = extractor.getPage(currentPage.getNextPageUrl());
currentPage = extractor.getPage(currentPage.getNextPage());
defaultTestListOfItems(SoundCloud, currentPage.getItems(), currentPage.getErrors());
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java
View File

@@ -119,7 +119,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
final InfoItemsPage<InfoItem> page1 = extractor.getInitialPage();
final InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPageUrl());
final InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPage());
assertNoDuplicatedItems(SoundCloud, page1, page2);
}

30
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View File

@@ -5,6 +5,7 @@ import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -22,39 +23,33 @@ import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeCommentsExtractorTest {
private static final String urlYT = "https://www.youtube.com/watch?v=D00Au7k3i6o";
private static final String urlInvidious = "https://invidio.us/watch?v=D00Au7k3i6o";
private static final String urlInvidioush = "https://invidiou.sh/watch?v=D00Au7k3i6o";
private static YoutubeCommentsExtractor extractorYT;
private static YoutubeCommentsExtractor extractorInvidious;
private static YoutubeCommentsExtractor extractorInvidioush;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractorYT = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(urlYT);
extractorYT.fetchPage();
extractorInvidious = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(urlInvidious);
extractorInvidioush = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(urlInvidioush);
}
@Test
public void testGetComments() throws IOException, ExtractionException {
assertTrue(getCommentsHelper(extractorYT));
assertTrue(getCommentsHelper(extractorInvidious));
assertTrue(getCommentsHelper(extractorInvidioush));
}
private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException {
boolean result;
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
result = findInComments(comments, "s1ck m3m3");
boolean result = findInComments(comments, "s1ck m3m3");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
comments = extractor.getPage(comments.getNextPage());
result = findInComments(comments, "s1ck m3m3");
}
@@ -65,20 +60,21 @@ public class YoutubeCommentsExtractorTest {
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
assertTrue(getCommentsFromCommentsInfoHelper(urlYT));
assertTrue(getCommentsFromCommentsInfoHelper(urlInvidious));
assertTrue(getCommentsFromCommentsInfoHelper(urlInvidioush));
}
private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException {
boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo(url);
result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
/* String nextPage = commentsInfo.getNextPageUrl();
while (!Utils.isBlank(nextPage) && !result) {
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
assertEquals("Comments", commentsInfo.getName());
boolean result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
Page nextPage = commentsInfo.getNextPage();
InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
while (moreItems.hasNextPage() && !result) {
moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "s1ck m3m3");
nextPage = moreItems.getNextPageUrl();
}*/
nextPage = moreItems.getNextPage();
}
return result;
}

9
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java
View File

@@ -5,9 +5,11 @@ import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class YoutubeParsingHelperTest {
@@ -27,4 +29,11 @@ public class YoutubeParsingHelperTest {
assertTrue("Hardcoded YouTube Music keys are not valid anymore",
YoutubeParsingHelper.areHardcodedYoutubeMusicKeysValid());
}
@Test
public void testParseDurationString() throws ParsingException {
assertEquals(1162567, YoutubeParsingHelper.parseDurationString("12:34:56:07"));
assertEquals(4445767, YoutubeParsingHelper.parseDurationString("1,234:56:07"));
assertEquals(754, YoutubeParsingHelper.parseDurationString("12:34 "));
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@@ -209,7 +209,7 @@ public class YoutubePlaylistExtractorTest {
// test for 2 more levels
for (int i = 0; i < 2; i++) {
currentPage = extractor.getPage(currentPage.getNextPageUrl());
currentPage = extractor.getPage(currentPage.getNextPage());
defaultTestListOfItems(YouTube, currentPage.getItems(), currentPage.getErrors());
}
}

14
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java
View File

@@ -19,8 +19,9 @@ import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS;
public class YoutubeSearchExtractorTest {
public static class All extends DefaultSearchExtractorTest {
@@ -186,15 +187,14 @@ public class YoutubeSearchExtractorTest {
@Test
public void testMoreRelatedItems() throws Exception {
final ListExtractor.InfoItemsPage<InfoItem> initialPage = extractor().getInitialPage();
// YouTube actually gives us an empty next page, but after that, no more pages.
assertTrue(extractor.hasNextPage());
final ListExtractor.InfoItemsPage<InfoItem> nextEmptyPage = extractor.getPage(extractor.getNextPageUrl());
assertTrue(initialPage.hasNextPage());
final ListExtractor.InfoItemsPage<InfoItem> nextEmptyPage = extractor.getPage(initialPage.getNextPage());
assertEquals(0, nextEmptyPage.getItems().size());
assertEmptyErrors("Empty page has errors", nextEmptyPage.getErrors());
assertFalse("More items available when it shouldn't", nextEmptyPage.hasNextPage());
final String nextPageUrl = nextEmptyPage.getNextPageUrl();
assertTrue("Next page is not empty or null", isNullOrEmpty(nextPageUrl));
}
}
@@ -206,7 +206,7 @@ public class YoutubeSearchExtractorTest {
extractor.fetchPage();
final ListExtractor.InfoItemsPage<InfoItem> page1 = extractor.getInitialPage();
final ListExtractor.InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPageUrl());
final ListExtractor.InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPage());
assertNoDuplicatedItems(YouTube, page1, page2);
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java
View File

@@ -48,6 +48,8 @@ public class YoutubeStreamExtractorAgeRestrictedTest {
public void testGetValidTimeStamp() throws IOException, ExtractionException {
StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
assertEquals(extractor.getTimeStamp() + "", "174");
extractor = YouTube.getStreamExtractor("https://youtube.com/embed/FmG385_uUys?start=174");
assertEquals(extractor.getTimeStamp() + "", "174");
}
@Test

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java
View File

@@ -129,11 +129,6 @@ public class YoutubeStreamExtractorUnlistedTest {
assertSame(StreamType.VIDEO_STREAM, extractor.getStreamType());
}
@Test
public void testGetNextVideo() throws ExtractionException {
assertNull(extractor.getNextStream());
}
@Test
public void testGetRelatedVideos() throws ExtractionException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();

10
extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java
View File

@@ -1,18 +1,24 @@
package org.schabi.newpipe.extractor.utils;
import com.grack.nanojson.JsonParserException;
import org.junit.Test;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.Arrays;
import static org.junit.Assert.assertEquals;
public class UtilsTest {
@Test
public void testMixedNumberWordToLong() throws JsonParserException, ParsingException {
public void testMixedNumberWordToLong() throws ParsingException {
assertEquals(10, Utils.mixedNumberWordToLong("10"));
assertEquals(10.5e3, Utils.mixedNumberWordToLong("10.5K"), 0.0);
assertEquals(10.5e6, Utils.mixedNumberWordToLong("10.5M"), 0.0);
assertEquals(10.5e6, Utils.mixedNumberWordToLong("10,5M"), 0.0);
assertEquals(1.5e9, Utils.mixedNumberWordToLong("1,5B"), 0.0);
}
@Test
public void testJoin() {
assertEquals("some,random,stuff", Utils.join(",", Arrays.asList("some", "random", "stuff")));
}
}