From 0fb73301e38c72d0e0b742deab5e6d9da4cde07c Mon Sep 17 00:00:00 2001 From: TobiGr Date: Wed, 5 Aug 2020 18:25:35 +0200 Subject: [PATCH 01/81] [YouTube] Fix crash on empty comment Closes #380 --- .../extractors/YoutubeCommentsExtractor.java | 16 +- .../YoutubeCommentsInfoItemExtractor.java | 10 +- .../youtube/YoutubeCommentsExtractorTest.java | 197 +++++++++++------- 3 files changed, 139 insertions(+), 84 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 287e7421..bd2af3c6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -47,13 +47,13 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}"); - String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\""); + final String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}"); + final String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\""); return getPage(getNextPage(commentsToken)); } private Page getNextPage(JsonObject ajaxJson) throws ParsingException { - JsonArray arr; + final JsonArray arr; try { arr = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations"); } catch (Exception e) { @@ -89,14 +89,14 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { throw new IllegalArgumentException("Page doesn't contain an URL"); } - String ajaxResponse = makeAjaxRequest(page.getUrl()); - JsonObject ajaxJson; + final String ajaxResponse = makeAjaxRequest(page.getUrl()); + final JsonObject ajaxJson; try { ajaxJson = JsonParser.array().from(ajaxResponse).getObject(1); } catch (Exception e) { throw new ParsingException("Could not parse json data for comments", e); } - CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); collectCommentsFrom(collector, ajaxJson); return new InfoItemsPage<>(collector, getNextPage(ajaxJson)); } @@ -160,8 +160,8 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { } private String findValue(String doc, String start, String end) { - int beginIndex = doc.indexOf(start) + start.length(); - int endIndex = doc.indexOf(end, beginIndex); + final int beginIndex = doc.indexOf(start) + start.length(); + final int endIndex = doc.indexOf(end, beginIndex); return doc.substring(beginIndex, endIndex); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index 91302244..d960bc2e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -34,7 +34,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract @Override public String getThumbnailUrl() throws ParsingException { try { - JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails"); + final JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails"); return JsonUtils.getString(arr.getObject(2), "url"); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); @@ -82,7 +82,13 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract @Override public String getCommentText() throws ParsingException { try { - String commentText = getTextFromObject(JsonUtils.getObject(json, "contentText")); + final JsonObject contentText = JsonUtils.getObject(json, "contentText"); + if (contentText.isEmpty()) { + // completely empty comments as described in + // https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584 + return ""; + } + final String commentText = getTextFromObject(contentText); // youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io return Utils.removeUTF8BOM(commentText); } catch (Exception e) { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index bb2b17be..b43dce67 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -23,91 +23,140 @@ import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; public class YoutubeCommentsExtractorTest { - private static final String urlYT = "https://www.youtube.com/watch?v=D00Au7k3i6o"; - private static final String urlInvidious = "https://invidio.us/watch?v=D00Au7k3i6o"; - private static YoutubeCommentsExtractor extractorYT; - private static YoutubeCommentsExtractor extractorInvidious; + /** + * Test a "normal" YouTube and Invidious page + */ + public static class Thomas { + private static final String urlYT = "https://www.youtube.com/watch?v=D00Au7k3i6o"; + private static final String urlInvidious = "https://invidio.us/watch?v=D00Au7k3i6o"; + private static YoutubeCommentsExtractor extractorYT; + private static YoutubeCommentsExtractor extractorInvidious; - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractorYT = (YoutubeCommentsExtractor) YouTube - .getCommentsExtractor(urlYT); - extractorYT.fetchPage(); - extractorInvidious = (YoutubeCommentsExtractor) YouTube - .getCommentsExtractor(urlInvidious); - } + private static final String commentContent = "sub 4 sub"; - @Test - public void testGetComments() throws IOException, ExtractionException { - assertTrue(getCommentsHelper(extractorYT)); - assertTrue(getCommentsHelper(extractorInvidious)); - } - - private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException { - InfoItemsPage comments = extractor.getInitialPage(); - boolean result = findInComments(comments, "s1ck m3m3"); - - while (comments.hasNextPage() && !result) { - comments = extractor.getPage(comments.getNextPage()); - result = findInComments(comments, "s1ck m3m3"); + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractorYT = (YoutubeCommentsExtractor) YouTube + .getCommentsExtractor(urlYT); + extractorYT.fetchPage(); + extractorInvidious = (YoutubeCommentsExtractor) YouTube + .getCommentsExtractor(urlInvidious); + extractorInvidious.fetchPage(); } - return result; - } - - @Test - public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { - assertTrue(getCommentsFromCommentsInfoHelper(urlYT)); - assertTrue(getCommentsFromCommentsInfoHelper(urlInvidious)); - } - - private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException { - CommentsInfo commentsInfo = CommentsInfo.getInfo(url); - - assertEquals("Comments", commentsInfo.getName()); - boolean result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3"); - - Page nextPage = commentsInfo.getNextPage(); - InfoItemsPage moreItems = new InfoItemsPage<>(null, nextPage, null); - while (moreItems.hasNextPage() && !result) { - moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage); - result = findInComments(moreItems.getItems(), "s1ck m3m3"); - nextPage = moreItems.getNextPage(); + @Test + public void testGetComments() throws IOException, ExtractionException { + assertTrue(getCommentsHelper(extractorYT)); + assertTrue(getCommentsHelper(extractorInvidious)); } - return result; - } - @Test - public void testGetCommentsAllData() throws IOException, ExtractionException { - InfoItemsPage comments = extractorYT.getInitialPage(); + private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException { + InfoItemsPage comments = extractor.getInitialPage(); + boolean result = findInComments(comments, commentContent); - DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); - for (CommentsInfoItem c : comments.getItems()) { - assertFalse(Utils.isBlank(c.getUploaderUrl())); - assertFalse(Utils.isBlank(c.getUploaderName())); - assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); - assertFalse(Utils.isBlank(c.getCommentId())); - assertFalse(Utils.isBlank(c.getCommentText())); - assertFalse(Utils.isBlank(c.getName())); - assertFalse(Utils.isBlank(c.getTextualUploadDate())); - assertNotNull(c.getUploadDate()); - assertFalse(Utils.isBlank(c.getThumbnailUrl())); - assertFalse(Utils.isBlank(c.getUrl())); - assertFalse(c.getLikeCount() < 0); + while (comments.hasNextPage() && !result) { + comments = extractor.getPage(comments.getNextPage()); + result = findInComments(comments, commentContent); + } + + return result; } - } - private boolean findInComments(InfoItemsPage comments, String comment) { - return findInComments(comments.getItems(), comment); - } + @Test + public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { + assertTrue(getCommentsFromCommentsInfoHelper(urlYT)); + assertTrue(getCommentsFromCommentsInfoHelper(urlInvidious)); + } - private boolean findInComments(List comments, String comment) { - for (CommentsInfoItem c : comments) { - if (c.getCommentText().contains(comment)) { - return true; + private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException { + final CommentsInfo commentsInfo = CommentsInfo.getInfo(url); + + assertEquals("Comments", commentsInfo.getName()); + boolean result = findInComments(commentsInfo.getRelatedItems(), commentContent); + + Page nextPage = commentsInfo.getNextPage(); + InfoItemsPage moreItems = new InfoItemsPage<>(null, nextPage, null); + while (moreItems.hasNextPage() && !result) { + moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage); + result = findInComments(moreItems.getItems(), commentContent); + nextPage = moreItems.getNextPage(); + } + return result; + } + + @Test + public void testGetCommentsAllData() throws IOException, ExtractionException { + InfoItemsPage comments = extractorYT.getInitialPage(); + + DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); + for (CommentsInfoItem c : comments.getItems()) { + assertFalse(Utils.isBlank(c.getUploaderUrl())); + assertFalse(Utils.isBlank(c.getUploaderName())); + assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + assertFalse(Utils.isBlank(c.getCommentId())); + assertFalse(Utils.isBlank(c.getCommentText())); + assertFalse(Utils.isBlank(c.getName())); + assertFalse(Utils.isBlank(c.getTextualUploadDate())); + assertNotNull(c.getUploadDate()); + assertFalse(Utils.isBlank(c.getThumbnailUrl())); + assertFalse(Utils.isBlank(c.getUrl())); + assertFalse(c.getLikeCount() < 0); } } - return false; + + private boolean findInComments(InfoItemsPage comments, String comment) { + return findInComments(comments.getItems(), comment); + } + + private boolean findInComments(List comments, String comment) { + for (CommentsInfoItem c : comments) { + if (c.getCommentText().contains(comment)) { + return true; + } + } + return false; + } + } + + /** + * Test a video with an empty comment + */ + public static class EmptyComment { + private static YoutubeCommentsExtractor extractor; + private final static String url = "https://www.youtube.com/watch?v=VM_6n762j6M"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeCommentsExtractor) YouTube + .getCommentsExtractor(url); + extractor.fetchPage(); + } + + @Test + public void testGetCommentsAllData() throws IOException, ExtractionException { + final InfoItemsPage comments = extractor.getInitialPage(); + + DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); + for (CommentsInfoItem c : comments.getItems()) { + assertFalse(Utils.isBlank(c.getUploaderUrl())); + assertFalse(Utils.isBlank(c.getUploaderName())); + assertFalse(Utils.isBlank(c.getUploaderAvatarUrl())); + assertFalse(Utils.isBlank(c.getCommentId())); + assertFalse(Utils.isBlank(c.getName())); + assertFalse(Utils.isBlank(c.getTextualUploadDate())); + assertNotNull(c.getUploadDate()); + assertFalse(Utils.isBlank(c.getThumbnailUrl())); + assertFalse(Utils.isBlank(c.getUrl())); + assertFalse(c.getLikeCount() < 0); + if (c.getCommentId().equals("Ugga_h1-EXdHB3gCoAEC")) { // comment without text + assertTrue(Utils.isBlank(c.getCommentText())); + } else { + assertFalse(Utils.isBlank(c.getCommentText())); + } + } + } + } } From f0f1c009b26fc7b8395cef7511d2a28e8591a578 Mon Sep 17 00:00:00 2001 From: mhmdanas <32234660+mhmdanas@users.noreply.github.com> Date: Mon, 10 Aug 2020 19:28:56 +0300 Subject: [PATCH 02/81] Remove SubtitlesStream#getURL() --- .../org/schabi/newpipe/extractor/stream/SubtitlesStream.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java index 8f16942e..c2b542c8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java @@ -42,10 +42,6 @@ public class SubtitlesStream extends Stream implements Serializable { return format.suffix; } - public String getURL() { - return url; - } - public boolean isAutoGenerated() { return autoGenerated; } From 1a63dcb3554ed2e040bcc2c6ef13118a5c3ac8b9 Mon Sep 17 00:00:00 2001 From: mhmdanas <32234660+mhmdanas@users.noreply.github.com> Date: Wed, 12 Aug 2020 22:37:36 +0300 Subject: [PATCH 03/81] Remove overriden field `url` --- .../org/schabi/newpipe/extractor/stream/SubtitlesStream.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java index c2b542c8..fe3b441e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java @@ -8,7 +8,6 @@ import java.util.Locale; public class SubtitlesStream extends Stream implements Serializable { private final MediaFormat format; private final Locale locale; - private final String url; private final boolean autoGenerated; private final String code; @@ -34,7 +33,6 @@ public class SubtitlesStream extends Stream implements Serializable { } this.code = languageCode; this.format = format; - this.url = url; this.autoGenerated = autoGenerated; } From 7657c2ed1a792407b0c79bc5a746ff68503b4a02 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Sat, 15 Aug 2020 17:08:07 +0200 Subject: [PATCH 04/81] Use initSafeStandardsObjects() --- .../youtube/extractors/YoutubeStreamExtractor.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 93bd2121..b5c6c796 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -627,7 +627,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { "yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(", "\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(" }; - ; private volatile String decryptionCode = ""; @@ -788,16 +787,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { } private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException { - Context context = Context.enter(); + final Context context = Context.enter(); context.setOptimizationLevel(-1); - Object result; + final Object result; try { - ScriptableObject scope = context.initStandardObjects(); + final ScriptableObject scope = context.initSafeStandardObjects(); context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null); - Function decryptionFunc = (Function) scope.get("decrypt", scope); + final Function decryptionFunc = (Function) scope.get("decrypt", scope); result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig}); } catch (Exception e) { - throw new DecryptException("could not get decrypt signature", e); + throw new DecryptException("Could not get decrypt signature", e); } finally { Context.exit(); } From ebbfe7f6d4dfd03a2f88367b33734989512b0181 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Tue, 29 Sep 2020 10:48:02 +0200 Subject: [PATCH 05/81] Skip YouTube's OTF streams --- .../youtube/extractors/YoutubeStreamExtractor.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index b5c6c796..0b18c4c2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -957,6 +957,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { ItagItem itagItem = ItagItem.getItag(itag); if (itagItem.itagType == itagTypeWanted) { + // Ignore streams that are delivered using YouTube's OTF format, + // as those only work with DASH and not with progressive HTTP. + if (formatData.getString("type", EMPTY_STRING) + .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) { + continue; + } + String streamUrl; if (formatData.has("url")) { streamUrl = formatData.getString("url"); From acb04eb351374229a023518605edf6360bc92e80 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 4 Oct 2020 13:37:19 +0200 Subject: [PATCH 06/81] Update extractor version to 0.20.0 --- README.md | 2 +- build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0f45a969..ec48cef8 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo. If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps: 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. -2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.19.7'`the `dependencies` in your `build.gradle`. Replace `v0.19.7` with the latest release. +2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.0'`the `dependencies` in your `build.gradle`. Replace `v0.20.0` with the latest release. ### Testing changes diff --git a/build.gradle b/build.gradle index 70da643b..3d579cfb 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.19.7' + version 'v0.20.0' group 'com.github.TeamNewPipe' repositories { From 4e04991762a6c3d2aa4fce126e8ef141a6bb0fce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Rumi=C5=84ski?= Date: Fri, 9 Oct 2020 00:37:34 +0200 Subject: [PATCH 07/81] Support short custom youtube channel urls --- .../YoutubeChannelLinkHandlerFactory.java | 27 +++++++++++++++---- .../YoutubeChannelLinkHandlerFactoryTest.java | 2 ++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index 0eb03085..89eaaed5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -49,6 +49,17 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { return "https://www.youtube.com/" + id; } + /** + * Returns true if path conform to + * custom short channel urls like youtube.com/yourcustomname + * + * @param splitPath path segments array + * @return true - if value conform to short channel url, false - not + */ + public boolean isCustomShortChannelUrl(String[] splitPath) { + return splitPath.length == 1 && !splitPath[0].matches("playlist|watch"); + } + @Override public String getId(String url) throws ParsingException { try { @@ -60,14 +71,20 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { throw new ParsingException("the URL given is not a Youtube-URL"); } - if (!path.startsWith("/user/") && !path.startsWith("/channel/") && !path.startsWith("/c/")) { + // remove leading "/" + path = path.substring(1); + String[] splitPath = path.split("/"); + + // Handle custom short channel urls like youtube.com/yourcustomname + if (isCustomShortChannelUrl(splitPath)) { + path = "c/" + path; + splitPath = path.split("/"); + } + + if (!path.startsWith("user/") && !path.startsWith("channel/") && !path.startsWith("c/")) { throw new ParsingException("the URL given is neither a channel nor an user"); } - // remove leading "/" - path = path.substring(1); - - String[] splitPath = path.split("/"); String id = splitPath[1]; if (id == null || !id.matches("[A-Za-z0-9_-]+")) { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java index fc409ffa..178d4b8a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java @@ -30,6 +30,8 @@ public class YoutubeChannelLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://www.youtube.com/c/creatoracademy")); + assertTrue(linkHandler.acceptUrl("https://youtube.com/DIMENSI0N")); + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA")); assertTrue(linkHandler.acceptUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); From 8c38a5509efad0ed4f2a4d84089ef3a74e113d1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Rumi=C5=84ski?= Date: Fri, 9 Oct 2020 19:07:38 +0200 Subject: [PATCH 08/81] Prevent attribution_link urls to be accepted by channel links handler --- .../linkHandler/YoutubeChannelLinkHandlerFactory.java | 5 +++-- .../youtube/YoutubeChannelLinkHandlerFactoryTest.java | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index 89eaaed5..ca163dd6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -56,8 +56,9 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { * @param splitPath path segments array * @return true - if value conform to short channel url, false - not */ - public boolean isCustomShortChannelUrl(String[] splitPath) { - return splitPath.length == 1 && !splitPath[0].matches("playlist|watch"); + private boolean isCustomShortChannelUrl(String[] splitPath) { + return splitPath.length == 1 && + !splitPath[0].matches("playlist|watch|attribution_link"); } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java index 178d4b8a..14c8b061 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -46,6 +47,8 @@ public class YoutubeChannelLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA")); assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); + + assertFalse(linkHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare")); } @Test From 9d63211a6694c6263bcc02dd4303aa3a4a4e9cbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Rumi=C5=84ski?= Date: Mon, 12 Oct 2020 19:56:53 +0200 Subject: [PATCH 09/81] Fix typos Co-authored-by: Tobias Groza --- .../youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index ca163dd6..aead6701 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -51,10 +51,10 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { /** * Returns true if path conform to - * custom short channel urls like youtube.com/yourcustomname + * custom short channel URLs like youtube.com/yourcustomname * * @param splitPath path segments array - * @return true - if value conform to short channel url, false - not + * @return true - if value conform to short channel URL, false - not */ private boolean isCustomShortChannelUrl(String[] splitPath) { return splitPath.length == 1 && From 7abb4b371381dea969cbd90f2f332e751047d3c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Rumi=C5=84ski?= Date: Mon, 12 Oct 2020 19:57:45 +0200 Subject: [PATCH 10/81] Update extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java Fix typos Co-authored-by: Tobias Groza --- .../youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index aead6701..6f1569e7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -76,7 +76,7 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { path = path.substring(1); String[] splitPath = path.split("/"); - // Handle custom short channel urls like youtube.com/yourcustomname + // Handle custom short channel URLs like youtube.com/yourcustomname if (isCustomShortChannelUrl(splitPath)) { path = "c/" + path; splitPath = path.split("/"); From 5ab1b053d2c8629076c3ac894af1df73916ee36e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Rumi=C5=84ski?= Date: Mon, 12 Oct 2020 20:11:28 +0200 Subject: [PATCH 11/81] Update youtube channel link handler tests Co-authored-by: Tobias Groza --- .../youtube/YoutubeChannelLinkHandlerFactoryTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java index 14c8b061..d2de6d29 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java @@ -48,7 +48,13 @@ public class YoutubeChannelLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA")); assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); + // do not accept URLs which are not channels + assertFalse(linkHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI&t=100")); + assertFalse(linkHandler.acceptUrl("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY")); assertFalse(linkHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare")); + assertFalse(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1d")); + assertFalse(linkHandler.acceptUrl("https://www.youtube.com/embed/jZViOEv90dI")); + assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); } @Test From e3f996e014a0b0d06f6183bf3ae5a195be2cf6fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Rumi=C5=84ski?= Date: Mon, 12 Oct 2020 20:59:56 +0200 Subject: [PATCH 12/81] Exlude links which are not channels --- .../youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index 6f1569e7..55e3e665 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -58,7 +58,7 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { */ private boolean isCustomShortChannelUrl(String[] splitPath) { return splitPath.length == 1 && - !splitPath[0].matches("playlist|watch|attribution_link"); + !splitPath[0].matches("playlist|watch|attribution_link|watch_popup|embed|feed"); } @Override From 19e862657a821d133aae38a3011235a584c740cd Mon Sep 17 00:00:00 2001 From: Stypox Date: Sun, 4 Oct 2020 15:30:17 +0200 Subject: [PATCH 13/81] [YouTube] Fix some decryption exceptions by retrying --- .../extractors/YoutubeStreamExtractor.java | 69 +++++++++++-------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 0b18c4c2..8364246c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -633,11 +633,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; + final String playerUrl; initialAjaxJson = getJsonResponse(url, getExtractorLocalization()); - final String playerUrl; - if (initialAjaxJson.getObject(2).has("response")) { // age-restricted videos initialData = initialAjaxJson.getObject(2).getObject("response"); ageLimit = 18; @@ -647,12 +646,31 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody(); videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse)); playerUrl = info.url; - } else { - initialData = initialAjaxJson.getObject(3).getObject("response"); - ageLimit = NO_AGE_LIMIT; - playerArgs = getPlayerArgs(initialAjaxJson.getObject(2).getObject("player")); - playerUrl = getPlayerUrl(initialAjaxJson.getObject(2).getObject("player")); + } else { + ageLimit = NO_AGE_LIMIT; + JsonObject playerConfig; + + // sometimes at random YouTube does not provide the player config, + // so just retry the same request three times + int attempts = 2; + while (true) { + playerConfig = initialAjaxJson.getObject(2).getObject("player", null); + if (playerConfig != null) { + break; + } + + if (attempts <= 0) { + throw new ParsingException( + "YouTube did not provide player config even after three attempts"); + } + initialAjaxJson = getJsonResponse(url, getExtractorLocalization()); + --attempts; + } + initialData = initialAjaxJson.getObject(3).getObject("response"); + + playerArgs = getPlayerArgs(playerConfig); + playerUrl = getPlayerUrl(playerConfig); } playerResponse = getPlayerResponse(); @@ -674,36 +692,27 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException { - JsonObject playerArgs; - + private JsonObject getPlayerArgs(final JsonObject playerConfig) throws ParsingException { //attempt to load the youtube js player JSON arguments - try { - playerArgs = playerConfig.getObject("args"); - } catch (Exception e) { - throw new ParsingException("Could not parse yt player config", e); + final JsonObject playerArgs = playerConfig.getObject("args", null); + if (playerArgs == null) { + throw new ParsingException("Could not extract args from YouTube player config"); } - return playerArgs; } - private String getPlayerUrl(JsonObject playerConfig) throws ParsingException { - try { - // The Youtube service needs to be initialized by downloading the - // js-Youtube-player. This is done in order to get the algorithm - // for decrypting cryptic signatures inside certain stream urls. - String playerUrl; + private String getPlayerUrl(final JsonObject playerConfig) throws ParsingException { + // The Youtube service needs to be initialized by downloading the + // js-Youtube-player. This is done in order to get the algorithm + // for decrypting cryptic signatures inside certain stream URLs. + final String playerUrl = playerConfig.getObject("assets").getString("js"); - JsonObject ytAssets = playerConfig.getObject("assets"); - playerUrl = ytAssets.getString("js"); - - if (playerUrl.startsWith("//")) { - playerUrl = HTTPS + playerUrl; - } - return playerUrl; - } catch (Exception e) { - throw new ParsingException("Could not load decryption code for the Youtube service.", e); + if (playerUrl == null) { + throw new ParsingException("Could not extract js URL from YouTube player config"); + } else if (playerUrl.startsWith("//")) { + return HTTPS + playerUrl; } + return playerUrl; } private JsonObject getPlayerResponse() throws ParsingException { From 538f5d3973258cc1b2943e0e4ccc9caf169a4c32 Mon Sep 17 00:00:00 2001 From: Scratch Date: Fri, 16 Oct 2020 13:45:05 +1100 Subject: [PATCH 14/81] Fix SoundCloud test artist account name --- .../services/soundcloud/SoundcloudChannelExtractorTest.java | 2 +- .../soundcloud/SoundcloudStreamExtractorDefaultTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java index 1877e11f..f9c71b75 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java @@ -41,7 +41,7 @@ public class SoundcloudChannelExtractorTest { @Test public void testName() { - assertEquals("LIL UZI VERT", extractor.getName()); + assertEquals("Lil Uzi Vert", extractor.getName()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 300d3757..794a1a6b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -63,7 +63,7 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetUploaderName() throws ParsingException { - assertEquals("LIL UZI VERT", extractor.getUploaderName()); + assertEquals("Lil Uzi Vert", extractor.getUploaderName()); } @Test From 6a70cb9d50132c1cf1bf59478f029b199d65fefc Mon Sep 17 00:00:00 2001 From: Scratch Date: Fri, 16 Oct 2020 13:39:58 +1100 Subject: [PATCH 15/81] Remove tailing slash from SoundCloud URLs Fixes #412 --- .../soundcloud/extractors/SoundcloudStreamExtractor.java | 2 +- .../linkHandler/SoundcloudStreamLinkHandlerFactory.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 6aee297d..d909acfc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -46,7 +46,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl()); + track = SoundcloudParsingHelper.resolveFor(downloader, getUrl()); String policy = track.getString("policy", EMPTY_STRING); if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java index e271345d..c70eff01 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java @@ -30,6 +30,8 @@ public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public String getId(String url) throws ParsingException { Utils.checkUrl(URL_PATTERN, url); + // Remove the tailing slash from URLs due to issues with the SoundCloud API + if (url.charAt(url.length() -1) == '/') url = url.substring(0, url.length()-1); try { return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); From be9a6f931ca1c517aa4eaf1eb172b593e24d9238 Mon Sep 17 00:00:00 2001 From: wb9688 Date: Fri, 16 Oct 2020 20:27:40 +0200 Subject: [PATCH 16/81] Fix parsing new ytInitialData --- .../extractor/services/youtube/YoutubeParsingHelper.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index e124d020..af10efa2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -197,8 +197,13 @@ public class YoutubeParsingHelper { public static JsonObject getInitialData(String html) throws ParsingException { try { - String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html); - return JsonParser.object().from(initialData); + try { + final String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html); + return JsonParser.object().from(initialData); + } catch (Parser.RegexException e) { + final String initialData = Parser.matchGroup1("var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html); + return JsonParser.object().from(initialData); + } } catch (JsonParserException | Parser.RegexException e) { throw new ParsingException("Could not get ytInitialData", e); } From 350eed6214b93255d788dfa208b1e9a5e5da91e6 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 16 Oct 2020 22:03:57 +0200 Subject: [PATCH 17/81] Release v0.20.1 --- README.md | 2 +- build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ec48cef8..36e2f7b5 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo. If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps: 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. -2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.0'`the `dependencies` in your `build.gradle`. Replace `v0.20.0` with the latest release. +2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.1'`the `dependencies` in your `build.gradle`. Replace `v0.20.1` with the latest release. ### Testing changes diff --git a/build.gradle b/build.gradle index 3d579cfb..f9d80df1 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.20.0' + version 'v0.20.1' group 'com.github.TeamNewPipe' repositories { From e945f711c1b0f79d8db6e0fb48d028965e53ec00 Mon Sep 17 00:00:00 2001 From: Scratch Date: Sat, 17 Oct 2020 08:42:46 +1100 Subject: [PATCH 18/81] Fix SoundCloud test artist account name (#416) --- .../services/soundcloud/SoundcloudPlaylistExtractorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index de9094c0..fa99a7df 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -98,7 +98,7 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testUploaderName() { - assertTrue(extractor.getUploaderName().contains("LIL UZI VERT")); + assertTrue(extractor.getUploaderName().contains("Lil Uzi Vert")); } @Test From 6887d595705a57f635151a0586b19669da862b71 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sun, 18 Oct 2020 12:03:01 +0200 Subject: [PATCH 19/81] [YouTube] Handle urls for Shorts --- .../youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java | 2 +- .../services/youtube/YoutubeStreamLinkHandlerFactoryTest.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index 596de2da..48fdc286 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -153,7 +153,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { return assertIsId(viewQueryValue); } - if (path.startsWith("embed/")) { + if (path.startsWith("embed/") || path.startsWith("shorts/")) { String id = path.split("/")[1]; return assertIsId(id); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index 62398d59..965306d5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -81,6 +81,7 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId()); assertEquals("n8X9_MgEdCg", linkHandler.fromUrl("vnd.youtube://n8X9_MgEdCg").getId()); assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId()); + assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("http://www.youtube.com/shorts/IOS2fqxwYbA").getId()); } @Test @@ -101,6 +102,7 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI")); assertTrue(linkHandler.acceptUrl("vnd.youtube.launch:jZViOEv90dI")); assertTrue(linkHandler.acceptUrl("https://music.youtube.com/watch?v=O0EDx9WAelc")); + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/shorts/IOS2fqxwYbA")); } @Test From d3f80d1538286434c32fa3d2cfe7a8fe987c09a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Rumi=C5=84ski?= Date: Tue, 20 Oct 2020 20:06:06 +0200 Subject: [PATCH 20/81] Exlude links which are not channels --- .../youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java | 2 +- .../youtube/YoutubeChannelLinkHandlerFactoryTest.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index 55e3e665..288c28b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -58,7 +58,7 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { */ private boolean isCustomShortChannelUrl(String[] splitPath) { return splitPath.length == 1 && - !splitPath[0].matches("playlist|watch|attribution_link|watch_popup|embed|feed"); + !splitPath[0].matches("playlist|watch|attribution_link|watch_popup|embed|feed|select_site"); } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java index d2de6d29..12bcb804 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java @@ -47,6 +47,8 @@ public class YoutubeChannelLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA")); assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watchismo")); + // do not accept URLs which are not channels assertFalse(linkHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI&t=100")); @@ -55,6 +57,8 @@ public class YoutubeChannelLinkHandlerFactoryTest { assertFalse(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1d")); assertFalse(linkHandler.acceptUrl("https://www.youtube.com/embed/jZViOEv90dI")); assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertFalse(linkHandler.acceptUrl("https://www.youtube.com/?app=desktop&persist_app=1")); + assertFalse(linkHandler.acceptUrl("https://m.youtube.com/select_site")); } @Test From 0e67d820bcbe65d1c7de0e8d4cbf607b13317fa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Rumi=C5=84ski?= Date: Thu, 22 Oct 2020 20:13:29 +0200 Subject: [PATCH 21/81] Use static regex pattern for excluded path segments --- .../YoutubeChannelLinkHandlerFactory.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index 288c28b2..af6bd12c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import java.util.regex.Pattern; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; @@ -49,6 +50,7 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { return "https://www.youtube.com/" + id; } + /** * Returns true if path conform to * custom short channel URLs like youtube.com/yourcustomname @@ -56,15 +58,17 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { * @param splitPath path segments array * @return true - if value conform to short channel URL, false - not */ - private boolean isCustomShortChannelUrl(String[] splitPath) { - return splitPath.length == 1 && - !splitPath[0].matches("playlist|watch|attribution_link|watch_popup|embed|feed|select_site"); + private boolean isCustomShortChannelUrl(final String[] splitPath) { + return splitPath.length == 1 && !excludedSegments.matcher(splitPath[0]).matches(); } + private static final Pattern excludedSegments = + Pattern.compile("playlist|watch|attribution_link|watch_popup|embed|feed|select_site"); + @Override public String getId(String url) throws ParsingException { try { - URL urlObj = Utils.stringToURL(url); + final URL urlObj = Utils.stringToURL(url); String path = urlObj.getPath(); if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) || @@ -86,7 +90,7 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { throw new ParsingException("the URL given is neither a channel nor an user"); } - String id = splitPath[1]; + final String id = splitPath[1]; if (id == null || !id.matches("[A-Za-z0-9_-]+")) { throw new ParsingException("The given id is not a Youtube-Video-ID"); From 29695aed0a295f7bf84fea435a7dfd304989fcd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Rumi=C5=84ski?= Date: Fri, 23 Oct 2020 16:42:13 +0200 Subject: [PATCH 22/81] Small field refactor --- .../linkHandler/YoutubeChannelLinkHandlerFactory.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index af6bd12c..2dc8fc42 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -33,6 +33,9 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { private static final YoutubeChannelLinkHandlerFactory instance = new YoutubeChannelLinkHandlerFactory(); + private static final Pattern excludedSegments = + Pattern.compile("playlist|watch|attribution_link|watch_popup|embed|feed|select_site"); + public static YoutubeChannelLinkHandlerFactory getInstance() { return instance; } @@ -49,8 +52,7 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { public String getUrl(String id, List contentFilters, String searchFilter) { return "https://www.youtube.com/" + id; } - - + /** * Returns true if path conform to * custom short channel URLs like youtube.com/yourcustomname @@ -62,9 +64,6 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { return splitPath.length == 1 && !excludedSegments.matcher(splitPath[0]).matches(); } - private static final Pattern excludedSegments = - Pattern.compile("playlist|watch|attribution_link|watch_popup|embed|feed|select_site"); - @Override public String getId(String url) throws ParsingException { try { From 1d7a86e664717d09e33371bc87c289903b19feb2 Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 9 Apr 2020 13:51:47 +0200 Subject: [PATCH 23/81] [Test] Add base classes for stream extractor tests Refactor all stream extractor tests to use new base class. Remove check if upload date is in the past: this does not have to hold true: youtube premieres turn up in search results even though they are in the future --- .../newpipe/extractor/ExtractorAsserts.java | 14 +- .../services/BaseStreamExtractorTest.java | 24 ++ .../services/DefaultStreamExtractorTest.java | 286 +++++++++++++++ .../extractor/services/DefaultTests.java | 4 +- .../MediaCCCStreamExtractorTest.java | 214 +++++------ .../PeertubeStreamExtractorDefaultTest.java | 180 ---------- .../peertube/PeertubeStreamExtractorTest.java | 165 +++++++++ .../PeertubeTrendingExtractorTest.java | 4 +- .../SoundcloudChartsExtractorTest.java | 2 +- .../SoundcloudStreamExtractorDefaultTest.java | 163 --------- .../SoundcloudStreamExtractorTest.java | 81 +++++ ...utubeStreamExtractorAgeRestrictedTest.java | 157 ++------- ...utubeStreamExtractorControversialTest.java | 138 ++------ .../YoutubeStreamExtractorDefaultTest.java | 333 ++++++------------ .../YoutubeStreamExtractorLivestreamTest.java | 155 ++------ .../YoutubeStreamExtractorUnlistedTest.java | 169 ++------- 16 files changed, 883 insertions(+), 1206 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java index 1006f7b3..a0a887d7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java @@ -1,12 +1,16 @@ package org.schabi.newpipe.extractor; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.net.MalformedURLException; import java.net.URL; import java.util.List; -import static org.junit.Assert.*; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; public class ExtractorAsserts { public static void assertEmptyErrors(String message, List errors) { @@ -56,4 +60,8 @@ public class ExtractorAsserts { assertTrue(message, stringToCheck.isEmpty()); } } + + public static void assertAtLeast(long expected, long actual) { + assertTrue(actual + " is not at least " + expected, actual >= expected); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java new file mode 100644 index 00000000..fb555a03 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java @@ -0,0 +1,24 @@ +package org.schabi.newpipe.extractor.services; + +public interface BaseStreamExtractorTest extends BaseExtractorTest { + void testStreamType() throws Exception; + void testUploaderName() throws Exception; + void testUploaderUrl() throws Exception; + void testUploaderAvatarUrl() throws Exception; + void testThumbnailUrl() throws Exception; + void testDescription() throws Exception; + void testLength() throws Exception; + void testTimestamp() throws Exception; + void testViewCount() throws Exception; + void testUploadDate() throws Exception; + void testTextualUploadDate() throws Exception; + void testLikeCount() throws Exception; + void testDislikeCount() throws Exception; + void testRelatedStreams() throws Exception; + void testAgeLimit() throws Exception; + void testErrorMessage() throws Exception; + void testAudioStreams() throws Exception; + void testVideoStreams() throws Exception; + void testSubtitles() throws Exception; + void testFrames() throws Exception; +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java new file mode 100644 index 00000000..406177f7 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -0,0 +1,286 @@ +package org.schabi.newpipe.extractor.services; + +import org.junit.Test; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Description; +import org.schabi.newpipe.extractor.stream.Frameset; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; + +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.List; +import java.util.TimeZone; + +import javax.annotation.Nullable; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertAtLeast; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems; + +/** + * Test for {@link StreamExtractor} + */ +public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest + implements BaseStreamExtractorTest { + + public abstract StreamType expectedStreamType(); + public abstract String expectedUploaderName(); + public abstract String expectedUploaderUrl(); + public abstract List expectedDescriptionContains(); // e.g. for full links + public abstract long expectedLength(); + public long expectedTimestamp() { return 0; }; // default: there is no timestamp + public abstract long expectedViewCountAtLeast(); + @Nullable public abstract String expectedUploadDate(); // format: "yyyy-MM-dd HH:mm:ss.SSS" + @Nullable public abstract String expectedTextualUploadDate(); + public abstract long expectedLikeCountAtLeast(); // return -1 if ratings are disabled + public abstract long expectedDislikeCountAtLeast(); // return -1 if ratings are disabled + public boolean expectedHasRelatedStreams() { return true; } // default: there are related videos + public int expectedAgeLimit() { return StreamExtractor.NO_AGE_LIMIT; } // default: no limit + @Nullable public String expectedErrorMessage() { return null; } // default: no error message + public boolean expectedHasVideoStreams() { return true; } // default: there are video streams + public boolean expectedHasAudioStreams() { return true; } // default: there are audio streams + public boolean expectedHasSubtitles() { return true; } // default: there are subtitles streams + public boolean expectedHasFrames() { return true; } // default: there are frames + + @Test + @Override + public void testStreamType() throws Exception { + assertEquals(expectedStreamType(), extractor().getStreamType()); + } + + @Test + @Override + public void testUploaderName() throws Exception { + assertEquals(expectedUploaderName(), extractor().getUploaderName()); + } + + @Test + @Override + public void testUploaderUrl() throws Exception { + final String uploaderUrl = extractor().getUploaderUrl(); + assertIsSecureUrl(uploaderUrl); + assertEquals(expectedUploaderUrl(), uploaderUrl); + } + + @Test + @Override + public void testUploaderAvatarUrl() throws Exception { + assertIsSecureUrl(extractor().getUploaderAvatarUrl()); + } + + @Test + @Override + public void testThumbnailUrl() throws Exception { + assertIsSecureUrl(extractor().getThumbnailUrl()); + } + + @Test + @Override + public void testDescription() throws Exception { + final Description description = extractor().getDescription(); + assertNotNull(description); + assertFalse("description is empty", description.getContent().isEmpty()); + + for (String s : expectedDescriptionContains()) { + assertThat(description.getContent(), containsString(s)); + } + } + + @Test + @Override + public void testLength() throws Exception { + assertEquals(expectedLength(), extractor().getLength()); + } + + @Test + @Override + public void testTimestamp() throws Exception { + assertEquals(expectedTimestamp(), extractor().getTimeStamp()); + } + + @Test + @Override + public void testViewCount() throws Exception { + assertAtLeast(expectedViewCountAtLeast(), extractor().getViewCount()); + } + + @Test + @Override + public void testUploadDate() throws Exception { + final DateWrapper dateWrapper = extractor().getUploadDate(); + + if (expectedUploadDate() == null) { + assertNull(dateWrapper); + } else { + assertNotNull(dateWrapper); + + final Calendar expectedDate = Calendar.getInstance(); + final Calendar actualDate = dateWrapper.date(); + expectedDate.setTimeZone(TimeZone.getTimeZone("GMT")); + actualDate.setTimeZone(TimeZone.getTimeZone("GMT")); + + final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S"); + expectedDate.setTime(sdf.parse(expectedUploadDate())); + assertEquals(expectedDate, actualDate); + } + } + + @Test + @Override + public void testTextualUploadDate() throws Exception { + assertEquals(expectedTextualUploadDate(), extractor().getTextualUploadDate()); + } + + @Test + @Override + public void testLikeCount() throws Exception { + if (expectedLikeCountAtLeast() == -1) { + assertEquals(-1, extractor().getLikeCount()); + } else { + assertAtLeast(expectedLikeCountAtLeast(), extractor().getLikeCount()); + } + } + + @Test + @Override + public void testDislikeCount() throws Exception { + if (expectedDislikeCountAtLeast() == -1) { + assertEquals(-1, extractor().getDislikeCount()); + } else { + assertAtLeast(expectedDislikeCountAtLeast(), extractor().getDislikeCount()); + } + } + + @Test + @Override + public void testRelatedStreams() throws Exception { + final StreamInfoItemsCollector relatedStreams = extractor().getRelatedStreams(); + + if (expectedHasRelatedStreams()) { + defaultTestListOfItems(extractor().getService(), relatedStreams.getItems(), + relatedStreams.getErrors()); + } else { + assertNull(relatedStreams); + } + } + + @Test + @Override + public void testAgeLimit() throws Exception { + assertEquals(expectedAgeLimit(), extractor().getAgeLimit()); + } + + @Test + @Override + public void testErrorMessage() throws Exception { + assertEquals(expectedErrorMessage(), extractor().getErrorMessage()); + } + + @Test + @Override + public void testVideoStreams() throws Exception { + List videoStreams = extractor().getVideoStreams(); + final List videoOnlyStreams = extractor().getVideoOnlyStreams(); + assertNotNull(videoStreams); + assertNotNull(videoOnlyStreams); + videoStreams.addAll(videoOnlyStreams); + + if (expectedHasVideoStreams()) { + assertFalse(videoStreams.isEmpty()); + + for (VideoStream stream : videoStreams) { + assertIsSecureUrl(stream.getUrl()); + assertFalse(stream.getResolution().isEmpty()); + + int formatId = stream.getFormatId(); + assertTrue("format id does not fit a video stream: " + formatId, + 0 <= formatId && formatId < 0x100); + } + } else { + assertTrue(videoStreams.isEmpty()); + } + } + + @Test + @Override + public void testAudioStreams() throws Exception { + final List audioStreams = extractor().getAudioStreams(); + assertNotNull(audioStreams); + + if (expectedHasAudioStreams()) { + assertFalse(audioStreams.isEmpty()); + + for (AudioStream stream : audioStreams) { + assertIsSecureUrl(stream.getUrl()); + + int formatId = stream.getFormatId(); + assertTrue("format id does not fit an audio stream: " + formatId, + 0x100 <= formatId && formatId < 0x1000); + } + } else { + assertTrue(audioStreams.isEmpty()); + } + } + + @Test + @Override + public void testSubtitles() throws Exception { + List subtitles = extractor().getSubtitlesDefault(); + assertNotNull(subtitles); + + if (expectedHasSubtitles()) { + assertFalse(subtitles.isEmpty()); + + for (SubtitlesStream stream : subtitles) { + assertIsSecureUrl(stream.getUrl()); + + int formatId = stream.getFormatId(); + assertTrue("format id does not fit an audio stream: " + formatId, + 0x1000 <= formatId && formatId < 0x10000); + } + } else { + assertTrue(subtitles.isEmpty()); + + MediaFormat[] formats = {MediaFormat.VTT, MediaFormat.TTML, MediaFormat.TRANSCRIPT1, + MediaFormat.TRANSCRIPT2, MediaFormat.TRANSCRIPT3, MediaFormat.SRT}; + for (MediaFormat format : formats) { + subtitles = extractor().getSubtitles(format); + assertNotNull(subtitles); + assertTrue(subtitles.isEmpty()); + } + } + } + + @Test + @Override + public void testFrames() throws Exception { + final List frames = extractor().getFrames(); + assertNotNull(frames); + + if (expectedHasFrames()) { + assertFalse(frames.isEmpty()); + for (final Frameset f : frames) { + for (final String url : f.getUrls()) { + assertIsValidUrl(url); + assertIsSecureUrl(url); + } + } + } else { + assertTrue(frames.isEmpty()); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java index a335b0ec..d2b9ca4a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java @@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import java.util.Calendar; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -42,7 +41,7 @@ public final class DefaultTests { StreamInfoItem streamInfoItem = (StreamInfoItem) item; assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName()); -// assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl()); + // assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl()); final String uploaderUrl = streamInfoItem.getUploaderUrl(); if (!isNullOrEmpty(uploaderUrl)) { assertIsSecureUrl(uploaderUrl); @@ -54,7 +53,6 @@ public final class DefaultTests { if (!isNullOrEmpty(streamInfoItem.getTextualUploadDate())) { final DateWrapper uploadDate = streamInfoItem.getUploadDate(); assertNotNull("No parsed upload date", uploadDate); - assertTrue("Upload date not in the past", uploadDate.date().before(Calendar.getInstance())); } } else if (item instanceof ChannelInfoItem) { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index 95c88229..6a4c4c0e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -1,204 +1,148 @@ package org.schabi.newpipe.extractor.services.media_ccc; -import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; +import java.util.Arrays; import java.util.List; -import static java.util.Objects.requireNonNull; +import javax.annotation.Nullable; + import static junit.framework.TestCase.assertEquals; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** * Test {@link MediaCCCStreamExtractor} */ public class MediaCCCStreamExtractorTest { - public static class Gpn18Tmux { - private static MediaCCCStreamExtractor extractor; + private static final String BASE_URL = "https://media.ccc.de/v/"; + + public static class Gpn18Tmux extends DefaultStreamExtractorTest { + private static final String ID = "gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht"; + private static final String URL = BASE_URL + ID; + private static StreamExtractor extractor; @BeforeClass - public static void setUpClass() throws Exception { + public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - - extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht"); + extractor = MediaCCC.getStreamExtractor(URL); extractor.fetchPage(); } - @Test - public void testServiceId() throws Exception { - assertEquals(2, extractor.getServiceId()); - } + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return MediaCCC; } + @Override public String expectedName() { return "tmux - Warum ein schwarzes Fenster am Bildschirm reicht"; } + @Override public String expectedId() { return ID; } + @Override public String expectedUrlContains() { return URL; } + @Override public String expectedOriginalUrlContains() { return URL; } - @Test - public void testName() throws Exception { - assertEquals("tmux - Warum ein schwarzes Fenster am Bildschirm reicht", extractor.getName()); - } + @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } + @Override public String expectedUploaderName() { return "gpn18"; } + @Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/gpn18"; } + @Override public List expectedDescriptionContains() { return Arrays.asList("SSH-Sessions", "\"Terminal Multiplexer\""); } + @Override public long expectedLength() { return 3097; } + @Override public long expectedViewCountAtLeast() { return 2380; } + @Nullable @Override public String expectedUploadDate() { return "2018-05-11 00:00:00.000"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2018-05-11T02:00:00.000+02:00"; } + @Override public long expectedLikeCountAtLeast() { return -1; } + @Override public long expectedDislikeCountAtLeast() { return -1; } + @Override public boolean expectedHasSubtitles() { return false; } + @Override public boolean expectedHasFrames() { return false; } + @Override @Test - public void testId() throws Exception { - assertEquals("gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getId()); - } - - @Test - public void testUrl() throws Exception { - assertIsSecureUrl(extractor.getUrl()); - assertEquals("https://media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl()); - } - - @Test - public void testOriginalUrl() throws Exception { - assertIsSecureUrl(extractor.getOriginalUrl()); - assertEquals("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getOriginalUrl()); - } - - @Test - public void testThumbnail() throws Exception { - assertIsSecureUrl(extractor.getThumbnailUrl()); + public void testThumbnailUrl() throws Exception { + super.testThumbnailUrl(); assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl()); } - @Test - public void testUploaderName() throws Exception { - assertEquals("gpn18", extractor.getUploaderName()); - } - - @Test - public void testUploaderUrl() throws Exception { - assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); - } - + @Override @Test public void testUploaderAvatarUrl() throws Exception { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + super.testUploaderAvatarUrl(); assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl()); } + @Override @Test public void testVideoStreams() throws Exception { - List videoStreamList = extractor.getVideoStreams(); - assertEquals(4, videoStreamList.size()); - for (VideoStream stream : videoStreamList) { - assertIsSecureUrl(stream.getUrl()); - } + super.testVideoStreams(); + assertEquals(4, extractor.getVideoStreams().size()); } + @Override @Test public void testAudioStreams() throws Exception { - List audioStreamList = extractor.getAudioStreams(); - assertEquals(2, audioStreamList.size()); - for (AudioStream stream : audioStreamList) { - assertIsSecureUrl(stream.getUrl()); - } - } - - @Test - public void testGetTextualUploadDate() throws ParsingException { - Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate()); - } - - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2018-05-11")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + super.testAudioStreams(); + assertEquals(2, extractor.getAudioStreams().size()); } } - public static class _36c3PrivacyMessaging { - private static MediaCCCStreamExtractor extractor; + public static class _36c3PrivacyMessaging extends DefaultStreamExtractorTest { + private static final String ID = "36c3-10565-what_s_left_for_private_messaging"; + private static final String URL = BASE_URL + ID; + private static StreamExtractor extractor; @BeforeClass - public static void setUpClass() throws Exception { + public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging"); + extractor = MediaCCC.getStreamExtractor(URL); extractor.fetchPage(); } - @Test - public void testName() throws Exception { - assertEquals("What's left for private messaging?", extractor.getName()); - } + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return MediaCCC; } + @Override public String expectedName() { return "What's left for private messaging?"; } + @Override public String expectedId() { return ID; } + @Override public String expectedUrlContains() { return URL; } + @Override public String expectedOriginalUrlContains() { return URL; } - @Test - public void testId() throws Exception { - assertEquals("36c3-10565-what_s_left_for_private_messaging", extractor.getId()); - } + @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } + @Override public String expectedUploaderName() { return "36c3"; } + @Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/36c3"; } + @Override public List expectedDescriptionContains() { return Arrays.asList("WhatsApp", "Signal"); } + @Override public long expectedLength() { return 3603; } + @Override public long expectedViewCountAtLeast() { return 2380; } + @Nullable @Override public String expectedUploadDate() { return "2020-01-11 00:00:00.000"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2020-01-11T01:00:00.000+01:00"; } + @Override public long expectedLikeCountAtLeast() { return -1; } + @Override public long expectedDislikeCountAtLeast() { return -1; } + @Override public boolean expectedHasSubtitles() { return false; } + @Override public boolean expectedHasFrames() { return false; } + @Override @Test - public void testUrl() throws Exception { - assertIsSecureUrl(extractor.getUrl()); - assertEquals("https://media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl()); - } - - @Test - public void testOriginalUrl() throws Exception { - assertIsSecureUrl(extractor.getOriginalUrl()); - assertEquals("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging", extractor.getOriginalUrl()); - } - - @Test - public void testThumbnail() throws Exception { - assertIsSecureUrl(extractor.getThumbnailUrl()); + public void testThumbnailUrl() throws Exception { + super.testThumbnailUrl(); assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl()); } - @Test - public void testUploaderName() throws Exception { - assertEquals("36c3", extractor.getUploaderName()); - } - - @Test - public void testUploaderUrl() throws Exception { - assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl()); - } - + @Override @Test public void testUploaderAvatarUrl() throws Exception { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + super.testUploaderAvatarUrl(); assertEquals("https://static.media.ccc.de/media/congress/2019/logo.png", extractor.getUploaderAvatarUrl()); } + @Override @Test public void testVideoStreams() throws Exception { - List videoStreamList = extractor.getVideoStreams(); - assertEquals(8, videoStreamList.size()); - for (VideoStream stream : videoStreamList) { - assertIsSecureUrl(stream.getUrl()); - } + super.testVideoStreams(); + assertEquals(8, extractor.getVideoStreams().size()); } + @Override @Test public void testAudioStreams() throws Exception { - List audioStreamList = extractor.getAudioStreams(); - assertEquals(2, audioStreamList.size()); - for (AudioStream stream : audioStreamList) { - assertIsSecureUrl(stream.getUrl()); - } - } - - @Test - public void testGetTextualUploadDate() throws ParsingException { - Assert.assertEquals("2020-01-11T01:00:00.000+01:00", extractor.getTextualUploadDate()); - } - - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2020-01-11")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + super.testAudioStreams(); + assertEquals(2, extractor.getAudioStreams().size()); } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java deleted file mode 100644 index c3d8c716..00000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ /dev/null @@ -1,180 +0,0 @@ -package org.schabi.newpipe.extractor.services.peertube; - -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.StreamType; - -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Locale; -import java.util.TimeZone; - -import static java.util.Objects.requireNonNull; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.PeerTube; - -/** - * Test for {@link StreamExtractor} - */ -public class PeertubeStreamExtractorDefaultTest { - private static PeertubeStreamExtractor extractor; - private static final String expectedLargeDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; - private static final String expectedSmallDescription = "https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary"; - - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - // setting instance might break test when running in parallel - PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube")); - extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d"); - extractor.fetchPage(); - } - - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'"); - sdf.setTimeZone(TimeZone.getTimeZone("GMT")); - instance.setTime(sdf.parse("2018-10-01T10:52:46.396Z")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); - - } - - @Test - public void testGetInvalidTimeStamp() throws ParsingException { - assertTrue(extractor.getTimeStamp() + "", - extractor.getTimeStamp() <= 0); - } - - @Test - public void testGetTitle() throws ParsingException { - assertEquals("What is PeerTube?", extractor.getName()); - } - - @Test - public void testGetLargeDescription() throws ParsingException { - assertEquals(expectedLargeDescription, extractor.getDescription().getContent()); - } - - @Test - public void testGetEmptyDescription() throws Exception { - PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); - extractorEmpty.fetchPage(); - assertEquals("", extractorEmpty.getDescription().getContent()); - } - - @Test - public void testGetSmallDescription() throws Exception { - PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server")); - PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea"); - extractorSmall.fetchPage(); - assertEquals(expectedSmallDescription, extractorSmall.getDescription().getContent()); - } - - @Test - public void testGetUploaderName() throws ParsingException { - assertEquals("Framasoft", extractor.getUploaderName()); - } - - @Test - public void testGetUploaderUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl()); - } - - @Test - public void testGetUploaderAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); - } - - @Test - public void testGetSubChannelName() throws ParsingException { - assertEquals("Les vidéos de Framasoft", extractor.getSubChannelName()); - } - - @Test - public void testGetSubChannelUrl() throws ParsingException { - assertIsSecureUrl(extractor.getSubChannelUrl()); - assertEquals("https://framatube.org/video-channels/bf54d359-cfad-4935-9d45-9d6be93f63e8", extractor.getSubChannelUrl()); - } - - @Test - public void testGetSubChannelAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getSubChannelAvatarUrl()); - } - - @Test - public void testGetLength() throws ParsingException { - assertEquals(113, extractor.getLength()); - } - - @Test - public void testGetViewCount() throws ParsingException { - assertTrue(Long.toString(extractor.getViewCount()), - extractor.getViewCount() > 10); - } - - @Test - public void testGetThumbnailUrl() throws ParsingException { - assertIsSecureUrl(extractor.getThumbnailUrl()); - } - - @Test - public void testGetVideoStreams() throws IOException, ExtractionException { - assertFalse(extractor.getVideoStreams().isEmpty()); - } - - @Test - public void testStreamType() throws ParsingException { - assertTrue(extractor.getStreamType() == StreamType.VIDEO_STREAM); - } - - @Ignore - @Test - public void testGetRelatedVideos() throws ExtractionException, IOException { - StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); - assertFalse(relatedVideos.getItems().isEmpty()); - assertTrue(relatedVideos.getErrors().isEmpty()); - } - - @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - assertFalse(extractor.getSubtitlesDefault().isEmpty()); - } - - @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { - assertFalse(extractor.getSubtitlesDefault().isEmpty()); - } - - @Test - public void testGetAgeLimit() throws ExtractionException, IOException { - assertEquals(0, extractor.getAgeLimit()); - PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://nocensoring.net/videos/embed/dbd8e5e1-c527-49b6-b70c-89101dbb9c08"); - ageLimit.fetchPage(); - assertEquals(18, ageLimit.getAgeLimit()); - } - - @Test - public void testGetSupportInformation() throws ExtractionException, IOException { - PeertubeStreamExtractor supportInfoExtractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37"); - supportInfoExtractor.fetchPage(); - assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo()); - } - - @Test - public void testGetLanguageInformation() throws ParsingException { - assertEquals(new Locale("en"), extractor.getLanguageInfo()); - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java new file mode 100644 index 00000000..bddb75bd --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java @@ -0,0 +1,165 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import javax.annotation.Nullable; + +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +public class PeertubeStreamExtractorTest { + private static final String BASE_URL = "/videos/watch/"; + + public static class WhatIsPeertube extends DefaultStreamExtractorTest { + private static final String ID = "9c9de5e8-0a1e-484a-b099-e80766180a6d"; + private static final String INSTANCE = "https://framatube.org"; + private static final String URL = INSTANCE + BASE_URL + ID; + private static StreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel (!) + PeerTube.setInstance(new PeertubeInstance(INSTANCE, "FramaTube")); + extractor = PeerTube.getStreamExtractor(URL); + extractor.fetchPage(); + } + + @Test + public void testGetLanguageInformation() throws ParsingException { + assertEquals(new Locale("en"), extractor.getLanguageInfo()); + } + + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return PeerTube; } + @Override public String expectedName() { return "What is PeerTube?"; } + @Override public String expectedId() { return ID; } + @Override public String expectedUrlContains() { return BASE_URL + ID; } + @Override public String expectedOriginalUrlContains() { return URL; } + + @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } + @Override public String expectedUploaderName() { return "Framasoft"; } + @Override public String expectedUploaderUrl() { return "https://framatube.org/accounts/framasoft"; } + @Override public List expectedDescriptionContains() { // CRLF line ending + return Arrays.asList("**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n" + + "\r\n" + + "**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n" + + "*A decentralized video hosting network, based on free/libre software!*\r\n" + + "\r\n" + + "**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n" + + "*Directed by* Aryeom\r\n" + + "*Assistant* Jehan\r\n" + + "**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n" + + "\r\n" + + "**Sponsored by** [Framasoft](https://framasoft.org)\r\n" + + "\r\n" + + "**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n" + + "\r\n" + + "**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n" + + "\r\n" + + "**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"); + } + @Override public long expectedLength() { return 113; } + @Override public long expectedViewCountAtLeast() { return 38600; } + @Nullable @Override public String expectedUploadDate() { return "2018-10-01 12:52:46.396"; } // GMT (!) + @Nullable @Override public String expectedTextualUploadDate() { return "2018-10-01T10:52:46.396Z"; } + @Override public long expectedLikeCountAtLeast() { return 120; } + @Override public long expectedDislikeCountAtLeast() { return 0; } + @Override public boolean expectedHasAudioStreams() { return false; } + @Override public boolean expectedHasFrames() { return false; } + + } + + public static class AgeRestricted extends DefaultStreamExtractorTest { + private static final String ID = "0d501633-f2d9-4476-87c6-71f1c02402a4"; + private static final String INSTANCE = "https://peertube.co.uk"; + private static final String URL = INSTANCE + BASE_URL + ID; + private static StreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance());; + // setting instance might break test when running in parallel (!) + PeerTube.setInstance(new PeertubeInstance(INSTANCE)); + extractor = PeerTube.getStreamExtractor(URL); + extractor.fetchPage(); + } + + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return PeerTube; } + @Override public String expectedName() { return "A DPR Combatant Describes how Orders are Given through Russian Officers"; } + @Override public String expectedId() { return ID; } + @Override public String expectedUrlContains() { return BASE_URL + ID; } + @Override public String expectedOriginalUrlContains() { return URL; } + + @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } + @Override public String expectedUploaderName() { return "Tomas Berezovskiy"; } + @Override public String expectedUploaderUrl() { return "https://peertube.iriseden.eu/accounts/tomas_berezovskiy"; } + @Override public List expectedDescriptionContains() { // LF line ending + return Arrays.asList("https://en.informnapalm.org/dpr-combatant-describes-orders-given-russian-officers/ " + + " The InformNapalm team received another video of a separatist prisoner of war telling about his " + + "activities in `Dontesk People’s Republic’ (DPR) structures. The video is old, as the interrogation" + + " date is September, but it is the situation described is still relevant and interesting today. In " + + "this recording the combatant re-tells how he came to be recruited into the DPR forces, and how " + + "they are operating under Russian military command. He expresses remorse for his stupidity. Perhaps" + + " he is just saying what he thinks his interrogator wants to hear, perhaps he is speaking from a " + + "new understanding?\n" + + "\n" + + "The video contains a lot of cut and paste (stitching) in places where intelligence data or valuable" + + " information has been deleted because it cannot be shared publically. We trust you will understand " + + "this necessity."); + } + @Override public long expectedLength() { return 512; } + @Override public long expectedViewCountAtLeast() { return 7; } + @Nullable @Override public String expectedUploadDate() { return "2019-10-22 08:16:48.982"; } // GMT (!) + @Nullable @Override public String expectedTextualUploadDate() { return "2019-10-22T06:16:48.982Z"; } + @Override public long expectedLikeCountAtLeast() { return 3; } + @Override public long expectedDislikeCountAtLeast() { return 0; } + @Override public int expectedAgeLimit() { return 18; } + @Override public boolean expectedHasAudioStreams() { return false; } + @Override public boolean expectedHasSubtitles() { return false; } + @Override public boolean expectedHasFrames() { return false; } + } + + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server")); + } + + @Test + public void testGetEmptyDescription() throws Exception { + StreamExtractor extractorEmpty = PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); + extractorEmpty.fetchPage(); + assertEquals("", extractorEmpty.getDescription().getContent()); + } + + @Test + public void testGetSmallDescription() throws Exception { + StreamExtractor extractorSmall = PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea"); + extractorSmall.fetchPage(); + assertEquals("https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary", extractorSmall.getDescription().getContent()); + } + + @Test + public void testGetSupportInformation() throws ExtractionException, IOException { + StreamExtractor supportInfoExtractor = PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37"); + supportInfoExtractor.fetchPage(); + assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java index 60f72f9d..924dbbc1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java @@ -8,8 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseListExtractorTest; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ServiceList.*; +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; import static org.schabi.newpipe.extractor.services.DefaultTests.*; public class PeertubeTrendingExtractorTest { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java index 8abfb4d3..54c48431 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseListExtractorTest; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChartsExtractor; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.services.DefaultTests.*; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java deleted file mode 100644 index 794a1a6b..00000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ /dev/null @@ -1,163 +0,0 @@ -package org.schabi.newpipe.extractor.services.soundcloud; - -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.StreamType; - -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.TimeZone; - -import static java.util.Objects.requireNonNull; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; - -/** - * Test for {@link StreamExtractor} - */ -public class SoundcloudStreamExtractorDefaultTest { - - public static class LilUziVertDoWhatIWant { - private static SoundcloudStreamExtractor extractor; - - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (SoundcloudStreamExtractor) SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon"); - extractor.fetchPage(); - } - - @Test - public void testGetInvalidTimeStamp() throws ParsingException { - assertTrue(extractor.getTimeStamp() + "", - extractor.getTimeStamp() <= 0); - } - - @Test - public void testGetValidTimeStamp() throws IOException, ExtractionException { - StreamExtractor extractor = SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon#t=69"); - assertEquals("69", extractor.getTimeStamp() + ""); - } - - @Test - public void testGetTitle() throws ParsingException { - assertEquals("Do What I Want [Produced By Maaly Raw + Don Cannon]", extractor.getName()); - } - - @Test - public void testGetDescription() throws ParsingException { - assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent()); - } - - @Test - public void testGetUploaderName() throws ParsingException { - assertEquals("Lil Uzi Vert", extractor.getUploaderName()); - } - - @Test - public void testGetLength() throws ParsingException { - assertEquals(175, extractor.getLength()); - } - - @Test - public void testGetViewCount() throws ParsingException { - assertTrue(Long.toString(extractor.getViewCount()), - extractor.getViewCount() > 44227978); - } - - @Test - public void testGetTextualUploadDate() throws ParsingException { - Assert.assertEquals("2016-07-31 18:18:07", extractor.getTextualUploadDate()); - } - - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000"); - sdf.setTimeZone(TimeZone.getTimeZone("GMT")); - instance.setTime(sdf.parse("2016/07/31 18:18:07 +0000")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); - } - - @Test - public void testGetUploaderUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://soundcloud.com/liluzivert", extractor.getUploaderUrl()); - } - - @Test - public void testGetThumbnailUrl() throws ParsingException { - assertIsSecureUrl(extractor.getThumbnailUrl()); - } - - @Test - public void testGetUploaderAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); - } - - @Test - public void testGetAudioStreams() throws IOException, ExtractionException { - assertFalse(extractor.getAudioStreams().isEmpty()); - } - - @Test - public void testStreamType() throws ParsingException { - assertTrue(extractor.getStreamType() == StreamType.AUDIO_STREAM); - } - - @Test - public void testGetRelatedVideos() throws ExtractionException, IOException { - StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); - assertFalse(relatedVideos.getItems().isEmpty()); - assertTrue(relatedVideos.getErrors().isEmpty()); - } - - @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitlesDefault().isEmpty()); - } - - @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitlesDefault().isEmpty()); - } - } - - public static class ContentNotSupported { - @BeforeClass - public static void setUp() { - NewPipe.init(DownloaderTestImpl.getInstance()); - } - - @Test(expected = ContentNotSupportedException.class) - public void hlsAudioStream() throws Exception { - final StreamExtractor extractor = - SoundCloud.getStreamExtractor("https://soundcloud.com/dualipa/cool"); - extractor.fetchPage(); - extractor.getAudioStreams(); - } - - @Test(expected = ContentNotSupportedException.class) - public void bothHlsAndOpusAudioStreams() throws Exception { - final StreamExtractor extractor = - SoundCloud.getStreamExtractor("https://soundcloud.com/lil-baby-4pf/no-sucker"); - extractor.fetchPage(); - extractor.getAudioStreams(); - } - } -} - diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java new file mode 100644 index 00000000..28c8d0b2 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -0,0 +1,81 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; +import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +import java.util.Arrays; +import java.util.List; + +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; + +public class SoundcloudStreamExtractorTest { + + public static class LilUziVertDoWhatIWant extends DefaultStreamExtractorTest { + private static final String ID = "do-what-i-want-produced-by-maaly-raw-don-cannon"; + private static final String UPLOADER = "https://soundcloud.com/liluzivert"; + private static final int TIMESTAMP = 69; + private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP; + private static StreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = SoundCloud.getStreamExtractor(URL); + extractor.fetchPage(); + } + + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return SoundCloud; } + @Override public String expectedName() { return "Do What I Want [Produced By Maaly Raw + Don Cannon]"; } + @Override public String expectedId() { return "276206960"; } + @Override public String expectedUrlContains() { return UPLOADER + "/" + ID; } + @Override public String expectedOriginalUrlContains() { return URL; } + + @Override public StreamType expectedStreamType() { return StreamType.AUDIO_STREAM; } + @Override public String expectedUploaderName() { return "Lil Uzi Vert"; } + @Override public String expectedUploaderUrl() { return UPLOADER; } + @Override public List expectedDescriptionContains() { return Arrays.asList("The Perfect LUV Tape®"); } + @Override public long expectedLength() { return 175; } + @Override public long expectedTimestamp() { return TIMESTAMP; } + @Override public long expectedViewCountAtLeast() { return 75413600; } + @Nullable @Override public String expectedUploadDate() { return "2016-07-31 18:18:07.000"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2016-07-31 18:18:07"; } + @Override public long expectedLikeCountAtLeast() { return -1; } + @Override public long expectedDislikeCountAtLeast() { return -1; } + @Override public boolean expectedHasVideoStreams() { return false; } + @Override public boolean expectedHasSubtitles() { return false; } + @Override public boolean expectedHasFrames() { return false; } + } + + public static class ContentNotSupported { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = ContentNotSupportedException.class) + public void hlsAudioStream() throws Exception { + final StreamExtractor extractor = + SoundCloud.getStreamExtractor("https://soundcloud.com/dualipa/cool"); + extractor.fetchPage(); + extractor.getAudioStreams(); + } + + @Test(expected = ContentNotSupportedException.class) + public void bothHlsAndOpusAudioStreams() throws Exception { + final StreamExtractor extractor = + SoundCloud.getStreamExtractor("https://soundcloud.com/lil-baby-4pf/no-sucker"); + extractor.fetchPage(); + extractor.getAudioStreams(); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index 95c8aeb4..6844ca2e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -1,144 +1,53 @@ package org.schabi.newpipe.extractor.services.youtube.stream; import org.junit.BeforeClass; -import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.stream.StreamType; -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; +import java.util.Arrays; import java.util.List; -import static java.util.Objects.requireNonNull; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import javax.annotation.Nullable; + import static org.schabi.newpipe.extractor.ServiceList.YouTube; -/** - * Test for {@link YoutubeStreamLinkHandlerFactory} - */ -public class YoutubeStreamExtractorAgeRestrictedTest { - public static final String HTTPS = "https://"; - private static YoutubeStreamExtractor extractor; +public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtractorTest { + private static final String ID = "MmBeUZqv1QA"; + private static final int TIMESTAMP = 196; + private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID + "&t=" + TIMESTAMP; + private static StreamExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=MmBeUZqv1QA"); + extractor = YouTube.getStreamExtractor(URL); extractor.fetchPage(); } - @Test - public void testGetInvalidTimeStamp() throws ParsingException { - assertTrue(extractor.getTimeStamp() + "", extractor.getTimeStamp() <= 0); - } + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return "FINGERING PORNSTARS @ AVN Expo 2017 In Las Vegas!"; } + @Override public String expectedId() { return ID; } + @Override public String expectedUrlContains() { return YoutubeStreamExtractorDefaultTest.BASE_URL + ID; } + @Override public String expectedOriginalUrlContains() { return URL; } - @Test - public void testGetValidTimeStamp() throws IOException, ExtractionException { - StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174"); - assertEquals(extractor.getTimeStamp() + "", "174"); - extractor = YouTube.getStreamExtractor("https://youtube.com/embed/FmG385_uUys?start=174"); - assertEquals(extractor.getTimeStamp() + "", "174"); - } - - @Test - public void testGetAgeLimit() throws ParsingException { - assertEquals(18, extractor.getAgeLimit()); - } - - @Test - public void testGetName() throws ParsingException { - assertNotNull("name is null", extractor.getName()); - assertFalse("name is empty", extractor.getName().isEmpty()); - } - - @Test - public void testGetDescription() throws ParsingException { - assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().getContent().isEmpty()); - } - - @Test - public void testGetUploaderName() throws ParsingException { - assertNotNull(extractor.getUploaderName()); - assertFalse(extractor.getUploaderName().isEmpty()); - } - - @Test - public void testGetLength() throws ParsingException { - assertEquals(1790, extractor.getLength()); - } - - @Test - public void testGetViews() throws ParsingException { - assertTrue(extractor.getViewCount() > 0); - } - - @Test - public void testGetTextualUploadDate() throws ParsingException { - assertEquals("2017-01-25", extractor.getTextualUploadDate()); - } - - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2017-01-25")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); - } - - @Test - public void testGetThumbnailUrl() throws ParsingException { - assertIsSecureUrl(extractor.getThumbnailUrl()); - } - - @Test - public void testGetUploaderAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); - } - - @Test - public void testGetAudioStreams() throws IOException, ExtractionException { - // audio streams are not always necessary - assertFalse(extractor.getAudioStreams().isEmpty()); - } - - @Test - public void testGetVideoStreams() throws IOException, ExtractionException { - List streams = new ArrayList<>(); - streams.addAll(extractor.getVideoStreams()); - streams.addAll(extractor.getVideoOnlyStreams()); - - assertTrue(Integer.toString(streams.size()), streams.size() > 0); - for (VideoStream s : streams) { - assertTrue(s.getUrl(), - s.getUrl().contains(HTTPS)); - assertTrue(s.resolution.length() > 0); - assertTrue(Integer.toString(s.getFormatId()), - 0 <= s.getFormatId() && s.getFormatId() <= 0x100); - } - } - - - @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitlesDefault().isEmpty()); - } - - @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); - } + @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } + @Override public String expectedUploaderName() { return "EpicFiveTV"; } + @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCuPUHlLP5POZphOIrjrNxiw"; } + @Override public List expectedDescriptionContains() { return Arrays.asList("http://instagram.com/Ruben_Sole", "AVN"); } + @Override public long expectedLength() { return 1790; } + @Override public long expectedTimestamp() { return TIMESTAMP; } + @Override public long expectedViewCountAtLeast() { return 28500000; } + @Nullable @Override public String expectedUploadDate() { return "2017-01-25 00:00:00.000"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2017-01-25"; } + @Override public long expectedLikeCountAtLeast() { return 149000; } + @Override public long expectedDislikeCountAtLeast() { return 38000; } + @Override public boolean expectedHasRelatedStreams() { return false; } // no related videos (!) + @Override public int expectedAgeLimit() { return 18; } + @Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; } + @Override public boolean expectedHasSubtitles() { return false; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java index 915f42ae..a090460a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java @@ -1,134 +1,54 @@ package org.schabi.newpipe.extractor.services.youtube.stream; import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.stream.StreamType; -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; +import java.util.Arrays; import java.util.List; -import static java.util.Objects.requireNonNull; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import javax.annotation.Nullable; + import static org.schabi.newpipe.extractor.ServiceList.YouTube; /** * Test for {@link YoutubeStreamLinkHandlerFactory} */ -public class YoutubeStreamExtractorControversialTest { - private static YoutubeStreamExtractor extractor; +public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtractorTest { + private static final String ID = "T4XJQO3qol8"; + private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID; + private static StreamExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=T4XJQO3qol8"); + extractor = YouTube.getStreamExtractor(URL); extractor.fetchPage(); } - @Test - public void testGetInvalidTimeStamp() throws ParsingException { - assertTrue(extractor.getTimeStamp() + "", extractor.getTimeStamp() <= 0); - } + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return "Burning Everyone's Koran"; } + @Override public String expectedId() { return ID; } + @Override public String expectedUrlContains() { return URL; } + @Override public String expectedOriginalUrlContains() { return URL; } - @Test - public void testGetValidTimeStamp() throws IOException, ExtractionException { - StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174"); - assertEquals(extractor.getTimeStamp() + "", "174"); - } - - @Test - @Ignore - public void testGetAgeLimit() throws ParsingException { - assertEquals(18, extractor.getAgeLimit()); - } - - @Test - public void testGetName() throws ParsingException { - assertNotNull("name is null", extractor.getName()); - assertFalse("name is empty", extractor.getName().isEmpty()); - } - - @Test - public void testGetDescription() throws ParsingException { - assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().getContent().isEmpty()); - } - - @Test - public void testGetUploaderName() throws ParsingException { - assertNotNull(extractor.getUploaderName()); - assertFalse(extractor.getUploaderName().isEmpty()); - } - - @Test - public void testGetLength() throws ParsingException { - assertEquals(219, extractor.getLength()); - } - - @Test - public void testGetViews() throws ParsingException { - assertTrue(extractor.getViewCount() > 0); - } - - @Test - public void testGetTextualUploadDate() throws ParsingException { - assertEquals("2010-09-09", extractor.getTextualUploadDate()); - } - - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2010-09-09")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); - } - - @Test - public void testGetThumbnailUrl() throws ParsingException { - assertIsSecureUrl(extractor.getThumbnailUrl()); - } - - @Test - public void testGetUploaderAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); - } - - @Test - public void testGetAudioStreams() throws IOException, ExtractionException { - // audio streams are not always necessary - assertFalse(extractor.getAudioStreams().isEmpty()); - } - - @Test - public void testGetVideoStreams() throws IOException, ExtractionException { - List streams = new ArrayList<>(); - streams.addAll(extractor.getVideoStreams()); - streams.addAll(extractor.getVideoOnlyStreams()); - assertTrue(streams.size() > 0); - } - - @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - // Video (/view?v=T4XJQO3qol8) set in the setUp() method has at least auto-generated (English) captions - assertFalse(extractor.getSubtitlesDefault().isEmpty()); - } - - @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { - // Video (/view?v=T4XJQO3qol8) set in the setUp() method has at least auto-generated (English) captions - assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); + @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } + @Override public String expectedUploaderName() { return "Amazing Atheist"; } + @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCjNxszyFPasDdRoD9J6X-sw"; } + @Override public List expectedDescriptionContains() { + return Arrays.asList("http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html", + "freedom"); } + @Override public long expectedLength() { return 219; } + @Override public long expectedViewCountAtLeast() { return 285000; } + @Nullable @Override public String expectedUploadDate() { return "2010-09-09 00:00:00.000"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2010-09-09"; } + @Override public long expectedLikeCountAtLeast() { return 13300; } + @Override public long expectedDislikeCountAtLeast() { return 2600; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 0d36fd9a..49369831 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -1,35 +1,22 @@ package org.schabi.newpipe.extractor.services.youtube.stream; -import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.ExtractorAsserts; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.stream.Frameset; +import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.VideoStream; -import org.schabi.newpipe.extractor.utils.Utils; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; +import java.util.Arrays; import java.util.List; -import static java.util.Objects.requireNonNull; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import javax.annotation.Nullable; + import static org.junit.Assert.fail; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; /* @@ -51,11 +38,8 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube; * You should have received a copy of the GNU General Public License * along with NewPipe. If not, see . */ - -/** - * Test for {@link StreamExtractor} - */ public class YoutubeStreamExtractorDefaultTest { + static final String BASE_URL = "https://www.youtube.com/watch?v="; public static class NotAvailable { @BeforeClass @@ -66,266 +50,163 @@ public class YoutubeStreamExtractorDefaultTest { @Test(expected = ContentNotAvailableException.class) public void nonExistentFetch() throws Exception { final StreamExtractor extractor = - YouTube.getStreamExtractor("https://www.youtube.com/watch?v=don-t-exist"); + YouTube.getStreamExtractor(BASE_URL + "don-t-exist"); extractor.fetchPage(); } @Test(expected = ParsingException.class) public void invalidId() throws Exception { final StreamExtractor extractor = - YouTube.getStreamExtractor("https://www.youtube.com/watch?v=INVALID_ID_INVALID_ID"); + YouTube.getStreamExtractor(BASE_URL + "INVALID_ID_INVALID_ID"); extractor.fetchPage(); } } - /** - * Test for {@link StreamExtractor} - */ - public static class AdeleHello { - private static YoutubeStreamExtractor extractor; + public static class AdeleHello extends DefaultStreamExtractorTest { + private static final String ID = "YQHsXMglC9A"; + private static final String URL = BASE_URL + ID; + private static StreamExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A"); + extractor = YouTube.getStreamExtractor(URL); extractor.fetchPage(); } @Test - public void testGetInvalidTimeStamp() throws ParsingException { - assertTrue(extractor.getTimeStamp() + "", - extractor.getTimeStamp() <= 0); - } - - @Test - public void testGetValidTimeStamp() throws ExtractionException { - StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174"); - assertEquals(extractor.getTimeStamp() + "", "174"); - } - - @Test - public void testGetTitle() throws ParsingException { - assertFalse(extractor.getName().isEmpty()); - } - - @Test - public void testGetDescription() throws ParsingException { - assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().getContent().isEmpty()); - } - - @Test - public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().getContent().contains("http://adele.com")); - } - - @Test - public void testGetUploaderName() throws ParsingException { - assertNotNull(extractor.getUploaderName()); - assertFalse(extractor.getUploaderName().isEmpty()); - } - - - @Test - public void testGetLength() throws ParsingException { - assertEquals(367, extractor.getLength()); - } - - @Test - public void testGetViewCount() throws ParsingException { - Long count = extractor.getViewCount(); - assertTrue(Long.toString(count), count >= /* specific to that video */ 1220025784); - } - - @Test - public void testGetTextualUploadDate() throws ParsingException { - Assert.assertEquals("2015-10-22", extractor.getTextualUploadDate()); - } - - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2015-10-22")); - assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); - } - - @Test - public void testGetUploaderUrl() throws ParsingException { - String url = extractor.getUploaderUrl(); + @Override + public void testUploaderUrl() throws ParsingException { + String url = extractor().getUploaderUrl(); if (!url.equals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw") && !url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) { fail("Uploader url is neither the music channel one nor the Vevo one"); } } - @Test - public void testGetThumbnailUrl() throws ParsingException { - assertIsSecureUrl(extractor.getThumbnailUrl()); - } + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return "Adele - Hello"; } + @Override public String expectedId() { return ID; } + @Override public String expectedUrlContains() { return URL; } + @Override public String expectedOriginalUrlContains() { return URL; } - @Test - public void testGetUploaderAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); - } - - @Test - public void testGetAudioStreams() throws ExtractionException { - assertFalse(extractor.getAudioStreams().isEmpty()); - } - - @Test - public void testGetVideoStreams() throws ExtractionException { - for (VideoStream s : extractor.getVideoStreams()) { - assertIsSecureUrl(s.url); - assertTrue(s.resolution.length() > 0); - assertTrue(Integer.toString(s.getFormatId()), - 0 <= s.getFormatId() && s.getFormatId() <= 0x100); - } - } - - @Test - public void testStreamType() throws ParsingException { - assertTrue(extractor.getStreamType() == StreamType.VIDEO_STREAM); - } - - @Test - public void testGetDashMpd() throws ParsingException { - // we dont expect this particular video to have a DASH file. For this purpouse we use a different test class. - assertTrue(extractor.getDashMpdUrl(), - extractor.getDashMpdUrl() != null && extractor.getDashMpdUrl().isEmpty()); - } - - @Test - public void testGetRelatedVideos() throws ExtractionException { - StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); - Utils.printErrors(relatedVideos.getErrors()); - assertFalse(relatedVideos.getItems().isEmpty()); - assertTrue(relatedVideos.getErrors().isEmpty()); - } - - @Test - public void testGetSubtitlesListDefault() { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitlesDefault().isEmpty()); - } - - @Test - public void testGetSubtitlesList() { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); - } - - @Test - public void testGetLikeCount() throws ParsingException { - long likeCount = extractor.getLikeCount(); - assertTrue("" + likeCount, likeCount >= 15000000); - } - - @Test - public void testGetDislikeCount() throws ParsingException { - long dislikeCount = extractor.getDislikeCount(); - assertTrue("" + dislikeCount, dislikeCount >= 818000); - } + @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } + @Override public String expectedUploaderName() { return "Adele"; } + @Override public String expectedUploaderUrl() { return null; } // overridden above + @Override public List expectedDescriptionContains() { return Arrays.asList("http://adele.com", "https://www.facebook.com/Adele"); } + @Override public long expectedLength() { return 367; } + @Override public long expectedViewCountAtLeast() { return 1220025784; } + @Nullable @Override public String expectedUploadDate() { return "2015-10-22 00:00:00.000"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2015-10-22"; } + @Override public long expectedLikeCountAtLeast() { return 15289000; } + @Override public long expectedDislikeCountAtLeast() { return 826000; } + @Override public boolean expectedHasSubtitles() { return false; } } - public static class DescriptionTestPewdiepie { - private static YoutubeStreamExtractor extractor; + public static class DescriptionTestPewdiepie extends DefaultStreamExtractorTest { + private static final String ID = "fBc4Q_htqPg"; + private static final int TIMESTAMP = 17; + private static final String URL = BASE_URL + ID + "&t=" + TIMESTAMP; + private static StreamExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=fBc4Q_htqPg"); + extractor = YouTube.getStreamExtractor(URL); extractor.fetchPage(); } - @Test - public void testGetDescription() throws ParsingException { - assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().getContent().isEmpty()); - } + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return "Dr. Phil DESTROYS spoiled brat!!!! . -- Dr Phil #7"; } + @Override public String expectedId() { return ID; } + @Override public String expectedUrlContains() { return BASE_URL + ID; } + @Override public String expectedOriginalUrlContains() { return URL; } - @Test - public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); - assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); - assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); + @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } + @Override public String expectedUploaderName() { return "PewDiePie"; } + @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw"; } + @Override public List expectedDescriptionContains() { + return Arrays.asList("https://www.reddit.com/r/PewdiepieSubmissions/", + "https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng", + "https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/"); } + @Override public long expectedLength() { return 1165; } + @Override public long expectedTimestamp() { return TIMESTAMP; } + @Override public long expectedViewCountAtLeast() { return 26682500; } + @Nullable @Override public String expectedUploadDate() { return "2018-09-12 00:00:00.000"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2018-09-12"; } + @Override public long expectedLikeCountAtLeast() { return 1166000; } + @Override public long expectedDislikeCountAtLeast() { return 16900; } } - public static class DescriptionTestUnboxing { - private static YoutubeStreamExtractor extractor; + public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest { + private static final String ID = "cV5TjZCJkuA"; + private static final String URL = BASE_URL + ID; + private static StreamExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=cV5TjZCJkuA"); + extractor = YouTube.getStreamExtractor(URL); extractor.fetchPage(); } - @Test - public void testGetDescription() throws ParsingException { - assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().getContent().isEmpty()); - } + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return "This Smartphone Changes Everything..."; } + @Override public String expectedId() { return ID; } + @Override public String expectedUrlContains() { return URL; } + @Override public String expectedOriginalUrlContains() { return URL; } - @Test - public void testGetFullLinksInDescription() throws ParsingException { - final String description = extractor.getDescription().getContent(); - assertTrue(description.contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(description.contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(description.contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(description.contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } + @Override public String expectedUploaderName() { return "Unbox Therapy"; } + @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCsTcErHg8oDvUnTzoqsYeNw"; } + @Override public List expectedDescriptionContains() { + return Arrays.asList("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34", + "https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34", + "https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34", + "https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"); } + @Override public long expectedLength() { return 434; } + @Override public long expectedViewCountAtLeast() { return 21229200; } + @Nullable @Override public String expectedUploadDate() { return "2018-06-19 00:00:00.000"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2018-06-19"; } + @Override public long expectedLikeCountAtLeast() { return 340100; } + @Override public long expectedDislikeCountAtLeast() { return 18700; } } - public static class RatingsDisabledTest { - private static YoutubeStreamExtractor extractor; + public static class RatingsDisabledTest extends DefaultStreamExtractorTest { + private static final String ID = "HRKu0cvrr_o"; + private static final int TIMESTAMP = 17; + private static final String URL = BASE_URL + ID + "&t=" + TIMESTAMP; + private static StreamExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=HRKu0cvrr_o"); + extractor = YouTube.getStreamExtractor(URL); extractor.fetchPage(); } - @Test - public void testGetLikeCount() throws ParsingException { - assertEquals(-1, extractor.getLikeCount()); - } + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return "AlphaOmegaSin Fanboy Logic: Likes/Dislikes Disabled = Point Invalid Lol wtf?"; } + @Override public String expectedId() { return ID; } + @Override public String expectedUrlContains() { return BASE_URL + ID; } + @Override public String expectedOriginalUrlContains() { return URL; } - @Test - public void testGetDislikeCount() throws ParsingException { - assertEquals(-1, extractor.getDislikeCount()); - } - - } - - public static class FramesTest { - private static YoutubeStreamExtractor extractor; - - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=HoK9shIJ2xQ"); - extractor.fetchPage(); - } - - @Test - public void testGetFrames() throws ExtractionException { - final List frames = extractor.getFrames(); - assertNotNull(frames); - assertFalse(frames.isEmpty()); - for (final Frameset f : frames) { - for (final String url : f.getUrls()) { - ExtractorAsserts.assertIsValidUrl(url); - ExtractorAsserts.assertIsSecureUrl(url); - } - } - } + @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } + @Override public String expectedUploaderName() { return "YouTuber PrinceOfFALLEN"; } + @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCQT2yul0lr6Ie9qNQNmw-sg"; } + @Override public List expectedDescriptionContains() { return Arrays.asList("dislikes", "Alpha", "wrong"); } + @Override public long expectedLength() { return 84; } + @Override public long expectedTimestamp() { return TIMESTAMP; } + @Override public long expectedViewCountAtLeast() { return 190; } + @Nullable @Override public String expectedUploadDate() { return "2019-01-02 00:00:00.000"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2019-01-02"; } + @Override public long expectedLikeCountAtLeast() { return -1; } + @Override public long expectedDislikeCountAtLeast() { return -1; } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index 0e17fe8d..6a675487 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -1,139 +1,54 @@ package org.schabi.newpipe.extractor.services.youtube.stream; import org.junit.BeforeClass; -import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; +import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.VideoStream; -import org.schabi.newpipe.extractor.utils.Utils; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import java.util.Arrays; +import java.util.List; + +import javax.annotation.Nullable; + import static org.schabi.newpipe.extractor.ServiceList.YouTube; -public class YoutubeStreamExtractorLivestreamTest { - private static YoutubeStreamExtractor extractor; +public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractorTest { + private static final String ID = "5qap5aO4i9A"; + private static final int TIMESTAMP = 1737; + private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID + "&t=" + TIMESTAMP; + private static StreamExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=5qap5aO4i9A"); + extractor = YouTube.getStreamExtractor(URL); extractor.fetchPage(); } - @Test - public void testGetInvalidTimeStamp() throws ParsingException { - assertTrue(extractor.getTimeStamp() + "", - extractor.getTimeStamp() <= 0); - } + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return "lofi hip hop radio - beats to relax/study to"; } + @Override public String expectedId() { return ID; } + @Override public String expectedUrlContains() { return YoutubeStreamExtractorDefaultTest.BASE_URL + ID; } + @Override public String expectedOriginalUrlContains() { return URL; } - @Test - public void testGetTitle() throws ParsingException { - assertFalse(extractor.getName().isEmpty()); - } - - @Test - public void testGetDescription() throws ParsingException { - assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().getContent().isEmpty()); - } - - @Test - public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().getContent().contains("https://bit.ly/chilledcow-playlists")); - } - - @Test - public void testGetUploaderName() throws ParsingException { - assertNotNull(extractor.getUploaderName()); - assertFalse(extractor.getUploaderName().isEmpty()); - } - - - @Test - public void testGetLength() throws ParsingException { - assertEquals(0, extractor.getLength()); - } - - @Test - public void testGetViewCount() throws ParsingException { - long count = extractor.getViewCount(); - assertTrue(Long.toString(count), count > -1); - } - - @Test - public void testGetUploadDate() throws ParsingException { - assertNull(extractor.getUploadDate()); - assertNull(extractor.getTextualUploadDate()); - } - - @Test - public void testGetUploaderUrl() throws ParsingException { - assertEquals("https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow", extractor.getUploaderUrl()); - } - - @Test - public void testGetThumbnailUrl() throws ParsingException { - assertIsSecureUrl(extractor.getThumbnailUrl()); - } - - @Test - public void testGetUploaderAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); - } - - @Test - public void testGetAudioStreams() throws ExtractionException { - assertFalse(extractor.getAudioStreams().isEmpty()); - } - - @Test - public void testGetVideoStreams() throws ExtractionException { - for (VideoStream s : extractor.getVideoStreams()) { - assertIsSecureUrl(s.url); - assertTrue(s.resolution.length() > 0); - assertTrue(Integer.toString(s.getFormatId()), - 0 <= s.getFormatId() && s.getFormatId() <= 0x100); - } - } - - @Test - public void testStreamType() throws ParsingException { - assertSame(extractor.getStreamType(), StreamType.LIVE_STREAM); - } - - @Test - public void testGetDashMpd() throws ParsingException { - assertTrue(extractor.getDashMpdUrl().startsWith("https://manifest.googlevideo.com/api/manifest/dash/")); - } - - @Test - public void testGetRelatedVideos() throws ExtractionException { - StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); - Utils.printErrors(relatedVideos.getErrors()); - assertFalse(relatedVideos.getItems().isEmpty()); - assertTrue(relatedVideos.getErrors().isEmpty()); - } - - @Test - public void testGetSubtitlesListDefault() { - assertTrue(extractor.getSubtitlesDefault().isEmpty()); - } - - @Test - public void testGetSubtitlesList() { - assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); + @Override public StreamType expectedStreamType() { return StreamType.LIVE_STREAM; } + @Override public String expectedUploaderName() { return "ChilledCow"; } + @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow"; } + @Override public List expectedDescriptionContains() { + return Arrays.asList("https://bit.ly/chilledcow-playlists", + "https://bit.ly/chilledcow-submissions"); } + @Override public long expectedLength() { return 0; } + @Override public long expectedTimestamp() { return TIMESTAMP; } + @Override public long expectedViewCountAtLeast() { return 0; } + @Nullable @Override public String expectedUploadDate() { return null; } + @Nullable @Override public String expectedTextualUploadDate() { return null; } + @Override public long expectedLikeCountAtLeast() { return 825000; } + @Override public long expectedDislikeCountAtLeast() { return 15600; } + @Override public boolean expectedHasSubtitles() { return false; } + @Override public boolean expectedHasFrames() { return false; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java index 49a851f4..22b537dd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java @@ -1,161 +1,50 @@ package org.schabi.newpipe.extractor.services.youtube.stream; -import org.junit.Assert; import org.junit.BeforeClass; -import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; +import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.VideoStream; -import org.schabi.newpipe.extractor.utils.Utils; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; +import java.util.Arrays; import java.util.List; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import javax.annotation.Nullable; + import static org.schabi.newpipe.extractor.ServiceList.YouTube; -public class YoutubeStreamExtractorUnlistedTest { - private static YoutubeStreamExtractor extractor; +public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTest { + static final String ID = "udsB8KnIJTg"; + static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID; + private static StreamExtractor extractor; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=udsB8KnIJTg"); + extractor = YouTube.getStreamExtractor(URL); extractor.fetchPage(); } - @Test - public void testGetInvalidTimeStamp() throws ParsingException { - assertTrue(extractor.getTimeStamp() + "", - extractor.getTimeStamp() <= 0); - } + @Override public StreamExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return "Praise the Casual: Ein Neuling trifft Dark Souls - Folge 5"; } + @Override public String expectedId() { return ID; } + @Override public String expectedUrlContains() { return URL; } + @Override public String expectedOriginalUrlContains() { return URL; } - @Test - public void testGetTitle() throws ParsingException { - assertFalse(extractor.getName().isEmpty()); - } - - @Test - public void testGetDescription() throws ParsingException { - assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().getContent().isEmpty()); - } - - @Test - public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/user/Roccowschiptune")); - } - - @Test - public void testGetUploaderName() throws ParsingException { - assertNotNull(extractor.getUploaderName()); - assertFalse(extractor.getUploaderName().isEmpty()); - } - - - @Test - public void testGetLength() throws ParsingException { - assertEquals(2488, extractor.getLength()); - } - - @Test - public void testGetViewCount() throws ParsingException { - long count = extractor.getViewCount(); - assertTrue(Long.toString(count), count >= /* specific to that video */ 1225); - } - - @Test - public void testGetTextualUploadDate() throws ParsingException { - Assert.assertEquals("2017-09-22", extractor.getTextualUploadDate()); - } - - @Test - public void testGetUploadDate() throws ParsingException, ParseException { - final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2017-09-22")); - assertNotNull(extractor.getUploadDate()); - assertEquals(instance, extractor.getUploadDate().date()); - } - - @Test - public void testGetUploaderUrl() throws ParsingException { - assertEquals("https://www.youtube.com/channel/UCPysfiuOv4VKBeXFFPhKXyw", extractor.getUploaderUrl()); - } - - @Test - public void testGetThumbnailUrl() throws ParsingException { - assertIsSecureUrl(extractor.getThumbnailUrl()); - } - - @Test - public void testGetUploaderAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); - } - - @Test - public void testGetAudioStreams() throws ExtractionException { - List audioStreams = extractor.getAudioStreams(); - assertFalse(audioStreams.isEmpty()); - for (AudioStream s : audioStreams) { - assertIsSecureUrl(s.url); - assertTrue(Integer.toString(s.getFormatId()), - 0x100 <= s.getFormatId() && s.getFormatId() < 0x1000); - } - } - - @Test - public void testGetVideoStreams() throws ExtractionException { - for (VideoStream s : extractor.getVideoStreams()) { - assertIsSecureUrl(s.url); - assertTrue(s.resolution.length() > 0); - assertTrue(Integer.toString(s.getFormatId()), - 0 <= s.getFormatId() && s.getFormatId() < 0x100); - } - } - - @Test - public void testStreamType() throws ParsingException { - assertSame(StreamType.VIDEO_STREAM, extractor.getStreamType()); - } - - @Test - public void testGetRelatedVideos() throws ExtractionException { - StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); - Utils.printErrors(relatedVideos.getErrors()); - assertFalse(relatedVideos.getItems().isEmpty()); - assertTrue(relatedVideos.getErrors().isEmpty()); - } - - @Test - public void testGetSubtitlesListDefault() { - assertFalse(extractor.getSubtitlesDefault().isEmpty()); - } - - @Test - public void testGetSubtitlesList() { - assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); - } - - @Test - public void testGetLikeCount() throws ParsingException { - long likeCount = extractor.getLikeCount(); - assertTrue("" + likeCount, likeCount >= 96); - } - - @Test - public void testGetDislikeCount() throws ParsingException { - long dislikeCount = extractor.getDislikeCount(); - assertTrue("" + dislikeCount, dislikeCount >= 0); + @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } + @Override public String expectedUploaderName() { return "Hooked"; } + @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCPysfiuOv4VKBeXFFPhKXyw"; } + @Override public List expectedDescriptionContains() { + return Arrays.asList("https://www.youtube.com/user/Roccowschiptune", + "https://www.facebook.com/HookedMagazinDE"); } + @Override public long expectedLength() { return 2488; } + @Override public long expectedViewCountAtLeast() { return 1500; } + @Nullable @Override public String expectedUploadDate() { return "2017-09-22 00:00:00.000"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2017-09-22"; } + @Override public long expectedLikeCountAtLeast() { return 110; } + @Override public long expectedDislikeCountAtLeast() { return 0; } } From 7fb867c1662457c51d6579f2803c913f356126bf Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 9 Apr 2020 14:11:42 +0200 Subject: [PATCH 24/81] [YouTube] Fix error message obtaining when there is none --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 8364246c..08e43d6f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -606,7 +606,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus") .getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason")); - } catch (ParsingException e) { + } catch (Exception e) { return null; } } From 7cd410f3fc304a9eb620fa28ba39f938e4565c2b Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 9 Apr 2020 14:45:33 +0200 Subject: [PATCH 25/81] [YouTube] Return 0 when there is no timestamp, not -2, as per javadoc --- .../youtube/extractors/YoutubeStreamExtractor.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 08e43d6f..bdaf4cad 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -253,7 +253,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { */ @Override public long getTimeStamp() throws ParsingException { - return getTimestampSeconds("((#|&|\\?)(t|start)=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); + long timestamp = getTimestampSeconds("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); + + if (timestamp == -2) { + // regex for timestamp was not found + return 0; + } else { + return timestamp; + } } @Override From 072bae321fdf9cca340d154d02b388623fadcc8a Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 9 Apr 2020 15:52:42 +0200 Subject: [PATCH 26/81] [YouTube] Fix frame extraction for livestreams Use saved playerResponse instead of parsing json every time --- .../youtube/extractors/YoutubeStreamExtractor.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index bdaf4cad..b2b5e4c4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1006,12 +1006,18 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public List getFrames() throws ExtractionException { try { - JsonObject jo = initialAjaxJson.getObject(2).getObject("player"); - final String resp = jo.getObject("args").getString("player_response"); - jo = JsonParser.object().from(resp); - final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|"); + final JsonObject storyboards = playerResponse.getObject("storyboards"); + final JsonObject storyboardsRenderer; + if (storyboards.has("playerLiveStoryboardSpecRenderer")) { + storyboardsRenderer = storyboards.getObject("playerLiveStoryboardSpecRenderer"); + } else { + storyboardsRenderer = storyboards.getObject("playerStoryboardSpecRenderer"); + } + + final String[] spec = storyboardsRenderer.getString("spec").split("\\|"); final String url = spec[0]; final ArrayList result = new ArrayList<>(spec.length - 1); + for (int i = 1; i < spec.length; ++i) { final String[] parts = spec[i].split("#"); if (parts.length != 8) { From 3b2cfb4ca2e9fbe9457e2a4551d97f7fa8bc6fec Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 9 Apr 2020 18:15:34 +0200 Subject: [PATCH 27/81] [SoundCloud] Return empty video stream list instead of null Also replace every instance of `return new ArrayList<>();` with `return Collections.emptyList();` --- .../media_ccc/extractors/MediaCCCStreamExtractor.java | 4 ++-- .../soundcloud/extractors/SoundcloudStreamExtractor.java | 6 +++--- .../services/youtube/extractors/YoutubeStreamExtractor.java | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 042c5cd1..18aa9708 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -277,8 +277,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public List getTags() { - return new ArrayList<>(); + public List getTags() throws ParsingException { + return Collections.emptyList(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index d909acfc..0eaf1d9d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -235,12 +235,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public List getVideoStreams() throws IOException, ExtractionException { - return null; + return Collections.emptyList(); } @Override public List getVideoOnlyStreams() throws IOException, ExtractionException { - return null; + return Collections.emptyList(); } @Override @@ -304,7 +304,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override public List getTags() throws ParsingException { - return new ArrayList<>(); + return Collections.emptyList(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index b2b5e4c4..f49812fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1087,7 +1087,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public List getTags() { - return new ArrayList<>(); + return Collections.emptyList(); } @Nonnull From 4349be13af66aaf9af4c244d53cce1702b3a0782 Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 9 Apr 2020 18:18:43 +0200 Subject: [PATCH 28/81] [PeerTube] Return empty audio stream list instead of null --- .../extractors/PeertubeStreamExtractor.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 4fda7827..74d646bf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -41,7 +41,7 @@ import javax.annotation.Nonnull; public class PeertubeStreamExtractor extends StreamExtractor { private final String baseUrl; private JsonObject json; - private List subtitles = new ArrayList<>(); + private final List subtitles = new ArrayList<>(); public PeertubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) throws ParsingException { super(service, linkHandler); @@ -64,11 +64,13 @@ public class PeertubeStreamExtractor extends StreamExtractor { return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } + @Nonnull @Override public String getThumbnailUrl() throws ParsingException { return baseUrl + JsonUtils.getString(json, "previewPath"); } + @Nonnull @Override public Description getDescription() throws ParsingException { String text; @@ -127,6 +129,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { return json.getLong("dislikes"); } + @Nonnull @Override public String getUploaderUrl() throws ParsingException { final String name = JsonUtils.getString(json, "account.name"); @@ -134,11 +137,13 @@ public class PeertubeStreamExtractor extends StreamExtractor { return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } + @Nonnull @Override public String getUploaderName() throws ParsingException { return JsonUtils.getString(json, "account.displayName"); } + @Nonnull @Override public String getUploaderAvatarUrl() { String value; @@ -150,6 +155,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { return baseUrl + value; } + @Nonnull @Override public String getSubChannelUrl() throws ParsingException { return JsonUtils.getString(json, "channel.url"); @@ -173,11 +179,13 @@ public class PeertubeStreamExtractor extends StreamExtractor { return baseUrl + value; } + @Nonnull @Override public String getDashMpdUrl() { return ""; } + @Nonnull @Override public String getHlsUrl() { return ""; @@ -185,7 +193,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public List getAudioStreams() { - return null; + return Collections.emptyList(); } @Override @@ -220,11 +228,13 @@ public class PeertubeStreamExtractor extends StreamExtractor { return Collections.emptyList(); } + @Nonnull @Override public List getSubtitlesDefault() { return subtitles; } + @Nonnull @Override public List getSubtitles(final MediaFormat format) { final List filteredSubs = new ArrayList<>(); @@ -256,6 +266,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { return collector; } + @Nonnull @Override public List getTags() { try { @@ -370,31 +381,37 @@ public class PeertubeStreamExtractor extends StreamExtractor { } } + @Nonnull @Override public String getName() throws ParsingException { return JsonUtils.getString(json, "name"); } + @Nonnull @Override public String getOriginalUrl() throws ParsingException { return baseUrl + "/videos/watch/" + getId(); } + @Nonnull @Override public String getHost() throws ParsingException { return JsonUtils.getString(json, "account.host"); } + @Nonnull @Override public String getPrivacy() throws ParsingException { return JsonUtils.getString(json, "privacy.label"); } + @Nonnull @Override public String getCategory() throws ParsingException { return JsonUtils.getString(json, "category.label"); } + @Nonnull @Override public String getLicence() throws ParsingException { return JsonUtils.getString(json, "licence.label"); From 7ae3cb6d07cea6abb71d9c69eb2b1b276e5b795f Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 9 Apr 2020 19:46:20 +0200 Subject: [PATCH 29/81] [PeerTube] Fix link handler inconsistency providing API links --- .../extractors/PeertubeAccountExtractor.java | 17 ++++++++------- .../extractors/PeertubeChannelExtractor.java | 17 ++++++++------- .../extractors/PeertubeStreamExtractor.java | 21 ++++++++++--------- .../PeertubeChannelLinkHandlerFactory.java | 7 +++---- .../PeertubeStreamLinkHandlerFactory.java | 5 +++-- .../PeertubeAccountExtractorTest.java | 10 ++++----- .../PeertubeChannelExtractorTest.java | 8 +++---- .../peertube/PeertubeStreamExtractorTest.java | 4 ++-- 8 files changed, 46 insertions(+), 43 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index 49978e88..aaf645d8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; @@ -20,6 +21,8 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import javax.annotation.Nonnull; + import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; @@ -85,10 +88,11 @@ public class PeertubeAccountExtractor extends ChannelExtractor { return ""; } + @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - final String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; - return getPage(new Page(pageUrl)); + return getPage(new Page( + getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE)); } @Override @@ -123,7 +127,8 @@ public class PeertubeAccountExtractor extends ChannelExtractor { @Override public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException { - final Response response = downloader.get(getUrl()); + final Response response = downloader.get( + baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT + getId()); if (response != null && response.responseBody() != null) { setInitialData(response.responseBody()); } else { @@ -140,13 +145,9 @@ public class PeertubeAccountExtractor extends ChannelExtractor { if (json == null) throw new ExtractionException("Unable to extract PeerTube account data"); } + @Nonnull @Override public String getName() throws ParsingException { return JsonUtils.getString(json, "displayName"); } - - @Override - public String getOriginalUrl() throws ParsingException { - return baseUrl + "/" + getId(); - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index e6cc66b6..432433cd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; @@ -20,6 +21,8 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import javax.annotation.Nonnull; + import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; @@ -92,10 +95,11 @@ public class PeertubeChannelExtractor extends ChannelExtractor { return baseUrl + value; } + @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - final String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; - return getPage(new Page(pageUrl)); + return getPage(new Page( + getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE)); } @Override @@ -130,7 +134,8 @@ public class PeertubeChannelExtractor extends ChannelExtractor { @Override public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException { - final Response response = downloader.get(getUrl()); + final Response response = downloader.get( + baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT + getId()); if (response != null && response.responseBody() != null) { setInitialData(response.responseBody()); } else { @@ -147,13 +152,9 @@ public class PeertubeChannelExtractor extends ChannelExtractor { if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data"); } + @Nonnull @Override public String getName() throws ParsingException { return JsonUtils.getString(json, "displayName"); } - - @Override - public String getOriginalUrl() throws ParsingException { - return baseUrl + "/" + getId(); - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 74d646bf..b67daca1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Stream; @@ -83,7 +84,9 @@ public class PeertubeStreamExtractor extends StreamExtractor { //if description is shortened, get full description final Downloader dl = NewPipe.getDownloader(); try { - final Response response = dl.get(getUrl() + "/description"); + final Response response = dl.get(baseUrl + + PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + + getId() + "/description"); final JsonObject jsonObject = JsonParser.object().from(response.responseBody()); text = JsonUtils.getString(jsonObject, "description"); } catch (ReCaptchaException | IOException | JsonParserException e) { @@ -338,7 +341,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException { - final Response response = downloader.get(getUrl()); + final Response response = downloader.get(baseUrl + PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + getId()); if (response != null && response.responseBody() != null) { setInitialData(response.responseBody()); } else { @@ -354,14 +357,18 @@ public class PeertubeStreamExtractor extends StreamExtractor { } catch (JsonParserException e) { throw new ExtractionException("Unable to extract PeerTube stream data", e); } - if (json == null) throw new ExtractionException("Unable to extract PeerTube stream data"); + if (json == null) { + throw new ExtractionException("Unable to extract PeerTube stream data"); + } PeertubeParsingHelper.validate(json); } private void loadSubtitles() { if (subtitles.isEmpty()) { try { - final Response response = getDownloader().get(getUrl() + "/captions"); + final Response response = getDownloader().get(baseUrl + + PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + + getId() + "/captions"); final JsonObject captionsJson = JsonParser.object().from(response.responseBody()); final JsonArray captions = JsonUtils.getArray(captionsJson, "data"); for (final Object c : captions) { @@ -387,12 +394,6 @@ public class PeertubeStreamExtractor extends StreamExtractor { return JsonUtils.getString(json, "name"); } - @Nonnull - @Override - public String getOriginalUrl() throws ParsingException { - return baseUrl + "/videos/watch/" + getId(); - } - @Nonnull @Override public String getHost() throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java index ef81ca4b..2903c295 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -11,7 +11,7 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory(); private static final String ID_PATTERN = "(accounts|video-channels)/([^/?&#]*)"; - private static final String API_ENDPOINT = "/api/v1/"; + public static final String API_ENDPOINT = "/api/v1/"; public static PeertubeChannelLinkHandlerFactory getInstance() { return instance; @@ -31,12 +31,11 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { - if (id.matches(ID_PATTERN)) { - return baseUrl + API_ENDPOINT + id; + return baseUrl + "/" + id; } else { // This is needed for compatibility with older versions were we didn't support video channels yet - return baseUrl + API_ENDPOINT + "accounts/" + id; + return baseUrl + "/accounts/" + id; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java index d3f369fc..3e16b509 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java @@ -10,7 +10,8 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory { private static final PeertubeStreamLinkHandlerFactory instance = new PeertubeStreamLinkHandlerFactory(); private static final String ID_PATTERN = "/videos/(watch/|embed/)?([^/?&#]*)"; - private static final String VIDEO_ENDPOINT = "/api/v1/videos/"; + public static final String VIDEO_API_ENDPOINT = "/api/v1/videos/"; + private static final String VIDEO_PATH = "/videos/watch/"; private PeertubeStreamLinkHandlerFactory() { } @@ -27,7 +28,7 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public String getUrl(String id, String baseUrl) { - return baseUrl + VIDEO_ENDPOINT + id; + return baseUrl + VIDEO_PATH + id; } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java index fd944f49..1c315ff1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java @@ -28,7 +28,7 @@ public class PeertubeAccountExtractorTest { // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeAccountExtractor) PeerTube - .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde"); + .getChannelExtractor("https://peertube.mastodon.host/accounts/kde"); extractor.fetchPage(); } @@ -53,7 +53,7 @@ public class PeertubeAccountExtractorTest { @Test public void testUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl()); + assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getUrl()); } @Test @@ -115,7 +115,7 @@ public class PeertubeAccountExtractorTest { // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeAccountExtractor) PeerTube - .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille"); + .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/booteille"); extractor.fetchPage(); } @@ -150,12 +150,12 @@ public class PeertubeAccountExtractorTest { @Test public void testUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl()); + assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getUrl()); } @Test public void testOriginalUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 010dbd7b..68ae7673 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -53,7 +53,7 @@ public class PeertubeChannelExtractorTest { @Test public void testUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getUrl()); + assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getUrl()); } @Test @@ -130,7 +130,7 @@ public class PeertubeChannelExtractorTest { // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeChannelExtractor) PeerTube - .getChannelExtractor("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457"); + .getChannelExtractor("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457"); extractor.fetchPage(); } @@ -165,12 +165,12 @@ public class PeertubeChannelExtractorTest { @Test public void testUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getUrl()); + assertEquals("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getUrl()); } @Test public void testOriginalUrl() throws ParsingException { - assertEquals("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getOriginalUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java index bddb75bd..1ecfc726 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java @@ -53,7 +53,7 @@ public class PeertubeStreamExtractorTest { @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Framasoft"; } - @Override public String expectedUploaderUrl() { return "https://framatube.org/accounts/framasoft"; } + @Override public String expectedUploaderUrl() { return "https://framatube.org/accounts/framasoft@framatube.org"; } @Override public List expectedDescriptionContains() { // CRLF line ending return Arrays.asList("**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n" + "\r\n" @@ -108,7 +108,7 @@ public class PeertubeStreamExtractorTest { @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Tomas Berezovskiy"; } - @Override public String expectedUploaderUrl() { return "https://peertube.iriseden.eu/accounts/tomas_berezovskiy"; } + @Override public String expectedUploaderUrl() { return "https://peertube.co.uk/accounts/tomas_berezovskiy@peertube.iriseden.eu"; } @Override public List expectedDescriptionContains() { // LF line ending return Arrays.asList("https://en.informnapalm.org/dpr-combatant-describes-orders-given-russian-officers/ " + " The InformNapalm team received another video of a separatist prisoner of war telling about his " From aeeae87641f8ca41e62b985ad20f0cf1ed565aa6 Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 9 Apr 2020 20:02:41 +0200 Subject: [PATCH 30/81] [PeerTube] Parse timestamp from url (previously unimplemented) --- .../extractors/PeertubeStreamExtractor.java | 13 ++++++++++--- .../peertube/PeertubeStreamExtractorTest.java | 9 ++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index b67daca1..7e1861f7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -112,9 +112,16 @@ public class PeertubeStreamExtractor extends StreamExtractor { } @Override - public long getTimeStamp() { - //TODO fetch timestamp from url if present; - return 0; + public long getTimeStamp() throws ParsingException { + long timestamp = + getTimestampSeconds("((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); + + if (timestamp == -2) { + // regex for timestamp was not found + return 0; + } else { + return timestamp; + } } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java index 1ecfc726..f78b96e1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java @@ -27,7 +27,9 @@ public class PeertubeStreamExtractorTest { public static class WhatIsPeertube extends DefaultStreamExtractorTest { private static final String ID = "9c9de5e8-0a1e-484a-b099-e80766180a6d"; private static final String INSTANCE = "https://framatube.org"; - private static final String URL = INSTANCE + BASE_URL + ID; + private static final int TIMESTAMP_MINUTE = 1; + private static final int TIMESTAMP_SECOND = 21; + private static final String URL = INSTANCE + BASE_URL + ID + "?start=" + TIMESTAMP_MINUTE + "m" + TIMESTAMP_SECOND + "s"; private static StreamExtractor extractor; @BeforeClass @@ -48,7 +50,7 @@ public class PeertubeStreamExtractorTest { @Override public StreamingService expectedService() { return PeerTube; } @Override public String expectedName() { return "What is PeerTube?"; } @Override public String expectedId() { return ID; } - @Override public String expectedUrlContains() { return BASE_URL + ID; } + @Override public String expectedUrlContains() { return INSTANCE + BASE_URL + ID; } @Override public String expectedOriginalUrlContains() { return URL; } @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @@ -74,6 +76,7 @@ public class PeertubeStreamExtractorTest { + "**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"); } @Override public long expectedLength() { return 113; } + @Override public long expectedTimestamp() { return TIMESTAMP_MINUTE*60 + TIMESTAMP_SECOND; } @Override public long expectedViewCountAtLeast() { return 38600; } @Nullable @Override public String expectedUploadDate() { return "2018-10-01 12:52:46.396"; } // GMT (!) @Nullable @Override public String expectedTextualUploadDate() { return "2018-10-01T10:52:46.396Z"; } @@ -103,7 +106,7 @@ public class PeertubeStreamExtractorTest { @Override public StreamingService expectedService() { return PeerTube; } @Override public String expectedName() { return "A DPR Combatant Describes how Orders are Given through Russian Officers"; } @Override public String expectedId() { return ID; } - @Override public String expectedUrlContains() { return BASE_URL + ID; } + @Override public String expectedUrlContains() { return URL; } @Override public String expectedOriginalUrlContains() { return URL; } @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } From b461da792f375a8527587ff13ff8a6ed8c6f43b4 Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 9 Apr 2020 20:25:22 +0200 Subject: [PATCH 31/81] [MediaCCC] Fix link handler inconsistency providing API links --- .../MediaCCCConferenceExtractor.java | 12 +++------ .../extractors/MediaCCCStreamExtractor.java | 27 +++++++++++-------- .../MediaCCCConferenceLinkHandlerFactory.java | 20 ++++++++------ .../MediaCCCStreamLinkHandlerFactory.java | 7 +++-- .../MediaCCCConferenceExtractorTest.java | 4 +-- 5 files changed, 39 insertions(+), 31 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java index 7d39eacb..5a7256a9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; @@ -87,10 +88,11 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String conferenceUrl = MediaCCCConferenceLinkHandlerFactory.CONFERENCE_API_ENDPOINT + getId(); try { - conferenceData = JsonParser.object().from(downloader.get(getUrl()).responseBody()); + conferenceData = JsonParser.object().from(downloader.get(conferenceUrl).responseBody()); } catch (JsonParserException jpe) { - throw new ExtractionException("Could not parse json returnd by url: " + getUrl()); + throw new ExtractionException("Could not parse json returnd by url: " + conferenceUrl); } } @@ -99,10 +101,4 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor { public String getName() throws ParsingException { return conferenceData.getString("title"); } - - @Nonnull - @Override - public String getOriginalUrl() { - return "https://media.ccc.de/c/" + conferenceData.getString("acronym"); - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 18aa9708..6512f092 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -19,6 +19,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory; import java.io.IOException; import java.util.ArrayList; @@ -93,7 +95,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override public String getUploaderUrl() { - return data.getString("conference_url"); + return MediaCCCConferenceLinkHandlerFactory.CONFERENCE_PATH + getUploaderName(); } @Nonnull @@ -111,25 +113,25 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getSubChannelUrl() throws ParsingException { + public String getSubChannelUrl() { return ""; } @Nonnull @Override - public String getSubChannelName() throws ParsingException { + public String getSubChannelName() { return ""; } @Nonnull @Override - public String getSubChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() { return ""; } @Nonnull @Override - public String getDashMpdUrl() throws ParsingException { + public String getDashMpdUrl() { return ""; } @@ -227,14 +229,13 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + final String videoUrl = MediaCCCStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + getId(); try { - data = JsonParser.object().from( - downloader.get(getLinkHandler().getUrl()).responseBody()); + data = JsonParser.object().from(downloader.get(videoUrl).responseBody()); conferenceData = JsonParser.object() - .from(downloader.get(getUploaderUrl()).responseBody()); + .from(downloader.get(data.getString("conference_url")).responseBody()); } catch (JsonParserException jpe) { - throw new ExtractionException("Could not parse json returned by url: " - + getLinkHandler().getUrl(), jpe); + throw new ExtractionException("Could not parse json returned by url: " + videoUrl, jpe); } } @@ -250,21 +251,25 @@ public class MediaCCCStreamExtractor extends StreamExtractor { return data.getString("frontend_link"); } + @Nonnull @Override public String getHost() { return ""; } + @Nonnull @Override public String getPrivacy() { return ""; } + @Nonnull @Override public String getCategory() { return ""; } + @Nonnull @Override public String getLicence() { return ""; @@ -277,7 +282,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public List getTags() throws ParsingException { + public List getTags() { return Collections.emptyList(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java index 5dc903d8..57591ae1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java @@ -7,19 +7,23 @@ import org.schabi.newpipe.extractor.utils.Parser; import java.util.List; public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory { + public static final String CONFERENCE_API_ENDPOINT = "https://api.media.ccc.de/public/conferences/"; + public static final String CONFERENCE_PATH = "https://media.ccc.de/c/"; + @Override - public String getUrl(final String id, final List contentFilter, final String sortFilter) - throws ParsingException { - return "https://media.ccc.de/public/conferences/" + id; + public String getUrl(final String id, final List contentFilter, + final String sortFilter) throws ParsingException { + return CONFERENCE_PATH + id; } @Override public String getId(final String url) throws ParsingException { - if (url.startsWith("https://media.ccc.de/public/conferences/") - || url.startsWith("https://api.media.ccc.de/public/conferences/")) { - return url.replaceFirst("https://(api\\.)?media\\.ccc\\.de/public/conferences/", ""); - } else if (url.startsWith("https://media.ccc.de/c/")) { - return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url); + if (url.startsWith(CONFERENCE_API_ENDPOINT)) { + return url.replace(CONFERENCE_API_ENDPOINT, ""); + } else if (url.startsWith("https://media.ccc.de/public/conferences/")) { + return url.replace("https://media.ccc.de/public/conferences/", ""); + } else if (url.startsWith(CONFERENCE_PATH)) { + return Parser.matchGroup1(CONFERENCE_PATH + "([^?#]*)", url); } else if (url.startsWith("https://media.ccc.de/b/")) { return Parser.matchGroup1("https://media.ccc.de/b/([^?#]*)", url); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java index a335abf8..d7ecdc3c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java @@ -8,6 +8,9 @@ import java.net.MalformedURLException; import java.net.URL; public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { + public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/"; + private static final String VIDEO_PATH = "https://media.ccc.de/v/"; + @Override public String getId(final String urlString) throws ParsingException { if (urlString.startsWith("https://media.ccc.de/public/events/") @@ -15,7 +18,7 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { return urlString.substring(35); //remove …/public/events part } - if (urlString.startsWith("https://api.media.ccc.de/public/events/") + if (urlString.startsWith(VIDEO_API_ENDPOINT) && !urlString.contains("?q=")) { return urlString.substring(39); //remove api…/public/events part } @@ -42,7 +45,7 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public String getUrl(final String id) throws ParsingException { - return "https://media.ccc.de/public/events/" + id; + return VIDEO_PATH + id; } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java index 98b6203c..f189497d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java @@ -31,7 +31,7 @@ public class MediaCCCConferenceExtractorTest { @Test public void testGetUrl() throws Exception { - assertEquals("https://media.ccc.de/public/conferences/froscon2017", extractor.getUrl()); + assertEquals("https://media.ccc.de/c/froscon2017", extractor.getUrl()); } @Test @@ -67,7 +67,7 @@ public class MediaCCCConferenceExtractorTest { @Test public void testGetUrl() throws Exception { - assertEquals("https://media.ccc.de/public/conferences/oscal19", extractor.getUrl()); + assertEquals("https://media.ccc.de/c/oscal19", extractor.getUrl()); } @Test From 3b2a1829d466e0f1b3d0b6c5a43de49bfde5a287 Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 9 Apr 2020 21:28:13 +0200 Subject: [PATCH 32/81] [MediaCCC] Extract tags --- .../media_ccc/extractors/MediaCCCStreamExtractor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 6512f092..2fbeb1ad 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -24,6 +24,7 @@ import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStrea import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -282,8 +283,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public List getTags() { - return Collections.emptyList(); + public List getTags() throws ParsingException { + return Arrays.asList(data.getArray("tags").toArray(new String[0])); } @Nonnull From 492db83ccff20a71a24aaab94e8968be3d4fffcd Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 9 Apr 2020 21:33:28 +0200 Subject: [PATCH 33/81] [MediaCCC] Return null instead of empty items collector As per the documentation in the base getRelatedStreams() --- .../services/media_ccc/extractors/MediaCCCStreamExtractor.java | 2 +- .../services/media_ccc/MediaCCCStreamExtractorTest.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 2fbeb1ad..193f044d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -219,7 +219,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Override public StreamInfoItemsCollector getRelatedStreams() { - return new StreamInfoItemsCollector(getServiceId()); + return null; } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index 6a4c4c0e..14111304 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -53,6 +53,7 @@ public class MediaCCCStreamExtractorTest { @Nullable @Override public String expectedTextualUploadDate() { return "2018-05-11T02:00:00.000+02:00"; } @Override public long expectedLikeCountAtLeast() { return -1; } @Override public long expectedDislikeCountAtLeast() { return -1; } + @Override public boolean expectedHasRelatedStreams() { return false; } @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } @@ -114,6 +115,7 @@ public class MediaCCCStreamExtractorTest { @Nullable @Override public String expectedTextualUploadDate() { return "2020-01-11T01:00:00.000+01:00"; } @Override public long expectedLikeCountAtLeast() { return -1; } @Override public long expectedDislikeCountAtLeast() { return -1; } + @Override public boolean expectedHasRelatedStreams() { return false; } @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } From d130fd79c31338293134f8416500f135fc57a63a Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 11 Apr 2020 15:52:59 +0200 Subject: [PATCH 34/81] [PeerTube] Prepend "accounts/" to channel id for backward compatibility --- .../extractors/PeertubeAccountExtractor.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index aaf645d8..9f04ffc2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -126,9 +126,15 @@ public class PeertubeAccountExtractor extends ChannelExtractor { } @Override - public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException { - final Response response = downloader.get( - baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT + getId()); + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + String accountUrl = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT; + if (getId().contains("accounts/")) { + accountUrl += getId(); + } else { + accountUrl += "accounts/" + getId(); + } + + final Response response = downloader.get(accountUrl); if (response != null && response.responseBody() != null) { setInitialData(response.responseBody()); } else { From 0c980b2d648b7eb72fa22482a390114da1d3f5ae Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 11 Apr 2020 15:53:40 +0200 Subject: [PATCH 35/81] [PeerTube] Improve channel and stream link handler tests --- ...PeertubeChannelLinkHandlerFactoryTest.java | 34 ++++++++++++++----- .../PeertubeStreamLinkHandlerFactoryTest.java | 28 ++++++++++++--- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java index d47dc6f6..6696ad63 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -28,20 +28,36 @@ public class PeertubeChannelLinkHandlerFactoryTest { @Test public void acceptUrlTest() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net")); - assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa")); + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos")); + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net/videos")); + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa")); } @Test - public void getIdFromUrl() throws ParsingException { - assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId()); - assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); - assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos").getId()); + public void getId() throws ParsingException { + assertEquals("accounts/kranti@videos.squat.net", + linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId()); + assertEquals("accounts/kranti@videos.squat.net", + linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); + assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", + linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos").getId()); + assertEquals("accounts/kranti@videos.squat.net", + linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net").getId()); + assertEquals("accounts/kranti@videos.squat.net", + linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net/videos").getId()); + assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", + linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getId()); } @Test - public void getUrlFromId() throws ParsingException { - assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromId("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl()); - assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("accounts/kranti@videos.squat.net").getUrl()); - assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("kranti@videos.squat.net").getUrl()); + public void getUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", + linkHandler.fromId("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl()); + assertEquals("https://peertube.mastodon.host/accounts/kranti@videos.squat.net", + linkHandler.fromId("accounts/kranti@videos.squat.net").getUrl()); + assertEquals("https://peertube.mastodon.host/accounts/kranti@videos.squat.net", + linkHandler.fromId("kranti@videos.squat.net").getUrl()); + assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", + linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java index f217b8e3..e60db3ba 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java @@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStream import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; /** * Test for {@link PeertubeStreamLinkHandlerFactory} @@ -17,16 +18,34 @@ public class PeertubeStreamLinkHandlerFactoryTest { private static PeertubeStreamLinkHandlerFactory linkHandler; @BeforeClass - public static void setUp() throws Exception { + public static void setUp() { + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); linkHandler = PeertubeStreamLinkHandlerFactory.getInstance(); NewPipe.init(DownloaderTestImpl.getInstance()); } @Test public void getId() throws Exception { - assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId()); - assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId()); - assertEquals("9c9de5e8-0a1e-484a-b099-e80766180a6d", linkHandler.fromUrl("https://framatube.org/videos/embed/9c9de5e8-0a1e-484a-b099-e80766180a6d").getId()); + assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", + linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId()); + assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", + linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId()); + assertEquals("9c9de5e8-0a1e-484a-b099-e80766180a6d", + linkHandler.fromUrl("https://framatube.org/videos/embed/9c9de5e8-0a1e-484a-b099-e80766180a6d").getId()); + assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", + linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId()); + assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", + linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId()); + } + + @Test + public void getUrl() throws Exception { + assertEquals("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60", + linkHandler.fromId("986aac60-1263-4f73-9ce5-36b18225cb60").getUrl()); + assertEquals("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60", + linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getUrl()); + assertEquals("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60", + linkHandler.fromUrl("https://peertube.mastodon.host/videos/embed/986aac60-1263-4f73-9ce5-36b18225cb60").getUrl()); } @@ -35,5 +54,6 @@ public class PeertubeStreamLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60")); assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa")); assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/embed/9c9de5e8-0a1e-484a-b099-e80766180a6d")); + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa")); } } \ No newline at end of file From 07a90d116a0059a027f158f59b5fe9d2720f361f Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 11 Apr 2020 16:57:31 +0200 Subject: [PATCH 36/81] [MediaCCC] Use regex to parse stream and conference urls --- .../MediaCCCConferenceLinkHandlerFactory.java | 18 +++------ .../MediaCCCStreamLinkHandlerFactory.java | 40 +++---------------- 2 files changed, 10 insertions(+), 48 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java index 57591ae1..e5e9b158 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java @@ -9,32 +9,24 @@ import java.util.List; public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory { public static final String CONFERENCE_API_ENDPOINT = "https://api.media.ccc.de/public/conferences/"; public static final String CONFERENCE_PATH = "https://media.ccc.de/c/"; + private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/conferences/)|(?:media\\.ccc\\.de/[bc]/))([^/?&#]*)"; @Override - public String getUrl(final String id, final List contentFilter, + public String getUrl(final String id, + final List contentFilter, final String sortFilter) throws ParsingException { return CONFERENCE_PATH + id; } @Override public String getId(final String url) throws ParsingException { - if (url.startsWith(CONFERENCE_API_ENDPOINT)) { - return url.replace(CONFERENCE_API_ENDPOINT, ""); - } else if (url.startsWith("https://media.ccc.de/public/conferences/")) { - return url.replace("https://media.ccc.de/public/conferences/", ""); - } else if (url.startsWith(CONFERENCE_PATH)) { - return Parser.matchGroup1(CONFERENCE_PATH + "([^?#]*)", url); - } else if (url.startsWith("https://media.ccc.de/b/")) { - return Parser.matchGroup1("https://media.ccc.de/b/([^?#]*)", url); - } - throw new ParsingException("Could not get id from url: " + url); + return Parser.matchGroup1(ID_PATTERN, url); } @Override public boolean onAcceptUrl(final String url) { try { - getId(url); - return true; + return getId(url) != null; } catch (ParsingException e) { return false; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java index d7ecdc3c..fa9ac482 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java @@ -2,45 +2,16 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; -import org.schabi.newpipe.extractor.utils.Utils; - -import java.net.MalformedURLException; -import java.net.URL; +import org.schabi.newpipe.extractor.utils.Parser; public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/"; private static final String VIDEO_PATH = "https://media.ccc.de/v/"; + private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/events/)|(?:media\\.ccc\\.de/v/))([^/?&#]*)"; @Override - public String getId(final String urlString) throws ParsingException { - if (urlString.startsWith("https://media.ccc.de/public/events/") - && !urlString.contains("?q=")) { - return urlString.substring(35); //remove …/public/events part - } - - if (urlString.startsWith(VIDEO_API_ENDPOINT) - && !urlString.contains("?q=")) { - return urlString.substring(39); //remove api…/public/events part - } - - URL url; - try { - url = Utils.stringToURL(urlString); - } catch (MalformedURLException e) { - throw new IllegalArgumentException("The given URL is not valid"); - } - - String path = url.getPath(); - // remove leading "/" of URL-path if URL-path is given - if (!path.isEmpty()) { - path = path.substring(1); - } - - if (path.startsWith("v/")) { - return path.substring(2); - } - - throw new ParsingException("Could not get id from url: " + url); + public String getId(final String url) throws ParsingException { + return Parser.matchGroup1(ID_PATTERN, url); } @Override @@ -51,8 +22,7 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public boolean onAcceptUrl(final String url) { try { - getId(url); - return true; + return getId(url) != null; } catch (ParsingException e) { return false; } From af5b8b1915a5ca4d7a79341e550ca64eda5e94e4 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 11 Apr 2020 16:57:56 +0200 Subject: [PATCH 37/81] [MediaCCC] Add tests for stream and conference link handlers --- ...iaCCCConferenceLinkHandlerFactoryTest.java | 42 +++++++++++++++++++ .../MediaCCCStreamLinkHandlerFactoryTest.java | 42 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamLinkHandlerFactoryTest.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceLinkHandlerFactoryTest.java new file mode 100644 index 00000000..945bbc35 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceLinkHandlerFactoryTest.java @@ -0,0 +1,42 @@ +package org.schabi.newpipe.extractor.services.media_ccc; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; + +import static org.junit.Assert.assertEquals; + +public class MediaCCCConferenceLinkHandlerFactoryTest { + private static MediaCCCConferenceLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = new MediaCCCConferenceLinkHandlerFactory(); + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void getId() throws ParsingException { + assertEquals("jh20", + linkHandler.fromUrl("https://media.ccc.de/c/jh20#278").getId()); + assertEquals("jh20", + linkHandler.fromUrl("https://media.ccc.de/b/jh20?a=b").getId()); + assertEquals("jh20", + linkHandler.fromUrl("https://api.media.ccc.de/public/conferences/jh20&a=b&b=c").getId()); + } + + @Test + public void getUrl() throws ParsingException { + assertEquals("https://media.ccc.de/c/jh20", + linkHandler.fromUrl("https://media.ccc.de/c/jh20#278").getUrl()); + assertEquals("https://media.ccc.de/c/jh20", + linkHandler.fromUrl("https://media.ccc.de/b/jh20?a=b").getUrl()); + assertEquals("https://media.ccc.de/c/jh20", + linkHandler.fromUrl("https://api.media.ccc.de/public/conferences/jh20&a=b&b=c").getUrl()); + assertEquals("https://media.ccc.de/c/jh20", + linkHandler.fromId("jh20").getUrl()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamLinkHandlerFactoryTest.java new file mode 100644 index 00000000..79228ae6 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamLinkHandlerFactoryTest.java @@ -0,0 +1,42 @@ +package org.schabi.newpipe.extractor.services.media_ccc; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory; + +import static org.junit.Assert.assertEquals; + +public class MediaCCCStreamLinkHandlerFactoryTest { + private static MediaCCCStreamLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = new MediaCCCStreamLinkHandlerFactory(); + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void getId() throws ParsingException { + assertEquals("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020", + linkHandler.fromUrl("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020").getId()); + assertEquals("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020", + linkHandler.fromUrl("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020?a=b").getId()); + assertEquals("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020", + linkHandler.fromUrl("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020#3").getId()); + assertEquals("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020", + linkHandler.fromUrl("https://api.media.ccc.de/public/events/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020&a=b").getId()); + } + + @Test + public void getUrl() throws ParsingException { + assertEquals("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020", + linkHandler.fromUrl("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020").getUrl()); + assertEquals("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020", + linkHandler.fromUrl("https://api.media.ccc.de/public/events/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020?b=a&a=b").getUrl()); + assertEquals("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020", + linkHandler.fromId("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020").getUrl()); + } +} From fcb9b6f855ddd5345a6cefcc9b82a0f5138719dd Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 11 Apr 2020 17:17:14 +0200 Subject: [PATCH 38/81] [MediaCCC] Use final when possible, ide refactorings Refactorings on `throws` clause --- .../media_ccc/extractors/MediaCCCConferenceExtractor.java | 4 ++-- .../media_ccc/extractors/MediaCCCStreamExtractor.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java index 5a7256a9..77ea3ad1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java @@ -72,8 +72,8 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray events = conferenceData.getArray("events"); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final JsonArray events = conferenceData.getArray("events"); for (int i = 0; i < events.size(); i++) { collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i))); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 193f044d..cb5e1011 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -283,7 +283,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Nonnull @Override - public List getTags() throws ParsingException { + public List getTags() { return Arrays.asList(data.getArray("tags").toArray(new String[0])); } From 06430c4749295309727f72c6ac18f036f22bdc19 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 11 Apr 2020 17:17:47 +0200 Subject: [PATCH 39/81] [PeerTube] Use final when possible, ide refactorings --- .../extractors/PeertubeStreamInfoItemExtractor.java | 6 +++--- .../linkHandler/PeertubeChannelLinkHandlerFactory.java | 3 +-- .../linkHandler/PeertubeStreamLinkHandlerFactory.java | 3 +-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index edb72c16..489f5fe0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -27,8 +27,7 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor @Override public String getThumbnailUrl() throws ParsingException { - final String value = JsonUtils.getString(item, "thumbnailPath"); - return baseUrl + value; + return baseUrl + JsonUtils.getString(item, "thumbnailPath"); } @Override @@ -51,7 +50,8 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor final String name = JsonUtils.getString(item, "account.name"); final String host = JsonUtils.getString(item, "account.host"); - return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); + return ServiceList.PeerTube.getChannelLHFactory() + .fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java index 2903c295..9134c14e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -24,8 +24,7 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getUrl(String id, List contentFilters, String searchFilter) throws ParsingException { - String baseUrl = ServiceList.PeerTube.getBaseUrl(); - return getUrl(id, contentFilters, searchFilter, baseUrl); + return getUrl(id, contentFilters, searchFilter, ServiceList.PeerTube.getBaseUrl()); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java index 3e16b509..16b3fed3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java @@ -22,8 +22,7 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public String getUrl(String id) { - String baseUrl = ServiceList.PeerTube.getBaseUrl(); - return getUrl(id, baseUrl); + return getUrl(id, ServiceList.PeerTube.getBaseUrl()); } @Override From 55bc01d1ce48404d6cb11cc419685696ba02eac5 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 11 Apr 2020 17:18:05 +0200 Subject: [PATCH 40/81] [SoundCloud] Use final when possible, ide refactorings --- .../extractors/SoundcloudStreamExtractor.java | 82 ++++++++++--------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 0eaf1d9d..c545ea80 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -68,8 +68,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getTextualUploadDate() throws ParsingException { - return track.getString("created_at").replace("T"," ").replace("Z", ""); + public String getTextualUploadDate() { + return track.getString("created_at") + .replace("T"," ") + .replace("Z", ""); } @Nonnull @@ -85,10 +87,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor { if (artworkUrl.isEmpty()) { artworkUrl = track.getObject("user").getString("avatar_url", EMPTY_STRING); } - String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); - return artworkUrlBetterResolution; + return artworkUrl.replace("large.jpg", "crop.jpg"); } + @Nonnull @Override public Description getDescription() { return new Description(track.getString("description"), Description.PLAIN_TEXT); @@ -144,19 +146,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getSubChannelUrl() throws ParsingException { + public String getSubChannelUrl() { return ""; } @Nonnull @Override - public String getSubChannelName() throws ParsingException { + public String getSubChannelName() { return ""; } @Nonnull @Override - public String getSubChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() { return ""; } @@ -168,14 +170,14 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getHlsUrl() throws ParsingException { + public String getHlsUrl() { return ""; } @Override public List getAudioStreams() throws IOException, ExtractionException { List audioStreams = new ArrayList<>(); - Downloader dl = NewPipe.getDownloader(); + final Downloader dl = NewPipe.getDownloader(); // Streams can be streamable and downloadable - or explicitly not. // For playing the track, it is only necessary to have a streamable track. @@ -183,12 +185,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor { if (!track.getBoolean("streamable")) return audioStreams; try { - JsonArray transcodings = track.getObject("media").getArray("transcodings"); + final JsonArray transcodings = track.getObject("media").getArray("transcodings"); // get information about what stream formats are available for (Object transcoding : transcodings) { - JsonObject t = (JsonObject) transcoding; + final JsonObject t = (JsonObject) transcoding; String url = t.getString("url"); if (!isNullOrEmpty(url)) { @@ -200,7 +202,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { // This url points to the endpoint which generates a unique and short living url to the stream. // TODO: move this to a separate method to generate valid urls when needed (e.g. resuming a paused stream) url += "?client_id=" + SoundcloudParsingHelper.clientId(); - String res = dl.get(url).responseBody(); + final String res = dl.get(url).responseBody(); try { JsonObject mp3UrlObject = JsonParser.object().from(res); @@ -234,24 +236,24 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Override - public List getVideoStreams() throws IOException, ExtractionException { + public List getVideoStreams() { return Collections.emptyList(); } @Override - public List getVideoOnlyStreams() throws IOException, ExtractionException { + public List getVideoOnlyStreams() { return Collections.emptyList(); } @Override @Nonnull - public List getSubtitlesDefault() throws IOException, ExtractionException { + public List getSubtitlesDefault() { return Collections.emptyList(); } @Override @Nonnull - public List getSubtitles(MediaFormat format) throws IOException, ExtractionException { + public List getSubtitles(MediaFormat format) { return Collections.emptyList(); } @@ -262,10 +264,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Override public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) + "/related" - + "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId()); + final String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) + + "/related?client_id=" + urlEncode(SoundcloudParsingHelper.clientId()); SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl); return collector; @@ -276,40 +278,44 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return null; } + @Nonnull @Override - public String getHost() throws ParsingException { + public String getHost() { + return ""; + } + + @Nonnull + @Override + public String getPrivacy() { + return ""; + } + + @Nonnull + @Override + public String getCategory() { + return ""; + } + + @Nonnull + @Override + public String getLicence() { return ""; } @Override - public String getPrivacy() throws ParsingException { - return ""; - } - - @Override - public String getCategory() throws ParsingException { - return ""; - } - - @Override - public String getLicence() throws ParsingException { - return ""; - } - - @Override - public Locale getLanguageInfo() throws ParsingException { + public Locale getLanguageInfo() { return null; } @Nonnull @Override - public List getTags() throws ParsingException { + public List getTags() { return Collections.emptyList(); } @Nonnull @Override - public String getSupportInfo() throws ParsingException { + public String getSupportInfo() { return ""; } } From 3191bd6c703e1a410a03b50c2d0e4cbe7b30d7b8 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 11 Apr 2020 17:18:17 +0200 Subject: [PATCH 41/81] [YouTube] Use final when possible --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index f49812fc..d4d618b5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -253,7 +253,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { */ @Override public long getTimeStamp() throws ParsingException { - long timestamp = getTimestampSeconds("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); + final long timestamp = + getTimestampSeconds("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); if (timestamp == -2) { // regex for timestamp was not found From 68d23defba04e3db08462d8268903ec23056fd5f Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 11 Apr 2020 17:26:31 +0200 Subject: [PATCH 42/81] [YouTube] Do not catch every exception on getErrorMessage @B0pol suggestion --- .../youtube/extractors/YoutubeStreamExtractor.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index d4d618b5..8ae88c39 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -612,10 +612,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getErrorMessage() { try { - return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus") - .getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason")); - } catch (Exception e) { - return null; + return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse") + .getObject("playabilityStatus").getObject("errorScreen") + .getObject("playerErrorMessageRenderer").getObject("reason")); + } catch (ParsingException | NullPointerException e) { + return null; // no error message } } From 8dc3f28618d38c692f3abee1ae612e54ac1de3ee Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 11 Apr 2020 17:42:15 +0200 Subject: [PATCH 43/81] [PeerTube] Test one channel url with api and one without --- .../services/peertube/PeertubeChannelExtractorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index 68ae7673..f9e686c8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -28,7 +28,7 @@ public class PeertubeChannelExtractorTest { // setting instance might break test when running in parallel PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); extractor = (PeertubeChannelExtractor) PeerTube - .getChannelExtractor("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa"); + .getChannelExtractor("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa"); extractor.fetchPage(); } From a4097d8d0119cfc71a3b969aa117df924af60e3d Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 11 Apr 2020 17:43:26 +0200 Subject: [PATCH 44/81] [MediaCCC] Return empty list of video-only streams instead of null --- .../services/media_ccc/extractors/MediaCCCStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index cb5e1011..92a049c5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -197,7 +197,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Override public List getVideoOnlyStreams() { - return null; + return Collections.emptyList(); } @Nonnull From a087b092b401977bd58b279a3f285549db0e2160 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 16 May 2020 16:07:25 +0200 Subject: [PATCH 45/81] [Test] Improve code style and add final --- .../services/DefaultStreamExtractorTest.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index 406177f7..fc2ebf43 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -95,7 +95,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest videoStreams = extractor().getVideoStreams(); + final List videoStreams = extractor().getVideoStreams(); final List videoOnlyStreams = extractor().getVideoOnlyStreams(); assertNotNull(videoStreams); assertNotNull(videoOnlyStreams); @@ -202,11 +202,11 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest subtitles = extractor().getSubtitlesDefault(); + final List subtitles = extractor().getSubtitlesDefault(); assertNotNull(subtitles); if (expectedHasSubtitles()) { assertFalse(subtitles.isEmpty()); - for (SubtitlesStream stream : subtitles) { + for (final SubtitlesStream stream : subtitles) { assertIsSecureUrl(stream.getUrl()); - int formatId = stream.getFormatId(); - assertTrue("format id does not fit an audio stream: " + formatId, + final int formatId = stream.getFormatId(); + assertTrue("format id does not fit a subtitles stream: " + formatId, 0x1000 <= formatId && formatId < 0x10000); } } else { assertTrue(subtitles.isEmpty()); - MediaFormat[] formats = {MediaFormat.VTT, MediaFormat.TTML, MediaFormat.TRANSCRIPT1, - MediaFormat.TRANSCRIPT2, MediaFormat.TRANSCRIPT3, MediaFormat.SRT}; - for (MediaFormat format : formats) { - subtitles = extractor().getSubtitles(format); - assertNotNull(subtitles); - assertTrue(subtitles.isEmpty()); + final MediaFormat[] formats = {MediaFormat.VTT, MediaFormat.TTML, MediaFormat.SRT, + MediaFormat.TRANSCRIPT1, MediaFormat.TRANSCRIPT2, MediaFormat.TRANSCRIPT3}; + for (final MediaFormat format : formats) { + final List formatSubtitles = extractor().getSubtitles(format); + assertNotNull(formatSubtitles); + assertTrue(formatSubtitles.isEmpty()); } } } From 6127826571a9041cc9d9d264ac97239249f2aff9 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 16 May 2020 20:07:12 +0200 Subject: [PATCH 46/81] [Test] Add stream metadata tests --- .../newpipe/extractor/ExtractorAsserts.java | 17 ++++++ .../services/BaseStreamExtractorTest.java | 7 +++ .../services/DefaultStreamExtractorTest.java | 54 ++++++++++++++++++- .../MediaCCCStreamExtractorTest.java | 2 + .../peertube/PeertubeStreamExtractorTest.java | 13 ++++- 5 files changed, 91 insertions(+), 2 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java index a0a887d7..69e10b0e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java @@ -2,6 +2,8 @@ package org.schabi.newpipe.extractor; import java.net.MalformedURLException; import java.net.URL; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import javax.annotation.Nonnull; @@ -10,6 +12,7 @@ import javax.annotation.Nullable; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class ExtractorAsserts { @@ -64,4 +67,18 @@ public class ExtractorAsserts { public static void assertAtLeast(long expected, long actual) { assertTrue(actual + " is not at least " + expected, actual >= expected); } + + // this assumes that sorting a and b in-place is not an issue, so it's only intended for tests + public static void assertEqualsOrderIndependent(List expected, List actual) { + if (expected == null) { + assertNull(actual); + return; + } else { + assertNotNull(actual); + } + + Collections.sort(expected); + Collections.sort(actual); + assertEquals(expected, actual); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java index fb555a03..6f3dc813 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java @@ -21,4 +21,11 @@ public interface BaseStreamExtractorTest extends BaseExtractorTest { void testVideoStreams() throws Exception; void testSubtitles() throws Exception; void testFrames() throws Exception; + void testHost() throws Exception; + void testPrivacy() throws Exception; + void testCategory() throws Exception; + void testLicence() throws Exception; + void testLanguageInfo() throws Exception; + void testTags() throws Exception; + void testSupportInfo() throws Exception; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index fc2ebf43..29d9b15d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -14,7 +14,9 @@ import org.schabi.newpipe.extractor.stream.VideoStream; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.TimeZone; import javax.annotation.Nullable; @@ -27,6 +29,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertAtLeast; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEqualsOrderIndependent; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl; import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems; @@ -42,7 +45,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest expectedDescriptionContains(); // e.g. for full links public abstract long expectedLength(); - public long expectedTimestamp() { return 0; }; // default: there is no timestamp + public long expectedTimestamp() { return 0; } // default: there is no timestamp public abstract long expectedViewCountAtLeast(); @Nullable public abstract String expectedUploadDate(); // format: "yyyy-MM-dd HH:mm:ss.SSS" @Nullable public abstract String expectedTextualUploadDate(); @@ -55,6 +58,13 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest expectedTags() { return Collections.emptyList(); } // default: no tags + public String expectedSupportInfo() { return ""; } // default: no support info available @Test @Override @@ -283,4 +293,46 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest expectedTags() { return Arrays.asList("gpn18", "105"); } @Override @Test @@ -118,6 +119,7 @@ public class MediaCCCStreamExtractorTest { @Override public boolean expectedHasRelatedStreams() { return false; } @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } + @Override public List expectedTags() { return Arrays.asList("36c3", "10565", "2019", "Security", "Main"); } @Override @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java index f78b96e1..40b19871 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java @@ -84,7 +84,12 @@ public class PeertubeStreamExtractorTest { @Override public long expectedDislikeCountAtLeast() { return 0; } @Override public boolean expectedHasAudioStreams() { return false; } @Override public boolean expectedHasFrames() { return false; } - + @Override public String expectedHost() { return "framatube.org"; } + @Override public String expectedPrivacy() { return "Public"; } + @Override public String expectedCategory() { return "Science & Technology"; } + @Override public String expectedLicence() { return "Attribution - Share Alike"; } + @Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); } + @Override public List expectedTags() { return Arrays.asList("framasoft", "peertube"); } } public static class AgeRestricted extends DefaultStreamExtractorTest { @@ -136,6 +141,12 @@ public class PeertubeStreamExtractorTest { @Override public boolean expectedHasAudioStreams() { return false; } @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } + @Override public String expectedHost() { return "peertube.iriseden.eu"; } + @Override public String expectedPrivacy() { return "Public"; } + @Override public String expectedCategory() { return "News & Politics"; } + @Override public String expectedLicence() { return "Attribution - Share Alike"; } + @Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("ru"); } + @Override public List expectedTags() { return Arrays.asList("ДНР", "ЛНР", "Кремль", "Новороссия", "ФСБ"); } } From 8ce711f40fe3667a9184da513a794d83c098985d Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 16 May 2020 20:27:58 +0200 Subject: [PATCH 47/81] [Test] Add sub channel name, url and thumbnail tests --- .../services/BaseStreamExtractorTest.java | 3 ++ .../services/DefaultStreamExtractorTest.java | 32 +++++++++++++++++++ .../peertube/PeertubeStreamExtractorTest.java | 4 +++ 3 files changed, 39 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java index 6f3dc813..fe7abc67 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java @@ -5,6 +5,9 @@ public interface BaseStreamExtractorTest extends BaseExtractorTest { void testUploaderName() throws Exception; void testUploaderUrl() throws Exception; void testUploaderAvatarUrl() throws Exception; + void testSubChannelName() throws Exception; + void testSubChannelUrl() throws Exception; + void testSubChannelAvatarUrl() throws Exception; void testThumbnailUrl() throws Exception; void testDescription() throws Exception; void testLength() throws Exception; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index 29d9b15d..860dfd48 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -43,6 +43,8 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest expectedDescriptionContains(); // e.g. for full links public abstract long expectedLength(); public long expectedTimestamp() { return 0; } // default: there is no timestamp @@ -92,6 +94,36 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest expectedDescriptionContains() { // CRLF line ending return Arrays.asList("**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n" + "\r\n" @@ -117,6 +119,8 @@ public class PeertubeStreamExtractorTest { @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Tomas Berezovskiy"; } @Override public String expectedUploaderUrl() { return "https://peertube.co.uk/accounts/tomas_berezovskiy@peertube.iriseden.eu"; } + @Override public String expectedSubChannelName() { return "smm.expx3"; } + @Override public String expectedSubChannelUrl() { return "https://peertube.iriseden.eu/video-channels/smm.expx3"; } @Override public List expectedDescriptionContains() { // LF line ending return Arrays.asList("https://en.informnapalm.org/dpr-combatant-describes-orders-given-russian-officers/ " + " The InformNapalm team received another video of a separatist prisoner of war telling about his " From d0b14644bb9937d73137a4b8c0121a13defe9b56 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 16 May 2020 21:14:10 +0200 Subject: [PATCH 48/81] [YouTube/MediaCCC] Consider dates as GMT and not as local --- .../services/media_ccc/extractors/MediaCCCParsingHelper.java | 5 ++++- .../extractor/services/youtube/YoutubeParsingHelper.java | 4 +++- .../extractor/services/DefaultStreamExtractorTest.java | 3 +-- .../services/peertube/PeertubeStreamExtractorTest.java | 4 ++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java index 9e5baaf9..dfdceead 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java @@ -6,6 +6,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import java.util.TimeZone; public final class MediaCCCParsingHelper { private MediaCCCParsingHelper() { } @@ -13,7 +14,9 @@ public final class MediaCCCParsingHelper { public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException { Date date; try { - date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + date = sdf.parse(textualUploadDate); } catch (ParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index af10efa2..cefab49d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -185,7 +185,9 @@ public class YoutubeParsingHelper { public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { Date date; try { - date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + sdf.setTimeZone(TimeZone.getTimeZone("GMT")); + date = sdf.parse(textualUploadDate); } catch (ParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index 860dfd48..81d9dced 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -172,10 +172,9 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest Date: Sun, 14 Jun 2020 10:41:13 +0200 Subject: [PATCH 49/81] [Test] Add stream dash mpd url test --- .../services/BaseStreamExtractorTest.java | 1 + .../services/DefaultStreamExtractorTest.java | 15 ++++++++++++++- .../YoutubeStreamExtractorLivestreamTest.java | 1 + 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java index fe7abc67..40356ceb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java @@ -23,6 +23,7 @@ public interface BaseStreamExtractorTest extends BaseExtractorTest { void testAudioStreams() throws Exception; void testVideoStreams() throws Exception; void testSubtitles() throws Exception; + void testGetDashMpdUrl() throws Exception; void testFrames() throws Exception; void testHost() throws Exception; void testPrivacy() throws Exception; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index 81d9dced..8ab642d9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -22,11 +22,11 @@ import java.util.TimeZone; import javax.annotation.Nullable; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertAtLeast; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEqualsOrderIndependent; @@ -59,6 +59,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest Date: Sun, 14 Jun 2020 19:51:54 +0200 Subject: [PATCH 50/81] [PeerTube] Change age restricted video in tests The old one wasn't available anymore --- .../peertube/PeertubeStreamExtractorTest.java | 51 ++++++++----------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java index 8b9550b8..f8c72957 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java @@ -95,9 +95,9 @@ public class PeertubeStreamExtractorTest { } public static class AgeRestricted extends DefaultStreamExtractorTest { - private static final String ID = "0d501633-f2d9-4476-87c6-71f1c02402a4"; - private static final String INSTANCE = "https://peertube.co.uk"; - private static final String URL = INSTANCE + BASE_URL + ID; + private static final String ID = "dbd8e5e1-c527-49b6-b70c-89101dbb9c08"; + private static final String INSTANCE = "https://nocensoring.net"; + private static final String URL = INSTANCE + "/videos/embed/" + ID; private static StreamExtractor extractor; @BeforeClass @@ -111,46 +111,35 @@ public class PeertubeStreamExtractorTest { @Override public StreamExtractor extractor() { return extractor; } @Override public StreamingService expectedService() { return PeerTube; } - @Override public String expectedName() { return "A DPR Combatant Describes how Orders are Given through Russian Officers"; } + @Override public String expectedName() { return "Covid-19 ? [Court-métrage]"; } @Override public String expectedId() { return ID; } - @Override public String expectedUrlContains() { return URL; } + @Override public String expectedUrlContains() { return INSTANCE + BASE_URL + ID; } @Override public String expectedOriginalUrlContains() { return URL; } @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } - @Override public String expectedUploaderName() { return "Tomas Berezovskiy"; } - @Override public String expectedUploaderUrl() { return "https://peertube.co.uk/accounts/tomas_berezovskiy@peertube.iriseden.eu"; } - @Override public String expectedSubChannelName() { return "smm.expx3"; } - @Override public String expectedSubChannelUrl() { return "https://peertube.iriseden.eu/video-channels/smm.expx3"; } + @Override public String expectedUploaderName() { return "Résilience humaine"; } + @Override public String expectedUploaderUrl() { return "https://nocensoring.net/accounts/gmt@nocensoring.net"; } + @Override public String expectedSubChannelName() { return "SYSTEM FAILURE Quel à-venir ?"; } + @Override public String expectedSubChannelUrl() { return "https://nocensoring.net/video-channels/systemfailure_quel"; } @Override public List expectedDescriptionContains() { // LF line ending - return Arrays.asList("https://en.informnapalm.org/dpr-combatant-describes-orders-given-russian-officers/ " - + " The InformNapalm team received another video of a separatist prisoner of war telling about his " - + "activities in `Dontesk People’s Republic’ (DPR) structures. The video is old, as the interrogation" - + " date is September, but it is the situation described is still relevant and interesting today. In " - + "this recording the combatant re-tells how he came to be recruited into the DPR forces, and how " - + "they are operating under Russian military command. He expresses remorse for his stupidity. Perhaps" - + " he is just saying what he thinks his interrogator wants to hear, perhaps he is speaking from a " - + "new understanding?\n" - + "\n" - + "The video contains a lot of cut and paste (stitching) in places where intelligence data or valuable" - + " information has been deleted because it cannot be shared publically. We trust you will understand " - + "this necessity."); + return Arrays.asList("2020, le monde est frappé par une pandémie, beaucoup d'humains sont confinés.", + "System Failure Quel à-venir ? - Covid-19 / 2020"); } - @Override public long expectedLength() { return 512; } - @Override public long expectedViewCountAtLeast() { return 7; } - @Nullable @Override public String expectedUploadDate() { return "2019-10-22 06:16:48.982"; } - @Nullable @Override public String expectedTextualUploadDate() { return "2019-10-22T06:16:48.982Z"; } - @Override public long expectedLikeCountAtLeast() { return 3; } + @Override public long expectedLength() { return 667; } + @Override public long expectedViewCountAtLeast() { return 138; } + @Nullable @Override public String expectedUploadDate() { return "2020-05-14 17:24:35.580"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2020-05-14T17:24:35.580Z"; } + @Override public long expectedLikeCountAtLeast() { return 1; } @Override public long expectedDislikeCountAtLeast() { return 0; } @Override public int expectedAgeLimit() { return 18; } @Override public boolean expectedHasAudioStreams() { return false; } @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } - @Override public String expectedHost() { return "peertube.iriseden.eu"; } + @Override public String expectedHost() { return "nocensoring.net"; } @Override public String expectedPrivacy() { return "Public"; } - @Override public String expectedCategory() { return "News & Politics"; } - @Override public String expectedLicence() { return "Attribution - Share Alike"; } - @Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("ru"); } - @Override public List expectedTags() { return Arrays.asList("ДНР", "ЛНР", "Кремль", "Новороссия", "ФСБ"); } + @Override public String expectedCategory() { return "Art"; } + @Override public String expectedLicence() { return "Attribution"; } + @Override public List expectedTags() { return Arrays.asList("Covid-19", "Gérôme-Mary trebor", "Horreur et beauté", "court-métrage", "nue artistique"); } } From f11fe87688beb27a051608c4f0facb2d8214b6d1 Mon Sep 17 00:00:00 2001 From: Stypox Date: Fri, 14 Aug 2020 11:09:29 +0200 Subject: [PATCH 51/81] [YouTube] Replace outdated PewDiePie video test with another one The old video was made private, and this video (wedding) is probably never going to be removed. --- .../YoutubeStreamExtractorDefaultTest.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 49369831..6a3aa4f2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -105,7 +105,7 @@ public class YoutubeStreamExtractorDefaultTest { } public static class DescriptionTestPewdiepie extends DefaultStreamExtractorTest { - private static final String ID = "fBc4Q_htqPg"; + private static final String ID = "7PIMiDcwNvc"; private static final int TIMESTAMP = 17; private static final String URL = BASE_URL + ID + "&t=" + TIMESTAMP; private static StreamExtractor extractor; @@ -119,7 +119,7 @@ public class YoutubeStreamExtractorDefaultTest { @Override public StreamExtractor extractor() { return extractor; } @Override public StreamingService expectedService() { return YouTube; } - @Override public String expectedName() { return "Dr. Phil DESTROYS spoiled brat!!!! . -- Dr Phil #7"; } + @Override public String expectedName() { return "Marzia & Felix - Wedding 19.08.2019"; } @Override public String expectedId() { return ID; } @Override public String expectedUrlContains() { return BASE_URL + ID; } @Override public String expectedOriginalUrlContains() { return URL; } @@ -128,17 +128,16 @@ public class YoutubeStreamExtractorDefaultTest { @Override public String expectedUploaderName() { return "PewDiePie"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw"; } @Override public List expectedDescriptionContains() { - return Arrays.asList("https://www.reddit.com/r/PewdiepieSubmissions/", - "https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng", - "https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/"); + return Arrays.asList("https://www.youtube.com/channel/UC7l23W7gFi4Uho6WSzckZRA", + "https://www.handcraftpictures.com/"); } - @Override public long expectedLength() { return 1165; } + @Override public long expectedLength() { return 381; } @Override public long expectedTimestamp() { return TIMESTAMP; } @Override public long expectedViewCountAtLeast() { return 26682500; } - @Nullable @Override public String expectedUploadDate() { return "2018-09-12 00:00:00.000"; } - @Nullable @Override public String expectedTextualUploadDate() { return "2018-09-12"; } - @Override public long expectedLikeCountAtLeast() { return 1166000; } - @Override public long expectedDislikeCountAtLeast() { return 16900; } + @Nullable @Override public String expectedUploadDate() { return "2019-08-24 00:00:00.000"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2019-08-24"; } + @Override public long expectedLikeCountAtLeast() { return 5212900; } + @Override public long expectedDislikeCountAtLeast() { return 30600; } } public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest { From 57e7994c9e46d068ba53cb320c91c22ec24f76de Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 24 Oct 2020 21:17:58 +0200 Subject: [PATCH 52/81] Add some missing finals, nullables and comments --- .../extractors/MediaCCCParsingHelper.java | 4 ++-- .../extractors/MediaCCCStreamExtractor.java | 2 ++ .../extractors/PeertubeAccountExtractor.java | 6 +++-- .../extractors/PeertubeStreamExtractor.java | 22 ++++++++++++------- .../extractors/SoundcloudStreamExtractor.java | 2 ++ .../youtube/YoutubeParsingHelper.java | 4 ++-- .../extractors/YoutubeStreamExtractor.java | 5 ++++- .../extractor/stream/StreamExtractor.java | 1 + .../services/DefaultStreamExtractorTest.java | 3 +++ 9 files changed, 34 insertions(+), 15 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java index dfdceead..14ca5c52 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java @@ -12,9 +12,9 @@ public final class MediaCCCParsingHelper { private MediaCCCParsingHelper() { } public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException { - Date date; + final Date date; try { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); sdf.setTimeZone(TimeZone.getTimeZone("GMT")); date = sdf.parse(textualUploadDate); } catch (ParseException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 92a049c5..96b1a5cc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Locale; import javax.annotation.Nonnull; +import javax.annotation.Nullable; public class MediaCCCStreamExtractor extends StreamExtractor { private JsonObject data; @@ -217,6 +218,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { return StreamType.VIDEO_STREAM; } + @Nullable @Override public StreamInfoItemsCollector getRelatedStreams() { return null; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index 9f04ffc2..b41bf205 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -96,7 +96,8 @@ public class PeertubeAccountExtractor extends ChannelExtractor { } @Override - public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + public InfoItemsPage getPage(final Page page) + throws IOException, ExtractionException { if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } @@ -126,7 +127,8 @@ public class PeertubeAccountExtractor extends ChannelExtractor { } @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { String accountUrl = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT; if (getId().contains("accounts/")) { accountUrl += getId(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 7e1861f7..fa02d473 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Locale; import javax.annotation.Nonnull; +import javax.annotation.Nullable; public class PeertubeStreamExtractor extends StreamExtractor { private final String baseUrl; @@ -113,7 +114,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public long getTimeStamp() throws ParsingException { - long timestamp = + final long timestamp = getTimestampSeconds("((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); if (timestamp == -2) { @@ -261,19 +262,24 @@ public class PeertubeStreamExtractor extends StreamExtractor { return StreamType.VIDEO_STREAM; } + @Nullable @Override public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final List tags = getTags(); final String apiUrl; - if (!tags.isEmpty()) { - apiUrl = getRelatedStreamsUrl(tags); - - } else { + if (tags.isEmpty()) { apiUrl = getUploaderUrl() + "/videos?start=0&count=8"; + } else { + apiUrl = getRelatedStreamsUrl(tags); + } + + if (Utils.isBlank(apiUrl)) { + return null; + } else { + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + getStreamsFromApi(collector, apiUrl); + return collector; } - if (!Utils.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); - return collector; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index c545ea80..bc1d5902 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Locale; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -262,6 +263,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return StreamType.AUDIO_STREAM; } + @Nullable @Override public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index cefab49d..ac7db985 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -183,9 +183,9 @@ public class YoutubeParsingHelper { } public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { - Date date; + final Date date; try { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); sdf.setTimeZone(TimeZone.getTimeZone("GMT")); date = sdf.parse(textualUploadDate); } catch (ParseException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 8ae88c39..0077a163 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -576,11 +576,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + @Nullable @Override public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException { assertPageFetched(); - if (getAgeLimit() != NO_AGE_LIMIT) return null; + if (getAgeLimit() != NO_AGE_LIMIT) { + return null; + } try { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index f48bb913..4e109bba 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -321,6 +321,7 @@ public abstract class StreamExtractor extends Extractor { * @throws IOException * @throws ExtractionException */ + @Nullable public abstract StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException; /** diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index 8ab642d9..a70b38a8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -249,6 +249,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest Date: Thu, 3 Sep 2020 02:29:18 +0200 Subject: [PATCH 53/81] [YouTube] Ignore leading characters in video id --- .../YoutubeStreamLinkHandlerFactory.java | 25 +++++++++++++------ .../YoutubeStreamLinkHandlerFactoryTest.java | 8 ++++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index 48fdc286..bcc1fac5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -12,6 +12,8 @@ import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL; @@ -37,6 +39,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { + private static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN = Pattern.compile("([a-zA-Z0-9_-]{11})"); private static final YoutubeStreamLinkHandlerFactory instance = new YoutubeStreamLinkHandlerFactory(); private YoutubeStreamLinkHandlerFactory() { @@ -46,13 +49,19 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { return instance; } - private static boolean isId(@Nullable String id) { - return id != null && id.matches("[a-zA-Z0-9_-]{11}"); + @Nullable + private static String extractId(@Nullable final String id) { + if (id != null) { + final Matcher m = YOUTUBE_VIDEO_ID_REGEX_PATTERN.matcher(id); + return m.find() ? m.group(1) : null; + } + return null; } - private static String assertIsId(@Nullable String id) throws ParsingException { - if (isId(id)) { - return id; + private static String assertIsId(@Nullable final String id) throws ParsingException { + final String extractedId = extractId(id); + if (extractedId != null) { + return extractedId; } else { throw new ParsingException("The given string is not a Youtube-Video-ID"); } @@ -81,9 +90,9 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { if (scheme != null && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) { String schemeSpecificPart = uri.getSchemeSpecificPart(); if (schemeSpecificPart.startsWith("//")) { - final String possiblyId = schemeSpecificPart.substring(2); - if (isId(possiblyId)) { - return possiblyId; + final String extractedId = extractId(schemeSpecificPart.substring(2)); + if (extractedId != null) { + return extractedId; } urlString = "https:" + schemeSpecificPart; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index 965306d5..e51f68db 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -81,6 +81,14 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId()); assertEquals("n8X9_MgEdCg", linkHandler.fromUrl("vnd.youtube://n8X9_MgEdCg").getId()); assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId()); + assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://m.youtube.com/watch?v=-cdveCh1kQk)").getId()); + assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://www.youtube.com/watch?v=-cdveCh1kQk-").getId()); + assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://WWW.YouTube.com/watch?v=-cdveCh1kQkwhatever").getId()); + assertEquals("O0EDx9WAelc", linkHandler.fromUrl("HTTPS://www.youtube.com/watch?v=O0EDx9WAelc]").getId()); + assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://youtu.be/-cdveCh1kQk)hello").getId()); + assertEquals("OGS7c0-CmRs", linkHandler.fromUrl("https://YouTu.be/OGS7c0-CmRswhatever)").getId()); + assertEquals("-cdveCh1kQk", linkHandler.fromUrl("HTTPS://youtu.be/-cdveCh1kQk)").getId()); + assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("https://www.youtube.com/shorts/IOS2fqxwYbAhi").getId()); assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("http://www.youtube.com/shorts/IOS2fqxwYbA").getId()); } From 01f49e8f668dc06355d9a3c9dc9cd5ce7c1501ac Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 26 Oct 2020 16:32:39 +0100 Subject: [PATCH 54/81] polish strings --- .../extractors/YoutubeStreamExtractor.java | 81 ++++++++++--------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 0077a163..8b60a41e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; - import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; @@ -36,6 +35,8 @@ import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.text.SimpleDateFormat; @@ -49,9 +50,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; @@ -84,8 +82,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { // Exceptions //////////////////////////////////////////////////////////////////////////*/ - public class DecryptException extends ParsingException { - DecryptException(String message, Throwable cause) { + public class DeobfuscateException extends ParsingException { + DeobfuscateException(String message, Throwable cause) { super(message, cause); } } @@ -156,12 +154,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); Calendar parsedTime = timeAgoParser.parse(time).date(); return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime()); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } try { // Premiered Feb 21, 2020 Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time); return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime()); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } } try { @@ -169,7 +169,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))); return new SimpleDateFormat("yyyy-MM-dd").format(d); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } throw new ParsingException("Could not get upload date"); } @@ -368,7 +369,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") .getObject("videoOwnerRenderer").getObject("title")); - } catch (ParsingException ignored) { } + } catch (ParsingException ignored) { + } if (isNullOrEmpty(uploaderName)) { uploaderName = playerResponse.getObject("videoDetails").getString("author"); @@ -436,11 +438,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { } if (!dashManifestUrl.contains("/signature/")) { - String encryptedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl); - String decryptedSig; + String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl); + String deobfuscatedSig; - decryptedSig = decryptSignature(encryptedSig, decryptionCode); - dashManifestUrl = dashManifestUrl.replace("/s/" + encryptedSig, "/signature/" + decryptedSig); + deobfuscatedSig = deobfuscateSignature(obfuscatedSig, deobfuscationCode); + dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig, "/signature/" + deobfuscatedSig); } return dashManifestUrl; @@ -630,7 +632,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String FORMATS = "formats"; private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String HTTPS = "https:"; - private static final String DECRYPTION_FUNC_NAME = "decrypt"; + private static final String DEOBFUSCATION_FUNC_NAME = "decrypt"; private final static String[] REGEXES = { "(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)", @@ -640,7 +642,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { "\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(" }; - private volatile String decryptionCode = ""; + private volatile String deobfuscationCode = ""; @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { @@ -695,8 +697,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new ContentNotAvailableException("Got error: \"" + reason + "\""); } - if (decryptionCode.isEmpty()) { - decryptionCode = loadDecryptionCode(playerUrl); + if (deobfuscationCode.isEmpty()) { + deobfuscationCode = loadDeobfuscationCode(playerUrl); } if (subtitlesInfos.isEmpty()) { @@ -716,7 +718,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private String getPlayerUrl(final JsonObject playerConfig) throws ParsingException { // The Youtube service needs to be initialized by downloading the // js-Youtube-player. This is done in order to get the algorithm - // for decrypting cryptic signatures inside certain stream URLs. + // for deobfuscating cryptic signatures inside certain stream URLs. final String playerUrl = playerConfig.getObject("assets").getString("js"); if (playerUrl == null) { @@ -768,11 +770,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (IOException e) { throw new ParsingException( - "Could load decryption code form restricted video for the Youtube service.", e); + "Could load deobfuscation code form restricted video for the Youtube service.", e); } } - private String loadDecryptionCode(String playerUrl) throws DecryptException { + private String loadDeobfuscationCode(String playerUrl) throws DeobfuscateException { try { Downloader downloader = NewPipe.getDownloader(); if (!playerUrl.contains("https://youtube.com")) { @@ -782,49 +784,49 @@ public class YoutubeStreamExtractor extends StreamExtractor { } final String playerCode = downloader.get(playerUrl, getExtractorLocalization()).responseBody(); - final String decryptionFunctionName = getDecryptionFuncName(playerCode); + final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode); final String functionPattern = "(" - + decryptionFunctionName.replace("$", "\\$") + + deobfuscationFunctionName.replace("$", "\\$") + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})"; - final String decryptionFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";"; + final String deobfuscateFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";"; final String helperObjectName = - Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunction); + Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", deobfuscateFunction); final String helperPattern = "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; final String helperObject = Parser.matchGroup1(helperPattern, playerCode.replace("\n", "")); final String callerFunction = - "function " + DECRYPTION_FUNC_NAME + "(a){return " + decryptionFunctionName + "(a);}"; + "function " + DEOBFUSCATION_FUNC_NAME + "(a){return " + deobfuscationFunctionName + "(a);}"; - return helperObject + decryptionFunction + callerFunction; + return helperObject + deobfuscateFunction + callerFunction; } catch (IOException ioe) { - throw new DecryptException("Could not load decrypt function", ioe); + throw new DeobfuscateException("Could not load deobfuscate function", ioe); } catch (Exception e) { - throw new DecryptException("Could not parse decrypt function ", e); + throw new DeobfuscateException("Could not parse deobfuscate function ", e); } } - private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException { + private String deobfuscateSignature(String obfuscatedSig, String deobfuscationCode) throws DeobfuscateException { final Context context = Context.enter(); context.setOptimizationLevel(-1); final Object result; try { final ScriptableObject scope = context.initSafeStandardObjects(); - context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null); - final Function decryptionFunc = (Function) scope.get("decrypt", scope); - result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig}); + context.evaluateString(scope, deobfuscationCode, "decryptionCode", 1, null); + final Function deobfuscateFunc = (Function) scope.get("decrypt", scope); + result = deobfuscateFunc.call(context, scope, scope, new Object[]{obfuscatedSig}); } catch (Exception e) { - throw new DecryptException("Could not get decrypt signature", e); + throw new DeobfuscateException("Could not get deobfuscate signature", e); } finally { Context.exit(); } return result == null ? "" : result.toString(); } - private String getDecryptionFuncName(final String playerCode) throws DecryptException { + private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException { Parser.RegexException exception = null; for (final String regex : REGEXES) { try { @@ -835,7 +837,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } } - throw new DecryptException("Could not find decrypt function with any of the given patterns.", exception); + throw new DeobfuscateException("Could not find deobfuscate function with any of the given patterns.", exception); } @Nonnull @@ -989,18 +991,19 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (formatData.has("url")) { streamUrl = formatData.getString("url"); } else { - // this url has an encrypted signature + // this url has an obfuscated signature final String cipherString = formatData.has("cipher") ? formatData.getString("cipher") : formatData.getString("signatureCipher"); final Map cipher = Parser.compatParseMap(cipherString); streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" - + decryptSignature(cipher.get("s"), decryptionCode); + + deobfuscateSignature(cipher.get("s"), deobfuscationCode); } urlAndItags.put(streamUrl, itagItem); } - } catch (UnsupportedEncodingException ignored) {} + } catch (UnsupportedEncodingException ignored) { + } } } From 0a12300c5e1ecb4048d5968f69b52dad8c80838f Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 26 Oct 2020 16:57:37 +0100 Subject: [PATCH 55/81] polish tests --- .../SoundcloudStreamExtractorTest.java | 45 ++--- .../youtube/YoutubeChannelExtractorTest.java | 184 ------------------ .../YoutubeMusicSearchExtractorTest.java | 4 +- .../YoutubeStreamExtractorDefaultTest.java | 42 ---- 4 files changed, 14 insertions(+), 261 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java index 28c8d0b2..6e386b1d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -19,9 +19,9 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; public class SoundcloudStreamExtractorTest { - public static class LilUziVertDoWhatIWant extends DefaultStreamExtractorTest { - private static final String ID = "do-what-i-want-produced-by-maaly-raw-don-cannon"; - private static final String UPLOADER = "https://soundcloud.com/liluzivert"; + public static class CreativeCommonsPlaysWellWithOthers extends DefaultStreamExtractorTest { + private static final String ID = "plays-well-with-others-ep-2-what-do-an-army-of-ants-and-an-online-encyclopedia-have-in-common"; + private static final String UPLOADER = "https://soundcloud.com/wearecc"; private static final int TIMESTAMP = 69; private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP; private static StreamExtractor extractor; @@ -35,20 +35,21 @@ public class SoundcloudStreamExtractorTest { @Override public StreamExtractor extractor() { return extractor; } @Override public StreamingService expectedService() { return SoundCloud; } - @Override public String expectedName() { return "Do What I Want [Produced By Maaly Raw + Don Cannon]"; } - @Override public String expectedId() { return "276206960"; } + @Override public String expectedName() { return "Plays Well with Others, Ep 2: What Do an Army of Ants and an Online Encyclopedia Have in Common?"; } + @Override public String expectedId() { return "597253485"; } @Override public String expectedUrlContains() { return UPLOADER + "/" + ID; } @Override public String expectedOriginalUrlContains() { return URL; } @Override public StreamType expectedStreamType() { return StreamType.AUDIO_STREAM; } - @Override public String expectedUploaderName() { return "Lil Uzi Vert"; } + @Override public String expectedUploaderName() { return "Creative Commons"; } @Override public String expectedUploaderUrl() { return UPLOADER; } - @Override public List expectedDescriptionContains() { return Arrays.asList("The Perfect LUV Tape®"); } - @Override public long expectedLength() { return 175; } + @Override public List expectedDescriptionContains() { return Arrays.asList("Stigmergy is a mechanism of indirect coordination", + "All original content in Plays Well with Others is available under a Creative Commons BY license."); } + @Override public long expectedLength() { return 1400; } @Override public long expectedTimestamp() { return TIMESTAMP; } - @Override public long expectedViewCountAtLeast() { return 75413600; } - @Nullable @Override public String expectedUploadDate() { return "2016-07-31 18:18:07.000"; } - @Nullable @Override public String expectedTextualUploadDate() { return "2016-07-31 18:18:07"; } + @Override public long expectedViewCountAtLeast() { return 27000; } + @Nullable @Override public String expectedUploadDate() { return "2019-03-28 13:36:18.000"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2019-03-28 13:36:18"; } @Override public long expectedLikeCountAtLeast() { return -1; } @Override public long expectedDislikeCountAtLeast() { return -1; } @Override public boolean expectedHasVideoStreams() { return false; } @@ -56,26 +57,4 @@ public class SoundcloudStreamExtractorTest { @Override public boolean expectedHasFrames() { return false; } } - public static class ContentNotSupported { - @BeforeClass - public static void setUp() { - NewPipe.init(DownloaderTestImpl.getInstance()); - } - - @Test(expected = ContentNotSupportedException.class) - public void hlsAudioStream() throws Exception { - final StreamExtractor extractor = - SoundCloud.getStreamExtractor("https://soundcloud.com/dualipa/cool"); - extractor.fetchPage(); - extractor.getAudioStreams(); - } - - @Test(expected = ContentNotSupportedException.class) - public void bothHlsAndOpusAudioStreams() throws Exception { - final StreamExtractor extractor = - SoundCloud.getStreamExtractor("https://soundcloud.com/lil-baby-4pf/no-sucker"); - extractor.fetchPage(); - extractor.getAudioStreams(); - } - } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index a916c560..1c461fa8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -432,190 +432,6 @@ public class YoutubeChannelExtractorTest { } } - // this channel has no "Subscribe" button - public static class EminemVEVO implements BaseChannelExtractorTest { - private static YoutubeChannelExtractor extractor; - - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeChannelExtractor) YouTube - .getChannelExtractor("https://www.youtube.com/user/EminemVEVO/"); - extractor.fetchPage(); - } - - /*////////////////////////////////////////////////////////////////////////// - // Extractor - //////////////////////////////////////////////////////////////////////////*/ - - @Test - public void testServiceId() { - assertEquals(YouTube.getServiceId(), extractor.getServiceId()); - } - - @Test - public void testName() throws Exception { - assertEquals("EminemVEVO", extractor.getName()); - } - - @Test - public void testId() throws Exception { - assertEquals("UC20vb-R_px4CguHzzBPhoyQ", extractor.getId()); - } - - @Test - public void testUrl() throws ParsingException { - assertEquals("https://www.youtube.com/channel/UC20vb-R_px4CguHzzBPhoyQ", extractor.getUrl()); - } - - @Test - public void testOriginalUrl() throws ParsingException { - assertEquals("https://www.youtube.com/user/EminemVEVO/", extractor.getOriginalUrl()); - } - - /*////////////////////////////////////////////////////////////////////////// - // ListExtractor - //////////////////////////////////////////////////////////////////////////*/ - - @Test - public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor); - } - - @Test - public void testMoreRelatedItems() throws Exception { - defaultTestMoreItems(extractor); - } - - /*////////////////////////////////////////////////////////////////////////// - // ChannelExtractor - //////////////////////////////////////////////////////////////////////////*/ - - @Test - public void testDescription() throws Exception { - final String description = extractor.getDescription(); - assertTrue(description, description.contains("Eminem on Vevo")); - } - - @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertTrue(avatarUrl, avatarUrl.contains("yt3")); - } - - @Test - public void testBannerUrl() throws Exception { - String bannerUrl = extractor.getBannerUrl(); - assertIsSecureUrl(bannerUrl); - assertTrue(bannerUrl, bannerUrl.contains("yt3")); - } - - @Test - public void testFeedUrl() throws Exception { - assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC20vb-R_px4CguHzzBPhoyQ", extractor.getFeedUrl()); - } - - @Test - public void testSubscriberCount() throws Exception { - // there is no "Subscribe" button - long subscribers = extractor.getSubscriberCount(); - assertEquals("Wrong subscriber count", -1, subscribers); - } - } - - /** - * Some VEVO channels will redirect to a new page with a new channel id. - *

- * Though, it isn't a simple redirect, but a redirect instruction embed in the response itself, this - * test assure that we account for that. - */ - public static class RedirectedChannel implements BaseChannelExtractorTest { - private static YoutubeChannelExtractor extractor; - - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeChannelExtractor) YouTube - .getChannelExtractor("https://www.youtube.com/channel/UCITk7Ky4iE5_xISw9IaHqpQ"); - extractor.fetchPage(); - } - - /*////////////////////////////////////////////////////////////////////////// - // Extractor - //////////////////////////////////////////////////////////////////////////*/ - - @Test - public void testServiceId() { - assertEquals(YouTube.getServiceId(), extractor.getServiceId()); - } - - @Test - public void testName() throws Exception { - assertEquals("LordiVEVO", extractor.getName()); - } - - @Test - public void testId() throws Exception { - assertEquals("UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getId()); - } - - @Test - public void testUrl() throws ParsingException { - assertEquals("https://www.youtube.com/channel/UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getUrl()); - } - - @Test - public void testOriginalUrl() throws ParsingException { - assertEquals("https://www.youtube.com/channel/UCITk7Ky4iE5_xISw9IaHqpQ", extractor.getOriginalUrl()); - } - - /*////////////////////////////////////////////////////////////////////////// - // ListExtractor - //////////////////////////////////////////////////////////////////////////*/ - - @Test - public void testRelatedItems() throws Exception { - defaultTestRelatedItems(extractor); - } - - @Test - public void testMoreRelatedItems() throws Exception { - assertNoMoreItems(extractor); - } - - /*////////////////////////////////////////////////////////////////////////// - // ChannelExtractor - //////////////////////////////////////////////////////////////////////////*/ - - @Test - public void testDescription() throws Exception { - assertEmpty(extractor.getDescription()); - } - - @Test - public void testAvatarUrl() throws Exception { - String avatarUrl = extractor.getAvatarUrl(); - assertIsSecureUrl(avatarUrl); - assertTrue(avatarUrl, avatarUrl.contains("yt3")); - } - - @Test - public void testBannerUrl() throws Exception { - assertEmpty(extractor.getBannerUrl()); - } - - @Test - public void testFeedUrl() throws Exception { - assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getFeedUrl()); - } - - @Test - public void testSubscriberCount() throws Exception { - assertEquals(-1, extractor.getSubscriberCount()); - } - } - public static class RandomChannel implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java index eeac8c49..28c4519e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -153,8 +153,8 @@ public class YoutubeMusicSearchExtractorTest { public static class CorrectedSearch extends DefaultSearchExtractorTest { private static SearchExtractor extractor; - private static final String QUERY = "duo lipa"; - private static final String EXPECTED_SUGGESTION = "dua lipa"; + private static final String QUERY = "nocopyrigh sounds"; + private static final String EXPECTED_SUGGESTION = "nocopyrightsounds"; @BeforeClass public static void setUp() throws Exception { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 6a3aa4f2..cc627f7f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -62,48 +62,6 @@ public class YoutubeStreamExtractorDefaultTest { } } - public static class AdeleHello extends DefaultStreamExtractorTest { - private static final String ID = "YQHsXMglC9A"; - private static final String URL = BASE_URL + ID; - private static StreamExtractor extractor; - - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = YouTube.getStreamExtractor(URL); - extractor.fetchPage(); - } - - @Test - @Override - public void testUploaderUrl() throws ParsingException { - String url = extractor().getUploaderUrl(); - if (!url.equals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw") && - !url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) { - fail("Uploader url is neither the music channel one nor the Vevo one"); - } - } - - @Override public StreamExtractor extractor() { return extractor; } - @Override public StreamingService expectedService() { return YouTube; } - @Override public String expectedName() { return "Adele - Hello"; } - @Override public String expectedId() { return ID; } - @Override public String expectedUrlContains() { return URL; } - @Override public String expectedOriginalUrlContains() { return URL; } - - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } - @Override public String expectedUploaderName() { return "Adele"; } - @Override public String expectedUploaderUrl() { return null; } // overridden above - @Override public List expectedDescriptionContains() { return Arrays.asList("http://adele.com", "https://www.facebook.com/Adele"); } - @Override public long expectedLength() { return 367; } - @Override public long expectedViewCountAtLeast() { return 1220025784; } - @Nullable @Override public String expectedUploadDate() { return "2015-10-22 00:00:00.000"; } - @Nullable @Override public String expectedTextualUploadDate() { return "2015-10-22"; } - @Override public long expectedLikeCountAtLeast() { return 15289000; } - @Override public long expectedDislikeCountAtLeast() { return 826000; } - @Override public boolean expectedHasSubtitles() { return false; } - } - public static class DescriptionTestPewdiepie extends DefaultStreamExtractorTest { private static final String ID = "7PIMiDcwNvc"; private static final int TIMESTAMP = 17; From a39a2cca8261c7a6fe0286a503c3e16b8e22875d Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 25 Oct 2020 20:29:47 +0100 Subject: [PATCH 56/81] fix redirect channels --- .../extractors/YoutubeChannelExtractor.java | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 45d2ac36..beed3e87 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; - import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -18,13 +17,10 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; import java.io.IOException; -import javax.annotation.Nonnull; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -87,7 +83,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final String browseId = endpoint.getObject("browseEndpoint").getString("browseId", EMPTY_STRING); - if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") && !browseId.isEmpty()) { + if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") + || webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL") && !browseId.isEmpty()) { if (!browseId.startsWith("UC")) { throw new ExtractionException("Redirected id is not pointing to a channel"); } @@ -191,12 +188,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { throw new ParsingException("Could not get subscriber count", e); } } else { - // If there's no subscribe button, the channel has the subscriber count disabled - if (c4TabbedHeaderRenderer.has("subscribeButton")) { - return 0; - } else { - return -1; - } + return ITEM_COUNT_UNKNOWN; } } From c1e98579603412118d022333ba5dd1ce15fbb8fa Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 26 Oct 2020 18:48:20 +0100 Subject: [PATCH 57/81] fix subscriber count when subscribe is disabled fixes #305 --- .../services/youtube/extractors/YoutubeChannelExtractor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index beed3e87..338f2294 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -20,7 +20,9 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import java.io.IOException; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; From db0ef83d6bc89cda92ca7084b21d51cd68dbdb0d Mon Sep 17 00:00:00 2001 From: bopol Date: Tue, 27 Oct 2020 13:40:09 +0100 Subject: [PATCH 58/81] fix youtube decryption and three attemps bug fixes teamnewpipe/newpipe#4572 fixes #439 --- .../extractors/YoutubeStreamExtractor.java | 60 ++++++++++--------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 8364246c..44a43483 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; - import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; @@ -36,6 +35,8 @@ import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.text.SimpleDateFormat; @@ -49,9 +50,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; @@ -102,6 +100,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private JsonObject videoPrimaryInfoRenderer; private JsonObject videoSecondaryInfoRenderer; private int ageLimit; + private boolean newJsonScheme; @Nonnull private List subtitlesInfos = new ArrayList<>(); @@ -156,12 +155,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); Calendar parsedTime = timeAgoParser.parse(time).date(); return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime()); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } try { // Premiered Feb 21, 2020 Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time); return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime()); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } } try { @@ -169,7 +170,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))); return new SimpleDateFormat("yyyy-MM-dd").format(d); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } throw new ParsingException("Could not get upload date"); } @@ -360,7 +362,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") .getObject("videoOwnerRenderer").getObject("title")); - } catch (ParsingException ignored) { } + } catch (ParsingException ignored) { + } if (isNullOrEmpty(uploaderName)) { uploaderName = playerResponse.getObject("videoDetails").getString("author"); @@ -650,27 +653,23 @@ public class YoutubeStreamExtractor extends StreamExtractor { } else { ageLimit = NO_AGE_LIMIT; JsonObject playerConfig; - - // sometimes at random YouTube does not provide the player config, - // so just retry the same request three times - int attempts = 2; - while (true) { - playerConfig = initialAjaxJson.getObject(2).getObject("player", null); - if (playerConfig != null) { - break; - } - - if (attempts <= 0) { - throw new ParsingException( - "YouTube did not provide player config even after three attempts"); - } - initialAjaxJson = getJsonResponse(url, getExtractorLocalization()); - --attempts; - } initialData = initialAjaxJson.getObject(3).getObject("response"); - playerArgs = getPlayerArgs(playerConfig); - playerUrl = getPlayerUrl(playerConfig); + // sometimes at random YouTube does not provide the player config + playerConfig = initialAjaxJson.getObject(2).getObject("player", null); + + if (playerConfig == null) { + newJsonScheme = true; + final EmbeddedInfo info = getEmbeddedInfo(); + final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts); + final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody(); + videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse)); + playerUrl = info.url; + } else { + playerArgs = getPlayerArgs(playerConfig); + playerUrl = getPlayerUrl(playerConfig); + } + } playerResponse = getPlayerResponse(); @@ -718,6 +717,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { private JsonObject getPlayerResponse() throws ParsingException { try { String playerResponseStr; + if (newJsonScheme) { + return initialAjaxJson.getObject(2).getObject("playerResponse"); + } + if (playerArgs != null) { playerResponseStr = playerArgs.getString("player_response"); } else { @@ -988,7 +991,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { urlAndItags.put(streamUrl, itagItem); } - } catch (UnsupportedEncodingException ignored) {} + } catch (UnsupportedEncodingException ignored) { + } } } From 6dc5ab4015699488f4377b4b30ad8e30afc9f1a9 Mon Sep 17 00:00:00 2001 From: bopol Date: Tue, 27 Oct 2020 13:48:58 +0100 Subject: [PATCH 59/81] find playerUrl in another place when assetsPattern regex fails --- .../extractors/YoutubeStreamExtractor.java | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 44a43483..5ce57dc9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -3,6 +3,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; @@ -740,11 +744,30 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String embedPageContent = downloader.get(embedUrl, getExtractorLocalization()).responseBody(); // Get player url - final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")"; - String playerUrl = Parser.matchGroup1(assetsPattern, embedPageContent) - .replace("\\", "").replace("\"", ""); + String playerUrl = null; + try { + final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")"; + playerUrl = Parser.matchGroup1(assetsPattern, embedPageContent) + .replace("\\", "").replace("\"", ""); + } catch (Parser.RegexException ex) { + // playerUrl is still available in the file, just somewhere else + final Document doc = Jsoup.parse(embedPageContent); + final Elements elems = doc.select("script").attr("name", "player_ias/base"); + for (Element elem : elems) { + if (elem.attr("src").contains("base.js")) { + playerUrl = elem.attr("src"); + } + } + + if (playerUrl == null) { + throw new ParsingException("Could not get playerUrl"); + } + } + if (playerUrl.startsWith("//")) { playerUrl = HTTPS + playerUrl; + } else if (playerUrl.startsWith("/")) { + playerUrl = HTTPS + "//youtube.com" + playerUrl; } try { From 947ce3ee10d3a68947cfd7f3a3df6196f8414174 Mon Sep 17 00:00:00 2001 From: Scratch Date: Sun, 11 Oct 2020 14:12:38 +1100 Subject: [PATCH 60/81] Fix parsing Soundcloud tracks that contain the term 'sets' --- .../soundcloud/SoundcloudParsingHelper.java | 17 ++++++++++++++--- .../SoundcloudStreamLinkHandlerFactory.java | 2 -- .../SoundcloudStreamLinkHandlerFactoryTest.java | 2 ++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 29a625a9..0a3c400a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -25,6 +25,8 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; import java.net.URLEncoder; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -148,12 +150,21 @@ public class SoundcloudParsingHelper { * * @return the resolved id */ - public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException { + public static String resolveIdWithEmbedPlayer(String urlString) throws IOException, ReCaptchaException, ParsingException { + // Remove the tailing slash from URLs due to issues with the SoundCloud API + if (urlString.charAt(urlString.length() -1) == '/') urlString = urlString.substring(0, urlString.length()-1); + + URL url; + try { + url = Utils.stringToURL(urlString); + } catch (MalformedURLException e){ + throw new IllegalArgumentException("The given URL is not valid"); + } String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" - + URLEncoder.encode(url, "UTF-8"), SoundCloud.getLocalization()).responseBody(); + + URLEncoder.encode(url.toString(), "UTF-8"), SoundCloud.getLocalization()).responseBody(); // handle playlists / sets different and get playlist id via uir field in JSON - if (url.contains("sets") && !url.endsWith("sets") && !url.endsWith("sets/")) + if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets")) return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response); return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java index c70eff01..e271345d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java @@ -30,8 +30,6 @@ public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory { @Override public String getId(String url) throws ParsingException { Utils.checkUrl(URL_PATTERN, url); - // Remove the tailing slash from URLs due to issues with the SoundCloud API - if (url.charAt(url.length() -1) == '/') url = url.substring(0, url.length()-1); try { return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java index db2ae2b2..33137b0d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java @@ -53,6 +53,7 @@ public class SoundcloudStreamLinkHandlerFactoryTest { assertEquals("309689103", linkHandler.fromUrl("https://soundcloud.com/liluzivert/15-ysl").getId()); assertEquals("309689082", linkHandler.fromUrl("https://www.soundcloud.com/liluzivert/15-luv-scars-ko").getId()); assertEquals("309689035", linkHandler.fromUrl("http://soundcloud.com/liluzivert/15-boring-shit").getId()); + assertEquals("259273264", linkHandler.fromUrl("https://soundcloud.com/liluzivert/ps-qs-produced-by-don-cannon/").getId()); assertEquals("294488599", linkHandler.fromUrl("http://www.soundcloud.com/liluzivert/secure-the-bag-produced-by-glohan-beats").getId()); assertEquals("294488438", linkHandler.fromUrl("HtTpS://sOuNdClOuD.cOm/LiLuZiVeRt/In-O4-pRoDuCeD-bY-dP-bEaTz").getId()); assertEquals("294488147", linkHandler.fromUrl("https://soundcloud.com/liluzivert/fresh-produced-by-zaytoven#t=69").getId()); @@ -60,6 +61,7 @@ public class SoundcloudStreamLinkHandlerFactoryTest { assertEquals("294487684", linkHandler.fromUrl("https://soundcloud.com/liluzivert/blonde-brigitte-produced-manny-fresh#t=1:9").getId()); assertEquals("294487428", linkHandler.fromUrl("https://soundcloud.com/liluzivert/today-produced-by-c-note#t=1m9s").getId()); assertEquals("294487157", linkHandler.fromUrl("https://soundcloud.com/liluzivert/changed-my-phone-produced-by-c-note#t=1m09s").getId()); + assertEquals("44556776", linkHandler.fromUrl("https://soundcloud.com/kechuspider-sets-1/last-days").getId()); } From 19c0e8700db3af35e5dfc58e4eea50e75d7c4c61 Mon Sep 17 00:00:00 2001 From: Tobias Groza Date: Wed, 28 Oct 2020 07:29:42 +0100 Subject: [PATCH 61/81] 0.20.2 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f9d80df1..4701f58e 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.20.1' + version 'v0.20.2' group 'com.github.TeamNewPipe' repositories { From 30ed4f2d636dbb6112ae1951e646f8756ae93ff4 Mon Sep 17 00:00:00 2001 From: Stypox Date: Mon, 26 Oct 2020 21:22:21 +0100 Subject: [PATCH 62/81] Remove any reference to decrypt and improve error message --- .../youtube/extractors/YoutubeStreamExtractor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 0b7e8f86..708613f0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -637,7 +637,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String FORMATS = "formats"; private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String HTTPS = "https:"; - private static final String DEOBFUSCATION_FUNC_NAME = "decrypt"; + private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate"; private final static String[] REGEXES = { "(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)", @@ -794,7 +794,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (IOException e) { throw new ParsingException( - "Could load deobfuscation code form restricted video for the Youtube service.", e); + "Could not load deobfuscation code from YouTube video embed", e); } } @@ -839,8 +839,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { final Object result; try { final ScriptableObject scope = context.initSafeStandardObjects(); - context.evaluateString(scope, deobfuscationCode, "decryptionCode", 1, null); - final Function deobfuscateFunc = (Function) scope.get("decrypt", scope); + context.evaluateString(scope, deobfuscationCode, "deobfuscationCode", 1, null); + final Function deobfuscateFunc = (Function) scope.get(DEOBFUSCATION_FUNC_NAME, scope); result = deobfuscateFunc.call(context, scope, scope, new Object[]{obfuscatedSig}); } catch (Exception e) { throw new DeobfuscateException("Could not get deobfuscate signature", e); From b21e59925d05198f4860c0a658da2f9b1ecc73d5 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 29 Oct 2020 19:52:29 +0100 Subject: [PATCH 63/81] [PeerTube] fix account and channel extractors --- .../peertube/extractors/PeertubeAccountExtractor.java | 11 +++-------- .../peertube/extractors/PeertubeChannelExtractor.java | 11 +++-------- .../peertube/PeertubeAccountExtractorTest.java | 5 ++--- .../peertube/PeertubeChannelExtractorTest.java | 3 +-- 4 files changed, 9 insertions(+), 21 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index b41bf205..79876765 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; - import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -19,14 +18,10 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; import java.io.IOException; -import javax.annotation.Nonnull; - -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class PeertubeAccountExtractor extends ChannelExtractor { @@ -92,7 +87,7 @@ public class PeertubeAccountExtractor extends ChannelExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { return getPage(new Page( - getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE)); + baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE)); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index 432433cd..dc5a54fd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; - import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -19,14 +18,10 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; import java.io.IOException; -import javax.annotation.Nonnull; - -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -99,7 +94,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { return getPage(new Page( - getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE)); + baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE)); } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java index 1c315ff1..44da4b42 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java @@ -89,10 +89,9 @@ public class PeertubeAccountExtractorTest { assertIsSecureUrl(extractor.getAvatarUrl()); } - @Ignore @Test - public void testBannerUrl() throws ParsingException { - assertIsSecureUrl(extractor.getBannerUrl()); + public void testBannerUrl() { + assertNull(extractor.getBannerUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index f9e686c8..a3cd46dd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -104,10 +104,9 @@ public class PeertubeChannelExtractorTest { assertIsSecureUrl(extractor.getAvatarUrl()); } - @Ignore @Test public void testBannerUrl() throws ParsingException { - assertIsSecureUrl(extractor.getBannerUrl()); + assertNull(extractor.getBannerUrl()); } @Test From c190a3029bf0a3ae36e584f1e74e83e4ba8137c4 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 27 Jun 2020 22:58:12 +0200 Subject: [PATCH 64/81] Consider protocol as base url when it is a custom one (e.g. vnd.youtube) --- .../services/youtube/YoutubeParsingHelper.java | 6 ------ .../YoutubeCommentsLinkHandlerFactory.java | 12 ------------ .../linkHandler/YoutubeStreamLinkHandlerFactory.java | 12 ------------ .../org/schabi/newpipe/extractor/utils/Utils.java | 11 ++++++++--- .../schabi/newpipe/extractor/utils/UtilsTest.java | 9 +++++++++ 5 files changed, 17 insertions(+), 33 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index ac7db985..a8ca2a03 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -55,12 +55,6 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } - /** - * The official youtube app supports intents in this format, where after the ':' is the videoId. - * Accordingly there are other apps sharing streams in this format. - */ - public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube"; - private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static String clientVersion; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java index 15bc31b6..421fc13f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java @@ -1,10 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL; - import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import java.util.List; @@ -17,15 +14,6 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { return instance; } - @Override - public ListLinkHandler fromUrl(String url) throws ParsingException { - if (url.startsWith(BASE_YOUTUBE_INTENT_URL)){ - return super.fromUrl(url, BASE_YOUTUBE_INTENT_URL); - } else { - return super.fromUrl(url); - } - } - @Override public String getUrl(String id) { return "https://m.youtube.com/watch?v=" + id; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index bcc1fac5..efc06da2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; @@ -15,8 +14,6 @@ import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL; - /* * Created by Christian Schabesberger on 02.02.16. * @@ -67,15 +64,6 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { } } - @Override - public LinkHandler fromUrl(String url) throws ParsingException { - if (url.startsWith(BASE_YOUTUBE_INTENT_URL)) { - return super.fromUrl(url, BASE_YOUTUBE_INTENT_URL); - } else { - return super.fromUrl(url); - } - } - @Override public String getUrl(String id) { return "https://www.youtube.com/watch?v=" + id; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 288e401c..c6bd508a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -182,13 +182,18 @@ public class Utils { } public static String getBaseUrl(String url) throws ParsingException { - URL uri; try { - uri = stringToURL(url); + final URL uri = stringToURL(url); + return uri.getProtocol() + "://" + uri.getAuthority(); } catch (MalformedURLException e) { + final String message = e.getMessage(); + if (message.startsWith("unknown protocol: ")) { + System.out.println(message.substring(18)); + return message.substring(18); // return just the protocol (e.g. vnd.youtube) + } + throw new ParsingException("Malformed url: " + url, e); } - return uri.getProtocol() + "://" + uri.getAuthority(); } public static boolean isNullOrEmpty(final String str) { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java index 5b0dfdb3..dcccc16d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java @@ -21,4 +21,13 @@ public class UtilsTest { public void testJoin() { assertEquals("some,random,stuff", Utils.join(",", Arrays.asList("some", "random", "stuff"))); } + + @Test + public void testGetBaseUrl() throws ParsingException { + assertEquals("https://www.youtube.com", Utils.getBaseUrl("https://www.youtube.com/watch?v=Hu80uDzh8RY")); + assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI")); + assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube:jZViOEv90dI")); + assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube://n8X9_MgEdCg")); + assertEquals("https://music.youtube.com", Utils.getBaseUrl("https://music.youtube.com/watch?v=O0EDx9WAelc")); + } } From 3fe55b30ba43ff44241e394d8b021b8c78c16e18 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 27 Jun 2020 15:47:49 +0200 Subject: [PATCH 65/81] Add support for Google search redirect url --- .../newpipe/extractor/StreamingService.java | 10 ++++++---- .../linkhandler/LinkHandlerFactory.java | 18 +++++++++++++++++ .../linkhandler/ListLinkHandlerFactory.java | 3 ++- .../schabi/newpipe/extractor/utils/Utils.java | 20 +++++++++++++++++++ .../schabi/newpipe/extractor/NewPipeTest.java | 5 ++++- .../newpipe/extractor/utils/UtilsTest.java | 15 ++++++++++++++ 6 files changed, 65 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index e21b17f3..3d09d509 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; +import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nullable; import java.util.Collections; @@ -277,12 +278,13 @@ public abstract class StreamingService { * Figures out where the link is pointing to (a channel, a video, a playlist, etc.) * @param url the url on which it should be decided of which link type it is * @return the link type of url - * @throws ParsingException */ public final LinkType getLinkTypeByUrl(String url) throws ParsingException { - LinkHandlerFactory sH = getStreamLHFactory(); - LinkHandlerFactory cH = getChannelLHFactory(); - LinkHandlerFactory pH = getPlaylistLHFactory(); + url = Utils.followGoogleRedirectIfNeeded(url); + + final LinkHandlerFactory sH = getStreamLHFactory(); + final LinkHandlerFactory cH = getChannelLHFactory(); + final LinkHandlerFactory pH = getPlaylistLHFactory(); if (sH != null && sH.acceptUrl(url)) { return LinkType.STREAM; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java index 6bba7b4e..7dcfe5f4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java @@ -42,12 +42,30 @@ public abstract class LinkHandlerFactory { // Logic /////////////////////////////////// + /** + * Builds a {@link LinkHandler} from a url.
+ * Be sure to call {@link Utils#followGoogleRedirectIfNeeded(String)} on the url if overriding + * this function. + * @param url the url to extract path and id from + * @return a {@link LinkHandler} complete with information + */ public LinkHandler fromUrl(String url) throws ParsingException { if (url == null) throw new IllegalArgumentException("url can not be null"); + url = Utils.followGoogleRedirectIfNeeded(url); final String baseUrl = Utils.getBaseUrl(url); return fromUrl(url, baseUrl); } + /** + * Builds a {@link LinkHandler} from a url and a base url. The url is expected to be already + * polished from google search redirects (otherwise how could {@code baseUrl} have been + * extracted?).
+ * So do not call {@link Utils#followGoogleRedirectIfNeeded(String)} on the url if overriding + * this function, since that should be done in {@link #fromUrl(String)}. + * @param url the url without google search redirects to extract id from + * @param baseUrl the base url + * @return a {@link LinkHandler} complete with information + */ public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException { if (url == null) throw new IllegalArgumentException("url can not be null"); if (!acceptUrl(url)) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java index 9ea478b0..cdbbab4f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java @@ -32,7 +32,8 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { @Override public ListLinkHandler fromUrl(String url) throws ParsingException { - String baseUrl = Utils.getBaseUrl(url); + url = Utils.followGoogleRedirectIfNeeded(url); + final String baseUrl = Utils.getBaseUrl(url); return fromUrl(url, baseUrl); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index c6bd508a..e8823af8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -196,6 +196,26 @@ public class Utils { } } + /** + * If the provided url is a Google search redirect, then the actual url is extracted from the + * {@code url=} query value and returned, otherwise the original url is returned. + * @param url the url which can possibly be a Google search redirect + * @return an url with no Google search redirects + */ + public static String followGoogleRedirectIfNeeded(final String url) { + // if the url is a redirect from a Google search, extract the actual url + try { + final URL decoded = Utils.stringToURL(url); + if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) { + return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), "UTF-8"); + } + } catch (final Exception ignored) { + } + + // url is not a google search redirect + return url; + } + public static boolean isNullOrEmpty(final String str) { return str == null || str.isEmpty(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java index bdad6cb6..5dbc4317 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java @@ -6,6 +6,7 @@ import java.util.HashSet; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.YouTube; public class NewPipeTest { @@ -39,8 +40,10 @@ public class NewPipeTest { assertEquals(getServiceByUrl("https://www.youtube.com/watch?v=_r6CgaFNAGg"), YouTube); assertEquals(getServiceByUrl("https://www.youtube.com/channel/UCi2bIyFtz-JdI-ou8kaqsqg"), YouTube); assertEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), YouTube); + assertEquals(getServiceByUrl("https://www.google.it/url?sa=t&rct=j&q=&esrc=s&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DHu80uDzh8RY&source=video"), YouTube); - assertNotEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), YouTube); + assertEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), SoundCloud); + assertEquals(getServiceByUrl("https://www.google.com/url?sa=t&url=https%3A%2F%2Fsoundcloud.com%2Fciaoproduction&rct=j&q=&esrc=s&source=web&cd="), SoundCloud); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java index dcccc16d..e4a65505 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java @@ -30,4 +30,19 @@ public class UtilsTest { assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube://n8X9_MgEdCg")); assertEquals("https://music.youtube.com", Utils.getBaseUrl("https://music.youtube.com/watch?v=O0EDx9WAelc")); } + + @Test + public void testFollowGoogleRedirect() { + assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY", + Utils.followGoogleRedirectIfNeeded("https://www.google.it/url?sa=t&rct=j&q=&esrc=s&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DHu80uDzh8RY&source=video")); + assertEquals("https://www.youtube.com/watch?v=0b6cFWG45kA", + Utils.followGoogleRedirectIfNeeded("https://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=video&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3D0b6cFWG45kA")); + assertEquals("https://soundcloud.com/ciaoproduction", + Utils.followGoogleRedirectIfNeeded("https://www.google.com/url?sa=t&url=https%3A%2F%2Fsoundcloud.com%2Fciaoproduction&rct=j&q=&esrc=s&source=web&cd=")); + + assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY¶m=xyz", + Utils.followGoogleRedirectIfNeeded("https://www.youtube.com/watch?v=Hu80uDzh8RY¶m=xyz")); + assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY&url=hello", + Utils.followGoogleRedirectIfNeeded("https://www.youtube.com/watch?v=Hu80uDzh8RY&url=hello")); + } } From 9e53cf0b56652c37e68d3c5a03624f0b6d8fff05 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sun, 28 Jun 2020 22:44:16 +0200 Subject: [PATCH 66/81] Fix parameter reassignment and other style issues Also remove left-behind debug statement --- .../org/schabi/newpipe/extractor/StreamingService.java | 10 +++++----- .../extractor/linkhandler/LinkHandlerFactory.java | 9 ++++----- .../extractor/linkhandler/ListLinkHandlerFactory.java | 8 ++++---- .../java/org/schabi/newpipe/extractor/utils/Utils.java | 8 ++++---- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 3d09d509..dcde0aff 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -279,18 +279,18 @@ public abstract class StreamingService { * @param url the url on which it should be decided of which link type it is * @return the link type of url */ - public final LinkType getLinkTypeByUrl(String url) throws ParsingException { - url = Utils.followGoogleRedirectIfNeeded(url); + public final LinkType getLinkTypeByUrl(final String url) throws ParsingException { + final String polishedUrl = Utils.followGoogleRedirectIfNeeded(url); final LinkHandlerFactory sH = getStreamLHFactory(); final LinkHandlerFactory cH = getChannelLHFactory(); final LinkHandlerFactory pH = getPlaylistLHFactory(); - if (sH != null && sH.acceptUrl(url)) { + if (sH != null && sH.acceptUrl(polishedUrl)) { return LinkType.STREAM; - } else if (cH != null && cH.acceptUrl(url)) { + } else if (cH != null && cH.acceptUrl(polishedUrl)) { return LinkType.CHANNEL; - } else if (pH != null && pH.acceptUrl(url)) { + } else if (pH != null && pH.acceptUrl(polishedUrl)) { return LinkType.PLAYLIST; } else { return LinkType.NONE; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java index 7dcfe5f4..ca428b70 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java @@ -49,11 +49,10 @@ public abstract class LinkHandlerFactory { * @param url the url to extract path and id from * @return a {@link LinkHandler} complete with information */ - public LinkHandler fromUrl(String url) throws ParsingException { - if (url == null) throw new IllegalArgumentException("url can not be null"); - url = Utils.followGoogleRedirectIfNeeded(url); - final String baseUrl = Utils.getBaseUrl(url); - return fromUrl(url, baseUrl); + public LinkHandler fromUrl(final String url) throws ParsingException { + final String polishedUrl = Utils.followGoogleRedirectIfNeeded(url); + final String baseUrl = Utils.getBaseUrl(polishedUrl); + return fromUrl(polishedUrl, baseUrl); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java index cdbbab4f..4980c319 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java @@ -31,10 +31,10 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { /////////////////////////////////// @Override - public ListLinkHandler fromUrl(String url) throws ParsingException { - url = Utils.followGoogleRedirectIfNeeded(url); - final String baseUrl = Utils.getBaseUrl(url); - return fromUrl(url, baseUrl); + public ListLinkHandler fromUrl(final String url) throws ParsingException { + final String polishedUrl = Utils.followGoogleRedirectIfNeeded(url); + final String baseUrl = Utils.getBaseUrl(polishedUrl); + return fromUrl(polishedUrl, baseUrl); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index e8823af8..95920270 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -181,15 +181,15 @@ public class Utils { return s; } - public static String getBaseUrl(String url) throws ParsingException { + public static String getBaseUrl(final String url) throws ParsingException { try { final URL uri = stringToURL(url); return uri.getProtocol() + "://" + uri.getAuthority(); - } catch (MalformedURLException e) { + } catch (final MalformedURLException e) { final String message = e.getMessage(); if (message.startsWith("unknown protocol: ")) { - System.out.println(message.substring(18)); - return message.substring(18); // return just the protocol (e.g. vnd.youtube) + // return just the protocol (e.g. vnd.youtube) + return message.substring("unknown protocol: ".length()); } throw new ParsingException("Malformed url: " + url, e); From 0526a5148dfc6b4b8955574c8f0e861aa60ac2b1 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Thu, 1 Oct 2020 15:22:21 +0530 Subject: [PATCH 67/81] Set source and target versions to Java 8. --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 4701f58e..070ed43f 100644 --- a/build.gradle +++ b/build.gradle @@ -2,8 +2,8 @@ allprojects { apply plugin: 'java-library' apply plugin: 'maven' - sourceCompatibility = 1.7 - targetCompatibility = 1.7 + sourceCompatibility = 1.8 + targetCompatibility = 1.8 version 'v0.20.2' group 'com.github.TeamNewPipe' From ee3af63c0475bb2cd96be7675c671a32edff2c18 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Sun, 18 Oct 2020 07:52:28 +0530 Subject: [PATCH 68/81] Switch to ChronoUnit. --- .../extractor/localization/TimeAgoParser.java | 24 +++++++------- .../extractor/timeago/PatternsHolder.java | 32 ++++++++----------- .../extractor/timeago/TimeAgoUnit.java | 11 ------- .../extractor/timeago/patterns/iw.java | 13 ++++---- 4 files changed, 33 insertions(+), 47 deletions(-) delete mode 100644 timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/TimeAgoUnit.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java index fe20135f..83b4aed2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java @@ -2,9 +2,9 @@ package org.schabi.newpipe.extractor.localization; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.timeago.PatternsHolder; -import org.schabi.newpipe.extractor.timeago.TimeAgoUnit; import org.schabi.newpipe.extractor.utils.Parser; +import java.time.temporal.ChronoUnit; import java.util.Calendar; import java.util.Collection; import java.util.Map; @@ -42,14 +42,14 @@ public class TimeAgoParser { * @throws ParsingException if the time unit could not be recognized */ public DateWrapper parse(String textualDate) throws ParsingException { - for (Map.Entry> caseUnitEntry : patternsHolder.specialCases().entrySet()) { - final TimeAgoUnit timeAgoUnit = caseUnitEntry.getKey(); + for (Map.Entry> caseUnitEntry : patternsHolder.specialCases().entrySet()) { + final ChronoUnit chronoUnit = caseUnitEntry.getKey(); for (Map.Entry caseMapToAmountEntry : caseUnitEntry.getValue().entrySet()) { final String caseText = caseMapToAmountEntry.getKey(); final Integer caseAmount = caseMapToAmountEntry.getValue(); if (textualDateMatches(textualDate, caseText)) { - return getResultFor(caseAmount, timeAgoUnit); + return getResultFor(caseAmount, chronoUnit); } } } @@ -63,8 +63,8 @@ public class TimeAgoParser { timeAgoAmount = 1; } - final TimeAgoUnit timeAgoUnit = parseTimeAgoUnit(textualDate); - return getResultFor(timeAgoAmount, timeAgoUnit); + final ChronoUnit chronoUnit = parseChronoUnit(textualDate); + return getResultFor(timeAgoAmount, chronoUnit); } private int parseTimeAgoAmount(String textualDate) throws NumberFormatException { @@ -72,13 +72,13 @@ public class TimeAgoParser { return Integer.parseInt(timeValueStr); } - private TimeAgoUnit parseTimeAgoUnit(String textualDate) throws ParsingException { - for (Map.Entry> entry : patternsHolder.asMap().entrySet()) { - final TimeAgoUnit timeAgoUnit = entry.getKey(); + private ChronoUnit parseChronoUnit(String textualDate) throws ParsingException { + for (Map.Entry> entry : patternsHolder.asMap().entrySet()) { + final ChronoUnit chronoUnit = entry.getKey(); for (String agoPhrase : entry.getValue()) { if (textualDateMatches(textualDate, agoPhrase)) { - return timeAgoUnit; + return chronoUnit; } } } @@ -112,11 +112,11 @@ public class TimeAgoParser { } } - private DateWrapper getResultFor(int timeAgoAmount, TimeAgoUnit timeAgoUnit) { + private DateWrapper getResultFor(int timeAgoAmount, ChronoUnit chronoUnit) { final Calendar calendarTime = getNow(); boolean isApproximation = false; - switch (timeAgoUnit) { + switch (chronoUnit) { case SECONDS: calendarTime.add(Calendar.SECOND, -timeAgoAmount); break; diff --git a/timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/PatternsHolder.java b/timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/PatternsHolder.java index 91f93102..f7dc9963 100644 --- a/timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/PatternsHolder.java +++ b/timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/PatternsHolder.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.timeago; +import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; @@ -16,7 +17,7 @@ public abstract class PatternsHolder { private final Collection months; private final Collection years; - private final Map> specialCases = new LinkedHashMap<>(); + private final Map> specialCases = new LinkedHashMap<>(); protected PatternsHolder(String wordSeparator, Collection seconds, Collection minutes, Collection hours, Collection days, @@ -69,30 +70,25 @@ public abstract class PatternsHolder { return years; } - public Map> specialCases() { + public Map> specialCases() { return specialCases; } - protected void putSpecialCase(TimeAgoUnit unit, String caseText, int caseAmount) { - Map item = specialCases.get(unit); - - if (item == null) { - item = new LinkedHashMap<>(); - specialCases.put(unit, item); - } + protected void putSpecialCase(ChronoUnit unit, String caseText, int caseAmount) { + Map item = specialCases.computeIfAbsent(unit, k -> new LinkedHashMap<>()); item.put(caseText, caseAmount); } - public Map> asMap() { - final Map> returnMap = new LinkedHashMap<>(); - returnMap.put(TimeAgoUnit.SECONDS, seconds()); - returnMap.put(TimeAgoUnit.MINUTES, minutes()); - returnMap.put(TimeAgoUnit.HOURS, hours()); - returnMap.put(TimeAgoUnit.DAYS, days()); - returnMap.put(TimeAgoUnit.WEEKS, weeks()); - returnMap.put(TimeAgoUnit.MONTHS, months()); - returnMap.put(TimeAgoUnit.YEARS, years()); + public Map> asMap() { + final Map> returnMap = new LinkedHashMap<>(); + returnMap.put(ChronoUnit.SECONDS, seconds()); + returnMap.put(ChronoUnit.MINUTES, minutes()); + returnMap.put(ChronoUnit.HOURS, hours()); + returnMap.put(ChronoUnit.DAYS, days()); + returnMap.put(ChronoUnit.WEEKS, weeks()); + returnMap.put(ChronoUnit.MONTHS, months()); + returnMap.put(ChronoUnit.YEARS, years()); return returnMap; } diff --git a/timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/TimeAgoUnit.java b/timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/TimeAgoUnit.java deleted file mode 100644 index 16e05c24..00000000 --- a/timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/TimeAgoUnit.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.schabi.newpipe.extractor.timeago; - -public enum TimeAgoUnit { - SECONDS, - MINUTES, - HOURS, - DAYS, - WEEKS, - MONTHS, - YEARS -} diff --git a/timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/patterns/iw.java b/timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/patterns/iw.java index a04bf74f..6f89e025 100644 --- a/timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/patterns/iw.java +++ b/timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/patterns/iw.java @@ -5,7 +5,8 @@ package org.schabi.newpipe.extractor.timeago.patterns; import org.schabi.newpipe.extractor.timeago.PatternsHolder; -import org.schabi.newpipe.extractor.timeago.TimeAgoUnit; + +import java.time.temporal.ChronoUnit; public class iw extends PatternsHolder { private static final String WORD_SEPARATOR = " "; @@ -26,10 +27,10 @@ public class iw extends PatternsHolder { private iw() { super(WORD_SEPARATOR, SECONDS, MINUTES, HOURS, DAYS, WEEKS, MONTHS, YEARS); - putSpecialCase(TimeAgoUnit.HOURS, "שעתיים", 2); - putSpecialCase(TimeAgoUnit.DAYS, "יומיים", 2); - putSpecialCase(TimeAgoUnit.WEEKS, "שבועיים", 2); - putSpecialCase(TimeAgoUnit.MONTHS, "חודשיים", 2); - putSpecialCase(TimeAgoUnit.YEARS, "שנתיים", 2); + putSpecialCase(ChronoUnit.HOURS, "שעתיים", 2); + putSpecialCase(ChronoUnit.DAYS, "יומיים", 2); + putSpecialCase(ChronoUnit.WEEKS, "שבועיים", 2); + putSpecialCase(ChronoUnit.MONTHS, "חודשיים", 2); + putSpecialCase(ChronoUnit.YEARS, "שנתיים", 2); } } \ No newline at end of file From 4f04cfcccaaf49bb5362bb7a02f6450c446934b6 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Sun, 18 Oct 2020 09:18:14 +0530 Subject: [PATCH 69/81] Switch from Calendar to OffsetDateTime in DateWrapper. --- .../extractor/localization/DateWrapper.java | 47 ++++++++++++++--- .../extractor/localization/TimeAgoParser.java | 51 ++++--------------- .../extractors/MediaCCCParsingHelper.java | 21 ++------ .../peertube/PeertubeParsingHelper.java | 23 +++------ .../soundcloud/SoundcloudParsingHelper.java | 27 ++++------ .../youtube/YoutubeParsingHelper.java | 27 +++++----- .../YoutubeFeedInfoItemExtractor.java | 20 ++------ .../extractors/YoutubeStreamExtractor.java | 21 ++++---- .../YoutubeStreamInfoItemExtractor.java | 22 ++++---- .../services/DefaultStreamExtractorTest.java | 18 +++---- .../YoutubeChannelLocalizationTest.java | 19 ++++--- 11 files changed, 130 insertions(+), 166 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java index 43328785..0390ee12 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java @@ -3,30 +3,61 @@ package org.schabi.newpipe.extractor.localization; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.Serializable; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.Calendar; +import java.util.GregorianCalendar; /** - * A wrapper class that provides a field to describe if the date is precise or just an approximation. + * A wrapper class that provides a field to describe if the date/time is precise or just an approximation. */ public class DateWrapper implements Serializable { - @NonNull private final Calendar date; + @NonNull private final OffsetDateTime offsetDateTime; private final boolean isApproximation; - public DateWrapper(@NonNull Calendar date) { - this(date, false); + /** + * @deprecated Use {@link #DateWrapper(OffsetDateTime)} instead. + */ + @Deprecated + public DateWrapper(@NonNull Calendar calendar) { + this(calendar, false); } - public DateWrapper(@NonNull Calendar date, boolean isApproximation) { - this.date = date; + /** + * @deprecated Use {@link #DateWrapper(OffsetDateTime, boolean)} instead. + */ + @Deprecated + public DateWrapper(@NonNull Calendar calendar, boolean isApproximation) { + offsetDateTime = OffsetDateTime.ofInstant(calendar.toInstant(), ZoneOffset.UTC); + this.isApproximation = isApproximation; + } + + public DateWrapper(@NonNull OffsetDateTime offsetDateTime) { + this(offsetDateTime, false); + } + + public DateWrapper(@NonNull OffsetDateTime offsetDateTime, boolean isApproximation) { + this.offsetDateTime = offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC); this.isApproximation = isApproximation; } /** - * @return the wrapped date. + * @return the wrapped date/time as a {@link Calendar}. + * + * @deprecated use {@link #offsetDateTime()} instead. */ + @Deprecated @NonNull public Calendar date() { - return date; + return GregorianCalendar.from(offsetDateTime.toZonedDateTime()); + } + + /** + * @return the wrapped date/time. + */ + @NonNull + public OffsetDateTime offsetDateTime() { + return offsetDateTime; } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java index 83b4aed2..dd74f9aa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java @@ -4,8 +4,9 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.timeago.PatternsHolder; import org.schabi.newpipe.extractor.utils.Parser; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; -import java.util.Calendar; import java.util.Collection; import java.util.Map; import java.util.regex.Pattern; @@ -16,7 +17,7 @@ import java.util.regex.Pattern; */ public class TimeAgoParser { private final PatternsHolder patternsHolder; - private final Calendar consistentNow; + private final OffsetDateTime now; /** * Creates a helper to parse upload dates in the format '2 days ago'. @@ -28,7 +29,7 @@ public class TimeAgoParser { */ public TimeAgoParser(PatternsHolder patternsHolder) { this.patternsHolder = patternsHolder; - consistentNow = Calendar.getInstance(); + now = OffsetDateTime.now(ZoneOffset.UTC); } /** @@ -113,64 +114,34 @@ public class TimeAgoParser { } private DateWrapper getResultFor(int timeAgoAmount, ChronoUnit chronoUnit) { - final Calendar calendarTime = getNow(); + OffsetDateTime offsetDateTime = now; boolean isApproximation = false; switch (chronoUnit) { case SECONDS: - calendarTime.add(Calendar.SECOND, -timeAgoAmount); - break; - case MINUTES: - calendarTime.add(Calendar.MINUTE, -timeAgoAmount); - break; - case HOURS: - calendarTime.add(Calendar.HOUR_OF_DAY, -timeAgoAmount); + offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit); break; case DAYS: - calendarTime.add(Calendar.DAY_OF_MONTH, -timeAgoAmount); - isApproximation = true; - break; - case WEEKS: - calendarTime.add(Calendar.WEEK_OF_YEAR, -timeAgoAmount); - isApproximation = true; - break; - case MONTHS: - calendarTime.add(Calendar.MONTH, -timeAgoAmount); + offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit); isApproximation = true; break; case YEARS: - calendarTime.add(Calendar.YEAR, -timeAgoAmount); - // Prevent `PrettyTime` from showing '12 months ago'. - calendarTime.add(Calendar.DAY_OF_MONTH, -1); + // minusDays is needed to prevent `PrettyTime` from showing '12 months ago'. + offsetDateTime = offsetDateTime.minusYears(timeAgoAmount).minusDays(1); isApproximation = true; break; } if (isApproximation) { - markApproximatedTime(calendarTime); + offsetDateTime = offsetDateTime.truncatedTo(ChronoUnit.HOURS); } - return new DateWrapper(calendarTime, isApproximation); - } - - private Calendar getNow() { - return (Calendar) consistentNow.clone(); - } - - /** - * Marks the time as approximated by setting minutes, seconds and milliseconds to 0. - * - * @param calendarTime Time to be marked as approximated - */ - private void markApproximatedTime(Calendar calendarTime) { - calendarTime.set(Calendar.MINUTE, 0); - calendarTime.set(Calendar.SECOND, 0); - calendarTime.set(Calendar.MILLISECOND, 0); + return new DateWrapper(offsetDateTime, isApproximation); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java index 14ca5c52..b6879fcd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java @@ -2,28 +2,17 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.TimeZone; +import java.time.OffsetDateTime; +import java.time.format.DateTimeParseException; public final class MediaCCCParsingHelper { private MediaCCCParsingHelper() { } - public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException { - final Date date; + public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException { try { - final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - sdf.setTimeZone(TimeZone.getTimeZone("GMT")); - date = sdf.parse(textualUploadDate); - } catch (ParseException e) { + return OffsetDateTime.parse(textualUploadDate); + } catch (DateTimeParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } - - final Calendar uploadDate = Calendar.getInstance(); - uploadDate.setTime(date); - return uploadDate; } - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index 5c6ceac4..42d183f4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.peertube; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; - import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; @@ -12,11 +11,10 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.TimeZone; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeParseException; public class PeertubeParsingHelper { public static final String START_KEY = "start"; @@ -34,19 +32,12 @@ public class PeertubeParsingHelper { } } - public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException { - final Date date; + public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException { try { - final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'"); - sdf.setTimeZone(TimeZone.getTimeZone("GMT")); - date = sdf.parse(textualUploadDate); - } catch (ParseException e) { + return OffsetDateTime.ofInstant(Instant.parse(textualUploadDate), ZoneOffset.UTC); + } catch (DateTimeParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } - - final Calendar uploadDate = Calendar.getInstance(); - uploadDate.setTime(date); - return uploadDate; } public static Page getNextPage(final String prevPageUrl, final long total) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 0a3c400a..566acd39 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -21,16 +21,18 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStr import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; -import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLEncoder; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.*; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; import static java.util.Collections.singletonList; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; @@ -95,23 +97,16 @@ public class SoundcloudParsingHelper { } } - public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { - Date date; + public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException { try { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - sdf.setTimeZone(TimeZone.getTimeZone("GMT")); - date = sdf.parse(textualUploadDate); - } catch (ParseException e1) { + return OffsetDateTime.parse(textualUploadDate); + } catch (DateTimeParseException e1) { try { - date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(textualUploadDate); - } catch (ParseException e2) { + return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss +0000")); + } catch (DateTimeParseException e2) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + ", " + e1.getMessage(), e2); } } - - final Calendar uploadDate = Calendar.getInstance(); - uploadDate.setTime(date); - return uploadDate; } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index a8ca2a03..3fb279b8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -5,7 +5,6 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; - import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; @@ -22,13 +21,18 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.*; +import java.time.OffsetDateTime; +import java.time.format.DateTimeParseException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; -import static org.schabi.newpipe.extractor.utils.Utils.*; +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; /* * Created by Christian Schabesberger on 02.03.16. @@ -176,19 +180,12 @@ public class YoutubeParsingHelper { } } - public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { - final Date date; + public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException { try { - final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - sdf.setTimeZone(TimeZone.getTimeZone("GMT")); - date = sdf.parse(textualUploadDate); - } catch (ParseException e) { + return OffsetDateTime.parse(textualUploadDate); + } catch (DateTimeParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } - - final Calendar uploadDate = Calendar.getInstance(); - uploadDate.setTime(date); - return uploadDate; } public static JsonObject getInitialData(String html) throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java index aadc8022..cea38518 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java @@ -7,11 +7,9 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import javax.annotation.Nullable; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.TimeZone; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { private final Element entryElement; @@ -62,19 +60,11 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - final Date date; try { - final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+00:00"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - date = dateFormat.parse(getTextualUploadDate()); - } catch (ParseException e) { + return new DateWrapper(OffsetDateTime.parse(getTextualUploadDate(), DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + } catch (DateTimeParseException e) { throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", e); } - - final Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - - return new DateWrapper(calendar); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 708613f0..d86c63d9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -43,11 +43,11 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Calendar; import java.util.Collections; -import java.util.Date; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -157,23 +157,24 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { // Premiered 20 hours ago TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); - Calendar parsedTime = timeAgoParser.parse(time).date(); - return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime()); + OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime(); + return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime); } catch (Exception ignored) { } try { // Premiered Feb 21, 2020 - Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time); - return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime()); + LocalDate localDate = LocalDate.parse(time, + DateTimeFormatter.ofPattern("MMM dd, YYYY", Locale.ENGLISH)); + return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate); } catch (Exception ignored) { } } try { // TODO: this parses English formatted dates only, we need a better approach to parse the textual date - Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( - getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))); - return new SimpleDateFormat("yyyy-MM-dd").format(d); + LocalDate localDate = LocalDate.parse(getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")), + DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH)); + return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate); } catch (Exception ignored) { } throw new ParsingException("Could not get upload date"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 0757208e..ae784f20 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -12,11 +12,14 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nullable; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -165,8 +168,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { } if (isPremiere()) { - final Date date = getDateFromPremiere().getTime(); - return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date); + return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(getDateFromPremiere()); } final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText")); @@ -250,15 +252,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { return videoInfo.has("upcomingEventData"); } - private Calendar getDateFromPremiere() throws ParsingException { + private OffsetDateTime getDateFromPremiere() throws ParsingException { final JsonObject upcomingEventData = videoInfo.getObject("upcomingEventData"); final String startTime = upcomingEventData.getString("startTime"); try { - final long startTimeTimestamp = Long.parseLong(startTime); - final Calendar calendar = Calendar.getInstance(); - calendar.setTime(new Date(startTimeTimestamp * 1000L)); - return calendar; + return OffsetDateTime.ofInstant(Instant.ofEpochSecond(Long.parseLong(startTime)), + ZoneOffset.UTC); } catch (Exception e) { throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\""); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index a70b38a8..19c767ae 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -12,14 +12,12 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; -import java.text.SimpleDateFormat; -import java.util.Calendar; +import javax.annotation.Nullable; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.Collections; import java.util.List; import java.util.Locale; -import java.util.TimeZone; - -import javax.annotation.Nullable; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.MatcherAssert.assertThat; @@ -171,13 +169,11 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest Date: Sun, 18 Oct 2020 12:49:19 +0530 Subject: [PATCH 70/81] Update README.md. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 36e2f7b5..11e616f3 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ If you're using Gradle, you could add NewPipe Extractor as a dependency with the 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. 2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.1'`the `dependencies` in your `build.gradle`. Replace `v0.20.1` with the latest release. +**Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required. + ### Testing changes To test changes quickly you can build the library locally. A good approach would be to add something like the following to your `settings.gradle`: From be9e160333758d87f939be66093513939c2a6bdb Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 1 Nov 2020 17:04:02 +0100 Subject: [PATCH 71/81] Fix build --- .../extractor/services/soundcloud/SoundcloudParsingHelper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 566acd39..3cc39d50 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -21,6 +21,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStr import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import java.io.IOException; From f69b3ef05d2b6ce7840c7393be01d4fdc04fe638 Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 1 Nov 2020 01:05:32 +0100 Subject: [PATCH 72/81] create FileUtils --- .../java/org/schabi/newpipe/FileUtils.java | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 extractor/src/test/java/org/schabi/newpipe/FileUtils.java diff --git a/extractor/src/test/java/org/schabi/newpipe/FileUtils.java b/extractor/src/test/java/org/schabi/newpipe/FileUtils.java new file mode 100644 index 00000000..3242275d --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/FileUtils.java @@ -0,0 +1,79 @@ +package org.schabi.newpipe; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonWriter; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +/** + * Util class to write file to disk + *

+ * Can be used to debug and test, for example writing a service's JSON response + * (especially useful if the response provided by the service is not documented) + */ +public class FileUtils { + + public static void createFile(String path, JsonObject content) throws IOException { + createFile(path, jsonObjToString(content)); + } + + public static void createFile(String path, JsonArray array) throws IOException { + createFile(path, jsonArrayToString(array)); + } + + /** + * Create a file given a path and its content. Create subdirectories if needed + * + * @param path the path to write the file, including the filename (and its extension) + * @param content the content to write + * @throws IOException + */ + public static void createFile(final String path, final String content) throws IOException { + final String[] dirs = path.split("/"); + if (dirs.length > 1) { + String pathWithoutFileName = path.replace(dirs[dirs.length - 1], ""); + if (!Files.exists(Paths.get(pathWithoutFileName))) { //create dirs if they don't exist + if (!new File(pathWithoutFileName).mkdirs()) { + throw new IOException("An error occurred while creating directories"); + } + } + } + writeFile(path, content); + } + + /** + * Write a file to disk + * + * @param filename the file name (and its extension if wanted) + * @param content the content to write + * @throws IOException + */ + private static void writeFile(final String filename, final String content) throws IOException { + final BufferedWriter writer = new BufferedWriter(new FileWriter(filename)); + writer.write(content); + writer.flush(); + writer.close(); + } + + /** + * Convert a JSON object to String + * toString() does not produce a valid JSON string + */ + public static String jsonObjToString(JsonObject object) { + return JsonWriter.string(object); + } + + /** + * Convert a JSON array to String + * toString() does not produce a valid JSON string + */ + public static String jsonArrayToString(JsonArray array) { + return JsonWriter.string(array); + } +} From 82746d172f470dec6f88ff49c0815b8041d9f2d7 Mon Sep 17 00:00:00 2001 From: "Bri@n" Date: Sun, 1 Nov 2020 17:34:34 -0500 Subject: [PATCH 73/81] Fix typo in DonationLinkHelper and rewrote swtich statement --- .../schabi/newpipe/extractor/utils/DonationLinkHelper.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java index 817595ad..eb7f35b1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java @@ -15,16 +15,14 @@ public class DonationLinkHelper { AMAZON, } - public static DonationService getDonatoinServiceByLink(String link) throws MalformedURLException { + public static DonationService getDonationServiceByLink(String link) throws MalformedURLException { URL url = new URL(fixLink(link)); switch (url.getHost()) { case "www.patreon.com": - return DonationService.PATREON; case "patreon.com": return DonationService.PATREON; - case "paypal.me": - return DonationService.PAYPAL; case "www.paypal.me": + case "paypal.me": return DonationService.PAYPAL; default: return DonationService.NO_DONATION; From 501ec30152642ad49ce0a1825410d200942d174c Mon Sep 17 00:00:00 2001 From: Stypox Date: Sun, 1 Nov 2020 19:11:08 +0100 Subject: [PATCH 74/81] Implement youtube subscription import from Google takeout --- .../YoutubeSubscriptionExtractor.java | 129 +++-------- .../subscription/SubscriptionExtractor.java | 8 +- .../java/org/schabi/newpipe/FileUtils.java | 15 ++ .../YoutubeSubscriptionExtractorTest.java | 84 ++++--- .../test/resources/youtube_export_test.xml | 23 -- .../youtube_takeout_import_test.json | 211 ++++++++++++++++++ 6 files changed, 309 insertions(+), 161 deletions(-) delete mode 100644 extractor/src/test/resources/youtube_export_test.xml create mode 100644 extractor/src/test/resources/youtube_takeout_import_test.json diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java index 55a9c7c1..ffdde814 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java @@ -1,126 +1,71 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.services.youtube.YoutubeService; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionItem; -import org.schabi.newpipe.extractor.utils.Parser; -import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import javax.annotation.Nonnull; + import static org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.ContentSource.INPUT_STREAM; /** - * Extract subscriptions from a YouTube export (OPML format supported) + * Extract subscriptions from a Google takout export (the user has to get the JSON out of the zip) */ public class YoutubeSubscriptionExtractor extends SubscriptionExtractor { + private static final String BASE_CHANNEL_URL = "https://www.youtube.com/channel/"; - public YoutubeSubscriptionExtractor(YoutubeService service) { - super(service, Collections.singletonList(INPUT_STREAM)); + public YoutubeSubscriptionExtractor(final YoutubeService youtubeService) { + super(youtubeService, Collections.singletonList(INPUT_STREAM)); } @Override public String getRelatedUrl() { - return "https://www.youtube.com/subscription_manager?action_takeout=1"; + return "https://takeout.google.com/takeout/custom/youtube"; } @Override - public List fromInputStream(InputStream contentInputStream) throws ExtractionException { - if (contentInputStream == null) throw new InvalidSourceException("input stream is null"); - - return getItemsFromOPML(contentInputStream); - } - - /*////////////////////////////////////////////////////////////////////////// - // OPML implementation - //////////////////////////////////////////////////////////////////////////*/ - - private static final String ID_PATTERN = "/videos.xml\\?channel_id=([A-Za-z0-9_-]*)"; - private static final String BASE_CHANNEL_URL = "https://www.youtube.com/channel/"; - - private List getItemsFromOPML(InputStream contentInputStream) throws ExtractionException { - final List result = new ArrayList<>(); - - final String contentString = readFromInputStream(contentInputStream); - Document document = Jsoup.parse(contentString, "", org.jsoup.parser.Parser.xmlParser()); - - if (document.select("opml").isEmpty()) { - throw new InvalidSourceException("document does not have OPML tag"); - } - - if (document.select("outline").isEmpty()) { - throw new InvalidSourceException("document does not have at least one outline tag"); - } - - for (Element outline : document.select("outline[type=rss]")) { - String title = outline.attr("title"); - String xmlUrl = outline.attr("abs:xmlUrl"); - - try { - String id = Parser.matchGroup1(ID_PATTERN, xmlUrl); - result.add(new SubscriptionItem(service.getServiceId(), BASE_CHANNEL_URL + id, title)); - } catch (Parser.RegexException ignored) { /* ignore invalid subscriptions */ } - } - - return result; - } - - /*////////////////////////////////////////////////////////////////////////// - // Utils - //////////////////////////////////////////////////////////////////////////*/ - - /** - * Throws an exception if the string does not have the right tag/string from a valid export. - */ - private void throwIfTagIsNotFound(String content) throws InvalidSourceException { - if (!content.trim().contains(" fromInputStream(@Nonnull final InputStream contentInputStream) + throws ExtractionException { + final JsonArray subscriptions; try { - byte[] buffer = new byte[16 * 1024]; - int read; - while ((read = inputStream.read(buffer)) != -1) { - String currentPartOfContent = new String(buffer, 0, read, "UTF-8"); - contentBuilder.append(currentPartOfContent); + subscriptions = JsonParser.array().from(contentInputStream); + } catch (JsonParserException e) { + throw new InvalidSourceException("Invalid json input stream", e); + } - // Fail-fast in case of reading a long unsupported input stream - if (!hasTag && contentBuilder.length() > 128) { - throwIfTagIsNotFound(contentBuilder.toString()); - hasTag = true; - } + boolean foundInvalidSubscription = false; + final List subscriptionItems = new ArrayList<>(); + for (final Object subscriptionObject : subscriptions) { + if (!(subscriptionObject instanceof JsonObject)) { + foundInvalidSubscription = true; + continue; } - } catch (InvalidSourceException e) { - throw e; - } catch (Throwable e) { - throw new InvalidSourceException(e); - } finally { - try { - inputStream.close(); - } catch (IOException ignored) { + + final JsonObject subscription = ((JsonObject) subscriptionObject).getObject("snippet"); + final String id = subscription.getObject("resourceId").getString("channelId", ""); + if (id.length() != 24) { // e.g. UCsXVk37bltHxD1rDPwtNM8Q + foundInvalidSubscription = true; + continue; } + + subscriptionItems.add(new SubscriptionItem(service.getServiceId(), + BASE_CHANNEL_URL + id, subscription.getString("title", ""))); } - final String fileContent = contentBuilder.toString().trim(); - if (fileContent.isEmpty()) { - throw new InvalidSourceException("Empty input stream"); + if (foundInvalidSubscription && subscriptionItems.isEmpty()) { + throw new InvalidSourceException("Found only invalid channel ids"); } - - if (!hasTag) { - throwIfTagIsNotFound(fileContent); - } - - return fileContent; + return subscriptionItems; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionExtractor.java index ce91b1e6..8a31b0f7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionExtractor.java @@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; import java.io.InputStream; @@ -71,8 +72,9 @@ public abstract class SubscriptionExtractor { * * @throws InvalidSourceException when the content read from the InputStream is invalid and can not be parsed */ - @SuppressWarnings("RedundantThrows") - public List fromInputStream(InputStream contentInputStream) throws IOException, ExtractionException { - throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName() + " doesn't support extracting from an InputStream"); + public List fromInputStream(@Nonnull final InputStream contentInputStream) + throws ExtractionException { + throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName() + + " doesn't support extracting from an InputStream"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/FileUtils.java b/extractor/src/test/java/org/schabi/newpipe/FileUtils.java index 3242275d..5c55a2fb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/FileUtils.java +++ b/extractor/src/test/java/org/schabi/newpipe/FileUtils.java @@ -61,6 +61,21 @@ public class FileUtils { writer.close(); } + /** + * Resolves the test resource file based on its filename. Looks in + * {@code extractor/src/test/resources/} and {@code src/test/resources/} + * @param filename the resource filename + * @return the resource file + */ + public static File resolveTestResource(final String filename) { + final File file = new File("extractor/src/test/resources/" + filename); + if (file.exists()) { + return file; + } else { + return new File("src/test/resources/" + filename); + } + } + /** * Convert a JSON object to String * toString() does not produce a valid JSON string diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java index 80821d26..931e12de 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java @@ -11,12 +11,16 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionItem; import java.io.ByteArrayInputStream; -import java.io.File; import java.io.FileInputStream; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.schabi.newpipe.FileUtils.resolveTestResource; /** * Test for {@link YoutubeSubscriptionExtractor} @@ -34,54 +38,48 @@ public class YoutubeSubscriptionExtractorTest { @Test public void testFromInputStream() throws Exception { - File testFile = new File("extractor/src/test/resources/youtube_export_test.xml"); - if (!testFile.exists()) testFile = new File("src/test/resources/youtube_export_test.xml"); + final List subscriptionItems = subscriptionExtractor.fromInputStream( + new FileInputStream(resolveTestResource("youtube_takeout_import_test.json"))); + assertEquals(7, subscriptionItems.size()); - List subscriptionItems = subscriptionExtractor.fromInputStream(new FileInputStream(testFile)); - assertTrue("List doesn't have exactly 8 items (had " + subscriptionItems.size() + ")", subscriptionItems.size() == 8); - - for (SubscriptionItem item : subscriptionItems) { + for (final SubscriptionItem item : subscriptionItems) { assertNotNull(item.getName()); assertNotNull(item.getUrl()); assertTrue(urlHandler.acceptUrl(item.getUrl())); - assertFalse(item.getServiceId() == -1); + assertEquals(ServiceList.YouTube.getServiceId(), item.getServiceId()); } } @Test public void testEmptySourceException() throws Exception { - String emptySource = "" + - "" + - ""; - - List items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(emptySource.getBytes("UTF-8"))); + final List items = subscriptionExtractor.fromInputStream( + new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8))); assertTrue(items.isEmpty()); } @Test public void testSubscriptionWithEmptyTitleInSource() throws Exception { - String channelId = "AA0AaAa0AaaaAAAAAA0aa0AA"; - String source = "" + - "" + - ""; + final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]"; + final List items = subscriptionExtractor.fromInputStream( + new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))); - List items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8"))); - assertTrue("List doesn't have exactly 1 item (had " + items.size() + ")", items.size() == 1); - assertTrue("Item does not have an empty title (had \"" + items.get(0).getName() + "\")", items.get(0).getName().isEmpty()); - assertTrue("Item does not have the right channel id \"" + channelId + "\" (the whole url is \"" + items.get(0).getUrl() + "\")", items.get(0).getUrl().endsWith(channelId)); + assertEquals(1, items.size()); + assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId()); + assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", items.get(0).getUrl()); + assertEquals("", items.get(0).getName()); } @Test public void testSubscriptionWithInvalidUrlInSource() throws Exception { - String source = "" + - "" + - "" + - "" + - "" + - ""; + final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," + + "{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]"; + final List items = subscriptionExtractor.fromInputStream( + new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))); - List items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8"))); - assertTrue(items.isEmpty()); + assertEquals(1, items.size()); + assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId()); + assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", items.get(0).getUrl()); + assertEquals("name2", items.get(0).getName()); } @Test @@ -89,26 +87,26 @@ public class YoutubeSubscriptionExtractorTest { List invalidList = Arrays.asList( "", "", - "", + "{\"a\":\"b\"}", + "[{}]", + "[\"\", 5]", + "[{\"snippet\":{\"title\":\"name\"}}]", + "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"}}}]", "", - null, "\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28", "gibberish"); for (String invalidContent : invalidList) { try { - if (invalidContent != null) { - byte[] bytes = invalidContent.getBytes("UTF-8"); - subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes)); - fail("Extracting from \"" + invalidContent + "\" didn't throw an exception"); - } else { - subscriptionExtractor.fromInputStream(null); - fail("Extracting from null String didn't throw an exception"); + byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8); + subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes)); + fail("Extracting from \"" + invalidContent + "\" didn't throw an exception"); + } catch (final Exception e) { + boolean correctType = e instanceof SubscriptionExtractor.InvalidSourceException; + if (!correctType) { + e.printStackTrace(); } - } catch (Exception e) { - // System.out.println(" -> " + e); - boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException; - assertTrue("\"" + e.getClass().getSimpleName() + "\" is not the expected exception", isExpectedException); + assertTrue(e.getClass().getSimpleName() + " is not InvalidSourceException", correctType); } } } diff --git a/extractor/src/test/resources/youtube_export_test.xml b/extractor/src/test/resources/youtube_export_test.xml deleted file mode 100644 index 4092f98a..00000000 --- a/extractor/src/test/resources/youtube_export_test.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/extractor/src/test/resources/youtube_takeout_import_test.json b/extractor/src/test/resources/youtube_takeout_import_test.json new file mode 100644 index 00000000..0dbc5c72 --- /dev/null +++ b/extractor/src/test/resources/youtube_takeout_import_test.json @@ -0,0 +1,211 @@ +[ { + "contentDetails" : { + "activityType" : "all", + "newItemCount" : 0, + "totalItemCount" : 229 + }, + "etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs", + "id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y", + "kind" : "youtube#subscription", + "snippet" : { + "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw", + "description" : "The official YouTube home of Gorillaz.", + "publishedAt" : "2020-11-01T17:24:34.498Z", + "resourceId" : { + "channelId" : "UCfIXdjDQH9Fau7y99_Orpjw", + "kind" : "youtube#channel" + }, + "thumbnails" : { + "default" : { + "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "high" : { + "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "medium" : { + "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg" + } + }, + "title" : "Gorillaz" + } +}, { + "contentDetails" : { + "activityType" : "all", + "newItemCount" : 0, + "totalItemCount" : 3502 + }, + "etag" : "wUgip-X0qBlnjj0frSTwP6B8XoY", + "id" : "qUAzBV8xkoLYOP-1gwzy63zpjj8SMTtDReGwIa2sHp8", + "kind" : "youtube#subscription", + "snippet" : { + "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw", + "description" : "The TED Talks channel features the best talks and performances from the TED Conference, where the world's leading thinkers and doers give the talk of their lives in 18 minutes (or less). Look for talks on Technology, Entertainment and Design -- plus science, business, global issues, the arts and more. You're welcome to link to or embed these videos, forward them to others and share these ideas with people you know.\n\nTED's videos may be used for non-commercial purposes under a Creative Commons License, Attribution–Non Commercial–No Derivatives (or the CC BY – NC – ND 4.0 International) and in accordance with our TED Talks Usage Policy (https://www.ted.com/about/our-organization/our-policies-terms/ted-talks-usage-policy). For more information on using TED for commercial purposes (e.g. employee learning, in a film or online course), please submit a Media Request at https://media-requests.ted.com", + "publishedAt" : "2020-11-01T17:24:11.769Z", + "resourceId" : { + "channelId" : "UCAuUUnT6oDeKwE6v1NGQxug", + "kind" : "youtube#channel" + }, + "thumbnails" : { + "default" : { + "url" : "https://yt3.ggpht.com/-bonZt347bMc/AAAAAAAAAAI/AAAAAAAAAAA/lR8QEKnqqHk/s88-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "high" : { + "url" : "https://yt3.ggpht.com/-bonZt347bMc/AAAAAAAAAAI/AAAAAAAAAAA/lR8QEKnqqHk/s800-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "medium" : { + "url" : "https://yt3.ggpht.com/-bonZt347bMc/AAAAAAAAAAI/AAAAAAAAAAA/lR8QEKnqqHk/s240-c-k-no-mo-rj-c0xffffff/photo.jpg" + } + }, + "title" : "TED" + } +}, { + "contentDetails" : { + "activityType" : "all", + "newItemCount" : 0, + "totalItemCount" : 98 + }, + "etag" : "M3Hl6FQUAD3e-fH9pcvcE9aPSWQ", + "id" : "qUAzBV8xkoLYOP-1gwzy64Vo-PpWMPDyIYBM1JUfepk", + "kind" : "youtube#subscription", + "snippet" : { + "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw", + "description" : "In a world where the content of digital images and videos can no longer be taken at face value, an unlikely hero fights for the acceptance of truth.\r\n\r\nCaptain Disillusion guides children of all ages through the maze of visual fakery to the open spaces of reality and peace of mind.\r\n\r\nSubscribe to get fun and detailed explanations of current \"unbelievable\" viral videos that fool the masses!", + "publishedAt" : "2020-11-01T17:23:52.909Z", + "resourceId" : { + "channelId" : "UCEOXxzW2vU0P-0THehuIIeg", + "kind" : "youtube#channel" + }, + "thumbnails" : { + "default" : { + "url" : "https://yt3.ggpht.com/-7f3hUYZP5Sc/AAAAAAAAAAI/AAAAAAAAAAA/4cpBHKDlbYQ/s88-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "high" : { + "url" : "https://yt3.ggpht.com/-7f3hUYZP5Sc/AAAAAAAAAAI/AAAAAAAAAAA/4cpBHKDlbYQ/s800-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "medium" : { + "url" : "https://yt3.ggpht.com/-7f3hUYZP5Sc/AAAAAAAAAAI/AAAAAAAAAAA/4cpBHKDlbYQ/s240-c-k-no-mo-rj-c0xffffff/photo.jpg" + } + }, + "title" : "Captain Disillusion" + } +}, { + "contentDetails" : { + "activityType" : "all", + "newItemCount" : 0, + "totalItemCount" : 130 + }, + "etag" : "crkTVZbDHS3arRZErMaLMnNqtac", + "id" : "qUAzBV8xkoLYOP-1gwzy66EVopYHE34m06PVw8Pvheg", + "kind" : "youtube#subscription", + "snippet" : { + "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw", + "description" : "Videos explaining things with optimistic nihilism. \n\nWe are a small team who want to make science look beautiful. Because it is beautiful. \n\nCurrently we make one animation video per month. Follow us on Twitter, Facebook to get notified when a new one comes out.\n\nFAQ:\n \n- We do the videos with After Effects and Illustrator.", + "publishedAt" : "2020-11-01T17:23:39.659Z", + "resourceId" : { + "channelId" : "UCsXVk37bltHxD1rDPwtNM8Q", + "kind" : "youtube#channel" + }, + "thumbnails" : { + "default" : { + "url" : "https://yt3.ggpht.com/-UwENvFjc4vI/AAAAAAAAAAI/AAAAAAAAAAA/04dXvZ_jl0I/s88-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "high" : { + "url" : "https://yt3.ggpht.com/-UwENvFjc4vI/AAAAAAAAAAI/AAAAAAAAAAA/04dXvZ_jl0I/s800-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "medium" : { + "url" : "https://yt3.ggpht.com/-UwENvFjc4vI/AAAAAAAAAAI/AAAAAAAAAAA/04dXvZ_jl0I/s240-c-k-no-mo-rj-c0xffffff/photo.jpg" + } + }, + "title" : "Kurzgesagt – In a Nutshell" + } +}, { + "contentDetails" : { + "activityType" : "all", + "newItemCount" : 0, + "totalItemCount" : 229 + }, + "etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs", + "id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y", + "kind" : "youtube#subscription", + "snippet" : { + "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw", + "description" : "ⓤⓝⓘⓒⓞⓓⓔ", + "publishedAt" : "2020-11-01T17:24:34.498Z", + "resourceId" : { + "channelId" : "UCfIXdjDQH9Fau7y99_Orpjw", + "kind" : "youtube#channel" + }, + "thumbnails" : { + "default" : { + "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "high" : { + "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "medium" : { + "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg" + } + }, + "title" : "ⓤⓝⓘⓒⓞⓓⓔ" + } +}, { + "contentDetails" : { + "activityType" : "all", + "newItemCount" : 0, + "totalItemCount" : 229 + }, + "etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs", + "id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y", + "kind" : "youtube#subscription", + "snippet" : { + "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw", + "description" : "中文", + "publishedAt" : "2020-11-01T17:24:34.498Z", + "resourceId" : { + "channelId" : "UCfIXdjDQH9Fau7y99_Orpjw", + "kind" : "youtube#channel" + }, + "thumbnails" : { + "default" : { + "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "high" : { + "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "medium" : { + "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg" + } + }, + "title" : "中文" + } +}, { + "contentDetails" : { + "activityType" : "all", + "newItemCount" : 0, + "totalItemCount" : 229 + }, + "etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs", + "id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y", + "kind" : "youtube#subscription", + "snippet" : { + "channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw", + "description" : "हिंदी", + "publishedAt" : "2020-11-01T17:24:34.498Z", + "resourceId" : { + "channelId" : "UCfIXdjDQH9Fau7y99_Orpjw", + "kind" : "youtube#channel" + }, + "thumbnails" : { + "default" : { + "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "high" : { + "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg" + }, + "medium" : { + "url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg" + } + }, + "title" : "हिंदी" + } +} ] \ No newline at end of file From 345e136f6c823d3ac6035bd7bcb18c2cc44edb6e Mon Sep 17 00:00:00 2001 From: bopol Date: Tue, 3 Nov 2020 19:10:10 +0100 Subject: [PATCH 75/81] create YouTubeCommentsLinkHandlerFactoryTest and remove invidious test from YouTubeCommentsExtractorTest, because it was just testing if the URL is accepted, then the extractor does the same thing, we don't need to test the same thing twice --- ...YouTubeCommentsLinkHandlerFactoryTest.java | 64 +++++++++++++++++++ .../youtube/YoutubeCommentsExtractorTest.java | 25 +++----- 2 files changed, 73 insertions(+), 16 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeCommentsLinkHandlerFactoryTest.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeCommentsLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeCommentsLinkHandlerFactoryTest.java new file mode 100644 index 00000000..b8d7b55e --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeCommentsLinkHandlerFactoryTest.java @@ -0,0 +1,64 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class YouTubeCommentsLinkHandlerFactoryTest { + + private static YoutubeCommentsLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + linkHandler = YoutubeCommentsLinkHandlerFactory.getInstance(); + } + + @Test(expected = IllegalArgumentException.class) + public void getIdWithNullAsUrl() throws ParsingException { + linkHandler.fromId(null); + } + + @Test + public void getIdFromYt() throws ParsingException { + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://www.youtube.com/watch?v=VM_6n762j6M").getId()); + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://m.youtube.com/watch?v=VM_6n762j6M").getId()); + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://youtube.com/watch?v=VM_6n762j6M").getId()); + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://WWW.youtube.com/watch?v=VM_6n762j6M").getId()); + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://youtu.be/VM_6n762j6M").getId()); + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://youtu.be/VM_6n762j6M&t=20").getId()); + } + + @Test + public void testAcceptUrl() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=VM_6n762j6M&t=20")); + assertTrue(linkHandler.acceptUrl("https://WWW.youtube.com/watch?v=VM_6n762j6M&t=20")); + assertTrue(linkHandler.acceptUrl("https://youtube.com/watch?v=VM_6n762j6M&t=20")); + assertTrue(linkHandler.acceptUrl("https://youtu.be/VM_6n762j6M&t=20")); + } + + @Test + public void testDeniesUrl() throws ParsingException { + assertFalse(linkHandler.acceptUrl("https://www.you com/watch?v=VM_6n762j6M")); + assertFalse(linkHandler.acceptUrl("https://com/watch?v=VM_6n762j6M")); + assertFalse(linkHandler.acceptUrl("htt ://com/watch?v=VM_6n762j6M")); + assertFalse(linkHandler.acceptUrl("ftp://www.youtube.com/watch?v=VM_6n762j6M")); + } + + @Test + public void getIdFromInvidious() throws ParsingException { + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://www.invidio.us/watch?v=VM_6n762j6M").getId()); + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidio.us/watch?v=VM_6n762j6M").getId()); + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://INVIDIO.US/watch?v=VM_6n762j6M").getId()); + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidio.us/VM_6n762j6M").getId()); + assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidio.us/VM_6n762j6M&t=20").getId()); + } + +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index b43dce67..62da5041 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -24,31 +24,25 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube; public class YoutubeCommentsExtractorTest { /** - * Test a "normal" YouTube and Invidious page + * Test a "normal" YouTube */ public static class Thomas { - private static final String urlYT = "https://www.youtube.com/watch?v=D00Au7k3i6o"; - private static final String urlInvidious = "https://invidio.us/watch?v=D00Au7k3i6o"; - private static YoutubeCommentsExtractor extractorYT; - private static YoutubeCommentsExtractor extractorInvidious; + private static final String url = "https://www.youtube.com/watch?v=D00Au7k3i6o"; + private static YoutubeCommentsExtractor extractor; private static final String commentContent = "sub 4 sub"; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); - extractorYT = (YoutubeCommentsExtractor) YouTube - .getCommentsExtractor(urlYT); - extractorYT.fetchPage(); - extractorInvidious = (YoutubeCommentsExtractor) YouTube - .getCommentsExtractor(urlInvidious); - extractorInvidious.fetchPage(); + extractor = (YoutubeCommentsExtractor) YouTube + .getCommentsExtractor(url); + extractor.fetchPage(); } @Test public void testGetComments() throws IOException, ExtractionException { - assertTrue(getCommentsHelper(extractorYT)); - assertTrue(getCommentsHelper(extractorInvidious)); + assertTrue(getCommentsHelper(extractor)); } private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException { @@ -65,8 +59,7 @@ public class YoutubeCommentsExtractorTest { @Test public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { - assertTrue(getCommentsFromCommentsInfoHelper(urlYT)); - assertTrue(getCommentsFromCommentsInfoHelper(urlInvidious)); + assertTrue(getCommentsFromCommentsInfoHelper(url)); } private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException { @@ -87,7 +80,7 @@ public class YoutubeCommentsExtractorTest { @Test public void testGetCommentsAllData() throws IOException, ExtractionException { - InfoItemsPage comments = extractorYT.getInitialPage(); + InfoItemsPage comments = extractor.getInitialPage(); DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); for (CommentsInfoItem c : comments.getItems()) { From fe31a90cb349c98f2199b5bc5b55a9904bb56115 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Mon, 2 Nov 2020 09:14:52 +0530 Subject: [PATCH 76/81] Remove DateTimeFormatter.ISO_OFFSET_DATE_TIME usage. --- .../youtube/extractors/YoutubeFeedInfoItemExtractor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java index cea38518..3945ff17 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java @@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.stream.StreamType; import javax.annotation.Nullable; import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { @@ -61,7 +60,7 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { @Override public DateWrapper getUploadDate() throws ParsingException { try { - return new DateWrapper(OffsetDateTime.parse(getTextualUploadDate(), DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + return new DateWrapper(OffsetDateTime.parse(getTextualUploadDate())); } catch (DateTimeParseException e) { throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", e); } From 9cf9e7e98044ef85fb72737824e99b7000771423 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Mon, 2 Nov 2020 09:15:55 +0530 Subject: [PATCH 77/81] Call existing constructor in DateWrapper. --- .../org/schabi/newpipe/extractor/localization/DateWrapper.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java index 0390ee12..f86efa9b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java @@ -28,8 +28,7 @@ public class DateWrapper implements Serializable { */ @Deprecated public DateWrapper(@NonNull Calendar calendar, boolean isApproximation) { - offsetDateTime = OffsetDateTime.ofInstant(calendar.toInstant(), ZoneOffset.UTC); - this.isApproximation = isApproximation; + this(OffsetDateTime.ofInstant(calendar.toInstant(), ZoneOffset.UTC), isApproximation); } public DateWrapper(@NonNull OffsetDateTime offsetDateTime) { From 4fe28d7e3a3cf6ba18d4d74219fad22adcba4911 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Tue, 3 Nov 2020 16:24:46 +0530 Subject: [PATCH 78/81] Fix YouTube parse error when only a date is present. --- .../extractor/services/youtube/YoutubeParsingHelper.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 3fb279b8..8b924cf9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -21,7 +21,9 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; +import java.time.LocalDate; import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.time.format.DateTimeParseException; import java.util.Collections; import java.util.HashMap; @@ -184,7 +186,11 @@ public class YoutubeParsingHelper { try { return OffsetDateTime.parse(textualUploadDate); } catch (DateTimeParseException e) { - throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); + try { + return LocalDate.parse(textualUploadDate).atStartOfDay().atOffset(ZoneOffset.UTC); + } catch (DateTimeParseException e1) { + throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e1); + } } } From 827f7bd13750b29e2382c6681f23b707f76e5f33 Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 29 Oct 2020 18:44:05 +0100 Subject: [PATCH 79/81] [YouTube] Cache deobfuscation and improve requests made Fix age restriction extraction Automatically fixes more things --- .../extractor/services/youtube/ItagItem.java | 2 +- .../extractors/YoutubeStreamExtractor.java | 430 ++++++++---------- .../services/DefaultStreamExtractorTest.java | 1 + 3 files changed, 180 insertions(+), 253 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java index a3de19e2..0b367686 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java @@ -94,7 +94,7 @@ public class ItagItem { return item; } } - throw new ParsingException("itag=" + Integer.toString(itagId) + " not supported"); + throw new ParsingException("itag=" + itagId + " not supported"); } /*////////////////////////////////////////////////////////////////////////// diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index d86c63d9..fb82d341 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -42,10 +44,10 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; +import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -86,7 +88,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { // Exceptions //////////////////////////////////////////////////////////////////////////*/ - public class DeobfuscateException extends ParsingException { + public static class DeobfuscateException extends ParsingException { DeobfuscateException(String message, Throwable cause) { super(message, cause); } @@ -94,20 +96,17 @@ public class YoutubeStreamExtractor extends StreamExtractor { /*//////////////////////////////////////////////////////////////////////////*/ + @Nullable private static String cachedDeobfuscationCode = null; + @Nullable private String playerJsUrl = null; + private JsonArray initialAjaxJson; - @Nullable - private JsonObject playerArgs; - @Nonnull - private final Map videoInfoPage = new HashMap<>(); - private JsonObject playerResponse; private JsonObject initialData; + @Nonnull private final Map videoInfoPage = new HashMap<>(); + private JsonObject playerResponse; private JsonObject videoPrimaryInfoRenderer; private JsonObject videoSecondaryInfoRenderer; - private int ageLimit; - private boolean newJsonScheme; - - @Nonnull - private List subtitlesInfos = new ArrayList<>(); + private int ageLimit = -1; + @Nullable private List subtitles = null; public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) { super(service, linkHandler); @@ -163,8 +162,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { } try { // Premiered Feb 21, 2020 - LocalDate localDate = LocalDate.parse(time, - DateTimeFormatter.ofPattern("MMM dd, YYYY", Locale.ENGLISH)); + final LocalDate localDate = LocalDate.parse(time, + DateTimeFormatter.ofPattern("MMM dd, yyyy", Locale.ENGLISH)); return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate); } catch (Exception ignored) { } @@ -224,9 +223,28 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public int getAgeLimit() { - if (isNullOrEmpty(initialData)) throw new IllegalStateException("initialData is not parsed yet"); + public int getAgeLimit() throws ParsingException { + if (ageLimit == -1) { + ageLimit = NO_AGE_LIMIT; + final JsonArray metadataRows = getVideoSecondaryInfoRenderer() + .getObject("metadataRowContainer").getObject("metadataRowContainerRenderer") + .getArray("rows"); + for (final Object metadataRow : metadataRows) { + final JsonArray contents = ((JsonObject) metadataRow) + .getObject("metadataRowRenderer").getArray("contents"); + for (final Object content : contents) { + final JsonArray runs = ((JsonObject) content).getArray("runs"); + for (final Object run : runs) { + final String rowText = ((JsonObject) run).getString("text", EMPTY_STRING); + if (rowText.contains("Age-restricted")) { + ageLimit = 18; + return ageLimit; + } + } + } + } + } return ageLimit; } @@ -313,8 +331,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (NumberFormatException nfe) { throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer", nfe); } catch (Exception e) { - if (ageLimit == 18) return -1; - throw new ParsingException("Could not get like count", e); + if (getAgeLimit() == NO_AGE_LIMIT) { + throw new ParsingException("Could not get like count", e); + } + return -1; } } @@ -337,8 +357,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (NumberFormatException nfe) { throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer", nfe); } catch (Exception e) { - if (ageLimit == 18) return -1; - throw new ParsingException("Could not get dislike count", e); + if (getAgeLimit() == NO_AGE_LIMIT) { + throw new ParsingException("Could not get dislike count", e); + } + return -1; } } @@ -402,8 +424,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { } if (isNullOrEmpty(url)) { - if (ageLimit == 18) return ""; - throw new ParsingException("Could not get uploader avatar URL"); + if (ageLimit == NO_AGE_LIMIT) { + throw new ParsingException("Could not get uploader avatar URL"); + } + return ""; } return fixThumbnailUrl(url); @@ -411,19 +435,19 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getSubChannelUrl() throws ParsingException { + public String getSubChannelUrl() { return ""; } @Nonnull @Override - public String getSubChannelName() throws ParsingException { + public String getSubChannelName() { return ""; } @Nonnull @Override - public String getSubChannelAvatarUrl() throws ParsingException { + public String getSubChannelAvatarUrl() { return ""; } @@ -437,8 +461,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { return playerResponse.getObject("streamingData").getString("dashManifestUrl"); } else if (videoInfoPage.containsKey("dashmpd")) { dashManifestUrl = videoInfoPage.get("dashmpd"); - } else if (playerArgs != null && playerArgs.isString("dashmpd")) { - dashManifestUrl = playerArgs.getString("dashmpd", EMPTY_STRING); } else { return ""; } @@ -447,7 +469,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl); String deobfuscatedSig; - deobfuscatedSig = deobfuscateSignature(obfuscatedSig, deobfuscationCode); + deobfuscatedSig = deobfuscateSignature(obfuscatedSig); dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig, "/signature/" + deobfuscatedSig); } @@ -465,11 +487,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { return playerResponse.getObject("streamingData").getString("hlsManifestUrl"); } catch (Exception e) { - if (playerArgs != null && playerArgs.isString("hlsvp")) { - return playerArgs.getString("hlsvp"); - } else { - throw new ParsingException("Could not get hls manifest url", e); - } + throw new ParsingException("Could not get hls manifest url", e); } } @@ -535,18 +553,46 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override @Nonnull - public List getSubtitlesDefault() { + public List getSubtitlesDefault() throws ParsingException { return getSubtitles(MediaFormat.TTML); } @Override @Nonnull - public List getSubtitles(final MediaFormat format) { + public List getSubtitles(final MediaFormat format) throws ParsingException { assertPageFetched(); - List subtitles = new ArrayList<>(); - for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) { - subtitles.add(subtitlesInfo.getSubtitle(format)); + // If the video is age restricted getPlayerConfig will fail + if (getAgeLimit() != NO_AGE_LIMIT) { + return Collections.emptyList(); } + if (subtitles != null) { + // already calculated + return subtitles; + } + + final JsonObject renderer = playerResponse.getObject("captions") + .getObject("playerCaptionsTracklistRenderer"); + final JsonArray captionsArray = renderer.getArray("captionTracks"); + // TODO: use this to apply auto translation to different language from a source language + // final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages"); + + subtitles = new ArrayList<>(); + for (int i = 0; i < captionsArray.size(); i++) { + final String languageCode = captionsArray.getObject(i).getString("languageCode"); + final String baseUrl = captionsArray.getObject(i).getString("baseUrl"); + final String vssId = captionsArray.getObject(i).getString("vssId"); + + if (languageCode != null && baseUrl != null && vssId != null) { + final boolean isAutoGenerated = vssId.startsWith("a."); + final String cleanUrl = baseUrl + .replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists + .replaceAll("&tlang=[^&]*", ""); // Remove translation language + + subtitles.add(new SubtitlesStream(format, languageCode, + cleanUrl + "&fmt=" + format.getSuffix(), isAutoGenerated)); + } + } + return subtitles; } @@ -554,14 +600,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { public StreamType getStreamType() throws ParsingException { assertPageFetched(); try { - if (!playerResponse.getObject("streamingData").has(FORMATS) || - (playerArgs != null && playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))) { - return StreamType.LIVE_STREAM; - } + return playerResponse.getObject("videoDetails").getBoolean("isLiveContent") + ? StreamType.LIVE_STREAM : StreamType.VIDEO_STREAM; } catch (Exception e) { throw new ParsingException("Could not get stream type", e); } - return StreamType.VIDEO_STREAM; } private StreamInfoItemExtractor getNextStream() throws ExtractionException { @@ -648,48 +691,24 @@ public class YoutubeStreamExtractor extends StreamExtractor { "\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(" }; - private volatile String deobfuscationCode = ""; - @Override - public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl() + "&pbj=1"; - final String playerUrl; + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { + initialAjaxJson = getJsonResponse(getUrl() + "&pbj=1", getExtractorLocalization()); - initialAjaxJson = getJsonResponse(url, getExtractorLocalization()); - - if (initialAjaxJson.getObject(2).has("response")) { // age-restricted videos - initialData = initialAjaxJson.getObject(2).getObject("response"); - ageLimit = 18; - - final EmbeddedInfo info = getEmbeddedInfo(); - final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts); - final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody(); - videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse)); - playerUrl = info.url; - - } else { - ageLimit = NO_AGE_LIMIT; - JsonObject playerConfig; - initialData = initialAjaxJson.getObject(3).getObject("response"); - - // sometimes at random YouTube does not provide the player config - playerConfig = initialAjaxJson.getObject(2).getObject("player", null); - - if (playerConfig == null) { - newJsonScheme = true; - final EmbeddedInfo info = getEmbeddedInfo(); - final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts); - final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody(); - videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse)); - playerUrl = info.url; - } else { - playerArgs = getPlayerArgs(playerConfig); - playerUrl = getPlayerUrl(playerConfig); + initialData = initialAjaxJson.getObject(3).getObject("response", null); + if (initialData == null) { + initialData = initialAjaxJson.getObject(2).getObject("response", null); + if (initialData == null) { + throw new ParsingException("Could not get initial data"); } - } - playerResponse = getPlayerResponse(); + playerResponse = initialAjaxJson.getObject(2).getObject("playerResponse", null); + if (playerResponse == null || !playerResponse.has("streamingData")) { + // try to get player response by fetching video info page + fetchVideoInfoPage(); + } final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus"); final String status = playabilityStatus.getString("status"); @@ -698,117 +717,77 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String reason = playabilityStatus.getString("reason"); throw new ContentNotAvailableException("Got error: \"" + reason + "\""); } - - if (deobfuscationCode.isEmpty()) { - deobfuscationCode = loadDeobfuscationCode(playerUrl); - } - - if (subtitlesInfos.isEmpty()) { - subtitlesInfos.addAll(getAvailableSubtitlesInfo()); - } } - private JsonObject getPlayerArgs(final JsonObject playerConfig) throws ParsingException { - //attempt to load the youtube js player JSON arguments - final JsonObject playerArgs = playerConfig.getObject("args", null); - if (playerArgs == null) { - throw new ParsingException("Could not extract args from YouTube player config"); - } - return playerArgs; - } + private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException { + final String sts = getEmbeddedInfoStsAndStorePlayerJsUrl(); + final String videoInfoUrl = getVideoInfoUrl(getId(), sts); + final String infoPageResponse = NewPipe.getDownloader() + .get(videoInfoUrl, getExtractorLocalization()).responseBody(); + videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse)); - private String getPlayerUrl(final JsonObject playerConfig) throws ParsingException { - // The Youtube service needs to be initialized by downloading the - // js-Youtube-player. This is done in order to get the algorithm - // for deobfuscating cryptic signatures inside certain stream URLs. - final String playerUrl = playerConfig.getObject("assets").getString("js"); - - if (playerUrl == null) { - throw new ParsingException("Could not extract js URL from YouTube player config"); - } else if (playerUrl.startsWith("//")) { - return HTTPS + playerUrl; - } - return playerUrl; - } - - private JsonObject getPlayerResponse() throws ParsingException { try { - String playerResponseStr; - if (newJsonScheme) { - return initialAjaxJson.getObject(2).getObject("playerResponse"); - } - - if (playerArgs != null) { - playerResponseStr = playerArgs.getString("player_response"); - } else { - playerResponseStr = videoInfoPage.get("player_response"); - } - return JsonParser.object().from(playerResponseStr); - } catch (Exception e) { - throw new ParsingException("Could not parse yt player response", e); + playerResponse = JsonParser.object().from(videoInfoPage.get("player_response")); + } catch (JsonParserException e) { + throw new ParsingException( + "Could not parse YouTube player response from video info page", e); } } @Nonnull - private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException { + private String getEmbeddedInfoStsAndStorePlayerJsUrl() { try { - final Downloader downloader = NewPipe.getDownloader(); final String embedUrl = "https://www.youtube.com/embed/" + getId(); - final String embedPageContent = downloader.get(embedUrl, getExtractorLocalization()).responseBody(); + final String embedPageContent = NewPipe.getDownloader() + .get(embedUrl, getExtractorLocalization()).responseBody(); - // Get player url - String playerUrl = null; try { final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")"; - playerUrl = Parser.matchGroup1(assetsPattern, embedPageContent) + playerJsUrl = Parser.matchGroup1(assetsPattern, embedPageContent) .replace("\\", "").replace("\"", ""); } catch (Parser.RegexException ex) { - // playerUrl is still available in the file, just somewhere else + // playerJsUrl is still available in the file, just somewhere else TODO + // it is ok not to find it, see how that's handled in getDeobfuscationCode() final Document doc = Jsoup.parse(embedPageContent); final Elements elems = doc.select("script").attr("name", "player_ias/base"); for (Element elem : elems) { if (elem.attr("src").contains("base.js")) { - playerUrl = elem.attr("src"); + playerJsUrl = elem.attr("src"); + break; } } - - if (playerUrl == null) { - throw new ParsingException("Could not get playerUrl"); - } } - if (playerUrl.startsWith("//")) { - playerUrl = HTTPS + playerUrl; - } else if (playerUrl.startsWith("/")) { - playerUrl = HTTPS + "//youtube.com" + playerUrl; - } - - try { - // Get embed sts - final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)"; - final String sts = Parser.matchGroup1(stsPattern, embedPageContent); - return new EmbeddedInfo(playerUrl, sts); - } catch (Exception i) { - // if it fails we simply reply with no sts as then it does not seem to be necessary - return new EmbeddedInfo(playerUrl, ""); - } - - } catch (IOException e) { - throw new ParsingException( - "Could not load deobfuscation code from YouTube video embed", e); + // Get embed sts + return Parser.matchGroup1("\"sts\"\\s*:\\s*(\\d+)", embedPageContent); + } catch (Exception i) { + // if it fails we simply reply with no sts as then it does not seem to be necessary + return ""; } } - private String loadDeobfuscationCode(String playerUrl) throws DeobfuscateException { - try { - Downloader downloader = NewPipe.getDownloader(); - if (!playerUrl.contains("https://youtube.com")) { - //sometimes the https://youtube.com part does not get send with - //than we have to add it by hand - playerUrl = "https://youtube.com" + playerUrl; - } - final String playerCode = downloader.get(playerUrl, getExtractorLocalization()).responseBody(); + + + private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException { + Parser.RegexException exception = null; + for (final String regex : REGEXES) { + try { + return Parser.matchGroup1(regex, playerCode); + } catch (Parser.RegexException re) { + if (exception == null) { + exception = re; + } + } + } + throw new DeobfuscateException("Could not find deobfuscate function with any of the given patterns.", exception); + } + + private String loadDeobfuscationCode(@Nonnull final String playerJsUrl) + throws DeobfuscateException { + try { + final String playerCode = NewPipe.getDownloader() + .get(playerJsUrl, getExtractorLocalization()).responseBody(); final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode); final String functionPattern = "(" @@ -834,7 +813,34 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - private String deobfuscateSignature(String obfuscatedSig, String deobfuscationCode) throws DeobfuscateException { + @Nonnull + private String getDeobfuscationCode() throws ParsingException { + if (cachedDeobfuscationCode == null) { + if (playerJsUrl == null) { + // the currentPlayerJsUrl was not found in any page fetched so far and there is + // nothing cached, so try fetching embedded info + getEmbeddedInfoStsAndStorePlayerJsUrl(); + if (playerJsUrl == null) { + throw new ParsingException( + "Embedded info did not provide YouTube player js url"); + } + } + + if (playerJsUrl.startsWith("//")) { + playerJsUrl = HTTPS + playerJsUrl; + } else if (playerJsUrl.startsWith("/")) { + // sometimes https://youtube.com part has to be added manually + playerJsUrl = HTTPS + "//youtube.com" + playerJsUrl; + } + + cachedDeobfuscationCode = loadDeobfuscationCode(playerJsUrl); + } + return cachedDeobfuscationCode; + } + + private String deobfuscateSignature(final String obfuscatedSig) throws ParsingException { + final String deobfuscationCode = getDeobfuscationCode(); + final Context context = Context.enter(); context.setOptimizationLevel(-1); final Object result; @@ -851,88 +857,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { return result == null ? "" : result.toString(); } - private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException { - Parser.RegexException exception = null; - for (final String regex : REGEXES) { - try { - return Parser.matchGroup1(regex, playerCode); - } catch (Parser.RegexException re) { - if (exception == null) { - exception = re; - } - } - } - throw new DeobfuscateException("Could not find deobfuscate function with any of the given patterns.", exception); - } - - @Nonnull - private List getAvailableSubtitlesInfo() { - // If the video is age restricted getPlayerConfig will fail - if (getAgeLimit() != NO_AGE_LIMIT) return Collections.emptyList(); - - final JsonObject captions; - if (!playerResponse.has("captions")) { - // Captions does not exist - return Collections.emptyList(); - } - captions = playerResponse.getObject("captions"); - - final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer"); - final JsonArray captionsArray = renderer.getArray("captionTracks"); - // todo: use this to apply auto translation to different language from a source language -// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages"); - - // This check is necessary since there may be cases where subtitles metadata do not contain caption track info - // e.g. https://www.youtube.com/watch?v=-Vpwatutnko - final int captionsSize = captionsArray.size(); - if (captionsSize == 0) return Collections.emptyList(); - - List result = new ArrayList<>(); - for (int i = 0; i < captionsSize; i++) { - final String languageCode = captionsArray.getObject(i).getString("languageCode"); - final String baseUrl = captionsArray.getObject(i).getString("baseUrl"); - final String vssId = captionsArray.getObject(i).getString("vssId"); - - if (languageCode != null && baseUrl != null && vssId != null) { - final boolean isAutoGenerated = vssId.startsWith("a."); - result.add(new SubtitlesInfo(baseUrl, languageCode, isAutoGenerated)); - } - } - - return result; - } - /*////////////////////////////////////////////////////////////////////////// - // Data Class - //////////////////////////////////////////////////////////////////////////*/ - - private class EmbeddedInfo { - final String url; - final String sts; - - EmbeddedInfo(final String url, final String sts) { - this.url = url; - this.sts = sts; - } - } - - private class SubtitlesInfo { - final String cleanUrl; - final String languageCode; - final boolean isGenerated; - - public SubtitlesInfo(final String baseUrl, final String languageCode, final boolean isGenerated) { - this.cleanUrl = baseUrl - .replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists - .replaceAll("&tlang=[^&]*", ""); // Remove translation language - this.languageCode = languageCode; - this.isGenerated = isGenerated; - } - - public SubtitlesStream getSubtitle(final MediaFormat format) { - return new SubtitlesStream(format, languageCode, cleanUrl + "&fmt=" + format.getSuffix(), isGenerated); - } - } - /*////////////////////////////////////////////////////////////////////////// // Utils //////////////////////////////////////////////////////////////////////////*/ @@ -989,14 +913,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { "&sts=" + sts + "&ps=default&gl=US&hl=en"; } - private Map getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException { - Map urlAndItags = new LinkedHashMap<>(); - JsonObject streamingData = playerResponse.getObject("streamingData"); + private Map getItags(final String streamingDataKey, + final ItagItem.ItagType itagTypeWanted) + throws ParsingException { + final Map urlAndItags = new LinkedHashMap<>(); + final JsonObject streamingData = playerResponse.getObject("streamingData"); if (!streamingData.has(streamingDataKey)) { return urlAndItags; } - JsonArray formats = streamingData.getArray(streamingDataKey); + final JsonArray formats = streamingData.getArray(streamingDataKey); for (int i = 0; i != formats.size(); ++i) { JsonObject formatData = formats.getObject(i); int itag = formatData.getInt("itag"); @@ -1022,7 +948,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { : formatData.getString("signatureCipher"); final Map cipher = Parser.compatParseMap(cipherString); streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" - + deobfuscateSignature(cipher.get("s"), deobfuscationCode); + + deobfuscateSignature(cipher.get("s")); } urlAndItags.put(streamUrl, itagItem); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index 19c767ae..837ce603 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -209,6 +209,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest Date: Wed, 4 Nov 2020 14:50:35 +0100 Subject: [PATCH 80/81] [YouTube] Fix detection of ended livestreams and parse livestream upload date --- .../extractors/YoutubeStreamExtractor.java | 37 +++++++++++-------- .../YoutubeStreamExtractorLivestreamTest.java | 4 +- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index fb82d341..608ae47c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -137,18 +137,27 @@ public class YoutubeStreamExtractor extends StreamExtractor { return title; } + @Nullable @Override public String getTextualUploadDate() throws ParsingException { - if (getStreamType().equals(StreamType.LIVE_STREAM)) { - return null; - } - - JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer"); - if (micro.isString("uploadDate") && !micro.getString("uploadDate").isEmpty()) { + final JsonObject micro = + playerResponse.getObject("microformat").getObject("playerMicroformatRenderer"); + if (!micro.getString("uploadDate", EMPTY_STRING).isEmpty()) { return micro.getString("uploadDate"); - } - if (micro.isString("publishDate") && !micro.getString("publishDate").isEmpty()) { + } else if (!micro.getString("publishDate", EMPTY_STRING).isEmpty()) { return micro.getString("publishDate"); + } else { + final JsonObject liveDetails = micro.getObject("liveBroadcastDetails"); + if (!liveDetails.getString("endTimestamp", EMPTY_STRING).isEmpty()) { + // an ended live stream + return liveDetails.getString("endTimestamp"); + } else if (!liveDetails.getString("startTimestamp", EMPTY_STRING).isEmpty()) { + // a running live stream + return liveDetails.getString("startTimestamp"); + } else if (getStreamType() == StreamType.LIVE_STREAM) { + // this should never be reached, but a live stream without upload date is valid + return null; + } } if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) { @@ -176,6 +185,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate); } catch (Exception ignored) { } + throw new ParsingException("Could not get upload date"); } @@ -597,16 +607,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public StreamType getStreamType() throws ParsingException { + public StreamType getStreamType() { assertPageFetched(); - try { - return playerResponse.getObject("videoDetails").getBoolean("isLiveContent") - ? StreamType.LIVE_STREAM : StreamType.VIDEO_STREAM; - } catch (Exception e) { - throw new ParsingException("Could not get stream type", e); - } + return playerResponse.getObject("streamingData").has(FORMATS) + ? StreamType.VIDEO_STREAM : StreamType.LIVE_STREAM; } + @Nullable private StreamInfoItemExtractor getNextStream() throws ExtractionException { try { final JsonObject firstWatchNextItem = initialData.getObject("contents") diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index c1c84ed4..5c53a69e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -45,8 +45,8 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor @Override public long expectedLength() { return 0; } @Override public long expectedTimestamp() { return TIMESTAMP; } @Override public long expectedViewCountAtLeast() { return 0; } - @Nullable @Override public String expectedUploadDate() { return null; } - @Nullable @Override public String expectedTextualUploadDate() { return null; } + @Nullable @Override public String expectedUploadDate() { return "2020-02-22 00:00:00.000"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2020-02-22"; } @Override public long expectedLikeCountAtLeast() { return 825000; } @Override public long expectedDislikeCountAtLeast() { return 15600; } @Override public boolean expectedHasSubtitles() { return false; } From 3582a5189f43178d586c5b08772cc8acd8589325 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Wed, 18 Nov 2020 20:41:25 +0100 Subject: [PATCH 81/81] Release 0.20.3 --- README.md | 2 +- build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 11e616f3..1911c753 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo. If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps: 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. -2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.1'`the `dependencies` in your `build.gradle`. Replace `v0.20.1` with the latest release. +2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.3'`the `dependencies` in your `build.gradle`. Replace `v0.20.3` with the latest release. **Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required. diff --git a/build.gradle b/build.gradle index 070ed43f..c20b35c9 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.8 targetCompatibility = 1.8 - version 'v0.20.2' + version 'v0.20.3' group 'com.github.TeamNewPipe' repositories {