PaperTowel/NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #306 from B0pol/metadata

Browse Source 
Extract metadata for youtube, soundcloud & mediaccc
This commit is contained in:
Stypox
2021-03-27 08:45:47 +01:00
committed by GitHub
parent e61ceef005 152221c7fb
commit b4dee6d08f
22 changed files with 313 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java
View File

@@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -9,6 +10,7 @@ import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -69,7 +71,8 @@ public class ExtractorAsserts {
}
// this assumes that sorting a and b in-place is not an issue, so it's only intended for tests
public static void assertEqualsOrderIndependent(List<String> expected, List<String> actual) {
public static void assertEqualsOrderIndependent(final List<String> expected,
final List<String> actual) {
if (expected == null) {
assertNull(actual);
return;
@@ -79,6 +82,7 @@ public class ExtractorAsserts {
Collections.sort(expected);
Collections.sort(actual);
assertEquals(expected, actual);
// using new ArrayList<> to make sure the type is the same
assertEquals(new ArrayList<>(expected), new ArrayList<>(actual));
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java
View File

@@ -66,7 +66,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
@Nullable public String expectedDashMpdUrlContains() { return null; } // default: no dash mpd
public boolean expectedHasFrames() { return true; } // default: there are frames
public String expectedHost() { return ""; } // default: no host for centralized platforms
public String expectedPrivacy() { return ""; } // default: no privacy policy available
public StreamExtractor.Privacy expectedPrivacy() { return StreamExtractor.Privacy.PUBLIC; } // default: public
public String expectedCategory() { return ""; } // default: no category
public String expectedLicence() { return ""; } // default: no licence
public Locale expectedLanguageInfo() { return null; } // default: no language info available

36
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java
View File

@@ -5,15 +5,17 @@ import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.annotation.Nullable;
import java.util.Locale;
import static junit.framework.TestCase.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
@@ -42,7 +44,6 @@ public class MediaCCCStreamExtractorTest {
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return URL; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "gpn18"; }
@Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/gpn18"; }
@@ -58,10 +59,10 @@ public class MediaCCCStreamExtractorTest {
@Override public boolean expectedHasFrames() { return false; }
@Override public List<String> expectedTags() { return Arrays.asList("gpn18", "105"); }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Override public Locale expectedLanguageInfo() { return new Locale("de"); }
@Override
@Test
public void testThumbnailUrl() throws Exception {
@Test public void testThumbnailUrl() throws Exception {
super.testThumbnailUrl();
assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl());
}
@@ -100,13 +101,20 @@ public class MediaCCCStreamExtractorTest {
extractor.fetchPage();
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return MediaCCC; }
@Override public String expectedName() { return "What's left for private messaging?"; }
@Override public String expectedId() { return ID; }
@Override public StreamExtractor extractor() {
return extractor;
}
@Override public StreamingService expectedService() {
return MediaCCC;
}
@Override public String expectedName() {
return "What's left for private messaging?";
}
@Override public String expectedId() {
return ID;
}
@Override public String expectedUrlContains() { return URL; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "36c3"; }
@Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/36c3"; }
@@ -123,8 +131,7 @@ public class MediaCCCStreamExtractorTest {
@Override public List<String> expectedTags() { return Arrays.asList("36c3", "10565", "2019", "Security", "Main"); }
@Override
@Test
public void testThumbnailUrl() throws Exception {
@Test public void testThumbnailUrl() throws Exception {
super.testThumbnailUrl();
assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl());
}
@@ -149,5 +156,10 @@ public class MediaCCCStreamExtractorTest {
super.testAudioStreams();
assertEquals(2, extractor.getAudioStreams().size());
}
@Override
public Locale expectedLanguageInfo() {
return new Locale("en");
}
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java
View File

@@ -88,7 +88,6 @@ public class PeertubeStreamExtractorTest {
@Override public boolean expectedHasAudioStreams() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public String expectedHost() { return "framatube.org"; }
@Override public String expectedPrivacy() { return "Public"; }
@Override public String expectedCategory() { return "Science & Technology"; }
@Override public String expectedLicence() { return "Attribution - Share Alike"; }
@Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); }
@@ -139,7 +138,6 @@ public class PeertubeStreamExtractorTest {
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public String expectedHost() { return "nocensoring.net"; }
@Override public String expectedPrivacy() { return "Public"; }
@Override public String expectedCategory() { return "Art"; }
@Override public String expectedLicence() { return "Attribution"; }
@Override public List<String> expectedTags() { return Arrays.asList("Covid-19", "Gérôme-Mary trebor", "Horreur et beauté", "court-métrage", "nue artistique"); }

29
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java
View File

@@ -34,11 +34,15 @@ public class SoundcloudStreamExtractorTest {
private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP;
private static StreamExtractor extractor;
@Test(expected = GeographicRestrictionException.class)
public void geoRestrictedContent() throws Exception {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = SoundCloud.getStreamExtractor(URL);
extractor.fetchPage();
try {
extractor.fetchPage();
} catch (final GeographicRestrictionException e) {
// expected
}
}
@Override public StreamExtractor extractor() { return extractor; }
@@ -67,6 +71,8 @@ public class SoundcloudStreamExtractorTest {
@Override public boolean expectedHasFrames() { return false; }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Override public boolean expectedHasRelatedStreams() { return false; }
@Override public String expectedLicence() { return "all-rights-reserved"; }
@Override public String expectedCategory() { return "Pop"; }
}
public static class SoundcloudGoPlusTrack extends DefaultStreamExtractorTest {
@@ -76,11 +82,15 @@ public class SoundcloudStreamExtractorTest {
private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP;
private static StreamExtractor extractor;
@Test(expected = SoundCloudGoPlusContentException.class)
public void goPlusContent() throws Exception {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = SoundCloud.getStreamExtractor(URL);
extractor.fetchPage();
try {
extractor.fetchPage();
} catch (final SoundCloudGoPlusContentException e) {
// expected
}
}
@Override public StreamExtractor extractor() { return extractor; }
@@ -109,6 +119,8 @@ public class SoundcloudStreamExtractorTest {
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Override public String expectedLicence() { return "all-rights-reserved"; }
@Override public String expectedCategory() { return "Dance"; }
}
public static class CreativeCommonsPlaysWellWithOthers extends DefaultStreamExtractorTest {
@@ -148,6 +160,11 @@ public class SoundcloudStreamExtractorTest {
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Override public String expectedLicence() { return "cc-by"; }
@Override public String expectedCategory() { return "Podcast"; }
@Override public List<String> expectedTags() {
return Arrays.asList("ants", "collaboration", "creative commons", "stigmergy", "storytelling", "wikipedia");
}
@Override
@Test

12
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java
View File

@@ -54,4 +54,16 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
@Override public int expectedAgeLimit() { return 18; }
@Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public String expectedCategory() {return "Entertainment"; }
@Override public String expectedLicence() { return "YouTube licence"; }
@Override
public List<String> expectedTags() {
return Arrays.asList("AEE", "AEE 2017", "AVN", "AVN 2016", "AVN 2017", "AVN 2017 Expo In Las Vegas",
"AVN Awards Show", "AVN Expo", "AVN Las Vegas", "AVN Magazine", "AVN Vlog", "Ariana Marie",
"August Ames", "Brenna Sparks", "CeCe Capella", "Cindy Starfall", "Elsa Jean", "Emma Hix",
"FINGERING", "FINGERING P0RNSTARS", "FINGERING PORNSTARS", "Kaho Shibuya", "Keisha Grey",
"Kimberly Chi", "Las Vegas", "Mia Martinez", "Pornstar", "Pornstars", "Riley Reid",
"Samantha Saint", "Vegas", "Vicki Chase");
}
}

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java
View File

@@ -56,4 +56,8 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
@Nullable @Override public String expectedTextualUploadDate() { return "2010-09-09"; }
@Override public long expectedLikeCountAtLeast() { return 13300; }
@Override public long expectedDislikeCountAtLeast() { return 2600; }
@Override public List<String> expectedTags() { return Arrays.asList("Books", "Burning", "Jones", "Koran", "Qur'an", "Terry", "the amazing atheist"); }
@Override public String expectedCategory() { return "Entertainment"; }
@Override public String expectedLicence() { return "YouTube licence"; }
}

90
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View File

@@ -4,6 +4,7 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
@@ -15,6 +16,7 @@ import org.schabi.newpipe.extractor.exceptions.PrivateContentException;
import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamSegment;
@@ -56,6 +58,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeStreamExtractorDefaultTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
static final String BASE_URL = "https://www.youtube.com/watch?v=";
public static final String YOUTUBE_LICENCE = "YouTube licence";
public static class NotAvailable {
@BeforeClass
@@ -145,6 +148,8 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedLikeCountAtLeast() { return 5212900; }
@Override public long expectedDislikeCountAtLeast() { return 30600; }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
@Override public String expectedCategory() { return "Entertainment"; }
// @formatter:on
}
@@ -185,6 +190,16 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedLikeCountAtLeast() { return 340100; }
@Override public long expectedDislikeCountAtLeast() { return 18700; }
@Override public boolean expectedUploaderVerified() { return true; }
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
@Override public String expectedCategory() { return "Science & Technology"; }
@Override public List<String> expectedTags() {
return Arrays.asList("2018", "8 plus", "apple", "apple iphone", "apple iphone x", "best", "best android",
"best smartphone", "cool gadgets", "find", "find x", "find x review", "find x unboxing", "findx",
"galaxy s9", "galaxy s9+", "hands on", "iphone 8", "iphone 8 plus", "iphone x", "new iphone", "nex",
"oneplus 6", "oppo", "oppo find x", "oppo find x hands on", "oppo find x review",
"oppo find x unboxing", "oppo findx", "pixel 2 xl", "review", "samsung", "samsung galaxy",
"samsung galaxy s9", "smartphone", "unbox therapy", "unboxing", "vivo", "vivo apex", "vivo nex");
}
// @formatter:on
}
@@ -260,8 +275,17 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedLikeCountAtLeast() { return 32100; }
@Override public long expectedDislikeCountAtLeast() { return 750; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public int expectedStreamSegmentsCount() { return 17; }
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
@Override public String expectedCategory() { return "Music"; }
@Override public List<String> expectedTags() {
return Arrays.asList("2019", "2019 anime", "Anime OST", "Epic anime ost", "OST Anime",
"anime epic soundtrack", "armin", "attack on titan", "battle anime ost", "battle anime soundtracks",
"combat anime ost", "epic soundtrack", "eren", "mikasa", "motivational anime ost",
"motivational anime soundtracks", "shingeki no kyojin");
}
// @formatter:on
@Test
public void testStreamSegment() throws Exception {
final StreamSegment segment = extractor.getStreamSegments().get(3);
@@ -270,7 +294,6 @@ public class YoutubeStreamExtractorDefaultTest {
assertEquals(BASE_URL + ID + "?t=589", segment.getUrl());
assertNotNull(segment.getPreviewUrl());
}
// @formatter:on
}
public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest {
@@ -308,6 +331,16 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedDislikeCountAtLeast() { return 20000; }
@Override public boolean expectedHasSubtitles() { return true; }
@Override public int expectedStreamSegmentsCount() { return 7; }
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
@Override public String expectedCategory() { return "Science & Technology"; }
@Override public List<String> expectedTags() {
return Arrays.asList("Diabetes", "Erkältung", "Gesundheit", "Immunabwehr", "Immunsystem", "Infektion",
"Komisch alles chemisch", "Krebs", "Lab", "Lesch", "Mai", "Mai Thi", "Mai Thi Nguyen-Kim",
"Mangel", "Nahrungsergänzungsmittel", "Nguyen", "Nguyen Kim", "Nguyen-Kim", "Quarks", "Sommer",
"Supplemente", "Supplements", "Tabletten", "Terra X", "TerraX", "The Secret Life Of Scientists",
"Tropfen", "Vitamin D", "Vitamin-D-Mangel", "Vitamine", "Winter", "einnehmen", "maiLab", "nehmen",
"supplementieren", "Überdosis", "Überschuss");
}
// @formatter:on
@Test
@@ -322,9 +355,12 @@ public class YoutubeStreamExtractorDefaultTest {
@Override
@Test
@Ignore("encoding problem")
public void testName() throws Exception {
super.testName();
}
public void testName() {}
@Override
@Test
@Ignore("encoding problem")
public void testTags() {}
}
public static class PublicBroadcasterTest extends DefaultStreamExtractorTest {
@@ -369,7 +405,51 @@ public class YoutubeStreamExtractorDefaultTest {
));
}
@Override public boolean expectedUploaderVerified() { return true; }
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
@Override public String expectedCategory() { return "Education"; }
@Override public List<String> expectedTags() {
return Arrays.asList("Abgrund", "Algen", "Bakterien", "Challengertief", "Dumbooktopus",
"Dunkel", "Dunkelheit", "Fische", "Flohkrebs", "Hadal-Zone", "Kontinentalschelf",
"Licht", "Mariannengraben", "Meer", "Meeresbewohner", "Meeresschnee", "Mesopelagial",
"Ozean", "Photosynthese", "Plankton", "Plastik", "Polypen", "Pottwale",
"Staatsquelle", "Tauchen", "Tauchgang", "Tentakel", "Tiefe", "Tiefsee", "Tintenfische",
"Titanic", "Vampirtintenfisch", "Verschmutzung", "Viperfisch", "Wale");
}
// @formatter:on
}
public static class UnlistedTest {
private static YoutubeStreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=tjz2u2DiveM");
extractor.fetchPage();
}
@Test
public void testGetUnlisted() {
assertEquals(StreamExtractor.Privacy.UNLISTED, extractor.getPrivacy());
}
}
public static class CCLicensed {
private static final String ID = "M4gD1WSo5mA";
private static final String URL = BASE_URL + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@Test
public void testGetLicence() throws ParsingException {
assertEquals("Creative Commons Attribution licence (reuse allowed)", extractor.getLicence());
}
}
}

14
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java
View File

@@ -39,13 +39,13 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
@Override public String expectedOriginalUrlContains() { return URL; }
@Override public StreamType expectedStreamType() { return StreamType.LIVE_STREAM; }
@Override public String expectedUploaderName() { return "ChilledCow"; }
@Override public String expectedUploaderName() { return "Lofi Girl"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow"; }
@Override public List<String> expectedDescriptionContains() {
return Arrays.asList("https://bit.ly/chilledcow-playlists",
"https://bit.ly/chilledcow-submissions");
}
@Override public boolean expectedUploaderVerified() { return true; }
@Override public boolean expectedUploaderVerified() { return false; }
@Override public long expectedLength() { return 0; }
@Override public long expectedTimestamp() { return TIMESTAMP; }
@Override public long expectedViewCountAtLeast() { return 0; }
@@ -56,4 +56,14 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
@Override public boolean expectedHasSubtitles() { return false; }
@Nullable @Override public String expectedDashMpdUrlContains() { return "https://manifest.googlevideo.com/api/manifest/dash/"; }
@Override public boolean expectedHasFrames() { return false; }
@Override public String expectedLicence() { return "YouTube licence"; }
@Override public String expectedCategory() { return "Music"; }
@Override public List<String> expectedTags() {
return Arrays.asList("beats to relax", "chilled cow", "chilled cow radio", "chilledcow", "chilledcow radio",
"chilledcow station", "chillhop", "hip hop", "hiphop", "lo fi", "lo fi hip hop", "lo fi hip hop radio",
"lo fi hiphop", "lo fi radio", "lo-fi", "lo-fi hip hop", "lo-fi hip hop radio", "lo-fi hiphop",
"lo-fi radio", "lofi", "lofi hip hop", "lofi hip hop radio", "lofi hiphop", "lofi radio", "music",
"lofi radio chilledcow", "music to study", "playlist", "radio", "relaxing music", "study music",
"lofi hip hop radio - beats to relax\\/study to");
}
}

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java
View File

@@ -15,6 +15,7 @@ import java.util.List;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.stream.StreamExtractor.Privacy.UNLISTED;
public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
@@ -50,4 +51,8 @@ public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTe
@Nullable @Override public String expectedTextualUploadDate() { return "2017-09-22"; }
@Override public long expectedLikeCountAtLeast() { return 110; }
@Override public long expectedDislikeCountAtLeast() { return 0; }
@Override public StreamExtractor.Privacy expectedPrivacy() { return UNLISTED; }
@Override public String expectedLicence() { return "YouTube licence"; }
@Override public String expectedCategory() { return "Gaming"; }
@Override public List<String> expectedTags() { return Arrays.asList("dark souls", "hooked", "praise the casual"); }
}