PaperTowel/NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement youtube subscription import from Google takeout

Browse Source
This commit is contained in:
Stypox
2020-11-01 19:11:08 +01:00
parent 2f02c0e6a4
commit 501ec30152
6 changed files with 309 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
extractor/src/test/java/org/schabi/newpipe/FileUtils.java
View File

@@ -61,6 +61,21 @@ public class FileUtils {
writer.close();
}
/**
* Resolves the test resource file based on its filename. Looks in
* {@code extractor/src/test/resources/} and {@code src/test/resources/}
* @param filename the resource filename
* @return the resource file
*/
public static File resolveTestResource(final String filename) {
final File file = new File("extractor/src/test/resources/" + filename);
if (file.exists()) {
return file;
} else {
return new File("src/test/resources/" + filename);
}
}
/**
* Convert a JSON object to String
* toString() does not produce a valid JSON string

84
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java
View File

@@ -11,12 +11,16 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.FileUtils.resolveTestResource;
/**
* Test for {@link YoutubeSubscriptionExtractor}
@@ -34,54 +38,48 @@ public class YoutubeSubscriptionExtractorTest {
@Test
public void testFromInputStream() throws Exception {
File testFile = new File("extractor/src/test/resources/youtube_export_test.xml");
if (!testFile.exists()) testFile = new File("src/test/resources/youtube_export_test.xml");
final List<SubscriptionItem> subscriptionItems = subscriptionExtractor.fromInputStream(
new FileInputStream(resolveTestResource("youtube_takeout_import_test.json")));
assertEquals(7, subscriptionItems.size());
List<SubscriptionItem> subscriptionItems = subscriptionExtractor.fromInputStream(new FileInputStream(testFile));
assertTrue("List doesn't have exactly 8 items (had " + subscriptionItems.size() + ")", subscriptionItems.size() == 8);
for (SubscriptionItem item : subscriptionItems) {
for (final SubscriptionItem item : subscriptionItems) {
assertNotNull(item.getName());
assertNotNull(item.getUrl());
assertTrue(urlHandler.acceptUrl(item.getUrl()));
assertFalse(item.getServiceId() == -1);
assertEquals(ServiceList.YouTube.getServiceId(), item.getServiceId());
}
}
@Test
public void testEmptySourceException() throws Exception {
String emptySource = "<opml version=\"1.1\"><body>" +
"<outline text=\"Testing\" title=\"123\" />" +
"</body></opml>";
List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(emptySource.getBytes("UTF-8")));
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)));
assertTrue(items.isEmpty());
}
@Test
public void testSubscriptionWithEmptyTitleInSource() throws Exception {
String channelId = "AA0AaAa0AaaaAAAAAA0aa0AA";
String source = "<opml version=\"1.1\"><body><outline text=\"YouTube Subscriptions\" title=\"YouTube Subscriptions\">" +
"<outline text=\"\" title=\"\" type=\"rss\" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_id=" + channelId + "\" />" +
"</outline></body></opml>";
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8")));
assertTrue("List doesn't have exactly 1 item (had " + items.size() + ")", items.size() == 1);
assertTrue("Item does not have an empty title (had \"" + items.get(0).getName() + "\")", items.get(0).getName().isEmpty());
assertTrue("Item does not have the right channel id \"" + channelId + "\" (the whole url is \"" + items.get(0).getUrl() + "\")", items.get(0).getUrl().endsWith(channelId));
assertEquals(1, items.size());
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", items.get(0).getUrl());
assertEquals("", items.get(0).getName());
}
@Test
public void testSubscriptionWithInvalidUrlInSource() throws Exception {
String source = "<opml version=\"1.1\"><body><outline text=\"YouTube Subscriptions\" title=\"YouTube Subscriptions\">" +
"<outline text=\"invalid\" title=\"url\" type=\"rss\" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_not_id=|||||||\"/>" +
"<outline text=\"fail\" title=\"fail\" type=\"rss\" xmlUgrl=\"invalidTag\"/>" +
"<outline text=\"invalid\" title=\"url\" type=\"rss\" xmlUrl=\"\"/>" +
"<outline text=\"\" title=\"\" type=\"rss\" xmlUrl=\"\"/>" +
"</outline></body></opml>";
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," +
"{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8")));
assertTrue(items.isEmpty());
assertEquals(1, items.size());
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", items.get(0).getUrl());
assertEquals("name2", items.get(0).getName());
}
@Test
@@ -89,26 +87,26 @@ public class YoutubeSubscriptionExtractorTest {
List<String> invalidList = Arrays.asList(
"<xml><notvalid></notvalid></xml>",
"<opml><notvalid></notvalid></opml>",
"<opml><body></body></opml>",
"{\"a\":\"b\"}",
"[{}]",
"[\"\", 5]",
"[{\"snippet\":{\"title\":\"name\"}}]",
"[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"}}}]",
"",
null,
"\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28",
"gibberish");
for (String invalidContent : invalidList) {
try {
if (invalidContent != null) {
byte[] bytes = invalidContent.getBytes("UTF-8");
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
} else {
subscriptionExtractor.fromInputStream(null);
fail("Extracting from null String didn't throw an exception");
byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
} catch (final Exception e) {
boolean correctType = e instanceof SubscriptionExtractor.InvalidSourceException;
if (!correctType) {
e.printStackTrace();
}
} catch (Exception e) {
// System.out.println(" -> " + e);
boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException;
assertTrue("\"" + e.getClass().getSimpleName() + "\" is not the expected exception", isExpectedException);
assertTrue(e.getClass().getSimpleName() + " is not InvalidSourceException", correctType);
}
}
}

23
extractor/src/test/resources/youtube_export_test.xml
View File

@@ -1,23 +0,0 @@
<opml version="1.1">
<body>
<outline text="YouTube Subscriptions" title="YouTube Subscriptions">
<outline text="Kurzgesagt – In a Nutshell" title="Kurzgesagt – In a Nutshell" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
<outline text="CaptainDisillusion" title="CaptainDisillusion" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCEOXxzW2vU0P-0THehuIIeg"/>
<outline text="TED" title="TED" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCAuUUnT6oDeKwE6v1NGQxug"/>
<outline text="Gorillaz" title="Gorillaz" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCfIXdjDQH9Fau7y99_Orpjw"/>
<outline text="ElectroBOOM" title="ElectroBOOM" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCJ0-OtVpF0wOKEqT2Z1HEtA"/>
<outline text="ⓤⓝⓘⓒⓞⓓⓔ" title="ⓤⓝⓘⓒⓞⓓⓔ" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
<outline text="中文" title="中文" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
<outline text="हिंदी" title="हिंदी" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
</outline>
</body>
</opml>

211
extractor/src/test/resources/youtube_takeout_import_test.json Normal file
View File

@@ -0,0 +1,211 @@
[ {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 229
},
"etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs",
"id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "The official YouTube home of Gorillaz.",
"publishedAt" : "2020-11-01T17:24:34.498Z",
"resourceId" : {
"channelId" : "UCfIXdjDQH9Fau7y99_Orpjw",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "Gorillaz"
}
}, {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 3502
},
"etag" : "wUgip-X0qBlnjj0frSTwP6B8XoY",
"id" : "qUAzBV8xkoLYOP-1gwzy63zpjj8SMTtDReGwIa2sHp8",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "The TED Talks channel features the best talks and performances from the TED Conference, where the world's leading thinkers and doers give the talk of their lives in 18 minutes (or less). Look for talks on Technology, Entertainment and Design -- plus science, business, global issues, the arts and more. You're welcome to link to or embed these videos, forward them to others and share these ideas with people you know.\n\nTED's videos may be used for non-commercial purposes under a Creative Commons License, Attribution–Non Commercial–No Derivatives (or the CC BY – NC – ND 4.0 International) and in accordance with our TED Talks Usage Policy (https://www.ted.com/about/our-organization/our-policies-terms/ted-talks-usage-policy). For more information on using TED for commercial purposes (e.g. employee learning, in a film or online course), please submit a Media Request at https://media-requests.ted.com",
"publishedAt" : "2020-11-01T17:24:11.769Z",
"resourceId" : {
"channelId" : "UCAuUUnT6oDeKwE6v1NGQxug",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-bonZt347bMc/AAAAAAAAAAI/AAAAAAAAAAA/lR8QEKnqqHk/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-bonZt347bMc/AAAAAAAAAAI/AAAAAAAAAAA/lR8QEKnqqHk/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-bonZt347bMc/AAAAAAAAAAI/AAAAAAAAAAA/lR8QEKnqqHk/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "TED"
}
}, {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 98
},
"etag" : "M3Hl6FQUAD3e-fH9pcvcE9aPSWQ",
"id" : "qUAzBV8xkoLYOP-1gwzy64Vo-PpWMPDyIYBM1JUfepk",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "In a world where the content of digital images and videos can no longer be taken at face value, an unlikely hero fights for the acceptance of truth.\r\n\r\nCaptain Disillusion guides children of all ages through the maze of visual fakery to the open spaces of reality and peace of mind.\r\n\r\nSubscribe to get fun and detailed explanations of current \"unbelievable\" viral videos that fool the masses!",
"publishedAt" : "2020-11-01T17:23:52.909Z",
"resourceId" : {
"channelId" : "UCEOXxzW2vU0P-0THehuIIeg",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-7f3hUYZP5Sc/AAAAAAAAAAI/AAAAAAAAAAA/4cpBHKDlbYQ/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-7f3hUYZP5Sc/AAAAAAAAAAI/AAAAAAAAAAA/4cpBHKDlbYQ/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-7f3hUYZP5Sc/AAAAAAAAAAI/AAAAAAAAAAA/4cpBHKDlbYQ/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "Captain Disillusion"
}
}, {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 130
},
"etag" : "crkTVZbDHS3arRZErMaLMnNqtac",
"id" : "qUAzBV8xkoLYOP-1gwzy66EVopYHE34m06PVw8Pvheg",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "Videos explaining things with optimistic nihilism. \n\nWe are a small team who want to make science look beautiful. Because it is beautiful. \n\nCurrently we make one animation video per month. Follow us on Twitter, Facebook to get notified when a new one comes out.\n\nFAQ:\n \n- We do the videos with After Effects and Illustrator.",
"publishedAt" : "2020-11-01T17:23:39.659Z",
"resourceId" : {
"channelId" : "UCsXVk37bltHxD1rDPwtNM8Q",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-UwENvFjc4vI/AAAAAAAAAAI/AAAAAAAAAAA/04dXvZ_jl0I/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-UwENvFjc4vI/AAAAAAAAAAI/AAAAAAAAAAA/04dXvZ_jl0I/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-UwENvFjc4vI/AAAAAAAAAAI/AAAAAAAAAAA/04dXvZ_jl0I/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "Kurzgesagt – In a Nutshell"
}
}, {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 229
},
"etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs",
"id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "ⓤⓝⓘⓒⓞⓓⓔ",
"publishedAt" : "2020-11-01T17:24:34.498Z",
"resourceId" : {
"channelId" : "UCfIXdjDQH9Fau7y99_Orpjw",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "ⓤⓝⓘⓒⓞⓓⓔ"
}
}, {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 229
},
"etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs",
"id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "中文",
"publishedAt" : "2020-11-01T17:24:34.498Z",
"resourceId" : {
"channelId" : "UCfIXdjDQH9Fau7y99_Orpjw",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "中文"
}
}, {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 229
},
"etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs",
"id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "हिंदी",
"publishedAt" : "2020-11-01T17:24:34.498Z",
"resourceId" : {
"channelId" : "UCfIXdjDQH9Fau7y99_Orpjw",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "हिंदी"
}
} ]