PaperTowel/NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor extractor

Browse Source 
- Refactor info classes and extractors
- Reformat and fix indentation
- Organize packages and classes
- Rename variables/methods and fix regex
- Change the capitalization
- Add methods to playlist extractor
This commit is contained in:
Mauricio Colli
2017-06-29 15:12:55 -03:00
parent 7581c1200b
commit 21e542e7d2
53 changed files with 1043 additions and 902 deletions
Download Patch File Download Diff File Expand all files Collapse all files

304
services/youtube/YoutubeChannelExtractor.java
View File

@@ -1,27 +1,26 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.AbstractStreamInfo;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.utils.Parser;
import java.io.IOException;
/**
/*
* Created by Christian Schabesberger on 25.07.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@@ -41,79 +40,55 @@ import java.io.IOException;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
@SuppressWarnings("WeakerAccess")
public class YoutubeChannelExtractor extends ChannelExtractor {
private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String TAG = YoutubeChannelExtractor.class.toString();
private Document doc;
/**
* It's lazily initialized (when getNextStreams is called)
*/
private Document nextStreamsAjax;
private String nextStreamsUrl = "";
// private CSSOMParser cssParser = new CSSOMParser(new SACParserCSS3());
/*//////////////////////////////////////////////////////////////////////////
// Variables for cache purposes (not "select" the current document all over again)
//////////////////////////////////////////////////////////////////////////*/
private String channelId;
private String channelName;
private String avatarUrl;
private String bannerUrl;
private String feedUrl;
private long subscriberCount = -1;
private Document doc = null;
private boolean isAjaxPage = false;
private static String userUrl = "";
private static String channelName = "";
private static String avatarUrl = "";
private static String bannerUrl = "";
private static String feedUrl = "";
private static long subscriberCount = -1;
// the fist page is html all other pages are ajax. Every new page can be requested by sending
// this request url.
private static String nextPageUrl = "";
public YoutubeChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
super(urlIdHandler, urlIdHandler.cleanUrl(url), serviceId);
fetchDocument();
}
public YoutubeChannelExtractor(UrlIdHandler urlIdHandler, String url, int page, int serviceId)
throws ExtractionException, IOException {
super(urlIdHandler, url, page, serviceId);
Downloader downloader = NewPipe.getDownloader();
url = urlIdHandler.cleanUrl(url) ; //+ "/video?veiw=0&flow=list&sort=dd";
if(page == 0) {
if (isUserUrl(url)) {
userUrl = url;
} else {
// we first need to get the user url. Otherwise we can't find videos
String channelPageContent = downloader.download(url);
Document channelDoc = Jsoup.parse(channelPageContent, url);
userUrl = getUserUrl(channelDoc);
@Override
public String getChannelId() throws ParsingException {
try {
if (channelId == null) {
channelId = getUrlIdHandler().getId(getUrl());
}
userUrl = userUrl + "/videos?veiw=0&flow=list&sort=dd&live_view=10000";
String pageContent = downloader.download(userUrl);
doc = Jsoup.parse(pageContent, userUrl);
nextPageUrl = getNextPageUrl(doc);
isAjaxPage = false;
} else {
String ajaxDataRaw = downloader.download(nextPageUrl);
JSONObject ajaxData;
try {
ajaxData = new JSONObject(ajaxDataRaw);
String htmlDataRaw = ajaxData.getString("content_html");
doc = Jsoup.parse(htmlDataRaw, nextPageUrl);
String nextPageHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if(!nextPageHtmlDataRaw.isEmpty()) {
Document nextPageData = Jsoup.parse(nextPageHtmlDataRaw, nextPageUrl);
nextPageUrl = getNextPageUrl(nextPageData);
} else {
nextPageUrl = "";
}
} catch (JSONException e) {
throw new ParsingException("Could not parse json data for next page", e);
}
isAjaxPage = true;
return channelId;
} catch (Exception e) {
throw new ParsingException("Could not get channel id");
}
}
@Override
public String getChannelName() throws ParsingException {
try {
if(!isAjaxPage) {
channelName = doc.select("span[class=\"qualified-channel-title-text\"]").first()
.select("a").first().text();
if (channelName == null) {
channelName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
}
return channelName;
} catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("Could not get channel name");
}
}
@@ -121,12 +96,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getAvatarUrl() throws ParsingException {
try {
if(!isAjaxPage) {
avatarUrl = doc.select("img[class=\"channel-header-profile-image\"]")
.first().attr("abs:src");
if (avatarUrl == null) {
avatarUrl = doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src");
}
return avatarUrl;
} catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("Could not get avatar", e);
}
}
@@ -134,19 +109,16 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getBannerUrl() throws ParsingException {
try {
if(!isAjaxPage) {
if (bannerUrl == null) {
Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first();
String cssContent = el.html();
String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent);
if (url.contains("s.ytimg.com") || url.contains("default_banner")) {
bannerUrl = null;
} else {
bannerUrl = url;
}
bannerUrl = url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url;
}
return bannerUrl;
} catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("Could not get Banner", e);
}
}
@@ -154,14 +126,105 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public StreamInfoItemCollector getStreams() throws ParsingException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Element ul;
if(isAjaxPage) {
ul = doc.select("body").first();
} else {
ul = doc.select("ul[id=\"browse-items-primary\"]").first();
Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
collectStreamsFrom(collector, ul);
return collector;
}
@Override
public long getSubscriberCount() throws ParsingException {
if (subscriberCount == -1) {
Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
if (el != null) {
subscriberCount = Long.parseLong(el.text().replaceAll("\\D+", ""));
} else {
throw new ParsingException("Could not get subscriber count");
}
}
for(final Element li : ul.children()) {
return subscriberCount;
}
@Override
public String getFeedUrl() throws ParsingException {
try {
if (feedUrl == null) {
String channelId = doc.getElementsByClass("yt-uix-subscription-button").first().attr("data-channel-external-id");
feedUrl = channelId == null ? "" : CHANNEL_FEED_BASE + channelId;
}
return feedUrl;
} catch (Exception e) {
throw new ParsingException("Could not get feed url", e);
}
}
@Override
public boolean hasMoreStreams() {
return nextStreamsUrl != null && !nextStreamsUrl.isEmpty();
}
@Override
public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException {
if (!hasMoreStreams()) throw new ExtractionException("Channel doesn't have more streams");
StreamInfoItemCollector collector = new StreamInfoItemCollector(getUrlIdHandler(), getServiceId());
setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
return collector;
}
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
String ajaxDataRaw = downloader.download(nextStreamsUrl);
try {
JSONObject ajaxData = new JSONObject(ajaxDataRaw);
String htmlDataRaw = ajaxData.getString("content_html");
nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextStreamsUrl);
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if (!nextStreamsHtmlDataRaw.isEmpty()) {
Document nextStreamsData = Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl);
nextStreamsUrl = getNextStreamsUrl(nextStreamsData);
} else {
nextStreamsUrl = "";
}
} catch (JSONException e) {
throw new ParsingException("Could not parse json data for next streams", e);
}
}
private String getNextStreamsUrl(Document d) throws ParsingException {
try {
Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
if (button != null) {
return button.attr("abs:data-uix-load-more-href");
} else {
// Sometimes channels are simply so small, they don't have a more streams/videos
return "";
}
} catch (Exception e) {
throw new ParsingException("could not get next streams' url", e);
}
}
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
Downloader downloader = NewPipe.getDownloader();
String userUrl = getUrl() + "/videos?view=0&flow=list&sort=dd&live_view=10000";
String pageContent = downloader.download(userUrl);
doc = Jsoup.parse(pageContent, userUrl);
nextStreamsUrl = getNextStreamsUrl(doc);
nextStreamsAjax = null;
}
private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException {
collector.getItemList().clear();
for (final Element li : element.children()) {
if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
collector.commit(new StreamInfoItemExtractor() {
@Override
@@ -201,8 +264,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
try {
return YoutubeParsingHelper.parseDurationString(
li.select("span[class=\"video-time\"]").first().text());
} catch(Exception e) {
if(isLiveStream(li)) {
} catch (Exception e) {
if (isLiveStream(li)) {
// -1 for no duration
return -1;
} else {
@@ -221,13 +284,13 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
try {
Element meta = li.select("div[class=\"yt-lockup-meta\"]").first();
Element li = meta.select("li").first();
if (li == null && meta != null) {
if (li == null) {
//this means we have a youtube red video
return "";
}else {
} else {
return li.text();
}
} catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("Could not get upload date", e);
}
}
@@ -244,16 +307,13 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return -1;
}
output = Parser.matchGroup1("([0-9,\\. ]*)", input)
.replace(" ", "")
.replace(".", "")
.replace(",", "");
output = input.replaceAll("\\D+", "");
try {
return Long.parseLong(output);
} catch (NumberFormatException e) {
// if this happens the video probably has no views
if(!input.isEmpty()) {
if (!input.isEmpty()) {
return 0;
} else {
throw new ParsingException("Could not handle input: " + input, e);
@@ -283,10 +343,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
private boolean isLiveStream(Element item) {
Element bla = item.select("span[class*=\"yt-badge-live\"]").first();
if(bla == null) {
if (bla == null) {
// sometimes livestreams dont have badges but sill are live streams
// if video time is not available we most likly have an offline livestream
if(item.select("span[class*=\"video-time\"]").first() == null) {
if (item.select("span[class*=\"video-time\"]").first() == null) {
return true;
}
}
@@ -295,63 +355,5 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
});
}
}
return collector;
}
@Override
public long getSubscriberCount() throws ParsingException {
Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]")
.first();
if(el != null) {
subscriberCount = Long.parseLong(el.text().replaceAll("\\D+",""));
} else if(el == null && subscriberCount == -1) {
throw new ParsingException("Could not get subscriber count");
}
return subscriberCount;
}
@Override
public String getFeedUrl() throws ParsingException {
try {
if(userUrl.contains("channel")) {
//channels don't have feeds in youtube, only user can provide such
return "";
}
if(!isAjaxPage) {
feedUrl = doc.select("link[title=\"RSS\"]").first().attr("abs:href");
}
return feedUrl;
} catch(Exception e) {
throw new ParsingException("Could not get feed url", e);
}
}
@Override
public boolean hasNextPage() throws ParsingException {
return !nextPageUrl.isEmpty();
}
private String getUserUrl(Document d) throws ParsingException {
return d.select("span[class=\"qualified-channel-title-text\"]").first()
.select("a").first().attr("abs:href");
}
private boolean isUserUrl(String url) throws ParsingException {
return url.contains("/user/");
}
private String getNextPageUrl(Document d) throws ParsingException {
try {
Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
if(button != null) {
return button.attr("abs:data-uix-load-more-href");
} else {
// sometimes channels are simply so small, they don't have a second/next4q page
return "";
}
} catch(Exception e) {
throw new ParsingException("could not load next page url", e);
}
}
}

25
services/youtube/YoutubeChannelInfoItemExtractor.java
View File

@@ -1,11 +1,10 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.Parser;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.jsoup.nodes.Element;
/**
/*
* Created by Christian Schabesberger on 12.02.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
@@ -32,49 +31,55 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
this.el = el;
}
@Override
public String getThumbnailUrl() throws ParsingException {
Element img = el.select("span[class*=\"yt-thumb-simple\"]").first()
.select("img").first();
String url = img.attr("abs:src");
if(url.contains("gif")) {
if (url.contains("gif")) {
url = img.attr("abs:data-thumb");
}
return url;
}
@Override
public String getChannelName() throws ParsingException {
return el.select("a[class*=\"yt-uix-tile-link\"]").first()
.text();
}
@Override
public String getWebPageUrl() throws ParsingException {
return el.select("a[class*=\"yt-uix-tile-link\"]").first()
.attr("abs:href");
}
@Override
public long getSubscriberCount() throws ParsingException {
Element subsEl = el.select("span[class*=\"yt-subscriber-count\"]").first();
if(subsEl == null) {
if (subsEl == null) {
return 0;
} else {
return Integer.parseInt(subsEl.text().replaceAll("\\D+",""));
return Long.parseLong(subsEl.text().replaceAll("\\D+", ""));
}
}
public int getVideoAmount() throws ParsingException {
@Override
public long getViewCount() throws ParsingException {
Element metaEl = el.select("ul[class*=\"yt-lockup-meta-info\"]").first();
if(metaEl == null) {
if (metaEl == null) {
return 0;
} else {
return Integer.parseInt(metaEl.text().replaceAll("\\D+",""));
return Long.parseLong(metaEl.text().replaceAll("\\D+", ""));
}
}
@Override
public String getDescription() throws ParsingException {
Element desEl = el.select("div[class*=\"yt-lockup-description\"]").first();
if(desEl == null) {
if (desEl == null) {
return "";
} else {
return desEl.text();

8
services/youtube/YoutubeChannelUrlIdHandler.java
View File

@@ -1,10 +1,10 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
/**
/*
* Created by Christian Schabesberger on 25.07.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@@ -35,13 +35,13 @@ public class YoutubeChannelUrlIdHandler implements UrlIdHandler {
}
public String cleanUrl(String siteUrl) throws ParsingException {
return getUrl(getId(siteUrl));
return getUrl(getId(siteUrl));
}
public boolean acceptUrl(String videoUrl) {
return (videoUrl.contains("youtube") ||
videoUrl.contains("youtu.be")) &&
( videoUrl.contains("/user/") ||
(videoUrl.contains("/user/") ||
videoUrl.contains("/channel/"));
}
}

4
services/youtube/YoutubeParsingHelper.java
View File

@@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
/**
/*
* Created by Christian Schabesberger on 02.03.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@@ -36,7 +36,7 @@ public class YoutubeParsingHelper {
String minutes = "0";
String seconds;
switch(splitInput.length) {
switch (splitInput.length) {
case 4:
days = splitInput[0];
hours = splitInput[1];

223
services/youtube/YoutubePlayListExtractor.java
View File

@@ -1,223 +0,0 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.AbstractStreamInfo;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlayListExtractor;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor;
import java.io.IOException;
public class YoutubePlayListExtractor extends PlayListExtractor {
private String TAG = YoutubePlayListExtractor.class.toString();
private Document doc = null;
private boolean isAjaxPage = false;
private static String name = "";
private static String feedUrl = "";
private static String avatarUrl = "";
private static String bannerUrl = "";
private static String nextPageUrl = "";
public YoutubePlayListExtractor(UrlIdHandler urlIdHandler,
String url, int page, int serviceId) throws IOException, ExtractionException {
super(urlIdHandler, url, page, serviceId);
Downloader downloader = NewPipe.getDownloader();
url = urlIdHandler.cleanUrl(url);
if(page == 0) {
String channelPageContent = downloader.download(url);
doc = Jsoup.parse(channelPageContent, url);
nextPageUrl = getNextPageUrl(doc);
isAjaxPage = false;
} else {
String ajaxDataRaw = downloader.download(nextPageUrl);
JSONObject ajaxData;
try {
ajaxData = new JSONObject(ajaxDataRaw);
final String htmlDataRaw = "<table><tbody id=\"pl-load-more-destination\">" + ajaxData.getString("content_html") + "</tbody></table>";
doc = Jsoup.parse(htmlDataRaw, nextPageUrl);
final String nextPageHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if(!nextPageHtmlDataRaw.isEmpty()) {
final Document nextPageData = Jsoup.parse(nextPageHtmlDataRaw, nextPageUrl);
nextPageUrl = getNextPageUrl(nextPageData);
} else {
nextPageUrl = "";
}
} catch (JSONException e) {
throw new ParsingException("Could not parse json data for next page", e);
}
isAjaxPage = true;
}
}
@Override
public String getName() throws ParsingException {
try {
if (!isAjaxPage) {
name = doc.select("span[class=\"qualified-channel-title-text\"]").first()
.select("a").first().text() + " - " +
doc.select("meta[name=title]").first().attr("content");
}
return name;
} catch (Exception e) {
throw new ParsingException("Could not get playlist name");
}
}
@Override
public String getAvatarUrl() throws ParsingException {
try {
if(!isAjaxPage) {
avatarUrl = doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("src");
if(avatarUrl.startsWith("//")) {
avatarUrl = "https:" + avatarUrl;
}
}
return avatarUrl;
} catch(Exception e) {
throw new ParsingException("Could not get playlist Avatar");
}
}
@Override
public String getBannerUrl() throws ParsingException {
try {
if(!isAjaxPage) {
Element el = doc.select("div[id=\"gh-banner\"] style").first();
String cssContent = el.html();
String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent);
if (url.contains("s.ytimg.com")) {
bannerUrl = null;
} else {
bannerUrl = url.substring(0, url.indexOf(");"));
}
}
return bannerUrl;
} catch(Exception e) {
throw new ParsingException("Could not get playlist Banner");
}
}
@Override
public StreamInfoItemCollector getStreams() throws ParsingException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first();
final YoutubeStreamUrlIdHandler youtubeStreamUrlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
for(final Element li : tbody.children()) {
collector.commit(new StreamInfoItemExtractor() {
@Override
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
}
@Override
public String getWebPageUrl() throws ParsingException {
try {
return youtubeStreamUrlIdHandler.getUrl(li.attr("data-video-id"));
} catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e);
}
}
@Override
public String getTitle() throws ParsingException {
try {
return li.attr("data-title");
} catch (Exception e) {
throw new ParsingException("Could not get title", e);
}
}
@Override
public int getDuration() throws ParsingException {
try {
return YoutubeParsingHelper.parseDurationString(
li.select("div[class=\"timestamp\"] span").first().text().trim());
} catch(Exception e) {
if(isLiveStream(li)) {
// -1 for no duration
return -1;
} else {
throw new ParsingException("Could not get Duration: " + getTitle(), e);
}
}
}
@Override
public String getUploader() throws ParsingException {
return li.select("div[class=pl-video-owner] a").text();
}
@Override
public String getUploadDate() throws ParsingException {
return "";
}
@Override
public long getViewCount() throws ParsingException {
return -1;
}
@Override
public String getThumbnailUrl() throws ParsingException {
try {
return "https://i.ytimg.com/vi/" + youtubeStreamUrlIdHandler.getId(getWebPageUrl()) + "/hqdefault.jpg";
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
@Override
public boolean isAd() throws ParsingException {
return false;
}
private boolean isLiveStream(Element item) {
Element bla = item.select("span[class*=\"yt-badge-live\"]").first();
if(bla == null) {
// sometimes livestreams dont have badges but sill are live streams
// if video time is not available we most likly have an offline livestream
if(item.select("span[class*=\"video-time\"]").first() == null) {
return true;
}
}
return bla != null;
}
});
}
return collector;
}
@Override
public boolean hasNextPage() throws ParsingException {
return nextPageUrl != null && !nextPageUrl.isEmpty();
}
private String getNextPageUrl(Document d) throws ParsingException {
try {
Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
if(button != null) {
return "https://www.youtube.com" + button.attr("data-uix-load-more-href");
} else {
// sometimes channels are simply so small, they don't have a second/next4q page
return "";
}
} catch(Exception e) {
throw new ParsingException("could not load next page url", e);
}
}
}

332
services/youtube/YoutubePlaylistExtractor.java Normal file
View File

@@ -0,0 +1,332 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.utils.Parser;
import java.io.IOException;
@SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor {
private Document doc = null;
/**
* It's lazily initialized (when getNextStreams is called)
*/
private Document nextStreamsAjax = null;
private String nextStreamsUrl = "";
/*//////////////////////////////////////////////////////////////////////////
// Variables for cache purposes (not "select" the current document all over again)
//////////////////////////////////////////////////////////////////////////*/
private String playlistId;
private String playlistName;
private String avatarUrl;
private String bannerUrl;
private long streamsCount;
private String uploaderUrl;
private String uploaderName;
private String uploaderAvatarUrl;
public YoutubePlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws IOException, ExtractionException {
super(urlIdHandler, urlIdHandler.cleanUrl(url), serviceId);
fetchDocument();
}
@Override
public String getPlaylistId() throws ParsingException {
try {
if (playlistId == null) {
playlistId = getUrlIdHandler().getId(getUrl());
}
return playlistId;
} catch (Exception e) {
throw new ParsingException("Could not get playlist id");
}
}
@Override
public String getPlaylistName() throws ParsingException {
try {
if (playlistName == null) {
playlistName = doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
}
return playlistName;
} catch (Exception e) {
throw new ParsingException("Could not get playlist name");
}
}
@Override
public String getAvatarUrl() throws ParsingException {
try {
if (avatarUrl == null) {
avatarUrl = doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
}
return avatarUrl;
} catch (Exception e) {
throw new ParsingException("Could not get playlist avatar");
}
}
@Override
public String getBannerUrl() throws ParsingException {
try {
if (bannerUrl == null) {
Element el = doc.select("div[id=\"gh-banner\"] style").first();
String cssContent = el.html();
String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent);
if (url.contains("s.ytimg.com")) {
bannerUrl = null;
} else {
bannerUrl = url.substring(0, url.indexOf(");"));
}
}
return bannerUrl;
} catch (Exception e) {
throw new ParsingException("Could not get playlist Banner");
}
}
@Override
public String getUploaderUrl() throws ParsingException {
try {
if (uploaderUrl == null) {
uploaderUrl = doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href");
}
return uploaderUrl;
} catch (Exception e) {
throw new ParsingException("Could not get playlist uploader name");
}
}
@Override
public String getUploaderName() throws ParsingException {
try {
if (uploaderName == null) {
uploaderName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
}
return uploaderName;
} catch (Exception e) {
throw new ParsingException("Could not get playlist uploader name");
}
}
@Override
public String getUploaderAvatarUrl() throws ParsingException {
try {
if (uploaderAvatarUrl == null) {
uploaderAvatarUrl = doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
}
return uploaderAvatarUrl;
} catch (Exception e) {
throw new ParsingException("Could not get playlist uploader avatar");
}
}
@Override
public long getStreamsCount() throws ParsingException {
if (streamsCount <= 0) {
String input;
try {
input = doc.select("ul[class=\"pl-header-details\"] li").get(1).text();
} catch (IndexOutOfBoundsException e) {
throw new ParsingException("Could not get video count from playlist", e);
}
try {
streamsCount = Long.parseLong(input.replaceAll("\\D+", ""));
} catch (NumberFormatException e) {
// When there's no videos in a playlist, there's no number in the "innerHtml",
// all characters that is not a number is removed, so we try to parse a empty string
if (!input.isEmpty()) {
streamsCount = 0;
} else {
throw new ParsingException("Could not handle input: " + input, e);
}
}
}
return streamsCount;
}
@Override
public StreamInfoItemCollector getStreams() throws ParsingException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first();
collectStreamsFrom(collector, tbody);
return collector;
}
@Override
public boolean hasMoreStreams() {
return nextStreamsUrl != null && !nextStreamsUrl.isEmpty();
}
@Override
public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException {
if (!hasMoreStreams()) throw new ExtractionException("Playlist doesn't have more streams");
StreamInfoItemCollector collector = new StreamInfoItemCollector(getUrlIdHandler(), getServiceId());
setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
return collector;
}
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
String ajaxDataRaw = downloader.download(nextStreamsUrl);
try {
JSONObject ajaxData = new JSONObject(ajaxDataRaw);
String htmlDataRaw = "<table><tbody id=\"pl-load-more-destination\">" + ajaxData.getString("content_html") + "</tbody></table>";
nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextStreamsUrl);
String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if (!nextStreamsHtmlDataRaw.isEmpty()) {
final Document nextStreamsData = Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl);
nextStreamsUrl = getNextStreamsUrl(nextStreamsData);
} else {
nextStreamsUrl = "";
}
} catch (JSONException e) {
throw new ParsingException("Could not parse json data for next streams", e);
}
}
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
Downloader downloader = NewPipe.getDownloader();
String pageContent = downloader.download(getUrl());
doc = Jsoup.parse(pageContent, getUrl());
nextStreamsUrl = getNextStreamsUrl(doc);
nextStreamsAjax = null;
}
private String getNextStreamsUrl(Document d) throws ParsingException {
try {
Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
if (button != null) {
return button.attr("abs:data-uix-load-more-href");
} else {
// Sometimes playlists are simply so small, they don't have a more streams/videos
return "";
}
} catch (Exception e) {
throw new ParsingException("could not get next streams' url", e);
}
}
private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException {
collector.getItemList().clear();
final YoutubeStreamUrlIdHandler youtubeStreamUrlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
for (final Element li : element.children()) {
collector.commit(new StreamInfoItemExtractor() {
@Override
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
}
@Override
public String getWebPageUrl() throws ParsingException {
try {
return youtubeStreamUrlIdHandler.getUrl(li.attr("data-video-id"));
} catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e);
}
}
@Override
public String getTitle() throws ParsingException {
try {
return li.attr("data-title");
} catch (Exception e) {
throw new ParsingException("Could not get title", e);
}
}
@Override
public int getDuration() throws ParsingException {
try {
return YoutubeParsingHelper.parseDurationString(
li.select("div[class=\"timestamp\"] span").first().text().trim());
} catch (Exception e) {
if (isLiveStream(li)) {
// -1 for no duration
return -1;
} else {
throw new ParsingException("Could not get Duration: " + getTitle(), e);
}
}
}
@Override
public String getUploader() throws ParsingException {
return li.select("div[class=pl-video-owner] a").text();
}
@Override
public String getUploadDate() throws ParsingException {
return "";
}
@Override
public long getViewCount() throws ParsingException {
return -1;
}
@Override
public String getThumbnailUrl() throws ParsingException {
try {
return "https://i.ytimg.com/vi/" + youtubeStreamUrlIdHandler.getId(getWebPageUrl()) + "/hqdefault.jpg";
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
@Override
public boolean isAd() throws ParsingException {
return false;
}
private boolean isLiveStream(Element item) {
Element bla = item.select("span[class*=\"yt-badge-live\"]").first();
if (bla == null) {
// sometimes livestreams dont have badges but sill are live streams
// if video time is not available we most likly have an offline livestream
if (item.select("span[class*=\"video-time\"]").first() == null) {
return true;
}
}
return bla != null;
}
});
}
}
}

4
services/youtube/YoutubePlayListUrlIdHandler.java → services/youtube/YoutubePlaylistUrlIdHandler.java
View File

@@ -1,11 +1,11 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
public class YoutubePlayListUrlIdHandler implements UrlIdHandler {
public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{34})";

19
services/youtube/YoutubeSearchEngine.java
View File

@@ -10,12 +10,12 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine;
import java.net.URLEncoder;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.EnumSet;
/**
/*
* Created by Christian Schabesberger on 09.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@@ -58,20 +58,19 @@ public class YoutubeSearchEngine extends SearchEngine {
String url = "https://www.youtube.com/results"
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+ "&page=" + Integer.toString(page + 1);
if(filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) {
if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) {
url += "&sp=EgIQAQ%253D%253D";
} else if(!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) {
} else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) {
url += "&sp=EgIQAg%253D%253D";
}
String site;
//String url = builder.build().toString();
//if we've been passed a valid language code, append it to the URL
if(!languageCode.isEmpty()) {
if (!languageCode.isEmpty()) {
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = downloader.download(url, languageCode);
}
else {
site = downloader.download(url, languageCode);
} else {
site = downloader.download(url);
}
@@ -95,7 +94,7 @@ public class YoutubeSearchEngine extends SearchEngine {
// both types of spell correction item
if ((el = item.select("div[class*=\"spell-correction\"]").first()) != null) {
collector.setSuggestion(el.select("a").first().text());
if(list.children().size() == 1) {
if (list.children().size() == 1) {
throw new NothingFoundException("Did you mean: " + el.select("a").first().text());
}
// search message item
@@ -105,7 +104,7 @@ public class YoutubeSearchEngine extends SearchEngine {
// video item type
} else if ((el = item.select("div[class*=\"yt-lockup-video\"]").first()) != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(el));
} else if((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) {
} else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) {
collector.commit(new YoutubeChannelInfoItemExtractor(el));
} else {
// noinspection ConstantConditions

30
services/youtube/YoutubeService.java
View File

@@ -1,18 +1,18 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.playlist.PlayListExtractor;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.stream_info.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import java.io.IOException;
/**
/*
* Created by Christian Schabesberger on 23.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@@ -44,17 +44,18 @@ public class YoutubeService extends StreamingService {
serviceInfo.name = "Youtube";
return serviceInfo;
}
@Override
public StreamExtractor getExtractorInstance(String url)
throws ExtractionException, IOException {
UrlIdHandler urlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
if(urlIdHandler.acceptUrl(url)) {
if (urlIdHandler.acceptUrl(url)) {
return new YoutubeStreamExtractor(urlIdHandler, url, getServiceId());
}
else {
} else {
throw new IllegalArgumentException("supplied String is not a valid Youtube URL");
}
}
@Override
public SearchEngine getSearchEngineInstance() {
return new YoutubeSearchEngine(getStreamUrlIdHandlerInstance(), getServiceId());
@@ -72,19 +73,18 @@ public class YoutubeService extends StreamingService {
@Override
public UrlIdHandler getPlayListUrlIdHandlerInstance() {
return new YoutubePlayListUrlIdHandler();
public UrlIdHandler getPlaylistUrlIdHandlerInstance() {
return new YoutubePlaylistUrlIdHandler();
}
@Override
public ChannelExtractor getChannelExtractorInstance(String url, int page)
throws ExtractionException, IOException {
return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, page, getServiceId());
public ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException {
return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, getServiceId());
}
public PlayListExtractor getPlayListExtractorInstance(String url, int page)
throws ExtractionException, IOException {
return new YoutubePlayListExtractor(getPlayListUrlIdHandlerInstance(), url, page, getServiceId());
@Override
public PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException {
return new YoutubePlaylistExtractor(getPlaylistUrlIdHandlerInstance(), url, getServiceId());
}
@Override

105
services/youtube/YoutubeStreamExtractor.java
View File

@@ -8,21 +8,21 @@ import org.jsoup.nodes.Element;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.AbstractStreamInfo;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream_info.AudioStream;
import org.schabi.newpipe.extractor.stream_info.StreamExtractor;
import org.schabi.newpipe.extractor.stream_info.StreamInfo;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream_info.VideoStream;
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Parser;
import java.io.IOException;
import java.util.List;
@@ -31,7 +31,7 @@ import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
/*
* Created by Christian Schabesberger on 06.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@@ -129,6 +129,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public int bandWidth = -1;
}
/**
* List can be found here https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L360
*/
private static final ItagItem[] itagList = {
//////////////////////////////////////////////////////////////////////////
// VIDEO ID ItagType Format Resolution FPS ///
@@ -186,13 +189,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
new ItagItem(315, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p60" , 60)
};
/**These lists only contain itag formats that are supported by the common Android Video player.
However if you are looking for a list showing all itag formats, look at
https://github.com/rg3/youtube-dl/issues/1687 */
public static boolean itagIsSupported(int itag) {
for(ItagItem item : itagList) {
if(itag == item.id) {
for (ItagItem item : itagList) {
if (itag == item.id) {
return true;
}
}
@@ -200,8 +199,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
public static ItagItem getItagItem(int itag) throws ParsingException {
for(ItagItem item : itagList) {
if(itag == item.id) {
for (ItagItem item : itagList) {
if (itag == item.id) {
return item;
}
}
@@ -215,7 +214,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private Map<String, String> videoInfoPage;
// static values
private static final String DECRYPTION_FUNC_NAME="decrypt";
private static final String DECRYPTION_FUNC_NAME = "decrypt";
// cached values
private static volatile String decryptionCode = "";
@@ -249,7 +248,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
isAgeRestricted = false;
}
if(decryptionCode.isEmpty()) {
if (decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
}
}
@@ -283,11 +282,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
playerArgs = playerConfig.getJSONObject("args");
// check if we have a live stream. We need to filter it, since its not yet supported.
if((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
isLiveStream = true;
}
} catch (JSONException e) {
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
if (isLiveStream) {
@@ -352,7 +351,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
//json player args method
return playerArgs.getString("title");
} catch(JSONException je) {//html <meta> method
} catch (JSONException je) {//html <meta> method
je.printStackTrace();
System.err.println("failed to load title from JSON args; trying to extract it from HTML");
try { // fall through to fall-back
@@ -380,11 +379,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
//json player args method
return playerArgs.getString("author");
} catch(JSONException je) {
} catch (JSONException je) {
je.printStackTrace();
System.err.println(
"failed to load uploader name from JSON args; trying to extract it from HTML");
} try {//fall through to fallback HTML method
}
try {//fall through to fallback HTML method
return doc.select("div.yt-user-info").first().text();
} catch (Exception e) {
throw new ParsingException("failed permanently to load uploader name.", e);
@@ -429,7 +429,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// Try to get high resolution thumbnail if it fails use low res from the player instead
try {
return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
} catch(Exception e) {
} catch (Exception e) {
System.err.println("Could not find high res Thumbnail. Using low res instead");
}
try { //fall through to fallback
@@ -458,14 +458,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getDashMpdUrl() throws ParsingException {
try {
String dashManifestUrl = "";
if(videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) {
if (videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) {
dashManifestUrl = videoInfoPage.get("dashmpd");
} else if (playerArgs.has("dashmpd")) {
dashManifestUrl = playerArgs.getString("dashmpd");
} else {
return "";
}
if(!dashManifestUrl.contains("/signature/")) {
if (!dashManifestUrl.contains("/signature/")) {
String encryptedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
String decryptedSig;
@@ -483,23 +483,23 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public List<AudioStream> getAudioStreams() throws ParsingException {
Vector<AudioStream> audioStreams = new Vector<>();
try{
try {
String encodedUrlMap;
// playerArgs could be null if the video is age restricted
if (playerArgs == null) {
if(videoInfoPage.containsKey("adaptive_fmts")) {
if (videoInfoPage.containsKey("adaptive_fmts")) {
encodedUrlMap = videoInfoPage.get("adaptive_fmts");
} else {
return null;
}
} else {
if(playerArgs.has("adaptive_fmts")) {
if (playerArgs.has("adaptive_fmts")) {
encodedUrlMap = playerArgs.getString("adaptive_fmts");
} else {
return null;
}
}
for(String url_data_str : encodedUrlMap.split(",")) {
for (String url_data_str : encodedUrlMap.split(",")) {
// This loop iterates through multiple streams, therefor tags
// is related to one and the same stream at a time.
Map<String, String> tags = Parser.compatParseMap(
@@ -535,7 +535,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public List<VideoStream> getVideoStreams() throws ParsingException {
Vector<VideoStream> videoStreams = new Vector<>();
try{
try {
String encodedUrlMap;
// playerArgs could be null if the video is age restricted
if (playerArgs == null) {
@@ -543,7 +543,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} else {
encodedUrlMap = playerArgs.getString(URL_ENCODED_FMT_STREAM_MAP);
}
for(String url_data_str : encodedUrlMap.split(",")) {
for (String url_data_str : encodedUrlMap.split(",")) {
try {
// This loop iterates through multiple streams, therefor tags
// is related to one and the same stream at a time.
@@ -554,7 +554,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (itagIsSupported(itag)) {
ItagItem itagItem = getItagItem(itag);
if(itagItem.itagType == ItagType.VIDEO) {
if (itagItem.itagType == ItagType.VIDEO) {
String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
@@ -578,7 +578,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new ParsingException("Failed to get video streams", e);
}
if(videoStreams.isEmpty()) {
if (videoStreams.isEmpty()) {
throw new ParsingException("Failed to get any video stream");
}
return videoStreams;
@@ -640,8 +640,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return videoOnlyStreams;
}
/**Attempts to parse (and return) the offset to start playing the video from.
* @return the offset (in seconds), or 0 if no timestamp is found.*/
/**
* Attempts to parse (and return) the offset to start playing the video from.
*
* @return the offset (in seconds), or 0 if no timestamp is found.
*/
@Override
public int getTimeStamp() throws ParsingException {
String timeStamp;
@@ -655,7 +658,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return -2;
}
if(!timeStamp.isEmpty()) {
if (!timeStamp.isEmpty()) {
try {
String secondsString = "";
String minutesString = "";
@@ -748,10 +751,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return -1;
}
return Integer.parseInt(dislikesString.replaceAll(REGEX_INT, ""));
} catch(NumberFormatException nfe) {
} catch (NumberFormatException nfe) {
throw new ParsingException(
"failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe);
} catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("Could not get dislike count", e);
}
}
@@ -761,7 +764,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
.select("li").first());
} catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("Could not get next video", e);
}
}
@@ -771,7 +774,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Element ul = doc.select("ul[id=\"watch-related\"]").first();
if(ul != null) {
if (ul != null) {
for (Element li : ul.children()) {
// first check if we have a playlist. If so leave them out
if (li.select("a[class*=\"content-link\"]").first() != null) {
@@ -780,7 +783,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
return collector;
} catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("Could not get related videos", e);
}
}
@@ -795,7 +798,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
return doc.select("div[class=\"yt-user-info\"]").first().children()
.select("a").first().attr("abs:href");
} catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("Could not get channel link", e);
}
}
@@ -806,9 +809,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return StreamInfo.StreamType.VIDEO_STREAM;
}
/**Provides information about links to other videos on the video page, such as related videos.
/**
* Provides information about links to other videos on the video page, such as related videos.
* This is encapsulated in a StreamInfoItem object,
* which is a subset of the fields in a full StreamInfo.*/
* which is a subset of the fields in a full StreamInfo.
*/
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
return new StreamInfoItemExtractor() {
@Override
@@ -896,7 +901,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
Downloader downloader = NewPipe.getDownloader();
if(!playerUrl.contains("https://youtube.com")) {
if (!playerUrl.contains("https://youtube.com")) {
//sometimes the https://youtube.com part does not get send with
//than we have to add it by hand
playerUrl = "https://youtube.com" + playerUrl;
@@ -921,9 +926,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
callerFunc = callerFunc.replace("%%", decryptionFuncName);
decryptionCode = helperObject + decryptionFunc + callerFunc;
} catch(IOException ioe) {
} catch (IOException ioe) {
throw new DecryptException("Could not load decrypt function", ioe);
} catch(Exception e) {
} catch (Exception e) {
throw new DecryptException("Could not parse decrypt function ", e);
}
@@ -931,7 +936,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
private String decryptSignature(String encryptedSig, String decryptionCode)
throws DecryptException{
throws DecryptException {
Context context = Context.enter();
context.setOptimizationLevel(-1);
Object result = null;

27
services/youtube/YoutubeStreamInfoItemExtractor.java
View File

@@ -1,13 +1,12 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.AbstractStreamInfo;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
/**
/*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeStreamInfoItemExtractor.java is part of NewPipe.
*
@@ -60,8 +59,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
try {
return YoutubeParsingHelper.parseDurationString(
item.select("span[class=\"video-time\"]").first().text());
} catch(Exception e) {
if(isLiveStream(item)) {
} catch (Exception e) {
if (isLiveStream(item)) {
// -1 for no duration
return -1;
} else {
@@ -85,12 +84,12 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
public String getUploadDate() throws ParsingException {
try {
Element div = item.select("div[class=\"yt-lockup-meta\"]").first();
if(div == null) {
if (div == null) {
return null;
} else {
return div.select("li").first().text();
}
} catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("Could not get upload date", e);
}
}
@@ -101,14 +100,14 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
String input;
try {
Element div = item.select("div[class=\"yt-lockup-meta\"]").first();
if(div == null) {
if (div == null) {
return -1;
} else {
input = div.select("li").get(1)
.text();
}
} catch (IndexOutOfBoundsException e) {
if(isLiveStream(item)) {
if (isLiveStream(item)) {
// -1 for no view count
return -1;
} else {
@@ -123,7 +122,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return Long.parseLong(output);
} catch (NumberFormatException e) {
// if this happens the video probably has no views
if(!input.isEmpty()) {
if (!input.isEmpty()) {
return 0;
} else {
throw new ParsingException("Could not handle input: " + input, e);
@@ -152,7 +151,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public AbstractStreamInfo.StreamType getStreamType() {
if(isLiveStream(item)) {
if (isLiveStream(item)) {
return AbstractStreamInfo.StreamType.LIVE_STREAM;
} else {
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
@@ -167,10 +166,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
private boolean isLiveStream(Element item) {
Element bla = item.select("span[class*=\"yt-badge-live\"]").first();
if(bla == null) {
if (bla == null) {
// sometimes livestreams dont have badges but sill are live streams
// if video time is not available we most likly have an offline livestream
if(item.select("span[class*=\"video-time\"]").first() == null) {
if (item.select("span[class*=\"video-time\"]").first() == null) {
return true;
}
}

34
services/youtube/YoutubeStreamUrlIdHandler.java
View File

@@ -2,11 +2,11 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.Parser;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
@@ -14,7 +14,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
/**
/*
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@@ -39,7 +39,8 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
private static final YoutubeStreamUrlIdHandler instance = new YoutubeStreamUrlIdHandler();
private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{11})";
private YoutubeStreamUrlIdHandler() {}
private YoutubeStreamUrlIdHandler() {
}
public static YoutubeStreamUrlIdHandler getInstance() {
return instance;
@@ -52,13 +53,13 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
@Override
public String getId(String url) throws ParsingException, IllegalArgumentException {
if(url.isEmpty()) {
if (url.isEmpty()) {
throw new IllegalArgumentException("The url parameter should not be empty");
}
String id;
String lowercaseUrl = url.toLowerCase();
if(lowercaseUrl.contains("youtube")) {
if (lowercaseUrl.contains("youtube")) {
if (url.contains("attribution_link")) {
try {
String escapedQuery = Parser.matchGroup1("u=(.[^&|$]*)", url);
@@ -67,31 +68,29 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
} catch (UnsupportedEncodingException uee) {
throw new ParsingException("Could not parse attribution_link", uee);
}
} else if(lowercaseUrl.contains("youtube.com/shared?ci=")) {
} else if (lowercaseUrl.contains("youtube.com/shared?ci=")) {
return getRealIdFromSharedLink(url);
} else if (url.contains("vnd.youtube")) {
id = Parser.matchGroup1(ID_PATTERN, url);
} else if (url.contains("embed")) {
id = Parser.matchGroup1("embed/" + ID_PATTERN, url);
} else if(url.contains("googleads")) {
} else if (url.contains("googleads")) {
throw new FoundAdException("Error found add: " + url);
} else {
id = Parser.matchGroup1("[?&]v=" + ID_PATTERN, url);
}
}
else if(lowercaseUrl.contains("youtu.be")) {
if(url.contains("v=")) {
} else if (lowercaseUrl.contains("youtu.be")) {
if (url.contains("v=")) {
id = Parser.matchGroup1("v=" + ID_PATTERN, url);
} else {
id = Parser.matchGroup1("[Yy][Oo][Uu][Tt][Uu]\\.[Bb][Ee]/" + ID_PATTERN, url);
}
}
else {
} else {
throw new ParsingException("Error no suitable url: " + url);
}
if(!id.isEmpty()){
if (!id.isEmpty()) {
return id;
} else {
throw new ParsingException("Error could not parse url: " + url);
@@ -100,12 +99,13 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
/**
* Get the real url from a shared uri.
*
* <p>
* Shared URI's look like this:
* <pre>
* * https://www.youtube.com/shared?ci=PJICrTByb3E
* * vnd.youtube://www.youtube.com/shared?ci=PJICrTByb3E&feature=twitter-deep-link
* </pre>
*
* @param url The shared url
* @return the id of the stream
* @throws ParsingException
@@ -127,8 +127,8 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
}
// is this bad? is this fragile?:
String realId = Parser.matchGroup1("rel=\"shortlink\" href=\"https://youtu.be/" + ID_PATTERN, content);
if(sharedId.equals(realId)) {
throw new ParsingException("Got same id for as shared id: " + sharedId);
if (sharedId.equals(realId)) {
throw new ParsingException("Got same id for as shared info_id: " + sharedId);
}
return realId;
}
@@ -147,7 +147,7 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
@Override
public boolean acceptUrl(String videoUrl) {
String lowercaseUrl = videoUrl.toLowerCase();
if(lowercaseUrl.contains("youtube") ||
if (lowercaseUrl.contains("youtube") ||
lowercaseUrl.contains("youtu.be")) {
// bad programming I know
try {

6
services/youtube/YoutubeSuggestionExtractor.java
View File

@@ -2,9 +2,9 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
@@ -20,7 +20,7 @@ import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
/**
/*
* Created by Christian Schabesberger on 28.09.16.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@@ -92,7 +92,7 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
}
}
return suggestions;
} catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("Could not get suggestions form document.", e);
}
}