PaperTowel/NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix dash parser and more refactor

Browse Source 
- Add new itags
This commit is contained in:
Mauricio Colli
2017-07-11 00:08:03 -03:00
parent bda65e83d6
commit b1989c0a83
41 changed files with 802 additions and 751 deletions
Download Patch File Download Diff File Expand all files Collapse all files

692
services/youtube/YoutubeStreamExtractor.java
View File

@@ -9,20 +9,21 @@ import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.List;
@@ -52,12 +53,11 @@ import java.util.regex.Pattern;
*/
public class YoutubeStreamExtractor extends StreamExtractor {
public static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
public static final String HTTPS = "https:";
public static final String CONTENT = "content";
public static final String REGEX_INT = "[^\\d]";
private static final String TAG = YoutubeStreamExtractor.class.getSimpleName();
// exceptions
/*//////////////////////////////////////////////////////////////////////////
// Exceptions
//////////////////////////////////////////////////////////////////////////*/
public class DecryptException extends ParsingException {
DecryptException(String message, Throwable cause) {
@@ -65,8 +65,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
// special content not available exceptions
public class GemaException extends ContentNotAvailableException {
GemaException(String message) {
super(message);
@@ -79,267 +77,27 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
// ----------------
/*//////////////////////////////////////////////////////////////////////////*/
// Sometimes if the html page of youtube is already downloaded, youtube web page will internally
// download the /get_video_info page. Since a certain date dashmpd url is only available over
// this /get_video_info page, so we always need to download this one to.
// %%video_id%% will be replaced by the actual video id
// $$el_type$$ will be replaced by the actual el_type (se the declarations below)
private static final String GET_VIDEO_INFO_URL =
"https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en";
// eltype is necessary for the url above
private static final String EL_INFO = "el=info";
private Document doc;
private final String dirtyUrl;
public enum ItagType {
AUDIO,
VIDEO,
VIDEO_ONLY
public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException {
super(urlIdHandler, urlIdHandler.cleanUrl(pageUrl), serviceId);
dirtyUrl = pageUrl;
fetchDocument();
}
private static class ItagItem {
public ItagItem(int id, ItagType type, MediaFormat format, String res, int fps) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.resolutionString = res;
this.fps = fps;
}
/*//////////////////////////////////////////////////////////////////////////
// Impl
//////////////////////////////////////////////////////////////////////////*/
public ItagItem(int id, ItagType type, MediaFormat format, int samplingRate, int bandWidth) {
this(id, type, format, 0, samplingRate, bandWidth);
}
public ItagItem(int id, ItagType type, MediaFormat format, int avgBitrate, int samplingRate, int bandWidth) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.avgBitrate = avgBitrate;
this.samplingRate = samplingRate;
this.bandWidth = bandWidth;
}
public int id;
public ItagType itagType;
public int mediaFormatId;
public String resolutionString;
public int fps = -1;
public int avgBitrate = -1;
public int samplingRate = -1;
public int bandWidth = -1;
}
/**
* List can be found here https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L360
*/
private static final ItagItem[] itagList = {
//////////////////////////////////////////////////////////////////////////
// VIDEO ID ItagType Format Resolution FPS ///
////////////////////////////////////////////////////////////////////////
new ItagItem(17, ItagType.VIDEO, MediaFormat.v3GPP, "144p" , 12),
new ItagItem(18, ItagType.VIDEO, MediaFormat.MPEG_4, "360p" , 24),
new ItagItem(22, ItagType.VIDEO, MediaFormat.MPEG_4, "720p" , 24),
new ItagItem(36, ItagType.VIDEO, MediaFormat.v3GPP, "240p" , 24),
new ItagItem(37, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p" , 24),
new ItagItem(38, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p" , 24),
new ItagItem(43, ItagType.VIDEO, MediaFormat.WEBM, "360p" , 24),
new ItagItem(44, ItagType.VIDEO, MediaFormat.WEBM, "480p" , 24),
new ItagItem(45, ItagType.VIDEO, MediaFormat.WEBM, "720p" , 24),
new ItagItem(46, ItagType.VIDEO, MediaFormat.WEBM, "1080p" , 24),
//////////////////////////////////////////////////////////////////////////////////////////
// AUDIO ID ItagType Format Bitrate SamplingR Bandwidth ///
////////////////////////////////////////////////////////////////////////////////////////
// Disable Opus codec as it's not well supported in older devices
// new ItagItem(249, ItagType.AUDIO, MediaFormat.WEBMA, 50, 0, 0),
// new ItagItem(250, ItagType.AUDIO, MediaFormat.WEBMA, 70, 0, 0),
// new ItagItem(251, ItagType.AUDIO, MediaFormat.WEBMA, 160, 0, 0),
new ItagItem(171, ItagType.AUDIO, MediaFormat.WEBMA, 128, 0, 0),
new ItagItem(172, ItagType.AUDIO, MediaFormat.WEBMA, 256, 0, 0),
new ItagItem(140, ItagType.AUDIO, MediaFormat.M4A, 128, 0, 0),
new ItagItem(141, ItagType.AUDIO, MediaFormat.M4A, 256, 0, 0),
/// VIDEO ONLY ///////////////////////////////////////////////////////////////////
// ID ItagType Format Resolution FPS ///
////////////////////////////////////////////////////////////////////////////////
// Don't add VideoOnly streams that have normal variants
// new ItagItem(160, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "144p" , 24),
// new ItagItem(133, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "240p" , 24),
// new ItagItem(134, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "360p" , 24),
new ItagItem(135, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "480p" , 30),
// new ItagItem(136, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p" , 30),
new ItagItem(298, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p60" , 60),
new ItagItem(137, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p" , 30),
new ItagItem(299, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p60" , 60),
new ItagItem(266, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "2160p" , 30),
// new ItagItem(243, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "360p" , 30),
new ItagItem(244, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
new ItagItem(245, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
new ItagItem(246, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
new ItagItem(247, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "720p" , 30),
new ItagItem(248, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1080p" , 30),
new ItagItem(271, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1440p" , 30),
// #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
new ItagItem(272, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p" , 30),
new ItagItem(302, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "720p60" , 60),
new ItagItem(303, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1080p60" , 60),
new ItagItem(308, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1440p60" , 60),
new ItagItem(313, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p" , 30),
new ItagItem(315, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p60" , 60)
};
public static boolean itagIsSupported(int itag) {
for (ItagItem item : itagList) {
if (itag == item.id) {
return true;
}
}
return false;
}
public static ItagItem getItagItem(int itag) throws ParsingException {
for (ItagItem item : itagList) {
if (itag == item.id) {
return item;
}
}
throw new ParsingException("itag=" + Integer.toString(itag) + " not supported");
}
private static final String TAG = YoutubeStreamExtractor.class.toString();
private final Document doc;
private JSONObject playerArgs;
private boolean isAgeRestricted;
private Map<String, String> videoInfoPage;
// static values
private static final String DECRYPTION_FUNC_NAME = "decrypt";
// cached values
private static volatile String decryptionCode = "";
UrlIdHandler urlidhandler = YoutubeStreamUrlIdHandler.getInstance();
String pageUrl = "";
public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId)
throws ExtractionException, IOException {
super(urlIdHandler, pageUrl, serviceId);
//most common videoInfo fields are now set in our superclass, for all services
this.pageUrl = pageUrl;
Downloader downloader = NewPipe.getDownloader();
String pageContent = downloader.download(urlidhandler.cleanUrl(pageUrl));
doc = Jsoup.parse(pageContent, pageUrl);
JSONObject ytPlayerConfig;
String playerUrl;
// Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%",
urlidhandler.getId(pageUrl)).replace("$$el_type$$", "&" + EL_INFO);
String videoInfoPageString = downloader.download(videoInfoUrl);
videoInfoPage = Parser.compatParseMap(videoInfoPageString);
playerUrl = getPlayerUrlFromRestrictedVideo(pageUrl);
isAgeRestricted = true;
} else {
ytPlayerConfig = getPlayerConfig(pageContent);
playerArgs = getPlayerArgs(ytPlayerConfig);
playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false;
}
if (decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
}
}
private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
@Override
public String getId() throws ParsingException {
try {
String ytPlayerConfigRaw =
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
return new JSONObject(ytPlayerConfigRaw);
} catch (Parser.RegexException e) {
String errorReason = getErrorMessage();
switch(errorReason) {
case "GEMA":
throw new GemaException(errorReason);
case "":
throw new ContentNotAvailableException("Content not available: player config empty", e);
default:
throw new ContentNotAvailableException("Content not available", e);
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
}
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
JSONObject playerArgs;
//attempt to load the youtube js player JSON arguments
boolean isLiveStream = false; //used to determine if this is a livestream or not
try {
playerArgs = playerConfig.getJSONObject("args");
// check if we have a live stream. We need to filter it, since its not yet supported.
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
isLiveStream = true;
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
if (isLiveStream) {
throw new LiveStreamException("This is a Life stream. Can't use those right now.");
}
return playerArgs;
}
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
try {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
String playerUrl = "";
JSONObject ytAssets = playerConfig.getJSONObject("assets");
playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (JSONException e) {
throw new ParsingException(
"Could not load decryption code for the Youtube service.", e);
}
}
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
try {
Downloader downloader = NewPipe.getDownloader();
String playerUrl = "";
String videoId = urlidhandler.getId(pageUrl);
String embedUrl = "https://www.youtube.com/embed/" + videoId;
String embedPageContent = downloader.download(embedUrl);
//todo: find out if this can be reapaced by Parser.matchGroup1()
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
while (patternMatcher.find()) {
playerUrl = patternMatcher.group(1);
}
playerUrl = playerUrl.replace("\\", "").replace("\"", "");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge requested");
return getUrlIdHandler().getId(getUrl());
} catch (Exception e) {
throw new ParsingException("Could not get stream id");
}
}
@@ -457,7 +215,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getDashMpdUrl() throws ParsingException {
try {
String dashManifestUrl = "";
String dashManifestUrl;
if (videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) {
dashManifestUrl = videoInfoPage.get("dashmpd");
} else if (playerArgs.has("dashmpd")) {
@@ -479,7 +237,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
@Override
public List<AudioStream> getAudioStreams() throws ParsingException {
Vector<AudioStream> audioStreams = new Vector<>();
@@ -507,9 +264,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
int itag = Integer.parseInt(tags.get("itag"));
if (itagIsSupported(itag)) {
ItagItem itagItem = getItagItem(itag);
if (itagItem.itagType == ItagType.AUDIO) {
if (ItagItem.isSupported(itag)) {
ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
@@ -517,11 +274,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
+ decryptSignature(tags.get("s"), decryptionCode);
}
audioStreams.add(new AudioStream(streamUrl,
itagItem.mediaFormatId,
itagItem.avgBitrate,
itagItem.bandWidth,
itagItem.samplingRate));
AudioStream audioStream = new AudioStream(streamUrl, itagItem.mediaFormatId, itagItem.avgBitrate);
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
audioStreams.add(audioStream);
}
}
}
}
@@ -552,19 +308,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
int itag = Integer.parseInt(tags.get("itag"));
if (itagIsSupported(itag)) {
ItagItem itagItem = getItagItem(itag);
if (itagItem.itagType == ItagType.VIDEO) {
if (ItagItem.isSupported(itag)) {
ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == ItagItem.ItagType.VIDEO) {
String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
streamUrl = streamUrl + "&signature="
+ decryptSignature(tags.get("s"), decryptionCode);
}
videoStreams.add(new VideoStream(
streamUrl,
itagItem.mediaFormatId,
itagItem.resolutionString));
VideoStream videoStream = new VideoStream(streamUrl, itagItem.mediaFormatId, itagItem.resolutionString);
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
videoStreams.add(videoStream);
}
}
}
} catch (Exception e) {
@@ -612,9 +369,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
int itag = Integer.parseInt(tags.get("itag"));
if (itagIsSupported(itag)) {
ItagItem itagItem = getItagItem(itag);
if (itagItem.itagType == ItagType.VIDEO_ONLY) {
if (ItagItem.isSupported(itag)) {
ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY) {
String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
@@ -622,11 +379,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
+ decryptSignature(tags.get("s"), decryptionCode);
}
videoOnlyStreams.add(new VideoStream(
true, //isVideoOnly
streamUrl,
itagItem.mediaFormatId,
itagItem.resolutionString));
VideoStream videoStream = new VideoStream(streamUrl, itagItem.mediaFormatId, itagItem.resolutionString, true);
if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) {
videoOnlyStreams.add(videoStream);
}
}
}
}
@@ -649,7 +405,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public int getTimeStamp() throws ParsingException {
String timeStamp;
try {
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", dirtyUrl);
} catch (Parser.RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp
@@ -730,7 +486,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
//if this ckicks in our button has no content and thefore likes/dislikes are disabled
return -1;
}
return Integer.parseInt(likesString.replaceAll(REGEX_INT, ""));
return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
} catch (NumberFormatException nfe) {
throw new ParsingException(
"failed to parse likesString \"" + likesString + "\" as integers", nfe);
@@ -750,7 +506,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
//if this kicks in our button has no content and therefore likes/dislikes are disabled
return -1;
}
return Integer.parseInt(dislikesString.replaceAll(REGEX_INT, ""));
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
} catch (NumberFormatException nfe) {
throw new ParsingException(
"failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe);
@@ -788,11 +544,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
@Override
public String getPageUrl() {
return pageUrl;
}
@Override
public String getChannelUrl() throws ParsingException {
try {
@@ -804,92 +555,186 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Override
public StreamInfo.StreamType getStreamType() throws ParsingException {
public StreamType getStreamType() throws ParsingException {
//todo: if implementing livestream support this value should be generated dynamically
return StreamInfo.StreamType.VIDEO_STREAM;
return StreamType.VIDEO_STREAM;
}
/**
* Provides information about links to other videos on the video page, such as related videos.
* This is encapsulated in a StreamInfoItem object,
* which is a subset of the fields in a full StreamInfo.
* {@inheritDoc}
*/
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
return new StreamInfoItemExtractor() {
@Override
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
}
@Override
public String getErrorMessage() {
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
StringBuilder errorReason;
@Override
public boolean isAd() throws ParsingException {
return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
}
if (errorMessage == null || errorMessage.isEmpty()) {
errorReason = null;
} else if (errorMessage.contains("GEMA")) {
// Gema sometimes blocks youtube music content in germany:
// https://www.gema.de/en/
// Detailed description:
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
errorReason = new StringBuilder("GEMA");
} else {
errorReason = new StringBuilder(errorMessage);
errorReason.append(" ");
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
}
@Override
public String getWebPageUrl() throws ParsingException {
return li.select("a.content-link").first().attr("abs:href");
}
@Override
public String getTitle() throws ParsingException {
//todo: check NullPointerException causing
return li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
}
@Override
public int getDuration() throws ParsingException {
return YoutubeParsingHelper.parseDurationString(
li.select("span.video-time").first().text());
}
@Override
public String getUploader() throws ParsingException {
return li.select("span.g-hovercard").first().text();
}
@Override
public String getUploadDate() throws ParsingException {
return null;
}
@Override
public long getViewCount() throws ParsingException {
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
return Long.parseLong(li.select("span.view-count")
.first().text().replaceAll(REGEX_INT, ""));
} catch (Exception e) {
//related videos sometimes have no view count
return 0;
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
Element img = li.select("img").first();
String thumbnailUrl = img.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we caught such an item.
if (thumbnailUrl.contains(".gif")) {
thumbnailUrl = img.attr("data-thumb");
}
if (thumbnailUrl.startsWith("//")) {
thumbnailUrl = HTTPS + thumbnailUrl;
}
return thumbnailUrl;
}
};
return errorReason != null ? errorReason.toString() : null;
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
private JSONObject playerArgs;
private boolean isAgeRestricted;
private Map<String, String> videoInfoPage;
private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
private static final String HTTPS = "https:";
private static final String CONTENT = "content";
/**
* Sometimes if the html page of youtube is already downloaded, youtube web page will internally
* download the /get_video_info page. Since a certain date dashmpd url is only available over
* this /get_video_info page, so we always need to download this one to.
* <p>
* %%video_id%% will be replaced by the actual video id
* $$el_type$$ will be replaced by the actual el_type (se the declarations below)
*/
private static final String GET_VIDEO_INFO_URL =
"https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en";
// eltype is necessary for the url above
private static final String EL_INFO = "el=info";
// static values
private static final String DECRYPTION_FUNC_NAME = "decrypt";
// cached values
private static volatile String decryptionCode = "";
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
Downloader downloader = NewPipe.getDownloader();
String pageContent = downloader.download(getUrl());
doc = Jsoup.parse(pageContent, getUrl());
JSONObject ytPlayerConfig;
String playerUrl;
String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%", getId()).replace("$$el_type$$", "&" + EL_INFO);
String videoInfoPageString = downloader.download(videoInfoUrl);
videoInfoPage = Parser.compatParseMap(videoInfoPageString);
// Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
playerUrl = getPlayerUrlFromRestrictedVideo(getUrl());
isAgeRestricted = true;
} else {
ytPlayerConfig = getPlayerConfig(pageContent);
playerArgs = getPlayerArgs(ytPlayerConfig);
playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false;
}
if (decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
}
}
private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
try {
String ytPlayerConfigRaw =
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
return new JSONObject(ytPlayerConfigRaw);
} catch (Parser.RegexException e) {
String errorReason = getErrorMessage();
switch (errorReason) {
case "GEMA":
throw new GemaException(errorReason);
case "":
throw new ContentNotAvailableException("Content not available: player config empty", e);
default:
throw new ContentNotAvailableException("Content not available", e);
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
}
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
JSONObject playerArgs;
//attempt to load the youtube js player JSON arguments
boolean isLiveStream = false; //used to determine if this is a livestream or not
try {
playerArgs = playerConfig.getJSONObject("args");
// check if we have a live stream. We need to filter it, since its not yet supported.
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
isLiveStream = true;
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
if (isLiveStream) {
throw new LiveStreamException("This is a Life stream. Can't use those right now.");
}
return playerArgs;
}
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
try {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
String playerUrl;
JSONObject ytAssets = playerConfig.getJSONObject("assets");
playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (JSONException e) {
throw new ParsingException(
"Could not load decryption code for the Youtube service.", e);
}
}
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
try {
Downloader downloader = NewPipe.getDownloader();
String playerUrl = "";
String videoId = getUrlIdHandler().getId(pageUrl);
String embedUrl = "https://www.youtube.com/embed/" + videoId;
String embedPageContent = downloader.download(embedUrl);
//todo: find out if this can be reapaced by Parser.matchGroup1()
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
while (patternMatcher.find()) {
playerUrl = patternMatcher.group(1);
}
playerUrl = playerUrl.replace("\\", "").replace("\"", "");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge requested");
}
}
private String loadDecryptionCode(String playerUrl) throws DecryptException {
String decryptionFuncName;
@@ -935,8 +780,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return decryptionCode;
}
private String decryptSignature(String encryptedSig, String decryptionCode)
throws DecryptException {
private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException {
Context context = Context.enter();
context.setOptimizationLevel(-1);
Object result = null;
@@ -953,28 +797,84 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return result == null ? "" : result.toString();
}
/**
* {@inheritDoc}
* Provides information about links to other videos on the video page, such as related videos.
* This is encapsulated in a StreamInfoItem object,
* which is a subset of the fields in a full StreamInfo.
*/
public String getErrorMessage() {
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
StringBuilder errorReason;
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
return new StreamInfoItemExtractor() {
@Override
public StreamType getStreamType() throws ParsingException {
return StreamType.VIDEO_STREAM;
}
if (errorMessage == null || errorMessage.isEmpty()) {
errorReason = null;
} else if(errorMessage.contains("GEMA")) {
// Gema sometimes blocks youtube music content in germany:
// https://www.gema.de/en/
// Detailed description:
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
errorReason = new StringBuilder("GEMA");
} else {
errorReason = new StringBuilder(errorMessage);
errorReason.append(" ");
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
}
@Override
public boolean isAd() throws ParsingException {
return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
}
return errorReason != null ? errorReason.toString() : null;
@Override
public String getWebPageUrl() throws ParsingException {
return li.select("a.content-link").first().attr("abs:href");
}
@Override
public String getTitle() throws ParsingException {
//todo: check NullPointerException causing
return li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
}
@Override
public int getDuration() throws ParsingException {
return YoutubeParsingHelper.parseDurationString(
li.select("span.video-time").first().text());
}
@Override
public String getUploader() throws ParsingException {
return li.select("span.g-hovercard").first().text();
}
@Override
public String getUploadDate() throws ParsingException {
return null;
}
@Override
public long getViewCount() throws ParsingException {
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
return Long.parseLong(Utils.removeNonDigitCharacters(
li.select("span.view-count").first().text()));
} catch (Exception e) {
//related videos sometimes have no view count
return 0;
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
Element img = li.select("img").first();
String thumbnailUrl = img.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we caught such an item.
if (thumbnailUrl.contains(".gif")) {
thumbnailUrl = img.attr("data-thumb");
}
if (thumbnailUrl.startsWith("//")) {
thumbnailUrl = HTTPS + thumbnailUrl;
}
return thumbnailUrl;
}
};
}
}