From fbb6a039b6056417166835a8ae9dffc748b8d6ba Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Tue, 11 Apr 2017 21:55:53 -0300 Subject: [PATCH] Add resolution support up to 4k --- DashMpdParser.java | 2 +- .../youtube/YoutubePlayListUrlIdHandler.java | 1 - services/youtube/YoutubeStreamExtractor.java | 151 ++++++++++++++---- stream_info/AudioStream.java | 16 +- stream_info/VideoStream.java | 30 ++-- 5 files changed, 148 insertions(+), 52 deletions(-) diff --git a/DashMpdParser.java b/DashMpdParser.java index 527750a8..66fd9424 100644 --- a/DashMpdParser.java +++ b/DashMpdParser.java @@ -81,7 +81,7 @@ public class DashMpdParser { } else if(memeType.equals(MediaFormat.M4A.mimeType)) { format = MediaFormat.M4A.id; } - audioStreams.add(new AudioStream(url, format, bandwidth, samplingRate)); + audioStreams.add(new AudioStream(url, format, 0, bandwidth, samplingRate)); } } } diff --git a/services/youtube/YoutubePlayListUrlIdHandler.java b/services/youtube/YoutubePlayListUrlIdHandler.java index dbe1be65..db909cd8 100644 --- a/services/youtube/YoutubePlayListUrlIdHandler.java +++ b/services/youtube/YoutubePlayListUrlIdHandler.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube; -import android.net.UrlQuerySanitizer; import org.schabi.newpipe.extractor.Parser; import org.schabi.newpipe.extractor.UrlIdHandler; diff --git a/services/youtube/YoutubeStreamExtractor.java b/services/youtube/YoutubeStreamExtractor.java index 2df36164..543c1fe0 100644 --- a/services/youtube/YoutubeStreamExtractor.java +++ b/services/youtube/YoutubeStreamExtractor.java @@ -9,15 +9,15 @@ import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; import org.schabi.newpipe.extractor.AbstractStreamInfo; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Parser; +import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.stream_info.AudioStream; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.Parser; -import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.stream_info.StreamExtractor; import org.schabi.newpipe.extractor.stream_info.StreamInfo; import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector; @@ -105,49 +105,84 @@ public class YoutubeStreamExtractor extends StreamExtractor { this.resolutionString = res; this.fps = fps; } + public ItagItem(int id, ItagType type, MediaFormat format, int samplingRate, int bandWidth) { + this(id, type, format, 0, samplingRate, bandWidth); + } + + public ItagItem(int id, ItagType type, MediaFormat format, int avgBitrate, int samplingRate, int bandWidth) { this.id = id; this.itagType = type; this.mediaFormatId = format.id; + this.avgBitrate = avgBitrate; this.samplingRate = samplingRate; this.bandWidth = bandWidth; } + public int id; public ItagType itagType; public int mediaFormatId; public String resolutionString; public int fps = -1; + public int avgBitrate = -1; public int samplingRate = -1; public int bandWidth = -1; } private static final ItagItem[] itagList = { - // video streams - // id, ItagType, MediaFormat, Resolution, fps - new ItagItem(17, ItagType.VIDEO, MediaFormat.v3GPP, "144p", 12), - new ItagItem(18, ItagType.VIDEO, MediaFormat.MPEG_4, "360p", 24), - new ItagItem(22, ItagType.VIDEO, MediaFormat.MPEG_4, "720p", 24), - new ItagItem(36, ItagType.VIDEO, MediaFormat.v3GPP, "240p", 24), - new ItagItem(37, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p", 24), - new ItagItem(38, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p", 24), - new ItagItem(43, ItagType.VIDEO, MediaFormat.WEBM, "360p", 24), - new ItagItem(44, ItagType.VIDEO, MediaFormat.WEBM, "480p", 24), - new ItagItem(45, ItagType.VIDEO, MediaFormat.WEBM, "720p", 24), - new ItagItem(46, ItagType.VIDEO, MediaFormat.WEBM, "1080p", 24), - // audio streams - // id, ItagType, MediaFormat, samplingR, bandwidth - new ItagItem(249, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0), // bandwith/samplingR 0 because not known - new ItagItem(250, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0), - new ItagItem(171, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0), - new ItagItem(140, ItagType.AUDIO, MediaFormat.M4A, 0, 0), - new ItagItem(251, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0), - // video only streams - new ItagItem(160, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "144p", 24), - new ItagItem(133, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "240p", 24), - new ItagItem(134, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "360p", 24), - new ItagItem(135, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "480p", 24), - new ItagItem(136, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p", 24), - new ItagItem(137, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p", 24), + ////////////////////////////////////////////////////////////////////////// + // VIDEO ID ItagType Format Resolution FPS /// + //////////////////////////////////////////////////////////////////////// + new ItagItem(17, ItagType.VIDEO, MediaFormat.v3GPP, "144p" , 12), + new ItagItem(18, ItagType.VIDEO, MediaFormat.MPEG_4, "360p" , 24), + new ItagItem(22, ItagType.VIDEO, MediaFormat.MPEG_4, "720p" , 24), + new ItagItem(36, ItagType.VIDEO, MediaFormat.v3GPP, "240p" , 24), + new ItagItem(37, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p" , 24), + new ItagItem(38, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p" , 24), + new ItagItem(43, ItagType.VIDEO, MediaFormat.WEBM, "360p" , 24), + new ItagItem(44, ItagType.VIDEO, MediaFormat.WEBM, "480p" , 24), + new ItagItem(45, ItagType.VIDEO, MediaFormat.WEBM, "720p" , 24), + new ItagItem(46, ItagType.VIDEO, MediaFormat.WEBM, "1080p" , 24), + + ////////////////////////////////////////////////////////////////////////////////////////// + // AUDIO ID ItagType Format Bitrate SamplingR Bandwidth /// + //////////////////////////////////////////////////////////////////////////////////////// + new ItagItem(249, ItagType.AUDIO, MediaFormat.WEBMA, 50, 0, 0), + new ItagItem(250, ItagType.AUDIO, MediaFormat.WEBMA, 70, 0, 0), + new ItagItem(251, ItagType.AUDIO, MediaFormat.WEBMA, 160, 0, 0), + new ItagItem(171, ItagType.AUDIO, MediaFormat.WEBMA, 128, 0, 0), + new ItagItem(172, ItagType.AUDIO, MediaFormat.WEBMA, 256, 0, 0), + new ItagItem(140, ItagType.AUDIO, MediaFormat.M4A, 128, 0, 0), + new ItagItem(141, ItagType.AUDIO, MediaFormat.M4A, 256, 0, 0), + + /// VIDEO ONLY /////////////////////////////////////////////////////////////////// + // ID ItagType Format Resolution FPS /// + //////////////////////////////////////////////////////////////////////////////// + // Don't add VideoOnly streams that have normal variants +// new ItagItem(160, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "144p" , 24), +// new ItagItem(133, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "240p" , 24), +// new ItagItem(134, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "360p" , 24), + new ItagItem(135, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "480p" , 30), +// new ItagItem(136, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p" , 30), + new ItagItem(298, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p60" , 60), + new ItagItem(137, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p" , 30), + new ItagItem(299, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p60" , 60), + new ItagItem(266, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "2160p" , 30), + +// new ItagItem(243, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "360p" , 30), + new ItagItem(244, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30), + new ItagItem(245, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30), + new ItagItem(246, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30), + new ItagItem(247, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "720p" , 30), + new ItagItem(248, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1080p" , 30), + new ItagItem(271, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1440p" , 30), + // #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) + new ItagItem(272, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p" , 30), + new ItagItem(302, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "720p60" , 60), + new ItagItem(303, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1080p60" , 60), + new ItagItem(308, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1440p60" , 60), + new ItagItem(313, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p" , 30), + new ItagItem(315, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p60" , 60) }; /**These lists only contain itag formats that are supported by the common Android Video player. @@ -483,6 +518,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { audioStreams.add(new AudioStream(streamUrl, itagItem.mediaFormatId, + itagItem.avgBitrate, itagItem.bandWidth, itagItem.samplingRate)); } @@ -549,7 +585,58 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public List getVideoOnlyStreams() throws ParsingException { - return null; + Vector videoOnlyStreams = new Vector<>(); + + try { + String encodedUrlMap; + // playerArgs could be null if the video is age restricted + if (playerArgs == null) { + if (videoInfoPage.containsKey("adaptive_fmts")) { + encodedUrlMap = videoInfoPage.get("adaptive_fmts"); + } else { + return null; + } + } else { + if (playerArgs.has("adaptive_fmts")) { + encodedUrlMap = playerArgs.getString("adaptive_fmts"); + } else { + return null; + } + } + for (String url_data_str : encodedUrlMap.split(",")) { + // This loop iterates through multiple streams, therefor tags + // is related to one and the same stream at a time. + Map tags = Parser.compatParseMap( + org.jsoup.parser.Parser.unescapeEntities(url_data_str, true)); + + int itag = Integer.parseInt(tags.get("itag")); + + if (itagIsSupported(itag)) { + ItagItem itagItem = getItagItem(itag); + if (itagItem.itagType == ItagType.VIDEO_ONLY) { + String streamUrl = tags.get("url"); + // if video has a signature: decrypt it and add it to the url + if (tags.get("s") != null) { + streamUrl = streamUrl + "&signature=" + + decryptSignature(tags.get("s"), decryptionCode); + } + + videoOnlyStreams.add(new VideoStream( + true, //isVideoOnly + streamUrl, + itagItem.mediaFormatId, + itagItem.resolutionString)); + } + } + } + } catch (Exception e) { + throw new ParsingException("Failed to get video only streams", e); + } + + if (videoOnlyStreams.isEmpty()) { + throw new ParsingException("Failed to get any video only stream"); + } + return videoOnlyStreams; } /**Attempts to parse (and return) the offset to start playing the video from. diff --git a/stream_info/AudioStream.java b/stream_info/AudioStream.java index b476cad6..1495644a 100644 --- a/stream_info/AudioStream.java +++ b/stream_info/AudioStream.java @@ -27,22 +27,26 @@ public class AudioStream implements Serializable{ public int format = -1; public int bandwidth = -1; public int sampling_rate = -1; + public int avgBitrate = -1; - public AudioStream(String url, int format, int bandwidth, int samplingRate) { - this.url = url; this.format = format; - this.bandwidth = bandwidth; this.sampling_rate = samplingRate; + public AudioStream(String url, int format, int avgBitrate, int bandwidth, int samplingRate) { + this.url = url; + this.format = format; + this.avgBitrate = avgBitrate; + this.bandwidth = bandwidth; + this.sampling_rate = samplingRate; } // reveals whether two streams are the same, but have different urls public boolean equalStats(AudioStream cmp) { return format == cmp.format && bandwidth == cmp.bandwidth - && sampling_rate == cmp.sampling_rate; + && sampling_rate == cmp.sampling_rate + && avgBitrate == cmp.avgBitrate; } // reveals whether two streams are equal public boolean equals(AudioStream cmp) { - return cmp != null && equalStats(cmp) - && url == cmp.url; + return cmp != null && equalStats(cmp) && url.equals(cmp.url); } } diff --git a/stream_info/VideoStream.java b/stream_info/VideoStream.java index 97877357..cff4c228 100644 --- a/stream_info/VideoStream.java +++ b/stream_info/VideoStream.java @@ -4,43 +4,49 @@ import java.io.Serializable; /** * Created by Christian Schabesberger on 04.03.16. - * + *

* Copyright (C) Christian Schabesberger 2016 * VideoStream.java is part of NewPipe. - * + *

* NewPipe is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + *

* NewPipe is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + *

* You should have received a copy of the GNU General Public License * along with NewPipe. If not, see . */ -public class VideoStream implements Serializable{ +public class VideoStream implements Serializable { //url of the stream public String url = ""; public int format = -1; public String resolution = ""; + public boolean isVideoOnly = false; public VideoStream(String url, int format, String res) { - this.url = url; this.format = format; resolution = res; + this(false, url, format, res); } - // reveals wether two streams are the same, but have diferent urls + public VideoStream(boolean isVideoOnly, String url, int format, String res) { + this.url = url; + this.format = format; + this.resolution = res; + this.isVideoOnly = isVideoOnly; + } + + // reveals whether two streams are the same, but have diferent urls public boolean equalStats(VideoStream cmp) { - return format == cmp.format - && resolution == cmp.resolution; + return format == cmp.format && resolution.equals(cmp.resolution); } - // revelas wether two streams are equal + // reveals whether two streams are equal public boolean equals(VideoStream cmp) { - return cmp != null && equalStats(cmp) - && url == cmp.url; + return cmp != null && equalStats(cmp) && url.equals(cmp.url); } }