PaperTowel/NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix dash parser and more refactor

Browse Source 
- Add new itags
This commit is contained in:
Mauricio Colli
2017-07-11 00:08:03 -03:00
parent bda65e83d6
commit b1989c0a83
41 changed files with 802 additions and 751 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
stream/AbstractStreamInfo.java
View File

@@ -1,41 +0,0 @@
package org.schabi.newpipe.extractor.stream;
/*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* AbstractStreamInfo.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.Info;
/**
* Common properties between StreamInfo and StreamInfoItem.
*/
public abstract class AbstractStreamInfo extends Info {
public enum StreamType {
NONE, // placeholder to check if stream type was checked or not
VIDEO_STREAM,
AUDIO_STREAM,
LIVE_STREAM,
AUDIO_LIVE_STREAM,
FILE
}
public StreamType stream_type;
public String uploader = "";
public String thumbnail_url = "";
public String upload_date = "";
public long view_count = -1;
}

34
stream/AudioStream.java
View File

@@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
/*
* Created by Christian Schabesberger on 04.03.16.
*
@@ -22,31 +20,17 @@ import java.io.Serializable;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class AudioStream implements Serializable {
public String url = "";
public int format = -1;
public int bandwidth = -1;
public int sampling_rate = -1;
public int avgBitrate = -1;
public class AudioStream extends Stream {
public int average_bitrate = -1;
public AudioStream(String url, int format, int avgBitrate, int bandwidth, int samplingRate) {
this.url = url;
this.format = format;
this.avgBitrate = avgBitrate;
this.bandwidth = bandwidth;
this.sampling_rate = samplingRate;
public AudioStream(String url, int format, int averageBitrate) {
super(url, format);
this.average_bitrate = averageBitrate;
}
// reveals whether two streams are the same, but have different urls
public boolean equalStats(AudioStream cmp) {
return format == cmp.format
&& bandwidth == cmp.bandwidth
&& sampling_rate == cmp.sampling_rate
&& avgBitrate == cmp.avgBitrate;
}
// reveals whether two streams are equal
public boolean equals(AudioStream cmp) {
return cmp != null && equalStats(cmp) && url.equals(cmp.url);
@Override
public boolean equalStats(Stream cmp) {
return super.equalStats(cmp) && cmp instanceof AudioStream &&
average_bitrate == ((AudioStream) cmp).average_bitrate;
}
}

39
stream/Stream.java Normal file
View File

@@ -0,0 +1,39 @@
package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
import java.util.List;
public abstract class Stream implements Serializable {
public String url;
public int format = -1;
public Stream(String url, int format) {
this.url = url;
this.format = format;
}
/**
* Reveals whether two streams are the same, but have different urls
*/
public boolean equalStats(Stream cmp) {
return cmp != null && format == cmp.format;
}
/**
* Reveals whether two Streams are equal
*/
public boolean equals(Stream cmp) {
return equalStats(cmp) && url.equals(cmp.url);
}
/**
* Check if the list already contains one stream with equals stats
*/
public static boolean containSimilarStream(Stream stream, List<? extends Stream> streamList) {
if (stream == null || streamList == null) return false;
for (Stream cmpStream : streamList) {
if (stream.equalStats(cmpStream)) return true;
}
return false;
}
}

14
stream/StreamExtractor.java
View File

@@ -31,20 +31,11 @@ import java.util.List;
*/
public abstract class StreamExtractor extends Extractor {
public static class ContentNotAvailableException extends ParsingException {
public ContentNotAvailableException(String message) {
super(message);
}
public ContentNotAvailableException(String message, Throwable cause) {
super(message, cause);
}
}
public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) {
super(urlIdHandler, serviceId, url);
}
public abstract String getId() throws ParsingException;
public abstract int getTimeStamp() throws ParsingException;
public abstract String getTitle() throws ParsingException;
public abstract String getDescription() throws ParsingException;
@@ -65,8 +56,7 @@ public abstract class StreamExtractor extends Extractor {
public abstract int getDislikeCount() throws ParsingException;
public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException;
public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException;
public abstract String getPageUrl();
public abstract StreamInfo.StreamType getStreamType() throws ParsingException;
public abstract StreamType getStreamType() throws ParsingException;
/**
* Analyses the webpage's document and extracts any error message there might be.

146
stream/StreamInfo.java
View File

@@ -1,7 +1,8 @@
package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.DashMpdParser;
@@ -31,11 +32,11 @@ import java.util.Vector;
/**
* Info object for opened videos, ie the video ready to play.
*/
@SuppressWarnings("ALL")
public class StreamInfo extends AbstractStreamInfo {
@SuppressWarnings("WeakerAccess")
public class StreamInfo extends Info {
public static class StreamExctractException extends ExtractionException {
StreamExctractException(String message) {
public static class StreamExtractException extends ExtractionException {
StreamExtractException(String message) {
super(message);
}
}
@@ -43,43 +44,11 @@ public class StreamInfo extends AbstractStreamInfo {
public StreamInfo() {
}
/**
* Creates a new StreamInfo object from an existing AbstractVideoInfo.
* All the shared properties are copied to the new StreamInfo.
*/
@SuppressWarnings("WeakerAccess")
public StreamInfo(AbstractStreamInfo avi) {
this.id = avi.id;
this.url = avi.url;
this.name = avi.name;
this.uploader = avi.uploader;
this.thumbnail_url = avi.thumbnail_url;
this.upload_date = avi.upload_date;
this.upload_date = avi.upload_date;
this.view_count = avi.view_count;
//todo: better than this
if (avi instanceof StreamInfoItem) {
//shitty String to convert code
/*
String dur = ((StreamInfoItem)avi).duration;
int minutes = Integer.parseInt(dur.substring(0, dur.indexOf(":")));
int seconds = Integer.parseInt(dur.substring(dur.indexOf(":")+1, dur.length()));
*/
this.duration = ((StreamInfoItem) avi).duration;
}
}
public void addException(Exception e) {
errors.add(e);
}
/**
* Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses
*/
public static StreamInfo getVideoInfo(StreamExtractor extractor)
throws ExtractionException, StreamExtractor.ContentNotAvailableException {
public static StreamInfo getVideoInfo(StreamExtractor extractor) throws ExtractionException {
StreamInfo streamInfo = new StreamInfo();
try {
@@ -87,15 +56,15 @@ public class StreamInfo extends AbstractStreamInfo {
streamInfo = extractStreams(streamInfo, extractor);
streamInfo = extractOptionalData(streamInfo, extractor);
} catch (ExtractionException e) {
// Currently YouTube does not distinguish between age restricted videos and videos blocked
// by country. This means that during the initialisation of the extractor, the extractor
// will assume that a video is age restricted while in reality it it blocked by country.
//
// We will now detect whether the video is blocked by country or not.
// Currently YouTube does not distinguish between age restricted videos and videos blocked
// by country. This means that during the initialisation of the extractor, the extractor
// will assume that a video is age restricted while in reality it it blocked by country.
//
// We will now detect whether the video is blocked by country or not.
String errorMsg = extractor.getErrorMessage();
if (errorMsg != null) {
throw new StreamExtractor.ContentNotAvailableException(errorMsg);
throw new ContentNotAvailableException(errorMsg);
} else {
throw e;
}
@@ -104,18 +73,14 @@ public class StreamInfo extends AbstractStreamInfo {
return streamInfo;
}
private static StreamInfo extractImportantData(
StreamInfo streamInfo, StreamExtractor extractor)
throws ExtractionException {
private static StreamInfo extractImportantData(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
/* ---- important data, withoug the video can't be displayed goes here: ---- */
// if one of these is not available an exception is meant to be thrown directly into the frontend.
UrlIdHandler uiconv = extractor.getUrlIdHandler();
streamInfo.service_id = extractor.getServiceId();
streamInfo.url = extractor.getPageUrl();
streamInfo.url = extractor.getUrl();
streamInfo.stream_type = extractor.getStreamType();
streamInfo.id = uiconv.getId(extractor.getPageUrl());
streamInfo.id = extractor.getId();
streamInfo.name = extractor.getTitle();
streamInfo.age_limit = extractor.getAgeLimit();
@@ -130,9 +95,7 @@ public class StreamInfo extends AbstractStreamInfo {
return streamInfo;
}
private static StreamInfo extractStreams(
StreamInfo streamInfo, StreamExtractor extractor)
throws ExtractionException {
private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
/* ---- stream extraction goes here ---- */
// At least one type of stream has to be available,
// otherwise an exception will be thrown directly into the frontend.
@@ -149,34 +112,33 @@ public class StreamInfo extends AbstractStreamInfo {
} catch (Exception e) {
streamInfo.addException(new ExtractionException("Couldn't get audio streams", e));
}
// also try to get streams from the dashMpd
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
if (streamInfo.audio_streams == null) {
streamInfo.audio_streams = new Vector<>();
}
//todo: make this quick and dirty solution a real fallback
// same as the quick and dirty above
try {
streamInfo.audio_streams.addAll(
DashMpdParser.getAudioStreams(streamInfo.dashMpdUrl));
} catch (Exception e) {
streamInfo.addException(
new ExtractionException("Couldn't get audio streams from dash mpd", e));
}
}
/* Extract video stream url*/
try {
streamInfo.video_streams = extractor.getVideoStreams();
} catch (Exception e) {
streamInfo.addException(
new ExtractionException("Couldn't get video streams", e));
streamInfo.addException(new ExtractionException("Couldn't get video streams", e));
}
/* Extract video only stream url*/
try {
streamInfo.video_only_streams = extractor.getVideoOnlyStreams();
} catch (Exception e) {
streamInfo.addException(
new ExtractionException("Couldn't get video only streams", e));
streamInfo.addException(new ExtractionException("Couldn't get video only streams", e));
}
// Lists can be null if a exception was thrown during extraction
if (streamInfo.video_streams == null) streamInfo.video_streams = new Vector<>();
if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new Vector<>();
if (streamInfo.audio_streams == null) streamInfo.audio_streams = new Vector<>();
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
try {
// Will try to find in the dash manifest for any stream that the ItagItem has (by the id),
// it has video, video only and audio streams and will only add to the list if it don't
// find a similar stream in the respective lists (calling Stream#equalStats).
DashMpdParser.getStreams(streamInfo);
} catch (Exception e) {
streamInfo.addException(new ExtractionException("Couldn't get streams from dash mpd", e));
}
}
// either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream,
@@ -184,15 +146,14 @@ public class StreamInfo extends AbstractStreamInfo {
if ((streamInfo.video_streams == null || streamInfo.video_streams.isEmpty())
&& (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty())
&& (streamInfo.dashMpdUrl == null || streamInfo.dashMpdUrl.isEmpty())) {
throw new StreamExctractException(
throw new StreamExtractException(
"Could not get any stream. See error variable to get further details.");
}
return streamInfo;
}
private static StreamInfo extractOptionalData(
StreamInfo streamInfo, StreamExtractor extractor) {
private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtractor extractor) {
/* ---- optional data goes here: ---- */
// If one of these fails, the frontend needs to handle that they are not available.
// Exceptions are therefore not thrown into the frontend, but stored into the error List,
@@ -259,8 +220,7 @@ public class StreamInfo extends AbstractStreamInfo {
streamInfo.addException(e);
}
try {
StreamInfoItemCollector c = new StreamInfoItemCollector(
extractor.getUrlIdHandler(), extractor.getServiceId());
StreamInfoItemCollector c = new StreamInfoItemCollector(extractor.getServiceId());
StreamInfoItemExtractor nextVideo = extractor.getNextVideo();
c.commit(nextVideo);
if (c.getItemList().size() != 0) {
@@ -282,26 +242,36 @@ public class StreamInfo extends AbstractStreamInfo {
return streamInfo;
}
public String uploader_thumbnail_url = "";
public String channel_url = "";
public String description = "";
public void addException(Exception e) {
errors.add(e);
}
public List<VideoStream> video_streams = null;
public List<AudioStream> audio_streams = null;
public List<VideoStream> video_only_streams = null;
public StreamType stream_type;
public String uploader;
public String thumbnail_url;
public String upload_date;
public long view_count = -1;
public String uploader_thumbnail_url;
public String channel_url;
public String description;
public List<VideoStream> video_streams;
public List<AudioStream> audio_streams;
public List<VideoStream> video_only_streams;
// video streams provided by the dash mpd do not need to be provided as VideoStream.
// Later on this will also aplly to audio streams. Since dash mpd is standarized,
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
// providing the dash mpd fille will be possible in the future.
public String dashMpdUrl = "";
public String dashMpdUrl;
public int duration = -1;
public int age_limit = -1;
public int like_count = -1;
public int dislike_count = -1;
public String average_rating = "";
public StreamInfoItem next_video = null;
public List<InfoItem> related_streams = null;
public String average_rating;
public StreamInfoItem next_video;
public List<InfoItem> related_streams = new Vector<>();
//in seconds. some metadata is not passed using a StreamInfo object!
public int start_position = 0;
}

20
stream/StreamInfoItem.java
View File

@@ -25,18 +25,16 @@ import org.schabi.newpipe.extractor.InfoItem;
/**
* Info object for previews of unopened videos, eg search results, related videos
*/
public class StreamInfoItem extends AbstractStreamInfo implements InfoItem {
public int duration;
public class StreamInfoItem extends InfoItem {
public StreamType stream_type;
public InfoType infoType() {
return InfoType.STREAM;
}
public String uploader;
public String thumbnail_url;
public String upload_date;
public long view_count = -1;
public int duration = -1;
public String getTitle() {
return name;
}
public String getLink() {
return url;
public StreamInfoItem() {
super(InfoType.STREAM);
}
}

19
stream/StreamInfoItemCollector.java
View File

@@ -1,8 +1,6 @@
package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -28,15 +26,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class StreamInfoItemCollector extends InfoItemCollector {
private UrlIdHandler urlIdHandler;
public StreamInfoItemCollector(UrlIdHandler handler, int serviceId) {
public StreamInfoItemCollector(int serviceId) {
super(serviceId);
urlIdHandler = handler;
}
private UrlIdHandler getUrlIdHandler() {
return urlIdHandler;
}
public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws Exception {
@@ -48,13 +39,7 @@ public class StreamInfoItemCollector extends InfoItemCollector {
// important information
resultItem.service_id = getServiceId();
resultItem.url = extractor.getWebPageUrl();
if (getUrlIdHandler() == null) {
throw new ParsingException("Error: UrlIdHandler not set");
} else if (!resultItem.url.isEmpty()) {
resultItem.id = NewPipe.getService(getServiceId())
.getStreamUrlIdHandlerInstance()
.getId(resultItem.url);
}
resultItem.name = extractor.getTitle();
resultItem.stream_type = extractor.getStreamType();

2
stream/StreamInfoItemExtractor.java
View File

@@ -23,7 +23,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
*/
public interface StreamInfoItemExtractor {
AbstractStreamInfo.StreamType getStreamType() throws ParsingException;
StreamType getStreamType() throws ParsingException;
String getWebPageUrl() throws ParsingException;
String getTitle() throws ParsingException;
int getDuration() throws ParsingException;

10
stream/StreamType.java Normal file
View File

@@ -0,0 +1,10 @@
package org.schabi.newpipe.extractor.stream;
public enum StreamType {
NONE, // placeholder to check if stream type was checked or not
VIDEO_STREAM,
AUDIO_STREAM,
LIVE_STREAM,
AUDIO_LIVE_STREAM,
FILE
}

31
stream/VideoStream.java
View File

@@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
/*
* Created by Christian Schabesberger on 04.03.16.
*
@@ -22,31 +20,24 @@ import java.io.Serializable;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class VideoStream implements Serializable {
//url of the stream
public String url = "";
public int format = -1;
public String resolution = "";
public boolean isVideoOnly = false;
public class VideoStream extends Stream {
public String resolution;
public boolean isVideoOnly;
public VideoStream(String url, int format, String res) {
this(false, url, format, res);
this(url, format, res, false);
}
public VideoStream(boolean isVideoOnly, String url, int format, String res) {
this.url = url;
this.format = format;
public VideoStream(String url, int format, String res, boolean isVideoOnly) {
super(url, format);
this.resolution = res;
this.isVideoOnly = isVideoOnly;
}
// reveals whether two streams are the same, but have different urls
public boolean equalStats(VideoStream cmp) {
return format == cmp.format && resolution.equals(cmp.resolution);
}
// reveals whether two streams are equal
public boolean equals(VideoStream cmp) {
return cmp != null && equalStats(cmp) && url.equals(cmp.url);
@Override
public boolean equalStats(Stream cmp) {
return super.equalStats(cmp) && cmp instanceof VideoStream &&
resolution.equals(((VideoStream) cmp).resolution) &&
isVideoOnly == ((VideoStream) cmp).isVideoOnly;
}
}