PaperTowel/NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added comments extractor

Browse Source
This commit is contained in:
Ritvik Saraf
2018-08-20 04:22:19 +05:30
parent 1e7bcfbd66
commit 53d3997904
17 changed files with 1007 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

236
extractor/src/test/java/org/schabi/newpipe/Downloader.java
View File

@@ -1,16 +1,18 @@
package org.schabi.newpipe;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import javax.net.ssl.HttpsURLConnection;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.net.ssl.HttpsURLConnection;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
/*
* Created by Christian Schabesberger on 28.01.16.
@@ -34,124 +36,144 @@ import java.util.Map;
public class Downloader implements org.schabi.newpipe.extractor.Downloader {
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private static String mCookies = "";
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private static String mCookies = "";
private static Downloader instance = null;
private static Downloader instance = null;
private Downloader() {
}
private Downloader() {
}
public static Downloader getInstance() {
if (instance == null) {
synchronized (Downloader.class) {
if (instance == null) {
instance = new Downloader();
}
}
}
return instance;
}
public static Downloader getInstance() {
if (instance == null) {
synchronized (Downloader.class) {
if (instance == null) {
instance = new Downloader();
}
}
}
return instance;
}
public static synchronized void setCookies(String cookies) {
Downloader.mCookies = cookies;
}
public static synchronized void setCookies(String cookies) {
Downloader.mCookies = cookies;
}
public static synchronized String getCookies() {
return Downloader.mCookies;
}
public static synchronized String getCookies() {
return Downloader.mCookies;
}
/**
* Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file
*/
public String download(String siteUrl, String language) throws IOException, ReCaptchaException {
Map<String, String> requestProperties = new HashMap<>();
requestProperties.put("Accept-Language", language);
return download(siteUrl, requestProperties);
}
/**
* Download the text file at the supplied URL as in download(String), but set
* the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the
* preferred language
* @return the contents of the specified text file
*/
public String download(String siteUrl, String language) throws IOException, ReCaptchaException {
Map<String, String> requestProperties = new HashMap<>();
requestProperties.put("Accept-Language", language);
return download(siteUrl, requestProperties);
}
/**
* Download the text file at the supplied URL as in download(String), but set
* the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException
*/
public String download(String siteUrl, Map<String, String> customProperties)
throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
for (Map.Entry<String, String> pair : customProperties.entrySet()) {
con.setRequestProperty(pair.getKey(), pair.getValue());
}
return dl(con);
}
/**
* Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException
*/
public String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
for (Map.Entry<String, String> pair: customProperties.entrySet()) {
con.setRequestProperty(pair.getKey(), pair.getValue());
}
return dl(con);
}
/**
* Common functionality between download(String url) and download(String url,
* String language)
*/
private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
StringBuilder response = new StringBuilder();
BufferedReader in = null;
/**
* Common functionality between download(String url) and download(String url, String language)
*/
private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
StringBuilder response = new StringBuilder();
BufferedReader in = null;
try {
con.setConnectTimeout(30 * 1000);// 30s
con.setReadTimeout(30 * 1000);// 30s
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);
try {
con.setConnectTimeout(30 * 1000);// 30s
con.setReadTimeout(30 * 1000);// 30s
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);
if (getCookies().length() > 0) {
con.addRequestProperty("Cookie", getCookies());
}
if (getCookies().length() > 0) {
con.setRequestProperty("Cookie", getCookies());
}
in = new BufferedReader(new InputStreamReader(con.getInputStream()));
String inputLine;
in = new BufferedReader(
new InputStreamReader(con.getInputStream()));
String inputLine;
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
} catch (UnknownHostException uhe) {// thrown when there's no internet connection
throw new IOException("unknown host or no network", uhe);
// Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
} catch (Exception e) {
/*
* HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge
* request See : https://github.com/rg3/youtube-dl/issues/5138
*/
if (con.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested");
}
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
} catch (UnknownHostException uhe) {//thrown when there's no internet connection
throw new IOException("unknown host or no network", uhe);
//Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
} catch (Exception e) {
/*
* HTTP 429 == Too Many Request
* Receive from Youtube.com = ReCaptcha challenge request
* See : https://github.com/rg3/youtube-dl/issues/5138
*/
if (con.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested");
}
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
} finally {
if (in != null) {
in.close();
}
}
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
} finally {
if (in != null) {
in.close();
}
}
return response.toString();
}
return response.toString();
}
/**
* Download (via HTTP) the text file located at the supplied URL, and return its
* contents. Primarily intended for downloading web pages.
*
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file
*/
public String download(String siteUrl) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
// HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
return dl(con);
}
/**
* Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
*
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file
*/
public String download(String siteUrl) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
return dl(con);
}
@Override
public DownloadResponse downloadWithHeaders(String siteUrl, Map<String, List<String>> requestHeaders)
throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value));
}
String responseBody = dl(con);
return new DownloadResponse(responseBody, con.getHeaderFields());
}
@Override
public DownloadResponse downloadWithHeaders(String siteUrl) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
String responseBody = dl(con);
return new DownloadResponse(responseBody, con.getHeaderFields());
}
}

46
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java Normal file
View File

@@ -0,0 +1,46 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import java.io.IOException;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
public class YoutubeCommentsExtractorTest {
private static YoutubeCommentsExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor("https://www.youtube.com/watch?v=rrgFN3AxGfs");
extractor.fetchPage();
}
@Test
public void testGetComments() throws IOException, ExtractionException {
boolean result = false;
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
result = findInComments(comments, "i should really be in the top comment.lol");
while (comments.hasNextPage()) {
comments = extractor.getPage(comments.getNextPageUrl());
result = findInComments(comments, "i should really be in the top comment.lol");
}
assertTrue(result);
}
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
return comments.getItems().stream().filter(c -> c.getCommentText().contains(comment)).findAny().isPresent();
}
}