added http post method in downloader, formatting
This commit is contained in:
Ritvik Saraf
@@ -36,144 +36,184 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
|
||||
public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
||||
|
||||
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
|
||||
private static String mCookies = "";
|
||||
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
|
||||
private static String mCookies = "";
|
||||
|
||||
private static Downloader instance = null;
|
||||
private static Downloader instance = null;
|
||||
|
||||
private Downloader() {
|
||||
}
|
||||
private Downloader() {
|
||||
}
|
||||
|
||||
public static Downloader getInstance() {
|
||||
if (instance == null) {
|
||||
synchronized (Downloader.class) {
|
||||
if (instance == null) {
|
||||
instance = new Downloader();
|
||||
}
|
||||
}
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
public static Downloader getInstance() {
|
||||
if (instance == null) {
|
||||
synchronized (Downloader.class) {
|
||||
if (instance == null) {
|
||||
instance = new Downloader();
|
||||
}
|
||||
}
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
public static synchronized void setCookies(String cookies) {
|
||||
Downloader.mCookies = cookies;
|
||||
}
|
||||
public static synchronized void setCookies(String cookies) {
|
||||
Downloader.mCookies = cookies;
|
||||
}
|
||||
|
||||
public static synchronized String getCookies() {
|
||||
return Downloader.mCookies;
|
||||
}
|
||||
public static synchronized String getCookies() {
|
||||
return Downloader.mCookies;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download the text file at the supplied URL as in download(String), but set
|
||||
* the HTTP header field "Accept-Language" to the supplied string.
|
||||
*
|
||||
* @param siteUrl the URL of the text file to return the contents of
|
||||
* @param language the language (usually a 2-character code) to set as the
|
||||
* preferred language
|
||||
* @return the contents of the specified text file
|
||||
*/
|
||||
public String download(String siteUrl, String language) throws IOException, ReCaptchaException {
|
||||
Map<String, String> requestProperties = new HashMap<>();
|
||||
requestProperties.put("Accept-Language", language);
|
||||
return download(siteUrl, requestProperties);
|
||||
}
|
||||
/**
|
||||
* Download the text file at the supplied URL as in download(String), but set
|
||||
* the HTTP header field "Accept-Language" to the supplied string.
|
||||
*
|
||||
* @param siteUrl the URL of the text file to return the contents of
|
||||
* @param language the language (usually a 2-character code) to set as the
|
||||
* preferred language
|
||||
* @return the contents of the specified text file
|
||||
*/
|
||||
public String download(String siteUrl, String language) throws IOException, ReCaptchaException {
|
||||
Map<String, String> requestProperties = new HashMap<>();
|
||||
requestProperties.put("Accept-Language", language);
|
||||
return download(siteUrl, requestProperties);
|
||||
}
|
||||
|
||||
/**
|
||||
* Download the text file at the supplied URL as in download(String), but set
|
||||
* the HTTP header field "Accept-Language" to the supplied string.
|
||||
*
|
||||
* @param siteUrl the URL of the text file to return the contents of
|
||||
* @param customProperties set request header properties
|
||||
* @return the contents of the specified text file
|
||||
* @throws IOException
|
||||
*/
|
||||
public String download(String siteUrl, Map<String, String> customProperties)
|
||||
throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
for (Map.Entry<String, String> pair : customProperties.entrySet()) {
|
||||
con.setRequestProperty(pair.getKey(), pair.getValue());
|
||||
}
|
||||
return dl(con);
|
||||
}
|
||||
/**
|
||||
* Download the text file at the supplied URL as in download(String), but set
|
||||
* the HTTP header field "Accept-Language" to the supplied string.
|
||||
*
|
||||
* @param siteUrl the URL of the text file to return the contents of
|
||||
* @param customProperties set request header properties
|
||||
* @return the contents of the specified text file
|
||||
* @throws IOException
|
||||
*/
|
||||
public String download(String siteUrl, Map<String, String> customProperties)
|
||||
throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
for (Map.Entry<String, String> pair : customProperties.entrySet()) {
|
||||
con.setRequestProperty(pair.getKey(), pair.getValue());
|
||||
}
|
||||
return dl(con);
|
||||
}
|
||||
|
||||
/**
|
||||
* Common functionality between download(String url) and download(String url,
|
||||
* String language)
|
||||
*/
|
||||
private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
|
||||
StringBuilder response = new StringBuilder();
|
||||
BufferedReader in = null;
|
||||
/**
|
||||
* Common functionality between download(String url) and download(String url,
|
||||
* String language)
|
||||
*/
|
||||
private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
|
||||
StringBuilder response = new StringBuilder();
|
||||
BufferedReader in = null;
|
||||
|
||||
try {
|
||||
con.setConnectTimeout(30 * 1000);// 30s
|
||||
con.setReadTimeout(30 * 1000);// 30s
|
||||
con.setRequestMethod("GET");
|
||||
con.setRequestProperty("User-Agent", USER_AGENT);
|
||||
try {
|
||||
|
||||
if (getCookies().length() > 0) {
|
||||
con.addRequestProperty("Cookie", getCookies());
|
||||
}
|
||||
con.setRequestMethod("GET");
|
||||
setDefaults(con);
|
||||
|
||||
in = new BufferedReader(new InputStreamReader(con.getInputStream()));
|
||||
String inputLine;
|
||||
in = new BufferedReader(new InputStreamReader(con.getInputStream()));
|
||||
String inputLine;
|
||||
|
||||
while ((inputLine = in.readLine()) != null) {
|
||||
response.append(inputLine);
|
||||
}
|
||||
} catch (UnknownHostException uhe) {// thrown when there's no internet connection
|
||||
throw new IOException("unknown host or no network", uhe);
|
||||
// Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
|
||||
} catch (Exception e) {
|
||||
/*
|
||||
* HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge
|
||||
* request See : https://github.com/rg3/youtube-dl/issues/5138
|
||||
*/
|
||||
if (con.getResponseCode() == 429) {
|
||||
throw new ReCaptchaException("reCaptcha Challenge requested");
|
||||
}
|
||||
while ((inputLine = in.readLine()) != null) {
|
||||
response.append(inputLine);
|
||||
}
|
||||
} catch (UnknownHostException uhe) {// thrown when there's no internet
|
||||
// connection
|
||||
throw new IOException("unknown host or no network", uhe);
|
||||
// Toast.makeText(getActivity(), uhe.getMessage(),
|
||||
// Toast.LENGTH_LONG).show();
|
||||
} catch (Exception e) {
|
||||
/*
|
||||
* HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge
|
||||
* request See : https://github.com/rg3/youtube-dl/issues/5138
|
||||
*/
|
||||
if (con.getResponseCode() == 429) {
|
||||
throw new ReCaptchaException("reCaptcha Challenge requested");
|
||||
}
|
||||
|
||||
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
|
||||
} finally {
|
||||
if (in != null) {
|
||||
in.close();
|
||||
}
|
||||
}
|
||||
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
|
||||
} finally {
|
||||
if (in != null) {
|
||||
in.close();
|
||||
}
|
||||
}
|
||||
|
||||
return response.toString();
|
||||
}
|
||||
return response.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Download (via HTTP) the text file located at the supplied URL, and return its
|
||||
* contents. Primarily intended for downloading web pages.
|
||||
*
|
||||
* @param siteUrl the URL of the text file to download
|
||||
* @return the contents of the specified text file
|
||||
*/
|
||||
public String download(String siteUrl) throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
// HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
|
||||
return dl(con);
|
||||
}
|
||||
private static void setDefaults(HttpsURLConnection con) {
|
||||
|
||||
@Override
|
||||
public DownloadResponse downloadWithHeaders(String siteUrl, Map<String, List<String>> requestHeaders)
|
||||
throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
|
||||
pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value));
|
||||
}
|
||||
String responseBody = dl(con);
|
||||
return new DownloadResponse(responseBody, con.getHeaderFields());
|
||||
}
|
||||
con.setConnectTimeout(30 * 1000);// 30s
|
||||
con.setReadTimeout(30 * 1000);// 30s
|
||||
|
||||
@Override
|
||||
public DownloadResponse downloadWithHeaders(String siteUrl) throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
String responseBody = dl(con);
|
||||
return new DownloadResponse(responseBody, con.getHeaderFields());
|
||||
}
|
||||
// set default user agent
|
||||
if (null == con.getRequestProperty("User-Agent")) {
|
||||
con.setRequestProperty("User-Agent", USER_AGENT);
|
||||
}
|
||||
|
||||
// add default cookies
|
||||
if (getCookies().length() > 0) {
|
||||
con.addRequestProperty("Cookie", getCookies());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Download (via HTTP) the text file located at the supplied URL, and return its
|
||||
* contents. Primarily intended for downloading web pages.
|
||||
*
|
||||
* @param siteUrl the URL of the text file to download
|
||||
* @return the contents of the specified text file
|
||||
*/
|
||||
public String download(String siteUrl) throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
// HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
|
||||
return dl(con);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DownloadResponse get(String siteUrl, Map<String, List<String>> requestHeaders)
|
||||
throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
|
||||
pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value));
|
||||
}
|
||||
String responseBody = dl(con);
|
||||
return new DownloadResponse(responseBody, con.getHeaderFields());
|
||||
}
|
||||
|
||||
@Override
|
||||
public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
String responseBody = dl(con);
|
||||
return new DownloadResponse(responseBody, con.getHeaderFields());
|
||||
}
|
||||
|
||||
@Override
|
||||
public DownloadResponse post(String siteUrl, String requestBody, Map<String, List<String>> requestHeaders)
|
||||
throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
con.setRequestMethod("POST");
|
||||
for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
|
||||
pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value));
|
||||
}
|
||||
// set fields to default if not set already
|
||||
setDefaults(con);
|
||||
|
||||
byte[] postDataBytes = requestBody.toString().getBytes("UTF-8");
|
||||
con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
|
||||
|
||||
con.setDoOutput(true);
|
||||
con.getOutputStream().write(postDataBytes);
|
||||
|
||||
BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String inputLine;
|
||||
while ((inputLine = in.readLine()) != null) {
|
||||
sb.append(inputLine);
|
||||
}
|
||||
return new DownloadResponse(sb.toString(), con.getHeaderFields());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,31 +16,31 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsE
|
||||
|
||||
public class YoutubeCommentsExtractorTest {
|
||||
|
||||
private static YoutubeCommentsExtractor extractor;
|
||||
private static YoutubeCommentsExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
extractor = (YoutubeCommentsExtractor) YouTube
|
||||
.getCommentsExtractor("https://www.youtube.com/watch?v=rrgFN3AxGfs");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance());
|
||||
extractor = (YoutubeCommentsExtractor) YouTube
|
||||
.getCommentsExtractor("https://www.youtube.com/watch?v=rrgFN3AxGfs");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetComments() throws IOException, ExtractionException {
|
||||
boolean result = false;
|
||||
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
|
||||
result = findInComments(comments, "i should really be in the top comment.lol");
|
||||
@Test
|
||||
public void testGetComments() throws IOException, ExtractionException {
|
||||
boolean result = false;
|
||||
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
|
||||
result = findInComments(comments, "i should really be in the top comment.lol");
|
||||
|
||||
while (comments.hasNextPage()) {
|
||||
comments = extractor.getPage(comments.getNextPageUrl());
|
||||
result = findInComments(comments, "i should really be in the top comment.lol");
|
||||
}
|
||||
while (comments.hasNextPage()) {
|
||||
comments = extractor.getPage(comments.getNextPageUrl());
|
||||
result = findInComments(comments, "i should really be in the top comment.lol");
|
||||
}
|
||||
|
||||
assertTrue(result);
|
||||
}
|
||||
assertTrue(result);
|
||||
}
|
||||
|
||||
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
|
||||
return comments.getItems().stream().filter(c -> c.getCommentText().contains(comment)).findAny().isPresent();
|
||||
}
|
||||
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
|
||||
return comments.getItems().stream().filter(c -> c.getCommentText().contains(comment)).findAny().isPresent();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user