PaperTowel/NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor and improvements

Browse Source
This commit is contained in:
Mauricio Colli
2017-08-06 17:20:15 -03:00
parent 731102be39
commit ea5b08db4c
61 changed files with 1458 additions and 857 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/test/java/org/schabi/newpipe/Downloader.java
View File

@@ -14,7 +14,7 @@ import java.util.Map;
import javax.net.ssl.HttpsURLConnection;
/**
/*
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@@ -35,16 +35,17 @@ import javax.net.ssl.HttpsURLConnection;
*/
public class Downloader implements org.schabi.newpipe.extractor.Downloader {
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private static String mCookies = "";
private static Downloader instance = null;
private Downloader() {}
private Downloader() {
}
public static Downloader getInstance() {
if(instance == null) {
if (instance == null) {
synchronized (Downloader.class) {
if (instance == null) {
instance = new Downloader();
@@ -62,11 +63,14 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
return Downloader.mCookies;
}
/**Download the text file at the supplied URL as in download(String),
/**
* Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
*
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
* @return the contents of the specified text file
*/
public String download(String siteUrl, String language) throws IOException, ReCaptchaException {
Map<String, String> requestProperties = new HashMap<>();
requestProperties.put("Accept-Language", language);
@@ -74,29 +78,35 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
}
/**Download the text file at the supplied URL as in download(String),
/**
* Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
*
* @param siteUrl the URL of the text file to return the contents of
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException*/
* @throws IOException
*/
public String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
Iterator it = customProperties.entrySet().iterator();
while(it.hasNext()) {
Map.Entry pair = (Map.Entry)it.next();
con.setRequestProperty((String)pair.getKey(), (String)pair.getValue());
while (it.hasNext()) {
Map.Entry pair = (Map.Entry) it.next();
con.setRequestProperty((String) pair.getKey(), (String) pair.getValue());
}
return dl(con);
}
/**Common functionality between download(String url) and download(String url, String language)*/
/**
* Common functionality between download(String url) and download(String url, String language)
*/
private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
StringBuilder response = new StringBuilder();
BufferedReader in = null;
try {
con.setReadTimeout(30 * 1000);// 30s
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);
@@ -108,13 +118,13 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
new InputStreamReader(con.getInputStream()));
String inputLine;
while((inputLine = in.readLine()) != null) {
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
} catch(UnknownHostException uhe) {//thrown when there's no internet connection
} catch (UnknownHostException uhe) {//thrown when there's no internet connection
throw new IOException("unknown host or no network", uhe);
//Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
} catch(Exception e) {
} catch (Exception e) {
/*
* HTTP 429 == Too Many Request
* Receive from Youtube.com = ReCaptcha challenge request
@@ -123,9 +133,10 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
if (con.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested");
}
throw new IOException(e);
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
} finally {
if(in != null) {
if (in != null) {
in.close();
}
}
@@ -133,10 +144,13 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
return response.toString();
}
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
/**
* Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
*
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
* @return the contents of the specified text file
*/
public String download(String siteUrl) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();