Merge branch 'master' into invidious-support
This commit is contained in:
Connectety
GitHub
@@ -1,17 +1,20 @@
|
||||
package org.schabi.newpipe;
|
||||
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.utils.Localization;
|
||||
|
||||
import javax.net.ssl.HttpsURLConnection;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URL;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.net.ssl.HttpsURLConnection;
|
||||
|
||||
import org.schabi.newpipe.extractor.DownloadRequest;
|
||||
import org.schabi.newpipe.extractor.DownloadResponse;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.utils.Localization;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 28.01.16.
|
||||
@@ -63,10 +66,11 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
||||
}
|
||||
|
||||
/**
|
||||
* Download the text file at the supplied URL as in download(String),
|
||||
* but set the HTTP header field "Accept-Language" to the supplied string.
|
||||
* Download the text file at the supplied URL as in download(String), but set
|
||||
* the HTTP header field "Accept-Language" to the supplied string.
|
||||
*
|
||||
* @param siteUrl the URL of the text file to return the contents of
|
||||
|
||||
* @param localization the language and country (usually a 2-character code for both values)
|
||||
* @return the contents of the specified text file
|
||||
*/
|
||||
@@ -76,57 +80,53 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
||||
return download(siteUrl, requestProperties);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Download the text file at the supplied URL as in download(String),
|
||||
* but set the HTTP header field "Accept-Language" to the supplied string.
|
||||
* Download the text file at the supplied URL as in download(String), but set
|
||||
* the HTTP header field "Accept-Language" to the supplied string.
|
||||
*
|
||||
* @param siteUrl the URL of the text file to return the contents of
|
||||
* @param customProperties set request header properties
|
||||
* @return the contents of the specified text file
|
||||
* @throws IOException
|
||||
*/
|
||||
public String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException {
|
||||
public String download(String siteUrl, Map<String, String> customProperties)
|
||||
throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
for (Map.Entry<String, String> pair: customProperties.entrySet()) {
|
||||
for (Map.Entry<String, String> pair : customProperties.entrySet()) {
|
||||
con.setRequestProperty(pair.getKey(), pair.getValue());
|
||||
}
|
||||
return dl(con);
|
||||
}
|
||||
|
||||
/**
|
||||
* Common functionality between download(String url) and download(String url, String language)
|
||||
* Common functionality between download(String url) and download(String url,
|
||||
* String language)
|
||||
*/
|
||||
private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
|
||||
StringBuilder response = new StringBuilder();
|
||||
BufferedReader in = null;
|
||||
|
||||
try {
|
||||
con.setConnectTimeout(30 * 1000);// 30s
|
||||
con.setReadTimeout(30 * 1000);// 30s
|
||||
|
||||
con.setRequestMethod("GET");
|
||||
con.setRequestProperty("User-Agent", USER_AGENT);
|
||||
setDefaults(con);
|
||||
|
||||
if (getCookies().length() > 0) {
|
||||
con.setRequestProperty("Cookie", getCookies());
|
||||
}
|
||||
|
||||
in = new BufferedReader(
|
||||
new InputStreamReader(con.getInputStream()));
|
||||
in = new BufferedReader(new InputStreamReader(con.getInputStream()));
|
||||
String inputLine;
|
||||
|
||||
while ((inputLine = in.readLine()) != null) {
|
||||
response.append(inputLine);
|
||||
}
|
||||
} catch (UnknownHostException uhe) {//thrown when there's no internet connection
|
||||
} catch (UnknownHostException uhe) {// thrown when there's no internet
|
||||
// connection
|
||||
throw new IOException("unknown host or no network", uhe);
|
||||
//Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
|
||||
// Toast.makeText(getActivity(), uhe.getMessage(),
|
||||
// Toast.LENGTH_LONG).show();
|
||||
} catch (Exception e) {
|
||||
/*
|
||||
* HTTP 429 == Too Many Request
|
||||
* Receive from Youtube.com = ReCaptcha challenge request
|
||||
* See : https://github.com/rg3/youtube-dl/issues/5138
|
||||
* HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge
|
||||
* request See : https://github.com/rg3/youtube-dl/issues/5138
|
||||
*/
|
||||
if (con.getResponseCode() == 429) {
|
||||
throw new ReCaptchaException("reCaptcha Challenge requested");
|
||||
@@ -142,9 +142,25 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
||||
return response.toString();
|
||||
}
|
||||
|
||||
private static void setDefaults(HttpsURLConnection con) {
|
||||
|
||||
con.setConnectTimeout(30 * 1000);// 30s
|
||||
con.setReadTimeout(30 * 1000);// 30s
|
||||
|
||||
// set default user agent
|
||||
if (null == con.getRequestProperty("User-Agent")) {
|
||||
con.setRequestProperty("User-Agent", USER_AGENT);
|
||||
}
|
||||
|
||||
// add default cookies
|
||||
if (getCookies().length() > 0) {
|
||||
con.addRequestProperty("Cookie", getCookies());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Download (via HTTP) the text file located at the supplied URL, and return its contents.
|
||||
* Primarily intended for downloading web pages.
|
||||
* Download (via HTTP) the text file located at the supplied URL, and return its
|
||||
* contents. Primarily intended for downloading web pages.
|
||||
*
|
||||
* @param siteUrl the URL of the text file to download
|
||||
* @return the contents of the specified text file
|
||||
@@ -152,7 +168,57 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
|
||||
public String download(String siteUrl) throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
|
||||
// HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
|
||||
return dl(con);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DownloadResponse get(String siteUrl, DownloadRequest request)
|
||||
throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
for (Map.Entry<String, List<String>> pair : request.getRequestHeaders().entrySet()) {
|
||||
for(String value: pair.getValue()) {
|
||||
con.addRequestProperty(pair.getKey(), value);
|
||||
}
|
||||
}
|
||||
String responseBody = dl(con);
|
||||
return new DownloadResponse(responseBody, con.getHeaderFields());
|
||||
}
|
||||
|
||||
@Override
|
||||
public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException {
|
||||
return get(siteUrl, DownloadRequest.emptyRequest);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DownloadResponse post(String siteUrl, DownloadRequest request)
|
||||
throws IOException, ReCaptchaException {
|
||||
URL url = new URL(siteUrl);
|
||||
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
|
||||
con.setRequestMethod("POST");
|
||||
for (Map.Entry<String, List<String>> pair : request.getRequestHeaders().entrySet()) {
|
||||
for(String value: pair.getValue()) {
|
||||
con.addRequestProperty(pair.getKey(), value);
|
||||
}
|
||||
}
|
||||
// set fields to default if not set already
|
||||
setDefaults(con);
|
||||
|
||||
if(null != request.getRequestBody()) {
|
||||
byte[] postDataBytes = request.getRequestBody().getBytes("UTF-8");
|
||||
con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
|
||||
con.setDoOutput(true);
|
||||
con.getOutputStream().write(postDataBytes);
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try (BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()))) {
|
||||
String inputLine;
|
||||
while ((inputLine = in.readLine()) != null) {
|
||||
sb.append(inputLine);
|
||||
}
|
||||
}
|
||||
return new DownloadResponse(sb.toString(), con.getHeaderFields());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.jsoup.helper.StringUtil;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.comments.CommentsInfo;
|
||||
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
|
||||
import org.schabi.newpipe.extractor.utils.Localization;
|
||||
|
||||
public class YoutubeCommentsExtractorTest {
|
||||
|
||||
private static YoutubeCommentsExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
extractor = (YoutubeCommentsExtractor) YouTube
|
||||
.getCommentsExtractor("https://www.youtube.com/watch?v=rrgFN3AxGfs");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetComments() throws IOException, ExtractionException {
|
||||
boolean result = false;
|
||||
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
|
||||
result = findInComments(comments, "i should really be in the top comment.lol");
|
||||
|
||||
while (comments.hasNextPage() && !result) {
|
||||
comments = extractor.getPage(comments.getNextPageUrl());
|
||||
result = findInComments(comments, "i should really be in the top comment.lol");
|
||||
}
|
||||
|
||||
assertTrue(result);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
|
||||
boolean result = false;
|
||||
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=rrgFN3AxGfs");
|
||||
assertTrue("what the fuck am i doing with my life.wmv".equals(commentsInfo.getName()));
|
||||
result = findInComments(commentsInfo.getRelatedItems(), "i should really be in the top comment.lol");
|
||||
|
||||
String nextPage = commentsInfo.getNextPageUrl();
|
||||
while (!StringUtil.isBlank(nextPage) && !result) {
|
||||
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
|
||||
result = findInComments(moreItems.getItems(), "i should really be in the top comment.lol");
|
||||
nextPage = moreItems.getNextPageUrl();
|
||||
}
|
||||
|
||||
assertTrue(result);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetCommentsAllData() throws IOException, ExtractionException {
|
||||
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
|
||||
for(CommentsInfoItem c: comments.getItems()) {
|
||||
assertFalse(StringUtil.isBlank(c.getAuthorEndpoint()));
|
||||
assertFalse(StringUtil.isBlank(c.getAuthorName()));
|
||||
assertFalse(StringUtil.isBlank(c.getAuthorThumbnail()));
|
||||
assertFalse(StringUtil.isBlank(c.getCommentId()));
|
||||
assertFalse(StringUtil.isBlank(c.getCommentText()));
|
||||
assertFalse(StringUtil.isBlank(c.getName()));
|
||||
assertFalse(StringUtil.isBlank(c.getPublishedTime()));
|
||||
assertFalse(StringUtil.isBlank(c.getThumbnailUrl()));
|
||||
assertFalse(StringUtil.isBlank(c.getUrl()));
|
||||
assertFalse(c.getLikeCount() == null);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
|
||||
return findInComments(comments.getItems(), comment);
|
||||
}
|
||||
|
||||
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
|
||||
for(CommentsInfoItem c: comments) {
|
||||
if(c.getCommentText().contains(comment)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package org.schabi.newpipe.extractor.utils;
|
||||
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParser;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
|
||||
|
||||
public class JsonUtilsTest {
|
||||
|
||||
@Test
|
||||
public void testGetValueFlat() throws JsonParserException, ParsingException {
|
||||
JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}");
|
||||
assertTrue("John".equals(JsonUtils.getValue(obj, "name")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetValueNested() throws JsonParserException, ParsingException {
|
||||
JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}");
|
||||
assertTrue("BMW".equals(JsonUtils.getValue(obj, "cars.car2")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetArray() throws JsonParserException, ParsingException {
|
||||
JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}");
|
||||
JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "batters.batter");
|
||||
assertTrue(!arr.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetValues() throws JsonParserException, ParsingException {
|
||||
JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}");
|
||||
JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "topping");
|
||||
List<Object> types = JsonUtils.getValues(arr, "type");
|
||||
assertTrue(types.contains("Chocolate with Sprinkles"));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user