Update client version, fix some tests, update mocks and do some improvements

Add the origin and the referer headers with the https://www.youtube.com value for YouTube JSON POST requests.
Don't add the consent cookie header for the requests which use the youtubei/innertube API because it's uneeded.
Fix some tests and update YouTube mocks

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java

@@ -101,14 +101,14 @@ public class YoutubeMixPlaylistExtractorTest {
             InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
             final Set<String> urls = new HashSet<>();
 
-            //Should work infinitely, but for testing purposes only 3 times
+            // Should work infinitely, but for testing purposes only 3 times
             for (int i = 0; i < 3; i++) {
                 assertTrue(streams.hasNextPage());
                 assertFalse(streams.getItems().isEmpty());
 
                 for (final StreamInfoItem item : streams.getItems()) {
-//                    TODO Duplicates are appearing
-//                    assertFalse(urls.contains(item.getUrl()));
+                    // TODO Duplicates are appearing
+                    // assertFalse(urls.contains(item.getUrl()));
                     urls.add(item.getUrl());
                 }
 
@@ -179,13 +179,13 @@ public class YoutubeMixPlaylistExtractorTest {
             InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
             final Set<String> urls = new HashSet<>();
 
-            //Should work infinitely, but for testing purposes only 3 times
+            // Should work infinitely, but for testing purposes only 3 times
             for (int i = 0; i < 3; i++) {
                 assertTrue(streams.hasNextPage());
                 assertFalse(streams.getItems().isEmpty());
                 for (final StreamInfoItem item : streams.getItems()) {
-//                    TODO Duplicates are appearing
-//                    assertFalse(urls.contains(item.getUrl()));
+                    // TODO Duplicates are appearing
+                    // assertFalse(urls.contains(item.getUrl()));
                     urls.add(item.getUrl());
                 }
 
@@ -255,14 +255,14 @@ public class YoutubeMixPlaylistExtractorTest {
             InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
             final Set<String> urls = new HashSet<>();
 
-            //Should work infinitely, but for testing purposes only 3 times
+            // Should work infinitely, but for testing purposes only 3 times
             for (int i = 0; i < 3; i++) {
                 assertTrue(streams.hasNextPage());
                 assertFalse(streams.getItems().isEmpty());
 
                 for (final StreamInfoItem item : streams.getItems()) {
                     // TODO Duplicates are appearing
-//                    assertFalse(urls.contains(item.getUrl()));
+                    // assertFalse(urls.contains(item.getUrl()));
                     urls.add(item.getUrl());
                 }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java

@@ -54,7 +54,8 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
     @Override public long expectedDislikeCountAtLeast() { return 38000; }
     @Override public boolean expectedHasRelatedItems() { return false; } // no related videos (!)
     @Override public int expectedAgeLimit() { return 18; }
-    @Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; }
+    // Broken, video is available with the embedded player
+    // @Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; }
     @Override public boolean expectedHasSubtitles() { return false; }
 
     @Override public String expectedCategory() { return ""; } // Unavailable on age restricted videos

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.stream.StreamType;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.Random;
 
 import javax.annotation.Nullable;
 
@@ -22,7 +23,6 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
  * Test for {@link YoutubeStreamLinkHandlerFactory}
  */
 
-@Ignore("Video is not available in specific countries. Someone else has to generate mocks")
 public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtractorTest {
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
     private static final String ID = "T4XJQO3qol8";
@@ -32,6 +32,7 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
     @BeforeClass
     public static void setUp() throws Exception {
         YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeParsingHelper.setNumberGenerator(new Random(1));
         NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "controversial"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java

@@ -37,7 +37,6 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
 
     @Override
     @Test
-    @Ignore("When visiting website it shows 'Lofi Girl', unknown why it's different in tests")
     public void testUploaderName() throws Exception {
         super.testUploaderName();
     }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_9.json

@@ -1,176 +0,0 @@
-{
-  "request": {
-    "httpMethod": "POST",
-    "url": "https://youtubei.googleapis.com/youtubei/v1/browse?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
-    "headers": {
-      "Accept-Language": [
-        "en-GB, en;q\u003d0.9"
-      ]
-    },
-    "dataToSend": [
-      123,
-      34,
-      98,
-      114,
-      111,
-      119,
-      115,
-      101,
-      73,
-      100,
-      34,
-      58,
-      34,
-      68,
-      79,
-      69,
-      83,
-      78,
-      84,
-      45,
-      69,
-      88,
-      73,
-      83,
-      84,
-      34,
-      44,
-      34,
-      99,
-      111,
-      110,
-      116,
-      101,
-      120,
-      116,
-      34,
-      58,
-      123,
-      34,
-      99,
-      108,
-      105,
-      101,
-      110,
-      116,
-      34,
-      58,
-      123,
-      34,
-      99,
-      108,
-      105,
-      101,
-      110,
-      116,
-      78,
-      97,
-      109,
-      101,
-      34,
-      58,
-      34,
-      49,
-      34,
-      44,
-      34,
-      99,
-      108,
-      105,
-      101,
-      110,
-      116,
-      86,
-      101,
-      114,
-      115,
-      105,
-      111,
-      110,
-      34,
-      58,
-      34,
-      50,
-      46,
-      50,
-      48,
-      50,
-      49,
-      48,
-      54,
-      48,
-      54,
-      34,
-      125,
-      125,
-      44,
-      34,
-      112,
-      97,
-      114,
-      97,
-      109,
-      115,
-      34,
-      58,
-      34,
-      69,
-      103,
-      90,
-      50,
-      97,
-      87,
-      82,
-      108,
-      98,
-      51,
-      77,
-      37,
-      51,
-      68,
-      34,
-      125
-    ],
-    "localization": {
-      "languageCode": "en",
-      "countryCode": "GB"
-    }
-  },
-  "response": {
-    "responseCode": 400,
-    "responseMessage": "",
-    "responseHeaders": {
-      "alt-svc": [
-        "h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
-      ],
-      "cache-control": [
-        "private"
-      ],
-      "content-type": [
-        "application/json; charset\u003dUTF-8"
-      ],
-      "date": [
-        "Tue, 08 Jun 2021 13:18:50 GMT"
-      ],
-      "server": [
-        "ESF"
-      ],
-      "vary": [
-        "Origin",
-        "X-Origin",
-        "Referer"
-      ],
-      "x-content-type-options": [
-        "nosniff"
-      ],
-      "x-frame-options": [
-        "SAMEORIGIN"
-      ],
-      "x-xss-protection": [
-        "0"
-      ]
-    },
-    "responseBody": "{\n  \"error\": {\n    \"code\": 400,\n    \"message\": \"Request contains an invalid argument.\",\n    \"errors\": [\n      {\n        \"message\": \"Request contains an invalid argument.\",\n        \"domain\": \"global\",\n        \"reason\": \"badRequest\"\n      }\n    ],\n    \"status\": \"INVALID_ARGUMENT\"\n  }\n}\n",
-    "latestUrl": "https://youtubei.googleapis.com/youtubei/v1/browse?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
-  }
-}

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/mix/invalid/generated_mock_1.json

@@ -6,14 +6,20 @@
       "Accept-Language": [
         "en-GB, en;q\u003d0.9"
       ],
+      "Origin": [
+        "https://www.youtube.com"
+      ],
       "Cookie": [
         "CONSENT\u003dPENDING+100285"
       ],
       "X-YouTube-Client-Name": [
         "1"
       ],
+      "Referer": [
+        "https://www.youtube.com"
+      ],
       "X-YouTube-Client-Version": [
-        "2.20210408.08.00"
+        "2.20210420.07.00"
       ]
     },
     "localization": {
@@ -38,7 +44,7 @@
         "application/json; charset\u003dutf-8"
       ],
       "date": [
-        "Sun, 11 Apr 2021 15:57:14 GMT"
+        "Wed, 21 Apr 2021 17:45:11 GMT"
       ],
       "expires": [
         "Mon, 01 Jan 1990 00:00:00 GMT"
@@ -53,8 +59,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003ddhXDf1Ev5bE; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "CONSENT\u003dPENDING+790; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com"
+        "YSC\u003d55xotGigykc; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "CONSENT\u003dPENDING+084; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"
@@ -75,7 +81,7 @@
         "0"
       ]
     },
-    "responseBody": "[\r\n{\"page\": \"watch\",\"rootVe\": \"3832\"},\r\n{\"page\": \"watch\",\"preconnect\": [\"https:\\/\\/r4---sn-hgn7rn7k.googlevideo.com\\/generate_204\",\"https:\\/\\/r4---sn-hgn7rn7k.googlevideo.com\\/generate_204?conn2\"]},\r\n{\"page\": \"watch\",\"playerResponse\": {\"responseContext\":{\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"is_viewed_live\",\"value\":\"False\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"23970530,24005646,23968386,24005602,23918597,24011119,23934970,23744176,24022914,23890959,23974595,23983296,23857950,24007246,9407156,24016478,1714250,24022308,23976696,24006670,24009750,24001373,24022875,24006795,24012117,23885487,23940238,23891346,24014268,24002697,23966208,23987676,23804281,23946420,23884386,23882502,24024495,23969934,24025870,23986021,23944779,24002011,23748146,24022616,23891344,24010576,23981192,24021968\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20210408.08.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlayer_rid\",\"value\":\"0xce87950a1c7b0640\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20210408\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"playabilityStatus\":{\"status\":\"ERROR\",\"reason\":\"Video unavailable\",\"errorScreen\":{\"playerErrorMessageRenderer\":{\"reason\":{\"simpleText\":\"Video unavailable\"},\"thumbnail\":{\"thumbnails\":[{\"url\":\"//s.ytimg.com/yts/img/meh7-vflGevej7.png\",\"width\":140,\"height\":100}]},\"icon\":{\"iconType\":\"ERROR_OUTLINE\"}}},\"contextParams\":\"Q0FBU0FnZ0E\u003d\"},\"trackingParams\":\"CAAQu2kiEwijlIybx_bvAhWnF_EFHZCGD0U\u003d\"}},\r\n{\"page\": \"watch\",\"response\": {\"responseContext\":{\"webResponseContextExtensionData\":{\"ytConfigData\":{\"visitorData\":\"CgtjdWVaRlB4MmUxRSjauMyDBg%3D%3D\",\"rootVisualElementType\":3832}}}},\"xsrf_token\": \"QUFFLUhqbjU0XzV6c3N4VDlZNXpGU3RuVW5IZ18wSkdsd3xBQ3Jtc0tuMkpBMjhqbEY2WEc5VTd3Mk1XZ3pMWXdVMmhQZzlXY3pOY05mV0FQQURJLVh3U3NqVHFSLUpsdmxtZnJURkhrOXdhTFE5T2RCV3dXSHQ2UlZVRHJnRjRmYkpvN1BGUEI3TllwZktBbnZyQ29LRVZwZw\\u003d\\u003d\",\"url\": \"/watch?v\\u003dabcde\\u0026list\\u003dRDabcde\",\"endpoint\": {\"clickTrackingParams\":\"IhMIs62Lm8f27wIVST_xBR3LIARZMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/watch?v\u003dabcde\",\"webPageType\":\"WEB_PAGE_TYPE_WATCH\",\"rootVe\":3832}},\"watchEndpoint\":{\"videoId\":\"abcde\"}}},\r\n{\"page\": \"watch\",\"timing\": {\"info\": {\"st\":  0.0 }}}]\r\n",
+    "responseBody": "[\r\n{\"page\": \"watch\",\"rootVe\": \"3832\"},\r\n{\"page\": \"watch\",\"preconnect\": [\"https:\\/\\/r4---sn-hgn7rn7k.googlevideo.com\\/generate_204\",\"https:\\/\\/r4---sn-hgn7rn7k.googlevideo.com\\/generate_204?conn2\"]},\r\n{\"page\": \"watch\",\"playerResponse\": {\"responseContext\":{\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"is_viewed_live\",\"value\":\"False\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"23804281,23884386,23882502,24010466,23934970,23969934,23918597,24005646,23983296,23966208,9405964,24021967,23891346,23970529,23946420,23975058,23891344,24007246,24007613,24022927,24031409,24011119,1714255,24002010,23890959,23974595,24022914,23944779,23987676,23744176,23857949,24001373,24028580,23968386,24006795,24005802,24012117,24014268,24022875,23995927,24014442\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20210420.07.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlayer_rid\",\"value\":\"0xbb641a161cfd5b01\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20210420\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"playabilityStatus\":{\"status\":\"ERROR\",\"reason\":\"Video unavailable\",\"errorScreen\":{\"playerErrorMessageRenderer\":{\"reason\":{\"simpleText\":\"Video unavailable\"},\"thumbnail\":{\"thumbnails\":[{\"url\":\"//s.ytimg.com/yts/img/meh7-vflGevej7.png\",\"width\":140,\"height\":100}]},\"icon\":{\"iconType\":\"ERROR_OUTLINE\"}}},\"contextParams\":\"Q0FBU0FnZ0E\u003d\"},\"trackingParams\":\"CAAQu2kiEwiMt9z-8Y_wAhUpNPEFHfXjBV0\u003d\"}},\r\n{\"page\": \"watch\",\"response\": {\"responseContext\":{\"webResponseContextExtensionData\":{\"ytConfigData\":{\"visitorData\":\"CgtUcGFUdS05TW5OSSinyYGEBg%3D%3D\",\"rootVisualElementType\":3832}}}},\"xsrf_token\": \"QUFFLUhqbHlBTEFxZDFZQlV2c2FKYjZyRDZkT0g3a0Y1Z3xBQ3Jtc0trTXNGLW5KSEhpUkJoR0xjcUN1RGZLUVROeDY2NWluUnBubGxWR0IzNUkwNXRyZlctRm1XaEs0dk9pVk9QM09iUzZRVFRRYVl3MVdvZ3BEVFBCcWRtd0MwdjZwOW5zenlVNlNNbzdaWXlJVmRLbC1tcw\\u003d\\u003d\",\"url\": \"/watch?v\\u003dabcde\\u0026list\\u003dRDabcde\",\"endpoint\": {\"clickTrackingParams\":\"IhMIlvHb_vGP8AIVBxjxBR0nGg62MghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/watch?v\u003dabcde\",\"webPageType\":\"WEB_PAGE_TYPE_WATCH\",\"rootVe\":3832}},\"watchEndpoint\":{\"videoId\":\"abcde\"}}},\r\n{\"page\": \"watch\",\"timing\": {\"info\": {\"st\":  0.0 }}}]\r\n",
     "latestUrl": "https://www.youtube.com/watch?v\u003dabcde\u0026list\u003dRDabcde\u0026pbj\u003d1"
   }
 }