Merge remote-tracking branch 'origin/master'

Add demo server discord invite

README.md

@@ -1,8 +1,12 @@
 <img align="right" src="https://i.imgur.com/zrE80HY.png" height="200" width="200">
 
+## Demo
+
+[https://discord.gg/4cp9CvjwaW](https://discord.gg/4cp9CvjwaW)
+
 # This fork
 
-Essentially, this is just a merge of two pull requests:
+Essentially, this is just a merge of three pull requests:
 
 ### Support for logging into your (burner) youtube account:
 
@@ -11,8 +15,23 @@ Essentially, this is just a merge of two pull requests:
 ### To allow building the project from source
 [PR-1703](https://github.com/jagrosh/MusicBot/pull/1703)
 
+### Add YouTube PO token & IP rotation
+https://github.com/jagrosh/MusicBot/pull/1772
+
 And I've now bumped the versions up to the newest [lavaplayer](https://mvnrepository.com/artifact/dev.arbjerg/lavaplayer)/[lavalink](https://maven.lavalink.dev/#/releases/dev/lavalink/youtube/common) versions. I will try to update this automatically in the future.
 
+
+### For PO_Tokens: In your bot folder, edit "config.txt" and add
+```
+ytpotoken = "PO_TOKEN_HERE"
+ytvisitordata = "VISITOR_DATA_HERE"
+```
+
+To get po_token and visitor_data, use [youtube-trusted-session-generator](https://github.com/iv-org/youtube-trusted-session-generator)
+
+
+If you have problems with this fork, you could also check out [this other fork](https://github.com/SeVile/MusicBot).
+
 ## Easy deployment via docker
 
 Additionally, i thought a corresponding docker image would make things better:

pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.jagrosh</groupId>
     <artifactId>JMusicBot</artifactId>
-    <version>0.4.6</version>
+    <version>0.4.9</version>
     <packaging>jar</packaging>
 
     <repositories>
@@ -56,10 +56,15 @@
             <artifactId>lavaplayer</artifactId>
             <version>2.2.2</version>
         </dependency>
+        <dependency>
+            <groupId>dev.arbjerg</groupId>
+            <artifactId>lavaplayer-ext-youtube-rotator</artifactId>
+            <version>2.2.3</version>
+        </dependency>
         <dependency>
             <groupId>dev.lavalink.youtube</groupId>
             <artifactId>common</artifactId>
-            <version>1.11.1</version>
+            <version>1.11.5</version>
         </dependency>
 	<dependency>
 	    <groupId>com.github.jagrosh</groupId>
@@ -88,6 +93,33 @@
             <artifactId>jsoup</artifactId>
             <version>1.15.3</version>
         </dependency>
+        <dependency>
+            <groupId>org.seleniumhq.selenium</groupId>
+            <artifactId>selenium-java</artifactId>
+            <version>4.27.0</version>
+            <!-- https://github.com/SeleniumHQ/selenium/issues/13791 -->
+            <exclusions>
+                <exclusion>
+                    <groupId>org.seleniumhq.selenium</groupId>
+                    <artifactId>selenium-devtools-v85</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.seleniumhq.selenium</groupId>
+                    <artifactId>selenium-devtools-v129</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.seleniumhq.selenium</groupId>
+                    <artifactId>selenium-devtools-v130</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>io.github.bonigarcia</groupId>
+            <artifactId>webdrivermanager</artifactId>
+            <version>5.9.2</version>
+        </dependency>
+
+
 
         <!-- Testing Dependencies -->
         <dependency>

src/main/java/com/jagrosh/jmusicbot/Bot.java

@@ -60,7 +60,7 @@ public class Bot
         this.threadpool = Executors.newSingleThreadScheduledExecutor();
         this.youTubeOauth2TokenHandler = new YoutubeOauth2TokenHandler();
         this.youTubeOauth2TokenHandler.init();
-        this.players = new PlayerManager(this);
+        this.players = new PlayerManager(this, config);
         this.players.init();
         this.nowplaying = new NowplayingHandler(this);
         this.nowplaying.init();

src/main/java/com/jagrosh/jmusicbot/BotConfig.java

@@ -18,11 +18,16 @@ package com.jagrosh.jmusicbot;
 import com.jagrosh.jmusicbot.entities.Prompt;
 import com.jagrosh.jmusicbot.utils.OtherUtil;
 import com.jagrosh.jmusicbot.utils.TimeUtil;
+import com.jagrosh.jmusicbot.utils.YouTubeUtil;
 import com.sedmelluq.discord.lavaplayer.track.AudioTrack;
+import com.sedmelluq.lava.extensions.youtuberotator.tools.ip.IpBlock;
 import com.typesafe.config.*;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+
 import net.dv8tion.jda.api.OnlineStatus;
 import net.dv8tion.jda.api.entities.Activity;
 
@@ -41,8 +46,10 @@ public class BotConfig
     private Path path = null;
     private String token, prefix, altprefix, helpWord, playlistsFolder, logLevel,
             successEmoji, warningEmoji, errorEmoji, loadingEmoji, searchingEmoji,
-            evalEngine;
-    private boolean youtubeOauth2, stayInChannel, songInGame, npImages, updatealerts, useEval, dbots;
+            chromePath, chromeDriverPath, evalEngine;
+    private YouTubeUtil.RoutingPlanner ytRoutingPlanner;
+    private List<IpBlock> ytIpBlocks;
+    private boolean youtubeOauth2, stayInChannel, songInGame, npImages, updatealerts, chromeHeadless, useEval, dbots;
     private long owner, maxSeconds, aloneTimeUntilStop;
     private int maxYTPlaylistPages;
     private double skipratio;
@@ -97,6 +104,11 @@ public class BotConfig
             aloneTimeUntilStop = config.getLong("alonetimeuntilstop");
             playlistsFolder = config.getString("playlistsfolder");
             aliases = config.getConfig("aliases");
+            chromePath = config.getString("chromepath");
+            chromeDriverPath = config.getString("chromedriverpath");
+            chromeHeadless = config.getBoolean("chromeheadless");
+            ytRoutingPlanner = config.getEnum(YouTubeUtil.RoutingPlanner.class, "ytroutingplanner");
+            ytIpBlocks = config.getStringList("ytipblocks").stream().map(YouTubeUtil::parseIpBlock).collect(Collectors.toList());
             transforms = config.getConfig("transforms");
             skipratio = config.getDouble("skipratio");
             dbots = owner == 113156185389092864L;
@@ -329,6 +341,31 @@ public class BotConfig
         return logLevel;
     }
 
+    public String getChromePath()
+    {
+        return chromePath.equals("AUTO") ? null : chromePath;
+    }
+
+    public String getChromeDriverPath()
+    {
+        return chromeDriverPath.equals("AUTO") ? null : chromeDriverPath;
+    }
+    
+    public boolean getChromeHeadless()
+    {
+        return chromeHeadless;
+    }
+
+    public YouTubeUtil.RoutingPlanner getYTRoutingPlanner()
+    {
+        return ytRoutingPlanner;
+    }
+    
+    public List<IpBlock> getYTIpBlocks()
+    {
+        return ytIpBlocks;
+    }
+    
     public boolean useEval()
     {
         return useEval;

src/main/java/com/jagrosh/jmusicbot/JMusicBot.java

@@ -40,6 +40,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import ch.qos.logback.classic.Level;
 
+import static com.jagrosh.jmusicbot.utils.YouTubeUtil.GetPoTokenAndVisitorData;
+
 /**
  *
  * @author John Grosh (jagrosh)
@@ -166,6 +168,8 @@ public class JMusicBot
                     + "attempting to connect, please make sure you're connected to the internet");
             System.exit(1);
         }
+
+        GetPoTokenAndVisitorData(config.getChromePath(), config.getChromeDriverPath(), config.getChromeHeadless());
     }
     
     private static CommandClient createCommandClient(BotConfig config, SettingsManager settings, Bot bot)

src/main/java/com/jagrosh/jmusicbot/audio/PlayerManager.java

@@ -17,7 +17,9 @@ package com.jagrosh.jmusicbot.audio;
 
 import com.dunctebot.sourcemanagers.DuncteBotSources;
 import com.jagrosh.jmusicbot.Bot;
+import com.jagrosh.jmusicbot.BotConfig;
 import com.jagrosh.jmusicbot.utils.OtherUtil;
+import com.jagrosh.jmusicbot.utils.YouTubeUtil;
 import com.sedmelluq.discord.lavaplayer.container.MediaContainerRegistry;
 import com.sedmelluq.discord.lavaplayer.player.AudioPlayer;
 import com.sedmelluq.discord.lavaplayer.player.DefaultAudioPlayerManager;
@@ -30,7 +32,15 @@ import com.sedmelluq.discord.lavaplayer.source.nico.NicoAudioSourceManager;
 import com.sedmelluq.discord.lavaplayer.source.soundcloud.SoundCloudAudioSourceManager;
 import com.sedmelluq.discord.lavaplayer.source.twitch.TwitchStreamAudioSourceManager;
 import com.sedmelluq.discord.lavaplayer.source.vimeo.VimeoAudioSourceManager;
+import com.sedmelluq.lava.extensions.youtuberotator.YoutubeIpRotatorSetup;
+import com.sedmelluq.lava.extensions.youtuberotator.planner.AbstractRoutePlanner;
+import com.sedmelluq.lava.extensions.youtuberotator.planner.BalancingIpRoutePlanner;
+import com.sedmelluq.lava.extensions.youtuberotator.planner.NanoIpRoutePlanner;
+import com.sedmelluq.lava.extensions.youtuberotator.tools.ip.IpBlock;
+import com.sedmelluq.lava.extensions.youtuberotator.tools.ip.Ipv4Block;
+import com.sedmelluq.lava.extensions.youtuberotator.tools.ip.Ipv6Block;
 import dev.lavalink.youtube.YoutubeAudioSourceManager;
+import dev.lavalink.youtube.clients.Web;
 import net.dv8tion.jda.api.entities.Guild;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -46,17 +56,30 @@ public class PlayerManager extends DefaultAudioPlayerManager
 {
     private final static Logger LOGGER = LoggerFactory.getLogger(PlayerManager.class);
     private final Bot bot;
-    
-    public PlayerManager(Bot bot)
+    private final BotConfig config;
+
+    public PlayerManager(Bot bot, BotConfig config)
     {
         this.bot = bot;
+        this.config = config;
     }
     
     public void init()
     {
         TransformativeAudioSourceManager.createTransforms(bot.getConfig().getTransforms()).forEach(t -> registerSourceManager(t));
+        
+        YoutubeAudioSourceManager yt = new YoutubeAudioSourceManager(true);
+        if (config.getYTRoutingPlanner() != YouTubeUtil.RoutingPlanner.NONE)
+        {
+            AbstractRoutePlanner routePlanner = YouTubeUtil.createRouterPlanner(config.getYTRoutingPlanner(), config.getYTIpBlocks());
+            YoutubeIpRotatorSetup rotator = new YoutubeIpRotatorSetup(routePlanner);
+            
+            rotator.forConfiguration(yt.getHttpInterfaceManager(), false)
+                    .withMainDelegateFilter(yt.getContextFilter())
+                    .setup();
+        }
 
-        YoutubeAudioSourceManager yt = setupYoutubeAudioSourceManager();
+        yt.setPlaylistPageCount(bot.getConfig().getMaxYTPlaylistPages());
         registerSourceManager(yt);
 
         registerSourceManager(SoundCloudAudioSourceManager.createDefault());

src/main/java/com/jagrosh/jmusicbot/utils/YouTubeUtil.java

@@ -0,0 +1,193 @@
+package com.jagrosh.jmusicbot.utils;
+
+import com.sedmelluq.lava.extensions.youtuberotator.planner.*;
+import com.sedmelluq.lava.extensions.youtuberotator.tools.ip.IpBlock;
+import com.sedmelluq.lava.extensions.youtuberotator.tools.ip.Ipv4Block;
+import com.sedmelluq.lava.extensions.youtuberotator.tools.ip.Ipv6Block;
+import dev.lavalink.youtube.clients.Web;
+import org.json.JSONObject;
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeDriverService;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.openqa.selenium.chromium.ChromiumDriverLogLevel;
+import org.openqa.selenium.devtools.NetworkInterceptor;
+import org.openqa.selenium.os.ExecutableFinder;
+import org.openqa.selenium.remote.http.Contents;
+import org.openqa.selenium.remote.http.Filter;
+import org.openqa.selenium.remote.http.HttpMethod;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import io.github.bonigarcia.wdm.WebDriverManager;
+
+public class YouTubeUtil {
+    private final static Logger LOGGER = LoggerFactory.getLogger(YouTubeUtil.class);
+    
+    public enum RoutingPlanner {
+        NONE,
+        ROTATE_ON_BAN,
+        LOAD_BALANCE,
+        NANO_SWITCH,
+        ROTATING_NANO_SWITCH
+    }
+    
+    public static IpBlock parseIpBlock(String cidr) {
+        if (Ipv6Block.isIpv6CidrBlock(cidr))
+            return new Ipv6Block(cidr);
+
+        if (Ipv4Block.isIpv4CidrBlock(cidr))
+            return new Ipv4Block(cidr);
+        
+        throw new IllegalArgumentException("Could not parse CIDR " + cidr);
+    }
+    
+    public static AbstractRoutePlanner createRouterPlanner(RoutingPlanner routingPlanner, List<IpBlock> ipBlocks) {
+        
+        switch (routingPlanner) {
+            case NONE:
+                return null;
+            case ROTATE_ON_BAN:
+                return new RotatingIpRoutePlanner(ipBlocks);
+            case LOAD_BALANCE:
+                return new BalancingIpRoutePlanner(ipBlocks);
+            case NANO_SWITCH:
+                return new NanoIpRoutePlanner(ipBlocks, true);
+            case ROTATING_NANO_SWITCH:
+                return new RotatingNanoIpRoutePlanner(ipBlocks);
+            default:
+                throw new IllegalArgumentException("Unknown RoutingPlanner value provided");
+        }
+    }
+
+    public static void GetPoTokenAndVisitorData(String userChromePath, String userChromeDriverPath, boolean headless)
+    {
+        //
+        // 1. Find a Chrome install & launch the WebDriver
+        //
+        WebDriverManager wdm = WebDriverManager.chromedriver();
+        if (userChromePath != null) {
+            wdm.browserVersionDetectionCommand(userChromePath + " --version");
+        }
+        else if (wdm.getBrowserPath().isEmpty())
+        {
+            // try finding a Chromium browser instead (Linux users usually install Chromium instead of Google Chrome)
+            wdm = WebDriverManager.chromiumdriver();
+            if (wdm.getBrowserPath().isEmpty()) {
+                LOGGER.error("Could not obtain a PO token for YouTube playback, because no Chrome browser could be found. Please install Google Chrome or Chromium!");
+                return;
+            }
+        }
+        if (userChromeDriverPath == null)
+            // Automatically download & set up ChromeDriver for the Chrome/Chromium version that the user has installed
+            wdm.setup();
+        else {
+            // Find & use the ChromeDriver executable the user provided
+            String path = new ExecutableFinder().find(userChromeDriverPath);
+            if (path == null) {
+                LOGGER.error("Could not obtain a PO token for YouTube playback, because the specified ChromeDriver could not be found. " +
+                        "Please check your config's value of the chromedriverpath, or set it to \"AUTO\" to have JMusicBot automatically download ChromeDriver");
+                return;
+            }
+            LOGGER.info("Using ChromeDriver at {}", path);
+            System.setProperty("webdriver.chrome.driver", path);
+        }
+
+        ChromeOptions chromeOptions = new ChromeOptions();
+
+        chromeOptions.setBinary(userChromePath != null ? userChromePath : wdm.getBrowserPath().get().toString());
+
+        if (!headless)
+            chromeOptions.addArguments("--auto-open-devtools-for-tabs");
+        else
+            chromeOptions.addArguments("--headless=new");
+
+        ChromeDriverService.Builder chromeDriverBuilder = new ChromeDriverService.Builder();
+        if (LOGGER.isDebugEnabled())
+            chromeDriverBuilder.withLogLevel(ChromiumDriverLogLevel.DEBUG);
+
+        WebDriver driver = new ChromeDriver(chromeDriverBuilder.build(), chromeOptions);
+
+        //
+        // 2. Setup a network interceptor to intercept the player API request & obtain po token and visitor data
+        //
+
+        // as the network interceptor below runs on a different thread, we'll have it notify us on this/current thread
+        // whether retrieving the data failed (completes with the Exception) or succeeded (completes with null)
+        CompletableFuture<Exception> interceptorFuture = new CompletableFuture<>();
+
+        NetworkInterceptor interceptor = new NetworkInterceptor(driver,
+                (Filter) next -> req -> {
+            
+            if (req.getMethod() != HttpMethod.POST || !req.getUri().contains("/youtubei/v1/player"))
+                return next.execute(req);
+    
+            try {
+                String rawBody = Contents.string(req);
+                JSONObject body = new JSONObject(rawBody);
+
+                String visitorData = body
+                        .getJSONObject("context")
+                        .getJSONObject("client")
+                        .getString("visitorData");
+
+                String poToken = body
+                        .getJSONObject("serviceIntegrityDimensions")
+                        .getString("poToken");
+
+
+                LOGGER.info("Successfully retrieved PO token & visitor data for YouTube playback.");
+                LOGGER.debug("PO token: {}", poToken);
+                LOGGER.debug("Visitor Data: {}", visitorData);
+
+                if (poToken.length() < 160)
+                    LOGGER.warn("There is a high chance that the retrieved PO token will not work. Try on a different network.");
+
+                Web.setPoTokenAndVisitorData(poToken, visitorData);
+
+                interceptorFuture.complete(null);
+            } catch (Exception e) {
+                interceptorFuture.complete(e);
+            }
+
+            return next.execute(req);
+        });
+
+        //
+        // 3. Navigate to YouTube & start the player
+        //
+        try {
+            driver.get("https://www.youtube.com/embed/aqz-KE-bpKQ");
+            driver.manage().timeouts().implicitlyWait(Duration.ofSeconds(15));
+            WebElement moviePlayer = driver.findElement(By.cssSelector("#movie_player"));
+            moviePlayer.click();
+
+            try {
+                // Wait for the network interceptor to tell us if it got the data
+                Exception result = interceptorFuture.get(20, TimeUnit.SECONDS);
+                if (result != null)
+                    throw new Exception("Failed to read YouTube player request body", result);
+            } catch (TimeoutException e) {
+                throw new Exception("Timed out waiting for YouTube player request");
+            }
+
+
+        } catch (Exception e) {
+            LOGGER.error("Failed to obtain PO token for YouTube playback: {}", e.getMessage());
+            LOGGER.debug("Exception:", e);
+        } finally {
+            interceptor.close();
+            // Only quit the browser when not in headless mode. Helps with troubleshooting.
+            if (headless)
+                driver.quit();
+        }
+    }
+}

src/main/resources/reference.conf

@@ -208,6 +208,49 @@ aliases {
 loglevel = info
 
 
+// This sets the executable path of where a Chrome or Chromium installation is located JMusicBot can use.
+// JMusicBot may launch a web browser to get around anti-bot checks.
+// Set this to "AUTO" to make JMusicBot attempt to find a Chrome/Chromium installation itself.
+
+chromepath = "AUTO"
+
+
+// This sets the executable path of where a ChromeDriver installation is located JMusicBot can use.
+// Set this to "AUTO" to make JMusicBot automatically download the ChromeDriver version
+// which matches the installed Chrome version.
+
+chromedriverpath = "AUTO"
+
+
+// This sets whether JMusicBot should launch Chrome/Chromium instances without a GUI.
+// It is recommended to leave this set to true.
+// If set to false, JMusicBot will not close the browser automatically (even when finished)
+// and will automatically open DevTools for troubleshooting purposes.
+
+chromeheadless = true
+
+
+// This configures the routing planner to use for IP rotation on YouTube.
+// When set, you need to also set an IP block below.
+// The following routing strategies are available:
+// NONE = Disable
+// ROTATE_ON_BAN = Switch IP when currently used address gets banned.
+// LOAD_BALANCE = Selects random IP for each track playback.
+// NANO_SWITCH = Selects IPv6 address based on current nanosecond. Recommended for one /64 IPv6 blocks.
+// ROTATING_NANO_SWITCH = Same as above, but uses the next /64 IPv6 block in the list in case of a ban.
+
+ytroutingplanner = NONE
+
+
+// The IPv4 and IPv6 blocks to use when using the above routing planner.
+// This does nothing when ytroutingplanner is set to none.
+// The list accepts a list of IPv4 or IPv6 CIDR blocks (not a mix of them however)
+// IPv4 example: [ "192.0.2.183", "198.51.100.153/29" ]
+// IPv6 example: [ "2a02:1234:5678:9abc::/64" ]
+
+ytipblocks = [  ]
+
+
 // Transforms are used to modify specific play inputs and convert them to different kinds of inputs
 // These are quite complicated to use, and have limited use-cases, but in theory allow for rough
 // whitelists or blacklists, roundabout loading from some sources, and customization of how things are