Merge pull request anime-dl#2 from AbdullahM0hamed/configurator-2

Rewrite

README.md

@@ -63,6 +63,7 @@ Yeah. Me too! That's why this tool exists.
 - Watchmovie
 - Nyaa.si
 - Animedaisuki
+- Justdubs
 - twist.moe - requires Node.js
 - Kissanime - requires Node.js
 - Kisscartoon - requires Node.js

anime_downloader/commands/config.py

@@ -6,6 +6,37 @@
 import logging
 from tabulate import tabulate
 from anime_downloader.sites import ALL_ANIME_SITES
+from anime_downloader.config import Config
+from anime_downloader import config
+
+logger = logging.getLogger(__name__)
+sitenames = [*config.DEFAULT_CONFIG["siteconfig"]]
+
+data = Config._CONFIG
+
+def create_table(_list):
+    newList = [(x, y) for x, y in zip(range(1,len(_list) + 1), _list)]
+    table = tabulate(newList, tablefmt = "psql")
+    table = "\n".join(table.split("\n")[::-1])
+    return table
+
+def traverse_json(data):
+    click.clear()
+    keys = [*data.keys()]
+    click.echo(create_table(keys))
+    val = click.prompt("Select Option", type = int, default = 1) - 1
+
+    if type(data[keys[val]]) == dict:
+        traverse_json(data[keys[val]])
+    else:
+        click.echo(f"Current value: {data[keys[val]]}")
+        data[keys[val]] = click.prompt(f"Input new value for {keys[val]}", type = type(data[keys[val]]))
+        return data
+
+traverse_json(data)
+Config._CONFIG = data
+Config.write()
+=======
 sitenames = [v[1] for v in ALL_ANIME_SITES]
 def clear():    
     os.system('cls||clear')

anime_downloader/config.py

@@ -28,7 +28,7 @@
         'provider': 'twist.moe',
     },
     "siteconfig": {
-        "nineanime": {
+        'nineanime': {
             "server": "mp4upload",
         },
         'anistream.xyz': {
@@ -85,6 +85,9 @@
         },
         'vidstream': {
             "servers": ["vidstream","gcloud","mp4upload","cloud9","hydrax"]
+        },
+        'justdubs': {
+            "servers": ["mp4upload","gcloud"]
         }
     }
 }

anime_downloader/extractors/gcloud.py

@@ -10,6 +10,11 @@
 class Gcloud(BaseExtractor):
     def _get_data(self):
         url = self.url
+
+        """gcloud uses the same video ID as other sites"""
+        url = url.replace('fembed.com','gcloud.live')
+        url = url.replace('feurl.com','gcloud.live')
+
         url = url.replace('gcloud.live/v/','gcloud.live/api/source/')
         if url.find('#') != -1:url = url[:url.find('#')] 
         url = (url[-url[::-1].find('/'):])

anime_downloader/extractors/kwik.py

@@ -19,17 +19,24 @@ def _get_data(self):
         # Kwik servers don't have direct link access you need to be referred
         # from somewhere, I will just use the url itself. We then
         # have to rebuild the url. Hopefully kwik doesn't block this too
-        eval_re = re.compile(r';(eval.*\))')
-        stream_parts_re = re.compile(r'https:\/\/(.*?)\..*\/(\d+)\/(.*)\/.*token=(.*)&expires=([^\']+)')
+
+        #Necessary
+        self.url = self.url.replace(".cx/e/", ".cx/f/")
+
         title_re = re.compile(r'title>(.*)<')
 
-        kwik_text = helpers.get(self.url, referer=self.url).text
-        obsfucated_js = eval_re.search(kwik_text).group(1)
-        deobsfucated_js = util.deobfuscate_packed_js(obsfucated_js)
+        resp = helpers.get(self.url, headers={"referer": self.url})
+        kwik_text = resp.text
+        cookies = resp.cookies
 
         title = title_re.search(kwik_text).group(1)
-        cdn, digits, file, token, expires = stream_parts_re.search(deobsfucated_js).group(1, 2, 3, 4, 5)
-        stream_url = f'https://{cdn}.nextstream.org/get/{token}/{expires}/mp4/{digits}/{file}/{title}'
+        deobfuscated = helpers.soupify(util.deobfuscate_packed_js(re.search(r'<(script).*(var\s+_.*escape.*?)</\1>(?s)', kwik_text).group(2)))
+
+        post_url = deobfuscated.form["action"]
+        token = deobfuscated.input["value"]
+
+        resp = helpers.post(post_url, headers={"referer": self.url}, params={"_token": token}, cookies=cookies, allow_redirects = False)
+        stream_url = resp.headers["Location"]
 
         logger.debug('Stream URL: %s' % stream_url)
         return {

anime_downloader/extractors/vidstream.py

@@ -11,7 +11,7 @@
 
 class VidStream(BaseExtractor):
     def _get_data(self):
-        
+
         '''
         Config:
         List of servers. Will use servers in order. 
@@ -28,30 +28,34 @@ def _get_data(self):
 
         url = self.url.replace('https:////','https://')
         url = url.replace('https://vidstreaming.io/download','https://vidstreaming.io/server.php')
-
         soup = helpers.soupify(helpers.get(url))
-
         servers = Config._read_config()['siteconfig']['vidstream']['servers']
-        sources_regex = r'sources:(\[{.*?}])'
-        sources = re.search(sources_regex,str(soup))
 
         linkserver = soup.select('li.linkserver')
         for a in servers:
             if a == 'vidstream':
-                return self._get_link(sources)
+                return self._get_link(soup)
             for b in linkserver:
                 if b.get('data-video').startswith(links.get(a,'None')):
-                    self.url = b.get('data-video')
-                    return extractors.get_extractor(a)._get_data(self)
+                    """
+                    Another class needs to get created instead of using self not to impact future loops
+                    If the extractor fails vidstream.py will get run again with changed self
+                    """
+                    info = self.__dict__.copy()
+                    info['url'] = b.get('data-video')
+                    _self = Extractor(info)  
+                    return extractors.get_extractor(a)._get_data(_self)
 
-    def _get_link(self,sources):
+    def _get_link(self,soup):
         QUALITIES = {
             "360":[],
             "480":[],
             "720":[],
             "1080":[],
         }
-        sources = sources.group(1)
+
+        sources_regex = r'sources:(\[{.*?}])'
+        sources = re.search(sources_regex,str(soup)).group(1)
         sources = sources.replace("'",'"') #json only accepts ""
 
         regex = r"[{|,][\n]*?[ ]*?[\t]*?[A-z]*?[^\"]:"
@@ -75,3 +79,10 @@ def _get_link(self,sources):
             'stream_url': stream_url,
             'referer': self.url
         }
+
+
+"""dummy class to prevent changing self"""
+class Extractor: 
+    def __init__(self, dictionary):
+        for k, v in dictionary.items():
+            setattr(self, k, v)

anime_downloader/sites/animeflix.py

@@ -39,8 +39,11 @@ def _scrape_episodes(self):
             # find a way to pass some values within the class
             episodes = helpers.get(self.episodeList_url,
                                    params={'slug': self.slug}).json()
+
+            if episodes.get('@type','') == 'Movie': #different response if movies
+                return [episodes['potentialAction']['target']]
             return [ self.anime_url + episode['url'] for episode in episodes['episodes'] ]
-        
+
         def _scrape_metadata(self):
             self.slug = self.url.strip('/').split('/')[-1]
             meta = helpers.get(self.meta_url, 

anime_downloader/sites/animeout.py

@@ -36,6 +36,6 @@ class AnimeOutEpisode(AnimeEpisode, sitename='animeout'):
         def _get_sources(self):
             soup = helpers.soupify(helpers.get(self.url))
             link = soup.select('div.Center > p > h2 > a')[0].get('href')
-            script = helpers.soupify(helpers.get(link)).select('script')[2].text
-            url = re.search(r'http[^"]*',script).group()
+            script = helpers.soupify(helpers.get(link)).select('script')[2]
+            url = re.search(r'http[^"]*',str(script)).group()
             return [('no_extractor', url,)]

anime_downloader/sites/animepahe.py

@@ -39,7 +39,7 @@ def _get_sources(self):
         sources = []
 
         server_list = re.findall(r'data-provider="([^"]+)', source_text)
-        episode_id, session_id = re.search(r'getEmbeds\((\d+), "([^"]+)', source_text).groups()
+        episode_id, session_id = re.search("getUrls\((\d+?), \"(.*)?\"", source_text).groups()
 
         for server in server_list:
             if server not in supported_servers:

anime_downloader/sites/dreamanime.py

@@ -22,9 +22,8 @@ class DreamAnime(Anime, sitename='dreamanime'):
 
     @classmethod
     def search(cls, query):
-        results = helpers.get("https://dreamanime.fun/search", params = {"term" : query}).text
-        soup = helpers.soupify(results)
-        result_data = soup.find_all("a", {"id":"epilink"})
+        soup = helpers.soupify(helpers.get("https://dreamanime.fun/search", params = {"term" : query}))
+        result_data = soup.select("a#epilink")
 
         search_results = [
             SearchResult(
@@ -42,7 +41,7 @@ def _scrape_episodes(self):
 
         episodes = []
 
-        _all = soup.find_all("div", {"class":"episode-wrap"})
+        _all = soup.select("div.episode-wrap")
         for i in _all:
             ep_type = i.find("div", {"class":re.compile("ep-type type-.* dscd")}).text
             if ep_type == 'Sub':
@@ -70,8 +69,8 @@ def getLink(self, name, _id):
 
     def _get_sources(self):
         server = self.config.get("server", "trollvid")
-        soup = helpers.soupify(helpers.get(self.url))
-        hosts = json.loads(soup.find("div", {"class":"spatry"}).previous_sibling.previous_sibling.text[21:-2])["videos"]
+        resp = helpers.get(self.url).text
+        hosts = json.loads(re.search("var\s+episode\s+=\s+({.*})", resp).group(1))["videos"]
         _type = hosts[0]["type"]
         try:
             host = list(filter(lambda video: video["host"] == server and video["type"] == _type, hosts))[0]

anime_downloader/sites/dubbedanime.py

@@ -55,7 +55,12 @@ def _get_sources(self):
             api = json.loads(re.search(episode_regex,soup).group(1))
             slug = api['slug']
             sources = api['videos']
-            vidstream = helpers.get(f'https://vid.xngine.com/api/episode/{slug}',referer = self.url).json()
+
+            try: #Continues even if vidstream api fails
+                vidstream = helpers.get(f'https://vid.xngine.com/api/episode/{slug}',referer = self.url).json()
+            except:
+                vidstream = []
+
             for a in vidstream:
                 if a['host'] == 'vidstreaming' and 'id' in a and 'type' in a:
                     sources.append(a)
@@ -70,7 +75,7 @@ def _get_sources(self):
                                 provider = a[:]
                             embed = server_links.get(provider,'{}').format(b['id'],x)
                             return [(provider, embed,)]
-            
+
             logger.debug('No servers found in selected language. Trying all supported servers')
 
             for a in servers: #trying all supported servers in order
@@ -84,11 +89,11 @@ def _get_sources(self):
                         return [(provider, embed,)]
 
             logger.debug('No supported servers found, trying mp4sh')
+
             if re.search(r'"trollvid","id":"([^"]*)', soup):
                 token = re.search(r'"trollvid","id":"([^"]*)', soup).group(1)
                 embed = server_links.get('mp4sh','{}').format(token,x)
                 return [('mp4sh', embed,)]
             else:
                 logger.debug('No servers found')
                 return [('no_extractor', '',)]
-

anime_downloader/sites/erairaws.py

@@ -2,7 +2,7 @@
 from anime_downloader.sites.anime import Anime, AnimeEpisode, SearchResult
 from anime_downloader.sites import helpers
 from difflib import get_close_matches
-import base64
+import re
 
 class EraiRaws(Anime, sitename='erai-raws'):
     sitename='erai-raws'
@@ -11,11 +11,9 @@ class EraiRaws(Anime, sitename='erai-raws'):
     #Bypass DDosGuard
     def bypass(self):
         host = "https://erai-raws.info"
-        url = "https://erai-raws.info/anime-list/"
-        u = base64.b64encode(url.encode('utf-8'))
-        h = base64.b64encode(host.encode('utf-8'))
-        bypass_link = helpers.post('https://ddgu.ddos-guard.net/ddgu/', data = {'u':u, 'h':h, 'p':''}, headers = {'Referer': url}, allow_redirects = False).headers["Location"]
-        helpers.get(bypass_link, allow_redirects = False)
+        resp = helpers.get("https://check.ddos-guard.net/check.js").text
+        ddosBypassPath = re.search("'(.*?)'", resp).groups()[0]
+        helpers.get(host + ddosBypassPath)
 
     def parse(self, rows, url):
         episodes = []

anime_downloader/sites/init.py

@@ -22,7 +22,8 @@
     ('watchmovie','watchmovie','WatchMovie'),
     ('animekisa','animekisa','AnimeKisa'),
     ('nyaa','nyaa','Nyaa'),
-    ('animedaisuki','animedaisuki','Animedaisuki')
+    ('animedaisuki','animedaisuki','Animedaisuki'),
+    ('justdubs','justdubs','JustDubs')
 ]
 
 

anime_downloader/sites/justdubs.py

@@ -0,0 +1,63 @@
+import logging
+import json
+import re
+
+from anime_downloader.sites.exceptions import AnimeDLError, NotFoundError
+from anime_downloader import util
+from anime_downloader.sites.anime import Anime, AnimeEpisode, SearchResult
+from anime_downloader.sites import helpers
+
+logger = logging.getLogger(__name__)
+
+class JustDubs(Anime, sitename='justdubs'):
+    sitename = 'justdubs'
+    @classmethod
+    def search(cls, query):
+        results = helpers.get(f"http://justdubs.org/search/node/{query}").text
+        soup = helpers.soupify(results)
+        results_data = soup.select("li.search-result a[href*='http://justdubs.org/watch-']")
+        logger.debug(results_data)
+        search_results = [
+            SearchResult(
+                title = result.text,
+                url = result.get("href")
+            )
+            for result in results_data
+        ]
+        return search_results
+
+    def _scrape_episodes(self):
+        soup = helpers.soupify(helpers.get(self.url))
+        ret = [str(a['href'])
+                for a in soup.find_all('a', {'class' : 'list-group-item'})]
+        if ret == []:
+            err = 'No Episodes Found in url "{}"'.format(self.url)
+            args = [self.url]
+            raise NotFoundError(err, *args)
+        return list(reversed(ret))
+
+    def _scrape_metadata(self):
+        soup = helpers.soupify(helpers.get(self.url))
+        self.title = soup.select('h1.page-header')[0].text
+
+class JustDubsEpisode(AnimeEpisode, sitename='justdubs'):
+    def _get_sources(self):
+        servers = self.config['servers']
+
+        """maps urls to extractors"""
+        server_links =  { 
+        'mp4upload':'mp4upload.com',
+        'gcloud':'gcloud.live',
+        'gcloud':'fembed.com'
+        }
+
+        soup = helpers.soupify(helpers.get(self.url)).select('iframe')
+
+        for a in servers:
+            for b in soup:
+                for c in server_links:
+                    if server_links[c] in b.get('src') and a == c:
+                        return [(c, b.get('src'))]
+
+        logger.warn("Unsupported URL")
+        return ""

anime_downloader/sites/nyaa.py

@@ -13,8 +13,10 @@ def search(cls,query):
 
         search_results = [
             SearchResult(
-                title = i.select("a:not(.comments)")[1].get("title") + ' | '+ i.find_all('td',class_ = 'text-center')[1].text,
-                url = i.find_all('a',{'href':re.compile(rex)})[0].get('href'))
+                title = i.select("a:not(.comments)")[1].get("title"),
+                url = i.find_all('a',{'href':re.compile(rex)})[0].get('href'),
+                meta= {'peers':i.find_all('td',class_ = 'text-center')[3].text + ' peers','size':i.find_all('td',class_ = 'text-center')[1].text})
+
             for i in search_results.select("tr.default,tr.success")
             ]