From 6f2080b7f7afe3581097ecb9698ccc3f4f85576a Mon Sep 17 00:00:00 2001 From: Hafitz Setya <71178188+breakdowns@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:14:40 +0700 Subject: [PATCH] v4.6.9 (#66) - Merged mega-rest from lzzy - Fix /watch download msg progress - Change auto stop to 20 minute - Fix stop duplicate mirror (#65) - Fix tar name - Change tracker Co-authored-by: Shivam Jha Co-authored-by: Dev Singh Rajput Co-authored-by: FA Tulloh <77918734+yourtulloh@users.noreply.github.com> --- .gitignore | 3 +- Dockerfile | 23 +-- README.md | 10 +- app.json | 8 +- aria.sh | 6 +- bot/__init__.py | 54 +++--- bot/__main__.py | 2 + bot/helper/ext_utils/bot_utils.py | 19 +- bot/helper/ext_utils/fs_utils.py | 2 +- .../download_utils/aria2_download.py | 21 ++- .../download_utils/download_helper.py | 4 +- .../download_utils/mega_download.py | 108 +++++++++++ .../download_utils/mega_downloader.py | 172 ------------------ .../youtube_dl_download_helper.py | 4 +- ...mega_download_status.py => mega_status.py} | 66 +++---- bot/helper/telegram_helper/message_utils.py | 4 - bot/modules/mirror.py | 24 +-- config_sample.env | 5 +- .netrc => netrc | 0 requirements.txt | 1 + 20 files changed, 232 insertions(+), 304 deletions(-) create mode 100644 bot/helper/mirror_utils/download_utils/mega_download.py delete mode 100644 bot/helper/mirror_utils/download_utils/mega_downloader.py rename bot/helper/mirror_utils/status_utils/{mega_download_status.py => mega_status.py} (54%) rename .netrc => netrc (100%) diff --git a/.gitignore b/.gitignore index 86facd3c158..cf822f393a3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ data* *.pickle authorized_chats.txt log.txt -accounts/* \ No newline at end of file +accounts/* +venv/ diff --git a/Dockerfile b/Dockerfile index d18e7458571..6a381f413e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,29 @@ -FROM lzzy12/mega-sdk-python:latest +FROM ubuntu:20.04 WORKDIR /usr/src/app RUN chmod 777 /usr/src/app - RUN apt-get -qq update && \ - apt-get install -y software-properties-common && \ - rm -rf /var/lib/apt/lists/* && \ - apt-add-repository non-free && \ - apt-get -qq update && \ - apt-get -qq install -y p7zip-full p7zip-rar aria2 curl pv jq ffmpeg locales python3-lxml && \ - apt-get purge -y software-properties-common + DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y tzdata aria2 git python3 python3-pip \ + locales python3-lxml \ + curl pv jq ffmpeg \ + p7zip-full p7zip-rar \ + libcrypto++-dev libssl-dev \ + libc-ares-dev libcurl4-openssl-dev \ + libsqlite3-dev libsodium-dev && \ + curl -L https://github.com/jaskaranSM/megasdkrest/releases/download/v0.1/megasdkrest -o /usr/local/bin/megasdkrest && \ + chmod +x /usr/local/bin/megasdkrest COPY requirements.txt . COPY extract /usr/local/bin COPY pextract /usr/local/bin RUN chmod +x /usr/local/bin/extract && chmod +x /usr/local/bin/pextract RUN pip3 install --no-cache-dir -r requirements.txt -RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \ -locale-gen +RUN locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 COPY . . -COPY .netrc /root/.netrc +COPY netrc /root/.netrc RUN chmod +x aria.sh CMD ["bash","start.sh"] diff --git a/README.md b/README.md index 4d8fbb7ae53..6b69c48d987 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,11 @@ This is a telegram bot writen in python for mirroring files on the internet to o - Check Heroku dynos stats - Add stickers to your pack - Shell and Executor +- racaty.net support ## From Source Repos - Mirroring direct download links, Torrent, and Telegram files to Google Drive -- Mirroring Mega.nz links to Google Drive (In development stage) +- Mirroring Mega.nz links to google drive - Copy files from someone's drive to your drive (Using Autorclone) - Download/upload progress, speeds and ETAs - Mirror all youtube-dl supported links @@ -90,8 +91,8 @@ Fill up rest of the fields. Meaning of each fields are discussed below: - **INDEX_URL**: (Optional field) Refer to https://github.com/maple3142/GDIndex/ The URL should not have any trailing '/' - **API_KEY**: This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org DO NOT put this in quotes. - **API_HASH**: This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org -- **MEGA_API_KEY**: Mega.nz api key to mirror mega.nz links. Get it from [Mega SDK Page](https://mega.nz/sdk) -- **MEGA_EMAIL_ID**: Your email id you used to sign up on mega.nz for using premium accounts (Leave th) +- **MEGA_KEY**: Mega.nz api key to mirror mega.nz links. Get it from [Mega SDK Page](https://mega.nz/sdk) +- **MEGA_USERNAME**: Your email id you used to sign up on mega.nz for using premium accounts (Leave th) - **MEGA_PASSWORD**: Your password for your mega.nz account - **STOP_DUPLICATE_MIRROR**: (Optional field) (Leave empty if unsure) if this field is set to `True` , bot will check file in drive, if it is present in drive, downloading will ne stopped. (Note - File will be checked using filename, not using filehash, so this feature is not perfect yet) - **ENABLE_FILESIZE_LIMIT**: Set it to `True` if you want to use `MAX_TORRENT_SIZE`. @@ -247,7 +248,7 @@ python3 add_to_team_drive.py -d SharedTeamDriveSrcID ``` ## Youtube-dl authentication using .netrc file -For using your premium accounts in youtube-dl, edit the [.netrc](https://github.com/breakdowns/slam-mirrorbot/blob/master/.netrc) file according to following format: +For using your premium accounts in youtube-dl, edit the netrc file according to following format: ``` machine host login username password my_youtube_password ``` @@ -264,5 +265,6 @@ Thanks to: - [Dank-del](https://github.com/Dank-del/) for base repo - [magneto261290](https://github.com/magneto261290/) for some features - [SVR666](https://github.com/SVR666/) for some features & fixes +- [breakdowns](https://github.com/breakdowns/) me (lol) and many more people who aren't mentioned here, but may be found in [Contributors](https://github.com/breakdowns/slam-mirrorbot/graphs/contributors). diff --git a/app.json b/app.json index 3b206a5e21e..d57ed98efe1 100644 --- a/app.json +++ b/app.json @@ -78,11 +78,11 @@ "description": "Uptobox premium token to mirror uptobox links. Get it from https://uptobox.com/my_account.", "required": false }, - "MEGA_API_KEY": { + "MEGA_KEY": { "description": "Mega.nz api key to mirror mega.nz links. Get it from https://mega.nz/sdk.", "required": false }, - "MEGA_EMAIL_ID": { + "MEGA_USERNAME": { "description": "Your email id you used to sign up on mega.nz.", "required": false }, @@ -90,10 +90,6 @@ "description": "Your password for your mega.nz account.", "required": false }, - "BLOCK_MEGA_FOLDER": { - "description": "If you want to remove mega.nz folder support, set it to True.", - "required": false - }, "BLOCK_MEGA_LINKS": { "description": "If you want to remove mega.nz mirror support, set it to True.", "required": false diff --git a/aria.sh b/aria.sh index 580a4863c8f..27cc601c9c6 100755 --- a/aria.sh +++ b/aria.sh @@ -1,10 +1,10 @@ export MAX_DOWNLOAD_SPEED=0 -tracker_list=$(curl -Ns https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt | awk '$1' | tr '\n' ',') +tracker_list=$(curl -Ns https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/all.txt | awk '$1' | tr '\n' ',') export MAX_CONCURRENT_DOWNLOADS=7 -aria2c --enable-rpc --rpc-listen-all=false --rpc-listen-port 6800 --check-certificate=false\ +aria2c --enable-rpc --rpc-listen-all=false --rpc-listen-port 6800 --check-certificate=false \ --max-connection-per-server=10 --rpc-max-request-size=1024M \ - --bt-tracker="[$tracker_list]" --bt-max-peers=0 --bt-tracker-connect-timeout=300 --bt-stop-timeout=300 --seed-time=0.01 --min-split-size=10M \ + --bt-tracker="[$tracker_list]" --bt-max-peers=0 --bt-tracker-connect-timeout=300 --bt-stop-timeout=1200 --seed-time=0.01 --min-split-size=10M \ --follow-torrent=mem --split=10 \ --daemon=true --allow-overwrite=true --max-overall-download-limit=$MAX_DOWNLOAD_SPEED \ --max-overall-upload-limit=1K --max-concurrent-downloads=$MAX_CONCURRENT_DOWNLOADS \ diff --git a/bot/__init__.py b/bot/__init__.py index 134fb86db43..50e650987d4 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -10,6 +10,8 @@ from pyrogram import Client from telegraph import Telegraph import socket +from megasdkrestclient import MegaSdkRestClient, errors as mega_err +import subprocess socket.setdefaulttimeout(600) @@ -102,6 +104,32 @@ def getConfig(name: str): telegraph_token = telegraph.get_access_token() LOGGER.info("Telegraph Token Generated: '" + telegraph_token + "'") +try: + MEGA_KEY = getConfig('MEGA_KEY') + +except KeyError: + MEGA_KEY = None + LOGGER.info('MEGA API KEY NOT AVAILABLE') +if MEGA_KEY is not None: + try: + MEGA_USERNAME = getConfig('MEGA_USERNAME') + MEGA_PASSWORD = getConfig('MEGA_PASSWORD') + # Start megasdkrest binary + subprocess.Popen(["megasdkrest", "--apikey", MEGA_KEY]) + time.sleep(3) + mega_client = MegaSdkRestClient('http://localhost:6090') + try: + mega_client.login(MEGA_USERNAME, MEGA_PASSWORD) + except mega_err.MegaSdkRestClientException as e: + logging.error(e.message['message']) + exit(0) + except KeyError: + LOGGER.info("Mega API KEY provided but credentials not provided. Starting mega in anonymous mode!") + MEGA_USERNAME = None + MEGA_PASSWORD = None +else: + MEGA_USERNAME = None + MEGA_PASSWORD = None try: HEROKU_API_KEY = getConfig('HEROKU_API_KEY') except KeyError: @@ -124,25 +152,15 @@ def getConfig(name: str): ENABLE_FILESIZE_LIMIT = False except KeyError: ENABLE_FILESIZE_LIMIT = False +try: + IMAGE_URL = getConfig('IMAGE_URL') +except KeyError: + IMAGE_URL = None try: UPTOBOX_TOKEN = getConfig('UPTOBOX_TOKEN') except KeyError: logging.warning('UPTOBOX_TOKEN not provided!') UPTOBOX_TOKEN = None -try: - MEGA_API_KEY = getConfig('MEGA_API_KEY') -except KeyError: - logging.warning('MEGA API KEY not provided!') - MEGA_API_KEY = None -try: - MEGA_EMAIL_ID = getConfig('MEGA_EMAIL_ID') - MEGA_PASSWORD = getConfig('MEGA_PASSWORD') - if len(MEGA_EMAIL_ID) == 0 or len(MEGA_PASSWORD) == 0: - raise KeyError -except KeyError: - logging.warning('MEGA Credentials not provided!') - MEGA_EMAIL_ID = None - MEGA_PASSWORD = None try: INDEX_URL = getConfig('INDEX_URL') if len(INDEX_URL) == 0: @@ -197,14 +215,6 @@ def getConfig(name: str): USE_SERVICE_ACCOUNTS = False except KeyError: USE_SERVICE_ACCOUNTS = False -try: - BLOCK_MEGA_FOLDER = getConfig('BLOCK_MEGA_FOLDER') - if BLOCK_MEGA_FOLDER.lower() == 'true': - BLOCK_MEGA_FOLDER = True - else: - BLOCK_MEGA_FOLDER = False -except KeyError: - BLOCK_MEGA_FOLDER = False try: BLOCK_MEGA_LINKS = getConfig('BLOCK_MEGA_LINKS') if BLOCK_MEGA_LINKS.lower() == 'true': diff --git a/bot/__main__.py b/bot/__main__.py index 47cf0c32cb3..21337862b69 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -20,6 +20,7 @@ now=datetime.now(pytz.timezone('Asia/Jakarta')) + @run_async def stats(update, context): currentTime = get_readable_time((time.time() - botStartTime)) @@ -54,6 +55,7 @@ def start(update, context): ''' update.effective_message.reply_photo("https://telegra.ph/file/db03910496f06094f1f7a.jpg", start_string, parse_mode=ParseMode.MARKDOWN) + @run_async def chat_list(update, context): chatlist ='' diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index d75ba33fa26..e8f09982c88 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -64,7 +64,7 @@ def getDownloadByGid(gid): with download_dict_lock: for dl in download_dict.values(): status = dl.status() - if status != MirrorStatus.STATUS_UPLOADING and status != MirrorStatus.STATUS_ARCHIVING\ + if status != MirrorStatus.STATUS_UPLOADING and status != MirrorStatus.STATUS_ARCHIVING \ and status != MirrorStatus.STATUS_EXTRACTING: if dl.gid() == gid: return dl @@ -133,6 +133,10 @@ def get_readable_time(seconds: int) -> str: return result +def is_mega_link(url: str): + return "mega.nz" in url + + def is_url(url: str): url = re.findall(URL_REGEX, url) if url: @@ -140,24 +144,13 @@ def is_url(url: str): return False -def is_mega_link(url: str): - return "mega.nz" in url - -def get_mega_link_type(url: str): - if "folder" in url: - return "folder" - elif "file" in url: - return "file" - elif "/#F!" in url: - return "folder" - return "file" - def is_magnet(url: str): magnet = re.findall(MAGNET_REGEX, url) if magnet: return True return False + def new_thread(fn): """To use as decorator to make a function call threaded. Needs import diff --git a/bot/helper/ext_utils/fs_utils.py b/bot/helper/ext_utils/fs_utils.py index 52c10c385f5..af89947dc31 100644 --- a/bot/helper/ext_utils/fs_utils.py +++ b/bot/helper/ext_utils/fs_utils.py @@ -55,7 +55,7 @@ def tar(org_path): path = pathlib.PurePath(org_path) LOGGER.info(f'Tar: orig_path: {org_path}, tar_path: {tar_path}') tar = tarfile.open(tar_path, "w") - tar.add(org_path, arcname=path.name) + tar.add(org_path, arcname=os.path.basename(org_path)) tar.close() return tar_path diff --git a/bot/helper/mirror_utils/download_utils/aria2_download.py b/bot/helper/mirror_utils/download_utils/aria2_download.py index d2e66904aba..087dbe015ba 100644 --- a/bot/helper/mirror_utils/download_utils/aria2_download.py +++ b/bot/helper/mirror_utils/download_utils/aria2_download.py @@ -22,14 +22,6 @@ def __onDownloadStarted(self, api, gid): download = api.get_download(gid) self.name = download.name sname = download.name - size = download.total_length - if ENABLE_FILESIZE_LIMIT: - if size / 1024 / 1024 / 1024 > MAX_TORRENT_SIZE: - LOGGER.info(f" Download size Exceeded: {gid}") - dl.getListener().onDownloadError(f'File size {get_readable_file_size(size)} larger than Maximum Allowed size {MAX_TORRENT_SIZE}') - aria2.remove([download]) - return - update_all_messages() if STOP_DUPLICATE_MIRROR: if dl.getListener().isTar == True: sname = sname + ".tar" @@ -40,7 +32,14 @@ def __onDownloadStarted(self, api, gid): smsg, button = gdrive.drive_list(sname) if smsg: dl.getListener().onDownloadError(f'😡 File is already available in drive. You should have search before mirror any file. You might get ban if you do this again. This download has been stopped.\n\n') - sendMarkup(" Here are the search results:👇", dl.getListener().bot, dl.getListener().update, button) + sendMarkup("Here are the search results:👇", dl.getListener().bot, dl.getListener().update, button) + aria2.remove([download]) + + size = download.total_length + if ENABLE_FILESIZE_LIMIT: + if size / 1024 / 1024 / 1024 > MAX_TORRENT_SIZE: + LOGGER.info(f" Download size Exceeded: {gid}") + dl.getListener().onDownloadError(f'File size {get_readable_file_size(size)} larger than Maximum Allowed size {MAX_TORRENT_SIZE}GB') aria2.remove([download]) return update_all_messages() @@ -72,7 +71,9 @@ def __onDownloadPause(self, api, gid): def __onDownloadStopped(self, api, gid): LOGGER.info(f"onDownloadStop: {gid}") dl = getDownloadByGid(gid) - if dl: dl.getListener().onDownloadError('Dead torrent!') + dl = getDownloadByGid(gid) + if dl: + dl.getListener().onDownloadError('Dead Torrent!') @new_thread def __onDownloadError(self, api, gid): diff --git a/bot/helper/mirror_utils/download_utils/download_helper.py b/bot/helper/mirror_utils/download_utils/download_helper.py index ebcca1444a5..2edb6c54b41 100644 --- a/bot/helper/mirror_utils/download_utils/download_helper.py +++ b/bot/helper/mirror_utils/download_utils/download_helper.py @@ -9,8 +9,8 @@ def __init__(self): class DownloadHelper: def __init__(self): - self.name = '' # Name of the download; empty string if no download has been started - self.size = 0.0 # Size of the download + self._name = '' # Name of the download; empty string if no download has been started + self._size = 0.0 # Size of the download self.downloaded_bytes = 0.0 # Bytes downloaded self.speed = 0.0 # Download speed in bytes per second self.progress = 0.0 diff --git a/bot/helper/mirror_utils/download_utils/mega_download.py b/bot/helper/mirror_utils/download_utils/mega_download.py new file mode 100644 index 00000000000..559b2467503 --- /dev/null +++ b/bot/helper/mirror_utils/download_utils/mega_download.py @@ -0,0 +1,108 @@ +import threading +from bot import LOGGER, download_dict, download_dict_lock +from .download_helper import DownloadHelper +from ..status_utils.mega_status import MegaDownloadStatus +from megasdkrestclient import MegaSdkRestClient, constants +from bot.helper.ext_utils.bot_utils import setInterval +from pathlib import Path + + +class MegaDownloader: + POLLING_INTERVAL = 2 + + def __init__(self, listener): + super().__init__() + self.__listener = listener + self.__name = "" + self.__gid = '' + self.__resource_lock = threading.Lock() + self.__mega_client = MegaSdkRestClient('http://localhost:6090') + self.__periodic = None + self.__downloaded_bytes = 0 + self.__progress = 0 + self.__size = 0 + + @property + def progress(self): + with self.__resource_lock: + return self.__progress + + @property + def downloaded_bytes(self): + with self.__resource_lock: + return self.__downloaded_bytes + + @property + def size(self): + with self.__resource_lock: + return self.__size + + @property + def gid(self): + with self.__resource_lock: + return self.__gid + + @property + def name(self): + with self.__resource_lock: + return self.__name + + @property + def download_speed(self): + if self.gid is not None: + return self.__mega_client.getDownloadInfo(self.gid)['speed'] + + def __onDownloadStart(self, name, size, gid): + self.__periodic = setInterval(self.POLLING_INTERVAL, self.__onInterval) + with download_dict_lock: + download_dict[self.__listener.uid] = MegaDownloadStatus(self, self.__listener) + with self.__resource_lock: + self.__name = name + self.__size = size + self.__gid = gid + self.__listener.onDownloadStarted() + + def __onInterval(self): + dlInfo = self.__mega_client.getDownloadInfo(self.gid) + if (dlInfo['state'] == constants.State.TYPE_STATE_COMPLETED or dlInfo[ + 'state'] == constants.State.TYPE_STATE_CANCELED or dlInfo[ + 'state'] == constants.State.TYPE_STATE_FAILED) and self.__periodic is not None: + self.__periodic.cancel() + if dlInfo['state'] == constants.State.TYPE_STATE_COMPLETED: + self.__onDownloadComplete() + return + if dlInfo['state'] == constants.State.TYPE_STATE_CANCELED: + self.__onDownloadError('Cancelled by user') + return + if dlInfo['state'] == constants.State.TYPE_STATE_FAILED: + self.__onDownloadError(dlInfo['error_string']) + return + self.__onDownloadProgress(dlInfo['completed_length'], dlInfo['total_length']) + + def __onDownloadProgress(self, current, total): + with self.__resource_lock: + self.__downloaded_bytes = current + try: + self.__progress = current / total * 100 + except ZeroDivisionError: + self.__progress = 0 + + def __onDownloadError(self, error): + self.__listener.onDownloadError(error) + + def __onDownloadComplete(self): + self.__listener.onDownloadComplete() + + def add_download(self, link, path): + Path(path).mkdir(parents=True, exist_ok=True) + dl = self.__mega_client.addDl(link, path) + gid = dl['gid'] + info = self.__mega_client.getDownloadInfo(gid) + file_name = info['name'] + file_size = info['total_length'] + self.__onDownloadStart(file_name, file_size, gid) + LOGGER.info(f'Started mega download with gid: {gid}') + + def cancel_download(self): + LOGGER.info(f'Cancelling download on user request: {self.gid}') + self.__mega_client.cancelDl(self.gid) diff --git a/bot/helper/mirror_utils/download_utils/mega_downloader.py b/bot/helper/mirror_utils/download_utils/mega_downloader.py deleted file mode 100644 index e1f43a5b764..00000000000 --- a/bot/helper/mirror_utils/download_utils/mega_downloader.py +++ /dev/null @@ -1,172 +0,0 @@ -from bot import LOGGER, MEGA_API_KEY, download_dict_lock, download_dict, MEGA_EMAIL_ID, MEGA_PASSWORD -import threading -from mega import (MegaApi, MegaListener, MegaRequest, MegaTransfer, MegaError) -from bot.helper.telegram_helper.message_utils import update_all_messages -import os -from bot.helper.ext_utils.bot_utils import new_thread, get_mega_link_type -from bot.helper.mirror_utils.status_utils.mega_download_status import MegaDownloadStatus -import random -import string - -class MegaDownloaderException(Exception): - pass - - -class MegaAppListener(MegaListener): - _NO_EVENT_ON = (MegaRequest.TYPE_LOGIN,MegaRequest.TYPE_FETCH_NODES) - NO_ERROR = "no error" - - def __init__(self, continue_event: threading.Event, listener): - self.continue_event = continue_event - self.node = None - self.public_node = None - self.listener = listener - self.uid = listener.uid - self.__bytes_transferred = 0 - self.is_cancelled = False - self.__speed = 0 - self.__name = '' - self.__size = 0 - self.error = None - self.gid = "" - super(MegaAppListener, self).__init__() - - @property - def speed(self): - """Returns speed of the download in bytes/second""" - return self.__speed - - @property - def name(self): - """Returns name of the download""" - return self.__name - - def setValues(self, name, size, gid): - self.__name = name - self.__size = size - self.gid = gid - - @property - def size(self): - """Size of download in bytes""" - return self.__size - - @property - def downloaded_bytes(self): - return self.__bytes_transferred - - def onRequestStart(self, api, request): - LOGGER.info('Request start ({})'.format(request)) - - def onRequestFinish(self, api, request, error): - LOGGER.info('Mega Request finished ({}); Result: {}' - .format(request, error)) - if str(error).lower() != "no error": - self.error = error.copy() - return - request_type = request.getType() - if request_type == MegaRequest.TYPE_LOGIN: - api.fetchNodes() - elif request_type == MegaRequest.TYPE_GET_PUBLIC_NODE: - self.public_node = request.getPublicMegaNode() - elif request_type == MegaRequest.TYPE_FETCH_NODES: - LOGGER.info("Fetching Root Node.") - self.node = api.getRootNode() - LOGGER.info(f"Node Name: {self.node.getName()}") - if request_type not in self._NO_EVENT_ON or self.node and "cloud drive" not in self.node.getName().lower(): - self.continue_event.set() - - def onRequestTemporaryError(self, api, request, error: MegaError): - LOGGER.info(f'Mega Request error in {error}') - if not self.is_cancelled: - self.listener.onDownloadError("RequestTempError: " + error.toString()) - self.is_cancelled = True - self.error = error.toString() - self.continue_event.set() - - def onTransferStart(self, api: MegaApi, transfer: MegaTransfer): - LOGGER.info(f"Transfer Started: {transfer.getFileName()}") - - def onTransferUpdate(self, api: MegaApi, transfer: MegaTransfer): - if self.is_cancelled: - api.cancelTransfer(transfer, None) - self.__speed = transfer.getSpeed() - self.__bytes_transferred = transfer.getTransferredBytes() - - def onTransferFinish(self, api: MegaApi, transfer: MegaTransfer, error): - try: - LOGGER.info(f'Transfer finished ({transfer}); Result: {transfer.getFileName()}') - if transfer.isFolderTransfer() and transfer.isFinished() or transfer.getFileName() == self.name and not self.is_cancelled: - self.listener.onDownloadComplete() - self.continue_event.set() - except Exception as e: - LOGGER.error(e) - - def onTransferTemporaryError(self, api, transfer, error): - filen = transfer.getFileName() - state = transfer.getState() - errStr = error.toString() - LOGGER.info(f'Mega download error in file {transfer} {filen}: {error}') - - if state == 1 or state == 4: - # Sometimes MEGA (offical client) can't stream a node either and raises a temp failed error. - # Don't break the transfer queue if transfer's in queued (1) or retrying (4) state [causes seg fault] - return - - self.error = errStr - if not self.is_cancelled: - self.is_cancelled = True - self.listener.onDownloadError(f"TransferTempError: {errStr} ({filen})") - - def cancel_download(self): - self.is_cancelled = True - self.listener.onDownloadError("Download Canceled by user") - - -class AsyncExecutor: - - def __init__(self): - self.continue_event = threading.Event() - - def do(self, function, args): - self.continue_event.clear() - function(*args) - self.continue_event.wait() - -listeners = [] - -class MegaDownloadHelper: - def __init__(self): - pass - - @staticmethod - @new_thread - def add_download(mega_link: str, path: str, listener): - if MEGA_API_KEY is None: - raise MegaDownloaderException('Mega API KEY not provided! Cannot mirror mega links') - executor = AsyncExecutor() - api = MegaApi(MEGA_API_KEY, None, None, 'telegram-mirror-bot') - global listeners - mega_listener = MegaAppListener(executor.continue_event, listener) - listeners.append(mega_listener) - with download_dict_lock: - download_dict[listener.uid] = MegaDownloadStatus(mega_listener, listener) - os.makedirs(path) - api.addListener(mega_listener) - if MEGA_EMAIL_ID is not None and MEGA_PASSWORD is not None: - executor.do(api.login, (MEGA_EMAIL_ID, MEGA_PASSWORD)) - link_type = get_mega_link_type(mega_link) - if link_type == "file": - executor.do(api.getPublicNode, (mega_link,)) - node = mega_listener.public_node - else: - LOGGER.info("Logging into mega folder") - folder_api = MegaApi(MEGA_API_KEY,None,None,'TgBot') - folder_api.addListener(mega_listener) - executor.do(folder_api.loginToFolder, (mega_link,)) - node = folder_api.authorizeNode(mega_listener.node) - if mega_listener.error is not None: - return listener.onDownloadError(str(mega_listener.error)) - gid = ''.join(random.SystemRandom().choices(string.ascii_letters + string.digits, k=8)) - mega_listener.setValues(node.getName(), api.getSize(node), gid) - executor.do(api.startDownload,(node,path)) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index e4f87483421..d590f981f35 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -36,7 +36,7 @@ def error(msg): class YoutubeDLHelper(DownloadHelper): def __init__(self, listener): super().__init__() - self.__name = "" + self.name = "" self.__start_time = time.time() self.__listener = listener self.__gid = "" @@ -102,7 +102,7 @@ def onDownloadError(self, error): self.__listener.onDownloadError(error) def extractMetaData(self, link, qual, name): - if 'hotstar' or 'sonyliv' in link: + if "hotstar" or "sonyliv" in link: self.opts['geo_bypass_country'] = 'IN' with YoutubeDL(self.opts) as ydl: diff --git a/bot/helper/mirror_utils/status_utils/mega_download_status.py b/bot/helper/mirror_utils/status_utils/mega_status.py similarity index 54% rename from bot/helper/mirror_utils/status_utils/mega_download_status.py rename to bot/helper/mirror_utils/status_utils/mega_status.py index e4f5656794a..cfcd54f04b5 100644 --- a/bot/helper/mirror_utils/status_utils/mega_download_status.py +++ b/bot/helper/mirror_utils/status_utils/mega_status.py @@ -1,62 +1,56 @@ -from bot.helper.ext_utils.bot_utils import get_readable_file_size,MirrorStatus, get_readable_time from bot import DOWNLOAD_DIR +from bot.helper.ext_utils.bot_utils import MirrorStatus, get_readable_file_size, get_readable_time from .status import Status class MegaDownloadStatus(Status): - def __init__(self, obj, listener): - self.uid = obj.uid - self.listener = listener self.obj = obj + self.uid = listener.uid self.message = listener.message - def name(self) -> str: - return self.obj.name - - def progress_raw(self): - try: - return round(self.processed_bytes() / self.obj.size * 100,2) - except ZeroDivisionError: - return 0.0 - - def progress(self): - """Progress of download in percentage""" - return f"{self.progress_raw()}%" + def gid(self): + return self.obj.gid - def status(self) -> str: - return MirrorStatus.STATUS_DOWNLOADING + def path(self): + return f"{DOWNLOAD_DIR}{self.uid}" def processed_bytes(self): return self.obj.downloaded_bytes - def eta(self): - try: - seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw() - return f'{get_readable_time(seconds)}' - except ZeroDivisionError: - return '-' - def size_raw(self): return self.obj.size - def size(self) -> str: + def size(self): return get_readable_file_size(self.size_raw()) - def downloaded(self) -> str: - return get_readable_file_size(self.obj.downloadedBytes) + def status(self): + return MirrorStatus.STATUS_DOWNLOADING - def speed_raw(self): - return self.obj.speed + def name(self): + return self.obj.name - def speed(self) -> str: - return f'{get_readable_file_size(self.speed_raw())}/s' + def progress_raw(self): + return self.obj.progress - def gid(self) -> str: - return self.obj.gid + def progress(self): + return f'{round(self.progress_raw(), 2)}%' - def path(self) -> str: - return f"{DOWNLOAD_DIR}{self.uid}" + def speed_raw(self): + """ + :return: Download speed in Bytes/Seconds + """ + return self.obj.download_speed + + def speed(self): + return f'{get_readable_file_size(self.speed_raw())}/s' + + def eta(self): + try: + seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw() + return f'{get_readable_time(seconds)}' + except ZeroDivisionError: + return '-' def download(self): return self.obj diff --git a/bot/helper/telegram_helper/message_utils.py b/bot/helper/telegram_helper/message_utils.py index be0f6d57d46..9cbb40d0716 100644 --- a/bot/helper/telegram_helper/message_utils.py +++ b/bot/helper/telegram_helper/message_utils.py @@ -101,7 +101,6 @@ def update_all_messages(): status_reply_dict[chat_id].text = msg - def sendStatusMessage(msg, bot): progress = get_readable_message() progress += f"CPU: {psutil.cpu_percent()}%\n" \ @@ -134,8 +133,5 @@ def sendStatusMessage(msg, bot): except Exception as e: LOGGER.error(str(e)) del status_reply_dict[msg.message.chat.id] - pass - if len(progress) == 0: - progress = "Starting DL" message = sendMessage(progress, bot, msg) status_reply_dict[msg.message.chat.id] = message diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 9cc2120fdf9..fe24519ae59 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -2,13 +2,12 @@ from telegram.ext import CommandHandler, run_async from telegram import InlineKeyboardMarkup -from bot import Interval, INDEX_URL, BUTTON_THREE_NAME, BUTTON_THREE_URL, BUTTON_FOUR_NAME, BUTTON_FOUR_URL, BUTTON_FIVE_NAME, BUTTON_FIVE_URL, BLOCK_MEGA_LINKS, BLOCK_MEGA_FOLDER +from bot import Interval, INDEX_URL, LOGGER, MEGA_KEY, BUTTON_THREE_NAME, BUTTON_THREE_URL, BUTTON_FOUR_NAME, BUTTON_FOUR_URL, BUTTON_FIVE_NAME, BUTTON_FIVE_URL, BLOCK_MEGA_LINKS from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, download_dict, download_dict_lock, SHORTENER, SHORTENER_API from bot.helper.ext_utils import fs_utils, bot_utils -from bot.helper.ext_utils.bot_utils import setInterval, get_mega_link_type +from bot.helper.ext_utils.bot_utils import setInterval from bot.helper.ext_utils.exceptions import DirectDownloadLinkException, NotSupportedExtractionArchive from bot.helper.mirror_utils.download_utils.aria2_download import AriaDownloadHelper -from bot.helper.mirror_utils.download_utils.mega_downloader import MegaDownloadHelper from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator from bot.helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper from bot.helper.mirror_utils.status_utils import listeners @@ -20,6 +19,7 @@ from bot.helper.telegram_helper.filters import CustomFilters from bot.helper.telegram_helper.message_utils import * from bot.helper.telegram_helper import button_build +from bot.helper.mirror_utils.download_utils.mega_download import MegaDownloader import urllib import pathlib import os @@ -269,18 +269,14 @@ def _mirror(bot, update, isTar=False, extract=False): except DirectDownloadLinkException as e: LOGGER.info(f'{link}: {e}') listener = MirrorListener(bot, update, pswd, isTar, tag, extract) - if bot_utils.is_mega_link(link): - link_type = get_mega_link_type(link) - if link_type == "folder" and BLOCK_MEGA_FOLDER: - sendMessage("Mega folder are blocked!", bot, update) - elif BLOCK_MEGA_LINKS: - sendMessage("Mega links are blocked bcoz mega downloading is too much unstable and buggy. mega support will be added back after fix", bot, update) - else: - mega_dl = MegaDownloadHelper() - mega_dl.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener) - sendStatusMessage(update, bot) + if bot_utils.is_mega_link(link) and MEGA_KEY is not None and not BLOCK_MEGA_LINKS: + mega_dl = MegaDownloader(listener) + mega_dl.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}/') + sendStatusMessage(update, bot) + elif bot_utils.is_mega_link(link) and BLOCK_MEGA_LINKS: + sendMessage("Mega links are blocked. Dont try to mirror mega links.", bot, update) else: - ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener, name) + ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}/', listener, name) sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) diff --git a/config_sample.env b/config_sample.env index 299e117c865..404a3e49102 100644 --- a/config_sample.env +++ b/config_sample.env @@ -16,10 +16,9 @@ AUTHORIZED_CHATS = "" USE_SERVICE_ACCOUNTS = "" INDEX_URL = "" UPTOBOX_TOKEN = "" -MEGA_API_KEY = "" -MEGA_EMAIL_ID = "" +MEGA_KEY = "" +MEGA_USERNAME = "" MEGA_PASSWORD = "" -BLOCK_MEGA_FOLDER = "" BLOCK_MEGA_LINKS = "" STOP_DUPLICATE_MIRROR = "" SHORTENER = "" diff --git a/.netrc b/netrc similarity index 100% rename from .netrc rename to netrc diff --git a/requirements.txt b/requirements.txt index f105d2dbbeb..c49715717bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,4 @@ js2py lxml telegraph pytz +megasdkrestclient>=0.1.1,<1.0.0