From 107c79a5600d2c95094d1551fd49fc8f936e6fc2 Mon Sep 17 00:00:00 2001 From: Mahesh0253 Date: Fri, 1 Jan 2021 22:22:07 +0530 Subject: [PATCH 01/21] Updated to pyrogram v1.1.7 --- requirements.txt | 2 +- utils.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5f0a6a2e1..3ca23d975 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -https://github.com/Mahesh0253/pyrogram/archive/beta.zip +https://github.com/Mahesh0253/pyrogram/archive/inline.zip tgcrypto umongo motor diff --git a/utils.py b/utils.py index 8d8630573..730cb99f7 100644 --- a/utils.py +++ b/utils.py @@ -1,9 +1,14 @@ import re +import base64 import logging +from struct import pack + +from pyrogram.file_id import FileId from pymongo.errors import DuplicateKeyError from umongo import Instance, Document, fields from motor.motor_asyncio import AsyncIOMotorClient from marshmallow.exceptions import ValidationError + from info import DATABASE_URI, DATABASE_NAME, COLLECTION_NAME, USE_CAPTION_FILTER logger = logging.getLogger(__name__) @@ -31,10 +36,13 @@ class Meta: async def save_file(media): """Save file in database""" + # TODO: Find better way to get same file_id for same media to avoid duplicates + file_id, file_ref = unpack_new_file_id(media.file_id) + try: file = Media( - file_id=media.file_id, - file_ref=media.file_ref, + file_id=file_id, + file_ref=file_ref, file_name=media.file_name, file_size=media.file_size, file_type=media.file_type, @@ -90,3 +98,40 @@ async def get_search_results(query, file_type=None, max_results=10, offset=0): files = await cursor.to_list(length=max_results) return files, next_offset + + +def encode_file_id(s: bytes) -> str: + r = b"" + n = 0 + + for i in s + bytes([22]) + bytes([4]): + if i == 0: + n += 1 + else: + if n: + r += b"\x00" + bytes([n]) + n = 0 + + r += bytes([i]) + + return base64.urlsafe_b64encode(r).decode().rstrip("=") + + +def encode_file_ref(file_ref: bytes) -> str: + return base64.urlsafe_b64encode(file_ref).decode().rstrip("=") + + +def unpack_new_file_id(new_file_id): + """Return file_id, file_ref""" + decoded = FileId.decode(new_file_id) + file_id = encode_file_id( + pack( + " Date: Thu, 14 Jan 2021 13:58:47 +0530 Subject: [PATCH 02/21] added AUTH_CHANNEL --- .gitignore | 2 +- README.md | 2 ++ app.json | 10 ++++++++++ info.py | 11 ++++++++--- plugins/commands.py | 17 ++++++++++------- plugins/inline.py | 29 ++++++++++++++++++++++++++++- sample_info.py | 4 +++- 7 files changed, 62 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 327eae423..306cd6985 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ *.session *.session-journal .vscode -*test.py +*test*.py setup.cfg # Byte-compiled / optimized / DLL files diff --git a/README.md b/README.md index 7b454dcd6..19f8002b5 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ Check `sample_info.py` before editing `info.py` file * `CACHE_TIME`: The maximum amount of time in seconds that the result of the inline query may be cached on the server * `USE_CAPTION_FILTER`: Whether bot should use captions to improve search results. (True/False) * `AUTH_USERS`: Username or ID of users to give access of inline search. Separate multiple users by space. Leave it empty if you don't want to restrict bot usage. +* `AUTH_CHANNEL`: Username or ID of channel. Without subscribing this channel users cannot use bot. +* `INVITE_MSG`: Auth channel invitation message. ### Admin commands ``` diff --git a/app.json b/app.json index efe135154..8cb00a9da 100644 --- a/app.json +++ b/app.json @@ -40,6 +40,16 @@ "value": "", "required": false }, + "AUTH_CHANNEL": { + "description": "Username or ID of channel. Without subscribing this channel users cannot use bot.", + "value": "", + "required": false + }, + "INVITE_MSG": { + "description": "Auth channel invitation message", + "value": "'Please join @.... to use this bot'", + "required": false + }, "USE_CAPTION_FILTER": { "description": "Whether bot should use captions to improve search results. (True False)", "value": "False", diff --git a/info.py b/info.py index 232aaa6db..db4326da6 100644 --- a/info.py +++ b/info.py @@ -1,6 +1,8 @@ import re from os import environ +id_pattern = re.compile(r'^.\d+$') + # Bot information SESSION = environ.get('SESSION', 'Media_search') USER_SESSION = environ.get('USER_SESSION', 'User_Bot') @@ -14,10 +16,12 @@ USE_CAPTION_FILTER = bool(environ.get('USE_CAPTION_FILTER', False)) # Admins, Channels & Users -ADMINS = [int(admin) if re.search('^\d+$', admin) else admin for admin in environ['ADMINS'].split()] -CHANNELS = [int(ch) if re.search('^.\d+$', ch) else ch for ch in environ['CHANNELS'].split()] -auth_users = [int(user) if re.search('^\d+$', user) else user for user in environ['AUTH_USERS'].split()] +ADMINS = [int(admin) if id_pattern.search(admin) else admin for admin in environ['ADMINS'].split()] +CHANNELS = [int(ch) if id_pattern.search(ch) else ch for ch in environ['CHANNELS'].split()] +auth_users = [int(user) if id_pattern.search(user) else user for user in environ['AUTH_USERS'].split()] AUTH_USERS = (auth_users + ADMINS) if auth_users else [] +auth_channel = environ.get('AUTH_CHANNEL') +AUTH_CHANNEL = int(auth_channel) if auth_channel and id_pattern.search(auth_channel) else auth_channel # MongoDB information DATABASE_URI = environ['DATABASE_URI'] @@ -32,3 +36,4 @@ """ SHARE_BUTTON_TEXT = 'Checkout {username} for searching files' +INVITE_MSG = environ.get('INVITE_MSG', 'Please join @.... to use this bot') \ No newline at end of file diff --git a/plugins/commands.py b/plugins/commands.py index dc7b85691..43dee8358 100644 --- a/plugins/commands.py +++ b/plugins/commands.py @@ -2,7 +2,7 @@ import logging from pyrogram import Client, filters from pyrogram.types import InlineKeyboardButton, InlineKeyboardMarkup -from info import START_MSG, CHANNELS, ADMINS +from info import START_MSG, CHANNELS, ADMINS, INVITE_MSG from utils import Media logger = logging.getLogger(__name__) @@ -11,12 +11,15 @@ @Client.on_message(filters.command('start')) async def start(bot, message): """Start command handler""" - buttons = [[ - InlineKeyboardButton('Search Here', switch_inline_query_current_chat=''), - InlineKeyboardButton('Go Inline', switch_inline_query=''), - ]] - reply_markup = InlineKeyboardMarkup(buttons) - await message.reply(START_MSG, reply_markup=reply_markup) + if len(message.command) > 1 and message.command[1] == 'subscribe': + await message.reply(INVITE_MSG) + else: + buttons = [[ + InlineKeyboardButton('Search Here', switch_inline_query_current_chat=''), + InlineKeyboardButton('Go Inline', switch_inline_query=''), + ]] + reply_markup = InlineKeyboardMarkup(buttons) + await message.reply(START_MSG, reply_markup=reply_markup) @Client.on_message(filters.command('channel') & filters.user(ADMINS)) diff --git a/plugins/inline.py b/plugins/inline.py index 6d2c5429a..fea074589 100644 --- a/plugins/inline.py +++ b/plugins/inline.py @@ -1,14 +1,27 @@ +import logging from urllib.parse import quote + from pyrogram import Client, emoji, filters +from pyrogram.errors import UserNotParticipant from pyrogram.types import InlineKeyboardButton, InlineKeyboardMarkup, InlineQueryResultCachedDocument + from utils import get_search_results -from info import MAX_RESULTS, CACHE_TIME, SHARE_BUTTON_TEXT, AUTH_USERS +from info import MAX_RESULTS, CACHE_TIME, SHARE_BUTTON_TEXT, AUTH_USERS, AUTH_CHANNEL + +logger = logging.getLogger(__name__) @Client.on_inline_query(filters.user(AUTH_USERS) if AUTH_USERS else None) async def answer(bot, query): """Show search results for given inline query""" + if AUTH_CHANNEL and not await is_subscribed(bot, query): + await query.answer(results=[], + cache_time=0, + switch_pm_text='You have to subscribe channel', + switch_pm_parameter="subscribe") + return + results = [] if '|' in query.query: string, file_type = query.query.split('|', maxsplit=1) @@ -75,3 +88,17 @@ def get_size(size): i += 1 size /= 1024.0 return "%.2f %s" % (size, units[i]) + + +async def is_subscribed(bot, query): + try: + user = await bot.get_chat_member(AUTH_CHANNEL, query.from_user.id) + except UserNotParticipant: + pass + except Exception as e: + logger.exception(e) + else: + if not user.status == 'kicked': + return True + + return False \ No newline at end of file diff --git a/sample_info.py b/sample_info.py index f26b8f09c..afca3c11c 100644 --- a/sample_info.py +++ b/sample_info.py @@ -14,6 +14,7 @@ ADMINS = [12345789, 'admin123', 98765432] CHANNELS = [-10012345678, -100987654321, 'channelusername'] AUTH_USERS = [] +AUTH_CHANNEL = None # MongoDB information DATABASE_URI = "mongodb://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb]?retryWrites=true&w=majority" @@ -27,4 +28,5 @@ Here you can search files in inline mode. Just press follwing buttons and start searching. """ -SHARE_BUTTON_TEXT = 'Checkout {username} for searching files' \ No newline at end of file +SHARE_BUTTON_TEXT = 'Checkout {username} for searching files' +INVITE_MSG = 'Please join @.... to use this bot' \ No newline at end of file From 7aadce482e13e7c291a3fa7e681588c59b41e48f Mon Sep 17 00:00:00 2001 From: Mahesh0253 Date: Thu, 14 Jan 2021 14:18:47 +0530 Subject: [PATCH 03/21] updated channel command --- plugins/commands.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/plugins/commands.py b/plugins/commands.py index 43dee8358..4f7fa977a 100644 --- a/plugins/commands.py +++ b/plugins/commands.py @@ -32,17 +32,22 @@ async def channel_info(bot, message): else: raise ValueError("Unexpected type of CHANNELS") + text = '📑 **Indexed channels/groups**\n' for channel in channels: - channel_info = await bot.get_chat(channel) - string = str(channel_info) - if len(string) > 4096: - filename = (channel_info.title or channel_info.first_name) + ".txt" - with open(filename, 'w') as f: - f.write(string) - await message.reply_document(filename) - os.remove(filename) + chat = await bot.get_chat(channel) + if chat.username: + text += '\n@' + chat.username else: - await message.reply(str(channel_info)) + text += '\n' + chat.title or chat.first_name + + if len(text) < 4096: + await message.reply(text) + else: + file = 'Indexed channels.txt' + with open(file, 'w') as f: + f.write(text) + await message.reply_document(file) + os.remove(file) @Client.on_message(filters.command('total') & filters.user(ADMINS)) From b5ce309f908c04718c9999233574216dda3ff4ad Mon Sep 17 00:00:00 2001 From: Mahesh0253 Date: Thu, 14 Jan 2021 14:29:16 +0530 Subject: [PATCH 04/21] small change --- plugins/commands.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/commands.py b/plugins/commands.py index 4f7fa977a..f55e4c712 100644 --- a/plugins/commands.py +++ b/plugins/commands.py @@ -40,6 +40,8 @@ async def channel_info(bot, message): else: text += '\n' + chat.title or chat.first_name + text += f'\n\n**Total:** {len(CHANNELS)}' + if len(text) < 4096: await message.reply(text) else: From 658db17849c9eb6aa6a6fb19a5c1587568478efe Mon Sep 17 00:00:00 2001 From: Mahesh <44301650+Mahesh0253@users.noreply.github.com> Date: Thu, 14 Jan 2021 15:28:25 +0530 Subject: [PATCH 05/21] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3ca23d975..74d815bd0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ https://github.com/Mahesh0253/pyrogram/archive/inline.zip tgcrypto -umongo +umongo==2.3.0 motor dnspython From 53cc0d90f066f18703cf88cd9d487590d3bfe838 Mon Sep 17 00:00:00 2001 From: Mahesh <44301650+Mahesh0253@users.noreply.github.com> Date: Thu, 14 Jan 2021 15:30:23 +0530 Subject: [PATCH 06/21] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 74d815bd0..aa2cef5c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ https://github.com/Mahesh0253/pyrogram/archive/inline.zip tgcrypto umongo==2.3.0 -motor +motor==2.3.0 dnspython From 7a79054ce59717a9137657716b34b5b5f4b30c00 Mon Sep 17 00:00:00 2001 From: Mahesh0253 Date: Thu, 14 Jan 2021 15:38:21 +0530 Subject: [PATCH 07/21] small fix --- info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/info.py b/info.py index db4326da6..af0b25182 100644 --- a/info.py +++ b/info.py @@ -18,7 +18,7 @@ # Admins, Channels & Users ADMINS = [int(admin) if id_pattern.search(admin) else admin for admin in environ['ADMINS'].split()] CHANNELS = [int(ch) if id_pattern.search(ch) else ch for ch in environ['CHANNELS'].split()] -auth_users = [int(user) if id_pattern.search(user) else user for user in environ['AUTH_USERS'].split()] +auth_users = [int(user) if id_pattern.search(user) else user for user in environ.get('AUTH_USERS', '').split()] AUTH_USERS = (auth_users + ADMINS) if auth_users else [] auth_channel = environ.get('AUTH_CHANNEL') AUTH_CHANNEL = int(auth_channel) if auth_channel and id_pattern.search(auth_channel) else auth_channel From 359f3b28a19f52b9385b89d0cd4ed9094ee9c0c5 Mon Sep 17 00:00:00 2001 From: Mahesh0253 Date: Fri, 15 Jan 2021 19:43:25 +0530 Subject: [PATCH 08/21] updated --- README.md | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 19f8002b5..02a4ef9dc 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,36 @@ -## [Media Search bot](https://github.com/Mahesh0253/Media-Search-bot) +# [Media Search bot](https://github.com/Mahesh0253/Media-Search-bot) -* Index channel/group files for inline search. -* When you going to post file on telegram channel/group this bot will save that in database, So you can search that easily in inline mode. -* Supports document, video and audio file formats with caption. +* Index channel or group files for inline search. +* When you post file on telegram channel or group this bot will save that file in database, so you can search easily in inline mode. +* Supports document, video and audio file formats with caption support. -### Installation +## Installation -#### Easy Way +### Easy Way [![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy) -#### Watch this video to create bot - https://youtu.be/dsuTn4qV2GA +### Watch this video to create bot - https://youtu.be/dsuTn4qV2GA -#### Hard Way +### Hard Way ```sh python3 -m venv env -. ./env/bin/activate + +# Activate virtual environment +tutorial-env\Scripts\activate.bat # For Windows +source env/bin/activate # For Linux or MacOS + +# Install Packages pip3 install -r requirements.txt + # Edit info.py with variables as given below python3 bot.py ``` Check `sample_info.py` before editing `info.py` file -#### Variables +## Variables -##### Required Variables +### Required Variables * `BOT_TOKEN`: Create a bot using [@BotFather](https://telegram.dog/BotFather), and get the Telegram API token. * `API_ID`: Get this value from [telegram.org](https://my.telegram.org/apps) * `API_HASH`: Get this value from [telegram.org](https://my.telegram.org/apps) @@ -33,7 +39,7 @@ Check `sample_info.py` before editing `info.py` file * `DATABASE_URI`: [mongoDB](https://www.mongodb.com) URI. Get this value from [mongoDB](https://www.mongodb.com). For more help watch this [video](https://youtu.be/dsuTn4qV2GA) * `DATABASE_NAME`: Name of the database in [mongoDB](https://www.mongodb.com). For more help watch this [video](https://youtu.be/dsuTn4qV2GA) -##### Optional Variables +### Optional Variables * `COLLECTION_NAME`: Name of the collections. Defaults to Telegram_files. If you going to use same database, then use different collection name for each bot * `MAX_RESULTS`: Maximum limit for inline search results * `CACHE_TIME`: The maximum amount of time in seconds that the result of the inline query may be cached on the server @@ -42,7 +48,7 @@ Check `sample_info.py` before editing `info.py` file * `AUTH_CHANNEL`: Username or ID of channel. Without subscribing this channel users cannot use bot. * `INVITE_MSG`: Auth channel invitation message. -### Admin commands +## Admin commands ``` channel - Get basic infomation about channels total - Show total of saved files @@ -50,15 +56,18 @@ delete - Delete file from database logger - Get log file ``` -### Tips +## Tips * Run [one_time_indexer.py](one_time_indexer.py) file to save old files in the database that are not indexed yet. * You can use `|` to separate query and file type while searching for specific type of file. For example: `Avengers | video` * If you don't want to create a channel or group, use your chat ID / username as the channel ID. When you send a file to a bot, it will be saved in the database. -### Contributions -Contributions are welcome. +## Contributions +> Contributions are welcome. + +## Thanks to [Pyrogram](https://github.com/pyrogram/pyrogram) -### Thanks to [Pyrogram](https://github.com/pyrogram/pyrogram) +## Support +> [Update Channel](https://t.me/botxupdates) and [Support Group](https://t.me/botxsupport) -### License +## License Code released under [The GNU General Public License](LICENSE). From 2376a9645fa8a0c5c7f1dec1dd4a91a25ff3a82a Mon Sep 17 00:00:00 2001 From: Mahesh0253 Date: Fri, 15 Jan 2021 19:47:55 +0530 Subject: [PATCH 09/21] small change --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 02a4ef9dc..fd7bacd0a 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,15 @@ ## Installation +### Watch this video to create bot - https://youtu.be/dsuTn4qV2GA ### Easy Way [![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy) -### Watch this video to create bot - https://youtu.be/dsuTn4qV2GA ### Hard Way ```sh +# Create virtual environment python3 -m venv env # Activate virtual environment @@ -26,7 +27,7 @@ pip3 install -r requirements.txt # Edit info.py with variables as given below python3 bot.py ``` -Check `sample_info.py` before editing `info.py` file +Check [`sample_info.py`](sample_info.py) before editing [`info.py`](info.py) file ## Variables @@ -62,12 +63,12 @@ logger - Get log file * If you don't want to create a channel or group, use your chat ID / username as the channel ID. When you send a file to a bot, it will be saved in the database. ## Contributions -> Contributions are welcome. +Contributions are welcome. ## Thanks to [Pyrogram](https://github.com/pyrogram/pyrogram) ## Support -> [Update Channel](https://t.me/botxupdates) and [Support Group](https://t.me/botxsupport) +[Update Channel](https://t.me/botxupdates) and [Support Group](https://t.me/botxsupport) ## License Code released under [The GNU General Public License](LICENSE). From fa02b001547b377dc9ef6a583c645065470dd56a Mon Sep 17 00:00:00 2001 From: Mahesh <44301650+Mahesh0253@users.noreply.github.com> Date: Sat, 16 Jan 2021 11:38:07 +0530 Subject: [PATCH 10/21] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fd7bacd0a..7d3142afa 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ python3 -m venv env # Activate virtual environment -tutorial-env\Scripts\activate.bat # For Windows +env\Scripts\activate.bat # For Windows source env/bin/activate # For Linux or MacOS # Install Packages From ea0adf310d36b8abea8d58fd0c73418836eb3730 Mon Sep 17 00:00:00 2001 From: Mahesh <44301650+Mahesh0253@users.noreply.github.com> Date: Sat, 16 Jan 2021 11:40:05 +0530 Subject: [PATCH 11/21] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d3142afa..20f045ba8 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ source env/bin/activate # For Linux or MacOS # Install Packages pip3 install -r requirements.txt -# Edit info.py with variables as given below +# Edit info.py with variables as given below then run bot python3 bot.py ``` Check [`sample_info.py`](sample_info.py) before editing [`info.py`](info.py) file From e5398e76b3d05421c2741212b8d6e5bd84fc2f4e Mon Sep 17 00:00:00 2001 From: Mahesh0253 Date: Fri, 5 Mar 2021 11:51:15 +0530 Subject: [PATCH 12/21] added index command --- README.md | 3 ++- app.json | 5 +++++ info.py | 1 + plugins/userbot.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++ sample_info.py | 1 + 5 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 plugins/userbot.py diff --git a/README.md b/README.md index 20f045ba8..c51a03f11 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ ### Hard Way -```sh +```bash # Create virtual environment python3 -m venv env @@ -48,6 +48,7 @@ Check [`sample_info.py`](sample_info.py) before editing [`info.py`](info.py) fil * `AUTH_USERS`: Username or ID of users to give access of inline search. Separate multiple users by space. Leave it empty if you don't want to restrict bot usage. * `AUTH_CHANNEL`: Username or ID of channel. Without subscribing this channel users cannot use bot. * `INVITE_MSG`: Auth channel invitation message. +* `USERBOT_STRING_SESSION`: User bot string session. ## Admin commands ``` diff --git a/app.json b/app.json index 8cb00a9da..ea4aa4254 100644 --- a/app.json +++ b/app.json @@ -19,6 +19,11 @@ "description": "Your bot token.", "value": "" }, + "USERBOT_STRING_SESSION": { + "description": "User bot string session.", + "value": "", + "required": false + }, "API_ID": { "description": "Get this value from https://my.telegram.org", "value": "" diff --git a/info.py b/info.py index af0b25182..79f9ce0e1 100644 --- a/info.py +++ b/info.py @@ -9,6 +9,7 @@ API_ID = int(environ['API_ID']) API_HASH = environ['API_HASH'] BOT_TOKEN = environ['BOT_TOKEN'] +USERBOT_STRING_SESSION = environ.get('USERBOT_STRING_SESSION') # Bot settings MAX_RESULTS = int(environ.get('MAX_RESULTS', 10)) diff --git a/plugins/userbot.py b/plugins/userbot.py new file mode 100644 index 000000000..7e939c7d2 --- /dev/null +++ b/plugins/userbot.py @@ -0,0 +1,53 @@ +import logging +import asyncio +from pyrogram import Client, filters +from info import USERBOT_STRING_SESSION, API_ID, API_HASH, ADMINS, id_pattern +from utils import save_file + +logger = logging.getLogger(__name__) +lock = asyncio.Lock() + + +@Client.on_message(filters.command(['index', 'indexfiles']) & filters.user(ADMINS)) +async def index_files(bot, message): + """Save channel or group files with the help of user bot""" + + if not USERBOT_STRING_SESSION: + await message.reply('Set `USERBOT_STRING_SESSION` in info.py file or in environment variables.') + elif len(message.command) == 1: + await message.reply('Please specify channel username or id in command.\n\n' + 'Example: `/index -10012345678`') + elif lock.locked(): + await message.reply('Wait until previous process complete.') + else: + msg = await message.reply('Processing...⏳') + raw_data = message.command[1:] + user_bot = Client(USERBOT_STRING_SESSION, API_ID, API_HASH) + chats = [int(chat) if id_pattern.search(chat) else chat for chat in raw_data] + total_files = 0 + + async with lock: + try: + async with user_bot: + for chat in chats: + async for user_message in user_bot.iter_history(chat): + message = await bot.get_messages( + chat, + user_message.message_id, + replies=0, + ) + for file_type in ("document", "video", "audio"): + media = getattr(message, file_type, None) + if media is not None: + break + else: + continue + media.file_type = file_type + media.caption = message.caption + await save_file(media) + total_files += 1 + except Exception as e: + logger.exception(e) + await msg.edit(f'Error: {e}') + else: + await msg.edit(f'Total {total_files} checked!') \ No newline at end of file diff --git a/sample_info.py b/sample_info.py index afca3c11c..071bfc32f 100644 --- a/sample_info.py +++ b/sample_info.py @@ -4,6 +4,7 @@ API_ID = 12345 API_HASH = '0123456789abcdef0123456789abcdef' BOT_TOKEN = '123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11' +USERBOT_STRING_SESSION = '' # Bot settings MAX_RESULTS = 10 From 09bb4fa9c418519485dca53eae6682cc71d21c14 Mon Sep 17 00:00:00 2001 From: Mahesh <44301650+Mahesh0253@users.noreply.github.com> Date: Sat, 6 Mar 2021 19:39:39 +0530 Subject: [PATCH 13/21] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c51a03f11..70d7e8406 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ Check [`sample_info.py`](sample_info.py) before editing [`info.py`](info.py) fil channel - Get basic infomation about channels total - Show total of saved files delete - Delete file from database +index - Index all files from channel or group logger - Get log file ``` From 0e617d1f184039e98b225618d1feba2dfdddd153 Mon Sep 17 00:00:00 2001 From: Mahesh <44301650+Mahesh0253@users.noreply.github.com> Date: Sat, 6 Mar 2021 19:43:03 +0530 Subject: [PATCH 14/21] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 70d7e8406..2be4467f2 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ logger - Get log file ``` ## Tips -* Run [one_time_indexer.py](one_time_indexer.py) file to save old files in the database that are not indexed yet. +* Use `index` command or run [one_time_indexer.py](one_time_indexer.py) file to save old files in the database that are not indexed yet. * You can use `|` to separate query and file type while searching for specific type of file. For example: `Avengers | video` * If you don't want to create a channel or group, use your chat ID / username as the channel ID. When you send a file to a bot, it will be saved in the database. From 6d47e657601a27cdba14066d5a1b3466fa46b02b Mon Sep 17 00:00:00 2001 From: Mahesh <44301650+Mahesh0253@users.noreply.github.com> Date: Sat, 6 Mar 2021 21:32:33 +0530 Subject: [PATCH 15/21] cache inline results according to configuration --- plugins/inline.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/inline.py b/plugins/inline.py index fea074589..25caf3e0c 100644 --- a/plugins/inline.py +++ b/plugins/inline.py @@ -9,6 +9,7 @@ from info import MAX_RESULTS, CACHE_TIME, SHARE_BUTTON_TEXT, AUTH_USERS, AUTH_CHANNEL logger = logging.getLogger(__name__) +cache_time = 0 if AUTH_USERS or AUTH_CHANNEL else CACHE_TIME @Client.on_inline_query(filters.user(AUTH_USERS) if AUTH_USERS else None) @@ -53,7 +54,7 @@ async def answer(bot, query): switch_pm_text += f" for {string}" await query.answer(results=results, - cache_time=CACHE_TIME, + cache_time=cache_time, switch_pm_text=switch_pm_text, switch_pm_parameter="start", next_offset=str(next_offset)) @@ -64,7 +65,7 @@ async def answer(bot, query): switch_pm_text += f' for "{string}"' await query.answer(results=[], - cache_time=CACHE_TIME, + cache_time=cache_time, switch_pm_text=switch_pm_text, switch_pm_parameter="okay") @@ -101,4 +102,4 @@ async def is_subscribed(bot, query): if not user.status == 'kicked': return True - return False \ No newline at end of file + return False From 1c1179dff8d21b1c0806aa91448abaab7ab94aee Mon Sep 17 00:00:00 2001 From: Mahesh <44301650+Mahesh0253@users.noreply.github.com> Date: Fri, 12 Mar 2021 19:23:12 +0530 Subject: [PATCH 16/21] Handle FloodWait errors --- plugins/userbot.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/plugins/userbot.py b/plugins/userbot.py index 7e939c7d2..bc87d5720 100644 --- a/plugins/userbot.py +++ b/plugins/userbot.py @@ -1,6 +1,7 @@ import logging import asyncio from pyrogram import Client, filters +from pyrogram.errors import FloodWait from info import USERBOT_STRING_SESSION, API_ID, API_HASH, ADMINS, id_pattern from utils import save_file @@ -30,12 +31,22 @@ async def index_files(bot, message): try: async with user_bot: for chat in chats: + async for user_message in user_bot.iter_history(chat): - message = await bot.get_messages( - chat, - user_message.message_id, - replies=0, - ) + try: + message = await bot.get_messages( + chat, + user_message.message_id, + replies=0, + ) + except FloodWait as e: + await asyncio.sleep(e.x) + message = await bot.get_messages( + chat, + user_message.message_id, + replies=0, + ) + for file_type in ("document", "video", "audio"): media = getattr(message, file_type, None) if media is not None: @@ -50,4 +61,4 @@ async def index_files(bot, message): logger.exception(e) await msg.edit(f'Error: {e}') else: - await msg.edit(f'Total {total_files} checked!') \ No newline at end of file + await msg.edit(f'Total {total_files} checked!') From 445ec0f6389c9ee11c4eae50e876e5fb725b82e3 Mon Sep 17 00:00:00 2001 From: Mahesh0253 Date: Sat, 17 Apr 2021 12:21:33 +0530 Subject: [PATCH 17/21] Improved search again button --- plugins/inline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/inline.py b/plugins/inline.py index fea074589..ec84485c0 100644 --- a/plugins/inline.py +++ b/plugins/inline.py @@ -32,7 +32,7 @@ async def answer(bot, query): file_type = None offset = int(query.offset or 0) - reply_markup = get_reply_markup(bot.username) + reply_markup = get_reply_markup(bot.username, query=string) files, next_offset = await get_search_results(string, file_type=file_type, max_results=MAX_RESULTS, @@ -69,10 +69,10 @@ async def answer(bot, query): switch_pm_parameter="okay") -def get_reply_markup(username): +def get_reply_markup(username, query): url = 't.me/share/url?url=' + quote(SHARE_BUTTON_TEXT.format(username=username)) buttons = [[ - InlineKeyboardButton('Search again', switch_inline_query_current_chat=''), + InlineKeyboardButton('Search again', switch_inline_query_current_chat=query), InlineKeyboardButton('Share bot', url=url), ]] return InlineKeyboardMarkup(buttons) From 4f3ed50d9ec07b215fd4aab2e20678d5ed9c5ac2 Mon Sep 17 00:00:00 2001 From: Mahesh <44301650+Mahesh0253@users.noreply.github.com> Date: Tue, 18 May 2021 15:01:58 +0530 Subject: [PATCH 18/21] updated search pattern --- utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils.py b/utils.py index 730cb99f7..c33c6f08b 100644 --- a/utils.py +++ b/utils.py @@ -66,7 +66,7 @@ async def save_file(media): async def get_search_results(query, file_type=None, max_results=10, offset=0): """For given query return (results, next_offset)""" - raw_pattern = query.lower().strip().replace(' ', '.*') + raw_pattern = query.lower().strip().replace(' ', '[\s\.\+\-_]') if not raw_pattern: raw_pattern = '.' @@ -134,4 +134,4 @@ def unpack_new_file_id(new_file_id): ) ) file_ref = encode_file_ref(decoded.file_reference) - return file_id, file_ref \ No newline at end of file + return file_id, file_ref From 651847594fde31a99a90fc34495b46eb2687f387 Mon Sep 17 00:00:00 2001 From: Mahesh <44301650+Mahesh0253@users.noreply.github.com> Date: Tue, 18 May 2021 15:18:29 +0530 Subject: [PATCH 19/21] update regex --- utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/utils.py b/utils.py index c33c6f08b..e46906cdd 100644 --- a/utils.py +++ b/utils.py @@ -66,9 +66,13 @@ async def save_file(media): async def get_search_results(query, file_type=None, max_results=10, offset=0): """For given query return (results, next_offset)""" - raw_pattern = query.lower().strip().replace(' ', '[\s\.\+\-_]') - if not raw_pattern: + query = query.strip() + if not query: raw_pattern = '.' + elif ' ' not in query: + raw_pattern = f'[\b\.\+\-_]{query}[b\.\+\-_]' + else: + raw_pattern = query.replace(' ', '[\s\.\+\-_]') try: regex = re.compile(raw_pattern, flags=re.IGNORECASE) From 4e8f396e5829ae07ec3dc526118b66e5addf1525 Mon Sep 17 00:00:00 2001 From: Mahesh <44301650+Mahesh0253@users.noreply.github.com> Date: Tue, 18 May 2021 21:47:13 +0530 Subject: [PATCH 20/21] Fix regex pattern --- utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils.py b/utils.py index e46906cdd..072ec7cf8 100644 --- a/utils.py +++ b/utils.py @@ -70,9 +70,9 @@ async def get_search_results(query, file_type=None, max_results=10, offset=0): if not query: raw_pattern = '.' elif ' ' not in query: - raw_pattern = f'[\b\.\+\-_]{query}[b\.\+\-_]' + raw_pattern =r'(\b|[\.\+\-_])' + query + r'(\b|[\.\+\-_])' else: - raw_pattern = query.replace(' ', '[\s\.\+\-_]') + raw_pattern = query.replace(' ', r'[\s\.\+\-_]') try: regex = re.compile(raw_pattern, flags=re.IGNORECASE) From aaf0af0819379aed581dcb32a66bc4bebe565ca2 Mon Sep 17 00:00:00 2001 From: Mahesh0253 Date: Wed, 19 May 2021 15:38:15 +0530 Subject: [PATCH 21/21] update requirements --- requirements.txt | 7 +++---- utils.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index aa2cef5c5..40cbb3c4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ https://github.com/Mahesh0253/pyrogram/archive/inline.zip -tgcrypto -umongo==2.3.0 -motor==2.3.0 -dnspython +tgcrypto==1.2.2 +umongo[motor]==3.0.0 +dnspython==2.1.0 \ No newline at end of file diff --git a/utils.py b/utils.py index 072ec7cf8..64d4e98c1 100644 --- a/utils.py +++ b/utils.py @@ -16,7 +16,7 @@ client = AsyncIOMotorClient(DATABASE_URI) db = client[DATABASE_NAME] -instance = Instance(db) +instance = Instance.from_db(db) @instance.register