Merge pull request #101 from TopRealm/merge-conflict

Merge conflict

.github/scripts/build_copy_files.py

@@ -0,0 +1,32 @@
+import shutil
+import os
+
+if __name__ == "__main__":
+    os.makedirs('output', exist_ok=True)
+    shutil.copytree('assets', 'output/assets')
+    os.makedirs('output/config', exist_ok=True)
+    shutil.copyfile('config/config.toml.example', 'output/config/config.toml.example')
+    shutil.copytree('locales', 'output/locales')
+    shutil.copytree('modules', 'output/modules')
+
+    def remove_py_files(path):
+
+        for root, dirs, files in os.walk(path):
+            for file in files:
+                if (file.endswith('.py') or file.endswith('.pyc') or file.endswith(
+                        '.pyo') or file.endswith('.pyd') or file.endswith('.pyw')):
+                    os.remove(os.path.join(root, file))
+
+        for root, dirs, files in os.walk(path):
+            for dir in dirs:
+                if dir == '__pycache__':
+                    shutil.rmtree(os.path.join(root, dir))
+
+    remove_py_files('output/modules')
+
+    build_paths = ['launcher-build', 'wrapper-build']
+    for build_path in build_paths:
+        if os.path.exists(build_path):
+            for file in os.listdir(build_path):
+                if file.endswith('.exe') or file.endswith('.bin') or file.endswith('.app'):
+                    shutil.copyfile(os.path.join(build_path, file), os.path.join('output', file))

.github/scripts/pre_build.py

@@ -0,0 +1,14 @@
+import shutil
+import json
+import os
+
+if __name__ == "__main__":
+    modules_folder_list = os.listdir('modules')
+    with open('assets/modules_list.json', 'w') as f:
+        json.dump(modules_folder_list, f)
+    schedulers_folder_list = os.listdir('schedulers')
+    with open('assets/schedulers_list.json', 'w') as f:
+        json.dump(schedulers_folder_list, f)
+    slash_list = os.listdir('bots/discord/slash')
+    with open('assets/discord_slash_list.json', 'w') as f:
+        json.dump(slash_list, f)

.github/workflows/build.yml

@@ -0,0 +1,72 @@
+name: "Build Nightly x64 Executables (3:00, UTC+8)"
+
+on:
+  schedule:
+    - cron: "00 19 * * *"
+  workflow_dispatch:
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [macos-latest, ubuntu-latest, windows-latest]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Check-out repository
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12' # Version range or exact version of a Python version to use, using SemVer's version range syntax
+          architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
+          cache: 'pip'
+          cache-dependency-path: |
+              **/requirements*.txt
+
+      - name: Install Dependencies
+        run: |
+          pip install -r requirements.txt
+
+      - name: Prepare Build
+        run: |
+          python .github/scripts/pre_build.py
+        
+
+      - name: Build Launcher Executable
+        uses: Nuitka/Nuitka-Action@main
+        with:
+          nuitka-version: main
+          script-name: launcher.py
+          onefile: true
+          standalone: true
+          output-dir: launcher-build
+          include-plugin-directory: |
+            modules
+            bots
+            schedulers
+          include-package-data: |
+            oss2
+            langconv
+            aliyunsdkcore
+            
+
+      - name: Build Wrapper Executable
+        uses: Nuitka/Nuitka-Action@main
+        with:
+          nuitka-version: main
+          script-name: bot.py
+          onefile: true
+          standalone: true
+          output-dir: wrapper-build
+
+      - name: Copy Assets
+        run: |
+          python .github/scripts/build_copy_files.py
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ runner.os }} Build
+          path: |
+            output/**

.gitignore

@@ -23,3 +23,6 @@ version*
 *.log
 matrix_store
 modules/ask/tools/self_knowledge
+output/*
+assets/modules_list.json
+assets/discord_slash_list.json

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
       rev: "1.6.1"
       hooks:
           - id: poetry-export
-            args: ["-f", "requirements.txt", "-o", "requirements.txt"]
+            args: ["--without-hashes", "-f", "requirements.txt", "-o", "requirements.txt"]
             verbose: true
     - repo: https://github.com/pre-commit/mirrors-autopep8
       rev: "v2.0.4"

bot.py

@@ -2,6 +2,7 @@
 import shutil
 import subprocess
 import sys
+import traceback
 from queue import Queue, Empty
 from threading import Thread
 from time import sleep
@@ -14,7 +15,7 @@
 
 encode = 'UTF-8'
 
-bots_required_configs = {
+bots_and_required_configs = {
     'aiocqhttp': [
         'qq_host',
         'qq_account'],
@@ -25,7 +26,8 @@
         'matrix_homeserver',
         'matrix_user',
         'matrix_device_id',
-        'matrix_token']
+        'matrix_token'],
+    'api': []
 }
 
 
@@ -62,49 +64,38 @@ def run_bot():
     cache_path = os.path.abspath(Config('cache_path', './cache/'))
     if os.path.exists(cache_path):
         shutil.rmtree(cache_path)
-    os.mkdir(cache_path)
-
-    pid_cache = os.path.abspath('.pid_last')
-    if os.path.exists(pid_cache):
-        with open(pid_cache, 'r') as f:
-            pid_last = f.read().split('\n')
-            running_pids = get_pid('python')
-            for pid in pid_last:
-                if int(pid) in running_pids:
-                    try:
-                        os.kill(int(pid), 9)
-                    except (PermissionError, ProcessLookupError):
-                        pass
-        os.remove(pid_cache)
+    os.makedirs(cache_path, exist_ok=True)
     envs = os.environ.copy()
     envs['PYTHONIOENCODING'] = 'UTF-8'
     envs['PYTHONPATH'] = os.path.abspath('.')
-    botdir = './bots/'
-    lst = os.listdir(botdir)
+    lst = bots_and_required_configs.keys()
     runlst = []
     for bl in lst:
         if bl in disabled_bots:
             continue
-        if bl in bots_required_configs:
+        if bl in bots_and_required_configs:
             abort = False
-            for c in bots_required_configs[bl]:
+            for c in bots_and_required_configs[bl]:
                 if not Config(c):
                     logger.error(f'Bot {bl} requires config {c} but not found, abort to launch.')
                     abort = True
                     break
             if abort:
                 continue
 
-        bot = os.path.abspath(f'{botdir}{bl}/bot.py')
-        if os.path.exists(bot):
-            p = subprocess.Popen([sys.executable, bot, 'subprocess'], shell=False, stdout=subprocess.PIPE,
-                                 stderr=subprocess.STDOUT,
-                                 cwd=os.path.abspath('.'), env=envs)
-            runlst.append(p)
-            pidlst.append(p.pid)
+        launch_args = [sys.executable, 'launcher.py', 'subprocess', bl]
 
-    with open(pid_cache, 'w') as c:
-        c.write('\n'.join(str(p) for p in pidlst))
+        if sys.platform == 'win32' and 'launcher.exe' in os.listdir('.') and not sys.argv[0].endswith('.py'):
+            launch_args = ['launcher.exe', 'subprocess', bl]
+
+        elif 'launcher.bin' in os.listdir('.') and not sys.argv[0].endswith('.py'):
+            launch_args = ['./launcher.bin', 'subprocess', bl]
+
+        p = subprocess.Popen(launch_args, shell=False, stdout=subprocess.PIPE,
+                             stderr=subprocess.STDOUT,
+                             cwd=os.path.abspath('.'), env=envs)
+        runlst.append(p)
+        pidlst.append(p.pid)
 
     q = Queue()
     threads = []
@@ -180,7 +171,11 @@ def run_bot():
                 pidlst.clear()
                 sleep(5)
                 continue
-    except KeyboardInterrupt:
+            except Exception as e:
+                logger.critical('An error occurred, please check the output.')
+                traceback.print_exc()
+                break
+    except (KeyboardInterrupt, SystemExit):
         for x in pidlst:
             try:
                 os.kill(x, 9)

bots/aiocqhttp/bot.py

@@ -20,7 +20,7 @@
 from core.utils.i18n import Locale, default_locale
 from database import BotDBUtil
 
-PrivateAssets.set(os.path.abspath(os.path.dirname(__file__) + '/assets'))
+PrivateAssets.set('assets/private/aiocqhttp')
 EnableDirtyWordCheck.status = True if Config('enable_dirty_check', False) else False
 Url.disable_mm = False if Config('enable_urlmanager', False) else True
 qq_account = str(Config("qq_account", cfg_type=(int, str)))

bots/aiogram/bot.py

@@ -14,7 +14,7 @@
 from core.utils.bot import load_prompt, init_async
 from core.utils.info import Info
 
-PrivateAssets.set(os.path.abspath(os.path.dirname(__file__) + '/assets'))
+PrivateAssets.set('assets/private/aiogram')
 Url.disable_mm = True
 
 

bots/api/bot.py

@@ -9,6 +9,7 @@
 
 from core.queue import JobQueue
 from core.scheduler import Scheduler
+from core.utils.info import Info
 
 sys.path.append(os.getcwd())
 
@@ -179,9 +180,9 @@ async def get_locale(locale: str, string: str):
             "detail": "Not Found"
         })
 
-if __name__ == "__main__":
+if __name__ == "__main__" or Info.subprocess:
     while True:
-        uvicorn.run("bot:app", port=Config('api_port', 5000), log_level="info")
+        uvicorn.run(app, port=Config('api_port', 5000), log_level="info")
         Logger.error('API Server crashed, is the port occupied?')
         Logger.error('Retrying in 5 seconds...')
         time.sleep(5)

bots/discord/bot.py

@@ -1,4 +1,5 @@
 import importlib
+import json
 import os
 import re
 import sys
@@ -17,7 +18,7 @@
 from core.utils.bot import init_async, load_prompt
 from core.utils.info import Info
 
-PrivateAssets.set(os.path.abspath(os.path.dirname(__file__) + '/assets'))
+PrivateAssets.set('assets/private/discord')
 Url.disable_mm = True
 
 count = 0
@@ -40,15 +41,32 @@ async def on_ready():
 
 def load_slashcommands():
     fun_file = None
-    dir_list = os.listdir(slash_load_dir)
+    if not Info.binary_mode:
+        dir_list = os.listdir(slash_load_dir)
+
+    else:
+        try:
+            Logger.warning('Binary mode detected, trying to load pre-built slash list...')
+            js = 'assets/discord_slash_list.json'
+            with open(js, 'r', encoding='utf-8') as f:
+                dir_list = json.load(f)
+        except Exception:
+            Logger.error('Failed to load pre-built slash list, using default list.')
+            dir_list = os.listdir(slash_load_dir)
     for file_name in dir_list:
         try:
             file_path = os.path.join(slash_load_dir, file_name)
             fun_file = None
-            if os.path.isdir(file_path):
+            if not Info.binary_mode:
+                if os.path.isdir(file_path):
+                    if file_name[0] != '_':
+                        fun_file = file_name
+                elif os.path.isfile(file_path):
+                    if file_name[0] != '_' and file_name.endswith('.py'):
+                        fun_file = file_name[:-3]
+            else:
                 if file_name[0] != '_':
                     fun_file = file_name
-            elif os.path.isfile(file_path):
                 if file_name[0] != '_' and file_name.endswith('.py'):
                     fun_file = file_name[:-3]
             if fun_file:

bots/kook/bot.py

@@ -13,7 +13,7 @@
 from core.utils.bot import load_prompt, init_async
 from core.utils.info import Info
 
-PrivateAssets.set(os.path.abspath(os.path.dirname(__file__) + '/assets'))
+PrivateAssets.set('assets/private/kook')
 EnableDirtyWordCheck.status = True if Config('enable_dirty_check', False) else False
 Url.disable_mm = False if Config('enable_urlmanager', False) else True
 Url.md_format = True

bots/matrix/bot.py

@@ -17,7 +17,7 @@
 from core.utils.bot import load_prompt, init_async
 from core.utils.info import Info
 
-PrivateAssets.set(os.path.abspath(os.path.dirname(__file__) + "/assets"))
+PrivateAssets.set('assets/private/matrix')
 Url.disable_mm = True
 
 

console.py

@@ -41,8 +41,8 @@
 
 EnableDirtyWordCheck.status = True
 Url.disable_mm = True
-PrivateAssets.set(os.path.abspath(os.path.dirname(__file__) + '/assets'))
-console_history_path = os.path.abspath(os.path.dirname(__file__) + '/.console_history')
+PrivateAssets.set('assets/private/console')
+console_history_path = os.path.abspath('assets/private/console/.console_history')
 if os.path.exists(console_history_path):
     os.remove(console_history_path)