diff --git a/modules/__init__.py b/modules/__init__.py index afb61e14..3d09bb9e 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -58,7 +58,9 @@ class Setting: 'debug': Setting(None, False, 'Debug mtc console mode. Prints Traceback on errors'), 'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'), 'BotToken': Setting('alert-bot', None, 'Alerting Telegram bot token'), - 'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id') + 'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id'), + 'auto_backup': Setting('validator', None, 'Make validator backup every election'), + 'auto_backup_path': Setting('validator', '/tmp/mytoncore/auto_backups/', 'Path to store auto-backups'), } diff --git a/modules/alert_bot.py b/modules/alert_bot.py index e9dafed3..2ebf4034 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -48,12 +48,12 @@ class Alert: "out_of_sync": Alert( "critical", "Node is out of sync on {sync} sec.", - 0 + 300 ), "service_down": Alert( "critical", "validator.service is down.", - 0 + 300 ), "adnl_connection_failed": Alert( "high", @@ -83,6 +83,7 @@ def __init__(self, ton, local, *args, **kwargs): self.validator_module = None self.inited = False self.hostname = None + self.ip = None self.token = None self.chat_id = None self.last_db_check = 0 @@ -113,6 +114,7 @@ def send_alert(self, alert_name: str, *args, **kwargs): ❗️ MyTonCtrl Alert {alert_name} ❗️ Hostname: {self.hostname} +Node IP: {self.ip} Time: {time_} ({int(time.time())}) Severity: {alert.severity} @@ -140,6 +142,7 @@ def init(self): from modules.validator import ValidatorModule self.validator_module = ValidatorModule(self.ton, self.local) self.hostname = get_hostname() + self.ip = self.ton.get_validator_engine_ip() self.set_global_vars() self.inited = True @@ -189,6 +192,8 @@ def print_alerts(self, args): print_table(table) def test_alert(self, args): + if not self.inited: + self.init() self.send_message('Test alert') def check_db_usage(self): @@ -204,6 +209,9 @@ def check_db_usage(self): def check_validator_wallet_balance(self): if not self.ton.using_validator(): return + validator_status = self.ton.GetValidatorStatus() + if not validator_status.is_working or validator_status.out_of_sync >= 20: + return validator_wallet = self.ton.GetValidatorWallet() validator_account = self.ton.GetAccount(validator_wallet.addrB64) if validator_account.balance < 10: diff --git a/modules/backups.py b/modules/backups.py new file mode 100644 index 00000000..7ed70954 --- /dev/null +++ b/modules/backups.py @@ -0,0 +1,80 @@ +import os +import shutil +import subprocess +import time + +import pkg_resources + +from modules.module import MtcModule +from mypylib.mypylib import color_print, ip2int, run_as_root, parse +from mytoninstaller.config import get_own_ip + + +class BackupModule(MtcModule): + + def create_keyring(self, dir_name): + keyring_dir = dir_name + '/keyring' + self.ton.validatorConsole.Run(f'exportallprivatekeys {keyring_dir}') + + def create_tmp_ton_dir(self): + result = self.ton.validatorConsole.Run("getconfig") + text = parse(result, "---------", "--------") + dir_name = self.ton.tempDir + f'/ton_backup_{int(time.time() * 1000)}' + dir_name_db = dir_name + '/db' + os.makedirs(dir_name_db) + with open(dir_name_db + '/config.json', 'w') as f: + f.write(text) + self.create_keyring(dir_name_db) + return dir_name + + def create_backup(self, args): + if len(args) > 1: + color_print("{red}Bad args. Usage:{endc} create_backup [filename]") + return + tmp_dir = self.create_tmp_ton_dir() + command_args = ["-m", self.ton.local.buffer.my_work_dir, "-t", tmp_dir] + if len(args) == 1: + command_args += ["-d", args[0]] + backup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/create_backup.sh') + process = subprocess.run(["bash", backup_script_path] + command_args, timeout=5) + + if process.returncode == 0: + color_print("create_backup - {green}OK{endc}") + else: + color_print("create_backup - {red}Error{endc}") + shutil.rmtree(tmp_dir) + return process.returncode + # end define + + def restore_backup(self, args): + if len(args) == 0 or len(args) > 2: + color_print("{red}Bad args. Usage:{endc} restore_backup [-y]") + return + if '-y' not in args: + res = input( + f'This action will overwrite existing configuration with contents of backup archive, please make sure that donor node is not in operation prior to this action. Proceed [y/n]') + if res.lower() != 'y': + print('aborted.') + return + else: + args.pop(args.index('-y')) + print('Before proceeding, mtc will create a backup of current configuration.') + try: + self.create_backup([]) + except: + color_print("{red}Could not create backup{endc}") + + ip = str(ip2int(get_own_ip())) + command_args = ["-m", self.ton.local.buffer.my_work_dir, "-n", args[0], "-i", ip] + + restore_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/restore_backup.sh') + if run_as_root(["bash", restore_script_path] + command_args) == 0: + color_print("restore_backup - {green}OK{endc}") + self.local.exit() + else: + color_print("restore_backup - {red}Error{endc}") + # end define + + def add_console_commands(self, console): + console.AddItem("create_backup", self.create_backup, self.local.translate("create_backup_cmd")) + console.AddItem("restore_backup", self.restore_backup, self.local.translate("restore_backup_cmd")) diff --git a/modules/nominator_pool.py b/modules/nominator_pool.py index 14f4fbdd..d00708b0 100644 --- a/modules/nominator_pool.py +++ b/modules/nominator_pool.py @@ -87,7 +87,45 @@ def update_validator_set(self, args): self.ton.PoolUpdateValidatorSet(pool_addr, wallet) color_print("UpdateValidatorSet - {green}OK{endc}") + def do_deposit_to_pool(self, pool_addr, amount): + wallet = self.ton.GetValidatorWallet() + bocPath = self.ton.local.buffer.my_temp_dir + wallet.name + "validator-deposit-query.boc" + fiftScript = self.ton.contractsDir + "nominator-pool/func/validator-deposit.fif" + args = [fiftScript, bocPath] + result = self.ton.fift.Run(args) + resultFilePath = self.ton.SignBocWithWallet(wallet, bocPath, pool_addr, amount) + self.ton.SendFile(resultFilePath, wallet) + + def deposit_to_pool(self, args): + try: + poll_addr = args[0] + amount = float(args[1]) + except: + color_print("{red}Bad args. Usage:{endc} deposit_to_pool ") + return + self.do_deposit_to_pool(poll_addr, amount) + color_print("DepositToPool - {green}OK{endc}") + + def do_withdraw_from_pool(self, pool_addr, amount): + pool_data = self.ton.GetPoolData(pool_addr) + if pool_data["state"] == 0: + self.ton.WithdrawFromPoolProcess(pool_addr, amount) + else: + self.ton.PendWithdrawFromPool(pool_addr, amount) + + def withdraw_from_pool(self, args): + try: + pool_addr = args[0] + amount = float(args[1]) + except: + color_print("{red}Bad args. Usage:{endc} withdraw_from_pool ") + return + self.do_withdraw_from_pool(pool_addr, amount) + color_print("WithdrawFromPool - {green}OK{endc}") + def add_console_commands(self, console): console.AddItem("new_pool", self.new_pool, self.local.translate("new_pool_cmd")) console.AddItem("activate_pool", self.activate_pool, self.local.translate("activate_pool_cmd")) console.AddItem("update_validator_set", self.update_validator_set, self.local.translate("update_validator_set_cmd")) + console.AddItem("withdraw_from_pool", self.withdraw_from_pool, self.local.translate("withdraw_from_pool_cmd")) + console.AddItem("deposit_to_pool", self.deposit_to_pool, self.local.translate("deposit_to_pool_cmd")) diff --git a/modules/pool.py b/modules/pool.py index ba37b16c..16b481c5 100644 --- a/modules/pool.py +++ b/modules/pool.py @@ -58,45 +58,7 @@ def check_download_pool_contract_scripts(self): if not os.path.isdir(contract_path): self.ton.DownloadContract("https://github.com/ton-blockchain/nominator-pool") - def do_deposit_to_pool(self, pool_addr, amount): - wallet = self.ton.GetValidatorWallet() - bocPath = self.ton.local.buffer.my_temp_dir + wallet.name + "validator-deposit-query.boc" - fiftScript = self.ton.contractsDir + "nominator-pool/func/validator-deposit.fif" - args = [fiftScript, bocPath] - result = self.ton.fift.Run(args) - resultFilePath = self.ton.SignBocWithWallet(wallet, bocPath, pool_addr, amount) - self.ton.SendFile(resultFilePath, wallet) - - def deposit_to_pool(self, args): - try: - poll_addr = args[0] - amount = float(args[1]) - except: - color_print("{red}Bad args. Usage:{endc} deposit_to_pool ") - return - self.do_deposit_to_pool(poll_addr, amount) - color_print("DepositToPool - {green}OK{endc}") - - def do_withdraw_from_pool(self, pool_addr, amount): - pool_data = self.ton.GetPoolData(pool_addr) - if pool_data["state"] == 0: - self.ton.WithdrawFromPoolProcess(pool_addr, amount) - else: - self.ton.PendWithdrawFromPool(pool_addr, amount) - - def withdraw_from_pool(self, args): - try: - pool_addr = args[0] - amount = float(args[1]) - except: - color_print("{red}Bad args. Usage:{endc} withdraw_from_pool ") - return - self.do_withdraw_from_pool(pool_addr, amount) - color_print("WithdrawFromPool - {green}OK{endc}") - def add_console_commands(self, console): console.AddItem("pools_list", self.print_pools_list, self.local.translate("pools_list_cmd")) console.AddItem("delete_pool", self.delete_pool, self.local.translate("delete_pool_cmd")) console.AddItem("import_pool", self.import_pool, self.local.translate("import_pool_cmd")) - console.AddItem("deposit_to_pool", self.deposit_to_pool, self.local.translate("deposit_to_pool_cmd")) - console.AddItem("withdraw_from_pool", self.withdraw_from_pool, self.local.translate("withdraw_from_pool_cmd")) diff --git a/modules/utilities.py b/modules/utilities.py index da108152..62c83efb 100644 --- a/modules/utilities.py +++ b/modules/utilities.py @@ -313,13 +313,14 @@ def print_validator_list(self, args): print(text) else: table = list() - table += [["id", "ADNL", "Pubkey", "Wallet", "Efficiency", "Online"]] + table += [["id", "ADNL", "Pubkey", "Wallet", "Stake", "Efficiency", "Online"]] for i, item in enumerate(data): adnl = item.get("adnlAddr") pubkey = item.get("pubkey") walletAddr = item.get("walletAddr") efficiency = item.get("efficiency") online = item.get("online") + stake = item.get("stake") if "adnl" not in args: adnl = self.reduct(adnl) if "pubkey" not in args: @@ -332,7 +333,7 @@ def print_validator_list(self, args): online = bcolors.green_text("true") if not online: online = bcolors.red_text("false") - table += [[str(i), adnl, pubkey, walletAddr, efficiency, online]] + table += [[str(i), adnl, pubkey, walletAddr, stake, efficiency, online]] print_table(table) # end define diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 76cf58cd..6896bf79 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -30,7 +30,7 @@ get_timestamp, timestamp2datetime, dec2hex, - Dict + Dict, int2ip ) @@ -115,6 +115,7 @@ def CheckConfigFile(self, fift, liteClient): self.local.add_log("Restoring the configuration file", "info") args = ["cp", backup_path, mconfig_path] subprocess.run(args) + self.dbFile = mconfig_path self.Refresh() elif os.path.isfile(backup_path) == False: self.local.add_log("Create backup config file", "info") @@ -1454,21 +1455,47 @@ def ElectionEntry(self, args=None): self.local.add_log("ElectionEntry completed. Start work time: " + str(startWorkTime)) self.clear_tmp() + self.make_backup(startWorkTime) #end define - def clear_tmp(self): + def clear_dir(self, dir_name): start = time.time() count = 0 week_ago = 60 * 60 * 24 * 7 - dir = self.tempDir - for f in os.listdir(dir): - ts = os.path.getmtime(os.path.join(dir, f)) + for f in os.listdir(dir_name): + ts = os.path.getmtime(os.path.join(dir_name, f)) if ts < time.time() - week_ago: count += 1 - os.remove(os.path.join(dir, f)) + if os.path.isfile(os.path.join(dir_name, f)): + os.remove(os.path.join(dir_name, f)) + self.local.add_log(f"Removed {count} old files from {dir_name} directory for {int(time.time() - start)} seconds", "info") + + def clear_tmp(self): + self.clear_dir(self.tempDir) - self.local.add_log(f"Removed {count} old files from tmp dir for {int(time.time() - start)} seconds", "info") + def make_backup(self, election_id: str): + if not self.local.db.get("auto_backup"): + return + from modules.backups import BackupModule + module = BackupModule(self, self.local) + args = [] + name = f"/mytonctrl_backup_elid{election_id}.zip" + backups_dir = self.tempDir + "/auto_backups" + if self.local.db.get("auto_backup_path"): + backups_dir = self.local.db.get("auto_backup_path") + os.makedirs(backups_dir, exist_ok=True) + args.append(backups_dir + name) + self.clear_dir(backups_dir) + exit_code = module.create_backup(args) + if exit_code != 0: + self.local.add_log(f"Backup failed with exit code {exit_code}", "error") + # try one more time + exit_code = module.create_backup(args) + if exit_code != 0: + self.local.add_log(f"Backup failed with exit code {exit_code}", "error") + if exit_code == 0: + self.local.add_log(f"Backup created successfully", "info") def GetValidatorKeyByTime(self, startWorkTime, endWorkTime): self.local.add_log("start GetValidatorKeyByTime function", "debug") @@ -2490,6 +2517,8 @@ def GetValidatorsList(self, past=False, fast=False, start=None, end=None): validator["efficiency"] = round(validator["wr"] * 100, 2) if saveElectionEntries and adnlAddr in saveElectionEntries: validator["walletAddr"] = saveElectionEntries[adnlAddr]["walletAddr"] + validator["stake"] = saveElectionEntries[adnlAddr].get("stake") + validator["stake"] = int(validator["stake"]) if validator["stake"] else None #end for # Set buffer @@ -3791,6 +3820,13 @@ def GetNetworkName(self): return "unknown" #end define + def get_validator_engine_ip(self): + try: + config = self.GetValidatorConfig() + return int2ip(config['addrs'][0]['ip']) + except: + return None + def GetFunctionBuffer(self, name, timeout=10): timestamp = get_timestamp() buff = self.local.buffer.get(name) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 4b3dd433..3263b3b2 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -85,8 +85,6 @@ def inject_globals(func): console.AddItem("get", inject_globals(GetSettings), local.translate("get_cmd")) console.AddItem("set", inject_globals(SetSettings), local.translate("set_cmd")) console.AddItem("rollback", inject_globals(rollback_to_mtc1), local.translate("rollback_cmd")) - console.AddItem("create_backup", inject_globals(create_backup), local.translate("create_backup_cmd")) - console.AddItem("restore_backup", inject_globals(restore_backup), local.translate("restore_backup_cmd")) #console.AddItem("xrestart", inject_globals(Xrestart), local.translate("xrestart_cmd")) #console.AddItem("xlist", inject_globals(Xlist), local.translate("xlist_cmd")) @@ -94,6 +92,10 @@ def inject_globals(func): #console.AddItem("ssoc", inject_globals(SignShardOverlayCert), local.translate("ssoc_cmd")) #console.AddItem("isoc", inject_globals(ImportShardOverlayCert), local.translate("isoc_cmd")) + from modules.backups import BackupModule + module = BackupModule(ton, local) + module.add_console_commands(console) + from modules.custom_overlays import CustomOverlayModule module = CustomOverlayModule(ton, local) module.add_console_commands(console) @@ -140,7 +142,6 @@ def inject_globals(func): module = AlertBotModule(ton, local) module.add_console_commands(console) - console.AddItem("cleanup", inject_globals(cleanup_validator_db), local.translate("cleanup_cmd")) console.AddItem("benchmark", inject_globals(run_benchmark), local.translate("benchmark_cmd")) # console.AddItem("activate_ton_storage_provider", inject_globals(activate_ton_storage_provider), local.translate("activate_ton_storage_provider_cmd")) @@ -403,12 +404,6 @@ def rollback_to_mtc1(local, ton, args): local.exit() #end define -def cleanup_validator_db(ton, args): - cleanup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/cleanup.sh') - run_args = ["bash", cleanup_script_path] - exit_code = run_as_root(run_args) -#end define - def run_benchmark(ton, args): timeout = 200 benchmark_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/benchmark.sh') @@ -939,53 +934,6 @@ def disable_mode(local, ton, args): #end define -def create_backup(local, ton, args): - if len(args) > 2: - color_print("{red}Bad args. Usage:{endc} create_backup [path_to_archive] [-y]") - return - if '-y' not in args: - res = input(f'Node and Mytoncore services will be stopped for few seconds while backup is created, Proceed [y/n]?') - if res.lower() != 'y': - print('aborted.') - return - else: - args.pop(args.index('-y')) - command_args = ["-m", ton.local.buffer.my_work_dir] - if len(args) == 1: - command_args += ["-d", args[0]] - backup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/create_backup.sh') - if run_as_root(["bash", backup_script_path] + command_args) == 0: - color_print("create_backup - {green}OK{endc}") - else: - color_print("create_backup - {red}Error{endc}") -#end define - - -def restore_backup(local, ton, args): - if len(args) == 0 or len(args) > 2: - color_print("{red}Bad args. Usage:{endc} restore_backup [-y]") - return - if '-y' not in args: - res = input(f'This action will overwrite existing configuration with contents of backup archive, please make sure that donor node is not in operation prior to this action. Proceed [y/n]') - if res.lower() != 'y': - print('aborted.') - return - else: - args.pop(args.index('-y')) - print('Before proceeding, mtc will create a backup of current configuration.') - create_backup(local, ton, ['-y']) - ip = str(ip2int(get_own_ip())) - command_args = ["-m", ton.local.buffer.my_work_dir, "-n", args[0], "-i", ip] - - restore_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/restore_backup.sh') - if run_as_root(["bash", restore_script_path] + command_args) == 0: - color_print("restore_backup - {green}OK{endc}") - local.exit() - else: - color_print("restore_backup - {red}Error{endc}") -#end define - - def Xrestart(inputArgs): if len(inputArgs) < 2: color_print("{red}Bad args. Usage:{endc} xrestart ") diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 22915e59..83752783 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -484,11 +484,6 @@ "ru": "Отправить тестовое оповещение через Telegram Bot", "zh_TW": "通過 Telegram Bot 發送測試警報" }, - "cleanup_cmd": { - "en": "Clean node old logs and temp files", - "ru": "Очистить старые логи и временные файлы ноды", - "zh_TW": "清理節點舊日誌和臨時文件" - }, "benchmark_cmd": { "en": "Run benchmark", "ru": "Запустить бенчмарк", diff --git a/mytonctrl/scripts/cleanup.sh b/mytonctrl/scripts/cleanup.sh deleted file mode 100644 index d3d10a2e..00000000 --- a/mytonctrl/scripts/cleanup.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -set -e - -# Проверить sudo -if [ "$(id -u)" != "0" ]; then - echo "Please run script as root" - exit 1 -fi - -# Цвета -COLOR='\033[92m' -ENDC='\033[0m' - -db_path=/var/ton-work/db - -function get_directory_size { - buff=$(du -sh ${db_path} | awk '{print $1}') - echo ${buff} -} - -echo -e "${COLOR}[1/7]${ENDC} Start node/validator DB cleanup process" -echo -e "${COLOR}[2/7]${ENDC} Stop node/validator" -systemctl stop validator - -echo -e "${COLOR}[3/7]${ENDC} Node/validator DB size before cleanup = $(get_directory_size)" -find /var/ton-work/db -name 'LOG.old*' -exec rm {} + - -echo -e "${COLOR}[4/7]${ENDC} Node/validator DB size after deleting old files = $(get_directory_size)" -rm -r /var/ton-work/db/files/packages/temp.archive.* - -echo -e "${COLOR}[5/7]${ENDC} Node/validator DB size after deleting temporary files = $(get_directory_size)" -echo -e "${COLOR}[6/7]${ENDC} Start node/validator" -systemctl start validator - -echo -e "${COLOR}[7/7]${ENDC} Node/validator DB cleanup process completed" diff --git a/mytonctrl/scripts/create_backup.sh b/mytonctrl/scripts/create_backup.sh index 9bab1355..dbcfe5b8 100644 --- a/mytonctrl/scripts/create_backup.sh +++ b/mytonctrl/scripts/create_backup.sh @@ -1,12 +1,16 @@ dest="mytonctrl_backup_$(hostname)_$(date +%s).tar.gz" mtc_dir="$HOME/.local/share/mytoncore" user=$(logname) +ton_dir="/var/ton-work" +keys_dir="/var/ton-work/keys" # Get arguments -while getopts d:m: flag +while getopts d:m:t:k: flag do case "${flag}" in d) dest=${OPTARG};; m) mtc_dir=${OPTARG};; + t) ton_dir=${OPTARG};; + k) keys_dir=${OPTARG};; *) echo "Flag -${flag} is not recognized. Aborting" exit 1 ;; @@ -16,32 +20,27 @@ done COLOR='\033[92m' ENDC='\033[0m' -systemctl stop validator -systemctl stop mytoncore - -echo -e "${COLOR}[1/4]${ENDC} Stopped validator and mytoncore" - - -tmp_dir="/tmp/mytoncore/backup" +tmp_dir="/tmp/mytoncore/backupv2" rm -rf $tmp_dir mkdir $tmp_dir +mkdir $tmp_dir/db -cp /var/ton-work/db/config.json ${tmp_dir} -cp -r /var/ton-work/db/keyring ${tmp_dir} -cp -r /var/ton-work/keys ${tmp_dir} +cp $ton_dir/db/config.json ${tmp_dir}/db +cp -r $ton_dir/db/keyring ${tmp_dir}/db +cp -r $keys_dir ${tmp_dir} cp -r $mtc_dir $tmp_dir -echo -e "${COLOR}[2/4]${ENDC} Copied files to ${tmp_dir}" - +python3 -c "import json;f=open('${tmp_dir}/db/config.json');json.load(f);f.close()" || exit 1 # Check if config.json is copied correctly +python3 -c "import json;f=open('${tmp_dir}/mytoncore/mytoncore.db');json.load(f);f.close()" || exit 2 # Check if mytoncore.db is copied correctly -systemctl start validator -systemctl start mytoncore - -echo -e "${COLOR}[3/4]${ENDC} Started validator and mytoncore" +echo -e "${COLOR}[1/2]${ENDC} Copied files to ${tmp_dir}" tar -zcf $dest -C $tmp_dir . chown $user:$user $dest -echo -e "${COLOR}[4/4]${ENDC} Backup successfully created in ${dest}!" +echo -e "${COLOR}[2/2]${ENDC} Backup successfully created in ${dest}!" + +rm -rf $tmp_dir + echo -e "If you wish to use archive package to migrate node to different machine please make sure to stop validator and mytoncore on donor (this) host prior to migration." diff --git a/mytonctrl/scripts/restore_backup.sh b/mytonctrl/scripts/restore_backup.sh index 361c7ff9..ca53f0de 100644 --- a/mytonctrl/scripts/restore_backup.sh +++ b/mytonctrl/scripts/restore_backup.sh @@ -29,9 +29,16 @@ rm -rf $tmp_dir mkdir $tmp_dir tar -xvzf $name -C $tmp_dir +if [ ! -d ${tmp_dir}/db ]; then + echo "Old version of backup detected" + mkdir ${tmp_dir}/db + mv ${tmp_dir}/config.json ${tmp_dir}/db + mv ${tmp_dir}/keyring ${tmp_dir}/db + +fi + rm -rf /var/ton-work/db/keyring -cp -f ${tmp_dir}/config.json /var/ton-work/db/ -cp -rf ${tmp_dir}/keyring /var/ton-work/db/ +cp -rf ${tmp_dir}/db /var/ton-work cp -rf ${tmp_dir}/keys /var/ton-work cp -rfT ${tmp_dir}/mytoncore $mtc_dir diff --git a/mytonctrl/scripts/upgrade.sh b/mytonctrl/scripts/upgrade.sh index 81a9fd0b..c3c4c52b 100644 --- a/mytonctrl/scripts/upgrade.sh +++ b/mytonctrl/scripts/upgrade.sh @@ -29,7 +29,7 @@ COLOR='\033[92m' ENDC='\033[0m' # Установить дополнительные зависимости -apt-get install -y libsecp256k1-dev libsodium-dev ninja-build fio rocksdb-tools liblz4-dev libjemalloc-dev +apt-get install -y libsecp256k1-dev libsodium-dev ninja-build fio rocksdb-tools liblz4-dev libjemalloc-dev automake libtool # bugfix if the files are in the wrong place wget "https://ton-blockchain.github.io/global.config.json" -O global.config.json @@ -67,6 +67,10 @@ ls -A1 | xargs rm -rf echo "https://github.com/${author}/${repo}.git -> ${branch}" git clone --recursive https://github.com/${author}/${repo}.git . git checkout ${branch} + +git submodule sync --recursive +git submodule update + export CC=/usr/bin/clang export CXX=/usr/bin/clang++ export CCACHE_DISABLE=1 diff --git a/mytoninstaller/mytoninstaller.py b/mytoninstaller/mytoninstaller.py index c5e60012..270afe97 100644 --- a/mytoninstaller/mytoninstaller.py +++ b/mytoninstaller/mytoninstaller.py @@ -194,7 +194,6 @@ def PrintLiteServerConfig(local, args): def CreateLocalConfigFile(local, args): initBlock = GetInitBlock() initBlock_b64 = dict2b64(initBlock) - user = local.buffer["user"] args = ["python3", "-m", "mytoninstaller", "-u", local.buffer.user, "-e", "clc", "-i", initBlock_b64] run_as_root(args) #end define diff --git a/mytoninstaller/scripts/jsonrpcinstaller.sh b/mytoninstaller/scripts/jsonrpcinstaller.sh index e6338480..926d61e1 100755 --- a/mytoninstaller/scripts/jsonrpcinstaller.sh +++ b/mytoninstaller/scripts/jsonrpcinstaller.sh @@ -28,7 +28,7 @@ ENDC='\033[0m' # Установка компонентов python3 echo -e "${COLOR}[1/4]${ENDC} Installing required packages" -pip3 install Werkzeug json-rpc cloudscraper pyotp +pip3 install Werkzeug json-rpc cloudscraper pyotp jsonpickle # todo: set versions # Клонирование репозиториев с github.com echo -e "${COLOR}[2/4]${ENDC} Cloning github repository" diff --git a/mytoninstaller/scripts/ton_http_api_installer.sh b/mytoninstaller/scripts/ton_http_api_installer.sh index fa87c42c..a5eab69c 100644 --- a/mytoninstaller/scripts/ton_http_api_installer.sh +++ b/mytoninstaller/scripts/ton_http_api_installer.sh @@ -12,7 +12,7 @@ COLOR='\033[92m' ENDC='\033[0m' # install python3 packages -pip3 install virtualenv +pip3 install virtualenv==20.27.1 # prepare the virtual environment echo -e "${COLOR}[1/4]${ENDC} Preparing the virtual environment" diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index 76e92908..cba219ea 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -462,6 +462,9 @@ def EnableJsonRpc(local): def enable_ton_http_api(local): local.add_log("start EnableTonHttpApi function", "debug") + if not os.path.exists('/usr/bin/ton/local.config.json'): + from mytoninstaller.mytoninstaller import CreateLocalConfigFile + CreateLocalConfigFile(local, []) ton_http_api_installer_path = pkg_resources.resource_filename('mytoninstaller.scripts', 'ton_http_api_installer.sh') exit_code = run_as_root(["bash", ton_http_api_installer_path]) if exit_code == 0: diff --git a/requirements.txt b/requirements.txt index f7705c47..01f7a310 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ -crc16 -requests -psutil -fastcrc -jsonpickle -pynacl +crc16==0.1.1 +requests==2.32.3 +psutil==6.1.0 +fastcrc==0.3.2 +pynacl==1.5.0 diff --git a/scripts/install.sh b/scripts/install.sh index dffe1134..1da6e42c 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -72,7 +72,7 @@ done if [ "${mode}" = "" ]; then # no mode echo "Running cli installer" wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/install.py - pip3 install inquirer + pip3 install inquirer==3.4.0 python3 install.py exit fi diff --git a/scripts/ton_installer.sh b/scripts/ton_installer.sh index 3ae64874..6454fd2a 100644 --- a/scripts/ton_installer.sh +++ b/scripts/ton_installer.sh @@ -55,7 +55,7 @@ if [ "$OSTYPE" == "linux-gnu" ]; then elif [ -f /etc/debian_version ]; then echo "Ubuntu/Debian Linux detected." apt-get update - apt-get install -y build-essential curl git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev libjemalloc-dev + apt-get install -y build-essential curl git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev libjemalloc-dev automake libtool # Install ninja apt-get install -y ninja-build @@ -92,7 +92,7 @@ else fi # Установка компонентов python3 -pip3 install psutil crc16 requests +pip3 install psutil==6.1.0 crc16==0.1.1 requests==2.32.3 # build openssl 3.0 echo -e "${COLOR}[2/6]${ENDC} Building OpenSSL 3.0" @@ -118,6 +118,9 @@ if [ "${ton_node_version}" != "master" ]; then cd ../ fi +git submodule sync --recursive +git submodule update + git config --global --add safe.directory $SOURCES_DIR/ton # Подготавливаем папки для компиляции diff --git a/tools/extract_backup_node_keys.sh b/tools/extract_backup_node_keys.sh new file mode 100644 index 00000000..adc4448c --- /dev/null +++ b/tools/extract_backup_node_keys.sh @@ -0,0 +1,48 @@ +name="backup.tar.gz" +dest="cleared_backup_$(hostname)_$(date +%s).tar.gz" +ton_db="" +tmp_dir="tmp/backup" +user=$(logname) + + +# Get arguments +while getopts n:d:t: flag +do + case "${flag}" in + n) name=${OPTARG};; + d) dest=${OPTARG};; + t) ton_db=${OPTARG};; + *) + echo "Flag -${flag} is not recognized. Aborting" + exit 1 ;; + esac +done + +rm -rf $tmp_dir +mkdir tmp/backup -p + +if [ ! -z "$ton_db" ]; then + mkdir ${tmp_dir}/db + cp -r "$ton_db"/db/keyring ${tmp_dir}/db + cp "$ton_db"/db/config.json ${tmp_dir}/db +else + tar -xzf $name -C $tmp_dir +fi + +rm -rf ${tmp_dir}/mytoncore +rm -rf ${tmp_dir}/keys +mv ${tmp_dir}/db/keyring ${tmp_dir}/db/old_keyring +mkdir ${tmp_dir}/db/keyring + +keys=$(python3 -c "import json;import base64;f=open('${tmp_dir}/db/config.json');config=json.load(f);f.close();keys=set();[([keys.add(base64.b64decode(key['key']).hex().upper()) for key in v['temp_keys']], [keys.add(base64.b64decode(adnl['id']).hex().upper()) for adnl in v['adnl_addrs']]) for v in config['validators']];print('\n'.join(list(keys)))") + +for key in $keys; do + mv ${tmp_dir}/db/old_keyring/${key} ${tmp_dir}/db/keyring +done + +rm -rf ${tmp_dir}/db/old_keyring + +tar -zcf $dest -C $tmp_dir . +chown $user:$user $dest + +echo -e "Node keys backup successfully created in ${dest}!" diff --git a/tools/inject_backup_node_keys.py b/tools/inject_backup_node_keys.py new file mode 100644 index 00000000..9566c788 --- /dev/null +++ b/tools/inject_backup_node_keys.py @@ -0,0 +1,49 @@ +import argparse +import base64 +import json +import subprocess +tmp_dir = "tmp/cleared_backup" + + +def b64tohex(b64str: str): + return base64.b64decode(b64str).hex().upper() + + +def run_vc(cmd: str): + args = ['/usr/bin/ton/validator-engine-console/validator-engine-console', '-k', '/var/ton-work/keys/client', '-p', '/var/ton-work/keys/server.pub', '-a', vc_address, '--cmd', cmd] + subprocess.run(args) + + +parser = argparse.ArgumentParser() +parser.add_argument('-n') +parser.add_argument('-a') + +args = parser.parse_args() +name = args.n +vc_address = args.a + +if not name or not vc_address: + print("Usage: inject_backup_node_keys.py -n -a ") + exit(1) + + +subprocess.run(f"rm -rf {tmp_dir}", shell=True) +subprocess.run(f"mkdir -p {tmp_dir}", shell=True) + +subprocess.run(f'tar -xzf {name} -C {tmp_dir}', shell=True) + +subprocess.run(f'cp -rf {tmp_dir}/db/keyring /var/ton-work/db/', shell=True) +subprocess.run(f'chown -R validator:validator /var/ton-work/db/keyring', shell=True) + +with open(f'{tmp_dir}/db/config.json', 'r') as f: + config = json.load(f) + +for v in config['validators']: + run_vc(f'addpermkey {b64tohex(v["id"])} {v["election_date"]} {v["expire_at"]}') + for tkey in v['temp_keys']: + run_vc(f'addtempkey {b64tohex(v["id"])} {b64tohex(tkey["key"])} {v["expire_at"]}') + for adnl in v['adnl_addrs']: + run_vc(f'addadnl {b64tohex(adnl["id"])} 0') + run_vc(f'addvalidatoraddr {b64tohex(v["id"])} {b64tohex(adnl["id"])} {v["expire_at"]}') + +subprocess.run(f'systemctl restart validator', shell=True)