From 57a1e223a9e42e9d1d7779ea4d0c5d7c9aab8378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 6 Aug 2024 14:09:07 +0100 Subject: [PATCH] Add new templating feature to allow to quickly override the default values of settings and custom configurations. You can also precise steps to follow in the UI to help the user configure services. --- CHANGELOG.md | 1 + docs/assets/img/bunkerweb_db.svg | 4 +- src/autoconf/Config.py | 7 +- .../confs/server-http/securitytxt.conf | 2 +- .../{templates => files}/security.txt | 0 src/common/core/templates/plugin.json | 8 + src/common/core/templates/templates/high.json | 1 + src/common/core/templates/templates/low.json | 144 ++ .../low/configs/modsec/anomaly_score.conf | 9 + .../core/templates/templates/medium.json | 1 + src/common/db/Database.py | 1693 +++++++++++------ src/common/db/model.py | 98 +- src/common/gen/Configurator.py | 181 +- src/common/gen/main.py | 21 +- src/common/gen/save_config.py | 53 +- src/common/settings.json | 9 + src/common/templates/high.json | 129 -- src/common/templates/low.json | 111 -- src/common/templates/medium.json | 110 -- src/ui/Dockerfile | 1 - 20 files changed, 1495 insertions(+), 1088 deletions(-) rename src/common/core/securitytxt/{templates => files}/security.txt (100%) create mode 100644 src/common/core/templates/plugin.json create mode 100644 src/common/core/templates/templates/high.json create mode 100644 src/common/core/templates/templates/low.json create mode 100644 src/common/core/templates/templates/low/configs/modsec/anomaly_score.conf create mode 100644 src/common/core/templates/templates/medium.json delete mode 100644 src/common/templates/high.json delete mode 100644 src/common/templates/low.json delete mode 100644 src/common/templates/medium.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d4021dcab..80dc505431 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - [FEATURE] Add new `REVERSE_PROXY_PASS_REQUEST_BODY` setting to control if the request body should be passed to the upstream server (default is yes) - [FEATURE] Jobs now have an history which the size can be controlled via the `DATABASE_MAX_JOBS_RUNS` setting (default is 10000) and it will be possible to see it in the web UI in a future release - [FEATURE] Add support for HTTP/3 connections limiting via the `HTTP3_CONNECTIONS_LIMIT` setting (default is 100) in the `limit` plugin +- [FEATURE] Add new templating feature to allow to quickly override the default values of settings and custom configurations. You can also precise steps to follow in the UI to help the user configure services. - [SCHEDULER] Refactor the scheduler to use the `BUNKERWEB_INSTANCES` (previously known as `OVERRIDE_INSTANCES`) environment variable instead of an integration specific system - [AUTOCONF] Add new `NAMESPACES` environment variable to allow setting the namespaces to watch for the autoconf feature which makes it possible to use multiple autoconf instances in the same cluster while keeping the configuration separated - [UI] Start refactoring the UI to make it more modular and easier to maintain with migration from Jinja to Vue.js diff --git a/docs/assets/img/bunkerweb_db.svg b/docs/assets/img/bunkerweb_db.svg index 56ce132cf2..fd15eb8262 100644 --- a/docs/assets/img/bunkerweb_db.svg +++ b/docs/assets/img/bunkerweb_db.svg @@ -1,4 +1,4 @@ -1*1*1*1*1*1*1*1*1*1*1*1*1*1*1*1*1*1*1*1*1*1*1*1*1*bw_instanceshostnamevarchar[256]portintserver_namevarchar[256]statusinstance_status_enummethodmethodscreation_datedatetimelast_seendatetimebw_services_settingsservice_idvarchar[64]setting_idvarchar[256]valuetextsuffixintmethodmethodsbw_servicesidvarchar[64]methodmethodsis_draftbooleanbw_jobs_cacheidintjob_namevarchar[128]service_idvarchar[64]file_namevarchar[256]datalongbloblast_updatedatetimechecksumvarchar[128]bw_jobs_runsidintjob_namevarchar[128]successbooleanstart_datedatetimeend_datedatetimebw_auditsidintlevelaudit_level_enummessagetextcreateddatetimetagtags_enumbw_global_valuessetting_idvarchar[256]valuetextsuffixintmethodmethodsbw_settingsidvarchar[256]namevarchar[256]plugin_idvarchar[64]contextcontexesdefaulttexthelpvarchar[512]labelvarchar[256]regexvarchar[1024]typesettings_typesmultiplevarchar[128]orderintbw_custom_configsidintservice_idvarchar[64]typecustom_config_typesnamevarchar[256]datalongblobchecksumvarchar[128]methodmethodsbw_jobsnamevarchar[128]plugin_idvarchar[64]file_namevarchar[256]everyschedulesreloadbooleanbw_ui_user_recovery_codesidintuser_namevarchar[256]codetinytextbw_template_settingsidinttemplate_idvarchar[256]setting_idvarchar[256]step_idintdefaulttextsuffixintbw_templatesidvarchar[256]namevarchar[256]plugin_idvarchar[64]bw_pluginsidvarchar[64]namevarchar[128]descriptionvarchar[256]versionvarchar[32]streamstream_typestypeplugin_typesmethodmethodsdatalongblobchecksumvarchar[128]config_changedbooleanlast_config_changedatetimebw_cli_commandsidintnamevarchar[64]plugin_idvarchar[64]file_namevarchar[256]bw_ui_user_actionsidintuser_namevarchar[256]createddatetimemessagetexttagtags_enumbw_template_custom_configsidinttemplate_idvarchar[256]step_idintnamevarchar[256]typecustom_config_typesdatalongblobchecksumvarchar[128]bw_selectssetting_idvarchar[256]valuevarchar[256]bw_ui_rolesnamevarchar[80]descriptionvarchar[256]update_datetimedatetimebw_plugin_pagesidintplugin_idvarchar[64]datalongblobchecksumvarchar[128]bw_ui_usersusernamevarchar[256]emailvarchar[256]passwordvarchar[256]methodmethodsadminboollast_login_atdatetimelast_login_ipvarchar[39]login_countinttotp_secretvarchar[256]totp_refreshedboolcreation_datedatetimeupdate_datedatetimebw_ui_roles_usersuser_namevarchar[256]role_namevarchar[80]bw_template_stepsidinttemplate_idvarchar[256]titletextsubtitletextbw_ui_roles_permissionsidintrole_namevarchar[80]permission_namevarchar[256]bw_ui_permissionsnamevarchar[256]bw_metadataidintis_initializedbooleanis_probooleanpro_licensevarchar[128]pro_expiredatetimepro_statuspro_status_enumpro_servicesintpro_overlappedbooleanlast_pro_checkdatetimefirst_config_savedbooleanautoconf_loadedbooleanscheduler_first_startbooleancustom_configs_changedbooleanlast_custom_configs_changedatetimeexternal_plugins_changedbooleanlast_external_plugins_changedatetimepro_plugins_changedbooleanlast_pro_plugins_changedatetimeinstances_changedbooleanlast_instances_changedatetimefailoverbooleanintegrationintegrationsversionvarchar \ No newline at end of file diff --git a/src/autoconf/Config.py b/src/autoconf/Config.py index 08db00806a..2c452a3da8 100644 --- a/src/autoconf/Config.py +++ b/src/autoconf/Config.py @@ -50,9 +50,10 @@ def __get_full_env(self) -> dict: for variable, value in service.items(): if variable == "NAMESPACE" or variable.startswith("CUSTOM_CONF") or not variable.isupper(): continue - if not self._db.is_setting(variable, multisite=True): - if variable in service: - self.__logger.warning(f"Variable {variable}: {value} is not a valid multisite setting, ignoring it") + + success, err = self._db.is_valid_setting(variable, value=value, multisite=True) + if not success: + self.__logger.warning(f"Variable {variable}: {value} is not a valid autoconf setting ({err}), ignoring it") continue config[f"{server_name}_{variable}"] = value config["SERVER_NAME"] += f" {server_name}" diff --git a/src/common/core/securitytxt/confs/server-http/securitytxt.conf b/src/common/core/securitytxt/confs/server-http/securitytxt.conf index c31122cd90..3cc1511d4b 100644 --- a/src/common/core/securitytxt/confs/server-http/securitytxt.conf +++ b/src/common/core/securitytxt/confs/server-http/securitytxt.conf @@ -1,7 +1,7 @@ {% if USE_SECURITYTXT == "yes" and SECURITYTXT_CONTACT != "" +%} location = {{ SECURITYTXT_URI }} { default_type 'text/plain; charset=utf-8'; - root /usr/share/bunkerweb/core/securitytxt/templates; + root /usr/share/bunkerweb/core/securitytxt/files; content_by_lua_block { local logger = require "bunkerweb.logger":new("SECURITYTXT") local helpers = require "bunkerweb.helpers" diff --git a/src/common/core/securitytxt/templates/security.txt b/src/common/core/securitytxt/files/security.txt similarity index 100% rename from src/common/core/securitytxt/templates/security.txt rename to src/common/core/securitytxt/files/security.txt diff --git a/src/common/core/templates/plugin.json b/src/common/core/templates/plugin.json new file mode 100644 index 0000000000..8dd97c2c55 --- /dev/null +++ b/src/common/core/templates/plugin.json @@ -0,0 +1,8 @@ +{ + "id": "templates", + "name": "Templates", + "description": "Fake core plugin for internal templates.", + "version": "1.0", + "stream": "yes", + "settings": {} +} diff --git a/src/common/core/templates/templates/high.json b/src/common/core/templates/templates/high.json new file mode 100644 index 0000000000..de42f72dbd --- /dev/null +++ b/src/common/core/templates/templates/high.json @@ -0,0 +1 @@ +{} // TODO diff --git a/src/common/core/templates/templates/low.json b/src/common/core/templates/templates/low.json new file mode 100644 index 0000000000..938c5b5623 --- /dev/null +++ b/src/common/core/templates/templates/low.json @@ -0,0 +1,144 @@ +{ + "name": "Basic security level", + "settings": { + "SERVER_NAME": "www.example.com", + "USE_REVERSE_PROXY": "yes", + "REVERSE_PROXY_HOST": "http://upstream-server:8080", + "REVERSE_PROXY_URL": "/", + "REVERSE_PROXY_CUSTOM_HOST": "", + "REVERSE_PROXY_SSL_SNI": "no", + "REVERSE_PROXY_SSL_SNI_NAME": "", + "REVERSE_PROXY_WS": "no", + "REVERSE_PROXY_KEEPALIVE": "no", + "AUTO_LETS_ENCRYPT": "yes", + "USE_LETS_ENCRYPT_STAGING": "no", + "ALLOWED_METHODS": "GET|POST|HEAD|OPTIONS|PUT|DELETE|PATCH", + "MAX_CLIENT_SIZE": "100m", + "HTTP2": "yes", + "HTTP3": "yes", + "SSL_PROTOCOLS": "TLSv1.2 TLSv1.3", + "COOKIE_FLAGS": "* SameSite=Lax", + "CONTENT_SECURITY_POLICY": "", + "PERMISSIONS_POLICY": "", + "KEEP_UPSTREAM_HEADERS": "*", + "REFERRER_POLICY": "no-referrer-when-downgrade", + "USE_CORS": "yes", + "CORS_ALLOW_ORIGIN": "*", + "USE_BAD_BEHAVIOR": "yes", + "BAD_BEHAVIOR_STATUS_CODES": "400 401 403 404 405 429 444", + "BAD_BEHAVIOR_BAN_TIME": "3600", + "BAD_BEHAVIOR_THRESHOLD": "30", + "BAD_BEHAVIOR_COUNT_TIME": "60", + "USE_ANTIBOT": "no", + "ANTIBOT_URI": "/challenge", + "ANTIBOT_RECAPTCHA_SCORE": "0.7", + "ANTIBOT_RECAPTCHA_SITEKEY": "", + "ANTIBOT_RECAPTCHA_SECRET": "", + "ANTIBOT_HCAPTCHA_SITEKEY": "", + "ANTIBOT_HCAPTCHA_SECRET": "", + "ANTIBOT_TURNSTILE_SITEKEY": "", + "ANTIBOT_TURNSTILE_SECRET": "", + "USE_BLACKLIST": "yes", + "USE_DNSBL": "no", + "USE_LIMIT_CONN": "yes", + "LIMIT_CONN_MAX_HTTP1": "25", + "LIMIT_CONN_MAX_HTTP2": "200", + "USE_LIMIT_REQ": "yes", + "LIMIT_REQ_URL": "/", + "LIMIT_REQ_RATE": "5r/s" + }, + "configs": ["modsec/anomaly_score.conf"], + "steps": [ + { + "title": "Web service - Front service", + "subtitle": "Configure your web service facing your clients", + "settings": [ + "SERVER_NAME", + "AUTO_LETS_ENCRYPT", + "USE_LETS_ENCRYPT_STAGING" + ] + }, + { + "title": "Web service - Upstream server", + "subtitle": "Configure the upstream server to be protected by BunkerWeb", + "settings": [ + "USE_REVERSE_PROXY", + "REVERSE_PROXY_HOST", + "REVERSE_PROXY_URL", + "REVERSE_PROXY_CUSTOM_HOST", + "REVERSE_PROXY_SSL_SNI", + "REVERSE_PROXY_SSL_SNI_NAME", + "REVERSE_PROXY_WS", + "REVERSE_PROXY_KEEPALIVE" + ] + }, + { + "title": "HTTP - General", + "subtitle": "Configure the settings related to the HTTP(S) protocol", + "settings": [ + "MAX_CLIENT_SIZE", + "ALLOWED_METHODS", + "HTTP2", + "HTTP3", + "SSL_PROTOCOLS" + ] + }, + { + "title": "HTTP - Headers", + "subtitle": "Configure the settings related to the HTTP headers", + "settings": [ + "COOKIE_FLAGS", + "CONTENT_SECURITY_POLICY", + "PERMISSIONS_POLICY", + "USE_CORS", + "CORS_ALLOW_ORIGIN", + "KEEP_UPSTREAM_HEADERS", + "REFERRER_POLICY" + ] + }, + { + "title": "Security - Bad behavior", + "subtitle": "Configure the settings related to the automatic ban when a bad behavior is detected.", + "settings": [ + "USE_BAD_BEHAVIOR", + "BAD_BEHAVIOR_STATUS_CODES", + "BAD_BEHAVIOR_BAN_TIME", + "BAD_BEHAVIOR_THRESHOLD", + "BAD_BEHAVIOR_COUNT_TIME" + ] + }, + { + "title": "Security - Blacklisting", + "subtitle": "Configure the settings related to the external blacklists.", + "settings": ["USE_BLACKLIST", "USE_DNSBL"] + }, + { + "title": "Security - Limiting", + "subtitle": "Configure the settings related to limiting requests and connections.", + "settings": [ + "USE_LIMIT_CONN", + "LIMIT_CONN_MAX_HTTP1", + "LIMIT_CONN_MAX_HTTP2", + "LIMIT_CONN_MAX_HTTP3", + "USE_LIMIT_REQ", + "LIMIT_REQ_URL", + "LIMIT_REQ_RATE" + ] + }, + { + "title": "Security - Antibot", + "subtitle": "Configure the settings about bot detection", + "settings": [ + "USE_ANTIBOT", + "ANTIBOT_URI", + "ANTIBOT_RECAPTCHA_SCORE", + "ANTIBOT_RECAPTCHA_SITEKEY", + "ANTIBOT_RECAPTCHA_SECRET", + "ANTIBOT_HCAPTCHA_SITEKEY", + "ANTIBOT_HCAPTCHA_SECRET", + "ANTIBOT_TURNSTILE_SITEKEY", + "ANTIBOT_TURNSTILE_SECRET" + ] + } + ] +} diff --git a/src/common/core/templates/templates/low/configs/modsec/anomaly_score.conf b/src/common/core/templates/templates/low/configs/modsec/anomaly_score.conf new file mode 100644 index 0000000000..14562163a9 --- /dev/null +++ b/src/common/core/templates/templates/low/configs/modsec/anomaly_score.conf @@ -0,0 +1,9 @@ +SecAction \ + "id:900110,\ + phase:1,\ + pass,\ + t:none,\ + nolog,\ + tag:'OWASP_CRS',\ + setvar:tx.inbound_anomaly_score_threshold=6,\ + setvar:tx.outbound_anomaly_score_threshold=5" diff --git a/src/common/core/templates/templates/medium.json b/src/common/core/templates/templates/medium.json new file mode 100644 index 0000000000..de42f72dbd --- /dev/null +++ b/src/common/core/templates/templates/medium.json @@ -0,0 +1 @@ +{} // TODO diff --git a/src/common/db/Database.py b/src/common/db/Database.py index f08b11bab7..a1f86c0ab5 100644 --- a/src/common/db/Database.py +++ b/src/common/db/Database.py @@ -4,17 +4,18 @@ from copy import deepcopy from datetime import datetime from io import BytesIO +from json import JSONDecodeError, loads from logging import Logger -from os import _exit, getenv, listdir, sep +from os import _exit, getenv, sep from os.path import join as os_join from pathlib import Path -from re import compile as re_compile, escape, search +from re import compile as re_compile, escape, error as RegexError, search from sys import argv, path as sys_path +from tarfile import open as tar_open from threading import Lock from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union from time import sleep from uuid import uuid4 -from zipfile import ZIP_DEFLATED, ZipFile from model import ( Base, @@ -30,7 +31,11 @@ Jobs_runs, Custom_configs, Selects, - BwcliCommands, + Bw_cli_commands, + Templates, + Template_steps, + Template_settings, + Template_custom_configs, Metadata, ) @@ -64,7 +69,6 @@ def set_sqlite_pragma(dbapi_connection, _): if isinstance(dbapi_connection, SQLiteConnection): cursor = dbapi_connection.cursor() - cursor.execute("PRAGMA foreign_keys=ON") cursor.execute("PRAGMA journal_mode=WAL") cursor.close() @@ -72,6 +76,7 @@ def set_sqlite_pragma(dbapi_connection, _): class Database: DB_STRING_RX = re_compile(r"^(?P(mariadb|mysql)(\+pymysql)?|sqlite(\+pysqlite)?|postgresql(\+psycopg)?):/+(?P/[^\s]+)") READONLY_ERROR = ("readonly", "read-only", "command denied", "Access denied") + RESTRICTED_TEMPLATE_SETTINGS = ("USE_TEMPLATE", "IS_DRAFT") def __init__( self, logger: Logger, sqlalchemy_string: Optional[str] = None, *, ui: bool = False, pool: Optional[bool] = None, log: bool = True, **kwargs @@ -300,9 +305,12 @@ def _db_session(self) -> Any: if session: session.remove() - def is_setting(self, setting: str, *, multisite: bool = False) -> bool: + def is_valid_setting( + self, setting: str, *, value: Optional[str] = None, multisite: bool = False, session: Optional[scoped_session] = None + ) -> Tuple[bool, str]: """Check if the setting exists in the database and optionally if it's multisite""" - with self._db_session() as session: + + def check_setting(session: scoped_session, setting: str, value: Optional[str], multisite: bool = False) -> Tuple[bool, str]: try: multiple = False if self.suffix_rx.search(setting): @@ -312,14 +320,36 @@ def is_setting(self, setting: str, *, multisite: bool = False) -> bool: db_setting = session.query(Settings).filter_by(id=setting).first() if not db_setting: - return False - elif multisite and db_setting.context != "multisite": - return False + for service in session.query(Services).with_entities(Services.id): + if setting.startswith(f"{service.id}_"): + db_setting = session.query(Settings).filter_by(id=setting.replace(f"{service.id}_", "")).first() + multisite = True + break + + if not db_setting: + return False, "missing" + + if multisite and db_setting.context != "multisite": + return False, "not multisite" elif multiple and db_setting.multiple is None: - return False - return True - except (ProgrammingError, OperationalError): - return False + return False, "not multiple" + + if value is not None: + try: + if search(db_setting.regex, value) is None: + return False, f"not matching regex: {db_setting.regex!r}" + except RegexError: + return False, f"invalid regex: {db_setting.regex!r}" + + return True, "" + except (ProgrammingError, OperationalError) as e: + return False, str(e) + + if session: + return check_setting(session, setting, value, multisite) + + with self._db_session() as session: + return check_setting(session, setting, value, multisite) def set_failover(self, value: bool = True) -> str: """Set the failover value""" @@ -568,63 +598,12 @@ def init_tables(self, default_plugins: List[dict], bunkerweb_version: str) -> Tu to_put = [] with self._db_session() as session: - db_plugins = old_data.get("bw_plugins", []) - - db_ids = [] - if db_plugins: - db_ids = [plugin.id for plugin in db_plugins] - ids = [plugin["id"] for plugin in default_plugins if "id" in plugin] - ids.append("general") - missing_ids = [plugin for plugin in db_ids if plugin not in ids] - - if missing_ids: - # Remove plugins that are no longer in the list - session.query(Plugins).filter(Plugins.id.in_(missing_ids)).delete() - session.query(Plugin_pages).filter(Plugin_pages.plugin_id.in_(missing_ids)).delete() - session.query(BwcliCommands).filter(BwcliCommands.plugin_id.in_(missing_ids)).delete() - - for plugin_job in session.query(Jobs).with_entities(Jobs.name).filter(Jobs.plugin_id.in_(missing_ids)): - session.query(Jobs_runs).filter(Jobs_runs.job_name == plugin_job.name).delete() - session.query(Jobs_cache).filter(Jobs_cache.job_name == plugin_job.name).delete() - session.query(Jobs).filter(Jobs.name == plugin_job.name).delete() - - for plugin_setting in session.query(Settings).with_entities(Settings.id).filter(Settings.plugin_id.in_(missing_ids)): - session.query(Selects).filter(Selects.setting_id == plugin_setting.id).delete() - session.query(Services_settings).filter(Services_settings.setting_id == plugin_setting.id).delete() - session.query(Global_values).filter(Global_values.setting_id == plugin_setting.id).delete() - session.query(Settings).filter(Settings.id == plugin_setting.id).delete() + saved_settings = set() for plugins in default_plugins: if not isinstance(plugins, list): plugins = [plugins] - plugin_filter = [plugin["id"] for plugin in plugins if "id" in plugin] - db_values = [plugin.id for plugin in old_data.get("bw_plugins", []) if plugin.id in plugin_filter] - missing_values = [plugin for plugin in db_values if plugin not in plugin_filter] - - if missing_values: - # Remove plugins that are no longer in the list - session.query(Plugins).filter(Plugins.id.in_(missing_values)).delete() - session.query(Plugin_pages).filter(Plugin_pages.plugin_id.in_(missing_values)).delete() - session.query(BwcliCommands).filter(BwcliCommands.plugin_id.in_(missing_values)).delete() - - for plugin_job in session.query(Jobs).with_entities(Jobs.name).filter(Jobs.plugin_id.in_(missing_values)): - session.query(Jobs_runs).filter(Jobs_runs.job_name == plugin_job.name).delete() - session.query(Jobs_cache).filter(Jobs_cache.job_name == plugin_job.name).delete() - session.query(Jobs).filter(Jobs.name == plugin_job.name).delete() - - for plugin_setting in session.query(Settings).with_entities(Settings.id).filter(Settings.plugin_id.in_(missing_values)): - session.query(Selects).filter(Selects.setting_id == plugin_setting.id).delete() - session.query(Services_settings).filter(Services_settings.setting_id == plugin_setting.id).delete() - session.query(Global_values).filter(Global_values.setting_id == plugin_setting.id).delete() - session.query(Settings).filter(Settings.id == plugin_setting.id).delete() - - if "bw_plugins" in old_data: - indexes = [i for i, plugin in enumerate(old_data["bw_plugins"]) if plugin.id in missing_values] - if indexes: - for i in indexes: - del old_data["bw_plugins"][i] - for plugin in plugins: settings = {} jobs = [] @@ -646,379 +625,536 @@ def init_tables(self, default_plugins: List[dict], bunkerweb_version: str) -> Tu if not isinstance(commands, dict): commands = {} - if "bw_plugins" in old_data: - found = False - for i, old_plugin in enumerate(old_data["bw_plugins"]): - if old_plugin.id == plugin["id"]: - found = True - break - - if found: + plugin_found = False + for i, db_plugin in enumerate(old_data.get("bw_plugins", [])): + if db_plugin.id == plugin["id"]: + plugin_found = True + if any(getattr(db_plugin, key, None) != plugin.get(key) for key in ("name", "description", "version", "stream", "type", "method")): + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}" already exists, updating it with the new values' + ) del old_data["bw_plugins"][i] + break + + if old_data and not plugin_found: + self.logger.warning(f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}" does not exist, creating it') + + to_put.append( + Plugins( + id=plugin["id"], + name=plugin["name"], + description=plugin["description"], + version=plugin["version"], + stream=plugin["stream"], + type=plugin.get("type", "core"), + method=plugin.get("method"), + data=plugin.get("data"), + checksum=plugin.get("checksum"), + ) + ) - db_plugin = session.query(Plugins).filter_by(id=plugin["id"]).first() - if db_plugin: - updates = {} + order = 0 + for setting, value in settings.items(): + value.update({"plugin_id": plugin["id"], "name": value["id"], "id": setting}) + select_values = value.pop("select", []) - if plugin["name"] != db_plugin.name: - updates[Plugins.name] = plugin["name"] + setting_found = False + if plugin_found: + for i, old_setting in enumerate(old_data.get("bw_settings", [])): + if old_setting.plugin_id == plugin["id"] and (old_setting.id == setting or old_setting.name == value["name"]): + setting_found = True + if any(getattr(old_setting, key, None) != data for key, data in value.items()): + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Setting "{setting}" already exists, updating it with the new values' + ) + del old_data["bw_settings"][i] + break - if plugin["description"] != db_plugin.description: - updates[Plugins.description] = plugin["description"] + if not setting_found: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Setting "{setting}" does not exist, creating it' + ) - if plugin["version"] != db_plugin.version: - updates[Plugins.version] = plugin["version"] + to_put.append(Settings(**value | {"order": order})) - if plugin["stream"] != db_plugin.stream: - updates[Plugins.stream] = plugin["stream"] + for select in select_values: + if plugin_found and setting_found: + select_found = False + for i, db_setting_select in enumerate(old_data.get("bw_selects", [])): + if db_setting_select.setting_id == value["id"] and db_setting_select.value == select: + select_found = True + del old_data["bw_selects"][i] + break - if plugin.get("type", "core") != db_plugin.type: - updates[Plugins.type] = plugin.get("type", "core") + if not select_found: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Setting "{setting}"\'s Select "{select}" does not exist, creating it' + ) - if plugin.get("method", "manual") != db_plugin.method: - updates[Plugins.method] = plugin.get("method", "manual") + to_put.append(Selects(setting_id=value["id"], value=select)) - if plugin.get("data") != db_plugin.data: - updates[Plugins.data] = plugin.get("data") + for i, old_select in enumerate(old_data.get("bw_selects", [])): + if old_select.setting_id == value["id"]: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Setting "{setting}"\'s Select "{old_select.value}" has been removed, deleting it' + ) + del old_data["bw_selects"][i] - if plugin.get("checksum") != db_plugin.checksum: - updates[Plugins.checksum] = plugin.get("checksum") + order += 1 + saved_settings.add(setting) - if updates: - self.logger.warning(f'Plugin "{plugin["id"]}" already exists, updating it with the new values') - session.query(Plugins).filter(Plugins.id == plugin["id"]).update(updates) - else: - to_put.append( - Plugins( - id=plugin["id"], - name=plugin["name"], - description=plugin["description"], - version=plugin["version"], - stream=plugin["stream"], - type=plugin.get("type", "core"), - method=plugin.get("method"), - data=plugin.get("data"), - checksum=plugin.get("checksum"), + for i, old_setting in enumerate(old_data.get("bw_settings", [])): + if old_setting.plugin_id == plugin["id"]: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Setting "{old_setting.id}" has been removed, deleting it' ) - ) - - db_values = [old_setting.id for old_setting in old_data.get("bw_settings", []) if old_setting.plugin_id == plugin["id"]] - missing_values = [setting for setting in db_values if setting not in settings] - if missing_values: - # Remove settings that are no longer in the list - self.logger.warning(f'Removing {len(missing_values)} settings from plugin "{plugin["id"]}" as they are no longer in the list') - session.query(Settings).filter(Settings.id.in_(missing_values)).delete() - session.query(Selects).filter(Selects.setting_id.in_(missing_values)).delete() - session.query(Services_settings).filter(Services_settings.setting_id.in_(missing_values)).delete() - session.query(Global_values).filter(Global_values.setting_id.in_(missing_values)).delete() - - if "bw_settings" in old_data: - indexes = [ - i for i, setting in enumerate(old_data["bw_settings"]) if setting.plugin_id == plugin["id"] and setting.id in missing_values - ] - if indexes: - for i in indexes: - del old_data["bw_settings"][i] + for j, old_select in enumerate(old_data.get("bw_selects", [])): + if old_select.setting_id == old_setting.id: + del old_data["bw_selects"][j] - order = 0 - for setting, value in settings.items(): - value.update({"plugin_id": plugin["id"], "name": value["id"], "id": setting}) + for j, old_global_value in enumerate(old_data.get("bw_global_values", [])): + if old_global_value.setting_id == old_setting.id: + del old_data["bw_global_values"][j] - if "bw_settings" in old_data: - found = False - for i, old_setting in enumerate(old_data["bw_settings"]): - if old_setting.id == value["id"]: - found = True - break + for j, old_service_setting in enumerate(old_data.get("bw_services_settings", [])): + if old_service_setting.setting_id == old_setting.id: + del old_data["bw_services_settings"][j] - if found: - del old_data["bw_settings"][i] + del old_data["bw_settings"][i] - db_setting = session.query(Settings).filter_by(id=setting).first() - select_values = value.pop("select", []) + for job in jobs: + job["file_name"] = job.pop("file") + job["reload"] = job.get("reload", False) + + if plugin_found: + job_found = False + for i, old_job in enumerate(old_data.get("bw_jobs", [])): + if old_job.plugin_id == plugin["id"] and old_job.name == job["name"]: + job_found = True + if any(getattr(old_job, key, None) != data for key, data in job.items()): + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Job "{job["name"]}" already exists, updating it with the new values' + ) + del old_data["bw_jobs"][i] + break - if db_setting: - updates = {} + if not job_found: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Job "{job["name"]}" does not exist, creating it' + ) - if value["plugin_id"] != db_setting.plugin_id: - updates[Settings.plugin_id] = value["plugin_id"] + to_put.append(Jobs(plugin_id=plugin["id"], **job)) - if value["name"] != db_setting.name: - updates[Settings.name] = value["name"] + for i, old_job in enumerate(old_data.get("bw_jobs", [])): + if old_job.plugin_id == plugin["id"]: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Job "{old_job.name}" has been removed, deleting it' + ) - if value["context"] != db_setting.context: - updates[Settings.context] = value["context"] + for j, old_cache in enumerate(old_data.get("bw_jobs_cache", [])): + if old_cache.job_id == old_job.id: + del old_data["bw_jobs_cache"][j] - if value["default"] != db_setting.default: - updates[Settings.default] = value["default"] + for j, old_run in enumerate(old_data.get("bw_jobs_runs", [])): + if old_run.job_id == old_job.id: + del old_data["bw_jobs_runs"][j] - if value["help"] != db_setting.help: - updates[Settings.help] = value["help"] + del old_data["bw_jobs"][i] - if value["label"] != db_setting.label: - updates[Settings.label] = value["label"] + plugin_path = ( + Path(sep, "usr", "share", "bunkerweb", "core", plugin["id"]) + if plugin.get("type", "core") == "core" + else ( + Path(sep, "etc", "bunkerweb", "plugins", plugin["id"]) + if plugin.get("type", "core") == "external" + else Path(sep, "etc", "bunkerweb", "pro", "plugins", plugin["id"]) + ) + ) + path_ui = plugin_path.joinpath("ui") - if value["regex"] != db_setting.regex: - updates[Settings.regex] = value["regex"] + if path_ui.is_dir(): + with BytesIO() as plugin_page_content: + with tar_open(fileobj=plugin_page_content, mode="w:gz", compresslevel=9) as tar: + tar.add(path_ui, arcname=path_ui.name, recursive=True) + plugin_page_content.seek(0) + checksum = bytes_hash(plugin_page_content, algorithm="sha256") + + if plugin_found: + page_found = False + for i, plugin_page in enumerate(old_data.get("bw_plugin_pages", [])): + if plugin_page.plugin_id == plugin["id"]: + page_found = True + if getattr(plugin_page, "checksum", None) != checksum or getattr(plugin_page, "template_file", None): + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Page already exists, updating it with the new values' + ) + del old_data["bw_plugin_pages"][i] + break + + if not page_found: + self.logger.warning(f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Page does not exist, creating it') + + to_put.append( + Plugin_pages( + plugin_id=plugin["id"], + data=plugin_page_content.getvalue(), + checksum=checksum, + ) + ) - if value["type"] != db_setting.type: - updates[Settings.type] = value["type"] + for i, old_plugin_page in enumerate(old_data.get("bw_plugin_pages", [])): + if old_plugin_page.plugin_id == plugin["id"]: + self.logger.warning(f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Page has been removed, deleting it') + del old_data["bw_plugin_pages"][i] - if value.get("multiple") != db_setting.multiple: - updates[Settings.multiple] = value.get("multiple") + for command, file_name in commands.items(): + if plugin_found: + command_found = False + for i, old_command in enumerate(old_data.get("bw_cli_commands", [])): + if old_command.plugin_id == plugin["id"] and old_command.name == command: + command_found = True + if old_command.file_name != file_name: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Command "{command}" already exists, updating it with the new file name' + ) + del old_data["bw_cli_commands"][i] + break - if order != db_setting.order: - updates[Settings.order] = order + if not command_found: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Command "{command}" does not exist, creating it' + ) - if updates: - self.logger.warning(f'Setting "{setting}" already exists, updating it with the new values') - session.query(Settings).filter(Settings.id == setting).update(updates) - else: - if db_plugin: - self.logger.warning(f'Setting "{setting}" does not exist, creating it') - to_put.append(Settings(**value | {"order": order})) + to_put.append(Bw_cli_commands(name=command, plugin_id=plugin["id"], file_name=file_name)) - db_values = [select.value for select in old_data.get("bw_selects", []) if select.setting_id == value["id"]] - missing_values = [select for select in db_values if select not in select_values] + for i, old_command in enumerate(old_data.get("bw_cli_commands", [])): + if old_command.plugin_id == plugin["id"]: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Command "{old_command.name}" has been removed, deleting it' + ) + del old_data["bw_cli_commands"][i] - if missing_values: - # Remove selects that are no longer in the list - self.logger.warning(f'Removing {len(missing_values)} selects from setting "{setting}" as they are no longer in the list') - session.query(Selects).filter(Selects.value.in_(missing_values)).delete() + # ? Add potential external/pro settings to saved_settings + for i, old_plugin in enumerate(old_data.get("bw_plugins", [])): + if getattr(old_plugin, "external", False) or getattr(old_plugin, "type", "core") != "core": + for j, old_setting in enumerate(old_data.get("bw_settings", [])): + if old_setting.plugin_id == old_plugin.id: + saved_settings.add(old_setting.id) - if "bw_selects" in old_data: - indexes = [ - i for i, select in enumerate(old_data["bw_selects"]) if select.setting_id == value["id"] and select.value in missing_values - ] - if indexes: - for i in indexes: - del old_data["bw_selects"][i] + for plugins in default_plugins: + if not isinstance(plugins, list): + plugins = [plugins] - for select in select_values: - if select not in db_values: - to_put.append(Selects(setting_id=value["id"], value=select)) + for plugin in plugins: + plugin_path = ( + Path(sep, "usr", "share", "bunkerweb", "core", plugin.get("id", "general")) + if plugin.get("type", "core") == "core" + else ( + Path(sep, "etc", "bunkerweb", "plugins", plugin.get("id", "general")) + if plugin.get("type", "core") == "external" + else Path(sep, "etc", "bunkerweb", "pro", "plugins", plugin.get("id", "general")) + ) + ) + templates_path = plugin_path.joinpath("templates") - order += 1 + if not templates_path.is_dir(): + continue - db_names = [job.name for job in old_data.get("bw_jobs", []) if job.plugin_id == plugin["id"]] - job_names = [job["name"] for job in jobs] - missing_names = [job for job in db_names if job not in job_names] + for template_file in templates_path.iterdir(): + if template_file.is_dir(): + continue - if missing_names: - # Remove jobs that are no longer in the list - self.logger.warning(f'Removing {len(missing_names)} jobs from plugin "{plugin["id"]}" as they are no longer in the list') - session.query(Jobs).filter(Jobs.name.in_(missing_names), Jobs.plugin_id == plugin["id"]).delete() - session.query(Jobs_cache).filter(Jobs_cache.job_name.in_(missing_names)).delete() - session.query(Jobs_runs).filter(Jobs_runs.job_name.in_(missing_names)).delete() + try: + template_data = loads(template_file.read_text()) + except JSONDecodeError: + self.logger.error( + f"{plugin.get('type', 'core').title()} Plugin \"{plugin['id']}\"'s Template file \"{template_file}\" is not a valid JSON file" + ) + continue - if "bw_jobs" in old_data: - indexes = [i for i, job in enumerate(old_data["bw_jobs"]) if job.plugin_id == plugin["id"] and job.name in missing_names] - if indexes: - for i in indexes: - del old_data["bw_jobs"][i] + template_id = template_file.stem - for job in jobs: - if "bw_jobs" in old_data: - found = False - for i, old_job in enumerate(old_data["bw_jobs"]): - if old_job.name == job["name"]: - found = True + if plugin_found: + template_found = False + for i, old_template in enumerate(old_data.get("bw_templates", [])): + if old_template.plugin_id == plugin.get("id", "general") and old_template.id == template_id: + template_found = True + del old_data["bw_templates"][i] break - if found: - del old_data["bw_jobs"][i] - - db_job = ( - session.query(Jobs) - .with_entities(Jobs.file_name, Jobs.every, Jobs.reload) - .filter_by(name=job["name"], plugin_id=plugin["id"]) - .first() - ) + if not template_found: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}" does not exist, creating it' + ) - if job["name"] not in db_names or not db_job: - job["file_name"] = job.pop("file") - job["reload"] = job.get("reload", False) - if db_plugin: - self.logger.warning(f'Job "{job["name"]}" does not exist, creating it') - to_put.append(Jobs(plugin_id=plugin["id"], **job)) - else: - updates = {} + to_put.append(Templates(id=template_id, plugin_id=plugin.get("id", "general"), name=template_data.get("name", template_id))) + + steps_settings = {} + steps_configs = {} + for step_id, step in enumerate(template_data.get("steps", []), start=1): + if plugin_found and template_found: + step_found = False + for i, old_step in enumerate(old_data.get("bw_template_steps", [])): + if old_step.template_id == template_id and old_step.id == step_id: + step_found = True + if old_step.title != step["title"] or old_step.subtitle != step["subtitle"]: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Step "{step["name"]}" already exists, updating it with the new values' + ) + del old_data["bw_template_steps"][i] + break + + if not step_found: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Step "{step["name"]}" does not exist, creating it' + ) - if job["file"] != db_job.file_name: - updates[Jobs.file_name] = job["file"] + to_put.append(Template_steps(id=step_id, template_id=template_id, title=step["title"], subtitle=step["subtitle"])) - if job["every"] != db_job.every: - updates[Jobs.every] = job["every"] + for setting in step.get("settings", []): + if step_id not in steps_settings: + steps_settings[step_id] = [] + steps_settings[step_id].append(setting) - if job.get("reload", None) != db_job.reload: - updates[Jobs.reload] = job.get("reload", False) + for config in step.get("configs", []): + if step_id not in steps_configs: + steps_configs[step_id] = [] + steps_configs[step_id].append(config) - if updates: - self.logger.warning(f'Job "{job["name"]}" already exists, updating it with the new values') - updates[Jobs.last_run] = None - session.query(Jobs_runs).filter(Jobs_runs.job_name == job["name"]).delete() - session.query(Jobs_cache).filter(Jobs_cache.job_name == job["name"]).delete() - session.query(Jobs).filter(Jobs.name == job["name"]).update(updates) + for i, old_step in enumerate(old_data.get("bw_template_steps", [])): + if old_step.template_id == template_id: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Step "{old_step.id}" has been removed, deleting it' + ) - if "bw_plugin_pages" in old_data: - found = False - for i, plugin_page in enumerate(old_data["bw_plugin_pages"]): - if plugin_page.plugin_id == plugin["id"]: - found = True - break + for j, old_setting in enumerate(old_data.get("bw_template_settings", [])): + if old_setting.step_id == old_step.id: + old_data["bw_template_settings"][j]["step_id"] = None - if found: - del old_data["bw_plugin_pages"][i] + for j, old_config in enumerate(old_data.get("bw_template_configs", [])): + if old_config.step_id == old_step.id: + old_data["bw_template_configs"][j]["step_id"] = None - plugin_path = ( - Path(sep, "usr", "share", "bunkerweb", "core", plugin["id"]) - if plugin.get("type", "core") == "core" - else ( - Path(sep, "etc", "bunkerweb", "plugins", plugin["id"]) - if plugin.get("type", "core") == "external" - else Path(sep, "etc", "bunkerweb", "pro", "plugins", plugin["id"]) - ) - ) + del old_data["bw_template_steps"][i] - path_ui = plugin_path.joinpath("ui") + for setting, default in template_data.get("settings", {}).items(): + setting_id, suffix = setting.rsplit("_", 1) if self.suffix_rx.search(setting) else (setting, None) - db_plugin_page = ( - session.query(Plugin_pages) - .with_entities( - Plugin_pages.template_checksum, - Plugin_pages.actions_checksum, - Plugin_pages.obfuscation_checksum, - ) - .filter_by(plugin_id=plugin["id"]) - .first() - ) - remove = not path_ui.is_dir() and db_plugin_page + if setting_id in self.RESTRICTED_TEMPLATE_SETTINGS: + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template {template_id} has a restricted setting: {setting}, skipping it' + ) + continue + elif setting_id not in saved_settings: + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template {template_id} has an invalid setting: {setting}, skipping it' + ) + continue - if path_ui.is_dir(): - remove = True - if {"template.html", "actions.py"}.issubset(listdir(str(path_ui))): - template = path_ui.joinpath("template.html").read_bytes() - actions = path_ui.joinpath("actions.py").read_bytes() - template_checksum = bytes_hash(template, algorithm="sha256") - actions_checksum = bytes_hash(actions, algorithm="sha256") - - obfuscation_file = None - obfuscation_checksum = None - obfuscation_dir = path_ui.joinpath("pyarmor_runtime_000000") - if obfuscation_dir.is_dir(): - obfuscation_file = BytesIO() - with ZipFile(obfuscation_file, "w", ZIP_DEFLATED) as zip_file: - for path in obfuscation_dir.rglob("*"): - if path.is_file(): - zip_file.write(path, path.relative_to(path_ui)) - obfuscation_file.seek(0, 0) - obfuscation_file = obfuscation_file.getvalue() - obfuscation_checksum = bytes_hash(obfuscation_file, algorithm="sha256") - - if db_plugin_page: - updates = {} - if template_checksum != db_plugin_page.template_checksum: - updates.update( - { - Plugin_pages.template_file: template, - Plugin_pages.template_checksum: template_checksum, - } + if plugin_found and template_found: + setting_found = False + for i, old_setting in enumerate(old_data.get("bw_template_settings", [])): + if old_setting.template_id == template_id and old_setting.id == setting_id and old_setting.suffix == suffix: + setting_found = True + if old_setting.default != default: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Setting "{setting}" already exists, updating it with the new default value' + ) + del old_data["bw_template_settings"][i] + break + + if not setting_found: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Setting "{setting}" does not exist, creating it' ) - if actions_checksum != db_plugin_page.actions_checksum: - updates.update( - { - Plugin_pages.actions_file: actions, - Plugin_pages.actions_checksum: actions_checksum, - } + step_id = None + for step, settings in steps_settings.items(): + if setting in settings: + step_id = step + break + + if step_id: + to_put.append( + Template_settings( + template_id=template_id, + setting_id=setting_id, + step_id=step_id, + default=default, + suffix=suffix, ) + ) + continue + + to_put.append( + Template_settings( + template_id=template_id, + setting_id=setting_id, + default=default, + suffix=suffix, + ) + ) + + for i, old_setting in enumerate(old_data.get("bw_template_settings", [])): + if old_setting.template_id == template_id: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Setting "{old_setting.id}" has been removed, deleting it' + ) + del old_data["bw_template_settings"][i] + + for config in template_data.get("configs", []): + try: + config_type, config_name = config.split("/", 1) + except ValueError: + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template {template_id} has an invalid config: {config}' + ) + continue + + if not templates_path.joinpath(template_id, "configs", config_type, config_name).is_file(): + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template {template_id} has a missing config: {config}' + ) + continue - if obfuscation_checksum != db_plugin_page.obfuscation_checksum: - updates.update( - { - Plugin_pages.obfuscation_file: obfuscation_file, - Plugin_pages.obfuscation_checksum: obfuscation_checksum, - } + content = templates_path.joinpath(template_id, "configs", config_type, config_name).read_bytes() + checksum = bytes_hash(content, algorithm="sha256") + + config_name = config_name.replace(".conf", "") + + if plugin_found and template_found: + config_found = False + for i, old_config in enumerate(old_data.get("bw_template_configs", [])): + if old_config.template_id == template_id and old_config.name == config_name and old_config.type == config_type: + config_found = True + if old_config.checksum != checksum: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Custom config "{config}" already exists, updating it with the new values' + ) + del old_data["bw_template_configs"][i] + break + + if not config_found: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Custom config "{config}" does not exist, creating it' ) - if updates: - self.logger.warning(f'Page for plugin "{plugin["id"]}" already exists, updating it with the new values') - session.query(Plugin_pages).filter(Plugin_pages.plugin_id == plugin["id"]).update(updates) - remove = False - else: - if db_plugin: - self.logger.warning(f'Page for plugin "{plugin["id"]}" does not exist, creating it') + step_id = None + for step, configs in steps_configs.items(): + if config in configs: + step_id = step + break + if step_id: to_put.append( - Plugin_pages( - plugin_id=plugin["id"], - template_file=template, - template_checksum=template_checksum, - actions_file=actions, - actions_checksum=actions_checksum, - obfuscation_file=obfuscation_file, - obfuscation_checksum=obfuscation_checksum, + Template_custom_configs( + template_id=template_id, + step_id=step_id, + type=config_type, + name=config_name, + data=content, + checksum=checksum, ) ) - remove = False + continue - if db_plugin_page and remove: - self.logger.warning(f'Removing page for plugin "{plugin["id"]}" as it no longer exists') - session.query(Plugin_pages).filter_by(plugin_id=plugin["id"]).delete() + to_put.append( + Template_custom_configs( + template_id=template_id, + type=config_type, + name=config_name, + data=content, + checksum=checksum, + ) + ) - db_names = [command.name for command in old_data.get("bwcli_commands", []) if command.plugin_id == plugin["id"]] - missing_names = [command for command in db_names if command not in commands] + for i, old_config in enumerate(old_data.get("bw_template_configs", [])): + if old_config.template_id == template_id: + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Custom config "{old_config.name}" has been removed, deleting it' + ) + del old_data["bw_template_configs"][i] - if missing_names: - # Remove commands that are no longer in the list - self.logger.warning(f'Removing {len(missing_names)} commands from plugin "{plugin["id"]}" as they are no longer in the list') - session.query(BwcliCommands).filter(BwcliCommands.name.in_(missing_names), BwcliCommands.plugin_id == plugin["id"]).delete() + for i, old_template in enumerate(old_data.get("bw_templates", [])): + if old_template.plugin_id == plugin.get("id", "general"): + self.logger.warning( + f'{plugin.get("type", "core").title()} Plugin "{plugin.get("id", "general")}"\'s Template "{old_template.id}" has been removed, deleting it' + ) - if "bwcli_commands" in old_data: - indexes = [ - i for i, command in enumerate(old_data["bwcli_commands"]) if command.plugin_id == plugin["id"] and command.name in missing_names - ] - if indexes: - for i in indexes: - del old_data["bwcli_commands"][i] + for j, old_step in enumerate(old_data.get("bw_template_steps", [])): + if old_step.template_id == old_template.id: + del old_data["bw_template_steps"][j] - for command, file_name in commands.items(): - if "bwcli_commands" in old_data: - found = False - for i, old_command in enumerate(old_data["bwcli_commands"]): - if old_command.name == command: - found = True - break + for j, old_setting in enumerate(old_data.get("bw_template_settings", [])): + if old_setting.template_id == old_template.id: + del old_data["bw_template_settings"][j] - if found: - del old_data["bwcli_commands"][i] + for j, old_config in enumerate(old_data.get("bw_template_configs", [])): + if old_config.template_id == old_template.id: + del old_data["bw_template_configs"][j] - db_command = session.query(BwcliCommands).with_entities(BwcliCommands.file_name).filter_by(name=command, plugin_id=plugin["id"]).first() - command_path = plugin_path.joinpath("bwcli", file_name) + del old_data["bw_templates"][i] - if command not in db_names or not db_command: - if db_plugin: - self.logger.warning(f'Command "{command}" does not exist, creating it') + for i, old_plugin in enumerate(old_data.get("bw_plugins", [])): + if not getattr(old_plugin, "external", False) and getattr(old_plugin, "type", "core") == "core": + self.logger.warning(f'Core plugin "{old_plugin.id}" has been removed, deleting it') - if not command_path.is_file(): - self.logger.warning(f'Command "{command}"\'s file "{file_name}" does not exist in the plugin directory, skipping it') - continue + for j, old_setting in enumerate(old_data.get("bw_settings", [])): + if old_setting.plugin_id == old_plugin.id: + for k, old_select in enumerate(old_data.get("bw_selects", [])): + if old_select.setting_id == old_setting.id: + del old_data["bw_selects"][k] - to_put.append(BwcliCommands(name=command, plugin_id=plugin["id"], file_name=file_name)) - else: - updates = {} + for k, old_global_value in enumerate(old_data.get("bw_global_values", [])): + if old_global_value.setting_id == old_setting.id: + del old_data["bw_global_values"][k] - if file_name != db_command.file_name: - updates[BwcliCommands.file_name] = file_name + for k, old_service_setting in enumerate(old_data.get("bw_services_settings", [])): + if old_service_setting.setting_id == old_setting.id: + del old_data["bw_services_settings"][k] - if updates: - self.logger.warning(f'Command "{command}" already exists, updating it with the new values') - if not command_path.is_file(): - self.logger.warning(f'Command "{command}"\'s file "{file_name}" does not exist in the plugin directory, removing it') - session.query(BwcliCommands).filter_by(name=command, plugin_id=plugin["id"]).delete() - continue - session.query(BwcliCommands).filter_by(name=command, plugin_id=plugin["id"]).update(updates) + del old_data["bw_settings"][j] + + for j, old_job in enumerate(old_data.get("bw_jobs", [])): + if old_job.plugin_id == old_plugin.id: + for k, old_cache in enumerate(old_data.get("bw_jobs_cache", [])): + if old_cache.job_id == old_job.id: + del old_data["bw_jobs_cache"][k] + + for k, old_run in enumerate(old_data.get("bw_jobs_runs", [])): + if old_run.job_id == old_job.id: + del old_data["bw_jobs_runs"][k] + + del old_data["bw_jobs"][j] + + for j, old_page in enumerate(old_data.get("bw_plugin_pages", [])): + if old_page.plugin_id == old_plugin.id: + del old_data["bw_plugin_pages"][j] + + for j, old_command in enumerate(old_data.get("bw_cli_commands", [])): + if old_command.plugin_id == old_plugin.id: + del old_data["bw_cli_commands"][j] + + for j, old_template in enumerate(old_data.get("bw_templates", [])): + if old_template.plugin_id == old_plugin.id: + for k, old_step in enumerate(old_data.get("bw_template_steps", [])): + if old_step.template_id == old_template.id: + del old_data["bw_template_steps"][k] + + for k, old_setting in enumerate(old_data.get("bw_template_settings", [])): + if old_setting.template_id == old_template.id: + del old_data["bw_template_settings"][k] + + for k, old_config in enumerate(old_data.get("bw_template_configs", [])): + if old_config.template_id == old_template.id: + del old_data["bw_template_configs"][k] + + del old_data["bw_templates"][j] + + del old_data["bw_plugins"][i] + + self.logger.debug(f"Remaining data: {old_data}") try: session.add_all(to_put) @@ -1034,26 +1170,18 @@ def init_tables(self, default_plugins: List[dict], bunkerweb_version: str) -> Tu self.logger.warning(f'Restoring data for table "{table_name}"') self.logger.debug(f"Data: {data}") for row in data: - has_external_column = "external" in row - row = { - column: getattr(row, column) - for column in Base.metadata.tables[table_name].columns.keys() + (["external"] if has_external_column else []) - if hasattr(row, column) - } + external_column = getattr(row, "external", None) + row = {column: getattr(row, column) for column in Base.metadata.tables[table_name].columns.keys() if hasattr(row, column)} # ? As the external column has been replaced by the type column, we need to update the data if the column exists - if table_name == "bw_plugins" and "external" in row: - row["type"] = "external" if row.pop("external") else "core" + if table_name == "bw_plugins" and external_column is not None: + row["type"] = "external" if external_column else "core" with self._db_session() as session: try: if table_name == "bw_metadata": - existing_row = session.query(Metadata).filter_by(id=1).first() - if not existing_row: - session.add(Metadata(**(row | {"ui_version": db_ui_version}))) - session.commit() - continue - session.query(Metadata).filter_by(id=1).update(row | {"ui_version": db_ui_version}) + session.add(Metadata(**(row | {"ui_version": db_ui_version}))) + session.commit() continue # Check if the row already exists in the table @@ -1068,6 +1196,26 @@ def init_tables(self, default_plugins: List[dict], bunkerweb_version: str) -> Tu continue self.logger.debug(e) + with self._db_session() as session: + for template_setting in session.query(Template_settings): + success, err = self.is_valid_setting( + f"{template_setting.setting_id}_{template_setting.suffix}" if template_setting.suffix else template_setting.setting_id, + value=template_setting.default, + multisite=True, + session=session, + ) + + if not success: + self.logger.warning( + f'Template "{template_setting.template_id}"\'s Setting "{template_setting.setting_id}" isn\'t a valid template setting ({err}), deleting it' + ) + session.query(Template_settings).filter_by(id=template_setting.id).delete() + + try: + session.commit() + except BaseException as e: + return False, str(e) + return True, "" def save_config(self, config: Dict[str, Any], method: str, changed: Optional[bool] = True) -> Union[str, Set[str]]: @@ -1166,8 +1314,15 @@ def save_config(self, config: Dict[str, Any], method: str, changed: Optional[boo session.query(Services).filter(Services.id == draft).update({Services.is_draft: True}) changed_services = True + template = config.get("USE_TEMPLATE", "") + if config.get("MULTISITE", "no") == "yes": self.logger.debug("Checking if the multisite settings have changed") + + service_templates = {} + for service in services: + service_templates[service] = config.get(f"{service}_USE_TEMPLATE", template) + global_values = [] for key, value in config.copy().items(): suffix = 0 @@ -1206,9 +1361,25 @@ def save_config(self, config: Dict[str, Any], method: str, changed: Optional[boo .first() ) + template_setting = None + if service_templates[server_name]: + template_setting = ( + session.query(Template_settings) + .with_entities(Template_settings.default) + .filter_by(template_id=service_templates[server_name], setting_id=key, suffix=suffix) + .first() + ) + if not service_setting: if key != "SERVER_NAME" and ( - (original_key not in config and original_key not in db_config and value == setting.default) + ( + original_key not in config + and original_key not in db_config + and ( + (service_templates[server_name] and value == template_setting.default) + or (not service_templates[server_name] and value == setting.default) + ) + ) or (original_key in config and value == config[original_key]) or (original_key in db_config and value == db_config[original_key]) ): @@ -1228,7 +1399,14 @@ def save_config(self, config: Dict[str, Any], method: str, changed: Optional[boo ) if key != "SERVER_NAME" and ( - (original_key not in config and original_key not in db_config and value == setting.default) + ( + original_key not in config + and original_key not in db_config + and ( + (service_templates[server_name] and value == template_setting.default) + or (not service_templates[server_name] and value == setting.default) + ) + ) or (original_key in config and value == config[original_key]) or (original_key in db_config and value == db_config[original_key]) ): @@ -1247,8 +1425,17 @@ def save_config(self, config: Dict[str, Any], method: str, changed: Optional[boo .first() ) + template_setting = None + if template: + template_setting = ( + session.query(Template_settings) + .with_entities(Template_settings.default) + .filter_by(template_id=template, setting_id=key, suffix=suffix) + .first() + ) + if not global_value: - if value == setting.default: + if (template_setting and value == template_setting.default) or (not template_setting and value == setting.default): continue self.logger.debug(f"Adding global setting {key}") @@ -1260,7 +1447,7 @@ def save_config(self, config: Dict[str, Any], method: str, changed: Optional[boo changed_plugins.add(setting.plugin_id) query = session.query(Global_values).filter(Global_values.setting_id == key, Global_values.suffix == suffix) - if value == setting.default: + if (template_setting and value == template_setting.default) or (not template_setting and value == setting.default): self.logger.debug(f"Removing global setting {key}") query.delete() continue @@ -1269,16 +1456,24 @@ def save_config(self, config: Dict[str, Any], method: str, changed: Optional[boo query.update({Global_values.value: value, Global_values.method: method}) elif method != "autoconf": self.logger.debug("Checking if non multisite settings have changed") - if ( - config.get("SERVER_NAME", "www.example.com") - and not session.query(Services) - .with_entities(Services.id) - .filter_by(id=config.get("SERVER_NAME", "www.example.com").split(" ")[0]) - .first() - ): - self.logger.debug("Adding service www.example.com") - to_put.append(Services(id=config.get("SERVER_NAME", "www.example.com").split(" ")[0], method=method)) - changed_services = True + + server_name = config.get("SERVER_NAME", None) + if template and server_name is None: + server_name = ( + session.query(Template_settings) + .with_entities(Template_settings.value) + .filter_by(template_id=template, setting_id="SERVER_NAME") + .first() + ) + + if server_name is None or server_name: + server_name = server_name or "www.example.com" + first_server = server_name.split(" ")[0] + + if not session.query(Services).with_entities(Services.id).filter_by(id=first_server).first(): + self.logger.debug(f"Adding service {first_server}") + to_put.append(Services(id=first_server, method=method)) + changed_services = True for key, value in config.items(): suffix = 0 @@ -1298,8 +1493,17 @@ def save_config(self, config: Dict[str, Any], method: str, changed: Optional[boo .first() ) + template_setting = None + if template: + template_setting = ( + session.query(Template_settings) + .with_entities(Template_settings.default) + .filter_by(template_id=template, setting_id=key, suffix=suffix) + .first() + ) + if not global_value: - if value == setting.default: + if (template_setting and value == template_setting.default) or (not template_setting and value == setting.default): continue self.logger.debug(f"Adding global setting {key}") @@ -1307,11 +1511,11 @@ def save_config(self, config: Dict[str, Any], method: str, changed: Optional[boo to_put.append(Global_values(setting_id=key, value=value, suffix=suffix, method=method)) elif ( method == global_value.method or (global_value.method not in ("scheduler", "autoconf") and method == "autoconf") - ) and value != global_value.value: + ) and global_value.value != value: changed_plugins.add(setting.plugin_id) query = session.query(Global_values).filter(Global_values.setting_id == key, Global_values.suffix == suffix) - if value == setting.default: + if (template_setting and value == template_setting.default) or (not template_setting and value == setting.default): self.logger.debug(f"Removing global setting {key}") query.delete() continue @@ -1472,11 +1676,40 @@ def get_non_default_settings( for global_value in results: setting_id = global_value.setting_id + (f"_{global_value.suffix}" if global_value.multiple and global_value.suffix > 0 else "") - config[setting_id] = global_value.value if not methods else {"value": global_value.value, "global": True, "method": global_value.method} + config[setting_id] = { + "value": global_value.value, + "global": True, + "method": global_value.method, + "template": None, + } + if global_value.context == "multisite": multisite.add(setting_id) - is_multisite = config.get("MULTISITE", {"value": "no"})["value"] == "yes" if methods else config.get("MULTISITE", "no") == "yes" + template_used = config.get("USE_TEMPLATE", {"value": ""})["value"] + if template_used: + query = ( + session.query(Template_settings) + .with_entities(Template_settings.setting_id, Template_settings.default, Template_settings.suffix) + .filter_by(template_id=template_used) + ) + + if filtered_settings: + query = query.filter(Template_settings.setting_id.in_(filtered_settings)) + + for template_setting in query: + key = template_setting.setting_id + (f"_{template_setting.suffix}" if template_setting.suffix > 0 else "") + if key in config and config[key]["method"] != "default": + continue + + config[key] = { + "value": template_setting.default, + "global": True, + "method": "default", + "template": template_used, + } + + is_multisite = config.get("MULTISITE", {"value": "no"})["value"] == "yes" services = session.query(Services).with_entities(Services.id, Services.is_draft) @@ -1488,9 +1721,12 @@ def get_non_default_settings( for service in services: for key in multisite: config[f"{service.id}_{key}"] = config[key] - config[f"{service.id}_IS_DRAFT"] = "yes" if service.is_draft else "no" - if methods: - config[f"{service.id}_IS_DRAFT"] = {"value": config[f"{service.id}_IS_DRAFT"], "global": False, "method": "default"} + config[f"{service.id}_IS_DRAFT"] = { + "value": "yes" if service.is_draft else "no", + "global": False, + "method": "default", + "template": None, + } servers += f"{service.id} " servers = servers.strip() @@ -1529,13 +1765,50 @@ def get_non_default_settings( split.discard(result.service_id) value = result.service_id + " " + " ".join(split) - config[f"{result.service_id}_{result.setting_id}" + (f"_{result.suffix}" if result.multiple and result.suffix else "")] = ( - value if not methods else {"value": value, "global": False, "method": result.method} - ) + config[f"{result.service_id}_{result.setting_id}" + (f"_{result.suffix}" if result.multiple and result.suffix else "")] = { + "value": value, + "global": False, + "method": result.method, + "template": None, + } + + for service in services: + template_used = config.get(f"{service.id}_USE_TEMPLATE", {"value": ""})["value"] + if template_used and template_used != config.get("USE_TEMPLATE", {"value": ""})["value"]: + query = ( + session.query(Template_settings) + .with_entities(Template_settings.setting_id, Template_settings.default, Template_settings.suffix) + .filter_by(template_id=template_used) + ) + + if filtered_settings: + query = query.filter(Template_settings.setting_id.in_(filtered_settings)) + + for setting in query: + key = f"{service.id}_{setting.setting_id}" + (f"_{setting.suffix}" if setting.suffix > 0 else "") + if key in config and config[key]["method"] != "default": + continue + + config[key] = { + "value": setting.default, + "global": False, + "method": "default", + "template": template_used, + } + else: servers = " ".join(service.id for service in services) - config["SERVER_NAME"] = servers if not methods else {"value": servers, "global": True, "method": "default"} + config["SERVER_NAME"] = { + "value": servers, + "global": True, + "method": "default", + "template": None, + } + + if not methods: + for key, value in config.copy().items(): + config[key] = value["value"] return config @@ -1566,8 +1839,7 @@ def get_config( query = query.filter(Settings.id.in_(filtered_settings)) for setting in query: - default = setting.default or "" - config[setting.id] = default if not methods else {"value": default, "global": True, "method": "default"} + config[setting.id] = {"value": setting.default or "", "global": True, "method": "default", "template": None} if setting.context == "multisite": multisite.add(setting.id) @@ -1619,7 +1891,9 @@ def get_services_settings(self, methods: bool = False, with_drafts: bool = False elif any(key.startswith(f"{s}_") for s in service_names): tmp_config.pop(key) elif key not in service_settings: - tmp_config[key] = {"value": value["value"], "global": value["global"], "method": value["method"]} if methods else value + tmp_config[key] = ( + {"value": value["value"], "global": value["global"], "method": value["method"], "template": value["template"]} if methods else value + ) services.append(tmp_config) @@ -1736,7 +2010,7 @@ def update_external_plugins(self, plugins: List[Dict[str, Any]], *, _type: Liter # Remove plugins that are no longer in the list session.query(Plugins).filter(Plugins.id.in_(missing_ids)).delete() session.query(Plugin_pages).filter(Plugin_pages.plugin_id.in_(missing_ids)).delete() - session.query(BwcliCommands).filter(BwcliCommands.plugin_id.in_(missing_ids)).delete() + session.query(Bw_cli_commands).filter(Bw_cli_commands.plugin_id.in_(missing_ids)).delete() for plugin_job in session.query(Jobs).with_entities(Jobs.name).filter(Jobs.plugin_id.in_(missing_ids)): session.query(Jobs_runs).filter(Jobs_runs.job_name == plugin_job.name).delete() @@ -1749,6 +2023,14 @@ def update_external_plugins(self, plugins: List[Dict[str, Any]], *, _type: Liter session.query(Global_values).filter(Global_values.setting_id == plugin_setting.id).delete() session.query(Settings).filter(Settings.id == plugin_setting.id).delete() + for plugin_template in session.query(Templates).with_entities(Templates.id).filter(Templates.plugin_id.in_(missing_ids)): + session.query(Template_steps).filter(Template_steps.template_id == plugin_template.id).delete() + session.query(Template_settings).filter(Template_settings.template_id == plugin_template.id).delete() + session.query(Template_custom_configs).filter(Template_custom_configs.template_id == plugin_template.id).delete() + session.query(Templates).filter(Templates.id == plugin_template.id).delete() + + db_settings = [setting.id for setting in session.query(Settings).with_entities(Settings.id)] + for plugin in plugins: settings = plugin.pop("settings", {}) jobs = plugin.pop("jobs", []) @@ -1825,10 +2107,14 @@ def update_external_plugins(self, plugins: List[Dict[str, Any]], *, _type: Liter session.query(Selects).filter(Selects.setting_id.in_(missing_ids)).delete() session.query(Services_settings).filter(Services_settings.setting_id.in_(missing_ids)).delete() session.query(Global_values).filter(Global_values.setting_id.in_(missing_ids)).delete() + session.query(Template_settings).filter(Template_settings.setting_id.in_(missing_ids)).delete() order = 0 + plugin_settings = set() for setting, value in settings.items(): value.update({"plugin_id": plugin["id"], "name": value["id"], "id": setting}) + plugin_settings.add(setting) + db_setting = ( session.query(Settings) .with_entities( @@ -1972,107 +2258,299 @@ def update_external_plugins(self, plugins: List[Dict[str, Any]], *, _type: Liter if path_ui.is_dir(): remove = True - if {"template.html", "actions.py"}.issubset(listdir(str(path_ui))): - template = path_ui.joinpath("template.html").read_bytes() - actions = path_ui.joinpath("actions.py").read_bytes() - template_checksum = bytes_hash(template, algorithm="sha256") - actions_checksum = bytes_hash(actions, algorithm="sha256") - - obfuscation_file = None - obfuscation_checksum = None - obfuscation_dir = path_ui.joinpath("pyarmor_runtime_000000") - if obfuscation_dir.is_dir(): - obfuscation_file = BytesIO() - with ZipFile(obfuscation_file, "w", ZIP_DEFLATED) as zip_file: - for path in obfuscation_dir.rglob("*"): - if path.is_file(): - zip_file.write(path, path.relative_to(path_ui)) - obfuscation_file.seek(0, 0) - obfuscation_file = obfuscation_file.getvalue() - obfuscation_checksum = bytes_hash(obfuscation_file, algorithm="sha256") - - if not db_plugin_page: - changes = True - - to_put.append( - Plugin_pages( - plugin_id=plugin["id"], - template_file=template, - template_checksum=template_checksum, - actions_file=actions, - actions_checksum=actions_checksum, - obfuscation_file=obfuscation_file, - obfuscation_checksum=obfuscation_checksum, - ) - ) - remove = False - else: - updates = {} - - if template_checksum != db_plugin_page.template_checksum: - updates.update( - { - Plugin_pages.template_file: template, - Plugin_pages.template_checksum: template_checksum, - } - ) - - if actions_checksum != db_plugin_page.actions_checksum: - updates.update( - { - Plugin_pages.actions_file: actions, - Plugin_pages.actions_checksum: actions_checksum, - } - ) - - if obfuscation_checksum != db_plugin_page.obfuscation_checksum: - updates.update( - { - Plugin_pages.obfuscation_file: obfuscation_file, - Plugin_pages.obfuscation_checksum: obfuscation_checksum, - } - ) - - if updates: - changes = True - session.query(Plugin_pages).filter(Plugin_pages.plugin_id == plugin["id"]).update(updates) - - remove = False + with BytesIO() as plugin_page_content: + with tar_open(fileobj=plugin_page_content, mode="w:gz", compresslevel=9) as tar: + tar.add(path_ui, arcname=path_ui.name, recursive=True) + plugin_page_content.seek(0) + checksum = bytes_hash(plugin_page_content, algorithm="sha256") + content = plugin_page_content.getvalue() + + if not db_plugin_page: + changes = True + to_put.append(Plugin_pages(plugin_id=plugin["id"], data=content, checksum=checksum)) + remove = False + elif checksum != db_plugin_page.checksum: + changes = True + session.query(Plugin_pages).filter(Plugin_pages.plugin_id == plugin["id"]).update( + {Plugin_pages.data: content, Plugin_pages.checksum: checksum} + ) + remove = False if db_plugin_page and remove: changes = True session.query(Plugin_pages).filter(Plugin_pages.plugin_id == plugin["id"]).delete() - db_names = [command.name for command in session.query(BwcliCommands).with_entities(BwcliCommands.name).filter_by(plugin_id=plugin["id"])] + db_names = [ + command.name for command in session.query(Bw_cli_commands).with_entities(Bw_cli_commands.name).filter_by(plugin_id=plugin["id"]) + ] missing_names = [command for command in db_names if command not in commands] if missing_names: # Remove commands that are no longer in the list - session.query(BwcliCommands).filter(BwcliCommands.name.in_(missing_names), BwcliCommands.plugin_id == plugin["id"]).delete() + session.query(Bw_cli_commands).filter(Bw_cli_commands.name.in_(missing_names), Bw_cli_commands.plugin_id == plugin["id"]).delete() for command, file_name in commands.items(): - db_command = session.query(BwcliCommands).with_entities(BwcliCommands.file_name).filter_by(name=command, plugin_id=plugin["id"]).first() + db_command = ( + session.query(Bw_cli_commands).with_entities(Bw_cli_commands.file_name).filter_by(name=command, plugin_id=plugin["id"]).first() + ) command_path = plugin_path.joinpath("bwcli", file_name) if command not in db_names or not db_command: if not command_path.is_file(): - self.logger.warning(f'Command "{command}"\'s file "{file_name}" does not exist in the plugin directory, skipping it') + self.logger.warning( + f'Plugin "{plugin["id"]}"\'s Command "{command}"\'s file "{file_name}" does not exist in the plugin directory, skipping it' + ) continue changes = True - to_put.append(BwcliCommands(name=command, plugin_id=plugin["id"], file_name=file_name)) + to_put.append(Bw_cli_commands(name=command, plugin_id=plugin["id"], file_name=file_name)) else: updates = {} if file_name != db_command.file_name: - updates[BwcliCommands.file_name] = file_name + updates[Bw_cli_commands.file_name] = file_name if updates: changes = True if not command_path.is_file(): - session.query(BwcliCommands).filter_by(name=command, plugin_id=plugin["id"]).delete() + session.query(Bw_cli_commands).filter_by(name=command, plugin_id=plugin["id"]).delete() + continue + session.query(Bw_cli_commands).filter_by(name=command, plugin_id=plugin["id"]).update(updates) + + db_names = [template.id for template in session.query(Templates).with_entities(Templates.id).filter_by(plugin_id=plugin["id"])] + templates_path = plugin_path.joinpath("templates") + + if not templates_path.is_dir(): + if db_names: + self.logger.warning(f'Plugin "{plugin["id"]}"\'s templates directory does not exist, removing all templates') + for template in db_names: + session.query(Templates).filter_by(id=template, plugin_id=plugin["id"]).delete() + session.query(Template_steps).filter_by(template_id=template).delete() + session.query(Template_settings).filter_by(template_id=template).delete() + session.query(Template_custom_configs).filter_by(template_id=template).delete() + continue + + saved_templates = set() + for template_file in templates_path.iterdir(): + if template_file.is_dir(): + continue + + try: + template_data = loads(template_file.read_text()) + except JSONDecodeError: + self.logger.error( + f"{plugin.get('type', 'core').title()} Plugin \"{plugin['id']}\"'s Template file \"{template_file}\" is not a valid JSON file" + ) + continue + + template_id = template_file.stem + + db_template = session.query(Templates).with_entities(Templates.id).filter_by(id=template_id, plugin_id=plugin["id"]).first() + + if not db_template: + changes = True + to_put.append(Templates(id=template_id, plugin_id=plugin["id"], name=template_data.get("name", template_id))) + + saved_templates.add(template_id) + + db_ids = [step.id for step in session.query(Template_steps).with_entities(Template_steps.id).filter_by(template_id=template_id)] + missing_ids = [x for x in range(1, len(template.get("steps", [])) + 1) if x not in db_ids] + + if missing_ids: + changes = True + session.query(Template_settings).filter(Template_settings.step_id.in_(missing_ids)).update({Template_settings.step_id: None}) + session.query(Template_custom_configs).filter(Template_custom_configs.step_id.in_(missing_ids)).update( + {Template_custom_configs.step_id: None} + ) + session.query(Template_steps).filter(Template_steps.id.in_(missing_ids)).delete() + + steps_settings = {} + steps_configs = {} + for step_id, step in enumerate(template.get("steps", []), start=1): + db_step = session.query(Template_steps).with_entities(Template_steps.id).filter_by(template_id=template_id, id=step_id).first() + if not db_step: + changes = True + to_put.append( + Template_steps(template_id=template_id, id=step_id, name=step["name"], title=step["title"], subtitle=step["subtitle"]) + ) + else: + updates = {} + + if step["title"] != db_step.title: + updates[Template_steps.title] = step["title"] + + if step["subtitle"] != db_step.subtitle: + updates[Template_steps.subtitle] = step["subtitle"] + + if updates: + changes = True + session.query(Template_steps).filter(Template_steps.id == db_step.id).update(updates) + + for setting in step.get("settings", []): + if step_id not in steps_settings: + steps_settings[step_id] = [] + steps_settings[step_id].append(setting) + + for config in step.get("configs", []): + if step_id not in steps_configs: + steps_configs[step_id] = [] + steps_configs[step_id].append(config) + + db_template_settings = [ + f"{setting.setting_id}_{setting.suffix}" if setting.suffix else setting.setting_id + for setting in session.query(Template_settings).with_entities(Template_settings.id).filter_by(template_id=template_id) + ] + missing_ids = [setting for setting in template.get("settings", {}) if setting not in db_template_settings] + + if missing_ids: + changes = True + session.query(Template_settings).filter(Template_settings.id.in_(missing_ids)).delete() + + for setting, default in template.get("settings", {}).items(): + setting_id, suffix = setting.rsplit("_", 1) if self.suffix_rx.search(setting) else (setting, None) + + if setting_id in self.RESTRICTED_TEMPLATE_SETTINGS: + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Setting "{setting}" is restricted, skipping it' + ) + session.query(Template_settings).filter_by(template_id=template_id, setting_id=setting_id, suffix=suffix).delete() + continue + elif setting_id not in plugin_settings and setting_id not in db_settings: + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Setting "{setting}" does not exist, skipping it' + ) + session.query(Template_settings).filter_by(template_id=template_id, setting_id=setting_id, suffix=suffix).delete() + continue + + success, err = self.is_valid_setting(setting_id, value=default, multisite=True, session=session) + if not success: + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Setting "{setting}" is not a valid template setting ({err}), skipping it' + ) + session.query(Template_settings).filter_by(template_id=template_id, setting_id=setting_id, suffix=suffix).delete() + continue + + step_id = None + for step, settings in steps_settings.items(): + if setting in settings: + step_id = step + break + + template_setting = ( + session.query(Template_settings) + .with_entities(Template_settings.id) + .filter_by(template_id=template_id, setting_id=setting_id, step_id=step_id, suffix=suffix) + .first() + ) + + if not template_setting: + changes = True + if step_id: + to_put.append( + Template_settings( + template_id=template_id, + setting_id=setting_id, + step_id=step_id, + suffix=suffix, + default=default, + ) + ) + continue + + to_put.append( + Template_settings( + template_id=template_id, + setting_id=setting_id, + suffix=suffix, + default=default, + ) + ) + elif default != template_setting.default: + changes = True + session.query(Template_settings).filter_by(id=template_setting.id).update({Template_settings.default: default}) + + db_template_configs = [ + f"{config.type}/{config.name}.conf" + for config in session.query(Template_custom_configs) + .with_entities(Template_custom_configs.type, Template_custom_configs.name) + .filter_by(template_id=template_id) + ] + missing_ids = [config for config in template.get("configs", {}) if config not in db_template_configs] + + if missing_ids: + changes = True + session.query(Template_custom_configs).filter(Template_custom_configs.name.in_(missing_ids)).delete() + + for config in template.get("configs", []): + try: + config_type, config_name = config.split("/", 1) + except ValueError: + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Custom config "{config}" is invalid, skipping it' + ) + continue + + if not templates_path.joinpath(template_id, "configs", config_type, config_name).is_file(): + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Custom config "{config}" does not exist, skipping it' + ) + continue + + content = templates_path.joinpath(template_id, "configs", config_type, config_name).read_bytes() + checksum = bytes_hash(content, algorithm="sha256") + + config_name = config_name.replace(".conf", "") + + step_id = None + for step, configs in steps_configs.items(): + if config in configs: + step_id = step + break + + template_config = ( + session.query(Template_custom_configs) + .with_entities(Template_custom_configs.id) + .filter_by(template_id=template_id, step_id=step_id, type=config_type, name=config_name) + .first() + ) + + if not template_config: + changes = True + if step_id: + to_put.append( + Template_custom_configs( + template_id=template_id, + step_id=step_id, + type=config_type, + name=config_name, + data=content, + checksum=checksum, + ) + ) continue - session.query(BwcliCommands).filter_by(name=command, plugin_id=plugin["id"]).update(updates) + + to_put.append( + Template_custom_configs( + template_id=template_id, + type=config_type, + name=config_name, + data=content, + checksum=checksum, + ) + ) + elif checksum != template_config.checksum: + changes = True + session.query(Template_custom_configs).filter_by(id=template_config.id).update( + {Template_custom_configs.data: content, Template_custom_configs.checksum: checksum} + ) + + for template in db_names: + if template not in saved_templates: + changes = True + session.query(Template_steps).filter_by(template_id=template).delete() + session.query(Template_settings).filter_by(template_id=template).delete() + session.query(Template_custom_configs).filter_by(template_id=template).delete() + session.query(Templates).filter_by(id=template, plugin_id=plugin["id"]).delete() continue @@ -2092,6 +2570,7 @@ def update_external_plugins(self, plugins: List[Dict[str, Any]], *, _type: Liter ) order = 0 + plugin_settings = set() for setting, value in settings.items(): db_setting = session.query(Settings).filter_by(id=setting).first() @@ -2106,6 +2585,7 @@ def update_external_plugins(self, plugins: List[Dict[str, Any]], *, _type: Liter to_put.append(Settings(**value | {"order": order})) order += 1 + plugin_settings.add(setting) for job in jobs: db_job = ( @@ -2133,85 +2613,152 @@ def update_external_plugins(self, plugins: List[Dict[str, Any]], *, _type: Liter if page: path_ui = plugin_path.joinpath("ui") - if path_ui.exists(): - if {"template.html", "actions.py"}.issubset(listdir(str(path_ui))): - db_plugin_page = ( - session.query(Plugin_pages) - .with_entities( - Plugin_pages.template_checksum, - Plugin_pages.actions_checksum, - Plugin_pages.obfuscation_checksum, - ) - .filter_by(plugin_id=plugin["id"]) - .first() + if not path_ui.is_dir(): + with BytesIO() as plugin_page_content: + with tar_open(fileobj=plugin_page_content, mode="w:gz", compresslevel=9) as tar: + tar.add(path_ui, arcname=path_ui.name, recursive=True) + plugin_page_content.seek(0) + checksum = bytes_hash(plugin_page_content, algorithm="sha256") + + to_put.append(Plugin_pages(plugin_id=plugin["id"], data=plugin_page_content.getvalue(), checksum=checksum)) + + for command, file_name in commands.items(): + if not plugin_path.joinpath("bwcli", file_name).is_file(): + self.logger.warning(f'Command "{command}"\'s file "{file_name}" does not exist in the plugin directory, skipping it') + continue + + to_put.append(Bw_cli_commands(name=command, plugin_id=plugin["id"], file_name=file_name)) + + templates_path = plugin_path.joinpath("templates") + + if not templates_path.is_dir(): + continue + + for template_file in plugin_path.joinpath("templates").iterdir(): + if template_file.is_dir(): + continue + + try: + template_data = loads(template_file.read_text()) + except JSONDecodeError: + self.logger.error(f'Template file "{template_file}" is not a valid JSON file') + continue + + template_id = template_file.stem + + to_put.append(Templates(id=template_id, plugin_id=plugin["id"], name=template_data.get("name", template_id))) + + steps_settings = {} + steps_configs = {} + for step_id, step in enumerate(template_data.get("steps", []), start=1): + to_put.append(Template_steps(template_id=template_id, id=step_id, title=step["title"], subtitle=step["subtitle"])) + + for setting in step.get("settings", []): + if step_id not in steps_settings: + steps_settings[step_id] = [] + steps_settings[step_id].append(setting) + + for config in step.get("configs", []): + if step_id not in steps_configs: + steps_configs[step_id] = [] + steps_configs[step_id].append(config) + + for setting, default in template_data.get("settings", {}).items(): + setting_id, suffix = setting.rsplit("_", 1) if self.suffix_rx.search(setting) else (setting, None) + + if setting_id in self.RESTRICTED_TEMPLATE_SETTINGS: + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Setting "{setting}" is restricted, skipping it' + ) + continue + elif setting_id not in plugin_settings and setting_id not in db_settings: + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Setting "{setting}" does not exist, skipping it' ) - template = path_ui.joinpath("template.html").read_bytes() - actions = path_ui.joinpath("actions.py").read_bytes() - template_checksum = bytes_hash(template, algorithm="sha256") - actions_checksum = bytes_hash(actions, algorithm="sha256") - - obfuscation_file = None - obfuscation_checksum = None - obfuscation_dir = path_ui.joinpath("pyarmor_runtime_000000") - if obfuscation_dir.is_dir(): - obfuscation_file = BytesIO() - with ZipFile(obfuscation_file, "w", ZIP_DEFLATED) as zip_file: - for path in obfuscation_dir.rglob("*"): - if path.is_file(): - zip_file.write(path, path.relative_to(path_ui)) - obfuscation_file.seek(0, 0) - obfuscation_file = obfuscation_file.getvalue() - obfuscation_checksum = bytes_hash(obfuscation_file, algorithm="sha256") - - if not db_plugin_page: + continue - to_put.append( - Plugin_pages( - plugin_id=plugin["id"], - template_file=template, - template_checksum=template_checksum, - actions_file=actions, - actions_checksum=actions_checksum, - obfuscation_file=obfuscation_file, - obfuscation_checksum=obfuscation_checksum, - ) + success, err = self.is_valid_setting(setting_id, value=default, multisite=True, session=session) + if not success: + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Setting "{setting}" is not a valid template setting ({err}), skipping it' + ) + continue + + step_id = None + for step, settings in steps_settings.items(): + if setting in settings: + step_id = step + break + + if step_id: + to_put.append( + Template_settings( + template_id=template_id, + setting_id=setting_id, + step_id=step_id, + default=default, + suffix=suffix, ) - else: - updates = {} + ) + continue - if template_checksum != db_plugin_page.template_checksum: - updates.update( - { - Plugin_pages.template_file: template, - Plugin_pages.template_checksum: template_checksum, - } - ) + to_put.append( + Template_settings( + template_id=template_id, + setting_id=setting_id, + default=default, + suffix=suffix, + ) + ) - if actions_checksum != db_plugin_page.actions_checksum: - updates.update( - { - Plugin_pages.actions_file: actions, - Plugin_pages.actions_checksum: actions_checksum, - } - ) + for config in template_data.get("configs", []): + try: + config_type, config_name = config.split("/", 1) + except ValueError: + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Custom config "{config}" is invalid, skipping it' + ) + continue - if obfuscation_checksum != db_plugin_page.obfuscation_checksum: - updates.update( - { - Plugin_pages.obfuscation_file: obfuscation_file, - Plugin_pages.obfuscation_checksum: obfuscation_checksum, - } - ) + if not templates_path.joinpath(template_id, "configs", config_type, config_name).is_file(): + self.logger.error( + f'{plugin.get("type", "core").title()} Plugin "{plugin["id"]}"\'s Template "{template_id}"\'s Custom config "{config}" does not exist, skipping it' + ) + continue - if updates: - session.query(Plugin_pages).filter(Plugin_pages.plugin_id == plugin["id"]).update(updates) + content = templates_path.joinpath(template_id, "configs", config_type, config_name).read_bytes() + checksum = bytes_hash(content, algorithm="sha256") - for command, file_name in commands.items(): - if not plugin_path.joinpath("bwcli", file_name).is_file(): - self.logger.warning(f'Command "{command}"\'s file "{file_name}" does not exist in the plugin directory, skipping it') - continue + config_name = config_name.replace(".conf", "") - to_put.append(BwcliCommands(name=command, plugin_id=plugin["id"], file_name=file_name)) + step_id = None + for step, configs in steps_configs.items(): + if config in configs: + step_id = step + break + + if step_id: + to_put.append( + Template_custom_configs( + template_id=template_id, + step_id=step_id, + type=config_type, + name=config_name, + data=content, + checksum=checksum, + ) + ) + continue + + to_put.append( + Template_custom_configs( + template_id=template_id, + type=config_type, + name=config_name, + data=content, + checksum=checksum, + ) + ) if changes: with suppress(ProgrammingError, OperationalError): @@ -2290,7 +2837,7 @@ def get_plugins(self, *, _type: Literal["all", "external", "pro"] = "all", with_ select.value for select in session.query(Selects).with_entities(Selects.value).filter_by(setting_id=setting.id) ] - for command in session.query(BwcliCommands).with_entities(BwcliCommands.name, BwcliCommands.file_name).filter_by(plugin_id=plugin.id): + for command in session.query(Bw_cli_commands).with_entities(Bw_cli_commands.name, Bw_cli_commands.file_name).filter_by(plugin_id=plugin.id): if "bwcli" not in data: data["bwcli"] = {} data["bwcli"][command.name] = command.file_name @@ -2536,32 +3083,24 @@ def get_instance(self, hostname: str, *, method: Optional[str] = None) -> Dict[s "last_seen": instance.last_seen, } - def get_plugin_actions(self, plugin: str) -> Optional[Any]: - """get actions file for the plugin""" + def get_plugin_page(self, plugin_id: str) -> Optional[bytes]: + """Get plugin page.""" with self._db_session() as session: - page = session.query(Plugin_pages).with_entities(Plugin_pages.actions_file).filter_by(plugin_id=plugin).first() + page = session.query(Plugin_pages).with_entities(Plugin_pages.data).filter_by(plugin_id=plugin_id).first() if not page: return None - return page.actions_file + return page.data - def get_plugin_template(self, plugin: str) -> Optional[Any]: - """get template file for the plugin""" + def get_template_settings(self, template_id: str) -> Dict[str, Any]: + """Get templates settings.""" with self._db_session() as session: - page = session.query(Plugin_pages).with_entities(Plugin_pages.template_file).filter_by(plugin_id=plugin).first() - - if not page: - return None - - return page.template_file - - def get_plugin_obfuscation(self, plugin: str) -> Optional[Any]: - """get obfuscation file for the plugin""" - with self._db_session() as session: - page = session.query(Plugin_pages).with_entities(Plugin_pages.obfuscation_file).filter_by(plugin_id=plugin).first() - - if not page: - return None - - return page.obfuscation_file + settings = {} + for setting in ( + session.query(Template_settings) + .with_entities(Template_settings.setting_id, Template_settings.default, Template_settings.suffix) + .filter_by(template_id=template_id) + ): + settings[f"{setting.setting_id}_{setting.suffix}" if setting.suffix else setting.setting_id] = setting.default + return settings diff --git a/src/common/db/model.py b/src/common/db/model.py index b6af1769f3..49d439698c 100644 --- a/src/common/db/model.py +++ b/src/common/db/model.py @@ -2,20 +2,7 @@ from datetime import datetime, timezone from functools import partial -from sqlalchemy import ( - TEXT, - Boolean, - Column, - DateTime, - Enum, - ForeignKey, - Identity, - Integer, - LargeBinary, - PrimaryKeyConstraint, - String, - func, -) +from sqlalchemy import TEXT, Boolean, Column, DateTime, Enum, ForeignKey, Identity, Integer, LargeBinary, String, func from sqlalchemy.orm import declarative_base, relationship from sqlalchemy.schema import UniqueConstraint @@ -71,21 +58,18 @@ class Plugins(Base): settings = relationship("Settings", back_populates="plugin", cascade="all, delete-orphan") jobs = relationship("Jobs", back_populates="plugin", cascade="all, delete-orphan") pages = relationship("Plugin_pages", back_populates="plugin", cascade="all") - commands = relationship("BwcliCommands", back_populates="plugin", cascade="all") + commands = relationship("Bw_cli_commands", back_populates="plugin", cascade="all") + templates = relationship("Templates", back_populates="plugin", cascade="all") class Settings(Base): __tablename__ = "bw_settings" - __table_args__ = ( - PrimaryKeyConstraint("id", "name"), - UniqueConstraint("id"), - ) id = Column(String(256), primary_key=True) - name = Column(String(256), primary_key=True) + name = Column(String(256), unique=True, nullable=False) plugin_id = Column(String(64), ForeignKey("bw_plugins.id", onupdate="cascade", ondelete="cascade"), nullable=False) context = Column(CONTEXTS_ENUM, nullable=False) - default = Column(String(4096), nullable=True, default="") + default = Column(TEXT, nullable=True, default="") help = Column(String(512), nullable=False) label = Column(String(256), nullable=True) regex = Column(String(1024), nullable=False) @@ -96,6 +80,7 @@ class Settings(Base): selects = relationship("Selects", back_populates="setting", cascade="all") services = relationship("Services_settings", back_populates="setting", cascade="all") global_value = relationship("Global_values", back_populates="setting", cascade="all") + templates = relationship("Template_settings", back_populates="setting", cascade="all") plugin = relationship("Plugins", back_populates="settings") @@ -162,14 +147,9 @@ class Plugin_pages(Base): __tablename__ = "bw_plugin_pages" id = Column(Integer, Identity(start=1, increment=1), primary_key=True) - plugin_id = Column(String(64), ForeignKey("bw_plugins.id", onupdate="cascade", ondelete="cascade"), nullable=False) - # TODO: replace with a raw data that gets extracted by the plugin - template_file = Column(LargeBinary(length=(2**32) - 1), nullable=False) - template_checksum = Column(String(128), nullable=False) - actions_file = Column(LargeBinary(length=(2**32) - 1), nullable=False) - actions_checksum = Column(String(128), nullable=False) - obfuscation_file = Column(LargeBinary(length=(2**32) - 1), default=None, nullable=True) - obfuscation_checksum = Column(String(128), default=None, nullable=True) + plugin_id = Column(String(64), ForeignKey("bw_plugins.id", onupdate="cascade", ondelete="cascade"), unique=True, nullable=False) + data = Column(LargeBinary(length=(2**32) - 1), nullable=False) + checksum = Column(String(128), nullable=False) plugin = relationship("Plugins", back_populates="pages") @@ -228,7 +208,7 @@ class Instances(Base): last_seen = Column(DateTime, nullable=True, server_default=func.now(), onupdate=partial(datetime.now, timezone.utc)) -class BwcliCommands(Base): +class Bw_cli_commands(Base): __tablename__ = "bw_cli_commands" __table_args__ = (UniqueConstraint("plugin_id", "name"),) @@ -240,6 +220,64 @@ class BwcliCommands(Base): plugin = relationship("Plugins", back_populates="commands") +class Templates(Base): + __tablename__ = "bw_templates" + + id = Column(String(256), primary_key=True) + name = Column(String(256), unique=True, nullable=False) + plugin_id = Column(String(64), ForeignKey("bw_plugins.id", onupdate="cascade", ondelete="cascade"), nullable=False) + + plugin = relationship("Plugins", back_populates="templates") + steps = relationship("Template_steps", back_populates="template", cascade="all") + settings = relationship("Template_settings", back_populates="template", cascade="all") + custom_configs = relationship("Template_custom_configs", back_populates="template", cascade="all") + + +class Template_steps(Base): + __tablename__ = "bw_template_steps" + + id = Column(Integer, primary_key=True) + template_id = Column(String(256), ForeignKey("bw_templates.id", onupdate="cascade", ondelete="cascade"), primary_key=True) + title = Column(TEXT, nullable=False) + subtitle = Column(TEXT, nullable=True) + + template = relationship("Templates", back_populates="steps") + settings = relationship("Template_settings", back_populates="step", cascade="all") + custom_configs = relationship("Template_custom_configs", back_populates="step", cascade="all") + + +class Template_settings(Base): + __tablename__ = "bw_template_settings" + __table_args__ = (UniqueConstraint("template_id", "setting_id", "step_id", "suffix"),) + + id = Column(Integer, Identity(start=1, increment=1), primary_key=True) + template_id = Column(String(256), ForeignKey("bw_templates.id", onupdate="cascade", ondelete="cascade"), nullable=False) + setting_id = Column(String(256), ForeignKey("bw_settings.id", onupdate="cascade", ondelete="cascade"), nullable=False) + step_id = Column(Integer, ForeignKey("bw_template_steps.id", onupdate="cascade", ondelete="cascade"), nullable=True) + default = Column(TEXT, nullable=False) + suffix = Column(Integer, nullable=True, default=0) + + template = relationship("Templates", back_populates="settings") + step = relationship("Template_steps", back_populates="settings") + setting = relationship("Settings", back_populates="templates") + + +class Template_custom_configs(Base): + __tablename__ = "bw_template_custom_configs" + __table_args__ = (UniqueConstraint("template_id", "step_id", "type", "name"),) + + id = Column(Integer, Identity(start=1, increment=1), primary_key=True) + template_id = Column(String(256), ForeignKey("bw_templates.id", onupdate="cascade", ondelete="cascade"), nullable=False) + step_id = Column(Integer, ForeignKey("bw_template_steps.id", onupdate="cascade", ondelete="cascade"), nullable=True) + type = Column(CUSTOM_CONFIGS_TYPES_ENUM, nullable=False) + name = Column(String(256), nullable=False) + data = Column(LargeBinary(length=(2**32) - 1), nullable=False) + checksum = Column(String(128), nullable=False) + + template = relationship("Templates", back_populates="custom_configs") + step = relationship("Template_steps", back_populates="custom_configs") + + class Metadata(Base): __tablename__ = "bw_metadata" diff --git a/src/common/gen/Configurator.py b/src/common/gen/Configurator.py index 237342aee1..d412f52189 100644 --- a/src/common/gen/Configurator.py +++ b/src/common/gen/Configurator.py @@ -1,103 +1,94 @@ #!/usr/bin/env python3 -from glob import glob -from hashlib import sha256 +from copy import deepcopy +from functools import cache from io import BytesIO from json import loads from logging import Logger -from os import cpu_count, listdir, sep -from os.path import basename, dirname, join +from os import listdir, sep +from os.path import join from pathlib import Path from re import compile as re_compile, error as RegexError, search as re_search from sys import path as sys_path from tarfile import open as tar_open -from threading import Lock, Semaphore, Thread -from traceback import format_exc -from typing import Any, Dict, List, Literal, Optional, Tuple, Union +from typing import Dict, List, Literal, Optional, Tuple, Union if join(sep, "usr", "share", "bunkerweb", "utils") not in sys_path: sys_path.append(join(sep, "usr", "share", "bunkerweb", "utils")) +from common_utils import bytes_hash # type: ignore + class Configurator: def __init__( self, settings: str, core: str, - external_plugins: Union[str, List[Dict[str, Any]]], - pro_plugins: Union[str, List[Dict[str, Any]]], - variables: Union[str, Dict[str, Any]], + external_plugins: Union[str, List[Dict[str, str]]], + pro_plugins: Union[str, List[Dict[str, str]]], + variables: Union[str, Dict[str, str]], logger: Logger, ): self.__logger = logger - self.__thread_lock = Lock() - self.__semaphore = Semaphore(cpu_count() or 1) self.__plugin_id_rx = re_compile(r"^[\w.-]{1,64}$") self.__plugin_version_rx = re_compile(r"^\d+\.\d+(\.\d+)?$") self.__setting_id_rx = re_compile(r"^[A-Z0-9_]{1,256}$") self.__name_rx = re_compile(r"^[\w.-]{1,128}$") self.__job_file_rx = re_compile(r"^[\w./-]{1,256}$") - self.__settings = self.__load_settings(settings) + self.__settings = self.__load_settings(Path(settings)) self.__core_plugins = [] - self.__load_plugins(core) + self.__load_plugins(Path(core)) if isinstance(external_plugins, str): self.__external_plugins = [] - self.__load_plugins(external_plugins, "external") + self.__load_plugins(Path(external_plugins), "external") else: self.__external_plugins = external_plugins if isinstance(pro_plugins, str): self.__pro_plugins = [] - self.__load_plugins(pro_plugins, "pro") + self.__load_plugins(Path(pro_plugins), "pro") else: self.__pro_plugins = pro_plugins if isinstance(variables, str): - self.__variables = self.__load_variables(variables) + self.__variables = self.__load_variables(Path(variables)) else: self.__variables = variables self.__multisite = self.__variables.get("MULTISITE", "no") == "yes" self.__servers = self.__map_servers() - def get_settings(self) -> Dict[str, Any]: - return self.__settings + def get_settings(self) -> Dict[str, str]: + return self.__settings.copy() - def get_plugins(self, _type: Literal["core", "external", "pro"]) -> List[Dict[str, Any]]: - return {"core": self.__core_plugins, "external": self.__external_plugins, "pro": self.__pro_plugins}[_type] + def get_plugins(self, _type: Literal["core", "external", "pro"]) -> List[Dict[str, str]]: + return {"core": deepcopy(self.__core_plugins), "external": deepcopy(self.__external_plugins), "pro": deepcopy(self.__pro_plugins)}.get(_type, []) - def get_plugins_settings(self, _type: Literal["core", "external", "pro"]) -> Dict[str, Any]: - if _type == "core": - plugins = self.__core_plugins - elif _type == "pro": - plugins = self.__pro_plugins - else: - plugins = self.__external_plugins + @cache + def get_plugins_settings(self, _type: Literal["core", "external", "pro"]) -> Dict[str, str]: plugins_settings = {} - - for plugin in plugins: - plugins_settings.update(plugin["settings"]) - + for plugin in self.get_plugins(_type): + plugins_settings.update(plugin.get("settings", {})) return plugins_settings + @cache def __map_servers(self) -> Dict[str, List[str]]: if not self.__multisite or "SERVER_NAME" not in self.__variables: return {} + servers = {} for server_name in self.__variables["SERVER_NAME"].strip().split(" "): if not server_name: continue - if not re_search(self.__settings["SERVER_NAME"]["regex"], server_name): + if re_search(self.__settings["SERVER_NAME"]["regex"], server_name) is None: self.__logger.warning(f"Ignoring server name {server_name} because regex is not valid") continue + names = [server_name] if f"{server_name}_SERVER_NAME" in self.__variables: - if not re_search( - self.__settings["SERVER_NAME"]["regex"], - self.__variables[f"{server_name}_SERVER_NAME"], - ): + if re_search(self.__settings["SERVER_NAME"]["regex"], self.__variables[f"{server_name}_SERVER_NAME"]) is None: self.__logger.warning(f"Ignoring {server_name}_SERVER_NAME because regex is not valid") else: names = self.__variables[f"{server_name}_SERVER_NAME"].strip().split(" ") @@ -105,21 +96,19 @@ def __map_servers(self) -> Dict[str, List[str]]: servers[server_name] = names return servers - def __load_settings(self, path: str) -> Dict[str, Any]: - return loads(Path(path).read_text()) + def __load_settings(self, path: Path) -> Dict[str, str]: + return loads(path.read_text()) - def __load_plugins(self, path: str, _type: Literal["core", "external", "pro"] = "core"): - threads = [] - for file in glob(join(path, "*", "plugin.json")): - thread = Thread(target=self.__load_plugin, args=(file, _type)) - thread.start() - threads.append(thread) + def __load_plugins(self, path: Path, _type: Literal["core", "external", "pro"] = "core"): + x = 0 + for file in path.glob("*/plugin.json"): + self.__logger.debug(f"Loading {_type} plugin {file}") + self.__load_plugin(file, _type) + x += 1 - for thread in threads: - thread.join() + self.__logger.info(f"Computed {x} {_type} plugin{'s' if x > 1 else ''}") - def __load_plugin(self, file: str, _type: Literal["core", "external", "pro"] = "core"): - self.__semaphore.acquire(timeout=60) + def __load_plugin(self, file: Path, _type: Literal["core", "external", "pro"] = "core"): try: data = self.__load_settings(file) @@ -128,39 +117,32 @@ def __load_plugin(self, file: str, _type: Literal["core", "external", "pro"] = " self.__logger.warning(f"Ignoring {_type} plugin {file} : {msg}") return - data["page"] = "ui" in listdir(dirname(file)) + data["page"] = "ui" in listdir(file.parent) if _type != "core": - plugin_content = BytesIO() - with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar: - tar.add(dirname(file), arcname=basename(dirname(file)), recursive=True) - plugin_content.seek(0, 0) - value = plugin_content.getvalue() - - data.update( - { - "type": _type, - "method": "manual", - "data": value, - "checksum": sha256(value).hexdigest(), - } - ) + with BytesIO() as plugin_content: + with tar_open(fileobj=plugin_content, mode="w:gz", compresslevel=9) as tar: + tar.add(file.parent, arcname=file.parent.name, recursive=True) + plugin_content.seek(0) + checksum = bytes_hash(plugin_content, algorithm="sha256") + value = plugin_content.getvalue() + + data.update({"type": _type, "method": "manual", "data": value, "checksum": checksum}) + + if _type == "pro": + self.__pro_plugins.append(data) + else: + self.__external_plugins.append(data) + self.__logger.debug(f"Loaded {_type} plugin {file} with {len(data.get('settings', {}))} setting(s)") + return + self.__core_plugins.append(data) + self.__logger.debug(f"Loaded core plugin {file} with {len(data.get('settings', {}))} setting(s)") + except BaseException as e: + self.__logger.error(f"Exception while loading JSON from {file} : {e}") - with self.__thread_lock: - if _type == "pro": - self.__pro_plugins.append(data) - else: - self.__external_plugins.append(data) - else: - with self.__thread_lock: - self.__core_plugins.append(data) - except: - self.__logger.error(f"Exception while loading JSON from {file} : {format_exc()}") - self.__semaphore.release() - - def __load_variables(self, path: str) -> Dict[str, Any]: + def __load_variables(self, path: Path) -> Dict[str, str]: variables = {} - with open(path) as f: + with path.open("r", encoding="utf-8") as f: lines = f.readlines() for line in lines: line = line.strip() @@ -170,18 +152,34 @@ def __load_variables(self, path: str) -> Dict[str, Any]: variables[split[0]] = split[1] return variables - def get_config(self) -> Dict[str, Any]: + def get_config(self, db=None) -> Dict[str, str]: config = {} + template = self.__variables.get("USE_TEMPLATE", "") + # Extract default settings default_settings = [ - self.__settings, + self.get_settings(), self.get_plugins_settings("core"), self.get_plugins_settings("external"), self.get_plugins_settings("pro"), ] + + if not default_settings[0]: + self.__logger.error("No settings found, exiting") + exit(1) + elif not default_settings[1]: + self.__logger.error("No core plugins found, exiting") + exit(1) + + # Extract template overridden settings + template_settings = {} + if template and db: + self.__logger.info(f"Using template {template}") + template_settings = db.get_template_settings(template) + for settings in default_settings: for setting, data in settings.items(): - config[setting] = data["default"] + config[setting] = template_settings.get(setting, data["default"]) # Override with variables for variable, value in self.__variables.items(): @@ -213,14 +211,20 @@ def get_config(self) -> Dict[str, Any]: "NAMESPACE", ) ): - self.__logger.warning(f"Ignoring variable {variable} : {err}") + self.__logger.warning(f"Ignoring variable {variable} : {err} - {value = !r}") + # Expand variables to each sites if MULTISITE=yes and if not present if config.get("MULTISITE", "no") == "yes": - for server_name in config["SERVER_NAME"].split(" "): + for server_name in config["SERVER_NAME"].strip().split(" "): server_name = server_name.strip() if not server_name: continue + service_template = config.get(f"{server_name}_USE_TEMPLATE", template) + service_template_settings = {} + if service_template != template and db: + service_template_settings = db.get_template_settings(service_template) + for settings in default_settings: for setting, data in settings.items(): if data["context"] == "global": @@ -231,7 +235,8 @@ def get_config(self) -> Dict[str, Any]: if setting == "SERVER_NAME": config[key] = server_name elif setting in config: - config[key] = config[setting] + config[key] = service_template_settings.get(setting, config[setting]) + return config def __check_var(self, variable: str) -> Tuple[bool, str]: @@ -243,7 +248,7 @@ def __check_var(self, variable: str) -> Tuple[bool, str]: return False, f"variable name {variable} doesn't exist" try: - if not re_search(where[real_var]["regex"], value): + if re_search(where[real_var]["regex"], value) is None: return (False, f"value {value} doesn't match regex {where[real_var]['regex']}") except RegexError: self.__logger.warning(f"Invalid regex for {variable} : {where[real_var]['regex']}, ignoring regex check") @@ -265,9 +270,9 @@ def __check_var(self, variable: str) -> Tuple[bool, str]: return True, "ok" - def __find_var(self, variable: str) -> Tuple[Optional[Dict[str, Any]], str]: + def __find_var(self, variable: str) -> Tuple[Optional[Dict[str, str]], str]: targets = [ - self.__settings, + self.get_settings(), self.get_plugins_settings("core"), self.get_plugins_settings("external"), self.get_plugins_settings("pro"), @@ -301,7 +306,7 @@ def __validate_plugin(self, plugin: dict) -> Tuple[bool, str]: elif plugin["stream"] not in ("yes", "no", "partial"): return (False, f"Invalid stream for plugin {plugin['id']} (Must be yes, no or partial)") - for setting, data in plugin["settings"].items(): + for setting, data in plugin.get("settings", {}).items(): if not all(key in data.keys() for key in ("context", "default", "help", "id", "label", "regex", "type")): return (False, f"missing keys for setting {setting} in plugin {plugin['id']}, must have context, default, help, id, label, regex and type") diff --git a/src/common/gen/main.py b/src/common/gen/main.py index 61f794a7d5..93f6edeebe 100644 --- a/src/common/gen/main.py +++ b/src/common/gen/main.py @@ -21,6 +21,7 @@ from Configurator import Configurator from Templator import Templator +DB_PATH = Path(sep, "usr", "share", "bunkerweb", "db") if __name__ == "__main__": logger = setup_logger("Generator", getenv("LOG_LEVEL", "INFO")) @@ -68,6 +69,15 @@ integration = get_integration() + db = None + if DB_PATH.is_dir(): + if DB_PATH.as_posix() not in sys_path: + sys_path.append(DB_PATH.as_posix()) + + from Database import Database # type: ignore + + db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None)) + if args.variables: variables_path = Path(args.variables) variables_path.parent.mkdir(parents=True, exist_ok=True) @@ -105,17 +115,8 @@ logger.info("Computing config ...") config: Dict[str, Any] = Configurator( str(settings_path), str(core_path), str(plugins_path), str(pro_plugins_path), str(variables_path), logger - ).get_config() + ).get_config(db) else: - if join(sep, "usr", "share", "bunkerweb", "db") not in sys_path: - sys_path.append(join(sep, "usr", "share", "bunkerweb", "db")) - - from Database import Database # type: ignore - - db = Database( - logger, - sqlalchemy_string=getenv("DATABASE_URI", None), - ) config: Dict[str, Any] = db.get_config() # Remove old files diff --git a/src/common/gen/save_config.py b/src/common/gen/save_config.py index 548871cd95..ef5ecf4eff 100644 --- a/src/common/gen/save_config.py +++ b/src/common/gen/save_config.py @@ -101,31 +101,6 @@ config = Configurator( str(settings_path), str(core_path), external_plugins, pro_plugins, str(variables_path) if args.variables else environ.copy(), LOGGER ) - settings = config.get_config() - - # Parse BunkerWeb instances from environment - apis = [] - hostnames = set() - for bw_instance in settings.get("BUNKERWEB_INSTANCES", "").split(" "): - if not bw_instance: - continue - - match = BUNKERWEB_STATIC_INSTANCES_RX.search(bw_instance) - if match: - if match.group("hostname") in hostnames: - LOGGER.warning(f"Duplicate BunkerWeb instance hostname {match.group('hostname')}, skipping it") - - hostnames.add(match.group("hostname")) - apis.append( - API( - f"http://{match.group('hostname')}:{match.group('port') or settings.get('API_HTTP_PORT', '5000')}", - host=settings.get("API_SERVER_NAME", "bwapi"), - ) - ) - else: - LOGGER.warning( - f"Invalid BunkerWeb instance {bw_instance}, it should match the following regex: (http://)(:) ({BUNKERWEB_STATIC_INSTANCES_RX.pattern}), skipping it" - ) custom_confs = [] for k, v in environ.items(): @@ -150,7 +125,7 @@ bunkerweb_version = get_version() db_metadata = db.get_metadata() - db_initialized = isinstance(db_metadata, str) or not db_metadata["is_initialized"] + db_initialized = not isinstance(db_metadata, str) and db_metadata["is_initialized"] if not db_initialized: LOGGER.info("Database not initialized, initializing ...") @@ -192,6 +167,32 @@ if args.init: sys_exit(0) + settings = config.get_config(db) + + # Parse BunkerWeb instances from environment + apis = [] + hostnames = set() + for bw_instance in settings.get("BUNKERWEB_INSTANCES", "").split(" "): + if not bw_instance: + continue + + match = BUNKERWEB_STATIC_INSTANCES_RX.search(bw_instance) + if match: + if match.group("hostname") in hostnames: + LOGGER.warning(f"Duplicate BunkerWeb instance hostname {match.group('hostname')}, skipping it") + + hostnames.add(match.group("hostname")) + apis.append( + API( + f"http://{match.group('hostname')}:{match.group('port') or settings.get('API_HTTP_PORT', '5000')}", + host=settings.get("API_SERVER_NAME", "bwapi"), + ) + ) + else: + LOGGER.warning( + f"Invalid BunkerWeb instance {bw_instance}, it should match the following regex: (http://)(:) ({BUNKERWEB_STATIC_INSTANCES_RX.pattern}), skipping it" + ) + changes = [] changed_plugins = set() err = db.save_config(settings, args.method, changed=False) diff --git a/src/common/settings.json b/src/common/settings.json index 308415e495..17d8a9c161 100644 --- a/src/common/settings.json +++ b/src/common/settings.json @@ -325,5 +325,14 @@ "label": "BunkerWeb instances", "regex": "^.*$", "type": "text" + }, + "USE_TEMPLATE": { + "context": "multisite", + "default": "", + "help": "Config template to use that will override the default values of specific settings.", + "id": "use-template", + "label": "Use template", + "regex": "^.*$", + "type": "text" } } diff --git a/src/common/templates/high.json b/src/common/templates/high.json deleted file mode 100644 index 5b299cbb4e..0000000000 --- a/src/common/templates/high.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "name": "medium", - "description": "Generic settings template with high security level required for your web service. False positives will certainly appear without any custom edit.", - "steps": [ - { - "name": "Server configuration", - "description": "Configure your server name and reverse proxy settings. Don't forget to add the corresponding DNS A entry pointing to your BunkerWeb IP.", - "settings": { - "SERVER_NAME": "www.example.com", - "USE_REVERSE_PROXY": "yes", - "REVERSE_PROXY_HOST": "http://my-upstream-server:8080", - "REVERSE_PROXY_URL": "/", - "REVERSE_PROXY_INTERCEPT_ERRORS": "yes", - "REVERSE_PROXY_WS": "no", - "REVERSE_PROXY_CUSTOM_HOST": "", - "REVERSE_PROXY_HEADERS": "Accept-Encoding ''", - "SERVE_FILES": "no" - } - }, - { - "name": "HTTPS", - "description": "Enable/disable and configure HTTPS for your service.", - "settings": { - "AUTO_LETS_ENCRYPT": "yes", - "SSL_PROTOCOLS": "TLSv1.3" - } - }, - { - "name": "HTTP configuration", - "description": "Miscellaneous settings related to HTTP protocol.", - "settings": { - "DENY_HTTP_STATUS": "444", - "USE_GZIP": "yes", - "USE_BROTLI": "yes", - "ALLOWED_METHODS": "GET|POST|HEAD", - "MAX_SIZES": "10m", - "COOKIE_FLAGS": "* HttpOnly SameSite=Lax", - "CONTENT_SECURITY_POLICY": "object-src 'none'; form-action 'self'; frame-ancestors 'self';", - "X_FRAME_OPTIONS": "SAMEORIGIN", - "PERMISSIONS_POLICY": "accelerometer=(), ambient-light-sensor=(), autoplay=(), battery=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), execution-while-not-rendered=(), execution-while-out-of-viewport=(), fullscreen=(), geolocation=(), gyroscope=(), hid=(), idle-detection=(), magnetometer=(), microphone=(), midi=(), navigation-override=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), serial=(), usb=(), web-share=(), xr-spatial-tracking=()", - "FEATURE_POLICY": "accelerometer 'none'; ambient-light-sensor 'none'; autoplay 'none'; battery 'none'; camera 'none'; display-capture 'none'; document-domain 'none'; encrypted-media 'none'; execution-while-not-rendered 'none'; execution-while-out-of-viewport 'none'; fullscreen 'none'; geolocation 'none'; gyroscope 'none'; layout-animation 'none'; legacy-image-formats 'none'; magnetometer 'none'; microphone 'none'; midi 'none'; navigation-override 'none'; payment 'none'; picture-in-picture 'none'; publickey-credentials-get 'none'; speaker-selection 'none'; sync-xhr 'none'; unoptimized-images 'none'; unsized-media 'none'; usb 'none'; screen-wake-lock 'none'; web-share 'none'; xr-spatial-tracking 'none';" - } - }, - { - "name": "Bad behavior", - "description": "Configure automatic bans when detecting bad behaviors on your web service.", - "settings": { - "USE_BAD_BEHAVIOR": "yes", - "BAD_BEHAVIOR_STATUS_CODES": "400 401 403 404 405 429 444", - "BAD_BEHAVIOR_BAN_TIME": "86400", - "BAD_BEHAVIOR_THRESHOLD": "5", - "BAD_BEHAVIOR_COUNT_TIME": "60" - } - }, - { - "name": "Limit", - "description": "Configure requests and connections limits on your web service.", - "settings": { - "USE_LIMIT_CONN": "yes", - "LIMIT_CONN_MAX_HTTP1": "10", - "LIMIT_CONN_MAX_HTTP2": "100", - "USE_LIMIT_REQ": "yes", - "LIMIT_REQ_URL": "/", - "LIMIT_REQ_RATE": "2r/s" - } - }, - { - "name": "DNSBL", - "description": "Enable/disable DNSBL protection. Might generate false positives especially if you have a worldwide audience.", - "settings": { - "USE_DNSBL": "yes" - } - }, - { - "name": "Country", - "description": "Configure allowed countries to reach out your web service. Recommended if you protect a restricted area such as extranet or administration panel.", - "settings": { - "WHITELIST_COUNTRY": "" - } - }, - { - "name": "Antibot", - "description": "Enable/disable and configure antibot protection globally on your web service.", - "settings": { - "USE_ANTIBOT": "captcha", - "ANTIBOT_TIME_RESOLVE": "120", - "ANTIBOT_TIME_VALID": "86400", - "ANTIBOT_RECAPTCHA_SCORE": "0.7", - "ANTIBOT_RECAPTCHA_SITEKEY": "", - "ANTIBOT_RECAPTCHA_SECRET": "", - "ANTIBOT_HCAPTCHA_SITEKEY": "", - "ANTIBOT_HCAPTCHA_SECRET": "", - "ANTIBOT_TURNSTILE_SITEKEY": "", - "ANTIBOT_TURNSTILE_SECRET": "" - } - }, - { - "name": "CORS", - "description": "Configure Cross-Origin Resource Sharing (CORS) to allow/deny external requests to your web service.", - "settings": { - "USE_CORS": "yes", - "CORS_ALLOW_ORIGIN": "" - } - }, - { - "name": "Reverse scan", - "description": "Configure reverse scan of client to detect open proxy or datacenter connections.", - "settings": { - "USE_REVERSE_SCAN": "yes", - "REVERSE_SCAN_PORTS": "22 80 443 3128 8000 8080" - } - }, - { - "name": "ModSecurity", - "description": "Enable/disable and configure ModSecurity on your web service.", - "settings": { - "USE_MODSECURITY": "yes", - "MODSECURITY_CRS_VERSION": "4" - }, - "configs": [ - { - "name": "template-high", - "type": "modsec-crs", - "data": "SecAction \"id:900000,phase:1,pass,t:none,nolog,tag:'OWASP_CRS',ver:'OWASP_CRS/4.2.0',setvar:tx.blocking_paranoia_level=4\"" - } - ] - } - ] -} diff --git a/src/common/templates/low.json b/src/common/templates/low.json deleted file mode 100644 index e077f89acc..0000000000 --- a/src/common/templates/low.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "name": "low", - "description": "Generic settings template with low security level to avoid false positives and get started with BunkerWeb.", - "steps": [ - { - "name": "Server configuration", - "description": "Configure your server name and reverse proxy settings. Don't forget to add the corresponding DNS A entry pointing to your BunkerWeb IP.", - "settings": { - "SERVER_NAME": "www.example.com", - "USE_REVERSE_PROXY": "yes", - "REVERSE_PROXY_HOST": "http://my-upstream-server:8080", - "REVERSE_PROXY_URL": "/", - "REVERSE_PROXY_INTERCEPT_ERRORS": "no", - "REVERSE_PROXY_WS": "yes", - "REVERSE_PROXY_CUSTOM_HOST": "", - "REVERSE_PROXY_HEADERS": "Accept-Encoding ''" - } - }, - { - "name": "HTTPS", - "description": "Enable/disable HTTPS for your service.", - "settings": { - "AUTO_LETS_ENCRYPT": "yes" - } - }, - { - "name": "HTTP configuration", - "description": "Miscellaneous settings related to HTTP protocol.", - "settings": { - "USE_GZIP": "yes", - "USE_BROTLI": "yes", - "ALLOWED_METHODS": "GET|POST|HEAD|PUT|PATCH|OPTIONS|DELETE", - "MAX_SIZES": "50m", - "COOKIE_FLAGS": "* SameSite=Lax", - "CONTENT_SECURITY_POLICY": "", - "X_FRAME_OPTIONS": "", - "PERMISSIONS_POLICY": "", - "FEATURE_POLICY": "", - "KEEP_UPSTREAM_HEADERS": "*" - } - }, - { - "name": "Bad behavior", - "description": "Configure automatic bans when detecting bad behaviors on your web service.", - "settings": { - "USE_BAD_BEHAVIOR": "yes", - "BAD_BEHAVIOR_STATUS_CODES": "400 401 403 405 429 444", - "BAD_BEHAVIOR_BAN_TIME": "3600", - "BAD_BEHAVIOR_THRESHOLD": "20", - "BAD_BEHAVIOR_COUNT_TIME": "60" - } - }, - { - "name": "Limit", - "description": "Configure requests and connections limits on your web service.", - "settings": { - "USE_LIMIT_CONN": "yes", - "LIMIT_CONN_MAX_HTTP1": 20, - "LIMIT_CONN_MAX_HTTP2": 200, - "USE_LIMIT_REQ": "yes", - "LIMIT_REQ_URL": "/", - "LIMIT_REQ_RATE": "5r/s" - } - }, - { - "name": "DNSBL", - "description": "Enable/disable DNSBL protection. Might generate false positives especially if you have a worldwide audience.", - "settings": { - "USE_DNSBL": "no" - } - }, - { - "name": "Country", - "description": "Configure allowed countries to reach out your web service. Recommended if you protect a restricted area such as extranet or administration panel.", - "settings": { - "WHITELIST_COUNTRY": "" - } - }, - { - "name": "Antibot", - "description": "Enable/disable and configure antibot protection globally on your web service.", - "settings": { - "USE_ANTIBOT": "no", - "ANTIBOT_TIME_RESOLVE": "120", - "ANTIBOT_TIME_VALID": "86400", - "ANTIBOT_RECAPTCHA_SCORE": "0.7", - "ANTIBOT_RECAPTCHA_SITEKEY": "", - "ANTIBOT_RECAPTCHA_SECRET": "", - "ANTIBOT_HCAPTCHA_SITEKEY": "", - "ANTIBOT_HCAPTCHA_SECRET": "", - "ANTIBOT_TURNSTILE_SITEKEY": "", - "ANTIBOT_TURNSTILE_SECRET": "" - } - }, - { - "name": "ModSecurity", - "description": "Enable/disable and configure ModSecurity on your web service.", - "settings": { - "USE_MODSECURITY": "yes" - }, - "configs": [ - { - "name": "template-low", - "type": "modsec-crs", - "description": "Override ModSecurity CRS settings.", - "data": "SecAction \"id:900110,phase:1,nolog,pass,t:none,setvar:tx.inbound_anomaly_score_threshold=7,setvar:tx.outbound_anomaly_score_threshold=4\"" - } - ] - } - ] -} diff --git a/src/common/templates/medium.json b/src/common/templates/medium.json deleted file mode 100644 index b9bb90f975..0000000000 --- a/src/common/templates/medium.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "name": "medium", - "description": "Generic settings template with medium security level aimed for average web service in production. False positives may appear depending on your environment.", - "steps": [ - { - "name": "Server configuration", - "description": "Configure your server name and reverse proxy settings. Don't forget to add the corresponding DNS A entry pointing to your BunkerWeb IP.", - "settings": { - "SERVER_NAME": "www.example.com", - "USE_REVERSE_PROXY": "yes", - "REVERSE_PROXY_HOST": "http://my-upstream-server:8080", - "REVERSE_PROXY_URL": "/", - "REVERSE_PROXY_INTERCEPT_ERRORS": "yes", - "REVERSE_PROXY_WS": "no", - "REVERSE_PROXY_CUSTOM_HOST": "", - "REVERSE_PROXY_HEADERS": "Accept-Encoding ''" - } - }, - { - "name": "HTTPS", - "description": "Enable/disable HTTPS for your service.", - "settings": { - "AUTO_LETS_ENCRYPT": "yes" - } - }, - { - "name": "HTTP configuration", - "description": "Miscellaneous settings related to HTTP protocol.", - "settings": { - "USE_GZIP": "yes", - "USE_BROTLI": "yes", - "ALLOWED_METHODS": "GET|POST|HEAD", - "MAX_SIZES": "10m", - "COOKIE_FLAGS": "* HttpOnly SameSite=Lax", - "CONTENT_SECURITY_POLICY": "object-src 'none'; form-action 'self'; frame-ancestors 'self';", - "X_FRAME_OPTIONS": "SAMEORIGIN", - "PERMISSIONS_POLICY": "accelerometer=(), ambient-light-sensor=(), autoplay=(), battery=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), execution-while-not-rendered=(), execution-while-out-of-viewport=(), fullscreen=(), geolocation=(), gyroscope=(), hid=(), idle-detection=(), magnetometer=(), microphone=(), midi=(), navigation-override=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), serial=(), usb=(), web-share=(), xr-spatial-tracking=()", - "FEATURE_POLICY": "accelerometer 'none'; ambient-light-sensor 'none'; autoplay 'none'; battery 'none'; camera 'none'; display-capture 'none'; document-domain 'none'; encrypted-media 'none'; execution-while-not-rendered 'none'; execution-while-out-of-viewport 'none'; fullscreen 'none'; geolocation 'none'; gyroscope 'none'; layout-animation 'none'; legacy-image-formats 'none'; magnetometer 'none'; microphone 'none'; midi 'none'; navigation-override 'none'; payment 'none'; picture-in-picture 'none'; publickey-credentials-get 'none'; speaker-selection 'none'; sync-xhr 'none'; unoptimized-images 'none'; unsized-media 'none'; usb 'none'; screen-wake-lock 'none'; web-share 'none'; xr-spatial-tracking 'none';" - } - }, - { - "name": "Bad behavior", - "description": "Configure automatic bans when detecting bad behaviors on your web service.", - "settings": { - "USE_BAD_BEHAVIOR": "yes", - "BAD_BEHAVIOR_STATUS_CODES": "400 401 403 404 405 429 444", - "BAD_BEHAVIOR_BAN_TIME": "86400", - "BAD_BEHAVIOR_THRESHOLD": "10", - "BAD_BEHAVIOR_COUNT_TIME": "60" - } - }, - { - "name": "Limit", - "description": "Configure requests and connections limits on your web service.", - "settings": { - "USE_LIMIT_CONN": "yes", - "LIMIT_CONN_MAX_HTTP1": "10", - "LIMIT_CONN_MAX_HTTP2": "100", - "USE_LIMIT_REQ": "yes", - "LIMIT_REQ_URL": "/", - "LIMIT_REQ_RATE": "2r/s" - } - }, - { - "name": "DNSBL", - "description": "Enable/disable DNSBL protection. Might generate false positives especially if you have a worldwide audience.", - "settings": { - "USE_DNSBL": "yes" - } - }, - { - "name": "Country", - "description": "Configure allowed countries to reach out your web service. Recommended if you protect a restricted area such as extranet or administration panel.", - "settings": { - "WHITELIST_COUNTRY": "" - } - }, - { - "name": "Antibot", - "description": "Enable/disable and configure antibot protection globally on your web service.", - "settings": { - "USE_ANTIBOT": "javascript", - "ANTIBOT_TIME_RESOLVE": "120", - "ANTIBOT_TIME_VALID": "86400", - "ANTIBOT_RECAPTCHA_SCORE": "0.7", - "ANTIBOT_RECAPTCHA_SITEKEY": "", - "ANTIBOT_RECAPTCHA_SECRET": "", - "ANTIBOT_HCAPTCHA_SITEKEY": "", - "ANTIBOT_HCAPTCHA_SECRET": "", - "ANTIBOT_TURNSTILE_SITEKEY": "", - "ANTIBOT_TURNSTILE_SECRET": "" - } - }, - { - "name": "CORS", - "description": "Configure Cross-Origin Resource Sharing (CORS) to allow/deny external requests to your web service.", - "settings": { - "USE_CORS": "no", - "CORS_ALLOW_ORIGIN": "*" - } - }, - { - "name": "ModSecurity", - "description": "Enable/disable and configure ModSecurity on your web service.", - "settings": { - "USE_MODSECURITY": "yes" - } - } - ] -} diff --git a/src/ui/Dockerfile b/src/ui/Dockerfile index 17a09510d0..99566da8ec 100644 --- a/src/ui/Dockerfile +++ b/src/ui/Dockerfile @@ -33,7 +33,6 @@ COPY src/common/gen gen COPY src/common/settings.json settings.json COPY src/common/utils utils COPY src/common/helpers helpers -COPY src/common/templates templates COPY src/VERSION VERSION COPY src/ui/builder ui/builder