From bfcc317cef33ac7de02de6a64c5f6fe543393384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?An=C5=BEe=20Pe=C4=8Dar?= Date: Sat, 2 Nov 2024 01:12:35 +0000 Subject: [PATCH 1/3] Improve SQLite default settings These settings make SQLite scale better in production environments with higher concurrency. --- src/dj_beat_drop/new.py | 43 +++++++++++++++++++++++++++++++++++++++ tests/test_new_command.py | 36 +++++++++++++++++++++++++++++--- 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src/dj_beat_drop/new.py b/src/dj_beat_drop/new.py index df0872c..6ced892 100644 --- a/src/dj_beat_drop/new.py +++ b/src/dj_beat_drop/new.py @@ -2,8 +2,10 @@ import re import shutil from pathlib import Path +from textwrap import dedent from InquirerPy import inquirer +from packaging.version import Version from dj_beat_drop import utils from dj_beat_drop.utils import color @@ -18,6 +20,35 @@ def rename_template_files(project_dir): os.rename(file, file.with_name(file.name[:-4])) +def replace_sqlite_config(content: str, django_version: str) -> str: + if Version(django_version) < Version("5.1"): + return content + + rtn_val = content + rtn_val = re.sub( + r"^DATABASES\s*=\s*\{.+?\}\n\}", + dedent(r'''DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + 'OPTIONS': { + 'transaction_mode': 'IMMEDIATE', + 'init_command': """ + PRAGMA journal_mode=WAL; + PRAGMA synchronous=NORMAL; + PRAGMA mmap_size = 134217728; + PRAGMA journal_size_limit = 27103364; + PRAGMA cache_size=2000; + """, + }, + } + }'''), + rtn_val, + flags=re.MULTILINE | re.DOTALL, + ) + return rtn_val + + def replace_settings_with_environs(content: str) -> str: rtn_val = content init_env = "# Initialize environs\n" "env = Env()\n" "env.read_env()" @@ -53,6 +84,16 @@ def create_dot_envfile(project_dir, context: dict[str, str]): f'ALLOWED_HOSTS=\n' f"DATABASE_URL=sqlite:///{project_dir / 'db.sqlite3'}" ) + if Version(context["django_version"]) >= Version("5.1"): + env_content += ( + "?transaction_mode=immediate" + "&init_command=PRAGMA journal_mode=WAL" + ";PRAGMA synchronous=NORMAL" + ";PRAGMA mmap_size=134217728" + ";PRAGMA journal_size_limit=27103364" + ";PRAGMA cache_size=2000" + ) + with open(env_file_path, "w") as f: f.write(env_content) @@ -68,6 +109,8 @@ def replace_variables(project_dir, context: dict[str, str], initialize_env): if str(file.relative_to(project_dir)) == "config/settings.py" and initialize_env is True: content = replace_settings_with_environs(content) create_dot_envfile(project_dir, context) + if str(file.relative_to(project_dir)) == "config/settings.py" and initialize_env is False: + content = replace_sqlite_config(content, context["django_version"]) with file.open("w") as f: f.write(content) diff --git a/tests/test_new_command.py b/tests/test_new_command.py index f089522..41b0cb1 100644 --- a/tests/test_new_command.py +++ b/tests/test_new_command.py @@ -3,11 +3,31 @@ import shutil import string from pathlib import Path +from textwrap import dedent from unittest import TestCase +from packaging.version import Version + from dj_beat_drop.new import create_new_project ENV_SECRET_KEY_PATTERN = 'SECRET_KEY = env.str("SECRET_KEY")' # noqa: S105 +SQLITE_OPTIONS_ENV = ( + "?transaction_mode=immediate" + "&init_command=PRAGMA journal_mode=WAL" + ";PRAGMA synchronous=NORMAL" + ";PRAGMA mmap_size=134217728" + ";PRAGMA journal_size_limit=27103364" + ";PRAGMA cache_size=2000" +) +SQLITE_OPTIONS = dedent(''' + 'transaction_mode': 'IMMEDIATE', + 'init_command': """ + PRAGMA journal_mode=WAL; + PRAGMA synchronous=NORMAL; + PRAGMA mmap_size = 134217728; + PRAGMA journal_size_limit = 27103364; + PRAGMA cache_size=2000; +''') FILE_ASSERTIONS = { "manage.py": [ "os.environ.setdefault('DJANGO_SETTINGS_MODULE', '{{ project_name }}.settings')", @@ -47,6 +67,7 @@ 'SECRET_KEY="{{ secret_key }}"', "ALLOWED_HOSTS=", "DATABASE_URL=sqlite:///{{ project_dir }}/db.sqlite3", + "{% 5.1 %}" + SQLITE_OPTIONS_ENV, ], "config/settings.py": [ "from environs import Env", @@ -56,6 +77,9 @@ 'DATABASES = {"default": env.dj_db_url("DATABASE_URL")}', ], } +NO_ENV_ASSERTIONS = { + "config/settings.py": ["{% 5.1 %}" + option for option in SQLITE_OPTIONS.splitlines()], +} class SafeDict(dict): @@ -112,6 +136,8 @@ def assert_files_are_correct( assertions.extend(uv_assertions) if initialize_env is True: assertions.extend(env_assertions) + else: + assertions.extend(NO_ENV_ASSERTIONS.get(relative_path, [])) with open(file) as f: content = f.read() for assertion_pattern in assertions: @@ -121,11 +147,15 @@ def assert_files_are_correct( and relative_path == "config/settings.py" ): assertion_pattern = ENV_SECRET_KEY_PATTERN - if re.match(r".*{{\s[_a-z]+\s}}.*", assertion_pattern) is None: - assertion = assertion_pattern - else: + if re.match(r".*{{\s[_a-z]+\s}}.*", assertion_pattern): formatted_assertion = assertion_pattern.replace("{{ ", "{").replace(" }}", "}") assertion = formatted_assertion.format_map(SafeDict(assertion_context)) + elif rematch := re.match(r".*{%\s(.*)\s%}.*", assertion_pattern): + if Version(template_context["django_version"]) < Version(rematch.group(1)): + continue + assertion = assertion_pattern.replace(f"{{% {rematch.group(1)} %}}", "") + else: + assertion = assertion_pattern assert assertion in content, f"Assertion failed for {relative_path}: {assertion}" def test_new_command_with_defaults(self): From a9e41cefb530d33de59a5422da2be54fb1470126 Mon Sep 17 00:00:00 2001 From: Brent O'Connor Date: Sat, 2 Nov 2024 18:02:40 -0500 Subject: [PATCH 2/3] Urlencode option params and move params to one constant --- src/dj_beat_drop/new.py | 62 ++++++++++++++++++++------------------- tests/test_new_command.py | 29 +++++++++--------- 2 files changed, 47 insertions(+), 44 deletions(-) diff --git a/src/dj_beat_drop/new.py b/src/dj_beat_drop/new.py index 6ced892..089455d 100644 --- a/src/dj_beat_drop/new.py +++ b/src/dj_beat_drop/new.py @@ -1,6 +1,7 @@ import os import re import shutil +import urllib.parse from pathlib import Path from textwrap import dedent @@ -10,6 +11,17 @@ from dj_beat_drop import utils from dj_beat_drop.utils import color +EXTRA_SQLITE_PARAMS = { + "transaction_mode": "IMMEDIATE", + "init_command": ( + "PRAGMA journal_mode = WAL;" + "PRAGMA synchronous = NORMAL;" + "PRAGMA mmap_size = 134217728;" + "PRAGMA journal_size_limit = 27103364;" + "PRAGMA cache_size = 2000" + ), +} + def rename_template_files(project_dir): # Rename .py-tpl files to .py @@ -24,25 +36,26 @@ def replace_sqlite_config(content: str, django_version: str) -> str: if Version(django_version) < Version("5.1"): return content + init_command_str = "".join( + [f' "{param};"\n' for param in EXTRA_SQLITE_PARAMS["init_command"].split(";")] + ) rtn_val = content rtn_val = re.sub( r"^DATABASES\s*=\s*\{.+?\}\n\}", - dedent(r'''DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': BASE_DIR / 'db.sqlite3', - 'OPTIONS': { - 'transaction_mode': 'IMMEDIATE', - 'init_command': """ - PRAGMA journal_mode=WAL; - PRAGMA synchronous=NORMAL; - PRAGMA mmap_size = 134217728; - PRAGMA journal_size_limit = 27103364; - PRAGMA cache_size=2000; - """, - }, - } - }'''), + dedent( + "DATABASES = {\n" + " 'default': {\n" + " 'ENGINE': 'django.db.backends.sqlite3',\n" + " 'NAME': BASE_DIR / 'db.sqlite3',\n" + " 'OPTIONS': {\n" + f" 'transaction_mode': '{EXTRA_SQLITE_PARAMS['transaction_mode']}',\n" + ' \'init_command\': (\n' + f'{init_command_str}' + ' )\n' + " }\n" + " }\n" + "}\n" + ), rtn_val, flags=re.MULTILINE | re.DOTALL, ) @@ -78,21 +91,10 @@ def replace_settings_with_environs(content: str) -> str: def create_dot_envfile(project_dir, context: dict[str, str]): env_file_path = project_dir / ".env" - env_content = ( - "DEBUG=True\n" - f"SECRET_KEY=\"{context['secret_key']}\"\n" - f'ALLOWED_HOSTS=\n' - f"DATABASE_URL=sqlite:///{project_dir / 'db.sqlite3'}" - ) + sqlite_url = f"sqlite:///{project_dir / 'db.sqlite3'}" if Version(context["django_version"]) >= Version("5.1"): - env_content += ( - "?transaction_mode=immediate" - "&init_command=PRAGMA journal_mode=WAL" - ";PRAGMA synchronous=NORMAL" - ";PRAGMA mmap_size=134217728" - ";PRAGMA journal_size_limit=27103364" - ";PRAGMA cache_size=2000" - ) + sqlite_url += "?" + urllib.parse.urlencode(EXTRA_SQLITE_PARAMS) + env_content = f"DEBUG=True\nSECRET_KEY=\"{context['secret_key']}\"\nALLOWED_HOSTS=\nDATABASE_URL={sqlite_url}\n" with open(env_file_path, "w") as f: f.write(env_content) diff --git a/tests/test_new_command.py b/tests/test_new_command.py index 41b0cb1..01a6ae0 100644 --- a/tests/test_new_command.py +++ b/tests/test_new_command.py @@ -12,22 +12,23 @@ ENV_SECRET_KEY_PATTERN = 'SECRET_KEY = env.str("SECRET_KEY")' # noqa: S105 SQLITE_OPTIONS_ENV = ( - "?transaction_mode=immediate" - "&init_command=PRAGMA journal_mode=WAL" - ";PRAGMA synchronous=NORMAL" - ";PRAGMA mmap_size=134217728" - ";PRAGMA journal_size_limit=27103364" - ";PRAGMA cache_size=2000" + "?transaction_mode=IMMEDIATE" + "&init_command=PRAGMA+journal_mode+%3D+WAL" + "%3BPRAGMA+synchronous+%3D+NORMAL" + "%3BPRAGMA+mmap_size+%3D+134217728" + "%3BPRAGMA+journal_size_limit+%3D+27103364" + "%3BPRAGMA+cache_size+%3D+2000" ) -SQLITE_OPTIONS = dedent(''' +SQLITE_OPTIONS = dedent(""" 'transaction_mode': 'IMMEDIATE', - 'init_command': """ - PRAGMA journal_mode=WAL; - PRAGMA synchronous=NORMAL; - PRAGMA mmap_size = 134217728; - PRAGMA journal_size_limit = 27103364; - PRAGMA cache_size=2000; -''') + 'init_command': ( + "PRAGMA journal_mode = WAL;" + "PRAGMA synchronous = NORMAL;" + "PRAGMA mmap_size = 134217728;" + "PRAGMA journal_size_limit = 27103364;" + "PRAGMA cache_size = 2000;" + ) +""") FILE_ASSERTIONS = { "manage.py": [ "os.environ.setdefault('DJANGO_SETTINGS_MODULE', '{{ project_name }}.settings')", From d279e4ea203a76c5df3ef306fa9c056090630b16 Mon Sep 17 00:00:00 2001 From: Brent O'Connor Date: Sat, 2 Nov 2024 18:13:00 -0500 Subject: [PATCH 3/3] Switch to using a tuple for the version in an assertion --- tests/test_new_command.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_new_command.py b/tests/test_new_command.py index 01a6ae0..afec6fc 100644 --- a/tests/test_new_command.py +++ b/tests/test_new_command.py @@ -68,7 +68,8 @@ 'SECRET_KEY="{{ secret_key }}"', "ALLOWED_HOSTS=", "DATABASE_URL=sqlite:///{{ project_dir }}/db.sqlite3", - "{% 5.1 %}" + SQLITE_OPTIONS_ENV, + # If Django version is 5.1 or higher, the following option should be present in the DATABASE_URL + ("5.1", SQLITE_OPTIONS_ENV), ], "config/settings.py": [ "from environs import Env", @@ -79,7 +80,8 @@ ], } NO_ENV_ASSERTIONS = { - "config/settings.py": ["{% 5.1 %}" + option for option in SQLITE_OPTIONS.splitlines()], + # If Django version is 5.1 or higher, the following option should be present in the DATABASE_URL + "config/settings.py": [("5.1", option) for option in SQLITE_OPTIONS.splitlines()], } @@ -142,6 +144,11 @@ def assert_files_are_correct( with open(file) as f: content = f.read() for assertion_pattern in assertions: + version_str = None + if isinstance(assertion_pattern, list | tuple) is True: + version_str, assertion_pattern = assertion_pattern + if version_str and Version(template_context["django_version"]) < Version(version_str): + continue if ( assertion_pattern.startswith("SECRET_KEY =") and initialize_env is True @@ -151,10 +158,6 @@ def assert_files_are_correct( if re.match(r".*{{\s[_a-z]+\s}}.*", assertion_pattern): formatted_assertion = assertion_pattern.replace("{{ ", "{").replace(" }}", "}") assertion = formatted_assertion.format_map(SafeDict(assertion_context)) - elif rematch := re.match(r".*{%\s(.*)\s%}.*", assertion_pattern): - if Version(template_context["django_version"]) < Version(rematch.group(1)): - continue - assertion = assertion_pattern.replace(f"{{% {rematch.group(1)} %}}", "") else: assertion = assertion_pattern assert assertion in content, f"Assertion failed for {relative_path}: {assertion}"