Merge branch 'main' into playwright-e2e

.github/workflows/docker-ci-tests.yml

@@ -1,4 +1,4 @@
-name: 'Pytest on docker'
+name: Pytest Docker
 
 on:
   push:
@@ -25,7 +25,8 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 jobs:
-  test:
+  pytest-docker:
+    name: Pytest Docker
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

.github/workflows/pytest.yml

@@ -26,7 +26,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
+  pytest:
+    name: Pytest
+    timeout-minutes: 10
     runs-on: ${{ matrix.os }}
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -125,7 +127,11 @@ jobs:
           psql -h localhost -U postgres geoname_testing -c "grant all privileges on schema public to $(whoami); grant all privileges on all tables in schema public to $(whoami); grant all privileges on all sequences in schema public to $(whoami);"
       - name: Test with pytest
         run: |
-          pytest --disable-warnings --gherkin-terminal-reporter -vv --showlocals --cov=funnel
+          pytest --disable-warnings --gherkin-terminal-reporter -vv --showlocals --ignore=tests/e2e
+      - name: Browser tests with pytest
+        timeout-minutes: 5
+        run: |
+          pytest --disable-warnings --gherkin-terminal-reporter -vv --showlocals --cov-append --cov=funnel tests/e2e
       - name: Prepare coverage report
         run: |
           mkdir -p coverage
@@ -139,7 +145,7 @@ jobs:
           parallel: true
 
   finish:
-    needs: test
+    needs: pytest
     runs-on: ubuntu-latest
     steps:
       - name: Publish to Coveralls

.pre-commit-config.yaml

@@ -7,7 +7,6 @@ default_language_version:
 ci:
   skip: [
       'pip-audit',
-      'yesqa',
       'creosote',
       'no-commit-to-branch',
       # 'hadolint-docker',
@@ -29,7 +28,7 @@ repos:
       - id: pip-compile-multi-verify
         files: ^requirements/.*\.(in|txt)$
   - repo: https://github.com/pypa/pip-audit
-    rev: v2.7.0
+    rev: v2.7.3
     hooks:
       - id: pip-audit
         args: [
@@ -51,75 +50,18 @@ repos:
           ]
         files: ^requirements/.*\.txt$
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.0
+    rev: v3.15.2
     hooks:
       - id: pyupgrade
         args: ['--keep-runtime-typing', '--py311-plus']
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.2.0
+    rev: v0.4.6
     hooks:
       - id: ruff
         args: ['--fix', '--exit-non-zero-on-fix']
-        # Extra args, only after removing flake8 and yesqa: '--extend-select', 'RUF100'
-  - repo: https://github.com/lucasmbrown/mirrors-autoflake
-    rev: v1.3
-    hooks:
-      - id: autoflake
-        args:
-          [
-            '--in-place',
-            '--remove-all-unused-imports',
-            '--ignore-init-module-imports',
-            '--remove-unused-variables',
-            '--remove-duplicate-keys',
-          ]
-  - repo: https://github.com/asottile/yesqa
-    rev: v1.5.0
-    hooks:
-      - id: yesqa
-        additional_dependencies: &flake8deps
-          - bandit
-          # - flake8-annotations
-          - flake8-assertive
-          - flake8-blind-except
-          - flake8-bugbear
-          - flake8-builtins
-          - flake8-comprehensions
-          - flake8-docstrings
-          - flake8-isort
-          - flake8-logging-format
-          - flake8-mutable
-          - flake8-plugin-utils
-          - flake8-print
-          - flake8-pytest-style
-          - pep8-naming
-          - toml
-          - tomli
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
-        additional_dependencies:
-          - tomli
-  - repo: https://github.com/psf/black
-    rev: 24.1.1
-    hooks:
-      - id: black
-  - repo: https://github.com/PyCQA/flake8
-    rev: 7.0.0
-    hooks:
-      - id: flake8
-        additional_dependencies: *flake8deps
-  - repo: https://github.com/PyCQA/flake8
-    rev: 7.0.0
-    hooks:
-      - id: flake8
-        name: flake8-pyi
-        types: [pyi]
-        additional_dependencies:
-          - flake8-pyi
+      - id: ruff-format
   - repo: https://github.com/PyCQA/pylint
-    rev: v3.0.3
+    rev: v3.2.2
     hooks:
       - id: pylint
         args: [
@@ -130,26 +72,18 @@ repos:
           ]
         additional_dependencies:
           - tomli
-  - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.7
-    hooks:
-      - id: bandit
-        language_version: python3
-        args: ['-c', 'pyproject.toml']
-        additional_dependencies:
-          - 'bandit[toml]'
   - repo: https://github.com/fredrikaverpil/creosote
     rev: v3.0.0
     hooks:
       - id: creosote
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.9.0.6
+    rev: v0.10.0.1
     hooks:
       - id: shellcheck
         args:
           - --external-sources
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-ast
@@ -183,7 +117,7 @@ repos:
       - id: trailing-whitespace
         args: ['--markdown-linebreak-ext=md']
   - repo: https://github.com/Lucas-C/pre-commit-hooks
-    rev: v1.5.4
+    rev: v1.5.5
     hooks:
       - id: forbid-crlf
       - id: remove-crlf

.testenv

@@ -4,6 +4,8 @@
 FLASK_ENV=testing
 FLASK_TESTING=true
 FLASK_DEBUG_TB_ENABLED=false
+# Disable Recaptcha
+FLASK_RECAPTCHA_DISABLED=true
 # Enable CSRF so tests reflect production use
 FLASK_WTF_CSRF_ENABLED=true
 # Use Redis cache so that rate limit validation tests work, with Redis db

Makefile

@@ -120,30 +120,30 @@ initpy: initpy-models initpy-forms initpy-loginproviders initpy-transports initp
 
 initpy-models:
 	mkinit --inplace --relative --black --lazy_loader_typed funnel/models/__init__.py
-	isort funnel/models/__init__.py funnel/models/__init__.pyi
-	black funnel/models/__init__.py funnel/models/__init__.pyi
+	ruff check --fix funnel/models/__init__.py funnel/models/__init__.pyi
+	ruff format funnel/models/__init__.py funnel/models/__init__.pyi
 
 initpy-forms:
 	mkinit --inplace --relative --black funnel/forms/__init__.py
-	isort funnel/forms/__init__.py
-	black funnel/forms/__init__.py
+	ruff check --fix funnel/forms/__init__.py
+	ruff format funnel/forms/__init__.py
 
 initpy-loginproviders:
 	mkinit --inplace --relative --black funnel/loginproviders/__init__.py
-	isort funnel/loginproviders/__init__.py
-	black funnel/loginproviders/__init__.py
+	ruff check --fix funnel/loginproviders/__init__.py
+	ruff format funnel/loginproviders/__init__.py
 
 initpy-transports:
 	# Do not auto-gen funnel/transports/__init__.py, only sub-packages
 	mkinit --inplace --relative --black funnel/transports/email
-	mkinit --inplace --relative --black  funnel/transports/sms
-	isort funnel/transports/*/__init__.py
-	black funnel/transports/*/__init__.py
+	mkinit --inplace --relative --black funnel/transports/sms
+	ruff check --fix funnel/transports/*/__init__.py
+	ruff format funnel/transports/*/__init__.py
 
 initpy-utils:
 	mkinit --inplace --relative --black --recursive funnel/utils
-	isort funnel/utils/__init__.py funnel/utils/*/__init__.py funnel/utils/*/*/__init__.py
-	black funnel/utils/__init__.py funnel/utils/*/__init__.py funnel/utils/*/*/__init__.py
+	ruff check --fix funnel/utils/__init__.py funnel/utils/*/__init__.py funnel/utils/*/*/__init__.py
+	ruff format funnel/utils/__init__.py funnel/utils/*/__init__.py funnel/utils/*/*/__init__.py
 
 install-npm:
 	npm install

devserver.py

@@ -3,8 +3,10 @@
 
 import os
 import sys
-from typing import Any
+import warnings
+from typing import Any, Literal, cast
 
+import rich.traceback
 from flask.cli import load_dotenv
 from werkzeug import run_simple
 
@@ -19,13 +21,34 @@ def rq_background_worker(*args: Any, **kwargs: Any) -> Any:
 
 
 if __name__ == '__main__':
+    rich.traceback.install(show_locals=True, width=None)
     load_dotenv()
-    sys.path.insert(0, os.path.dirname(__file__))
+    script_dir = os.path.dirname(__file__)
+    if script_dir != '.' and not script_dir.endswith('/.'):
+        # If this script is not running from the current working directory, add it's
+        # path to the Python path so imports work
+        sys.path.insert(0, script_dir)
     os.environ['FLASK_ENV'] = 'development'  # Needed for coaster.app.init_app
     os.environ.setdefault('FLASK_DEBUG', '1')
     debug_mode = os.environ['FLASK_DEBUG'].lower() not in {'0', 'false', 'no'}
+    ssl_context: str | Literal['adhoc'] | tuple[str, str] | None  # noqa: PYI051
+    ssl_context = os.environ.get('FLASK_DEVSERVER_HTTPS')
+    if ssl_context is not None:
+        if not ssl_context:
+            ssl_context = None  # Recast empty string as None
+        elif ssl_context == 'adhoc':
+            # For type checkers to narrow to a literal value
+            ssl_context = cast(Literal['adhoc'], ssl_context)
+        elif ':' in ssl_context:
+            ssl_context = cast(tuple[str, str], tuple(ssl_context.split(':', 1)))
+        else:
+            warnings.warn(
+                f"FLASK_DEVSERVER_HTTPS env var has invalid value {ssl_context!r}",
+                stacklevel=1,
+            )
+            ssl_context = None
 
-    from funnel.devtest import BackgroundWorker, devtest_app
+    from funnel.devtest import BackgroundWorker, RichDebuggedApplication, devtest_app
 
     # Set debug mode on apps
     devtest_app.debug = debug_mode
@@ -35,18 +58,25 @@ def rq_background_worker(*args: Any, **kwargs: Any) -> Any:
         # Only start RQ worker within the reloader environment
         background_rq = BackgroundWorker(
             rq_background_worker,
-            mock_transports=bool(getbool(os.environ.get('MOCK_TRANSPORTS', False))),
+            mock_transports=bool(getbool(os.environ.get('MOCK_TRANSPORTS', True))),
         )
         background_rq.start()
 
+    if debug_mode:
+        run_app: Any = RichDebuggedApplication(
+            devtest_app, evalex=True, console_path='/_console'
+        )
+    else:
+        run_app = devtest_app
+
     run_simple(
         os.environ.get('FLASK_RUN_HOST', '127.0.0.1'),
         int(os.environ.get('FLASK_RUN_PORT', 3000)),
-        devtest_app,
+        run_app,
         use_reloader=True,
-        use_debugger=debug_mode,
-        use_evalex=debug_mode,
+        use_debugger=False,  # Since we've already wrapped the app in the debugger
         threaded=True,
+        ssl_context=ssl_context,
         extra_files=['funnel/static/build/manifest.json'],
     )
 

funnel/__init__.py

@@ -26,28 +26,28 @@
 
 #: Main app for hasgeek.com
 app = Flask(__name__, instance_relative_config=True)
-app.name = 'funnel'  # pyright: ignore[reportGeneralTypeIssues]
+app.name = 'funnel'  # pyright: ignore[reportAttributeAccessIssue]
 app.config['SITE_TITLE'] = __("Hasgeek")
 #: Short link app at has.gy
 shortlinkapp = Flask(__name__, static_folder=None, instance_relative_config=True)
-shortlinkapp.name = 'shortlink'  # pyright: ignore[reportGeneralTypeIssues]
+shortlinkapp.name = 'shortlink'  # pyright: ignore[reportAttributeAccessIssue]
 #: Unsubscribe app at bye.li
 unsubscribeapp = Flask(__name__, static_folder=None, instance_relative_config=True)
-unsubscribeapp.name = 'unsubscribe'  # pyright: ignore[reportGeneralTypeIssues]
+unsubscribeapp.name = 'unsubscribe'  # pyright: ignore[reportAttributeAccessIssue]
 
 all_apps = [app, shortlinkapp, unsubscribeapp]
 
 mail = Mail()
 pages = FlatPages()
-manifest = WebpackManifest(filepath='static/build/manifest.json')
+webpack = WebpackManifest(filepath='static/build/manifest.json', jinja_global='webpack')
 
 redis_store = FlaskRedis(decode_responses=True, config_prefix='CACHE_REDIS')
 rq = RQ()
 rq.job_class = 'rq.job.Job'
 rq.queues = ['funnel']  # Queues used in this app
 executor = Executor()
 
-# --- Assets ---------------------------------------------------------------------------
+# MARK: Assets -------------------------------------------------------------------------
 
 #: Theme files, for transitioning away from Baseframe templates. These are used by
 #: Baseframe's render_form and other form helper functions.
@@ -66,7 +66,7 @@
 assets['schedules.js'][version] = 'js/schedules.js'
 
 
-# --- Import rest of the app -----------------------------------------------------------
+# MARK: Import rest of the app ---------------------------------------------------------
 
 from . import (  # isort:skip  # noqa: F401  # pylint: disable=wrong-import-position
     geoip,
@@ -81,7 +81,7 @@
 )
 from .models import db, sa_orm  # isort:skip
 
-# --- Configuration---------------------------------------------------------------------
+# MARK: Configuration ------------------------------------------------------------------
 
 # Config is loaded from legacy Python settings files in the instance folder and then
 # overridden with values from the environment. Python config is pending deprecation
@@ -117,7 +117,7 @@
         phonenumbers.PhoneNumberFormat.INTERNATIONAL,
     )
     proxies.init_app(each_app)
-    manifest.init_app(each_app)
+    webpack.init_app(each_app)
     db.init_app(each_app)
     mail.init_app(each_app)
 
@@ -197,13 +197,13 @@
 
 views.siteadmin.init_rq_dashboard()
 
-# --- Serve static files with WhiteNoise -----------------------------------------------
+# MARK: Serve static files with WhiteNoise ---------------------------------------------
 
 _wn = WhiteNoise(app.wsgi_app, root=app.static_folder, prefix=app.static_url_path)
 _wn.add_files(baseframe.static_folder, prefix=baseframe.static_url_path)
 app.wsgi_app = _wn  # type: ignore[method-assign]
 
-# --- Init SQLAlchemy mappers ----------------------------------------------------------
+# MARK: Init SQLAlchemy mappers --------------------------------------------------------
 
 # Database model loading (from Funnel or extensions) is complete.
 # Configure database mappers now, before the process is forked for workers.

funnel/assets/js/app.js

@@ -13,6 +13,7 @@ import ReadStatus from './utils/read_status';
 import LazyLoadMenu from './utils/lazyloadmenu';
 import './utils/getDevicePixelRatio';
 import setTimezoneCookie from './utils/timezone';
+import './utils/follow_action';
 import 'muicss/dist/js/mui';
 
 const pace = require('pace-js');