initial commit

.dockerignore

@@ -0,0 +1,26 @@
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE

.gitignore

@@ -0,0 +1,133 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+node_modules
+
+
+tags
+cscope.out
+cscope.files
+
+# idea
+.idea/
+
+# dependencies
+node_modules/
+
+# testing
+/coverage
+
+# production
+build/
+
+# misc
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.DS_Store
+
+# pytest
+.pytest_cache/
+
+# ruff
+.ruff_cache
+
+.vscode/

.pre-commit-config.yaml

@@ -0,0 +1,15 @@
+repos:
+- repo: local 
+  hooks:
+    - id: black
+      name: Black
+      entry: /bin/sh -c 'cd ./proxy/ && black .'
+      language: system
+      types: [python]
+    - id: ruff
+      stages: [commit]
+      name: Ruff
+      types: [python]
+      language: system
+      entry: /bin/sh -c 'cd ./proxy/ && ruff check . --config ../pyproject.toml --show-source --fix'
+

Dockerfile

@@ -0,0 +1,40 @@
+FROM pypy:3.10 as prod
+
+ARG INSTALL_ADDONS=true
+
+ENV APP_HOME=/app
+ENV POETRY_OPTS=
+
+RUN pip install -U poetry
+
+RUN mkdir $APP_HOME
+
+COPY ./proxy $APP_HOME/proxy
+COPY ./README.md $APP_HOME/README.md
+COPY ./plugin_hooks/* $APP_HOME/proxy/
+COPY poetry.lock pyproject.toml $APP_HOME
+
+WORKDIR $APP_HOME
+
+# install optionally required libraries
+RUN [[ ${INSTALL_ADDONS} = "true" ]] || exit 0 && \
+    apt-get update && \
+    apt-get install -y libmagic1
+
+RUN poetry config virtualenvs.create false && if [ "$INSTALL_ADDONS" = "true" ]; then poetry install --with addons --without dev; else poetry install --only main; fi
+
+
+EXPOSE 8000
+
+CMD ["poetry", "run", "python", "-m", "proxy.app"]
+
+FROM prod as dev_image
+
+# rust is required for aiohttp-devtools
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+RUN poetry install
+
+CMD ["adev", "runserver", "proxy"]
+

README.md

@@ -0,0 +1,65 @@
+# s3-hooked
+
+s3 proxy written in Python/aiohttp that supports flexible integrations via hooks to
+achieve things like symmetric encryption (built-in), validations, virus scanning,
+triggering webhooks etc.
+
+## Pluggable Hooks
+
+The proxy forwards all traffic to the configured S3-store.
+
+When uploading and downloading you can inject hooks in three categories:
+ `pre_upload_before_check` that will be called before upload simultaneously
+ `pre_upload_unsafe` that will be called before upload but after
+    `pre_upload_before_check` has run successfully. You can run calls on the content
+    here that will actually interpret the data, assuming the previous call has checked
+    for malignous data.
+ `post_upload` will run after the upload has run successfully. It will not change the
+    existing state. Consider it side-effects.
+
+Hooks are named with the function name by default. This can be overriden when register-
+ing.
+
+The default hooks for encryption/decryption are `hook_encrypt_data` and 
+`hook_decrypt_data` respectively. If you override the hooks in your implementation 
+you should import them from the `proxy.default_hooks` module. You may of course
+override those, too.
+
+Every hook should take the current request plus optionally data or other params and
+return a tuple of the name, boolen of success and optionally resulting data or 
+the error message on failure.
+
+Per event all registered hooks must pass in order to proceed. If a hook fails the
+proxy will return an error message with name and reason.
+To register a procedure as a hook wrap it in a hook-function that takes
+the request and binary data.
+
+Unless overridden hooks will be read from `proxy.default_hooks`. Hooks are overridden
+by placing decorated functions in the `plugin_hooks/hooks.py` file. If you want to
+keep the default encryption mechanism import them event from `proxy.default_hooks`.
+If you wish to override them import the events from `proxy.events` directly.
+
+The `pos` parameter for registering hooks is only relevant if the registered hooks
+should be called in a predefined order, i. e. on an event that waits for each hook
+to be finished (i. e. `blocking=True`)
+
+
+## Incldued addon dependencies
+
+The `INSTALL_ADDONS` build arg (true/false) controls whether to install the following extra
+dependencies (default: true)
+
+ - `python-clamd`
+ - `python-magic`
+
+
+
+## start the development server
+
+```
+docker compose up -d --build 
+```
+
+This will pull up a local instance on :8080
+
+It will also create a minio service on :9000

docker-compose.override.yml

@@ -0,0 +1,31 @@
+version: '3.4'
+
+services:
+   proxy:
+     image: s3-hooked/proxy:dev
+     build:
+       target: dev_image
+     environment:
+       - PROXY_SECRET=dqxNYNhp9nfnYQjpUE2dzGnXBHem3shYd0lIaNKuyomem6SB
+       - PROXY_ENVIRONMENT=${ENVIRONMENT:-development}
+       - MINIO_ACCESS_KEY=minio-dev-key
+       - MINIO_SECRET_KEY=5hA9J2SW5syVch0u5oJtVwUJokR7loHHYh1rMV8V
+       - PROXY_OBJECT_STORE_HOST=minio
+       - PROXY_OBJECT_STORE_PORT=9000
+       - PROXY_OBJECT_STORE_SSL_ENABLED=false
+       - PYTHONBREAKPOINT=ipdb.set_trace
+     volumes:
+       - .:/app/
+     ports:
+      - "8080:8000"
+     command: adev runserver proxy
+
+   minio:
+     environment:
+       - MINIO_ROOT_USER=admin
+       - MINIO_ROOT_PASSWORD=minio-admin
+       - MINIO_BROWSER=on
+     ports:
+       - "9000:9000"
+       - "9090:9090"
+     command: server data --console-address :9090

docker-compose.yml

@@ -0,0 +1,27 @@
+version: '3.4'
+
+services:
+  proxy:
+    build:
+      dockerfile: Dockerfile
+      context: .
+      target: prod
+      args:
+        - INSTALL_ADDONS=true
+    image: s3-hooked/proxy:prod
+    environment:
+      - PROXY_SECRET
+      - PROXY_OBJECT_STORE_HOST=${PROXYO_OBJECT_STORE_HOST:-minio}
+      - PROXY_OBJECT_STORE_PORT=${PROXYO_OBJECT_STORE_PORT:-9000}
+      - PROXY_OBJECT_STORE_SSL_ENABLED=${PROXY_OJECT_STORE_SSL_ENABLED:-true}
+    ports:
+      - "8080:8000"
+  minio:
+    image: minio/minio:RELEASE.2023-09-16T01-01-47Z
+    volumes:
+      - minio_data:/data
+    command: server data
+
+volumes:
+  minio_data:
+