diff --git a/.gitignore b/.gitignore
index 126b2e4..d51687f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,10 @@
-.venv
-.envrc
-.terraform*
-data
-__pycache__
-result
-.direnv
+.venv
+.envrc
+.terraform*
+data
+__pycache__
+result
+.direnv
+
+# Noise from dynamodb-local program in the working directory
+dynamodb-local-metadata.json
diff --git a/README.md b/README.md
index c39e929..1197c00 100644
--- a/README.md
+++ b/README.md
@@ -56,14 +56,9 @@ Drawback: the call graph must be idempotent, meaning: for the same inputs, a tas
## Demo
-### Requirements
+Requires [Nix](https://nixos.org), with flakes enabled.
-- Docker
-- [Nix](https://nixos.org)
-
-### Instructions
-
-Make sure you have Docker running, and start the full demo:
+You can start the full demo without installation:
```
$ nix run github:nobssoftware/brrr#demo
diff --git a/brrr-demo.nix b/brrr-demo.service.nix
similarity index 90%
rename from brrr-demo.nix
rename to brrr-demo.service.nix
index 898fdab..792894a 100644
--- a/brrr-demo.nix
+++ b/brrr-demo.service.nix
@@ -12,7 +12,9 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
-# Brrr-demo module for process-compose-flake
+# Brrr-demo module for services-flake. Awkwardly named file because
+# services-flake insists on auto-deriving the module name from the filename.
+# Ok.
{ config, pkgs, name, lib, ... }: {
options = with lib.types; {
diff --git a/brrr_demo.py b/brrr_demo.py
index 7441b04..8c025f9 100755
--- a/brrr_demo.py
+++ b/brrr_demo.py
@@ -11,7 +11,7 @@
from brrr.backends import redis as redis_, dynamo
import brrr
-from brrr import task, wrrrk, setup, brrr
+from brrr import task
@bottle.route("/")
def get_or_schedule_task(task_name: str):
@@ -33,9 +33,6 @@ def get_or_schedule_task(task_name: str):
return {"status": "accepted"}
def init_brrr(reset_backends):
- # Check credentials
- boto3.client('sts').get_caller_identity()
-
redis_client = redis.Redis(decode_responses=True)
queue = redis_.RedisStream(redis_client, os.environ.get("REDIS_QUEUE_KEY", "r1"))
if reset_backends:
@@ -46,14 +43,15 @@ def init_brrr(reset_backends):
if reset_backends:
store.create_table()
- setup(queue, store)
+ brrr.setup(queue, store)
@task
def fib(n: int, salt=None):
match n:
- case 0: return 0
- case 1: return 1
- case _: return sum(fib.map([[n - 2, salt], [n - 1, salt]]))
+ case 0 | 1:
+ return n
+ case _:
+ return sum(fib.map([[n - 2, salt], [n - 1, salt]]))
@task
def fib_and_print(n: str, salt = None):
@@ -75,7 +73,7 @@ def cmd(f):
@cmd
def worker():
init_brrr(False)
- wrrrk(1)
+ brrr.wrrrk(1)
@cmd
def server():
diff --git a/dynamodb.service.nix b/dynamodb.service.nix
new file mode 100644
index 0000000..718824e
--- /dev/null
+++ b/dynamodb.service.nix
@@ -0,0 +1,41 @@
+# Copyright © 2024 Brrr Authors
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+# Dynamodb module for process-compose-flake
+
+{ config, pkgs, lib, ... }: {
+ options.services.dynamodb = with lib.types; {
+ enable = lib.mkEnableOption "Enable Dynamodb local service";
+ args = lib.mkOption {
+ default = [];
+ type = listOf str;
+ };
+ dataDir = lib.mkOption {
+ default = "data/dynamodb";
+ type = str;
+ };
+ };
+ config = let
+ cfg = config.services.dynamodb;
+ in
+ lib.mkIf cfg.enable {
+ settings.processes.dynamodb.command = let
+ bin = lib.getExe pkgs.dynamodb-local;
+ dir = lib.escapeShellArg cfg.dataDir;
+ in ''
+ mkdir -p ${dir}
+ ${bin} -dbPath ${dir} ${lib.escapeShellArgs cfg.args}
+ '';
+ };
+}
diff --git a/flake.nix b/flake.nix
index 85adcb7..b78086a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -46,33 +46,41 @@
inputs.devshell.flakeModule
];
# A reusable process-compose module (for flake-parts) with either a full
+
# demo environment, or just the dependencies if you want to run a server
# manually.
flake = {
- processComposeModules.default = { pkgs, ... }: {
- imports = [
- ./localstack.nix
- (inputs.services-flake.lib.multiService ./brrr-demo.nix)
- ];
- services = let
- demoEnv = {
- AWS_ENDPOINT_URL = "http://localhost:4566";
- AWS_ACCESS_KEY_ID = "000000000000";
- AWS_SECRET_ACCESS_KEY = "localstack-foo";
- AWS_DEFAULT_REGION = "us-east-1";
- };
- in {
- redis.r1.enable = true;
- localstack.enable = true;
- brrr-demo.worker = {
- package = self.packages.${pkgs.system}.brrr-demo;
- args = [ "worker" ];
- environment = demoEnv;
- };
- brrr-demo.server = {
- package = self.packages.${pkgs.system}.brrr-demo;
- args = [ "server" ];
- environment = demoEnv;
+ processComposeModules = {
+ brrr-demo = inputs.services-flake.lib.multiService ./brrr-demo.service.nix;
+ dynamodb = import ./dynamodb.service.nix;
+ localstack = import ./localstack.service.nix;
+ default = { pkgs, ... }: {
+ imports = with self.processComposeModules; [
+ brrr-demo
+ dynamodb
+ # Unused for now but will probably be reintroduced for an SQS demo
+ # soon.
+ localstack
+ ];
+ services = let
+ demoEnv = {
+ AWS_ENDPOINT_URL = "http://localhost:8000";
+ AWS_ACCESS_KEY_ID = "000000000000";
+ AWS_SECRET_ACCESS_KEY = "fake";
+ };
+ in {
+ redis.r1.enable = true;
+ dynamodb.enable = true;
+ brrr-demo.worker = {
+ package = self.packages.${pkgs.system}.brrr-demo;
+ args = [ "worker" ];
+ environment = demoEnv;
+ };
+ brrr-demo.server = {
+ package = self.packages.${pkgs.system}.brrr-demo;
+ args = [ "server" ];
+ environment = demoEnv;
+ };
};
};
};
@@ -95,6 +103,11 @@
);
in {
config = {
+ _module.args.pkgs = import inputs.nixpkgs {
+ inherit system;
+ # dynamodb
+ config.allowUnfree = true;
+ };
process-compose.demo = {
imports = [
inputs.services-flake.processComposeModules.default
@@ -149,16 +162,30 @@
config.Entrypoint = [ "${lib.getExe pkg}" ];
};
};
- checks.pytest = pkgs.stdenvNoCC.mkDerivation {
- name = "pytest";
- nativeBuildInputs = [ self'.packages.dev ];
- src = lib.cleanSource ./.;
- buildPhase = ''
- pytest
- '';
- installPhase = ''
- touch $out
- '';
+ checks = {
+ pytest = pkgs.stdenvNoCC.mkDerivation {
+ name = "pytest";
+ nativeBuildInputs = [ self'.packages.dev ];
+ src = lib.cleanSource ./.;
+ buildPhase = ''
+ pytest
+ '';
+ installPhase = ''
+ touch $out
+ '';
+ };
+ ruff = pkgs.stdenvNoCC.mkDerivation {
+ name = "ruff";
+ nativeBuildInputs = [ self'.packages.dev ];
+ src = lib.cleanSource ./.;
+ # Don’t check tests for now though we should
+ buildPhase = ''
+ ruff check src
+ '';
+ installPhase = ''
+ touch $out
+ '';
+ };
};
devshells = {
impure = {
diff --git a/localstack.nix b/localstack.service.nix
similarity index 100%
rename from localstack.nix
rename to localstack.service.nix
diff --git a/pyproject.toml b/pyproject.toml
index 6196631..f2c806c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "brrr"
-version = "0.0.3"
+version = "0.0.4"
description = "Horizontally scalable workflow scheduling with pluggable backends"
authors = [
{name = "Hraban Luyat", email = "hraban@0brg.net"},
diff --git a/src/brrr/__init__.py b/src/brrr/__init__.py
index b0f1e2f..9ca3265 100644
--- a/src/brrr/__init__.py
+++ b/src/brrr/__init__.py
@@ -1,10 +1,12 @@
from .brrr import Brrr
-# For ergonomics, we provide a singleton and a bunch of proxies as the module interface
-brrr = Brrr()
+# For ergonomics, we provide a singleton and a bunch of proxies as the module interface.
+_brrr = Brrr()
-setup = brrr.setup
-gather = brrr.gather
-wrrrk = brrr.wrrrk
-task = brrr.register_task
-schedule = brrr.schedule
+setup = _brrr.setup
+gather = _brrr.gather
+read = _brrr.read
+wrrrk = _brrr.wrrrk
+task = _brrr.register_task
+tasks = _brrr.tasks
+schedule = _brrr.schedule
diff --git a/src/brrr/asyncrrr.py b/src/brrr/asyncrrr.py
deleted file mode 100644
index 082a6ea..0000000
--- a/src/brrr/asyncrrr.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import asyncio
-
-async def async_wrrrk(workers: int = 1):
- """
- Spin up a number of worker threads
- """
- return asyncio.gather(
- *[asyncio.create_task(asyncio_worker()) for _ in range(workers)]
- )
diff --git a/src/brrr/backends/dynamo.py b/src/brrr/backends/dynamo.py
index 3a5132e..e5901ed 100644
--- a/src/brrr/backends/dynamo.py
+++ b/src/brrr/backends/dynamo.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import os
import typing
from ..store import CompareMismatch, MemKey, Store
@@ -103,7 +102,7 @@ def compare_and_delete(self, key: MemKey, expected: bytes):
ExpressionAttributeNames={"#value": "value"},
ExpressionAttributeValues={":expected": {"B": expected}},
)
- except self.client.exceptions.ConditionalCheckFailedException as e:
+ except self.client.exceptions.ConditionalCheckFailedException:
raise CompareMismatch
def create_table(self):
diff --git a/src/tests/__init__.py b/tests/__init__.py
similarity index 100%
rename from src/tests/__init__.py
rename to tests/__init__.py
diff --git a/src/tests/test_queue.py b/tests/test_queue.py
similarity index 100%
rename from src/tests/test_queue.py
rename to tests/test_queue.py
diff --git a/src/tests/test_store.py b/tests/test_store.py
similarity index 100%
rename from src/tests/test_store.py
rename to tests/test_store.py