Dockerize setup

.env.sample

@@ -0,0 +1,4 @@
+BN_URL=
+BP_URL=
+DATA_DIR=
+PROXY_PORT=

Caddyfile.sample

@@ -0,0 +1,89 @@
+# Access to everything.
+(authorised-super-users) {
+}
+
+# Access to blockprint's private API and nothing else.
+(authorised-blockprint-users) {
+}
+
+# Access to the gauge /classify method, but not blockprint's private API.
+(authorised-blockprint-workers) {
+}
+
+http:// {
+	# tls /certs/cert.pem /certs/key.pem
+	log {
+		output stderr
+		format filter {
+			wrap console
+			fields {
+				request>headers>Authorization delete
+			}
+		}
+	}
+
+	encode gzip zstd
+
+	@public {
+		path /blocks_per_client/*
+		path /sync/status
+		path /sync/gaps
+	}
+
+	@private {
+		path /validator/*
+		path /blocks/*
+		path /confusion/*
+	}
+
+	@gauge-classify {
+		path /gauge/classify
+	}
+
+	@gauge-accuracy {
+		path /confusion
+		path /accuracy
+		path /gauge/accuracy
+		path /gauge/confusion
+	}
+
+	@eleel {
+		path /eleel
+		path /eleel/
+	}
+	@eleel-canonical {
+		path /eleel/canonical
+	}
+
+	reverse_proxy @public bp:8000
+	reverse_proxy @private bp:8000
+
+	reverse_proxy @gauge-accuracy blockgauge:8002 {
+		rewrite /accuracy
+	}
+
+	reverse_proxy @gauge-classify blockgauge:8002 {
+		rewrite /classify
+	}
+
+	reverse_proxy @eleel localhost:8552 {
+		rewrite /
+	}
+	respond @eleel-canonical 400 {
+		body "{\"error\": \"nice try\"}"
+		close
+	}
+
+	handle_errors {
+		respond "{\"error\": \"{http.error.status_code} {http.error.status_text}\"}"
+	}
+
+	basicauth @private {
+		import authorised-super-users
+		import authorised-blockprint-users
+	}
+	basicauth @gauge-classify {
+		import authorised-super-users
+		import authorised-blockprint-workers
+	}
+}

Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.9
+# Or any preferred Python version.
+WORKDIR /app
+COPY ./*.py .
+ADD requirements.txt .
+RUN pip install -r requirements.txt
+# Or enter the name of your unique directory and parameter set.
+CMD ["gunicorn", "--timeout", "1800", "--bind", "0.0.0.0:8000", "api_server:app"]

api_server.py

@@ -1,7 +1,6 @@
 import os
 import json
 import falcon
-
 from multi_classifier import MultiClassifier
 from build_db import (
     open_block_db,
@@ -17,15 +16,22 @@
     count_false_positives,
     count_false_negatives,
 )
-from classifier import * # We need this one for the persisted model
+import __main__
+from classifier import (
+    Classifier,
+    import_classifier,
+)
+
+__main__.Classifier = Classifier
 
 
 DATA_DIR = "./data/mainnet/training"
 BLOCK_DB = os.environ.get("BLOCK_DB") or "./block_db.sqlite"
 BN_URL = "http://localhost:5052"
 SELF_URL = "http://localhost:8000"
 DISABLE_CLASSIFIER = "DISABLE_CLASSIFIER" in os.environ
-MODEL_PATH=os.environ.get("MODEL_PATH") or ""
+MODEL_PATH = os.environ.get("MODEL_PATH") or ""
+
 
 class Classify:
     def __init__(self, classifier, block_db):
@@ -205,22 +211,22 @@ def on_get(self, req, resp, client, start_slot, end_slot=None):
 classifier = None
 if not DISABLE_CLASSIFIER:
     if MODEL_PATH != "":
-        if MODEL_PATH.endswith('.pkl'):
-            classifier = import_classifier(MODEL_PATH)
-
+        if MODEL_PATH.endswith(".pkl"):
+            try:
+                classifier = import_classifier(MODEL_PATH)
+            except Exception as e:
+                print(f"Failed to persist classifier due to {e}")
+                exit(1)
+
         else:
             print("model path must end with .pkl")
-            exit(0)
-            
+            exit(1)
+
     else:
         print("Initialising classifier, this could take a moment...")
         classifier = MultiClassifier(DATA_DIR) if not DISABLE_CLASSIFIER else None
         print("Done")
 
-if classifier == None:
-    print("The classifier was not loaded")
-    exit(0)
-
 block_db = open_block_db(BLOCK_DB)
 
 app.add_route("/classify/no_store", ClassifyNoStore(classifier))

classifier.py

@@ -301,6 +301,17 @@ def import_classifier(model_path: str) -> Classifier:
         print(f"Failed to import classifier due to {e}")
 
 
+def import_classifier(model_path: str) -> Classifier:
+    """Load a pickled classifier.
+
+    This function may throw an exception if the data is corrupt or the file does not exist.
+    """
+    print(f"""Loading classifier from {model_path}""")
+    classifier = pickle.load(open(model_path, "rb"))
+    print("Loaded classifier into memory")
+    return classifier
+
+
 def main():
     args = parse_args()
     data_dir = args.data_dir

docker-compose.yml

@@ -0,0 +1,84 @@
+networks:
+  bp_cluster:
+
+services:
+  bp:
+    profiles:
+      - ""
+      - "server"
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      - BLOCK_DB=/app/block_db.sqlite
+      - BN_URL=${BN_URL:-localhost:5052}
+      - GUNICORN_CMD_ARGS="--bind=0.0.0.0"
+      - PYTHONUNBUFFERED=1
+#      - MODEL_PATH=/app/classifier.pkl
+    volumes:
+      - ${DATA_DIR:-./training_data}:/app/data/mainnet/training
+      - ${BLOCK_DB:-./block_db.sqlite}:/app/block_db.sqlite
+#      - ./example.pkl:/app/classifier.pkl
+    networks:
+      - bp_cluster
+    entrypoint: ["gunicorn", "--timeout", "1800", "api_server:app"]                
+  bp-bg:
+    profiles:
+      - ""
+      - "server"
+    build:
+      context: .
+      dockerfile: Dockerfile
+    networks:
+      - bp_cluster
+    environment:
+      - PYTHONUNBUFFERED=1
+      - BN_URL=${BN_URL:-http://localhost:5052}
+      - BP_URL=http://bp:8000
+    entrypoint: ["./background_tasks.py"]
+
+  blockgauge:
+    profiles:
+      - ""
+      - "server"
+    image: ghcr.io/blockprint-collective/blockgauge
+    command: >-
+      blockgauge
+      --lighthouse-url ${BN_URL:-localhost:5052} 
+      --blockprint-url http://bp:8000
+      --listen-address 0.0.0.0  
+    networks:
+      - bp_cluster
+
+  blockdreamer:
+    profiles:
+      - "dreamer"
+    image: ghcr.io/blockprint-collective/blockdreamer
+    volumes:
+      - ./dreamer:/mnt/dreamer
+    working_dir: "/mnt/dreamer"
+    command: >-
+      blockdreamer 
+      --config /mnt/dreamer/config.toml
+
+  caddy:
+    profiles:
+      - ""
+      - "server"
+    image: caddy:2.7.5
+    ports:
+    - "${PROXY_PORT:-80}:80"
+    - "443:443"
+    volumes:
+    - ./Caddyfile:/etc/caddy/Caddyfile:ro
+    - ./data/caddy/data:/data
+    - ./data/caddy/config:/config
+    - ./certs:/certs
+    restart: unless-stopped
+    networks:
+      - bp_cluster
+
+
+
+
+

dreamer/example.toml

@@ -0,0 +1,16 @@
+network = "mainnet"
+canonical_bn = "http://localhost:5052"
+
+[[post_endpoints]]
+name = "blockgauge"
+url = "http://localhost:5052/lighthouse/analysis/block_rewards"
+extra_data = false
+compare_rewards = true
+
+[[nodes]]
+name = "lighthouse-subscribe-none"
+label = "Lighthouse"
+url = "http://localhost:5052"
+v3 = true
+ssz = false
+skip_randao_verification = true