make workers more resillient

prefetch and concurrency were pinning workers to
jobs that weren't going to terminate. this should
alleviate some worker pool exhaustion by cancelling
deadlocked tasks and allowing workers to immediately pick up
tasks from queue if they're available.

Signed-off-by: James Kunstle <[email protected]>

8Knot/_celery.py

@@ -16,10 +16,11 @@
 )
 
 celery_app.conf.update(
-    task_time_limit=84600,
+    task_time_limit=2700, # 45 minutes
     task_acks_late=True,
     task_track_started=True,
     result_extended=True,
+    worker_prefetch_multiplier=1,
 )
 
 celery_manager = CeleryManager(celery_app=celery_app)

docker-compose.yml

@@ -55,7 +55,7 @@ services:
     build:
       context: .
       dockerfile: ./docker/Dockerfile
-    command: ["celery", "-A", "app:celery_app", "worker", "--loglevel=INFO"]
+    command: ["celery", "-A", "app:celery_app", "worker", "--loglevel=INFO", "--concurrency=1"]
     depends_on:
       - redis-cache
       - redis-users
@@ -77,6 +77,7 @@ services:
         "--loglevel=INFO",
         "-Q",
         "data",
+        "--concurrency=1"
       ]
     depends_on:
       - redis-cache

openshift/base/8k-autoscale.yaml

@@ -34,7 +34,7 @@ spec:
     kind: Deployment
     name: eightknot-worker-callback
   minReplicas: 1
-  maxReplicas: 20
+  maxReplicas: 30 # each is its own process
   metrics:
     - type: Resource
       resource:
@@ -59,7 +59,7 @@ spec:
     kind: Deployment
     name: eightknot-worker-query
   minReplicas: 1
-  maxReplicas: 8
+  maxReplicas: 20 
   metrics:
     - type: Resource
       resource:

openshift/overlays/prod/kustomization.yaml

@@ -26,14 +26,14 @@ patches:
     patch: |-
       - op: add
         path: /spec/replicas
-        value: 3
+        value: 15 
   - target:
       kind: Deployment
       name: eightknot-worker-query
     patch: |-
       - op: add
         path: /spec/replicas
-        value: 2
+        value: 10 
   - target:
       kind: HorizontalPodAutoscaler
     patch: |-