From 3f8a41eef9537c25bf5f9402c52948d4aec767a0 Mon Sep 17 00:00:00 2001
From: James Kunstle <jkunstle@redhat.com>
Date: Tue, 23 Jan 2024 12:24:24 -0500
Subject: [PATCH] make workers more resillient

prefetch and concurrency were pinning workers to
jobs that weren't going to terminate. this should
alleviate some worker pool exhaustion by cancelling
deadlocked tasks and allowing workers to immediately pick up
tasks from queue if they're available.

Signed-off-by: James Kunstle <jkunstle@redhat.com>
---
 8Knot/_celery.py                           | 3 ++-
 docker-compose.yml                         | 3 ++-
 openshift/base/8k-autoscale.yaml           | 4 ++--
 openshift/overlays/prod/kustomization.yaml | 4 ++--
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/8Knot/_celery.py b/8Knot/_celery.py
index e4463539..c2d86135 100644
--- a/8Knot/_celery.py
+++ b/8Knot/_celery.py
@@ -16,10 +16,11 @@
 )
 
 celery_app.conf.update(
-    task_time_limit=84600,
+    task_time_limit=2700, # 45 minutes
     task_acks_late=True,
     task_track_started=True,
     result_extended=True,
+    worker_prefetch_multiplier=1,
 )
 
 celery_manager = CeleryManager(celery_app=celery_app)
diff --git a/docker-compose.yml b/docker-compose.yml
index 3cbd7956..6e9c3280 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -55,7 +55,7 @@ services:
     build:
       context: .
       dockerfile: ./docker/Dockerfile
-    command: ["celery", "-A", "app:celery_app", "worker", "--loglevel=INFO"]
+    command: ["celery", "-A", "app:celery_app", "worker", "--loglevel=INFO", "--concurrency=1"]
     depends_on:
       - redis-cache
       - redis-users
@@ -77,6 +77,7 @@ services:
         "--loglevel=INFO",
         "-Q",
         "data",
+        "--concurrency=1"
       ]
     depends_on:
       - redis-cache
diff --git a/openshift/base/8k-autoscale.yaml b/openshift/base/8k-autoscale.yaml
index 5ef9e8be..07a285f8 100644
--- a/openshift/base/8k-autoscale.yaml
+++ b/openshift/base/8k-autoscale.yaml
@@ -34,7 +34,7 @@ spec:
     kind: Deployment
     name: eightknot-worker-callback
   minReplicas: 1
-  maxReplicas: 20
+  maxReplicas: 30 # each is its own process
   metrics:
     - type: Resource
       resource:
@@ -59,7 +59,7 @@ spec:
     kind: Deployment
     name: eightknot-worker-query
   minReplicas: 1
-  maxReplicas: 8
+  maxReplicas: 20 
   metrics:
     - type: Resource
       resource:
diff --git a/openshift/overlays/prod/kustomization.yaml b/openshift/overlays/prod/kustomization.yaml
index 50af66d6..c07444b6 100644
--- a/openshift/overlays/prod/kustomization.yaml
+++ b/openshift/overlays/prod/kustomization.yaml
@@ -26,14 +26,14 @@ patches:
     patch: |-
       - op: add
         path: /spec/replicas
-        value: 3
+        value: 15 
   - target:
       kind: Deployment
       name: eightknot-worker-query
     patch: |-
       - op: add
         path: /spec/replicas
-        value: 2
+        value: 10 
   - target:
       kind: HorizontalPodAutoscaler
     patch: |-