From b7010f17f9e20bb83abeb685913627f02750cae5 Mon Sep 17 00:00:00 2001
From: joschrew <jonas.schrewe@gwdg.de>
Date: Fri, 7 Jul 2023 16:01:32 +0200
Subject: [PATCH 01/19] Draft for slim containers

---
 Makefile-slim                                 | 41 +++++++++++++++++
 slim-containers-files/Dummy-Core-Dockerfile   | 13 ++++++
 slim-containers-files/delegator_template.py   | 24 ++++++++++
 .../docker-compose.processor.template.yaml    | 14 ++++++
 .../docker-compose.template.yaml              | 34 ++++++++++++++
 slim-containers-files/ocrd_cis/Dockerfile     | 15 +++++++
 .../ocrd_cis/docker-compose.yaml              | 14 ++++++
 .../ocrd_tesserocr/Dockerfile                 | 44 +++++++++++++++++++
 .../ocrd_tesserocr/docker-compose.yaml        | 14 ++++++
 slim-containers-files/ps-config.yaml          | 12 +++++
 10 files changed, 225 insertions(+)
 create mode 100644 Makefile-slim
 create mode 100644 slim-containers-files/Dummy-Core-Dockerfile
 create mode 100755 slim-containers-files/delegator_template.py
 create mode 100644 slim-containers-files/docker-compose.processor.template.yaml
 create mode 100644 slim-containers-files/docker-compose.template.yaml
 create mode 100644 slim-containers-files/ocrd_cis/Dockerfile
 create mode 100644 slim-containers-files/ocrd_cis/docker-compose.yaml
 create mode 100644 slim-containers-files/ocrd_tesserocr/Dockerfile
 create mode 100644 slim-containers-files/ocrd_tesserocr/docker-compose.yaml
 create mode 100644 slim-containers-files/ps-config.yaml

diff --git a/Makefile-slim b/Makefile-slim
new file mode 100644
index 00000000..469aa580
--- /dev/null
+++ b/Makefile-slim
@@ -0,0 +1,41 @@
+export PYTHON ?= python3
+VIRTUAL_ENV = $(CURDIR)/venv2
+BIN = $(VIRTUAL_ENV)/bin
+ACTIVATE_VENV = $(BIN)/activate
+OCRD_MODULES = OCRD_CIS OCRD_TESSEROCR
+OCRD_CIS = ocrd-cis-ocropy-binarize ocrd-cis-ocropy-dewarp
+OCRD_TESSEROCR = ocrd-tesserocr-recognize ocrd-tesserocr-segment-region
+PROCESSORS = $(foreach mod,$(OCRD_MODULES),$(foreach proc,$($(mod)), $(proc) ))
+DELEGATORS = $(foreach proc,$(PROCESSORS),$(BIN)/$(proc))
+
+slim-venv: docker-compose.yaml .env $(DELEGATORS) | $(VIRTUAL_ENV)
+
+
+# create a delegator to the processing server for the processor
+$(BIN)/ocrd-%: | $(VIRTUAL_ENV)
+	@sed "s/{{\s*processor_name\s*}}/$(subst $(BIN)/,,$@)/" slim-containers-files/delegator_template.py > $@;
+	@chmod u+x $@
+
+
+$(VIRTUAL_ENV): $(ACTIVATE_VENV)
+	. $(ACTIVATE_VENV) && $(MAKE) -C core install
+
+%/bin/activate:
+	$(PYTHON) -m venv $(subst /bin/activate,,$@)
+	. $@ && pip install --upgrade pip setuptools wheel
+
+# append the service to docker-compose for a processor
+add_proc = sed -e "s/{{\s*processor_name\s*}}/$1/" -e "s/{{\s*processor_group_name\s*}}/\L$2/" \
+            slim-containers-files/docker-compose.processor.template.yaml >> docker-compose.yaml;
+
+docker-compose.yaml:
+	@cat slim-containers-files/docker-compose.template.yaml > docker-compose.yaml
+	@$(foreach mod,$(OCRD_MODULES),$(foreach proc,$($(mod)),$(call add_proc,$(proc),$(mod))))
+
+.env:
+	@rm -rf .env
+	@echo OCRD_PS_PORT=8000 >> .env
+	@echo OCRD_PS_MTU=1300 >> .env
+	@echo MONGODB_URL=mongodb://ocrd-mongodb:27017 >> .env
+	@echo RABBITMQ_URL=amqp://admin:admin@ocrd-rabbitmq:5672 >> .env
+
diff --git a/slim-containers-files/Dummy-Core-Dockerfile b/slim-containers-files/Dummy-Core-Dockerfile
new file mode 100644
index 00000000..fc667591
--- /dev/null
+++ b/slim-containers-files/Dummy-Core-Dockerfile
@@ -0,0 +1,13 @@
+# I need this because i need the network-for-slim-branch and this contains a comment how to make
+# pudb run.
+FROM ocrd/core:latest AS base
+WORKDIR /build-ocrd
+RUN apt install vim-tiny -y
+RUN git clone https://github.com/ocr-d/core.git && \
+	cd core && \
+	git checkout network-for-slim-prep && \
+	#sed -i "290 i \        from pudb.remote import set_trace; set_trace(term_size=(160, 40), host='0.0.0.0', port=6900)" ocrd_network/ocrd_network/processing_server.py && \
+	make install-dev && \
+	pip install pudb
+EXPOSE 6900
+WORKDIR /data
diff --git a/slim-containers-files/delegator_template.py b/slim-containers-files/delegator_template.py
new file mode 100755
index 00000000..499047eb
--- /dev/null
+++ b/slim-containers-files/delegator_template.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+import sys
+from pathlib import Path
+import subprocess
+
+# Later the address (or rather the port) should be dynamic
+processing_server_address = "http://localhost:8000"
+processor_name = "{{ processor_name }}"
+
+args = list(sys.argv)
+if "-m" in args:
+    idx = args.index("-m")
+    metspath = args[idx + 1]
+    if Path(metspath).is_absolute():
+        print("absolute path is not supported")
+        exit(1)
+    args[idx + 1] = f"/data/{metspath}"
+
+
+cmd = [
+    "ocrd", "network", "client", "processing", "processor",
+    processor_name, "--address", processing_server_address
+]
+subprocess.run(cmd + args[1:])
diff --git a/slim-containers-files/docker-compose.processor.template.yaml b/slim-containers-files/docker-compose.processor.template.yaml
new file mode 100644
index 00000000..01122f2b
--- /dev/null
+++ b/slim-containers-files/docker-compose.processor.template.yaml
@@ -0,0 +1,14 @@
+
+  {{ processor_name }}:
+    extends:
+      file: slim-containers-files/{{ processor_group_name}}/docker-compose.yaml
+      service: {{ processor_name }}
+    command: ocrd network processing-worker --database $MONGODB_URL --queue $RABBITMQ_URL --create-queue {{ processor_name }}
+    depends_on:
+      - ocrd-processing-server
+      - ocrd-mongodb
+      - ocrd-rabbitmq
+    # restart: The worker creates its queue but rabbitmq needs a few seconds to be available
+    restart: on-failure:3
+    volumes:
+      - "$PWD/data:/data"
diff --git a/slim-containers-files/docker-compose.template.yaml b/slim-containers-files/docker-compose.template.yaml
new file mode 100644
index 00000000..d4b4f2f5
--- /dev/null
+++ b/slim-containers-files/docker-compose.template.yaml
@@ -0,0 +1,34 @@
+networks:
+  default:
+    driver: bridge
+    driver_opts:
+      com.docker.network.driver.mtu: ${OCRD_PS_MTU}
+
+services:
+  ocrd-processing-server:
+    build:
+      # later real ocrd-core image should be referenced here
+      dockerfile: slim-containers-files/Dummy-Core-Dockerfile
+      args:
+        BASE_IMAGE: ubuntu:20.04
+    ports:
+      - ${OCRD_PS_PORT}:8000
+    volumes:
+      - "./slim-containers-files/ps-config.yaml:/ocrd-processing-server-config.yaml"
+    command: ocrd network processing-server -a 0.0.0.0:8000 /ocrd-processing-server-config.yaml
+
+  ocrd-mongodb:
+    image: mongo
+    # Ports are only needed during the implementation phase to test. To be removed later
+    ports:
+      - "27018:27017"
+
+  ocrd-rabbitmq:
+    image: rabbitmq:3-management
+    # Ports are only needed during the implementation phase to test. To be removed later
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+    environment:
+      - "RABBITMQ_DEFAULT_USER=admin"
+      - "RABBITMQ_DEFAULT_PASS=admin"
diff --git a/slim-containers-files/ocrd_cis/Dockerfile b/slim-containers-files/ocrd_cis/Dockerfile
new file mode 100644
index 00000000..5839c70a
--- /dev/null
+++ b/slim-containers-files/ocrd_cis/Dockerfile
@@ -0,0 +1,15 @@
+FROM ocrd/core:latest AS base
+WORKDIR /build-ocrd
+# Remove the next RUN, this is only to checkout my branch while the changes are not in core yet
+RUN git clone https://github.com/ocr-d/core.git && \
+	cd core && \
+	git checkout network-for-slim-prep && \
+	make install
+
+# Not based on ocrd_cis "original" Dockerfile. That seems out of date and in ocrd_all ocrd_cis is
+# simply installed with pip so I do the same here
+COPY ocrd_cis/ ./ocrd_cis/
+COPY setup.py README.md LICENSE ocrd-tool.json Manifest.in ./
+RUN pip install . && rm -rf /build-ocrd
+# TODO: install models for ocrd-cis
+WORKDIR /data
diff --git a/slim-containers-files/ocrd_cis/docker-compose.yaml b/slim-containers-files/ocrd_cis/docker-compose.yaml
new file mode 100644
index 00000000..b61da055
--- /dev/null
+++ b/slim-containers-files/ocrd_cis/docker-compose.yaml
@@ -0,0 +1,14 @@
+services:
+  ocrd-cis-ocropy-binarize:
+    build:
+      context: ../../ocrd_cis
+      dockerfile: ../slim-containers-files/ocrd_cis/Dockerfile
+    command:
+      ocrd network processing-worker ocrd-cis-ocropy-binarize --database $MONGODB_URL --queue $RABBITMQ_URL --create-queue
+
+  ocrd-cis-ocropy-dewarp:
+    build:
+      context: ../../ocrd_cis
+      dockerfile: ../slim-containers-files/ocrd_cis/Dockerfile
+    command:
+      ocrd network processing-worker ocrd-cis-ocropy-dewarp --database $MONGODB_URL --queue $RABBITMQ_URL --create-queue
diff --git a/slim-containers-files/ocrd_tesserocr/Dockerfile b/slim-containers-files/ocrd_tesserocr/Dockerfile
new file mode 100644
index 00000000..2ffc7544
--- /dev/null
+++ b/slim-containers-files/ocrd_tesserocr/Dockerfile
@@ -0,0 +1,44 @@
+FROM ocrd/core:latest AS base
+WORKDIR /build-ocrd-core
+# Remove the next RUN, this is only to checkout my branch while the changes are not in core yet
+RUN git clone https://github.com/ocr-d/core.git && \
+	cd core && \
+	git checkout network-for-slim-prep && \
+	make install
+
+# copied from https://github.com/OCR-D/ocrd_tesserocr/blob/master/Dockerfile and modified
+ARG VCS_REF
+ARG BUILD_DATE
+LABEL \
+    maintainer="https://ocr-d.de/kontakt" \
+    org.label-schema.vcs-ref=$VCS_REF \
+    org.label-schema.vcs-url="https://github.com/OCR-D/ocrd_tesserocr" \
+    org.label-schema.build-date=$BUILD_DATE
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV PYTHONIOENCODING utf8
+
+# avoid HOME/.local/share (hard to predict USER here)
+# so let XDG_DATA_HOME coincide with fixed system location
+# (can still be overridden by derived stages)
+ENV XDG_DATA_HOME /usr/local/share
+
+WORKDIR /build-ocrd
+COPY setup.py .
+COPY ocrd_tesserocr/ocrd-tool.json .
+COPY README.md .
+COPY requirements.txt .
+COPY requirements_test.txt .
+COPY ocrd_tesserocr ./ocrd_tesserocr
+COPY Makefile .
+RUN make deps-ubuntu && \
+    apt-get install -y --no-install-recommends \
+    g++ \
+    && make deps install \
+    && rm -rf /build-ocrd \
+    && apt-get -y remove --auto-remove g++ libtesseract-dev make
+RUN ocrd resmgr download ocrd-tesserocr-recognize Fraktur.traineddata
+RUN ocrd resmgr download ocrd-tesserocr-recognize deu.traineddata
+
+WORKDIR /data
+VOLUME /data
diff --git a/slim-containers-files/ocrd_tesserocr/docker-compose.yaml b/slim-containers-files/ocrd_tesserocr/docker-compose.yaml
new file mode 100644
index 00000000..a29a51cd
--- /dev/null
+++ b/slim-containers-files/ocrd_tesserocr/docker-compose.yaml
@@ -0,0 +1,14 @@
+services:
+  ocrd-tesserocr-recognize:
+    build:
+      context: ../../ocrd_tesserocr
+      dockerfile: ../slim-containers-files/ocrd_tesserocr/Dockerfile
+    command:
+      ocrd network processing-worker ocrd-tesseroc-recognize --database $MONGODB_URL --queue $RABBITMQ_URL --create-queue
+
+  ocrd-tesserocr-segment-region:
+    build:
+      context: ../../ocrd_tesserocr
+      dockerfile: ../slim-containers-files/ocrd_tesserocr/Dockerfile
+    command:
+      ocrd network processing-worker ocrd-tesserocr-segment-region --database $MONGODB_URL --queue $RABBITMQ_URL --create-queue
diff --git a/slim-containers-files/ps-config.yaml b/slim-containers-files/ps-config.yaml
new file mode 100644
index 00000000..2d8b1de2
--- /dev/null
+++ b/slim-containers-files/ps-config.yaml
@@ -0,0 +1,12 @@
+process_queue:
+  address: ocrd-rabbitmq
+  port: 5672
+  skip_deployment: true
+  credentials:
+    username: admin
+    password: admin
+database:
+  address: ocrd-mongodb
+  port: 27017
+  skip_deployment: true
+hosts: []

From 5e71acf785a4478961cf4c9bab77e13a26c25d76 Mon Sep 17 00:00:00 2001
From: joschrew <jonas.schrewe@gwdg.de>
Date: Wed, 12 Jul 2023 16:49:23 +0200
Subject: [PATCH 02/19] Change submodule core head, for now

Changes not yet merged to core are needed for this PR to work. This
must be reset later
---
 core | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core b/core
index 552cfcd7..ed55b894 160000
--- a/core
+++ b/core
@@ -1 +1 @@
-Subproject commit 552cfcd72aec38b23a856924391602acc7689267
+Subproject commit ed55b894b09f38d1e870d373efdbe6cc7f5c40cd

From 2fb23471b3cda22f9123fbed8aaf8e2d26bfc99d Mon Sep 17 00:00:00 2001
From: joschrew <jonas.schrewe@gwdg.de>
Date: Wed, 12 Jul 2023 16:51:51 +0200
Subject: [PATCH 03/19] Use core Dockerfile from submodule

---
 slim-containers-files/Dummy-Core-Dockerfile        | 13 -------------
 slim-containers-files/docker-compose.template.yaml |  3 +--
 2 files changed, 1 insertion(+), 15 deletions(-)
 delete mode 100644 slim-containers-files/Dummy-Core-Dockerfile

diff --git a/slim-containers-files/Dummy-Core-Dockerfile b/slim-containers-files/Dummy-Core-Dockerfile
deleted file mode 100644
index fc667591..00000000
--- a/slim-containers-files/Dummy-Core-Dockerfile
+++ /dev/null
@@ -1,13 +0,0 @@
-# I need this because i need the network-for-slim-branch and this contains a comment how to make
-# pudb run.
-FROM ocrd/core:latest AS base
-WORKDIR /build-ocrd
-RUN apt install vim-tiny -y
-RUN git clone https://github.com/ocr-d/core.git && \
-	cd core && \
-	git checkout network-for-slim-prep && \
-	#sed -i "290 i \        from pudb.remote import set_trace; set_trace(term_size=(160, 40), host='0.0.0.0', port=6900)" ocrd_network/ocrd_network/processing_server.py && \
-	make install-dev && \
-	pip install pudb
-EXPOSE 6900
-WORKDIR /data
diff --git a/slim-containers-files/docker-compose.template.yaml b/slim-containers-files/docker-compose.template.yaml
index d4b4f2f5..7825d1e7 100644
--- a/slim-containers-files/docker-compose.template.yaml
+++ b/slim-containers-files/docker-compose.template.yaml
@@ -7,8 +7,7 @@ networks:
 services:
   ocrd-processing-server:
     build:
-      # later real ocrd-core image should be referenced here
-      dockerfile: slim-containers-files/Dummy-Core-Dockerfile
+      context: core
       args:
         BASE_IMAGE: ubuntu:20.04
     ports:

From fa2a47715f56b0f64d19db0e8fbca692189d198e Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Wed, 12 Jul 2023 17:15:56 +0200
Subject: [PATCH 04/19] Remove metspath-check from delegator-template

Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com>
---
 slim-containers-files/delegator_template.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/slim-containers-files/delegator_template.py b/slim-containers-files/delegator_template.py
index 499047eb..66734623 100755
--- a/slim-containers-files/delegator_template.py
+++ b/slim-containers-files/delegator_template.py
@@ -8,13 +8,6 @@
 processor_name = "{{ processor_name }}"
 
 args = list(sys.argv)
-if "-m" in args:
-    idx = args.index("-m")
-    metspath = args[idx + 1]
-    if Path(metspath).is_absolute():
-        print("absolute path is not supported")
-        exit(1)
-    args[idx + 1] = f"/data/{metspath}"
 
 
 cmd = [

From 7736969458dc24281ff87d189167d426209a02be Mon Sep 17 00:00:00 2001
From: joschrew <jonas.schrewe@gwdg.de>
Date: Thu, 13 Jul 2023 09:02:28 +0200
Subject: [PATCH 05/19] Make port for processing server dynamic

---
 Makefile-slim                               | 5 +++--
 slim-containers-files/delegator_template.py | 3 +--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile-slim b/Makefile-slim
index 469aa580..b300407c 100644
--- a/Makefile-slim
+++ b/Makefile-slim
@@ -7,13 +7,14 @@ OCRD_CIS = ocrd-cis-ocropy-binarize ocrd-cis-ocropy-dewarp
 OCRD_TESSEROCR = ocrd-tesserocr-recognize ocrd-tesserocr-segment-region
 PROCESSORS = $(foreach mod,$(OCRD_MODULES),$(foreach proc,$($(mod)), $(proc) ))
 DELEGATORS = $(foreach proc,$(PROCESSORS),$(BIN)/$(proc))
+OCRD_PS_PORT = 8000
 
 slim-venv: docker-compose.yaml .env $(DELEGATORS) | $(VIRTUAL_ENV)
 
 
 # create a delegator to the processing server for the processor
 $(BIN)/ocrd-%: | $(VIRTUAL_ENV)
-	@sed "s/{{\s*processor_name\s*}}/$(subst $(BIN)/,,$@)/" slim-containers-files/delegator_template.py > $@;
+	@sed -e "s/{{\s*processor_name\s*}}/$(subst $(BIN)/,,$@)/" -e "s/{{\s*OCRD_PS_PORT\s*}}/$(OCRD_PS_PORT)/" slim-containers-files/delegator_template.py > $@;
 	@chmod u+x $@
 
 
@@ -34,7 +35,7 @@ docker-compose.yaml:
 
 .env:
 	@rm -rf .env
-	@echo OCRD_PS_PORT=8000 >> .env
+	@echo OCRD_PS_PORT=$(OCRD_PS_PORT) >> .env
 	@echo OCRD_PS_MTU=1300 >> .env
 	@echo MONGODB_URL=mongodb://ocrd-mongodb:27017 >> .env
 	@echo RABBITMQ_URL=amqp://admin:admin@ocrd-rabbitmq:5672 >> .env
diff --git a/slim-containers-files/delegator_template.py b/slim-containers-files/delegator_template.py
index 66734623..a5ad9fde 100755
--- a/slim-containers-files/delegator_template.py
+++ b/slim-containers-files/delegator_template.py
@@ -1,10 +1,9 @@
 #!/usr/bin/env python
 import sys
-from pathlib import Path
 import subprocess
 
 # Later the address (or rather the port) should be dynamic
-processing_server_address = "http://localhost:8000"
+processing_server_address = "http://localhost:{{ OCRD_PS_PORT }}"
 processor_name = "{{ processor_name }}"
 
 args = list(sys.argv)

From e86611765afc7828a7ff0d44aa4a6c266e7b0d79 Mon Sep 17 00:00:00 2001
From: joschrew <jonas.schrewe@gwdg.de>
Date: Wed, 19 Jul 2023 10:02:19 +0200
Subject: [PATCH 06/19] Make delegator blocking with a waiting http server

---
 slim-containers-files/delegator_template.py | 46 +++++++++++++++++----
 1 file changed, 39 insertions(+), 7 deletions(-)

diff --git a/slim-containers-files/delegator_template.py b/slim-containers-files/delegator_template.py
index a5ad9fde..22081139 100755
--- a/slim-containers-files/delegator_template.py
+++ b/slim-containers-files/delegator_template.py
@@ -1,16 +1,48 @@
 #!/usr/bin/env python
 import sys
 import subprocess
+from http.server import BaseHTTPRequestHandler, HTTPServer
 
-# Later the address (or rather the port) should be dynamic
 processing_server_address = "http://localhost:{{ OCRD_PS_PORT }}"
 processor_name = "{{ processor_name }}"
 
-args = list(sys.argv)
+STOP_WAITING_SERVER = False
 
 
-cmd = [
-    "ocrd", "network", "client", "processing", "processor",
-    processor_name, "--address", processing_server_address
-]
-subprocess.run(cmd + args[1:])
+class CallbackReceiver(BaseHTTPRequestHandler):
+    """Simple http-server to wait for a processor to finish working.
+
+    The OCR-D processor is started with a callback-url which points to this server. The processor
+    starts working in the background but the call is returned immediately. When the processor is
+    finished it sends a request to the callback-url. The purpose of this server is to wait for the
+    processor to finish and print its result.
+    """
+    def do_POST(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain")
+        self.end_headers()
+        self.wfile.write("finished".encode("utf-8"))
+        len = int(self.headers.get("Content-Length", 0))
+        data = self.rfile.read(len).decode("utf-8")
+        # TODO: how should the callback-content be handled/printed
+        print(f"Processor finished: {data}")
+        global STOP_WAITING_SERVER
+        STOP_WAITING_SERVER = True
+
+
+def call_processor_and_wait():
+    server = HTTPServer(("0.0.0.0", 0), CallbackReceiver)
+    callback_url = f"http://172.17.0.1:{server.server_address[1]}"
+    cmd = [
+        "ocrd", "network", "client", "processing", "processor",
+        processor_name, "--address", processing_server_address,
+        "--callback-url", callback_url
+    ]
+    args = list(sys.argv)
+    subprocess.run(cmd + args[1:])
+    while not STOP_WAITING_SERVER:
+        server.handle_request()
+
+
+if __name__ == "__main__":
+    call_processor_and_wait()

From 10725b91a2e09a0a4f136578162300799eb4278d Mon Sep 17 00:00:00 2001
From: joschrew <jonas.schrewe@gwdg.de>
Date: Wed, 19 Jul 2023 14:53:58 +0200
Subject: [PATCH 07/19] Get processor name in delegator from argv

---
 Makefile-slim                               | 2 +-
 slim-containers-files/delegator_template.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Makefile-slim b/Makefile-slim
index b300407c..f358550a 100644
--- a/Makefile-slim
+++ b/Makefile-slim
@@ -14,7 +14,7 @@ slim-venv: docker-compose.yaml .env $(DELEGATORS) | $(VIRTUAL_ENV)
 
 # create a delegator to the processing server for the processor
 $(BIN)/ocrd-%: | $(VIRTUAL_ENV)
-	@sed -e "s/{{\s*processor_name\s*}}/$(subst $(BIN)/,,$@)/" -e "s/{{\s*OCRD_PS_PORT\s*}}/$(OCRD_PS_PORT)/" slim-containers-files/delegator_template.py > $@;
+	@sed -e "s/{{\s*OCRD_PS_PORT\s*}}/$(OCRD_PS_PORT)/" slim-containers-files/delegator_template.py > $@;
 	@chmod u+x $@
 
 
diff --git a/slim-containers-files/delegator_template.py b/slim-containers-files/delegator_template.py
index 22081139..34f06577 100755
--- a/slim-containers-files/delegator_template.py
+++ b/slim-containers-files/delegator_template.py
@@ -2,9 +2,10 @@
 import sys
 import subprocess
 from http.server import BaseHTTPRequestHandler, HTTPServer
+from os.path import basename
 
 processing_server_address = "http://localhost:{{ OCRD_PS_PORT }}"
-processor_name = "{{ processor_name }}"
+processor_name = basename(sys.argv[0])
 
 STOP_WAITING_SERVER = False
 

From c6da76e4e8fced0bd7d64348c8d3422e4b56d065 Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Thu, 20 Jul 2023 09:02:10 +0200
Subject: [PATCH 08/19] Don't publish docker ports for mongo

Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com>
---
 slim-containers-files/docker-compose.template.yaml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/slim-containers-files/docker-compose.template.yaml b/slim-containers-files/docker-compose.template.yaml
index 7825d1e7..432ee55e 100644
--- a/slim-containers-files/docker-compose.template.yaml
+++ b/slim-containers-files/docker-compose.template.yaml
@@ -24,10 +24,6 @@ services:
 
   ocrd-rabbitmq:
     image: rabbitmq:3-management
-    # Ports are only needed during the implementation phase to test. To be removed later
-    ports:
-      - "5672:5672"
-      - "15672:15672"
     environment:
       - "RABBITMQ_DEFAULT_USER=admin"
       - "RABBITMQ_DEFAULT_PASS=admin"

From 25260061bf196942d3e43cc30db6c8a1e1f4f8f7 Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Thu, 20 Jul 2023 09:03:35 +0200
Subject: [PATCH 09/19] Don't publish docker ports for mongodb and rabbitmq

Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com>
---
 slim-containers-files/docker-compose.template.yaml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/slim-containers-files/docker-compose.template.yaml b/slim-containers-files/docker-compose.template.yaml
index 432ee55e..ff6bedc9 100644
--- a/slim-containers-files/docker-compose.template.yaml
+++ b/slim-containers-files/docker-compose.template.yaml
@@ -18,9 +18,6 @@ services:
 
   ocrd-mongodb:
     image: mongo
-    # Ports are only needed during the implementation phase to test. To be removed later
-    ports:
-      - "27018:27017"
 
   ocrd-rabbitmq:
     image: rabbitmq:3-management

From 3ca07bd9c17240c0b2a27ac3fae66863420895dc Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Thu, 20 Jul 2023 09:51:09 +0200
Subject: [PATCH 10/19] Remove redundant call from Makefile-slim

Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com>
---
 Makefile-slim | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Makefile-slim b/Makefile-slim
index f358550a..e98fc4e0 100644
--- a/Makefile-slim
+++ b/Makefile-slim
@@ -34,7 +34,6 @@ docker-compose.yaml:
 	@$(foreach mod,$(OCRD_MODULES),$(foreach proc,$($(mod)),$(call add_proc,$(proc),$(mod))))
 
 .env:
-	@rm -rf .env
 	@echo OCRD_PS_PORT=$(OCRD_PS_PORT) >> .env
 	@echo OCRD_PS_MTU=1300 >> .env
 	@echo MONGODB_URL=mongodb://ocrd-mongodb:27017 >> .env

From b0017771dbd1c54b3d036935ecc75dd62b0cff32 Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Thu, 20 Jul 2023 10:58:36 +0200
Subject: [PATCH 11/19] Update env-var-handling for rabbitmq 1

Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com>
---
 slim-containers-files/docker-compose.template.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/slim-containers-files/docker-compose.template.yaml b/slim-containers-files/docker-compose.template.yaml
index ff6bedc9..b0fa540e 100644
--- a/slim-containers-files/docker-compose.template.yaml
+++ b/slim-containers-files/docker-compose.template.yaml
@@ -22,5 +22,6 @@ services:
   ocrd-rabbitmq:
     image: rabbitmq:3-management
     environment:
-      - "RABBITMQ_DEFAULT_USER=admin"
-      - "RABBITMQ_DEFAULT_PASS=admin"
+      environment:
+        RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-admin}
+        RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASS:-admin}

From 1047ba2b540f5c6d9cc8744542ba0d235c979208 Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Thu, 20 Jul 2023 10:59:01 +0200
Subject: [PATCH 12/19] Update env-var-handling for rabbitmq 2

Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com>
---
 Makefile-slim | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile-slim b/Makefile-slim
index e98fc4e0..120b78e1 100644
--- a/Makefile-slim
+++ b/Makefile-slim
@@ -37,5 +37,7 @@ docker-compose.yaml:
 	@echo OCRD_PS_PORT=$(OCRD_PS_PORT) >> .env
 	@echo OCRD_PS_MTU=1300 >> .env
 	@echo MONGODB_URL=mongodb://ocrd-mongodb:27017 >> .env
-	@echo RABBITMQ_URL=amqp://admin:admin@ocrd-rabbitmq:5672 >> .env
+	@echo RABBITMQ_USER=admin >> .env
+	@echo RABBITMQ_PASS=admin >> .env
+	@echo 'RABBITMQ_URL=amqp://$${RABBITMQ_USER}:$${RABBITMQ_PASS}@ocrd-rabbitmq:5672' >> .env
 

From ab2d1c3207919c2bc701eea8428ba4ce2b22b5f2 Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Thu, 20 Jul 2023 11:04:08 +0200
Subject: [PATCH 13/19] Update env-var-handling for mongodb 1

Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com>
---
 slim-containers-files/docker-compose.template.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/slim-containers-files/docker-compose.template.yaml b/slim-containers-files/docker-compose.template.yaml
index b0fa540e..0b0e0d5d 100644
--- a/slim-containers-files/docker-compose.template.yaml
+++ b/slim-containers-files/docker-compose.template.yaml
@@ -17,7 +17,10 @@ services:
     command: ocrd network processing-server -a 0.0.0.0:8000 /ocrd-processing-server-config.yaml
 
   ocrd-mongodb:
-    image: mongo
+    image: mongo:latest
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: ${MONGODB_USER:-admin}
+      MONGO_INITDB_ROOT_PASSWORD: ${MONGODB_PASS:-admin}
 
   ocrd-rabbitmq:
     image: rabbitmq:3-management

From 78346862fdbda3277fae19efed283a4972e87b72 Mon Sep 17 00:00:00 2001
From: joschrew <91774427+joschrew@users.noreply.github.com>
Date: Thu, 20 Jul 2023 11:04:38 +0200
Subject: [PATCH 14/19] Update env-var-handling for mongodb 2

Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com>
---
 Makefile-slim | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile-slim b/Makefile-slim
index 120b78e1..03193b95 100644
--- a/Makefile-slim
+++ b/Makefile-slim
@@ -36,7 +36,9 @@ docker-compose.yaml:
 .env:
 	@echo OCRD_PS_PORT=$(OCRD_PS_PORT) >> .env
 	@echo OCRD_PS_MTU=1300 >> .env
-	@echo MONGODB_URL=mongodb://ocrd-mongodb:27017 >> .env
+	@echo MONGODB_USER=admin >> .env
+	@echo MONGODB_PASS=admin >> .env
+	@echo 'MONGODB_URL=mongodb://$${MONGODB_USER}:$${MONGODB_PASS}@ocrd-mongodb:27017' >> .env
 	@echo RABBITMQ_USER=admin >> .env
 	@echo RABBITMQ_PASS=admin >> .env
 	@echo 'RABBITMQ_URL=amqp://$${RABBITMQ_USER}:$${RABBITMQ_PASS}@ocrd-rabbitmq:5672' >> .env

From d53366c9fcbc8bc21a5b6805236216e34a8ab133 Mon Sep 17 00:00:00 2001
From: joschrew <jonas.schrewe@gwdg.de>
Date: Thu, 20 Jul 2023 11:37:04 +0200
Subject: [PATCH 15/19] Fix env-var-handling for mongodb and rabbitmq

---
 slim-containers-files/docker-compose.template.yaml | 5 ++---
 slim-containers-files/ps-config.yaml               | 3 +++
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/slim-containers-files/docker-compose.template.yaml b/slim-containers-files/docker-compose.template.yaml
index 0b0e0d5d..a97036fd 100644
--- a/slim-containers-files/docker-compose.template.yaml
+++ b/slim-containers-files/docker-compose.template.yaml
@@ -25,6 +25,5 @@ services:
   ocrd-rabbitmq:
     image: rabbitmq:3-management
     environment:
-      environment:
-        RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-admin}
-        RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASS:-admin}
+      RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-admin}
+      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASS:-admin}
diff --git a/slim-containers-files/ps-config.yaml b/slim-containers-files/ps-config.yaml
index 2d8b1de2..7450400e 100644
--- a/slim-containers-files/ps-config.yaml
+++ b/slim-containers-files/ps-config.yaml
@@ -9,4 +9,7 @@ database:
   address: ocrd-mongodb
   port: 27017
   skip_deployment: true
+  credentials:
+    username: admin
+    password: admin
 hosts: []

From c6e67d6a9c1844a398c57b43745c73e7353391bf Mon Sep 17 00:00:00 2001
From: joschrew <jonas.schrewe@gwdg.de>
Date: Fri, 21 Jul 2023 16:03:59 +0200
Subject: [PATCH 16/19] Generate ps-config dynamically with .env settings

---
 .../docker-compose.template.yaml              | 26 ++++++++++++++++---
 slim-containers-files/ps-config.yaml          | 15 -----------
 2 files changed, 23 insertions(+), 18 deletions(-)
 delete mode 100644 slim-containers-files/ps-config.yaml

diff --git a/slim-containers-files/docker-compose.template.yaml b/slim-containers-files/docker-compose.template.yaml
index a97036fd..cf1a0be7 100644
--- a/slim-containers-files/docker-compose.template.yaml
+++ b/slim-containers-files/docker-compose.template.yaml
@@ -12,9 +12,29 @@ services:
         BASE_IMAGE: ubuntu:20.04
     ports:
       - ${OCRD_PS_PORT}:8000
-    volumes:
-      - "./slim-containers-files/ps-config.yaml:/ocrd-processing-server-config.yaml"
-    command: ocrd network processing-server -a 0.0.0.0:8000 /ocrd-processing-server-config.yaml
+    environment:
+      MONGODB_USER: ${MONGODB_USER:-admin}
+      MONGODB_PASS: ${MONGODB_PASS:-admin}
+      RABBITMQ_USER: ${RABBITMQ_USER:-admin}
+      RABBITMQ_PASS: ${RABBITMQ_PASS:-admin}
+    command: |
+      /bin/bash -c "echo -e \"
+        process_queue:
+          address: ocrd-rabbitmq
+          port: 5672
+          skip_deployment: true
+          credentials:
+            username: ${RABBITMQ_USER}
+            password: ${RABBITMQ_PASS}
+        database:
+          address: ocrd-mongodb
+          port: 27017
+          skip_deployment: true
+          credentials:
+            username: ${MONGODB_USER}
+            password: ${MONGODB_PASS}
+        hosts: []\" > /ocrd-processing-server-config.yaml && \
+        ocrd network processing-server -a 0.0.0.0:8000 /ocrd-processing-server-config.yaml"
 
   ocrd-mongodb:
     image: mongo:latest
diff --git a/slim-containers-files/ps-config.yaml b/slim-containers-files/ps-config.yaml
deleted file mode 100644
index 7450400e..00000000
--- a/slim-containers-files/ps-config.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-process_queue:
-  address: ocrd-rabbitmq
-  port: 5672
-  skip_deployment: true
-  credentials:
-    username: admin
-    password: admin
-database:
-  address: ocrd-mongodb
-  port: 27017
-  skip_deployment: true
-  credentials:
-    username: admin
-    password: admin
-hosts: []

From cfd1a79f1ff47252dea8fe23956470a8a3182c0b Mon Sep 17 00:00:00 2001
From: joschrew <jonas.schrewe@gwdg.de>
Date: Thu, 27 Jul 2023 13:18:48 +0200
Subject: [PATCH 17/19] Read port for delegator from .env

---
 Makefile-slim                               |  2 +-
 slim-containers-files/delegator_template.py | 11 ++++++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/Makefile-slim b/Makefile-slim
index 03193b95..4f6eb7c1 100644
--- a/Makefile-slim
+++ b/Makefile-slim
@@ -14,7 +14,7 @@ slim-venv: docker-compose.yaml .env $(DELEGATORS) | $(VIRTUAL_ENV)
 
 # create a delegator to the processing server for the processor
 $(BIN)/ocrd-%: | $(VIRTUAL_ENV)
-	@sed -e "s/{{\s*OCRD_PS_PORT\s*}}/$(OCRD_PS_PORT)/" slim-containers-files/delegator_template.py > $@;
+	@sed -e "s|{{\s*OCRD_SLIM_ENV_PATH\s*}}|$(CURDIR)/.env|" slim-containers-files/delegator_template.py > $@;
 	@chmod u+x $@
 
 
diff --git a/slim-containers-files/delegator_template.py b/slim-containers-files/delegator_template.py
index 34f06577..066561fb 100755
--- a/slim-containers-files/delegator_template.py
+++ b/slim-containers-files/delegator_template.py
@@ -3,8 +3,17 @@
 import subprocess
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from os.path import basename
+import re
+from pathlib import Path
 
-processing_server_address = "http://localhost:{{ OCRD_PS_PORT }}"
+env_path = "{{ OCRD_SLIM_ENV_PATH }}"
+try:
+    port = re.search(r"OCRD_PS_PORT=(?P<port>[0-9]+)",
+                     Path(f'{env_path}').read_text()).group('port')
+except:
+    raise ValueError("Variable OCRD_PS_PORT not found in .env")
+
+processing_server_address = f"http://localhost:{port}"
 processor_name = basename(sys.argv[0])
 
 STOP_WAITING_SERVER = False

From 7ce609667c82d8421a17352111bbb7a4c794a0e2 Mon Sep 17 00:00:00 2001
From: joschrew <jonas.schrewe@gwdg.de>
Date: Tue, 1 Aug 2023 10:42:20 +0200
Subject: [PATCH 18/19] Revert last commit: read port from .env"

This reverts commit cfd1a79f1ff47252dea8fe23956470a8a3182c0b. Goal is
to read .env when activating the venv and use the env-variable to set
the processing server port.
---
 Makefile-slim                               |  2 +-
 slim-containers-files/delegator_template.py | 11 +----------
 2 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/Makefile-slim b/Makefile-slim
index 4f6eb7c1..03193b95 100644
--- a/Makefile-slim
+++ b/Makefile-slim
@@ -14,7 +14,7 @@ slim-venv: docker-compose.yaml .env $(DELEGATORS) | $(VIRTUAL_ENV)
 
 # create a delegator to the processing server for the processor
 $(BIN)/ocrd-%: | $(VIRTUAL_ENV)
-	@sed -e "s|{{\s*OCRD_SLIM_ENV_PATH\s*}}|$(CURDIR)/.env|" slim-containers-files/delegator_template.py > $@;
+	@sed -e "s/{{\s*OCRD_PS_PORT\s*}}/$(OCRD_PS_PORT)/" slim-containers-files/delegator_template.py > $@;
 	@chmod u+x $@
 
 
diff --git a/slim-containers-files/delegator_template.py b/slim-containers-files/delegator_template.py
index 066561fb..34f06577 100755
--- a/slim-containers-files/delegator_template.py
+++ b/slim-containers-files/delegator_template.py
@@ -3,17 +3,8 @@
 import subprocess
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from os.path import basename
-import re
-from pathlib import Path
 
-env_path = "{{ OCRD_SLIM_ENV_PATH }}"
-try:
-    port = re.search(r"OCRD_PS_PORT=(?P<port>[0-9]+)",
-                     Path(f'{env_path}').read_text()).group('port')
-except:
-    raise ValueError("Variable OCRD_PS_PORT not found in .env")
-
-processing_server_address = f"http://localhost:{port}"
+processing_server_address = "http://localhost:{{ OCRD_PS_PORT }}"
 processor_name = basename(sys.argv[0])
 
 STOP_WAITING_SERVER = False

From a7c1328d93d26ad0fdda5a834b051f8be400363c Mon Sep 17 00:00:00 2001
From: joschrew <jonas.schrewe@gwdg.de>
Date: Tue, 1 Aug 2023 11:39:38 +0200
Subject: [PATCH 19/19] Read and set .env when activating venv

---
 Makefile-slim                               | 1 +
 slim-containers-files/delegator_template.py | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile-slim b/Makefile-slim
index 03193b95..71bf5657 100644
--- a/Makefile-slim
+++ b/Makefile-slim
@@ -24,6 +24,7 @@ $(VIRTUAL_ENV): $(ACTIVATE_VENV)
 %/bin/activate:
 	$(PYTHON) -m venv $(subst /bin/activate,,$@)
 	. $@ && pip install --upgrade pip setuptools wheel
+	@echo "set -a; source $(CURDIR)/.env; set +a" >> $@
 
 # append the service to docker-compose for a processor
 add_proc = sed -e "s/{{\s*processor_name\s*}}/$1/" -e "s/{{\s*processor_group_name\s*}}/\L$2/" \
diff --git a/slim-containers-files/delegator_template.py b/slim-containers-files/delegator_template.py
index 34f06577..3d389f6a 100755
--- a/slim-containers-files/delegator_template.py
+++ b/slim-containers-files/delegator_template.py
@@ -3,8 +3,9 @@
 import subprocess
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from os.path import basename
+from os import environ
 
-processing_server_address = "http://localhost:{{ OCRD_PS_PORT }}"
+processing_server_address = f"http://localhost:{environ['OCRD_PS_PORT']}"
 processor_name = basename(sys.argv[0])
 
 STOP_WAITING_SERVER = False