Draft for slim containers

Makefile-slim

@@ -0,0 +1,41 @@
+export PYTHON ?= python3
+VIRTUAL_ENV = $(CURDIR)/venv2
+BIN = $(VIRTUAL_ENV)/bin
+ACTIVATE_VENV = $(BIN)/activate
+OCRD_MODULES = OCRD_CIS OCRD_TESSEROCR
+OCRD_CIS = ocrd-cis-ocropy-binarize ocrd-cis-ocropy-dewarp
+OCRD_TESSEROCR = ocrd-tesserocr-recognize ocrd-tesserocr-segment-region
+PROCESSORS = $(foreach mod,$(OCRD_MODULES),$(foreach proc,$($(mod)), $(proc) ))
+DELEGATORS = $(foreach proc,$(PROCESSORS),$(BIN)/$(proc))
+
+slim-venv: docker-compose.yaml .env $(DELEGATORS) | $(VIRTUAL_ENV)
+
+
+# create a delegator to the processing server for the processor
+$(BIN)/ocrd-%: | $(VIRTUAL_ENV)
+	@sed "s/{{\s*processor_name\s*}}/$(subst $(BIN)/,,$@)/" slim-containers-files/delegator_template.py > $@;
+	@chmod u+x $@
+
+
+$(VIRTUAL_ENV): $(ACTIVATE_VENV)
+	. $(ACTIVATE_VENV) && $(MAKE) -C core install
+
+%/bin/activate:
+	$(PYTHON) -m venv $(subst /bin/activate,,$@)
+	. $@ && pip install --upgrade pip setuptools wheel
+
+# append the service to docker-compose for a processor
+add_proc = sed -e "s/{{\s*processor_name\s*}}/$1/" -e "s/{{\s*processor_group_name\s*}}/\L$2/" \
+            slim-containers-files/docker-compose.processor.template.yaml >> docker-compose.yaml;
+
+docker-compose.yaml:
+	@cat slim-containers-files/docker-compose.template.yaml > docker-compose.yaml
+	@$(foreach mod,$(OCRD_MODULES),$(foreach proc,$($(mod)),$(call add_proc,$(proc),$(mod))))
+
+.env:
+	@rm -rf .env
+	@echo OCRD_PS_PORT=8000 >> .env
+	@echo OCRD_PS_MTU=1300 >> .env
+	@echo MONGODB_URL=mongodb://ocrd-mongodb:27017 >> .env
+	@echo RABBITMQ_URL=amqp://admin:admin@ocrd-rabbitmq:5672 >> .env
+

slim-containers-files/Dummy-Core-Dockerfile

@@ -0,0 +1,13 @@
+# I need this because i need the network-for-slim-branch and this contains a comment how to make
+# pudb run.
+FROM ocrd/core:latest AS base
+WORKDIR /build-ocrd
+RUN apt install vim-tiny -y
+RUN git clone https://github.com/ocr-d/core.git && \
+	cd core && \
+	git checkout network-for-slim-prep && \
+	#sed -i "290 i \        from pudb.remote import set_trace; set_trace(term_size=(160, 40), host='0.0.0.0', port=6900)" ocrd_network/ocrd_network/processing_server.py && \
+	make install-dev && \
+	pip install pudb
+EXPOSE 6900
+WORKDIR /data

slim-containers-files/delegator_template.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+import sys
+from pathlib import Path
+import subprocess
+
+# Later the address (or rather the port) should be dynamic
+processing_server_address = "http://localhost:8000"
+processor_name = "{{ processor_name }}"
+
+args = list(sys.argv)
+if "-m" in args:
+    idx = args.index("-m")
+    metspath = args[idx + 1]
+    if Path(metspath).is_absolute():
+        print("absolute path is not supported")
+        exit(1)
+    args[idx + 1] = f"/data/{metspath}"
+
+
+cmd = [
+    "ocrd", "network", "client", "processing", "processor",
+    processor_name, "--address", processing_server_address
+]
+subprocess.run(cmd + args[1:])

slim-containers-files/docker-compose.processor.template.yaml

@@ -0,0 +1,14 @@
+
+  {{ processor_name }}:
+    extends:
+      file: slim-containers-files/{{ processor_group_name}}/docker-compose.yaml
+      service: {{ processor_name }}
+    command: ocrd network processing-worker --database $MONGODB_URL --queue $RABBITMQ_URL --create-queue {{ processor_name }}
+    depends_on:
+      - ocrd-processing-server
+      - ocrd-mongodb
+      - ocrd-rabbitmq
+    # restart: The worker creates its queue but rabbitmq needs a few seconds to be available
+    restart: on-failure:3
+    volumes:
+      - "$PWD/data:/data"

slim-containers-files/docker-compose.template.yaml

@@ -0,0 +1,34 @@
+networks:
+  default:
+    driver: bridge
+    driver_opts:
+      com.docker.network.driver.mtu: ${OCRD_PS_MTU}
+
+services:
+  ocrd-processing-server:
+    build:
+      # later real ocrd-core image should be referenced here
+      dockerfile: slim-containers-files/Dummy-Core-Dockerfile
+      args:
+        BASE_IMAGE: ubuntu:20.04
+    ports:
+      - ${OCRD_PS_PORT}:8000
+    volumes:
+      - "./slim-containers-files/ps-config.yaml:/ocrd-processing-server-config.yaml"
+    command: ocrd network processing-server -a 0.0.0.0:8000 /ocrd-processing-server-config.yaml
+
+  ocrd-mongodb:
+    image: mongo
+    # Ports are only needed during the implementation phase to test. To be removed later
+    ports:
+      - "27018:27017"
+
+  ocrd-rabbitmq:
+    image: rabbitmq:3-management
+    # Ports are only needed during the implementation phase to test. To be removed later
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+    environment:
+      - "RABBITMQ_DEFAULT_USER=admin"
+      - "RABBITMQ_DEFAULT_PASS=admin"

slim-containers-files/ocrd_cis/Dockerfile

@@ -0,0 +1,15 @@
+FROM ocrd/core:latest AS base
+WORKDIR /build-ocrd
+# Remove the next RUN, this is only to checkout my branch while the changes are not in core yet
+RUN git clone https://github.com/ocr-d/core.git && \
+	cd core && \
+	git checkout network-for-slim-prep && \
+	make install
+
+# Not based on ocrd_cis "original" Dockerfile. That seems out of date and in ocrd_all ocrd_cis is
+# simply installed with pip so I do the same here
+COPY ocrd_cis/ ./ocrd_cis/
+COPY setup.py README.md LICENSE ocrd-tool.json Manifest.in ./
+RUN pip install . && rm -rf /build-ocrd
+# TODO: install models for ocrd-cis
+WORKDIR /data

slim-containers-files/ocrd_cis/docker-compose.yaml

@@ -0,0 +1,14 @@
+services:
+  ocrd-cis-ocropy-binarize:
+    build:
+      context: ../../ocrd_cis
+      dockerfile: ../slim-containers-files/ocrd_cis/Dockerfile
+    command:
+      ocrd network processing-worker ocrd-cis-ocropy-binarize --database $MONGODB_URL --queue $RABBITMQ_URL --create-queue
+
+  ocrd-cis-ocropy-dewarp:
+    build:
+      context: ../../ocrd_cis
+      dockerfile: ../slim-containers-files/ocrd_cis/Dockerfile
+    command:
+      ocrd network processing-worker ocrd-cis-ocropy-dewarp --database $MONGODB_URL --queue $RABBITMQ_URL --create-queue

slim-containers-files/ocrd_tesserocr/Dockerfile

@@ -0,0 +1,44 @@
+FROM ocrd/core:latest AS base
+WORKDIR /build-ocrd-core
+# Remove the next RUN, this is only to checkout my branch while the changes are not in core yet
+RUN git clone https://github.com/ocr-d/core.git && \
+	cd core && \
+	git checkout network-for-slim-prep && \
+	make install
+
+# copied from https://github.com/OCR-D/ocrd_tesserocr/blob/master/Dockerfile and modified
+ARG VCS_REF
+ARG BUILD_DATE
+LABEL \
+    maintainer="https://ocr-d.de/kontakt" \
+    org.label-schema.vcs-ref=$VCS_REF \
+    org.label-schema.vcs-url="https://github.com/OCR-D/ocrd_tesserocr" \
+    org.label-schema.build-date=$BUILD_DATE
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV PYTHONIOENCODING utf8
+
+# avoid HOME/.local/share (hard to predict USER here)
+# so let XDG_DATA_HOME coincide with fixed system location
+# (can still be overridden by derived stages)
+ENV XDG_DATA_HOME /usr/local/share
+
+WORKDIR /build-ocrd
+COPY setup.py .
+COPY ocrd_tesserocr/ocrd-tool.json .
+COPY README.md .
+COPY requirements.txt .
+COPY requirements_test.txt .
+COPY ocrd_tesserocr ./ocrd_tesserocr
+COPY Makefile .
+RUN make deps-ubuntu && \
+    apt-get install -y --no-install-recommends \
+    g++ \
+    && make deps install \
+    && rm -rf /build-ocrd \
+    && apt-get -y remove --auto-remove g++ libtesseract-dev make
+RUN ocrd resmgr download ocrd-tesserocr-recognize Fraktur.traineddata
+RUN ocrd resmgr download ocrd-tesserocr-recognize deu.traineddata
+
+WORKDIR /data
+VOLUME /data

slim-containers-files/ocrd_tesserocr/docker-compose.yaml

@@ -0,0 +1,14 @@
+services:
+  ocrd-tesserocr-recognize:
+    build:
+      context: ../../ocrd_tesserocr
+      dockerfile: ../slim-containers-files/ocrd_tesserocr/Dockerfile
+    command:
+      ocrd network processing-worker ocrd-tesseroc-recognize --database $MONGODB_URL --queue $RABBITMQ_URL --create-queue
+
+  ocrd-tesserocr-segment-region:
+    build:
+      context: ../../ocrd_tesserocr
+      dockerfile: ../slim-containers-files/ocrd_tesserocr/Dockerfile
+    command:
+      ocrd network processing-worker ocrd-tesserocr-segment-region --database $MONGODB_URL --queue $RABBITMQ_URL --create-queue

slim-containers-files/ps-config.yaml

@@ -0,0 +1,12 @@
+process_queue:
+  address: ocrd-rabbitmq
+  port: 5672
+  skip_deployment: true
+  credentials:
+    username: admin
+    password: admin
+database:
+  address: ocrd-mongodb
+  port: 27017
+  skip_deployment: true
+hosts: []