Merge pull request #1087 from OCR-D/worker-cli

replace --agent-type with subcommand, #1032, #1085

ocrd/ocrd/cli/ocrd_tool.py

@@ -6,6 +6,7 @@
     :nested: full
 
 """
+from inspect import getmodule
 from json import dumps
 import codecs
 import sys
@@ -115,16 +116,18 @@ def moduledir(self):
                   show_resource=res_name)
 
 @ocrd_tool_tool.command('help', help="Generate help for processors")
+@click.argument('subcommand', required=False)
 @pass_ocrd_tool
-def ocrd_tool_tool_params_help(ctx):
+def ocrd_tool_tool_params_help(ctx, subcommand):
     class BashProcessor(Processor):
         # set docstrings to empty
-        # fixme: override the module-level docstring, too
         __doc__ = None
+        # HACK: override the module-level docstring, too
+        getmodule(OcrdToolCtx).__doc__ = None
         def process(self):
             return super()
     BashProcessor(None, ocrd_tool=ctx.json['tools'][ctx.tool_name],
-                  show_help=True)
+                  show_help=True, subcommand=subcommand)
 
 # ----------------------------------------------------------------------
 # ocrd ocrd-tool tool categories

ocrd/ocrd/decorators/__init__.py

@@ -20,6 +20,8 @@
 from .ocrd_cli_options import ocrd_cli_options
 from .mets_find_options import mets_find_options
 
+SUBCOMMANDS = ['worker', 'server']
+
 def ocrd_cli_wrap_processor(
     processorClass,
     mets=None,
@@ -35,8 +37,8 @@ def ocrd_cli_wrap_processor(
     show_resource=None,
     list_resources=False,
     # ocrd_network params start #
-    agent_type=None,
-    agent_address=None,
+    subcommand=None,
+    address=None,
     queue=None,
     database=None,
     # ocrd_network params end #
@@ -51,17 +53,20 @@ def ocrd_cli_wrap_processor(
             dump_json=dump_json,
             dump_module_dir=dump_module_dir,
             show_help=help,
+            subcommand=subcommand,
             show_version=version,
             show_resource=show_resource,
             list_resources=list_resources
         )
         sys.exit()
+    if subcommand:
+        # Used for checking/starting network agents for the WebAPI architecture
+        check_and_run_network_agent(processorClass, subcommand, address, database, queue)
+    elif address or queue or database:
+        raise ValueError(f"Subcommand options --adress --queue and --database are only valid for subcommands 'worker' or 'server'")
 
-    initLogging()
 
-    # Used for checking/starting network agents for the WebAPI architecture
-    # Has no side effects if neither of the 4 ocrd_network parameters are passed
-    check_and_run_network_agent(processorClass, agent_type, agent_address, database, queue)
+    initLogging()
 
     LOG = getLogger('ocrd_cli_wrap_processor')
     # LOG.info('kwargs=%s' % kwargs)
@@ -128,59 +133,50 @@ def exit():
     run_processor(processorClass, mets_url=mets, workspace=workspace, **kwargs)
 
 
-def check_and_run_network_agent(ProcessorClass, agent_type: str, agent_address: str, database: str, queue: str):
-    if not agent_type and (agent_address or database or queue):
-        raise ValueError("Options '--database', '--queue', and '--address' are valid only with '--type'")
-    if not agent_type:
-        return
+def check_and_run_network_agent(ProcessorClass, subcommand: str, address: str, database: str, queue: str):
+    """
+    """
+    if subcommand not in SUBCOMMANDS:
+        raise ValueError(f"SUBCOMMAND can only be one of {SUBCOMMANDS}")
 
     if not database:
-        raise ValueError("Options '--type' and '--database' are mutually inclusive")
-    allowed_agent_types = ['server', 'worker']
-    if agent_type not in allowed_agent_types:
-        agents_str = ', '.join(allowed_agent_types)
-        raise ValueError(f"Wrong type parameter. Allowed types: {agents_str}")
-    if agent_type == 'server':
-        if not agent_address:
-            raise ValueError("Options '--type=server' and '--address' are mutually inclusive")
+        raise ValueError(f"Option '--database' is invalid for subcommand {subcommand}")
+
+    if subcommand == 'server':
+        if not address:
+            raise ValueError(f"Option '--address' required for subcommand {subcommand}")
         if queue:
-            raise ValueError("Options '--type=server' and '--queue' are mutually exclusive")
-    if agent_type == 'worker':
+            raise ValueError(f"Option '--queue' invalid for subcommand {subcommand}")
+    if subcommand == 'worker':
+        if address:
+            raise ValueError(f"Option '--address' invalid for subcommand {subcommand}")
         if not queue:
-            raise ValueError("Options '--type=worker' and '--queue' are mutually inclusive")
-        if agent_address:
-            raise ValueError("Options '--type=worker' and '--address' are mutually exclusive")
+            raise ValueError(f"Option '--queue' required for subcommand {subcommand}")
 
     import logging
     logging.getLogger('ocrd.network').setLevel(logging.DEBUG)
 
     processor = ProcessorClass(workspace=None, dump_json=True)
-    if agent_type == 'worker':
-        try:
-            # TODO: Passing processor_name and ocrd_tool is reduntant
-            processing_worker = ProcessingWorker(
-                rabbitmq_addr=queue,
-                mongodb_addr=database,
-                processor_name=processor.ocrd_tool['executable'],
-                ocrd_tool=processor.ocrd_tool,
-                processor_class=ProcessorClass,
-            )
-            # The RMQConsumer is initialized and a connection to the RabbitMQ is performed
-            processing_worker.connect_consumer()
-            # Start consuming from the queue with name `processor_name`
-            processing_worker.start_consuming()
-        except Exception as e:
-            sys.exit(f"Processing worker has failed with error: {e}")
-    if agent_type == 'server':
-        try:
-            # TODO: Better validate that inside the ProcessorServer itself
-            host, port = agent_address.split(':')
-            processor_server = ProcessorServer(
-                mongodb_addr=database,
-                processor_name=processor.ocrd_tool['executable'],
-                processor_class=ProcessorClass,
-            )
-            processor_server.run_server(host=host, port=int(port))
-        except Exception as e:
-            sys.exit(f"Processor server has failed with error: {e}")
+    if subcommand == 'worker':
+        # TODO: Passing processor_name and ocrd_tool is reduntant
+        processing_worker = ProcessingWorker(
+            rabbitmq_addr=queue,
+            mongodb_addr=database,
+            processor_name=processor.ocrd_tool['executable'],
+            ocrd_tool=processor.ocrd_tool,
+            processor_class=ProcessorClass,
+        )
+        # The RMQConsumer is initialized and a connection to the RabbitMQ is performed
+        processing_worker.connect_consumer()
+        # Start consuming from the queue with name `processor_name`
+        processing_worker.start_consuming()
+    elif subcommand == 'server':
+        # TODO: Better validate that inside the ProcessorServer itself
+        host, port = address.split(':')
+        processor_server = ProcessorServer(
+            mongodb_addr=database,
+            processor_name=processor.ocrd_tool['executable'],
+            processor_class=ProcessorClass,
+        )
+        processor_server.run_server(host=host, port=int(port))
     sys.exit(0)

ocrd/ocrd/decorators/ocrd_cli_options.py

@@ -1,5 +1,5 @@
 import click
-from click import option, Path
+from click import option, Path, group, command, argument
 from .parameter_option import parameter_option, parameter_override_option
 from .loglevel_option import loglevel_option
 from ocrd_network import (
@@ -39,8 +39,7 @@ def cli(mets_url):
         parameter_option,
         parameter_override_option,
         loglevel_option,
-        option('--type', 'agent_type', type=click.Choice(['worker', 'server'])),
-        option('--address', 'agent_address', type=ServerAddressParamType()),
+        option('--address', type=ServerAddressParamType()),
         option('--queue', type=QueueServerParamType()),
         option('--database', type=DatabaseParamType()),
         option('-C', '--show-resource'),
@@ -49,6 +48,12 @@ def cli(mets_url):
         option('-D', '--dump-module-dir', is_flag=True, default=False),
         option('-h', '--help', is_flag=True, default=False),
         option('-V', '--version', is_flag=True, default=False),
+        # Subcommand, only used for 'worker'/'server'. Cannot be handled in
+        # click because processors use the @command decorator and even if they
+        # were using `group`, you cannot combine have a command with
+        # subcommands. So we have to work around that by creating a
+        # pseudo-subcommand handled in ocrd_cli_wrap_processor
+        argument('subcommand', nargs=1, required=False, type=click.Choice(['worker', 'server'])),
     ]
     for param in params:
         param(f)

ocrd/ocrd/lib.bash

@@ -87,7 +87,7 @@ ocrd__list_resources () {
 ##
 ocrd__usage () {
 
-    ocrd ocrd-tool "$OCRD_TOOL_JSON" tool "$OCRD_TOOL_NAME" help
+    ocrd ocrd-tool "$OCRD_TOOL_JSON" tool "$OCRD_TOOL_NAME" help $ocrd__subcommand
 
 }
 
@@ -124,6 +124,10 @@ ocrd__parse_argv () {
     local __parameters=()
     local __parameter_overrides=()
 
+    if [[ $1 == 'worker' || $1 == 'server' ]];then
+         ocrd__subcommand="$1" ; shift ;
+    fi
+
     while [[ "${1:-}" = -* ]];do
         case "$1" in
             -l|--log-level) ocrd__argv[log_level]=$2 ; shift ;;
@@ -146,31 +150,30 @@ ocrd__parse_argv () {
             -V|--version) ocrd ocrd-tool "$OCRD_TOOL_JSON" version; exit ;;
             --queue) ocrd__worker_queue="$2" ; shift ;;
             --database) ocrd__worker_database="$2" ; shift ;;
-            --type) ocrd__worker_type="$2" ; shift ;;
             --address) ocrd__worker_address="$2" ; shift ;;
             *) ocrd__raise "Unknown option '$1'" ;;
         esac
         shift
     done
 
-    if [ -v ocrd__worker_queue -o -v ocrd__worker_database -o -v ocrd__worker_type -o -v ocrd__worker_address ]; then
-        if ! [ -v ocrd__worker_type ] ; then
-            ocrd__raise "For Processing Worker / Processor Server --type is required"
+    if [ -v ocrd__worker_queue -o -v ocrd__worker_database -o -v ocrd__subcommand -o -v ocrd__worker_address ]; then
+        if ! [ -v ocrd__subcommand ] ; then
+            ocrd__raise "Provide subcommand 'worker' or 'server' for Processing Worker / Processor Server"
         elif ! [ -v ocrd__worker_database ]; then
             ocrd__raise "For the Processing Worker / Processor Server --database is required"
         fi
-        if [ ${ocrd__worker_type} = "worker" ]; then
+        if [ ${ocrd__subcommand} = "worker" ]; then
             if ! [ -v ocrd__worker_queue ]; then
                 ocrd__raise "For the Processing Worker --queue is required"
             fi
             ocrd network processing-worker $OCRD_TOOL_NAME --queue "${ocrd__worker_queue}" --database "${ocrd__worker_database}"
-        elif [ ${ocrd__worker_type} = "server" ]; then
+        elif [ ${ocrd__subcommand} = "server" ]; then
             if ! [ -v ocrd__worker_address ]; then
                 ocrd__raise "For the Processor Server --address is required"
             fi
             ocrd network processor-server $OCRD_TOOL_NAME --database "${ocrd__worker_database}" --address "${ocrd__worker_address}"
         else
-            ocrd__raise "--type must be either 'worker' or 'server' not '${ocrd__worker_type}'"
+            ocrd__raise "subcommand must be either 'worker' or 'server' not '${ocrd__subcommand}'"
         fi
         exit
     fi

ocrd/ocrd/processor/base.py

@@ -61,6 +61,7 @@ def __init__(
             show_resource=None,
             list_resources=False,
             show_help=False,
+            subcommand=None,
             show_version=False,
             dump_json=False,
             dump_module_dir=False,
@@ -92,6 +93,7 @@ def __init__(
              show_help (boolean): If true, then instead of processing, print a usage description \
                  including the standard CLI and all of this processor's ocrd-tool parameters and \
                  docstrings.
+             subcommand (string): 'worker' or 'server', only used here for the right --help output
              show_version (boolean): If true, then instead of processing, print information on \
                  this processor's version and OCR-D version. Exit afterwards.
              dump_json (boolean): If true, then instead of processing, print :py:attr:`ocrd_tool` \
@@ -127,7 +129,7 @@ def __init__(
                 sys.stdout.buffer.write(fpath.read_bytes())
             return
         if show_help:
-            self.show_help()
+            self.show_help(subcommand=subcommand)
             return
         self.version = version
         if show_version:
@@ -149,8 +151,8 @@ def __init__(
             raise Exception("Invalid parameters %s" % report.errors)
         self.parameter = parameter
 
-    def show_help(self):
-        print(generate_processor_help(self.ocrd_tool, processor_instance=self))
+    def show_help(self, subcommand=None):
+        print(generate_processor_help(self.ocrd_tool, processor_instance=self, subcommand=subcommand))
 
     def show_version(self):
         print("Version %s, ocrd/core %s" % (self.version, OCRD_VERSION))