Merge pull request #323 from deeppavlov/dev

Release v0.14.1
deeppavlov · Feb 14, 2023 · 4c17e6f · 4c17e6f
2 parents 598e490 + 5edea73
commit 4c17e6f
Show file tree

Hide file tree

Showing 15 changed files with 71 additions and 60 deletions.
diff --git a/annotators/entity_linking_deepy/Dockerfile b/annotators/entity_linking_deepy/Dockerfile
@@ -1,5 +1,4 @@
 FROM deeppavlov/base-gpu:0.12.1
-RUN pip install --upgrade pip && pip install git+https://github.com/deeppavlov/[email protected]
 
 ARG CONFIG
 ARG COMMIT
@@ -9,7 +8,11 @@ ARG SRC_DIR
 ENV CONFIG=$CONFIG
 ENV PORT=$PORT
 
-RUN pip install sentry-sdk==0.13.0
+RUN pip install pybind11==2.2.4
+RUN pip install hdt==2.3
+
+COPY ./annotators/entity_linking_deepy/requirements.txt /src/requirements.txt
+RUN pip install --upgrade pip && pip install -r /src/requirements.txt
 
 RUN cd DeepPavlov && \
     git config --global user.email "[email protected]" && \

diff --git a/annotators/entity_linking_deepy/requirements.txt b/annotators/entity_linking_deepy/requirements.txt
@@ -1,5 +1,7 @@
 aiohttp
 jinja2<=3.0.3
 Werkzeug<=2.0.3
+git+https://github.com/deeppavlov/[email protected]
 git+https://github.com/deeppavlov/bert.git@feat/multi_gpu
-tensorflow==1.15.5
+sentry-sdk==0.13.0
+rapidfuzz==0.7.6
diff --git a/assistant_dists/deepy_adv/docker-compose.override.yml b/assistant_dists/deepy_adv/docker-compose.override.yml
@@ -1,11 +1,8 @@
 services:
   agent:
+    command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/deepy_adv/pipeline_conf.json'
     volumes:
-      - "./assistant_dists/deepy_adv:/dp-agent"
-      - "./skill_selectors/:/dp-agent/skill_selectors/"
-      - "./state_formatters/:/dp-agent/state_formatters/"
-      - "./skills/:/dp-agent/skills/"
-      - "./common/:/dp-agent/common/"
+      - ".:/dp-agent"
     environment:
       WAIT_HOSTS: "spelling-preprocessing:8074, harvesters-maintenance-gobot-skill:3882,
         rule-based-response-selector:8005, emotion-classification-deepy:8015, dff-program-y-skill:8008, entity-linking-deepy:8075,

diff --git a/assistant_dists/deepy_adv/pipeline_conf.json b/assistant_dists/deepy_adv/pipeline_conf.json
@@ -34,8 +34,8 @@
                     "protocol": "http",
                     "url": "http://entity-linking-deepy:8075/model"
                 },
-                "dialog_formatter": "dp_formatters:last_utt_dialog",
-                "response_formatter": "dp_formatters:entity_linking_formatter",
+                "dialog_formatter": "state_formatters.dp_formatters:last_utt_dialog",
+                "response_formatter": "state_formatters.dp_formatters:entity_linking_formatter",
                 "state_manager_method": "add_annotation"
             },
             "sentseg": {
@@ -44,8 +44,8 @@
                     "timeout": 1.5,
                     "url": "http://sentseg:8011/sentseg"
                 },
-                "dialog_formatter": "dp_formatters:last_utt_dialog",
-                "response_formatter": "dp_formatters:simple_formatter_service",
+                "dialog_formatter": "state_formatters.dp_formatters:last_utt_dialog",
+                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
                 "state_manager_method": "add_annotation"
             },
             "intent_catcher": {
@@ -69,8 +69,8 @@
                     "protocol": "http",
                     "url": "http://harvesters-maintenance-gobot-skill:3882/respond"
                 },
-                "dialog_formatter": "dp_formatters:full_dialog",
-                "response_formatter": "dp_formatters:base_skill_formatter",
+                "dialog_formatter": "state_formatters.dp_formatters:full_dialog",
+                "response_formatter": "state_formatters.dp_formatters:base_skill_formatter",
                 "state_manager_method": "add_hypothesis",
                 "previous_services": ["annotators"]
             },
@@ -92,11 +92,11 @@
             "emotion_classification": {
                 "connector": {
                     "protocol": "python",
-                    "class_name": "connectors:BatchConnector",
+                    "class_name": "assistant_dists.deepy_adv.connectors:BatchConnector",
                     "url": "http://emotion-classification-deepy:8015/model"
                 },
-              "dialog_formatter": "dp_formatters:hypotheses_list",
-              "response_formatter": "dp_formatters:simple_formatter_service",
+              "dialog_formatter": "state_formatters.dp_formatters:hypotheses_list",
+              "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
               "previous_services": ["skills"],
               "state_manager_method": "add_hypothesis_annotation_batch"
             }
@@ -107,8 +107,8 @@
                     "protocol": "http",
                     "url": "http://rule-based-response-selector:8005/respond"
                 },
-                "dialog_formatter": "dp_formatters:full_dialog",
-                "response_formatter": "dp_formatters:base_response_selector_formatter_service",
+                "dialog_formatter": "state_formatters.dp_formatters:full_dialog",
+                "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service",
                 "previous_services": ["response_annotators"],
                 "state_manager_method": "add_bot_utterance"
             }

diff --git a/assistant_dists/deepy_base/docker-compose.override.yml b/assistant_dists/deepy_base/docker-compose.override.yml
@@ -1,11 +1,8 @@
 services:
   agent:
+    command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/deepy_base/pipeline_conf.json'
     volumes:
-      - "./assistant_dists/deepy_base:/dp-agent"
-      - "./skill_selectors/:/dp-agent/skill_selectors/"
-      - "./state_formatters/:/dp-agent/state_formatters/"
-      - "./skills/:/dp-agent/skills/"
-      - "./common/:/dp-agent/common/"
+      - ".:/dp-agent"
     environment:
       WAIT_HOSTS: "spelling-preprocessing:8074, harvesters-maintenance-skill:3662, rule-based-response-selector:8005,
         emotion-classification-deepy:8015, dff-program-y-skill:8008"

diff --git a/assistant_dists/deepy_base/pipeline_conf.json b/assistant_dists/deepy_base/pipeline_conf.json
@@ -36,8 +36,8 @@
                     "protocol": "http",
                     "url": "http://harvesters-maintenance-skill:3662/respond"
                 },
-                "dialog_formatter": "dp_formatters:full_dialog",
-                "response_formatter": "dp_formatters:base_skill_formatter",
+                "dialog_formatter": "state_formatters.dp_formatters:full_dialog",
+                "response_formatter": "state_formatters.dp_formatters:base_skill_formatter",
                 "state_manager_method": "add_hypothesis",
                 "previous_services": ["annotators"]
             },
@@ -59,11 +59,11 @@
             "emotion_classification": {
                 "connector": {
                     "protocol": "python",
-                    "class_name": "connectors:BatchConnector",
+                    "class_name": "assistant_dists.deepy_base.connectors:BatchConnector",
                     "url": "http://emotion-classification-deepy:8015/model"
                 },
-              "dialog_formatter": "dp_formatters:hypotheses_list",
-              "response_formatter": "dp_formatters:simple_formatter_service",
+              "dialog_formatter": "state_formatters.dp_formatters:hypotheses_list",
+              "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
               "previous_services": ["skills"],
               "state_manager_method": "add_hypothesis_annotation_batch"
             }
@@ -74,8 +74,8 @@
                     "protocol": "http",
                     "url": "http://rule-based-response-selector:8005/respond"
                 },
-                "dialog_formatter": "dp_formatters:full_dialog",
-                "response_formatter": "dp_formatters:base_response_selector_formatter_service",
+                "dialog_formatter": "state_formatters.dp_formatters:full_dialog",
+                "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service",
                 "previous_services": ["response_annotators"],
                 "state_manager_method": "add_bot_utterance"
             }

diff --git a/assistant_dists/deepy_faq/docker-compose.override.yml b/assistant_dists/deepy_faq/docker-compose.override.yml
@@ -1,11 +1,8 @@
 services:
   agent:
+    command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/deepy_faq/pipeline_conf.json'
     volumes:
-      - "./assistant_dists/deepy_faq:/dp-agent"
-      - "./skill_selectors/:/dp-agent/skill_selectors/"
-      - "./state_formatters/:/dp-agent/state_formatters/"
-      - "./skills/:/dp-agent/skills/"
-      - "./common/:/dp-agent/common/"
+      - ".:/dp-agent"
     environment:
       WAIT_HOSTS: "spelling-preprocessing:8074, faq-skill:3772, rule-based-response-selector:8005,
         emotion-classification-deepy:8015, dff-program-y-skill:8008"

diff --git a/assistant_dists/deepy_faq/pipeline_conf.json b/assistant_dists/deepy_faq/pipeline_conf.json
@@ -36,8 +36,8 @@
                     "protocol": "http",
                     "url": "http://faq-skill:3772/respond"
                 },
-                "dialog_formatter": "dp_formatters:full_dialog",
-                "response_formatter": "dp_formatters:base_skill_formatter",
+                "dialog_formatter": "state_formatters.dp_formatters:full_dialog",
+                "response_formatter": "state_formatters.dp_formatters:base_skill_formatter",
                 "state_manager_method": "add_hypothesis",
                 "previous_services": ["annotators"]
             },
@@ -59,11 +59,11 @@
             "emotion_classification": {
                 "connector": {
                     "protocol": "python",
-                    "class_name": "connectors:BatchConnector",
+                    "class_name": "assistant_dists.deepy_faq.connectors:BatchConnector",
                     "url": "http://emotion-classification-deepy:8015/model"
                 },
-              "dialog_formatter": "dp_formatters:hypotheses_list",
-              "response_formatter": "dp_formatters:simple_formatter_service",
+              "dialog_formatter": "state_formatters.dp_formatters:hypotheses_list",
+              "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
               "previous_services": ["skills"],
               "state_manager_method": "add_hypothesis_annotation_batch"
             }
@@ -74,8 +74,8 @@
                     "protocol": "http",
                     "url": "http://rule-based-response-selector:8005/respond"
                 },
-                "dialog_formatter": "dp_formatters:full_dialog",
-                "response_formatter": "dp_formatters:base_response_selector_formatter_service",
+                "dialog_formatter": "state_formatters.dp_formatters:full_dialog",
+                "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service",
                 "previous_services": ["response_annotators"],
                 "state_manager_method": "add_bot_utterance"
             }

diff --git a/assistant_dists/deepy_gobot_base/docker-compose.override.yml b/assistant_dists/deepy_gobot_base/docker-compose.override.yml
@@ -1,11 +1,8 @@
 services:
   agent:
+    command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/deepy_gobot_base/pipeline_conf.json'
     volumes:
-      - "./assistant_dists/deepy_gobot_base:/dp-agent"
-      - "./skill_selectors/:/dp-agent/skill_selectors/"
-      - "./state_formatters/:/dp-agent/state_formatters/"
-      - "./skills/:/dp-agent/skills/"
-      - "./common/:/dp-agent/common/"
+      - ".:/dp-agent"
     environment:
       WAIT_HOSTS: "spelling-preprocessing:8074, harvesters-maintenance-gobot-skill:3882,
         rule-based-response-selector:8005, emotion-classification-deepy:8015, dff-program-y-skill:8008"

diff --git a/assistant_dists/deepy_gobot_base/pipeline_conf.json b/assistant_dists/deepy_gobot_base/pipeline_conf.json
@@ -36,8 +36,8 @@
                     "protocol": "http",
                     "url": "http://harvesters-maintenance-gobot-skill:3882/respond"
                 },
-                "dialog_formatter": "dp_formatters:full_dialog",
-                "response_formatter": "dp_formatters:base_skill_formatter",
+                "dialog_formatter": "state_formatters.dp_formatters:full_dialog",
+                "response_formatter": "state_formatters.dp_formatters:base_skill_formatter",
                 "state_manager_method": "add_hypothesis",
                 "previous_services": ["annotators"]
             },
@@ -59,11 +59,11 @@
             "emotion_classification": {
                 "connector": {
                     "protocol": "python",
-                    "class_name": "connectors:BatchConnector",
+                    "class_name": "assistant_dists.deepy_gobot_base.connectors:BatchConnector",
                     "url": "http://emotion-classification-deepy:8015/model"
                 },
-              "dialog_formatter": "dp_formatters:hypotheses_list",
-              "response_formatter": "dp_formatters:simple_formatter_service",
+              "dialog_formatter": "state_formatters.dp_formatters:hypotheses_list",
+              "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
               "previous_services": ["skills"],
               "state_manager_method": "add_hypothesis_annotation_batch"
             }
@@ -74,8 +74,8 @@
                     "protocol": "http",
                     "url": "http://rule-based-response-selector:8005/respond"
                 },
-                "dialog_formatter": "dp_formatters:full_dialog",
-                "response_formatter": "dp_formatters:base_response_selector_formatter_service",
+                "dialog_formatter": "state_formatters.dp_formatters:full_dialog",
+                "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service",
                 "previous_services": ["response_annotators"],
                 "state_manager_method": "add_bot_utterance"
             }

diff --git a/services/transformers_lm/Dockerfile b/services/transformers_lm/Dockerfile
@@ -10,6 +10,8 @@ ARG CONFIG_NAME
 ENV CONFIG_NAME ${CONFIG_NAME}
 ARG HALF_PRECISION
 ENV HALF_PRECISION ${HALF_PRECISION}
+ARG MAX_LEN_GEN_TEXT
+ENV MAX_LEN_GEN_TEXT ${MAX_LEN_GEN_TEXT}
 
 COPY ./services/transformers_lm/requirements.txt /src/requirements.txt
 RUN pip install -r /src/requirements.txt

diff --git a/services/transformers_lm/server.py b/services/transformers_lm/server.py
@@ -17,13 +17,18 @@
 
 PRETRAINED_MODEL_NAME_OR_PATH = os.environ.get("PRETRAINED_MODEL_NAME_OR_PATH")
 CONFIG_NAME = os.environ.get("CONFIG_NAME")
-HALF_PRECISION = bool(os.environ.get("HALF_PRECISION", 0))
+HALF_PRECISION = os.environ.get("HALF_PRECISION", 0)
+HALF_PRECISION = 0 if HALF_PRECISION is None else bool(int(HALF_PRECISION))
 logging.info(f"PRETRAINED_MODEL_NAME_OR_PATH = {PRETRAINED_MODEL_NAME_OR_PATH}")
 NAMING = ["AI", "Human"]
+MAX_LEN_GEN_TEXT = os.environ.get("MAX_LEN_GEN_TEXT", 0)
 
 with open(CONFIG_NAME, "r") as f:
     generation_params = json.load(f)
-max_length = generation_params.get("max_length", 50)
+if not MAX_LEN_GEN_TEXT:
+    max_length = generation_params.get("max_length", 50)
+else:
+    max_length = int(MAX_LEN_GEN_TEXT)
 del generation_params["max_length"]
 
 app = Flask(__name__)

diff --git a/skills/harvesters_maintenance_gobot_skill_deepy/Dockerfile b/skills/harvesters_maintenance_gobot_skill_deepy/Dockerfile
@@ -2,8 +2,12 @@ FROM python:3.7.4
 
 RUN mkdir /src
 
+RUN rm -r /etc/apt/sources.list.d && \
+    apt update && apt install -y libsndfile-dev
+
 COPY ./requirements.txt /src/requirements.txt
-RUN pip install -r /src/requirements.txt
+RUN pip install --upgrade pip && pip install -r /src/requirements.txt
+RUN pip install pyopenssl --upgrade
 RUN python -m deeppavlov install gobot_md_yaml_minimal && \
     python -m deeppavlov download gobot_md_yaml_minimal
 

diff --git a/skills/harvesters_maintenance_gobot_skill_deepy/requirements.txt b/skills/harvesters_maintenance_gobot_skill_deepy/requirements.txt
@@ -2,9 +2,8 @@ flask==1.1.1
 itsdangerous==2.0.1
 gunicorn==19.9.0
 requests==2.22.0
-numpy==1.17.2
 sentry-sdk==0.12.3
 ruamel.yaml==0.15.100
-deeppavlov==0.13.0
+deeppavlov==0.14.0
 jinja2<=3.0.3
 Werkzeug<=2.0.3
diff --git a/state_formatters/dp_formatters.py b/state_formatters/dp_formatters.py
@@ -497,6 +497,10 @@ def persona_bot_formatter(dialog: Dict):
     ]
 
 
+def full_dialog(dialog: Dict):
+    return [{"dialogs": [dialog]}]
+
+
 def full_history_dialog(dialog: Dict):
     """
     Used ONLY by: response selector
@@ -525,6 +529,10 @@ def utt_sentrewrite_modified_last_dialog_emotion_skill(dialog: Dict):
     return [{"dialogs": [dialog]}]
 
 
+def base_skill_formatter(payload: Dict):
+    return [{"text": payload[0], "confidence": payload[1]}]
+
+
 def skill_with_attributes_formatter_service(payload: List):
     """
     Formatter should use `"state_manager_method": "add_hypothesis"` in config!!!