diff --git a/annotators/entity_linking_deepy/Dockerfile b/annotators/entity_linking_deepy/Dockerfile index 651e9b7813..46bd4768d2 100644 --- a/annotators/entity_linking_deepy/Dockerfile +++ b/annotators/entity_linking_deepy/Dockerfile @@ -1,5 +1,4 @@ FROM deeppavlov/base-gpu:0.12.1 -RUN pip install --upgrade pip && pip install git+https://github.com/deeppavlov/DeepPavlov.git@0.12.1 ARG CONFIG ARG COMMIT @@ -9,7 +8,11 @@ ARG SRC_DIR ENV CONFIG=$CONFIG ENV PORT=$PORT -RUN pip install sentry-sdk==0.13.0 +RUN pip install pybind11==2.2.4 +RUN pip install hdt==2.3 + +COPY ./annotators/entity_linking_deepy/requirements.txt /src/requirements.txt +RUN pip install --upgrade pip && pip install -r /src/requirements.txt RUN cd DeepPavlov && \ git config --global user.email "you@example.com" && \ diff --git a/annotators/entity_linking_deepy/requirements.txt b/annotators/entity_linking_deepy/requirements.txt index cfc8cfb2ca..aa4798d7c0 100644 --- a/annotators/entity_linking_deepy/requirements.txt +++ b/annotators/entity_linking_deepy/requirements.txt @@ -1,5 +1,7 @@ aiohttp jinja2<=3.0.3 Werkzeug<=2.0.3 +git+https://github.com/deeppavlov/DeepPavlov.git@0.12.1 git+https://github.com/deeppavlov/bert.git@feat/multi_gpu -tensorflow==1.15.5 \ No newline at end of file +sentry-sdk==0.13.0 +rapidfuzz==0.7.6 diff --git a/assistant_dists/deepy_adv/docker-compose.override.yml b/assistant_dists/deepy_adv/docker-compose.override.yml index b2cc82dbf5..05052bfee8 100644 --- a/assistant_dists/deepy_adv/docker-compose.override.yml +++ b/assistant_dists/deepy_adv/docker-compose.override.yml @@ -1,11 +1,8 @@ services: agent: + command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/deepy_adv/pipeline_conf.json' volumes: - - "./assistant_dists/deepy_adv:/dp-agent" - - "./skill_selectors/:/dp-agent/skill_selectors/" - - "./state_formatters/:/dp-agent/state_formatters/" - - "./skills/:/dp-agent/skills/" - - "./common/:/dp-agent/common/" + - ".:/dp-agent" environment: WAIT_HOSTS: "spelling-preprocessing:8074, harvesters-maintenance-gobot-skill:3882, rule-based-response-selector:8005, emotion-classification-deepy:8015, dff-program-y-skill:8008, entity-linking-deepy:8075, diff --git a/assistant_dists/deepy_adv/pipeline_conf.json b/assistant_dists/deepy_adv/pipeline_conf.json index 67e889654d..be417441c5 100644 --- a/assistant_dists/deepy_adv/pipeline_conf.json +++ b/assistant_dists/deepy_adv/pipeline_conf.json @@ -34,8 +34,8 @@ "protocol": "http", "url": "http://entity-linking-deepy:8075/model" }, - "dialog_formatter": "dp_formatters:last_utt_dialog", - "response_formatter": "dp_formatters:entity_linking_formatter", + "dialog_formatter": "state_formatters.dp_formatters:last_utt_dialog", + "response_formatter": "state_formatters.dp_formatters:entity_linking_formatter", "state_manager_method": "add_annotation" }, "sentseg": { @@ -44,8 +44,8 @@ "timeout": 1.5, "url": "http://sentseg:8011/sentseg" }, - "dialog_formatter": "dp_formatters:last_utt_dialog", - "response_formatter": "dp_formatters:simple_formatter_service", + "dialog_formatter": "state_formatters.dp_formatters:last_utt_dialog", + "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", "state_manager_method": "add_annotation" }, "intent_catcher": { @@ -69,8 +69,8 @@ "protocol": "http", "url": "http://harvesters-maintenance-gobot-skill:3882/respond" }, - "dialog_formatter": "dp_formatters:full_dialog", - "response_formatter": "dp_formatters:base_skill_formatter", + "dialog_formatter": "state_formatters.dp_formatters:full_dialog", + "response_formatter": "state_formatters.dp_formatters:base_skill_formatter", "state_manager_method": "add_hypothesis", "previous_services": ["annotators"] }, @@ -92,11 +92,11 @@ "emotion_classification": { "connector": { "protocol": "python", - "class_name": "connectors:BatchConnector", + "class_name": "assistant_dists.deepy_adv.connectors:BatchConnector", "url": "http://emotion-classification-deepy:8015/model" }, - "dialog_formatter": "dp_formatters:hypotheses_list", - "response_formatter": "dp_formatters:simple_formatter_service", + "dialog_formatter": "state_formatters.dp_formatters:hypotheses_list", + "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", "previous_services": ["skills"], "state_manager_method": "add_hypothesis_annotation_batch" } @@ -107,8 +107,8 @@ "protocol": "http", "url": "http://rule-based-response-selector:8005/respond" }, - "dialog_formatter": "dp_formatters:full_dialog", - "response_formatter": "dp_formatters:base_response_selector_formatter_service", + "dialog_formatter": "state_formatters.dp_formatters:full_dialog", + "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": ["response_annotators"], "state_manager_method": "add_bot_utterance" } diff --git a/assistant_dists/deepy_base/docker-compose.override.yml b/assistant_dists/deepy_base/docker-compose.override.yml index 2fdddba7df..6b557abae1 100644 --- a/assistant_dists/deepy_base/docker-compose.override.yml +++ b/assistant_dists/deepy_base/docker-compose.override.yml @@ -1,11 +1,8 @@ services: agent: + command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/deepy_base/pipeline_conf.json' volumes: - - "./assistant_dists/deepy_base:/dp-agent" - - "./skill_selectors/:/dp-agent/skill_selectors/" - - "./state_formatters/:/dp-agent/state_formatters/" - - "./skills/:/dp-agent/skills/" - - "./common/:/dp-agent/common/" + - ".:/dp-agent" environment: WAIT_HOSTS: "spelling-preprocessing:8074, harvesters-maintenance-skill:3662, rule-based-response-selector:8005, emotion-classification-deepy:8015, dff-program-y-skill:8008" diff --git a/assistant_dists/deepy_base/pipeline_conf.json b/assistant_dists/deepy_base/pipeline_conf.json index 794f755bdc..7c405b7453 100644 --- a/assistant_dists/deepy_base/pipeline_conf.json +++ b/assistant_dists/deepy_base/pipeline_conf.json @@ -36,8 +36,8 @@ "protocol": "http", "url": "http://harvesters-maintenance-skill:3662/respond" }, - "dialog_formatter": "dp_formatters:full_dialog", - "response_formatter": "dp_formatters:base_skill_formatter", + "dialog_formatter": "state_formatters.dp_formatters:full_dialog", + "response_formatter": "state_formatters.dp_formatters:base_skill_formatter", "state_manager_method": "add_hypothesis", "previous_services": ["annotators"] }, @@ -59,11 +59,11 @@ "emotion_classification": { "connector": { "protocol": "python", - "class_name": "connectors:BatchConnector", + "class_name": "assistant_dists.deepy_base.connectors:BatchConnector", "url": "http://emotion-classification-deepy:8015/model" }, - "dialog_formatter": "dp_formatters:hypotheses_list", - "response_formatter": "dp_formatters:simple_formatter_service", + "dialog_formatter": "state_formatters.dp_formatters:hypotheses_list", + "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", "previous_services": ["skills"], "state_manager_method": "add_hypothesis_annotation_batch" } @@ -74,8 +74,8 @@ "protocol": "http", "url": "http://rule-based-response-selector:8005/respond" }, - "dialog_formatter": "dp_formatters:full_dialog", - "response_formatter": "dp_formatters:base_response_selector_formatter_service", + "dialog_formatter": "state_formatters.dp_formatters:full_dialog", + "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": ["response_annotators"], "state_manager_method": "add_bot_utterance" } diff --git a/assistant_dists/deepy_faq/docker-compose.override.yml b/assistant_dists/deepy_faq/docker-compose.override.yml index a1b37bf797..d49bebe53d 100644 --- a/assistant_dists/deepy_faq/docker-compose.override.yml +++ b/assistant_dists/deepy_faq/docker-compose.override.yml @@ -1,11 +1,8 @@ services: agent: + command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/deepy_faq/pipeline_conf.json' volumes: - - "./assistant_dists/deepy_faq:/dp-agent" - - "./skill_selectors/:/dp-agent/skill_selectors/" - - "./state_formatters/:/dp-agent/state_formatters/" - - "./skills/:/dp-agent/skills/" - - "./common/:/dp-agent/common/" + - ".:/dp-agent" environment: WAIT_HOSTS: "spelling-preprocessing:8074, faq-skill:3772, rule-based-response-selector:8005, emotion-classification-deepy:8015, dff-program-y-skill:8008" diff --git a/assistant_dists/deepy_faq/pipeline_conf.json b/assistant_dists/deepy_faq/pipeline_conf.json index 2dab76d415..c0331aa39f 100644 --- a/assistant_dists/deepy_faq/pipeline_conf.json +++ b/assistant_dists/deepy_faq/pipeline_conf.json @@ -36,8 +36,8 @@ "protocol": "http", "url": "http://faq-skill:3772/respond" }, - "dialog_formatter": "dp_formatters:full_dialog", - "response_formatter": "dp_formatters:base_skill_formatter", + "dialog_formatter": "state_formatters.dp_formatters:full_dialog", + "response_formatter": "state_formatters.dp_formatters:base_skill_formatter", "state_manager_method": "add_hypothesis", "previous_services": ["annotators"] }, @@ -59,11 +59,11 @@ "emotion_classification": { "connector": { "protocol": "python", - "class_name": "connectors:BatchConnector", + "class_name": "assistant_dists.deepy_faq.connectors:BatchConnector", "url": "http://emotion-classification-deepy:8015/model" }, - "dialog_formatter": "dp_formatters:hypotheses_list", - "response_formatter": "dp_formatters:simple_formatter_service", + "dialog_formatter": "state_formatters.dp_formatters:hypotheses_list", + "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", "previous_services": ["skills"], "state_manager_method": "add_hypothesis_annotation_batch" } @@ -74,8 +74,8 @@ "protocol": "http", "url": "http://rule-based-response-selector:8005/respond" }, - "dialog_formatter": "dp_formatters:full_dialog", - "response_formatter": "dp_formatters:base_response_selector_formatter_service", + "dialog_formatter": "state_formatters.dp_formatters:full_dialog", + "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": ["response_annotators"], "state_manager_method": "add_bot_utterance" } diff --git a/assistant_dists/deepy_gobot_base/docker-compose.override.yml b/assistant_dists/deepy_gobot_base/docker-compose.override.yml index ccab28b6d1..516a0fadd3 100644 --- a/assistant_dists/deepy_gobot_base/docker-compose.override.yml +++ b/assistant_dists/deepy_gobot_base/docker-compose.override.yml @@ -1,11 +1,8 @@ services: agent: + command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/deepy_gobot_base/pipeline_conf.json' volumes: - - "./assistant_dists/deepy_gobot_base:/dp-agent" - - "./skill_selectors/:/dp-agent/skill_selectors/" - - "./state_formatters/:/dp-agent/state_formatters/" - - "./skills/:/dp-agent/skills/" - - "./common/:/dp-agent/common/" + - ".:/dp-agent" environment: WAIT_HOSTS: "spelling-preprocessing:8074, harvesters-maintenance-gobot-skill:3882, rule-based-response-selector:8005, emotion-classification-deepy:8015, dff-program-y-skill:8008" diff --git a/assistant_dists/deepy_gobot_base/pipeline_conf.json b/assistant_dists/deepy_gobot_base/pipeline_conf.json index 398f66a519..1840d5ab4a 100644 --- a/assistant_dists/deepy_gobot_base/pipeline_conf.json +++ b/assistant_dists/deepy_gobot_base/pipeline_conf.json @@ -36,8 +36,8 @@ "protocol": "http", "url": "http://harvesters-maintenance-gobot-skill:3882/respond" }, - "dialog_formatter": "dp_formatters:full_dialog", - "response_formatter": "dp_formatters:base_skill_formatter", + "dialog_formatter": "state_formatters.dp_formatters:full_dialog", + "response_formatter": "state_formatters.dp_formatters:base_skill_formatter", "state_manager_method": "add_hypothesis", "previous_services": ["annotators"] }, @@ -59,11 +59,11 @@ "emotion_classification": { "connector": { "protocol": "python", - "class_name": "connectors:BatchConnector", + "class_name": "assistant_dists.deepy_gobot_base.connectors:BatchConnector", "url": "http://emotion-classification-deepy:8015/model" }, - "dialog_formatter": "dp_formatters:hypotheses_list", - "response_formatter": "dp_formatters:simple_formatter_service", + "dialog_formatter": "state_formatters.dp_formatters:hypotheses_list", + "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", "previous_services": ["skills"], "state_manager_method": "add_hypothesis_annotation_batch" } @@ -74,8 +74,8 @@ "protocol": "http", "url": "http://rule-based-response-selector:8005/respond" }, - "dialog_formatter": "dp_formatters:full_dialog", - "response_formatter": "dp_formatters:base_response_selector_formatter_service", + "dialog_formatter": "state_formatters.dp_formatters:full_dialog", + "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": ["response_annotators"], "state_manager_method": "add_bot_utterance" } diff --git a/services/transformers_lm/Dockerfile b/services/transformers_lm/Dockerfile index ac9aec7346..34793d5f38 100644 --- a/services/transformers_lm/Dockerfile +++ b/services/transformers_lm/Dockerfile @@ -10,6 +10,8 @@ ARG CONFIG_NAME ENV CONFIG_NAME ${CONFIG_NAME} ARG HALF_PRECISION ENV HALF_PRECISION ${HALF_PRECISION} +ARG MAX_LEN_GEN_TEXT +ENV MAX_LEN_GEN_TEXT ${MAX_LEN_GEN_TEXT} COPY ./services/transformers_lm/requirements.txt /src/requirements.txt RUN pip install -r /src/requirements.txt diff --git a/services/transformers_lm/server.py b/services/transformers_lm/server.py index 1422cde62a..fc4a7d4eb6 100644 --- a/services/transformers_lm/server.py +++ b/services/transformers_lm/server.py @@ -17,13 +17,18 @@ PRETRAINED_MODEL_NAME_OR_PATH = os.environ.get("PRETRAINED_MODEL_NAME_OR_PATH") CONFIG_NAME = os.environ.get("CONFIG_NAME") -HALF_PRECISION = bool(os.environ.get("HALF_PRECISION", 0)) +HALF_PRECISION = os.environ.get("HALF_PRECISION", 0) +HALF_PRECISION = 0 if HALF_PRECISION is None else bool(int(HALF_PRECISION)) logging.info(f"PRETRAINED_MODEL_NAME_OR_PATH = {PRETRAINED_MODEL_NAME_OR_PATH}") NAMING = ["AI", "Human"] +MAX_LEN_GEN_TEXT = os.environ.get("MAX_LEN_GEN_TEXT", 0) with open(CONFIG_NAME, "r") as f: generation_params = json.load(f) -max_length = generation_params.get("max_length", 50) +if not MAX_LEN_GEN_TEXT: + max_length = generation_params.get("max_length", 50) +else: + max_length = int(MAX_LEN_GEN_TEXT) del generation_params["max_length"] app = Flask(__name__) diff --git a/skills/harvesters_maintenance_gobot_skill_deepy/Dockerfile b/skills/harvesters_maintenance_gobot_skill_deepy/Dockerfile index 57b6b67eab..ae826f873a 100644 --- a/skills/harvesters_maintenance_gobot_skill_deepy/Dockerfile +++ b/skills/harvesters_maintenance_gobot_skill_deepy/Dockerfile @@ -2,8 +2,12 @@ FROM python:3.7.4 RUN mkdir /src +RUN rm -r /etc/apt/sources.list.d && \ + apt update && apt install -y libsndfile-dev + COPY ./requirements.txt /src/requirements.txt -RUN pip install -r /src/requirements.txt +RUN pip install --upgrade pip && pip install -r /src/requirements.txt +RUN pip install pyopenssl --upgrade RUN python -m deeppavlov install gobot_md_yaml_minimal && \ python -m deeppavlov download gobot_md_yaml_minimal diff --git a/skills/harvesters_maintenance_gobot_skill_deepy/requirements.txt b/skills/harvesters_maintenance_gobot_skill_deepy/requirements.txt index badfb037b0..1b471d563c 100644 --- a/skills/harvesters_maintenance_gobot_skill_deepy/requirements.txt +++ b/skills/harvesters_maintenance_gobot_skill_deepy/requirements.txt @@ -2,9 +2,8 @@ flask==1.1.1 itsdangerous==2.0.1 gunicorn==19.9.0 requests==2.22.0 -numpy==1.17.2 sentry-sdk==0.12.3 ruamel.yaml==0.15.100 -deeppavlov==0.13.0 +deeppavlov==0.14.0 jinja2<=3.0.3 Werkzeug<=2.0.3 diff --git a/state_formatters/dp_formatters.py b/state_formatters/dp_formatters.py index d2fcb946f2..889c1cd3f7 100755 --- a/state_formatters/dp_formatters.py +++ b/state_formatters/dp_formatters.py @@ -497,6 +497,10 @@ def persona_bot_formatter(dialog: Dict): ] +def full_dialog(dialog: Dict): + return [{"dialogs": [dialog]}] + + def full_history_dialog(dialog: Dict): """ Used ONLY by: response selector @@ -525,6 +529,10 @@ def utt_sentrewrite_modified_last_dialog_emotion_skill(dialog: Dict): return [{"dialogs": [dialog]}] +def base_skill_formatter(payload: Dict): + return [{"text": payload[0], "confidence": payload[1]}] + + def skill_with_attributes_formatter_service(payload: List): """ Formatter should use `"state_manager_method": "add_hypothesis"` in config!!!