Merge pull request #340 from deeppavlov/dev

Release v0.16.0

.gitignore

@@ -142,3 +142,6 @@ kubernetes/models
 docker-compose-one-replica.yml
 
 *.pem
+
+# personal env keys and tokens
+.env_secret

README_multilingual.md

@@ -17,9 +17,9 @@ Dream Architecture is presented in the following image:
 | Sentence Ranker          | 2.5 GB RAM, 1.8 GB GPU   | for a pair of sentences predicts a floating point value. For multilingual version, return cosine similarity between embeddings from multilingual sentence BERT |
 
 ## Skills & Services
-| Name                | Requirements          | Description                                                                                                                                       |
-|---------------------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|
-| GPT-2 Multilingual  | 5 GB RAM, 6.5 GB GPU  | GPT2-based generative model. For Multilingual distribution we propose mgpt by Sberbank [from HugginFace](https://huggingface.co/sberbank-ai/mGPT) |
+| Name               | Requirements          | Description                                                                                                                                       |
+|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|
+| GPT-2 Multilingual | 5 GB RAM, 6.5 GB GPU  | GPT2-based generative model. For Multilingual distribution we propose mgpt by Sberbank [from HugginFace](https://huggingface.co/sberbank-ai/mGPT) |
 
 # Papers
 

README_ru.md

@@ -214,6 +214,7 @@ docker-compose -f docker-compose.yml -f assistant_dists/dream/docker-compose.ove
 | DFF Intent Responder | 50 MB RAM                | provides template-based replies for some of the intents detected by Intent Catcher annotator                                        |
 | DFF Program Y Skill  | 80 MB RAM                | **[New DFF version]** Chatbot Program Y (https://github.com/keiffster/program-y) adapted for Dream socialbot                        |
 | DFF Friendship Skill | 70 MB RAM                | **[New DFF version]** DFF-based skill to greet the user in the beginning of the dialog, and forward the user to some scripted skill |
+| DFF Template Skill   | 50 MB RAM                | **[New DFF version]** DFF-based skill that provides an example of DFF usage                                                         |
 | Text QA              | 3.8 GiB RAM, 5.2 GiB GPU | Навык для ответа на вопросы по тексту.                                                                                              |
 
 

annotators/ConversationEvaluator/Dockerfile

@@ -2,9 +2,9 @@ FROM deeppavlov/base-gpu:0.12.0
 RUN pip install --upgrade pip && pip install git+https://github.com/deeppavlov/[email protected]
 
 ARG CONFIG
-ARG PORT
+ARG SERVICE_PORT
 ENV CONFIG=$CONFIG
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 
 WORKDIR /src
 RUN mkdir common

annotators/NER_deeppavlov/Dockerfile

@@ -2,12 +2,12 @@ FROM deeppavlov/base-gpu:1.0.0rc1
 RUN pip install --upgrade pip && pip install git+https://github.com/deeppavlov/[email protected]
 
 ARG CONFIG
-ARG PORT
+ARG SERVICE_PORT
 ARG SRC_DIR
 ARG SED_ARG=" | "
 
 ENV CONFIG=$CONFIG
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 
 COPY ./annotators/NER_deeppavlov/requirements.txt /src/requirements.txt
 RUN pip install --upgrade pip && pip install -r /src/requirements.txt

annotators/combined_classification/Dockerfile

@@ -6,9 +6,9 @@ WORKDIR /base/DeepPavlov
 
 ARG CONFIG
 
-ARG PORT
+ARG SERVICE_PORT
 ENV CONFIG=$CONFIG
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 
 WORKDIR /src
 RUN mkdir common

annotators/combined_classification/load_test.sh

@@ -1,2 +1,2 @@
 pip install -r requirements_load_test.txt
-locust -f load_test.py --headless -u 10 -r 2 --host http://0.0.0.0:$PORT/model
+locust -f load_test.py --headless -u 10 -r 2 --host http://0.0.0.0:$SERVICE_PORT/model

annotators/dialog_breakdown/Dockerfile

@@ -2,11 +2,11 @@ FROM deeppavlov/base-gpu:0.12.0
 RUN pip install --upgrade pip && pip install git+https://github.com/deeppavlov/[email protected]
 
 ARG CONFIG
-ARG PORT
+ARG SERVICE_PORT
 ARG SED_ARG=" | "
 
 ENV CONFIG=$CONFIG
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 
 WORKDIR /src
 RUN mkdir common

annotators/entity_detection/Dockerfile

@@ -5,13 +5,13 @@ RUN apt-get update && apt-get install git -y
 ARG SEQ_TAG_CONFIG
 ARG CONFIG
 ARG FINEGRAINED
-ARG PORT
+ARG SERVICE_PORT
 ARG SRC_DIR
 
 ENV SEQ_TAG_CONFIG=$SEQ_TAG_CONFIG
 ENV CONFIG=$CONFIG
 ENV FINEGRAINED=$FINEGRAINED
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 
 COPY ./annotators/entity_detection/requirements.txt /src/requirements.txt
 RUN pip install -r /src/requirements.txt
@@ -23,4 +23,4 @@ COPY $SRC_DIR /src
 WORKDIR /src
 RUN python -m deeppavlov install $SEQ_TAG_CONFIG
 
-CMD gunicorn  --workers=1 --timeout 500 server:app -b 0.0.0.0:$PORT
+CMD gunicorn  --workers=1 --timeout 500 server:app -b 0.0.0.0:$SERVICE_PORT

annotators/entity_detection_rus/Dockerfile

@@ -2,15 +2,15 @@ FROM deeppavlov/base-gpu:0.12.1
 RUN pip install --upgrade pip && pip install git+https://github.com/deeppavlov/[email protected]
 
 ARG CONFIG
-ARG PORT
+ARG SERVICE_PORT
 ARG SRC_DIR
 ARG SED_ARG=" | "
 
 ARG LANGUAGE=EN
 ENV LANGUAGE ${LANGUAGE}
 
 ENV CONFIG=$CONFIG
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 
 COPY ./annotators/entity_detection_rus/requirements.txt /src/requirements.txt
 RUN pip install -r /src/requirements.txt

annotators/entity_linking/Dockerfile

@@ -19,12 +19,12 @@ ARG LANGUAGE=EN
 ENV LANGUAGE ${LANGUAGE}
 
 ARG CONFIG
-ARG PORT
+ARG SERVICE_PORT
 ARG SRC_DIR
 ARG SED_ARG=" | "
 
 ENV CONFIG=$CONFIG
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 
 COPY ./annotators/entity_linking/requirements.txt /src/requirements.txt
 RUN pip install -r /src/requirements.txt

annotators/entity_linking_deepy/Dockerfile

@@ -2,11 +2,11 @@ FROM deeppavlov/base-gpu:0.12.1
 
 ARG CONFIG
 ARG COMMIT
-ARG PORT
+ARG SERVICE_PORT
 ARG SRC_DIR
 
 ENV CONFIG=$CONFIG
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 
 RUN pip install pybind11==2.2.4
 RUN pip install hdt==2.3
@@ -25,4 +25,4 @@ COPY $SRC_DIR /src
 
 WORKDIR /src
 
-CMD python -m deeppavlov riseapi $CONFIG -p $PORT -d
+CMD python -m deeppavlov riseapi $CONFIG -p $SERVICE_PORT -d

annotators/entity_linking_rus/Dockerfile

@@ -17,12 +17,12 @@ ARG LANGUAGE=EN
 ENV LANGUAGE ${LANGUAGE}
 
 ARG CONFIG
-ARG PORT
+ARG SERVICE_PORT
 ARG SRC_DIR
 ARG SED_ARG=" | "
 
 ENV CONFIG=$CONFIG
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 
 RUN pip install pybind11==2.2.4
 RUN pip install hdt==2.3

annotators/fact_retrieval/Dockerfile

@@ -6,15 +6,15 @@ ARG CONFIG
 ARG CONFIG_WIKI
 ARG CONFIG_WHOW
 ARG COMMIT=0.13.0
-ARG PORT
+ARG SERVICE_PORT
 ARG SRC_DIR
 ARG SED_ARG=" | "
 ARG N_FACTS
 
 ENV CONFIG=$CONFIG
 ENV CONFIG_WIKI=$CONFIG_WIKI
 ENV CONFIG_WHOW=$CONFIG_WHOW
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 ENV N_FACTS=$N_FACTS
 
 COPY ./annotators/fact_retrieval/requirements.txt /src/requirements.txt

annotators/fact_retrieval/test_server.py

@@ -5,9 +5,9 @@
 import common.test_utils as test_utils
 
 
-PORT = int(os.getenv("PORT"))
+SERVICE_PORT = int(os.getenv("SERVICE_PORT"))
 RANDOM_SEED = int(os.getenv("RANDOM_SEED", 2718))
-URL = f"http://0.0.0.0:{PORT}/model"
+URL = f"http://0.0.0.0:{SERVICE_PORT}/model"
 
 
 def handler(requested_data, random_seed):

annotators/fact_retrieval_rus/Dockerfile

@@ -4,13 +4,13 @@ RUN apt-get update && apt-get install git -y
 
 ARG COMMIT=0.13.0
 ARG CONFIG
-ARG PORT
+ARG SERVICE_PORT
 ARG SRC_DIR
 ARG TOP_N
 
 ENV COMMIT=$COMMIT
 ENV CONFIG=$CONFIG
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 ENV TOP_N=$TOP_N
 
 COPY ./annotators/fact_retrieval_rus/requirements.txt /src/requirements.txt

annotators/kbqa/Dockerfile

@@ -3,13 +3,13 @@ RUN pip install --upgrade pip && pip install git+https://github.com/deeppavlov/D
 
 ARG CONFIG
 ARG COMMIT
-ARG PORT
+ARG SERVICE_PORT
 ARG SRC_DIR
 
 ARG SED_ARG=" | "
 
 ENV CONFIG=$CONFIG
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 ENV COMMIT=$COMMIT
 
 COPY ./annotators/kbqa/requirements.txt /src/requirements.txt

annotators/spelling_preprocessing_dp/Dockerfile

@@ -15,12 +15,12 @@ ENV LANGUAGE ${LANGUAGE}
 
 ARG CONFIG
 ARG COMMIT=0.13.0
-ARG PORT
+ARG SERVICE_PORT
 ARG SRC_DIR
 ARG SED_ARG=" | "
 
 ENV CONFIG=$CONFIG
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 
 COPY ./annotators/spelling_preprocessing_dp/requirements.txt /src/requirements.txt
 RUN pip install --upgrade pip && pip install -r /src/requirements.txt

annotators/spelling_preprocessing_ru/Dockerfile

@@ -15,12 +15,12 @@ ENV LANGUAGE ${LANGUAGE}
 
 ARG CONFIG
 ARG COMMIT=0.13.0
-ARG PORT
+ARG SERVICE_PORT
 ARG SRC_DIR
 ARG SED_ARG=" | "
 
 ENV CONFIG=$CONFIG
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 
 COPY ./annotators/spelling_preprocessing_dp/requirements.txt /src/requirements.txt
 RUN pip install --upgrade pip && pip install -r /src/requirements.txt

annotators/wiki_parser/Dockerfile

@@ -3,14 +3,14 @@ FROM python:3.9.1
 ARG CONFIG
 ARG COMMIT
 ARG FAST=1
-ARG PORT
+ARG SERVICE_PORT
 ARG SRC_DIR
 ARG LANGUAGE=EN
 ENV LANGUAGE ${LANGUAGE}
 
 ENV CONFIG=$CONFIG
 ENV FAST=$FAST
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 ENV COMMIT=$COMMIT
 ENV LANGUAGE=$LANGUAGE
 

assistant_dists/ai_faq_assistant/cpu.yml

@@ -0,0 +1,14 @@
+version: '3.7'
+services:
+  combined-classification:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""
+  sentence-ranker:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""
+  transformers-lm-gptj:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""

assistant_dists/ai_faq_assistant/db_conf.json

@@ -0,0 +1,6 @@
+{
+    "host": "DB_HOST",
+    "port": "DB_PORT",
+    "name": "DB_NAME",
+    "env": true
+}

assistant_dists/ai_faq_assistant/dev.yml

@@ -0,0 +1,62 @@
+# С такими volumes удобно дебажить, не нужно пересобирать контейнер каждый раз при изменении кода
+services:
+  agent:
+    volumes:
+      - ".:/dp-agent"
+    ports:
+      - 4242:4242
+  sentseg:
+    volumes:
+      - "./annotators/SentSeg:/src"
+    ports:
+      - 8011:8011
+  convers-evaluation-no-scripts-selector:
+    volumes:
+      - "./response_selectors/convers_evaluation_based_selector:/src"
+      - "./common:/src/common"
+    ports:
+      - 8009:8009
+  badlisted-words:
+    volumes:
+      - "./annotators/BadlistedWordsDetector:/src"
+      - "./common:/src/common"
+    ports:
+      - 8018:8018
+  spelling-preprocessing:
+    volumes:
+      - "./annotators/spelling_preprocessing:/src"
+    ports:
+      - 8074:8074
+  combined-classification:
+    volumes:
+      - "./common:/src/common"
+      - "./annotators/combined_classification:/src"
+    ports:
+      - 8087:8087
+  sentence-ranker:
+    volumes:
+      - "./services/sentence_ranker:/src"
+      - "~/.deeppavlov/cache:/root/.cache"
+    ports:
+      - 8128:8128
+  prompt-selector:
+    volumes:
+      - "./annotators/prompt_selector:/src"
+      - "./common:/src/common"
+    ports:
+      - 8135:8135
+  transformers-lm-gptj:
+    volumes:
+      - "./services/transformers_lm:/src"
+      - "./common:/src/common"
+      - "~/.deeppavlov/cache:/root/.cache"
+    ports:
+      - 8130:8130
+  dff-ai-faq-prompted-skill:
+    volumes:
+      - "./skills/dff_template_prompted_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8143:8143
+
+version: "3.7"