Fromage annotator and dff-fromage-image-skill

assistant_dists/dream_multimodal/cpu.yml

@@ -1,14 +1,14 @@
 version: '3.7'
 services:
-  dialogpt:
+  intent-catcher:
     environment:
       DEVICE: cpu
       CUDA_VISIBLE_DEVICES: ""
-  intent-catcher:
+  sentence-ranker:
     environment:
       DEVICE: cpu
       CUDA_VISIBLE_DEVICES: ""
-  sentence-ranker:
+  fromage:
     environment:
       DEVICE: cpu
       CUDA_VISIBLE_DEVICES: ""

assistant_dists/dream_multimodal/dev.yml

@@ -10,23 +10,17 @@ services:
       - 3000:3000
     volumes:
       - "~/.deeppavlov/file_server:/tmp"
-  dff-program-y-skill:
-    volumes:
-      - "./skills/dff_program_y_skill:/src"
-      - "./common:/src/common"
-    ports:
-      - 8008:8008
   sentseg:
     volumes:
       - "./annotators/SentSeg:/src"
     ports:
       - 8011:8011
-  convers-evaluation-selector:
+  dff-program-y-skill:
     volumes:
-      - "./response_selectors/convers_evaluation_based_selector:/src"
+      - "./skills/dff_program_y_skill:/src"
       - "./common:/src/common"
     ports:
-      - 8009:8009
+      - 8008:8008
   dff-intent-responder-skill:
     volumes:
       - "./skills/dff_intent_responder_skill:/src"
@@ -46,18 +40,6 @@ services:
       - "./common:/src/common"
     ports:
       - 8018:8018
-  spelling-preprocessing:
-    volumes:
-      - "./annotators/spelling_preprocessing:/src"
-    ports:
-      - 8074:8074
-  dialogpt:
-    volumes:
-      - "./services/dialogpt:/src"
-      - "./common:/src/common"
-      - "~/.deeppavlov/cache:/root/.cache"
-    ports:
-      - 8125:8125
   sentence-ranker:
     volumes:
       - "./services/sentence_ranker:/src"
@@ -76,4 +58,17 @@ services:
       - "./common:/src/common"
     ports:
       - 8124:8124
+  fromage:
+    volumes:
+      - "./services/fromage:/src"
+      - "./common:/src/common"
+      - "~/.deeppavlov/cache:/root/.cache"
+    ports:
+      - 8069:8069
+  dff-fromage-image-skill:
+    volumes:
+      - "./skills/dff_fromage_image_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8070:8070
 version: "3.7"

assistant_dists/dream_multimodal/docker-compose.override.yml

@@ -2,9 +2,10 @@ services:
   agent:
     command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_multimodal/pipeline_conf.json'
     environment:
-      WAIT_HOSTS: "dff-program-y-skill:8008, sentseg:8011, convers-evaluation-selector:8009, 
+      WAIT_HOSTS: "dff-program-y-skill:8008, sentseg:8011, ranking-based-response-selector:8002,
           dff-intent-responder-skill:8012, intent-catcher:8014, badlisted-words:8018,
-          spelling-preprocessing:8074, dialogpt:8125, sentence-ranker:8128, image-captioning:8123, dff-image-skill:8124"
+          sentence-ranker:8128, image-captioning:8123, dff-image-skill:8124,
+          fromage:8069, dff-fromage-image-skill:8070"
       WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1200}
       HIGH_PRIORITY_INTENTS: 1
       RESTRICTION_FOR_SENSITIVE_CASE: 1
@@ -15,26 +16,33 @@ services:
   files:
     image: julienmeerschart/simple-file-upload-download-server
 
-  dff-program-y-skill:
-    env_file: [.env]
+  ranking-based-response-selector:
+    env_file: [ .env ]
     build:
       args:
-        SERVICE_PORT: 8008
-        SERVICE_NAME: dff_program_y_skill
+        SERVICE_PORT: 8002
+        SERVICE_NAME: response_selector
         LANGUAGE: EN
+        SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker
+        SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond
+        SENTENCE_RANKER_TIMEOUT: 3
+        N_UTTERANCES_CONTEXT: 5
+        FILTER_TOXIC_OR_BADLISTED: 1
+        FALLBACK_FILE: fallbacks_dream_en.json
       context: .
-      dockerfile: ./skills/dff_program_y_skill/Dockerfile
-    command: gunicorn --workers=1 server:app -b 0.0.0.0:8008 --reload
+      dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile
+    command: flask run -h 0.0.0.0 -p 8002
+    environment:
+      - FLASK_APP=server
     deploy:
       resources:
         limits:
-          memory: 1024M
+          memory: 100M
         reservations:
-          memory: 1024M
-
+          memory: 100M
 
   sentseg:
-    env_file: [.env]
+    env_file: [ .env ]
     build:
       context: ./annotators/SentSeg/
     command: flask run -h 0.0.0.0 -p 8011
@@ -47,41 +55,22 @@ services:
         reservations:
           memory: 1.5G
 
-  convers-evaluation-selector:
+  dff-program-y-skill:
     env_file: [.env]
     build:
       args:
-        TAG_BASED_SELECTION: 1
-        CALL_BY_NAME_PROBABILITY: 0.5
-        PROMPT_PROBA: 0.1
-        ACKNOWLEDGEMENT_PROBA: 0.3
-        PRIORITIZE_WITH_REQUIRED_ACT: 0
-        PRIORITIZE_NO_DIALOG_BREAKDOWN: 0
-        PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0
-        IGNORE_DISLIKED_SKILLS: 0
-        GREETING_FIRST: 1
-        RESTRICTION_FOR_SENSITIVE_CASE: 1
-        PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 1
-        MAX_TURNS_WITHOUT_SCRIPTS: 7
-        ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 1
-        PRIORITIZE_SCRIPTED_SKILLS: 0
-        CONFIDENCE_STRENGTH: 0.8
-        CONV_EVAL_STRENGTH: 0.4
-        PRIORITIZE_HUMAN_INITIATIVE: 1
-        QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8
+        SERVICE_PORT: 8008
+        SERVICE_NAME: dff_program_y_skill
         LANGUAGE: EN
-        FALLBACK_FILE: fallbacks_dream_en.json
       context: .
-      dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile
-    command: flask run -h 0.0.0.0 -p 8009
-    environment:
-      - FLASK_APP=server
+      dockerfile: ./skills/dff_program_y_skill/Dockerfile
+    command: gunicorn --workers=1 server:app -b 0.0.0.0:8008 --reload
     deploy:
       resources:
         limits:
-          memory: 256M
+          memory: 1024M
         reservations:
-          memory: 256M
+          memory: 1024M
 
   dff-intent-responder-skill:
     env_file: [ .env ]
@@ -137,46 +126,6 @@ services:
         reservations:
           memory: 256M
 
-  spelling-preprocessing:
-    env_file: [ .env ]
-    build:
-      args:
-        SERVICE_PORT: 8074
-        SERVICE_NAME: spelling_preprocessing
-      context: ./annotators/spelling_preprocessing/
-    command: flask run -h 0.0.0.0 -p 8074
-    environment:
-      - FLASK_APP=server
-    deploy:
-      resources:
-        limits:
-          memory: 100M
-        reservations:
-          memory: 100M
-
-  dialogpt:
-    env_file: [ .env ]
-    build:
-      args:
-        SERVICE_PORT: 8125
-        SERVICE_NAME: dialogpt
-        PRETRAINED_MODEL_NAME_OR_PATH: microsoft/DialoGPT-medium
-        N_HYPOTHESES_TO_GENERATE: 5
-        CONFIG_NAME: dialogpt_en.json
-        MAX_HISTORY_DEPTH: 2
-      context: .
-      dockerfile: ./services/dialogpt/Dockerfile
-    command: flask run -h 0.0.0.0 -p 8125
-    environment:
-      - CUDA_VISIBLE_DEVICES=0
-      - FLASK_APP=server
-    deploy:
-      resources:
-        limits:
-          memory: 2G
-        reservations:
-          memory: 2G
-
   sentence-ranker:
     env_file: [ .env ]
     build:
@@ -230,5 +179,39 @@ services:
           memory: 1024M
         reservations:
           memory: 1024M
-
-version: '3.7'
+
+  fromage:
+    env_file: [ .env ]
+    build:
+      args:
+        SERVICE_PORT: 8069
+        SERVICE_NAME: fromage
+      context: .
+      dockerfile: ./services/fromage/Dockerfile
+    command: flask run -h 0.0.0.0 -p 8069
+    environment:
+      - CUDA_VISIBLE_DEVICES=0
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 5G
+        reservations:
+          memory: 5G
+
+  dff-fromage-image-skill:
+    env_file: [.env]
+    build:
+      args:
+        SERVICE_PORT: 8070
+        SERVICE_NAME: dff_fromage_image_skill
+      context: .
+      dockerfile: ./skills/dff_fromage_image_skill/Dockerfile
+    command: gunicorn --workers=1 server:app -b 0.0.0.0:8070 --reload
+    deploy:
+      resources:
+        limits:
+          memory: 1024M
+        reservations:
+          memory: 1024M
+version: '3.7'

assistant_dists/dream_multimodal/pipeline_conf.json

@@ -58,22 +58,6 @@
             }
         },
         "annotators": {
-            "spelling_preprocessing": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 1.0,
-                    "url": "http://spelling-preprocessing:8074/respond"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:last_utt_dialog",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [],
-                "state_manager_method": "add_annotation_and_reset_human_attributes_for_first_turn",
-                "is_enabled": true,
-                "source": {
-                    "component": "components/pGxj32ic41pvquRXUdqc7A.yml",
-                    "service": "annotators/spelling_preprocessing/service_configs/spelling-preprocessing"
-                }
-            },
             "sentseg": {
                 "connector": {
                     "protocol": "http",
@@ -143,6 +127,21 @@
                     "component": "components/f2hECzXDTTfljm5mweoMFA.yml",
                     "service": "services/image_captioning/service_configs/image-captioning"
                 }
+            },
+            "fromage": {
+                "connector": {
+                    "protocol": "http",
+                    "timeout": 90.0,
+                    "url": "http://fromage:8069/respond"
+                },
+                "dialog_formatter": "state_formatters.dp_formatters:fromage_formatter",
+                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
+                "state_manager_method": "add_annotation",
+                "is_enabled": true,
+                "source": {
+                    "component": "components/8iHHdjsnfhewkl.yml",
+                    "service": "services/fromage/service_configs/fromage"
+                }
             }
         },
         "response_annotators": {
@@ -222,10 +221,10 @@
             }
         },
         "skill_selectors": {
-            "rule_based_selector": {
+            "description_based_skill_selector": {
                 "connector": {
                     "protocol": "python",
-                    "class_name": "skill_selectors.rule_based_selector.connector:RuleBasedSkillSelectorConnector"
+                    "class_name": "skill_selectors.description_based_skill_selector.connector:DescriptionBasedSkillSelectorConnector"
                 },
                 "dialog_formatter": "state_formatters.dp_formatters:base_skill_selector_formatter_dialog",
                 "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
@@ -237,8 +236,8 @@
                 ],
                 "is_enabled": true,
                 "source": {
-                    "component": "components/xSwFvtAUdvtQosvzpb7oMg.yml",
-                    "service": "skill_selectors/rule_based_selector/service_configs/agent"
+                    "component": "components/dfsw4bji8bgjq2.yml",
+                    "service": "skill_selectors/description_based_skill_selector/service_configs/agent"
                 }
             }
         },
@@ -296,22 +295,22 @@
                     "service": "skills/dummy_skill/service_configs/agent"
                 }
             },
-            "dialogpt": {
+            "dff_fromage_image_skill": {
                 "connector": {
                     "protocol": "http",
-                    "timeout": 3.0,
-                    "url": "http://dialogpt:8125/respond"
+                    "timeout": 2.0,
+                    "url": "http://dff-fromage-image-skill:8070/respond"
                 },
-                "dialog_formatter": "state_formatters.dp_formatters:last_utt_and_history_dialog",
+                "dialog_formatter": "state_formatters.dp_formatters:dff_fromage_image_skill_formatter",
                 "response_formatter": "state_formatters.dp_formatters:skill_with_attributes_formatter_service",
                 "previous_services": [
                     "skill_selectors"
                 ],
                 "state_manager_method": "add_hypothesis",
                 "is_enabled": true,
                 "source": {
-                    "component": "components/7Dxjy1ZpQEAoPcEeABGBLg.yml",
-                    "service": "services/dialogpt/service_configs/dialogpt"
+                    "component": "components/8jfFjmYnbdeH.yml",
+                    "service": "services/dff_fromage_image_skill/service_configs/dff-fromage-image-skill"
                 }
             },
             "dff_image_skill": {
@@ -338,18 +337,18 @@
                 "connector": {
                     "protocol": "http",
                     "timeout": 1.0,
-                    "url": "http://convers-evaluation-selector:8009/respond"
+                    "url": "http://ranking-based-response-selector:8002/respond"
                 },
-                "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog",
+                "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog",
                 "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service",
                 "previous_services": [
                     "candidate_annotators"
                 ],
                 "state_manager_method": "add_bot_utterance",
                 "is_enabled": true,
                 "source": {
-                    "component": "components/ly2AVNtIcJpTWz1qJ1mvKQ.yml",
-                    "service": "response_selectors/convers_evaluation_based_selector/service_configs/convers-evaluation-selector"
+                    "component": "components/YJzc7NwGrLmKp6gfZJh7X1.yml",
+                    "service": "response_selectors/ranking_based_response_selector/service_configs/ranking-based-response-selector"
                 }
             }
         }