add fusion in decoder

deeppavlov/configs/generative_qa/nq_fid.json

@@ -0,0 +1,97 @@
+{
+  "dataset_reader": {
+    "class_name": "json_reader",
+    "data_path": "{DATASET_PATH}/natural_questions_dataset.json"
+  },
+  "dataset_iterator": {
+    "class_name": "data_learning_iterator",
+    "seed": 42,
+    "shuffle": true
+  },
+  "chainer": {
+    "in": ["question", "contexts", "titles"],
+    "in_y": ["target", "gold_answers"],
+    "pipe": [
+      {
+        "class_name": "fid_input_preprocessor",
+        "vocab_file": "{TRANSFORMER}",
+        "max_seq_length": 200,
+        "in": ["question", "contexts"],
+        "out": ["input_ids", "attention_mask"]
+      },
+      {
+        "class_name": "fid_target_preprocessor",
+        "vocab_file": "{TRANSFORMER}",
+        "answer_maxlength" : 50,
+        "in": ["target"],
+        "out": ["target_ids"]
+      },
+      {
+        "class_name": "torch_generative_qa_fid",
+        "pretrained_transformer": "{TRANSFORMER}",
+        "save_path": "{MODEL_PATH}",
+        "load_path": "{MODEL_PATH}",
+        "optimizer": "AdamW",
+        "optimizer_parameters": {
+          "lr": 3e-04,
+          "weight_decay": 0.01,
+          "betas": [0.9, 0.999],
+          "eps": 1e-08
+        },
+        "learning_rate_drop_patience": 24,
+        "learning_rate_drop_div": 2,
+        "min_learning_rate": 1e-5,
+        "generate_max_length" : 50,
+        "in": ["input_ids", "attention_mask"],
+        "in_y": ["target_ids"],
+        "out": ["model_answer"]
+      }
+    ],
+    "out": ["model_answer"]
+  },
+  "train": {
+    "show_examples": false,
+    "evaluation_targets": [
+      "valid"
+    ],
+    "log_every_n_batches": 100,
+    "val_every_n_batches": 600,
+    "batch_size": 1,
+    "validation_patience": 100,
+    "metrics": [
+      {
+        "name": "squad_v2_em",
+        "inputs": ["gold_answers", "model_answer"]
+      },
+      {
+        "name": "squad_v2_f1",
+        "inputs": ["gold_answers", "model_answer"]
+      }
+    ],
+    "class_name": "torch_trainer"
+  },
+  "metadata": {
+    "variables": {
+      "TRANSFORMER": "t5-base",
+      "ROOT_PATH": "~/.deeppavlov",
+      "DOWNLOADS_PATH": "{ROOT_PATH}/downloads",
+      "MODELS_PATH": "{ROOT_PATH}/models",
+      "MODEL_PATH": "{MODELS_PATH}/generative_qa/fusion_in_decoder/natural_questions",
+      "DATASET_PATH": "{DOWNLOADS_PATH}/natural_questions"
+    },
+    "download": [
+      {
+        "url": "http://files.deeppavlov.ai/deeppavlov_data/generative_qa/datasets/natural_questions/natural_questions_dataset.json",
+        "subdir": "{DATASET_PATH}"
+      },
+      {
+        "url": "http://files.deeppavlov.ai/deeppavlov_data/generative_qa/models/fusion_in_decoder/natural_questions/config.json",
+        "subdir": "{MODEL_PATH}"
+      },
+      {
+        "url": "http://files.deeppavlov.ai/deeppavlov_data/generative_qa/models/fusion_in_decoder/natural_questions/pytorch_model.bin",
+        "subdir": "{MODEL_PATH}"
+      }
+    ]
+  }
+}

deeppavlov/configs/generative_qa/tqa_fid.json

@@ -0,0 +1,97 @@
+{
+    "dataset_reader": {
+      "class_name": "json_reader",
+      "data_path": "{DATASET_PATH}/trivia_qa_dataset.json"
+    },
+    "dataset_iterator": {
+      "class_name": "data_learning_iterator",
+      "seed": 42,
+      "shuffle": true
+    },
+    "chainer": {
+      "in": ["question", "contexts", "titles"],
+      "in_y": ["target", "gold_answers"],
+      "pipe": [
+        {
+          "class_name": "fid_input_preprocessor",
+          "vocab_file": "{TRANSFORMER}",
+          "max_seq_length": 200,
+          "in": ["question", "contexts"],
+          "out": ["input_ids", "attention_mask"]
+        },
+        {
+          "class_name": "fid_target_preprocessor",
+          "vocab_file": "{TRANSFORMER}",
+          "answer_maxlength" : 50,
+          "in": ["target"],
+          "out": ["target_ids"]
+        },
+        {
+          "class_name": "torch_generative_qa_fid",
+          "pretrained_transformer": "{TRANSFORMER}",
+          "save_path": "{MODEL_PATH}",
+          "load_path": "{MODEL_PATH}",
+          "optimizer": "AdamW",
+          "optimizer_parameters": {
+            "lr": 3e-04,
+            "weight_decay": 0.01,
+            "betas": [0.9, 0.999],
+            "eps": 1e-08
+          },
+          "learning_rate_drop_patience": 24,
+          "learning_rate_drop_div": 2,
+          "min_learning_rate": 1e-5,
+          "generate_max_length" : 50,
+          "in": ["input_ids", "attention_mask"],
+          "in_y": ["target_ids"],
+          "out": ["model_answer"]
+        }
+      ],
+      "out": ["model_answer"]
+    },
+    "train": {
+      "show_examples": false,
+      "evaluation_targets": [
+        "valid"
+      ],
+      "log_every_n_batches": 100,
+      "val_every_n_batches": 600,
+      "batch_size": 1,
+      "validation_patience": 100,
+      "metrics": [
+        {
+          "name": "squad_v2_em",
+          "inputs": ["gold_answers", "model_answer"]
+        },
+        {
+          "name": "squad_v2_f1",
+          "inputs": ["gold_answers", "model_answer"]
+        }
+      ],
+      "class_name": "torch_trainer"
+    },
+    "metadata": {
+      "variables": {
+        "TRANSFORMER": "t5-base",
+        "ROOT_PATH": "~/.deeppavlov",
+        "DOWNLOADS_PATH": "{ROOT_PATH}/downloads",
+        "MODELS_PATH": "{ROOT_PATH}/models",
+        "MODEL_PATH": "{MODELS_PATH}/generative_qa/fusion_in_decoder/trivia_qa",
+        "DATASET_PATH": "{DOWNLOADS_PATH}/trivia_qa"
+      },
+      "download": [
+        {
+          "url": "http://files.deeppavlov.ai/deeppavlov_data/generative_qa/datasets/trivia_qa/trivia_qa_dataset.json",
+          "subdir": "{DATASET_PATH}"
+        },
+        {
+          "url": "http://files.deeppavlov.ai/deeppavlov_data/generative_qa/models/fusion_in_decoder/trivia_qa/config.json",
+          "subdir": "{MODEL_PATH}"
+        },
+        {
+          "url": "http://files.deeppavlov.ai/deeppavlov_data/generative_qa/models/fusion_in_decoder/trivia_qa/pytorch_model.bin",
+          "subdir": "{MODEL_PATH}"
+        }
+      ]
+    }
+  }

deeppavlov/core/common/registry.json

@@ -16,11 +16,14 @@
   "entity_linker": "deeppavlov.models.entity_extraction.entity_linking:EntityLinker",
   "faq_reader": "deeppavlov.dataset_readers.faq_reader:FaqDatasetReader",
   "fasttext": "deeppavlov.models.embedders.fasttext_embedder:FasttextEmbedder",
+  "fid_input_preprocessor": "deeppavlov.models.preprocessors.torch_transformers_preprocessor:FiDInputPreprocessor",
+  "fid_target_preprocessor": "deeppavlov.models.preprocessors.torch_transformers_preprocessor:FiDTargetPreprocessor",
   "fit_trainer": "deeppavlov.core.trainers.fit_trainer:FitTrainer",
   "hashing_tfidf_vectorizer": "deeppavlov.models.vectorizers.hashing_tfidf_vectorizer:HashingTfIdfVectorizer",
   "huggingface_dataset_iterator": "deeppavlov.dataset_iterators.huggingface_dataset_iterator:HuggingFaceDatasetIterator",
   "huggingface_dataset_reader": "deeppavlov.dataset_readers.huggingface_dataset_reader:HuggingFaceDatasetReader",
   "imdb_reader": "deeppavlov.dataset_readers.imdb_reader:ImdbReader",
+  "json_reader": "deeppavlov.dataset_readers.json_reader:JsonReader",
   "kenlm_elector": "deeppavlov.models.spelling_correction.electors.kenlm_elector:KenlmElector",
   "line_reader": "deeppavlov.dataset_readers.line_reader:LineReader",
   "logit_ranker": "deeppavlov.models.doc_retrieval.logit_ranker:LogitRanker",
@@ -81,6 +84,7 @@
   "top1_elector": "deeppavlov.models.spelling_correction.electors.top1_elector:TopOneElector",
   "torch_bert_ranker": "deeppavlov.models.torch_bert.torch_bert_ranker:TorchBertRankerModel",
   "torch_bert_ranker_preprocessor": "deeppavlov.models.preprocessors.torch_transformers_preprocessor:TorchBertRankerPreprocessor",
+  "torch_generative_qa_fid": "deeppavlov.models.torch_bert.torch_generative_qa:TorchFiD",
   "torch_record_postprocessor": "deeppavlov.models.preprocessors.torch_transformers_preprocessor:TorchRecordPostprocessor",
   "torch_squad_transformers_preprocessor": "deeppavlov.models.preprocessors.torch_transformers_preprocessor:TorchSquadTransformersPreprocessor",
   "torch_text_classification_model": "deeppavlov.models.classifiers.torch_classification_model:TorchTextClassificationModel",

deeppavlov/core/common/requirements_registry.json

@@ -86,6 +86,10 @@
     "{DEEPPAVLOV_PATH}/requirements/pytorch.txt",
     "{DEEPPAVLOV_PATH}/requirements/transformers.txt"
   ],
+  "torch_generative_qa_fid": [
+    "{DEEPPAVLOV_PATH}/requirements/pytorch.txt",
+    "{DEEPPAVLOV_PATH}/requirements/transformers_3.0.2.txt"
+  ],
   "torch_record_postprocessor": [
     "{DEEPPAVLOV_PATH}/requirements/pytorch.txt",
     "{DEEPPAVLOV_PATH}/requirements/transformers.txt"
@@ -113,6 +117,14 @@
     "{DEEPPAVLOV_PATH}/requirements/pytorch.txt",
     "{DEEPPAVLOV_PATH}/requirements/transformers.txt"
   ],
+  "fid_input_preprocessor": [
+    "{DEEPPAVLOV_PATH}/requirements/pytorch.txt",
+    "{DEEPPAVLOV_PATH}/requirements/transformers_3.0.2.txt"
+  ],
+  "fid_target_preprocessor": [
+    "{DEEPPAVLOV_PATH}/requirements/pytorch.txt",
+    "{DEEPPAVLOV_PATH}/requirements/transformers_3.0.2.txt"
+  ],
   "torch_transformers_multiplechoice": [
     "{DEEPPAVLOV_PATH}/requirements/pytorch.txt",
     "{DEEPPAVLOV_PATH}/requirements/transformers.txt"

deeppavlov/dataset_readers/json_reader.py

@@ -0,0 +1,30 @@
+# Copyright 2017 Neural Networks and Deep Learning lab, MIPT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from typing import Dict, Optional
+
+from deeppavlov.core.common.registry import register
+from deeppavlov.core.data.dataset_reader import DatasetReader
+
+@register('json_reader')
+class JsonReader(DatasetReader):
+
+    def read(self, data_path: str, valid_size: Optional[int] = None) -> Dict:
+        with open(data_path, 'r') as f:
+            dataset = json.load(f)
+        if valid_size is not None:
+            dataset["valid"] = dataset["valid"][:valid_size]
+
+        return dataset

deeppavlov/metrics/bleu.py

@@ -20,6 +20,8 @@
 from deeppavlov.core.common.metrics_registry import register_metric
 from deeppavlov.metrics.google_bleu import compute_bleu
 
+import numpy as np
+
 SMOOTH = SmoothingFunction()
 
 

deeppavlov/models/preprocessors/torch_transformers_preprocessor.py

@@ -421,6 +421,69 @@ def __call__(self, questions_batch: List[List[str]], rels_batch: List[List[str]]
         return input_features
 
 
+@register('fid_input_preprocessor')
+class FiDInputPreprocessor(Component):
+    def __init__(self,
+                 vocab_file: str,
+                 do_lower_case: bool = True,
+                 max_seq_length: int = 512,
+                 **kwargs) -> None:
+        self.max_seq_length = max_seq_length
+
+        if Path(vocab_file).is_file():
+            vocab_file = str(expand_path(vocab_file))
+            self.tokenizer = AutoTokenizer(vocab_file=vocab_file, do_lower_case=do_lower_case)
+        else:
+            self.tokenizer = AutoTokenizer.from_pretrained(vocab_file, do_lower_case=do_lower_case)
+
+    def __call__(self, questions_batch: List[str], contexts_batch: List[List[str]]):
+        prepare_data = lambda q, c,: f"question: {q} context: {c}"
+        passages_batch = [[prepare_data(question, context) for context in contexts] 
+                            for (question, contexts) in zip(questions_batch, contexts_batch)]
+
+        passage_ids, passage_masks = [], []
+        for text_passages in passages_batch:
+            passages_encoding = self.tokenizer(
+                text_passages,
+                max_length=self.max_seq_length if self.max_seq_length > 0 else None,
+                pad_to_max_length=True,
+                return_tensors='pt',
+                truncation=True if self.max_seq_length > 0 else False,
+            )
+            passage_ids.append(passages_encoding['input_ids'][None])
+            passage_masks.append(passages_encoding['attention_mask'][None])
+
+        passage_ids = torch.cat(passage_ids, dim=0)
+        passage_masks = torch.cat(passage_masks, dim=0)
+
+        return passage_ids, passage_masks
+
+@register('fid_target_preprocessor')
+class FiDTargetPreprocessor(Component):
+    def __init__(self,
+                 vocab_file: str,
+                 do_lower_case: bool = True,
+                 answer_maxlength: int = 50,
+                 **kwargs) -> None:
+        self.answer_maxlength = answer_maxlength
+        if Path(vocab_file).is_file():
+            vocab_file = str(expand_path(vocab_file))
+            self.tokenizer = AutoTokenizer(vocab_file=vocab_file, do_lower_case=do_lower_case)
+        else:
+            self.tokenizer = AutoTokenizer.from_pretrained(vocab_file, do_lower_case=do_lower_case)
+
+
+    def __call__(self, targets_batch: List[str]):
+        target_encoding = self.tokenizer(
+            targets_batch,
+            max_length=self.answer_maxlength if self.answer_maxlength > 0 else None,
+            pad_to_max_length=True,
+            return_tensors='pt',
+            truncation=True if self.answer_maxlength > 0 else False,
+        )
+        target_ids = target_encoding["input_ids"]
+        return target_ids
+
 @register('torch_transformers_ner_preprocessor')
 class TorchTransformersNerPreprocessor(Component):
     """