From 0121f6e7efa3c1f19cc4704ac6f69747e1baa028 Mon Sep 17 00:00:00 2001
From: ali abedzadeh <abedzadeh.ali.a@gmail.com>
Date: Fri, 26 Mar 2021 15:56:03 +0430
Subject: [PATCH] Validate and log validation schedule (#141)

* Add sanity check for `validatefreq < niters`
* Add log message for validation schedule
* Add sanity check for `niters<validatefreq` in tensorflow trainer
---
 capreolus/trainer/pytorch.py    | 20 ++++++++++++++++++++
 capreolus/trainer/tensorflow.py |  2 ++
 2 files changed, 22 insertions(+)

diff --git a/capreolus/trainer/pytorch.py b/capreolus/trainer/pytorch.py
index 406d2c7f4..da54708a7 100644
--- a/capreolus/trainer/pytorch.py
+++ b/capreolus/trainer/pytorch.py
@@ -55,6 +55,9 @@ def build(self):
         if self.config["niters"] <= 0:
             raise ValueError("niters must be > 0")
 
+        if self.config["niters"] < self.config["validatefreq"]:
+            raise ValueError("niters must be equal or greater than validatefreq")
+
         if self.config["itersize"] < self.config["batch"]:
             raise ValueError("itersize must be >= batch")
 
@@ -161,6 +164,22 @@ def fastforward_training(self, reranker, weights_path, loss_fn, best_metric_fn):
             logger.info("attempted to load weights from %s but failed, starting at iteration 0", weights_fn)
             return default_return_values
 
+    def get_validation_schedule_msg(self, initial_iter=0):
+        """Describe validation schedule considering `niters` and `validatefreq`
+
+        Args:
+            initial_iter (int): starting point of iteration. defined by train method.
+
+        Example:
+            Assuming self.config["niters"] = 20 and self.config["validatefreq"] = 3, this method will return:
+            `Validation is scheduled on iterations: [3, 6, 9, 12, 15, 18]` given initial_iter in [0, 1, 2]
+            `Validation is scheduled on iterations: [6, 9, 12, 15, 18]` given initial_iter == 3
+        """
+        validation_schedule = [validate for validate in range(initial_iter+1, self.config["niters"]+1)
+                               if validate % self.config["validatefreq"] == 0]
+        msg = f"Validation is scheduled on iterations: {validation_schedule}"
+        return msg
+
     def train(self, reranker, train_dataset, train_output_path, dev_data, dev_output_path, qrels, metric, relevance_level=1):
         """Train a model following the trainer's config (specifying batch size, number of iterations, etc).
 
@@ -225,6 +244,7 @@ def train(self, reranker, train_dataset, train_output_path, dev_data, dev_output
                 logger.debug("fastforwarding train_dataloader to iteration %s", initial_iter)
                 self.exhaust_used_train_data(train_dataloader, n_batch_to_exhaust=initial_iter * self.n_batch_per_iter)
 
+        logger.info(self.get_validation_schedule_msg(initial_iter))
         train_start_time = time.time()
         for niter in range(initial_iter, self.config["niters"]):
             niter = niter + 1  # index from 1
diff --git a/capreolus/trainer/tensorflow.py b/capreolus/trainer/tensorflow.py
index 6619c511e..279f0a8f2 100644
--- a/capreolus/trainer/tensorflow.py
+++ b/capreolus/trainer/tensorflow.py
@@ -101,6 +101,8 @@ def validate(self):
             raise ValueError("storage, tpuname and tpuzone configs must be provided when training on TPU")
         if self.tpu and self.config["storage"] and not self.config["storage"].startswith("gs://"):
             raise ValueError("For TPU utilization, the storage config should start with 'gs://'")
+        if self.config["niters"] < self.config["validatefreq"]:
+            raise ValueError("niters must be equal or greater than validatefreq")
 
     def train(self, reranker, train_dataset, train_output_path, dev_data, dev_output_path, qrels, metric, relevance_level=1):
         if self.tpu: