CMCCallback fix (#941)

* fix args order * fix conformity matrix shape * fix * CHANGELOG.md * manually checked convergence of samplers and cmc score. * fix cmc05 threshold
catalyst-team · Sep 25, 2020 · 19ae864 · 19ae864
1 parent c816f3d
commit 19ae864
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 7 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 - Logging double logging :) ([#936](https://github.com/catalyst-team/catalyst/pull/936))
 
+- CMCCallback ([#941](https://github.com/catalyst-team/catalyst/pull/941))
 
 ## [20.09] - 2020-09-07
 

diff --git a/bin/tests/check_dl_core_callbacks.sh b/bin/tests/check_dl_core_callbacks.sh
@@ -798,7 +798,7 @@ metrics = utils.load_config('$LOGFILE')
 
 EPS = 0.00001
 assert metrics['last']['cmc01'] > 0.1  # slightly better then random
-assert metrics['last']['cmc05'] > 0.5
+assert metrics['last']['cmc05'] > 0.4
 """
 
 ################################  pipeline 22  ################################

diff --git a/catalyst/dl/callbacks/metrics/cmc_score.py b/catalyst/dl/callbacks/metrics/cmc_score.py
@@ -138,15 +138,15 @@ def on_loader_end(self, runner: "IRunner"):
             self._query_idx == self._query_size
         ), "An error occurred during the accumulation process."
 
-        conformity_matrix = self._query_labels == self._gallery_labels.reshape(
+        conformity_matrix = self._gallery_labels == self._query_labels.reshape(
             -1, 1
         )
         for key in self.list_args:
             metric = self._metric_fn(
-                self._gallery_embeddings,
-                self._query_embeddings,
-                conformity_matrix,
-                key,
+                query_embeddings=self._query_embeddings,
+                gallery_embeddings=self._gallery_embeddings,
+                conformity_matrix=conformity_matrix,
+                topk=key,
             )
             runner.loader_metrics[f"{self._prefix}{key:02}"] = metric
         self._gallery_embeddings = None

diff --git a/tests/_tests_scripts/dl_z_mvp_mnist_metric_learning.py b/tests/_tests_scripts/dl_z_mvp_mnist_metric_learning.py
@@ -79,7 +79,7 @@ def main() -> None:
     This function checks metric learning pipeline with
     different triplets samplers.
     """
-    cmc_score_th = 0.9
+    cmc_score_th = 0.85
 
     # Note! cmc_score should be > 0.97
     # after 600 epoch. Please check it mannually