From 98cad9b75a8b448de07d1b2f0d0f94dc24dbf3b0 Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@eduroam-int-dhcp-97-216-55.ucl.ac.uk>
Date: Wed, 6 Nov 2024 09:35:43 +0000
Subject: [PATCH 01/18] Adding some testing on boxes

---
 test/test_utility/test_utils.py | 76 +++++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)

diff --git a/test/test_utility/test_utils.py b/test/test_utility/test_utils.py
index e69de29..c016125 100644
--- a/test/test_utility/test_utils.py
+++ b/test/test_utility/test_utils.py
@@ -0,0 +1,76 @@
+import pytest
+import numpy as np
+from numpy.testing import assert_allclose, assert_array_equal
+from MetricsReloaded.utility.utils import intersection_boxes, guess_input_style, com_from_box, point_in_box, point_in_mask, area_box, compute_box, compute_center_of_mass, compute_skeleton, combine_df, distance_transform_edt, one_hot_encode, median_heuristic, box_ior, box_iou, union_boxes, max_x_at_y_less, min_x_at_y_less, skeletonize, trapezoidal_integration
+
+
+box3 = [2,3, 4,4]
+box4 = [4,4,5,6]
+
+
+def test_intersection_boxes_empty():
+    box1 = [2,3,5,7]
+    box2 = [6,8,10,10]
+    intersection = intersection_boxes(box1,box2)
+    assert_allclose(intersection, 0)
+
+
+def test_intersection_boxes_shared_corner():
+    box1 = [2,3,5,7]
+    box3 = [2,3, 4,4]
+    intersection = intersection_boxes(box1, box3)
+    assert_allclose(intersection, 6)
+
+def test_intersection_boxes_contained():
+    box1 = [2,3,5,7]
+    box4 = [4,4,5,6]
+    intersection = intersection_boxes(box1, box4)
+    assert_allclose(intersection, 6)
+
+def test_guess_input_style():
+    mask = np.zeros([4,5])
+    mask[2:3,1:4]=1
+    box = np.asarray([2,1,3,4])
+    com = np.asarray([2.5,2.5])
+    test_mask = guess_input_style(mask)
+    test_box = guess_input_style(box)
+    test_com = guess_input_style(com)
+    assert test_mask == 'mask'
+    assert test_box == 'box'
+    assert test_com == 'com'
+
+def test_com_from_box():
+    box_1 = [2,2,3,3]
+    box_2 = [1,2,1,2]
+    com_1 = com_from_box(np.asarray(box_1))
+    com_2 = com_from_box(np.asarray(box_2))
+    assert_array_equal(com_1, np.asarray([2.5,2.5])) 
+    assert_array_equal(com_2, np.asarray([1,2]))
+
+def test_point_in_box():
+    box = [2,1,5,8]
+    point1 = [3,6]
+    point2 = [1,9]
+    assert point_in_box(np.asarray(point1), np.asarray(box)) == True
+    assert point_in_box(np.asarray(point2), np.asarray(box)) == False
+
+def test_point_in_mask():
+    mask = np.zeros([10,10])
+    mask[2:6,1:9] = 1
+    point1 = [3,6]
+    point2 = [1,9]
+    assert point_in_mask(np.asarray(point1), np.asarray(mask)) == True
+    assert point_in_mask(np.asarray(point2), np.asarray(mask)) == False
+
+def test_area_box():
+    box = [1,2,3,1,3,5]
+    assert area_box(np.asarray(box)) == 6
+
+
+# def test_compute_box():
+# def test_compute_center_of_mass():
+# def test_box_ior():
+# def test_box_iou():
+# def test_union_boxes():
+# def 
+# point_in_box, point_in_mask, area_box, compute_box, compute_center_of_mass, compute_skeleton, combine_df, distance_transform_edt, one_hot_encode, median_heuristic, box_ior, box_iou, union_boxes, max_x_at_y_less, min_x_at_y_less, skeletonize, trapezoidal_integration
\ No newline at end of file

From f81d569a4636586be7e3bfbb4ff1ee9bff711d16 Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@eduroam-int-dhcp-97-216-55.ucl.ac.uk>
Date: Wed, 6 Nov 2024 16:37:57 +0000
Subject: [PATCH 02/18] More tests for utility functions

---
 test/test_utility/test_utils.py | 62 +++++++++++++++++++++++++++++----
 1 file changed, 55 insertions(+), 7 deletions(-)

diff --git a/test/test_utility/test_utils.py b/test/test_utility/test_utils.py
index c016125..2a365c3 100644
--- a/test/test_utility/test_utils.py
+++ b/test/test_utility/test_utils.py
@@ -67,10 +67,58 @@ def test_area_box():
     assert area_box(np.asarray(box)) == 6
 
 
-# def test_compute_box():
-# def test_compute_center_of_mass():
-# def test_box_ior():
-# def test_box_iou():
-# def test_union_boxes():
-# def 
-# point_in_box, point_in_mask, area_box, compute_box, compute_center_of_mass, compute_skeleton, combine_df, distance_transform_edt, one_hot_encode, median_heuristic, box_ior, box_iou, union_boxes, max_x_at_y_less, min_x_at_y_less, skeletonize, trapezoidal_integration
\ No newline at end of file
+def test_compute_box():
+    mask = np.zeros([10,10])
+    mask[2:5,3:8] = 1
+    mask[4:6,4:5] = 1
+    box1 = [2,3,5,7]
+    assert_array_equal(compute_box(mask),np.asarray(box1))
+
+def test_compute_center_of_mass():
+    mask = np.zeros([10,10])
+    mask[2:5,3:8] = 1
+    mask[4:6,4:5] = 1
+    assert_array_equal(compute_center_of_mass(mask),np.asarray([3.125,	4.9375]))
+
+def test_box_ior():
+    box1 = [3,5,5,7]
+    box2 = [3,4,4,6]
+    assert box_ior(np.asarray(box1),np.asarray(box2)) == 4.0/6.0
+
+def test_box_iou():
+    box1 = [3,5,5,7]
+    box2 = [3,4,4,6]
+    assert box_iou(np.asarray(box1),np.asarray(box2)) == 4.0/11.0
+
+def test_union_boxes():
+    box1 = [3,5,5,7]
+    box2 = [3,4,4,6]
+    assert union_boxes(np.asarray(box1),np.asarray(box2)) == 11
+
+def test_point_in_box():
+    box1 = [3,5,5,7]
+    point1 = [4,7]
+    point2 = [2,3]
+    assert point_in_box(np.asarray(point1), np.asarray(box1)) == 1
+    assert point_in_box(np.asarray(point2),np.asarray(box1)) == 0
+
+def test_point_in_mask():
+    mask = np.zeros([10,10])
+    mask[2:5,4:8] = 1
+    mask[4:6,4:5] = 1
+    point1 = [4,7]
+    point2 = [2,3]
+    assert point_in_mask(point1,mask) == 1
+    assert point_in_mask(point2, mask) == 0
+
+def test_max_x_at_y_less():
+    x = [1, 2, 1, 3, 4, 0, 1, 4, 5]
+    y = [1, 2, 3, 4, 5, 6, 7 ,8, 9]
+    assert max_x_at_y_less(np.asarray(x), np.asarray(y),6) == 4
+
+
+def test_min_x_at_y_less():
+    x = [1, 2, 1, 3, 4, 0, 1, 4, 5]
+    y = [1, 2, 3, 4, 5, 6, 7 ,8, 9]
+    assert min_x_at_y_less(np.asarray(x), np.asarray(y),6) == 0
+ #compute_skeleton, combine_df, distance_transform_edt, one_hot_encode, median_heuristic, max_x_at_y_less, min_x_at_y_less, skeletonize, trapezoidal_integration
\ No newline at end of file

From ec33ab8b9c7eff7e60e01340b8d934e941e84173 Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@eduroam-int-dhcp-97-216-55.ucl.ac.uk>
Date: Thu, 7 Nov 2024 10:47:21 +0000
Subject: [PATCH 03/18] Updating docs and warnings

---
 MetricsReloaded/metrics/pairwise_measures.py | 268 +++++++++++++------
 1 file changed, 185 insertions(+), 83 deletions(-)

diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py
index 8efc6db..2ff04d7 100755
--- a/MetricsReloaded/metrics/pairwise_measures.py
+++ b/MetricsReloaded/metrics/pairwise_measures.py
@@ -155,7 +155,7 @@ def chance_agreement_probability(self):
         """Determines the probability of agreeing by chance given two classifications.
         To be used for CK calculation
 
-
+        return: chance (probability for classification of agreeing by chance)
         """
         chance = 0
         for f in self.list_values:
@@ -189,7 +189,8 @@ def balanced_accuracy(self):
         col_sum = np.sum(cm, 0)
         numerator = np.sum(np.diag(cm) / col_sum)
         denominator = len(self.list_values)
-        return numerator / denominator
+        balanced_accuracy = numerator / denominator
+        return balanced_accuracy
 
     def expectation_matrix(self):
         """
@@ -201,10 +202,8 @@ def expectation_matrix(self):
         one_hot_ref = one_hot_encode(self.ref, len(self.list_values))
         pred_numb = np.sum(one_hot_pred, 0)
         ref_numb = np.sum(one_hot_ref, 0)
-        return (
-            np.matmul(np.reshape(pred_numb, [-1, 1]), np.reshape(ref_numb, [1, -1]))
-            / np.shape(one_hot_pred)[0]
-        )
+        expectation_matrix = np.matmul(np.reshape(pred_numb, [-1, 1]), np.reshape(ref_numb, [1, -1]))/ np.shape(one_hot_pred)[0]
+        return expectation_matrix
 
     def weighted_cohens_kappa(self):
         """
@@ -225,6 +224,7 @@ def weighted_cohens_kappa(self):
             )
         numerator = np.sum(weights * cm)
         denominator = np.sum(weights * exp)
+        weighted_cohens_kappa = 1 - numerator / denominator
         return 1 - numerator / denominator
 
     def to_dict_meas(self, fmt="{:.4f}"):
@@ -296,8 +296,9 @@ def __fp_map(self):
         """
         ref_float = np.asarray(self.ref, dtype=np.float32)
         pred_float = np.asarray(self.pred, dtype=np.float32)
-        return np.asarray((pred_float - ref_float) > 0.0, dtype=np.float32)
-
+        fp_map = np.asarray((pred_float - ref_float) > 0.0, dtype=np.float32)
+        return fp_map
+    
     def __fn_map(self):
         """
         This function calculates the false negative map
@@ -306,8 +307,9 @@ def __fn_map(self):
         """
         ref_float = np.asarray(self.ref, dtype=np.float32)
         pred_float = np.asarray(self.pred, dtype=np.float32)
-        return np.asarray((ref_float - pred_float) > 0.0, dtype=np.float32)
-
+        fn_map = np.asarray((ref_float - pred_float) > 0.0, dtype=np.float32)
+        return fn_map
+    
     def __tp_map(self):
         """
         This function calculates the true positive map
@@ -316,7 +318,8 @@ def __tp_map(self):
         """
         ref_float = np.asarray(self.ref, dtype=np.float32)
         pred_float = np.asarray(self.pred, dtype=np.float32)
-        return np.asarray((ref_float + pred_float) > 1.0, dtype=np.float32)
+        tp_map = np.asarray((ref_float + pred_float) > 1.0, dtype=np.float32)
+        return tp_map
 
     def __tn_map(self):
         """
@@ -326,8 +329,9 @@ def __tn_map(self):
         """
         ref_float = np.asarray(self.ref, dtype=np.float32)
         pred_float = np.asarray(self.pred, dtype=np.float32)
-        return np.asarray((ref_float + pred_float) < 0.5, dtype=np.float32)
-
+        tn_map = np.asarray((ref_float + pred_float) < 0.5, dtype=np.float32)
+        return tn_map
+    
     def __union_map(self):
         """
         This function calculates the union map between prediction and
@@ -344,70 +348,103 @@ def __intersection_map(self):
 
         :return: intersection map
         """
-        return np.multiply(self.ref, self.pred)
+        intersection_map = np.multiply(self.ref, self.pred)
+        return intersection_map
 
     @CacheFunctionOutput
     def n_pos_ref(self):
         """
         Returns the number of elements in ref
+        
+        :return: n_pos_ref
         """
-        return np.sum(self.ref)
+        n_pos_ref = np.sum(self.ref)
+        return n_pos_ref
 
     @CacheFunctionOutput
     def n_neg_ref(self):
         """
         Returns the number of negative elements in ref
+        
+        :return: n_neg_ref
         """
-        return np.sum(1 - self.ref)
+        n_neg_ref = np.sum(1-self.ref)
+        return n_neg_ref
 
     @CacheFunctionOutput
     def n_pos_pred(self):
         """
         Returns the number of positive elements in the prediction
+        
+        :return: n_pos_pred
         """
+        n_pos_pred = np.sum(self.pred)
         return np.sum(self.pred)
 
     @CacheFunctionOutput
     def n_neg_pred(self):
         """
         Returns the number of negative elements in the prediction
+        
+        :return: n_neg_pred
         """
-        return np.sum(1 - self.pred)
+        n_neg_pred = np.sum(1-self.pred)
+        return n_neg_pred
 
     @CacheFunctionOutput
     def fp(self):
         """
         Calculates the number of FP as sum of elements in FP_map
+        
+        :return: fp
         """
-        return np.sum(self.__fp_map())
+        fp = np.sum(self.__fp_map())
+        return fp
 
     @CacheFunctionOutput
     def fn(self):
         """
         Calculates the number of FN as sum of elements of FN_map
+        
+        :return: fn
         """
-        return np.sum(self.__fn_map())
+        fn = np.sum(self.__fn_map())
+        return fn
 
     @CacheFunctionOutput
     def tp(self):
         """
         Returns the number of true positive (TP) elements
+        
+        :return: tp
         """
-        return np.sum(self.__tp_map())
+        tp = np.sum(self.__tp_map())
+        return tp
 
     @CacheFunctionOutput
     def tn(self):
         """
         Returns the number of True Negative (TN) elements
+        
+        :return: tn
         """
-        return np.sum(self.__tn_map())
+        tn = np.sum(self.__tn_map())
+        return tn
 
     @CacheFunctionOutput
     def n_intersection(self):
         """
         Returns the number of elements in the intersection of reference and prediction (=TP)
+        
+        .. math::
+
+            I =  TP
+
+
+        :return: n_intersection
         """
-        return np.sum(self.__intersection_map())
+        n_intersection = np.sum(self.__intersection_map())
+        return n_intersection
 
     @CacheFunctionOutput
     def n_union(self):
@@ -418,8 +455,10 @@ def n_union(self):
 
             U = {\vert} Pred {\vert} + {\vert} Ref {\vert} - TP
 
+        :return: n_union
         """
-        return np.sum(self.__union_map())
+        n_union = np.sum(self.__union_map())
+        return n_union
 
     def youden_index(self):
         """
@@ -431,9 +470,10 @@ def youden_index(self):
 
         Youden, W.J, Index for rating diagnostic tests - 1950 Cancer 3 - 32,35
 
-        :return: Youden index
+        :return: youden_index
         """
-        return self.specificity() + self.sensitivity() - 1
+        youden_index =  self.specificity() + self.sensitivity() - 1
+        return youden_index
 
     def sensitivity(self):
         """
@@ -452,7 +492,8 @@ def sensitivity(self):
         if self.n_pos_ref() == 0:
             warnings.warn("reference empty, sensitivity not defined")
             return np.nan
-        return self.tp() / self.n_pos_ref()
+        sensitivity = self.tp() / self.n_pos_ref()
+        return sensitivity
 
     def specificity(self):
         """
@@ -472,7 +513,8 @@ def specificity(self):
         if self.n_neg_ref() == 0:
             warnings.warn("reference all positive, specificity not defined")
             return np.nan
-        return self.tn() / self.n_neg_ref()
+        specificity = self.tn() / self.n_neg_ref()
+        return specificity
 
     def balanced_accuracy(self):
         """
@@ -484,7 +526,8 @@ def balanced_accuracy(self):
 
         :return: balanced accuracy
         """
-        return 0.5 * self.sensitivity() + 0.5 * self.specificity()
+        balanced_accuracy = 0.5 * self.sensitivity() + 0.5 * self.specificity()
+        return balanced_accuracy
 
     def accuracy(self):
         """
@@ -499,7 +542,8 @@ def accuracy(self):
 
         :return: accuracy
         """
-        return (self.tn() + self.tp()) / (self.tn() + self.tp() + self.fn() + self.fp())
+        accuracy = (self.tn() + self.tp()) / (self.tn() + self.tp() + self.fn() + self.fp())
+        return accuracy
 
     def false_positive_rate(self):
         """
@@ -511,9 +555,10 @@ def false_positive_rate(self):
 
         Burke D, Brundage J, Redfield R., Measurement of the False positive rate in a screening Program for Human Immunodeficiency Virus Infections - 1988 - The New England Journal of Medicine 319 (15) 961-964
 
-        :return: false positive rate
+        :return: false_positive_rate
         """
-        return self.fp() / self.n_neg_ref()
+        false_positive_rate = self.fp() / self.n_neg_ref()
+        return false_positive_rate
 
     def normalised_expected_cost(self):
         """
@@ -521,7 +566,7 @@ def normalised_expected_cost(self):
 
         Luciana Ferrer. 2022. Analysis and Comparison of Classification Metrics. arXiv preprint arXiv:2209.05355 (2022).
 
-        :return: normalised expected cost
+        :return: normalised_expected_cost
         """
 
         prior_background = (self.tn() + self.fp()) / (np.size(self.ref))
@@ -541,10 +586,10 @@ def normalised_expected_cost(self):
         r_fp = self.fp() / self.n_neg_ref()
         r_fn = self.fn() / self.n_pos_ref()
         if alpha >= 1:
-            ecn = alpha * r_fp + r_fn
+            normalised_expected_cost = alpha * r_fp + r_fn
         else:
-            ecn = r_fp + 1 / alpha * r_fn
-        return ecn
+            normalised_expected_cost = r_fp + 1 / alpha * r_fn
+        return normalised_expected_cost
 
     def matthews_correlation_coefficient(self):
         """
@@ -556,7 +601,7 @@ def matthews_correlation_coefficient(self):
 
             MCC = \dfrac{TP * TN - FP * FN}{(TP+FP)*(TP+FN)*(TN+FP)*(TN+FN)}
 
-        :return: MCC
+        :return: mcc
         """
         numerator = self.tp() * self.tn() - self.fp() * self.fn()
         denominator = (
@@ -565,7 +610,8 @@ def matthews_correlation_coefficient(self):
             * (self.tn() + self.fp())
             * (self.tn() + self.fn())
         )
-        return numerator / np.sqrt(denominator)
+        mcc = numerator / np.sqrt(denominator)
+        return mcc
 
     def expected_matching_ck(self):
         list_values = np.unique(self.ref)
@@ -598,13 +644,14 @@ def cohens_kappa(self):
 
         Cohen, J. A coefficient of agreement for nominal scales - Educational and Psychological Measurement (1960) 20 37-46
 
-        :return: CK
+        :return: cohens_kappa
         """
         p_e = self.expected_matching_ck()
         p_o = self.accuracy()
         numerator = p_o - p_e
         denominator = 1 - p_e
-        return numerator / denominator
+        cohens_kappa = numerator / denominator
+        return cohens_kappa
 
     def positive_likelihood_ratio(self):
         """
@@ -617,11 +664,21 @@ def positive_likelihood_ratio(self):
 
             LR+ = \dfrac{Sensitivity}{1-Specificity}
 
-        :return: LR+
+        :return: positive_likelihood_ratio (LR+)
         """
         numerator = self.sensitivity()
         denominator = 1 - self.specificity()
-        return numerator / denominator
+        if self.n_neg_ref() == 0:
+            warnings.warn("reference all positive, specificity not defined")
+            return np.nan
+        if self.n_pos_ref() == 0:
+            warnings.warn("reference empty - sensitivity not defined")
+            return np.nan
+        if self.specificity() == 1:
+            warnings.warn("Perfect specifiicty - likelihood ratio not defined")
+            return np.nan
+        positive_likelihood_ratio = numerator / denominator
+        return positive_likelihood_ratio
 
     def pred_in_ref(self):
         """
@@ -648,7 +705,7 @@ def positive_predictive_values(self):
 
         Fletcher, R.H and Fletcher S.W (2005) - Clinical Epidemiology, the essentials p45
 
-        :return: PPV
+        :return: positive_predictive_value (PPV)
         """
         if self.flag_empty_pred:
             if self.flag_empty_ref:
@@ -657,7 +714,8 @@ def positive_predictive_values(self):
             else:
                 warnings.warn("prediction empty, ppv not defined but set to 0")
                 return 0
-        return self.tp() / (self.tp() + self.fp())
+        positive_predictive_value = self.tp() / (self.tp() + self.fp())
+        return positive_predictive_value
 
     def recall(self):
         """
@@ -673,7 +731,8 @@ def recall(self):
                 "prediction is empty but ref not, recall not defined but set to 0"
             )
             return 0
-        return self.tp() / (self.tp() + self.fn())
+        recall = self.tp() / (self.tp() + self.fn())
+        return recall
 
     def dsc(self):
         """
@@ -687,6 +746,7 @@ def dsc(self):
         
         This is also F:math:`{\\beta}` for :math:`{\\beta}`=1
 
+        :return: dsc
         """
 
         numerator = 2 * self.tp()
@@ -695,7 +755,8 @@ def dsc(self):
             warnings.warn("Both Prediction and Reference are empty - set to 1 as correct solution even if not defined")
             return 1
         else:
-            return numerator / denominator
+            dsc = numerator / denominator
+            return dsc
 
     def fbeta(self):
         """
@@ -718,6 +779,7 @@ def fbeta(self):
         if "beta" in self.dict_args.keys():
             beta = self.dict_args["beta"]
         else:
+            warnings.warn("beta value not specified in option - default set to 1")
             beta = 1
         numerator = (
             (1 + np.square(beta)) * self.positive_predictive_values() * self.recall()
@@ -736,7 +798,8 @@ def fbeta(self):
             else:
                 return 1  # Potentially modify to nan
         else:
-            return numerator / denominator
+            fbeta = numerator / denominator
+            return fbeta
 
     def net_benefit_treated(self):
         """
@@ -752,7 +815,7 @@ def net_benefit_treated(self):
         where ER relates to the exchange rate. For instance if a suitable exchange rate is to find
         1 positive case among 10 tested (1TP for 9 FP), the exchange rate would be 1/9
 
-        :return: NB
+        :return: net_benefit
         """
         if "exchange_rate" in self.dict_args.keys():
             er = self.dict_args["exchange_rate"]
@@ -761,8 +824,8 @@ def net_benefit_treated(self):
         n = np.size(self.pred)
         tp = self.tp()
         fp = self.fp()
-        nb = tp / n - fp / n * er
-        return nb
+        net_benefit = tp / n - fp / n * er
+        return net_benefit
 
     def negative_predictive_values(self):
         """
@@ -771,6 +834,10 @@ def negative_predictive_values(self):
 
         Fletcher, R.H and Fletcher S.W (2005) - Clinical Epidemiology, the essentials p45
 
+        .. math::
+
+            NPV = \dfrac{TN}{N}
+
         :return: NPV
         """
         if self.tn() + self.fn() == 0:
@@ -785,23 +852,9 @@ def negative_predictive_values(self):
                     "Nothing negative in pred but should be NPV not defined but set to 0"
                 )
                 return 0
-        return self.tn() / (self.fn() + self.tn())
-
-    # def dice_score(self):
-    #     """
-    #     This function returns the dice score coefficient between a reference
-    #     and prediction images
-
-    #     :return: dice score
-    #     """
-    #     if not "fbeta" in self.dict_args.keys():
-    #         self.dict_args["fbeta"] = 1
-    #     elif self.dict_args["fbeta"] != 1:
-    #         warnings.warn("Modifying fbeta option to get dice score")
-    #         self.dict_args["fbeta"] = 1
-    #     else:
-    #         print("Already correct value for fbeta option")
-    #     return self.fbeta()
+        negative_predictive_value = self.tn() / (self.fn() + self.tn())
+        return negative_predictive_value
+
 
     def fppi(self):
         """
@@ -815,7 +868,8 @@ def fppi(self):
         sum_per_image = np.sum(
             np.reshape(self.__fp_map(), -1, self.ref.shape[-1]), axis=0
         )
-        return np.mean(sum_per_image)
+        fppi = np.mean(sum_per_image)
+        return fppi
 
     def intersection_over_reference(self):
         """
@@ -824,12 +878,17 @@ def intersection_over_reference(self):
 
         Pavel Matula, Martin Maška, Dmitry V Sorokin, Petr Matula, Carlos Ortiz-de Solórzano, and Michal Kozubek. Cell tracking accuracy measurement based on comparison of acyclic oriented graphs. PloS one, 10(12):e0144959, 2015.
 
+        .. math::
+
+            IoR = \dfrac{\vert \text{Pred} \bigcap \text{Ref} \vert}{\vert Ref \vert}
+
         :return: IoR
         """
         if self.flag_empty_ref:
-            warnings.warn("Empty reference")
+            warnings.warn("IoR not defined - Empty reference")
             return np.nan
-        return self.n_intersection() / self.n_pos_ref()
+        ior = self.n_intersection()/self.n_pos_ref()
+        return ior
 
     def intersection_over_union(self):
         """
@@ -839,12 +898,17 @@ def intersection_over_union(self):
 
         Murphy, A.H. The Finley Affair: a signal event in the history of forecast verification - Weather and Forecasting (1996) 11
 
+        .. math::
+
+            IoU = \dfrac{\vert Pred \bigcap Ref\vert}{\vert Pred \bigcup Ref \vert}
+
         :return: IoU
         """
         if self.flag_empty_pred and self.flag_empty_ref:
-            warnings.warn("Both reference and prediction are empty")
+            warnings.warn("IoU not defined - Both reference and prediction are empty")
             return np.nan
-        return self.n_intersection() / self.n_union()
+        iou = self.n_intersection() / self.n_union()
+        return iou
 
     def com_dist(self):
         """
@@ -856,6 +920,7 @@ def com_dist(self):
         """
         
         if self.flag_empty_pred or self.flag_empty_ref:
+            warnings.warn('Impossible to calculate distance between centre of masses as either reference of prediction is empty')
             return -1
         else:
             com_ref = compute_center_of_mass(self.ref)
@@ -879,11 +944,13 @@ def com_ref(self):
         This function calculates the centre of mass of the reference
         prediction
 
-        :return: Centre of mass coordinates of reference when not empty, -1 otherwise
+        :return: com_ref - Centre of mass coordinates of reference when not empty, -1 otherwise
         """
         if self.flag_empty_ref:
+            warnings.warn('Empty reference - centre of mass not defined')
             return -1
-        return ndimage.center_of_mass(self.ref)
+        com_ref = ndimage.center_of_mass(self.ref)
+        return com_ref
 
     def com_pred(self):
         """
@@ -891,23 +958,38 @@ def com_pred(self):
         :returns: -1 if empty image, centre of mass of prediction otherwise
         """
         if self.flag_empty_pred:
+            warnings.warn('Empty prediction - centre of mass not defined')
             return -1
         else:
-            return ndimage.center_of_mass(self.pred)
+            com_pred = ndimage.center_of_mass(self.pred)
+            return com_pred
 
     def list_labels(self):
+        """
+        Creates the tuple with unique values of labels
+
+        return list_labels
+        """
         if self.list_labels is None:
             return ()
         return tuple(np.unique(self.list_labels))
 
-    def vol_diff(self):
+    def absolute_volume_difference_ratio(self):
         """
         This function calculates the ratio of difference in volume between
         the reference and prediction images.
 
-        :return: vol_diff
+        .. math::
+
+            AVDR = \dfrac{\vert Pred - Ref\vert}{\vert Ref \vert} 
+
+        :return: avdr
         """
-        return np.abs(self.n_pos_ref() - self.n_pos_pred()) / self.n_pos_ref()
+        if self.n_pos_ref() == 0:
+            warnings.warn('Empty reference - absolute volume difference ratio not defined')
+            return np.nan
+        avdr = np.abs(self.n_pos_ref() - self.n_pos_pred()) / self.n_pos_ref()
+        return avdr
 
     @CacheFunctionOutput
     def skeleton_versions(self):
@@ -935,7 +1017,11 @@ def topology_precision(self):
         skeleton_ref, skeleton_pred = self.skeleton_versions()
         numerator = np.sum(skeleton_pred * self.ref)
         denominator = np.sum(skeleton_pred)
-        return numerator / denominator
+        if denominator == 0:
+            warnings.warn('Empty prediction skeleton - topology precision not defined')
+            return np.nan
+        topology_precision = numerator / denominator
+        return topology_precision
 
     def topology_sensitivity(self):
         """
@@ -952,7 +1038,11 @@ def topology_sensitivity(self):
         skeleton_ref, skeleton_pred = self.skeleton_versions()
         numerator = np.sum(skeleton_ref * self.pred)
         denominator = np.sum(skeleton_ref)
-        return numerator / denominator
+        if denominator == 0:
+            warnings.warn("Reference skeleton empty - topology sensitivity not defined")
+            return np.nan
+        topology_sensitivity = numerator / denominator
+        return topology_sensitivity
 
     def centreline_dsc(self):
         """
@@ -972,7 +1062,11 @@ def centreline_dsc(self):
         top_sens = self.topology_sensitivity()
         numerator = 2 * top_sens * top_prec
         denominator = top_sens + top_prec
-        return numerator / denominator
+        if np.isnan(top_sens) or np.isnan(top_sens):
+            warnings.warn("Topology sensitivity or precision not defined")
+            return np.nan
+        cDSC = numerator / denominator
+        return cDSC
 
     def boundary_iou(self):
         """
@@ -1018,7 +1112,11 @@ def boundary_iou(self):
                 np.zeros_like(border_pred),
             )
         )
-        return intersect / union
+        if union == 0:
+            warnings.warn('Union empty for boundary iou - not defined')
+            return np.nan
+        boundary_iou = intersect / union
+        return boundary_iou
 
 
     @CacheFunctionOutput
@@ -1086,8 +1184,10 @@ def measured_distance(self):
         if "hd_perc" in self.dict_args.keys():
             perc = self.dict_args["hd_perc"]
         else:
+            warnings.warn('Percentile not specified in options for Hausdorff distance - default set to 95')
             perc = 95
         if np.sum(self.pred + self.ref) == 0:
+            warnings.warn("Prediction and reference empty - distances set to 0")
             return 0, 0, 0, 0
         (
             ref_border_dist,
@@ -1130,7 +1230,8 @@ def measured_average_distance(self):
 
         :return: assd
         """
-        return self.measured_distance()[1]
+        assd = self.measured_distance()[1]
+        return assd
 
     def measured_masd(self):
         """
@@ -1145,7 +1246,8 @@ def measured_masd(self):
         
         :return: masd
         """
-        return self.measured_distance()[3]
+        masd = self.measured_distance()[3]
+        return masd
 
     def measured_hausdorff_distance(self):
         """

From 98c6e7fab29bf185e83b805adf7c47b6587664e7 Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@eduroam-int-dhcp-97-216-55.ucl.ac.uk>
Date: Thu, 7 Nov 2024 10:48:45 +0000
Subject: [PATCH 04/18] Updating docs and warnings distance

---
 MetricsReloaded/metrics/pairwise_measures.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py
index 2ff04d7..f78838c 100755
--- a/MetricsReloaded/metrics/pairwise_measures.py
+++ b/MetricsReloaded/metrics/pairwise_measures.py
@@ -1259,7 +1259,8 @@ def measured_hausdorff_distance(self):
 
         :return: hausdorff_distance
         """
-        return self.measured_distance()[0]
+        hausdorff_distance = self.measured_distance()[0]
+        return hausdorff_distance
 
     def measured_hausdorff_distance_perc(self):
         """
@@ -1270,7 +1271,8 @@ def measured_hausdorff_distance_perc(self):
         
         :return: hausdorff_distance_perc
         """
-        return self.measured_distance()[2]
+        hausdorff_distance_perc = self.measured_distance()[2]
+        return hausdorff_distance_perc
 
     def to_dict_meas(self, fmt="{:.4f}"):
         result_dict = {}

From 781bd51a52a12e1b352d523ee9148c588ba25286 Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@eduroam-int-dhcp-97-216-55.ucl.ac.uk>
Date: Thu, 7 Nov 2024 10:50:42 +0000
Subject: [PATCH 05/18] Updating test for absolute volume difference ratio

---
 test/test_metrics/test_pairwise_measures.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_metrics/test_pairwise_measures.py b/test/test_metrics/test_pairwise_measures.py
index f16193c..296f0f2 100644
--- a/test/test_metrics/test_pairwise_measures.py
+++ b/test/test_metrics/test_pairwise_measures.py
@@ -263,7 +263,7 @@ def test_voldiff():
     ref[2:4, 2:4] = 1
     pred[3:5, 3:5] = 1
     bpm = PM(pred, ref)
-    value_test = bpm.vol_diff()
+    value_test = bpm.absolute_volume_difference_ratio()
     expected_vdiff = 0
     assert_allclose(value_test, expected_vdiff)
 

From 4dd00bffcd57dedfae3f671356b1c8b86b8ac0f1 Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@eduroam-int-dhcp-97-216-55.ucl.ac.uk>
Date: Thu, 7 Nov 2024 16:54:11 +0000
Subject: [PATCH 06/18] Adding formulas to calibration measures, correcting
 kernel calculation, creating test for confusion matrix

---
 .../metrics/calibration_measures.py           | 92 ++++++++++++++++++-
 MetricsReloaded/metrics/pairwise_measures.py  | 36 +++++---
 test/test_metrics/test_calibration_metrics.py | 12 +--
 test/test_metrics/test_pairwise_measures.py   | 18 +++-
 4 files changed, 134 insertions(+), 24 deletions(-)

diff --git a/MetricsReloaded/metrics/calibration_measures.py b/MetricsReloaded/metrics/calibration_measures.py
index 10b6a79..d8cd78c 100644
--- a/MetricsReloaded/metrics/calibration_measures.py
+++ b/MetricsReloaded/metrics/calibration_measures.py
@@ -91,7 +91,7 @@ def class_wise_expectation_calibration_error(self):
 
             cwECE = \dfrac{1}{K}\sum_{k=1}^{K}\sum_{i=1}^{N}\dfrac{\vert B_{i,k} \vert}{N} \left(y_{k}(B_{i,k}) - p_{k}(B_{i,k})\right)
 
-
+        :return: cwece
         """
 
         if "bins_ece" in self.dict_args:
@@ -139,6 +139,13 @@ def expectation_calibration_error(self):
         Derives the expectation calibration error in the case of binary task
         bins_ece is the key in the dictionary for the number of bins to consider
         Default is 10
+
+        .. math::
+
+            ECE = \sum_{m=1}^{M} \dfrac{|B_m|}{n}(\dfrac{1}{|B_m|}\sum_{i \in B_m}1(pred_ik==ref_ik)-\dfrac{1}{|B_m|}\sum_{i \in B_m}pred_i)
+
+        :return: ece
+
         """
         if "bins_ece" in self.dict_args:
             nbins = self.dict_args["bins_ece"]
@@ -179,22 +186,44 @@ def brier_score(self):
         Glenn W Brier et al. 1950. Verification of forecasts expressed in terms of probability. Monthly weather review 78, 1
         (1950), 1–3.
 
+        .. math::
+
+            BS = \dfrac{1}{N}\sum_{i=1}{N}\sum_{j=1}^{C}(p_{ic}-r_{ic})^2
+
+        where :math: `p_{ic}` is the probability for class c and :math: `r_{ic}` the binary reference for class c and element i
+
         :return: brier score (BS)
+
         """
         bs = np.mean(np.sum(np.square(self.one_hot_ref - self.pred),1))
         return bs
 
     def root_brier_score(self):
         """
+        Determines the root brier score
+
         Gruber S. and Buettner F., Better Uncertainty Calibration via Proper Scores
         for Classification and Beyond, In Proceedings of the 36th International
         Conference on  Neural Information Processing Systems, 2022
+
+        .. math::
+
+            RBS = \sqrt{BS}
+
+        :return: rbs
         """
-        return np.sqrt(self.brier_score())
+        rbs = np.sqrt(self.brier_score())
+        return rbs
 
     def logarithmic_score(self):
         """
         Calculation of the logarithmic score https://en.wikipedia.org/wiki/Scoring_rule
+        
+        .. math::
+
+            LS = 1/N\sum_{i=1}^{N}\log{pred_ik}ref_{ik}
+
+        :return: ls
         """
         eps = 1e-10
         log_pred = np.log(self.pred + eps)
@@ -204,6 +233,11 @@ def logarithmic_score(self):
         return ls
 
     def distance_ij(self,i,j):
+        """
+        Determines the euclidean distance between two vectors of prediction for two samples i and j
+
+        :return: distance
+        """
         pred_i = self.pred[i,:]
         pred_j = self.pred[j,:]
         distance = np.sqrt(np.sum(np.square(pred_i - pred_j)))
@@ -211,20 +245,36 @@ def distance_ij(self,i,j):
 
 
     def kernel_calculation(self, i,j):
+        """
+        Defines the kernel value for two samples i and j with the following definition for k(x_i,x_j)
+
+        .. math::
+
+            k(x_i,x_j) = exp(-||x_i-y_j||/ \\nu)I_{N}
+
+        where :math: `\\nu` is the bandwith defined as the median heuristic if not specified in the options and N the number of classes
+
+        :return: kernel_value
+
+        """
         distance = self.distance_ij(i,j)
         if 'bandwidth_kce' in self.dict_args.keys():
             bandwidth = self.dict_args['bandwidth_kce']
         else:
             bandwidth = median_heuristic(self.pred)
         value = np.exp(-distance/bandwidth)
-        identity = np.ones([self.pred.shape[1], self.pred.shape[1]])
-        return value * identity
+        identity = np.eye(self.pred.shape[1])
+        kernel_value = value*identity
+        return kernel_value
 
     def kernel_calibration_error(self):
         """
         Based on the paper Widmann, D., Lindsten, F., and Zachariah, D.
         Calibration tests in multi-class classification: A unifying framework.
         Advances in Neural Information Processing Systems, 32:12257–12267, 2019.
+
+        :return: kce
+
         """
         one_hot_ref = one_hot_encode(self.ref, self.pred.shape[1])
         numb_samples = self.pred.shape[0]
@@ -246,6 +296,9 @@ def top_label_classification_error(self):
         """
         Calculation of the top-label classification error. Assumes pred_proba a matrix K x Numb observations
         with probability to be in class k for observation i in position (k,i)
+
+        :return: tce
+
         """
         class_max = np.argmax(self.pred, 1)
         prob_pred_max = np.max(self.pred, 1)
@@ -271,7 +324,12 @@ def kernel_based_ece(self):
         Teodora Popordanoska, Raphael Sayer, and Matthew B Blaschko. 2022. A Consistent and Differentiable Lp Canonical
         Calibration Error Estimator. In Advances in Neural Information Processing Systems.
 
+        .. math::
+ 
+            ECE\_KDE = 1/N \sum_{j=1}^{N}||\dfrac{\sum_{i \\neq j}k_{Dir}(pred_j,pred_i)ref_i}{\sum_{i \\neq j}k_{Dir}(pred_j,pred_i)} - pred_j || 
+
         :return: ece_kde
+
         """
         ece_kde = 0
         one_hot_ref = one_hot_encode(self.ref, self.pred.shape[1])
@@ -298,6 +356,18 @@ def kernel_based_ece(self):
         return ece_kde
 
     def gamma_ik(self, i, k):
+        """
+        Definition of gamma value for sample i class k of the predictions
+
+        .. math::
+
+            gamma_{ik} = \Gamma(pred_{ik}/h + 1)
+
+        where h is the bandwidth value set as default to 0.5
+
+        :return gamma_ik
+
+        """
         pred_ik = self.pred[i, k]
         if "bandwidth" in self.dict_args.keys():
             h = self.dict_args["bandwidth"]
@@ -308,6 +378,16 @@ def gamma_ik(self, i, k):
         return gamma_ik
 
     def dirichlet_kernel(self, j, i):
+        """
+        Calculation of Dirichlet kernel value for predictions of samples i and j
+
+        .. math::
+
+            k_{Dir}(x_j,x_i) = \dfrac{\Gamma(\sum_{k=1}^{K}\\alpha_{ik})}{\prod_{k=1}^{K}\\alpha_{ik}}\prod_{k=1}^{K}x_jk^{\\alpha_{ik}-1}
+        
+        :return: kernel_value
+
+        """
         pred_i = self.pred[i, :]
         pred_j = self.pred[j, :]
         nclasses = self.pred.shape[1]
@@ -334,7 +414,9 @@ def negative_log_likelihood(self):
 
         .. math::
 
-            -\sum_{i=1}{N} log(p_{i,k} | y_i=k)
+            NLL = -\sum_{i=1}{N} log(p_{i,k} | y_i=k)
+
+        :return: NLL
 
         """
         log_pred = np.log(self.pred)
diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py
index f78838c..593ab00 100755
--- a/MetricsReloaded/metrics/pairwise_measures.py
+++ b/MetricsReloaded/metrics/pairwise_measures.py
@@ -197,6 +197,7 @@ def expectation_matrix(self):
         Determination of the expectation matrix to be used for CK derivation
 
         :return: expectation_matrix
+
         """
         one_hot_pred = one_hot_encode(self.pred, len(self.list_values))
         one_hot_ref = one_hot_encode(self.ref, len(self.list_values))
@@ -225,7 +226,7 @@ def weighted_cohens_kappa(self):
         numerator = np.sum(weights * cm)
         denominator = np.sum(weights * exp)
         weighted_cohens_kappa = 1 - numerator / denominator
-        return 1 - numerator / denominator
+        return weighted_cohens_kappa
 
     def to_dict_meas(self, fmt="{:.4f}"):
         """Given the selected metrics provides a dictionary with relevant metrics"""
@@ -272,6 +273,7 @@ def __init__(
             "hd_perc": (self.measured_hausdorff_distance_perc, "HDPerc"),
             "masd": (self.measured_masd, "MASD"),
             "nsd": (self.normalised_surface_distance, "NSD"),
+            "avdr": (self.absolute_volume_difference_ratio, "AVDR")
         }
 
         self.pred = pred
@@ -481,7 +483,7 @@ def sensitivity(self):
 
         .. math::
 
-            Sens = \dfrac{TP}{\sharp Ref}
+            Sens = \dfrac{TP}{|Ref|}
 
         Yerushalmy J., Statistical Problems in assessing Methods of Medical Diagnosis with Special reference to X-Ray Techniques, 1947, Public Health Reports, pp1432-1449
 
@@ -501,7 +503,7 @@ def specificity(self):
 
         .. math::
 
-            Spec = \dfrac{TN}{\sharp {1-Ref}}
+            Spec = \dfrac{TN}{|1-Ref|}
 
         Yerushalmy J., Statistical Problems in assessing Methods of Medical Diagnosis with Special reference to X-Ray Techniques, 1947, Public Health Reports, pp1432-1449
 
@@ -551,7 +553,7 @@ def false_positive_rate(self):
 
         .. math::
 
-            FPR = \dfrac{FP}{\sharp \bar{Ref}}
+            FPR = \dfrac{FP}{|1-Ref|}
 
         Burke D, Brundage J, Redfield R., Measurement of the False positive rate in a screening Program for Human Immunodeficiency Virus Infections - 1988 - The New England Journal of Medicine 319 (15) 961-964
 
@@ -638,13 +640,12 @@ def cohens_kappa(self):
 
             CK = \dfrac{p_o - p_e}{1-p_e}
 
-        where
-
-        :math:`p_e = ` expected chance matching and :math:`p_o = `observed accuracy
+        where :math: `p_e = ` expected chance matching and :math: `p_o = `observed accuracy
 
         Cohen, J. A coefficient of agreement for nominal scales - Educational and Psychological Measurement (1960) 20 37-46
 
         :return: cohens_kappa
+
         """
         p_e = self.expected_matching_ck()
         p_o = self.accuracy()
@@ -743,10 +744,12 @@ def dsc(self):
         ..math::
 
             DSC = \dfrac{2TP}{2TP+FP+FN}
+
         
         This is also F:math:`{\\beta}` for :math:`{\\beta}`=1
 
         :return: dsc
+
         """
 
         numerator = 2 * self.tp()
@@ -880,9 +883,10 @@ def intersection_over_reference(self):
 
         .. math::
 
-            IoR = \dfrac{\vert \text{Pred} \bigcap \text{Ref} \vert}{\vert Ref \vert}
+            IoR = \dfrac{| \text{Pred} \cap \text{Ref} |}{| Ref |}
 
         :return: IoR
+
         """
         if self.flag_empty_ref:
             warnings.warn("IoR not defined - Empty reference")
@@ -900,9 +904,10 @@ def intersection_over_union(self):
 
         .. math::
 
-            IoU = \dfrac{\vert Pred \bigcap Ref\vert}{\vert Pred \bigcup Ref \vert}
+            IoU = \dfrac{|Pred \cap Ref|}{| Pred \cup Ref |}
 
         :return: IoU
+
         """
         if self.flag_empty_pred and self.flag_empty_ref:
             warnings.warn("IoU not defined - Both reference and prediction are empty")
@@ -915,8 +920,10 @@ def com_dist(self):
         This function calculates the euclidean distance between the centres
         of mass of the reference and prediction.
 
+        
         :return: Euclidean distance between centre of mass when reference and prediction not empty
         -1 otherwise
+
         """
         
         if self.flag_empty_pred or self.flag_empty_ref:
@@ -981,9 +988,10 @@ def absolute_volume_difference_ratio(self):
 
         .. math::
 
-            AVDR = \dfrac{\vert Pred - Ref\vert}{\vert Ref \vert} 
+            AVDR = \dfrac{| Pred - Ref|}{| Ref |} 
 
         :return: avdr
+
         """
         if self.n_pos_ref() == 0:
             warnings.warn('Empty reference - absolute volume difference ratio not defined')
@@ -1082,6 +1090,7 @@ def boundary_iou(self):
 
         where :math:A_d are the pixels of A within a distance d of the boundary
         :return: boundary_iou
+
         """
         if "boundary_dist" in self.dict_args.keys():
             distance = self.dict_args["boundary_dist"]
@@ -1180,6 +1189,7 @@ def measured_distance(self):
 
         :return: hausdorff distance and average symmetric distance, hausdorff distance at perc
         and masd
+
         """
         if "hd_perc" in self.dict_args.keys():
             perc = self.dict_args["hd_perc"]
@@ -1223,7 +1233,7 @@ def measured_average_distance(self):
 
         .. math::
 
-            ASSD(A,B) = \dfrac{\sum_{a\inA}d(a,B) + \sum_{b\inB}d(b,A)}{|A|+ |B|}
+            ASSD(A,B) = \dfrac{\sum_{a\in A}d(a,B) + \sum_{b\in B}d(b,A)}{|A|+ |B|}
 
         Heimann, T., et al. (2009), Comparison and evaluation of methods for liver segmentation from CT datasets. IEEE Trans Med Imaging. 28(8): p. 1251-65.
         Varduhi Yeghiazaryan and Irina Voiculescu. An overview of current evaluation methods used in medical image segmentation. Department of Computer Science, University of Oxford, 2015.
@@ -1242,9 +1252,11 @@ def measured_masd(self):
 
         .. math::
 
-            MASD(A,B) = \dfrac{1}{2}\left(\dfrac{\sum_{a\in A}d(a,B)}{|A|} + \dfrac{\sum_{b\inB}d(b,A)}{|B|})
+            MASD(A,B) = \dfrac{1}{2}(\dfrac{\sum_{a\in A}d(a,B)}{|A|} + \dfrac{\sum_{b\in B}d(b,A)}{|B|})
         
+            
         :return: masd
+
         """
         masd = self.measured_distance()[3]
         return masd
diff --git a/test/test_metrics/test_calibration_metrics.py b/test/test_metrics/test_calibration_metrics.py
index 6eeb5ef..1dd3d9a 100644
--- a/test/test_metrics/test_calibration_metrics.py
+++ b/test/test_metrics/test_calibration_metrics.py
@@ -120,12 +120,12 @@ def test_kernel_calibration_error():
     expected_median_heuristic = 0.90
     value_median = median_heuristic(pred)
     assert_allclose(value_median, expected_median_heuristic, atol = 0.01)
-    kernel_01 = np.exp(-np.sqrt(0.78)/value_median) * np.ones([3,3])
-    kernel_02 = np.exp(-np.sqrt(0.86)/value_median) * np.ones([3,3])
-    kernel_03 = np.exp(-np.sqrt(0.02)/value_median) * np.ones([3,3])
-    kernel_12 = np.exp(-np.sqrt(1.26)/value_median) * np.ones([3,3])
-    kernel_13 = np.exp(-np.sqrt(0.86)/value_median) * np.ones([3,3])
-    kernel_23 = np.exp(-np.sqrt(1.14)/value_median) * np.ones([3,3])
+    kernel_01 = np.exp(-np.sqrt(0.78)/value_median) * np.eye(3)
+    kernel_02 = np.exp(-np.sqrt(0.86)/value_median) * np.eye(3)
+    kernel_03 = np.exp(-np.sqrt(0.02)/value_median) * np.eye(3)
+    kernel_12 = np.exp(-np.sqrt(1.26)/value_median) * np.eye(3)
+    kernel_13 = np.exp(-np.sqrt(0.86)/value_median) * np.eye(3)
+    kernel_23 = np.exp(-np.sqrt(1.14)/value_median) * np.eye(3)
 
     vect_0 = np.asarray([-0.1, 0.4, -0.3])
     vect_1 = np.asarray([0.2, -0.1, -0.1])
diff --git a/test/test_metrics/test_pairwise_measures.py b/test/test_metrics/test_pairwise_measures.py
index 296f0f2..ea51439 100644
--- a/test/test_metrics/test_pairwise_measures.py
+++ b/test/test_metrics/test_pairwise_measures.py
@@ -5,8 +5,9 @@
     MultiLabelLocSegPairwiseMeasure as MLIS,
 )
 import numpy as np
+
 from MetricsReloaded.utility.utils import one_hot_encode
-from numpy.testing import assert_allclose
+from numpy.testing import assert_allclose, assert_array_equal
 from sklearn.metrics import cohen_kappa_score as cks
 from sklearn.metrics import matthews_corrcoef as mcc
 
@@ -280,6 +281,21 @@ def test_matthews_correlation_coefficient():
     assert_allclose(value_test, expected_mcc, atol=0.001)
     assert_allclose(value_test2, expected_mcc, atol=0.001)
 
+def test_confusion_matrix():
+    """
+    Taking Figure SN3.39 as inspiration
+    """
+
+    mask_ref = np.zeros([5,6])
+    mask_ref[0:5,0:4] = 1
+    mask_pred = np.zeros([5,6])
+    mask_pred[1:4,1:6]=1
+    mpm = MPM(np.reshape(mask_pred,[-1]),np.reshape(mask_ref,[-1]),[0,1])
+    cm_test = mpm.confusion_matrix()
+    cm = np.asarray([[4,11],[6,9]])
+    print(cm_test)
+    assert_array_equal(cm_test,cm)
+
 
 def test_ec3():
     test_true = np.asarray([0, 1, 2, 3, 4])

From 9ff432874560c90c2b21b48b2219d2b59a978192 Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@eduroam-int-dhcp-97-216-55.ucl.ac.uk>
Date: Fri, 8 Nov 2024 10:07:09 +0000
Subject: [PATCH 07/18] Updating net benefit test to match paper

---
 test/test_metrics/test_pairwise_measures.py | 22 ++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/test/test_metrics/test_pairwise_measures.py b/test/test_metrics/test_pairwise_measures.py
index ea51439..66d59e9 100644
--- a/test/test_metrics/test_pairwise_measures.py
+++ b/test/test_metrics/test_pairwise_measures.py
@@ -308,27 +308,35 @@ def test_ec3():
 
 
 def test_netbenefit():
+    """
+    Taking as reference figure SN 2.11 p 51 of Pitfalls paper
+    """
     ref = np.concatenate([np.ones([30]), np.zeros([75])])
     pred = np.ones([105])
-    pred2 = np.concatenate(
-        [np.ones([22]), np.zeros([8]), np.ones([50]), np.zeros([25])]
-    )
+    # pred2 = np.concatenate(
+    #     [np.ones([22]), np.zeros([8]), np.ones([50]), np.zeros([25])]
+    # )
+    pred2 = np.concatenate([np.ones([20]),np.zeros([10]),np.ones([60]),np.zeros([15])])
     ppm = PM(pred, ref, dict_args={"exchange_rate": 1.0 / 9.0})
     value_test = ppm.net_benefit_treated()
     ppm2 = PM(pred2, ref, dict_args={"exchange_rate": 1.0 / 9.0})
     value_test2 = ppm2.net_benefit_treated()
-    ppm3 = PM(pred, ref)
+    ppm3 = PM(pred2, ref)
     value_test3 = ppm3.net_benefit_treated()
+    ppm4 = PM(pred,ref)
+    value_test4 = ppm4.net_benefit_treated()
     print(value_test, value_test2)
     expected_netbenefit1 = 0.206
-    expected_netbenefit2 = 0.157
-    expected_netbenefit3 = -0.429
+    expected_netbenefit2 = 0.127
+    expected_netbenefit3 = -0.381
+    expected_netbenefit4 = -0.429
     assert_allclose(value_test, expected_netbenefit1, atol=0.001)
     assert_allclose(value_test2, expected_netbenefit2, atol=0.001)
     assert_allclose(value_test3, expected_netbenefit3, atol=0.001)
-
+    assert_allclose(value_test4, expected_netbenefit4, atol=0.001)
 
 def test_cohenskappa2():
+
     bpm = PM(f38_pred, f38_ref)
     value_test = bpm.cohens_kappa()
     print("CK f38 ", value_test, cks(f38_pred, f38_ref))

From 0e126b54d7bc97731de7347d0234d7c676ebf91a Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@eduroam-int-dhcp-97-216-55.ucl.ac.uk>
Date: Fri, 8 Nov 2024 10:33:15 +0000
Subject: [PATCH 08/18] Updating accuracy test and documentation of net benefit

---
 MetricsReloaded/metrics/pairwise_measures.py |  2 +-
 test/test_metrics/test_pairwise_measures.py  | 34 ++++++++++++++------
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py
index 593ab00..53a225e 100755
--- a/MetricsReloaded/metrics/pairwise_measures.py
+++ b/MetricsReloaded/metrics/pairwise_measures.py
@@ -813,7 +813,7 @@ def net_benefit_treated(self):
 
         .. math::
 
-            NB = \dfrac{TP}{N} - \dfrac{FP}{N} * ER
+            NB = \dfrac{TP}{TP+TN+FP+FN} - \dfrac{FP}{TP+TN+FP+FN} * ER
 
         where ER relates to the exchange rate. For instance if a suitable exchange rate is to find
         1 positive case among 10 tested (1TP for 9 FP), the exchange rate would be 1/9
diff --git a/test/test_metrics/test_pairwise_measures.py b/test/test_metrics/test_pairwise_measures.py
index 66d59e9..f5a0b73 100644
--- a/test/test_metrics/test_pairwise_measures.py
+++ b/test/test_metrics/test_pairwise_measures.py
@@ -306,24 +306,38 @@ def test_ec3():
     expected_ec = 0.25
     assert_allclose(value_test, expected_ec, atol=0.01)
 
+def test_accuracy():
+    """
+    Taking as reference figure SN 2.11 p51 of Pitfalls paper
+    """
+    ref1 = np.concatenate([np.ones([30]), np.zeros([75])])
+    pred1 = np.ones([105])
+    ref2 = np.concatenate([np.ones([35]),np.zeros([70])])
+    pred2 = np.concatenate([np.ones([20]),np.zeros([15]),np.ones([60]),np.zeros([10])])
+    expected_accuracy1 = 0.286
+    expected_accuracy2 = 0.286
+    ppm1 = PM(pred1, ref1)
+    ppm2 = PM(pred2, ref2)
+    value_test1 = ppm1.accuracy()
+    value_test2 = ppm2.accuracy()
+    assert_allclose(value_test1, expected_accuracy1,atol=0.001)
+    assert_allclose(value_test2, expected_accuracy2,atol=0.001)
 
 def test_netbenefit():
     """
     Taking as reference figure SN 2.11 p 51 of Pitfalls paper
     """
-    ref = np.concatenate([np.ones([30]), np.zeros([75])])
-    pred = np.ones([105])
-    # pred2 = np.concatenate(
-    #     [np.ones([22]), np.zeros([8]), np.ones([50]), np.zeros([25])]
-    # )
-    pred2 = np.concatenate([np.ones([20]),np.zeros([10]),np.ones([60]),np.zeros([15])])
-    ppm = PM(pred, ref, dict_args={"exchange_rate": 1.0 / 9.0})
+    ref1 = np.concatenate([np.ones([30]), np.zeros([75])])
+    pred1 = np.ones([105])
+    ref2 = np.concatenate([np.ones([35]),np.zeros([70])])
+    pred2 = np.concatenate([np.ones([20]),np.zeros([15]),np.ones([60]),np.zeros([10])])
+    ppm = PM(pred1, ref1, dict_args={"exchange_rate": 1.0 / 9.0})
     value_test = ppm.net_benefit_treated()
-    ppm2 = PM(pred2, ref, dict_args={"exchange_rate": 1.0 / 9.0})
+    ppm2 = PM(pred2, ref2, dict_args={"exchange_rate": 1.0 / 9.0})
     value_test2 = ppm2.net_benefit_treated()
-    ppm3 = PM(pred2, ref)
+    ppm3 = PM(pred2, ref2)
     value_test3 = ppm3.net_benefit_treated()
-    ppm4 = PM(pred,ref)
+    ppm4 = PM(pred1,ref1)
     value_test4 = ppm4.net_benefit_treated()
     print(value_test, value_test2)
     expected_netbenefit1 = 0.206

From 09d63adfdd770ef75ab0aedcb8275b7ee39813f1 Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@eduroam-int-dhcp-97-216-55.ucl.ac.uk>
Date: Fri, 8 Nov 2024 10:48:49 +0000
Subject: [PATCH 09/18] updating test for PPV

---
 test/test_metrics/test_pairwise_measures.py | 31 +++++++++++++++++----
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/test/test_metrics/test_pairwise_measures.py b/test/test_metrics/test_pairwise_measures.py
index f5a0b73..52c2664 100644
--- a/test/test_metrics/test_pairwise_measures.py
+++ b/test/test_metrics/test_pairwise_measures.py
@@ -13,6 +13,12 @@
 
 from MetricsReloaded.metrics.prob_pairwise_measures import ProbabilityPairwiseMeasures
 
+#Data for figure SN 2.9 of Pitfalls p
+pred29_1 = np.concatenate([np.ones([45]),np.zeros([5]),np.ones([10]),np.zeros([40])])
+ref29_1 = np.concatenate([np.ones([50]),np.zeros([50])])
+pred29_2 = np.concatenate([np.ones([81]),np.zeros([9]),np.ones([2]),np.zeros([8])])
+ref29_2 = np.concatenate([np.ones([90]),np.zeros([10])])
+
 ### Small size of structures relative to pixel/voxel size (DSC)
 ## Larger structure
 p_large_ref = np.zeros((11, 11))
@@ -500,12 +506,25 @@ def test_sens():
 
 
 def test_ppv():
-    print(f27_pred1, f27_ref1)
-    pm = PM(f27_pred1, f27_ref1)
-    value_test = pm.positive_predictive_values()
-    print("PPV ", value_test)
-    expected_ppv = 0.975
-    assert_allclose(value_test, expected_ppv, atol=0.001)
+    """
+    Taking as inspiration figure SN2.9 p49 Pitfalls
+    """
+    # print(f27_pred1, f27_ref1)
+    # pm = PM(f27_pred1, f27_ref1)
+    # value_test = pm.positive_predictive_values()
+    # print("PPV ", value_test)
+    # expected_ppv = 0.975
+
+    ppm1 = PM(pred29_1, ref29_1)
+    ppm2 = PM(pred29_2, ref29_2)
+    value_test1 = ppm1.positive_predictive_values()
+    value_test2 = ppm2.positive_predictive_values()
+    expected_ppv1 = 0.82
+    expected_ppv2 = 0.98
+
+
+    assert_allclose(value_test1, expected_ppv1, atol=0.01)
+    assert_allclose(value_test2, expected_ppv2, atol=0.01)
 
 
 def test_positive_likelihood_ratio():

From f7bd33c0f603e784f64d6278062299bc2d7000bc Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@eduroam-int-dhcp-97-216-55.ucl.ac.uk>
Date: Fri, 8 Nov 2024 11:40:27 +0000
Subject: [PATCH 10/18] Using figure 2.9 as illustrative example for many tests

---
 test/test_metrics/test_pairwise_measures.py | 134 +++++++++++++-------
 1 file changed, 88 insertions(+), 46 deletions(-)

diff --git a/test/test_metrics/test_pairwise_measures.py b/test/test_metrics/test_pairwise_measures.py
index 52c2664..5bada67 100644
--- a/test/test_metrics/test_pairwise_measures.py
+++ b/test/test_metrics/test_pairwise_measures.py
@@ -18,6 +18,8 @@
 ref29_1 = np.concatenate([np.ones([50]),np.zeros([50])])
 pred29_2 = np.concatenate([np.ones([81]),np.zeros([9]),np.ones([2]),np.zeros([8])])
 ref29_2 = np.concatenate([np.ones([90]),np.zeros([10])])
+ppm29_1 = PM(pred29_1, ref29_1)
+ppm29_2 = PM(pred29_2, ref29_2)
 
 ### Small size of structures relative to pixel/voxel size (DSC)
 ## Larger structure
@@ -274,6 +276,17 @@ def test_voldiff():
     expected_vdiff = 0
     assert_allclose(value_test, expected_vdiff)
 
+def test_matthews_correlation_coefficient_29():
+    """
+    Taking SN 3.9 as figure illustration for MCC p49 Pitfalls
+    """
+    expected_mcc1 = 0.70
+    expected_mcc2 = 0.56
+    value_test1 = ppm29_1.matthews_correlation_coefficient()
+    value_test2 = ppm29_2.matthews_correlation_coefficient()
+    assert_allclose(value_test1, expected_mcc1, atol=0.01)
+    assert_allclose(value_test2, expected_mcc2, atol=0.02)
+
 
 def test_matthews_correlation_coefficient():
     bpm = PM(f38_pred, f38_ref)
@@ -365,15 +378,16 @@ def test_cohenskappa2():
 
 
 def test_negative_predictive_value():
-    f17_ref = np.concatenate([np.ones([50]), np.zeros([50])])
-    f17_pred = np.concatenate(
-        [np.ones([45]), np.zeros([5]), np.ones([10]), np.zeros(40)]
-    )
-    bpm = PM(f17_pred, f17_ref)
-    value_test = bpm.negative_predictive_values()
-    expected_npv = 0.889
-    assert_allclose(value_test, expected_npv, atol=0.001)
-    print("NPV", value_test)
+    """
+    Taking figure SN 2.9 as inspiration p49 Pitfalls
+    """
+    value_test1 = ppm29_1.negative_predictive_values()
+    value_test2 = ppm29_2.negative_predictive_values()
+    expected_npv1 = 0.889
+    expected_npv2 = 0.47
+    assert_allclose(value_test1, expected_npv1, atol=0.001)
+    assert_allclose(value_test2, expected_npv2, atol=0.01)
+    
 
 
 def test_expectedcost():
@@ -384,12 +398,27 @@ def test_expectedcost():
     assert_allclose(value_test, expected_ec, atol=0.01)
 
 
-def test_expectedcost2():
-    mpm = MPM(f27_pred, f27_ref, [0, 1])
-    value_test = mpm.normalised_expected_cost()
-    print("ECn", value_test)
+def test_normalised_expectedcost2():
+    """
+    TAking SN 3.9 as reference p49 pitfalls
+    """
+    value_test1 = ppm29_1.normalised_expected_cost()
+    value_test2 = ppm29_2.normalised_expected_cost()
     expected_ec = 0.30
-    assert_allclose(value_test, expected_ec, atol=0.01)
+    
+    assert_allclose(value_test2, expected_ec, atol=0.01)
+    assert_allclose(value_test1, expected_ec, atol=0.01)
+    
+def test_cohenskappa():
+    """
+    Taking SN 2.9 p49 Pitfalls as reference
+    """
+    value_test1 = ppm29_1.cohens_kappa()
+    value_test2 = ppm29_2.cohens_kappa()
+    expected_ck1 = 0.70
+    expected_ck2 = 0.53
+    assert_allclose(value_test1, expected_ck1, atol=0.01)
+    assert_allclose(value_test2, expected_ck2, atol=0.01)
 
 
 def test_cohenskappa3():
@@ -401,20 +430,27 @@ def test_cohenskappa3():
 
 
 def test_balanced_accuracy2():
-    bpm = PM(f38_pred, f38_ref)
-    value_test = bpm.balanced_accuracy()
-    print("BA f38 ", value_test)
-    expected_ba = 0.87
-    assert_allclose(value_test, expected_ba, atol=0.01)
+    """
+    Taking Figure SN 2.39 as inspiration p49 pitfalls
+    """
+    expected_ba1 = 0.85
+    expected_ba2 = 0.85
+    value_test1 = ppm29_1.balanced_accuracy()
+    value_test2 = ppm29_2.balanced_accuracy()
+    assert_allclose(value_test1, expected_ba1, atol=0.01)
+    assert_allclose(value_test2, expected_ba2, atol=0.01)
 
 
 def test_youden_index2():
-    bpm = PM(f38_pred, f38_ref)
-    value_test = bpm.youden_index()
-    print("J f38 ", value_test)
-    expected_yi = 0.75
-    assert_allclose(value_test, expected_yi, atol=0.01)
-
+    """
+    Taking as inspiration figure SN2.9 p49 Pitfalls
+    """
+    expected_yi1 = 0.70
+    expected_yi2 = 0.70
+    value_test1 = ppm29_1.youden_index()
+    value_test2 = ppm29_2.youden_index()
+    assert_allclose(value_test1, expected_yi1, atol=0.01)
+    assert_allclose(value_test2, expected_yi2, atol=0.01)
 
 def test_mcc():
     list_values = [0, 1, 2, 3]
@@ -425,6 +461,9 @@ def test_mcc():
 
 
 def test_distance_empty():
+    """
+    Testing that output is 0 when reference and prediction empty for calculation of distance
+    """
     pred = np.zeros([14, 14])
     ref = np.zeros([14, 14])
     bpm = PM(pred, ref)
@@ -432,6 +471,16 @@ def test_distance_empty():
     expected_dist = (0, 0, 0, 0)
     assert_allclose(value_test, expected_dist)
 
+def test_fbeta():
+    """
+    Taking inspiration from SN 2.9 - p49 Pitfalls
+    """
+    expected_f11 = 0.86
+    expected_f12 = 0.94
+    value_test1 = ppm29_1.fbeta()
+    value_test2 = ppm29_2.fbeta()
+    assert_allclose(value_test1, expected_f11, atol=0.01)
+    assert_allclose(value_test2, expected_f12, atol=0.01)
 
 def test_dsc_fbeta():
     bpm = PM(p_pred, p_ref)
@@ -509,34 +558,27 @@ def test_ppv():
     """
     Taking as inspiration figure SN2.9 p49 Pitfalls
     """
-    # print(f27_pred1, f27_ref1)
-    # pm = PM(f27_pred1, f27_ref1)
-    # value_test = pm.positive_predictive_values()
-    # print("PPV ", value_test)
-    # expected_ppv = 0.975
-
-    ppm1 = PM(pred29_1, ref29_1)
-    ppm2 = PM(pred29_2, ref29_2)
-    value_test1 = ppm1.positive_predictive_values()
-    value_test2 = ppm2.positive_predictive_values()
+    
+    value_test1 = ppm29_1.positive_predictive_values()
+    value_test2 = ppm29_2.positive_predictive_values()
     expected_ppv1 = 0.82
     expected_ppv2 = 0.98
-
-
     assert_allclose(value_test1, expected_ppv1, atol=0.01)
     assert_allclose(value_test2, expected_ppv2, atol=0.01)
 
 
 def test_positive_likelihood_ratio():
-    f17_ref = np.concatenate([np.ones([50]), np.zeros([50])])
-    f17_pred = np.concatenate(
-        [np.ones([45]), np.zeros([5]), np.ones([10]), np.zeros(40)]
-    )
-    bpm = PM(f17_pred, f17_ref)
-    value_test = bpm.positive_likelihood_ratio()
-    expected_plr = 4.5
-    assert_allclose(value_test, expected_plr, atol=0.01)
-
+    """
+    Taking as inspiration figure SN2.9 p49 Pitfalls
+    """
+    ppm1 = PM(pred29_1, ref29_1)
+    ppm2 = PM(pred29_2, ref29_2)
+    value_test1 = ppm1.positive_likelihood_ratio()
+    value_test2 = ppm2.positive_likelihood_ratio()
+    expected_plr1 = 4.50
+    expected_plr2 = 4.50
+    assert_allclose(value_test1, expected_plr1, atol=0.01)
+    assert_allclose(value_test2, expected_plr2, atol=0.01)
 
 def test_hd():
     f20_ref = np.zeros([14, 14])

From 2bb377ff5f806ef599c33eb4843552f062fb238d Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@eduroam-int-dhcp-97-216-55.ucl.ac.uk>
Date: Fri, 8 Nov 2024 13:30:04 +0000
Subject: [PATCH 11/18] Adding test based on S10

---
 test/test_metrics/test_pairwise_measures.py | 175 +++++++++++++++++++-
 1 file changed, 174 insertions(+), 1 deletion(-)

diff --git a/test/test_metrics/test_pairwise_measures.py b/test/test_metrics/test_pairwise_measures.py
index 5bada67..4183c7b 100644
--- a/test/test_metrics/test_pairwise_measures.py
+++ b/test/test_metrics/test_pairwise_measures.py
@@ -13,7 +13,7 @@
 
 from MetricsReloaded.metrics.prob_pairwise_measures import ProbabilityPairwiseMeasures
 
-#Data for figure SN 2.9 of Pitfalls p
+#Data for figure SN 2.9 of Pitfalls p49
 pred29_1 = np.concatenate([np.ones([45]),np.zeros([5]),np.ones([10]),np.zeros([40])])
 ref29_1 = np.concatenate([np.ones([50]),np.zeros([50])])
 pred29_2 = np.concatenate([np.ones([81]),np.zeros([9]),np.ones([2]),np.zeros([8])])
@@ -21,6 +21,24 @@
 ppm29_1 = PM(pred29_1, ref29_1)
 ppm29_2 = PM(pred29_2, ref29_2)
 
+#Data for figure SN 2.17 of pitfalls p59
+pred217_1 = np.concatenate([np.ones([6]), np.zeros([94])])
+pred217_2 = np.zeros([100])
+ref217 = np.concatenate([np.ones([3]),np.zeros([97])])
+ppm217_1 = PM(pred217_1, ref217)
+ppm217_2 = PM(pred217_2, ref217)
+
+#Data for figure SN 2.10 of pitfalls p50
+ref210 = np.zeros([14,14])
+ref210[5:9,5:9] = 1
+pred210_1 = np.zeros([14,14])
+pred210_1[6:8,6:8] = 1
+pred210_2 = np.zeros([14,14])
+pred210_2[4:10,4:10]=1
+ppm210_1 = PM(pred210_1, ref210)
+ppm210_2 = PM(pred210_2, ref210)
+
+
 ### Small size of structures relative to pixel/voxel size (DSC)
 ## Larger structure
 p_large_ref = np.zeros((11, 11))
@@ -252,6 +270,102 @@
 f59_pred2 = np.zeros([15, 15])
 f59_pred2[4:8, 5:9] = 1
 
+def test_fn_map():
+    """
+    Using SN2.10 as basis of illustration for FP TP FN TN calculation
+    """
+    fn1 = ppm210_1.fn()
+    fn2 = ppm210_2.fn()
+    expected_fn1 = 12
+    expected_fn2 = 0
+    # fn_map_1 = ppm210_1.__fn_map()
+    # expected_fn_map1 = np.zeros([14,14])
+    # expected_fn_map1[5:6,5:9] = 1
+    # expected_fn_map1[8:9,5:9] = 1 
+    # expected_fn_map1[5:9,5:6] = 1
+    # expected_fn_map1[5:9,8:9] = 1
+    # fn_map_2 = ppm210_2.__fn_map()
+    # expected_fn_map2 = np.zeros([14,14])
+    # assert_array_equal(fn_map_1, expected_fn_map1)
+    # assert_array_equal(fn_map_2, expected_fn_map2)
+    assert fn1 == 12
+    assert fn2 == 0 
+
+def test_fp():
+    """
+    Using SN2.10 as illustrative example for calculation of TP TN FP FN
+
+    """
+    fp1 = ppm210_1.fp()
+    fp2 = ppm210_2.fp()
+    expected_fp1 = 0
+    expected_fp2 = 20
+    assert fp1 == expected_fp1
+    assert fp2 == expected_fp2
+
+def test_tp():
+    """
+    Using SN2.10 as illustrative example p50 pitfalls
+    """
+
+    tp1 = ppm210_1.tp()
+    tp2 = ppm210_2.tp()
+    expected_tp1 = 4
+    expected_tp2 = 16
+    assert tp1 == expected_tp1
+    assert tp2 == expected_tp2
+
+def test_tn():
+    """
+    Using SN2.10 as illustrative example p50 Pitfalls paper
+    """
+
+    tn1 = ppm210_1.tn()
+    tn2 = ppm210_2.tn()
+    expected_tn1 = 180
+    expected_tn2 = 160
+    assert tn1 == expected_tn1
+    assert tn2 == expected_tn2
+
+def test_n_pos_ref():
+    expected_n_pos_ref = 16
+    n_pos_ref = ppm210_1.n_pos_ref()
+    assert expected_n_pos_ref == n_pos_ref
+
+def test_n_union():
+    expected_n_union1 = 16
+    expected_n_union2 = 36
+    n_union1 = ppm210_1.n_union()
+    n_union2 = ppm210_2.n_union()
+    assert expected_n_union1 == n_union1
+    assert expected_n_union2 == n_union2
+
+def test_n_intersection():
+    expected_n_intersection1 = 4
+    expected_n_intersection2 = 16
+    n_intersection1 = ppm210_1.n_intersection()
+    n_intersection2 = ppm210_2.n_intersection()
+    assert expected_n_intersection1 == n_intersection1
+    assert expected_n_intersection2 == n_intersection2
+
+def test_n_neg_ref():
+    expected_n_neg_ref = 180
+    n_neg_ref = ppm210_1.n_neg_ref()
+    assert expected_n_neg_ref == n_neg_ref
+
+def test_n_pos_pred():
+    expected_n_pos_pred1 = 4
+    expected_n_pos_pred2 = 36
+    n_pos_pred1 = ppm210_1.n_pos_pred()
+    n_pos_pred2 = ppm210_2.n_pos_pred()
+    assert expected_n_pos_pred2 == n_pos_pred2
+    assert expected_n_pos_pred1 == n_pos_pred1
+
+def test_n_neg_pred():
+    expected_n_neg_pred1 = 192
+    expected_n_neg_pred2 = 160
+    n_neg_pred1 = ppm210_1.n_neg_pred()
+    n_neg_pred2 = ppm210_2.n_neg_pred()
 
 def test_balanced_accuracy():
     list_values = [0, 1, 2, 3]
@@ -276,6 +390,18 @@ def test_voldiff():
     expected_vdiff = 0
     assert_allclose(value_test, expected_vdiff)
 
+def test_specificity():
+    """
+    Using figure 2.17 p59 as example test
+    """
+    value_test1 = ppm217_1.specificity()
+    value_test2 = ppm217_2.specificity()
+    expected_spec1 = 0.97
+    expected_spec2 = 1.00
+    assert_allclose(value_test1, expected_spec1, atol=0.01)
+    assert_allclose(value_test2, expected_spec2, atol=0.01)
+
+
 def test_matthews_correlation_coefficient_29():
     """
     Taking SN 3.9 as figure illustration for MCC p49 Pitfalls
@@ -545,6 +671,29 @@ def test_fbeta():
     assert_allclose(value_test, expected_fbeta, atol=0.001)
     assert_allclose(value_test2, expected_fbeta, atol=0.001)
 
+def test_sensitivity():
+    """
+    Using figure 2.17 p59 as example for test
+    """
+
+    value_test1 = ppm217_1.sensitivity()
+    value_test2 = ppm217_2.sensitivity()
+    expected_sens1 = 1.00
+    expected_sens2 = 0.00
+    assert_allclose(value_test1, expected_sens1, atol=0.01)
+    assert_allclose(value_test2, expected_sens2, atol=0.01)
+
+def test_recall():
+    """
+    Using figure 2.17 p59 as example for test
+    """
+
+    value_test1 = ppm217_1.recall()
+    value_test2 = ppm217_2.recall()
+    expected_rec1 = 1.00
+    expected_rec2 = 0.00
+    assert_allclose(value_test1, expected_rec1, atol=0.01)
+    assert_allclose(value_test2, expected_rec2, atol=0.01)
 
 def test_sens():
     pm = PM(f27_pred1, f27_ref1)
@@ -580,6 +729,30 @@ def test_positive_likelihood_ratio():
     assert_allclose(value_test1, expected_plr1, atol=0.01)
     assert_allclose(value_test2, expected_plr2, atol=0.01)
 
+def test_hausdorff_distances_s210():
+    """
+    Using Figure 2.10 as illustrative example
+    """
+    hausdorff_1 = ppm210_1.measured_hausdorff_distance()
+    hausdorff_2 = ppm210_2.measured_hausdorff_distance()
+    expected_hd1 = 1.41
+    expected_hd2 = 1.41
+    assert_allclose(hausdorff_1,expected_hd1,atol=0.01)
+    assert_allclose(hausdorff_2,expected_hd2,atol=0.01)
+
+def test_masd_s210():
+    """
+    Using Figure 2.10 as illustrative example
+    """
+    masd_1 = ppm210_1.measured_masd()
+    masd_2 = ppm210_2.measured_masd()
+    expected_masd1 = 1.07
+    expected_masd2 = 1.04
+    assert_allclose(masd_1,expected_masd1,atol=0.01)
+    assert_allclose(masd_2,expected_masd2,atol=0.01)
+
+
+
 def test_hd():
     f20_ref = np.zeros([14, 14])
     f20_ref[1, 1] = 1

From 361513567360f9b8195624ad79fced4a7675717d Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@eduroam-int-dhcp-97-216-55.ucl.ac.uk>
Date: Fri, 8 Nov 2024 14:35:33 +0000
Subject: [PATCH 12/18] Updating test distance to map figure 2.10

---
 MetricsReloaded/metrics/pairwise_measures.py |  4 ++++
 test/test_metrics/test_pairwise_measures.py  | 18 ++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py
index 53a225e..2851f8c 100755
--- a/MetricsReloaded/metrics/pairwise_measures.py
+++ b/MetricsReloaded/metrics/pairwise_measures.py
@@ -1170,6 +1170,7 @@ def normalised_surface_distance(self):
         if "nsd" in self.dict_args.keys():
             tau = self.dict_args["nsd"]
         else:
+            warnings.warn('No value set up for NSD tolerance - default to 1')
             tau = 1
         dist_ref, dist_pred, border_ref, border_pred = self.border_distance()
         reg_ref = np.where(
@@ -1178,8 +1179,11 @@ def normalised_surface_distance(self):
         reg_pred = np.where(
             dist_pred <= tau, np.ones_like(dist_pred), np.zeros_like(dist_pred)
         )
+        # print(np.sum(border_pred),np.sum(reg_ref),np.sum(border_ref),np.sum(reg_pred))
+        # print(np.sum(border_pred*reg_ref),np.sum(border_ref*reg_pred))
         numerator = np.sum(border_pred * reg_ref) + np.sum(border_ref * reg_pred)
         denominator = np.sum(border_ref) + np.sum(border_pred)
+        # print(numerator, denominator, tau)
         return numerator / denominator
 
     def measured_distance(self):
diff --git a/test/test_metrics/test_pairwise_measures.py b/test/test_metrics/test_pairwise_measures.py
index 4183c7b..55f9212 100644
--- a/test/test_metrics/test_pairwise_measures.py
+++ b/test/test_metrics/test_pairwise_measures.py
@@ -751,7 +751,25 @@ def test_masd_s210():
     assert_allclose(masd_1,expected_masd1,atol=0.01)
     assert_allclose(masd_2,expected_masd2,atol=0.01)
 
+def test_assd_s210():
+    assd_1 = ppm210_1.measured_average_distance()
+    assd_2 = ppm210_2.measured_average_distance()
+    expected_assd1 = 1.10
+    expected_assd2 = 1.05
+    assert_allclose(assd_1, expected_assd1,atol=0.01)
+    assert_allclose(assd_2, expected_assd2,atol=0.01)
 
+def test_nsd_s210():
+    """
+    Using Figure 2.10 as illustrative example
+    """
+    nsd_1 = ppm210_1.normalised_surface_distance()
+    nsd_2 = ppm210_2.normalised_surface_distance()
+    expected_nsd1 = 0.75
+    expected_nsd2 = 0.875
+    print(nsd_1,nsd_2)
+    assert_allclose(nsd_1,expected_nsd1,atol=0.01)
+    assert_allclose(nsd_2,expected_nsd2,atol=0.01)
 
 def test_hd():
     f20_ref = np.zeros([14, 14])

From cd4cccc68c6e61ad5d54f13dd31d03af2674a7be Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@eduroam-int-dhcp-97-216-55.ucl.ac.uk>
Date: Fri, 8 Nov 2024 16:28:32 +0000
Subject: [PATCH 13/18] updating cldice test based on figure 2.14

---
 test/test_metrics/test_pairwise_measures.py | 80 ++++++++++++++++++---
 1 file changed, 71 insertions(+), 9 deletions(-)

diff --git a/test/test_metrics/test_pairwise_measures.py b/test/test_metrics/test_pairwise_measures.py
index 55f9212..12ae0ad 100644
--- a/test/test_metrics/test_pairwise_measures.py
+++ b/test/test_metrics/test_pairwise_measures.py
@@ -38,6 +38,13 @@
 ppm210_1 = PM(pred210_1, ref210)
 ppm210_2 = PM(pred210_2, ref210)
 
+#Data for figure 2.12 p53
+ref212 = np.zeros([22, 22])
+ref212[2:21, 2:21] = 1
+pred212 = np.zeros([22, 22])
+pred212[3:21, 2:21] = 1
+ppm212_1 = PM(pred212, ref212)
+ppm212_2 = PM(pred212,ref212,dict_args={'boundary_dist':2})
 
 ### Small size of structures relative to pixel/voxel size (DSC)
 ## Larger structure
@@ -70,6 +77,45 @@
 f27_ref2 = f27_pred1
 f27_pred2 = f27_ref1
 
+# Figure ClDice p 53 S2.14
+ref214 = np.zeros([24,24])
+ref214[1:10,7:12]=1
+ref214[10:12,3:19]=1
+ref214[12:15,3:5]=1
+ref214[12:15,15:19]=1
+ref214[14:20,15:17]=1
+ref214[14:15,1:5]=1
+ref214[14:17,2:3]=1
+ref214[14:19,4:5]=1
+ref214[17:18,4:8]=1
+ref214[14:15,15:24]=1
+ref214[12:15,22:23]=1
+ref214[14:17,21:22]=1
+ref214[17:20,5:6]=1
+ref214[17:22,12:13]=1
+ref214[19:20,12:17]=1
+ref214[18:19,15:20]=1
+ref214[17,19]=1
+
+pred214_1 = np.zeros([24,24])
+pred214_1[1:10,7:12]=1
+pred214_1[10:12,3:15]=1
+
+pred214_2 = np.copy(ref214)
+pred214_2[10:14,3:4] = 0
+pred214_2[10:11,3:9] = 0
+pred214_2[10:11,10:19] = 0
+pred214_2[1:11,7:9] = 0
+pred214_2[1:11,10:12]=0
+pred214_2[10:14,18:19]=0
+pred214_2[12:14,15:17]=0
+pred214_2[14:19,15:16]=0
+
+ppm214_1 = PM(pred214_1, ref214)
+ppm214_2 = PM(pred214_2, ref214)
+
+
+
 # panoptic quality
 pq_pred1 = np.zeros([21, 21])
 pq_pred1[5:7, 2:5] = 1
@@ -790,16 +836,32 @@ def test_hd():
 
 
 def test_boundary_iou():
-    f21_ref = np.zeros([22, 22])
-    f21_ref[2:21, 2:21] = 1
-    f21_pred = np.zeros([22, 22])
-    f21_pred[3:21, 2:21] = 1
-    bpm = PM(f21_pred, f21_ref)
-    value_test = bpm.boundary_iou()
-    expected_biou = 0.6
-    assert_allclose(value_test, expected_biou, atol=0.1)
-    assert np.round(value_test, 1) == 0.6
+    """
+    Taking as inspiration figure S 2.12
+    """
+    
+    value_test1 = ppm212_1.boundary_iou()
+    expected_biou_1 = 0.6
+    expected_biou_2 = 0.8
+    value_test2 = ppm212_2.boundary_iou()
+    assert_allclose(value_test1, expected_biou_1, atol=0.1)
+    assert_allclose(value_test2, expected_biou_2, atol=0.1)
+    
+def test_cldsc_s214():
+    value_test1 = ppm214_1.centreline_dsc()
+    value_test2 = ppm214_2.centreline_dsc()
+    expected_cldsc1 = 0.475
+    expected_cldsc2 = 0.78
+    assert_allclose(value_test1, expected_cldsc1, atol=0.01)
+    assert_allclose(value_test2, expected_cldsc2, atol=0.01)
 
+def test_dsc_s214():
+    value_test1 = ppm214_1.dsc()
+    value_test2 = ppm214_2.dsc()
+    expected_dsc1 = 0.666
+    expected_dsc2 = 0.685
+    assert_allclose(value_test1, expected_dsc1, atol=0.01)
+    assert_allclose(value_test2, expected_dsc2, atol=0.01)
 
 def test_cldsc():
     pm1 = PM(pred_clDice_small1, ref_clDice_small)

From 1abd2752cd061601ce261d3b28d8014f4d17fe42 Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@Caroles-MacBook-Pro-3.local>
Date: Mon, 11 Nov 2024 11:59:39 +0000
Subject: [PATCH 14/18] Updating figures and tests

---
 .../metrics/calibration_measures.py           |  53 ++++++++-
 MetricsReloaded/metrics/pairwise_measures.py  |  13 ++-
 test/test_metrics/test_calibration_metrics.py |  37 ++++--
 test/test_metrics/test_pairwise_measures.py   | 109 +++++++++---------
 .../test_prob_pairwise_measures.py            |  10 +-
 5 files changed, 151 insertions(+), 71 deletions(-)

diff --git a/MetricsReloaded/metrics/calibration_measures.py b/MetricsReloaded/metrics/calibration_measures.py
index d8cd78c..938f8c0 100644
--- a/MetricsReloaded/metrics/calibration_measures.py
+++ b/MetricsReloaded/metrics/calibration_measures.py
@@ -31,7 +31,7 @@
 import numpy as np
 import math
 from scipy.special import gamma
-
+import warnings
 # from metrics.pairwise_measures import CacheFunctionOutput
 from MetricsReloaded.utility.utils import (
     CacheFunctionOutput,
@@ -150,6 +150,7 @@ def expectation_calibration_error(self):
         if "bins_ece" in self.dict_args:
             nbins = self.dict_args["bins_ece"]
         else:
+            warnings.warn("Bins ECE not specified in optional arguments dictionary - default set to 10")
             nbins = 10
         step = 1.0 / nbins
         range_values = np.arange(0, 1.00001, step)
@@ -176,7 +177,55 @@ def expectation_calibration_error(self):
             else:
                 list_values.append(nsamples * np.abs(prop - np.mean(pred_sel)))
             numb_samples += nsamples
-        return np.sum(np.asarray(list_values)) / numb_samples
+        ece = np.sum(np.asarray(list_values)) / numb_samples
+        return ece
+    
+
+    def maximum_calibration_error(self):
+        """
+        Derives the maximum calibration error in the case of binary task
+        bins_mce is the key in the dictionary for the number of bins to consider
+        Default is 10
+
+        .. math::
+
+            MCE = max(|\dfrac{1}{|B_m|}\sum_{i \in B_m}1(pred_ik==ref_ik)-\dfrac{1}{|B_m|}\sum_{i \in B_m}pred_i|)
+
+        :return: mce
+
+        """
+        if "bins_mce" in self.dict_args:
+            nbins = self.dict_args["bins_mce"]
+        else:
+            warnings.warn("Bins MCE not specified in optional arguments dictionary - default set to 10")
+            nbins = 10
+        step = 1.0 / nbins
+        range_values = np.arange(0, 1.00001, step)
+        list_values = []
+        numb_samples = 0
+        pred_prob = self.pred[:,1]
+        for (l, u) in zip(range_values[:-1], range_values[1:]):
+            ref_tmp = np.where(
+                np.logical_and(pred_prob > l, pred_prob <= u),
+                self.ref,
+                np.ones_like(self.ref) * -1,
+            )
+            ref_sel = ref_tmp[ref_tmp > -1]
+            nsamples = np.size(ref_sel)
+            prop = np.sum(ref_sel) / nsamples
+            pred_tmp = np.where(
+                np.logical_and(pred_prob > l, pred_prob <= u),
+                pred_prob,
+                np.ones_like(pred_prob) * -1,
+            )
+            pred_sel = pred_tmp[pred_tmp > -1]
+            if nsamples == 0:
+                list_values.append(0)
+            else:
+                list_values.append(np.abs(prop - np.mean(pred_sel)))
+        mce = np.max(np.asarray(list_values))
+        return mce
+
 
     def brier_score(self):
         """
diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py
index 2851f8c..85b8894 100755
--- a/MetricsReloaded/metrics/pairwise_measures.py
+++ b/MetricsReloaded/metrics/pairwise_measures.py
@@ -265,6 +265,11 @@ def __init__(
             "fbeta": (self.fbeta, "FBeta"),
             "dsc":(self.dsc, "DSC"),
             "youden_ind": (self.youden_index, "YoudenInd"),
+            "ppv":(self.positive_predictive_value,'PPV'),
+            "npv":(self.negative_predictive_value,'NPV'),
+            "ior":(self.intersection_over_reference,"IoR"),
+            "sensitivity":(self.sensitivity,"Sens"),
+            "specificity":(self.specificity,"Spec"),
             "mcc": (self.matthews_correlation_coefficient, "MCC"),
             "cldice": (self.centreline_dsc, "CentreLineDSC"),
             "assd": (self.measured_average_distance, "ASSD"),
@@ -693,7 +698,7 @@ def pred_in_ref(self):
         else:
             return 0
 
-    def positive_predictive_values(self):
+    def positive_predictive_value(self):
         """
         Calculates the positive predictive value
 
@@ -785,10 +790,10 @@ def fbeta(self):
             warnings.warn("beta value not specified in option - default set to 1")
             beta = 1
         numerator = (
-            (1 + np.square(beta)) * self.positive_predictive_values() * self.recall()
+            (1 + np.square(beta)) * self.positive_predictive_value() * self.recall()
         )
         denominator = (
-            np.square(beta) * self.positive_predictive_values() + self.recall()
+            np.square(beta) * self.positive_predictive_value() + self.recall()
         )
         if np.isnan(denominator):
             if self.fp() + self.fn() > 0:
@@ -830,7 +835,7 @@ def net_benefit_treated(self):
         net_benefit = tp / n - fp / n * er
         return net_benefit
 
-    def negative_predictive_values(self):
+    def negative_predictive_value(self):
         """
         This function calculates the negative predictive value ratio between
         the number of true negatives and the total number of negative elements
diff --git a/test/test_metrics/test_calibration_metrics.py b/test/test_metrics/test_calibration_metrics.py
index 1dd3d9a..8828f85 100644
--- a/test/test_metrics/test_calibration_metrics.py
+++ b/test/test_metrics/test_calibration_metrics.py
@@ -4,9 +4,7 @@
 from scipy.special import gamma
 from MetricsReloaded.utility.utils import median_heuristic
 
-
-def test_expected_calibration_error():
-    f40_pred = [[1-0.22, 0.22 ],
+pred_224 = [[1-0.22, 0.22 ],
                 [1-0.48, 0.48],
                 [0.51,0.49],
                 [0.04, 0.96],
@@ -17,15 +15,38 @@ def test_expected_calibration_error():
                 [0.66, 0.34],
                 [0.13, 0.87]]
     #f40_pred = [0.22, 0.48, 0.49, 0.96, 0.55, 0.64, 0.78, 0.82, 0.34, 0.87]
-    f40_ref = [0, 1, 0, 0, 1, 1, 1, 1, 1, 0]
-    ppm = CalibrationMeasures(f40_pred, f40_ref)
-    ppm1 = CalibrationMeasures(f40_pred, f40_ref, dict_args={"bins_ece": 2})
-    value_test2 = ppm.expectation_calibration_error()
+ref_224 = [0, 1, 0, 0, 1, 1, 1, 1, 1, 0]
+
+def test_expected_calibration_error():
+    """
+    Using as reference SN 2.24 p67
+    """
+    ppm1 = CalibrationMeasures(pred_224, ref_224, dict_args={"bins_ece": 2})
+    ppm2 = CalibrationMeasures(pred_224, ref_224, dict_args={'bins_ece':5})
+    ppm3 = CalibrationMeasures(pred_224, ref_224)
     value_test1 = ppm1.expectation_calibration_error()
+    value_test2 = ppm2.expectation_calibration_error()
+    value_test3 = ppm3.expectation_calibration_error()
     expected_ece1 = 0.11
-    expected_ece2 = 0.36
+    expected_ece2 = 0.32
+    expected_ece3 = 0.36
+    assert_allclose(value_test1, expected_ece1, atol=0.01)
+    assert_allclose(value_test2, expected_ece2, atol=0.01)
+    assert_allclose(value_test3, expected_ece3, atol=0.01)
+
+def test_maximum_calibration_error():
+    ppm1 = CalibrationMeasures(pred_224, ref_224, dict_args={"bins_mce": 2})
+    ppm2 = CalibrationMeasures(pred_224, ref_224, dict_args={'bins_mce':5})
+    ppm3 = CalibrationMeasures(pred_224, ref_224)
+    value_test1 = ppm1.maximum_calibration_error()
+    value_test2 = ppm2.maximum_calibration_error()
+    value_test3 = ppm3.maximum_calibration_error()
+    expected_ece1 = 0.12
+    expected_ece2 = 0.55
+    expected_ece3 = 0.96
     assert_allclose(value_test1, expected_ece1, atol=0.01)
     assert_allclose(value_test2, expected_ece2, atol=0.01)
+    assert_allclose(value_test3, expected_ece3, atol=0.01)
 
 
 def test_logarithmic_score():
diff --git a/test/test_metrics/test_pairwise_measures.py b/test/test_metrics/test_pairwise_measures.py
index 12ae0ad..4b8c32c 100644
--- a/test/test_metrics/test_pairwise_measures.py
+++ b/test/test_metrics/test_pairwise_measures.py
@@ -46,6 +46,14 @@
 ppm212_1 = PM(pred212, ref212)
 ppm212_2 = PM(pred212,ref212,dict_args={'boundary_dist':2})
 
+#Data for figure 5c (Hausdoff with annotation error p14 Pitfalls)
+ref5c = np.zeros([14, 14])
+ref5c[1, 1] = 1
+ref5c[9:12, 9:12] = 1
+pred5c = np.zeros([14, 14])
+pred5c [9:12, 9:12] = 1
+bpm5c = PM(pred5c, ref5c, dict_args={'hd_perc':95})
+
 ### Small size of structures relative to pixel/voxel size (DSC)
 ## Larger structure
 p_large_ref = np.zeros((11, 11))
@@ -77,7 +85,7 @@
 f27_ref2 = f27_pred1
 f27_pred2 = f27_ref1
 
-# Figure ClDice p 53 S2.14
+# Figure ClDice p 53 S2.14 pitfalls paper
 ref214 = np.zeros([24,24])
 ref214[1:10,7:12]=1
 ref214[10:12,3:19]=1
@@ -116,26 +124,26 @@
 
 
 
-# panoptic quality
-pq_pred1 = np.zeros([21, 21])
-pq_pred1[5:7, 2:5] = 1
-pq_pred2 = np.zeros([21, 21])
-pq_pred2[14:18, 4:6] = 1
-pq_pred2[16, 3] = 1
-pq_pred3 = np.zeros([21, 21])
-pq_pred3[14:18, 7:12] = 1
-pq_pred4 = np.zeros([21, 21])
-pq_pred4[2:8, 13:16] = 1
-pq_pred4[2:4, 12] = 1
-
-pq_ref1 = np.zeros([21, 21])
-pq_ref1[8:11, 3] = 1
-pq_ref1[9, 2:5] = 1
-pq_ref2 = np.zeros([21, 21])
-pq_ref2[14:19, 7:13] = 1
-pq_ref3 = np.zeros([21, 21])
-pq_ref3[2:7, 14:17] = 1
-pq_ref3[2:4, 12:14] = 1
+# panoptic quality Figure 3.51 p96
+pq_pred1 = np.zeros([18, 18])
+pq_pred1[ 3:7,1:3] = 1
+pq_pred1[3:6,3:7]=1
+pq_pred2 = np.zeros([18, 18])
+pq_pred2[13:16,4:6] = 1
+pq_pred3 = np.zeros([18, 18])
+pq_pred3[7:12,13:17] = 1
+pq_pred4 = np.zeros([18, 18])
+pq_pred4[13:15,13:17] = 1
+pq_pred4[15,15] = 1
+
+pq_ref1 = np.zeros([18, 18])
+pq_ref1[2:7, 1:3] = 1
+pq_ref1[2:5,3:6] = 1
+pq_ref2 = np.zeros([18, 18])
+pq_ref2[6:12,12:17] = 1
+pq_ref3 = np.zeros([18, 18])
+pq_ref3[14:15:,7:10] = 1
+pq_ref3[13:16,8:9] = 1
 
 f27_pred = np.concatenate([np.ones([81]), np.zeros([9]), np.ones([2]), np.zeros([8])])
 f27_ref = np.concatenate([np.ones([90]), np.zeros([10])])
@@ -324,16 +332,6 @@ def test_fn_map():
     fn2 = ppm210_2.fn()
     expected_fn1 = 12
     expected_fn2 = 0
-    # fn_map_1 = ppm210_1.__fn_map()
-    # expected_fn_map1 = np.zeros([14,14])
-    # expected_fn_map1[5:6,5:9] = 1
-    # expected_fn_map1[8:9,5:9] = 1 
-    # expected_fn_map1[5:9,5:6] = 1
-    # expected_fn_map1[5:9,8:9] = 1
-    # fn_map_2 = ppm210_2.__fn_map()
-    # expected_fn_map2 = np.zeros([14,14])
-    # assert_array_equal(fn_map_1, expected_fn_map1)
-    # assert_array_equal(fn_map_2, expected_fn_map2)
     assert fn1 == 12
     assert fn2 == 0 
 
@@ -553,8 +551,8 @@ def test_negative_predictive_value():
     """
     Taking figure SN 2.9 as inspiration p49 Pitfalls
     """
-    value_test1 = ppm29_1.negative_predictive_values()
-    value_test2 = ppm29_2.negative_predictive_values()
+    value_test1 = ppm29_1.negative_predictive_value()
+    value_test2 = ppm29_2.negative_predictive_value()
     expected_npv1 = 0.889
     expected_npv2 = 0.47
     assert_allclose(value_test1, expected_npv1, atol=0.001)
@@ -699,7 +697,7 @@ def test_nsd2():
     assert_allclose(value_test, expected_nsd2, atol=0.01)
 
 
-def test_iou():
+def test_intersection_over_union():
     bpm = PM(p_pred, p_ref)
     value_test = bpm.intersection_over_union()
     print("IoU ", value_test)
@@ -707,15 +705,19 @@ def test_iou():
     assert_allclose(value_test, expected_iou, atol=0.01)
 
 
-def test_fbeta():
-    pm = PM(p_large_pred1, p_large_ref)
-    pm2 = PM(p_large_pred1, p_large_ref, dict_args={"beta": 1})
-    value_test = pm.fbeta()
-    value_test2 = pm2.fbeta()
-    print(value_test)
-    expected_fbeta = 0.986
-    assert_allclose(value_test, expected_fbeta, atol=0.001)
-    assert_allclose(value_test2, expected_fbeta, atol=0.001)
+def test_fbeta_beta_value():
+    """
+    Taking inspiration from SN 2.9 - p49 Pitfalls
+    """
+    expected_f11 = 0.86
+    expected_f12 = 0.94
+    ppm29_1.dict_args={'beta':1}
+    ppm29_2.dict_args={'beta':1}
+    value_test1 = ppm29_1.fbeta()
+    value_test2 = ppm29_2.fbeta()
+    assert_allclose(value_test1, expected_f11, atol=0.01)
+    assert_allclose(value_test2, expected_f12, atol=0.01)
+
 
 def test_sensitivity():
     """
@@ -749,13 +751,13 @@ def test_sens():
     assert_allclose(value_test, expected_sens, atol=0.01)
 
 
-def test_ppv():
+def test_positive_predictive_value():
     """
     Taking as inspiration figure SN2.9 p49 Pitfalls
     """
     
-    value_test1 = ppm29_1.positive_predictive_values()
-    value_test2 = ppm29_2.positive_predictive_values()
+    value_test1 = ppm29_1.positive_predictive_value()
+    value_test2 = ppm29_2.positive_predictive_value()
     expected_ppv1 = 0.82
     expected_ppv2 = 0.98
     assert_allclose(value_test1, expected_ppv1, atol=0.01)
@@ -817,15 +819,12 @@ def test_nsd_s210():
     assert_allclose(nsd_1,expected_nsd1,atol=0.01)
     assert_allclose(nsd_2,expected_nsd2,atol=0.01)
 
-def test_hd():
-    f20_ref = np.zeros([14, 14])
-    f20_ref[1, 1] = 1
-    f20_ref[9:12, 9:12] = 1
-    f20_pred = np.zeros([14, 14])
-    f20_pred[9:12, 9:12] = 1
-    bpm = PM(f20_pred, f20_ref, dict_args={"hd_perc": 95})
-    hausdorff_distance = bpm.measured_hausdorff_distance()
-    hausdorff_distance_perc = bpm.measured_hausdorff_distance_perc()
+def test_hausdorff_distance_5c():
+    """
+    Using figure 5c p14 as illustration for calculation of HD and HD95
+    """
+    hausdorff_distance = bpm5c.measured_hausdorff_distance()
+    hausdorff_distance_perc = bpm5c.measured_hausdorff_distance_perc()
     print(hausdorff_distance_perc)
     expected_hausdorff_distance = 11.31
     expected_hausdorff_distance_perc = 6.79
diff --git a/test/test_metrics/test_prob_pairwise_measures.py b/test/test_metrics/test_prob_pairwise_measures.py
index f6bebbe..7ce4990 100644
--- a/test/test_metrics/test_prob_pairwise_measures.py
+++ b/test/test_metrics/test_prob_pairwise_measures.py
@@ -14,7 +14,10 @@
 from MetricsReloaded.metrics.prob_pairwise_measures import ProbabilityPairwiseMeasures
 
 
-def test_auc():
+def test_auroc():
+    """
+    Based on SN2.18 p60 of Pitfalls paper
+    """
     ref = np.asarray([0, 0, 0, 1, 1, 1])
     pred_proba = np.asarray([0.21, 0.35, 0.63, 0.92, 0.32, 0.79])
     ppm = ProbabilityPairwiseMeasures(pred_proba, ref)
@@ -24,7 +27,10 @@ def test_auc():
     assert_allclose(value_test, expected_auc, atol=0.01)
 
 
-def test_ap():
+def test_average_precision():
+    """
+    Based on SN2.18 p60 of pitfalls paper
+    """
     ref = np.asarray([0, 0, 0, 1, 1, 1])
     pred_proba = np.asarray([0.21, 0.35, 0.63, 0.92, 0.32, 0.79])
     ppm = ProbabilityPairwiseMeasures(pred_proba, ref)

From aa56a00e01f03b9d06070b533bda29be5f3954f7 Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@Caroles-MacBook-Pro-3.local>
Date: Mon, 11 Nov 2024 15:44:29 +0000
Subject: [PATCH 15/18] Update to include figure 6a

---
 .../utility/assignment_localization.py        |  2 ++
 .../test_assignment_localization.py           | 32 ++++++++++++++++++-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/MetricsReloaded/utility/assignment_localization.py b/MetricsReloaded/utility/assignment_localization.py
index 16cfea7..b120f12 100644
--- a/MetricsReloaded/utility/assignment_localization.py
+++ b/MetricsReloaded/utility/assignment_localization.py
@@ -437,10 +437,12 @@ def initial_mapping(self):
             possible_binary = np.where(
                 matrix < self.thresh, np.ones_like(matrix), np.zeros_like(matrix)
             )
+            print(np.sum(possible_binary), "Possible matches from com or dist")
         else:
             possible_binary = np.where(
                 matrix > self.thresh, np.ones_like(matrix), np.zeros_like(matrix)
             )
+            print(np.sum(possible_binary), "Possible matches")
 
         list_valid = []
         list_matching = []
diff --git a/test/test_utility/test_assignment_localization.py b/test/test_utility/test_assignment_localization.py
index df68cd6..991cb37 100644
--- a/test/test_utility/test_assignment_localization.py
+++ b/test/test_utility/test_assignment_localization.py
@@ -6,10 +6,28 @@
 )
 from MetricsReloaded.utility.assignment_localization import AssignmentMapping
 import numpy as np
-from numpy.testing import assert_allclose
+from numpy.testing import assert_allclose, assert_array_almost_equal
 from sklearn.metrics import cohen_kappa_score as cks
 from sklearn.metrics import matthews_corrcoef as mcc
 
+#Data for figure 6a testing of assignment and average precision
+ref6c1 = np.asarray([3,2,7,5])
+ref6c2 = np.asarray([7,9,8,11])
+ref6c3 = np.asarray([1,16,3,18])
+ref6c4 = np.asarray([14,14,16,18])
+
+pred6c1 = np.asarray([2,3,6,6])
+pred6c2 = np.asarray([2,15,4,17])
+pred6c3 = np.asarray([13,13,15,17])
+pred6c4 = np.asarray([16,7,19,10])
+pred6c5 = np.asarray([12,2,15,4])
+
+pred_proba_6c = [[0.05, 0.95],[0.30,0.70],[0.20,0.80],[0.20,0.80],[0.10,0.90]]
+
+pred_boxes_6c = [pred6c1, pred6c2, pred6c3, pred6c4, pred6c5]
+ref_boxes_6c = [ref6c1, ref6c2, ref6c3, ref6c4]
+
+
 
 ## Data for figure 59 and testing of localisation
 f59_ref1 = np.zeros([15, 15])
@@ -22,6 +40,18 @@
 f59_pred2 = np.zeros([15, 15])
 f59_pred2[4:8, 5:9] = 1
 
+def test_assignment_6c():
+    asm1 = AssignmentMapping(pred_loc=pred_boxes_6c, ref_loc=ref_boxes_6c, pred_prob=pred_proba_6c, thresh=0.1,localization='box_iou')
+    df_matching, df_fn, df_fp, list_valid = asm1.initial_mapping()
+    print(asm1.matrix, df_matching, df_fp, df_fn, list_valid)
+    numb_fn = df_fn.shape[0]
+    numb_fp = df_fp.shape[0]
+    expected_fn = 1
+    expected_fp = 2
+    assert expected_fn ==  numb_fn
+    assert expected_fp == numb_fp
+    assert_array_almost_equal(np.asarray(list_valid),np.asarray([0,1,2]))
+
 def test_check_localization():
     ref_box = [[2,2,4,4]]
     ref_com = [[3,3]]

From 663de491c65f9418a95db2af19c727f14d101f71 Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@Caroles-MacBook-Pro-3.local>
Date: Tue, 12 Nov 2024 09:54:43 +0000
Subject: [PATCH 16/18] Updating NLL and reference to cheat sheet

---
 .../metrics/calibration_measures.py           |   8 +-
 test/test_metrics/test_calibration_metrics.py |   5 +-
 .../test_assignment_localization.py           | 129 ++++++++++++++++--
 3 files changed, 126 insertions(+), 16 deletions(-)

diff --git a/MetricsReloaded/metrics/calibration_measures.py b/MetricsReloaded/metrics/calibration_measures.py
index 938f8c0..ca04ca5 100644
--- a/MetricsReloaded/metrics/calibration_measures.py
+++ b/MetricsReloaded/metrics/calibration_measures.py
@@ -460,10 +460,14 @@ def negative_log_likelihood(self):
 
         George Cybenko, Dianne P O’Leary, and Jorma Rissanen. 1998. The Mathematics of Information Coding, Extraction
         and Distribution. Vol. 107. Springer Science & Business Media.
+        Cheat Sheet p 116 - Figure SN 3.71
 
         .. math::
 
-            NLL = -\sum_{i=1}{N} log(p_{i,k} | y_i=k)
+            NLL = -\dfrac{1}{N}\sum_{i=1}^{N}\sum_{k=1}^{C} y_{ik} \dot log(p_{i,k})
+
+        where :math: `y_{ik}` the outcome is 1 if the class of :math: `y_{i}` is k and :math: `p_{ik}` is the predicted 
+        probability for sample :math: `x_i` and class k
 
         :return: NLL
 
@@ -471,7 +475,7 @@ def negative_log_likelihood(self):
         log_pred = np.log(self.pred)
         numb_samples = self.pred.shape[0]
         ll = np.sum(log_pred[range(numb_samples), self.ref])
-        nll = -1 * ll
+        nll = -1/numb_samples * ll
         return nll
 
     def to_dict_meas(self, fmt="{:.4f}"):
diff --git a/test/test_metrics/test_calibration_metrics.py b/test/test_metrics/test_calibration_metrics.py
index 8828f85..295ad42 100644
--- a/test/test_metrics/test_calibration_metrics.py
+++ b/test/test_metrics/test_calibration_metrics.py
@@ -35,6 +35,9 @@ def test_expected_calibration_error():
     assert_allclose(value_test3, expected_ece3, atol=0.01)
 
 def test_maximum_calibration_error():
+    """
+    Using figure 2.24 p67 of pitfalls as reference
+    """
     ppm1 = CalibrationMeasures(pred_224, ref_224, dict_args={"bins_mce": 2})
     ppm2 = CalibrationMeasures(pred_224, ref_224, dict_args={'bins_mce':5})
     ppm3 = CalibrationMeasures(pred_224, ref_224)
@@ -88,7 +91,7 @@ def test_negative_log_likelihood():
     pred_nll = [[0.1, 0.8, 0.05, 0.1], [0.6, 0.1, 0, 0.7], [0.3, 0.1, 0.95, 0.2]]
     ref_nll = np.asarray(ref_nll)
     pred_nll = np.asarray(pred_nll).T
-    expected_nll = -1 * (np.log(0.8) + np.log(0.6) + np.log(0.7) + np.log(0.95))
+    expected_nll = -1/4 * (np.log(0.8) + np.log(0.6) + np.log(0.7) + np.log(0.95))
     cm = CalibrationMeasures(pred_nll, ref_nll)
     value_test = cm.negative_log_likelihood()
     assert_allclose(value_test, expected_nll)
diff --git a/test/test_utility/test_assignment_localization.py b/test/test_utility/test_assignment_localization.py
index 991cb37..bae37eb 100644
--- a/test/test_utility/test_assignment_localization.py
+++ b/test/test_utility/test_assignment_localization.py
@@ -11,22 +11,46 @@
 from sklearn.metrics import matthews_corrcoef as mcc
 
 #Data for figure 6a testing of assignment and average precision
-ref6c1 = np.asarray([3,2,7,5])
-ref6c2 = np.asarray([7,9,8,11])
-ref6c3 = np.asarray([1,16,3,18])
-ref6c4 = np.asarray([14,14,16,18])
+ref6a1 = np.asarray([3,2,7,5])
+ref6a2 = np.asarray([7,9,8,11])
+ref6a3 = np.asarray([1,16,3,18])
+ref6a4 = np.asarray([14,14,16,18])
 
-pred6c1 = np.asarray([2,3,6,6])
-pred6c2 = np.asarray([2,15,4,17])
-pred6c3 = np.asarray([13,13,15,17])
-pred6c4 = np.asarray([16,7,19,10])
-pred6c5 = np.asarray([12,2,15,4])
+pred6a1 = np.asarray([2,3,6,6])
+pred6a2 = np.asarray([2,15,4,17])
+pred6a3 = np.asarray([13,13,15,17])
+pred6a4 = np.asarray([16,7,19,10])
+pred6a5 = np.asarray([12,2,15,4])
 
-pred_proba_6c = [[0.05, 0.95],[0.30,0.70],[0.20,0.80],[0.20,0.80],[0.10,0.90]]
+pred_proba_6a = [[0.05, 0.95],[0.30,0.70],[0.20,0.80],[0.20,0.80],[0.10,0.90]]
 
-pred_boxes_6c = [pred6c1, pred6c2, pred6c3, pred6c4, pred6c5]
-ref_boxes_6c = [ref6c1, ref6c2, ref6c3, ref6c4]
+pred_boxes_6a = [pred6a1, pred6a2, pred6a3, pred6a4, pred6a5]
+ref_boxes_6a = [ref6a1, ref6a2, ref6a3, ref6a4]
 
+#Data from Panoptic Quality - 3.51 p96
+#Figure 3.51 p96
+pq_pred1 = np.zeros([18, 18])
+pq_pred1[ 3:7,1:3] = 1
+pq_pred1[3:6,3:7]=1
+pq_pred2 = np.zeros([18, 18])
+pq_pred2[13:16,4:6] = 1
+pq_pred3 = np.zeros([18, 18])
+pq_pred3[7:12,13:17] = 1
+pq_pred4 = np.zeros([18, 18])
+pq_pred4[13:15,13:17] = 1
+pq_pred4[15,15] = 1
+
+pq_ref1 = np.zeros([18, 18])
+pq_ref1[2:7, 1:3] = 1
+pq_ref1[2:5,3:6] = 1
+pq_ref2 = np.zeros([18, 18])
+pq_ref2[6:12,12:17] = 1
+pq_ref3 = np.zeros([18, 18])
+pq_ref3[14:15:,7:10] = 1
+pq_ref3[13:16,8:9] = 1
+
+ref_351 = [pq_ref1, pq_ref2, pq_ref3]
+pred_351 = [pq_pred1, pq_pred2, pq_pred3,pq_pred4]
 
 
 ## Data for figure 59 and testing of localisation
@@ -41,7 +65,7 @@
 f59_pred2[4:8, 5:9] = 1
 
 def test_assignment_6c():
-    asm1 = AssignmentMapping(pred_loc=pred_boxes_6c, ref_loc=ref_boxes_6c, pred_prob=pred_proba_6c, thresh=0.1,localization='box_iou')
+    asm1 = AssignmentMapping(pred_loc=pred_boxes_6a, ref_loc=ref_boxes_6a, pred_prob=pred_proba_6a, thresh=0.1,localization='box_iou')
     df_matching, df_fn, df_fp, list_valid = asm1.initial_mapping()
     print(asm1.matrix, df_matching, df_fp, df_fn, list_valid)
     numb_fn = df_fn.shape[0]
@@ -52,6 +76,7 @@ def test_assignment_6c():
     assert expected_fp == numb_fp
     assert_array_almost_equal(np.asarray(list_valid),np.asarray([0,1,2]))
 
+
 def test_check_localization():
     ref_box = [[2,2,4,4]]
     ref_com = [[3,3]]
@@ -209,6 +234,84 @@ def test_pairwise_pointcomdist():
     expected_matrix = np.asarray([[0, 9.22],[8.49, 1]])
     assert_allclose(am.matrix, expected_matrix,atol=0.01)
 
+def test_pairwise_boxiou_6a():
+    """
+    Using figure 6a p14 pitfalls as illustration for boxiou pairwise example
+    """
+    asm1 = AssignmentMapping(pred_loc=pred_boxes_6a, ref_loc=ref_boxes_6a, pred_prob=pred_proba_6a, thresh=0.1,localization='box_iou')
+    df_matching, df_fn, df_fp, list_valid = asm1.initial_mapping()
+    expected_matrix = np.asarray([[0.42857143, 0.  ,       0.  ,       0.        ], 
+                                  [0. ,        0.  ,       0.28571429, 0.        ],
+                                  [0. ,        0.  ,       0.     ,    0.36363636],
+                                  [0.  ,       0.  ,       0.     ,    0.        ],
+                                  [0. ,        0.  ,       0.    ,     0.        ]])
+    assert_array_almost_equal(expected_matrix, asm1.matrix)
+
+def test_com_from_refbox_6a():
+    """
+    Using figure 6a as illustration
+    """
+
+    asm1 = AssignmentMapping(pred_loc=pred_boxes_6a, ref_loc=ref_boxes_6a, pred_prob=pred_proba_6a, thresh=0.1,localization='box_iou')
+    asm1.com_fromrefbox()
+    test_com = asm1.ref_loc_mod
+    expected_ref_com = [[5,3.5],[7.5,10],[2,17],[15,16]]
+    assert_array_almost_equal(np.asarray(expected_ref_com), test_com)
+
+def test_com_from_predbox_6a():
+    """
+    Using figure 6a as illustration
+    """
+    asm1 = AssignmentMapping(pred_loc=pred_boxes_6a, ref_loc=ref_boxes_6a, pred_prob=pred_proba_6a, thresh=0.1,localization='box_iou')
+    asm1.com_frompredbox()
+    test_com = asm1.pred_loc_mod
+    print(test_com)
+    expected_pred_com = [[4,4.5],[3,16],[14,15],[17.5,8.5],[13.5,3]]
+    assert_array_almost_equal(np.asarray(expected_pred_com), test_com,decimal=2)
+
+def test_comfrompredmask_351():
+    """
+    Using figure 3.51 (p96 Panoptic quality) as illustration
+    """
+    asm = AssignmentMapping(pred_loc=pred_351, ref_loc=ref_351,pred_prob=np.asarray([1,1,1,1]),  thresh=0.1,localization='mask_iou')
+    expected_predcom = [[4.2,3.3],[14,4.5],[9,14.5],[13.66,14.55]]
+    asm.com_frompredmask()
+    test_com = asm.pred_loc_mod
+    assert_array_almost_equal(np.asarray(expected_predcom),test_com,decimal=2)
+
+def test_comfromrefmask_351():
+    """
+    Using figure 3.51 (p96 Panoptic quality) as illustrative example
+    """
+    asm = AssignmentMapping(pred_loc=pred_351, ref_loc=ref_351,pred_prob=np.asarray([1,1,1,1]),  thresh=0.1,localization='mask_iou')
+    expected_refcom = [[3.53,2.68],[8.5,14],[14,8]]
+    asm.com_fromrefmask()
+    test_com = asm.ref_loc_mod
+    assert_array_almost_equal(np.asarray(expected_refcom),test_com,decimal=2)
+
+def test_box_fromrefmask():
+    """
+    Using fig 3.51 p96 as illustrative example
+    """
+    asm = AssignmentMapping(pred_loc=pred_351, ref_loc=ref_351,pred_prob=np.asarray([1,1,1,1]),  thresh=0.1,localization='mask_iou')
+    asm.box_fromrefmask()
+    test_box = asm.ref_loc_mod
+    expected_box = [[2,1,6,5],[6,12,11,16],[13,7,15,9]]
+    assert_array_almost_equal(np.asarray(expected_box),test_box)
+
+def test_box_frompredmask():
+    """
+    Using fig 3.51 p96 as illustrative example
+    """
+    asm = AssignmentMapping(pred_loc=pred_351, ref_loc=ref_351,pred_prob=np.asarray([1,1,1,1]),  thresh=0.1,localization='mask_iou')
+    asm.box_frompredmask()
+    test_box = asm.pred_loc_mod
+    expected_box = [[3,1,6,6],[13,4,15,5],[7,13,11,16],[13,13,15,16]]
+    assert_array_almost_equal(np.asarray(expected_box),test_box)
+
+
+
+
 
 def test_localization():
     ref = [f59_ref1, f59_ref2]

From 069ab10d05a67f849d2b1cf6e8d4c3abc7a64466 Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@Caroles-MacBook-Pro-3.local>
Date: Fri, 13 Dec 2024 14:08:20 +0000
Subject: [PATCH 17/18] Correction of warning for probabilistic input

---
 MetricsReloaded/metrics/calibration_measures.py       | 3 +++
 MetricsReloaded/processes/mixed_measures_processes.py | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/MetricsReloaded/metrics/calibration_measures.py b/MetricsReloaded/metrics/calibration_measures.py
index ca04ca5..44448ca 100644
--- a/MetricsReloaded/metrics/calibration_measures.py
+++ b/MetricsReloaded/metrics/calibration_measures.py
@@ -138,6 +138,9 @@ def expectation_calibration_error(self):
         """
         Derives the expectation calibration error in the case of binary task
         bins_ece is the key in the dictionary for the number of bins to consider
+        Cheat sheet SN 3.68 p113
+        Defined in Mahdi Pakdaman Naeini, Gregory Cooper, and Milos Hauskrecht. Obtaining well calibrated probabilities using
+        bayesian binning. In Twenty-Ninth AAAI Conference on Artificial Intelligence, 2015.
         Default is 10
 
         .. math::
diff --git a/MetricsReloaded/processes/mixed_measures_processes.py b/MetricsReloaded/processes/mixed_measures_processes.py
index d81d50f..228124d 100644
--- a/MetricsReloaded/processes/mixed_measures_processes.py
+++ b/MetricsReloaded/processes/mixed_measures_processes.py
@@ -776,7 +776,7 @@ def per_label_dict(self):
                         dict_mt["label"] = lab
                         dict_mt["case"] = name
                         list_mt.append(dict_mt)
-                    else:
+                    if not self.flag_valid_proba and len(self.measures_mt)>0:
                         warnings.warn('No probabilistic input or no probabilistic measure so impossible to get multi-threshold metric')
                 else:
                     list_pred.append(pred_tmp)

From a88e6eda15563233785145a3dd306dbf16d62ee3 Mon Sep 17 00:00:00 2001
From: Carole Sudre <carolesudre@Caroles-MacBook-Pro-3.local>
Date: Fri, 13 Dec 2024 14:12:52 +0000
Subject: [PATCH 18/18] Jupyter notebook to visualise relevant test figures

---
 examples/FiguresTestMetricsReloaded.ipynb | 332 ++++++++++++++++++++++
 1 file changed, 332 insertions(+)
 create mode 100644 examples/FiguresTestMetricsReloaded.ipynb

diff --git a/examples/FiguresTestMetricsReloaded.ipynb b/examples/FiguresTestMetricsReloaded.ipynb
new file mode 100644
index 0000000..9924a1d
--- /dev/null
+++ b/examples/FiguresTestMetricsReloaded.ipynb
@@ -0,0 +1,332 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "d0295d7c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "from matplotlib import pyplot as plt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "5ac4439c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Figure clDice p\n",
+    "\n",
+    "ref214 = np.zeros([24,24])\n",
+    "ref214[1:10,7:12]=1\n",
+    "ref214[10:12,3:19]=1\n",
+    "ref214[12:15,3:5]=1\n",
+    "ref214[12:15,15:19]=1\n",
+    "ref214[14:20,15:17]=1\n",
+    "ref214[14:15,1:5]=1\n",
+    "ref214[14:17,2:3]=1\n",
+    "ref214[14:19,4:5]=1\n",
+    "ref214[17:18,4:8]=1\n",
+    "ref214[14:15,15:24]=1\n",
+    "ref214[12:15,22:23]=1\n",
+    "ref214[14:17,21:22]=1\n",
+    "ref214[17:20,5:6]=1\n",
+    "ref214[17:22,12:13]=1\n",
+    "ref214[19:20,12:17]=1\n",
+    "ref214[18:19,15:20]=1\n",
+    "ref214[17,19]=1\n",
+    "\n",
+    "pred214_1 = np.zeros([24,24])\n",
+    "pred214_1[1:10,7:12]=1\n",
+    "pred214_1[10:12,3:15]=1\n",
+    "\n",
+    "pred214_2 = np.copy(ref214)\n",
+    "pred214_2[10:14,3:4] = 0\n",
+    "pred214_2[10:11,3:9] = 0\n",
+    "pred214_2[10:11,10:19] = 0\n",
+    "pred214_2[1:11,7:9] = 0\n",
+    "pred214_2[1:11,10:12]=0\n",
+    "pred214_2[10:14,18:19]=0\n",
+    "pred214_2[12:14,15:17]=0\n",
+    "pred214_2[14:19,15:16]=0\n",
+    "\n",
+    "test_comb1 = pred214_1 + ref214\n",
+    "test_comb2 = pred214_2 + ref214"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "41c9937b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Text(0.5, 1.0, 'Prediction 2 over reference')"
+      ]
+     },
+     "execution_count": 32,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzYAAAElCAYAAAA2knddAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvgUlEQVR4nO3deXQUdb7+8acTkoaEJBDIymYIICPrFQW5hk0iAZUji7KoY0AHlcU7wEVHVDZRM7ggjubCeOcqV8YF0QE3xIGwXRxARdBBNCcwcUCBANEsBAiQfH9/8EsPTRKSJh26v8n7dU6fQ1dVV31S3fVQn67qKocxxggAAAAALBbg6wIAAAAAoKZobAAAAABYj8YGAAAAgPVobAAAAABYj8YGAAAAgPVobAAAAABYj8YGAAAAgPVobAAAAABYj8YGAAAAgPVobOAmKytLgwYNUkREhBwOh1atWuXrkgDrXXHFFRo3bpzr+caNG+VwOLRx40avLcPhcGju3Llemx9qx5o1a9S9e3c1bNhQDodDeXl5vi4JliBHUIYcqRyNjcWWLl0qh8PhejRo0EAtWrTQuHHj9NNPP13SPFNTU/X3v/9dTz31lJYtW6ZrrrnGy1UDl9eF20nDhg3VoUMHTZkyRTk5Ob4uzyOrV6/2u52OQ4cO6ZFHHtGAAQMUFhbm9R2tuiQ3N1ejRo1So0aNlJ6ermXLlik0NNTXZaEayJHalZGRoXvuuUcdOnRQSEiI2rZtq9/85jc6dOiQr0vzO+TIxTXwdQGouSeeeEIJCQk6deqUtm3bpqVLl2rLli3avXu3GjZsWO35nDx5Ulu3btVjjz2mKVOm1GLFwOV3/nayZcsWLV68WKtXr9bu3bsVEhJyWWvp27evTp48qeDgYI9et3r1aqWnp1e4U3Ly5Ek1aHD5Iz0zM1MLFixQ+/bt1aVLF23duvWy12CLL774QoWFhZo/f76Sk5N9XQ4uATlSO373u9/p559/1u2336727dvrH//4h15++WV99NFH2rVrl2JjYy97Tf6KHLk4Gps6YMiQIa4jK7/5zW/UvHlzLViwQB988IFGjRpV7fkcPXpUktSkSROv1Xbq1CkFBwcrIICDg/CtC7eTZs2aaeHChXr//fc1duzYCl9TVFRUK9+EBQQEePSlQ3V4e37V1aNHD+Xm5ioyMlLvvvuubr/9dp/U4Q2evt+eTn/kyBFJ3s3Y2vqMomLkSO1YuHChkpKS3PYVBg8erH79+unll1/Wk08+6ZO6LgU54lvsbdZBffr0kSTt27fPNez777/XbbfdpsjISDVs2FDXXHONPvjgA9f4uXPnqk2bNpKkhx56SA6HQ1dccYVr/E8//aR77rlHMTExcjqd6tSpk1599VW35Zad7/v222/r8ccfV4sWLRQSEqKCggJJ0vbt2zV48GBFREQoJCRE/fr102effeY2j7lz58rhcGjv3r0aN26cmjRpooiICI0fP14nTpwo97f++c9/Vs+ePRUSEqKmTZuqb9+++utf/+o2zSeffKI+ffooNDRUYWFhuvnmm/Xtt99ewppFXXLDDTdIkrKzsyVJ48aNU+PGjbVv3z7ddNNNCgsL05133ilJKi0t1aJFi9SpUyc1bNhQMTExuv/++/XLL7+4zdMYoyeffFItW7ZUSEiIBgwYUOFnrbJz47dv366bbrpJTZs2VWhoqLp27aoXX3zRVV96erokuZ0SU6aic+N37typIUOGKDw8XI0bN9bAgQO1bds2t2nKTrH57LPPNH36dEVFRSk0NFTDhw93fdlxMWFhYYqMjKxyuotZsWKFevTooUaNGql58+a666673E6nfe655+RwOPTPf/6z3Gtnzpyp4OBgt/fCk6zZs2eP7rjjDjVt2lRJSUmV1li2njZt2qRJkyYpOjpaLVu2dI2vKmf69++v1NRUSdK1114rh8Ph9nsJb9T85z//2bUeIyMjNWbMGB04cMBtHv3791fnzp21Z88eDRgwQCEhIWrRooWeeeaZcn/zqVOnNHfuXHXo0EENGzZUXFycRowY4fZ/S3W3jbqKHDmnpjnSt2/fcl+A9u3bV5GRkfruu++qfL1EjnirZttzhMamDvrhhx8kSU2bNpUkffvtt7ruuuv03Xff6ZFHHtHzzz+v0NBQDRs2TCtXrpQkjRgxQi+88IIkaezYsVq2bJkWLVokScrJydF1112ndevWacqUKXrxxRfVrl073Xvvva5pzjd//nx9/PHHmjFjhp5++mkFBwdr/fr16tu3rwoKCjRnzhw9/fTTysvL0w033KDPP/+83DxGjRqlwsJCpaWladSoUVq6dKnmzZvnNs28efP061//WkFBQXriiSc0b948tWrVSuvXr3dNs2zZMt18881q3LixFixYoFmzZmnPnj1KSkpyrSfUT2Wh2qxZM9ews2fPKiUlRdHR0Xruuec0cuRISdL999+vhx56SNdff71efPFFjR8/Xm+88YZSUlJ05swZ1+tnz56tWbNmqVu3bnr22WfVtm1bDRo0SEVFRVXWs3btWvXt21d79uzRb3/7Wz3//PMaMGCAPvroI1cNN954o6Rzn+uyR2W+/fZb9enTR19//bUefvhhzZo1S9nZ2erfv7+2b99ebvoHH3xQX3/9tebMmaOJEyfqww8/vCynpC5dulSjRo1SYGCg0tLSNGHCBP3lL39RUlKS6wexo0aNksPh0DvvvFPu9e+8844GDRrkyjtPs+b222/XiRMn9PTTT2vChAlV1jtp0iTt2bNHs2fP1iOPPCKpejnz2GOP6b777pN07nSmZcuW6f777/dazU899ZTuvvtutW/fXgsXLtTUqVOVkZGhvn37lvth8S+//KLBgwerW7duev7559WxY0f97ne/0yeffOKapqSkRLfccovmzZunHj166Pnnn9dvf/tb5efna/fu3a7pqrtt1FXkiDtv5sjx48d1/PhxNW/evMppyRFyxMXAWq+99pqRZNatW2eOHj1qDhw4YN59910TFRVlnE6nOXDggDHGmIEDB5ouXbqYU6dOuV5bWlpq/v3f/920b9/eNSw7O9tIMs8++6zbcu69914TFxdnjh075jZ8zJgxJiIiwpw4ccIYY8yGDRuMJNO2bVvXsLJltW/f3qSkpJjS0lLX8BMnTpiEhARz4403uobNmTPHSDL33HOP27KGDx9umjVr5nqelZVlAgICzPDhw01JSYnbtGXLKCwsNE2aNDETJkxwG3/48GETERFRbjjqpoq2k7fffts0a9bMNGrUyPz444/GGGNSU1ONJPPII4+4vf7//u//jCTzxhtvuA1fs2aN2/AjR46Y4OBgc/PNN7t9zh999FEjyaSmprqGlW0rGzZsMMYYc/bsWZOQkGDatGljfvnlF7flnD+vyZMnm8piW5KZM2eO6/mwYcNMcHCw2bdvn2vYwYMHTVhYmOnbt2+59ZOcnOy2rGnTppnAwECTl5dX4fIqsmLFCre/qyqnT5820dHRpnPnzubkyZOu4R999JGRZGbPnu0a1rt3b9OjRw+313/++edGknn99deNMZeWNWPHjq1WrWXrKSkpyZw9e9Y13JOcKZvHF1984RrmjZp/+OEHExgYaJ566im34X//+99NgwYN3Ib369fPbZ0ZY0xxcbGJjY01I0eOdA179dVXjSSzcOHCcuuirM7qbht1ATly+XKkzPz5840kk5GRcdHpyBFy5HwcsakDkpOTFRUVpVatWum2225TaGioPvjgA7Vs2VI///yz1q9f7zoCcuzYMR07dky5ublKSUlRVlbWRa+gZozRe++9p6FDh8oY43r9sWPHlJKSovz8fH311Vdur0lNTVWjRo1cz3ft2qWsrCzdcccdys3Ndb2+qKhIAwcO1ObNm1VaWuo2jwceeMDteZ8+fZSbm+s6rW3VqlUqLS3V7Nmzyx2+LjusvnbtWuXl5Wns2LFudQcGBqpXr17asGGD5ysb1jp/OxkzZowaN26slStXqkWLFm7TTZw40e35ihUrFBERoRtvvNHtc9SjRw81btzY9Tlat26dTp8+rQcffNDt1I6pU6dWWdvOnTuVnZ2tqVOnljtv+vx5VVdJSYn++te/atiwYWrbtq1reFxcnO644w5t2bLFtS2Vue+++9yW1adPH5WUlFR42oa3fPnllzpy5IgmTZrkdm7/zTffrI4dO+rjjz92DRs9erR27NjhdvrC8uXL5XQ6deutt0ryTtZUZcKECQoMDHQ9r2nOeKPmv/zlLyotLdWoUaPcaoiNjVX79u3L1dC4cWPdddddrufBwcHq2bOn/vGPf7iGvffee2revLkefPDBcjWXfU6qu23UJeTI5cmRzZs3a968eRo1apTrdL/KkCPkyPm4eEAdkJ6erg4dOig/P1+vvvqqNm/eLKfTKUnau3evjDGaNWuWZs2aVeHrjxw5Ui6Uyxw9elR5eXl65ZVX9Morr1T6+vMlJCS4Pc/KypIk13mhFcnPz3cdApak1q1bu40vG/fLL78oPDxc+/btU0BAgK666qpK51m23MpCMTw8vNLXou4p204aNGigmJgYXXnlleWa4gYNGrid7yyd+xzl5+crOjq6wvmWff7L/uNu37692/ioqCi3z3ZFyv6T7dy5c/X/oIs4evSoTpw4oSuvvLLcuF/96lcqLS3VgQMH1KlTJ9fwi21ztaVsnVVUZ8eOHbVlyxbX89tvv13Tp0/X8uXL9eijj8oYoxUrVrjO/ZcuLWsuzKuqVJZvl5oz3qg5KytLxphyn70yQUFBbs9btmxZbke3adOm+uabb1zP9+3bpyuvvPKiV8iq7rZRl5Aj59Rmjnz//fcaPny4OnfurD/96U9VTk+OkCPno7GpA3r27Om6SsuwYcOUlJSkO+64Q5mZma4OfcaMGUpJSanw9e3atat03mWvv+uuuyrdYLp27er2/PyjNefP49lnn1X37t0rnEfjxo3dnp//Tcb5jDGV1nqhsuUuW7aswktF+uKSlvCd87eTyjidznI7KaWlpYqOjtYbb7xR4WuioqK8VqMveWObq03x8fHq06eP3nnnHT366KPatm2b9u/frwULFrimuZSsuTCvqlJZvl1qznij5tLSUjkcDn3yyScVvo+1ka9ly60P28b5yJGLq+ln68CBA66bhK9evVphYWHeLI8cqaKGupAj7NnVMWU/nBswYIBefvll3XPPPZLOddqXcr3zqKgohYWFqaSk5JKvl56YmCjp3DcO3rrmemJiokpLS7Vnz55KN+Ky5UZHR3Otd1yyxMRErVu3Ttdff/1F//Mqu6pgVlaW22kbR48erfLbyrLP6u7duy/6Wa3u6SRRUVEKCQlRZmZmuXHff/+9AgIC1KpVq2rNqzaVrbPMzMxy31RmZma6xpcZPXq0Jk2apMzMTC1fvlwhISEaOnSoa3xtZE1Vapoz3qg5MTFRxhglJCSoQ4cOlzSPiua5fft2nTlzptw3tedPU51tA+RIdeTm5mrQoEEqLi5WRkaG4uLiqvU6coQcOR+/samD+vfvr549e2rRokUKDw9X//799cc//rHCO/hWdRnGwMBAjRw5Uu+9957bFSyq+3rp3H0uEhMT9dxzz+n48eOXNI8LDRs2TAEBAXriiSfKnTda9m1BSkqKwsPD9fTTT1d4VY1LWS7qn1GjRqmkpETz588vN+7s2bOuK8UkJycrKChIL730kts3VhVdOfBCV199tRISErRo0aJyV545f15l9xm4cJoLBQYGatCgQXr//ffdrv6Xk5OjN998U0lJSX5xKuY111yj6OhoLVmyRMXFxa7hn3zyib777jvdfPPNbtOPHDlSgYGBeuutt7RixQrdcsstbvdeqI2sqUpNc8YbNY8YMUKBgYGaN29euW9LjTHKzc2tch4XGjlypI4dO6aXX3653LiyZVR32wA5UpWioiLddNNN+umnn7R69epKT4eqCDlCjpyPIzZ11EMPPaTbb79dS5cuVXp6upKSktSlSxdNmDBBbdu2VU5OjrZu3aoff/xRX3/99UXn9fvf/14bNmxQr169NGHCBF111VX6+eef9dVXX2ndunX6+eefL/r6gIAA/elPf9KQIUPUqVMnjR8/Xi1atNBPP/2kDRs2KDw8XB9++KFHf1+7du302GOPaf78+erTp49GjBghp9OpL774QvHx8UpLS1N4eLgWL16sX//617r66qs1ZswYRUVFaf/+/fr44491/fXXV7ixAefr16+f7r//fqWlpWnXrl0aNGiQgoKClJWVpRUrVujFF1/UbbfdpqioKM2YMUNpaWm65ZZbdNNNN2nnzp365JNPqrxcaUBAgBYvXqyhQ4eqe/fuGj9+vOLi4vT999/r22+/1aeffirp3H9ekvQf//EfSklJUWBgoMaMGVPhPJ988kmtXbtWSUlJmjRpkho0aKA//vGPKi4urvBeAzVRdvO8svstLFu2zHVe++OPP17p64KCgrRgwQKNHz9e/fr109ixY5WTk6MXX3xRV1xxhaZNm+Y2fXR0tAYMGKCFCxeqsLBQo0ePdhtfG1lTlZrmjDdqTkxM1JNPPqmZM2fqhx9+0LBhwxQWFqbs7GytXLlS9913n2bMmOHR33X33Xfr9ddf1/Tp0/X555+rT58+Kioq0rp16zRp0iTdeuut1d42QI5U5c4779Tnn3+ue+65R999953bvWsaN26sYcOGVfpacoQccePRNdTgVyq65F+ZkpISk5iYaBITE83Zs2fNvn37zN13321iY2NNUFCQadGihbnlllvMu+++63pNZZd7NsaYnJwcM3nyZNOqVSsTFBRkYmNjzcCBA80rr7zimqbs0pMrVqyosN6dO3eaESNGmGbNmhmn02natGljRo0a5XYpx7LLEB49erTCvzU7O9tt+Kuvvmr+7d/+zTidTtO0aVPTr18/s3btWrdpNmzYYFJSUkxERIRp2LChSUxMNOPGjTNffvll5SsXdcbFtpPzpaammtDQ0ErHv/LKK6ZHjx6mUaNGJiwszHTp0sU8/PDD5uDBg65pSkpKzLx580xcXJxp1KiR6d+/v9m9e7dp06bNRS/TWmbLli3mxhtvNGFhYSY0NNR07drVvPTSS67xZ8+eNQ8++KCJiooyDofD7ZKtuuAyrcYY89VXX5mUlBTTuHFjExISYgYMGGD+9re/VWv9VFZjRSRV+qiO5cuXu7bjyMhIc+edd7oun3uh//7v/zaSTFhYmNulXc9Xk6ypTFWfo+rkzMXm4Y2a33vvPZOUlGRCQ0NNaGio6dixo5k8ebLJzMx0TdOvXz/TqVOncq9NTU01bdq0cRt24sQJ89hjj5mEhARX7t92221ul/41pnrbhu3IkdrNkTZt2lSaIRd+LitDjpAjxhjjMMZPfhkKAAAAAJeI39gAAAAAsB6NDQAAAADr0dgAAAAAsB6NDQAAAADr0dgAAAAAsJ7f3cemtLRUBw8eVFhYWLXvjgugdhhjVFhYqPj4eAUE2PM9CDkC+A8bc4QMAfyHJxnid43NwYMH1apVK1+XAeA8Bw4cUMuWLX1dRrWRI4D/sSlHyBDA/1QnQ/yusQkLC5MkJekmNVCQj6sB6rezOqMtWu3aLm1BjgD+w8YcIUMA/+FJhtRaY5Oenq5nn31Whw8fVrdu3fTSSy+pZ8+eVb6u7JBvAwWpgYMwAXzq/9++1xenYlxqhkjkCOBXLMwRMgTwIx5kSK2c7Lp8+XJNnz5dc+bM0VdffaVu3bopJSVFR44cqY3FAahjyBAANUWOAPVPrTQ2Cxcu1IQJEzR+/HhdddVVWrJkiUJCQvTqq6+Wm7a4uFgFBQVuDwD1mycZIpEjAMpjXwSof7ze2Jw+fVo7duxQcnLyvxYSEKDk5GRt3bq13PRpaWmKiIhwPfixHlC/eZohEjkCwB37IkD95PXG5tixYyopKVFMTIzb8JiYGB0+fLjc9DNnzlR+fr7rceDAAW+XBMAinmaIRI4AcMe+CFA/+fyqaE6nU06n09dlALAYOQKgJsgQoG7w+hGb5s2bKzAwUDk5OW7Dc3JyFBsb6+3FAahjyBAANUWOAPWT14/YBAcHq0ePHsrIyNCwYcMknbuDb0ZGhqZMmeLtxdVpnx7c5esSaiQlvruvS4CFyBDvIkdQH5Ej3rP3heu8Nq99o5dUa7rE5Q94bZntpm3z2rzg/2rlVLTp06crNTVV11xzjXr27KlFixapqKhI48ePr43FAahjyBAANUWOAPVPrTQ2o0eP1tGjRzV79mwdPnxY3bt315o1a8r9iA8AKkKGAKgpcgSof2rt4gFTpkzhcC+AS0aGAKgpcgSoX2rlBp0AAAAAcDnR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwntcbm7lz58rhcLg9Onbs6O3FAKijyBAANUWOAPVTg9qYaadOnbRu3bp/LaRBrSwGQB1FhgCoKXIEqH9qZStv0KCBYmNja2PWAOoBMgRATZEjQP1TK7+xycrKUnx8vNq2bas777xT+/fvr3Ta4uJiFRQUuD0A1G+eZIhEjgAoj30RoP7xemPTq1cvLV26VGvWrNHixYuVnZ2tPn36qLCwsMLp09LSFBER4Xq0atXK2yUBsIinGSKRIwDcsS8C1E8OY4ypzQXk5eWpTZs2Wrhwoe69995y44uLi1VcXOx6XlBQoFatWqm/blUDR1Btlub3Pj24y9cl1EhKfHdfl4AaOmvOaKPeV35+vsLDw31SQ1UZIpEjF0OOwNdsyBEypHJ7X7jOa/PaN3pJtaZLXP6A15bZbto2r80LvuFJhtT6L+maNGmiDh06aO/evRWOdzqdcjqdtV0GAEtVlSESOQLg4tgXAeqHWr+PzfHjx7Vv3z7FxcXV9qIA1EFkCICaIkeA+sHrjc2MGTO0adMm/fDDD/rb3/6m4cOHKzAwUGPHjvX2ogDUQWQIgJoiR4D6yeunov34448aO3ascnNzFRUVpaSkJG3btk1RUVHeXhSAOogMAVBT5AhQP3m9sXn77be9PUsA9QgZAqCmyBGgfqr139gAAAAAQG2jsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANZr4OsC/MGnB3f5uoQ6ifV6Tkp8d1+XgMuAz3vtuNzrNXH5A5d1edXVbto2X5eAWrb3het8XUKdxHo9p75kCEdsAAAAAFjP48Zm8+bNGjp0qOLj4+VwOLRq1Sq38cYYzZ49W3FxcWrUqJGSk5OVlZXlrXoBWI4MAVBT5AiAinjc2BQVFalbt25KT0+vcPwzzzyjP/zhD1qyZIm2b9+u0NBQpaSk6NSpUzUuFoD9yBAANUWOAKiIx7+xGTJkiIYMGVLhOGOMFi1apMcff1y33nqrJOn1119XTEyMVq1apTFjxpR7TXFxsYqLi13PCwoKPC0JgEW8nSESOQLUN+yLAKiIV39jk52drcOHDys5Odk1LCIiQr169dLWrVsrfE1aWpoiIiJcj1atWnmzJAAWuZQMkcgRAP/CvghQf3m1sTl8+LAkKSYmxm14TEyMa9yFZs6cqfz8fNfjwIED3iwJgEUuJUMkcgTAv7AvAtRfPr/cs9PplNPp9HUZACxGjgCoCTIEqBu8esQmNjZWkpSTk+M2PCcnxzUOACpDhgCoKXIEqL+8esQmISFBsbGxysjIUPfu3SWd+wHe9u3bNXHiRG8uCkAdRIYAqKm6niP7Ri/xdQm1rj78jd7krzcW9gWPG5vjx49r7969rufZ2dnatWuXIiMj1bp1a02dOlVPPvmk2rdvr4SEBM2aNUvx8fEaNmyYN+sGYCkyBEBNkSMAKuJxY/Pll19qwIABrufTp0+XJKWmpmrp0qV6+OGHVVRUpPvuu095eXlKSkrSmjVr1LBhQ+9VDcBaZAiAmiJHAFTE48amf//+MsZUOt7hcOiJJ57QE088UaPCANRNZAiAmiJHAFTEqxcPAAAAAABfoLEBAAAAYD0aGwAAAADWo7EBAAAAYD0aGwAAAADWo7EBAAAAYD2PL/eMi0uJ7+7rEuqtTw/u8nUJfqM664LPqnd5887P7aZt89q84KEXfF2A/9j7wnVVTsNn1T+R775Tne2muvaNXuK1efmCL/ZFOGIDAAAAwHo0NgAAAACsR2MDAAAAwHo0NgAAAACsR2MDAAAAwHo0NgAAAACsR2MDAAAAwHo0NgAAAACsR2MDAAAAwHo0NgAAAACsR2MDAAAAwHo0NgAAAACsR2MDAAAAwHo0NgAAAACsR2MDAAAAwHo0NgAAAACsR2MDAAAAwHo0NgAAAACs18DXBVyqTw/u8nUJQLX462e1OnUVFJaqaYfar8VX9r5wna9LAKrFXz+r1amr9NQp6ZH3L0M1l593892b84K/2zd6yeWd12ivLc6rvL0vwhEbAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPRobAAAAANajsQEAAABgPWtv0JkS371a01Xnxj/enBfqhup+Ji636n4GvVX/WXNG0j+8Mi9/1G7atmpNV52bEHpzXqgbqvuZuNyq+xn0Vv1nzRnt98qc/A/7IqhN7Iuc48m+CEdsAAAAAFjP48Zm8+bNGjp0qOLj4+VwOLRq1Sq38ePGjZPD4XB7DB482Fv1ArAcGQKgpsgRABXxuLEpKipSt27dlJ6eXuk0gwcP1qFDh1yPt956q0ZFAqg7yBAANUWOAKiIx7+xGTJkiIYMGXLRaZxOp2JjYy+5KAB1FxkCoKbIEQAVqZXf2GzcuFHR0dG68sorNXHiROXm5lY6bXFxsQoKCtweAOo3TzJEIkcAlMe+CFD/eL2xGTx4sF5//XVlZGRowYIF2rRpk4YMGaKSkpIKp09LS1NERITr0apVK2+XBMAinmaIRI4AcMe+CFA/ef1yz2PGjHH9u0uXLuratasSExO1ceNGDRw4sNz0M2fO1PTp013PCwoKCBSgHvM0QyRyBIA79kWA+qnWL/fctm1bNW/eXHv37q1wvNPpVHh4uNsDAMpUlSESOQLg4tgXAeqHWm9sfvzxR+Xm5iouLq62FwWgDiJDANQUOQLUDx6finb8+HG3bzyys7O1a9cuRUZGKjIyUvPmzdPIkSMVGxurffv26eGHH1a7du2UkpLi1cIB2IkMAVBT5AiAinjc2Hz55ZcaMGCA63nZOampqalavHixvvnmG/3v//6v8vLyFB8fr0GDBmn+/PlyOp3eqxqAtcgQADVFjgCoiMeNTf/+/WWMqXT8p59+WqOCANRtZAiAmiJHAFSk1n9jAwAAAAC1jcYGAAAAgPVobAAAAABYj8YGAAAAgPVobAAAAABYj8YGAAAAgPU8vtyzbVLiu1/W5X16cNdlXZ50+f/G6qrOuvDX2r2pup+J6qyL+rC+/FG7adsu6/L2vnDdZV2edPn/xuqqzrrw19q9qbqfieqsi/qwvvwN+yK+w77IOfVlX4QjNgAAAACsR2MDAAAAwHo0NgAAAACsR2MDAAAAwHo0NgAAAACsR2MDAAAAwHo0NgAAAACsR2MDAAAAwHo0NgAAAACs18DXBdRX/nzXVvjmrs2Ap7iDvH/b+8J1vi4BuCj2Rfwb+yKe44gNAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHjfo9EB1bmTFzZQ846/ri5uWobZU56aa3NjRM/66vriBKmoD+yLe59315b15sS/iOY7YAAAAALAejQ0AAAAA69HYAAAAALAejQ0AAAAA69HYAAAAALAejQ0AAAAA69HYAAAAALAejQ0AAAAA69HYAAAAALBeA18XUNdwl1jPsL6A8rhjvWdYX4A7/m/1DOur7uCIDQAAAADredTYpKWl6dprr1VYWJiio6M1bNgwZWZmuk1z6tQpTZ48Wc2aNVPjxo01cuRI5eTkeLVoAPYiRwDUBBkCoDIeNTabNm3S5MmTtW3bNq1du1ZnzpzRoEGDVFRU5Jpm2rRp+vDDD7VixQpt2rRJBw8e1IgRI7xeOAA7kSMAaoIMAVAZj35js2bNGrfnS5cuVXR0tHbs2KG+ffsqPz9f//M//6M333xTN9xwgyTptdde069+9Stt27ZN1113Xbl5FhcXq7i42PW8oKDgUv4OAJYgRwDUBBkCoDI1+o1Nfn6+JCkyMlKStGPHDp05c0bJycmuaTp27KjWrVtr69atFc4jLS1NERERrkerVq1qUhIAy5AjAGqCDAFQ5pIbm9LSUk2dOlXXX3+9OnfuLEk6fPiwgoOD1aRJE7dpY2JidPjw4QrnM3PmTOXn57seBw4cuNSSAFiGHAFQE2QIgPNd8uWeJ0+erN27d2vLli01KsDpdMrpdNZoHgDsRI4AqAkyBMD5LumIzZQpU/TRRx9pw4YNatmypWt4bGysTp8+rby8PLfpc3JyFBsbW6NCAdQt5AiAmiBDAFzIo8bGGKMpU6Zo5cqVWr9+vRISEtzG9+jRQ0FBQcrIyHANy8zM1P79+9W7d2/vVAzAauQIgJogQwBUxqNT0SZPnqw333xT77//vsLCwlznqkZERKhRo0aKiIjQvffeq+nTpysyMlLh4eF68MEH1bt37wqvQgKg/iFHANQEGQKgMh41NosXL5Yk9e/f3234a6+9pnHjxkmSXnjhBQUEBGjkyJEqLi5WSkqK/uu//ssrxQKwHzkCoCbIEACV8aixMcZUOU3Dhg2Vnp6u9PT0Sy4KQN1FjgCoCTIEQGVqdB8bAAAAAPAHNDYAAAAArEdjAwAAAMB6NDYAAAAArEdjAwAAAMB6NDYAAAAArOfR5Z4BT6TEd/d1CQAs127aNl+XAMBi7IvULxyxAQAAAGA9GhsAAAAA1qOxAQAAAGA9GhsAAAAA1qOxAQAAAGA9GhsAAAAA1qOxAQAAAGA9GhsAAAAA1qOxAQAAAGA9GhsAAAAA1qOxAQAAAGA9GhsAAAAA1qOxAQAAAGA9GhsAAAAA1qOxAQAAAGA9GhsAAAAA1qOxAQAAAGC9Br4u4ELGGEnSWZ2RjI+LAeq5szoj6V/bpS3IEcB/2JgjZAjgPzzJEL9rbAoLCyVJW7Tax5UAKFNYWKiIiAhfl1Ft5Ajgf2zKETIE8D/VyRCH8bOvUEpLS3Xw4EGFhYXJ4XBIkgoKCtSqVSsdOHBA4eHhPq7QczbXb3Ptkt31+0PtxhgVFhYqPj5eAQH2nLla13LE5tolu+u3uXbJP+q3MUfqWoZIdtdvc+2S3fX7Q+2eZIjfHbEJCAhQy5YtKxwXHh5u3QfifDbXb3Ptkt31+7p2W75hPV9dzRGba5fsrt/m2iXf129bjtTVDJHsrt/m2iW76/d17dXNEDu+OgEAAACAi6CxAQAAAGA9Kxobp9OpOXPmyOl0+rqUS2Jz/TbXLtldv821+yOb16fNtUt2129z7ZL99fsT29elzfXbXLtkd/221e53Fw8AAAAAAE9ZccQGAAAAAC6GxgYAAACA9WhsAAAAAFiPxgYAAACA9WhsAAAAAFjPisYmPT1dV1xxhRo2bKhevXrp888/93VJVZo7d64cDofbo2PHjr4uq1KbN2/W0KFDFR8fL4fDoVWrVrmNN8Zo9uzZiouLU6NGjZScnKysrCzfFHuBqmofN25cufdi8ODBvin2Amlpabr22msVFham6OhoDRs2TJmZmW7TnDp1SpMnT1azZs3UuHFjjRw5Ujk5OT6q2E42ZohkV47YnCESOYKq2ZgjNmWIZHeOkCH+we8bm+XLl2v69OmaM2eOvvrqK3Xr1k0pKSk6cuSIr0urUqdOnXTo0CHXY8uWLb4uqVJFRUXq1q2b0tPTKxz/zDPP6A9/+IOWLFmi7du3KzQ0VCkpKTp16tRlrrS8qmqXpMGDB7u9F2+99dZlrLBymzZt0uTJk7Vt2zatXbtWZ86c0aBBg1RUVOSaZtq0afrwww+1YsUKbdq0SQcPHtSIESN8WLVdbM4QyZ4csTlDJHIEF2dzjtiSIZLdOUKG+Anj53r27GkmT57sel5SUmLi4+NNWlqaD6uq2pw5c0y3bt18XcYlkWRWrlzpel5aWmpiY2PNs88+6xqWl5dnnE6neeutt3xQYeUurN0YY1JTU82tt97qk3o8deTIESPJbNq0yRhzbj0HBQWZFStWuKb57rvvjCSzdetWX5VpFVszxBh7c8TmDDGGHEF5tuaIrRlijN05Qob4jl8fsTl9+rR27Nih5ORk17CAgAAlJydr69atPqyserKyshQfH6+2bdvqzjvv1P79+31d0iXJzs7W4cOH3d6HiIgI9erVy4r3QZI2btyo6OhoXXnllZo4caJyc3N9XVKF8vPzJUmRkZGSpB07dujMmTNu675jx45q3bq1Nevel2zPEKlu5EhdyBCJHKmvbM+RupAhUt3IETKk9vl1Y3Ps2DGVlJQoJibGbXhMTIwOHz7so6qqp1evXlq6dKnWrFmjxYsXKzs7W3369FFhYaGvS/NY2bq28X2Qzh36ff3115WRkaEFCxZo06ZNGjJkiEpKSnxdmpvS0lJNnTpV119/vTp37izp3LoPDg5WkyZN3Ka1Zd37ms0ZItWdHLE9QyRypD6zOUfqSoZI9ucIGXJ5NPB1AXXVkCFDXP/u2rWrevXqpTZt2uidd97Rvffe68PK6p8xY8a4/t2lSxd17dpViYmJ2rhxowYOHOjDytxNnjxZu3fv9uvzn3F5kSP+gxyBjcgQ/0GGXB5+fcSmefPmCgwMLHfVhZycHMXGxvqoqkvTpEkTdejQQXv37vV1KR4rW9d14X2QpLZt26p58+Z+9V5MmTJFH330kTZs2KCWLVu6hsfGxur06dPKy8tzm97WdX+51aUMkezNkbqWIRI5Up/UpRyxNUOkupcjZEjt8OvGJjg4WD169FBGRoZrWGlpqTIyMtS7d28fVua548ePa9++fYqLi/N1KR5LSEhQbGys2/tQUFCg7du3W/c+SNKPP/6o3Nxcv3gvjDGaMmWKVq5cqfXr1yshIcFtfI8ePRQUFOS27jMzM7V//34r1/3lVpcyRLI3R+pahkjkSH1Sl3LE1gyR6l6OkCG1xMcXL6jS22+/bZxOp1m6dKnZs2ePue+++0yTJk3M4cOHfV3aRf3nf/6n2bhxo8nOzjafffaZSU5ONs2bNzdHjhzxdWkVKiwsNDt37jQ7d+40kszChQvNzp07zT//+U9jjDG///3vTZMmTcz7779vvvnmG3PrrbeahIQEc/LkSR9XfvHaCwsLzYwZM8zWrVtNdna2Wbdunbn66qtN+/btzalTp3xdupk4caKJiIgwGzduNIcOHXI9Tpw44ZrmgQceMK1btzbr1683X375pendu7fp3bu3D6u2i60ZYoxdOWJzhhhDjuDibM0RmzLEGLtzhAzxD37f2BhjzEsvvWRat25tgoODTc+ePc22bdt8XVKVRo8ebeLi4kxwcLBp0aKFGT16tNm7d6+vy6rUhg0bjKRyj9TUVGPMucsszpo1y8TExBin02kGDhxoMjMzfVv0/3ex2k+cOGEGDRpkoqKiTFBQkGnTpo2ZMGGC3/xnVFHdksxrr73mmubkyZNm0qRJpmnTpiYkJMQMHz7cHDp0yHdFW8jGDDHGrhyxOUOMIUdQNRtzxKYMMcbuHCFD/IPDGGO8eQQIAAAAAC43v/6NDQAAAABUB40NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACwHo0NAAAAAOvR2AAAAACw3v8D+PG0ExPoFdUAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 1000x1500 with 3 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "fig,ax = plt.subplots(1,3,figsize=(10,15))\n",
+    "ax[0].imshow(ref214)\n",
+    "ax[1].imshow(test_comb1)\n",
+    "ax[2].imshow(test_comb2)\n",
+    "ax[0].set_title('Reference')\n",
+    "ax[1].set_title('Prediction 1 over reference')\n",
+    "ax[2].set_title('Prediction 2 over reference')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "612018d2",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(array([0., 1., 2.]), array([438,  66,  72]))"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.unique(test_comb2, return_counts=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "id": "ec9e9033",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Text(0.5, 1.0, 'Reference')"
+      ]
+     },
+     "execution_count": 38,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0UAAAGiCAYAAAAoWW2mAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyXUlEQVR4nO3de3hU5bn+8XuSkAlgMpySQCBApIiKHBQlWwEFSYkBEaSKINKAFqkFEakWqOUk2oi6vbDCBmwLsZ5QtwLWAwhIQBRQCbRCFQM7YhQ5ViYcJGLy/v7wlylDjpOsyQy+3891rUtnzbve9ayszDzcWXNwGWOMAAAAAMBSEaEuAAAAAABCiVAEAAAAwGqEIgAAAABWIxQBAAAAsBqhCAAAAIDVCEUAAAAArEYoAgAAAGA1QhEAAAAAqxGKAAAAAFiNUARIysvLU79+/eTxeORyubR8+fJQlwQAQEDatm2rUaNG+W7n5OTI5XIpJyfHsX24XC7NnDnTsfmAcEEowjknOztbLpfLt0RFRally5YaNWqUvv766xrNmZmZqU8++UQPP/ywnn32WV1++eUOVw0A+Kk7uz/FxMToggsu0Pjx43XgwIFQl1dtb731FsEH1okKdQFATT344INKSUnRqVOntHnzZmVnZ2vjxo3asWOHYmJiqj3Pd999p02bNumBBx7Q+PHjg1gxAMAGZ/anjRs3asGCBXrrrbe0Y8cONWjQoM7quPrqq/Xdd98pOjo6oO3eeustzZ8/v9xg9N133ykqin8+4qeH32qcszIyMnxXdH71q1+pWbNmmjNnjl5//XUNHTq02vMcOnRIktSoUSPHajt16pSio6MVEcHFWACwzdn9qWnTpnriiSe0YsUKDR8+vMz4EydOqGHDho7XEREREdAfCavD6fmAcMG/2PCT0atXL0nSnj17fOs+++wz3XTTTWrSpIliYmJ0+eWX6/XXX/fdP3PmTLVp00aSdP/998vlcqlt27a++7/++mvdfvvtSkxMlNvtVseOHbV48WK//Za+Znvp0qX6wx/+oJYtW6pBgwYqLCyUJG3ZskXXXXedPB6PGjRooGuuuUbvv/++3xwzZ86Uy+XS7t27NWrUKDVq1Egej0ejR4/WyZMnyxzrc889p+7du6tBgwZq3Lixrr76ar3zzjt+Y95++2316tVLDRs2VGxsrAYMGKCdO3fW4CcLAKiNa6+9VpKUn5+vUaNG6bzzztOePXvUv39/xcbGasSIEZKkkpISzZ07Vx07dlRMTIwSExM1duxYffvtt37zGWP00EMPqVWrVmrQoIH69OlT7vN7Re8p2rJli/r376/GjRurYcOG6ty5s5588klJ0qhRozR//nxJ8nspYKny3lO0bds2ZWRkKC4uTuedd5769u2rzZs3+40pfWnh+++/r0mTJik+Pl4NGzbUjTfe6PvjJBBKXCnCT8YXX3whSWrcuLEkaefOnerRo4datmypKVOmqGHDhnr55Zc1ePBgvfrqq7rxxhs1ZMgQNWrUSPfee6+GDx+u/v3767zzzpMkHThwQP/1X/8ll8ul8ePHKz4+Xm+//bbuuOMOFRYWauLEiX77nz17tqKjo3XfffepqKhI0dHRevfdd5WRkaFu3bppxowZioiI0JIlS3TttdfqvffeU/fu3f3mGDp0qFJSUpSVlaXc3Fz95S9/UUJCgubMmeMbM2vWLM2cOVNXXXWVHnzwQUVHR2vLli1699131a9fP0nSs88+q8zMTKWnp2vOnDk6efKkFixYoJ49e2rbtm1+wQ8AEFylf6xr2rSpJOmHH35Qenq6evbsqccff9z3krqxY8cqOztbo0eP1oQJE5Sfn6958+Zp27Ztev/991WvXj1J0vTp0/XQQw+pf//+6t+/v3Jzc9WvXz99//33VdayevVqXX/99WrRooXuueceNW/eXJ9++qneeOMN3XPPPRo7dqz27dun1atX69lnn61yvp07d6pXr16Ki4vT7373O9WrV0+LFi1S7969tX79eqWmpvqNv/vuu9W4cWPNmDFDX3zxhebOnavx48frpZdeCuhnCjjOAOeYJUuWGElmzZo15tChQ6agoMD87//+r4mPjzdut9sUFBQYY4zp27ev6dSpkzl16pRv25KSEnPVVVeZ9u3b+9bl5+cbSeaxxx7z288dd9xhWrRoYQ4fPuy3ftiwYcbj8ZiTJ08aY4xZt26dkWTOP/9837rSfbVv396kp6ebkpIS3/qTJ0+alJQU8/Of/9y3bsaMGUaSuf322/32deONN5qmTZv6bufl5ZmIiAhz4403muLiYr+xpfs4duyYadSokRkzZozf/fv37zcej6fMegCAM8rrT0uXLjVNmzY19evXN1999ZXJzMw0ksyUKVP8tn3vvfeMJPP888/7rV+5cqXf+oMHD5ro6GgzYMAAv97y+9//3kgymZmZvnWl/WndunXGGGN++OEHk5KSYtq0aWO+/fZbv/2cOde4ceNMRf9ElGRmzJjhuz148GATHR1t9uzZ41u3b98+Exsba66++uoyP5u0tDS/fd17770mMjLSHD16tNz9AXWFl8/hnJWWlqb4+HglJyfrpptuUsOGDfX666+rVatW+ve//613331XQ4cO1bFjx3T48GEdPnxYR44cUXp6uvLy8ir9pDpjjF599VUNHDhQxhjf9ocPH1Z6erq8Xq9yc3P9tsnMzFT9+vV9t7dv3668vDzdeuutOnLkiG/7EydOqG/fvtqwYYNKSkr85vj1r3/td7tXr146cuSI76V4y5cvV0lJiaZPn17m/UqlL29YvXq1jh49quHDh/vVHRkZqdTUVK1bty7wHzYAoNrO7E/Dhg3Teeedp2XLlqlly5a+MXfddZffNq+88oo8Ho9+/vOf+z13d+vWTeedd57vuXvNmjX6/vvvdffdd/u9rO3sVy+UZ9u2bcrPz9fEiRPLvI/2zLmqq7i4WO+8844GDx6s888/37e+RYsWuvXWW7Vx40Zf/yp15513+u2rV69eKi4u1t69ewPeP+AkXj6Hc9b8+fN1wQUXyOv1avHixdqwYYPcbrckaffu3TLGaNq0aZo2bVq52x88eNCvQZ3p0KFDOnr0qJ5++mk9/fTTFW5/ppSUFL/beXl5kn4MSxXxer2+l/tJUuvWrf3uL73v22+/VVxcnPbs2aOIiAhdfPHFFc5Zut/S17CfLS4ursJtAQC1V9qfoqKilJiYqA4dOvj9ISsqKkqtWrXy2yYvL09er1cJCQnlzlnac0rDQ/v27f3uj4+P9+sn5Sl9Gd8ll1wS2AFV4NChQzp58qQ6dOhQ5r6LLrpIJSUlKigoUMeOHX3rK+tzQCgRinDO6t69u+/TfQYPHqyePXvq1ltv1a5du3xXYO677z6lp6eXu/3PfvazCucu3f62226rMNR07tzZ7/aZV4nOnOOxxx5T165dy52j9P1LpSIjI8sdZ4ypsNazle732WefVfPmzcvcz0epAkBwndmfyuN2u8tc7S8pKVFCQoKef/75creJj493tMZQcaLPAcHAv47wkxAZGamsrCz16dNH8+bN0+233y5JqlevntLS0gKeLz4+XrGxsSouLq7R9pLUrl07ST9emanpHOXNWVJSon/9618VBq3S/SYkJDi2XwBAcLVr105r1qxRjx49yvyR7Uyln5ial5fn95K1Q4cOVXm1pbQ/7Nixo9L+UN2X0sXHx6tBgwbatWtXmfs+++wzRUREKDk5uVpzAaHGe4rwk9G7d291795dc+fOVVxcnHr37q1Fixbpm2++KTO2qo//jIyM1C9+8Qu9+uqr2rFjR8DbS1K3bt3Url07Pf744zp+/HiN5jjb4MGDFRERoQcffLDM+5FK/8qWnp6uuLg4/fGPf9Tp06cd2S8AILiGDh2q4uJizZ49u8x9P/zwg44ePSrpx/cr1atXT0899ZTf1ZW5c+dWuY/LLrtMKSkpmjt3rm++UmfOVfqdSWePOVtkZKT69eunFStW+D4BVvrx01tfeOEF9ezZk5ds45zBlSL8pNx///26+eablZ2drfnz56tnz57q1KmTxowZo/PPP18HDhzQpk2b9NVXX+kf//hHpXM98sgjWrdunVJTUzVmzBhdfPHF+ve//63c3FytWbNG//73vyvdPiIiQn/5y1+UkZGhjh07avTo0WrZsqW+/vprrVu3TnFxcfr73/8e0PH97Gc/0wMPPKDZs2erV69eGjJkiNxutz766CMlJSUpKytLcXFxWrBggUaOHKnLLrtMw4YNU3x8vL788ku9+eab6tGjh+bNmxfQfgEAwXXNNddo7NixysrK0vbt29WvXz/Vq1dPeXl5euWVV/Tkk0/qpptuUnx8vO677z5lZWXp+uuvV//+/bVt2za9/fbbatasWaX7iIiI0IIFCzRw4EB17dpVo0ePVosWLfTZZ59p586dWrVqlaQf/6gnSRMmTFB6eroiIyM1bNiwcud86KGHtHr1avXs2VO/+c1vFBUVpUWLFqmoqEiPPvqosz8kIIgIRfhJGTJkiO/qzJgxY/Txxx9r1qxZys7O1pEjR5SQkKBLL71U06dPr3KuxMREffjhh3rwwQf12muv6X/+53/UtGlTdezY0e97gyrTu3dvbdq0SbNnz9a8efN0/PhxNW/eXKmpqRo7dmyNjvHBBx9USkqKnnrqKT3wwANq0KCBOnfurJEjR/rG3HrrrUpKStIjjzyixx57TEVFRWrZsqV69eql0aNH12i/AIDgWrhwobp166ZFixbp97//vaKiotS2bVvddttt6tGjh2/cQw89pJiYGC1cuND3x7t33nlHAwYMqHIf6enpWrdunWbNmqX//u//VklJidq1a6cxY8b4xgwZMkR33323li5dqueee07GmApDUceOHfXee+9p6tSpysrKUklJiVJTU/Xcc8+V+Y4iIJy5DO9sAwAAAGAx3lMEAAAAwGqEIgAAAABWIxQBAAAAsBqhCAAAAIDVCEUAAAAArEYoAgAAAGC1sPueopKSEu3bt0+xsbFyuVyhLgcArGKM0bFjx5SUlKSICP5uVoreBAChUVd9KexC0b59+5ScnBzqMgDAagUFBWrVqlWoywgb9CYACK1g96WwC0WxsbGSpJ7qryjVC3E1AGCXH3RaG/WW77kYP6I3AUBo1FVfCrtQVPqyhCjVU5SLxgMAdcr8+B9eIuaP3gQAIVJHfYkXjAMAAACwGqEIAAAAgNUIRQAAAACsFrRQNH/+fLVt21YxMTFKTU3Vhx9+GKxdAQBQLfQmAEB5ghKKXnrpJU2aNEkzZsxQbm6uunTpovT0dB08eDAYuwMAoEr0JgBARYISip544gmNGTNGo0eP1sUXX6yFCxeqQYMGWrx4cTB2BwBAlehNAICKOB6Kvv/+e23dulVpaWn/2UlEhNLS0rRp06Yy44uKilRYWOi3AADgJHoTAKAyjoeiw4cPq7i4WImJiX7rExMTtX///jLjs7Ky5PF4fAvfGA4AcBq9CQBQmZB/+tzUqVPl9Xp9S0FBQahLAgBYjt4EAHaJcnrCZs2aKTIyUgcOHPBbf+DAATVv3rzMeLfbLbfb7XQZAAD40JsAAJVx/EpRdHS0unXrprVr1/rWlZSUaO3atbryyiud3h0AAFWiNwEAKuP4lSJJmjRpkjIzM3X55Zere/fumjt3rk6cOKHRo0cHY3cAAFSJ3gQAqEhQQtEtt9yiQ4cOafr06dq/f7+6du2qlStXlnmDKwAAdYXeBACoiMsYY0JdxJkKCwvl8XjUW4MU5aoX6nIAwCo/mNPK0Qp5vV7FxcWFupywQW8CgNCoq74U8k+fAwAAAIBQIhQBAAAAsBqhCAAAAIDVCEUAAAAArEYoAgAAAGA1QhEAAAAAqxGKAAAAAFiNUAQAAADAaoQiAAAAAFYjFAEAAACwGqEIAAAAgNUIRQAAAACsRigCAAAAYDVCEQAAAACrEYoAAAAAWI1QBAAAAMBqhCIAAAAAViMUAQAAALAaoQgAAACA1QhFAAAAAKxGKAIAAABgtahQF1AXVu3bHuoSICk9qWuoSwAAAADK4EoRAAAAAKsRigAAAABYjVAEAAAAwGqEIgAAAABWIxQBAAAAsBqhCAAAAIDVHA9FWVlZuuKKKxQbG6uEhAQNHjxYu3btcno3AABUG70JAFAZx0PR+vXrNW7cOG3evFmrV6/W6dOn1a9fP504ccLpXQEAUC30JgBAZRz/8taVK1f63c7OzlZCQoK2bt2qq6++2undAQBQJXoTAKAyjoeis3m9XklSkyZNyr2/qKhIRUVFvtuFhYXBLgkAYDl6EwDgTEH9oIWSkhJNnDhRPXr00CWXXFLumKysLHk8Ht+SnJwczJIAAJajNwEAzhbUUDRu3Djt2LFDS5curXDM1KlT5fV6fUtBQUEwSwIAWI7eBAA4W9BePjd+/Hi98cYb2rBhg1q1alXhOLfbLbfbHawyAADwoTcBAMrjeCgyxujuu+/WsmXLlJOTo5SUFKd3AQBAQOhNAIDKOB6Kxo0bpxdeeEErVqxQbGys9u/fL0nyeDyqX7++07sDAKBK9CYAQGUcf0/RggUL5PV61bt3b7Vo0cK3vPTSS07vCgCAaqE3AQAqE5SXzwEAEE7oTQCAygT10+cAAAAAINwRigAAAABYLWgfyQ0AACq2at/2UJcAh6UndQ11CQBqiCtFAAAAAKxGKAIAAABgNUIRAAAAAKsRigAAAABYjVAEAAAAwGqEIgAAAABWIxQBAAAAsBqhCAAAAIDVCEUAAAAArEYoAgAAAGA1QhEAAAAAqxGKAAAAAFiNUAQAAADAaoQiAAAAAFYjFAEAAACwGqEIAAAAgNUIRQAAAACsRigCAAAAYDVCEQAAAACrEYoAAAAAWI1QBAAAAMBqhCIAAAAAViMUAQAAALAaoQgAAACA1QhFAAAAAKwW9FD0yCOPyOVyaeLEicHeFQAAVaIvAQDOFtRQ9NFHH2nRokXq3LlzMHcDAEC10JcAAOUJWig6fvy4RowYoT//+c9q3LhxsHYDAEC10JcAABUJWigaN26cBgwYoLS0tErHFRUVqbCw0G8BAMBp1e1LEr0JAGwTFYxJly5dqtzcXH300UdVjs3KytKsWbOCUQYAAJIC60sSvQkAbOP4laKCggLdc889ev755xUTE1Pl+KlTp8rr9fqWgoICp0sCAFgs0L4k0ZsAwDaOXynaunWrDh48qMsuu8y3rri4WBs2bNC8efNUVFSkyMhI331ut1tut9vpMgAAkBR4X5LoTQBgG8dDUd++ffXJJ5/4rRs9erQuvPBCTZ48uUzjAQAgmOhLAICqOB6KYmNjdckll/ita9iwoZo2bVpmPQAAwUZfAgBUJehf3goAAAAA4Swonz53tpycnLrYDQAA1UJfAgCciStFAAAAAKxGKAIAAABgtTp5+Rwqlp7UNdQlAAAAAFbjShEAAAAAqxGKAAAAAFiNUAQAAADAaoQiAAAAAFYjFAEAAACwGqEIAAAAgNUIRQAAAACsRigCAAAAYDVCEQAAAACrEYoAAAAAWI1QBAAAAMBqhCIAAAAAViMUAQAAALAaoQgAAACA1QhFAAAAAKxGKAIAAABgNUIRAAAAAKsRigAAAABYjVAEAAAAwGpRoS4AwLlr1b7toS4BktKTuoa6BPzE8DsFwDZcKQIAAABgNUIRAAAAAKsRigAAAABYjVAEAAAAwGqEIgAAAABWC0oo+vrrr3XbbbepadOmql+/vjp16qSPP/44GLsCAKBa6E0AgIo4/pHc3377rXr06KE+ffro7bffVnx8vPLy8tS4cWOndwUAQLXQmwAAlXE8FM2ZM0fJyclasmSJb11KSorTuwEAoNroTQCAyjj+8rnXX39dl19+uW6++WYlJCTo0ksv1Z///OcKxxcVFamwsNBvAQDASfQmAEBlHA9F//d//6cFCxaoffv2WrVqle666y5NmDBBzzzzTLnjs7Ky5PF4fEtycrLTJQEALEdvAgBUxmWMMU5OGB0drcsvv1wffPCBb92ECRP00UcfadOmTWXGFxUVqaioyHe7sLBQycnJ6q1BinLVc6SmVfu2OzJPMKQndQ11CUCNhfNjyyZOPo/8YE4rRyvk9XoVFxfn2LyhRm8KDL0JQLioq77k+JWiFi1a6OKLL/Zbd9FFF+nLL78sd7zb7VZcXJzfAgCAk+hNAIDKOB6KevTooV27dvmt+/zzz9WmTRundwUAQLXQmwAAlXE8FN17773avHmz/vjHP2r37t164YUX9PTTT2vcuHFO7woAgGqhNwEAKuN4KLriiiu0bNkyvfjii7rkkks0e/ZszZ07VyNGjHB6VwAAVAu9CQBQGce/p0iSrr/+el1//fXBmBoAgBqhNwEAKuL4lSIAAAAAOJcQigAAAABYLSgvnws3fN8CACDc0JuAc0M4f6dYODvXnuO4UgQAAADAaoQiAAAAAFYjFAEAAACwGqEIAAAAgNUIRQAAAACsRigCAAAAYDVCEQAAAACrEYoAAAAAWI1QBAAAAMBqhCIAAAAAViMUAQAAALAaoQgAAACA1QhFAAAAAKxGKAIAAABgNUIRAAAAAKsRigAAAABYjVAEAAAAwGqEIgAAAABWIxQBAAAAsBqhCAAAAIDVCEUAAAAArEYoAgAAAGA1QhEAAAAAqxGKAAAAAFiNUAQAAADAao6HouLiYk2bNk0pKSmqX7++2rVrp9mzZ8sY4/SuAACoFnoTAKAyUU5POGfOHC1YsEDPPPOMOnbsqI8//lijR4+Wx+PRhAkTnN4dAABVojcBACrjeCj64IMPNGjQIA0YMECS1LZtW7344ov68MMPnd4VAADVQm8CAFTG8ZfPXXXVVVq7dq0+//xzSdI//vEPbdy4URkZGeWOLyoqUmFhod8CAICT6E0AgMo4fqVoypQpKiws1IUXXqjIyEgVFxfr4Ycf1ogRI8odn5WVpVmzZjldBgAAPvQmAEBlHL9S9PLLL+v555/XCy+8oNzcXD3zzDN6/PHH9cwzz5Q7furUqfJ6vb6loKDA6ZIAAJajNwEAKuP4laL7779fU6ZM0bBhwyRJnTp10t69e5WVlaXMzMwy491ut9xut9NlAADgQ28CAFTG8StFJ0+eVESE/7SRkZEqKSlxelcAAFQLvQkAUBnHrxQNHDhQDz/8sFq3bq2OHTtq27ZteuKJJ3T77bc7vSsAAKqF3gQAqIzjoeipp57StGnT9Jvf/EYHDx5UUlKSxo4dq+nTpzu9KwAAqoXeBACojOOhKDY2VnPnztXcuXOdnhoAgBqhNwEAKuP4e4oAAAAA4FxCKAIAAABgNUIRAAAAAKsRigAAAABYjVAEAAAAwGqEIgAAAABWIxQBAAAAsBqhCAAAAIDVCEUAAAAArEYoAgAAAGA1QhEAAAAAqxGKAAAAAFiNUAQAAADAaoQiAAAAAFYjFAEAAACwGqEIAAAAgNUIRQAAAACsRigCAAAAYDVCEQAAAACrEYoAAAAAWI1QBAAAAMBqhCIAAAAAViMUAQAAALAaoQgAAACA1QhFAAAAAKxGKAIAAABgNUIRAAAAAKsRigAAAABYLeBQtGHDBg0cOFBJSUlyuVxavny53/3GGE2fPl0tWrRQ/fr1lZaWpry8PKfqBQDAD30JAFBbAYeiEydOqEuXLpo/f3659z/66KP605/+pIULF2rLli1q2LCh0tPTderUqVoXCwDA2ehLAIDaigp0g4yMDGVkZJR7nzFGc+fO1R/+8AcNGjRIkvS3v/1NiYmJWr58uYYNG1a7agEAOAt9CQBQW46+pyg/P1/79+9XWlqab53H41Fqaqo2bdpU7jZFRUUqLCz0WwAAcEJN+pJEbwIA2zgaivbv3y9JSkxM9FufmJjou+9sWVlZ8ng8viU5OdnJkgAAFqtJX5LoTQBgm5B/+tzUqVPl9Xp9S0FBQahLAgBYjt4EAHZxNBQ1b95cknTgwAG/9QcOHPDddza32624uDi/BQAAJ9SkL0n0JgCwjaOhKCUlRc2bN9fatWt96woLC7VlyxZdeeWVTu4KAIAq0ZcAANUR8KfPHT9+XLt37/bdzs/P1/bt29WkSRO1bt1aEydO1EMPPaT27dsrJSVF06ZNU1JSkgYPHuxk3QAASKIvAQBqL+BQ9PHHH6tPnz6+25MmTZIkZWZmKjs7W7/73e904sQJ3XnnnTp69Kh69uyplStXKiYmxrmqAQD4/+hLAIDachljTKiLOFNhYaE8Ho96a5CiXPVCXQ6ASqzatz3UJUBSelJXx+b6wZxWjlbI6/XyPpoz0JsAe9Hrasap3lRXfSnknz4HAAAAAKFEKAIAAABgNUIRAAAAAKsRigAAAABYjVAEAAAAwGqEIgAAAABWIxQBAAAAsBqhCAAAAIDVCEUAAAAArEYoAgAAAGA1QhEAAAAAqxGKAAAAAFiNUAQAAADAaoQiAAAAAFYjFAEAAACwGqEIAAAAgNUIRQAAAACsRigCAAAAYDVCEQAAAACrEYoAAAAAWI1QBAAAAMBqhCIAAAAAVosKdQEAzl3pSV1DXQIAAEFFr7MDV4oAAAAAWI1QBAAAAMBqhCIAAAAAViMUAQAAALAaoQgAAACA1QIORRs2bNDAgQOVlJQkl8ul5cuX++47ffq0Jk+erE6dOqlhw4ZKSkrSL3/5S+3bt8/JmgEA8KEvAQBqK+BQdOLECXXp0kXz588vc9/JkyeVm5uradOmKTc3V6+99pp27dqlG264wZFiAQA4G30JAFBbAX9PUUZGhjIyMsq9z+PxaPXq1X7r5s2bp+7du+vLL79U69ata1YlAAAVoC8BAGor6O8p8nq9crlcatSoUbB3BQBAlehLAICzBXylKBCnTp3S5MmTNXz4cMXFxZU7pqioSEVFRb7bhYWFwSwJAGCx6vQlid4EALYJ2pWi06dPa+jQoTLGaMGCBRWOy8rKksfj8S3JycnBKgkAYLHq9iWJ3gQAtglKKCptPHv37tXq1asr/Wvc1KlT5fV6fUtBQUEwSgIAWCyQviTRmwDANo6/fK608eTl5WndunVq2rRppePdbrfcbrfTZQAAICnwviTRmwDANgGHouPHj2v37t2+2/n5+dq+fbuaNGmiFi1a6KabblJubq7eeOMNFRcXa//+/ZKkJk2aKDo62rnKAQAQfQkAUHsuY4wJZIOcnBz16dOnzPrMzEzNnDlTKSkp5W63bt069e7du8r5CwsL5fF41FuDFOWqF0hpAIBa+sGcVo5WyOv1VvkSs3AR7L4k0ZsAIFTqqi8FfKWod+/eqixHBZixAACoFfoSAKC2gv49RQAAAAAQzghFAAAAAKxGKAIAAABgNcc/khs4F63at92xudKTujo2FwCc65x8fg0GnrNRlXD/HQ5X59pjiytFAAAAAKxGKAIAAABgNUIRAAAAAKsRigAAAABYjVAEAAAAwGqEIgAAAABWIxQBAAAAsBqhCAAAAIDVCEUAAAAArEYoAgAAAGA1QhEAAAAAqxGKAAAAAFiNUAQAAADAaoQiAAAAAFYjFAEAAACwGqEIAAAAgNUIRQAAAACsRigCAAAAYDVCEQAAAACrRYW6AKAmVu3bHuoSKuR0belJXR2dDwAAAP64UgQAAADAaoQiAAAAAFYjFAEAAACwGqEIAAAAgNUIRQAAAACsFnAo2rBhgwYOHKikpCS5XC4tX768wrG//vWv5XK5NHfu3FqUCABAxehLAIDaCjgUnThxQl26dNH8+fMrHbds2TJt3rxZSUlJNS4OAICq0JcAALUV8PcUZWRkKCMjo9IxX3/9te6++26tWrVKAwYMqHFxAABUhb4EAKgtx7+8taSkRCNHjtT999+vjh07Vjm+qKhIRUVFvtuFhYVOlwQAsFigfUmiNwGAbRz/oIU5c+YoKipKEyZMqNb4rKwseTwe35KcnOx0SQAAiwXalyR6EwDYxtFQtHXrVj355JPKzs6Wy+Wq1jZTp06V1+v1LQUFBU6WBACwWE36kkRvAgDbOBqK3nvvPR08eFCtW7dWVFSUoqKitHfvXv32t79V27Zty93G7XYrLi7ObwEAwAk16UsSvQkAbOPoe4pGjhyptLQ0v3Xp6ekaOXKkRo8e7eSuAACoEn0JAFAdAYei48ePa/fu3b7b+fn52r59u5o0aaLWrVuradOmfuPr1aun5s2bq0OHDrWvFgCAs9CXAAC1FXAo+vjjj9WnTx/f7UmTJkmSMjMzlZ2d7VhhAABUB30JAFBbAYei3r17yxhT7fFffPFFoLsAAKDa6EsAgNpy/CO5AQAAAOBcQigCAAAAYDVHP30OqCvpSV0dnW/Vvu2OzeV0bQAAIHTCua87+e8XKbyPNdi4UgQAAADAaoQiAAAAAFYjFAEAAACwGqEIAAAAgNUIRQAAAACsRigCAAAAYDVCEQAAAACrEYoAAAAAWI1QBAAAAMBqhCIAAAAAViMUAQAAALAaoQgAAACA1QhFAAAAAKxGKAIAAABgNUIRAAAAAKsRigAAAABYjVAEAAAAwGqEIgAAAABWIxQBAAAAsFpUqAsAwkF6UtdQlwAAP0k8vwLBw+PLOVwpAgAAAGA1QhEAAAAAqxGKAAAAAFiNUAQAAADAaoQiAAAAAFYLOBRt2LBBAwcOVFJSklwul5YvX15mzKeffqobbrhBHo9HDRs21BVXXKEvv/zSiXoBAPBDXwIA1FbAoejEiRPq0qWL5s+fX+79e/bsUc+ePXXhhRcqJydH//znPzVt2jTFxMTUulgAAM5GXwIA1FbA31OUkZGhjIyMCu9/4IEH1L9/fz366KO+de3atatZdQAAVIG+BACoLUffU1RSUqI333xTF1xwgdLT05WQkKDU1NRyX8pQqqioSIWFhX4LAABOqElfkuhNAGAbR0PRwYMHdfz4cT3yyCO67rrr9M477+jGG2/UkCFDtH79+nK3ycrKksfj8S3JyclOlgQAsFhN+pJEbwIA2zh+pUiSBg0apHvvvVddu3bVlClTdP3112vhwoXlbjN16lR5vV7fUlBQ4GRJAACL1aQvSfQmALBNwO8pqkyzZs0UFRWliy++2G/9RRddpI0bN5a7jdvtltvtdrIMAAAk1awvSfQmALCNo1eKoqOjdcUVV2jXrl1+6z///HO1adPGyV0BAFAl+hIAoDoCvlJ0/Phx7d6923c7Pz9f27dvV5MmTdS6dWvdf//9uuWWW3T11VerT58+Wrlypf7+978rJyfHyboBAJBEXwIA1J7LGGMC2SAnJ0d9+vQpsz4zM1PZ2dmSpMWLFysrK0tfffWVOnTooFmzZmnQoEHVmr+wsFAej0e9NUhRrnqBlAYAqKUfzGnlaIW8Xq/i4uJCXU61BLsvSfQmAAiVuupLAYeiYKPxAEDonIuhqC7QmwAgNOqqLzn6niIAAAAAONcQigAAAABYjVAEAAAAwGqEIgAAAABWIxQBAAAAsBqhCAAAAIDVCEUAAAAArEYoAgAAAGA1QhEAAAAAqxGKAAAAAFiNUAQAAADAaoQiAAAAAFYjFAEAAACwGqEIAAAAgNUIRQAAAACsRigCAAAAYDVCEQAAAACrEYoAAAAAWI1QBAAAAMBqhCIAAAAAViMUAQAAALBaVKgLOJsxRpL0g05LJsTFAIBlftBpSf95LsaP6E0AEBp11ZfCLhQdO3ZMkrRRb4W4EgCw15EjR+TxeEJdRtigNwFAaAW7L7lMmP05sKSkRPv27VNsbKxcLleF4woLC5WcnKyCggLFxcXVYYXO4RjCA8cQHjiG8OD1etW6dWt9++23atSoUajLCRv0pnMLxxAeOIbwcK4fQ131pbC7UhQREaFWrVpVe3xcXNw5eYLPxDGEB44hPHAM4SEigrecnonedG7iGMIDxxAezvVjCHZfousBAAAAsBqhCAAAAIDVztlQ5Ha7NWPGDLnd7lCXUmMcQ3jgGMIDxxAefgrHEEo/hZ8fxxAeOIbwwDGEXl3VH3YftAAAAAAAdemcvVIEAAAAAE4gFAEAAACwGqEIAAAAgNUIRQAAAACsFtahaP78+Wrbtq1iYmKUmpqqDz/8sNLxr7zyii688ELFxMSoU6dOeuutt+qo0rKysrJ0xRVXKDY2VgkJCRo8eLB27dpV6TbZ2dlyuVx+S0xMTB1VXNbMmTPL1HPhhRdWuk04nQNJatu2bZljcLlcGjduXLnjw+EcbNiwQQMHDlRSUpJcLpeWL1/ud78xRtOnT1eLFi1Uv359paWlKS8vr8p5A3081UZlx3D69GlNnjxZnTp1UsOGDZWUlKRf/vKX2rdvX6Vz1uT3MVjHIEmjRo0qU891111X5bzhch4klfvYcLlceuyxxyqcs67PQziiN9Gbaove9B/h8pxIbwqP8yCFrjeFbSh66aWXNGnSJM2YMUO5ubnq0qWL0tPTdfDgwXLHf/DBBxo+fLjuuOMObdu2TYMHD9bgwYO1Y8eOOq78R+vXr9e4ceO0efNmrV69WqdPn1a/fv104sSJSreLi4vTN99841v27t1bRxWXr2PHjn71bNy4scKx4XYOJOmjjz7yq3/16tWSpJtvvrnCbUJ9Dk6cOKEuXbpo/vz55d7/6KOP6k9/+pMWLlyoLVu2qGHDhkpPT9epU6cqnDPQx1Mwj+HkyZPKzc3VtGnTlJubq9dee027du3SDTfcUOW8gfw+1lZV50GSrrvuOr96XnzxxUrnDKfzIMmv9m+++UaLFy+Wy+XSL37xi0rnrcvzEG7oTfQmJ9CbfhROz4n0pvA4D1IIe5MJU927dzfjxo3z3S4uLjZJSUkmKyur3PFDhw41AwYM8FuXmppqxo4dG9Q6q+vgwYNGklm/fn2FY5YsWWI8Hk/dFVWFGTNmmC5dulR7fLifA2OMueeee0y7du1MSUlJufeH2zmQZJYtW+a7XVJSYpo3b24ee+wx37qjR48at9ttXnzxxQrnCfTx5KSzj6E8H374oZFk9u7dW+GYQH8fnVTeMWRmZppBgwYFNE+4n4dBgwaZa6+9ttIxoTwP4YDeFHr0ptCjN/0Hval2wqk3heWVou+//15bt25VWlqab11ERITS0tK0adOmcrfZtGmT33hJSk9Pr3B8XfN6vZKkJk2aVDru+PHjatOmjZKTkzVo0CDt3LmzLsqrUF5enpKSknT++edrxIgR+vLLLyscG+7n4Pvvv9dzzz2n22+/XS6Xq8Jx4XYOzpSfn6/9+/f7/Zw9Ho9SU1Mr/DnX5PFU17xer1wulxo1alTpuEB+H+tCTk6OEhIS1KFDB9111106cuRIhWPD/TwcOHBAb775pu64444qx4bbeagr9KbweV6kN4X+HJyJ3hRez4n0ppqdh7AMRYcPH1ZxcbESExP91icmJmr//v3lbrN///6AxtelkpISTZw4UT169NAll1xS4bgOHTpo8eLFWrFihZ577jmVlJToqquu0ldffVWH1f5HamqqsrOztXLlSi1YsED5+fnq1auXjh07Vu74cD4HkrR8+XIdPXpUo0aNqnBMuJ2Ds5X+LAP5Odfk8VSXTp06pcmTJ2v48OGKi4urcFygv4/Bdt111+lvf/ub1q5dqzlz5mj9+vXKyMhQcXFxuePD/Tw888wzio2N1ZAhQyodF27noS7Rm8LjeZHeFPpzcDZ6U/g8J9Kban4eompbLKo2btw47dixo8rXNl555ZW68sorfbevuuoqXXTRRVq0aJFmz54d7DLLyMjI8P1/586dlZqaqjZt2ujll1+uVmIPN3/961+VkZGhpKSkCseE2zn4qTt9+rSGDh0qY4wWLFhQ6dhw+30cNmyY7/87deqkzp07q127dsrJyVHfvn3rvJ7aWrx4sUaMGFHlm7fD7Tyg5uhN4YHeFH7oTeGjLntTWF4patasmSIjI3XgwAG/9QcOHFDz5s3L3aZ58+YBja8r48eP1xtvvKF169apVatWAW1br149XXrppdq9e3eQqgtMo0aNdMEFF1RYT7ieA0nau3ev1qxZo1/96lcBbRdu56D0ZxnIz7kmj6e6UNp09u7dq9WrV1f6l7jyVPX7WNfOP/98NWvWrMJ6wvU8SNJ7772nXbt2Bfz4kMLvPAQTvelH4fa8SG8KPXrTf4TbcyK9qfrnISxDUXR0tLp166a1a9f61pWUlGjt2rV+fyk505VXXuk3XpJWr15d4fhgM8Zo/PjxWrZsmd59912lpKQEPEdxcbE++eQTtWjRIggVBu748ePas2dPhfWE2zk405IlS5SQkKABAwYEtF24nYOUlBQ1b97c7+dcWFioLVu2VPhzrsnjKdhKm05eXp7WrFmjpk2bBjxHVb+Pde2rr77SkSNHKqwnHM9Dqb/+9a/q1q2bunTpEvC24XYegone9KNwe16kN4Uevek/wu05kd4UwHmo1cc0BNHSpUuN2+022dnZ5l//+pe58847TaNGjcz+/fuNMcaMHDnSTJkyxTf+/fffN1FRUebxxx83n376qZkxY4apV6+e+eSTT0JS/1133WU8Ho/Jyckx33zzjW85efKkb8zZxzBr1iyzatUqs2fPHrN161YzbNgwExMTY3bu3BmKQzC//e1vTU5OjsnPzzfvv/++SUtLM82aNTMHDx4st/5wOweliouLTevWrc3kyZPL3BeO5+DYsWNm27ZtZtu2bUaSeeKJJ8y2bdt8n37zyCOPmEaNGpkVK1aYf/7zn2bQoEEmJSXFfPfdd745rr32WvPUU0/5blf1eKrLY/j+++/NDTfcYFq1amW2b9/u9/goKiqq8Biq+n2sy2M4duyYue+++8ymTZtMfn6+WbNmjbnssstM+/btzalTpyo8hnA6D6W8Xq9p0KCBWbBgQblzhPo8hBt6E73JKfSm8HpOpDeFx3koFYreFLahyBhjnnrqKdO6dWsTHR1tunfvbjZv3uy775prrjGZmZl+419++WVzwQUXmOjoaNOxY0fz5ptv1nHF/yGp3GXJkiW+MWcfw8SJE33Hm5iYaPr3729yc3Prvvj/75ZbbjEtWrQw0dHRpmXLluaWW24xu3fv9t0f7ueg1KpVq4wks2vXrjL3heM5WLduXbm/O6V1lpSUmGnTppnExETjdrtN3759yxxbmzZtzIwZM/zWVfZ4qstjyM/Pr/DxsW7dugqPoarfx7o8hpMnT5p+/fqZ+Ph4U69ePdOmTRszZsyYMg0knM9DqUWLFpn69eubo0ePljtHqM9DOKI30ZucQG/6Ubg8J9KbwuM8lApFb3IZY0z1rysBAAAAwE9LWL6nCAAAAADqCqEIAAAAgNUIRQAAAACsRigCAAAAYDVCEQAAAACrEYoAAAAAWI1QBAAAAMBqhCIAAAAAViMUAQAAALAaoQgAAACA1QhFAAAAAKxGKAIAAABgtf8Han81jzEP7hsAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 1000x1500 with 2 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "#Figure 3.51 p96\n",
+    "pq_pred1 = np.zeros([18, 18])\n",
+    "pq_pred1[ 3:7,1:3] = 1\n",
+    "pq_pred1[3:6,3:7]=1\n",
+    "pq_pred2 = np.zeros([18, 18])\n",
+    "pq_pred2[13:16,4:6] = 1\n",
+    "pq_pred3 = np.zeros([18, 18])\n",
+    "pq_pred3[7:12,13:17] = 1\n",
+    "pq_pred4 = np.zeros([18, 18])\n",
+    "pq_pred4[13:15,13:17] = 1\n",
+    "pq_pred4[15,15] = 1\n",
+    "\n",
+    "pq_ref1 = np.zeros([18, 18])\n",
+    "pq_ref1[2:7, 1:3] = 1\n",
+    "pq_ref1[2:5,3:6] = 1\n",
+    "pq_ref2 = np.zeros([18, 18])\n",
+    "pq_ref2[6:12,12:17] = 1\n",
+    "pq_ref3 = np.zeros([18, 18])\n",
+    "pq_ref3[14:15:,7:10] = 1\n",
+    "pq_ref3[13:16,8:9] = 1\n",
+    "\n",
+    "fig,ax = plt.subplots(1,2,figsize=(10,15))\n",
+    "ax[1].imshow(pq_pred1+pq_pred2+pq_pred3+pq_pred4)\n",
+    "ax[0].imshow(pq_ref1 + pq_ref2+pq_ref3)\n",
+    "ax[1].set_title('Prediction')\n",
+    "ax[0].set_title('Reference')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "id": "bc86ce0a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Text(0.5, 1.0, 'Prediction 2')"
+      ]
+     },
+     "execution_count": 35,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzYAAAElCAYAAAA2knddAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAoPUlEQVR4nO3de3RU5aH+8Wdym0BIBoIhFwghUKwVuSgUjnIRSiQNQgWPIkhPA1jLaUFFlhc4FQKCpqjLlVayotSleDje9QgeD2IxQikW8BLoKh6lgaaaEhMuSgJBIiTv7w9/GRkTIMPsYfLOfD9rzVrOnr33+zKQx/3M3tnjMsYYAQAAAIDFokI9AQAAAAAIFMUGAAAAgPUoNgAAAACsR7EBAAAAYD2KDQAAAADrUWwAAAAAWI9iAwAAAMB6FBsAAAAA1qPYAAAAALAexQY+ysvLNW7cOHk8HrlcLq1duzbUUwJguV69emnGjBne55s3b5bL5dLmzZsdG8PlcmnJkiWO7Q9A+0GGoK0oNhZbvXq1XC6X9xETE6Pu3btrxowZ2r9//3ntMz8/X3/961/1wAMPaM2aNRoyZIjDswZwIX03J+Lj43XxxRdr7ty5qqmpCfX0/LJ+/fp2d+Dx+eefa8GCBRozZowSExMdP9gCQo0MCa7S0lLNmjVLF198sTp27KjevXvr5z//uT7//PNQT81KMaGeAAJ3//33Kzs7WydOnND27du1evVqbd26Vbt371Z8fHyb9/PVV19p27Zt+vWvf625c+cGccYALrTTc2Lr1q0qKSnR+vXrtXv3bnXs2PGCzmXUqFH66quvFBcX59d269evV3FxcasHJl999ZViYi78/9L27NmjFStWqG/fvurfv7+2bdt2wecAXAhkSHDce++9+uKLL3TjjTeqb9+++vvf/66VK1fqjTfe0K5du5SWlnbB52Qzik0YyMvL855Z+fnPf66LLrpIK1as0Ouvv64pU6a0eT8HDx6UJHXu3NmxuZ04cUJxcXGKiuLkIBBK382Jrl276tFHH9W6des0bdq0Vrepr69XQkKC43OJiory60OXtnB6f201ePBgHT58WMnJyXrllVd04403hmQeQLCRIcHx6KOPasSIET7HST/+8Y919dVXa+XKlVq+fHlI5mUrjjbD0MiRIyVJ+/bt8y775JNPdMMNNyg5OVnx8fEaMmSIXn/9de/rS5YsUVZWliTp7rvvlsvlUq9evbyv79+/X7NmzVJqaqrcbrf69eunp556ymfc5mteX3jhBd13333q3r27OnbsqLq6OknSjh079OMf/1gej0cdO3bU1VdfrXfffddnH0uWLJHL5dLevXs1Y8YMde7cWR6PRzNnztTx48db/Fn/67/+S0OHDlXHjh3VpUsXjRo1Sn/4wx981nnzzTc1cuRIJSQkKDExUddee60++uij83hngfDxox/9SJJUUVEhSZoxY4Y6deqkffv2afz48UpMTNT06dMlSU1NTSoqKlK/fv0UHx+v1NRUzZ49W19++aXPPo0xWr58uXr06KGOHTtqzJgxrf6snen6+B07dmj8+PHq0qWLEhISNGDAAP32t7/1zq+4uFiSfC6Ladba9fE7d+5UXl6ekpKS1KlTJ40dO1bbt2/3Waf5Mpt3331X8+fPV0pKihISEjR58mTvhz1nk5iYqOTk5HOuB4QbMuQbgWbIqFGjWnz4O2rUKCUnJ+vjjz8+5/bwxRmbMPSPf/xDktSlSxdJ0kcffaThw4ere/fuWrBggRISEvTSSy9p0qRJevXVVzV58mRdf/316ty5s+68805NmzZN48ePV6dOnSRJNTU1+pd/+Re5XC7NnTtXKSkpevPNN3XLLbeorq5O8+bN8xl/2bJliouL01133aWGhgbFxcXpnXfeUV5engYPHqyCggJFRUXp6aef1o9+9CP96U9/0tChQ332MWXKFGVnZ6uwsFBlZWV68skn1a1bN61YscK7ztKlS7VkyRJdddVVuv/++xUXF6cdO3bonXfe0bhx4yRJa9asUX5+vnJzc7VixQodP35cJSUlGjFihHbu3OlT3oBI0vzBR9euXb3LTp06pdzcXI0YMUKPPPKI9/KS2bNna/Xq1Zo5c6Zuv/12VVRUaOXKldq5c6feffddxcbGSpIWL16s5cuXa/z48Ro/frzKyso0btw4ff311+ecz8aNGzVhwgSlp6frjjvuUFpamj7++GO98cYbuuOOOzR79mxVVVVp48aNWrNmzTn399FHH2nkyJFKSkrSPffco9jYWD3xxBMaPXq0/vjHP2rYsGE+6992223q0qWLCgoK9I9//ENFRUWaO3euXnzxxTa/p0AkIUOClyHHjh3TsWPHdNFFF/m9bcQzsNbTTz9tJJm3337bHDx40FRWVppXXnnFpKSkGLfbbSorK40xxowdO9b079/fnDhxwrttU1OTueqqq0zfvn29yyoqKowk8/DDD/uMc8stt5j09HRz6NAhn+VTp041Ho/HHD9+3BhjzKZNm4wk07t3b++y5rH69u1rcnNzTVNTk3f58ePHTXZ2trnmmmu8ywoKCowkM2vWLJ+xJk+ebLp27ep9Xl5ebqKioszkyZNNY2Ojz7rNYxw9etR07tzZ3HrrrT6vV1dXG4/H02I5EI5ay4kXXnjBdO3a1XTo0MH885//NMYYk5+fbySZBQsW+Gz/pz/9yUgyzz77rM/yDRs2+Cw/cOCAiYuLM9dee63Pz/l//Md/GEkmPz/fu6w5KzZt2mSMMebUqVMmOzvbZGVlmS+//NJnnNP3NWfOHHOm/21JMgUFBd7nkyZNMnFxcWbfvn3eZVVVVSYxMdGMGjWqxfuTk5PjM9add95poqOjzZEjR1odrzUvv/yyz58LCAdkyIXLkGbLli0zkkxpaanf20Y6LkULAzk5OUpJSVFmZqZuuOEGJSQk6PXXX1ePHj30xRdf6J133tGUKVN09OhRHTp0SIcOHdLhw4eVm5ur8vLys95BzRijV199VRMnTpQxxrv9oUOHlJubq9raWpWVlflsk5+frw4dOnif79q1S+Xl5br55pt1+PBh7/b19fUaO3astmzZoqamJp99/Pu//7vP85EjR+rw4cPey9rWrl2rpqYmLV68uMUp3OZTyxs3btSRI0c0bdo0n3lHR0dr2LBh2rRpk/9vNmCp03Ni6tSp6tSpk1577TV1797dZ71f/vKXPs9ffvlleTweXXPNNT4/R4MHD1anTp28P0dvv/22vv76a912220+l3d894xua3bu3KmKigrNmzevxe/4nb6vtmpsbNQf/vAHTZo0Sb179/YuT09P180336ytW7d6s6TZL37xC5+xRo4cqcbGRn366ad+jw+EIzLkwmTIli1btHTpUk2ZMsV7uR/ajkvRwkBxcbEuvvhi1dbW6qmnntKWLVvkdrslSXv37pUxRosWLdKiRYta3f7AgQMtgqnZwYMHdeTIEa1atUqrVq064/any87O9nleXl4u6ZvCcya1tbXeS+ckqWfPnj6vN7/25ZdfKikpSfv27VNUVJQuvfTSM+6zedwzBUNSUtIZtwXCTXNOxMTEKDU1Vd///vdbfCgQExOjHj16+CwrLy9XbW2tunXr1up+m3/+m//n3bdvX5/XU1JSfH62W9N8Sctll13W9j/QWRw8eFDHjx/X97///Rav/eAHP1BTU5MqKyvVr18/7/KzZQ4AMqRZMDPkk08+0eTJk3XZZZfpySefPM/ZRzaKTRgYOnSo904lkyZN0ogRI3TzzTdrz5493jMhd911l3Jzc1vd/nvf+94Z9928/U9/+tMzFpMBAwb4PD/9bM3p+3j44Yc1aNCgVvfR/Ps8zaKjo1tdzxhzxrl+V/O4a9asafV2iaG4rSMQKqfnxJm43e4WBypNTU3q1q2bnn322Va3SUlJcWyOoeRE5gDhjAw5u0AzpLKy0vsF6evXr1diYqKT04sYHNmFmejoaBUWFmrMmDFauXKlZs2aJUmKjY1VTk6O3/tLSUlRYmKiGhsbz2t7SerTp4+kb86QnO8+WttnU1OT/u///u+MZal53G7dujk2LhBp+vTpo7ffflvDhw9v8aHF6ZrvqlheXu5z6cbBgwfP+Yll88/q7t27z/qz2tZLSlJSUtSxY0ft2bOnxWuffPKJoqKilJmZ2aZ9AQgMGXJuhw8f1rhx49TQ0KDS0lKlp6c7tu9Iw+/YhKHRo0dr6NChKioqUlJSkkaPHq0nnnii1W+xPdetCKOjo/Wv//qvevXVV7V7926/t5e++Z6HPn366JFHHtGxY8fOax/fNWnSJEVFRen+++9v8fs5zZ+O5ObmKikpSQ8++KBOnjzpyLhApJkyZYoaGxu1bNmyFq+dOnVKR44ckfTN9fexsbF67LHHfD6hLCoqOucYV1xxhbKzs1VUVOTdX7PT99X8fRjfXee7oqOjNW7cOK1bt857l0jpmzs8PvfccxoxYgSXogIXCBlydvX19Ro/frz279+v9evXt7gUD/7hjE2Yuvvuu3XjjTdq9erVKi4u1ogRI9S/f3/deuut6t27t2pqarRt2zb985//1F/+8pez7us3v/mNNm3apGHDhunWW2/VpZdeqi+++EJlZWV6++239cUXX5x1+6ioKD355JPKy8tTv379NHPmTHXv3l379+/Xpk2blJSUpP/5n//x68/3ve99T7/+9a+1bNkyjRw5Utdff73cbrfef/99ZWRkqLCwUElJSSopKdG//du/6YorrtDUqVOVkpKizz77TP/7v/+r4cOHa+XKlX6NC0Saq6++WrNnz1ZhYaF27dqlcePGKTY2VuXl5Xr55Zf129/+VjfccINSUlJ01113qbCwUBMmTND48eO1c+dOvfnmm+e8ZWlUVJRKSko0ceJEDRo0SDNnzlR6ero++eQTffTRR3rrrbckffMhiSTdfvvtys3NVXR0tKZOndrqPpcvX66NGzdqxIgR+tWvfqWYmBg98cQTamho0EMPPeToe9T8BXrN37exZs0abd26VZJ03333OToWYBsy5OymT5+u9957T7NmzdLHH3/s8901nTp10qRJkxwbKyKE6G5scEDzLQbff//9Fq81NjaaPn36mD59+phTp06Zffv2mZ/97GcmLS3NxMbGmu7du5sJEyaYV155xbvNmW73bIwxNTU1Zs6cOSYzM9PExsaatLQ0M3bsWLNq1SrvOs23X3z55Zdbne/OnTvN9ddfb7p27WrcbrfJysoyU6ZM8bmdYfPtng8ePNjqn7WiosJn+VNPPWUuv/xy43a7TZcuXczVV19tNm7c6LPOpk2bTG5urvF4PCY+Pt706dPHzJgxw3zwwQdnfnOBMHG2nDhdfn6+SUhIOOPrq1atMoMHDzYdOnQwiYmJpn///uaee+4xVVVV3nUaGxvN0qVLTXp6uunQoYMZPXq02b17t8nKyjrrrVqbbd261VxzzTUmMTHRJCQkmAEDBpjHHnvM+/qpU6fMbbfdZlJSUozL5fK5bau+c6tWY4wpKyszubm5plOnTqZjx45mzJgx5s9//nOb3p8zzbE1ks74AGxHhgQ3Q7Kyss6YH1lZWWfdFi25jOE3IwEAAADYjd+xAQAAAGA9ig0AAAAA61FsAAAAAFiPYgMAAADAehQbAAAAANZrd99j09TUpKqqKiUmJrb5G2IBBIcxRkePHlVGRoaiouz5HIQcAdoPG3OEDAHaD38ypN0Vm6qqKmVmZoZ6GgBOU1lZqR49eoR6Gm1GjgDtj005QoYA7U9bMqTdFZvExERJ0giNV4xiQzwbILKd0klt1Xrvz6UtyBGg/bAxR8gQoP3wJ0PaXbFpPuUbo1jFuAgTIKT+/9f32nYpBjkCtCMW5ggZArQjfmSIHRe7AgAAAMBZUGwAAAAAWC9oxaa4uFi9evVSfHy8hg0bpvfeey9YQwEIQ2QIgECRI0BkCUqxefHFFzV//nwVFBSorKxMAwcOVG5urg4cOBCM4QCEGTIEQKDIESDyBKXYPProo7r11ls1c+ZMXXrppXr88cfVsWNHPfXUUy3WbWhoUF1dnc8DQGTzJ0MkcgRASxyLAJHH8WLz9ddf68MPP1ROTs63g0RFKScnR9u2bWuxfmFhoTwej/fBfeOByOZvhkjkCABfHIsAkcnxYnPo0CE1NjYqNTXVZ3lqaqqqq6tbrL9w4ULV1tZ6H5WVlU5PCYBF/M0QiRwB4ItjESAyhfx7bNxut9xud6inAcBi5AiAQJAhQHhw/IzNRRddpOjoaNXU1Pgsr6mpUVpamtPDAQgzZAiAQJEjQGRyvNjExcVp8ODBKi0t9S5rampSaWmprrzySqeHAxBmyBAAgSJHgMgUlEvR5s+fr/z8fA0ZMkRDhw5VUVGR6uvrNXPmzGAMByDMkCEAAkWOAJEnKMXmpptu0sGDB7V48WJVV1dr0KBB2rBhQ4tf4gOA1pAhAAJFjgCRx2WMMaGexOnq6urk8Xg0WtcpxhUb6ukAEe2UOanNWqfa2lolJSWFejptRo4A7YeNOUKGAO2HPxkSlC/oBAAAAIALiWIDAAAAwHoUGwAAAADWo9gAAAAAsB7FBgAAAID1KDYAAAAArEexAQAAAGA9ig0AAAAA61FsAAAAAFiPYgMAAADAehQbAAAAANaj2AAAAACwHsUGAAAAgPUoNgAAAACsR7EBAAAAYD2KDQAAAADrUWwAAAAAWI9iAwAAAMB6FBsAAAAA1qPYAAAAALAexQYAAACA9Sg2AAAAAKzneLEpLCzUD3/4QyUmJqpbt26aNGmS9uzZ4/QwAMIUGQIgUOQIEJkcLzZ//OMfNWfOHG3fvl0bN27UyZMnNW7cONXX1zs9FIAwRIYACBQ5AkSmGKd3uGHDBp/nq1evVrdu3fThhx9q1KhRLdZvaGhQQ0OD93ldXZ3TUwJgEX8zRCJHAPjiWASITEH/HZva2lpJUnJycquvFxYWyuPxeB+ZmZnBnhIAi5wrQyRyBMDZcSwCRAaXMcYEa+dNTU36yU9+oiNHjmjr1q2trtPapySZmZkaresU44oN1tQAtMEpc1KbtU61tbVKSkq64OO3JUMkcgRoz2zIETIEaL/8yRDHL0U73Zw5c7R79+6zHpC43W653e5gTgOApdqSIRI5AuDMOBYBIkfQis3cuXP1xhtvaMuWLerRo0ewhgEQpsgQAIEiR4DI4nixMcbotttu02uvvabNmzcrOzvb6SEAhDEyBECgyBEgMjlebObMmaPnnntO69atU2JioqqrqyVJHo9HHTp0cHo4AGGGDAEQKHIEiEyO3xWtpKREtbW1Gj16tNLT072PF1980emhAIQhMgRAoMgRIDIF5VI0ADhfZAiAQJEjQGQK+vfYAAAAAECwUWwAAAAAWI9iAwAAAMB6FBsAAAAA1qPYAAAAALAexQYAAACA9Sg2AAAAAKxHsQEAAABgPYoNAAAAAOtRbAAAAABYj2IDAAAAwHoUGwAAAADWo9gAAAAAsB7FBgAAAID1KDYAAAAArEexAQAAAGC9mFBPAAAAAO3PW1W7Qj0FSMrNGBTqKViDMzYAAAAArEexAQAAAGA9ig0AAAAA61FsAAAAAFgv6MXmN7/5jVwul+bNmxfsoQCEITIEQKDIESAyBLXYvP/++3riiSc0YMCAYA4DIEyRIQACRY4AkSNoxebYsWOaPn26fv/736tLly7BGgZAmCJDAASKHAEiS9CKzZw5c3TttdcqJyfnrOs1NDSorq7O5wEAbc0QiRwB0DqORYDIEpQv6HzhhRdUVlam999//5zrFhYWaunSpcGYBgBL+ZMhEjkCoCWORYDI4/gZm8rKSt1xxx169tlnFR8ff871Fy5cqNraWu+jsrLS6SkBsIi/GSKRIwB8cSwCRCbHz9h8+OGHOnDggK644grvssbGRm3ZskUrV65UQ0ODoqOjva+53W653W6npwHAUv5miESOAPDFsQgQmRwvNmPHjtVf//pXn2UzZ87UJZdconvvvbfFAQkAnI4MARAocgSITI4Xm8TERF122WU+yxISEtS1a9cWywHgu8gQAIEiR4DIFPQv6AQAAACAYAvKXdG+a/PmzRdiGABhigwBEChyBAh/F6TY4MJ7q2pXqKfQLuRmDAr1FAAAAHABcCkaAAAAAOtRbAAAAABYj2IDAAAAwHoUGwAAAADWo9gAAAAAsB7FBgAAAID1KDYAAAAArEexAQAAAGA9ig0AAAAA61FsAAAAAFiPYgMAAADAehQbAAAAANaj2AAAAACwHsUGAAAAgPUoNgAAAACsR7EBAAAAYL2YUE8AABC+3qraFdLxczMGhXR8AMCFwxkbAAAAANaj2AAAAACwHsUGAAAAgPUoNgAAAACsF5Ris3//fv30pz9V165d1aFDB/Xv318ffPBBMIYCEIbIEACBIkeAyOP4XdG+/PJLDR8+XGPGjNGbb76plJQUlZeXq0uXLk4PBSAMkSEAAkWOAJHJ8WKzYsUKZWZm6umnn/Yuy87OPuP6DQ0Namho8D6vq6tzekoALOJvhkjkCABfHIsAkcnxS9Fef/11DRkyRDfeeKO6deumyy+/XL///e/PuH5hYaE8Ho/3kZmZ6fSUAFjE3wyRyBEAvjgWASKT48Xm73//u0pKStS3b1+99dZb+uUvf6nbb79dzzzzTKvrL1y4ULW1td5HZWWl01MCYBF/M0QiRwD44lgEiEyOX4rW1NSkIUOG6MEHH5QkXX755dq9e7cef/xx5efnt1jf7XbL7XY7PQ0AlvI3QyRyBIAvjkWAyOT4GZv09HRdeumlPst+8IMf6LPPPnN6KABhiAwBEChyBIhMjheb4cOHa8+ePT7L/va3vykrK8vpoQCEITIEQKDIESAyOV5s7rzzTm3fvl0PPvig9u7dq+eee06rVq3SnDlznB4KQBgiQwAEihwBIpPjxeaHP/yhXnvtNT3//PO67LLLtGzZMhUVFWn69OlODwUgDJEhAAJFjgCRyfGbB0jShAkTNGHChGDsGkAEIEMABIocASKP42dsAAAAAOBCo9gAAAAAsB7FBgAAAID1KDYAAAAArEexAQAAAGA9ig0AAAAA61FsAAAAAFiPYgMAAADAehQbAAAAANaj2AAAAACwHsUGAAAAgPUoNgAAAACsR7EBAAAAYD2KDQAAAADrUWwAAAAAWI9iAwAAAMB6MaGeAAAgfOVmDAr1FAAAEYIzNgAAAACsR7EBAAAAYD2KDQAAAADrUWwAAAAAWM/xYtPY2KhFixYpOztbHTp0UJ8+fbRs2TIZY5weCkAYIkMABIocASKT43dFW7FihUpKSvTMM8+oX79++uCDDzRz5kx5PB7dfvvtTg8HIMyQIQACRY4AkcnxYvPnP/9Z1113na699lpJUq9evfT888/rvffec3ooAGGIDAEQKHIEiEyOX4p21VVXqbS0VH/7298kSX/5y1+0detW5eXltbp+Q0OD6urqfB4AIpe/GSKRIwB8cSwCRCbHz9gsWLBAdXV1uuSSSxQdHa3GxkY98MADmj59eqvrFxYWaunSpU5PA4Cl/M0QiRwB4ItjESAyOX7G5qWXXtKzzz6r5557TmVlZXrmmWf0yCOP6Jlnnml1/YULF6q2ttb7qKysdHpKACzib4ZI5AgAXxyLAJHJ8TM2d999txYsWKCpU6dKkvr3769PP/1UhYWFys/Pb7G+2+2W2+12ehoALOVvhkjkCABfHIsAkcnxMzbHjx9XVJTvbqOjo9XU1OT0UADCEBkCIFDkCBCZHD9jM3HiRD3wwAPq2bOn+vXrp507d+rRRx/VrFmznB4KQBgiQwAEihwBIpPjxeaxxx7TokWL9Ktf/UoHDhxQRkaGZs+ercWLFzs9FIAwRIYACBQ5AkQml2lnX8NbV1cnj8ej0bpOMa7YUE/HWm9V7Qr1FNqF3IxBoZ6C1U6Zk9qsdaqtrVVSUlKop9Nm5AjQftiYI2TINziWaB8i/VjGnwxx/HdsAAAAAOBCc/xSNLQPkd7uAQAAEFk4YwMAAADAehQbAAAAANaj2AAAAACwHsUGAAAAgPUoNgAAAACsR7EBAAAAYD2KDQAAAADrUWwAAAAAWI9iAwAAAMB6FBsAAAAA1qPYAAAAALAexQYAAACA9Sg2AAAAAKxHsQEAAABgPYoNAAAAAOvFhHoCAAAAaH9yMwaFegqAXzhjAwAAAMB6FBsAAAAA1qPYAAAAALAexQYAAACA9fwuNlu2bNHEiROVkZEhl8ultWvX+rxujNHixYuVnp6uDh06KCcnR+Xl5U7NF4DlyBAAgSJHALTG72JTX1+vgQMHqri4uNXXH3roIf3ud7/T448/rh07dighIUG5ubk6ceJEwJMFYD8yBECgyBEArfH7ds95eXnKy8tr9TVjjIqKinTffffpuuuukyT953/+p1JTU7V27VpNnTq1xTYNDQ1qaGjwPq+rq/N3SgAs4nSGSOQIEGk4FgHQGkd/x6aiokLV1dXKycnxLvN4PBo2bJi2bdvW6jaFhYXyeDzeR2ZmppNTAmCR88kQiRwB8C2ORYDI5Wixqa6uliSlpqb6LE9NTfW+9l0LFy5UbW2t91FZWenklABY5HwyRCJHAHyLYxEgcvl9KZrT3G633G53qKcBwGLkCIBAkCFAeHD0jE1aWpokqaamxmd5TU2N9zUAOBMyBECgyBEgcjlabLKzs5WWlqbS0lLvsrq6Ou3YsUNXXnmlk0MBCENkCIBAkSNA5PL7UrRjx45p79693ucVFRXatWuXkpOT1bNnT82bN0/Lly9X3759lZ2drUWLFikjI0OTJk1yct4ALEWGAAgUOQKgNX4Xmw8++EBjxozxPp8/f74kKT8/X6tXr9Y999yj+vp6/eIXv9CRI0c0YsQIbdiwQfHx8c7NGoC1yBAAgSJHALTGZYwxoZ7E6erq6uTxeDRa1ynGFRvq6QAR7ZQ5qc1ap9raWiUlJYV6Om1GjgDth405QoYA7Yc/GeLo79gAAAAAQChQbAAAAABYj2IDAAAAwHoUGwAAAADWo9gAAAAAsB7FBgAAAID1KDYAAAAArEexAQAAAGA9ig0AAAAA61FsAAAAAFiPYgMAAADAehQbAAAAANaj2AAAAACwHsUGAAAAgPUoNgAAAACsR7EBAAAAYD2KDQAAAADrUWwAAAAAWI9iAwAAAMB6FBsAAAAA1qPYAAAAALCe38Vmy5YtmjhxojIyMuRyubR27VrvaydPntS9996r/v37KyEhQRkZGfrZz36mqqoqJ+cMwGJkCIBAkSMAWuN3samvr9fAgQNVXFzc4rXjx4+rrKxMixYtUllZmf77v/9be/bs0U9+8hNHJgvAfmQIgECRIwBaE+PvBnl5ecrLy2v1NY/Ho40bN/osW7lypYYOHarPPvtMPXv2PL9ZAggbZAiAQJEjAFrjd7HxV21trVwulzp37tzq6w0NDWpoaPA+r6urC/aUAFjkXBkikSMAzo5jESAyBPXmASdOnNC9996radOmKSkpqdV1CgsL5fF4vI/MzMxgTgmARdqSIRI5AuDMOBYBIkfQis3Jkyc1ZcoUGWNUUlJyxvUWLlyo2tpa76OysjJYUwJgkbZmiESOAGgdxyJAZAnKpWjNQfLpp5/qnXfeOesnrW63W263OxjTAGApfzJEIkcAtMSxCBB5HC82zUFSXl6uTZs2qWvXrk4PASCMkSEAAkWOAJHJ72Jz7Ngx7d271/u8oqJCu3btUnJystLT03XDDTeorKxMb7zxhhobG1VdXS1JSk5OVlxcnHMzB2AlMgRAoMgRAK1xGWOMPxts3rxZY8aMabE8Pz9fS5YsUXZ2dqvbbdq0SaNHjz7n/uvq6uTxeDRa1ynGFevP1AA47JQ5qc1ap9ra2nNeDtZWwc4QiRwB2hMbc4QMAdoPfzLE7zM2o0eP1tm6kJ89CUCEIUMABIocAdCaoN7uGQAAAAAuBIoNAAAAAOtRbAAAAABYj2IDAAAAwHoUGwAAAADWo9gAAAAAsB7FBgAAAID1KDYAAAAArEexAQAAAGA9ig0AAAAA61FsAAAAAFiPYgMAAADAehQbAAAAANaj2AAAAACwHsUGAAAAgPUoNgAAAACsFxPqCXyXMUaSdEonJRPiyQAR7pROSvr259IW5AjQftiYI2QI0H74kyHtrtgcPXpUkrRV60M8EwDNjh49Ko/HE+pptBk5ArQ/NuUIGQK0P23JEJdpZx+hNDU1qaqqSomJiXK5XC1er6urU2ZmpiorK5WUlBSCGYYH3sfARcJ7aIzR0aNHlZGRoagoe65cJUeCj/fQGZHwPtqYI+fKECky/u6CjfcwcJHwHvqTIe3ujE1UVJR69OhxzvWSkpLC9i/wQuJ9DFy4v4e2fMJ6OnLkwuE9dEa4v4+25UhbM0QK/7+7C4H3MHDh/h62NUPs+OgEAAAAAM6CYgMAAADAetYVG7fbrYKCArnd7lBPxWq8j4HjPbQXf3eB4z10Bu+jvfi7CxzvYeB4D321u5sHAAAAAIC/rDtjAwAAAADfRbEBAAAAYD2KDQAAAADrUWwAAAAAWI9iAwAAAMB61hWb4uJi9erVS/Hx8Ro2bJjee++9UE/JKkuWLJHL5fJ5XHLJJaGeVru2ZcsWTZw4URkZGXK5XFq7dq3P68YYLV68WOnp6erQoYNycnJUXl4emsmiTciR80eGnB9yJLyQIYEhR/xHhrSNVcXmxRdf1Pz581VQUKCysjINHDhQubm5OnDgQKinZpV+/frp888/9z62bt0a6im1a/X19Ro4cKCKi4tbff2hhx7S7373Oz3++OPasWOHEhISlJubqxMnTlzgmaItyJHAkSH+I0fCBxniDHLEP2RIGxmLDB061MyZM8f7vLGx0WRkZJjCwsIQzsouBQUFZuDAgaGehrUkmddee837vKmpyaSlpZmHH37Yu+zIkSPG7Xab559/PgQzxLmQI4EhQwJHjtiNDAkcORIYMuTMrDlj8/XXX+vDDz9UTk6Od1lUVJRycnK0bdu2EM7MPuXl5crIyFDv3r01ffp0ffbZZ6GekrUqKipUXV3t8+/S4/Fo2LBh/Ltsh8gRZ5AhziJH7EGGOIcccQ4Z8i1ris2hQ4fU2Nio1NRUn+Wpqamqrq4O0azsM2zYMK1evVobNmxQSUmJKioqNHLkSB09ejTUU7NS8789/l3agRwJHBniPHLEHmSIM8gRZ5Eh34oJ9QRwYeXl5Xn/e8CAARo2bJiysrL00ksv6ZZbbgnhzADYgAwBEChyBMFizRmbiy66SNHR0aqpqfFZXlNTo7S0tBDNyn6dO3fWxRdfrL1794Z6KlZq/rfHv0s7kCPOI0MCR47YgwwJDnIkMGTIt6wpNnFxcRo8eLBKS0u9y5qamlRaWqorr7wyhDOz27Fjx7Rv3z6lp6eHeipWys7OVlpams+/y7q6Ou3YsYN/l+0QOeI8MiRw5Ig9yJDgIEcCQ4Z8y6pL0ebPn6/8/HwNGTJEQ4cOVVFRkerr6zVz5sxQT80ad911lyZOnKisrCxVVVWpoKBA0dHRmjZtWqin1m4dO3bM51OkiooK7dq1S8nJyerZs6fmzZun5cuXq2/fvsrOztaiRYuUkZGhSZMmhW7SOCNyJDBkyPkhR8IHGRI4csR/ZEgbhfq2bP567LHHTM+ePU1cXJwZOnSo2b59e6inZJWbbrrJpKenm7i4ONO9e3dz0003mb1794Z6Wu3apk2bjKQWj/z8fGPMN7dZXLRokUlNTTVut9uMHTvW7NmzJ7STxlmRI+ePDDk/5Eh4IUMCQ474jwxpG5cxxoSgTwEAAACAY6z5HRsAAAAAOBOKDQAAAADrUWwAAAAAWI9iAwAAAMB6FBsAAAAA1qPYAAAAALAexQYAAACA9Sg2AAAAAKxHsQEAAABgPYoNAAAAAOtRbAAAAABY7/8BVD0h9ahzriIAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 1000x1500 with 3 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "#Data for figure SN 2.10 of pitfalls p50\n",
+    "ref210 = np.zeros([14,14])\n",
+    "ref210[5:9,5:9] = 1\n",
+    "pred210_1 = np.zeros([14,14])\n",
+    "pred210_1[6:8,6:8] = 1\n",
+    "pred210_2 = np.zeros([14,14])\n",
+    "pred210_2[4:10,4:10]=1\n",
+    "\n",
+    "fig,ax = plt.subplots(1,3,figsize=(10,15))\n",
+    "ax[0].imshow(ref210)\n",
+    "ax[1].imshow(pred210_1)\n",
+    "ax[2].imshow(pred210_2)\n",
+    "ax[0].set_title('Reference')\n",
+    "ax[1].set_title('Prediction 1')\n",
+    "ax[2].set_title('Prediction 2')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "id": "44a021e8",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Text(0.5, 1.0, 'Prediction')"
+      ]
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0MAAAGiCAYAAAAlRx3hAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2c0lEQVR4nO3de3xU9Z3/8ffkNuGSDLdcIUBAFEUuFSUioFBShmhZA96ItgZE6tpgxRSt2AoIbOOl7dKWFOx2Ia2UomwLqEUUAgQpAcslu+JWHiEbDBQSLjUZEiSE5Pz+8JepAxNgkhkm4ft6Ph7nIXPO93vOZ47z4MN75pwZm2VZlgAAAADAMCHBLgAAAAAAgoEwBAAAAMBIhCEAAAAARiIMAQAAADASYQgAAACAkQhDAAAAAIxEGAIAAABgJMIQAAAAACMRhgAAAAAYiTAENFNxcbHGjRsnh8Mhm82mtWvXBrskAAB80rt3b02ZMsX9eOvWrbLZbNq6davfjmGz2TRv3jy/7Q/wJ8IQjJCXlyebzeZewsLC1L17d02ZMkV///vfm7XPzMxMffzxx/q3f/s3vfHGG7r11lv9XDUA4Fp3YX+KjIzU9ddfrxkzZqiioiLY5V2x9evXE3jQJoUFuwDgapo/f76Sk5N19uxZ7dy5U3l5edq+fbv279+vyMjIK97PF198ocLCQv3whz/UjBkzAlgxAMAEX+1P27dv15IlS7R+/Xrt379f7du3v2p13Hnnnfriiy8UERHh07z169crNzfXayD64osvFBbGPznROvHKhFHS0tLcn+A8/vjj6tatm1555RW9/fbbevDBB694PydOnJAkderUyW+1nT17VhEREQoJ4QNbADDNhf2pa9eu+tnPfqZ169YpIyPjovE1NTXq0KGD3+sICQnx6c3BK+Hv/QH+xL+6YLRRo0ZJkkpKStzrPv30U91///3q0qWLIiMjdeutt+rtt992b583b5569eolSXr22Wdls9nUu3dv9/a///3veuyxxxQXFye73a4BAwZo2bJlHsdtvCZ71apV+tGPfqTu3burffv2crlckqRdu3Zp/Pjxcjgcat++ve666y795S9/8djHvHnzZLPZdPDgQU2ZMkWdOnWSw+HQ1KlTdebMmYue64oVKzRs2DC1b99enTt31p133qkPPvjAY8x7772nUaNGqUOHDoqKitI999yjTz75pBlnFgDQEl//+tclSaWlpZoyZYo6duyokpIS3X333YqKitIjjzwiSWpoaNCiRYs0YMAARUZGKi4uTk888YQ+//xzj/1ZlqWFCxeqR48eat++vcaMGeP17/em7hnatWuX7r77bnXu3FkdOnTQoEGD9POf/1ySNGXKFOXm5kqSxyV/jbzdM7Rv3z6lpaUpOjpaHTt21NixY7Vz506PMY2XEP7lL39Rdna2YmJi1KFDB02cONH9piTQUnwyBKMdOnRIktS5c2dJ0ieffKIRI0aoe/fuev7559WhQwe99dZbSk9P1x//+EdNnDhRkyZNUqdOnfTMM88oIyNDd999tzp27ChJqqio0O233y6bzaYZM2YoJiZG7733nqZNmyaXy6WZM2d6HH/BggWKiIjQrFmzVFtbq4iICG3evFlpaWkaOnSo5s6dq5CQEC1fvlxf//rX9eGHH2rYsGEe+3jwwQeVnJysnJwc7d27V7/5zW8UGxurV155xT3mpZde0rx583THHXdo/vz5ioiI0K5du7R582aNGzdOkvTGG28oMzNTTqdTr7zyis6cOaMlS5Zo5MiR2rdvn0fgAwAEVuObdF27dpUknT9/Xk6nUyNHjtRPfvIT96VzTzzxhPLy8jR16lR973vfU2lpqRYvXqx9+/bpL3/5i8LDwyVJc+bM0cKFC3X33Xfr7rvv1t69ezVu3DidO3fusrVs3LhR3/zmN5WQkKCnn35a8fHx+tvf/qZ3331XTz/9tJ544gkdPXpUGzdu1BtvvHHZ/X3yyScaNWqUoqOj9dxzzyk8PFyvv/66Ro8erYKCAqWkpHiMf+qpp9S5c2fNnTtXhw4d0qJFizRjxgy9+eabPp1TwCsLMMDy5cstSdamTZusEydOWIcPH7b+67/+y4qJibHsdrt1+PBhy7Isa+zYsdbAgQOts2fPuuc2NDRYd9xxh9WvXz/3utLSUkuS9dprr3kcZ9q0aVZCQoJ18uRJj/WTJ0+2HA6HdebMGcuyLGvLli2WJKtPnz7udY3H6tevn+V0Oq2Ghgb3+jNnzljJycnWN77xDfe6uXPnWpKsxx57zONYEydOtLp27ep+XFxcbIWEhFgTJ0606uvrPcY2HuP06dNWp06drOnTp3tsLy8vtxwOx0XrAQD+4a0/rVq1yuratavVrl0768iRI1ZmZqYlyXr++ec95n744YeWJOv3v/+9x/oNGzZ4rD9+/LgVERFh3XPPPR695YUXXrAkWZmZme51jf1py5YtlmVZ1vnz563k5GSrV69e1ueff+5xnK/uKysry2rqn5WSrLlz57ofp6enWxEREVZJSYl73dGjR62oqCjrzjvvvOjcpKamehzrmWeesUJDQ63KykqvxwN8wWVyMEpqaqpiYmKUlJSk+++/Xx06dNDbb7+tHj166B//+Ic2b96sBx98UKdPn9bJkyd18uRJnTp1Sk6nU8XFxZf85jnLsvTHP/5REyZMkGVZ7vknT56U0+lUVVWV9u7d6zEnMzNT7dq1cz8uKipScXGxHn74YZ06dco9v6amRmPHjtW2bdvU0NDgsY9//dd/9Xg8atQonTp1yn3J3dq1a9XQ0KA5c+ZcdD9S42UMGzduVGVlpTIyMjzqDg0NVUpKirZs2eL7yQYAXLGv9qfJkyerY8eOWrNmjbp37+4e8+STT3rMWb16tRwOh77xjW94/N09dOhQdezY0f1396ZNm3Tu3Dk99dRTHpevXXi1gjf79u1TaWmpZs6cedF9sl/d15Wqr6/XBx98oPT0dPXp08e9PiEhQQ8//LC2b9/u7l+NvvOd73gca9SoUaqvr9dnn33m8/GBC3GZHIySm5ur66+/XlVVVVq2bJm2bdsmu90uSTp48KAsy9KLL76oF1980ev848ePezSmrzpx4oQqKyv161//Wr/+9a+bnP9VycnJHo+Li4slfRmSmlJVVeW+rE+Sevbs6bG9cdvnn3+u6OholZSUKCQkRDfddFOT+2w8buM16heKjo5uci4AoOUa+1NYWJji4uJ0ww03eLyBFRYWph49enjMKS4uVlVVlWJjY73us7HnNIaGfv36eWyPiYnx6CfeNF6ud/PNN/v2hJpw4sQJnTlzRjfccMNF22688UY1NDTo8OHDGjBggHv9pfoc0FKEIRhl2LBh7m/rSU9P18iRI/Xwww/rwIED7k9cZs2aJafT6XX+dddd1+S+G+d/61vfajLMDBo0yOPxVz8V+uo+XnvtNQ0ZMsTrPhrvT2oUGhrqdZxlWU3WeqHG477xxhuKj4+/aDtfiQoAgfXV/uSN3W6/6NP9hoYGxcbG6ve//73XOTExMX6tMVj80eeApvAvHBgrNDRUOTk5GjNmjBYvXqzHHntMkhQeHq7U1FSf9xcTE6OoqCjV19c3a74k9e3bV9KXn8Q0dx/e9tnQ0KD//d//bTJgNR43NjbWb8cFAARW3759tWnTJo0YMeKiN9e+qvEbUIuLiz0uTTtx4sRlP11p7A/79++/ZH+40kvmYmJi1L59ex04cOCibZ9++qlCQkKUlJR0RfsC/IF7hmC00aNHa9iwYVq0aJGio6M1evRovf766zp27NhFYy/3NZ6hoaG677779Mc//lH79+/3eb4kDR06VH379tVPfvITVVdXN2sfF0pPT1dISIjmz59/0f1Gje+qOZ1ORUdH68c//rHq6ur8clwAQGA9+OCDqq+v14IFCy7adv78eVVWVkr68n6k8PBw/fKXv/T4NGXRokWXPcYtt9yi5ORkLVq0yL2/Rl/dV+NvHl045kKhoaEaN26c1q1b5/5GV+nLb2NduXKlRo4cyaXZuKr4ZAjGe/bZZ/XAAw8oLy9Pubm5GjlypAYOHKjp06erT58+qqioUGFhoY4cOaL//u//vuS+Xn75ZW3ZskUpKSmaPn26brrpJv3jH//Q3r17tWnTJv3jH/+45PyQkBD95je/UVpamgYMGKCpU6eqe/fu+vvf/64tW7YoOjpa77zzjk/P77rrrtMPf/hDLViwQKNGjdKkSZNkt9v117/+VYmJicrJyVF0dLSWLFmib3/727rllls0efJkxcTEqKysTH/+8581YsQILV682KfjAgAC66677tITTzyhnJwcFRUVady4cQoPD1dxcbFWr16tn//857r//vsVExOjWbNmKScnR9/85jd19913a9++fXrvvffUrVu3Sx4jJCRES5Ys0YQJEzRkyBBNnTpVCQkJ+vTTT/XJJ5/o/fffl/Tlm3mS9L3vfU9Op1OhoaGaPHmy130uXLhQGzdu1MiRI/Xd735XYWFhev3111VbW6tXX33VvycJuAzCEIw3adIk96cx06dP1+7du/XSSy8pLy9Pp06dUmxsrL72ta9pzpw5l91XXFycPvroI82fP19/+tOf9Ktf/Updu3bVgAEDPH7351JGjx6twsJCLViwQIsXL1Z1dbXi4+OVkpKiJ554olnPcf78+UpOTtYvf/lL/fCHP1T79u01aNAgffvb33aPefjhh5WYmKiXX35Zr732mmpra9W9e3eNGjVKU6dObdZxAQCBtXTpUg0dOlSvv/66XnjhBYWFhal379761re+pREjRrjHLVy4UJGRkVq6dKn7TbsPPvhA99xzz2WP4XQ6tWXLFr300kv66U9/qoaGBvXt21fTp093j5k0aZKeeuoprVq1SitWrJBlWU2GoQEDBujDDz/U7NmzlZOTo4aGBqWkpGjFihUX/cYQEGg2i7vPAAAAABiIe4YAAAAAGIkwBAAAAMBIhCEAAAAARiIMAQAAADASYQgAAACAkQhDAAAAAIx0TfzOUENDg44ePaqoqCjZbLZglwMARrEsS6dPn1ZiYqJCQniPrRG9CQCCw5e+dE2EoaNHjyopKSnYZQCA0Q4fPqwePXoEu4xWg94EAMF1JX3pmghDUVFRkqSRulthCg9yNQBglvOq03atd/9djC/RmwAgOHzpS9dEGGq8/CBM4Qqz0XAA4KqyvvwPl4J5ojcBQJD40Je4uBsAAACAkQIWhnJzc9W7d29FRkYqJSVFH3300SXHr169Wv3791dkZKQGDhyo9evXB6o0AICB6EsAgAsFJAy9+eabys7O1ty5c7V3714NHjxYTqdTx48f9zp+x44dysjI0LRp07Rv3z6lp6crPT1d+/fvD0R5AADD0JcAAN7YLMuy/L3TlJQU3XbbbVq8eLGkL79eNCkpSU899ZSef/75i8Y/9NBDqqmp0bvvvuted/vtt2vIkCFaunTpZY/ncrnkcDg0WvdyXTYAXGXnrTpt1TpVVVUpOjo62OV4dbX7kkRvAoBg8aUv+f2ToXPnzmnPnj1KTU3950FCQpSamqrCwkKvcwoLCz3GS5LT6WxyfG1trVwul8cCAIA3V6MvSfQmAGiL/B6GTp48qfr6esXFxXmsj4uLU3l5udc55eXlPo3PycmRw+FwL/yOAwCgKVejL0n0JgBoi9rkt8nNnj1bVVVV7uXw4cPBLgkAYDh6EwC0PX7/naFu3bopNDRUFRUVHusrKioUHx/vdU58fLxP4+12u+x2u38KBgBc065GX5LoTQDQFvn9k6GIiAgNHTpU+fn57nUNDQ3Kz8/X8OHDvc4ZPny4x3hJ2rhxY5PjAQC4UvQlAEBT/P7JkCRlZ2crMzNTt956q4YNG6ZFixappqZGU6dOlSQ9+uij6t69u3JyciRJTz/9tO666y799Kc/1T333KNVq1Zp9+7d+vWvfx2I8gAAhqEvAQC8CUgYeuihh3TixAnNmTNH5eXlGjJkiDZs2OC+GbWsrEwhIf/8UOqOO+7QypUr9aMf/UgvvPCC+vXrp7Vr1+rmm28ORHkAAMPQlwAA3gTkd4auNn7LAQCCpy38zlAw0JsAIDiC+jtDAAAAANAWEIYAAAAAGIkwBAAAAMBIhCEAAAAARiIMAQAAADASYQgAAACAkQhDAAAAAIxEGAIAAABgJMIQAAAAACMRhgAAAAAYiTAEAAAAwEiEIQAAAABGIgwBAAAAMBJhCAAAAICRCEMAAAAAjEQYAgAAAGCksGAXcK14/2hRsEsAgGZzJg4JdgkAAFx1fDIEAAAAwEiEIQAAAABGIgwBAAAAMBJhCAAAAICRCEMAAAAAjEQYAgAAAGAkwhAAAAAAIxGGAAAAABiJMAQAAADASIQhAAAAAEYiDAEAAAAwEmEIAAAAgJH8HoZycnJ02223KSoqSrGxsUpPT9eBAwcuOScvL082m81jiYyM9HdpAABD0ZsAAN74PQwVFBQoKytLO3fu1MaNG1VXV6dx48appqbmkvOio6N17Ngx9/LZZ5/5uzQAgKHoTQAAb8L8vcMNGzZ4PM7Ly1NsbKz27NmjO++8s8l5NptN8fHx/i4HAAB6EwDAK7+HoQtVVVVJkrp06XLJcdXV1erVq5caGhp0yy236Mc//rEGDBjgdWxtba1qa2vdj10ul/8KBgBc80zpTe8fLQp2CQDQbM7EIQE/RkC/QKGhoUEzZ87UiBEjdPPNNzc57oYbbtCyZcu0bt06rVixQg0NDbrjjjt05MgRr+NzcnLkcDjcS1JSUqCeAgDgGkNvAgA0slmWZQVq508++aTee+89bd++XT169LjieXV1dbrxxhuVkZGhBQsWXLTd27tvSUlJGq17FWYL90vtvuLdNwBtWUvefTtv1Wmr1qmqqkrR0dH+KypA6E0A0DY0tzf50pcCdpncjBkz9O6772rbtm0+NRtJCg8P19e+9jUdPHjQ63a73S673e6PMgEABqE3AQC+yu+XyVmWpRkzZmjNmjXavHmzkpOTfd5HfX29Pv74YyUkJPi7PACAgehNAABv/P7JUFZWllauXKl169YpKipK5eXlkiSHw6F27dpJkh599FF1795dOTk5kqT58+fr9ttv13XXXafKykq99tpr+uyzz/T444/7uzwAgIHoTQAAb/wehpYsWSJJGj16tMf65cuXa8qUKZKksrIyhYT880Opzz//XNOnT1d5ebk6d+6soUOHaseOHbrpppv8XR4AwED0JgCANwH9AoWrxeVyyeFwcJMqADSTSV+gcLXQmwCgZa7GFygE9Ku1AQAAAKC1IgwBAAAAMBJhCAAAAICRCEMAAAAAjEQYAgAAAGAkwhAAAAAAIxGGAAAAABiJMAQAAADASIQhAAAAAEYiDAEAAAAwEmEIAAAAgJEIQwAAAACMRBgCAAAAYCTCEAAAAAAjEYYAAAAAGIkwBAAAAMBIhCEAAAAARiIMAQAAADASYQgAAACAkQhDAAAAAIxEGAIAAABgJMIQAAAAACMRhgAAAAAYiTAEAAAAwEiEIQAAAABGIgwBAAAAMBJhCAAAAICRCEMAAAAAjEQYAgAAAGAkv4ehefPmyWazeSz9+/e/5JzVq1erf//+ioyM1MCBA7V+/Xp/lwUAMBi9CQDgTUA+GRowYICOHTvmXrZv397k2B07digjI0PTpk3Tvn37lJ6ervT0dO3fvz8QpQEADEVvAgBcKCBhKCwsTPHx8e6lW7duTY79+c9/rvHjx+vZZ5/VjTfeqAULFuiWW27R4sWLA1EaAMBQ9CYAwIUCEoaKi4uVmJioPn366JFHHlFZWVmTYwsLC5Wamuqxzul0qrCwsMk5tbW1crlcHgsAAJdCbwIAXMjvYSglJUV5eXnasGGDlixZotLSUo0aNUqnT5/2Or68vFxxcXEe6+Li4lReXt7kMXJycuRwONxLUlKSX58DAODaQm8CAHjj9zCUlpamBx54QIMGDZLT6dT69etVWVmpt956y2/HmD17tqqqqtzL4cOH/bZvAMC1h94EAPAmLNAH6NSpk66//nodPHjQ6/b4+HhVVFR4rKuoqFB8fHyT+7Tb7bLb7X6tEwBgDnoTAEC6Cr8zVF1drZKSEiUkJHjdPnz4cOXn53us27hxo4YPHx7o0gAAhqI3AQCkAIShWbNmqaCgQIcOHdKOHTs0ceJEhYaGKiMjQ5L06KOPavbs2e7xTz/9tDZs2KCf/vSn+vTTTzVv3jzt3r1bM2bM8HdpAABD0ZsAAN74/TK5I0eOKCMjQ6dOnVJMTIxGjhypnTt3KiYmRpJUVlamkJB/ZrA77rhDK1eu1I9+9CO98MIL6tevn9auXaubb77Z36UBAAxFbwIAeGOzLMsKdhEt5XK55HA4NFr3KswWHpQa3j9aFJTjAoA/OBOHNHvueatOW7VOVVVVio6O9l9RbRy9CQBaprm9yZe+FPB7hgAAAACgNSIMAQAAADASYQgAAACAkQhDAAAAAIxEGAIAAABgJMIQAAAAACMRhgAAAAAYiTAEAAAAwEiEIQAAAABGIgwBAAAAMBJhCAAAAICRCEMAAAAAjEQYAgAAAGAkwhAAAAAAIxGGAAAAABiJMAQAAADASIQhAAAAAEYiDAEAAAAwEmEIAAAAgJEIQwAAAACMRBgCAAAAYCTCEAAAAAAjEYYAAAAAGIkwBAAAAMBIhCEAAAAARiIMAQAAADASYQgAAACAkQhDAAAAAIxEGAIAAABgJL+Hod69e8tms120ZGVleR2fl5d30djIyEh/lwUAMBi9CQDgTZi/d/jXv/5V9fX17sf79+/XN77xDT3wwANNzomOjtaBAwfcj202m7/LAgAYjN4EAPDG72EoJibG4/HLL7+svn376q677mpyjs1mU3x8vL9LAQBAEr0JAOBdQO8ZOnfunFasWKHHHnvsku+oVVdXq1evXkpKStK9996rTz755JL7ra2tlcvl8lgAALgS9CYAQKOAhqG1a9eqsrJSU6ZMaXLMDTfcoGXLlmndunVasWKFGhoadMcdd+jIkSNNzsnJyZHD4XAvSUlJAageAHAtojcBABrZLMuyArVzp9OpiIgIvfPOO1c8p66uTjfeeKMyMjK0YMECr2Nqa2tVW1vrfuxyuZSUlKTRuldhtvAW190c7x8tCspxAcAfnIlDmj33vFWnrVqnqqoqRUdH+6+oAKE3AUDb0Nze5Etf8vs9Q40+++wzbdq0SX/60598mhceHq6vfe1rOnjwYJNj7Ha77HZ7S0sEABiG3gQA+KqAXSa3fPlyxcbG6p577vFpXn19vT7++GMlJCQEqDIAgKnoTQCArwpIGGpoaNDy5cuVmZmpsDDPD58effRRzZ492/14/vz5+uCDD/R///d/2rt3r771rW/ps88+0+OPPx6I0gAAhqI3AQAuFJDL5DZt2qSysjI99thjF20rKytTSMg/M9jnn3+u6dOnq7y8XJ07d9bQoUO1Y8cO3XTTTYEoDQBgKHoTAOBCAf0ChavF5XLJ4XBwkyoANJNJX6BwtdCbAKBlrsYXKAT0q7UBAAAAoLUiDAEAAAAwEmEIAAAAgJEIQwAAAACMRBgCAAAAYCTCEAAAAAAjEYYAAAAAGIkwBAAAAMBIhCEAAAAARiIMAQAAADASYQgAAACAkQhDAAAAAIxEGAIAAABgJMIQAAAAACMRhgAAAAAYiTAEAAAAwEiEIQAAAABGIgwBAAAAMBJhCAAAAICRCEMAAAAAjEQYAgAAAGAkwhAAAAAAIxGGAAAAABiJMAQAAADASIQhAAAAAEYiDAEAAAAwEmEIAAAAgJEIQwAAAACMRBgCAAAAYCSfw9C2bds0YcIEJSYmymazae3atR7bLcvSnDlzlJCQoHbt2ik1NVXFxcWX3W9ubq569+6tyMhIpaSk6KOPPvK1NACAgehLAIDm8jkM1dTUaPDgwcrNzfW6/dVXX9UvfvELLV26VLt27VKHDh3kdDp19uzZJvf55ptvKjs7W3PnztXevXs1ePBgOZ1OHT9+3NfyAACGoS8BAJrLZlmW1ezJNpvWrFmj9PR0SV+++5aYmKjvf//7mjVrliSpqqpKcXFxysvL0+TJk73uJyUlRbfddpsWL14sSWpoaFBSUpKeeuopPf/885etw+VyyeFwaLTuVZgtvLlPp0XeP1oUlOMCgD84E4c0e+55q05btU5VVVWKjo72X1HN0Fr6kkRvAoCWam5v8qUv+fWeodLSUpWXlys1NdW9zuFwKCUlRYWFhV7nnDt3Tnv27PGYExISotTU1Cbn1NbWyuVyeSwAAFzoavUlid4EAG2RX8NQeXm5JCkuLs5jfVxcnHvbhU6ePKn6+nqf5uTk5MjhcLiXpKQkP1QPALjWXK2+JNGbAKAtapPfJjd79mxVVVW5l8OHDwe7JACA4ehNAND2+DUMxcfHS5IqKio81ldUVLi3Xahbt24KDQ31aY7dbld0dLTHAgDAha5WX5LoTQDQFvk1DCUnJys+Pl75+fnudS6XS7t27dLw4cO9zomIiNDQoUM95jQ0NCg/P7/JOQAAXAn6EgDgUsJ8nVBdXa2DBw+6H5eWlqqoqEhdunRRz549NXPmTC1cuFD9+vVTcnKyXnzxRSUmJrq/2UeSxo4dq4kTJ2rGjBmSpOzsbGVmZurWW2/VsGHDtGjRItXU1Gjq1Kktf4YAgGsafQkA0Fw+h6Hdu3drzJgx7sfZ2dmSpMzMTOXl5em5555TTU2NvvOd76iyslIjR47Uhg0bFBkZ6Z5TUlKikydPuh8/9NBDOnHihObMmaPy8nINGTJEGzZsuOjmVQAALkRfAgA0V4t+Z6i14LccAKBlrpXfGWpN6E0A0DJt7neGAAAAAKCtIAwBAAAAMBJhCAAAAICRCEMAAAAAjEQYAgAAAGAkwhAAAAAAIxGGAAAAABiJMAQAAADASIQhAAAAAEYiDAEAAAAwEmEIAAAAgJEIQwAAAACMRBgCAAAAYCTCEAAAAAAjEYYAAAAAGIkwBAAAAMBIhCEAAAAARiIMAQAAADASYQgAAACAkQhDAAAAAIxEGAIAAABgJMIQAAAAACMRhgAAAAAYiTAEAAAAwEiEIQAAAABGIgwBAAAAMBJhCAAAAICRCEMAAAAAjEQYAgAAAGAkn8PQtm3bNGHCBCUmJspms2nt2rXubXV1dfrBD36ggQMHqkOHDkpMTNSjjz6qo0ePXnKf8+bNk81m81j69+/v85MBAJiHvgQAaC6fw1BNTY0GDx6s3Nzci7adOXNGe/fu1Ysvvqi9e/fqT3/6kw4cOKB/+Zd/uex+BwwYoGPHjrmX7du3+1oaAMBA9CUAQHOF+TohLS1NaWlpXrc5HA5t3LjRY93ixYs1bNgwlZWVqWfPnk0XEham+Ph4X8sBABiOvgQAaK6A3zNUVVUlm82mTp06XXJccXGxEhMT1adPHz3yyCMqKytrcmxtba1cLpfHAgDAlQhEX5LoTQDQFgU0DJ09e1Y/+MEPlJGRoejo6CbHpaSkKC8vTxs2bNCSJUtUWlqqUaNG6fTp017H5+TkyOFwuJekpKRAPQUAwDUkUH1JojcBQFsUsDBUV1enBx98UJZlacmSJZccm5aWpgceeECDBg2S0+nU+vXrVVlZqbfeesvr+NmzZ6uqqsq9HD58OBBPAQBwDQlkX5LoTQDQFvl8z9CVaGw4n332mTZv3nzJd9+86dSpk66//nodPHjQ63a73S673e6PUgEABgh0X5LoTQDQFvn9k6HGhlNcXKxNmzapa9euPu+jurpaJSUlSkhI8Hd5AADD0JcAAE3xOQxVV1erqKhIRUVFkqTS0lIVFRWprKxMdXV1uv/++7V79279/ve/V319vcrLy1VeXq5z58659zF27FgtXrzY/XjWrFkqKCjQoUOHtGPHDk2cOFGhoaHKyMho+TMEAFzT6EsAgOby+TK53bt3a8yYMe7H2dnZkqTMzEzNmzdPb7/9tiRpyJAhHvO2bNmi0aNHS5JKSkp08uRJ97YjR44oIyNDp06dUkxMjEaOHKmdO3cqJibG1/IAAIahLwEAmsvnMDR69GhZltXk9ktta3To0CGPx6tWrfK1DAAAJNGXAADNF/DfGQIAAACA1ogwBAAAAMBIhCEAAAAARiIMAQAAADASYQgAAACAkQhDAAAAAIxEGAIAAABgJMIQAAAAACMRhgAAAAAYiTAEAAAAwEiEIQAAAABGIgwBAAAAMBJhCAAAAICRCEMAAAAAjEQYAgAAAGAkwhAAAAAAIxGGAAAAABiJMAQAAADASIQhAAAAAEYiDAEAAAAwEmEIAAAAgJEIQwAAAACMRBgCAAAAYCTCEAAAAAAjEYYAAAAAGIkwBAAAAMBIhCEAAAAARiIMAQAAADASYQgAAACAkXwOQ9u2bdOECROUmJgom82mtWvXemyfMmWKbDabxzJ+/PjL7jc3N1e9e/dWZGSkUlJS9NFHH/laGgDAQPQlAEBz+RyGampqNHjwYOXm5jY5Zvz48Tp27Jh7+cMf/nDJfb755pvKzs7W3LlztXfvXg0ePFhOp1PHjx/3tTwAgGHoSwCA5grzdUJaWprS0tIuOcZutys+Pv6K9/mzn/1M06dP19SpUyVJS5cu1Z///GctW7ZMzz//vK8lAgAMQl8CADRXQO4Z2rp1q2JjY3XDDTfoySef1KlTp5oce+7cOe3Zs0epqan/LCokRKmpqSosLPQ6p7a2Vi6Xy2MBAKApge5LEr0JANoiv4eh8ePH63e/+53y8/P1yiuvqKCgQGlpaaqvr/c6/uTJk6qvr1dcXJzH+ri4OJWXl3udk5OTI4fD4V6SkpL8/TQAANeIq9GXJHoTALRFPl8mdzmTJ092/3ngwIEaNGiQ+vbtq61bt2rs2LF+Ocbs2bOVnZ3tfuxyuWg6AACvrkZfkuhNANAWBfyrtfv06aNu3brp4MGDXrd369ZNoaGhqqio8FhfUVHR5PXddrtd0dHRHgsAAFciEH1JojcBQFsU8DB05MgRnTp1SgkJCV63R0REaOjQocrPz3eva2hoUH5+voYPHx7o8gAAhqEvAQAa+RyGqqurVVRUpKKiIklSaWmpioqKVFZWpurqaj377LPauXOnDh06pPz8fN1777267rrr5HQ63fsYO3asFi9e7H6cnZ2t//iP/9Bvf/tb/e1vf9OTTz6pmpoa97f4AADQFPoSAKC5fL5naPfu3RozZoz7ceP10ZmZmVqyZIn+53/+R7/97W9VWVmpxMREjRs3TgsWLJDdbnfPKSkp0cmTJ92PH3roIZ04cUJz5sxReXm5hgwZog0bNlx08yoAABeiLwEAmstmWZYV7CJayuVyyeFwaLTuVZgtPCg1vH+0KCjHBQB/cCYOafbc81adtmqdqqqquE/mK+hNANAyze1NvvSlgN8zBAAAAACtEWEIAAAAgJEIQwAAAACMRBgCAAAAYCTCEAAAAAAjEYYAAAAAGIkwBAAAAMBIhCEAAAAARiIMAQAAADASYQgAAACAkQhDAAAAAIxEGAIAAABgJMIQAAAAACMRhgAAAAAYiTAEAAAAwEiEIQAAAABGIgwBAAAAMBJhCAAAAICRCEMAAAAAjEQYAgAAAGAkwhAAAAAAIxGGAAAAABiJMAQAAADASIQhAAAAAEYiDAEAAAAwEmEIAAAAgJEIQwAAAACMRBgCAAAAYCTCEAAAAAAj+RyGtm3bpgkTJigxMVE2m01r16712G6z2bwur732WpP7nDdv3kXj+/fv7/OTAQCYh74EAGgun8NQTU2NBg8erNzcXK/bjx075rEsW7ZMNptN99133yX3O2DAAI9527dv97U0AICB6EsAgOYK83VCWlqa0tLSmtweHx/v8XjdunUaM2aM+vTpc+lCwsIumgsAwOXQlwAAzRXQe4YqKir05z//WdOmTbvs2OLiYiUmJqpPnz565JFHVFZW1uTY2tpauVwujwUAgMsJVF+S6E0A0BYFNAz99re/VVRUlCZNmnTJcSkpKcrLy9OGDRu0ZMkSlZaWatSoUTp9+rTX8Tk5OXI4HO4lKSkpEOUDAK4xgepLEr0JANqigIahZcuW6ZFHHlFkZOQlx6WlpemBBx7QoEGD5HQ6tX79elVWVuqtt97yOn727NmqqqpyL4cPHw5E+QCAa0yg+pJEbwKAtsjne4au1IcffqgDBw7ozTff9Hlup06ddP311+vgwYNet9vtdtnt9paWCAAwSCD7kkRvAoC2KGCfDP3nf/6nhg4dqsGDB/s8t7q6WiUlJUpISAhAZQAAE9GXAAAX8jkMVVdXq6ioSEVFRZKk0tJSFRUVedxY6nK5tHr1aj3++ONe9zF27FgtXrzY/XjWrFkqKCjQoUOHtGPHDk2cOFGhoaHKyMjwtTwAgGHoSwCA5vL5Mrndu3drzJgx7sfZ2dmSpMzMTOXl5UmSVq1aJcuymmwaJSUlOnnypPvxkSNHlJGRoVOnTikmJkYjR47Uzp07FRMT42t5AADD0JcAAM1lsyzLCnYRLeVyueRwODRa9yrMFh6UGt4/WhSU4wKAPzgThzR77nmrTlu1TlVVVYqOjvZfUW0cvQkAWqa5vcmXvhTQb5MDAAAAgNaKMAQAAADASIQhAAAAAEYiDAEAAAAwEmEIAAAAgJEIQwAAAACMRBgCAAAAYCTCEAAAAAAjEYYAAAAAGIkwBAAAAMBIhCEAAAAARiIMAQAAADASYQgAAACAkQhDAAAAAIxEGAIAAABgJMIQAAAAACMRhgAAAAAYiTAEAAAAwEiEIQAAAABGIgwBAAAAMBJhCAAAAICRCEMAAAAAjEQYAgAAAGAkwhAAAAAAIxGGAAAAABiJMAQAAADASIQhAAAAAEYiDAEAAAAwEmEIAAAAgJF8CkM5OTm67bbbFBUVpdjYWKWnp+vAgQMeY86ePausrCx17dpVHTt21H333aeKiopL7teyLM2ZM0cJCQlq166dUlNTVVxc7PuzAQAYh94EAGgun8JQQUGBsrKytHPnTm3cuFF1dXUaN26campq3GOeeeYZvfPOO1q9erUKCgp09OhRTZo06ZL7ffXVV/WLX/xCS5cu1a5du9ShQwc5nU6dPXu2ec8KAGAMehMAoLlslmVZzZ184sQJxcbGqqCgQHfeeaeqqqoUExOjlStX6v7775ckffrpp7rxxhtVWFio22+//aJ9WJalxMREff/739esWbMkSVVVVYqLi1NeXp4mT5582TpcLpccDodG616F2cKb+3Ra5P2jRUE5LgD4gzNxSLPnnrfqtFXrVFVVpejoaP8V1Uz0pn+iNwFoy5rbm3zpSy26Z6iqqkqS1KVLF0nSnj17VFdXp9TUVPeY/v37q2fPniosLPS6j9LSUpWXl3vMcTgcSklJaXJObW2tXC6XxwIAgERvAgBcuWaHoYaGBs2cOVMjRozQzTffLEkqLy9XRESEOnXq5DE2Li5O5eXlXvfTuD4uLu6K5+Tk5MjhcLiXpKSk5j4NAMA1hN4EAPBFs8NQVlaW9u/fr1WrVvmznisye/ZsVVVVuZfDhw9f9RoAAK0PvQkA4ItmhaEZM2bo3Xff1ZYtW9SjRw/3+vj4eJ07d06VlZUe4ysqKhQfH+91X43rL/xWn0vNsdvtio6O9lgAAGajNwEAfOVTGLIsSzNmzNCaNWu0efNmJScne2wfOnSowsPDlZ+f71534MABlZWVafjw4V73mZycrPj4eI85LpdLu3btanIOAACN6E0AgObyKQxlZWVpxYoVWrlypaKiolReXq7y8nJ98cUXkr68uXTatGnKzs7Wli1btGfPHk2dOlXDhw/3+Lae/v37a82aNZIkm82mmTNnauHChXr77bf18ccf69FHH1ViYqLS09P990wBANckehMAoLnCfBm8ZMkSSdLo0aM91i9fvlxTpkyRJP37v/+7QkJCdN9996m2tlZOp1O/+tWvPMYfOHDA/W0/kvTcc8+ppqZG3/nOd1RZWamRI0dqw4YNioyMbMZTAgCYhN4EAGiuFv3OUGvBbzkAQMtcS78z1FrQmwCgZVr97wwBAAAAQFvl02VyaFpL3lUFACAQ6E0AcGl8MgQAAADASIQhAAAAAEYiDAEAAAAwEmEIAAAAgJEIQwAAAACMRBgCAAAAYCTCEAAAAAAjEYYAAAAAGIkwBAAAAMBIhCEAAAAARiIMAQAAADASYQgAAACAkQhDAAAAAIxEGAIAAABgpLBgF+APlmVJks6rTrKCXAwAGOa86iT98+9ifIneBADB4UtfuibC0OnTpyVJ27U+yJUAgLlOnz4th8MR7DJaDXoTAATXlfQlm3UNvJXX0NCgo0ePKioqSjab7aLtLpdLSUlJOnz4sKKjo4NQYdvHOWwZzl/LcP5aJtDnz7IsnT59WomJiQoJ4errRvSmwOL8tQznr2U4fy3TmvrSNfHJUEhIiHr06HHZcdHR0bxgW4hz2DKcv5bh/LVMIM8fnwhdjN50dXD+Wobz1zKcv5ZpDX2Jt/AAAAAAGIkwBAAAAMBIRoQhu92uuXPnym63B7uUNotz2DKcv5bh/LUM56914v9Ly3D+Wobz1zKcv5ZpTefvmvgCBQAAAADwlRGfDAEAAADAhQhDAAAAAIxEGAIAAABgJMIQAAAAACMZEYZyc3PVu3dvRUZGKiUlRR999FGwS2oT5s2bJ5vN5rH0798/2GW1Wtu2bdOECROUmJgom82mtWvXemy3LEtz5sxRQkKC2rVrp9TUVBUXFwen2FbocudvypQpF70ex48fH5xiW6GcnBzddtttioqKUmxsrNLT03XgwAGPMWfPnlVWVpa6du2qjh076r777lNFRUWQKga9qXnoTb6hN7UMvall2kJvuubD0Jtvvqns7GzNnTtXe/fu1eDBg+V0OnX8+PFgl9YmDBgwQMeOHXMv27dvD3ZJrVZNTY0GDx6s3Nxcr9tfffVV/eIXv9DSpUu1a9cudejQQU6nU2fPnr3KlbZOlzt/kjR+/HiP1+Mf/vCHq1hh61ZQUKCsrCzt3LlTGzduVF1dncaNG6eamhr3mGeeeUbvvPOOVq9erYKCAh09elSTJk0KYtXmoje1DL3pytGbWobe1DJtojdZ17hhw4ZZWVlZ7sf19fVWYmKilZOTE8Sq2oa5c+dagwcPDnYZbZIka82aNe7HDQ0NVnx8vPXaa6+511VWVlp2u936wx/+EIQKW7cLz59lWVZmZqZ17733BqWetuj48eOWJKugoMCyrC9fb+Hh4dbq1avdY/72t79ZkqzCwsJglWkselPz0Zuaj97UMvSmlmuNvema/mTo3Llz2rNnj1JTU93rQkJClJqaqsLCwiBW1nYUFxcrMTFRffr00SOPPKKysrJgl9QmlZaWqry83OO16HA4lJKSwmvRB1u3blVsbKxuuOEGPfnkkzp16lSwS2q1qqqqJEldunSRJO3Zs0d1dXUer8H+/furZ8+evAavMnpTy9Gb/IPe5B/0pivXGnvTNR2GTp48qfr6esXFxXmsj4uLU3l5eZCqajtSUlKUl5enDRs2aMmSJSotLdWoUaN0+vTpYJfW5jS+3ngtNt/48eP1u9/9Tvn5+XrllVdUUFCgtLQ01dfXB7u0VqehoUEzZ87UiBEjdPPNN0v68jUYERGhTp06eYzlNXj10Ztaht7kP/SmlqM3XbnW2pvCrspR0CalpaW5/zxo0CClpKSoV69eeuuttzRt2rQgVgYTTZ482f3ngQMHatCgQerbt6+2bt2qsWPHBrGy1icrK0v79+/nPgpck+hNaE3oTVeutfama/qToW7duik0NPSib6SoqKhQfHx8kKpquzp16qTrr79eBw8eDHYpbU7j643Xov/06dNH3bp14/V4gRkzZujdd9/Vli1b1KNHD/f6+Ph4nTt3TpWVlR7jeQ1effQm/6I3NR+9yf/oTd615t50TYehiIgIDR06VPn5+e51DQ0Nys/P1/Dhw4NYWdtUXV2tkpISJSQkBLuUNic5OVnx8fEer0WXy6Vdu3bxWmymI0eO6NSpU7we/z/LsjRjxgytWbNGmzdvVnJyssf2oUOHKjw83OM1eODAAZWVlfEavMroTf5Fb2o+epP/0Zs8tYXedM1fJpedna3MzEzdeuutGjZsmBYtWqSamhpNnTo12KW1erNmzdKECRPUq1cvHT16VHPnzlVoaKgyMjKCXVqrVF1d7fFOUGlpqYqKitSlSxf17NlTM2fO1MKFC9WvXz8lJyfrxRdfVGJiotLT04NXdCtyqfPXpUsXvfTSS7rvvvsUHx+vkpISPffcc7ruuuvkdDqDWHXrkZWVpZUrV2rdunWKiopyX2vtcDjUrl07ORwOTZs2TdnZ2erSpYuio6P11FNPafjw4br99tuDXL156E3NR2/yDb2pZehNLdMmetNV+c66IPvlL39p9ezZ04qIiLCGDRtm7dy5M9gltQkPPfSQlZCQYEVERFjdu3e3HnroIevgwYPBLqvV2rJliyXpoiUzM9OyrC+/wvTFF1+04uLiLLvdbo0dO9Y6cOBAcItuRS51/s6cOWONGzfOiomJscLDw61evXpZ06dPt8rLy4Nddqvh7dxJspYvX+4e88UXX1jf/e53rc6dO1vt27e3Jk6caB07dix4RRuO3tQ89Cbf0Jtaht7UMm2hN9n+f6EAAAAAYJRr+p4hAAAAAGgKYQgAAACAkQhDAAAAAIxEGAIAAABgJMIQAAAAACMRhgAAAAAYiTAEAAAAwEiEIQAAAABGIgwBAAAAMBJhCAAAAICRCEMAAAAAjEQYAgAAAGCk/wer2ACI8csREAAAAABJRU5ErkJggg==",
+      "text/plain": [
+       "<Figure size 1000x1500 with 2 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "#Data for figure 2.12 p53\n",
+    "ref212 = np.zeros([22, 22])\n",
+    "ref212[2:21, 2:21] = 1\n",
+    "pred212 = np.zeros([22, 22])\n",
+    "pred212[3:21, 2:21] = 1\n",
+    "\n",
+    "fig,ax = plt.subplots(1,2,figsize=(10,15))\n",
+    "ax[0].imshow(ref212)\n",
+    "ax[1].imshow(pred212)\n",
+    "ax[0].set_title('Reference')\n",
+    "ax[1].set_title('Prediction')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "id": "684ab088",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Text(0.5, 1.0, 'Prediction')"
+      ]
+     },
+     "execution_count": 37,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzYAAAGiCAYAAAA1J1M9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAr6UlEQVR4nO3de5SVBb0//vcAMiDCqMhVUUezTLxL8E3QMEkOXgo95vFWiB3zFF7Qlal58K7jpVyUclBrpR7TtE55yZMWKmp2vCOttFQ0UtIjqOmMQiIyz+8Pf8xxBIWBPe55htdrrb1a+9nP5fNgw5v3fp69p6YoiiIAAAAl1qXaAwAAAKwpxQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQaSzJkzJ3vttVfq6upSU1OTm2++udojAUCbbL755jniiCNant9zzz2pqanJPffcU7Fj1NTU5Mwzz6zY/qCSFBtK5+qrr05NTU3Lo1u3btl4441zxBFH5MUXX1ytfU6YMCF//OMfc9555+Xaa6/NsGHDKjw1AJ3dB/OpR48e+eQnP5ljjjkm8+fPr/Z4q+zXv/618kIpdav2ALC6zj777NTX1+ftt9/Ogw8+mKuvvjr3339/nnjiifTo0WOV9/OPf/wjDzzwQE477bQcc8wx7TgxAGuD9+fT/fffn+nTp+fXv/51nnjiiay77rof2xy77757/vGPf6R79+5t2u7Xv/51pk2btsJy849//CPduvnnIx2T/2dSWuPGjWu5svKv//qv2WijjXLhhRfm1ltvzUEHHbTK+3nllVeSJOuvv37FZnv77bfTvXv3dOnioijA2uaD+dS3b99ccsklueWWW3LIIYcst/7ChQvTq1evis/RpUuXNr3RtyoqvT+oJP/qotPYbbfdkiTPPfdcy7KnnnoqBx54YDbccMP06NEjw4YNy6233try+plnnpnNNtssSXLSSSelpqYmm2++ecvrL774Yo488sgMGDAgtbW1GTp0aH784x+3Ou6ye5hvuOGG/Pu//3s23njjrLvuumlqakqSPPTQQ/mnf/qn1NXVZd11183nPve5/P73v2+1jzPPPDM1NTV59tlnc8QRR2T99ddPXV1dJk6cmEWLFi13rj/5yU8yfPjwrLvuutlggw2y++6757e//W2rdW6//fbstttu6dWrV3r37p199tknTz755Gr8yQKwJj7/+c8nSebOnZsjjjgi6623Xp577rnsvffe6d27dw477LAkSXNzc6ZOnZqhQ4emR48eGTBgQI4++ui8/vrrrfZXFEXOPffcbLLJJll33XWzxx57rPDv9w/7jM1DDz2UvffeOxtssEF69eqV7bffPt///veTJEcccUSmTZuWJK1uq1tmRZ+xefzxxzNu3Lj06dMn6623Xvbcc888+OCDrdZZdpve73//+5x44onp169fevXqlf3337/lDUZYU67Y0Gn89a9/TZJssMEGSZInn3wyI0eOzMYbb5xTTjklvXr1ys9+9rOMHz8+v/jFL7L//vvngAMOyPrrr58TTjghhxxySPbee++st956SZL58+fn//2//5eampocc8wx6devX26//fZ87WtfS1NTUyZPntzq+Oecc066d++eb33rW1m8eHG6d++eu+++O+PGjcsuu+ySM844I126dMlVV12Vz3/+8/nd736X4cOHt9rHQQcdlPr6+jQ0NGTWrFn50Y9+lP79++fCCy9sWeess87KmWeemV133TVnn312unfvnoceeih333139tprryTJtddemwkTJmTs2LG58MILs2jRokyfPj2jRo3K448/3qq8AdC+lr3h1rdv3yTJu+++m7Fjx2bUqFH57ne/23J72tFHH52rr746EydOzHHHHZe5c+fmsssuy+OPP57f//73WWeddZIkp59+es4999zsvffe2XvvvTNr1qzstddeeeedd1Y6y4wZM7Lvvvtm0KBBOf744zNw4MD8+c9/zm233Zbjjz8+Rx99dF566aXMmDEj11577Ur39+STT2a33XZLnz598u1vfzvrrLNOrrjiiowePTr33ntvRowY0Wr9Y489NhtssEHOOOOM/PWvf83UqVNzzDHH5MYbb2zTnymsUAElc9VVVxVJijvvvLN45ZVXinnz5hX/9V//VfTr16+ora0t5s2bVxRFUey5557FdtttV7z99tst2zY3Nxe77rprsdVWW7Usmzt3bpGkuPjii1sd52tf+1oxaNCg4tVXX221/OCDDy7q6uqKRYsWFUVRFDNnziySFFtssUXLsmXH2mqrrYqxY8cWzc3NLcsXLVpU1NfXF1/4whdalp1xxhlFkuLII49sdaz999+/6Nu3b8vzOXPmFF26dCn233//YunSpa3WXXaMN998s1h//fWLo446qtXrL7/8clFXV7fccgAqY0X5dMMNNxR9+/YtevbsWfztb38rJkyYUCQpTjnllFbb/u53vyuSFNddd12r5XfccUer5QsWLCi6d+9e7LPPPq2y5Tvf+U6RpJgwYULLsmX5NHPmzKIoiuLdd98t6uvri80226x4/fXXWx3n/fuaNGlS8WH/RExSnHHGGS3Px48fX3Tv3r147rnnWpa99NJLRe/evYvdd999uT+bMWPGtDrWCSecUHTt2rV44403Vng8aAu3olFaY8aMSb9+/TJkyJAceOCB6dWrV2699dZssskm+fvf/5677747Bx10UN588828+uqrefXVV/Paa69l7NixmTNnzkd+g1pRFPnFL36R/fbbL0VRtGz/6quvZuzYsWlsbMysWbNabTNhwoT07Nmz5fns2bMzZ86cHHrooXnttddatl+4cGH23HPP3HfffWlubm61j3/7t39r9Xy33XbLa6+91nJb280335zm5uacfvrpy31+Z9mtAjNmzMgbb7yRQw45pNXcXbt2zYgRIzJz5sy2/2EDsMren08HH3xw1ltvvdx0003ZeOONW9b5xje+0Wqbn//856mrq8sXvvCFVn9377LLLllvvfVa/u6+884788477+TYY49tdYvYB+8iWJHHH388c+fOzeTJk5f7XOn797Wqli5dmt/+9rcZP358tthii5blgwYNyqGHHpr777+/Jb+W+frXv97qWLvttluWLl2a559/vs3Hhw9yKxqlNW3atHzyk59MY2NjfvzjH+e+++5LbW1tkuTZZ59NURSZMmVKpkyZssLtFyxY0Cpk3u+VV17JG2+8kSuvvDJXXnnlh27/fvX19a2ez5kzJ8l7hefDNDY2ttw6lySbbrppq9eXvfb666+nT58+ee6559KlS5dss802H7rPZcdddk/3B/Xp0+dDtwVgzS3Lp27dumXAgAH51Kc+1erNqG7dumWTTTZptc2cOXPS2NiY/v37r3CfyzJnWQHYaqutWr3er1+/VnmyIstuidt2223bdkIf4pVXXsmiRYvyqU99arnXPv3pT6e5uTnz5s3L0KFDW5Z/VM7BmlJsKK3hw4e3fOvM+PHjM2rUqBx66KF5+umnW66EfOtb38rYsWNXuP0nPvGJD933su0PP/zwDy0m22+/favn779a8/59XHzxxdlxxx1XuI9ln+dZpmvXritcryiKD531g5Yd99prr83AgQOXe93XdAK0r/fn04rU1tYud9W9ubk5/fv3z3XXXbfCbfr161fRGaulEjkHH8a/cOgUunbtmoaGhuyxxx657LLLcuSRRyZJ1llnnYwZM6bN++vXr1969+6dpUuXrtb2SbLlllsmee8KyeruY0X7bG5uzp/+9KcPLUvLjtu/f/+KHReA9rXlllvmzjvvzMiRI5d7o+z9ln2T55w5c1rd/vXKK6+s9KrHsnx44oknPjIfVvW2tH79+mXdddfN008/vdxrTz31VLp06ZIhQ4as0r6gEnzGhk5j9OjRGT58eKZOnZo+ffpk9OjRueKKK/K///u/y627sq+W7Nq1a/75n/85v/jFL/LEE0+0efsk2WWXXbLlllvmu9/9bt56663V2scHjR8/Pl26dMnZZ5+93Odzlr3bNXbs2PTp0yfnn39+lixZUpHjAtC+DjrooCxdujTnnHPOcq+9++67eeONN5K89/mdddZZJ5deemmrqxxTp05d6TF23nnn1NfXZ+rUqS37W+b9+1r2O3U+uM4Hde3aNXvttVduueWWlm8mTd77VtHrr78+o0aNcvszHytXbOhUTjrppHz5y1/O1VdfnWnTpmXUqFHZbrvtctRRR2WLLbbI/Pnz88ADD+Rvf/tb/vCHP3zkvi644ILMnDkzI0aMyFFHHZVtttkmf//73zNr1qzceeed+fvf//6R23fp0iU/+tGPMm7cuAwdOjQTJ07MxhtvnBdffDEzZ85Mnz598qtf/apN5/eJT3wip512Ws4555zstttuOeCAA1JbW5tHHnkkgwcPTkNDQ/r06ZPp06fnK1/5SnbeeeccfPDB6devX1544YX893//d0aOHJnLLrusTccFoH197nOfy9FHH52GhobMnj07e+21V9ZZZ53MmTMnP//5z/P9738/Bx54YPr165dvfetbaWhoyL777pu99947jz/+eG6//fZstNFGH3mMLl26ZPr06dlvv/2y4447ZuLEiRk0aFCeeuqpPPnkk/nNb36T5L035pLkuOOOy9ixY9O1a9ccfPDBK9znueeemxkzZmTUqFH55je/mW7duuWKK67I4sWLc9FFF1X2DwlWQrGhUznggANarpIcddRRefTRR3PWWWfl6quvzmuvvZb+/ftnp512yumnn77SfQ0YMCAPP/xwzj777Pzyl7/Mf/zHf6Rv374ZOnRoq98r81FGjx6dBx54IOecc04uu+yyvPXWWxk4cGBGjBiRo48+erXO8eyzz059fX0uvfTSnHbaaVl33XWz/fbb5ytf+UrLOoceemgGDx6cCy64IBdffHEWL16cjTfeOLvttlsmTpy4WscFoH1dfvnl2WWXXXLFFVfkO9/5Trp165bNN988hx9+eEaOHNmy3rnnnpsePXrk8ssvb3kD7re//W322WeflR5j7NixmTlzZs4666x873vfS3Nzc7bccsscddRRLesccMABOfbYY3PDDTfkJz/5SYqi+NBiM3To0Pzud7/LqaeemoaGhjQ3N2fEiBH5yU9+stzvsIH2VlP4tBYAAFByPmMDAACUnmIDAACUnmIDAACUnmIDAACUnmIDAACUnmIDAACUXof7PTbNzc156aWX0rt379TU1FR7HIC1SlEUefPNNzN48OB06eK9r2VkE0B1tCWXOlyxeemllzJkyJBqjwGwVps3b1422WSTao/RYcgmgOpalVzqcMWmd+/eSZJR2Tvdsk6VpwFYu7ybJbk/v275u5j3yCaA6mhLLnW4YrPsEn+3rJNuNcID4GNVvPc/brdqTTYBVEkbcskN1AAAQOkpNgAAQOkpNgAAQOkpNgAAQOkpNgAAQOm1W7GZNm1aNt988/To0SMjRozIww8/3F6HAoCVkksAnVu7FJsbb7wxJ554Ys4444zMmjUrO+ywQ8aOHZsFCxa0x+EA4CPJJYDOr12KzSWXXJKjjjoqEydOzDbbbJPLL7886667bn784x+3x+EA4CPJJYDOr+LF5p133sljjz2WMWPG/N9BunTJmDFj8sADD1T6cADwkeQSwNqhW6V3+Oqrr2bp0qUZMGBAq+UDBgzIU089tdz6ixcvzuLFi1ueNzU1VXokANZibc2lRDYBlFHVvxWtoaEhdXV1LY8hQ4ZUeyQA1nKyCaB8Kl5sNtpoo3Tt2jXz589vtXz+/PkZOHDgcuufeuqpaWxsbHnMmzev0iMBsBZray4lsgmgjCpebLp3755ddtkld911V8uy5ubm3HXXXfnsZz+73Pq1tbXp06dPqwcAVEpbcymRTQBlVPHP2CTJiSeemAkTJmTYsGEZPnx4pk6dmoULF2bixIntcTgA+EhyCaDza5di8y//8i955ZVXcvrpp+fll1/OjjvumDvuuGO5D24CwMdBLgF0fjVFURTVHuL9mpqaUldXl9H5UrrVrFPtcQDWKu8WS3JPbkljY6Pbr95HNgFUR1tyqerfigYAALCmFBsAAKD0FBsAAKD0FBsAAKD0FBsAAKD02uXrnsvgNy/NbvdjjB28Y7sfAwAAcMUGAADoBBQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9LpVe4BqGTt4x2qPAAAAVIgrNgAAQOkpNgAAQOkpNgAAQOkpNgAAQOkpNgAAQOkpNgAAQOkpNgAAQOkpNgAAQOkpNgAAQOlVvNg0NDTkM5/5THr37p3+/ftn/Pjxefrppyt9GABYZbIJoPOreLG59957M2nSpDz44IOZMWNGlixZkr322isLFy6s9KEAYJXIJoDOr1uld3jHHXe0en711Venf//+eeyxx7L77rtX+nAAsFKyCaDzq3ix+aDGxsYkyYYbbrjC1xcvXpzFixe3PG9qamrvkQBYy8kmgM6nXb88oLm5OZMnT87IkSOz7bbbrnCdhoaG1NXVtTyGDBnSniMBsJaTTQCdU7sWm0mTJuWJJ57IDTfc8KHrnHrqqWlsbGx5zJs3rz1HAmAtJ5sAOqd2uxXtmGOOyW233Zb77rsvm2yyyYeuV1tbm9ra2vYaAwBayCaAzqvixaYoihx77LG56aabcs8996S+vr7ShwCANpFNAJ1fxYvNpEmTcv311+eWW25J79698/LLLydJ6urq0rNnz0ofDgBWSjYBdH4V/4zN9OnT09jYmNGjR2fQoEEtjxtvvLHShwKAVSKbADq/drkVDQA6EtkE0Pm167eiAQAAfBwUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPQUGwAAoPTavdhccMEFqampyeTJk9v7UACwUnIJoHNq12LzyCOP5Iorrsj222/fnocBgFUilwA6r3YrNm+99VYOO+yw/PCHP8wGG2zQXocBgFUilwA6t3YrNpMmTco+++yTMWPGfOR6ixcvTlNTU6sHAFTaquZSIpsAyqhbe+z0hhtuyKxZs/LII4+sdN2GhoacddZZ7TEGACRpWy4lsgmgjCp+xWbevHk5/vjjc91116VHjx4rXf/UU09NY2Njy2PevHmVHgmAtVhbcymRTQBlVPErNo899lgWLFiQnXfeuWXZ0qVLc9999+Wyyy7L4sWL07Vr15bXamtrU1tbW+kxACBJ23MpkU0AZVTxYrPnnnvmj3/8Y6tlEydOzNZbb52TTz55ufAAgPYklwDWDhUvNr179862227balmvXr3St2/f5ZYDQHuTSwBrh3b/BZ0AAADtrV2+Fe2D7rnnno/jMACwSuQSQOfjig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB6ig0AAFB67VJsXnzxxRx++OHp27dvevbsme222y6PPvpoexwKAFZKLgF0ft0qvcPXX389I0eOzB577JHbb789/fr1y5w5c7LBBhtU+lAAsFJyCWDtUPFic+GFF2bIkCG56qqrWpbV19dX+jAAsErkEsDaoeK3ot16660ZNmxYvvzlL6d///7Zaaed8sMf/rDShwGAVSKXANYOFS82f/nLXzJ9+vRstdVW+c1vfpNvfOMbOe6443LNNdescP3Fixenqamp1QMAKqWtuZTIJoAyqvitaM3NzRk2bFjOP//8JMlOO+2UJ554IpdffnkmTJiw3PoNDQ0566yzKj0GACRpey4lsgmgjCp+xWbQoEHZZpttWi379Kc/nRdeeGGF65966qlpbGxsecybN6/SIwGwFmtrLiWyCaCMKn7FZuTIkXn66adbLXvmmWey2WabrXD92tra1NbWVnoMAEjS9lxKZBNAGVX8is0JJ5yQBx98MOeff36effbZXH/99bnyyiszadKkSh8KAFZKLgGsHSpebD7zmc/kpptuyk9/+tNsu+22OeecczJ16tQcdthhlT4UAKyUXAJYO1T8VrQk2XfffbPvvvu2x64BoM3kEkDnV/ErNgAAAB83xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACg9xQYAACi9ihebpUuXZsqUKamvr0/Pnj2z5ZZb5pxzzklRFJU+FACslFwCWDt0q/QOL7zwwkyfPj3XXHNNhg4dmkcffTQTJ05MXV1djjvuuEofDgA+klwCWDtUvNj8z//8T770pS9ln332SZJsvvnm+elPf5qHH3640ocCgJWSSwBrh4rfirbrrrvmrrvuyjPPPJMk+cMf/pD7778/48aNW+H6ixcvTlNTU6sHAFRKW3MpkU0AZVTxKzannHJKmpqasvXWW6dr165ZunRpzjvvvBx22GErXL+hoSFnnXVWpccAgCRtz6VENgGUUcWv2PzsZz/Lddddl+uvvz6zZs3KNddck+9+97u55pprVrj+qaeemsbGxpbHvHnzKj0SAGuxtuZSIpsAyqjiV2xOOumknHLKKTn44IOTJNttt12ef/75NDQ0ZMKECcutX1tbm9ra2kqPAQBJ2p5LiWwCKKOKX7FZtGhRunRpvduuXbumubm50ocCgJWSSwBrh4pfsdlvv/1y3nnnZdNNN83QoUPz+OOP55JLLsmRRx5Z6UMBwErJJYC1Q8WLzaWXXpopU6bkm9/8ZhYsWJDBgwfn6KOPzumnn17pQwHASsklgLVDTdHBfvVyU1NT6urqMjpfSreadao9DsBa5d1iSe7JLWlsbEyfPn2qPU6HIZsAqqMtuVTxz9gAAAB83BQbAACg9BQbAACg9BQbAACg9BQbAACg9Cr+dc9A5/Gbl2ZXe4RSGTt4x2qPANDpyaa2WZuyyRUbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9BQbAACg9NpcbO67777st99+GTx4cGpqanLzzTe3er0oipx++ukZNGhQevbsmTFjxmTOnDmVmhcAWpFLACSrUWwWLlyYHXbYIdOmTVvh6xdddFF+8IMf5PLLL89DDz2UXr16ZezYsXn77bfXeFgA+CC5BECSdGvrBuPGjcu4ceNW+FpRFJk6dWr+/d//PV/60peSJP/5n/+ZAQMG5Oabb87BBx+8ZtMCwAfIJQCSCn/GZu7cuXn55ZczZsyYlmV1dXUZMWJEHnjggRVus3jx4jQ1NbV6AEAlrE4uJbIJoIwqWmxefvnlJMmAAQNaLR8wYEDLax/U0NCQurq6lseQIUMqORIAa7HVyaVENgGUUdW/Fe3UU09NY2Njy2PevHnVHgmAtZxsAiifihabgQMHJknmz5/favn8+fNbXvug2tra9OnTp9UDACphdXIpkU0AZVTRYlNfX5+BAwfmrrvualnW1NSUhx56KJ/97GcreSgAWCm5BLD2aPO3or311lt59tlnW57PnTs3s2fPzoYbbphNN900kydPzrnnnputttoq9fX1mTJlSgYPHpzx48dXcm4ASCKXAHhPm4vNo48+mj322KPl+YknnpgkmTBhQq6++up8+9vfzsKFC/P1r389b7zxRkaNGpU77rgjPXr0qNzUAPD/k0sAJElNURRFtYd4v6amptTV1WV0vpRuNetUexxYq/3mpdnVHqFUxg7esdojrLF3iyW5J7eksbHR50reRzZBxyGb2qbs2dSWXKr6t6IBAACsKcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAoPcUGAAAovW7VHgDouMYO3rHaIwBAK7KJD+OKDQAAUHqKDQAAUHqKDQAAUHqKDQAAUHqKDQAAUHqKDQAAUHqKDQAAUHptLjb33Xdf9ttvvwwePDg1NTW5+eabW15bsmRJTj755Gy33Xbp1atXBg8enK9+9at56aWXKjkzALSQSwAkq1FsFi5cmB122CHTpk1b7rVFixZl1qxZmTJlSmbNmpVf/vKXefrpp/PFL36xIsMCwAfJJQCSpFtbNxg3blzGjRu3wtfq6uoyY8aMVssuu+yyDB8+PC+88EI23XTT1ZsSAD6EXAIg+Rg+Y9PY2Jiampqsv/767X0oAFgpuQTQObX5ik1bvP322zn55JNzyCGHpE+fPitcZ/HixVm8eHHL86ampvYcCYC12KrkUiKbAMqo3a7YLFmyJAcddFCKosj06dM/dL2GhobU1dW1PIYMGdJeIwGwFlvVXEpkE0AZtUuxWRYezz//fGbMmPGR74qdeuqpaWxsbHnMmzevPUYCYC3WllxKZBNAGVX8VrRl4TFnzpzMnDkzffv2/cj1a2trU1tbW+kxACBJ23MpkU0AZdTmYvPWW2/l2WefbXk+d+7czJ49OxtuuGEGDRqUAw88MLNmzcptt92WpUuX5uWXX06SbLjhhunevXvlJgeAyCUA3lNTFEXRlg3uueee7LHHHsstnzBhQs4888zU19evcLuZM2dm9OjRK91/U1NT6urqMjpfSreaddoyGgBr6N1iSe7JLWlsbFzp7VodRXvnUiKbAKqlLbnU5is2o0ePzkd1oTb2JABYI3IJgORj+D02AAAA7U2xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASk+xAQAASq9btQf4oKIokiTvZklSVHkYgLXMu1mS5P/+LuY9sgmgOtqSSx2u2Lz55ptJkvvz6ypPArD2evPNN1NXV1ftMToM2QRQXauSSzVFB3tbrrm5OS+99FJ69+6dmpqaVdqmqakpQ4YMybx589KnT592nrD9dJbzSDrPuTiPjqeznEtHPY+iKPLmm29m8ODB6dLF3crLyCbn0ZF0lnNxHh1PRzyXtuRSh7ti06VLl2yyySartW2fPn06zH+ENdFZziPpPOfiPDqeznIuHfE8XKlZnmxyHh1RZzkX59HxdLRzWdVc8nYcAABQeooNAABQep2i2NTW1uaMM85IbW1ttUdZI53lPJLOcy7Oo+PpLOfSWc6DD9dZ/hs7j46ns5yL8+h4yn4uHe7LAwAAANqqU1yxAQAA1m6KDQAAUHqKDQAAUHqKDQAAUHqlLzbTpk3L5ptvnh49emTEiBF5+OGHqz1SmzU0NOQzn/lMevfunf79+2f8+PF5+umnqz3WGrvgggtSU1OTyZMnV3uUNnvxxRdz+OGHp2/fvunZs2e22267PProo9Ueq82WLl2aKVOmpL6+Pj179syWW26Zc845Jx39O0Puu+++7Lfffhk8eHBqampy8803t3q9KIqcfvrpGTRoUHr27JkxY8Zkzpw51Rl2JT7qXJYsWZKTTz452223XXr16pXBgwfnq1/9al566aXqDUxFyKaOSzZVn2yqrs6cS6UuNjfeeGNOPPHEnHHGGZk1a1Z22GGHjB07NgsWLKj2aG1y7733ZtKkSXnwwQczY8aMLFmyJHvttVcWLlxY7dFW2yOPPJIrrrgi22+/fbVHabPXX389I0eOzDrrrJPbb789f/rTn/K9730vG2ywQbVHa7MLL7ww06dPz2WXXZY///nPufDCC3PRRRfl0ksvrfZoH2nhwoXZYYcdMm3atBW+ftFFF+UHP/hBLr/88jz00EPp1atXxo4dm7fffvtjnnTlPupcFi1alFmzZmXKlCmZNWtWfvnLX+bpp5/OF7/4xSpMSqXIpo5LNnUMsqm6OnUuFSU2fPjwYtKkSS3Ply5dWgwePLhoaGio4lRrbsGCBUWS4t577632KKvlzTffLLbaaqtixowZxec+97ni+OOPr/ZIbXLyyScXo0aNqvYYFbHPPvsURx55ZKtlBxxwQHHYYYdVaaK2S1LcdNNNLc+bm5uLgQMHFhdffHHLsjfeeKOora0tfvrTn1ZhwlX3wXNZkYcffrhIUjz//PMfz1BUnGzqmGRTxyGbOo7OlkulvWLzzjvv5LHHHsuYMWNalnXp0iVjxozJAw88UMXJ1lxjY2OSZMMNN6zyJKtn0qRJ2WeffVr9tymTW2+9NcOGDcuXv/zl9O/fPzvttFN++MMfVnus1bLrrrvmrrvuyjPPPJMk+cMf/pD7778/48aNq/Jkq2/u3Ll5+eWXW/3/q66uLiNGjCj9z37y3s9/TU1N1l9//WqPwmqQTR2XbOo4ZFO5lCmXulV7gNX16quvZunSpRkwYECr5QMGDMhTTz1VpanWXHNzcyZPnpyRI0dm2223rfY4bXbDDTdk1qxZeeSRR6o9ymr7y1/+kunTp+fEE0/Md77znTzyyCM57rjj0r1790yYMKHa47XJKaeckqampmy99dbp2rVrli5dmvPOOy+HHXZYtUdbbS+//HKSrPBnf9lrZfX222/n5JNPziGHHJI+ffpUexxWg2zqmGRTxyKbyqNsuVTaYtNZTZo0KU888UTuv//+ao/SZvPmzcvxxx+fGTNmpEePHtUeZ7U1Nzdn2LBhOf/885MkO+20U5544olcfvnlpQuPn/3sZ7nuuuty/fXXZ+jQoZk9e3YmT56cwYMHl+5cOrslS5bkoIMOSlEUmT59erXHgVZkU/XJJj5uZcyl0t6KttFGG6Vr166ZP39+q+Xz58/PwIEDqzTVmjnmmGNy2223ZebMmdlkk02qPU6bPfbYY1mwYEF23nnndOvWLd26dcu9996bH/zgB+nWrVuWLl1a7RFXyaBBg7LNNtu0WvbpT386L7zwQpUmWn0nnXRSTjnllBx88MHZbrvt8pWvfCUnnHBCGhoaqj3aalv2892ZfvaXhcfzzz+fGTNmlOJdMVZMNnU8sqnjkU0dX1lzqbTFpnv37tlll11y1113tSxrbm7OXXfdlc9+9rNVnKztiqLIMccck5tuuil333136uvrqz3Satlzzz3zxz/+MbNnz255DBs2LIcddlhmz56drl27VnvEVTJy5MjlvtL0mWeeyWabbValiVbfokWL0qVL6x/zrl27prm5uUoTrbn6+voMHDiw1c9+U1NTHnroodL97Cf/Fx5z5szJnXfemb59+1Z7JNaAbOp4ZFPHI5s6tjLnUqlvRTvxxBMzYcKEDBs2LMOHD8/UqVOzcOHCTJw4sdqjtcmkSZNy/fXX55Zbbknv3r1b7sWsq6tLz549qzzdquvdu/dy91736tUrffv2LdU92SeccEJ23XXXnH/++TnooIPy8MMP58orr8yVV15Z7dHabL/99st5552XTTfdNEOHDs3jjz+eSy65JEceeWS1R/tIb731Vp599tmW53Pnzs3s2bOz4YYbZtNNN83kyZNz7rnnZquttkp9fX2mTJmSwYMHZ/z48dUb+kN81LkMGjQoBx54YGbNmpXbbrstS5cubfn533DDDdO9e/dqjc0akE0di2zqeGRTdXXqXKrul7KtuUsvvbTYdNNNi+7duxfDhw8vHnzwwWqP1GZJVvi46qqrqj3aGivjV2oWRVH86le/Krbddtuitra22HrrrYsrr7yy2iOtlqampuL4448vNt1006JHjx7FFltsUZx22mnF4sWLqz3aR5o5c+YKfyYmTJhQFMV7X6s5ZcqUYsCAAUVtbW2x5557Fk8//XR1h/4QH3Uuc+fO/dCf/5kzZ1Z7dNaAbOrYZFN1yabq6sy5VFMUHfzXvAIAAKxEaT9jAwAAsIxiAwAAlJ5iAwAAlJ5iAwAAlJ5iAwAAlJ5iAwAAlJ5iAwAAlJ5iAwAAlJ5iAwAAlJ5iAwAAlJ5iAwAAlJ5iAwAAlN7/B1rf+j7d5Sv2AAAAAElFTkSuQmCC",
+      "text/plain": [
+       "<Figure size 1000x1500 with 2 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "#Figure 5c p 14 Pitfalls\n",
+    "ref5c = np.zeros([14, 14])\n",
+    "ref5c[1, 1] = 1\n",
+    "ref5c[9:12, 9:12] = 1\n",
+    "pred5c = np.zeros([14, 14])\n",
+    "pred5c [9:12, 9:12] = 1\n",
+    "\n",
+    "fig,ax = plt.subplots(1,2,figsize=(10,15))\n",
+    "ax[0].imshow(ref5c)\n",
+    "ax[1].imshow(pred5c)\n",
+    "ax[0].set_title('Reference')\n",
+    "ax[1].set_title('Prediction')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ddbe8980",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}