From 439d7ed8346577954e103cf571d0bcf6735b0704 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 14 Mar 2024 12:47:25 +0100 Subject: [PATCH 001/101] Simplify annotations as python>=3.9 Starting from python 3.9, we can - replace Tuple, Dict, List, Type from typing by buitins tuple, dict, list, type - replace typing.Callable by collections.abc.Callable - replace typing.Sequence by collections.abc.Sequence See https://peps.python.org/pep-0585/ --- docs/generate_nb_index.py | 9 +-- src/decomon/backward_layers/activations.py | 55 +++++++------- .../backward_layers/backward_layers.py | 32 ++++---- .../backward_layers/backward_maxpooling.py | 16 ++-- src/decomon/backward_layers/backward_merge.py | 32 ++++---- src/decomon/backward_layers/convert.py | 4 +- src/decomon/backward_layers/core.py | 12 +-- src/decomon/backward_layers/utils.py | 54 ++++++------- src/decomon/core.py | 38 +++++----- src/decomon/keras_utils.py | 6 +- src/decomon/layers/activations.py | 59 +++++++------- src/decomon/layers/convert.py | 12 +-- src/decomon/layers/core.py | 22 +++--- src/decomon/layers/decomon_layers.py | 48 ++++++------ src/decomon/layers/decomon_merge_layers.py | 26 +++---- src/decomon/layers/decomon_reshape.py | 14 ++-- src/decomon/layers/maxpooling.py | 20 ++--- src/decomon/layers/utils.py | 76 +++++++++---------- src/decomon/layers/utils_pooling.py | 10 +-- src/decomon/metrics/loss.py | 33 ++++---- src/decomon/metrics/metric.py | 12 +-- src/decomon/metrics/utils.py | 6 +- src/decomon/models/backward_cloning.py | 43 ++++++----- src/decomon/models/convert.py | 23 +++--- src/decomon/models/crown.py | 28 +++---- src/decomon/models/forward_cloning.py | 31 ++++---- src/decomon/models/models.py | 20 ++--- src/decomon/models/utils.py | 32 ++++---- src/decomon/types/__init__.py | 4 +- src/decomon/utils.py | 55 +++++++------- src/decomon/wrapper.py | 17 +++-- tests/conftest.py | 38 +++++----- 32 files changed, 447 insertions(+), 440 deletions(-) diff --git a/docs/generate_nb_index.py b/docs/generate_nb_index.py index c69f6565..80befd97 100644 --- a/docs/generate_nb_index.py +++ b/docs/generate_nb_index.py @@ -4,7 +4,6 @@ import os import re import urllib.parse -from typing import List, Tuple NOTEBOOKS_LIST_PLACEHOLDER = "[[notebooks-list]]" NOTEBOOKS_PAGE_TEMPLATE_RELATIVE_PATH = "tutorials.template.md" @@ -25,7 +24,7 @@ def extract_notebook_title_n_description( notebook_filepath: str, -) -> Tuple[str, List[str]]: +) -> tuple[str, list[str]]: # load notebook with open(notebook_filepath, "rt", encoding="utf-8") as f: notebook = json.load(f) @@ -33,7 +32,7 @@ def extract_notebook_title_n_description( # find title + description: from first cell, h1 title + remaining text. # or title from filename else title = "" - description_lines: List[str] = [] + description_lines: list[str] = [] cell = notebook["cells"][0] if cell["cell_type"] == "markdown": firstline = cell["source"][0].strip() @@ -51,7 +50,7 @@ def extract_notebook_title_n_description( return title, description_lines -def filter_tags_from_description(description_lines: List[str], html_tag_to_remove: str) -> List[str]: +def filter_tags_from_description(description_lines: list[str], html_tag_to_remove: str) -> list[str]: description = "".join(description_lines) # opening/closing tags opening_tag = html_tag_to_remove @@ -147,7 +146,7 @@ def get_binder_link( return link -def get_repo_n_branches_for_binder_n_github_links() -> Tuple[bool, str, str, str, str, str, bool]: +def get_repo_n_branches_for_binder_n_github_links() -> tuple[bool, str, str, str, str, str, bool]: # repos + branches to use for binder environment and notebooks content. creating_links = True use_nbgitpuller = False diff --git a/src/decomon/backward_layers/activations.py b/src/decomon/backward_layers/activations.py index b53ade7d..153f1cd9 100644 --- a/src/decomon/backward_layers/activations.py +++ b/src/decomon/backward_layers/activations.py @@ -1,5 +1,6 @@ import warnings -from typing import Any, Callable, Dict, List, Optional, Union +from collections.abc import Callable +from typing import Any, Optional, Union import keras.ops as K import numpy as np @@ -39,7 +40,7 @@ def backward_relu( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, alpha: float = 0.0, @@ -48,7 +49,7 @@ def backward_relu( slope: Union[str, Slope] = Slope.V_SLOPE, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of relu Args: @@ -87,13 +88,13 @@ def backward_relu( def backward_sigmoid( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, slope: Union[str, Slope] = Slope.V_SLOPE, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of sigmoid Args: @@ -118,13 +119,13 @@ def backward_sigmoid( def backward_tanh( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, slope: Union[str, Slope] = Slope.V_SLOPE, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of tanh Args: @@ -149,13 +150,13 @@ def backward_tanh( def backward_hard_sigmoid( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, slope: Union[str, Slope] = Slope.V_SLOPE, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of hard sigmoid Args: @@ -178,13 +179,13 @@ def backward_hard_sigmoid( def backward_elu( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, slope: Union[str, Slope] = Slope.V_SLOPE, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of Exponential Linear Unit Args: @@ -208,13 +209,13 @@ def backward_elu( def backward_selu( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, slope: Union[str, Slope] = Slope.V_SLOPE, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of Scaled Exponential Linear Unit (SELU) Args: @@ -238,13 +239,13 @@ def backward_selu( def backward_linear( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, slope: Union[str, Slope] = Slope.V_SLOPE, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of linear Args: @@ -265,13 +266,13 @@ def backward_linear( def backward_exponential( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, slope: Union[str, Slope] = Slope.V_SLOPE, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPAof exponential Args: @@ -294,13 +295,13 @@ def backward_exponential( def backward_softplus( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, slope: Union[str, Slope] = Slope.V_SLOPE, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of softplus Args: @@ -325,13 +326,13 @@ def backward_softplus( def backward_softsign( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, slope: Union[str, Slope] = Slope.V_SLOPE, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of softsign Args: @@ -359,7 +360,7 @@ def backward_softsign( def backward_softsign_( - inputs: List[Tensor], + inputs: list[Tensor], w_u_out: Tensor, b_u_out: Tensor, w_l_out: Tensor, @@ -368,7 +369,7 @@ def backward_softsign_( mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: if perturbation_domain is None: perturbation_domain = BoxDomain() w_u_0, b_u_0, w_l_0, b_l_0 = get_linear_hull_s_shape( @@ -391,14 +392,14 @@ def backward_softsign_( def backward_softmax( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, slope: Union[str, Slope] = Slope.V_SLOPE, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, axis: int = -1, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of softmax Args: @@ -422,7 +423,7 @@ def backward_softmax( raise NotImplementedError() -def deserialize(name: str) -> Callable[..., List[Tensor]]: +def deserialize(name: str) -> Callable[..., list[Tensor]]: """Get the activation from name. Args: @@ -457,7 +458,7 @@ def deserialize(name: str) -> Callable[..., List[Tensor]]: raise ValueError("Could not interpret " "activation function identifier:", name) -def get(identifier: Any) -> Callable[..., List[Tensor]]: +def get(identifier: Any) -> Callable[..., list[Tensor]]: """Get the `identifier` activation function. Args: diff --git a/src/decomon/backward_layers/backward_layers.py b/src/decomon/backward_layers/backward_layers.py index 7b3cbf7f..76619e64 100644 --- a/src/decomon/backward_layers/backward_layers.py +++ b/src/decomon/backward_layers/backward_layers.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras import keras.ops as K @@ -65,7 +65,7 @@ def __init__( ) self.frozen_weights = False - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: if len(inputs) == 0: inputs = self.layer.input @@ -96,7 +96,7 @@ def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor return [w, b, w, b] - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: """ Args: input_shape: list of input shape @@ -166,7 +166,7 @@ def __init__( ) self.frozen_weights = False - def get_affine_components(self, inputs: List[BackendTensor]) -> Tuple[BackendTensor, BackendTensor]: + def get_affine_components(self, inputs: list[BackendTensor]) -> tuple[BackendTensor, BackendTensor]: """Express the implicit affine matrix of the convolution layer. Conv is a linear operator but its affine component is implicit @@ -210,7 +210,7 @@ def get_affine_components(self, inputs: List[BackendTensor]) -> Tuple[BackendTen return w_out_, b_out_ - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: weight_, bias_ = self.get_affine_components(inputs) return [weight_, bias_] * 2 @@ -252,13 +252,13 @@ def __init__( self.activation_name = layer.get_config()["activation"] self.slope = Slope(slope) self.finetune = finetune - self.finetune_param: List[keras.Variable] = [] + self.finetune_param: list[keras.Variable] = [] if self.finetune: self.frozen_alpha = False - self.grid_finetune: List[keras.Variable] = [] + self.grid_finetune: list[keras.Variable] = [] self.frozen_grid = False - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: config = super().get_config() config.update( { @@ -268,7 +268,7 @@ def get_config(self) -> Dict[str, Any]: ) return config - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: """ Args: input_shape: list of input shape @@ -371,7 +371,7 @@ def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: self.built = True - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: # infer the output dimension if self.activation_name != "linear": if self.finetune: @@ -452,7 +452,7 @@ def __init__( **kwargs, ) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: return get_identity_lirpa(inputs) @@ -477,7 +477,7 @@ def __init__( **kwargs, ) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: return get_identity_lirpa(inputs) @@ -504,7 +504,7 @@ def __init__( self.dims = layer.dims self.op = layer.call - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) # w_u_out (None, n_in, n_out) @@ -545,7 +545,7 @@ def __init__( **kwargs, ) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: return get_identity_lirpa(inputs) @@ -573,7 +573,7 @@ def __init__( self.axis = self.layer.axis self.op_flat = Flatten() - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: y = inputs[-1] n_out = int(np.prod(y.shape[1:])) @@ -625,5 +625,5 @@ def __init__( **kwargs, ) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: return get_identity_lirpa(inputs) diff --git a/src/decomon/backward_layers/backward_maxpooling.py b/src/decomon/backward_layers/backward_maxpooling.py index 823d8526..e1e5f567 100644 --- a/src/decomon/backward_layers/backward_maxpooling.py +++ b/src/decomon/backward_layers/backward_maxpooling.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras.ops as K import numpy as np @@ -14,8 +14,8 @@ class BackwardMaxPooling2D(BackwardLayer): """Backward LiRPA of MaxPooling2D""" - pool_size: Tuple[int, int] - strides: Tuple[int, int] + pool_size: tuple[int, int] + strides: tuple[int, int] padding: str data_format: str fast: bool @@ -41,12 +41,12 @@ def __init__( def _pooling_function_fast( self, - inputs: List[BackendTensor], + inputs: list[BackendTensor], w_u_out: BackendTensor, b_u_out: BackendTensor, w_l_out: BackendTensor, b_l_out: BackendTensor, - ) -> List[BackendTensor]: + ) -> list[BackendTensor]: x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) op_flat = Flatten() @@ -79,12 +79,12 @@ def _pooling_function_fast( def _pooling_function_not_fast( self, - inputs: List[BackendTensor], + inputs: list[BackendTensor], w_u_out: BackendTensor, b_u_out: BackendTensor, w_l_out: BackendTensor, b_l_out: BackendTensor, - ) -> List[BackendTensor]: + ) -> list[BackendTensor]: """ Args: inputs @@ -184,7 +184,7 @@ def _pooling_function_not_fast( return [w_u_out, b_u_out, w_l_out, b_l_out] - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) if self.fast: return self._pooling_function_fast( diff --git a/src/decomon/backward_layers/backward_merge.py b/src/decomon/backward_layers/backward_merge.py index acfdcd3e..0a9028a7 100644 --- a/src/decomon/backward_layers/backward_merge.py +++ b/src/decomon/backward_layers/backward_merge.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from itertools import chain -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras import keras.ops as K @@ -39,7 +39,7 @@ class BackwardMerge(ABC, Wrapper): layer: Layer - _trainable_weights: List[keras.Variable] + _trainable_weights: list[keras.Variable] def __init__( self, @@ -77,7 +77,7 @@ def ibp(self) -> bool: def affine(self) -> bool: return get_affine(self.mode) - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: config = super().get_config() config.update( { @@ -90,7 +90,7 @@ def get_config(self) -> Dict[str, Any]: return config @abstractmethod - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[List[BackendTensor]]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: """ Args: inputs @@ -100,7 +100,7 @@ def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[List[BackendT """ pass - def compute_output_shape(self, input_shape: List[Tuple[Optional[int], ...]]) -> List[Tuple[Optional[int], ...]]: + def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: """Compute expected output shape according to input shape Will be called by symbolic calls on Keras Tensors. @@ -113,7 +113,7 @@ def compute_output_shape(self, input_shape: List[Tuple[Optional[int], ...]]) -> """ raise NotImplementedError() - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: """ Args: input_shape @@ -164,7 +164,7 @@ def __init__( mode=self.mode, perturbation_domain=self.perturbation_domain, dc_decomp=self.dc_decomp ).call - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[List[BackendTensor]]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) @@ -211,7 +211,7 @@ def __init__( ) self.op = DecomonAdd(mode=self.mode, perturbation_domain=self.perturbation_domain, dc_decomp=False).call - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[List[BackendTensor]]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) @@ -220,8 +220,8 @@ def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[List[BackendT if n_elem == 1: # nothing to merge return [[w_u_out, b_u_out, w_l_out, b_l_out]] else: - bounds: List[List[BackendTensor]] = [] - input_bounds: List[List[BackendTensor]] = [] + bounds: list[list[BackendTensor]] = [] + input_bounds: list[list[BackendTensor]] = [] for j in range(n_elem - 1, 0, -1): inputs_1 = inputs_list[j] @@ -274,7 +274,7 @@ def __init__( if not isinstance(layer, DecomonSubtract): raise KeyError() - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[List[BackendTensor]]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) @@ -319,7 +319,7 @@ def __init__( if not isinstance(layer, DecomonMaximum): raise KeyError() - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[List[BackendTensor]]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) @@ -364,7 +364,7 @@ def __init__( if not isinstance(layer, DecomonMinimum): raise KeyError() - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[List[BackendTensor]]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) @@ -411,7 +411,7 @@ def __init__( self.axis = self.layer.axis - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[List[BackendTensor]]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) @@ -454,7 +454,7 @@ def __init__( if not isinstance(layer, DecomonMultiply): raise KeyError() - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[List[BackendTensor]]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) @@ -504,7 +504,7 @@ def __init__( raise NotImplementedError() - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[List[BackendTensor]]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) diff --git a/src/decomon/backward_layers/convert.py b/src/decomon/backward_layers/convert.py index a4116463..f1f6d08b 100644 --- a/src/decomon/backward_layers/convert.py +++ b/src/decomon/backward_layers/convert.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Union +from typing import Any, Optional, Union from keras.layers import Layer @@ -8,7 +8,7 @@ from decomon.backward_layers.core import BackwardLayer from decomon.core import BoxDomain, ForwardMode, PerturbationDomain, Slope -_mapping_name2class: Dict[str, Any] = vars(decomon.backward_layers.backward_layers) +_mapping_name2class: dict[str, Any] = vars(decomon.backward_layers.backward_layers) _mapping_name2class.update(vars(decomon.backward_layers.backward_merge)) _mapping_name2class.update(vars(decomon.backward_layers.backward_maxpooling)) diff --git a/src/decomon/backward_layers/core.py b/src/decomon/backward_layers/core.py index 92e98381..99baf276 100644 --- a/src/decomon/backward_layers/core.py +++ b/src/decomon/backward_layers/core.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras import numpy as np @@ -20,7 +20,7 @@ class BackwardLayer(ABC, Wrapper): layer: Layer - _trainable_weights: List[keras.Variable] + _trainable_weights: list[keras.Variable] def __init__( self, @@ -58,7 +58,7 @@ def ibp(self) -> bool: def affine(self) -> bool: return get_affine(self.mode) - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: config = super().get_config() config.update( { @@ -71,7 +71,7 @@ def get_config(self) -> Dict[str, Any]: return config @abstractmethod - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: """ Args: inputs @@ -81,7 +81,7 @@ def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor """ pass - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: """ Args: input_shape @@ -92,7 +92,7 @@ def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: # generic case: nothing to do before call pass - def compute_output_shape(self, input_shape: List[Tuple[Optional[int], ...]]) -> List[Tuple[Optional[int], ...]]: + def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: """Compute expected output shape according to input shape Will be called by symbolic calls on Keras Tensors. diff --git a/src/decomon/backward_layers/utils.py b/src/decomon/backward_layers/utils.py index a36425a7..1b6094e5 100644 --- a/src/decomon/backward_layers/utils.py +++ b/src/decomon/backward_layers/utils.py @@ -1,4 +1,4 @@ -from typing import Any, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras.ops as K import numpy as np @@ -22,8 +22,8 @@ def backward_add( - inputs_0: List[Tensor], - inputs_1: List[Tensor], + inputs_0: list[Tensor], + inputs_1: list[Tensor], w_u_out: Tensor, b_u_out: Tensor, w_l_out: Tensor, @@ -31,7 +31,7 @@ def backward_add( perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, dc_decomp: bool = False, -) -> List[List[Tensor]]: +) -> list[list[Tensor]]: """Backward LiRPA of inputs_0+inputs_1 Args: @@ -84,10 +84,10 @@ def backward_add( def backward_linear_prod( x_0: Tensor, - bounds_x: List[Tensor], - back_bounds: List[Tensor], + bounds_x: list[Tensor], + back_bounds: list[Tensor], perturbation_domain: Optional[PerturbationDomain] = None, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of a subroutine prod Args: @@ -158,8 +158,8 @@ def backward_linear_prod( def backward_maximum( - inputs_0: List[Tensor], - inputs_1: List[Tensor], + inputs_0: list[Tensor], + inputs_1: list[Tensor], w_u_out: Tensor, b_u_out: Tensor, w_l_out: Tensor, @@ -168,7 +168,7 @@ def backward_maximum( mode: Union[str, ForwardMode] = ForwardMode.HYBRID, dc_decomp: bool = False, **kwargs: Any, -) -> List[List[Tensor]]: +) -> list[list[Tensor]]: """Backward LiRPA of maximum(inputs_0, inputs_1) Args: @@ -228,7 +228,7 @@ def backward_maximum( # convex hull of the maximum between two functions def backward_max_( - inputs: List[Tensor], + inputs: list[Tensor], w_u_out: Tensor, b_u_out: Tensor, w_l_out: Tensor, @@ -238,7 +238,7 @@ def backward_max_( axis: int = -1, dc_decomp: bool = False, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of max Args: @@ -350,8 +350,8 @@ def backward_max_( def backward_minimum( - inputs_0: List[Tensor], - inputs_1: List[Tensor], + inputs_0: list[Tensor], + inputs_1: list[Tensor], w_u_out: Tensor, b_u_out: Tensor, w_l_out: Tensor, @@ -360,7 +360,7 @@ def backward_minimum( mode: Union[str, ForwardMode] = ForwardMode.HYBRID, dc_decomp: bool = False, **kwargs: Any, -) -> List[List[Tensor]]: +) -> list[list[Tensor]]: """Backward LiRPA of minimum(inputs_0, inputs_1) Args: @@ -407,7 +407,7 @@ def backward_minus( b_u_out: Tensor, w_l_out: Tensor, b_l_out: Tensor, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of -x Args: @@ -430,7 +430,7 @@ def backward_scale( b_u_out: Tensor, w_l_out: Tensor, b_l_out: Tensor, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of scale_factor*x Args: @@ -453,8 +453,8 @@ def backward_scale( def backward_subtract( - inputs_0: List[Tensor], - inputs_1: List[Tensor], + inputs_0: list[Tensor], + inputs_1: list[Tensor], w_u_out: Tensor, b_u_out: Tensor, w_l_out: Tensor, @@ -462,7 +462,7 @@ def backward_subtract( perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, dc_decomp: bool = False, -) -> List[List[Tensor]]: +) -> list[list[Tensor]]: """Backward LiRPA of inputs_0 - inputs_1 Args: @@ -499,8 +499,8 @@ def backward_subtract( def backward_multiply( - inputs_0: List[Tensor], - inputs_1: List[Tensor], + inputs_0: list[Tensor], + inputs_1: list[Tensor], w_u_out: Tensor, b_u_out: Tensor, w_l_out: Tensor, @@ -508,7 +508,7 @@ def backward_multiply( perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, dc_decomp: bool = False, -) -> List[List[Tensor]]: +) -> list[list[Tensor]]: """Backward LiRPA of element-wise multiply inputs_0*inputs_1 Args: @@ -582,7 +582,7 @@ def backward_multiply( def backward_sort( - inputs: List[Tensor], + inputs: list[Tensor], w_u_out: Tensor, b_u_out: Tensor, w_l_out: Tensor, @@ -591,7 +591,7 @@ def backward_sort( perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, dc_decomp: bool = False, -) -> List[Tensor]: +) -> list[Tensor]: """Backward LiRPA of sort Args: @@ -646,7 +646,7 @@ def backward_sort( return [w_u_out, b_u_out, w_l_out, b_l_out] -def get_identity_lirpa(inputs: List[Tensor]) -> List[Tensor]: +def get_identity_lirpa(inputs: list[Tensor]) -> list[Tensor]: y = inputs[-1] shape = int(np.prod(y.shape[1:])) @@ -658,7 +658,7 @@ def get_identity_lirpa(inputs: List[Tensor]) -> List[Tensor]: return [w_u_out, b_u_out, w_l_out, b_l_out] -def get_identity_lirpa_shapes(input_shapes: List[Tuple[Optional[int], ...]]) -> List[Tuple[Optional[int], ...]]: +def get_identity_lirpa_shapes(input_shapes: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: y_shape = input_shapes[-1] batch_size = y_shape[0] flatten_dim = int(np.prod(y_shape[1:])) # type: ignore diff --git a/src/decomon/core.py b/src/decomon/core.py index 52eedae5..74790f4a 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras.ops as K import numpy as np @@ -40,12 +40,12 @@ def get_lower(self, x: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: def get_nb_x_components(self) -> int: ... - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: return { "opt_option": self.opt_option, } - def get_x_input_shape_wo_batchsize(self, original_input_dim: int) -> Tuple[int, ...]: + def get_x_input_shape_wo_batchsize(self, original_input_dim: int) -> tuple[int, ...]: n_comp_x = self.get_nb_x_components() if n_comp_x == 1: return (original_input_dim,) @@ -92,7 +92,7 @@ def __init__(self, eps: float, p: float = 2, opt_option: Option = Option.milp): raise ValueError(p_error_msg) self.p = p - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: config = super().get_config() config.update( { @@ -201,20 +201,20 @@ def ibp(self) -> bool: def affine(self) -> bool: return get_affine(self.mode) - def get_kerasinputshape(self, inputsformode: List[Tensor]) -> Tuple[Optional[int], ...]: + def get_kerasinputshape(self, inputsformode: list[Tensor]) -> tuple[Optional[int], ...]: return inputsformode[-1].shape def get_kerasinputshape_from_inputshapesformode( - self, inputshapesformode: List[Tuple[Optional[int], ...]] - ) -> Tuple[Optional[int], ...]: + self, inputshapesformode: list[tuple[Optional[int], ...]] + ) -> tuple[Optional[int], ...]: return inputshapesformode[-1] def get_fullinputshapes_from_inputshapesformode( self, - inputshapesformode: List[Tuple[Optional[int], ...]], - ) -> List[Tuple[Optional[int], ...]]: + inputshapesformode: list[tuple[Optional[int], ...]], + ) -> list[tuple[Optional[int], ...]]: nb_tensors = self.nb_tensors - empty_shape: Tuple[Optional[int], ...] = tuple() + empty_shape: tuple[Optional[int], ...] = tuple() if self.dc_decomp: if self.mode == ForwardMode.HYBRID: ( @@ -276,8 +276,8 @@ def get_fullinputshapes_from_inputshapesformode( return [x_shape, u_c_shape, w_u_shape, b_u_shape, l_c_shape, w_l_shape, b_l_shape, h_shape, g_shape] def get_fullinputs_from_inputsformode( - self, inputsformode: List[Tensor], compute_ibp_from_affine: bool = True, tight: bool = True - ) -> List[Tensor]: + self, inputsformode: list[Tensor], compute_ibp_from_affine: bool = True, tight: bool = True + ) -> list[Tensor]: """ Args: @@ -347,8 +347,8 @@ def get_fullinputs_from_inputsformode( return [x, u_c, w_u, b_u, l_c, w_l, b_l, h, g] def get_fullinputs_by_type_from_inputsformode_to_merge( - self, inputsformode: List[Tensor], compute_ibp_from_affine: bool = False, tight: bool = True - ) -> List[List[Tensor]]: + self, inputsformode: list[Tensor], compute_ibp_from_affine: bool = False, tight: bool = True + ) -> list[list[Tensor]]: """ Args: @@ -419,11 +419,11 @@ def get_fullinputs_by_type_from_inputsformode_to_merge( return [inputs_x, inputs_u_c, inputs_w_u, inputs_b_u, inputs_l_c, inputs_w_l, inputs_b_l, inputs_h, inputs_g] - def split_inputsformode_to_merge(self, inputsformode: List[Any]) -> List[List[Any]]: + def split_inputsformode_to_merge(self, inputsformode: list[Any]) -> list[list[Any]]: n_comp = self.nb_tensors return [inputsformode[n_comp * i : n_comp * (i + 1)] for i in range(len(inputsformode) // n_comp)] - def extract_inputsformode_from_fullinputs(self, inputs: List[Tensor]) -> List[Tensor]: + def extract_inputsformode_from_fullinputs(self, inputs: list[Tensor]) -> list[Tensor]: x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs if self.mode == ForwardMode.HYBRID: inputsformode = [x, u_c, w_u, b_u, l_c, w_l, b_l] @@ -438,8 +438,8 @@ def extract_inputsformode_from_fullinputs(self, inputs: List[Tensor]) -> List[Te return inputsformode def extract_inputshapesformode_from_fullinputshapes( - self, inputshapes: List[Tuple[Optional[int], ...]] - ) -> List[Tuple[Optional[int], ...]]: + self, inputshapes: list[tuple[Optional[int], ...]] + ) -> list[tuple[Optional[int], ...]]: x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputshapes if self.mode == ForwardMode.HYBRID: inputshapesformode = [x, u_c, w_u, b_u, l_c, w_l, b_l] @@ -453,7 +453,7 @@ def extract_inputshapesformode_from_fullinputshapes( inputshapesformode += [h, g] return inputshapesformode - def extract_outputsformode_from_fulloutputs(self, outputs: List[Tensor]) -> List[Tensor]: + def extract_outputsformode_from_fulloutputs(self, outputs: list[Tensor]) -> list[Tensor]: return self.extract_inputsformode_from_fullinputs(outputs) @staticmethod diff --git a/src/decomon/keras_utils.py b/src/decomon/keras_utils.py index 0a6b6547..0209c0a8 100644 --- a/src/decomon/keras_utils.py +++ b/src/decomon/keras_utils.py @@ -1,4 +1,4 @@ -from typing import Any, List +from typing import Any import keras import keras.ops as K @@ -143,7 +143,7 @@ def get_weight_index_from_name(layer: Layer, weight_name: str) -> int: raise IndexError(f"The weight {weight_name} is not tracked by the layer {layer}.") -def reset_layer(new_layer: Layer, original_layer: Layer, weight_names: List[str]) -> None: +def reset_layer(new_layer: Layer, original_layer: Layer, weight_names: list[str]) -> None: """Reset some weights of a layer by using the weights of another layer. Args: @@ -199,7 +199,7 @@ def share_layer_all_weights( ) -def share_weights_and_build(original_layer: Layer, new_layer: Layer, weight_names: List[str]) -> None: +def share_weights_and_build(original_layer: Layer, new_layer: Layer, weight_names: list[str]) -> None: """Share the weights specidifed by names of an already built layer to another unbuilt layer. We assume that each weight is also an original_laer's attribute whose name is the weight name. diff --git a/src/decomon/layers/activations.py b/src/decomon/layers/activations.py index 128d449e..e773a147 100644 --- a/src/decomon/layers/activations.py +++ b/src/decomon/layers/activations.py @@ -1,5 +1,6 @@ import warnings -from typing import Any, Callable, Dict, List, Optional, Union +from collections.abc import Callable +from typing import Any, Optional, Union import keras.ops as K import numpy as np @@ -40,7 +41,7 @@ def relu( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, alpha: float = 0.0, @@ -49,7 +50,7 @@ def relu( mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """ Args: inputs: list of input tensors @@ -82,14 +83,14 @@ def relu( def linear_hull_s_shape( - inputs: List[Tensor], + inputs: list[Tensor], func: Callable[[Tensor], Tensor] = K.sigmoid, f_prime: Callable[[Tensor], Tensor] = sigmoid_prime, dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, -) -> List[Tensor]: +) -> list[Tensor]: """Computing the linear hull of s-shape functions Args: @@ -158,13 +159,13 @@ def linear_hull_s_shape( def sigmoid( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA for Sigmoid activation function . `1 / (1 + exp(-x))`. @@ -190,13 +191,13 @@ def sigmoid( def tanh( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA for Hyperbolic activation function. `tanh(x)=2*sigmoid(2*x)+1` @@ -223,13 +224,13 @@ def tanh( def hard_sigmoid( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA for Hard sigmoid activation function. Faster to compute than sigmoid activation. @@ -255,13 +256,13 @@ def hard_sigmoid( def elu( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA for Exponential linear unit. Args: @@ -286,13 +287,13 @@ def elu( def selu( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA for Scaled Exponential Linear Unit (SELU). SELU is equal to: `scale * elu(x, alpha)`, where alpha and scale @@ -323,13 +324,13 @@ def selu( def linear( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA foe Linear (i.e. identity) activation function. Args: @@ -348,13 +349,13 @@ def linear( def exponential( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA for Exponential activation function. Args: @@ -376,13 +377,13 @@ def exponential( def softplus( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA for Softplus activation function `log(exp(x) + 1)`. Args: @@ -406,13 +407,13 @@ def softplus( def softsign( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA for Softsign activation function `x / (abs(x) + 1)`. Args: @@ -438,7 +439,7 @@ def softsign( def softmax( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, @@ -446,7 +447,7 @@ def softmax( clip: bool = True, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA for Softmax activation function. Args: @@ -501,21 +502,21 @@ def softmax( def group_sort_2( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, data_format: str = "channels_last", slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: if perturbation_domain is None: perturbation_domain = BoxDomain() mode = ForwardMode(mode) raise NotImplementedError() -def deserialize(name: str) -> Callable[..., List[Tensor]]: +def deserialize(name: str) -> Callable[..., list[Tensor]]: """Get the activation from name. Args: @@ -553,7 +554,7 @@ def deserialize(name: str) -> Callable[..., List[Tensor]]: raise ValueError(f"Could not interpret activation function identifier: {name}") -def get(identifier: Any) -> Callable[..., List[Tensor]]: +def get(identifier: Any) -> Callable[..., list[Tensor]]: """Get the `identifier` activation function. Args: diff --git a/src/decomon/layers/convert.py b/src/decomon/layers/convert.py index b0d51e17..4b39d0c4 100644 --- a/src/decomon/layers/convert.py +++ b/src/decomon/layers/convert.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras from keras.layers import Activation, Input, Layer @@ -79,7 +79,7 @@ def to_decomon( def _to_decomon_wo_input_init( layer: Layer, - namespace: Dict[str, Any], + namespace: dict[str, Any], slope: Union[str, Slope] = Slope.V_SLOPE, dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, @@ -119,9 +119,9 @@ def _to_decomon_wo_input_init( def _prepare_input_tensors( layer: Layer, input_dim: int, dc_decomp: bool, perturbation_domain: PerturbationDomain, mode: ForwardMode -) -> List[keras.KerasTensor]: +) -> list[keras.KerasTensor]: original_input_shapes = get_layer_input_shape(layer) - decomon_input_shapes: List[List[Optional[int]]] = [list(input_shape[1:]) for input_shape in original_input_shapes] + decomon_input_shapes: list[list[Optional[int]]] = [list(input_shape[1:]) for input_shape in original_input_shapes] n_input = len(decomon_input_shapes) x_input_shape = perturbation_domain.get_x_input_shape_wo_batchsize(input_dim) x_input = Input(x_input_shape, dtype=layer.dtype) @@ -150,10 +150,10 @@ def _prepare_input_tensors( return flatten_input_list -SingleInputShapeType = Tuple[Optional[int], ...] +SingleInputShapeType = tuple[Optional[int], ...] -def get_layer_input_shape(layer: Layer) -> List[SingleInputShapeType]: +def get_layer_input_shape(layer: Layer) -> list[SingleInputShapeType]: """Retrieves the input shape(s) of a layer. Only applicable if the layer has exactly one input, diff --git a/src/decomon/layers/core.py b/src/decomon/layers/core.py index 9d2fb6cb..86ef87cb 100644 --- a/src/decomon/layers/core.py +++ b/src/decomon/layers/core.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional, Tuple, Type, Union +from typing import Any, Optional, Union import keras from keras.layers import Layer @@ -19,11 +19,11 @@ class DecomonLayer(ABC, Layer): """Abstract class that contains the common information of every implemented layers for Forward LiRPA""" - _trainable_weights: List[keras.Variable] + _trainable_weights: list[keras.Variable] @property @abstractmethod - def original_keras_layer_class(self) -> Type[Layer]: + def original_keras_layer_class(self) -> type[Layer]: """The keras layer class from which this class is the decomon equivalent.""" pass @@ -72,7 +72,7 @@ def ibp(self) -> bool: def affine(self) -> bool: return get_affine(self.mode) - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: config = super().get_config() config.update( { @@ -86,7 +86,7 @@ def get_config(self) -> Dict[str, Any]: ) return config - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: """ Args: input_shape @@ -99,7 +99,7 @@ def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: self.original_keras_layer_class.build(self, y_input_shape) @abstractmethod - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: """ Args: inputs @@ -109,8 +109,8 @@ def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor """ def compute_output_shape( - self, input_shape: Union[Tuple[Optional[int], ...], List[Tuple[Optional[int], ...]]] - ) -> Union[Tuple[Optional[int], ...], List[Tuple[Optional[int], ...]]]: + self, input_shape: Union[tuple[Optional[int], ...], list[tuple[Optional[int], ...]]] + ) -> Union[tuple[Optional[int], ...], list[tuple[Optional[int], ...]]]: """Compute expected output shape according to input shape Will be called by symbolic calls on Keras Tensors. @@ -139,7 +139,7 @@ def compute_output_shape( return input_shape # we know from here that we got a list of input shapes. Mypy does not. - input_shapes: List[Tuple[Optional[int], ...]] = input_shape # type: ignore + input_shapes: list[tuple[Optional[int], ...]] = input_shape # type: ignore y_shape = self.inputs_outputs_spec.get_kerasinputshape_from_inputshapesformode( input_shapes ) # input shape for the original layer @@ -203,7 +203,7 @@ def reset_layer(self, layer: Layer) -> None: reset_layer(new_layer=self, original_layer=layer, weight_names=weight_names) @property - def keras_weights_names(self) -> List[str]: + def keras_weights_names(self) -> list[str]: """Weights names of the corresponding Keras layer. Will be used to decide which weight to take from the keras layer in `reset_layer()` @@ -211,7 +211,7 @@ def keras_weights_names(self) -> List[str]: """ return [] - def join(self, bounds: List[BackendTensor]) -> List[BackendTensor]: + def join(self, bounds: list[BackendTensor]) -> list[BackendTensor]: """ Args: bounds diff --git a/src/decomon/layers/decomon_layers.py b/src/decomon/layers/decomon_layers.py index a89d5efe..d6eced0b 100644 --- a/src/decomon/layers/decomon_layers.py +++ b/src/decomon/layers/decomon_layers.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras import keras.ops as K @@ -41,7 +41,7 @@ class DecomonConv2D(DecomonLayer, Conv2D): def __init__( self, filters: int, - kernel_size: Union[int, Tuple[int, int]], + kernel_size: Union[int, tuple[int, int]], perturbation_domain: Optional[PerturbationDomain] = None, dc_decomp: bool = False, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, @@ -88,7 +88,7 @@ def __init__( if self.dc_decomp: self.input_spec += [InputSpec(min_ndim=4), InputSpec(min_ndim=4)] - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: """ Args: input_shape @@ -136,7 +136,7 @@ def share_weights(self, layer: Layer) -> None: self.kernel = layer.kernel self.bias = layer.bias - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: """computing the perturbation analysis of the operator without the activation function Args: @@ -218,10 +218,10 @@ def conv_neg(x: BackendTensor) -> BackendTensor: x_max = self.perturbation_domain.get_upper(x, w_u - w_l, b_u - b_l) mask_b = o_value - K.sign(x_max) - def step_pos(x: BackendTensor, _: List[BackendTensor]) -> Tuple[BackendTensor, List[BackendTensor]]: + def step_pos(x: BackendTensor, _: list[BackendTensor]) -> tuple[BackendTensor, list[BackendTensor]]: return conv_pos(x), [] - def step_neg(x: BackendTensor, _: List[BackendTensor]) -> Tuple[BackendTensor, List[BackendTensor]]: + def step_neg(x: BackendTensor, _: list[BackendTensor]) -> tuple[BackendTensor, list[BackendTensor]]: return conv_neg(x), [] b_u_out = conv_pos(b_u) + conv_neg(b_l) @@ -261,7 +261,7 @@ def step_neg(x: BackendTensor, _: List[BackendTensor]) -> Tuple[BackendTensor, L ) @property - def keras_weights_names(self) -> List[str]: + def keras_weights_names(self) -> list[str]: """Weights names of the corresponding Keras layer. Will be used to decide which weight to take from the keras layer in `reset_layer()` @@ -317,10 +317,10 @@ def __init__( **kwargs, ) self.input_spec = [InputSpec(min_ndim=2) for _ in range(self.nb_tensors)] - self.input_shape_build: Optional[List[Tuple[Optional[int], ...]]] = None + self.input_shape_build: Optional[list[tuple[Optional[int], ...]]] = None self.op_dot = K.dot - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: """ Args: input_shape: list of input shape @@ -418,7 +418,7 @@ def set_back_bounds(self, has_backward_bounds: bool) -> None: op = Dot(1) self.op_dot = lambda x, y: op([x, y]) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: """ Args: inputs @@ -511,7 +511,7 @@ def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor ) @property - def keras_weights_names(self) -> List[str]: + def keras_weights_names(self) -> list[str]: """Weights names of the corresponding Keras layer. Will be used to decide which weight to take from the keras layer in `reset_layer()` @@ -570,7 +570,7 @@ def __init__( self.activation = activations.get(activation) self.activation_name = activation - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: if self.finetune and self.mode in [ForwardMode.HYBRID, ForwardMode.AFFINE]: shape = input_shape[-1][1:] @@ -591,7 +591,7 @@ def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: constraint=ClipAlpha(), ) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: if self.finetune and self.mode in [ForwardMode.AFFINE, ForwardMode.HYBRID] and self.activation_name != "linear": if self.activation_name[:4] == "relu": return self.activation( @@ -707,7 +707,7 @@ def __init__( if self.dc_decomp: self.input_spec += [InputSpec(min_ndim=1), InputSpec(min_ndim=1)] - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: """ Args: self @@ -718,7 +718,7 @@ def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: """ return None - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: def op(x: BackendTensor) -> BackendTensor: return Flatten.call(self, x) @@ -808,11 +808,11 @@ def __init__( **kwargs, ) - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: super().build(input_shape) self.input_spec = [InputSpec(min_ndim=len(elem)) for elem in input_shape] - def call(self, inputs: List[BackendTensor], training: bool = False, **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], training: bool = False, **kwargs: Any) -> list[BackendTensor]: z_value = K.cast(0.0, self.dtype) if training: @@ -879,7 +879,7 @@ def call_op(x: BackendTensor, training: bool) -> BackendTensor: ) @property - def keras_weights_names(self) -> List[str]: + def keras_weights_names(self) -> list[str]: """Weights names of the corresponding Keras layer. Will be used to decide which weight to take from the keras layer in `reset_layer()` @@ -903,7 +903,7 @@ class DecomonDropout(DecomonLayer, Dropout): def __init__( self, rate: float, - noise_shape: Optional[Tuple[int, ...]] = None, + noise_shape: Optional[tuple[int, ...]] = None, seed: Optional[int] = None, perturbation_domain: Optional[PerturbationDomain] = None, dc_decomp: bool = False, @@ -926,11 +926,11 @@ def __init__( **kwargs, ) - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: super().build(input_shape) self.input_spec = [InputSpec(min_ndim=len(elem)) for elem in input_shape] - def call(self, inputs: List[BackendTensor], training: bool = False, **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], training: bool = False, **kwargs: Any) -> list[BackendTensor]: if training: raise NotImplementedError("not working during training") @@ -944,16 +944,16 @@ class DecomonInputLayer(DecomonLayer, InputLayer): original_keras_layer_class = InputLayer - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: return inputs def __init__( self, - shape: Optional[Tuple[int, ...]] = None, + shape: Optional[tuple[int, ...]] = None, batch_size: Optional[int] = None, dtype: Optional[str] = None, sparse: Optional[bool] = None, - batch_shape: Optional[Tuple[Optional[int], ...]] = None, + batch_shape: Optional[tuple[Optional[int], ...]] = None, input_tensor: Optional[keras.KerasTensor] = None, name: Optional[str] = None, perturbation_domain: Optional[PerturbationDomain] = None, diff --git a/src/decomon/layers/decomon_merge_layers.py b/src/decomon/layers/decomon_merge_layers.py index 8d925da3..8838cdb5 100644 --- a/src/decomon/layers/decomon_merge_layers.py +++ b/src/decomon/layers/decomon_merge_layers.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras.ops as K from keras.layers import ( @@ -26,7 +26,7 @@ class DecomonMerge(DecomonLayer): """Base class for Decomon layers based on Mergind Keras layers.""" - def compute_output_shape(self, input_shape: List[Tuple[Optional[int], ...]]) -> List[Tuple[Optional[int], ...]]: # type: ignore + def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: # type: ignore """Compute output shapes from input shapes. By default, we assume that all inputs will be merged into "one" (still a list of tensors though). @@ -81,7 +81,7 @@ def compute_output_shape(self, input_shape: List[Tuple[Optional[int], ...]]) -> ] return self.inputs_outputs_spec.extract_inputshapesformode_from_fullinputshapes(fulloutputshapes) - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: n_comp = self.nb_tensors input_shape_y = input_shape[n_comp - 1 :: n_comp] self.original_keras_layer_class.build(self, input_shape_y) @@ -115,7 +115,7 @@ def __init__( **kwargs, ) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: # splits the inputs ( inputs_x, @@ -186,7 +186,7 @@ def __init__( ) self.op = Lambda(lambda x: sum(x) / len(x)) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: # splits the inputs ( inputs_x, @@ -256,7 +256,7 @@ def __init__( **kwargs, ) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: if self.dc_decomp: raise NotImplementedError() @@ -305,7 +305,7 @@ def __init__( **kwargs, ) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: if self.dc_decomp: raise NotImplementedError() @@ -368,7 +368,7 @@ def __init__( **kwargs, ) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: if self.dc_decomp: raise NotImplementedError() @@ -428,7 +428,7 @@ def __init__( **kwargs, ) - def func(inputs: List[BackendTensor]) -> BackendTensor: + def func(inputs: list[BackendTensor]) -> BackendTensor: return Concatenate.call(self, inputs) self.op = func @@ -437,7 +437,7 @@ def func(inputs: List[BackendTensor]) -> BackendTensor: else: self.op_w = Concatenate(axis=self.axis + 1) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: # splits the inputs ( inputs_x, @@ -507,7 +507,7 @@ def __init__( **kwargs, ) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: if self.dc_decomp: raise NotImplementedError() @@ -547,7 +547,7 @@ class DecomonDot(DecomonMerge, Dot): def __init__( self, - axes: Union[int, Tuple[int, int]] = (-1, -1), + axes: Union[int, tuple[int, int]] = (-1, -1), perturbation_domain: Optional[PerturbationDomain] = None, dc_decomp: bool = False, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, @@ -571,7 +571,7 @@ def __init__( else: self.axes = axes - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: if self.dc_decomp: raise NotImplementedError() diff --git a/src/decomon/layers/decomon_reshape.py b/src/decomon/layers/decomon_reshape.py index f0937010..6327406e 100644 --- a/src/decomon/layers/decomon_reshape.py +++ b/src/decomon/layers/decomon_reshape.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple, Type, Union +from typing import Any, Optional, Union from keras.layers import InputSpec, Layer, Permute, Reshape from keras.src.backend import rnn @@ -17,7 +17,7 @@ class DecomonReshape(DecomonLayer, Reshape): def __init__( self, - target_shape: Tuple[int, ...], + target_shape: tuple[int, ...], perturbation_domain: Optional[PerturbationDomain] = None, dc_decomp: bool = False, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, @@ -69,7 +69,7 @@ def __init__( if self.dc_decomp: self.input_spec += [InputSpec(min_ndim=1), InputSpec(min_ndim=1)] - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: def op(x: BackendTensor) -> BackendTensor: return Reshape.call(self, x) @@ -101,7 +101,7 @@ def op(x: BackendTensor) -> BackendTensor: else: - def step_func(x: BackendTensor, _: List[BackendTensor]) -> Tuple[BackendTensor, List[BackendTensor]]: + def step_func(x: BackendTensor, _: list[BackendTensor]) -> tuple[BackendTensor, list[BackendTensor]]: return op(x), _ w_u_out = rnn(step_function=step_func, inputs=w_u, initial_states=[], unroll=False)[1] @@ -123,7 +123,7 @@ class DecomonPermute(DecomonLayer, Permute): def __init__( self, - dims: Tuple[int, ...], + dims: tuple[int, ...], perturbation_domain: Optional[PerturbationDomain] = None, dc_decomp: bool = False, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, @@ -177,7 +177,7 @@ def __init__( if self.dc_decomp: self.input_spec += [InputSpec(min_ndim=1), InputSpec(min_ndim=1)] - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: def op(x: BackendTensor) -> BackendTensor: return Permute.call(self, x) @@ -208,7 +208,7 @@ def op(x: BackendTensor) -> BackendTensor: w_l_out = op(w_l) else: - def step_func(x: BackendTensor, _: List[BackendTensor]) -> Tuple[BackendTensor, List[BackendTensor]]: + def step_func(x: BackendTensor, _: list[BackendTensor]) -> tuple[BackendTensor, list[BackendTensor]]: return op(x), _ w_u_out = rnn(step_function=step_func, inputs=w_u, initial_states=[], unroll=False)[1] diff --git a/src/decomon/layers/maxpooling.py b/src/decomon/layers/maxpooling.py index 759e0efb..c6998c20 100644 --- a/src/decomon/layers/maxpooling.py +++ b/src/decomon/layers/maxpooling.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras.ops as K import numpy as np @@ -19,15 +19,15 @@ class DecomonMaxPooling2D(DecomonLayer, MaxPooling2D): original_keras_layer_class = MaxPooling2D - pool_size: Tuple[int, int] - strides: Tuple[int, int] + pool_size: tuple[int, int] + strides: tuple[int, int] padding: str data_format: str def __init__( self, - pool_size: Union[int, Tuple[int, int]] = (2, 2), - strides: Optional[Union[int, Tuple[int, int]]] = None, + pool_size: Union[int, tuple[int, int]] = (2, 2), + strides: Optional[Union[int, tuple[int, int]]] = None, padding: str = "valid", data_format: Optional[str] = None, perturbation_domain: Optional[PerturbationDomain] = None, @@ -157,8 +157,8 @@ def conv_w_(x: BackendTensor) -> BackendTensor: def _pooling_function_fast( self, - inputs: List[BackendTensor], - ) -> List[BackendTensor]: + inputs: list[BackendTensor], + ) -> list[BackendTensor]: x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) dtype = x.dtype empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) @@ -186,8 +186,8 @@ def _pooling_function_fast( def _pooling_function_not_fast( self, - inputs: List[BackendTensor], - ) -> List[BackendTensor]: + inputs: list[BackendTensor], + ) -> list[BackendTensor]: x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode( inputs, compute_ibp_from_affine=False ) @@ -230,7 +230,7 @@ def _pooling_function_not_fast( outputs, axis=-1, dc_decomp=self.dc_decomp, mode=self.mode, perturbation_domain=self.perturbation_domain ) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: if self.fast: return self._pooling_function_fast(inputs) else: diff --git a/src/decomon/layers/utils.py b/src/decomon/layers/utils.py index b15bfa4f..7bb09d6a 100644 --- a/src/decomon/layers/utils.py +++ b/src/decomon/layers/utils.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras import keras.ops as K @@ -90,7 +90,7 @@ def __init__(self, initializer: Initializer, **kwargs: Any): super().__init__(**kwargs) self.initializer = initializer - def __call__(self, shape: Tuple[Optional[int], ...], dtype: Optional[str] = None, **kwargs: Any) -> BackendTensor: + def __call__(self, shape: tuple[Optional[int], ...], dtype: Optional[str] = None, **kwargs: Any) -> BackendTensor: w = self.initializer.__call__(shape, dtype) return K.maximum(K.cast(0.0, dtype=dtype), w) @@ -102,19 +102,19 @@ def __init__(self, initializer: Initializer, **kwargs: Any): super().__init__(**kwargs) self.initializer = initializer - def __call__(self, shape: Tuple[Optional[int], ...], dtype: Optional[str] = None, **kwargs: Any) -> BackendTensor: + def __call__(self, shape: tuple[Optional[int], ...], dtype: Optional[str] = None, **kwargs: Any) -> BackendTensor: w = self.initializer.__call__(shape, dtype) return K.minimum(K.cast(0.0, dtype=dtype), w) def softplus_( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: if perturbation_domain is None: perturbation_domain = BoxDomain() mode = ForwardMode(mode) @@ -147,13 +147,13 @@ def softplus_( def sum( - inputs: List[Tensor], + inputs: list[Tensor], axis: int = -1, dc_decomp: bool = False, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, perturbation_domain: Optional[PerturbationDomain] = None, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: if perturbation_domain is None: perturbation_domain = BoxDomain() mode = ForwardMode(mode) @@ -196,12 +196,12 @@ def sum( def frac_pos( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: if perturbation_domain is None: perturbation_domain = BoxDomain() @@ -244,14 +244,14 @@ def frac_pos( # convex hull of the maximum between two functions def max_( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, axis: int = -1, finetune: bool = False, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of max(x, axis) Args: @@ -396,7 +396,7 @@ def max_( ) -def softmax_to_linear(model: keras.Model) -> Tuple[keras.Model, bool]: +def softmax_to_linear(model: keras.Model) -> tuple[keras.Model, bool]: """linearize the softmax layer for verification Args: @@ -422,18 +422,18 @@ def softmax_to_linear(model: keras.Model) -> Tuple[keras.Model, bool]: return model, False -def linear_to_softmax(model: keras.Model) -> Tuple[keras.Model, bool]: +def linear_to_softmax(model: keras.Model) -> tuple[keras.Model, bool]: model.layers[-1].activation = keras.activations.get("softmax") return model def multiply( - inputs_0: List[Tensor], - inputs_1: List[Tensor], + inputs_0: list[Tensor], + inputs_1: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of (element-wise) multiply(x,y)=-x*y. Args: @@ -522,13 +522,13 @@ def multiply( def permute_dimensions( - inputs: List[Tensor], + inputs: list[Tensor], axis: int, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, axis_perm: int = 1, dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of (element-wise) permute(x,axis) Args: @@ -588,13 +588,13 @@ def permute_dimensions( def broadcast( - inputs: List[Tensor], + inputs: list[Tensor], n: int, axis: int, mode: Union[str, ForwardMode], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of broadcasting Args: @@ -643,12 +643,12 @@ def broadcast( def split( - inputs: List[Tensor], + inputs: list[Tensor], axis: int = -1, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, -) -> List[List[Tensor]]: +) -> list[list[Tensor]]: """LiRPA implementation of split Args: @@ -718,12 +718,12 @@ def split( def sort( - inputs: List[Tensor], + inputs: list[Tensor], axis: int = -1, dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of sort by selection Args: @@ -856,11 +856,11 @@ def sort( def pow( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of pow(x )=x**2 Args: @@ -879,11 +879,11 @@ def pow( def abs( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of |x| Args: @@ -914,11 +914,11 @@ def abs( def frac_pos_hull( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of 1/x for x>0 Args: @@ -953,14 +953,14 @@ def frac_pos_hull( # convex hull for min def min_( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, axis: int = -1, finetune: bool = False, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of min(x, axis=axis) Args: @@ -992,13 +992,13 @@ def min_( def expand_dims( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, axis: int = -1, perturbation_domain: Optional[PerturbationDomain] = None, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: if perturbation_domain is None: perturbation_domain = BoxDomain() mode = ForwardMode(mode) @@ -1044,12 +1044,12 @@ def expand_dims( def log( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Exponential activation function. Args: @@ -1108,13 +1108,13 @@ def log( def exp( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Exponential activation function. Args: diff --git a/src/decomon/layers/utils_pooling.py b/src/decomon/layers/utils_pooling.py index c1e7cf9b..4b7ecefd 100644 --- a/src/decomon/layers/utils_pooling.py +++ b/src/decomon/layers/utils_pooling.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Optional, Union import keras.ops as K import numpy as np @@ -18,13 +18,13 @@ def get_upper_linear_hull_max( - inputs: List[Tensor], + inputs: list[Tensor], mode: Union[str, ForwardMode] = ForwardMode.HYBRID, perturbation_domain: Optional[PerturbationDomain] = None, axis: int = -1, dc_decomp: bool = False, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Compute the linear hull that overapproximates max along the axis dimension Args: @@ -135,14 +135,14 @@ def get_upper_linear_hull_max( def get_lower_linear_hull_max( - inputs: List[Tensor], + inputs: list[Tensor], mode: Union[str, ForwardMode] = ForwardMode.HYBRID, perturbation_domain: Optional[PerturbationDomain] = None, axis: int = -1, finetune_lower: Optional[BackendTensor] = None, dc_decomp: bool = False, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Compute the linear hull that overapproximates max along the axis dimension Args: diff --git a/src/decomon/metrics/loss.py b/src/decomon/metrics/loss.py index aa4d00e6..fce6797d 100644 --- a/src/decomon/metrics/loss.py +++ b/src/decomon/metrics/loss.py @@ -1,4 +1,5 @@ -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from collections.abc import Callable +from typing import Any, Optional, Union import keras import keras.ops as K @@ -33,7 +34,7 @@ def get_model(model: DecomonModel) -> DecomonModel: if mode == ForwardMode.IBP: - def func(outputs: List[Tensor]) -> Tensor: + def func(outputs: list[Tensor]) -> Tensor: u_c, l_c = outputs return K.concatenate([K.expand_dims(u_c, -1), K.expand_dims(l_c, -1)], -1) @@ -41,7 +42,7 @@ def func(outputs: List[Tensor]) -> Tensor: elif mode == ForwardMode.AFFINE: - def func(outputs: List[Tensor]) -> Tensor: + def func(outputs: list[Tensor]) -> Tensor: x_0, w_u, b_u, w_l, b_l = outputs if len(x_0.shape) == 2: x_0_reshaped = x_0[:, :, None] @@ -60,7 +61,7 @@ def func(outputs: List[Tensor]) -> Tensor: elif mode == ForwardMode.HYBRID: - def func(outputs: List[Tensor]) -> Tensor: + def func(outputs: list[Tensor]) -> Tensor: x_0, u_c, w_u, b_u, l_c, w_l, b_l = outputs if len(x_0.shape) == 2: @@ -384,7 +385,7 @@ def __init__( self.asymptotic = asymptotic self.backward = backward - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: config = super().get_config() config.update( { @@ -394,7 +395,7 @@ def get_config(self) -> Dict[str, Any]: ) return config - def call_no_backward(self, inputs: List[BackendTensor], **kwargs: Any) -> BackendTensor: + def call_no_backward(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: if not self.asymptotic: u_c, l_c = self.convert2mode_layer(inputs) @@ -423,7 +424,7 @@ def adv_ibp(u_c: BackendTensor, l_c: BackendTensor, y_tensor: BackendTensor) -> score, K.cast(-1, dtype=score.dtype) ) # + 1e-3*K.maximum(K.max(K.abs(u_c), -1)[:,None], K.abs(l_c)) - def call_backward(self, inputs: List[BackendTensor], **kwargs: Any) -> BackendTensor: + def call_backward(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: if not self.asymptotic: u_c, l_c = self.convert2mode_layer(inputs) return K.softmax(u_c) @@ -431,16 +432,16 @@ def call_backward(self, inputs: List[BackendTensor], **kwargs: Any) -> BackendTe else: raise NotImplementedError() - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> BackendTensor: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: if self.backward: return self.call_backward(inputs, **kwargs) else: return self.call_no_backward(inputs, **kwargs) - def compute_output_shape(self, input_shape: List[Tuple[Optional[int], ...]]) -> Tuple[Optional[int], ...]: # type: ignore + def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> tuple[Optional[int], ...]: # type: ignore return input_shape[-1] - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: return None @@ -477,7 +478,7 @@ def __init__( self.backward = backward - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: config = super().get_config() config.update( { @@ -486,7 +487,7 @@ def get_config(self) -> Dict[str, Any]: ) return config - def call_no_backward(self, inputs: List[BackendTensor], **kwargs: Any) -> BackendTensor: + def call_no_backward(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: if self.mode == ForwardMode.HYBRID: x, _, w_u, b_u, _, w_l, b_l = inputs else: @@ -522,7 +523,7 @@ def radius_label(y_tensor: BackendTensor, backward: bool = False) -> BackendTens return K.concatenate([radius_label(source_tensor[:, i]) for i in range(shape)], -1) - def call_backward(self, inputs: List[BackendTensor], **kwargs: Any) -> BackendTensor: + def call_backward(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: if self.mode == ForwardMode.HYBRID: x, _, w_u, b_u, _, w_l, b_l = inputs else: @@ -552,16 +553,16 @@ def radius_label(y_tensor: BackendTensor) -> BackendTensor: return K.concatenate([radius_label(source_tensor[:, i]) for i in range(shape)], -1) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> BackendTensor: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: if self.backward: return self.call_backward(inputs, **kwargs) else: return self.call_no_backward(inputs, **kwargs) - def compute_output_shape(self, input_shape: Union[List[Tuple[Optional[int], ...]],]) -> Tuple[Optional[int], ...]: # type: ignore + def compute_output_shape(self, input_shape: Union[list[tuple[Optional[int], ...]],]) -> tuple[Optional[int], ...]: # type: ignore return input_shape[-1] - def build(self, input_shape: List[Tuple[Optional[int], ...]]) -> None: + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: return None diff --git a/src/decomon/metrics/metric.py b/src/decomon/metrics/metric.py index 729eb9fa..45b600d7 100644 --- a/src/decomon/metrics/metric.py +++ b/src/decomon/metrics/metric.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import Any, Dict, List, Optional, Union +from typing import Any, Optional, Union import keras import keras.ops as K @@ -49,7 +49,7 @@ def __init__( else: self.perturbation_domain = perturbation_domain - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: config = super().get_config() config.update( { @@ -62,7 +62,7 @@ def get_config(self) -> Dict[str, Any]: return config @abstractmethod - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> BackendTensor: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: """ Args: inputs @@ -114,7 +114,7 @@ def linear_adv( return K.max(adv_score, -1) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> BackendTensor: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: """ Args: inputs @@ -186,7 +186,7 @@ def __init__( """ super().__init__(ibp=ibp, affine=affine, mode=mode, perturbation_domain=perturbation_domain, **kwargs) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> BackendTensor: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: """ Args: inputs @@ -318,7 +318,7 @@ def linear_upper(self, z_tensor: Tensor, y_tensor: Tensor, w_u: Tensor, b_u: Ten return K.sum(upper_score, -1) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> BackendTensor: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: """ Args: inputs diff --git a/src/decomon/metrics/utils.py b/src/decomon/metrics/utils.py index 90f3c363..3cc9a7e5 100644 --- a/src/decomon/metrics/utils.py +++ b/src/decomon/metrics/utils.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Optional, Union from decomon.core import BoxDomain, ForwardMode, PerturbationDomain from decomon.layers.utils import exp, expand_dims, log, sum @@ -9,11 +9,11 @@ def categorical_cross_entropy( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, perturbation_domain: Optional[PerturbationDomain] = None, -) -> List[Tensor]: +) -> list[Tensor]: # step 1: exponential if perturbation_domain is None: perturbation_domain = BoxDomain() diff --git a/src/decomon/models/backward_cloning.py b/src/decomon/models/backward_cloning.py index bfc570e0..89137c77 100644 --- a/src/decomon/models/backward_cloning.py +++ b/src/decomon/models/backward_cloning.py @@ -1,5 +1,6 @@ +from collections.abc import Callable from copy import deepcopy -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras import keras.ops as K @@ -46,7 +47,7 @@ def get_disconnected_input( if dtype is None: dtype = floatx() - def disco_priv(inputs: List[Tensor]) -> List[Tensor]: + def disco_priv(inputs: list[Tensor]) -> list[Tensor]: x, u_c, w_f_u, b_f_u, l_c, w_f_l, b_f_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) dtype = x.dtype empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) @@ -66,7 +67,7 @@ def disco_priv(inputs: List[Tensor]) -> List[Tensor]: def retrieve_layer( node: Node, layer_fn: Callable[[Layer], BackwardLayer], - backward_map: Dict[int, BackwardLayer], + backward_map: dict[int, BackwardLayer], joint: bool = True, ) -> BackwardLayer: if id(node) in backward_map: @@ -83,17 +84,17 @@ def crown_( ibp: bool, affine: bool, perturbation_domain: PerturbationDomain, - input_map: Dict[int, List[keras.KerasTensor]], + input_map: dict[int, list[keras.KerasTensor]], layer_fn: Callable[[Layer], BackwardLayer], - backward_bounds: List[keras.KerasTensor], - backward_map: Optional[Dict[int, BackwardLayer]] = None, + backward_bounds: list[keras.KerasTensor], + backward_map: Optional[dict[int, BackwardLayer]] = None, joint: bool = True, fuse: bool = True, - output_map: Optional[Dict[int, List[keras.KerasTensor]]] = None, + output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, merge_layers: Optional[Layer] = None, fuse_layer: Optional[Layer] = None, **kwargs: Any, -) -> Tuple[List[keras.KerasTensor], Optional[Layer]]: +) -> tuple[list[keras.KerasTensor], Optional[Layer]]: """ @@ -216,25 +217,25 @@ def crown_( def get_input_nodes( model: Model, - dico_nodes: Dict[int, List[Node]], + dico_nodes: dict[int, list[Node]], ibp: bool, affine: bool, - input_tensors: List[keras.KerasTensor], + input_tensors: list[keras.KerasTensor], output_map: OutputMapDict, layer_fn: Callable[[Layer], BackwardLayer], joint: bool, set_mode_layer: Layer, perturbation_domain: Optional[PerturbationDomain] = None, **kwargs: Any, -) -> Tuple[Dict[int, List[keras.KerasTensor]], Dict[int, BackwardLayer], Dict[int, List[keras.KerasTensor]]]: +) -> tuple[dict[int, list[keras.KerasTensor]], dict[int, BackwardLayer], dict[int, list[keras.KerasTensor]]]: keys = [e for e in dico_nodes.keys()] keys.sort(reverse=True) fuse_layer = None - input_map: Dict[int, List[keras.KerasTensor]] = {} - backward_map: Dict[int, BackwardLayer] = {} + input_map: dict[int, list[keras.KerasTensor]] = {} + backward_map: dict[int, BackwardLayer] = {} if perturbation_domain is None: perturbation_domain = BoxDomain() - crown_map: Dict[int, List[keras.KerasTensor]] = {} + crown_map: dict[int, list[keras.KerasTensor]] = {} for depth in keys: nodes = dico_nodes[depth] for node in nodes: @@ -246,7 +247,7 @@ def get_input_nodes( # import pdb; pdb.set_trace() input_map[id(node)] = input_tensors else: - output: List[keras.KerasTensor] = [] + output: list[keras.KerasTensor] = [] for parent in parents: # do something if id(parent) in output_map.keys(): @@ -283,8 +284,8 @@ def get_input_nodes( def crown_model( model: Model, - input_tensors: List[keras.KerasTensor], - back_bounds: Optional[List[keras.KerasTensor]] = None, + input_tensors: list[keras.KerasTensor], + back_bounds: Optional[list[keras.KerasTensor]] = None, slope: Union[str, Slope] = Slope.V_SLOPE, ibp: bool = True, affine: bool = True, @@ -296,7 +297,7 @@ def crown_model( layer_fn: Callable[..., BackwardLayer] = to_backward, fuse: bool = True, **kwargs: Any, -) -> Tuple[List[keras.KerasTensor], List[keras.KerasTensor], Dict[int, BackwardLayer], None]: +) -> tuple[list[keras.KerasTensor], list[keras.KerasTensor], dict[int, BackwardLayer], None]: if back_bounds is None: back_bounds = [] if forward_map is None: @@ -400,8 +401,8 @@ def func(layer: Layer) -> Layer: def convert_backward( model: Model, - input_tensors: List[keras.KerasTensor], - back_bounds: Optional[List[keras.KerasTensor]] = None, + input_tensors: list[keras.KerasTensor], + back_bounds: Optional[list[keras.KerasTensor]] = None, slope: Union[str, Slope] = Slope.V_SLOPE, ibp: bool = True, affine: bool = True, @@ -415,7 +416,7 @@ def convert_backward( final_affine: bool = False, input_dim: int = -1, **kwargs: Any, -) -> Tuple[List[keras.KerasTensor], List[keras.KerasTensor], Dict[int, BackwardLayer], None]: +) -> tuple[list[keras.KerasTensor], list[keras.KerasTensor], dict[int, BackwardLayer], None]: model = ensure_functional_model(model) if input_dim == -1: input_dim = get_input_dim(model) diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index 1d5a25b3..2fc66445 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -1,4 +1,5 @@ -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from collections.abc import Callable +from typing import Any, Optional, Union import keras from keras.layers import InputLayer, Layer @@ -31,7 +32,7 @@ ) -def _clone_keras_model(model: Model, layer_fn: Callable[[Layer], List[Layer]]) -> Model: +def _clone_keras_model(model: Model, layer_fn: Callable[[Layer], list[Layer]]) -> Model: if model.inputs is None: raise ValueError("model.inputs must be not None. You should call the model on a batch of data.") @@ -79,11 +80,11 @@ def preprocess_keras_model( # create status def convert( model: Model, - input_tensors: List[keras.KerasTensor], + input_tensors: list[keras.KerasTensor], method: Union[str, ConvertMethod] = ConvertMethod.CROWN, ibp: bool = False, affine: bool = False, - back_bounds: Optional[List[keras.KerasTensor]] = None, + back_bounds: Optional[list[keras.KerasTensor]] = None, layer_fn: Callable[..., Layer] = to_decomon, slope: Union[str, Slope] = Slope.V_SLOPE, input_dim: int = -1, @@ -97,10 +98,10 @@ def convert( final_ibp: bool = False, final_affine: bool = False, **kwargs: Any, -) -> Tuple[ - List[keras.KerasTensor], - List[keras.KerasTensor], - Union[LayerMapDict, Dict[int, BackwardLayer]], +) -> tuple[ + list[keras.KerasTensor], + list[keras.KerasTensor], + Union[LayerMapDict, dict[int, BackwardLayer]], Optional[OutputMapDict], ]: if back_bounds is None: @@ -119,7 +120,7 @@ def convert( # prepare the Keras Model: split non-linear activation functions into separate Activation layers model = preprocess_keras_model(model) - layer_map: Union[LayerMapDict, Dict[int, BackwardLayer]] + layer_map: Union[LayerMapDict, dict[int, BackwardLayer]] if method != ConvertMethod.CROWN: input_tensors, output, layer_map, forward_map = convert_forward( @@ -172,12 +173,12 @@ def clone( slope: Union[str, Slope] = Slope.V_SLOPE, perturbation_domain: Optional[PerturbationDomain] = None, method: Union[str, ConvertMethod] = ConvertMethod.CROWN, - back_bounds: Optional[List[keras.KerasTensor]] = None, + back_bounds: Optional[list[keras.KerasTensor]] = None, finetune: bool = False, shared: bool = True, finetune_forward: bool = False, finetune_backward: bool = False, - extra_inputs: Optional[List[keras.KerasTensor]] = None, + extra_inputs: Optional[list[keras.KerasTensor]] = None, to_keras: bool = True, final_ibp: Optional[bool] = None, final_affine: Optional[bool] = None, diff --git a/src/decomon/models/crown.py b/src/decomon/models/crown.py index 2ee4a49c..6abbd7df 100644 --- a/src/decomon/models/crown.py +++ b/src/decomon/models/crown.py @@ -1,5 +1,5 @@ # extra layers necessary for backward LiRPA -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras.ops as K from keras.layers import InputSpec, Layer @@ -15,7 +15,7 @@ def __init__(self, mode: Union[str, ForwardMode], **kwargs: Any): super().__init__(**kwargs) self.mode = ForwardMode(mode) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: inputs_wo_backward_bounds = inputs[:-4] backward_bounds = inputs[-4:] @@ -28,7 +28,7 @@ def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor return merge_with_previous([w_f_u, b_f_u, w_f_l, b_f_l] + backward_bounds) - def compute_output_shape(self, input_shape: List[Tuple[Optional[int], ...]]) -> List[Tuple[Optional[int], ...]]: + def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: inputs_wo_backward_bounds_shapes = input_shape[:-4] backward_bounds_shapes = input_shape[-4:] @@ -51,7 +51,7 @@ def compute_output_shape(self, input_shape: List[Tuple[Optional[int], ...]]) -> [w_f_u_shape, b_f_u_shape, w_f_l_shape, b_f_l_shape] + backward_bounds_shapes ) - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: config = super().get_config() config.update({"mode": self.mode}) return config @@ -63,7 +63,7 @@ def __init__(self, mode: Union[str, ForwardMode], perturbation_domain: Perturbat self.mode = ForwardMode(mode) self.perturbation_domain = perturbation_domain - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: inputs_wo_backward_bounds = inputs[:-4] backward_bounds = inputs[-4:] w_u_out, b_u_out, w_l_out, b_l_out = backward_bounds @@ -91,7 +91,7 @@ def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor else: raise ValueError(f"Unknwon mode {self.mode}") - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: config = super().get_config() config.update({"mode": self.mode, "perturbation_domain": self.perturbation_domain}) return config @@ -100,8 +100,8 @@ def get_config(self) -> Dict[str, Any]: class MergeWithPrevious(Layer): def __init__( self, - input_shape_layer: Optional[Tuple[int, ...]] = None, - backward_shape_layer: Optional[Tuple[int, ...]] = None, + input_shape_layer: Optional[tuple[int, ...]] = None, + backward_shape_layer: Optional[tuple[int, ...]] = None, **kwargs: Any, ): super().__init__(**kwargs) @@ -117,13 +117,13 @@ def __init__( b_b_spec = InputSpec(ndim=2, axes={-1: n_out}) self.input_spec = [w_out_spec, b_out_spec] * 2 + [w_b_spec, b_b_spec] * 2 # - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: return merge_with_previous(inputs) - def compute_output_shape(self, input_shape: List[Tuple[Optional[int], ...]]) -> List[Tuple[Optional[int], ...]]: + def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: return merge_with_previous_compute_output_shape(input_shape) - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: config = super().get_config() config.update( { @@ -135,8 +135,8 @@ def get_config(self) -> Dict[str, Any]: def merge_with_previous_compute_output_shape( - input_shapes: List[Tuple[Optional[int], ...]] -) -> List[Tuple[Optional[int], ...]]: + input_shapes: list[tuple[Optional[int], ...]] +) -> list[tuple[Optional[int], ...]]: w_b_in_shape, b_b_in_shape = input_shapes[-2:] w_out_in_shape, b_out_in_shape = input_shapes[:2] batch_size, flattened_keras_input_shape, flattened_keras_output_shape = w_out_in_shape @@ -146,7 +146,7 @@ def merge_with_previous_compute_output_shape( return [w_b_out_shape, b_b_out_shape] * 2 -def merge_with_previous(inputs: List[BackendTensor]) -> List[BackendTensor]: +def merge_with_previous(inputs: list[BackendTensor]) -> list[BackendTensor]: w_u_out, b_u_out, w_l_out, b_l_out, w_b_u, b_b_u, w_b_l, b_b_l = inputs # w_u_out (None, n_h_in, n_h_out) diff --git a/src/decomon/models/forward_cloning.py b/src/decomon/models/forward_cloning.py index e2d8e7af..2c12946d 100644 --- a/src/decomon/models/forward_cloning.py +++ b/src/decomon/models/forward_cloning.py @@ -4,8 +4,9 @@ """ import inspect +from collections.abc import Callable from copy import deepcopy -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras from keras.layers import InputLayer, Layer @@ -26,11 +27,11 @@ ) OutputMapKey = Union[str, int] -OutputMapVal = Union[List[keras.KerasTensor], "OutputMapDict"] -OutputMapDict = Dict[OutputMapKey, OutputMapVal] +OutputMapVal = Union[list[keras.KerasTensor], "OutputMapDict"] +OutputMapDict = dict[OutputMapKey, OutputMapVal] -LayerMapVal = Union[List[DecomonLayer], "LayerMapDict"] -LayerMapDict = Dict[int, LayerMapVal] +LayerMapVal = Union[list[DecomonLayer], "LayerMapDict"] +LayerMapDict = dict[int, LayerMapVal] def include_dim_layer_fn( @@ -43,7 +44,7 @@ def include_dim_layer_fn( affine: bool = True, finetune: bool = False, shared: bool = True, -) -> Callable[[Layer], List[Layer]]: +) -> Callable[[Layer], list[Layer]]: """include external parameters inside the translation of a layer to its decomon counterpart Args: @@ -66,7 +67,7 @@ def include_dim_layer_fn( if "input_dim" in inspect.signature(layer_fn).parameters: - def func(layer: Layer) -> List[Layer]: + def func(layer: Layer) -> list[Layer]: return [ layer_fn_copy( layer, @@ -83,7 +84,7 @@ def func(layer: Layer) -> List[Layer]: else: - def func(layer: Layer) -> List[Layer]: + def func(layer: Layer) -> list[Layer]: return [layer_fn_copy(layer)] return func @@ -91,7 +92,7 @@ def func(layer: Layer) -> List[Layer]: def convert_forward( model: Model, - input_tensors: List[keras.KerasTensor], + input_tensors: list[keras.KerasTensor], layer_fn: Callable[..., Layer] = to_decomon, slope: Union[str, Slope] = Slope.V_SLOPE, input_dim: int = -1, @@ -103,7 +104,7 @@ def convert_forward( shared: bool = True, softmax_to_linear: bool = True, **kwargs: Any, -) -> Tuple[List[keras.KerasTensor], List[keras.KerasTensor], LayerMapDict, OutputMapDict]: +) -> tuple[list[keras.KerasTensor], list[keras.KerasTensor], LayerMapDict, OutputMapDict]: if perturbation_domain is None: perturbation_domain = BoxDomain() @@ -138,14 +139,14 @@ def convert_forward( def convert_forward_functional_model( model: Model, - layer_fn: Callable[[Layer], List[Layer]], - input_tensors: List[keras.KerasTensor], + layer_fn: Callable[[Layer], list[Layer]], + input_tensors: list[keras.KerasTensor], softmax_to_linear: bool = True, count: int = 0, output_map: Optional[OutputMapDict] = None, layer_map: Optional[LayerMapDict] = None, - layer2layer_map: Optional[Dict[int, List[Layer]]] = None, -) -> Tuple[List[keras.KerasTensor], List[keras.KerasTensor], LayerMapDict, OutputMapDict, Dict[int, List[Layer]]]: + layer2layer_map: Optional[dict[int, list[Layer]]] = None, +) -> tuple[list[keras.KerasTensor], list[keras.KerasTensor], LayerMapDict, OutputMapDict, dict[int, list[Layer]]]: if softmax_to_linear: model, has_softmax = softmax_2_linear(model) @@ -163,7 +164,7 @@ def convert_forward_functional_model( layer_map = {} if layer2layer_map is None: layer2layer_map = {} - output: List[keras.KerasTensor] = input_tensors + output: list[keras.KerasTensor] = input_tensors for depth in keys: nodes = dico_nodes[depth] for node in nodes: diff --git a/src/decomon/models/models.py b/src/decomon/models/models.py index b1ff067c..572ac072 100644 --- a/src/decomon/models/models.py +++ b/src/decomon/models/models.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Optional, Union import keras import keras.ops as K @@ -19,8 +19,8 @@ class DecomonModel(keras.Model): def __init__( self, - inputs: Union[keras.KerasTensor, List[keras.KerasTensor]], - outputs: Union[keras.KerasTensor, List[keras.KerasTensor]], + inputs: Union[keras.KerasTensor, list[keras.KerasTensor]], + outputs: Union[keras.KerasTensor, list[keras.KerasTensor]], perturbation_domain: Optional[PerturbationDomain] = None, dc_decomp: bool = False, method: Union[str, ConvertMethod] = ConvertMethod.FORWARD_AFFINE, @@ -43,7 +43,7 @@ def __init__( self.backward_bounds = backward_bounds self.shared = shared - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: # force having functional config which is skipped by default # because DecomonModel.__init__() has not same signature as Functional.__init__() config = Model(self.inputs, self.outputs).get_config() @@ -96,8 +96,8 @@ def reset_finetuning(self) -> None: layer.reset_finetuning() def predict_on_single_batch_np( - self, inputs: Union[np.ndarray, List[np.ndarray]] - ) -> Union[np.ndarray, List[np.ndarray]]: + self, inputs: Union[np.ndarray, list[np.ndarray]] + ) -> Union[np.ndarray, list[np.ndarray]]: """Make predictions on numpy arrays fitting in one batch Avoid using `self.predict()` known to be not designed for small arrays, @@ -128,8 +128,8 @@ def _check_domain( return perturbation_domain -def get_AB(model: DecomonModel) -> Dict[str, List[keras.Variable]]: - dico_AB: Dict[str, List[keras.Variable]] = {} +def get_AB(model: DecomonModel) -> dict[str, list[keras.Variable]]: + dico_AB: dict[str, list[keras.Variable]] = {} perturbation_domain = model.perturbation_domain if not (isinstance(perturbation_domain, GridDomain) and perturbation_domain.opt_option == Option.milp): return dico_AB @@ -144,8 +144,8 @@ def get_AB(model: DecomonModel) -> Dict[str, List[keras.Variable]]: return dico_AB -def get_AB_finetune(model: DecomonModel) -> Dict[str, keras.Variable]: - dico_AB: Dict[str, keras.Variable] = {} +def get_AB_finetune(model: DecomonModel) -> dict[str, keras.Variable]: + dico_AB: dict[str, keras.Variable] = {} perturbation_domain = model.perturbation_domain if not (isinstance(perturbation_domain, GridDomain) and perturbation_domain.opt_option == Option.milp): return dico_AB diff --git a/src/decomon/models/utils.py b/src/decomon/models/utils.py index f2adf005..f0ecd6aa 100644 --- a/src/decomon/models/utils.py +++ b/src/decomon/models/utils.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Optional, Union import keras import keras.ops as K @@ -41,7 +41,7 @@ class ConvertMethod(str, Enum): FORWARD_HYBRID = "forward-hybrid" -def get_ibp_affine_from_method(method: Union[str, ConvertMethod]) -> Tuple[bool, bool]: +def get_ibp_affine_from_method(method: Union[str, ConvertMethod]) -> tuple[bool, bool]: method = ConvertMethod(method) if method in [ConvertMethod.FORWARD_IBP, ConvertMethod.CROWN_FORWARD_IBP]: return True, False @@ -90,7 +90,7 @@ def get_input_tensors( perturbation_domain: PerturbationDomain, ibp: bool = True, affine: bool = True, -) -> Tuple[keras.KerasTensor, List[keras.KerasTensor]]: +) -> tuple[keras.KerasTensor, list[keras.KerasTensor]]: input_dim = get_input_dim(model) mode = get_mode(ibp=ibp, affine=affine) dc_decomp = False @@ -158,8 +158,8 @@ def get_input_dim(layer: Layer) -> int: def prepare_inputs_for_layer( - inputs: Union[Tuple[keras.KerasTensor, ...], List[keras.KerasTensor], keras.KerasTensor] -) -> Union[Tuple[keras.KerasTensor, ...], List[keras.KerasTensor], keras.KerasTensor]: + inputs: Union[tuple[keras.KerasTensor, ...], list[keras.KerasTensor], keras.KerasTensor] +) -> Union[tuple[keras.KerasTensor, ...], list[keras.KerasTensor], keras.KerasTensor]: """Prepare inputs for keras/decomon layers. Some Keras layers do not like list of tensors even with one single tensor. @@ -173,8 +173,8 @@ def prepare_inputs_for_layer( def wrap_outputs_from_layer_in_list( - outputs: Union[Tuple[keras.KerasTensor, ...], List[keras.KerasTensor], keras.KerasTensor] -) -> List[keras.KerasTensor]: + outputs: Union[tuple[keras.KerasTensor, ...], list[keras.KerasTensor], keras.KerasTensor] +) -> list[keras.KerasTensor]: if not isinstance(outputs, list): if isinstance(outputs, tuple): return list(outputs) @@ -184,7 +184,7 @@ def wrap_outputs_from_layer_in_list( return outputs -def split_activation(layer: Layer) -> List[Layer]: +def split_activation(layer: Layer) -> list[Layer]: # get activation config = layer.get_config() activation = config.pop("activation", None) @@ -218,7 +218,7 @@ def split_activation(layer: Layer) -> List[Layer]: return [layer_wo_activation, activation_layer] -def preprocess_layer(layer: Layer) -> List[Layer]: +def preprocess_layer(layer: Layer) -> list[Layer]: return split_activation(layer) @@ -226,16 +226,16 @@ def is_input_node(node: Node) -> bool: return len(node.input_tensors) == 0 -def get_depth_dict(model: Model) -> Dict[int, List[Node]]: +def get_depth_dict(model: Model) -> dict[int, list[Node]]: depth_keys = list(model._nodes_by_depth.keys()) depth_keys.sort(reverse=True) nodes_list = [] - dico_depth: Dict[int, int] = {} - dico_nodes: Dict[int, List[Node]] = {} + dico_depth: dict[int, int] = {} + dico_nodes: dict[int, list[Node]] = {} - def fill_dico(node: Node, dico_depth: Optional[Dict[int, int]] = None) -> Dict[int, int]: + def fill_dico(node: Node, dico_depth: Optional[dict[int, int]] = None) -> dict[int, int]: if dico_depth is None: dico_depth = {} @@ -311,7 +311,7 @@ def __init__( model_input_dim=self.input_dim, ) - def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor]: + def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: compute_ibp_from_affine = self.mode_from == ForwardMode.AFFINE and self.mode_to != ForwardMode.AFFINE tight = self.mode_from == ForwardMode.HYBRID and self.mode_to != ForwardMode.AFFINE compute_dummy_affine = self.mode_from == ForwardMode.IBP and self.mode_to != ForwardMode.IBP @@ -331,7 +331,7 @@ def call(self, inputs: List[BackendTensor], **kwargs: Any) -> List[BackendTensor [x, u_c, w_u, b_u, l_c, w_l, b_l, h, g] ) - def compute_output_shape(self, input_shape: List[Tuple[Optional[int], ...]]) -> List[Tuple[Optional[int], ...]]: + def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: ( x_shape, u_c_shape, @@ -347,7 +347,7 @@ def compute_output_shape(self, input_shape: List[Tuple[Optional[int], ...]]) -> [x_shape, u_c_shape, w_u_shape, b_u_shape, l_c_shape, w_l_shape, b_l_shape, h_shape, g_shape] ) - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> dict[str, Any]: config = super().get_config() config.update( {"mode_from": self.mode_from, "mode_to": self.mode_to, "perturbation_domain": self.perturbation_domain} diff --git a/src/decomon/types/__init__.py b/src/decomon/types/__init__.py index 0193c1f8..8f7c1d3e 100644 --- a/src/decomon/types/__init__.py +++ b/src/decomon/types/__init__.py @@ -1,7 +1,7 @@ """Typing module""" -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Union import keras @@ -20,4 +20,4 @@ """Type for any tensor, from keras or backend.""" -DecomonInputs = List[Tensor] +DecomonInputs = list[Tensor] diff --git a/src/decomon/utils.py b/src/decomon/utils.py index 756d70ab..a4d79448 100644 --- a/src/decomon/utils.py +++ b/src/decomon/utils.py @@ -1,4 +1,5 @@ -from typing import Any, Callable, List, Optional, Tuple, Union +from collections.abc import Callable +from typing import Any, Optional, Union import keras.ops as K import numpy as np @@ -91,7 +92,7 @@ def get_bound_grid( W_l: Tensor, b_l: Tensor, n: int, -) -> Tuple[Tensor, Tensor]: +) -> tuple[Tensor, Tensor]: upper = get_upper_bound_grid(x, W_u, b_u, n) lower = get_lower_bound_grid(x, W_l, b_l, n) @@ -99,7 +100,7 @@ def get_bound_grid( # convert max Wx +b s.t Wx+b<=0 into a subset-sum problem with positive values -def convert_lower_search_2_subset_sum(x: Tensor, W: Tensor, b: Tensor, n: int) -> Tuple[Tensor, Tensor]: +def convert_lower_search_2_subset_sum(x: Tensor, W: Tensor, b: Tensor, n: int) -> tuple[Tensor, Tensor]: x_min = x[:, 0] x_max = x[:, 1] @@ -131,7 +132,7 @@ def get_linear_hull_relu( upper_g: float = 0.0, lower_g: float = 0.0, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: slope = Slope(slope) # in case upper=lower, this cases are # considered with index_dead and index_linear @@ -200,17 +201,17 @@ def get_linear_hull_relu( return [w_u, b_u, w_l, b_l] -def get_linear_hull_sigmoid(upper: Tensor, lower: Tensor, slope: Union[str, Slope], **kwargs: Any) -> List[Tensor]: +def get_linear_hull_sigmoid(upper: Tensor, lower: Tensor, slope: Union[str, Slope], **kwargs: Any) -> list[Tensor]: x = [upper, lower] return get_linear_hull_s_shape(x, func=K.sigmoid, f_prime=sigmoid_prime, mode=ForwardMode.IBP, **kwargs) -def get_linear_hull_tanh(upper: Tensor, lower: Tensor, slope: Union[str, Slope], **kwargs: Any) -> List[Tensor]: +def get_linear_hull_tanh(upper: Tensor, lower: Tensor, slope: Union[str, Slope], **kwargs: Any) -> list[Tensor]: x = [upper, lower] return get_linear_hull_s_shape(x, func=K.tanh, f_prime=tanh_prime, mode=ForwardMode.IBP, **kwargs) -def get_linear_softplus_hull(upper: Tensor, lower: Tensor, slope: Union[str, Slope], **kwargs: Any) -> List[Tensor]: +def get_linear_softplus_hull(upper: Tensor, lower: Tensor, slope: Union[str, Slope], **kwargs: Any) -> list[Tensor]: slope = Slope(slope) # in case upper=lower, this cases are # considered with index_dead and index_linear @@ -265,12 +266,12 @@ def get_linear_softplus_hull(upper: Tensor, lower: Tensor, slope: Union[str, Slo def subtract( - inputs_0: List[Tensor], - inputs_1: List[Tensor], + inputs_0: list[Tensor], + inputs_1: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of inputs_0-inputs_1 Args: @@ -292,12 +293,12 @@ def subtract( def add( - inputs_0: List[Tensor], - inputs_1: List[Tensor], + inputs_0: list[Tensor], + inputs_1: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of inputs_0+inputs_1 Args: @@ -340,13 +341,13 @@ def add( def relu_( - inputs: List[Tensor], + inputs: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, slope: Union[str, Slope] = Slope.V_SLOPE, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: if perturbation_domain is None: perturbation_domain = BoxDomain() @@ -401,12 +402,12 @@ def relu_( def minus( - inputs: List[Tensor], + inputs: list[Tensor], mode: Union[str, ForwardMode] = ForwardMode.HYBRID, dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of minus(x)=-x. Args: @@ -440,14 +441,14 @@ def minus( def maximum( - inputs_0: List[Tensor], - inputs_1: List[Tensor], + inputs_0: list[Tensor], + inputs_1: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, finetune: bool = False, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of element-wise max Args: @@ -481,14 +482,14 @@ def maximum( def minimum( - inputs_0: List[Tensor], - inputs_1: List[Tensor], + inputs_0: list[Tensor], + inputs_1: list[Tensor], dc_decomp: bool = False, perturbation_domain: Optional[PerturbationDomain] = None, mode: Union[str, ForwardMode] = ForwardMode.HYBRID, finetune: bool = False, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """LiRPA implementation of element-wise min Args: @@ -521,7 +522,7 @@ def minimum( def get_linear_hull_s_shape( - inputs: List[Tensor], + inputs: list[Tensor], func: TensorFunction = K.sigmoid, f_prime: TensorFunction = sigmoid_prime, perturbation_domain: Optional[PerturbationDomain] = None, @@ -529,7 +530,7 @@ def get_linear_hull_s_shape( slope: Union[str, Slope] = Slope.V_SLOPE, dc_decomp: bool = False, **kwargs: Any, -) -> List[Tensor]: +) -> list[Tensor]: """Computing the linear hull of shape functions given the pre activation neurons Args: @@ -612,7 +613,7 @@ def get_t_upper( s_l: Tensor, func: TensorFunction = K.sigmoid, f_prime: TensorFunction = sigmoid_prime, -) -> List[Tensor]: +) -> list[Tensor]: """linear interpolation between lower and upper bounds on the function func to have a symbolic approximation of the best coefficient for the affine upper bound @@ -670,7 +671,7 @@ def get_t_lower( s_u: Tensor, func: TensorFunction = K.sigmoid, f_prime: TensorFunction = sigmoid_prime, -) -> List[Tensor]: +) -> list[Tensor]: """linear interpolation between lower and upper bounds on the function func to have a symbolic approximation of the best coefficient for the affine lower bound diff --git a/src/decomon/wrapper.py b/src/decomon/wrapper.py index d3952e48..2a5129b8 100644 --- a/src/decomon/wrapper.py +++ b/src/decomon/wrapper.py @@ -1,4 +1,5 @@ -from typing import Callable, List, Optional, Sequence, Tuple, Union +from collections.abc import Callable, Sequence +from typing import Optional, Union import keras import numpy as np @@ -146,7 +147,7 @@ def get_adv_box( n_label = source_labels.shape[-1] # two possitible cases: the model improves the bound based on the knowledge of the labels - output: List[npt.NDArray[np.float_]] + output: list[npt.NDArray[np.float_]] if decomon_model.backward_bounds: C = np.diag([1] * n_label)[None] - source_labels[:, :, None] output = decomon_model.predict_on_single_batch_np([z, C]) # type: ignore @@ -339,7 +340,7 @@ def check_adv_box( source_labels_list = [ source_labels[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r) ] - target_labels_list: List[Optional[npt.NDArray[np.int_]]] + target_labels_list: list[Optional[npt.NDArray[np.int_]]] if ( (target_labels is not None) and (not isinstance(target_labels, int)) @@ -470,7 +471,7 @@ def get_range_box( x_max: npt.NDArray[np.float_], batch_size: int = -1, n_sub_boxes: int = 1, -) -> Tuple[npt.NDArray[np.float_], npt.NDArray[np.float_]]: +) -> tuple[npt.NDArray[np.float_], npt.NDArray[np.float_]]: """bounding the outputs of a model in a given box if the constant is negative, then it is a formal guarantee that there is no adversarial examples @@ -652,7 +653,7 @@ def get_range_noise( eps: float, p: float = np.inf, batch_size: int = -1, -) -> Tuple[npt.NDArray[np.float_], npt.NDArray[np.float_]]: +) -> tuple[npt.NDArray[np.float_], npt.NDArray[np.float_]]: """Bounds the output of a model in an Lp Ball Args: @@ -756,7 +757,7 @@ def get_range_noise( def refine_boxes( x_min: npt.NDArray[np.float_], x_max: npt.NDArray[np.float_], n_sub_boxes: int = 10 -) -> Tuple[npt.NDArray[np.float_], npt.NDArray[np.float_]]: +) -> tuple[npt.NDArray[np.float_], npt.NDArray[np.float_]]: # flatten x_min and x_max shape = list(x_min.shape[1:]) output_dim = np.prod(shape) @@ -771,7 +772,7 @@ def refine_boxes( def split( x_min: npt.NDArray[np.float_], x_max: npt.NDArray[np.float_], j: npt.NDArray[np.int_] - ) -> Tuple[npt.NDArray[np.float_], npt.NDArray[np.float_]]: + ) -> tuple[npt.NDArray[np.float_], npt.NDArray[np.float_]]: n_0 = len(x_min) n_k = x_min.shape[1] @@ -977,7 +978,7 @@ def get_adv_noise( source_labels_list = [ source_labels[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r) ] - target_labels_list: List[Optional[npt.NDArray[np.int_]]] + target_labels_list: list[Optional[npt.NDArray[np.int_]]] if ( (target_labels is not None) and (not isinstance(target_labels, int)) diff --git a/tests/conftest.py b/tests/conftest.py index 84cb3a1b..7829181b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Union +from typing import Optional, Union import keras import keras.config as keras_config @@ -155,12 +155,12 @@ def method(request): class ModelNumpyFromKerasTensors: - def __init__(self, inputs: List[KerasTensor], outputs: List[KerasTensor]): + def __init__(self, inputs: list[KerasTensor], outputs: list[KerasTensor]): self.inputs = inputs self.outputs = outputs self._model = Model(inputs, outputs) - def __call__(self, inputs_: List[np.ndarray]): + def __call__(self, inputs_: list[np.ndarray]): output_tensors = self._model(inputs_) if isinstance(output_tensors, list): return [K.convert_to_numpy(output) for output in output_tensors] @@ -200,8 +200,8 @@ def is_method_mode_compatible(method, mode): @staticmethod def predict_on_small_numpy( - model: Model, x: Union[np.ndarray, List[np.ndarray]] - ) -> Union[np.ndarray, List[np.ndarray]]: + model: Model, x: Union[np.ndarray, list[np.ndarray]] + ) -> Union[np.ndarray, list[np.ndarray]]: """Make predictions for model directly on small numpy arrays Avoid using `model.predict()` known to be not designed for small arrays, @@ -347,10 +347,10 @@ def get_standard_values_1d_box(n, dc_decomp=True, grad_bounds=False, nb=100): @staticmethod def get_inputs_for_mode_from_full_inputs( - inputs: Union[List[Tensor], List[npt.NDArray[np.float_]]], + inputs: Union[list[Tensor], list[npt.NDArray[np.float_]]], mode: Union[str, ForwardMode] = ForwardMode.HYBRID, dc_decomp: bool = True, - ) -> Union[List[Tensor], List[npt.NDArray[np.float_]]]: + ) -> Union[list[Tensor], list[npt.NDArray[np.float_]]]: """Extract from full inputs the ones corresponding to the selected mode. Args: @@ -385,7 +385,7 @@ def get_inputs_for_mode_from_full_inputs( @staticmethod def get_inputs_np_for_decomon_model_from_full_inputs( - inputs: List[npt.NDArray[np.float_]], + inputs: list[npt.NDArray[np.float_]], ) -> npt.NDArray[np.float_]: """Extract from full numpy inputs the ones for a decomon model prediction. @@ -400,8 +400,8 @@ def get_inputs_np_for_decomon_model_from_full_inputs( @staticmethod def get_input_ref_bounds_from_full_inputs( - inputs: Union[List[Tensor], List[npt.NDArray[np.float_]]], - ) -> Union[List[Tensor], List[npt.NDArray[np.float_]]]: + inputs: Union[list[Tensor], list[npt.NDArray[np.float_]]], + ) -> Union[list[Tensor], list[npt.NDArray[np.float_]]]: """Extract lower and upper bound for input ref from full inputs Args: @@ -415,9 +415,9 @@ def get_input_ref_bounds_from_full_inputs( @staticmethod def prepare_full_np_inputs_for_convert_model( - inputs: List[npt.NDArray[np.float_]], + inputs: list[npt.NDArray[np.float_]], dc_decomp: bool = True, - ) -> List[npt.NDArray[np.float_]]: + ) -> list[npt.NDArray[np.float_]]: """Prepare full numpy inputs for convert_forward or convert_backward. W_u and W_l will be idendity matrices, and b_u, b_l zeros vectors. @@ -445,10 +445,10 @@ def prepare_full_np_inputs_for_convert_model( @staticmethod def get_input_tensors_for_decomon_convert_from_full_inputs( - inputs: List[Tensor], + inputs: list[Tensor], mode: Union[str, ForwardMode] = ForwardMode.HYBRID, dc_decomp: bool = True, - ) -> List[Tensor]: + ) -> list[Tensor]: """Extract from full tensor inputs the ones for a conversion to decomon model. Args: @@ -485,7 +485,7 @@ def get_input_tensors_for_decomon_convert_from_full_inputs( @staticmethod def get_input_ref_from_full_inputs( - inputs: Union[List[Tensor], List[npt.NDArray[np.float_]]] + inputs: Union[list[Tensor], list[npt.NDArray[np.float_]]] ) -> Union[Tensor, npt.NDArray[np.float_]]: """Extract from full inputs the input of reference for the original Keras layer. @@ -527,11 +527,11 @@ def get_tensor_decomposition_1d_box(dc_decomp=True): @staticmethod def get_full_outputs_from_outputs_for_mode( - outputs_for_mode: Union[List[Tensor], List[npt.NDArray[np.float_]]], + outputs_for_mode: Union[list[Tensor], list[npt.NDArray[np.float_]]], mode: Union[str, ForwardMode] = ForwardMode.HYBRID, dc_decomp: bool = True, - full_inputs: Optional[Union[List[Tensor], List[npt.NDArray[np.float_]]]] = None, - ) -> Union[List[Tensor], List[npt.NDArray[np.float_]]]: + full_inputs: Optional[Union[list[Tensor], list[npt.NDArray[np.float_]]]] = None, + ) -> Union[list[Tensor], list[npt.NDArray[np.float_]]]: mode = ForwardMode(mode) if dc_decomp: if mode == ForwardMode.HYBRID: @@ -577,7 +577,7 @@ def get_input_dim_images_box(odd): return 6 @staticmethod - def get_input_dim_from_full_inputs(inputs: Union[List[Tensor], List[npt.NDArray[np.float_]]]) -> int: + def get_input_dim_from_full_inputs(inputs: Union[list[Tensor], list[npt.NDArray[np.float_]]]) -> int: """Get input_dim for to_decomon or to_backward from full inputs Args: From bec8cd831d45e12498de1fceec7853fd4f18bec7 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 14 Mar 2024 12:54:48 +0100 Subject: [PATCH 002/101] Remove decomon.keras_utils.LinalgSolve as it exists now as keras.ops.solve() --- src/decomon/keras_utils.py | 27 ---------------------- src/decomon/layers/utils_pooling.py | 3 +-- tests/test_keras_utils.py | 35 +---------------------------- 3 files changed, 2 insertions(+), 63 deletions(-) diff --git a/src/decomon/keras_utils.py b/src/decomon/keras_utils.py index 0209c0a8..681f359c 100644 --- a/src/decomon/keras_utils.py +++ b/src/decomon/keras_utils.py @@ -13,33 +13,6 @@ BACKEND_JAX = "jax" -class LinalgSolve(keras.Operation): - """Keras operation mimicking tensorflow.linalg.solve().""" - - def compute_output_spec(self, matrix: keras.KerasTensor, rhs: keras.KerasTensor) -> keras.KerasTensor: - rhs_shape = rhs.shape - rhs_dtype = getattr(rhs, "dtype", type(rhs)) - rhs_sparse = getattr(rhs, "sparse", False) - return keras.KerasTensor( - shape=rhs_shape, - dtype=rhs_dtype, - sparse=rhs_sparse, - ) - - def call(self, matrix: BackendTensor, rhs: BackendTensor) -> BackendTensor: - backend = keras.config.backend() - if backend == BACKEND_TENSORFLOW: - import tensorflow as tf - - return tf.linalg.solve(matrix=matrix, rhs=rhs) - elif backend == BACKEND_PYTORCH: - import torch - - return torch.linalg.solve(A=matrix, B=rhs) - else: - raise NotImplementedError(f"linalg_solve() not yet implemented for backend {backend}.") - - class BatchedIdentityLike(keras.Operation): """Keras Operation creating an identity tensor with shape (including batch_size) based on input. diff --git a/src/decomon/layers/utils_pooling.py b/src/decomon/layers/utils_pooling.py index 4b7ecefd..541d5eb6 100644 --- a/src/decomon/layers/utils_pooling.py +++ b/src/decomon/layers/utils_pooling.py @@ -10,7 +10,6 @@ PerturbationDomain, get_affine, ) -from decomon.keras_utils import LinalgSolve from decomon.types import BackendTensor, Tensor # step 1: compute (x_i, y_i) such that x_i[j]=l_j if j==i else u_j @@ -112,7 +111,7 @@ def get_upper_linear_hull_max( if dtype != dtype32: corners_collapse = K.cast(corners_collapse, dtype32) corners_pred = K.cast(corners_pred, dtype32) - w_hull = LinalgSolve()(matrix=corners_collapse, rhs=K.expand_dims(corners_pred, -1)) # (None, shape_, n_dim+1, 1) + w_hull = K.solve(corners_collapse, K.expand_dims(corners_pred, -1)) # (None, shape_, n_dim+1, 1) if dtype != dtype32: w_hull = K.cast(w_hull, dtype=dtype) diff --git a/tests/test_keras_utils.py b/tests/test_keras_utils.py index c65ad19a..6fa5aaf3 100644 --- a/tests/test_keras_utils.py +++ b/tests/test_keras_utils.py @@ -5,13 +5,7 @@ from keras.layers import Dense, Input from numpy.testing import assert_almost_equal -from decomon.keras_utils import ( - BACKEND_PYTORCH, - BACKEND_TENSORFLOW, - LinalgSolve, - get_weight_index_from_name, - share_layer_all_weights, -) +from decomon.keras_utils import get_weight_index_from_name, share_layer_all_weights def test_get_weight_index_from_name_nok_attribute(): @@ -34,33 +28,6 @@ def test_get_weight_index_from_name_ok(): assert get_weight_index_from_name(layer=layer, weight_name="bias") in [0, 1] -def test_linalgsolve(floatx, decimal): - if keras.config.backend() in (BACKEND_TENSORFLOW, BACKEND_PYTORCH) and floatx == 16: - pytest.skip("LinalgSolve not implemented for float16 on torch and tensorflow") - - dtype = f"float{floatx}" - - matrix = np.array([[1, 0, 0], [2, 1, 0], [3, 2, 1]]) - matrix = np.repeat(matrix[None, None], 2, axis=0) - matrix_symbolic_tensor = keras.KerasTensor(shape=matrix.shape, dtype=dtype) - matrix_tensor = keras.ops.convert_to_tensor(matrix, dtype=dtype) - - rhs = np.array([[1, 0], [0, 0], [0, 1]]) - rhs = np.repeat(rhs[None, None], 2, axis=0) - rhs_symbolic_tensor = keras.KerasTensor(shape=rhs.shape, dtype=dtype) - rhs_tensor = keras.ops.convert_to_tensor(rhs, dtype=dtype) - - expected_sol = np.array([[1, 0], [-2, 0], [1, 1]]) - expected_sol = np.repeat(expected_sol[None, None], 2, axis=0) - - sol_symbolic_tensor = LinalgSolve()(matrix_symbolic_tensor, rhs_symbolic_tensor) - assert tuple(sol_symbolic_tensor.shape) == tuple(expected_sol.shape) - - sol_tensor = LinalgSolve()(matrix_tensor, rhs_tensor) - assert keras.backend.standardize_dtype(sol_tensor.dtype) == dtype - assert_almost_equal(expected_sol, keras.ops.convert_to_numpy(sol_tensor), decimal=decimal) - - def test_share_layer_all_weights_nok_original_layer_unbuilt(): original_layer = Dense(3) new_layer = original_layer.__class__.from_config(original_layer.get_config()) From 02f00eefdce4454e5da4cc9869b4fa30a904c37c Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 14 Mar 2024 13:51:08 +0100 Subject: [PATCH 003/101] Update github actions versions to use node 20 See https://github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/ --- .github/workflows/build-doc.yml | 8 +++----- .github/workflows/deploy-doc.yml | 10 +++++----- .github/workflows/python-publish.yml | 8 ++++---- .github/workflows/python-tests.yml | 22 +++++++++++----------- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build-doc.yml b/.github/workflows/build-doc.yml index 050f8148..9c394eb7 100644 --- a/.github/workflows/build-doc.yml +++ b/.github/workflows/build-doc.yml @@ -76,11 +76,9 @@ jobs: echo "AUTODOC_NOTEBOOKS_BRANCH=${AUTODOC_NOTEBOOKS_BRANCH}" >> $GITHUB_ENV # check computed variables echo "Notebooks source: ${AUTODOC_NOTEBOOKS_REPO_URL}/tree/${AUTODOC_NOTEBOOKS_BRANCH}" - - uses: actions/checkout@v3 - with: - submodules: true + - uses: actions/checkout@v4 - name: Setup python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ env.python-version }} cache: "pip" @@ -119,7 +117,7 @@ jobs: ') echo doc_version=${doc_version} >> $GITHUB_OUTPUT - name: upload as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ inputs.doc-artifact-name }} path: ${{ inputs.doc-path }} diff --git a/.github/workflows/deploy-doc.yml b/.github/workflows/deploy-doc.yml index ef6b6ba8..159e0bc0 100644 --- a/.github/workflows/deploy-doc.yml +++ b/.github/workflows/deploy-doc.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest if: inputs.binder-env-fullref != '' steps: - - uses: actions/checkout@v3 # checkout triggering branch to get scripts/trigger_binder.sh + - uses: actions/checkout@v4 # checkout triggering branch to get scripts/trigger_binder.sh - name: Trigger a build for default binder env ref on each BinderHub deployments in the mybinder.org federation run: | binder_env_full_ref=${{ inputs.binder-env-fullref }} @@ -39,9 +39,9 @@ jobs: deploy-doc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ inputs.doc-artifact-name }} path: ${{ inputs.doc-path }} @@ -59,10 +59,10 @@ jobs: needs: [deploy-doc] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: gh-pages - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" - name: Generate versions.json diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index f1be731e..af5bae1e 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -14,8 +14,8 @@ jobs: outputs: package_version: ${{ steps.get_package_version.outputs.version }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.9' - name: Install dependencies @@ -33,7 +33,7 @@ jobs: echo "version=$version" echo "version=$version" >> $GITHUB_OUTPUT - name: Upload as build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist path: dist @@ -59,7 +59,7 @@ jobs: outputs: tuto-tag-name: ${{ steps.push-tuto-release-tag.outputs.new_tag_name }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: replace decomon version to install in colab notebooks run: | version=${{ needs.deploy.outputs.package_version }} diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 3f419d1d..974dd92a 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -15,18 +15,18 @@ jobs: linters: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: create requirements.txt so that pip cache with setup-python works run: echo "pre-commit" > requirements_precommit.txt - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 cache: pip cache-dependency-path: requirements_precommit.txt - name: install pre-commit run: python -m pip install pre-commit - name: get cached pre-commit hooks - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/pre-commit key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} @@ -36,8 +36,8 @@ jobs: type-checking: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: 3.9 cache: pip @@ -48,7 +48,7 @@ jobs: python -m pip install --upgrade tox setuptools - name: Restore cached .tox id: cache-tox - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: .tox key: @@ -70,8 +70,8 @@ jobs: run: shell: bash steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: pip @@ -82,7 +82,7 @@ jobs: python -m pip install --upgrade tox setuptools - name: Restore cached .tox id: cache-tox - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: .tox key: @@ -123,7 +123,7 @@ jobs: echo "coverage_exists=${coverage_exists}" >> $GITHUB_OUTPUT - name: Export coverage report (if existing) if: steps.check-coverage-report.outputs.coverage_exists == 'true' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage path: | From 4f7bd2ff10f8dde8e6624a05a07f8000e7f123e5 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 15 Mar 2024 15:07:42 +0100 Subject: [PATCH 004/101] Use tensorflow >=2.16 to avoid having reinstall keras3 --- .github/workflows/build-doc.yml | 3 --- binder/environment.yml | 6 +++--- binder/postBuild | 6 ------ docs/requirements.txt | 2 +- tox.ini | 5 +---- tutorials/tutorial1_sinus-interactive.ipynb | 7 ++----- tutorials/tutorial2_noise_sensor.ipynb | 7 ++----- tutorials/tutorial3_adversarial_attack.ipynb | 7 ++----- tutorials/tutorial4_certified_over_estimation.ipynb | 7 ++----- tutorials/z_Advanced/tensorboard-and-decomon.ipynb | 8 ++------ 10 files changed, 15 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build-doc.yml b/.github/workflows/build-doc.yml index 9c394eb7..5aa1fcf1 100644 --- a/.github/workflows/build-doc.yml +++ b/.github/workflows/build-doc.yml @@ -90,9 +90,6 @@ jobs: python -m pip install -U pip setuptools pip install . pip install -r docs/requirements.txt - # ensure having keras 3 instead of keras 2.15 installed by tensorflow - pip uninstall -y keras - pip install "keras>=3" - name: generate documentation id: sphinx-build run: | diff --git a/binder/environment.yml b/binder/environment.yml index c56e1cf6..86ef75ed 100644 --- a/binder/environment.yml +++ b/binder/environment.yml @@ -8,7 +8,7 @@ dependencies: - kaggle - pandas - scipy - - tensorboard>=2.13 -# - tensorflow>=2.15 # conflicting dependencies with decomon (keras 2) + - tensorboard>=2.16 + - tensorflow>=2.16 - jupyter-server-proxy - - .. # decomon + - .. # decomon diff --git a/binder/postBuild b/binder/postBuild index 9bbeccbe..388e5f47 100644 --- a/binder/postBuild +++ b/binder/postBuild @@ -1,9 +1,3 @@ -# install tensorflow backend separately from decomon (dependencies conflict as tf 2.15 requires keras 2) -pip install tensorflow>=2.15 -# use keras 3 instead of keras 2 installed by tensorflow -pip uninstall -y keras -pip install "keras>=3" - # tensorboard launches at startup mv binder/tensorboardserverextension.py ${NB_PYTHON_PREFIX}/lib/python*/site-packages/ # enable tensorboard extension diff --git a/docs/requirements.txt b/docs/requirements.txt index 65da0bc8..4196e804 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ sphinx myst_parser sphinx_rtd_theme -tensorflow>=2.15 # install a backend for keras 3 (and tensorflow is the one needed by default) +tensorflow>=2.16 # install a backend for keras 3 (and tensorflow is the one needed by default) diff --git a/tox.ini b/tox.ini index 514286aa..e9574275 100644 --- a/tox.ini +++ b/tox.ini @@ -15,15 +15,12 @@ deps = pytest<8 pytest-cases py39-linux-tf: pytest-cov - tf: tensorflow>=2.15 # backend for keras 3 + tf: tensorflow>=2.16 # backend for keras 3 torch: torch>=2.1.0 # backend for keras 3 setenv = tf: KERAS_BACKEND=tensorflow torch: KERAS_BACKEND=torch commands = - # be sure to get keras 3 instead of keras 2 brought back by tensorflow 2.15 - pip uninstall keras -y - pip install "keras>=3" pip list python -c 'import keras; print(keras.config.backend())' pytest -v \ diff --git a/tutorials/tutorial1_sinus-interactive.ipynb b/tutorials/tutorial1_sinus-interactive.ipynb index 024c4bb5..fbc533b4 100644 --- a/tutorials/tutorial1_sinus-interactive.ipynb +++ b/tutorials/tutorial1_sinus-interactive.ipynb @@ -56,10 +56,7 @@ " !{sys.executable} -m pip install -U pip\n", " !{sys.executable} -m pip install git+https://github.com/airbus/decomon@main#egg=decomon\n", " # install desired backend (by default tensorflow)\n", - " !{sys.executable} -m pip install \"tensorflow>=2.15\"\n", - " # ensure having keras 3 and not keras 2.15\n", - " !{sys.executable} -m pip uninstall -y keras\n", - " !{sys.executable} -m pip install \"keras>=3\"" + " !{sys.executable} -m pip install \"tensorflow>=2.16\"" ] }, { @@ -423,7 +420,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.9.18" }, "toc": { "base_numbering": 1, diff --git a/tutorials/tutorial2_noise_sensor.ipynb b/tutorials/tutorial2_noise_sensor.ipynb index e1c62b4d..0f0fbf88 100644 --- a/tutorials/tutorial2_noise_sensor.ipynb +++ b/tutorials/tutorial2_noise_sensor.ipynb @@ -60,10 +60,7 @@ " !{sys.executable} -m pip install -U pip\n", " !{sys.executable} -m pip install git+https://github.com/airbus/decomon@main#egg=decomon\n", " # install desired backend (by default tensorflow)\n", - " !{sys.executable} -m pip install \"tensorflow>=2.15\"\n", - " # ensure having keras 3 and not keras 2.15\n", - " !{sys.executable} -m pip uninstall -y keras\n", - " !{sys.executable} -m pip install \"keras>=3\"" + " !{sys.executable} -m pip install \"tensorflow>=2.16\"" ] }, { @@ -492,7 +489,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.9.18" }, "toc": { "base_numbering": 1, diff --git a/tutorials/tutorial3_adversarial_attack.ipynb b/tutorials/tutorial3_adversarial_attack.ipynb index 3c8af6ca..a5eabfeb 100644 --- a/tutorials/tutorial3_adversarial_attack.ipynb +++ b/tutorials/tutorial3_adversarial_attack.ipynb @@ -58,10 +58,7 @@ " !{sys.executable} -m pip install -U pip\n", " !{sys.executable} -m pip install git+https://github.com/airbus/decomon@main#egg=decomon\n", " # install desired backend (by default tensorflow)\n", - " !{sys.executable} -m pip install \"tensorflow>=2.15\"\n", - " # ensure having keras 3 and not keras 2.15\n", - " !{sys.executable} -m pip uninstall -y keras\n", - " !{sys.executable} -m pip install \"keras>=3\"" + " !{sys.executable} -m pip install \"tensorflow>=2.16\"" ] }, { @@ -506,7 +503,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.9.18" }, "toc": { "base_numbering": 1, diff --git a/tutorials/tutorial4_certified_over_estimation.ipynb b/tutorials/tutorial4_certified_over_estimation.ipynb index 30d3309f..b2b3eb40 100644 --- a/tutorials/tutorial4_certified_over_estimation.ipynb +++ b/tutorials/tutorial4_certified_over_estimation.ipynb @@ -67,10 +67,7 @@ " !{sys.executable} -m pip install -U pip\n", " !{sys.executable} -m pip install git+https://github.com/airbus/decomon@main#egg=decomon\n", " # install desired backend (by default tensorflow)\n", - " !{sys.executable} -m pip install \"tensorflow>=2.15\"\n", - " # ensure having keras 3 and not keras 2.15\n", - " !{sys.executable} -m pip uninstall -y keras\n", - " !{sys.executable} -m pip install \"keras>=3\"" + " !{sys.executable} -m pip install \"tensorflow>=2.16\"" ] }, { @@ -735,7 +732,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.9.18" }, "toc": { "base_numbering": 1, diff --git a/tutorials/z_Advanced/tensorboard-and-decomon.ipynb b/tutorials/z_Advanced/tensorboard-and-decomon.ipynb index ed9d4d15..fcb6cd8f 100644 --- a/tutorials/z_Advanced/tensorboard-and-decomon.ipynb +++ b/tutorials/z_Advanced/tensorboard-and-decomon.ipynb @@ -54,14 +54,10 @@ " import sys # noqa: avoid having this import removed by pycln\n", "\n", " # install dev version for dev doc, or release version for release doc\n", - " !{sys.executable} -m pip install \"tensorflow>=2.13\" \"tensorboard>=2.13\" \"keras>=2.13\"\n", " !{sys.executable} -m pip install -U pip\n", " !{sys.executable} -m pip install git+https://github.com/airbus/decomon@main#egg=decomon\n", " # install desired backend (by default tensorflow)\n", - " !{sys.executable} -m pip install \"tensorflow>=2.15\"\n", - " # ensure having keras 3 and not keras 2.15\n", - " !{sys.executable} -m pip uninstall -y keras\n", - " !{sys.executable} -m pip install \"keras>=3\"" + " !{sys.executable} -m pip install \"tensorflow>=2.16 tensorboard>=2.16\"" ] }, { @@ -356,7 +352,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.9.18" }, "toc": { "base_numbering": 1, From 8fb5abf70efe77b1353adcdcabb8d58d02991050 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 30 Jan 2024 13:47:37 +0100 Subject: [PATCH 005/101] Prepare new structure of decomon.layers This will reflect the structure of keras.layers. For instance - Dense can be found in keras.layers.core.dense, - DecomonDense will be found in decomon.layers.core.dense --- src/decomon/layers/activations.py | 581 -------- src/decomon/layers/activations/__init__.py | 0 src/decomon/layers/activations/activation.py | 0 src/decomon/layers/convert.py | 186 --- src/decomon/layers/convolutional/__init__.py | 0 src/decomon/layers/convolutional/conv2d.py | 0 src/decomon/layers/core.py | 247 ---- src/decomon/layers/core/__init__.py | 0 src/decomon/layers/core/dense.py | 0 src/decomon/layers/decomon_layers.py | 982 -------------- src/decomon/layers/decomon_merge_layers.py | 633 --------- src/decomon/layers/decomon_reshape.py | 221 ---- src/decomon/layers/layer.py | 0 src/decomon/layers/maxpooling.py | 241 ---- src/decomon/layers/merging/__init__.py | 0 src/decomon/layers/merging/add.py | 0 src/decomon/layers/merging/average.py | 0 src/decomon/layers/merging/concatenate.py | 0 src/decomon/layers/merging/dot.py | 0 src/decomon/layers/merging/maximum.py | 0 src/decomon/layers/merging/minimum.py | 0 src/decomon/layers/merging/multiply.py | 0 src/decomon/layers/merging/subtract.py | 0 src/decomon/layers/normalization/__init__.py | 0 .../normalization/batch_normalization.py | 0 src/decomon/layers/pooling/__init__.py | 0 src/decomon/layers/pooling/max_pooling2d.py | 0 src/decomon/layers/regularization/__init__.py | 0 src/decomon/layers/regularization/dropout.py | 0 src/decomon/layers/reshaping/__init__.py | 0 src/decomon/layers/reshaping/flatten.py | 0 src/decomon/layers/reshaping/permute.py | 0 src/decomon/layers/reshaping/reshape.py | 0 src/decomon/layers/utils.py | 1170 ----------------- src/decomon/layers/utils_pooling.py | 194 --- 35 files changed, 4455 deletions(-) delete mode 100644 src/decomon/layers/activations.py create mode 100644 src/decomon/layers/activations/__init__.py create mode 100644 src/decomon/layers/activations/activation.py create mode 100644 src/decomon/layers/convolutional/__init__.py create mode 100644 src/decomon/layers/convolutional/conv2d.py delete mode 100644 src/decomon/layers/core.py create mode 100644 src/decomon/layers/core/__init__.py create mode 100644 src/decomon/layers/core/dense.py delete mode 100644 src/decomon/layers/decomon_layers.py delete mode 100644 src/decomon/layers/decomon_merge_layers.py delete mode 100644 src/decomon/layers/decomon_reshape.py create mode 100644 src/decomon/layers/layer.py delete mode 100644 src/decomon/layers/maxpooling.py create mode 100644 src/decomon/layers/merging/__init__.py create mode 100644 src/decomon/layers/merging/add.py create mode 100644 src/decomon/layers/merging/average.py create mode 100644 src/decomon/layers/merging/concatenate.py create mode 100644 src/decomon/layers/merging/dot.py create mode 100644 src/decomon/layers/merging/maximum.py create mode 100644 src/decomon/layers/merging/minimum.py create mode 100644 src/decomon/layers/merging/multiply.py create mode 100644 src/decomon/layers/merging/subtract.py create mode 100644 src/decomon/layers/normalization/__init__.py create mode 100644 src/decomon/layers/normalization/batch_normalization.py create mode 100644 src/decomon/layers/pooling/__init__.py create mode 100644 src/decomon/layers/pooling/max_pooling2d.py create mode 100644 src/decomon/layers/regularization/__init__.py create mode 100644 src/decomon/layers/regularization/dropout.py create mode 100644 src/decomon/layers/reshaping/__init__.py create mode 100644 src/decomon/layers/reshaping/flatten.py create mode 100644 src/decomon/layers/reshaping/permute.py create mode 100644 src/decomon/layers/reshaping/reshape.py delete mode 100644 src/decomon/layers/utils.py delete mode 100644 src/decomon/layers/utils_pooling.py diff --git a/src/decomon/layers/activations.py b/src/decomon/layers/activations.py deleted file mode 100644 index e773a147..00000000 --- a/src/decomon/layers/activations.py +++ /dev/null @@ -1,581 +0,0 @@ -import warnings -from collections.abc import Callable -from typing import Any, Optional, Union - -import keras.ops as K -import numpy as np - -from decomon.core import ( - BoxDomain, - ForwardMode, - InputsOutputsSpec, - PerturbationDomain, - Slope, - get_affine, - get_ibp, -) -from decomon.layers.core import DecomonLayer -from decomon.layers.utils import exp, expand_dims, frac_pos, multiply, softplus_, sum -from decomon.types import Tensor -from decomon.utils import ( - get_linear_hull_s_shape, - minus, - relu_, - sigmoid_prime, - softsign_prime, - tanh_prime, -) - -ELU = "elu" -SELU = "selu" -SOFTPLUS = "softplus" -SOFTSIGN = "softsign" -SOFTMAX = "softmax" -RELU = "relu" -SIGMOID = "sigmoid" -TANH = "tanh" -EXPONENTIAL = "exponential" -HARD_SIGMOID = "hard_sigmoid" -LINEAR = "linear" -GROUP_SORT_2 = "GroupSort2" - - -def relu( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - alpha: float = 0.0, - max_value: Optional[float] = None, - threshold: float = 0.0, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - """ - Args: - inputs: list of input tensors - dc_decomp: boolean that indicates - perturbation_domain: type of convex input domain (None or dict) - alpha: see Keras official documentation - max_value: see Keras official documentation - threshold: see Keras official documentation - mode: type of Forward propagation (ibp, affine, or hybrid) - slope: - **kwargs: see Keras official documentation - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if threshold != 0: - raise NotImplementedError() - - if not alpha and max_value is None: - # default values: return relu_(x) = max(x, 0) - return relu_( - inputs, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode, slope=slope, **kwargs - ) - - raise NotImplementedError() - - -def linear_hull_s_shape( - inputs: list[Tensor], - func: Callable[[Tensor], Tensor] = K.sigmoid, - f_prime: Callable[[Tensor], Tensor] = sigmoid_prime, - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, -) -> list[Tensor]: - """Computing the linear hull of s-shape functions - - Args: - inputs: list of input tensors - func: the function (sigmoid, tanh, softsign...) - f_prime: the derivative of the function (sigmoid_prime...) - dc_decomp: boolean that indicates - perturbation_domain: type of convex input domain (None or dict) - mode: type of Forward propagation (ibp, affine, or hybrid) - slope: - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - if ibp: - u_c_out = func(u_c) - l_c_out = func(l_c) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if affine: - w_u_0, b_u_0, w_l_0, b_l_0 = get_linear_hull_s_shape( - inputs, func=func, f_prime=f_prime, perturbation_domain=perturbation_domain, mode=mode - ) - if len(w_u.shape) == len(b_u.shape): - # it happens with the convert function to spare memory footprint - n_dim = int(np.prod(w_u.shape[1:])) - M = np.reshape( - np.diag([K.cast(1, dtype=w_u_0.dtype)] * n_dim), [1, n_dim] + list(w_u.shape[1:]) - ) # usage de numpy pb pour les types - w_u_out = M * K.concatenate([K.expand_dims(w_u_0, 1)] * n_dim, 1) - w_l_out = M * K.concatenate([K.expand_dims(w_l_0, 1)] * n_dim, 1) - b_u_out = b_u_0 - b_l_out = b_l_0 - else: - w_u_out = K.expand_dims(w_u_0, 1) * w_u # pour l'instant - b_u_out = b_u_0 + w_u_0 * b_u - w_l_out = K.expand_dims(w_l_0, 1) * w_l - b_l_out = b_l_0 + w_l_0 * b_l - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if dc_decomp: - raise NotImplementedError() - else: - h_out, g_out = empty_tensor, empty_tensor - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -def sigmoid( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA for Sigmoid activation function . - `1 / (1 + exp(-x))`. - - Args: - inputs: list of input tensors - dc_decomp: boolean that indicates - perturbation_domain: type of convex input domain (None or dict) - mode: type of Forward propagation (ibp, affine, or hybrid) - **kwargs: see Keras official documentation - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - func = K.sigmoid - f_prime = sigmoid_prime - return linear_hull_s_shape( - inputs, func, f_prime, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode, slope=slope - ) - - -def tanh( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA for Hyperbolic activation function. - `tanh(x)=2*sigmoid(2*x)+1` - - Args: - inputs: list of input tensors - dc_decomp: boolean that indicates - perturbation_domain: type of convex input domain (None or dict) - mode: type of Forward propagation (ibp, affine, or hybrid) - slope: - **kwargs: see Keras official documentation - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - func = K.tanh - f_prime = tanh_prime - return linear_hull_s_shape( - inputs, func, f_prime, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode, slope=slope - ) - - -def hard_sigmoid( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA for Hard sigmoid activation function. - Faster to compute than sigmoid activation. - - Args: - inputs: list of input tensors - dc_decomp: boolean that indicates - perturbation_domain: type of convex input domain (None or dict) - mode: type of Forward propagation (ibp, affine, or hybrid) - slope: - **kwargs: see Keras official documentation - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if dc_decomp: - raise NotImplementedError() - mode = ForwardMode(mode) - raise NotImplementedError() - - -def elu( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA for Exponential linear unit. - - Args: - inputs: list of input tensors - dc_decomp: boolean that indicates - perturbation_domain: type of convex input domain (None or dict) - mode: type of Forward propagation (ibp, affine, or hybrid) - slope: - **kwargs: see Keras official documentation - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if dc_decomp: - raise NotImplementedError() - mode = ForwardMode(mode) - raise NotImplementedError() - - -def selu( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA for Scaled Exponential Linear Unit (SELU). - - SELU is equal to: `scale * elu(x, alpha)`, where alpha and scale - are predefined constants. The values of `alpha` and `scale` are - chosen so that the mean and variance of the inputs are preserved - between two consecutive layers as long as the weights are initialized - correctly (see `lecun_normal` initialization) and the number of inputs - is "large enough" (see references for more information). - - Args: - inputs: list of input tensors - dc_decomp: boolean that indicates - perturbation_domain: type of convex input domain (None or dict) - mode: type of Forward propagation (ibp, affine, or hybrid) - slope: - **kwargs: see Keras official documentation - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if dc_decomp: - raise NotImplementedError() - mode = ForwardMode(mode) - raise NotImplementedError() - - -def linear( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA foe Linear (i.e. identity) activation function. - - Args: - inputs: list of input tensors - dc_decomp: boolean that indicates - perturbation_domain: type of convex input domain (None or dict) - mode: type of Forward propagation (ibp, affine, or hybrid) - slope: - **kwargs: see Keras official documentation - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - return inputs - - -def exponential( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA for Exponential activation function. - - Args: - inputs: list of input tensors - dc_decomp: boolean that indicates - perturbation_domain: type of convex input domain (None or dict) - mode: type of Forward propagation (ibp, affine, or hybrid) - slope: - **kwargs: see Keras official documentation - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - return exp(inputs, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode, slope=slope, **kwargs) - - -def softplus( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA for Softplus activation function `log(exp(x) + 1)`. - - Args: - inputs: list of input tensors - dc_decomp: boolean that indicates - perturbation_domain: type of convex input domain (None or dict) - mode: type of Forward propagation (ibp, affine, or hybrid) - slope: - **kwargs: see Keras official documentation - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if dc_decomp: - raise NotImplementedError() - - return softplus_(inputs, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode, slope=slope) - - -def softsign( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA for Softsign activation function `x / (abs(x) + 1)`. - - Args: - inputs: list of input tensors - dc_decomp: boolean that indicates - perturbation_domain: type of convex input domain (None or dict) - mode: type of Forward propagation (ibp, affine, or hybrid) - slope: - **kwargs: see Keras official documentation - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - func = K.softsign - f_prime = softsign_prime - return linear_hull_s_shape( - inputs, func, f_prime, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode, slope=slope - ) - - -def softmax( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - axis: int = -1, - clip: bool = True, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA for Softmax activation function. - - Args: - inputs: list of input tensors - dc_decomp: boolean that indicates - perturbation_domain: type of convex input domain (None or dict) - mode: type of Forward propagation (ibp, affine, or hybrid) - slope: - **kwargs: see Keras official documentation - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if dc_decomp: - raise NotImplementedError() - mode = ForwardMode(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - outputs_exp = exponential( - minus(inputs, mode=mode, perturbation_domain=perturbation_domain), - dc_decomp=dc_decomp, - perturbation_domain=perturbation_domain, - mode=mode, - slope=slope, - ) - outputs = expand_dims( - frac_pos( - sum(outputs_exp, axis=axis, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode), - dc_decomp=dc_decomp, - perturbation_domain=perturbation_domain, - mode=mode, - ), - mode=mode, - axis=axis, - perturbation_domain=perturbation_domain, - ) - outputs = multiply(outputs_exp, outputs, mode=mode, perturbation_domain=perturbation_domain) - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode( - outputs, compute_ibp_from_affine=False - ) - if ibp: - o_value = K.cast(1.0, dtype=u_c.dtype) - z_value = K.cast(0.0, dtype=u_c.dtype) - u_c = K.minimum(u_c, o_value) - l_c = K.maximum(l_c, z_value) - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs([x, u_c, w_u, b_u, l_c, w_l, b_l, h, g]) - - -def group_sort_2( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - data_format: str = "channels_last", - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - raise NotImplementedError() - - -def deserialize(name: str) -> Callable[..., list[Tensor]]: - """Get the activation from name. - - Args: - name: name of the method. - among the implemented Keras activation function. - - Returns: - the activation function - """ - name = name.lower() - - if name == SOFTMAX: - return softmax - elif name == ELU: - return elu - elif name == SELU: - return selu - elif name == SOFTPLUS: - return softplus - elif name == SOFTSIGN: - return softsign - elif name == SIGMOID: - return sigmoid - elif name == TANH: - return tanh - elif name == RELU: - return relu - elif name == EXPONENTIAL: - return exponential - elif name == LINEAR: - return linear - elif name == GROUP_SORT_2: - return group_sort_2 - else: - raise ValueError(f"Could not interpret activation function identifier: {name}") - - -def get(identifier: Any) -> Callable[..., list[Tensor]]: - """Get the `identifier` activation function. - - Args: - identifier: None or str, name of the function. - - Returns: - The activation function, `linear` if `identifier` is None. - - """ - if identifier is None: - return linear - elif isinstance(identifier, str): - return deserialize(identifier) - elif callable(identifier): - if isinstance(identifier, DecomonLayer): - warnings.warn( - "Do not pass a layer instance (such as {identifier}) as the " - "activation argument of another layer. Instead, advanced " - "activation layers should be used just like any other " - "layer in a model.".format(identifier=identifier.__class__.__name__) - ) - return identifier - else: - raise ValueError("Could not interpret " "activation function identifier:", identifier) diff --git a/src/decomon/layers/activations/__init__.py b/src/decomon/layers/activations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/activations/activation.py b/src/decomon/layers/activations/activation.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/convert.py b/src/decomon/layers/convert.py index 4b39d0c4..e69de29b 100644 --- a/src/decomon/layers/convert.py +++ b/src/decomon/layers/convert.py @@ -1,186 +0,0 @@ -from typing import Any, Optional, Union - -import keras -from keras.layers import Activation, Input, Layer - -import decomon.layers.decomon_layers -import decomon.layers.decomon_merge_layers -import decomon.layers.decomon_reshape -import decomon.layers.maxpooling -from decomon.core import BoxDomain, ForwardMode, PerturbationDomain, Slope, get_mode -from decomon.layers.core import DecomonLayer - -# mapping between decomon class names and actual classes -_mapping_name2class = vars(decomon.layers.decomon_layers) -_mapping_name2class.update(vars(decomon.layers.decomon_merge_layers)) -_mapping_name2class.update(vars(decomon.layers.decomon_reshape)) -_mapping_name2class.update(vars(decomon.layers.maxpooling)) - - -def to_decomon( - layer: Layer, - input_dim: int, - slope: Union[str, Slope] = Slope.V_SLOPE, - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - finetune: bool = False, - ibp: bool = True, - affine: bool = True, - shared: bool = True, - fast: bool = True, -) -> DecomonLayer: - """Transform a standard keras layer into a Decomon layer. - - Type of layer is tested to know how to transform it into a DecomonLayer of the good type. - If type is not treated yet, raises an TypeError - - Args: - layer: a Keras Layer - input_dim: an integer that represents the dim - of the input perturbation domain - slope: - dc_decomp: boolean that indicates whether we return a difference - of convex decomposition of our layer - perturbation_domain: the type of perturbation domain - ibp: boolean that indicates whether we propagate constant bounds - affine: boolean that indicates whether we propagate affine - bounds - - Returns: - the associated DecomonLayer - """ - - # get class name - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - mode = get_mode(ibp=ibp, affine=affine) - layer_decomon = _to_decomon_wo_input_init( - layer=layer, - namespace=_mapping_name2class, - slope=slope, - dc_decomp=dc_decomp, - perturbation_domain=perturbation_domain, - finetune=finetune, - mode=mode, - shared=shared, - fast=fast, - ) - - input_tensors = _prepare_input_tensors( - layer=layer, input_dim=input_dim, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode - ) - layer_decomon(input_tensors) - layer_decomon.reset_layer(layer) - - # return layer_decomon - return layer_decomon - - -def _to_decomon_wo_input_init( - layer: Layer, - namespace: dict[str, Any], - slope: Union[str, Slope] = Slope.V_SLOPE, - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - finetune: bool = False, - mode: ForwardMode = ForwardMode.HYBRID, - shared: bool = True, - fast: bool = True, -) -> DecomonLayer: - if perturbation_domain is None: - perturbation_domain = BoxDomain() - class_name = layer.__class__.__name__ - # check if layer has a built argument that built is set to True - if hasattr(layer, "built"): - if not layer.built: - raise ValueError(f"the layer {layer.name} has not been built yet") - decomon_class_name = f"Decomon{class_name}" - config_layer = layer.get_config() - config_layer["name"] = layer.name + "_decomon" - config_layer["dc_decomp"] = dc_decomp - config_layer["perturbation_domain"] = perturbation_domain - - config_layer["mode"] = mode - config_layer["finetune"] = finetune - config_layer["slope"] = slope - config_layer["shared"] = shared - config_layer["fast"] = fast - if not isinstance(layer, Activation): - config_layer.pop("activation", None) # Hyp: no non-linear activation in dense or conv2d layers - try: - layer_decomon = namespace[decomon_class_name].from_config(config_layer) - except: - raise NotImplementedError(f"The decomon version of {class_name} is not yet implemented.") - - layer_decomon.share_weights(layer) - return layer_decomon - - -def _prepare_input_tensors( - layer: Layer, input_dim: int, dc_decomp: bool, perturbation_domain: PerturbationDomain, mode: ForwardMode -) -> list[keras.KerasTensor]: - original_input_shapes = get_layer_input_shape(layer) - decomon_input_shapes: list[list[Optional[int]]] = [list(input_shape[1:]) for input_shape in original_input_shapes] - n_input = len(decomon_input_shapes) - x_input_shape = perturbation_domain.get_x_input_shape_wo_batchsize(input_dim) - x_input = Input(x_input_shape, dtype=layer.dtype) - w_input = [Input(tuple([input_dim] + decomon_input_shapes[i])) for i in range(n_input)] - y_input = [Input(tuple(decomon_input_shapes[i])) for i in range(n_input)] - - if mode == ForwardMode.HYBRID: - nested_input_list = [ - [x_input, y_input[i], w_input[i], y_input[i], y_input[i], w_input[i], y_input[i]] for i in range(n_input) - ] - elif mode == ForwardMode.IBP: - nested_input_list = [[y_input[i], y_input[i]] for i in range(n_input)] - elif mode == ForwardMode.AFFINE: - nested_input_list = [[x_input, w_input[i], y_input[i], w_input[i], y_input[i]] for i in range(n_input)] - else: - raise ValueError(f"Unknown mode {mode}") - - flatten_input_list = [tensor for input_list in nested_input_list for tensor in input_list] - - if dc_decomp: - if n_input == 1: - flatten_input_list += [y_input[0], y_input[0]] - else: - raise NotImplementedError() - - return flatten_input_list - - -SingleInputShapeType = tuple[Optional[int], ...] - - -def get_layer_input_shape(layer: Layer) -> list[SingleInputShapeType]: - """Retrieves the input shape(s) of a layer. - - Only applicable if the layer has exactly one input, - i.e. if it is connected to one incoming layer, or if all inputs - have the same shape. - - Args: - layer: - - Returns: - Input shape, as an integer shape tuple - (or list of shape tuples, one tuple per input tensor). - - Raises: - AttributeError: if the layer has no defined input_shape. - RuntimeError: if called in Eager mode. - """ - - if not layer._inbound_nodes: - raise AttributeError(f'The layer "{layer.name}" has never been called ' "and thus has no defined input shape.") - all_input_shapes = set([str([tensor.shape for tensor in node.input_tensors]) for node in layer._inbound_nodes]) - if len(all_input_shapes) == 1: - return [tensor.shape for tensor in layer._inbound_nodes[0].input_tensors] - else: - raise AttributeError( - 'The layer "' + str(layer.name) + '" has multiple inbound nodes, ' - "with different input shapes. Hence " - 'the notion of "input shape" is ' - "ill-defined for the layer. " - ) diff --git a/src/decomon/layers/convolutional/__init__.py b/src/decomon/layers/convolutional/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/convolutional/conv2d.py b/src/decomon/layers/convolutional/conv2d.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/core.py b/src/decomon/layers/core.py deleted file mode 100644 index 86ef87cb..00000000 --- a/src/decomon/layers/core.py +++ /dev/null @@ -1,247 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, Optional, Union - -import keras -from keras.layers import Layer - -from decomon.core import ( - BoxDomain, - ForwardMode, - InputsOutputsSpec, - PerturbationDomain, - get_affine, - get_ibp, -) -from decomon.keras_utils import check_if_single_shape, reset_layer -from decomon.types import BackendTensor - - -class DecomonLayer(ABC, Layer): - """Abstract class that contains the common information of every implemented layers for Forward LiRPA""" - - _trainable_weights: list[keras.Variable] - - @property - @abstractmethod - def original_keras_layer_class(self) -> type[Layer]: - """The keras layer class from which this class is the decomon equivalent.""" - pass - - def __init__( - self, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - """ - Args: - perturbation_domain: type of convex input domain (None or dict) - dc_decomp: boolean that indicates whether we return a - mode: type of Forward propagation (ibp, affine, or hybrid) - **kwargs: extra parameters - difference of convex decomposition of our layer - """ - kwargs.pop("slope", None) # remove it if not used by the decomon layer - super().__init__(**kwargs) - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - self.inputs_outputs_spec = InputsOutputsSpec( - dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain - ) - self.nb_tensors = self.inputs_outputs_spec.nb_tensors - self.dc_decomp = dc_decomp - self.perturbation_domain = perturbation_domain - self.mode = ForwardMode(mode) - self.finetune = finetune # extra optimization with hyperparameters - self.frozen_weights = False - self.frozen_alpha = False - self.shared = shared - self.fast = fast - self.has_backward_bounds = False # optimizing Forward LiRPA for adversarial perturbation - - @property - def ibp(self) -> bool: - return get_ibp(self.mode) - - @property - def affine(self) -> bool: - return get_affine(self.mode) - - def get_config(self) -> dict[str, Any]: - config = super().get_config() - config.update( - { - "dc_decomp": self.dc_decomp, - "shared": self.shared, - "fast": self.fast, - "finetune": self.finetune, - "mode": self.mode, - "perturbation_domain": self.perturbation_domain, - } - ) - return config - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - """ - Args: - input_shape - - Returns: - - """ - # generic case: call build from underlying keras layer with the proper intput_shape - y_input_shape = input_shape[-1] - self.original_keras_layer_class.build(self, y_input_shape) - - @abstractmethod - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - """ - Args: - inputs - - Returns: - - """ - - def compute_output_shape( - self, input_shape: Union[tuple[Optional[int], ...], list[tuple[Optional[int], ...]]] - ) -> Union[tuple[Optional[int], ...], list[tuple[Optional[int], ...]]]: - """Compute expected output shape according to input shape - - Will be called by symbolic calls on Keras Tensors. - - - We use the original (Keras) layer compute_output_shape() if available to update accordingly the input shapes. - - Else we simply return the input shapes - - Beware, compute_output_shape() is sometimes called by the original keras layer inside its `call()`, - which can be called inside the decomon layer call(). Check this by looking at input_shape - (list of shapes or simple shape?) - - Args: - input_shape - - Returns: - - """ - if check_if_single_shape(input_shape): - # call from a keras layer call() with a single shape - try: - return self.original_keras_layer_class.compute_output_shape( - self, input_shape - ) # output shape of the original layer - except NotImplementedError: - # no compute_output_shape implemented (e.g. InputLayer) - return input_shape - - # we know from here that we got a list of input shapes. Mypy does not. - input_shapes: list[tuple[Optional[int], ...]] = input_shape # type: ignore - y_shape = self.inputs_outputs_spec.get_kerasinputshape_from_inputshapesformode( - input_shapes - ) # input shape for the original layer - try: - y_out_shape = self.original_keras_layer_class.compute_output_shape( - self, y_shape - ) # output shape of the original layer - except NotImplementedError: - # no compute_output_shape implemented (e.g. InputLayer) - return input_shape - else: - # we now the original output shape => deduce the decomon output shapes - ( - x_shape, - u_c_shape, - w_u_shape, - b_u_shape, - l_c_shape, - w_l_shape, - b_l_shape, - h_shape, - g_shape, - ) = self.inputs_outputs_spec.get_fullinputshapes_from_inputshapesformode(input_shapes) - y_out_shape_wo_batchsize = y_out_shape[1:] - if self.inputs_outputs_spec.affine: - model_inputdim = x_shape[-1] - batchsize = x_shape[0] - w_out_shape = (batchsize, model_inputdim) + y_out_shape_wo_batchsize - else: - w_out_shape = tuple() - fulloutputshapes = [ - x_shape, - y_out_shape, - w_out_shape, - y_out_shape, - y_out_shape, - w_out_shape, - y_out_shape, - y_out_shape, - y_out_shape, - ] - return self.inputs_outputs_spec.extract_inputshapesformode_from_fullinputshapes(fulloutputshapes) - - def compute_output_spec(self, *args: Any, **kwargs: Any) -> keras.KerasTensor: - """Compute output spec from output shape in case of symbolic call.""" - return Layer.compute_output_spec(self, *args, **kwargs) - - def reset_layer(self, layer: Layer) -> None: - """Reset the weights by using the weights of another (a priori non-decomon) layer. - - It set the weights whose names are listed by `keras_weights_names`. - - Args: - layer - - Returns: - - """ - weight_names = self.keras_weights_names - if len(weight_names) > 0: - reset_layer(new_layer=self, original_layer=layer, weight_names=weight_names) - - @property - def keras_weights_names(self) -> list[str]: - """Weights names of the corresponding Keras layer. - - Will be used to decide which weight to take from the keras layer in `reset_layer()` - - """ - return [] - - def join(self, bounds: list[BackendTensor]) -> list[BackendTensor]: - """ - Args: - bounds - - Returns: - - """ - raise NotImplementedError() - - def freeze_weights(self) -> None: - pass - - def unfreeze_weights(self) -> None: - pass - - def freeze_alpha(self) -> None: - pass - - def unfreeze_alpha(self) -> None: - pass - - def reset_finetuning(self) -> None: - pass - - def share_weights(self, layer: Layer) -> None: - pass - - def split_kwargs(self, **kwargs: Any) -> None: - # necessary for InputLayer - pass - - def set_back_bounds(self, has_backward_bounds: bool) -> None: - pass diff --git a/src/decomon/layers/core/__init__.py b/src/decomon/layers/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/core/dense.py b/src/decomon/layers/core/dense.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/decomon_layers.py b/src/decomon/layers/decomon_layers.py deleted file mode 100644 index d6eced0b..00000000 --- a/src/decomon/layers/decomon_layers.py +++ /dev/null @@ -1,982 +0,0 @@ -from typing import Any, Optional, Union - -import keras -import keras.ops as K -import numpy as np -from keras.layers import ( - Activation, - BatchNormalization, - Conv2D, - Dense, - Dot, - Dropout, - Flatten, - InputLayer, - InputSpec, - Lambda, - Layer, -) -from keras.src.backend import rnn - -from decomon.core import ForwardMode, PerturbationDomain, Slope -from decomon.layers import activations -from decomon.layers.core import DecomonLayer -from decomon.layers.utils import ( - ClipAlpha, - ClipAlphaAndSumtoOne, - NonPos, - Project_initializer_pos, -) -from decomon.types import BackendTensor - - -class DecomonConv2D(DecomonLayer, Conv2D): - """Forward LiRPA implementation of Conv2d layers. - See Keras official documentation for further details on the Conv2d operator - - """ - - original_keras_layer_class = Conv2D - - def __init__( - self, - filters: int, - kernel_size: Union[int, tuple[int, int]], - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - kwargs.pop("activation", None) - super().__init__( - filters=filters, - kernel_size=kernel_size, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - if self.mode == ForwardMode.HYBRID: - self.input_spec = [ - InputSpec(min_ndim=2), # z - InputSpec(min_ndim=4), # u - InputSpec(min_ndim=4), # wu - InputSpec(min_ndim=4), # bu - InputSpec(min_ndim=4), # l - InputSpec(min_ndim=4), # wl - InputSpec(min_ndim=4), # bl - ] - elif self.mode == ForwardMode.IBP: - self.input_spec = [ - InputSpec(min_ndim=4), # u - InputSpec(min_ndim=4), # l - ] - elif self.mode == ForwardMode.AFFINE: - self.input_spec = [ - InputSpec(min_ndim=2), # z - InputSpec(min_ndim=4), # wu - InputSpec(min_ndim=4), # bu - InputSpec(min_ndim=4), # wl - InputSpec(min_ndim=4), # bl - ] - if self.dc_decomp: - self.input_spec += [InputSpec(min_ndim=4), InputSpec(min_ndim=4)] - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - """ - Args: - input_shape - - Returns: - - """ - - assert len(input_shape) == self.nb_tensors - if self.data_format == "channels_first": - channel_axis = 1 - else: - channel_axis = -1 - if input_shape[0][channel_axis] is None: - raise ValueError("The channel dimension of the inputs " "should be defined. Found `None`.") - input_dim = input_shape[-1][channel_axis] - - kernel_shape = self.kernel_size + (input_dim, self.filters) - - if not self.shared: - self.kernel = self.add_weight( - shape=kernel_shape, - initializer=self.kernel_initializer, - name="kernel_all", - regularizer=self.kernel_regularizer, - constraint=self.kernel_constraint, - ) - - if self.use_bias: - self.bias = self.add_weight( - shape=(self.filters,), - initializer=self.bias_initializer, - name="bias_pos", - regularizer=self.bias_regularizer, - constraint=self.bias_constraint, - ) - else: - self.bias = None - - self.built = True - - def share_weights(self, layer: Layer) -> None: - if not self.shared: - return - self.kernel = layer.kernel - self.bias = layer.bias - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - """computing the perturbation analysis of the operator without the activation function - - Args: - inputs: list of input tensors - **kwargs - - Returns: - List of updated tensors - """ - z_value = K.cast(0.0, self.dtype) - o_value = K.cast(1.0, self.dtype) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - def conv_pos(x: BackendTensor) -> BackendTensor: - return K.conv( - x, - K.maximum(z_value, self.kernel), - strides=list(self.strides), - padding=self.padding, - data_format=self.data_format, - dilation_rate=self.dilation_rate, - ) - - def conv_neg(x: BackendTensor) -> BackendTensor: - return K.conv( - x, - K.minimum(z_value, self.kernel), - strides=list(self.strides), - padding=self.padding, - data_format=self.data_format, - dilation_rate=self.dilation_rate, - ) - - if self.dc_decomp: - h_out = conv_pos(h) + conv_neg(g) - g_out = conv_pos(g) + conv_neg(h) - else: - h_out, g_out = empty_tensor, empty_tensor - - if self.ibp: - u_c_out = conv_pos(u_c) + conv_neg(l_c) - l_c_out = conv_pos(l_c) + conv_neg(u_c) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if self.affine: - y = K.conv( - b_u, - self.kernel, - strides=list(self.strides), - padding=self.padding, - data_format=self.data_format, - dilation_rate=self.dilation_rate, - ) - - if len(w_u.shape) == len(b_u.shape): - flatdim = int(np.prod(b_u.shape[1:])) - identity_tensor = K.reshape(K.identity(flatdim, dtype=self.dtype), (-1,) + tuple(b_u.shape[1:])) - w_u_out = K.conv( - identity_tensor, - self.kernel, - strides=list(self.strides), - padding=self.padding, - data_format=self.data_format, - dilation_rate=self.dilation_rate, - ) - w_u_out = K.expand_dims(w_u_out, 0) + z_value * K.expand_dims(y, 1) - w_l_out = w_u_out - b_u_out = 0 * y - b_l_out = 0 * y - - else: - # check for linearity - x_max = self.perturbation_domain.get_upper(x, w_u - w_l, b_u - b_l) - mask_b = o_value - K.sign(x_max) - - def step_pos(x: BackendTensor, _: list[BackendTensor]) -> tuple[BackendTensor, list[BackendTensor]]: - return conv_pos(x), [] - - def step_neg(x: BackendTensor, _: list[BackendTensor]) -> tuple[BackendTensor, list[BackendTensor]]: - return conv_neg(x), [] - - b_u_out = conv_pos(b_u) + conv_neg(b_l) - b_l_out = conv_pos(b_l) + conv_neg(b_u) - - w_u_out = ( - rnn(step_function=step_pos, inputs=w_u, initial_states=[], unroll=False)[1] - + rnn(step_function=step_neg, inputs=w_l, initial_states=[], unroll=False)[1] - ) - w_l_out = ( - rnn(step_function=step_pos, inputs=w_l, initial_states=[], unroll=False)[1] - + rnn(step_function=step_neg, inputs=w_u, initial_states=[], unroll=False)[1] - ) - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if self.use_bias: - # put bias to the correct shape - if self.data_format == "channels_last": - bias_shape = (1,) * (self.rank + 1) + (self.filters,) - else: - bias_shape = (1, self.filters) + (1,) * self.rank - reshaped_bias = K.reshape(self.bias, bias_shape) - - if self.dc_decomp: - g_out += K.reshape(K.minimum(z_value, self.bias), bias_shape) - h_out += K.reshape(K.maximum(z_value, self.bias), bias_shape) - if self.affine: - b_u_out += reshaped_bias - b_l_out += reshaped_bias - if self.ibp: - u_c_out += reshaped_bias - l_c_out += reshaped_bias - - return self.inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - @property - def keras_weights_names(self) -> list[str]: - """Weights names of the corresponding Keras layer. - - Will be used to decide which weight to take from the keras layer in `reset_layer()` - - """ - weight_names = ["kernel"] - if self.use_bias: - weight_names.append("bias") - return weight_names - - def freeze_weights(self) -> None: - if not self.frozen_weights: - self._trainable_weights = [] - self.frozen_weights = True - - def unfreeze_weights(self) -> None: - if self.frozen_weights: - if self.use_bias: - self._trainable_weights = [self.bias] + self._trainable_weights - self._trainable_weights = [self.kernel] + self._trainable_weights - self.frozen_weights = False - - -class DecomonDense(DecomonLayer, Dense): - """Forward LiRPA implementation of Dense layers. - See Keras official documentation for further details on the Dense operator - """ - - original_keras_layer_class = Dense - - def __init__( - self, - units: int, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - kwargs.pop("activation", None) - kwargs.pop("kernel_constraint", None) - super().__init__( - units=units, - kernel_constraint=None, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - self.input_spec = [InputSpec(min_ndim=2) for _ in range(self.nb_tensors)] - self.input_shape_build: Optional[list[tuple[Optional[int], ...]]] = None - self.op_dot = K.dot - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - """ - Args: - input_shape: list of input shape - - Returns: - - """ - - assert len(input_shape) >= self.nb_tensors - - input_dim = input_shape[-1][-1] - - if not self.shared: - kernel = self.add_weight( - shape=(input_dim, self.units), - initializer=Project_initializer_pos(self.kernel_initializer), - name="kernel_pos", - regularizer=self.kernel_regularizer, - ) - try: - self.kernel = kernel - except AttributeError: - # manage hidden weights introduced for LoRA https://github.com/keras-team/keras/pull/18942 - self._kernel = kernel - - if self.use_bias: - self.bias = self.add_weight( - shape=(self.units,), - initializer=self.bias_initializer, - name="bias_pos", - regularizer=self.bias_regularizer, - constraint=self.bias_constraint, - ) - else: - self.bias = None - - # 6 inputs tensors : h, g, x_min, x_max, W_u, b_u, W_l, b_l - if self.mode == ForwardMode.HYBRID: - self.input_spec = [ - InputSpec(min_ndim=2), # x_0 - InputSpec(min_ndim=2, axes={-1: input_dim}), # u_c - InputSpec(min_ndim=2, axes={-1: input_dim}), # W_u - InputSpec(min_ndim=2, axes={-1: input_dim}), # b_u - InputSpec(min_ndim=2, axes={-1: input_dim}), # l_c - InputSpec(min_ndim=2, axes={-1: input_dim}), # W_l - InputSpec(min_ndim=2, axes={-1: input_dim}), # b_l - ] - elif self.mode == ForwardMode.IBP: - self.input_spec = [ - InputSpec(min_ndim=2, axes={-1: input_dim}), # u_c - InputSpec(min_ndim=2, axes={-1: input_dim}), # l_c - ] - elif self.mode == ForwardMode.AFFINE: - self.input_spec = [ - InputSpec(min_ndim=2), # x_0 - InputSpec(min_ndim=2, axes={-1: input_dim}), # W_u - InputSpec(min_ndim=2, axes={-1: input_dim}), # b_u - InputSpec(min_ndim=2, axes={-1: input_dim}), # W_l - InputSpec(min_ndim=2, axes={-1: input_dim}), # b_l - ] - if self.dc_decomp: - self.input_spec += [ - InputSpec(min_ndim=2, axes={-1: input_dim}), # h - InputSpec(min_ndim=2, axes={-1: input_dim}), - ] # g - - if self.has_backward_bounds: - self.input_spec += [InputSpec(ndim=3, axes={-1: self.units, -2: self.units})] - - self.built = True - self.input_shape_build = input_shape - - def share_weights(self, layer: Layer) -> None: - if not self.shared: - return - try: - self.kernel = layer.kernel - except AttributeError: - # manage hidden weights introduced for LoRA https://github.com/keras-team/keras/pull/18942 - self._kernel = layer._kernel - if self.use_bias: - self.bias = layer.bias - - def set_back_bounds(self, has_backward_bounds: bool) -> None: - # check for activation - if self.activation is not None and self.activation_name != "linear" and has_backward_bounds: - raise ValueError() - self.has_backward_bounds = has_backward_bounds - if self.built and has_backward_bounds: - if self.input_shape_build is None: - raise ValueError("self.input_shape_build should not be None when calling set_back_bounds") - # rebuild with an additional input - self.build(self.input_shape_build) - if self.has_backward_bounds: - op = Dot(1) - self.op_dot = lambda x, y: op([x, y]) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - """ - Args: - inputs - - Returns: - - """ - if self.kernel is None: - raise RuntimeError("self.kernel cannot be None when calling call()") - - z_value = K.cast(0.0, self.dtype) - - if self.has_backward_bounds: - back_bound = inputs[-1] - inputs = inputs[:-1] - kernel = K.sum(self.kernel[None, :, :, None] * back_bound[:, None], 2) - kernel_pos_back = K.maximum(z_value, kernel) - kernel_neg_back = K.minimum(z_value, kernel) - - kernel_pos = K.maximum(z_value, self.kernel) - kernel_neg = K.minimum(z_value, self.kernel) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - if self.dc_decomp: - h_out = K.dot(h, kernel_pos) + K.dot(g, kernel_neg) - g_out = K.dot(g, kernel_pos) + K.dot(h, kernel_neg) - else: - h_out, g_out = empty_tensor, empty_tensor - - if self.ibp: - if not self.has_backward_bounds: - u_c_out = self.op_dot(u_c, kernel_pos) + self.op_dot(l_c, kernel_neg) - l_c_out = self.op_dot(l_c, kernel_pos) + self.op_dot(u_c, kernel_neg) - else: - u_c_out = self.op_dot(u_c, kernel_pos_back) + self.op_dot(l_c, kernel_neg_back) - l_c_out = self.op_dot(l_c, kernel_pos_back) + self.op_dot(u_c, kernel_neg_back) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if self.affine: - if len(w_u.shape) == len(b_u.shape): - y = K.dot(b_u, self.kernel) - w_u_out = K.expand_dims(0 * y, 1) + K.expand_dims(self.kernel, 0) - w_l_out = w_u_out - b_u_out = z_value * y - b_l_out = b_u_out - - if len(w_u.shape) != len(b_u.shape): - b_u_out = K.dot(b_u, kernel_pos) + K.dot(b_l, kernel_neg) - b_l_out = K.dot(b_l, kernel_pos) + K.dot(b_u, kernel_neg) - - if not self.has_backward_bounds: - w_u_out = K.dot(w_u, kernel_pos) + K.dot(w_l, kernel_neg) - w_l_out = K.dot(w_l, kernel_pos) + K.dot(w_u, kernel_neg) - else: - raise NotImplementedError() # bug somewhere - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if self.use_bias: - if not self.has_backward_bounds: - if self.ibp: - u_c_out += self.bias - l_c_out += self.bias - if self.affine: - b_u_out += self.bias - b_l_out += self.bias - else: - b = K.sum(back_bound * self.bias[None, None], 1) - if self.ibp: - u_c_out = u_c_out + b - l_c_out = l_c_out + b - if self.affine: - b_u_out = b_u_out + b - b_l_out = b_l_out + b - - if self.dc_decomp: - if self.has_backward_bounds: - raise NotImplementedError() - h_out += K.maximum(z_value, self.bias) - g_out += K.minimum(z_value, self.bias) - - return self.inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - @property - def keras_weights_names(self) -> list[str]: - """Weights names of the corresponding Keras layer. - - Will be used to decide which weight to take from the keras layer in `reset_layer()` - - """ - weight_names = ["kernel"] - if self.use_bias: - weight_names.append("bias") - return weight_names - - def freeze_weights(self) -> None: - if not self.frozen_weights: - self._trainable_weights = [] - self.frozen_weights = True - - def unfreeze_weights(self) -> None: - if self.frozen_weights: - if self.use_bias: - self._trainable_weights = [self.bias] + self._trainable_weights - self._trainable_weights = [self.kernel] + self._trainable_weights - self.frozen_weights = False - - -class DecomonActivation(DecomonLayer, Activation): - """Forward LiRPA implementation of Activation layers. - See Keras official documentation for further details on the Activation operator - """ - - original_keras_layer_class = Activation - - def __init__( - self, - activation: str, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - activation=activation, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - self.slope = Slope(slope) - self.supports_masking = True - self.activation = activations.get(activation) - self.activation_name = activation - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - if self.finetune and self.mode in [ForwardMode.HYBRID, ForwardMode.AFFINE]: - shape = input_shape[-1][1:] - - if self.activation_name != "linear" and self.mode != ForwardMode.IBP: - if self.activation_name[:4] != "relu": - self.beta_u_f = self.add_weight( - shape=shape, - initializer="ones", - name="beta_u_f", - regularizer=None, - constraint=ClipAlpha(), - ) - self.beta_l_f = self.add_weight( - shape=shape, - initializer="ones", - name="beta_l_f", - regularizer=None, - constraint=ClipAlpha(), - ) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - if self.finetune and self.mode in [ForwardMode.AFFINE, ForwardMode.HYBRID] and self.activation_name != "linear": - if self.activation_name[:4] == "relu": - return self.activation( - inputs, - mode=self.mode, - dc_decomp=self.dc_decomp, - perturbation_domain=self.perturbation_domain, - finetune=self.beta_l_f, - slope=self.slope, - ) - else: - return self.activation( - inputs, - mode=self.mode, - dc_decomp=self.dc_decomp, - perturbation_domain=self.perturbation_domain, - finetune=[self.beta_u_f, self.beta_l_f], - slope=self.slope, - ) - else: - output = self.activation( - inputs, - mode=self.mode, - perturbation_domain=self.perturbation_domain, - dc_decomp=self.dc_decomp, - slope=self.slope, - ) - return output - - def reset_finetuning(self) -> None: - if self.finetune and self.mode != ForwardMode.IBP: - if self.activation_name != "linear": - if self.activation_name[:4] == "relu": - self.beta_l_f.assign(np.ones_like(self.beta_l_f.value())) - else: - self.beta_u_f.assign(np.ones_like(self.beta_u_f.value())) - self.beta_l_f.assign(np.ones_like(self.beta_l_f.value())) - - def freeze_alpha(self) -> None: - if not self.frozen_alpha: - if self.finetune and self.mode in [ForwardMode.AFFINE, ForwardMode.HYBRID]: - self._trainable_weights = [] - self.frozen_alpha = True - - def unfreeze_alpha(self) -> None: - if self.frozen_alpha: - if self.finetune and self.mode in [ForwardMode.AFFINE, ForwardMode.HYBRID]: - if self.activation_name != "linear": - if self.activation_name[:4] != "relu": - self._trainable_weights += [self.beta_u_f, self.beta_l_f] - else: - self._trainable_weights += [self.beta_l_f] - self.frozen_alpha = False - - -class DecomonFlatten(DecomonLayer, Flatten): - """Forward LiRPA implementation of Flatten layers. - See Keras official documentation for further details on the Flatten operator - """ - - original_keras_layer_class = Flatten - - def __init__( - self, - data_format: Optional[str] = None, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - """ - Args: - data_format - **kwargs - """ - super().__init__( - data_format=data_format, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - if self.mode == ForwardMode.HYBRID: - self.input_spec = [ - InputSpec(min_ndim=1), # z - InputSpec(min_ndim=1), # u - InputSpec(min_ndim=2), # w_u - InputSpec(min_ndim=2), # b_u - InputSpec(min_ndim=1), # l - InputSpec(min_ndim=2), # w_l - InputSpec(min_ndim=2), # b_l - ] - elif self.mode == ForwardMode.IBP: - self.input_spec = [ - InputSpec(min_ndim=1), # u - InputSpec(min_ndim=1), # l - ] - elif self.mode == ForwardMode.AFFINE: - self.input_spec = [ - InputSpec(min_ndim=1), # z - InputSpec(min_ndim=2), # w_u - InputSpec(min_ndim=2), # b_u - InputSpec(min_ndim=2), # w_l - InputSpec(min_ndim=2), # b_l - ] - if self.dc_decomp: - self.input_spec += [InputSpec(min_ndim=1), InputSpec(min_ndim=1)] - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - """ - Args: - self - input_shape - - Returns: - - """ - return None - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - def op(x: BackendTensor) -> BackendTensor: - return Flatten.call(self, x) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - if self.dc_decomp: - h_out = op(h) - g_out = op(g) - else: - h_out, g_out = empty_tensor, empty_tensor - - if self.ibp: - u_c_out = op(u_c) - l_c_out = op(l_c) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if self.affine: - b_u_out = op(b_u) - b_l_out = op(b_l) - input_dim = x.shape[-1] - output_shape = int(np.prod(b_u_out.shape[1:])) - w_u_out = K.reshape(w_u, (-1, input_dim, output_shape)) - w_l_out = K.reshape(w_l, (-1, input_dim, output_shape)) - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - return self.inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -class DecomonBatchNormalization(DecomonLayer, BatchNormalization): - """Forward LiRPA implementation of Batch Normalization layers. - See Keras official documentation for further details on the BatchNormalization operator - """ - - original_keras_layer_class = BatchNormalization - - def __init__( - self, - axis: int = -1, - momentum: float = 0.99, - epsilon: float = 1e-3, - center: bool = True, - scale: bool = True, - beta_initializer: str = "zeros", - gamma_initializer: str = "ones", - moving_mean_initializer: str = "zeros", - moving_variance_initializer: str = "ones", - beta_regularizer: Optional[str] = None, - gamma_regularizer: Optional[str] = None, - beta_constraint: Optional[str] = None, - gamma_constraint: Optional[str] = None, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - axis=axis, - momentum=momentum, - epsilon=epsilon, - center=center, - scale=scale, - beta_initializer=beta_initializer, - gamma_initializer=gamma_initializer, - moving_mean_initializer=moving_mean_initializer, - moving_variance_initializer=moving_variance_initializer, - beta_regularizer=beta_regularizer, - gamma_regularizer=gamma_regularizer, - beta_constraint=beta_constraint, - gamma_constraint=gamma_constraint, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - super().build(input_shape) - self.input_spec = [InputSpec(min_ndim=len(elem)) for elem in input_shape] - - def call(self, inputs: list[BackendTensor], training: bool = False, **kwargs: Any) -> list[BackendTensor]: - z_value = K.cast(0.0, self.dtype) - - if training: - raise NotImplementedError("not working during training") - - def call_op(x: BackendTensor, training: bool) -> BackendTensor: - return BatchNormalization.call(self, x, training=training) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - y = call_op(inputs[-1], training=training) - - n_dim = len(y.shape) - shape = [1] * n_dim - shape[self.axis] = self.moving_mean.shape[0] - - if not hasattr(self, "gamma") or self.gamma is None: # scale = False - gamma = K.ones(shape) - else: # scale = True - gamma = K.reshape(self.gamma + z_value, shape) - if not hasattr(self, "beta") or self.beta is None: # center = False - beta = K.zeros(shape) - else: # center = True - beta = K.reshape(self.beta + z_value, shape) - moving_mean = K.reshape(self.moving_mean + z_value, shape) - moving_variance = K.reshape(self.moving_variance + z_value, shape) - - if self.ibp: - u_c_0 = (u_c - moving_mean) / K.sqrt(moving_variance + self.epsilon) - l_c_0 = (l_c - moving_mean) / K.sqrt(moving_variance + self.epsilon) - u_c_out = K.maximum(z_value, gamma) * u_c_0 + K.minimum(z_value, gamma) * l_c_0 + beta - l_c_out = K.maximum(z_value, gamma) * l_c_0 + K.minimum(z_value, gamma) * u_c_0 + beta - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if self.affine: - b_u_0 = (b_u - moving_mean) / K.sqrt(moving_variance + self.epsilon) - b_l_0 = (b_l - moving_mean) / K.sqrt(moving_variance + self.epsilon) - - b_u_out = K.maximum(z_value, gamma) * b_u_0 + K.minimum(z_value, gamma) * b_l_0 + beta - b_l_out = K.maximum(z_value, gamma) * b_l_0 + K.minimum(z_value, gamma) * b_u_0 + beta - - gamma = K.expand_dims(gamma, 1) - moving_variance = K.expand_dims(moving_variance, 1) - - w_u_0 = w_u / K.sqrt(moving_variance + self.epsilon) - w_l_0 = w_l / K.sqrt(moving_variance + self.epsilon) - w_u_out = K.maximum(z_value, gamma) * w_u_0 + K.minimum(z_value, gamma) * w_l_0 - w_l_out = K.maximum(z_value, gamma) * w_l_0 + K.minimum(z_value, gamma) * w_u_0 - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if self.dc_decomp: - raise NotImplementedError() - else: - h_out, g_out = empty_tensor, empty_tensor - - return self.inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - @property - def keras_weights_names(self) -> list[str]: - """Weights names of the corresponding Keras layer. - - Will be used to decide which weight to take from the keras layer in `reset_layer()` - - """ - weight_names = ["moving_mean", "moving_variance"] - if self.center: - weight_names.append("beta") - if self.scale: - weight_names.append("gamma") - return weight_names - - -class DecomonDropout(DecomonLayer, Dropout): - """Forward LiRPA implementation of Dropout layers. - See Keras official documentation for further details on the Dropout operator - """ - - original_keras_layer_class = Dropout - - def __init__( - self, - rate: float, - noise_shape: Optional[tuple[int, ...]] = None, - seed: Optional[int] = None, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - rate=rate, - noise_shape=noise_shape, - seed=seed, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - super().build(input_shape) - self.input_spec = [InputSpec(min_ndim=len(elem)) for elem in input_shape] - - def call(self, inputs: list[BackendTensor], training: bool = False, **kwargs: Any) -> list[BackendTensor]: - if training: - raise NotImplementedError("not working during training") - - return inputs - - -class DecomonInputLayer(DecomonLayer, InputLayer): - """Forward LiRPA implementation of Dropout layers. - See Keras official documentation for further details on the Dropout operator - """ - - original_keras_layer_class = InputLayer - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - return inputs - - def __init__( - self, - shape: Optional[tuple[int, ...]] = None, - batch_size: Optional[int] = None, - dtype: Optional[str] = None, - sparse: Optional[bool] = None, - batch_shape: Optional[tuple[Optional[int], ...]] = None, - input_tensor: Optional[keras.KerasTensor] = None, - name: Optional[str] = None, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - shape=shape, - batch_size=batch_size, - dtype=dtype, - input_tensor=input_tensor, - sparse=sparse, - name=name, - batch_shape=batch_shape, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) diff --git a/src/decomon/layers/decomon_merge_layers.py b/src/decomon/layers/decomon_merge_layers.py deleted file mode 100644 index 8838cdb5..00000000 --- a/src/decomon/layers/decomon_merge_layers.py +++ /dev/null @@ -1,633 +0,0 @@ -from typing import Any, Optional, Union - -import keras.ops as K -from keras.layers import ( - Add, - Average, - Concatenate, - Dot, - Lambda, - Layer, - Maximum, - Minimum, - Multiply, - Subtract, -) - -from decomon.core import ForwardMode, PerturbationDomain -from decomon.layers.core import DecomonLayer -from decomon.layers.utils import broadcast, multiply, permute_dimensions -from decomon.types import BackendTensor -from decomon.utils import maximum, minus, subtract - -##### Merge Layer #### - - -class DecomonMerge(DecomonLayer): - """Base class for Decomon layers based on Mergind Keras layers.""" - - def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: # type: ignore - """Compute output shapes from input shapes. - - By default, we assume that all inputs will be merged into "one" (still a list of tensors though). - - """ - # split inputs - input_shapes_list = self.inputs_outputs_spec.split_inputsformode_to_merge(input_shape) - - # compute original layer output shape - y_shapes = [ - self.inputs_outputs_spec.get_kerasinputshape_from_inputshapesformode(input_shapes) - for input_shapes in input_shapes_list - ] # input shapes for the original layer - y_out_shape = self.original_keras_layer_class.compute_output_shape( - self, y_shapes - ) # output shape of the original layer - - # same shape as one input or not? - if y_out_shape == y_shapes[0]: - # same output shape as one input - return input_shapes_list[0] - else: - # something change along the way (cf Concatenate), => we deduce decomon output shape - ( - x_shape, - u_c_shape, - w_u_shape, - b_u_shape, - l_c_shape, - w_l_shape, - b_l_shape, - h_shape, - g_shape, - ) = self.inputs_outputs_spec.get_fullinputshapes_from_inputshapesformode(input_shapes_list[0]) - y_out_shape_wo_batchsize = y_out_shape[1:] - if self.inputs_outputs_spec.affine: - model_inputdim = x_shape[-1] - batchsize = x_shape[0] - w_out_shape = (batchsize, model_inputdim) + y_out_shape_wo_batchsize - else: - w_out_shape = tuple() - fulloutputshapes = [ - x_shape, - y_out_shape, - w_out_shape, - y_out_shape, - y_out_shape, - w_out_shape, - y_out_shape, - y_out_shape, - y_out_shape, - ] - return self.inputs_outputs_spec.extract_inputshapesformode_from_fullinputshapes(fulloutputshapes) - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - n_comp = self.nb_tensors - input_shape_y = input_shape[n_comp - 1 :: n_comp] - self.original_keras_layer_class.build(self, input_shape_y) - - -class DecomonAdd(DecomonMerge, Add): - """LiRPA implementation of Add layers. - See Keras official documentation for further details on the Add operator - - """ - - original_keras_layer_class = Add - - def __init__( - self, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - # splits the inputs - ( - inputs_x, - inputs_u_c, - inputs_w_u, - inputs_b_u, - inputs_l_c, - inputs_w_l, - inputs_b_l, - inputs_h, - inputs_g, - ) = self.inputs_outputs_spec.get_fullinputs_by_type_from_inputsformode_to_merge(inputs) - x_out = inputs_x[0] - dtype = x_out.dtype - empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - # outputs - if self.ibp: - u_c_out = sum(inputs_u_c) - l_c_out = sum(inputs_l_c) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if self.affine: - b_u_out = sum(inputs_b_u) - b_l_out = sum(inputs_b_l) - w_u_out = sum(inputs_w_u) - w_l_out = sum(inputs_w_l) - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if self.dc_decomp: - raise NotImplementedError() - else: - h_out, g_out = empty_tensor, empty_tensor - - return self.inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x_out, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -class DecomonAverage(DecomonMerge, Average): - """LiRPA implementation of Average layers. - See Keras official documentation for further details on the Average operator - - """ - - original_keras_layer_class = Average - - def __init__( - self, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - self.op = Lambda(lambda x: sum(x) / len(x)) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - # splits the inputs - ( - inputs_x, - inputs_u_c, - inputs_w_u, - inputs_b_u, - inputs_l_c, - inputs_w_l, - inputs_b_l, - inputs_h, - inputs_g, - ) = self.inputs_outputs_spec.get_fullinputs_by_type_from_inputsformode_to_merge(inputs) - x_out = inputs_x[0] - dtype = x_out.dtype - empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - # outputs - if self.ibp: - u_c_out = self.op(inputs_u_c) - l_c_out = self.op(inputs_l_c) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if self.affine: - b_u_out = self.op(inputs_b_u) - b_l_out = self.op(inputs_b_l) - w_u_out = self.op(inputs_w_u) - w_l_out = self.op(inputs_w_l) - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if self.dc_decomp: - raise NotImplementedError() - else: - h_out, g_out = empty_tensor, empty_tensor - - return self.inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x_out, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -class DecomonSubtract(DecomonMerge, Subtract): - """LiRPA implementation of Subtract layers. - See Keras official documentation for further details on the Subtract operator - - """ - - original_keras_layer_class = Subtract - - def __init__( - self, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - if self.dc_decomp: - raise NotImplementedError() - - # splits the inputs - inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) - - # check number of inputs - if len(inputs_list) != 2: - raise ValueError("This layer is intended to merge only 2 layers.") - - output = subtract( - inputs_list[0], - inputs_list[1], - dc_decomp=self.dc_decomp, - perturbation_domain=self.perturbation_domain, - mode=self.mode, - ) - return output - - -class DecomonMinimum(DecomonMerge, Minimum): - """LiRPA implementation of Minimum layers. - See Keras official documentation for further details on the Minimum operator - - """ - - original_keras_layer_class = Minimum - - def __init__( - self, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - if self.dc_decomp: - raise NotImplementedError() - - # splits the inputs - inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) - # look at minus the input to apply maximum - inputs_list = [ - minus(single_inputs, mode=self.mode, dc_decomp=self.dc_decomp, perturbation_domain=self.perturbation_domain) - for single_inputs in inputs_list - ] - - #  check number of inputs - if len(inputs_list) == 1: # nothing to merge - return inputs - else: - output = maximum( - inputs_list[0], - inputs_list[1], - dc_decomp=self.dc_decomp, - perturbation_domain=self.perturbation_domain, - mode=self.mode, - ) - for j in range(2, len(inputs_list)): - output = maximum( - output, - inputs_list[j], - dc_decomp=self.dc_decomp, - perturbation_domain=self.perturbation_domain, - mode=self.mode, - ) - - return minus(output, mode=self.mode, dc_decomp=self.dc_decomp, perturbation_domain=self.perturbation_domain) - - -class DecomonMaximum(DecomonMerge, Maximum): - """LiRPA implementation of Maximum layers. - See Keras official documentation for further details on the Maximum operator - - """ - - original_keras_layer_class = Maximum - - def __init__( - self, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - if self.dc_decomp: - raise NotImplementedError() - - # splits the inputs - inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) - - #  check number of inputs - if len(inputs_list) == 1: # nothing to merge - return inputs - else: - output = maximum( - inputs_list[0], - inputs_list[1], - dc_decomp=self.dc_decomp, - perturbation_domain=self.perturbation_domain, - mode=self.mode, - ) - for j in range(2, len(inputs_list)): - output = maximum( - output, - inputs_list[j], - dc_decomp=self.dc_decomp, - perturbation_domain=self.perturbation_domain, - mode=self.mode, - ) - - return output - - -class DecomonConcatenate(DecomonMerge, Concatenate): - """LiRPA implementation of Concatenate layers. - See Keras official documentation for further details on the Concatenate operator - - """ - - original_keras_layer_class = Concatenate - - def __init__( - self, - axis: int = -1, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - axis=axis, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - def func(inputs: list[BackendTensor]) -> BackendTensor: - return Concatenate.call(self, inputs) - - self.op = func - if self.axis == -1: - self.op_w = self.op - else: - self.op_w = Concatenate(axis=self.axis + 1) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - # splits the inputs - ( - inputs_x, - inputs_u_c, - inputs_w_u, - inputs_b_u, - inputs_l_c, - inputs_w_l, - inputs_b_l, - inputs_h, - inputs_g, - ) = self.inputs_outputs_spec.get_fullinputs_by_type_from_inputsformode_to_merge(inputs) - x_out = inputs_x[0] - dtype = x_out.dtype - empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - # outputs - if self.ibp: - u_c_out = self.op(inputs_u_c) - l_c_out = self.op(inputs_l_c) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if self.affine: - b_u_out = self.op(inputs_b_u) - b_l_out = self.op(inputs_b_l) - w_u_out = self.op_w(inputs_w_u) - w_l_out = self.op_w(inputs_w_l) - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if self.dc_decomp: - raise NotImplementedError() - else: - h_out, g_out = empty_tensor, empty_tensor - - return self.inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x_out, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -class DecomonMultiply(DecomonMerge, Multiply): - """LiRPA implementation of Multiply layers. - See Keras official documentation for further details on the Multiply operator - - """ - - original_keras_layer_class = Multiply - - def __init__( - self, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - if self.dc_decomp: - raise NotImplementedError() - - # splits the inputs - inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) - - #  check number of inputs - if len(inputs_list) == 1: # nothing to merge - return inputs - else: - output = multiply( - inputs_list[0], - inputs_list[1], - dc_decomp=self.dc_decomp, - perturbation_domain=self.perturbation_domain, - mode=self.mode, - ) - for j in range(2, len(inputs_list)): - output = multiply( - output, - inputs_list[j], - dc_decomp=self.dc_decomp, - perturbation_domain=self.perturbation_domain, - mode=self.mode, - ) - - return output - - -class DecomonDot(DecomonMerge, Dot): - """LiRPA implementation of Dot layers. - See Keras official documentation for further details on the Dot operator - - """ - - original_keras_layer_class = Dot - - def __init__( - self, - axes: Union[int, tuple[int, int]] = (-1, -1), - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - axes=axes, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - if isinstance(axes, int): - self.axes = (axes, axes) - else: - self.axes = axes - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - if self.dc_decomp: - raise NotImplementedError() - - # splits the inputs - inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) - - #  check number of inputs - if len(inputs_list) == 1: # nothing to merge - return inputs - elif len(inputs_list) == 2: - inputs_0, inputs_1 = inputs_list - else: - raise NotImplementedError("This layer is not implemented to merge more than 2 layers.") - - input_shape_0 = self.inputs_outputs_spec.get_kerasinputshape(inputs_0) - input_shape_1 = self.inputs_outputs_spec.get_kerasinputshape(inputs_1) - n_0 = len(input_shape_0) - 2 - n_1 = len(input_shape_1) - 2 - - inputs_0 = permute_dimensions(inputs_0, self.axes[0], mode=self.mode, dc_decomp=self.dc_decomp) - inputs_1 = permute_dimensions(inputs_1, self.axes[1], mode=self.mode, dc_decomp=self.dc_decomp) - inputs_0 = broadcast( - inputs_0, n_1, -1, mode=self.mode, dc_decomp=self.dc_decomp, perturbation_domain=self.perturbation_domain - ) - inputs_1 = broadcast( - inputs_1, n_0, 2, mode=self.mode, dc_decomp=self.dc_decomp, perturbation_domain=self.perturbation_domain - ) - outputs_multiply = multiply( - inputs_0, inputs_1, dc_decomp=self.dc_decomp, perturbation_domain=self.perturbation_domain, mode=self.mode - ) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode( - outputs_multiply, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - if self.ibp: - u_c_out = K.sum(u_c, 1) - l_c_out = K.sum(l_c, 1) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if self.affine: - w_u_out = K.sum(w_u, 2) - b_u_out = K.sum(b_u, 1) - w_l_out = K.sum(w_l, 2) - b_l_out = K.sum(b_l, 1) - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if self.dc_decomp: - raise NotImplementedError() - else: - h_out, g_out = empty_tensor, empty_tensor - - return self.inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) diff --git a/src/decomon/layers/decomon_reshape.py b/src/decomon/layers/decomon_reshape.py deleted file mode 100644 index 6327406e..00000000 --- a/src/decomon/layers/decomon_reshape.py +++ /dev/null @@ -1,221 +0,0 @@ -from typing import Any, Optional, Union - -from keras.layers import InputSpec, Layer, Permute, Reshape -from keras.src.backend import rnn - -from decomon.core import ForwardMode, PerturbationDomain, get_affine, get_ibp -from decomon.layers.core import DecomonLayer -from decomon.types import BackendTensor - - -class DecomonReshape(DecomonLayer, Reshape): - """Forward LiRPA implementation of Reshape layers. - See Keras official documentation for further details on the Reshape operator - """ - - original_keras_layer_class = Reshape - - def __init__( - self, - target_shape: tuple[int, ...], - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - """ - Args: - data_format - **kwargs - """ - super().__init__( - target_shape=target_shape, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - if self.mode == ForwardMode.HYBRID: - self.input_spec = [ - InputSpec(min_ndim=1), # z - InputSpec(min_ndim=1), # u_c - InputSpec(min_ndim=1), # w_u - InputSpec(min_ndim=1), # b_u - InputSpec(min_ndim=1), # l_c - InputSpec(min_ndim=1), # w_l - InputSpec(min_ndim=1), # b_l - ] - elif self.mode == ForwardMode.IBP: - self.input_spec = [ - InputSpec(min_ndim=1), # u_c - InputSpec(min_ndim=1), # l_c - ] - if self.mode == ForwardMode.AFFINE: - self.input_spec = [ - InputSpec(min_ndim=1), # z - InputSpec(min_ndim=1), # w_u - InputSpec(min_ndim=1), # b_u - InputSpec(min_ndim=1), # w_l - InputSpec(min_ndim=1), # b_l - ] - - if self.dc_decomp: - self.input_spec += [InputSpec(min_ndim=1), InputSpec(min_ndim=1)] - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - def op(x: BackendTensor) -> BackendTensor: - return Reshape.call(self, x) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - if self.dc_decomp: - h_out = op(h) - g_out = op(g) - else: - h_out, g_out = empty_tensor, empty_tensor - - if self.ibp: - u_c_out = op(u_c) - l_c_out = op(l_c) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if self.affine: - b_u_out = op(b_u) - b_l_out = op(b_l) - - if len(w_u.shape) == len(b_u.shape): - w_u_out = op(w_u) - w_l_out = op(w_l) - - else: - - def step_func(x: BackendTensor, _: list[BackendTensor]) -> tuple[BackendTensor, list[BackendTensor]]: - return op(x), _ - - w_u_out = rnn(step_function=step_func, inputs=w_u, initial_states=[], unroll=False)[1] - w_l_out = rnn(step_function=step_func, inputs=w_l, initial_states=[], unroll=False)[1] - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - return self.inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -class DecomonPermute(DecomonLayer, Permute): - """Forward LiRPA implementation of Reshape layers. - See Keras official documentation for further details on the Reshape operator - """ - - original_keras_layer_class = Permute - - def __init__( - self, - dims: tuple[int, ...], - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - """ - Args: - data_format - **kwargs - """ - super().__init__( - dims=dims, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - if self.mode == ForwardMode.HYBRID: - self.input_spec = [ - InputSpec(min_ndim=1), # z - InputSpec(min_ndim=1), # u_c - InputSpec(min_ndim=1), # w_u - InputSpec(min_ndim=1), # b_u - InputSpec(min_ndim=1), # l_c - InputSpec(min_ndim=1), # w_l - InputSpec(min_ndim=1), # b_l - ] - elif self.mode == ForwardMode.IBP: - self.input_spec = [ - InputSpec(min_ndim=1), # u_c - InputSpec(min_ndim=1), # l_c - ] - elif self.mode == ForwardMode.AFFINE: - self.input_spec = [ - InputSpec(min_ndim=1), # z - InputSpec(min_ndim=1), # w_u - InputSpec(min_ndim=1), # b_u - InputSpec(min_ndim=1), # w_l - InputSpec(min_ndim=1), # b_l - ] - else: - raise ValueError(f"Unknown mode {mode}") - - if self.dc_decomp: - self.input_spec += [InputSpec(min_ndim=1), InputSpec(min_ndim=1)] - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - def op(x: BackendTensor) -> BackendTensor: - return Permute.call(self, x) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - if self.dc_decomp: - h_out = op(h) - g_out = op(g) - else: - h_out, g_out = empty_tensor, empty_tensor - - if self.ibp: - u_c_out = op(u_c) - l_c_out = op(l_c) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if self.affine: - b_u_out = op(b_u) - b_l_out = op(b_l) - - if len(w_u.shape) == len(b_u.shape): - w_u_out = op(w_u) - w_l_out = op(w_l) - else: - - def step_func(x: BackendTensor, _: list[BackendTensor]) -> tuple[BackendTensor, list[BackendTensor]]: - return op(x), _ - - w_u_out = rnn(step_function=step_func, inputs=w_u, initial_states=[], unroll=False)[1] - w_l_out = rnn(step_function=step_func, inputs=w_l, initial_states=[], unroll=False)[1] - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - return self.inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/maxpooling.py b/src/decomon/layers/maxpooling.py deleted file mode 100644 index c6998c20..00000000 --- a/src/decomon/layers/maxpooling.py +++ /dev/null @@ -1,241 +0,0 @@ -from typing import Any, Optional, Union - -import keras.ops as K -import numpy as np -from keras.layers import InputSpec, MaxPooling2D - -from decomon.core import ForwardMode, PerturbationDomain -from decomon.layers.core import DecomonLayer -from decomon.layers.utils import max_ -from decomon.types import BackendTensor - - -# step 1: compute the maximum -class DecomonMaxPooling2D(DecomonLayer, MaxPooling2D): - """LiRPA implementation of MaxPooling2D layers. - See Keras official documentation for further details on the MaxPooling2D operator - - """ - - original_keras_layer_class = MaxPooling2D - - pool_size: tuple[int, int] - strides: tuple[int, int] - padding: str - data_format: str - - def __init__( - self, - pool_size: Union[int, tuple[int, int]] = (2, 2), - strides: Optional[Union[int, tuple[int, int]]] = None, - padding: str = "valid", - data_format: Optional[str] = None, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - pool_size=pool_size, - strides=strides, - padding=padding, - data_format=data_format, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - if self.mode == ForwardMode.IBP: - self.input_spec = [ - InputSpec(ndim=4), # u - InputSpec(ndim=4), # l - ] - elif self.mode == ForwardMode.AFFINE: - self.input_spec = [ - InputSpec(min_ndim=2), # x - InputSpec(ndim=5), # w_u - InputSpec(ndim=4), # b_u - InputSpec(ndim=5), # w_l - InputSpec(ndim=4), - ] # b_l - elif self.mode == ForwardMode.HYBRID: - self.input_spec = [ - InputSpec(min_ndim=2), # x - InputSpec(ndim=4), # u - InputSpec(ndim=5), # w_u - InputSpec(ndim=4), # b_u - InputSpec(ndim=4), # l - InputSpec(ndim=5), # w_l - InputSpec(ndim=4), - ] # b_l - else: - raise ValueError(f"Unknown mode {self.mode}.") - - if self.dc_decomp: - self.input_spec += [InputSpec(ndim=4), InputSpec(ndim=4)] - - # express maxpooling with convolutions - self.filters = np.zeros( - (self.pool_size[0], self.pool_size[1], 1, int(np.prod(self.pool_size))), dtype=self.dtype - ) - for i in range(self.pool_size[0]): - for j in range(self.pool_size[1]): - self.filters[i, j, 0, i * self.pool_size[0] + j] = 1 - - self.filters_w = self.filters[None] - self.strides_w = (1,) + self.strides - - def conv_(x: BackendTensor) -> BackendTensor: - if self.data_format in [None, "channels_last"]: - return K.cast( - K.expand_dims( - K.conv( - x, - self.filters, - strides=list(self.strides), - padding=self.padding, - data_format=self.data_format, - ), - -2, - ), - self.dtype, - ) - else: - return K.cast( - K.expand_dims( - K.conv( - x, - self.filters, - strides=list(self.strides), - padding=self.padding, - data_format=self.data_format, - ), - 1, - ), - self.dtype, - ) - - def conv_w_(x: BackendTensor) -> BackendTensor: - if self.data_format in [None, "channels_last"]: - return K.cast( - K.expand_dims( - K.conv( - x, - self.filters_w, - strides=list(self.strides_w), - padding=self.padding, - data_format=self.data_format, - ), - -2, - ), - self.dtype, - ) - else: - return K.cast( - K.expand_dims( - K.conv( - x, - self.filters_w, - strides=list(self.strides_w), - padding=self.padding, - data_format=self.data_format, - ), - 1, - ), - self.dtype, - ) - - self.internal_op = conv_ - self.internal_op_w = conv_w_ - - def _pooling_function_fast( - self, - inputs: list[BackendTensor], - ) -> list[BackendTensor]: - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - dtype = x.dtype - empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - l_c_out = K.max_pool(l_c, self.pool_size, self.strides, self.padding, self.data_format) - u_c_out = K.max_pool(u_c, self.pool_size, self.strides, self.padding, self.data_format) - - if self.affine: - n_in = x.shape[-1] - w_u_out = K.concatenate([0 * K.expand_dims(u_c_out, 1)] * n_in, 1) - w_l_out = w_u_out - b_u_out = u_c_out - b_l_out = l_c_out - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if self.dc_decomp: - raise NotImplementedError() - else: - h_out, g_out = empty_tensor, empty_tensor - - return self.inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - def _pooling_function_not_fast( - self, - inputs: list[BackendTensor], - ) -> list[BackendTensor]: - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) - input_shape = self.inputs_outputs_spec.get_kerasinputshape(inputs) - n_split = input_shape[-1] - - if self.dc_decomp: - h_out = K.concatenate( - [self.internal_op(elem) for elem in K.split(h, n_split, -1)], - -2, - ) - g_out = K.concatenate( - [self.internal_op(elem) for elem in K.split(g, n_split, -1)], - -2, - ) - else: - h_out, g_out = empty_tensor, empty_tensor - - if self.ibp: - u_c_out = K.concatenate([self.internal_op(elem) for elem in K.split(u_c, n_split, -1)], -2) - l_c_out = K.concatenate([self.internal_op(elem) for elem in K.split(l_c, n_split, -1)], -2) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if self.affine: - b_u_out = K.concatenate([self.internal_op(elem) for elem in K.split(b_u, n_split, -1)], -2) - b_l_out = K.concatenate([self.internal_op(elem) for elem in K.split(b_l, n_split, -1)], -2) - w_u_out = K.concatenate([self.internal_op_w(elem) for elem in K.split(w_u, n_split, -1)], -2) - w_l_out = K.concatenate([self.internal_op_w(elem) for elem in K.split(w_l, n_split, -1)], -2) - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - outputs = self.inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - return max_( - outputs, axis=-1, dc_decomp=self.dc_decomp, mode=self.mode, perturbation_domain=self.perturbation_domain - ) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - if self.fast: - return self._pooling_function_fast(inputs) - else: - return self._pooling_function_not_fast(inputs) - - -# Aliases -DecomonMaxPool2d = DecomonMaxPooling2D diff --git a/src/decomon/layers/merging/__init__.py b/src/decomon/layers/merging/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/merging/add.py b/src/decomon/layers/merging/add.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/merging/average.py b/src/decomon/layers/merging/average.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/merging/concatenate.py b/src/decomon/layers/merging/concatenate.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/merging/dot.py b/src/decomon/layers/merging/dot.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/merging/maximum.py b/src/decomon/layers/merging/maximum.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/merging/minimum.py b/src/decomon/layers/merging/minimum.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/merging/multiply.py b/src/decomon/layers/merging/multiply.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/merging/subtract.py b/src/decomon/layers/merging/subtract.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/normalization/__init__.py b/src/decomon/layers/normalization/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/normalization/batch_normalization.py b/src/decomon/layers/normalization/batch_normalization.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/pooling/__init__.py b/src/decomon/layers/pooling/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/pooling/max_pooling2d.py b/src/decomon/layers/pooling/max_pooling2d.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/regularization/__init__.py b/src/decomon/layers/regularization/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/regularization/dropout.py b/src/decomon/layers/regularization/dropout.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/reshaping/__init__.py b/src/decomon/layers/reshaping/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/reshaping/flatten.py b/src/decomon/layers/reshaping/flatten.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/reshaping/permute.py b/src/decomon/layers/reshaping/permute.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/reshaping/reshape.py b/src/decomon/layers/reshaping/reshape.py new file mode 100644 index 00000000..e69de29b diff --git a/src/decomon/layers/utils.py b/src/decomon/layers/utils.py deleted file mode 100644 index 7bb09d6a..00000000 --- a/src/decomon/layers/utils.py +++ /dev/null @@ -1,1170 +0,0 @@ -from typing import Any, Optional, Union - -import keras -import keras.ops as K -import numpy as np -from keras.config import epsilon -from keras.constraints import Constraint -from keras.initializers import Initializer -from keras.layers import Layer - -from decomon.core import ( - BallDomain, - BoxDomain, - ForwardMode, - InputsOutputsSpec, - PerturbationDomain, - Slope, - get_affine, - get_ibp, -) -from decomon.types import BackendTensor, Tensor -from decomon.utils import add, get_linear_softplus_hull, maximum, minimum, minus, relu_ - - -def is_a_merge_layer(layer: Layer) -> bool: - return hasattr(layer, "_merge_function") - - -class NonPos(Constraint): - """Constrains the weights to be non-negative.""" - - def __call__(self, w: BackendTensor) -> BackendTensor: - return K.minimum(w, K.cast(0.0, dtype=w.dtype)) - - -class NonNeg(Constraint): - """Constrains the weights to be non-negative.""" - - def __call__(self, w: BackendTensor) -> BackendTensor: - return K.maximum(w, K.cast(0.0, dtype=w.dtype)) - - -class ClipAlpha(Constraint): - """Cosntraints the weights to be between 0 and 1.""" - - def __call__(self, w: BackendTensor) -> BackendTensor: - return K.clip(w, 0.0, 1.0) - - -class ClipAlphaGrid(Constraint): - """Cosntraints the weights to be between 0 and 1.""" - - def __call__(self, w: BackendTensor) -> BackendTensor: - w = K.clip(w, 0.0, 1.0) - w /= K.maximum(K.sum(w, 0), K.cast(1.0, dtype=w.dtype))[None] - return w - - -class ClipAlphaAndSumtoOne(Constraint): - """Cosntraints the weights to be between 0 and 1.""" - - def __call__(self, w: BackendTensor) -> BackendTensor: - w = K.clip(w, 0.0, 1.0) - # normalize the first colum to 1 - w_scale = K.maximum(K.sum(w, 0), K.cast(epsilon(), dtype=w.dtype)) - return w / w_scale[:, None, None, None] - - -class MultipleConstraint(Constraint): - """stacking multiple constraints""" - - def __init__(self, constraint_0: Optional[Constraint], constraint_1: Constraint, **kwargs: Any): - super().__init__(**kwargs) - if constraint_0: - self.constraints = [constraint_0, constraint_1] - else: - self.constraints = [constraint_1] - - def __call__(self, w: BackendTensor) -> BackendTensor: - for c in self.constraints: - w = c.__call__(w) - - return w - - -class Project_initializer_pos(Initializer): - """Initializer that generates tensors initialized to 1.""" - - def __init__(self, initializer: Initializer, **kwargs: Any): - super().__init__(**kwargs) - self.initializer = initializer - - def __call__(self, shape: tuple[Optional[int], ...], dtype: Optional[str] = None, **kwargs: Any) -> BackendTensor: - w = self.initializer.__call__(shape, dtype) - return K.maximum(K.cast(0.0, dtype=dtype), w) - - -class Project_initializer_neg(Initializer): - """Initializer that generates tensors initialized to 1.""" - - def __init__(self, initializer: Initializer, **kwargs: Any): - super().__init__(**kwargs) - self.initializer = initializer - - def __call__(self, shape: tuple[Optional[int], ...], dtype: Optional[str] = None, **kwargs: Any) -> BackendTensor: - w = self.initializer.__call__(shape, dtype) - return K.minimum(K.cast(0.0, dtype=dtype), w) - - -def softplus_( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - dtype = x.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - u_c_out = K.softplus(u_c) - l_c_out = K.softplus(l_c) - - if mode in [ForwardMode.AFFINE, ForwardMode.HYBRID]: - w_u_out, b_u_out, w_l_out, b_l_out = get_linear_softplus_hull(upper=u_c, lower=l_c, slope=slope, **kwargs) - b_u_out = w_u_out * b_u + b_u_out - b_l_out = w_l_out * b_l + b_l_out - w_u_out = K.expand_dims(w_u_out, 1) * w_u - w_l_out = K.expand_dims(w_l_out, 1) * w_l - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if dc_decomp: - raise NotImplementedError("Not yet implemented for dc_decomp=True") - else: - h_out, g_out = empty_tensor, empty_tensor - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -def sum( - inputs: list[Tensor], - axis: int = -1, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - **kwargs: Any, -) -> list[Tensor]: - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - ibp = get_ibp(mode) - affine = get_affine(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - if ibp: - u_c_out = K.sum(u_c, axis=axis) - l_c_out = K.sum(l_c, axis=axis) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if affine: - if axis == -1: - axis_w = -1 - else: - axis_w = axis + 1 - w_u_out = K.sum(w_u, axis=axis_w) - w_l_out = K.sum(w_l, axis=axis_w) - b_u_out = K.sum(b_u, axis=axis) - b_l_out = K.sum(b_l, axis=axis) - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if dc_decomp: - raise NotImplementedError("Not yet implemented for dc_decomp=True") - else: - h_out, g_out = empty_tensor, empty_tensor - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -def frac_pos( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - **kwargs: Any, -) -> list[Tensor]: - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - dtype = x.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - u_c_out = K.cast(1.0, dtype=l_c.dtype) / l_c - l_c_out = K.cast(1.0, dtype=u_c.dtype) / u_c - - if affine: - w_u_0 = (u_c_out - l_c_out) / K.maximum(u_c - l_c, K.cast(epsilon(), dtype=u_c.dtype)) - b_u_0 = l_c_out - w_u_0 * l_c - - y = (u_c + l_c) / 2.0 - b_l_0 = K.cast(2.0, dtype=y.dtype) / y - w_l_0 = -K.cast(1.0, dtype=y.dtype) / y**2 - - w_u_out = w_u_0[:, None] * w_l - b_u_out = b_u_0 * b_l + b_u_0 - w_l_out = w_l_0[:, None] * w_u - b_l_out = b_l_0 * b_u + b_l_0 - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if dc_decomp: - raise NotImplementedError("Not yet implemented for dc_decomp=True") - else: - h_out, g_out = empty_tensor, empty_tensor - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -# convex hull of the maximum between two functions -def max_( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - axis: int = -1, - finetune: bool = False, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA implementation of max(x, axis) - - Args: - inputs: list of tensors - dc_decomp: boolean that indicates - perturbation_domain: the type of perturbation domain - axis: axis to perform the maximum - whether we return a difference of convex decomposition of our layer - - Returns: - max operation along an axis - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - inputs_outputs_spec_no_dc = InputsOutputsSpec(dc_decomp=False, mode=mode, perturbation_domain=perturbation_domain) - - input_shape = inputs[-1].shape - max_dim = input_shape[axis] - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, tight=False, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - empty_tensors_list = [empty_tensor] * max_dim - - if mode == ForwardMode.IBP and not dc_decomp: - u_c_out = K.max(u_c, axis=axis) - l_c_out = K.max(l_c, axis=axis) - - return [u_c_out, l_c_out] - - # do some transpose so that the last axis is also at the end - - if ibp: - u_c_list = K.split(u_c, max_dim, axis) - l_c_list = K.split(l_c, max_dim, axis) - u_c_tmp = u_c_list[0] + 0 * (u_c_list[0]) - l_c_tmp = l_c_list[0] + 0 * (l_c_list[0]) - else: - u_c_tmp, l_c_tmp = empty_tensor, empty_tensor - u_c_list, l_c_list = empty_tensors_list, empty_tensors_list - - if affine: - b_u_list = K.split(b_u, max_dim, axis) - b_l_list = K.split(b_l, max_dim, axis) - b_u_tmp = b_u_list[0] + 0 * (b_u_list[0]) - b_l_tmp = b_l_list[0] + 0 * (b_l_list[0]) - - if axis == -1: - w_u_list = K.split(w_u, max_dim, axis) - w_l_list = K.split(w_l, max_dim, axis) - else: - w_u_list = K.split(w_u, max_dim, axis + 1) - w_l_list = K.split(w_l, max_dim, axis + 1) - w_u_tmp = w_u_list[0] + 0 * (w_u_list[0]) - w_l_tmp = w_l_list[0] + 0 * (w_l_list[0]) - - if finetune: - params = kwargs["finetune_params"] - params_split = [e[0] for e in K.split(params[None], max_dim, axis)] - else: - params_split = [empty_tensor] * max_dim - - else: - w_u_tmp, b_u_tmp, w_l_tmp, b_l_tmp = empty_tensor, empty_tensor, empty_tensor, empty_tensor - w_u_list, b_u_list, w_l_list, b_l_list = ( - empty_tensors_list, - empty_tensors_list, - empty_tensors_list, - empty_tensors_list, - ) - params_split = [False] * max_dim - - h_tmp, g_tmp = None, None - output_tmp = inputs_outputs_spec_no_dc.extract_outputsformode_from_fulloutputs( - [ - x, - u_c_tmp, - w_u_tmp, - b_u_tmp, - l_c_tmp, - w_l_tmp, - b_l_tmp, - h_tmp, - g_tmp, - ] - ) - for i in range(1, max_dim): - output_i = inputs_outputs_spec_no_dc.extract_outputsformode_from_fulloutputs( - [x, u_c_list[i], w_u_list[i], b_u_list[i], l_c_list[i], w_l_list[i], b_l_list[i], None, None] - ) - output_tmp = maximum( - output_tmp, - output_i, - dc_decomp=False, - mode=mode, - finetune=finetune, - finetune_params=params_split[i], - perturbation_domain=perturbation_domain, - ) - - # reduce the dimension - tight = not (mode == ForwardMode.AFFINE) # no need to compute u_c, l_c for pure affine mode - ( - _, - u_c_out, - w_u_out, - b_u_out, - l_c_out, - w_l_out, - b_l_out, - _, - _, - ) = inputs_outputs_spec_no_dc.get_fullinputs_from_inputsformode(output_tmp, tight=tight) - - if mode in [ForwardMode.IBP, ForwardMode.HYBRID]: - u_c_out = K.squeeze(u_c_out, axis) - l_c_out = K.squeeze(l_c_out, axis) - if mode in [ForwardMode.HYBRID, ForwardMode.AFFINE]: - b_u_out = K.squeeze(b_u_out, axis) - b_l_out = K.squeeze(b_l_out, axis) - if axis == -1: - w_u_out = K.squeeze(w_u_out, axis) - w_l_out = K.squeeze(w_l_out, axis) - else: - w_u_out = K.squeeze(w_u_out, axis + 1) - w_l_out = K.squeeze(w_l_out, axis + 1) - - if dc_decomp: - g_out = K.sum(g, axis=axis) - h_out = K.max(h + g, axis=axis) - g_out - else: - g_out, h_out = empty_tensor, empty_tensor - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -def softmax_to_linear(model: keras.Model) -> tuple[keras.Model, bool]: - """linearize the softmax layer for verification - - Args: - model: Keras Model - - Returns: - model without the softmax - """ - layer = model.layers[-1] - # check that layer is not an instance of the object Softmax - if isinstance(layer, keras.layers.Softmax): - model_normalize = keras.models.Model(model.get_input_at(0), keras.layers.Activation("linear")(layer.input)) - - return model_normalize, True - - if hasattr(layer, "activation"): - if not layer.get_config()["activation"] == "softmax": - return model, False - layer.get_config()["activation"] = "linear" - layer.activation = keras.activations.get("linear") - return model, True - - return model, False - - -def linear_to_softmax(model: keras.Model) -> tuple[keras.Model, bool]: - model.layers[-1].activation = keras.activations.get("softmax") - return model - - -def multiply( - inputs_0: list[Tensor], - inputs_1: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> list[Tensor]: - """LiRPA implementation of (element-wise) multiply(x,y)=-x*y. - - Args: - inputs_0: list of tensors - inputs_1: list of tensors - dc_decomp: boolean that indicates - perturbation_domain: the type of perturbation domain - mode: type of Forward propagation (ibp, affine, or hybrid) - whether we return a difference of convex decomposition of our layer - whether we propagate upper and lower bounds on the values of the gradient - - Returns: - maximum(inputs_0, inputs_1) - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x_0, u_c_0, w_u_0, b_u_0, l_c_0, w_l_0, b_l_0, h_0, g_0 = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs_0 - ) - x_1, u_c_1, w_u_1, b_u_1, l_c_1, w_l_1, b_l_1, h_1, g_1 = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs_1 - ) - dtype = x_0.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - # using McCormick's inequalities to derive bounds - # xy<= x_u*y + x*y_L - xU*y_L - # xy<= x*y_u + x_L*y - x_L*y_U - - # xy >=x_L*y + x*y_L -x_L*y_L - # xy >= x_U*y + x*y_U - x_U*y_U - - if ibp: - u_c_0_out = K.maximum(u_c_0 * u_c_1, u_c_0 * l_c_1) + K.maximum(u_c_0 * l_c_1, l_c_0 * l_c_1) - u_c_0 * l_c_1 - u_c_1_out = K.maximum(u_c_1 * u_c_0, u_c_1 * l_c_0) + K.maximum(u_c_1 * l_c_0, l_c_1 * l_c_0) - u_c_1 * l_c_0 - u_c_out = K.minimum(u_c_0_out, u_c_1_out) - l_c_out = K.minimum(l_c_0 * l_c_1, l_c_0 * u_c_1) + K.minimum(l_c_0 * l_c_1, u_c_0 * l_c_1) - l_c_0 * l_c_1 - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if affine: - z_value = K.cast(0.0, dtype=u_c_0.dtype) - # xy <= x_u * y + x * y_L - xU * y_L - cx_u_pos = K.maximum(u_c_0, z_value) - cx_u_neg = K.minimum(u_c_0, z_value) - - cy_l_pos = K.maximum(l_c_1, z_value) - cy_l_neg = K.minimum(l_c_1, z_value) - w_u_out = ( - cx_u_pos[:, None] * w_u_1 - + cx_u_neg[:, None] * w_l_1 - + cy_l_pos[:, None] * w_u_0 - + cy_l_neg[:, None] * w_l_0 - ) - b_u_out = cx_u_pos * b_u_1 + cx_u_neg * b_l_1 + cy_l_pos * b_u_0 + cy_l_neg * b_l_0 - u_c_0 * l_c_1 - - # xy >= x_U*y + x*y_U - x_U*y_U - cx_l_pos = K.maximum(l_c_0, z_value) - cx_l_neg = K.minimum(l_c_0, z_value) - - w_l_out = ( - cx_l_pos[:, None] * w_l_1 - + cx_l_neg[:, None] * w_u_1 - + cy_l_pos[:, None] * w_l_0 - + cy_l_neg[:, None] * w_u_0 - ) - b_l_out = cx_l_pos * b_l_1 + cx_l_neg * b_u_1 + cy_l_pos * b_l_0 + cy_l_neg * b_u_0 - l_c_0 * l_c_1 - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if dc_decomp: - raise NotImplementedError("Not yet implemented for dc_decomp=True") - else: - h_out, g_out = empty_tensor, empty_tensor - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x_0, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -def permute_dimensions( - inputs: list[Tensor], - axis: int, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - axis_perm: int = 1, - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, -) -> list[Tensor]: - """LiRPA implementation of (element-wise) permute(x,axis) - - Args: - inputs: list of input tensors - axis: axis on which we apply the permutation - mode: type of Forward propagation (ibp, affine, or hybrid) - axis_perm: see DecomonPermute operator - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, tight=False, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - input_shape = inputs_outputs_spec.get_kerasinputshape(inputs) - if len(input_shape) <= 2: - # not enough dim to permute - return inputs - index_np = np.arange(len(input_shape)) - index = tuple(np.insert(np.delete(index_np, axis), axis_perm, axis)) - index_w_np = np.arange(len(input_shape) + 1) - index_w = tuple(np.insert(np.delete(index_w_np, axis), axis_perm + 1, axis)) - - if ibp: - u_c_out = K.transpose(u_c, index) - l_c_out = K.transpose(l_c, index) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - if affine: - w_u_out = K.transpose(w_u, index_w) - b_u_out = K.transpose(b_u, index) - w_l_out = K.transpose(w_l, index_w) - b_l_out = K.transpose(b_l, index) - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if dc_decomp: - raise NotImplementedError("Not yet implemented for dc_decomp=True") - else: - h_out, g_out = empty_tensor, empty_tensor - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -def broadcast( - inputs: list[Tensor], - n: int, - axis: int, - mode: Union[str, ForwardMode], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, -) -> list[Tensor]: - """LiRPA implementation of broadcasting - - Args: - inputs - n - axis - mode - - Returns: - - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, tight=False, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - if ibp: - for _ in range(n): - u_c = K.expand_dims(u_c, axis) - l_c = K.expand_dims(l_c, axis) - - if axis != -1: - axis_w = axis + 1 - else: - axis_w = -1 - - if affine: - for _ in range(n): - b_u = K.expand_dims(b_u, axis) - b_l = K.expand_dims(b_l, axis) - w_u = K.expand_dims(w_u, axis_w) - w_l = K.expand_dims(w_l, axis_w) - - if dc_decomp: - raise NotImplementedError("Not yet implemented for dc_decomp=True") - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs([x, u_c, w_u, b_u, l_c, w_l, b_l, h, g]) - - -def split( - inputs: list[Tensor], - axis: int = -1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, -) -> list[list[Tensor]]: - """LiRPA implementation of split - - Args: - inputs - axis - mode - - Returns: - - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - input_shape = inputs_outputs_spec.get_kerasinputshape(inputs) - if axis == -1: - n = input_shape[-1] - axis = len(input_shape) - 1 - else: - n = input_shape[axis] - if n is None: - raise ValueError(f"Dimension {axis} corresponding to `axis` cannot be None") - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, tight=False, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - empty_tensor_list = [empty_tensor] * n - - if ibp: - u_c_list = K.split(u_c, num_or_size_splits=n, axis=axis) - l_c_list = K.split(l_c, num_or_size_splits=n, axis=axis) - else: - u_c_list, l_c_list = empty_tensor_list, empty_tensor_list - - if affine: - b_u_list = K.split(b_u, num_or_size_splits=n, axis=axis) - b_l_list = K.split(b_l, num_or_size_splits=n, axis=axis) - - if axis != -1: - axis += 1 - w_u_list = K.split(w_u, num_or_size_splits=n, axis=axis) - w_l_list = K.split(w_l, num_or_size_splits=n, axis=axis) - else: - w_u_list, b_u_list, w_l_list, b_l_list = ( - empty_tensor_list, - empty_tensor_list, - empty_tensor_list, - empty_tensor_list, - ) - - if dc_decomp: - raise NotImplementedError("Not yet implemented for dc_decomp=True") - else: - h_list, g_list = empty_tensor_list, empty_tensor_list - - return [ - inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_list[i], w_u_list[i], b_u_list[i], l_c_list[i], w_l_list[i], b_l_list[i], h_list[i], g_list[i]] - ) - for i in range(n) - ] - - -def sort( - inputs: list[Tensor], - axis: int = -1, - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> list[Tensor]: - """LiRPA implementation of sort by selection - - Args: - inputs - axis - dc_decomp - perturbation_domain - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, tight=False, compute_ibp_from_affine=False - ) - input_shape = inputs_outputs_spec.get_kerasinputshape(inputs) - dtype = x.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - if axis == -1: - n = input_shape[-1] - axis = len(input_shape) - 1 - else: - n = input_shape[axis] - if n is None: - raise ValueError(f"Dimension {axis} corresponding to `axis` cannot be None") - - empty_tensor_list = [empty_tensor] * n - - # what about splitting elements - op_split = lambda x: K.split(x, n, axis=axis) - if ibp: - u_c_list = op_split(u_c) - l_c_list = op_split(l_c) - else: - u_c_list, l_c_list = empty_tensor_list, empty_tensor_list - - if affine: - w_u_list = K.split(w_u, n, axis=axis + 1) - b_u_list = op_split(b_u) - w_l_list = K.split(w_l, n, axis=axis + 1) - b_l_list = op_split(b_l) - else: - w_u_list, b_u_list, w_l_list, b_l_list = ( - empty_tensor_list, - empty_tensor_list, - empty_tensor_list, - empty_tensor_list, - ) - - if dc_decomp: - raise NotImplementedError("Not yet implemented for dc_decomp=True") - else: - h_list, g_list = empty_tensor_list, empty_tensor_list - - # use selection sort - for i in range(n - 1): - for j in range(i + 1, n): - input_i = inputs_outputs_spec.extract_inputsformode_from_fullinputs( - [x, u_c_list[i], w_u_list[i], b_u_list[i], l_c_list[i], w_l_list[i], b_l_list[i], h_list[i], g_list[i]] - ) - input_j = inputs_outputs_spec.extract_inputsformode_from_fullinputs( - [x, u_c_list[j], w_u_list[j], b_u_list[j], l_c_list[j], w_l_list[j], b_l_list[j], h_list[j], g_list[j]] - ) - - # max and min of splitted inputs - output_a = maximum( - input_i, input_j, mode=mode, perturbation_domain=perturbation_domain, dc_decomp=dc_decomp - ) - output_b = minimum( - input_i, input_j, mode=mode, perturbation_domain=perturbation_domain, dc_decomp=dc_decomp - ) - - # update lists - ( - x, - u_c_list[j], - w_u_list[j], - b_u_list[j], - l_c_list[j], - w_l_list[j], - b_l_list[j], - h_list[j], - g_list[j], - ) = inputs_outputs_spec.get_fullinputs_from_inputsformode(output_a) - ( - x, - u_c_list[i], - w_u_list[i], - b_u_list[i], - l_c_list[i], - w_l_list[i], - b_l_list[i], - h_list[i], - g_list[i], - ) = inputs_outputs_spec.get_fullinputs_from_inputsformode(output_b) - - op_concatenate = lambda x: K.concatenate(x, axis) - # update the inputs - if ibp: - u_c_out = op_concatenate(u_c_list) - l_c_out = op_concatenate(l_c_list) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - if affine: - w_u_out = K.concatenate(w_u_list, axis + 1) - w_l_out = K.concatenate(w_l_list, axis + 1) - b_u_out = op_concatenate(b_u_list) - b_l_out = op_concatenate(b_l_list) - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if dc_decomp: - raise NotImplementedError("Not yet implemented for dc_decomp=True") - else: - h_out, g_out = empty_tensor, empty_tensor - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -def pow( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> list[Tensor]: - """LiRPA implementation of pow(x )=x**2 - - Args: - inputs - dc_decomp - perturbation_domain - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - return multiply(inputs, inputs, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode) - - -def abs( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> list[Tensor]: - """LiRPA implementation of |x| - - Args: - inputs - dc_decomp - perturbation_domain - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - inputs_0 = relu_(inputs, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode) - inputs_1 = minus( - relu_( - minus(inputs, mode=mode, perturbation_domain=perturbation_domain), - dc_decomp=dc_decomp, - perturbation_domain=perturbation_domain, - mode=mode, - ), - mode=mode, - perturbation_domain=perturbation_domain, - ) - - return add(inputs_0, inputs_1, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode) - - -def frac_pos_hull( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> list[Tensor]: - """LiRPA implementation of 1/x for x>0 - - Args: - inputs - dc_decomp - perturbation_domain - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - - l_c = K.maximum(l_c, K.cast(1.0, dtype=l_c.dtype)) - z = (u_c + l_c) / 2.0 - w_l = -K.cast(1.0, dtype=z.dtype) / K.power(z) - b_l = K.cast(2.0, dtype=z.dtype) / z - w_u = (K.cast(1.0, dtype=u_c.dtype) / u_c - K.cast(1.0, dtype=l_c.dtype) / l_c) / (u_c - l_c) - b_u = K.cast(1.0, dtype=u_c.dtype) / u_c - w_u * u_c - - return [w_u, b_u, w_l, b_l] - - -# convex hull for min -def min_( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - axis: int = -1, - finetune: bool = False, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA implementation of min(x, axis=axis) - - Args: - inputs - dc_decomp - perturbation_domain - mode - axis - - Returns: - - """ - # return - max - x - if perturbation_domain is None: - perturbation_domain = BoxDomain() - return minus( - max_( - minus(inputs, mode=mode, perturbation_domain=perturbation_domain), - dc_decomp=dc_decomp, - perturbation_domain=perturbation_domain, - mode=mode, - axis=axis, - finetune=finetune, - **kwargs, - ), - mode=mode, - perturbation_domain=perturbation_domain, - ) - - -def expand_dims( - inputs: list[Tensor], - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - axis: int = -1, - perturbation_domain: Optional[PerturbationDomain] = None, - **kwargs: Any, -) -> list[Tensor]: - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, tight=False, compute_ibp_from_affine=False - ) - dtype = x.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - if axis == -1: - axis_w = axis - else: - axis_w = axis + 1 - - op = lambda t: K.expand_dims(t, axis) - - if ibp: - u_c_out = op(u_c) - l_c_out = op(l_c) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if affine: - b_u_out = op(b_u) - b_l_out = op(b_l) - w_u_out = K.expand_dims(w_u, axis_w) - w_l_out = K.expand_dims(w_l, axis_w) - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if dc_decomp: - raise NotImplementedError("Not yet implemented for dc_decomp=True") - else: - h_out, g_out = empty_tensor, empty_tensor - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -def log( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - **kwargs: Any, -) -> list[Tensor]: - """Exponential activation function. - - Args: - inputs: list of input tensors - dc_decomp: boolean that indicates - perturbation_domain: the type of convex input domain - mode: type of Forward propagation (ibp, affine, or hybrid) - **kwargs: extra parameters - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - dtype = x.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - if mode == ForwardMode.AFFINE: - l_c = K.maximum(K.cast(epsilon(), dtype=l_c.dtype), l_c) - - u_c_out = K.log(u_c) - l_c_out = K.log(l_c) - - if affine: - y = (u_c + l_c) / 2.0 - - w_l_0 = (u_c_out - l_c_out) / K.maximum(u_c - l_c, K.cast(epsilon(), dtype=u_c.dtype)) - b_l_0 = l_c_out - w_l_0 * l_c - - w_u_0 = K.cast(1, dtype=y.dtype) / y - b_u_0 = K.log(y) - 1 - - w_u_out = w_u_0[:, None] * w_u - b_u_out = w_u_0 * b_u + b_u_0 - w_l_out = w_l_0[:, None] * w_l - b_l_out = w_l_0 * b_l + b_l_0 - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if dc_decomp: - raise NotImplementedError("Not yet implemented for dc_decomp=True") - else: - h_out, g_out = empty_tensor, empty_tensor - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -def exp( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - """Exponential activation function. - - Args: - inputs: list of input tensors - dc_decomp: boolean that indicates - perturbation_domain: the type of convex input domain - mode: type of Forward propagation (ibp, affine, or hybrid) - slope: - **kwargs: extra parameters - whether we return a difference of convex decomposition of our layer - - Returns: - the updated list of tensors - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - dtype = x.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - u_c_out = K.exp(u_c) - l_c_out = K.exp(l_c) - - if affine: - y = (u_c + l_c) / 2.0 # do finetuneting - - w_u_0 = (u_c_out - l_c_out) / K.maximum(u_c - l_c, K.cast(epsilon(), dtype=u_c.dtype)) - b_u_0 = l_c_out - w_u_0 * l_c - - w_l_0 = K.exp(y) - b_l_0 = w_l_0 * (-y + 1) - - w_u_out = w_u_0[:, None] * w_u - b_u_out = w_u_0 * b_u + b_u_0 - w_l_out = w_l_0[:, None] * w_l - b_l_out = w_l_0 * b_l + b_l_0 - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if dc_decomp: - raise NotImplementedError("Not yet implemented for dc_decomp=True") - else: - h_out, g_out = empty_tensor, empty_tensor - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) diff --git a/src/decomon/layers/utils_pooling.py b/src/decomon/layers/utils_pooling.py deleted file mode 100644 index 541d5eb6..00000000 --- a/src/decomon/layers/utils_pooling.py +++ /dev/null @@ -1,194 +0,0 @@ -from typing import Any, Optional, Union - -import keras.ops as K -import numpy as np - -from decomon.core import ( - BoxDomain, - ForwardMode, - InputsOutputsSpec, - PerturbationDomain, - get_affine, -) -from decomon.types import BackendTensor, Tensor - -# step 1: compute (x_i, y_i) such that x_i[j]=l_j if j==i else u_j -# dataset of size n+1 on which we can compute an affine bound - - -def get_upper_linear_hull_max( - inputs: list[Tensor], - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - axis: int = -1, - dc_decomp: bool = False, - **kwargs: Any, -) -> list[Tensor]: - """Compute the linear hull that overapproximates max along the axis dimension - - Args: - inputs: list of input tensors - mode: type of Forward propagation (ibp, affine, or hybrid). Default to hybrid. - perturbation_domain (optional): type of perturbation domain that encompass the set of perturbations. Defaults to None. - axis (optional): Defaults to -1. See Keras offical documentation backend.max(., axis) - - Raises: - NotImplementedError: axis <0 and axis!=-1 - - Returns: - list of output tensors. The upper linear relaxation of max(., axis) in the mode format - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - affine = get_affine(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - dtype = x.dtype - input_shape = inputs_outputs_spec.get_kerasinputshape(inputs) - - # attention if axis=-1 or axis=n - dtype32 = "float32" - o_value = K.cast(1.0, dtype) - z_value = K.cast(0.0, dtype) - if axis == len(input_shape): - axis = -1 - if axis != -1 and axis < 0: - raise NotImplementedError() # to do - - # get the shape of the dimension - n_dim = input_shape[axis] - if n_dim is None: - raise ValueError(f"Dimension {axis} corresponding to `axis` cannot be None") - - # expand dim/broadcast - mask = K.identity(n_dim, dtype=dtype) # (n_dim, n_dim) - mask_shape = [1] * (len(u_c.shape) + 1) - mask_shape[-1] = n_dim - if axis != -1: - mask_shape[axis] = n_dim - else: - mask_shape[-2] = n_dim - mask = K.reshape(mask, mask_shape) # (1, shape, n_dim, n_dim) - - l_c_reshaped = K.expand_dims(l_c, -1) # (1, shape, n_dim, 1) - u_c_reshaped = K.expand_dims(u_c, -1) # (1, shape, n_dim, 1) - - # detect collapsed dimensions: lower[i]==upper[i] - index_collapse = o_value - K.sign(l_c_reshaped - u_c_reshaped) - - corners = mask * l_c_reshaped + (o_value - mask) * (u_c_reshaped) # (1, shape, n_dim, n_dim) - corners_collapse = mask * l_c_reshaped + (o_value - mask) * (u_c_reshaped + index_collapse) - # add the corners containing all the upper bounds - corners_collapse = K.concatenate( - [corners_collapse, u_c_reshaped + index_collapse], axis=-1 - ) # (None, shape, n_dim, n_dim+1) - corners = K.concatenate([corners, u_c_reshaped], axis=-1) - - if axis != -1: - corners_pred = K.max(corners, axis=axis) # (None, shape_, n_dim+1) - else: - corners_pred = K.max(corners, axis=-2) # (None, shape_, n_dim+1) - - # include bias in corners - if axis != -1: - bias_corner = o_value + K.sum(z_value * corners, axis, keepdims=True) - corners_collapse = K.concatenate([corners_collapse, bias_corner], axis=axis) # (None, shape_, n_dim+1, n_dim+1) - - else: - bias_corner = o_value + K.sum(z_value * corners, -2, keepdims=True) - corners_collapse = K.concatenate([corners_collapse, bias_corner], axis=-2) - - dimensions = list(range(len(corners.shape))) - if axis != -1: - dim_permutation = dimensions[:axis] + dimensions[axis + 1 :] + [dimensions[axis]] - else: - dim_permutation = dimensions[:-2] + dimensions[-1:] + [dimensions[-2]] - - corners_collapse = K.transpose(corners_collapse, dim_permutation) - # tf.linalg.solve works only for float32 - if dtype != dtype32: - corners_collapse = K.cast(corners_collapse, dtype32) - corners_pred = K.cast(corners_pred, dtype32) - w_hull = K.solve(corners_collapse, K.expand_dims(corners_pred, -1)) # (None, shape_, n_dim+1, 1) - - if dtype != dtype32: - w_hull = K.cast(w_hull, dtype=dtype) - - shape_prev = int(np.prod(inputs[-1].shape[1:axis])) - if axis == -1: - shape_after = 1 - else: - shape_after = int(np.prod(inputs[-1].shape[axis + 1 :])) - # (-1, shape_prev, axis, shape_after, n_dim+1) - flatten_dim = int(np.prod(w_hull.shape[1:-2])) - w_hull_flat = K.reshape(w_hull, (-1, flatten_dim, n_dim + 1)) - w_u = K.reshape(w_hull_flat[:, :, :-1], (-1, shape_prev, shape_after, n_dim)) # (-1, shape_, n_dim) - w_u = K.reshape(K.transpose(w_u, (0, 1, 3, 2)), [-1] + list(inputs[-1].shape[1:])) - # reshape w_u - shape_max = K.max(inputs[-1], axis).shape[1:] - b_u = K.reshape(w_hull_flat[:, :, -1], [-1] + list(shape_max)) # (-1, shape_) - - return [w_u, b_u] - - -def get_lower_linear_hull_max( - inputs: list[Tensor], - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - axis: int = -1, - finetune_lower: Optional[BackendTensor] = None, - dc_decomp: bool = False, - **kwargs: Any, -) -> list[Tensor]: - """Compute the linear hull that overapproximates max along the axis dimension - - Args: - inputs: list of input tensors - mode: type of Forward propagation (ibp, affine, or hybrid). Default to hybrid. - perturbation_domain (optional): type of perturbation domain that encompass the set of perturbations. Defaults to None. - axis (optional): Defaults to -1. See Keras offical documentation backend.max(., axis) - finetune_lower: If not None, should be a constant tensor used to fine tune the lower relaxation. - - Raises: - NotImplementedError: axis <0 and axis!=-1 - - Returns: - list of output tensors. The lower linear relaxation of max(., axis) in the mode format - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - affine = get_affine(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - dtype = x.dtype - input_shape = inputs_outputs_spec.get_kerasinputshape(inputs) - - o_value = K.cast(1.0, dtype) - z_value = K.cast(0.0, dtype) - if axis == len(input_shape): - axis = -1 - if axis != -1 and axis < 0: - raise NotImplementedError() # to do - - V = u_c + l_c - M = -u_c + K.expand_dims(K.max(l_c, axis), axis) - # M>=0 means that the associated coordinate cannot be an argmax - # we can use M to add a penalty to the index that cannot be an argmax - - V += K.expand_dims(K.max(K.abs(V), axis), axis) * (K.sign(M) + o_value) - - w_l = K.one_hot(K.argmin(V, axis), num_classes=V.shape[axis], axis=axis, dtype=dtype) - b_l = K.sum(z_value * w_l, axis) - # without finetuning: consider a one hot vector with value one at index=argmin(V) - - if finetune_lower is not None: - alpha = finetune_lower[None] - y = alpha * u_c + (o_value - alpha) * l_c - w_l_alpha = K.one_hot(K.argmax(y, axis), num_classes=V.shape[axis], axis=axis, dtype=dtype) - w_l = (o_value - alpha) * w_l + alpha * w_l_alpha - - return [w_l, b_l] From 4bae8c20eb6823714e3f44964d870004dd8941dd Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 2 Feb 2024 21:25:43 +0100 Subject: [PATCH 006/101] Remove old tests --- tests/test_activation.py | 143 ------ tests/test_backward_activation.py | 101 ---- tests/test_backward_compute_output_shape.py | 79 --- tests/test_backward_conv.py | 85 ---- tests/test_backward_dense.py | 112 ----- tests/test_backward_layer_wo_merge.py | 115 ----- tests/test_backward_layers.py | 184 ------- tests/test_backward_layers_get_config.py | 120 ----- tests/test_backward_native_layers.py | 132 ----- tests/test_backward_utils.py | 91 ---- tests/test_clone.py | 287 ----------- tests/test_clone_backward.py | 68 --- tests/test_clone_forward.py | 45 -- tests/test_conv.py | 126 ----- tests/test_crown_layers_get_config.py | 23 - tests/test_decomon_activation_layer.py | 43 -- tests/test_decomon_compute_output_shape.py | 105 ---- tests/test_decomon_reset_layer.py | 218 --------- tests/test_dense_layer.py | 291 ----------- tests/test_layers_convert_utils.py | 25 - tests/test_layers_get_config.py | 111 ----- tests/test_merge.py | 246 ---------- tests/test_metric.py | 34 -- tests/test_metrics_get_config.py | 34 -- tests/test_models_get_config.py | 28 -- tests/test_models_utils.py | 170 ------- tests/test_pooling.py | 49 -- tests/test_preprocess_keras_model.py | 95 ---- tests/test_reshape.py | 137 ------ tests/test_to_decomon.py | 78 --- tests/test_utils.py | 503 -------------------- tests/test_utils_conv.py | 141 ------ tests/test_utils_pooling.py | 54 --- tests/test_wrapper.py | 152 ------ 34 files changed, 4225 deletions(-) delete mode 100644 tests/test_activation.py delete mode 100644 tests/test_backward_activation.py delete mode 100644 tests/test_backward_compute_output_shape.py delete mode 100644 tests/test_backward_conv.py delete mode 100644 tests/test_backward_dense.py delete mode 100644 tests/test_backward_layer_wo_merge.py delete mode 100644 tests/test_backward_layers.py delete mode 100644 tests/test_backward_layers_get_config.py delete mode 100644 tests/test_backward_native_layers.py delete mode 100644 tests/test_backward_utils.py delete mode 100644 tests/test_clone.py delete mode 100644 tests/test_clone_backward.py delete mode 100644 tests/test_clone_forward.py delete mode 100644 tests/test_conv.py delete mode 100644 tests/test_crown_layers_get_config.py delete mode 100644 tests/test_decomon_activation_layer.py delete mode 100644 tests/test_decomon_compute_output_shape.py delete mode 100644 tests/test_decomon_reset_layer.py delete mode 100644 tests/test_dense_layer.py delete mode 100644 tests/test_layers_convert_utils.py delete mode 100644 tests/test_layers_get_config.py delete mode 100644 tests/test_merge.py delete mode 100644 tests/test_metric.py delete mode 100644 tests/test_metrics_get_config.py delete mode 100644 tests/test_models_get_config.py delete mode 100644 tests/test_models_utils.py delete mode 100644 tests/test_pooling.py delete mode 100644 tests/test_preprocess_keras_model.py delete mode 100644 tests/test_reshape.py delete mode 100644 tests/test_to_decomon.py delete mode 100644 tests/test_utils.py delete mode 100644 tests/test_utils_conv.py delete mode 100644 tests/test_utils_pooling.py delete mode 100644 tests/test_wrapper.py diff --git a/tests/test_activation.py b/tests/test_activation.py deleted file mode 100644 index c41b7f36..00000000 --- a/tests/test_activation.py +++ /dev/null @@ -1,143 +0,0 @@ -# Test unit for decomon with Dense layers - - -import keras.ops as K -import numpy as np -import pytest -from numpy.testing import assert_almost_equal - -from decomon.core import ForwardMode, Slope -from decomon.layers.activations import relu, sigmoid, softmax, softsign, tanh - - -@pytest.mark.parametrize( - "activation_func, tensor_func, decimal", - [ - (sigmoid, K.sigmoid, 5), - (tanh, K.tanh, 5), - (softsign, K.softsign, 4), - (softmax, K.softmax, 4), - (relu, K.relu, 4), - ], -) -def test_activation_1D_box(n, mode, floatx, decimal, helpers, activation_func, tensor_func): - # softmax: test only n=0,3 - if activation_func is softmax: - if n not in {0, 3}: - pytest.skip("softmax activation only possible for n=0 or 3") - - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs=inputs_) - - # reference output - output_ref_ = K.convert_to_numpy(tensor_func(input_ref_)) - - # decomon output - output = activation_func(inputs_for_mode, dc_decomp=dc_decomp, mode=mode) - f_decomon = helpers.function(inputs, output) - outputs_ = f_decomon(inputs_) - - #  check bounds consistency - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - mode=mode, - dc_decomp=dc_decomp, - decimal=decimal, - ) - - -@pytest.mark.parametrize( - "n", - [ - 0, - 1, - 2, - ], -) -def test_activation_1D_box_slope(n, slope, helpers): - mode = ForwardMode.AFFINE - activation_func = relu - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - - # decomon output - outputs = activation_func(inputs_for_mode, dc_decomp=dc_decomp, mode=mode, slope=slope) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # full outputs & inputs - ( - z_output, - u_c_output, - w_u_output, - b_u_output, - l_c_output, - w_l_output, - b_l_output, - h_output, - g_output, - ) = helpers.get_full_outputs_from_outputs_for_mode( - outputs_for_mode=outputs_, - full_inputs=inputs_, - mode=mode, - dc_decomp=dc_decomp, - ) - - ( - x_0, - y_0, - z_0, - u_c_0, - W_u_0, - b_u_0, - l_c_0, - W_l_0, - b_l_0, - ) = inputs_ # numpy values - - # check bounds according to slope - slope = Slope(slope) - if n == 0: - assert_almost_equal(w_u_output, np.zeros(w_u_output.shape)) - assert_almost_equal(b_u_output, np.zeros(b_u_output.shape)) - assert_almost_equal(w_l_output, np.zeros(w_l_output.shape)) - assert_almost_equal(b_l_output, np.zeros(b_l_output.shape)) - elif n == 1: - assert_almost_equal(w_u_output, np.ones(w_u_output.shape)) - assert_almost_equal(b_u_output, np.zeros(b_u_output.shape)) - assert_almost_equal(w_l_output, W_l_0) - assert_almost_equal(b_l_output, np.zeros(b_l_output.shape)) - elif n == 2: - assert_almost_equal(w_u_output, 0.5 * np.ones(w_u_output.shape)) - assert_almost_equal(b_u_output, 0.5 * np.ones(b_u_output.shape)) - if slope == Slope.O_SLOPE: - assert_almost_equal(w_l_output, W_l_0) - assert_almost_equal(b_l_output, np.zeros(b_l_output.shape)) - elif slope == Slope.Z_SLOPE: - assert_almost_equal(w_l_output, np.zeros(w_l_output.shape)) - assert_almost_equal(b_l_output, np.zeros(b_l_output.shape)) - elif slope == Slope.V_SLOPE: - assert_almost_equal(w_l_output, np.zeros(w_l_output.shape)) - assert_almost_equal(b_l_output, np.zeros(b_l_output.shape)) - elif slope == Slope.S_SLOPE: - assert_almost_equal(w_l_output, w_u_output) - assert_almost_equal(b_l_output, np.zeros(b_l_output.shape)) - elif slope == Slope.A_SLOPE: - assert_almost_equal(b_l_output, np.zeros(b_l_output.shape)) - assert_almost_equal(w_l_output, np.zeros(w_l_output.shape)) diff --git a/tests/test_backward_activation.py b/tests/test_backward_activation.py deleted file mode 100644 index 598f805c..00000000 --- a/tests/test_backward_activation.py +++ /dev/null @@ -1,101 +0,0 @@ -import numpy as np -import pytest -from numpy.testing import assert_almost_equal - -from decomon.backward_layers.activations import backward_relu, backward_softsign -from decomon.core import ForwardMode, Slope - - -@pytest.mark.parametrize( - "activation_func", - [ - backward_relu, - backward_softsign, - ], -) -def test_activation_backward_1D_box(n, mode, floatx, decimal, activation_func, helpers): - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - - # backward outputs - outputs = activation_func(inputs_for_mode, mode=mode) - f_func = helpers.function(inputs, outputs) - outputs_ = f_func(inputs_) - - # check bounds consistency - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_, - backward_outputs=outputs_, - decimal=decimal, - ) - - -@pytest.mark.parametrize( - "n", - [ - 0, - 1, - 2, - ], -) -def test_activation_backward_1D_box_slope(n, slope, helpers): - mode = ForwardMode.AFFINE - activation_func = backward_relu - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - x_, y_, z_, u_c_, W_u_, B_u_, l_c_, W_l_, B_l_ = inputs_ - - # backward outputs - outputs = activation_func(inputs_for_mode, mode=mode) - f_func = helpers.function(inputs, outputs) - outputs_ = f_func(inputs_) - - # backward recomposition - w_u_, b_u_, w_l_, b_l_ = outputs_ - w_u_b = np.sum(np.maximum(w_u_, 0) * W_u_ + np.minimum(w_u_, 0) * W_l_, 1)[:, :, None] - b_u_b = b_u_ + np.sum(np.maximum(w_u_, 0) * B_u_[:, :, None], 1) + np.sum(np.minimum(w_u_, 0) * B_l_[:, :, None], 1) - w_l_b = np.sum(np.maximum(w_l_, 0) * W_l_ + np.minimum(w_l_, 0) * W_u_, 1)[:, :, None] - b_l_b = b_l_ + np.sum(np.maximum(w_l_, 0) * B_l_[:, :, None], 1) + np.sum(np.minimum(w_l_, 0) * B_u_[:, :, None], 1) - - # check bounds according to slope - slope = Slope(slope) - if n == 0: - assert_almost_equal(w_u_b, np.zeros(w_u_b.shape)) - assert_almost_equal(b_u_b, np.zeros(b_u_b.shape)) - assert_almost_equal(w_l_b, np.zeros(w_l_b.shape)) - assert_almost_equal(b_l_b, np.zeros(b_l_b.shape)) - elif n == 1: - assert_almost_equal(w_u_b, len(w_u_b) * np.ones(w_u_b.shape)) # * len ?? - assert_almost_equal(b_u_b, np.zeros(b_u_b.shape)) - assert_almost_equal(w_l_b, len(w_l_b) * np.ones(w_l_b.shape)) - assert_almost_equal(b_l_b, np.zeros(b_l_b.shape)) - elif n == 2: - assert_almost_equal(w_u_b, 0.5 * len(w_u_b) * np.ones(w_u_b.shape)) - assert_almost_equal(b_u_b, 0.5 * np.ones(b_u_b.shape)) - if slope == Slope.O_SLOPE: - assert_almost_equal(w_l_b, np.zeros(w_l_b.shape)) # ?? - assert_almost_equal(b_l_b, np.zeros(b_l_b.shape)) - elif slope == Slope.Z_SLOPE: - assert_almost_equal(w_l_b, np.zeros(w_l_b.shape)) - assert_almost_equal(b_l_b, np.zeros(b_l_b.shape)) - elif slope == Slope.V_SLOPE: - assert_almost_equal(w_l_b, np.zeros(w_l_b.shape)) - assert_almost_equal(b_l_b, np.zeros(b_l_b.shape)) - elif slope == Slope.S_SLOPE: - assert_almost_equal(w_l_b, np.zeros(w_l_b.shape)) # ?? - assert_almost_equal(b_l_b, np.zeros(b_l_b.shape)) - elif slope == Slope.A_SLOPE: - assert_almost_equal(b_l_b, np.zeros(b_l_b.shape)) - assert_almost_equal(w_l_b, np.zeros(w_l_b.shape)) diff --git a/tests/test_backward_compute_output_shape.py b/tests/test_backward_compute_output_shape.py deleted file mode 100644 index 8753ff01..00000000 --- a/tests/test_backward_compute_output_shape.py +++ /dev/null @@ -1,79 +0,0 @@ -import pytest -from keras.layers import Activation, InputLayer, Reshape - -from decomon.backward_layers.convert import to_backward -from decomon.layers.core import DecomonLayer -from decomon.layers.decomon_layers import ( - DecomonActivation, - DecomonBatchNormalization, - DecomonConv2D, - DecomonDense, - DecomonFlatten, -) -from decomon.layers.decomon_merge_layers import DecomonAdd, DecomonConcatenate -from decomon.layers.decomon_reshape import DecomonReshape -from decomon.layers.maxpooling import DecomonMaxPooling2D - - -@pytest.mark.parametrize( - "layer_class, layer_kwargs", - [ - (DecomonDense, dict(units=3, use_bias=True)), - (DecomonFlatten, dict()), - (DecomonBatchNormalization, dict(center=True, scale=True)), - (DecomonBatchNormalization, dict(center=False, scale=False)), - (DecomonActivation, dict(activation="linear")), - (DecomonActivation, dict(activation="relu")), - (Activation, dict(activation="linear")), - (Activation, dict(activation="relu")), - (DecomonConv2D, dict(filters=10, kernel_size=(3, 3))), - (DecomonReshape, dict(target_shape=(1, -1, 1))), - (Reshape, dict(target_shape=(1, -1, 1))), - ], -) -@pytest.mark.parametrize("dc_decomp", [False]) # limit dc_decomp -@pytest.mark.parametrize("n", [0]) # limit 1d cases -@pytest.mark.parametrize("odd", [0]) # limit multid cases -@pytest.mark.parametrize("data_format", ["channels_last"]) # limit images cases -def test_compute_output_shape( - helpers, - mode, - dc_decomp, - layer_class, - layer_kwargs, - inputs_for_mode, # decomon inputs: symbolic tensors - input_ref, # keras input: symbolic tensor - inputs_for_mode_, # decomon inputs: numpy arrays - inputs_metadata, # inputs metadata: data_format, ... -): - # skip nonsensical combinations - if (layer_class == DecomonBatchNormalization or layer_class == DecomonMaxPooling2D) and dc_decomp: - pytest.skip(f"{layer_class} with dc_decomp=True not yet implemented.") - if (layer_class == DecomonConv2D or layer_class == DecomonMaxPooling2D) and len(input_ref.shape) < 4: - pytest.skip(f"{layer_class} applies only on image-like inputs.") - - # add data_format for convolution and maxpooling - if layer_class in (DecomonConv2D, DecomonMaxPooling2D): - layer_kwargs["data_format"] = inputs_metadata["data_format"] - - # construct and build original layer (decomon or keras) - if issubclass(layer_class, DecomonLayer): - layer = layer_class(mode=mode, dc_decomp=dc_decomp, **layer_kwargs) - layer(inputs_for_mode) - else: # keras layer - layer = layer_class(**layer_kwargs) - layer(input_ref) - - # get backward layer - backward_layer = to_backward(layer, mode=mode) - - # check symbolic tensor output shapes - inputshapes = [i.shape for i in inputs_for_mode] - outputshapes = backward_layer.compute_output_shape(inputshapes) - outputs = backward_layer(inputs_for_mode) - assert [o.shape for o in outputs] == outputshapes - - # check output shapes for concrete call - outputs_ = backward_layer(inputs_for_mode_) - # compare without batch sizes - assert [tuple(o.shape)[1:] for o in outputs_] == [s[1:] for s in outputshapes] diff --git a/tests/test_backward_conv.py b/tests/test_backward_conv.py deleted file mode 100644 index 88bc3131..00000000 --- a/tests/test_backward_conv.py +++ /dev/null @@ -1,85 +0,0 @@ -# Test unit for decomon with Dense layers - - -import keras.config as keras_config -import numpy as np -import pytest - -from decomon.backward_layers.convert import to_backward -from decomon.layers.decomon_layers import DecomonConv2D - - -def test_Decomon_conv_box(data_format, padding, use_bias, mode, floatx, decimal, helpers): - # skip unavailable combinations - if floatx == 16 and keras_config.backend() == "torch" and not helpers.in_GPU_mode(): - pytest.skip("Pytorch does not implement conv2d for float16 in CPU mode.") - - if data_format == "channels_first" and not helpers.in_GPU_mode() and keras_config.backend() == "tensorflow": - pytest.skip("data format 'channels first' is possible only in GPU mode for tensorflow.") - - odd, m_0, m_1 = 0, 0, 1 - dc_decomp = False - - # tensor inputs - inputs = helpers.get_tensor_decomposition_images_box(data_format, odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_images_box(data_format, odd, m0=m_0, m1=m_1, dc_decomp=dc_decomp) - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ - - # decomon layer - decomon_layer = DecomonConv2D( - 10, - kernel_size=(3, 3), - dc_decomp=dc_decomp, - padding=padding, - use_bias=use_bias, - mode=mode, - dtype=keras_config.floatx(), - data_format=data_format, - ) - decomon_layer(inputs_for_mode) # init weights - - # get backward layer - backward_layer = to_backward(decomon_layer) - - # backward outputs - outputs = backward_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - w_u_, b_u_, w_l_, b_l_ = outputs_ - - # reshape the matrices - w_u_ = w_u_[:, None] - w_l_ = w_l_[:, None] - b_u_ = b_u_[:, None] - b_l_ = b_l_[:, None] - - b_l = b_l.reshape((len(b_l), -1)) - b_u = b_u.reshape((len(b_u), -1)) - W_u = W_u.reshape((len(W_u), W_u.shape[1], -1)) - W_l = W_l.reshape((len(W_l), W_l.shape[1], -1)) - - # backward recomposition - w_r_u = np.sum(np.maximum(0.0, w_u_) * np.expand_dims(W_u, -1), 2) + np.sum( - np.minimum(0.0, w_u_) * np.expand_dims(W_l, -1), 2 - ) - w_r_l = np.sum(np.maximum(0.0, w_l_) * np.expand_dims(W_l, -1), 2) + np.sum( - np.minimum(0.0, w_l_) * np.expand_dims(W_u, -1), 2 - ) - b_r_u = ( - np.sum(np.maximum(0, w_u_[:, 0]) * np.expand_dims(b_u, -1), 1)[:, None] - + np.sum(np.minimum(0, w_u_[:, 0]) * np.expand_dims(b_l, -1), 1)[:, None] - + b_u_ - ) - b_r_l = ( - np.sum(np.maximum(0, w_l_[:, 0]) * np.expand_dims(b_l, -1), 1)[:, None] - + np.sum(np.minimum(0, w_l_[:, 0]) * np.expand_dims(b_u, -1), 1)[:, None] - + b_l_ - ) - - # check bounds consistency - helpers.assert_output_properties_box_linear(x, None, z[:, 0], z[:, 1], None, w_r_u, b_r_u, None, w_r_l, b_r_l) diff --git a/tests/test_backward_dense.py b/tests/test_backward_dense.py deleted file mode 100644 index 4c236189..00000000 --- a/tests/test_backward_dense.py +++ /dev/null @@ -1,112 +0,0 @@ -import keras.config as keras_config -from keras.layers import Dense, Input - -from decomon.backward_layers.convert import to_backward -from decomon.core import ForwardMode -from decomon.layers.decomon_layers import DecomonDense - - -def test_Backward_Dense_1D_box(n, use_bias, mode, floatx, decimal, helpers): - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_dim = helpers.get_input_dim_from_full_inputs(inputs) - input_ref = helpers.get_input_ref_from_full_inputs(inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - - # keras layer - keras_layer = Dense(1, use_bias=use_bias, dtype=keras_config.floatx()) - keras_layer(input_ref) # init weights - - # get backward layer - backward_layer = to_backward(keras_layer, mode=mode, input_dim=input_dim) - - # backward outputs - outputs = backward_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_, - backward_outputs=outputs_, - decimal=decimal, - ) - - -def test_Backward_Dense_multiD_box(odd, floatx, decimal, mode, helpers): - dc_decomp = False - use_bias = True - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_multid_box(odd=odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_dim = helpers.get_input_dim_from_full_inputs(inputs) - input_ref = helpers.get_input_ref_from_full_inputs(inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - - # keras layer - keras_layer = Dense(1, use_bias=use_bias, dtype=keras_config.floatx()) - keras_layer(input_ref) # init weights - - # get backward layer - backward_layer = to_backward(keras_layer, mode=mode, input_dim=input_dim) - - # backward outputs - outputs = backward_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_, - backward_outputs=outputs_, - decimal=decimal, - ) - - -def test_Backward_DecomonDense_1D_box(n, helpers): - dc_decomp = False - use_bias = True - mode = ForwardMode.HYBRID - decimal = 5 - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_dim = helpers.get_input_dim_from_full_inputs(inputs) - input_ref = helpers.get_input_ref_from_full_inputs(inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - - # decomon layer - decomon_layer = DecomonDense(1, use_bias=use_bias, dc_decomp=dc_decomp, mode=mode, dtype=keras_config.floatx()) - decomon_layer(inputs_for_mode) # init weights - - # get backward layer - backward_layer = to_backward(decomon_layer, input_dim=input_dim) - - # backward outputs - outputs = backward_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_, - backward_outputs=outputs_, - decimal=decimal, - ) - - # try to call the backward layer with previous bounds - w_out = Input((1, 1)) - b_out = Input((1,)) - previous_bounds = [w_out, b_out, w_out, b_out] - backward_layer(inputs_for_mode + previous_bounds) diff --git a/tests/test_backward_layer_wo_merge.py b/tests/test_backward_layer_wo_merge.py deleted file mode 100644 index 90807e7c..00000000 --- a/tests/test_backward_layer_wo_merge.py +++ /dev/null @@ -1,115 +0,0 @@ -import keras.config as keras_config -import numpy as np -import pytest -from keras.layers import ( - Activation, - BatchNormalization, - Conv2D, - Dense, - Dropout, - Flatten, - Input, - MaxPooling2D, - Permute, - Reshape, -) -from keras.ops import convert_to_numpy -from keras.src.layers.merging.dot import batch_dot - -from decomon.backward_layers.convert import to_backward -from decomon.core import ForwardMode -from decomon.layers.core import DecomonLayer -from decomon.layers.decomon_layers import DecomonDense - - -@pytest.mark.parametrize( - "layer_class, layer_kwargs", - [ - (Dense, dict(units=3, use_bias=True)), - (Dense, dict(units=3, use_bias=False)), - (Activation, dict(activation="linear")), - (Activation, dict(activation="relu")), - (Reshape, dict(target_shape=(1, -1, 1))), - (BatchNormalization, dict()), - (BatchNormalization, dict(axis=1)), - (BatchNormalization, dict(center=False, scale=False)), - (BatchNormalization, dict(axis=2, center=False)), - (BatchNormalization, dict(axis=3)), - (Conv2D, dict(filters=10, kernel_size=(3, 3), data_format="channels_last")), - (Flatten, dict()), - (Dropout, dict(rate=0.9)), - (Permute, dict(dims=(1,))), - (Permute, dict(dims=(2, 1, 3))), - ], -) -@pytest.mark.parametrize("randomize_weights", [False, True]) -@pytest.mark.parametrize("floatx", [32]) # fix floatx -@pytest.mark.parametrize("data_format", ["channels_last"]) # limit images cases -@pytest.mark.parametrize("dc_decomp", [False]) # limit dc_decomp -def test_backward_layer( - helpers, - mode, - layer_class, - layer_kwargs, - randomize_weights, - decimal, - dc_decomp, - inputs_for_mode, # decomon inputs: symbolic tensors - inputs, # decomon inputs: symbolic tensors - input_ref, # keras input: symbolic tensor - inputs_, # decomon inputs: numpy arrays - input_ref_, # keras input: numpy array - inputs_metadata, # inputs metadata: data_format, ... -): - # skip nonsensical combinations - if ( - layer_class is BatchNormalization - and "axis" in layer_kwargs - and layer_kwargs["axis"] > 1 - and len(input_ref.shape) < 4 - ): - pytest.skip("batchnormalization on axis>1 possible only for image-like data") - if layer_class in (Conv2D, MaxPooling2D) and len(input_ref.shape) < 4: - pytest.skip("convolution and maxpooling2d possible only for image-like data") - if layer_class is Permute and len(layer_kwargs["dims"]) < 3 and len(input_ref.shape) >= 4: - pytest.skip("1d permutation not possible for image-like data") - if layer_class is Permute and len(layer_kwargs["dims"]) == 3 and len(input_ref.shape) < 4: - pytest.skip("3d permutation possible only for image-like data") - - # add data_format for convolution and maxpooling - if layer_class in (Conv2D,): - layer_kwargs["data_format"] = inputs_metadata["data_format"] - - # construct and build original layer (keras) - layer = layer_class(**layer_kwargs) - output_ref = layer(input_ref) - f_ref = helpers.function(inputs, output_ref) - - # randomize weights - if randomize_weights: - for weight in layer.weights: - weight.assign(np.random.random(tuple(weight.shape))) - - # get backward layer & function - backward_layer = to_backward(layer, mode=mode) - outputs = backward_layer(inputs_for_mode) - f_backward = helpers.function(inputs, outputs) - - # keras & decomon outputs - output_ref_ = f_ref(inputs_) - outputs_ = f_backward(inputs_) - - # flattened keras inputs/outputs - input_ref_flat_ = np.reshape(input_ref_, (input_ref_.shape[0], -1)) - output_ref_flat_ = np.reshape(output_ref_, (output_ref_.shape[0], -1)) - - # check bounds consistency - w_u_out_, b_u_out_, w_l_out_, b_l_out_ = outputs_ - upper_ = convert_to_numpy(batch_dot(w_u_out_, input_ref_flat_, axes=(-2, -1))) + b_u_out_ # batch mult - lower_ = convert_to_numpy(batch_dot(w_l_out_, input_ref_flat_, axes=(-2, -1))) + b_l_out_ # batch mult - np.testing.assert_almost_equal( - np.clip(lower_ - output_ref_flat_, 0, np.inf), 0.0, decimal=decimal, err_msg="x_min >x_" - ) - np.testing.assert_almost_equal( - np.clip(output_ref_flat_ - upper_, 0, np.inf), 0.0, decimal=decimal, err_msg="x_max < x_" - ) diff --git a/tests/test_backward_layers.py b/tests/test_backward_layers.py deleted file mode 100644 index 98353986..00000000 --- a/tests/test_backward_layers.py +++ /dev/null @@ -1,184 +0,0 @@ -import keras.config as keras_config -import pytest -from keras.layers import Layer, Reshape - -from decomon.backward_layers.convert import to_backward -from decomon.core import ForwardMode, Slope -from decomon.layers.decomon_layers import DecomonActivation, DecomonFlatten -from decomon.layers.decomon_reshape import DecomonReshape - - -def test_Backward_Activation_1D_box_model(n, activation, mode, floatx, decimal, helpers): - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - - # decomon layer - decomon_layer = DecomonActivation(activation, dc_decomp=dc_decomp, mode=mode, dtype=keras_config.floatx()) - decomon_layer(inputs_for_mode) # build it - - # get backward layer - backward_layer = to_backward(decomon_layer) - - # backward outputs - outputs = backward_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_, - backward_outputs=outputs_, - decimal=decimal, - ) - - -def test_Backward_Activation_1D_box_model_slope(helpers): - n = 2 - activation = "relu" - mode = ForwardMode.AFFINE - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - - #  decomon layer - decomon_layer = DecomonActivation(activation, dc_decomp=dc_decomp, mode=mode, dtype=keras_config.floatx()) - decomon_layer(inputs_for_mode) # build it - - # to_backward with a given slope => outputs - outputs_by_slope = {} - for slope in Slope: - layer_backward = to_backward(decomon_layer, slope=slope, mode=mode) - assert layer_backward.slope == slope - outputs = layer_backward(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_by_slope[slope] = f_decomon(inputs_) - - # check results - # O_Slope != Z_Slope - same_outputs_O_n_Z = [ - (a == b).all() for a, b in zip(outputs_by_slope[Slope.O_SLOPE], outputs_by_slope[Slope.Z_SLOPE]) - ] - assert not all(same_outputs_O_n_Z) - - # V_Slope == Z_Slope - for a, b in zip(outputs_by_slope[Slope.V_SLOPE], outputs_by_slope[Slope.Z_SLOPE]): - assert (a == b).all() - - -def test_Backward_Activation_multiD_box(odd, activation, floatx, decimal, mode, helpers): - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_multid_box(odd=odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - - #  decomon layer - decomon_layer = DecomonActivation(activation, dc_decomp=dc_decomp, mode=mode, dtype=keras_config.floatx()) - decomon_layer(inputs_for_mode) # build it - - # get backward layer - backward_layer = to_backward(decomon_layer) - - # backward outputs - outputs = backward_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_, - backward_outputs=outputs_, - decimal=decimal, - ) - - -def test_Backward_Flatten_multiD_box(odd, floatx, decimal, mode, data_format, helpers): - if data_format == "channels_first" and not helpers.in_GPU_mode() and keras_config.backend() == "tensorflow": - pytest.skip("data format 'channels first' is possible only in GPU mode for tensorflow.") - - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_multid_box(odd=odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - - #  decomon layer - decomon_layer = DecomonFlatten(data_format, dc_decomp=dc_decomp, mode=mode, dtype=keras_config.floatx()) - decomon_layer(inputs_for_mode) # build it - - # get backward layer - backward_layer = to_backward(decomon_layer) - - # backward outputs - outputs = backward_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_, - backward_outputs=outputs_, - decimal=decimal, - ) - - -def test_Backward_Reshape_multiD_box(odd, floatx, decimal, mode, helpers): - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_multid_box(odd=odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - - #  decomon layer - decomon_layer = DecomonReshape((-1,), dc_decomp=dc_decomp, mode=mode, dtype=keras_config.floatx()) - decomon_layer(inputs_for_mode) # build it - - # get backward layer - backward_layer = to_backward(decomon_layer) - - # backward outputs - outputs = backward_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_, - backward_outputs=outputs_, - decimal=decimal, - ) - - -def test_to_backward_ko(): - class MyLayer(Layer): - ... - - layer = MyLayer() - with pytest.raises(NotImplementedError): - to_backward(layer) - - -def test_name(): - layer = Reshape((1, 2)) - backward_layer = to_backward(layer) - backward_layer.name.startswith(f"{layer.name}_") diff --git a/tests/test_backward_layers_get_config.py b/tests/test_backward_layers_get_config.py deleted file mode 100644 index af7456bd..00000000 --- a/tests/test_backward_layers_get_config.py +++ /dev/null @@ -1,120 +0,0 @@ -import pytest -from keras.layers import Layer - -from decomon.backward_layers.backward_layers import ( - BackwardActivation, - BackwardBatchNormalization, - BackwardConv2D, - BackwardDense, - BackwardDropout, - BackwardFlatten, - BackwardInputLayer, - BackwardPermute, - BackwardReshape, -) -from decomon.backward_layers.backward_maxpooling import BackwardMaxPooling2D -from decomon.backward_layers.backward_merge import ( - BackwardAdd, - BackwardAverage, - BackwardConcatenate, - BackwardMaximum, - BackwardMinimum, - BackwardMultiply, - BackwardSubtract, -) -from decomon.layers.decomon_layers import ( - DecomonActivation, - DecomonBatchNormalization, - DecomonConv2D, - DecomonDense, - DecomonDropout, -) -from decomon.layers.decomon_merge_layers import ( - DecomonAdd, - DecomonAverage, - DecomonConcatenate, - DecomonMaximum, - DecomonMinimum, - DecomonMultiply, - DecomonSubtract, -) -from decomon.layers.decomon_reshape import DecomonPermute, DecomonReshape -from decomon.layers.maxpooling import DecomonMaxPooling2D - - -def test_backward_layers(): - activation = "linear" - sublayer = DecomonActivation(activation) - layer = BackwardActivation(layer=sublayer) - config = layer.get_config() - assert config["layer"]["class_name"] == sublayer.__class__.__name__ - - units = 2 - sublayer = DecomonDense(units=units, use_bias=False) - layer = BackwardDense(layer=sublayer) - config = layer.get_config() - assert config["layer"]["class_name"] == sublayer.__class__.__name__ - - filters = 2 - kernel_size = 3, 3 - sublayer = DecomonConv2D(filters=filters, kernel_size=kernel_size) - layer = BackwardConv2D(layer=sublayer) - config = layer.get_config() - assert config["layer"]["class_name"] == sublayer.__class__.__name__ - - rate = 0.9 - sublayer = DecomonDropout(rate=rate) - layer = BackwardDropout(layer=sublayer) - config = layer.get_config() - assert config["layer"]["class_name"] == sublayer.__class__.__name__ - - dims = (1, 2, 3) - sublayer = DecomonPermute(dims=dims) - layer = BackwardPermute(layer=sublayer) - config = layer.get_config() - assert config["layer"]["class_name"] == sublayer.__class__.__name__ - - shape = (1, 2, 3) - sublayer = DecomonReshape(target_shape=shape) - layer = BackwardReshape(layer=sublayer) - config = layer.get_config() - assert config["layer"]["class_name"] == sublayer.__class__.__name__ - - sublayer = DecomonBatchNormalization() - layer = BackwardBatchNormalization(layer=sublayer) - config = layer.get_config() - assert config["layer"]["class_name"] == sublayer.__class__.__name__ - - sublayer = Layer() - for cls in [BackwardFlatten, BackwardInputLayer]: - layer = cls(layer=sublayer) - config = layer.get_config() - assert config["layer"]["class_name"] == sublayer.__class__.__name__ - - -backward2decomon = { - BackwardAdd: DecomonAdd, - BackwardAverage: DecomonAverage, - BackwardConcatenate: DecomonConcatenate, - # BackwardDot: DecomonDot, # not yet implemented - BackwardMaximum: DecomonMaximum, - BackwardMinimum: DecomonMinimum, - BackwardMultiply: DecomonMultiply, - BackwardSubtract: DecomonSubtract, -} - - -def test_backward_merge(): - for backwardlayer_cls, decomonlayer_cls in backward2decomon.items(): - sublayer = decomonlayer_cls() - layer = backwardlayer_cls(layer=sublayer) - config = layer.get_config() - assert config["layer"]["class_name"] == sublayer.__class__.__name__ - - -@pytest.mark.skip("Not yet implemented.") -def test_backward_maxpooling(): - sublayer = DecomonMaxPooling2D() - layer = BackwardMaxPooling2D(layer=sublayer) - config = layer.get_config() - assert config["layer"]["class_name"] == sublayer.__class__.__name__ diff --git a/tests/test_backward_native_layers.py b/tests/test_backward_native_layers.py deleted file mode 100644 index 1ad25053..00000000 --- a/tests/test_backward_native_layers.py +++ /dev/null @@ -1,132 +0,0 @@ -import keras.config as keras_config -import pytest -from keras.layers import Activation, Flatten, Reshape - -from decomon.backward_layers.convert import to_backward - - -def test_Backward_NativeActivation_1D_box_model(n, activation, mode, floatx, decimal, helpers): - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - - # keras layer - keras_layer = Activation(activation, dtype=keras_config.floatx()) - keras_layer(input_ref) # build it - - # get backward layer - backward_layer = to_backward(keras_layer, mode=mode) - - # backward outputs - outputs = backward_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_, - backward_outputs=outputs_, - decimal=decimal, - ) - - -def test_Backward_NativeActivation_multiD_box(odd, activation, floatx, decimal, mode, helpers): - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_multid_box(odd=odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - - # keras layer - keras_layer = Activation(activation, dtype=keras_config.floatx()) - keras_layer(input_ref) # build it - - # get backward layer - backward_layer = to_backward(keras_layer, mode=mode) - - # backward outputs - outputs = backward_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_, - backward_outputs=outputs_, - decimal=decimal, - ) - - -def test_Backward_NativeFlatten_multiD_box(odd, floatx, decimal, mode, data_format, helpers): - if data_format == "channels_first" and not helpers.in_GPU_mode() and keras_config.backend() == "tensorflow": - pytest.skip("data format 'channels first' is possible only in GPU mode for tensorflow.") - - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_multid_box(odd=odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - - # keras layer - keras_layer = Flatten(data_format, dtype=keras_config.floatx()) - keras_layer(input_ref) # build it - - # get backward layer - backward_layer = to_backward(keras_layer, mode=mode) - - # backward outputs - outputs = backward_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_, - backward_outputs=outputs_, - decimal=decimal, - ) - - -def test_Backward_NativeReshape_multiD_box(odd, floatx, decimal, mode, helpers): - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_multid_box(odd=odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - - # keras layer - keras_layer = Reshape((-1,), dtype=keras_config.floatx()) - keras_layer(input_ref) # build it - - # get backward layer - backward_layer = to_backward(keras_layer, mode=mode) - - # backward outputs - outputs = backward_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_, - backward_outputs=outputs_, - decimal=decimal, - ) diff --git a/tests/test_backward_utils.py b/tests/test_backward_utils.py deleted file mode 100644 index 97cf7b07..00000000 --- a/tests/test_backward_utils.py +++ /dev/null @@ -1,91 +0,0 @@ -# Test unit for decomon with Dense layers - -import keras.config as keras_config -import numpy as np -import pytest -from keras.layers import Input - -from decomon.backward_layers.utils import ( - backward_add, - backward_maximum, - backward_subtract, -) - - -def add_op(x, y): - return x + y - - -def subtract_op(x, y): - return x - y - - -@pytest.mark.parametrize( - "n_0, n_1", - [ - (0, 3), - (1, 4), - (2, 5), - ], -) -@pytest.mark.parametrize( - "backward_func, tensor_op", - [ - (backward_add, add_op), - (backward_maximum, np.maximum), - (backward_subtract, subtract_op), - ], -) -def test_reduce_backward_1D_box(n_0, n_1, backward_func, tensor_op, mode, floatx, decimal, helpers): - dc_decomp = False - - #  tensor inputs - inputs_0 = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode_0 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_0, mode=mode, dc_decomp=dc_decomp) - inputs_1 = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode_1 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_1, mode=mode, dc_decomp=dc_decomp) - - # numpy inputs - inputs_0_ = helpers.get_standard_values_1d_box(n_0, dc_decomp=dc_decomp) - inputs_1_ = helpers.get_standard_values_1d_box(n_1, dc_decomp=dc_decomp) - input_ref_0_ = helpers.get_input_ref_from_full_inputs(inputs=inputs_0_) - batchsize = input_ref_0_.shape[0] - - x_0, y_0, z_0, u_c_0, W_u_0, b_u_0, l_c_0, W_l_0, b_l_0 = inputs_0_ - x_1, y_1, z_1, u_c_1, W_u_1, b_u_1, l_c_1, W_l_1, b_l_1 = inputs_1_ - - # backward outputs - w_out = Input((1, 1), dtype=keras_config.floatx()) - b_out = Input((1,), dtype=keras_config.floatx()) - back_bounds_0, back_bounds_1 = backward_func( - inputs_for_mode_0, inputs_for_mode_1, w_out, b_out, w_out, b_out, mode=mode - ) - f_add = helpers.function(inputs_0 + inputs_1 + [w_out, b_out], back_bounds_0 + back_bounds_1) - outputs_ = f_add(inputs_0_ + inputs_1_ + [np.ones((batchsize, 1, 1)), np.zeros((batchsize, 1))]) - # get back separate outputs - outputs_0_ = outputs_[: len(outputs_) // 2] - outputs_1_ = outputs_[len(outputs_) // 2 :] - - # reference output and constant lower and upper bounds - output_ref = tensor_op(y_0, y_1) - upper_constant_bound = tensor_op(u_c_0, u_c_1) - lower_constant_bound = tensor_op(l_c_0, l_c_1) - - #  check bounds consistency - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_0_, - backward_outputs=outputs_0_, - decimal=decimal, - output_ref=output_ref, - upper_constant_bound=upper_constant_bound, - lower_constant_bound=lower_constant_bound, - ) - - helpers.assert_backward_layer_output_properties_box_linear( - full_inputs=inputs_1_, - backward_outputs=outputs_1_, - decimal=decimal, - output_ref=output_ref, - upper_constant_bound=upper_constant_bound, - lower_constant_bound=lower_constant_bound, - ) diff --git a/tests/test_clone.py b/tests/test_clone.py deleted file mode 100644 index 0a4d355b..00000000 --- a/tests/test_clone.py +++ /dev/null @@ -1,287 +0,0 @@ -# creating toy network and assess that the decomposition is correct - - -import keras.config as keras_config -import keras.ops as K -import numpy as np -import pytest -from keras.layers import Activation, Dense, Flatten, Input, Reshape -from keras.models import Model, Sequential - -from decomon.core import ForwardMode, Slope, get_affine, get_ibp, get_mode -from decomon.layers.decomon_reshape import DecomonReshape -from decomon.models import clone -from decomon.models.convert import FeedDirection, get_direction -from decomon.models.utils import ( - ConvertMethod, - get_ibp_affine_from_method, - has_merge_layers, -) - - -def test_convert_nok_several_inputs(): - a = Input((1,)) - b = Input((2,)) - model = Model([a, b], a) - - with pytest.raises(ValueError, match="only 1 input"): - clone(model) - - -def test_convert_nok_unflattened_input(): - a = Input((1, 2)) - model = Model(a, a) - - with pytest.raises(ValueError, match="flattened input"): - clone(model) - - -def test_clone_with_backbounds(method, helpers): - dc_decomp = False - n = 0 - ibp, affine = get_ibp_affine_from_method(method) - mode = get_mode(ibp, affine) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs_) - input_decomon_wo_backbounds = helpers.get_inputs_np_for_decomon_model_from_full_inputs(inputs=inputs_) - - #  keras model and output of reference - ref_nn = helpers.toy_network_tutorial() - output_ref_ = helpers.predict_on_small_numpy(ref_nn, input_ref_) - - # create back_bounds for adversarial robustness like studies - output_dim = int(np.prod(ref_nn.output_shape[1:])) - C = Input((output_dim, output_dim)) - batchsize = input_decomon_wo_backbounds.shape[0] - C_ = np.repeat(np.identity(output_dim), repeats=batchsize, axis=0) - inputs_decomon_ = [input_decomon_wo_backbounds, C_] - - # decomon conversion - decomon_model = clone(ref_nn, method=method, final_ibp=ibp, final_affine=affine, back_bounds=[C]) - - #  decomon outputs - outputs_ = helpers.predict_on_small_numpy(decomon_model, inputs_decomon_) - - #  check bounds consistency - helpers.assert_decomon_model_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - mode=mode, - dc_decomp=dc_decomp, - ) - - -def test_convert_1D(n, method, mode, floatx, decimal, helpers): - if not helpers.is_method_mode_compatible(method=method, mode=mode): - # skip method=ibp/crown-ibp with mode=affine/hybrid - pytest.skip(f"output mode {mode} is not compatible with convert method {method}") - - dc_decomp = False - ibp = get_ibp(mode=mode) - affine = get_affine(mode=mode) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs_) - input_decomon_ = helpers.get_inputs_np_for_decomon_model_from_full_inputs(inputs=inputs_) - - #  keras model and output of reference - ref_nn = helpers.toy_network_tutorial(dtype=keras_config.floatx()) - output_ref_ = helpers.predict_on_small_numpy(ref_nn, input_ref_) - - # decomon conversion - decomon_model = clone(ref_nn, method=method, final_ibp=ibp, final_affine=affine) - - #  decomon outputs - outputs_ = helpers.predict_on_small_numpy(decomon_model, input_decomon_) - - #  check bounds consistency - helpers.assert_decomon_model_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - mode=mode, - dc_decomp=dc_decomp, - decimal=decimal, - ) - - -def test_convert_1D_forward_slope(slope, helpers): - ibp = True - affine = True - dc_decomp = False - - n, method, mode = 0, "forward-hybrid", "hybrid" - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - ref_nn = helpers.toy_network_tutorial(dtype=keras_config.floatx()) - ref_nn(input_ref) - - decomon_model = clone(ref_nn, method=method, final_ibp=ibp, final_affine=affine, slope=slope) - - # check slope of activation layers - for layer in decomon_model.layers: - if layer.__class__.__name__.endswith("Activation"): - assert layer.slope == Slope(slope) - - -def test_convert_1D_backward_slope(slope, helpers): - n, method, mode = 0, "crown-forward-hybrid", "hybrid" - ibp = True - affine = True - dc_decomp = False - - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - ref_nn = helpers.toy_network_tutorial(dtype=keras_config.floatx()) - ref_nn(input_ref) - - decomon_model = clone(ref_nn, method=method, final_ibp=ibp, final_affine=affine, slope=slope) - - # check slope of layers with activation - for layer in decomon_model.layers: - layer_class_name = layer.__class__.__name__ - if layer_class_name.endswith("Activation"): - assert layer.slope == Slope(slope) - - -def test_name_forward(): - layers = [] - layers.append(Input((1,))) - layers.append(Dense(1)) - layers.append(Dense(1, name="superman")) # specify the dimension of the input space - layers.append(Activation("relu")) - layers.append(Dense(1, activation="relu", name="batman")) - model = Sequential(layers) - - decomon_model_f = clone(model=model, method=ConvertMethod.FORWARD_HYBRID) - nb_superman_layers = len([layer for layer in decomon_model_f.layers if layer.name.startswith("superman_")]) - assert nb_superman_layers == 1 - nb_batman_layers = len([layer for layer in decomon_model_f.layers if layer.name.startswith("batman_")]) - assert nb_batman_layers == 2 - - -def test_name_backward(): - layers = [] - layers.append(Input((1,))) - layers.append(Dense(1)) - layers.append(Dense(1, name="superman")) # specify the dimension of the input space - layers.append(Activation("relu")) - layers.append(Dense(1, activation="relu", name="batman")) - model = Sequential(layers) - - decomon_model_b = clone(model=model, method=ConvertMethod.CROWN_FORWARD_HYBRID) - nb_superman_layers = len([layer for layer in decomon_model_b.layers if layer.name.startswith("superman_")]) - assert nb_superman_layers == 2 - nb_batman_layers = len([layer for layer in decomon_model_b.layers if layer.name.startswith("batman_")]) - assert nb_batman_layers == 3 - - -def test_convert_toy_models_1d(toy_model_1d, method, mode, helpers): - if not helpers.is_method_mode_compatible(method=method, mode=mode): - # skip method=ibp/crown-ibp with mode=affine/hybrid - pytest.skip(f"output mode {mode} is not compatible with convert method {method}") - - decimal = 4 - n = 6 - dc_decomp = False - ibp = get_ibp(mode=mode) - affine = get_affine(mode=mode) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs_) - input_decomon_ = helpers.get_inputs_np_for_decomon_model_from_full_inputs(inputs=inputs_) - - #  keras model and output of reference - ref_nn = toy_model_1d - output_ref_ = helpers.predict_on_small_numpy(ref_nn, input_ref_) - - # decomon conversion - if (get_direction(method) == FeedDirection.BACKWARD) and has_merge_layers(ref_nn): - # skip models with merge layers in backward direction as not yet implemented - with pytest.raises(NotImplementedError): - decomon_model = clone(ref_nn, method=method, final_ibp=ibp, final_affine=affine) - return - - decomon_model = clone(ref_nn, method=method, final_ibp=ibp, final_affine=affine) - - #  decomon outputs - outputs_ = helpers.predict_on_small_numpy(decomon_model, input_decomon_) - - #  check bounds consistency - helpers.assert_decomon_model_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - mode=mode, - dc_decomp=dc_decomp, - decimal=decimal, - ) - - -def test_convert_cnn(method, mode, helpers): - if not helpers.is_method_mode_compatible(method=method, mode=mode): - # skip method=ibp/crown-ibp with mode=affine/hybrid - pytest.skip(f"output mode {mode} is not compatible with convert method {method}") - - if get_direction(method) == FeedDirection.BACKWARD: - # skip as BackwardConv2D not yet ready - pytest.skip(f"BackwardConv2D not yet fully implemented") - - decimal = 4 - data_format = "channels_last" - odd, m_0, m_1 = 0, 0, 1 - - dc_decomp = False - ibp = get_ibp(mode=mode) - affine = get_affine(mode=mode) - - # numpy inputs - inputs_ = helpers.get_standard_values_images_box(data_format, odd, m0=m_0, m1=m_1, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs_) - input_ref_min_, input_ref_max_ = helpers.get_input_ref_bounds_from_full_inputs(inputs_) - - # flatten inputs - preprocess_layer = Flatten(data_format=data_format) - input_ref_reshaped_ = K.convert_to_numpy(preprocess_layer(input_ref_)) - input_ref_min_reshaped_ = K.convert_to_numpy(preprocess_layer(input_ref_min_)) - input_ref_max_reshaped_ = K.convert_to_numpy(preprocess_layer(input_ref_max_)) - - input_decomon_ = np.concatenate((input_ref_min_reshaped_[:, None], input_ref_max_reshaped_[:, None]), axis=1) - - #  keras model and output of reference - image_data_shape = input_ref_.shape[1:] # image shape: before flattening - ref_nn = helpers.toy_struct_cnn(dtype=keras_config.floatx(), image_data_shape=image_data_shape) - output_ref_ = helpers.predict_on_small_numpy(ref_nn, input_ref_reshaped_) - - # decomon conversion - decomon_model = clone(ref_nn, method=method, final_ibp=ibp, final_affine=affine) - - #  decomon outputs - outputs_ = helpers.predict_on_small_numpy(decomon_model, input_decomon_) - - #  check bounds consistency - z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_, h_, g_ = helpers.get_full_outputs_from_outputs_for_mode( - outputs_for_mode=outputs_, mode=mode, dc_decomp=dc_decomp, full_inputs=inputs_ - ) - helpers.assert_output_properties_box( - input_ref_reshaped_, - output_ref_, - h_, - g_, - input_ref_min_reshaped_, - input_ref_max_reshaped_, - u_c_, - w_u_, - b_u_, - l_c_, - w_l_, - b_l_, - decimal=decimal, - ) diff --git a/tests/test_clone_backward.py b/tests/test_clone_backward.py deleted file mode 100644 index 2202115e..00000000 --- a/tests/test_clone_backward.py +++ /dev/null @@ -1,68 +0,0 @@ -# creating toy network and assess that the decomposition is correct - - -import keras.config as keras_config - -from decomon.core import get_affine, get_ibp -from decomon.models.backward_cloning import convert_backward -from decomon.models.forward_cloning import convert_forward -from decomon.models.utils import ensure_functional_model - - -def test_convert_backward_1D(n, mode, floatx, decimal, helpers): - ibp = get_ibp(mode=mode) - affine = get_affine(mode=mode) - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - input_tensors = helpers.get_input_tensors_for_decomon_convert_from_full_inputs( - inputs=inputs, mode=mode, dc_decomp=dc_decomp - ) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - inputs_ = helpers.prepare_full_np_inputs_for_convert_model(inputs_, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs_) - - # keras model and output of reference - ref_nn = helpers.toy_network_tutorial(dtype=keras_config.floatx()) - output_ref_ = helpers.predict_on_small_numpy(ref_nn, input_ref_) - - # convert to functional - ref_nn = ensure_functional_model(ref_nn) - - # decomon conversion - back_bounds = [] - _, _, _, forward_map = convert_forward( - ref_nn, - ibp=ibp, - affine=affine, - shared=True, - input_tensors=input_tensors, - back_bounds=back_bounds, - ) - _, outputs, _, _ = convert_backward( - ref_nn, - input_tensors=input_tensors, - ibp=ibp, - affine=affine, - forward_map=forward_map, - final_ibp=ibp, - final_affine=affine, - back_bounds=back_bounds, - ) - - # decomon outputs - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_decomon_model_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - mode=mode, - dc_decomp=dc_decomp, - decimal=decimal, - ) diff --git a/tests/test_clone_forward.py b/tests/test_clone_forward.py deleted file mode 100644 index b7a5ce79..00000000 --- a/tests/test_clone_forward.py +++ /dev/null @@ -1,45 +0,0 @@ -# creating toy network and assess that the decomposition is correct - - -import keras.config as keras_config - -from decomon.core import get_affine, get_ibp -from decomon.models.forward_cloning import convert_forward - - -def test_convert_forward_1D(n, mode, floatx, decimal, helpers): - ibp = get_ibp(mode=mode) - affine = get_affine(mode=mode) - dc_decomp = False - - #  tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - input_tensors = helpers.get_input_tensors_for_decomon_convert_from_full_inputs( - inputs=inputs, mode=mode, dc_decomp=dc_decomp - ) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - inputs_ = helpers.prepare_full_np_inputs_for_convert_model(inputs_, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs_) - - # keras model and output of reference - ref_nn = helpers.toy_network_tutorial(dtype=keras_config.floatx()) - output_ref_ = helpers.predict_on_small_numpy(ref_nn, input_ref_) - - # decomon conversion - _, outputs, _, _ = convert_forward(ref_nn, ibp=ibp, affine=affine, shared=True, input_tensors=input_tensors) - - #  decomon outputs - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_decomon_model_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - mode=mode, - dc_decomp=dc_decomp, - decimal=decimal, - ) diff --git a/tests/test_conv.py b/tests/test_conv.py deleted file mode 100644 index 6317dfb5..00000000 --- a/tests/test_conv.py +++ /dev/null @@ -1,126 +0,0 @@ -# Test unit for decomon with Dense layers - - -import keras.config as keras_config -import numpy as np -import pytest -from keras.layers import Conv2D - -from decomon.core import ForwardMode, get_affine, get_ibp -from decomon.layers.convert import to_decomon -from decomon.layers.decomon_layers import DecomonConv2D - - -def test_Decomon_conv_box(data_format, mode, dc_decomp, floatx, decimal, helpers): - # skip unavailable combinations - if floatx == 16 and keras_config.backend() == "torch" and not helpers.in_GPU_mode(): - pytest.skip("Pytorch does not implement conv2d for float16 in CPU mode.") - - if data_format == "channels_first" and not helpers.in_GPU_mode() and keras_config.backend() == "tensorflow": - pytest.skip("data format 'channels first' is possible only in GPU mode for tensorflow.") - - odd, m_0, m_1 = 0, 0, 1 - kwargs_layer = dict(filters=10, kernel_size=(3, 3), dtype=keras_config.floatx(), data_format=data_format) - - # tensor inputs - inputs = helpers.get_tensor_decomposition_images_box(data_format, odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_images_box(data_format, odd, m0=m_0, m1=m_1, dc_decomp=dc_decomp) - - # decomon layer - decomon_layer = DecomonConv2D(dc_decomp=dc_decomp, mode=mode, **kwargs_layer) - - # original output (not computed here) - output_ref_ = None - - # decomon function - outputs = decomon_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - - # test with several kernel (negative or positive part) - W_, bias = decomon_layer.get_weights() - for np_function in [np.maximum, np.minimum]: - decomon_layer.set_weights([np_function(0.0, W_), bias]) - outputs_ = f_decomon(inputs_) - # check bounds consistency - if dc_decomp: - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - decimal=decimal, - mode=mode, - ) - else: - helpers.assert_decomon_layer_output_properties_box_linear( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - decimal=decimal, - mode=mode, - ) - - -def test_Decomon_conv_to_decomon_box(shared, floatx, dc_decomp, helpers): - # skip unavailable combinations - if floatx == 16 and keras_config.backend() == "torch" and not helpers.in_GPU_mode(): - pytest.skip("Pytorch does not implement conv2d for float16 in CPU mode.") - - data_format = "channels_last" - odd, m_0, m_1 = 0, 0, 1 - dc_decomp = True - mode = ForwardMode.HYBRID - ibp = get_ibp(mode=mode) - affine = get_affine(mode=mode) - kwargs_layer = dict(filters=10, kernel_size=(3, 3), dtype=keras_config.floatx(), data_format=data_format) - - if floatx == 16: - decimal = 1 - else: - decimal = 4 - - # tensor inputs - inputs = helpers.get_tensor_decomposition_images_box(data_format, odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_images_box(data_format, odd, m0=m_0, m1=m_1, dc_decomp=dc_decomp) - - # keras layer & function & output - keras_layer = Conv2D(**kwargs_layer) - output_ref = keras_layer(input_ref) - f_ref = helpers.function(inputs, output_ref) - output_ref_ = f_ref(inputs_) - - #  decomon layer & function & output via to_decomon - input_dim = helpers.get_input_dim_from_full_inputs(inputs) - decomon_layer = to_decomon(keras_layer, input_dim, dc_decomp=dc_decomp, shared=shared, ibp=ibp, affine=affine) - outputs = decomon_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - if dc_decomp: - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - decimal=decimal, - mode=mode, - ) - else: - helpers.assert_decomon_layer_output_properties_box_linear( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - decimal=decimal, - mode=mode, - ) diff --git a/tests/test_crown_layers_get_config.py b/tests/test_crown_layers_get_config.py deleted file mode 100644 index e5856e4f..00000000 --- a/tests/test_crown_layers_get_config.py +++ /dev/null @@ -1,23 +0,0 @@ -from decomon.core import BoxDomain, ForwardMode -from decomon.models.crown import Convert2BackwardMode, Fuse, MergeWithPrevious - - -def test_crown_layers(): - mode = ForwardMode.AFFINE - perturbation_domain = BoxDomain() - input_shape_layer = (1, 2, 4) - backward_shape_layer = (2, 5, 10) - - layer = Fuse(mode=mode) - config = layer.get_config() - assert config["mode"] == mode - - layer = Convert2BackwardMode(mode=mode, perturbation_domain=perturbation_domain) - config = layer.get_config() - assert config["mode"] == mode - assert "perturbation_domain" in config - - layer = MergeWithPrevious(input_shape_layer=input_shape_layer, backward_shape_layer=backward_shape_layer) - config = layer.get_config() - assert config["input_shape_layer"] == input_shape_layer - assert config["backward_shape_layer"] == backward_shape_layer diff --git a/tests/test_decomon_activation_layer.py b/tests/test_decomon_activation_layer.py deleted file mode 100644 index 7c2143b5..00000000 --- a/tests/test_decomon_activation_layer.py +++ /dev/null @@ -1,43 +0,0 @@ -from decomon.core import ForwardMode, Slope -from decomon.layers.decomon_layers import DecomonActivation - - -def test_decomon_activation_slope(helpers): - mode = ForwardMode.AFFINE - activation = "relu" - n = 2 - - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=False) - ( - x_0, - y_0, - z_0, - u_c_0, - W_u_0, - b_u_0, - l_c_0, - W_l_0, - b_l_0, - ) = inputs_ # numpy values - - outputs_by_slope = {} - for slope in Slope: - layer = DecomonActivation(activation, dc_decomp=False, mode=mode, slope=slope) - assert layer.slope == slope - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=False) - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs # tensors - inputs_for_mode = [z, W_u, b_u, W_l, b_l] - output = layer(inputs_for_mode) - f_func = helpers.function(inputs, output) - outputs_by_slope[slope] = f_func(inputs_) - - # check results - # O_Slope != Z_Slope - same_outputs_O_n_Z = [ - (a == b).all() for a, b in zip(outputs_by_slope[Slope.O_SLOPE], outputs_by_slope[Slope.Z_SLOPE]) - ] - assert not all(same_outputs_O_n_Z) - - # V_Slope == Z_Slope - for a, b in zip(outputs_by_slope[Slope.V_SLOPE], outputs_by_slope[Slope.Z_SLOPE]): - assert (a == b).all() diff --git a/tests/test_decomon_compute_output_shape.py b/tests/test_decomon_compute_output_shape.py deleted file mode 100644 index c8d0d4c8..00000000 --- a/tests/test_decomon_compute_output_shape.py +++ /dev/null @@ -1,105 +0,0 @@ -import pytest - -from decomon.layers.decomon_layers import ( - DecomonBatchNormalization, - DecomonConv2D, - DecomonDense, - DecomonFlatten, -) -from decomon.layers.decomon_merge_layers import DecomonAdd, DecomonConcatenate -from decomon.layers.decomon_reshape import DecomonReshape -from decomon.layers.maxpooling import DecomonMaxPooling2D - - -@pytest.mark.parametrize( - "layer_class, layer_kwargs", - [ - (DecomonDense, dict(units=3, use_bias=True)), - (DecomonFlatten, dict()), - (DecomonBatchNormalization, dict(center=True, scale=True)), - (DecomonBatchNormalization, dict(center=False, scale=False)), - (DecomonConv2D, dict(filters=10, kernel_size=(3, 3))), - (DecomonMaxPooling2D, dict(pool_size=(2, 2), strides=(2, 2), padding="valid")), - (DecomonReshape, dict(target_shape=(1, -1, 1))), - ], -) -@pytest.mark.parametrize("n", [0]) # limit 1d cases -@pytest.mark.parametrize("odd", [0]) # limit multid cases -@pytest.mark.parametrize("data_format", ["channels_last"]) # limit images cases -def test_compute_output_shape( - helpers, - mode, - dc_decomp, - layer_class, - layer_kwargs, - inputs_for_mode, # decomon inputs: symbolic tensors - input_ref, # keras input: symbolic tensor - inputs_for_mode_, # decomon inputs: numpy arrays - inputs_metadata, # inputs metadata: data_format, ... -): - # skip nonsensical combinations - if (layer_class == DecomonBatchNormalization or layer_class == DecomonMaxPooling2D) and dc_decomp: - pytest.skip(f"{layer_class} with dc_decomp=True not yet implemented.") - if (layer_class == DecomonConv2D or layer_class == DecomonMaxPooling2D) and len(input_ref.shape) < 4: - pytest.skip(f"{layer_class} applies only on image-like inputs.") - - # add data_format for convolution and maxpooling - if layer_class in (DecomonConv2D, DecomonMaxPooling2D): - layer_kwargs["data_format"] = inputs_metadata["data_format"] - - # construct layer - layer = layer_class(mode=mode, dc_decomp=dc_decomp, **layer_kwargs) - - # check symbolic tensor output shapes - inputshapes = [i.shape for i in inputs_for_mode] - outputshapes = layer.compute_output_shape(inputshapes) - outputs = layer(inputs_for_mode) - assert [o.shape for o in outputs] == outputshapes - - # check output shapes for concrete call - outputs_ = layer(inputs_for_mode_) - # compare without batch sizes - assert [tuple(o.shape)[1:] for o in outputs_] == [s[1:] for s in outputshapes] - - -@pytest.mark.parametrize( - "layer_class, layer_kwargs", - [ - (DecomonAdd, dict()), - (DecomonConcatenate, dict()), - ], -) -@pytest.mark.parametrize("n", [0]) # limit 1d cases -@pytest.mark.parametrize("odd", [0]) # limit multid cases -@pytest.mark.parametrize("data_format", ["channels_last"]) # limit images cases -def test_merge_layers_compute_output_shape( - helpers, - mode, - dc_decomp, - layer_class, - layer_kwargs, - inputs_for_mode, # decomon inputs: symbolic tensors - inputs_for_mode_, # decomon inputs: numpy arrays -): - if dc_decomp: - pytest.skip(f"{layer_class} with dc_decomp=True not yet implemented.") - - # tensors inputs - concatenated_inputs_for_mode = inputs_for_mode + inputs_for_mode - - # numpy inputs - concatenated_inputs_for_mode_ = inputs_for_mode_ + inputs_for_mode_ - - # construct layer - layer = layer_class(mode=mode, dc_decomp=dc_decomp, **layer_kwargs) - - # check symbolic tensor output shapes - inputshapes = [i.shape for i in concatenated_inputs_for_mode] - outputshapes = layer.compute_output_shape(inputshapes) - outputs = layer(concatenated_inputs_for_mode) - assert [o.shape for o in outputs] == outputshapes - - # check output shapes for concrete call - outputs_ = layer(concatenated_inputs_for_mode_) - # compare without batch sizes - assert [tuple(o.shape)[1:] for o in outputs_] == [s[1:] for s in outputshapes] diff --git a/tests/test_decomon_reset_layer.py b/tests/test_decomon_reset_layer.py deleted file mode 100644 index ce0634f4..00000000 --- a/tests/test_decomon_reset_layer.py +++ /dev/null @@ -1,218 +0,0 @@ -import keras.ops as K -import numpy as np -import pytest -from keras.layers import BatchNormalization, Conv2D, Dense, Flatten -from numpy.testing import assert_almost_equal - -from decomon.core import ForwardMode -from decomon.layers.decomon_layers import ( - DecomonBatchNormalization, - DecomonConv2D, - DecomonDense, - DecomonFlatten, -) - - -def test_decomondense_reset_layer(helpers, use_bias): - dc_decomp = False - input_dim = 1 - units = 3 - mode = ForwardMode.HYBRID - layer = Dense(units=units, use_bias=use_bias) - layer(K.zeros((2, input_dim))) - decomon_layer = DecomonDense(units=units, use_bias=use_bias, mode=mode, dc_decomp=dc_decomp) - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - decomon_layer(inputs_for_mode) - - kernel = layer.kernel - layer.kernel.assign(2 * np.ones_like(kernel)) - decomon_layer.kernel.assign(np.zeros_like(kernel)) - if use_bias: - bias = layer.bias - layer.bias.assign(np.ones_like(bias)) - decomon_layer.bias.assign(np.zeros_like(bias)) - - decomon_layer.reset_layer(layer) - assert decomon_layer.kernel is not layer.kernel - assert_almost_equal(K.convert_to_numpy(decomon_layer.kernel), K.convert_to_numpy(layer.kernel)) - if use_bias: - assert len(layer.weights) == 2 - assert decomon_layer.bias is not layer.bias - assert_almost_equal(K.convert_to_numpy(decomon_layer.bias), K.convert_to_numpy(layer.bias)) - else: - assert len(layer.weights) == 1 - - -def test_decomondense_reset_layer_decomon_with_new_weights(helpers): - dc_decomp = False - input_dim = 1 - units = 3 - mode = ForwardMode.HYBRID - layer = Dense(units=units) - layer(K.zeros((2, input_dim))) - decomon_layer = DecomonDense(units=units, mode=mode, dc_decomp=dc_decomp) - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - # Add new variables - decomon_layer.add_weight(shape=(input_dim, units), initializer="ones", name="alpha", trainable=True) - decomon_layer.add_weight(shape=(input_dim, units), initializer="ones", name="beta", trainable=False) - # Build layer - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - decomon_layer(inputs_for_mode) - assert len(decomon_layer.weights) == 4 - assert len(decomon_layer.trainable_weights) == 3 - - kernel, bias = layer.get_weights() - layer.set_weights([2 * np.ones_like(kernel), np.ones_like(bias)]) - decomon_layer.kernel.assign(np.zeros_like(kernel)) - decomon_layer.bias.assign(np.zeros_like(bias)) - - decomon_layer.reset_layer(layer) - assert decomon_layer.kernel is not layer.kernel - assert decomon_layer.bias is not layer.bias - assert_almost_equal(K.convert_to_numpy(decomon_layer.kernel), K.convert_to_numpy(layer.kernel)) - assert_almost_equal(K.convert_to_numpy(decomon_layer.bias), K.convert_to_numpy(layer.bias)) - - -def test_decomondense_reset_layer_keras_with_new_weights(helpers): - dc_decomp = False - input_dim = 1 - units = 3 - mode = ForwardMode.HYBRID - layer = Dense(units=units) - # Add new variables - layer.add_weight(shape=(input_dim, units), initializer="ones", name="alpha", trainable=False) - layer(K.zeros((2, input_dim))) - decomon_layer = DecomonDense(units=units, mode=mode, dc_decomp=dc_decomp) - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - decomon_layer(inputs_for_mode) - assert len(layer.weights) == 3 - - kernel = layer.kernel - layer.kernel.assign(2 * np.ones_like(kernel)) - decomon_layer.kernel.assign(np.zeros_like(kernel)) - bias = layer.bias - layer.bias.assign(np.ones_like(bias)) - decomon_layer.bias.assign(np.zeros_like(bias)) - - decomon_layer.reset_layer(layer) - assert decomon_layer.kernel is not layer.kernel - assert decomon_layer.bias is not layer.bias - assert_almost_equal(K.convert_to_numpy(decomon_layer.kernel), K.convert_to_numpy(layer.kernel)) - assert_almost_equal(K.convert_to_numpy(decomon_layer.bias), K.convert_to_numpy(layer.bias)) - - -def test_decomondense_reset_layer_ko_keraslayer_not_nuilt(): - dc_decomp = False - input_dim = 1 - units = 3 - mode = ForwardMode.HYBRID - layer = Dense(units=units) - decomon_layer = DecomonDense(units=units, mode=mode, dc_decomp=dc_decomp) - with pytest.raises(ValueError): - decomon_layer.reset_layer(layer) - - -def test_decomondense_reset_layer_ko_decomonlayer_not_nuilt(): - dc_decomp = False - input_dim = 1 - units = 3 - mode = ForwardMode.HYBRID - layer = Dense(units=units) - layer(K.zeros((2, input_dim))) - decomon_layer = DecomonDense(units=units, mode=mode, dc_decomp=dc_decomp) - with pytest.raises(ValueError): - decomon_layer.reset_layer(layer) - - -def test_decomonconv2d_reset_layer(helpers, use_bias): - dc_decomp = False - odd = 0 - data_format = "channels_last" - filters = 3 - kernel_size = (3, 3) - mode = ForwardMode.HYBRID - - inputs = helpers.get_tensor_decomposition_images_box(data_format, odd, dc_decomp=False) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - layer = Conv2D(filters=filters, kernel_size=kernel_size, use_bias=use_bias) - layer(input_ref) - decomon_layer = DecomonConv2D( - filters=filters, kernel_size=kernel_size, use_bias=use_bias, mode=mode, dc_decomp=dc_decomp - ) - decomon_layer(inputs_for_mode) - - kernel = layer.kernel - layer.kernel.assign(2 * np.ones_like(kernel)) - decomon_layer.kernel.assign(np.zeros_like(kernel)) - if use_bias: - bias = layer.bias - layer.bias.assign(np.ones_like(bias)) - decomon_layer.bias.assign(np.zeros_like(bias)) - - decomon_layer.reset_layer(layer) - assert decomon_layer.kernel is not layer.kernel - assert_almost_equal(K.convert_to_numpy(decomon_layer.kernel), K.convert_to_numpy(layer.kernel)) - if use_bias: - assert decomon_layer.bias is not layer.bias - assert_almost_equal(K.convert_to_numpy(decomon_layer.bias), K.convert_to_numpy(layer.bias)) - - -@pytest.mark.parametrize( - "center, scale", - [ - (True, True), - (False, False), - ], -) -def test_decomonbacthnormalization_reset_layer(helpers, center, scale): - dc_decomp = False - odd = 0 - mode = ForwardMode.HYBRID - inputs = helpers.get_tensor_decomposition_multid_box(odd=odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - layer = BatchNormalization(center=center, scale=scale) - layer(input_ref) - decomon_layer = DecomonBatchNormalization(center=center, scale=scale, mode=mode, dc_decomp=dc_decomp) - decomon_layer(inputs_for_mode) - - moving_mean = layer.moving_mean - layer.moving_mean.assign(np.ones_like(moving_mean)) - decomon_layer.moving_mean.assign(np.zeros_like(moving_mean)) - moving_variance = layer.moving_variance - layer.moving_variance.assign(np.ones_like(moving_variance)) - decomon_layer.moving_variance.assign(np.zeros_like(moving_variance)) - if center: - beta = layer.beta - layer.beta.assign(np.ones_like(beta)) - decomon_layer.beta.assign(np.zeros_like(beta)) - if scale: - gamma = layer.gamma - layer.gamma.assign(np.ones_like(gamma)) - decomon_layer.gamma.assign(np.zeros_like(gamma)) - - decomon_layer.reset_layer(layer) - - keras_weights = layer.get_weights() - decomon_weights = layer.get_weights() - for i in range(len(decomon_weights)): - assert_almost_equal(decomon_weights[i], keras_weights[i]) - - -def test_decomonflatten_reset_layer(helpers): - dc_decomp = False - input_dim = 1 - mode = ForwardMode.HYBRID - layer = Flatten() - layer(K.zeros((2, input_dim))) - decomon_layer = DecomonFlatten(mode=mode, dc_decomp=dc_decomp) - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - decomon_layer(inputs_for_mode) - - decomon_layer.reset_layer(layer) diff --git a/tests/test_dense_layer.py b/tests/test_dense_layer.py deleted file mode 100644 index 896fd151..00000000 --- a/tests/test_dense_layer.py +++ /dev/null @@ -1,291 +0,0 @@ -# Test unit for decomon with Dense layers - - -import keras.config as keras_config -import numpy as np -from keras.layers import Dense -from numpy.testing import assert_almost_equal - -from decomon.core import ForwardMode, get_affine, get_ibp -from decomon.layers.convert import to_decomon -from decomon.layers.decomon_layers import DecomonDense -from decomon.models.utils import split_activation - - -def test_DecomonDense_1D_box(n, mode, shared, floatx, decimal, helpers): - dc_decomp = True - kwargs_layer = dict(units=1, use_bias=True, dtype=keras_config.floatx()) - - # tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - - # keras layer & function - keras_layer = Dense(**kwargs_layer) - output_ref = keras_layer(input_ref) - f_ref = helpers.function(inputs, output_ref) - - # decomon layer & function - decomon_layer = DecomonDense(dc_decomp=dc_decomp, shared=shared, mode=mode, **kwargs_layer) - decomon_layer.share_weights(keras_layer) - outputs = decomon_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - - # test with several kernel values - kernel_coeffs = [2, -3] - W_, bias = decomon_layer.get_weights() - - for kernel_coeff in kernel_coeffs: - # set weights - if not shared: - decomon_layer.set_weights([2 * np.ones_like(W_), np.ones_like(bias)]) - keras_layer.set_weights([2 * np.ones_like(W_), np.ones_like(bias)]) - - # keras & decomon outputs - output_ref_ = f_ref(inputs_) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - decimal=decimal, - mode=mode, - ) - - -def test_DecomonDense_1D_box_nodc(n, helpers): - dc_decomp = False - shared = False - mode = ForwardMode.HYBRID - decimal = 5 - kwargs_layer = dict(units=1, use_bias=True, dtype=keras_config.floatx()) - - # tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - - # keras layer & function - keras_layer = Dense(**kwargs_layer) - output_ref = keras_layer(input_ref) - f_ref = helpers.function(inputs, output_ref) - - #  decomon layer & function - decomon_layer = DecomonDense(dc_decomp=dc_decomp, shared=shared, mode=mode, **kwargs_layer) - decomon_layer.share_weights(keras_layer) - outputs = decomon_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - - #  test with several kernel values - kernel_coeffs = [2, -3] - W_, bias = decomon_layer.get_weights() - - for kernel_coeff in kernel_coeffs: - # set weights - if not shared: - decomon_layer.set_weights([2 * np.ones_like(W_), np.ones_like(bias)]) - keras_layer.set_weights([2 * np.ones_like(W_), np.ones_like(bias)]) - - # keras & decomon outputs - output_ref_ = f_ref(inputs_) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_decomon_layer_output_properties_box_linear( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - decimal=decimal, - mode=mode, - ) - - -def test_DecomonDense_multiD_box(odd, mode, dc_decomp, helpers): - shared = False - kwargs_layer = dict(units=1, use_bias=True, dtype=keras_config.floatx()) - decimal = 5 - - # tensor inputs - inputs = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - - # keras layer & function - keras_layer = Dense(**kwargs_layer) - output_ref = keras_layer(input_ref) - f_ref = helpers.function(inputs, output_ref) - - # decomon layer & function - decomon_layer = DecomonDense(dc_decomp=dc_decomp, shared=shared, mode=mode, **kwargs_layer) - decomon_layer.share_weights(keras_layer) - outputs = decomon_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - - # test with several kernel values - kernel_coeffs = [2, -3] - W_, bias = decomon_layer.get_weights() - - for kernel_coeff in kernel_coeffs: - # set weights - if not shared: - decomon_layer.set_weights([2 * np.ones_like(W_), np.ones_like(bias)]) - keras_layer.set_weights([2 * np.ones_like(W_), np.ones_like(bias)]) - - # keras & decomon outputs - output_ref_ = f_ref(inputs_) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - if dc_decomp: - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - decimal=decimal, - mode=mode, - ) - else: - helpers.assert_decomon_layer_output_properties_box_linear( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - decimal=decimal, - mode=mode, - ) - - -def test_DecomonDense_1D_to_decomon_box(n, activation, mode, shared, helpers): - dc_decomp = True - kwargs_layer = dict(units=1, use_bias=True, dtype=keras_config.floatx()) - decimal = 5 - ibp = get_ibp(mode=mode) - affine = get_affine(mode=mode) - - # tensor inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - - # keras layer & function & output - keras_layer = Dense(**kwargs_layer) - output_ref = keras_layer(input_ref) - f_ref = helpers.function(inputs, output_ref) - output_ref_ = f_ref(inputs_) - - # decomon layer & function & output via to_decomon - input_dim = helpers.get_input_dim_from_full_inputs(inputs) - layers_ref = split_activation(keras_layer) - decomon_layers = [] - for layer in layers_ref: - decomon_layers.append(to_decomon(layer, input_dim, dc_decomp=dc_decomp, ibp=ibp, affine=affine, shared=shared)) - outputs = decomon_layers[0](inputs_for_mode) - if len(decomon_layers) > 1: - outputs = decomon_layers[1](outputs) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - decimal=decimal, - mode=mode, - ) - - # check same weights - W_, bias = decomon_layers[0].get_weights() - W_0, b_0 = keras_layer.get_weights() - assert_almost_equal(W_, W_0, decimal=decimal, err_msg="wrong decomposition") - assert_almost_equal(bias, b_0, decimal=decimal, err_msg="wrong decomposition") - - -def test_DecomonDense_multiD_to_decomon_box(odd, activation, mode, dc_decomp, helpers): - shared = False - kwargs_layer = dict(units=1, use_bias=True, dtype=keras_config.floatx()) - decimal = 5 - ibp = get_ibp(mode=mode) - affine = get_affine(mode=mode) - - # tensor inputs - inputs = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - - # keras layer & function & output - keras_layer = Dense(**kwargs_layer) - output_ref = keras_layer(input_ref) - W_0, b_0 = keras_layer.get_weights() - if odd == 0: - W_0[0] = 1.037377 - W_0[1] = -0.7575816 - - keras_layer.set_weights([W_0, b_0]) - if odd == 1: - W_0[0] = -0.1657672 - W_0[1] = -0.2613032 - W_0[2] = 0.08437371 - keras_layer.set_weights([W_0, b_0]) - f_ref = helpers.function(inputs, output_ref) - output_ref_ = f_ref(inputs_) - - #  decomon layer & function via to_decomon - input_dim = helpers.get_input_dim_from_full_inputs(inputs) - layers_ref = split_activation(keras_layer) - decomon_layers = [] - for layer in layers_ref: - decomon_layers.append(to_decomon(layer, input_dim, dc_decomp=dc_decomp, ibp=ibp, affine=affine, shared=shared)) - outputs = decomon_layers[0](inputs_for_mode) - if len(decomon_layers) > 1: - outputs = decomon_layers[1](outputs) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - if dc_decomp: - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - decimal=decimal, - mode=mode, - ) - else: - helpers.assert_decomon_layer_output_properties_box_linear( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - decimal=decimal, - mode=mode, - ) - - # check same weights - W_, bias = decomon_layers[0].get_weights() - W_0, b_0 = keras_layer.get_weights() - assert_almost_equal(W_, W_0, decimal=decimal, err_msg="wrong decomposition") - assert_almost_equal(bias, b_0, decimal=decimal, err_msg="wrong decomposition") diff --git a/tests/test_layers_convert_utils.py b/tests/test_layers_convert_utils.py deleted file mode 100644 index e318086d..00000000 --- a/tests/test_layers_convert_utils.py +++ /dev/null @@ -1,25 +0,0 @@ -import pytest -from keras.layers import Dense, Input - -from decomon.layers.convert import get_layer_input_shape - - -def test_get_layer_input_shape_nok_uncalled(): - layer = Dense(3) - with pytest.raises(AttributeError): - get_layer_input_shape(layer) - - -def test_get_layer_input_shape_nok_multiple_shapes(): - layer = Dense(3) - layer(Input((1,))) - layer(Input((1, 1))) - with pytest.raises(AttributeError): - get_layer_input_shape(layer) - - -def test_get_layer_input_shape_ok(): - layer = Dense(3) - layer(Input((1,))) - input_shape = get_layer_input_shape(layer) - assert input_shape == [(None, 1)] diff --git a/tests/test_layers_get_config.py b/tests/test_layers_get_config.py deleted file mode 100644 index ba672ca8..00000000 --- a/tests/test_layers_get_config.py +++ /dev/null @@ -1,111 +0,0 @@ -import pytest - -from decomon.core import ForwardMode -from decomon.layers.decomon_layers import ( - DecomonActivation, - DecomonBatchNormalization, - DecomonConv2D, - DecomonDense, - DecomonDropout, - DecomonFlatten, - DecomonInputLayer, -) -from decomon.layers.decomon_merge_layers import ( - DecomonAdd, - DecomonAverage, - DecomonConcatenate, - DecomonDot, - DecomonMaximum, - DecomonMinimum, - DecomonMultiply, - DecomonSubtract, -) -from decomon.layers.decomon_reshape import DecomonPermute, DecomonReshape -from decomon.layers.maxpooling import DecomonMaxPooling2D - - -def test_decomon_reshape(): - dims = (1, 2, 3) - mode = ForwardMode.AFFINE - layer = DecomonPermute(dims=dims, mode=mode) - config = layer.get_config() - assert config["dims"] == dims - assert config["mode"] == mode - - shape = (1, 2, 3) - layer = DecomonReshape(target_shape=shape, mode=mode) - config = layer.get_config() - assert config["target_shape"] == shape - assert config["mode"] == mode - - -def test_maxpooling(): - layer = DecomonMaxPooling2D() - config = layer.get_config() - assert "pool_size" in config - - -def test_decomon_merge_layers(): - for cls in [ - DecomonAdd, - DecomonAverage, - DecomonConcatenate, - DecomonDot, - DecomonMaximum, - DecomonMinimum, - DecomonMultiply, - DecomonSubtract, - ]: - layer = cls() - config = layer.get_config() - assert "mode" in config - if layer == DecomonConcatenate: - assert "axis" in config - elif layer == DecomonDot: - assert "axes" in config - - -def test_decomon_layers(): - activation_mapping = { - "relu": "relu", - "softmax": "softmax", - "linear": "linear", - None: "linear", - } - for activation_arg, activation_res in activation_mapping.items(): - layer = DecomonActivation(activation=activation_arg) - config = layer.get_config() - print(config) - print(layer.activation) - assert config["activation"] == activation_res - - units = 2 - layer = DecomonDense(units=units) - config = layer.get_config() - assert config["units"] == units - - filters = 2 - kernel_size = 3, 3 - layer = DecomonConv2D(filters=filters, kernel_size=kernel_size) - config = layer.get_config() - assert config["filters"] == filters - assert config["kernel_size"] == kernel_size - - rate = 0.9 - layer = DecomonDropout(rate=rate) - config = layer.get_config() - assert config["rate"] == rate - - shape = (2, 5) - layer = DecomonInputLayer(shape=shape) - config = layer.get_config() - assert config["batch_shape"] == (None,) + shape - - for cls in [DecomonBatchNormalization, DecomonFlatten]: - layer = cls() - config = layer.get_config() - assert "mode" in config - if layer == DecomonBatchNormalization: - assert "axis" in config - elif layer == DecomonFlatten: - assert "data_format" in config diff --git a/tests/test_merge.py b/tests/test_merge.py deleted file mode 100644 index c3514ca5..00000000 --- a/tests/test_merge.py +++ /dev/null @@ -1,246 +0,0 @@ -# Test unit for decomon with Dense layers - - -import keras.config as keras_config -import numpy as np -import pytest -from keras.layers import Add, Average, Concatenate, Maximum, Minimum, Multiply, Subtract - -from decomon.core import ForwardMode, get_affine, get_ibp -from decomon.layers.convert import to_decomon -from decomon.layers.decomon_merge_layers import ( - DecomonAdd, - DecomonAverage, - DecomonConcatenate, - DecomonMaximum, - DecomonMinimum, - DecomonMultiply, - DecomonSubtract, -) - - -def add_op(x, y): - return x + y - - -def subtract_op(x, y): - return x - y - - -def multiply_op(x, y): - return x * y - - -def average_op(x, y): - return (x + y) / 2.0 - - -def concatenate_op(x, y): - return np.concatenate([x, y], -1) - - -@pytest.mark.parametrize( - "decomon_op_class, tensor_op, decomon_op_kwargs", - [ - (DecomonAdd, add_op, {}), - (DecomonSubtract, subtract_op, {}), - (DecomonAverage, average_op, {}), - (DecomonMaximum, np.maximum, {}), - (DecomonMinimum, np.minimum, {}), - (DecomonConcatenate, concatenate_op, {"axis": -1}), - (DecomonMultiply, multiply_op, {}), - ], -) -def test_DecomonOp_1D_box(decomon_op_class, tensor_op, decomon_op_kwargs, n, mode, floatx, decimal, helpers): - dc_decomp = False - - #  tensor inputs - inputs_0 = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_1 = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode_0 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_0, mode=mode, dc_decomp=dc_decomp) - inputs_for_mode_1 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_1, mode=mode, dc_decomp=dc_decomp) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs=inputs_) - - # original output - output_ref_ = tensor_op(input_ref_, input_ref_) - - # decomon output - decomon_op = decomon_op_class(dc_decomp=dc_decomp, mode=mode, dtype=keras_config.floatx(), **decomon_op_kwargs) - output = decomon_op(inputs_for_mode_0 + inputs_for_mode_1) - f_decomon = helpers.function(inputs_0 + inputs_1, output) - outputs_ = f_decomon(inputs_ + inputs_) - - #  check bounds consistency - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - mode=mode, - dc_decomp=dc_decomp, - decimal=decimal, - ) - - -@pytest.mark.parametrize( - "decomon_op_class, tensor_op, decomon_op_kwargs", - [ - (DecomonAdd, add_op, {}), - (DecomonSubtract, subtract_op, {}), - (DecomonAverage, average_op, {}), - (DecomonMaximum, np.maximum, {}), - (DecomonMinimum, np.minimum, {}), - (DecomonConcatenate, concatenate_op, {"axis": -1}), - (DecomonMultiply, multiply_op, {}), - ], -) -def test_DecomonOp_multiD_box(decomon_op_class, tensor_op, decomon_op_kwargs, odd, mode, floatx, decimal, helpers): - dc_decomp = False - - #  tensor inputs - inputs_0 = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_1 = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_for_mode_0 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_0, mode=mode, dc_decomp=dc_decomp) - inputs_for_mode_1 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_1, mode=mode, dc_decomp=dc_decomp) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs=inputs_) - - # original output - output_ref_ = tensor_op(input_ref_, input_ref_) - - # decomon output - decomon_op = decomon_op_class(dc_decomp=dc_decomp, mode=mode, dtype=keras_config.floatx(), **decomon_op_kwargs) - output = decomon_op(inputs_for_mode_0 + inputs_for_mode_1) - f_decomon = helpers.function(inputs_0 + inputs_1, output) - outputs_ = f_decomon(inputs_ + inputs_) - - #  check bounds consistency - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - mode=mode, - dc_decomp=dc_decomp, - decimal=decimal, - ) - - -@pytest.mark.parametrize( - "layer_class, tensor_op, layer_kwargs", - [ - (Add, add_op, {}), - (Average, average_op, {}), - (Subtract, subtract_op, {}), - (Maximum, np.maximum, {}), - (Minimum, np.minimum, {}), - (Concatenate, concatenate_op, {"axis": -1}), - (Multiply, multiply_op, {}), - ], -) -def test_Decomon_1D_box_to_decomon(layer_class, tensor_op, layer_kwargs, n, helpers): - dc_decomp = False - mode = ForwardMode.HYBRID - ibp = get_ibp(mode=mode) - affine = get_affine(mode=mode) - decimal = 5 - - #  tensor inputs - inputs_0 = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_1 = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode_0 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_0, mode=mode, dc_decomp=dc_decomp) - inputs_for_mode_1 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_1, mode=mode, dc_decomp=dc_decomp) - input_ref_0 = helpers.get_input_ref_from_full_inputs(inputs=inputs_0) - input_ref_1 = helpers.get_input_ref_from_full_inputs(inputs=inputs_1) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs=inputs_) - - # original output - output_ref_ = tensor_op(input_ref_, input_ref_) - - # to_decomon - ref_op = layer_class(dtype=keras_config.floatx(), **layer_kwargs) - ref_op([input_ref_0, input_ref_1]) - decomon_op = to_decomon( - ref_op, input_dim=helpers.get_input_dim_from_full_inputs(inputs_0), dc_decomp=dc_decomp, affine=affine, ibp=ibp - ) - - # decomon output - output = decomon_op(inputs_for_mode_0 + inputs_for_mode_1) - f_decomon = helpers.function(inputs_0 + inputs_1, output) - outputs_ = f_decomon(inputs_ + inputs_) - - #  check bounds consistency - helpers.assert_decomon_layer_output_properties_box_linear( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - mode=mode, - dc_decomp=dc_decomp, - decimal=decimal, - ) - - -#### to_decomon multiD - - -@pytest.mark.parametrize( - "layer_class, tensor_op, layer_kwargs", - [ - (Add, add_op, {}), - (Average, average_op, {}), - (Subtract, subtract_op, {}), - (Maximum, np.maximum, {}), - (Minimum, np.minimum, {}), - (Concatenate, concatenate_op, {"axis": -1}), - (Multiply, multiply_op, {}), - ], -) -def test_Decomon_multiD_box_to_decomon(layer_class, tensor_op, layer_kwargs, odd, helpers): - dc_decomp = False - mode = ForwardMode.HYBRID - ibp = get_ibp(mode=mode) - affine = get_affine(mode=mode) - decimal = 5 - - #  tensor inputs - inputs_0 = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_1 = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_for_mode_0 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_0, mode=mode, dc_decomp=dc_decomp) - inputs_for_mode_1 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_1, mode=mode, dc_decomp=dc_decomp) - input_ref_0 = helpers.get_input_ref_from_full_inputs(inputs=inputs_0) - input_ref_1 = helpers.get_input_ref_from_full_inputs(inputs=inputs_1) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs=inputs_) - - # original output - output_ref_ = tensor_op(input_ref_, input_ref_) - - #  to_decomon - ref_op = layer_class(dtype=keras_config.floatx(), **layer_kwargs) - ref_op([input_ref_0, input_ref_1]) - decomon_op = to_decomon( - ref_op, input_dim=helpers.get_input_dim_from_full_inputs(inputs_0), dc_decomp=dc_decomp, affine=affine, ibp=ibp - ) - - # decomon output - output = decomon_op(inputs_for_mode_0 + inputs_for_mode_1) - f_decomon = helpers.function(inputs_0 + inputs_1, output) - outputs_ = f_decomon(inputs_ + inputs_) - - #  check bounds consistency - helpers.assert_decomon_layer_output_properties_box_linear( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - mode=mode, - dc_decomp=dc_decomp, - decimal=decimal, - ) diff --git a/tests/test_metric.py b/tests/test_metric.py deleted file mode 100644 index a7af674a..00000000 --- a/tests/test_metric.py +++ /dev/null @@ -1,34 +0,0 @@ -import keras.ops as K - -from decomon.metrics.utils import categorical_cross_entropy - - -def test_categorical_cross_entropy(odd, mode, floatx, decimal, helpers): - dc_decomp = False - - # tensor inputs - inputs = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - - # original output - f_ref = helpers.function(inputs, -input_ref + K.log(K.sum(K.exp(input_ref), -1))[:, None]) - output_ref_ = f_ref(inputs_) - - # decomon output - output = categorical_cross_entropy(inputs_for_mode, dc_decomp=dc_decomp, mode=mode) - f_entropy = helpers.function(inputs, output) - outputs_ = f_entropy(inputs_) - - # check bounds consistency - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - mode=mode, - decimal=decimal, - ) diff --git a/tests/test_metrics_get_config.py b/tests/test_metrics_get_config.py deleted file mode 100644 index 5d7fde03..00000000 --- a/tests/test_metrics_get_config.py +++ /dev/null @@ -1,34 +0,0 @@ -from decomon.core import ForwardMode -from decomon.metrics.loss import DecomonLossFusion, DecomonRadiusRobust -from decomon.metrics.metric import ( - AdversarialCheck, - AdversarialScore, - MetricMode, - UpperScore, -) - - -def test_decomon_loss(): - backward = True - mode = ForwardMode.AFFINE - - layer = DecomonLossFusion(mode=mode, backward=backward) - config = layer.get_config() - assert config["backward"] == backward - assert config["mode"] == mode - - layer = DecomonRadiusRobust(backward=backward, mode=mode) - config = layer.get_config() - assert config["backward"] == backward - assert config["mode"] == mode - - -def test_metric(): - ibp, affine, mode, perturbation_domain = True, False, MetricMode.BACKWARD, {} - for cls in [AdversarialCheck, AdversarialScore, UpperScore]: - layer = cls(ibp=ibp, affine=affine, mode=mode, perturbation_domain=perturbation_domain) - config = layer.get_config() - assert config["ibp"] == ibp - assert config["affine"] == affine - assert config["mode"] == mode - assert "perturbation_domain" in config diff --git a/tests/test_models_get_config.py b/tests/test_models_get_config.py deleted file mode 100644 index 43174d20..00000000 --- a/tests/test_models_get_config.py +++ /dev/null @@ -1,28 +0,0 @@ -from keras.layers import Dense, Input - -from decomon.models import DecomonModel - - -def test_get_config(): - input = Input((1,)) - output = Dense(2)(Dense(3)(input)) - decomon_model = DecomonModel(input, output) - config = decomon_model.get_config() - print(config) - expected_keys = [ - "layers", - "name", - "input_layers", - "output_layers", - "perturbation_domain", - "dc_decomp", - "method", - "ibp", - "affine", - "finetune", - "shared", - "backward_bounds", - ] - for k in expected_keys: - assert k in config - assert len(config["layers"]) == len(decomon_model.layers) diff --git a/tests/test_models_utils.py b/tests/test_models_utils.py deleted file mode 100644 index d15c344c..00000000 --- a/tests/test_models_utils.py +++ /dev/null @@ -1,170 +0,0 @@ -import keras -import keras.ops as K -import numpy as np -import pytest -from keras import Input -from keras.layers import Activation, Conv2D, Dense, Layer, Permute, PReLU -from numpy.testing import assert_almost_equal - -from decomon.keras_utils import get_weight_index_from_name -from decomon.models.utils import preprocess_layer, split_activation - - -@pytest.mark.parametrize( - "layer_class, layer_kwargs, input_shape_wo_batchsize, embedded_activation_layer_class", - [ - (Dense, {"units": 3, "activation": "relu"}, (1,), None), - (Dense, {"units": 3}, (1,), PReLU), - (Conv2D, {"filters": 2, "kernel_size": (3, 3), "activation": "relu"}, (64, 64, 3), None), - ], -) -def test_split_activation_do_split( - layer_class, layer_kwargs, input_shape_wo_batchsize, embedded_activation_layer_class, use_bias -): - # init layer - if embedded_activation_layer_class is not None: - layer = layer_class(use_bias=use_bias, activation=embedded_activation_layer_class(), **layer_kwargs) - else: - layer = layer_class(use_bias=use_bias, **layer_kwargs) - # init input_shape and weights - input_shape = input_shape_wo_batchsize - input_tensor = Input(input_shape) - layer(input_tensor) - # split - layers = split_activation(layer) - # check layer split - assert len(layers) == 2 - layer_wo_activation, activation_layer = layers - assert isinstance(layer_wo_activation, layer.__class__) - assert layer_wo_activation.get_config()["activation"] == "linear" - if isinstance(layer.activation, Layer): - assert activation_layer == layer.activation - else: - assert isinstance(activation_layer, Activation) - assert activation_layer.get_config()["activation"] == layer_kwargs["activation"] - # check names starts with original name + "_" - assert layer_wo_activation.name.startswith(f"{layer.name}_") - assert activation_layer.name.startswith(f"{layer.name}_") - # check already built - assert layer_wo_activation.built - assert activation_layer.built - # check same outputs - input_shape_with_batch_size = (5,) + input_shape_wo_batchsize - flatten_dim = np.prod(input_shape_with_batch_size) - inputs_np = np.linspace(-1, 1, flatten_dim).reshape(input_shape_with_batch_size) - output_np_ref = K.convert_to_numpy(layer(inputs_np)) - output_np_new = K.convert_to_numpy(activation_layer(layer_wo_activation(inputs_np))) - assert_almost_equal(output_np_new, output_np_ref) - # check same weights (really same objects) - for i in range(len(layer_wo_activation.weights)): - assert layer.weights[i] is layer_wo_activation.weights[i] - - -@pytest.mark.parametrize( - "layer_class, layer_kwargs", - [ - (Dense, {"units": 3}), - (Activation, {"activation": "relu"}), - (Permute, {"dims": (1, 2, 3)}), - ], -) -def test_split_activation_do_nothing(layer_class, layer_kwargs): - layer = layer_class(**layer_kwargs) - layers = split_activation(layer) - assert len(layers) == 1 - assert layers[0] == layer - - -def test_split_activation_uninitialized_layer_ko(): - layer = Dense(3, activation="relu") - with pytest.raises(ValueError): - layers = split_activation(layer) - - -@pytest.mark.parametrize( - "layer_class_name, layer_kwargs, input_shape_wo_batchsize", - [ - ("Dense", {"units": 3}, (1,)), - ("Activation", {"activation": "relu"}, (1,)), - ("Permute", {"dims": (1, 2, 3)}, (1, 1, 1)), - ], -) -def test_preprocess_layer_no_nonlinear_activation(layer_class_name, layer_kwargs, input_shape_wo_batchsize): - layer_class = globals()[layer_class_name] - layer = layer_class(**layer_kwargs) - # build layer - input_tensor = Input(input_shape_wo_batchsize) - layer(input_tensor) - # preprocess - layers = preprocess_layer(layer) - # check resulting layers - assert len(layers) == 1 - keras_layer = layers[0] - # check values - assert keras_layer is layer - # try to call the resulting layers 3 times - input_tensor = K.ones((5,) + input_shape_wo_batchsize) - for _ in range(3): - keras_layer(input_tensor) - - -@pytest.mark.parametrize( - "layer_class_name, " - "layer_kwargs, " - "input_shape_wo_batchsize, " - "embedded_activation_layer_class_name, " - "embedded_activation_layer_class_kwargs, ", - [ - ("Dense", {"units": 3, "activation": "relu"}, (1,), None, None), - ("Dense", {"units": 3}, (1,), "PReLU", {}), - ("Conv2D", {"filters": 2, "kernel_size": (3, 3), "activation": "relu"}, (64, 64, 3), None, None), - ], -) -def test_preprocess_layer_nonlinear_activation( - layer_class_name, - layer_kwargs, - input_shape_wo_batchsize, - embedded_activation_layer_class_name, - embedded_activation_layer_class_kwargs, - use_bias, -): - # init layer - layer_class = globals()[layer_class_name] - if embedded_activation_layer_class_name is not None: - embedded_activation_layer_class = globals()[embedded_activation_layer_class_name] - embedded_activation_layer = embedded_activation_layer_class(**embedded_activation_layer_class_kwargs) - layer = layer_class(use_bias=use_bias, activation=embedded_activation_layer, **layer_kwargs) - else: - layer = layer_class(use_bias=use_bias, **layer_kwargs) - # init input_shape and weights - input_shape = input_shape_wo_batchsize - input_tensor = Input(input_shape) - layer(input_tensor) - # split - layers = preprocess_layer(layer) - # check layer split - assert len(layers) == 2 - layer_wo_activation, activation_layer = layers - assert isinstance(layer_wo_activation, layer.__class__) - assert layer_wo_activation.get_config()["activation"] == "linear" - if isinstance(layer.activation, Layer): - assert activation_layer == layer.activation - else: - assert isinstance(activation_layer, Activation) - assert activation_layer.get_config()["activation"] == layer_kwargs["activation"] - # check names starts with with original name + "_" - assert layer_wo_activation.name.startswith(f"{layer.name}_") - assert activation_layer.name.startswith(f"{layer.name}_") - # check already built - assert layer_wo_activation.built - assert activation_layer.built - # check same outputs - input_shape_with_batch_size = (5,) + input_shape_wo_batchsize - flatten_dim = np.prod(input_shape_with_batch_size) - inputs_np = np.linspace(-1, 1, flatten_dim).reshape(input_shape_with_batch_size) - output_np_ref = K.convert_to_numpy(layer(inputs_np)) - output_np_new = K.convert_to_numpy(activation_layer(layer_wo_activation(inputs_np))) - assert_almost_equal(output_np_new, output_np_ref) - # check same weights (really same objects) - for i in range(len(layer_wo_activation.weights)): - assert layer.weights[i] is layer_wo_activation.weights[i] diff --git a/tests/test_pooling.py b/tests/test_pooling.py deleted file mode 100644 index aaa45296..00000000 --- a/tests/test_pooling.py +++ /dev/null @@ -1,49 +0,0 @@ -import keras.config as keras_config -import pytest -from keras.layers import MaxPooling2D - -from decomon.layers.maxpooling import DecomonMaxPooling2D - - -def test_MaxPooling2D_box(mode, floatx, decimal, helpers): - # skip unavailable combinations - if floatx == 16 and keras_config.backend() == "torch" and not helpers.in_GPU_mode(): - pytest.skip("Pytorch does not implement maxpooling for float16 in CPU mode.") - - odd, m_0, m_1 = 0, 0, 1 - data_format = "channels_last" - dc_decomp = True - fast = False - kwargs_layer = dict(pool_size=(2, 2), strides=(2, 2), padding="valid", dtype=keras_config.floatx()) - - # tensor inputs - inputs = helpers.get_tensor_decomposition_images_box(data_format, odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_images_box(data_format, odd, m0=m_0, m1=m_1, dc_decomp=dc_decomp) - - # keras & decomon layer - keras_layer = MaxPooling2D(**kwargs_layer) - decomon_layer = DecomonMaxPooling2D(dc_decomp=dc_decomp, fast=fast, mode=mode, **kwargs_layer) - - # original output - output_ref = keras_layer(input_ref) - f_ref = helpers.function(inputs, output_ref) - output_ref_ = f_ref(inputs_) - - # decomon outputs - outputs = decomon_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - decimal=decimal, - mode=mode, - ) diff --git a/tests/test_preprocess_keras_model.py b/tests/test_preprocess_keras_model.py deleted file mode 100644 index 7c40b5fe..00000000 --- a/tests/test_preprocess_keras_model.py +++ /dev/null @@ -1,95 +0,0 @@ -import keras.ops as K -import numpy as np -import pytest -from keras.layers import Activation, Conv2D, Dense, Flatten, Input, PReLU -from keras.models import Model, Sequential -from numpy.testing import assert_almost_equal - -from decomon.models.convert import ( - preprocess_keras_model, - split_activations_in_keras_model, -) - - -def test_split_activations_in_keras_model_no_inputshape_ko(): - layers = [ - Conv2D( - 10, - kernel_size=(3, 3), - activation="relu", - data_format="channels_last", - ), - Flatten(), - Dense(1), - ] - model = Sequential(layers) - with pytest.raises(ValueError): - converted_model = split_activations_in_keras_model(model) - - -def test_split_activations_in_keras_model(toy_model): - converted_model = split_activations_in_keras_model(toy_model) - assert isinstance(converted_model, Model) - # check no more activation functions in non-activation layers - for layer in converted_model.layers: - activation = layer.get_config().get("activation", None) - assert isinstance(layer, Activation) or activation is None or activation == "linear" - # check same outputs - input_shape_wo_batchsize = toy_model.input_shape[1:] - input_shape_with_batch_size = (5,) + input_shape_wo_batchsize - flatten_dim = np.prod(input_shape_with_batch_size) - inputs_np = np.linspace(-1, 1, flatten_dim).reshape(input_shape_with_batch_size) - output_np_ref = K.convert_to_numpy(toy_model(inputs_np)) - output_np_new = K.convert_to_numpy(converted_model(inputs_np)) - assert_almost_equal(output_np_new, output_np_ref, decimal=4) - - -@pytest.mark.parametrize( - "layer_class_name, " - "layer_kwargs, " - "input_shape_wo_batchsize, " - "embedded_activation_layer_class_name, " - "embedded_activation_layer_class_kwargs, ", - [ - ("Dense", {"units": 3, "activation": "relu"}, (1,), None, None), - ("Dense", {"units": 3}, (1,), "PReLU", {}), - ("Conv2D", {"filters": 2, "kernel_size": (3, 3), "activation": "relu"}, (64, 64, 3), None, None), - ], -) -def test_preprocess( - layer_class_name, - layer_kwargs, - input_shape_wo_batchsize, - embedded_activation_layer_class_name, - embedded_activation_layer_class_kwargs, - use_bias, -): - # init hidden layer - layer_class = globals()[layer_class_name] - if embedded_activation_layer_class_name is not None: - embedded_activation_layer_class = globals()[embedded_activation_layer_class_name] - embedded_activation_layer = embedded_activation_layer_class(**embedded_activation_layer_class_kwargs) - hidden_layer = layer_class(use_bias=use_bias, activation=embedded_activation_layer, **layer_kwargs) - else: - hidden_layer = layer_class(use_bias=use_bias, **layer_kwargs) - layers = [ - Input(shape=input_shape_wo_batchsize), - hidden_layer, - Dense(1), - ] - model = Sequential(layers) - converted_model = preprocess_keras_model(model) - # check no more embedded activation - for layer in converted_model.layers: - activation = layer.get_config().get("activation", None) - assert isinstance(layer, Activation) or activation is None or activation == "linear" - # check number of layers - assert len(converted_model.layers) == 4 - # check same outputs - input_shape_wo_batchsize = model.input_shape[1:] - input_shape_with_batch_size = (5,) + input_shape_wo_batchsize - flatten_dim = np.prod(input_shape_with_batch_size) - inputs_np = np.linspace(-1, 1, flatten_dim).reshape(input_shape_with_batch_size) - output_np_ref = K.convert_to_numpy(model(inputs_np)) - output_np_new = K.convert_to_numpy(converted_model(inputs_np)) - assert_almost_equal(output_np_new, output_np_ref, decimal=4) diff --git a/tests/test_reshape.py b/tests/test_reshape.py deleted file mode 100644 index 105337db..00000000 --- a/tests/test_reshape.py +++ /dev/null @@ -1,137 +0,0 @@ -import keras.config as keras_config -import numpy as np -import pytest -from keras.layers import Permute, Reshape - -from decomon.core import ForwardMode, get_affine, get_ibp -from decomon.layers.convert import to_decomon -from decomon.layers.decomon_reshape import DecomonPermute, DecomonReshape - - -def keras_target_shape_reshape(input_ref): - return (int(np.prod(input_ref.shape[1:])),) - - -def target_shape_keras2np_reshape(target_shape): - return (-1, target_shape[0]) - - -def keras_target_shape_permute(input_ref): - n_dim = len(input_ref.shape) - 1 - return np.random.permutation(n_dim) + 1 - - -def target_shape_keras2np_permute(target_shape): - return tuple([0] + list(target_shape)) - - -@pytest.mark.parametrize( - "decomon_layer_class, keras_target_shape_func, target_shape_keras2np_func, np_func", - [ - (DecomonReshape, keras_target_shape_reshape, target_shape_keras2np_reshape, np.reshape), - (DecomonPermute, keras_target_shape_permute, target_shape_keras2np_permute, np.transpose), - ], -) -def test_Decomon_reshape_n_permute_box( - decomon_layer_class, - keras_target_shape_func, - target_shape_keras2np_func, - np_func, - mode, - dc_decomp, - floatx, - decimal, - helpers, -): - odd, m_0, m_1 = 0, 0, 1 - data_format = "channels_last" - - # tensor inputs - inputs = helpers.get_tensor_decomposition_images_box(data_format, odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_images_box(data_format, odd, m0=m_0, m1=m_1, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs=inputs_) - - # target shape - target_shape = keras_target_shape_func(input_ref) - target_shape_ = target_shape_keras2np_func(target_shape) - - # original output - output_ref_ = np_func(input_ref_, target_shape_) - - # decomon layer - decomon_layer = decomon_layer_class(target_shape, dc_decomp=dc_decomp, mode=mode, dtype=keras_config.floatx()) - - # decomon output - output = decomon_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, output) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - mode=mode, - dc_decomp=dc_decomp, - decimal=decimal, - ) - - -@pytest.mark.parametrize( - "keras_target_shape_func, keras_layer_class", - [ - (keras_target_shape_reshape, Reshape), - (keras_target_shape_permute, Permute), - ], -) -def test_Decomon_reshape_n_permute_to_decomon_box( - keras_target_shape_func, keras_layer_class, shared, floatx, decimal, helpers -): - odd, m_0, m_1 = 0, 0, 1 - dc_decomp = True - data_format = "channels_last" - mode = ForwardMode.HYBRID - ibp = get_ibp(mode=mode) - affine = get_affine(mode=mode) - - # tensor inputs - inputs = helpers.get_tensor_decomposition_images_box(data_format, odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_images_box(data_format, odd, m0=m_0, m1=m_1, dc_decomp=dc_decomp) - - #  target shape - target_shape = keras_target_shape_func(input_ref) - - # keras layer - keras_layer = keras_layer_class(target_shape, dtype=keras_config.floatx()) - - # original output - output_ref = keras_layer(input_ref) - f_ref = helpers.function(inputs, output_ref) - output_ref_ = f_ref(inputs_) - - # conversion with to_decomon - input_dim = helpers.get_input_dim_from_full_inputs(inputs_) - decomon_layer = to_decomon(keras_layer, input_dim, dc_decomp=dc_decomp, shared=shared, ibp=ibp, affine=affine) - - # decomon outputs - outputs = decomon_layer(inputs_for_mode) - f_decomon = helpers.function(inputs, outputs) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - dc_decomp=dc_decomp, - mode=mode, - decimal=decimal, - ) diff --git a/tests/test_to_decomon.py b/tests/test_to_decomon.py deleted file mode 100644 index c6218d07..00000000 --- a/tests/test_to_decomon.py +++ /dev/null @@ -1,78 +0,0 @@ -import pytest -from keras.layers import Add, Conv2D, Dense, Input, Layer, Reshape - -from decomon.layers.convert import to_decomon -from decomon.layers.utils import is_a_merge_layer - - -class MyLayer(Layer): - """Mock layer unknown from decomon.""" - - ... - - -class MyMerge(Layer): - """Mock merge layer unknown from decomon.""" - - def _merge_function(self, inputs): - return inputs - - -def test_is_merge_layer(): - layer = MyMerge() - assert is_a_merge_layer(layer) - layer = MyLayer() - assert not is_a_merge_layer(layer) - - -def test_to_decomon_merge_not_built_ko(): - layer = MyMerge() - with pytest.raises(ValueError): - to_decomon(layer, input_dim=1) - - -def test_to_decomon_not_built_ko(): - layer = MyLayer() - with pytest.raises(ValueError): - to_decomon(layer, input_dim=1) - - -def test_to_decomon_merge_not_implemented_ko(): - layer = MyMerge() - layer.built = True - with pytest.raises(NotImplementedError): - to_decomon(layer, input_dim=1) - - -def test_to_decomon_not_implemented_ko(): - layer = MyLayer() - layer.built = True - with pytest.raises(NotImplementedError): - to_decomon(layer, input_dim=1) - - -@pytest.mark.parametrize( - "layer_class, layer_kwargs, input_shape_wo_batchsize, nb_inputs", - [ - (Dense, {"units": 3, "use_bias": True}, (1,), 1), - (Conv2D, {"filters": 2, "kernel_size": (3, 3), "use_bias": True}, (64, 64, 3), 1), - (Dense, {"units": 3, "use_bias": False}, (1,), 1), - (Conv2D, {"filters": 2, "kernel_size": (3, 3), "use_bias": False}, (64, 64, 3), 1), - (Reshape, {"target_shape": (72,)}, (6, 6, 2), 1), - (Add, {}, (1,), 2), - ], -) -def test_to_decomon_ok(layer_class, layer_kwargs, input_shape_wo_batchsize, nb_inputs): - layer = layer_class(**layer_kwargs) - # init input_shape and weights - # input_tensors + build layer - if nb_inputs == 1: - input_tensor = Input(input_shape_wo_batchsize) - layer(input_tensor) - else: - input_tensors = [Input(input_shape_wo_batchsize) for _ in range(nb_inputs)] - layer(input_tensors) - decomon_layer = to_decomon(layer, input_dim=1) - # check trainable weights - for i in range(len(layer.weights)): - assert layer.weights[i] is decomon_layer.weights[i] diff --git a/tests/test_utils.py b/tests/test_utils.py deleted file mode 100644 index 9856401a..00000000 --- a/tests/test_utils.py +++ /dev/null @@ -1,503 +0,0 @@ -# Test unit for decomon with Dense layers - - -import keras.ops as K -import numpy as np -import pytest -from numpy.testing import assert_allclose, assert_almost_equal - -from decomon.core import BoxDomain, ForwardMode -from decomon.layers.utils import add, max_, maximum, minus, relu_ -from decomon.utils import subtract - - -def test_get_upper_multi_box(odd, floatx, decimal, helpers): - inputs = helpers.get_tensor_decomposition_multid_box(odd) - inputs_ = helpers.get_standard_values_multid_box(odd) - - x, y, x_0, u_c, W_u, b_u, _, _, _, _, _ = inputs - x_, y_, x_0_, u_c_, W_u_, b_u_, _, _, _, _, _ = inputs_ - - # compute maximum - x_min_ = x_0_[:, 0][:, :, None] - x_max_ = x_0_[:, 1][:, :, None] - upper_pred = np.sum(np.minimum(W_u_, 0) * x_min_ + np.maximum(W_u_, 0) * x_max_, 1) + b_u_ - - upper = BoxDomain().get_upper(x_0, W_u, b_u) - - f_upper = helpers.function([x_0, W_u, b_u], upper) - upper_ = f_upper([x_0_, W_u_, b_u_]) - - assert_allclose(upper_pred, upper_) - - -def test_get_upper_box_numpy(n, floatx, decimal, helpers): - inputs = helpers.get_tensor_decomposition_1d_box() - inputs_ = helpers.get_standard_values_1d_box(n) - - x, y, x_0, u_c, W_u, b_u, _, _, _, _, _ = inputs - x_, y_, x_0_, u_c_, W_u_, b_u_, _, _, _, _, _ = inputs_ - - x_expand = x_ + np.zeros_like(x_) - n_expand = len(W_u_.shape) - len(x_expand.shape) - for i in range(n_expand): - x_expand = np.expand_dims(x_expand, -1) - - upper_pred = np.sum(W_u_ * x_expand, 1) + b_u_ - upper_pred = upper_pred.max(0) - - upper = BoxDomain().get_upper(x_0, W_u, b_u) - - f_upper = helpers.function([x_0, W_u, b_u], upper) - upper_ = f_upper([x_0_, W_u_, b_u_]).max() - - assert_allclose(upper_pred, upper_) - - -def test_get_upper_box(n, floatx, decimal, helpers): - inputs = helpers.get_tensor_decomposition_1d_box() - inputs_ = helpers.get_standard_values_1d_box(n) - - x, y, x_0, u_c, W_u, b_u, _, _, _, _, _ = inputs - _, _, x_0_, u_c_, W_u_, b_u_, _, _, _, _, _ = inputs_ - - upper = BoxDomain().get_upper(x_0, W_u, b_u) - - f_upper = helpers.function([x_0, W_u, b_u], upper) - f_u = helpers.function(inputs, u_c) - - output = inputs_[1] - - upper_ = f_upper([x_0_, W_u_, b_u_]) - u_c_ = f_u(inputs_) - - assert_almost_equal( - np.clip(output - upper_, 0, np.inf), - np.zeros_like(output), - decimal=decimal, - err_msg="upper_= 0: - # check that we find this case ! - if mode in [ForwardMode.AFFINE, ForwardMode.HYBRID]: - assert_almost_equal( - w_u_output, - W_u_input, - decimal=decimal, - err_msg="w_u_!=W_u but lower_>=0 in call {}".format(n), - ) - assert_almost_equal( - b_u_output, - b_u_input, - decimal=decimal, - err_msg="b_u_!=b_u but lower_>=0 in call {}".format(n), - ) - assert_almost_equal( - w_l_output, - W_l_input, - decimal=decimal, - err_msg="w_l_!=W_l but lower_>=0 upper_<=0 in call {}".format(n), - ) - assert_almost_equal( - b_l_output, - b_l_input, - decimal=decimal, - err_msg="b_l_!=b_l but lower_>=0 upper_<=0 in call {}".format(n), - ) - if mode in [ForwardMode.IBP, ForwardMode.HYBRID]: - assert_almost_equal( - u_c_output, - u_c_input, - decimal=decimal, - err_msg="u_c_!=u_c but lower_>=0 in call {}".format(n), - ) - assert_almost_equal( - l_c_output, - l_c_input, - decimal=decimal, - err_msg="l_c_!=l_c but lower_>=0 in call {}".format(n), - ) - - if mode != ForwardMode.IBP: - assert_almost_equal( - z_input[:, 0], z_output[:, 0], decimal=decimal, err_msg="the lower bound should be unchanged" - ) - assert_almost_equal( - z_input[:, 1], z_output[:, 1], decimal=decimal, err_msg="the upper bound should be unchanged" - ) - - -def add_op(x, y): - return x + y - - -def subtract_op(x, y): - return x - y - - -def minus_op(x): - return -x - - -@pytest.mark.parametrize( - "decomon_func, tensor_func", - [ - (add, add_op), - (subtract, subtract_op), - (maximum, K.maximum), - ], -) -def test_func_with_2_inputs(decomon_func, tensor_func, odd, mode, floatx, decimal, helpers): - dc_decomp = True - - #  tensor inputs - inputs_0 = helpers.get_tensor_decomposition_multid_box(odd) - inputs_1 = helpers.get_tensor_decomposition_multid_box(odd) - inputs_for_mode_0 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_0, mode=mode, dc_decomp=dc_decomp) - inputs_for_mode_1 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_1, mode=mode, dc_decomp=dc_decomp) - input_ref_0 = helpers.get_input_ref_from_full_inputs(inputs=inputs_0) - input_ref_1 = helpers.get_input_ref_from_full_inputs(inputs=inputs_1) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd) - - # original output - f_ref = helpers.function(inputs_0 + inputs_1, tensor_func(input_ref_0, input_ref_1)) - output_ref_ = f_ref(inputs_ + inputs_) - - # decomon output - output = decomon_func(inputs_for_mode_0, inputs_for_mode_1, dc_decomp=dc_decomp, mode=mode) - f_decomon = helpers.function(inputs_0 + inputs_1, output) - outputs_ = f_decomon(inputs_ + inputs_) - - #  check bounds consistency - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - mode=mode, - dc_decomp=dc_decomp, - decimal=decimal, - ) - - -@pytest.mark.parametrize( - "decomon_func, tensor_func, tensor_func_kwargs", - [ - (minus, minus_op, None), - (max_, K.max, {"axis": -1}), - ], -) -def test_func_with_1_input(decomon_func, tensor_func, tensor_func_kwargs, odd, mode, floatx, decimal, helpers): - dc_decomp = True - if tensor_func_kwargs is None: - tensor_func_kwargs = {} - - # tensor inputs - inputs = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - - # original output - f_ref = helpers.function(inputs, tensor_func(input_ref, **tensor_func_kwargs)) - output_ref_ = f_ref(inputs_) - - # decomon output - output = decomon_func(inputs_for_mode, dc_decomp=dc_decomp, mode=mode) - f_decomon = helpers.function(inputs, output) - outputs_ = f_decomon(inputs_) - - # check bounds consistency - helpers.assert_decomon_layer_output_properties_box( - full_inputs=inputs_, - output_ref=output_ref_, - outputs_for_mode=outputs_, - mode=mode, - dc_decomp=dc_decomp, - decimal=decimal, - ) - - -# DC_DECOMP = FALSE -def test_max_nodc(odd, helpers): - dc_decomp = False - mode = ForwardMode.HYBRID - - inputs = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - - output = max_(inputs_for_mode, dc_decomp=dc_decomp, mode=mode) - f_max = helpers.function(inputs, output) - assert len(f_max(inputs_)) == 7 - - -def test_maximum_nodc(odd, helpers): - dc_decomp = False - mode = ForwardMode.HYBRID - - inputs_0 = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_1 = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_for_mode_0 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_0, mode=mode, dc_decomp=dc_decomp) - inputs_for_mode_1 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_1, mode=mode, dc_decomp=dc_decomp) - input_ref_0 = helpers.get_input_ref_from_full_inputs(inputs=inputs_0) - input_ref_1 = helpers.get_input_ref_from_full_inputs(inputs=inputs_1) - - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - output = maximum(inputs_for_mode_0, inputs_for_mode_1, dc_decomp=dc_decomp, mode=mode) - - f_ref = helpers.function(inputs_0 + inputs_1, K.maximum(input_ref_0, input_ref_1)) - f_maximum = helpers.function(inputs_0 + inputs_1, output) - - assert len(f_maximum(inputs_ + inputs_)) == 7 - f_ref(inputs_ + inputs_) - - -def test_minus_nodc(odd, helpers): - dc_decomp = False - mode = ForwardMode.HYBRID - - inputs = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - output = minus(inputs_for_mode, dc_decomp=dc_decomp, mode=mode) - - f_ref = helpers.function(inputs, -inputs[1]) - f_minus = helpers.function(inputs, output) - - assert len(f_minus(inputs_)) == 7 - f_ref(inputs_) - - -def test_add_nodc(odd, helpers): - dc_decomp = False - mode = ForwardMode.HYBRID - - inputs_0 = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_1 = helpers.get_tensor_decomposition_multid_box(odd, dc_decomp=dc_decomp) - inputs_for_mode_0 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_0, mode=mode, dc_decomp=dc_decomp) - inputs_for_mode_1 = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_1, mode=mode, dc_decomp=dc_decomp) - input_ref_0 = helpers.get_input_ref_from_full_inputs(inputs=inputs_0) - input_ref_1 = helpers.get_input_ref_from_full_inputs(inputs=inputs_1) - - inputs_ = helpers.get_standard_values_multid_box(odd, dc_decomp=dc_decomp) - - output = add(inputs_for_mode_0, inputs_for_mode_1, dc_decomp=dc_decomp, mode=mode) - f_ref = helpers.function(inputs_0 + inputs_1, input_ref_0 + input_ref_1) - f_add = helpers.function(inputs_0 + inputs_1, output) - assert len(f_add(inputs_ + inputs_)) == 7 - f_ref(inputs_ + inputs_) - - -def test_relu_1D_box_nodc(n, helpers): - dc_decomp = False - mode = ForwardMode.HYBRID - - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs - - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=dc_decomp) - - output = relu_(inputs_for_mode, dc_decomp=dc_decomp, mode=mode) - lower = BoxDomain().get_lower(z, W_l, b_l) - upper = BoxDomain().get_upper(z, W_u, b_u) - - f_lower = helpers.function(inputs, lower) - f_upper = helpers.function(inputs, upper) - - np.min(f_lower(inputs_)) - np.max(f_upper(inputs_)) - - f_relu_ = helpers.function(inputs, output) - assert len(f_relu_(inputs_)) == 7 diff --git a/tests/test_utils_conv.py b/tests/test_utils_conv.py deleted file mode 100644 index bbd1eb57..00000000 --- a/tests/test_utils_conv.py +++ /dev/null @@ -1,141 +0,0 @@ -# Test unit for decomon with Dense layers -import keras.config -import keras.ops as K -import numpy as np -import pytest -from keras.layers import Conv2D -from numpy.testing import assert_almost_equal - -from decomon.backward_layers.utils_conv import get_toeplitz -from decomon.core import get_affine, get_ibp -from decomon.layers.convert import to_decomon - - -def test_toeplitz_from_Keras(channels, filter_size, strides, flatten, data_format, padding, floatx, decimal, helpers): - # skip unavailable combinations - if floatx == 16 and keras.config.backend() == "torch": - pytest.skip("Pytorch does not implement conv2d for float16") - - # filter_size, strides, flatten, - if floatx == 16: - decimal = 0 - - if data_format == "channels_first" and keras.config.backend() == "tensorflow" and not helpers.in_GPU_mode(): - pytest.skip("data format 'channels first' is possible only in GPU mode for tensorflow") - - if data_format == "channels_first": - pytest.xfail("get_toeplitz with channels_first is bugged for now.") - - dc_decomp = False - odd, m_0, m_1 = 0, 0, 1 - - inputs = helpers.get_tensor_decomposition_images_box(data_format, odd, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_images_box(data_format, odd, m0=m_0, m1=m_1, dc_decomp=dc_decomp) - - # should be working either with convolution of conv2D - kwargs_layer = dict( - data_format=data_format, - filters=channels, - kernel_size=(filter_size, filter_size), - strides=strides, - padding=padding, - use_bias=False, - dtype=input_ref.dtype, - ) - - layer = Conv2D(**kwargs_layer) - - result_ref = layer(input_ref) - W = get_toeplitz(layer, flatten) - - if not flatten: - w_in, h_in, c_in, w_out, h_out, c_out = W.shape - W = K.reshape(W, (w_in * h_in * c_in, w_out * h_out * c_out)) - - n_in, n_out = W.shape - y_flat = K.reshape(input_ref, (-1, n_in, 1)) - result_flat = K.reshape(result_ref, (-1, n_out)) - result_toeplitz = K.sum(W[None] * y_flat, 1) - output_test = K.sum((result_toeplitz - result_flat) ** 2) - f_test = helpers.function(inputs, output_test) - output_test_ = f_test(inputs_) - - assert_almost_equal( - output_test_, - np.zeros_like(output_test_), - decimal=decimal, - err_msg="wrong toeplitz matrix", - ) - - -def test_toeplitz_from_Decomon( - floatx, decimal, mode, channels, filter_size, strides, flatten, data_format, padding, helpers -): - # skip unavailable combinations - if floatx == 16 and keras.config.backend() == "torch" and not helpers.in_GPU_mode(): - pytest.skip("Pytorch does not implement conv2d for float16 in CPU mode.") - - if data_format == "channels_first" and keras.config.backend() == "tensorflow" and not helpers.in_GPU_mode(): - pytest.skip("data format 'channels first' is possible only in GPU mode for tensorflow") - - if data_format == "channels_first": - pytest.xfail("get_toeplitz with channels_first is bugged for now.") - - odd, m_0, m_1 = 0, 0, 1 - if floatx == 16: - decimal = 0 - - dc_decomp = False - ibp = get_ibp(mode=mode) - affine = get_affine(mode=mode) - - # tensor inputs - inputs = helpers.get_tensor_decomposition_images_box(data_format, odd, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs=inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_images_box(data_format, odd, m0=m_0, m1=m_1, dc_decomp=dc_decomp) - kwargs_layer = dict( - filters=channels, - kernel_size=(filter_size, filter_size), - strides=strides, - padding=padding, - use_bias=False, - dtype=input_ref.dtype, - data_format=data_format, - ) - - # should be working either with convolution of conv2D - layer = Conv2D(**kwargs_layer) - result_ref = layer(input_ref) - - # toeplitz matrix should be compatible with a DecomonLayer - input_dim = input_ref.shape[-1] - - decomon_layer = to_decomon(layer, input_dim, ibp=ibp, affine=affine) - - W = get_toeplitz(decomon_layer, flatten) - - if not flatten: - w_in, h_in, c_in, w_out, h_out, c_out = W.shape - W = K.reshape(W, (w_in * h_in * c_in, w_out * h_out * c_out)) - - n_in, n_out = W.shape - y_flat = K.reshape(input_ref, (-1, n_in, 1)) - result_flat = K.reshape(result_ref, (-1, n_out)) - result_toeplitz = K.sum(W[None] * y_flat, 1) - - output_test = K.sum((result_toeplitz - result_flat) ** 2) - - f_test = helpers.function(inputs, output_test) - output_test_ = f_test(inputs_) - - assert_almost_equal( - output_test_, - np.zeros_like(output_test_), - decimal=decimal, - err_msg="wrong toeplitz matrix", - ) diff --git a/tests/test_utils_pooling.py b/tests/test_utils_pooling.py deleted file mode 100644 index 3f776367..00000000 --- a/tests/test_utils_pooling.py +++ /dev/null @@ -1,54 +0,0 @@ -import keras.ops as K -import numpy as np -import pytest -from numpy.testing import assert_almost_equal - -from decomon.layers.utils_pooling import ( - get_lower_linear_hull_max, - get_upper_linear_hull_max, -) - - -@pytest.mark.parametrize( - "func, minmax, clipmin, clipmax", - [ - (get_lower_linear_hull_max, np.max, -np.inf, 0.0), - (get_upper_linear_hull_max, np.min, 0.0, np.inf), - ], -) -def test_get_lower_upper_linear_hull_max( - func, minmax, clipmin, clipmax, mode, floatx, decimal, axis, finetune_odd, helpers -): - if finetune_odd is not None and func is not get_lower_linear_hull_max: - # skip test with finetune if not get_lower - pytest.skip("finetune_odd is only intended for get_lower_linear_hull_max()") - if func is get_lower_linear_hull_max: - pytest.xfail("get_lower_linear_hull_max() is bugged") - - odd, m_0, m_1 = 0, 0, 1 - data_format = "channels_last" - dc_decomp = True - - # inputs - inputs = helpers.get_tensor_decomposition_images_box(data_format, odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - inputs_ = helpers.get_standard_values_images_box(data_format, odd, m0=m_0, m1=m_1, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs_) - - if finetune_odd is not None: - finetune_odd = K.convert_to_tensor(finetune_odd, dtype="float{}".format(floatx)) - - # decomon output - output = func(inputs_for_mode, mode=mode, axis=axis, finetune_lower=finetune_odd) - f_pooling = helpers.function(inputs, output) - w_, b_ = f_pooling(inputs_) - - # reference output - output_ref_ = np.max(input_ref_, axis=axis) - - assert_almost_equal( - minmax(np.clip(np.sum(w_ * input_ref_, axis) + b_ - output_ref_, clipmin, clipmax)), - 0.0, - decimal=decimal, - err_msg=f"linear hull for bounding max", - ) diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py deleted file mode 100644 index 8103d889..00000000 --- a/tests/test_wrapper.py +++ /dev/null @@ -1,152 +0,0 @@ -import numpy as np -import pytest -from keras.layers import Activation, Dense, Input -from keras.models import Sequential -from numpy.testing import assert_almost_equal - -from decomon import get_adv_box, get_lower_box, get_range_box, get_upper_box -from decomon.core import get_affine, get_ibp -from decomon.models import clone - - -@pytest.fixture() -def toy_model_1d(): - sequential = Sequential() - sequential.add(Input((1,))) - sequential.add(Dense(1, activation="linear")) - sequential.add(Activation("relu")) - sequential.add(Dense(1, activation="linear")) - return sequential - - -@pytest.fixture() -def toy_model_multid(odd, helpers): - input_dim = helpers.get_input_dim_multid_box(odd) - sequential = Sequential() - sequential.add(Input((input_dim,))) - sequential.add(Dense(1, activation="linear")) - sequential.add(Activation("relu")) - sequential.add(Dense(1, activation="linear")) - return sequential - - -def test_get_adv_box_1d(toy_model_1d, helpers): - inputs_ = helpers.get_standard_values_1d_box(n=0, dc_decomp=False) - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ - - score = get_adv_box(toy_model_1d, z[:, 0], z[:, 1], source_labels=0) - - -def test_get_upper_1d_box(toy_model_1d, n, method, mode, helpers): - if not helpers.is_method_mode_compatible(method=method, mode=mode): - # skip method=ibp/crown-ibp with mode=affine/hybrid - pytest.skip(f"output mode {mode} is not compatible with convert method {method}") - - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=False) - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ - - ibp = get_ibp(mode) - affine = get_affine(mode) - backward_model = clone(toy_model_1d, method=method, ibp=ibp, affine=affine, mode=mode) - upper = get_upper_box(backward_model, z[:, 0], z[:, 1]) - y_ref = helpers.predict_on_small_numpy(toy_model_1d, y) - - try: - assert (upper - y_ref).min() + 1e-6 >= 0.0 - except AssertionError: - toy_model_1d.save_weights("get_upper_1d_box_fail_{}_{}_{}.hd5".format(n, method, mode)) - raise AssertionError - - -def test_get_lower_1d_box(toy_model_1d, n, method, mode, helpers): - if not helpers.is_method_mode_compatible(method=method, mode=mode): - # skip method=ibp/crown-ibp with mode=affine/hybrid - pytest.skip(f"output mode {mode} is not compatible with convert method {method}") - - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=False) - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ - - ibp = get_ibp(mode) - affine = get_affine(mode) - backward_model = clone(toy_model_1d, method=method, final_ibp=ibp, final_affine=affine) - lower = get_lower_box(backward_model, z[:, 0], z[:, 1]) - y_ref = helpers.predict_on_small_numpy(toy_model_1d, y) - - try: - assert (y_ref - lower).min() + 1e-6 >= 0.0 - except AssertionError: - toy_model_1d.save_weights("get_lower_1d_box_fail_{}_{}_{}.hd5".format(n, method, mode)) - raise AssertionError - - -def test_get_range_1d_box(toy_model_1d, n, method, mode, helpers): - if not helpers.is_method_mode_compatible(method=method, mode=mode): - # skip method=ibp/crown-ibp with mode=affine/hybrid - pytest.skip(f"output mode {mode} is not compatible with convert method {method}") - - inputs_ = helpers.get_standard_values_1d_box(n, dc_decomp=False) - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ - - ibp = get_ibp(mode) - affine = get_affine(mode) - backward_model = clone(toy_model_1d, method=method, final_ibp=ibp, final_affine=affine) - upper, lower = get_range_box(backward_model, z[:, 0], z[:, 1]) - y_ref = helpers.predict_on_small_numpy(toy_model_1d, y) - - try: - assert (upper - y_ref).min() + 1e-6 >= 0.0 - assert (y_ref - lower).min() + 1e-6 >= 0.0 - except AssertionError: - toy_model_1d.save_weights("get_range_1d_box_fail_{}_{}_{}.hd5".format(n, method, mode)) - raise AssertionError - - -def test_get_upper_multid_box(toy_model_multid, odd, method, mode, helpers): - if not helpers.is_method_mode_compatible(method=method, mode=mode): - # skip method=ibp/crown-ibp with mode=affine/hybrid - pytest.skip(f"output mode {mode} is not compatible with convert method {method}") - - inputs_ = helpers.get_standard_values_multid_box_convert(odd, dc_decomp=False) - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ - ibp = get_ibp(mode) - affine = get_affine(mode) - backward_model = clone(toy_model_multid, method=method, final_ibp=ibp, final_affine=affine) - upper = get_upper_box(backward_model, z[:, 0], z[:, 1]) - y_ref = helpers.predict_on_small_numpy(toy_model_multid, y) - - assert (upper - y_ref).min() + 1e-6 >= 0.0 - - -def test_get_lower_multid_box(toy_model_multid, odd, method, mode, helpers): - if not helpers.is_method_mode_compatible(method=method, mode=mode): - # skip method=ibp/crown-ibp with mode=affine/hybrid - pytest.skip(f"output mode {mode} is not compatible with convert method {method}") - - inputs_ = helpers.get_standard_values_multid_box_convert(odd, dc_decomp=False) - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ - - ibp = get_ibp(mode) - affine = get_affine(mode) - backward_model = clone(toy_model_multid, method=method, final_ibp=ibp, final_affine=affine) - lower = get_lower_box(backward_model, z[:, 0], z[:, 1]) - y_ref = helpers.predict_on_small_numpy(toy_model_multid, y) - - assert (y_ref - lower).min() + 1e-6 >= 0.0 - - -def test_get_range_multid_box(toy_model_multid, odd, method, mode, helpers): - if not helpers.is_method_mode_compatible(method=method, mode=mode): - # skip method=ibp/crown-ibp with mode=affine/hybrid - pytest.skip(f"output mode {mode} is not compatible with convert method {method}") - - inputs_ = helpers.get_standard_values_multid_box_convert(odd, dc_decomp=False) - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ - - ibp = get_ibp(mode) - affine = get_affine(mode) - backward_model = clone(toy_model_multid, method=method, final_ibp=ibp, final_affine=affine) - upper, lower = get_range_box(backward_model, z[:, 0], z[:, 1]) - y_ref = helpers.predict_on_small_numpy(toy_model_multid, y) - - assert (upper - y_ref).min() + 1e-6 >= 0.0 - assert (y_ref - lower).min() + 1e-6 >= 0.0 From 127be44cef032ff26944d82eb1768ed4762e5f51 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 1 Feb 2024 23:17:56 +0100 Subject: [PATCH 007/101] Deactivate imports in decomon.__init__() Some imports are not working anymore because of the partial refactoring --- src/decomon/__init__.py | 34 +++++++++++++++++----------------- src/decomon/models/__init__.py | 4 ++-- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/decomon/__init__.py b/src/decomon/__init__.py index 87cd56bf..2debee7b 100644 --- a/src/decomon/__init__.py +++ b/src/decomon/__init__.py @@ -8,23 +8,23 @@ import keras -from . import layers, models -from .metrics.loss import get_adv_loss, get_lower_loss, get_model, get_upper_loss -from .models.models import get_AB as get_grid_params -from .models.models import get_AB_finetune as get_grid_slope -from .wrapper import ( - check_adv_box, - get_adv_box, - get_adv_noise, - get_lower_box, - get_lower_noise, - get_range_box, - get_range_noise, - get_upper_box, - get_upper_noise, - refine_box, -) -from .wrapper_with_tuning import get_lower_box_tuning, get_upper_box_tuning +# from . import layers, models +# from .metrics.loss import get_adv_loss, get_lower_loss, get_model, get_upper_loss +# from .models.models import get_AB as get_grid_params +# from .models.models import get_AB_finetune as get_grid_slope +# from .wrapper import ( +# check_adv_box, +# get_adv_box, +# get_adv_noise, +# get_lower_box, +# get_lower_noise, +# get_range_box, +# get_range_noise, +# get_upper_box, +# get_upper_noise, +# refine_box, +# ) +# from .wrapper_with_tuning import get_lower_box_tuning, get_upper_box_tuning try: __version__ = version("decomon") diff --git a/src/decomon/models/__init__.py b/src/decomon/models/__init__.py index 88415a00..d2ae1fe9 100644 --- a/src/decomon/models/__init__.py +++ b/src/decomon/models/__init__.py @@ -1,4 +1,4 @@ -from .convert import clone -from .models import DecomonModel +# from .convert import clone +# from .models import DecomonModel # from .decomon_sequential import DecomonModel From d3102db787154513c651f5baee288796815d562e Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 6 Feb 2024 11:24:56 +0100 Subject: [PATCH 008/101] Remove some of previous code in conftest.py With the refactoring, some code is not relevant anymore. --- tests/conftest.py | 1724 ++++++--------------------------------------- 1 file changed, 198 insertions(+), 1526 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7829181b..fafc9577 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,50 +1,53 @@ -from typing import Optional, Union +from typing import Union import keras import keras.config as keras_config import keras.ops as K import numpy as np -import numpy.typing as npt import pytest -from keras import KerasTensor -from keras.layers import ( - Activation, - Add, - Average, - Conv2D, - Dense, - Flatten, - Input, - Reshape, -) -from keras.models import Model, Sequential -from numpy.testing import assert_almost_equal -from pytest_cases import fixture, fixture_union, param_fixture +from keras import KerasTensor, Model +from keras.layers import Input +from pytest_cases import param_fixture, param_fixtures -from decomon.core import ForwardMode, Slope +from decomon.core import BoxDomain, Propagation, Slope from decomon.keras_utils import ( BACKEND_JAX, BACKEND_NUMPY, BACKEND_PYTORCH, BACKEND_TENSORFLOW, + batch_multid_dot, ) from decomon.models.utils import ConvertMethod -from decomon.types import Tensor - - -@pytest.fixture(params=[m.value for m in ForwardMode]) -def mode(request): - return request.param +from decomon.types import BackendTensor, Tensor + +ibp, affine, propagation = param_fixtures( + "ibp, affine, propagation", + [ + (True, True, Propagation.FORWARD), + (False, True, Propagation.FORWARD), + (True, False, Propagation.FORWARD), + (True, True, Propagation.BACKWARD), + ], + ids=["forward-hybrid", "forward-affine", "forward-ibp", "backward"], +) +slope = param_fixture("slope", [s.value for s in Slope]) +n = param_fixture("n", list(range(10))) +use_bias = param_fixture("use_bias", [True, False]) +randomize = param_fixture("randomize", [True, False]) +padding = param_fixture("padding", ["same", "valid"]) +activation = param_fixture("activation", [None, "relu"]) +data_format = param_fixture("data_format", ["channels_last", "channels_first"]) +method = param_fixture("method", [m.value for m in ConvertMethod]) -@pytest.fixture(params=[s.value for s in Slope]) -def slope(request): - return request.param +@pytest.fixture +def batchsize(): + return 10 -@pytest.fixture(params=list(range(10))) -def n(request): - return request.param +@pytest.fixture +def perturbation_domain(): + return BoxDomain() @pytest.fixture(params=[32, 64, 16]) @@ -71,89 +74,6 @@ def decimal(floatx): return 4 -@pytest.fixture(params=[True, False]) -def dc_decomp(request): - return request.param - - -@pytest.fixture(params=[True, False]) -def use_bias(request): - return request.param - - -@pytest.fixture(params=[True, False]) -def shared(request): - return request.param - - -@pytest.fixture(params=["same", "valid"]) -def padding(request): - return request.param - - -@pytest.fixture(params=[None, "linear", "relu"]) -def activation(request): - return request.param - - -data_format = param_fixture(argname="data_format", argvalues=["channels_last", "channels_first"]) - - -@pytest.fixture(params=[0, 1]) -def odd(request): - return request.param - - -@pytest.fixture(params=[1, 2, 3, -1]) -def axis(request): - return request.param - - -@pytest.fixture(params=[1, 2, 11]) -def channels(request): - return request.param - - -@pytest.fixture(params=[1, 2, 3]) -def filter_size(request): - return request.param - - -@pytest.fixture(params=[1, 2]) -def strides(request): - return request.param - - -@pytest.fixture(params=[True, False]) -def flatten(request): - return request.param - - -@pytest.fixture(params=[0, 1, 2, 3, 4, 5, 6]) -def finetune_odd(request) -> Optional[np.ndarray]: - # hard code several configuration of finetune for images odd=1 (6, 6, 2) - finetune_params = np.zeros((6, 6, 2)) - if request.param == 1: - finetune_params += 1 - elif request.param == 2: - finetune_params[0] = 1 - elif request.param == 3: - finetune_params[:, 0] = 1 - elif request.param == 4: - finetune_params[:, :, 0] = 1 - elif request.param == 5: - finetune_params[0, 0, 0] = 1 - else: - return None - - return finetune_params - - -@pytest.fixture(params=[m.value for m in ConvertMethod]) -def method(request): - return request.param - - class ModelNumpyFromKerasTensors: def __init__(self, inputs: list[KerasTensor], outputs: list[KerasTensor]): self.inputs = inputs @@ -169,6 +89,8 @@ def __call__(self, inputs_: list[np.ndarray]): class Helpers: + function = ModelNumpyFromKerasTensors + @staticmethod def in_GPU_mode() -> bool: backend = keras.config.backend() @@ -190,1482 +112,232 @@ def in_GPU_mode() -> bool: raise NotImplementedError(f"Not implemented for {backend} backend.") @staticmethod - def is_method_mode_compatible(method, mode): - return not ( - ConvertMethod(method) in {ConvertMethod.CROWN_FORWARD_IBP, ConvertMethod.FORWARD_IBP} - and ForwardMode(mode) != ForwardMode.IBP - ) - - function = ModelNumpyFromKerasTensors + def generate_random_tensor(shape_wo_batchsize, batchsize=10, dtype="float32"): + shape = (batchsize,) + shape_wo_batchsize + return K.convert_to_tensor(np.random.random(shape), dtype=dtype) @staticmethod - def predict_on_small_numpy( - model: Model, x: Union[np.ndarray, list[np.ndarray]] - ) -> Union[np.ndarray, list[np.ndarray]]: - """Make predictions for model directly on small numpy arrays - - Avoid using `model.predict()` known to be not designed for small arrays, - and leading to memory leaks when used in loops. - - See https://keras.io/api/models/model_training_apis/#predict-method and - https://github.com/tensorflow/tensorflow/issues/44711 - - Args: - model: - x: - - Returns: - - """ - output_tensors = model(x) - if isinstance(output_tensors, list): - return [K.convert_to_numpy(output) for output in output_tensors] - else: - return K.convert_to_numpy(output_tensors) - - @staticmethod - def get_standard_values_1d_box(n, dc_decomp=True, grad_bounds=False, nb=100): - """A set of functions with their monotonic decomposition for testing the activations""" - w_u_ = np.ones(nb, dtype=keras_config.floatx()) - b_u_ = np.zeros(nb, dtype=keras_config.floatx()) - w_l_ = np.ones(nb, dtype=keras_config.floatx()) - b_l_ = np.zeros(nb, dtype=keras_config.floatx()) - - if n == 0: - # identity - y_ = np.linspace(-2, -1, nb) - x_ = np.linspace(-2, -1, nb) - h_ = np.linspace(-2, -1, nb) - g_ = np.zeros_like(x_) - - elif n == 1: - y_ = np.linspace(1, 2, nb) - x_ = np.linspace(1, 2, nb) - h_ = np.linspace(1, 2, nb) - g_ = np.zeros_like(x_) - - elif n == 2: - y_ = np.linspace(-1, 1, nb) - x_ = np.linspace(-1, 1, nb) - h_ = np.linspace(-1, 1, nb) - g_ = np.zeros_like(x_) - - elif n == 3: - # identity - y_ = np.linspace(-2, -1, nb) - x_ = np.linspace(-2, -1, nb) - h_ = 2 * np.linspace(-2, -1, nb) - g_ = -np.linspace(-2, -1, nb) - - elif n == 4: - y_ = np.linspace(1, 2, nb) - x_ = np.linspace(1, 2, nb) - h_ = 2 * np.linspace(1, 2, nb) - g_ = -np.linspace(1, 2, nb) - - elif n == 5: - y_ = np.linspace(-1, 1, nb) - x_ = np.linspace(-1, 1, nb) - h_ = 2 * np.linspace(-1, 1, nb) - g_ = -np.linspace(-1, 1, nb) - - elif n == 6: - assert nb == 100, "expected nb=100 samples" - # cosine function - x_ = np.linspace(-np.pi, np.pi, 100) - y_ = np.cos(x_) - h_ = np.concatenate([y_[:50], np.ones((50,))]) - 0.5 - g_ = np.concatenate([np.ones((50,)), y_[50:]]) - 0.5 - w_u_ = np.zeros_like(x_) - w_l_ = np.zeros_like(x_) - b_u_ = np.ones_like(x_) - b_l_ = -np.ones_like(x_) - - elif n == 7: - # h and g >0 - h_ = np.linspace(0.5, 2, nb) - g_ = np.linspace(1, 2, nb)[::-1] - x_ = h_ + g_ - y_ = h_ + g_ - - elif n == 8: - # h <0 and g <0 - # h_max+g_max <=0 - h_ = np.linspace(-2, -1, nb) - g_ = np.linspace(-2, -1, nb)[::-1] - y_ = h_ + g_ - x_ = h_ + g_ - - elif n == 9: - # h >0 and g <0 - # h_min+g_min >=0 - h_ = np.linspace(4, 5, nb) - g_ = np.linspace(-2, -1, nb)[::-1] - y_ = h_ + g_ - x_ = h_ + g_ - - else: - raise ValueError("n must be between 0 and 9.") - - x_min_ = x_.min() + np.zeros_like(x_) - x_max_ = x_.max() + np.zeros_like(x_) - - x_0_ = np.concatenate([x_min_[:, None], x_max_[:, None]], 1) - - u_c_ = np.max(y_) * np.ones((nb,)) - l_c_ = np.min(y_) * np.ones((nb,)) - - if dc_decomp: - output = [ - x_[:, None], - y_[:, None], - x_0_[:, :, None], - u_c_[:, None], - w_u_[:, None, None], - b_u_[:, None], - l_c_[:, None], - w_l_[:, None, None], - b_l_[:, None], - h_[:, None], - g_[:, None], - ] - else: - output = [ - x_[:, None], - y_[:, None], - x_0_[:, :, None], - u_c_[:, None], - w_u_[:, None, None], - b_u_[:, None], - l_c_[:, None], - w_l_[:, None, None], - b_l_[:, None], - ] - - # cast element - return [e.astype(keras_config.floatx()) for e in output] - - @staticmethod - def get_inputs_for_mode_from_full_inputs( - inputs: Union[list[Tensor], list[npt.NDArray[np.float_]]], - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - dc_decomp: bool = True, - ) -> Union[list[Tensor], list[npt.NDArray[np.float_]]]: - """Extract from full inputs the ones corresponding to the selected mode. - - Args: - inputs: inputs from `get_standard_values_xxx()` or `get_tensor_decomposition_xxx()` - mode: - dc_decomp: - - Returns: - - """ - mode = ForwardMode(mode) - if dc_decomp: - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l, h, g = inputs - if mode == ForwardMode.HYBRID: - return inputs[2:] - elif mode == ForwardMode.AFFINE: - return [z, W_u, b_u, W_l, b_l, h, g] - elif mode == ForwardMode.IBP: - return [u_c, l_c, h, g] - else: - raise ValueError("Unknown mode.") - else: - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs - if mode == ForwardMode.HYBRID: - return inputs[2:] - elif mode == ForwardMode.AFFINE: - return [z, W_u, b_u, W_l, b_l] - elif mode == ForwardMode.IBP: - return [u_c, l_c] + def get_decomon_input_shapes( + model_input_shape, + model_output_shape, + layer_input_shape, + layer_output_shape, + ibp, + affine, + propagation, + perturbation_domain, + ): + x_shape = perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape) + if affine: + if propagation == Propagation.FORWARD: + b_in_shape = layer_input_shape + w_in_shape = model_input_shape + layer_input_shape else: - raise ValueError("Unknown mode.") - - @staticmethod - def get_inputs_np_for_decomon_model_from_full_inputs( - inputs: list[npt.NDArray[np.float_]], - ) -> npt.NDArray[np.float_]: - """Extract from full numpy inputs the ones for a decomon model prediction. - - Args: - inputs: inputs from `get_standard_values_xxx()` - - Returns: - - """ - l_c_, u_c_ = Helpers.get_input_ref_bounds_from_full_inputs(inputs=inputs) - return np.concatenate((l_c_[:, None], u_c_[:, None]), axis=1) - - @staticmethod - def get_input_ref_bounds_from_full_inputs( - inputs: Union[list[Tensor], list[npt.NDArray[np.float_]]], - ) -> Union[list[Tensor], list[npt.NDArray[np.float_]]]: - """Extract lower and upper bound for input ref from full inputs - - Args: - inputs: inputs from `get_standard_values_xxx()` or `get_tensor_decomposition_xxx()` - - Returns: + b_in_shape = model_output_shape + w_in_shape = layer_output_shape + model_output_shape - """ - u_c_, l_c_ = inputs[3], inputs[6] - return [l_c_, u_c_] - - @staticmethod - def prepare_full_np_inputs_for_convert_model( - inputs: list[npt.NDArray[np.float_]], - dc_decomp: bool = True, - ) -> list[npt.NDArray[np.float_]]: - """Prepare full numpy inputs for convert_forward or convert_backward. - - W_u and W_l will be idendity matrices, and b_u, b_l zeros vectors. - - Args: - inputs: inputs from `get_standard_values_xxx()` or `get_tensor_decomposition_xxx()` - - Returns: - - """ - if dc_decomp: - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l, h, g = inputs + affine_bounds_to_propagate_shape = [w_in_shape, b_in_shape, w_in_shape, b_in_shape] else: - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs - - b_u = np.zeros_like(b_u) - b_l = np.zeros_like(b_l) - W_u = np.repeat(np.identity(n=W_u.shape[-1])[None, :, :], repeats=W_u.shape[0], axis=0) - W_l = np.repeat(np.identity(n=W_l.shape[-1])[None, :, :], repeats=W_l.shape[0], axis=0) - - if dc_decomp: - return [x, y, z, u_c, W_u, b_u, l_c, W_l, b_l, h, g] + affine_bounds_to_propagate_shape = [] + if ibp: + constant_oracle_bounds_shape = [layer_input_shape, layer_input_shape] else: - return [x, y, z, u_c, W_u, b_u, l_c, W_l, b_l] - - @staticmethod - def get_input_tensors_for_decomon_convert_from_full_inputs( - inputs: list[Tensor], - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - dc_decomp: bool = True, - ) -> list[Tensor]: - """Extract from full tensor inputs the ones for a conversion to decomon model. - - Args: - inputs: inputs from `get_tensor_decomposition_xxx()` - mode: - dc_decomp: + constant_oracle_bounds_shape = [] - Returns: - - """ - mode = ForwardMode(mode) - if dc_decomp: - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l, h, g = inputs - input_box_tensor = K.concatenate([K.expand_dims(l_c, 1), K.expand_dims(u_c, 1)], 1) - if mode == ForwardMode.HYBRID: - return [input_box_tensor, u_c, W_u, b_u, l_c, W_l, b_l, h, g] - elif mode == ForwardMode.AFFINE: - return [input_box_tensor, W_u, b_u, W_l, b_l, h, g] - elif mode == ForwardMode.IBP: - return [u_c, l_c, h, g] - else: - raise ValueError("Unknown mode.") - else: - x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs - input_box_tensor = K.concatenate([K.expand_dims(l_c, 1), K.expand_dims(u_c, 1)], 1) - if mode == ForwardMode.HYBRID: - return [input_box_tensor, u_c, W_u, b_u, l_c, W_l, b_l] - elif mode == ForwardMode.AFFINE: - return [input_box_tensor, W_u, b_u, W_l, b_l] - elif mode == ForwardMode.IBP: - return [u_c, l_c] - else: - raise ValueError("Unknown mode.") + return [affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, x_shape] @staticmethod - def get_input_ref_from_full_inputs( - inputs: Union[list[Tensor], list[npt.NDArray[np.float_]]] - ) -> Union[Tensor, npt.NDArray[np.float_]]: - """Extract from full inputs the input of reference for the original Keras layer. + def get_decomon_symbolic_inputs( + model_input_shape, + model_output_shape, + layer_input_shape, + layer_output_shape, + ibp, + affine, + propagation, + perturbation_domain, + ): + """Generate decomon symbolic inputs for a decomon layer + + To be used as `decomon_layer(*decomon_inputs)`. Args: - inputs: inputs from `get_standard_values_xxx()` or `get_tensor_decomposition_xxx()` + model_input_shape: + model_output_shape: + layer_input_shape: + layer_output_shape: + ibp: + affine: + propagation: + perturbation_domain: Returns: """ - return inputs[1] - - @staticmethod - def get_tensor_decomposition_1d_box(dc_decomp=True): - if dc_decomp: - return [ - Input((1,), dtype=keras_config.floatx()), - Input((1,), dtype=keras_config.floatx()), - Input((2, 1), dtype=keras_config.floatx()), - Input((1,), dtype=keras_config.floatx()), - Input((1, 1), dtype=keras_config.floatx()), - Input((1,), dtype=keras_config.floatx()), - Input((1,), dtype=keras_config.floatx()), - Input((1, 1), dtype=keras_config.floatx()), - Input((1,), dtype=keras_config.floatx()), - Input((1,), dtype=keras_config.floatx()), - Input((1,), dtype=keras_config.floatx()), - ] - return [ - Input((1,), dtype=keras_config.floatx()), - Input((1,), dtype=keras_config.floatx()), - Input((2, 1), dtype=keras_config.floatx()), - Input((1,), dtype=keras_config.floatx()), - Input((1, 1), dtype=keras_config.floatx()), - Input((1,), dtype=keras_config.floatx()), - Input((1,), dtype=keras_config.floatx()), - Input((1, 1), dtype=keras_config.floatx()), - Input((1,), dtype=keras_config.floatx()), - ] + decomon_input_shape = Helpers.get_decomon_input_shapes( + model_input_shape, + model_output_shape, + layer_input_shape, + layer_output_shape, + ibp, + affine, + propagation, + perturbation_domain, + ) + affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, x_shape = decomon_input_shape + x = Input(x_shape) + affine_bounds_to_propagate = [Input(shape) for shape in affine_bounds_to_propagate_shape] + constant_oracle_bounds = [Input(shape) for shape in constant_oracle_bounds_shape] + return [affine_bounds_to_propagate, constant_oracle_bounds, x] @staticmethod - def get_full_outputs_from_outputs_for_mode( - outputs_for_mode: Union[list[Tensor], list[npt.NDArray[np.float_]]], - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - dc_decomp: bool = True, - full_inputs: Optional[Union[list[Tensor], list[npt.NDArray[np.float_]]]] = None, - ) -> Union[list[Tensor], list[npt.NDArray[np.float_]]]: - mode = ForwardMode(mode) - if dc_decomp: - if mode == ForwardMode.HYBRID: - z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_, h_, g_ = outputs_for_mode - elif mode == ForwardMode.AFFINE: - z_, w_u_, b_u_, w_l_, b_l_, h_, g_ = outputs_for_mode - u_c_, l_c_ = None, None - elif mode == ForwardMode.IBP: - u_c_, l_c_, h_, g_ = outputs_for_mode - z_, w_u_, b_u_, w_l_, b_l_ = None, None, None, None, None - if full_inputs is not None: - z_ = full_inputs[2] - else: - raise ValueError("Unknown mode.") - else: - if mode == ForwardMode.HYBRID: - z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_ = outputs_for_mode - elif mode == ForwardMode.AFFINE: - z_, w_u_, b_u_, w_l_, b_l_ = outputs_for_mode - u_c_, l_c_ = None, None - elif mode == ForwardMode.IBP: - u_c_, l_c_ = outputs_for_mode - z_, w_u_, b_u_, w_l_, b_l_ = None, None, None, None, None - if full_inputs is not None: - z_ = full_inputs[2] - else: - raise ValueError("Unknown mode.") - h_, g_ = None, None - return [z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_, h_, g_] + def generate_simple_decomon_layer_inputs_from_keras_input( + keras_input, layer_output_shape, ibp, affine, propagation, perturbation_domain + ): + """Generate simple decomon inputs for a layer from the corresponding keras input - @staticmethod - def get_input_dim_multid_box(odd): - if odd: - return 3 - else: - return 2 + Hypothesis: single-layer model => model input/output = layer input/output - @staticmethod - def get_input_dim_images_box(odd): - if odd: - return 7 - else: - return 6 + For affine bounds, weights= identity + bias = 0 + For constant bounds, (keras_input, keras_input) - @staticmethod - def get_input_dim_from_full_inputs(inputs: Union[list[Tensor], list[npt.NDArray[np.float_]]]) -> int: - """Get input_dim for to_decomon or to_backward from full inputs + To be used as `decomon_layer(*decomon_inputs)`. Args: - inputs: inputs from `get_standard_values_xxx()` or `get_tensor_decomposition_xxx()` + keras_input: + layer_output_shape: + ibp: + affine: + propagation: + perturbation_domain: Returns: """ - return inputs[0].shape[-1] - - @staticmethod - def get_tensor_decomposition_multid_box(odd=1, dc_decomp=True): - n = Helpers.get_input_dim_multid_box(odd) - - if dc_decomp: - # x, y, z, u, w_u, b_u, l, w_l, b_l, h, g - return [ - Input((n,), dtype=keras_config.floatx()), - Input((n,), dtype=keras_config.floatx()), - Input((2, n), dtype=keras_config.floatx()), - Input((n,), dtype=keras_config.floatx()), - Input((n, n), dtype=keras_config.floatx()), - Input((n,), dtype=keras_config.floatx()), - Input((n,), dtype=keras_config.floatx()), - Input((n, n), dtype=keras_config.floatx()), - Input((n,), dtype=keras_config.floatx()), - Input((n,), dtype=keras_config.floatx()), - Input((n,), dtype=keras_config.floatx()), - ] - return [ - Input((n,), dtype=keras_config.floatx()), - Input((n,), dtype=keras_config.floatx()), - Input((2, n), dtype=keras_config.floatx()), - Input((n,), dtype=keras_config.floatx()), - Input((n, n), dtype=keras_config.floatx()), - Input((n,), dtype=keras_config.floatx()), - Input((n,), dtype=keras_config.floatx()), - Input((n, n), dtype=keras_config.floatx()), - Input((n,), dtype=keras_config.floatx()), - ] - - @staticmethod - def get_standard_values_multid_box(odd=1, dc_decomp=True): - if dc_decomp: - ( - x_0, - y_0, - z_0, - u_c_0, - w_u_0, - b_u_0, - l_c_0, - w_l_0, - b_l_0, - h_0, - g_0, - ) = Helpers.get_standard_values_1d_box(0, dc_decomp) - ( - x_1, - y_1, - z_1, - u_c_1, - w_u_1, - b_u_1, - l_c_1, - w_l_1, - b_l_1, - h_1, - g_1, - ) = Helpers.get_standard_values_1d_box(1, dc_decomp) - else: - ( - x_0, - y_0, - z_0, - u_c_0, - w_u_0, - b_u_0, - l_c_0, - w_l_0, - b_l_0, - ) = Helpers.get_standard_values_1d_box(0, dc_decomp) - ( - x_1, - y_1, - z_1, - u_c_1, - w_u_1, - b_u_1, - l_c_1, - w_l_1, - b_l_1, - ) = Helpers.get_standard_values_1d_box(1, dc_decomp) - - if not odd: - # output (x_0+x_1, x_0+2*x_0) - x_ = np.concatenate([x_0, x_1], -1) - z_min_ = np.concatenate([z_0[:, 0], z_1[:, 0]], -1) - z_max_ = np.concatenate([z_0[:, 1], z_1[:, 1]], -1) - z_ = np.concatenate([z_min_[:, None], z_max_[:, None]], 1) - y_ = np.concatenate([y_0 + y_1, y_0 + 2 * y_1], -1) - b_u_ = np.concatenate([b_u_0 + b_u_1, b_u_0 + 2 * b_u_1], -1) - u_c_ = np.concatenate([u_c_0 + u_c_1, u_c_0 + 2 * u_c_1], -1) - b_l_ = np.concatenate([b_l_0 + b_l_1, b_l_0 + 2 * b_l_1], -1) - l_c_ = np.concatenate([l_c_0 + l_c_1, l_c_0 + 2 * l_c_1], -1) - - if dc_decomp: - h_ = np.concatenate([h_0 + h_1, h_0 + 2 * h_1], -1) - g_ = np.concatenate([g_0 + g_1, g_0 + 2 * g_1], -1) - - w_u_ = np.zeros((len(x_), 2, 2)) - w_u_[:, 0, 0] = w_u_0[:, 0, 0] - w_u_[:, 1, 0] = w_u_1[:, 0, 0] - w_u_[:, 0, 1] = w_u_0[:, 0, 0] - w_u_[:, 1, 1] = 2 * w_u_1[:, 0, 0] - - w_l_ = np.zeros((len(x_), 2, 2)) - w_l_[:, 0, 0] = w_l_0[:, 0, 0] - w_l_[:, 1, 0] = w_l_1[:, 0, 0] - w_l_[:, 0, 1] = w_l_0[:, 0, 0] - w_l_[:, 1, 1] = 2 * w_l_1[:, 0, 0] - - else: - ( - x_2, - y_2, - z_2, - u_c_2, - w_u_2, - b_u_2, - l_c_2, - w_l_2, - b_l_2, - h_2, - g_2, - ) = Helpers.get_standard_values_1d_box(2) - - # output (x_0+x_1, x_0+2*x_0, x_2) - x_ = np.concatenate([x_0, x_1, x_2], -1) - z_min_ = np.concatenate([z_0[:, 0], z_1[:, 0], z_2[:, 0]], -1) - z_max_ = np.concatenate([z_0[:, 1], z_1[:, 1], z_2[:, 1]], -1) - z_ = np.concatenate([z_min_[:, None], z_max_[:, None]], 1) - y_ = np.concatenate([y_0 + y_1, y_0 + 2 * y_1, y_2], -1) - b_u_ = np.concatenate([b_u_0 + b_u_1, b_u_0 + 2 * b_u_1, b_u_2], -1) - b_l_ = np.concatenate([b_l_0 + b_l_1, b_l_0 + 2 * b_l_1, b_l_2], -1) - u_c_ = np.concatenate([u_c_0 + u_c_1, u_c_0 + 2 * u_c_1, u_c_2], -1) - l_c_ = np.concatenate([l_c_0 + l_c_1, l_c_0 + 2 * l_c_1, l_c_2], -1) - - if dc_decomp: - h_ = np.concatenate([h_0 + h_1, h_0 + 2 * h_1, h_2], -1) - g_ = np.concatenate([g_0 + g_1, g_0 + 2 * g_1, g_2], -1) - - w_u_ = np.zeros((len(x_), 3, 3)) - w_u_[:, 0, 0] = w_u_0[:, 0, 0] - w_u_[:, 1, 0] = w_u_1[:, 0, 0] - w_u_[:, 0, 1] = w_u_0[:, 0, 0] - w_u_[:, 1, 1] = 2 * w_u_1[:, 0, 0] - w_u_[:, 2, 2] = w_u_2[:, 0, 0] - - w_l_ = np.zeros((len(x_), 3, 3)) - w_l_[:, 0, 0] = w_l_0[:, 0, 0] - w_l_[:, 1, 0] = w_l_1[:, 0, 0] - w_l_[:, 0, 1] = w_l_0[:, 0, 0] - w_l_[:, 1, 1] = 2 * w_l_1[:, 0, 0] - w_l_[:, 2, 2] = w_l_2[:, 0, 0] - - if dc_decomp: - return [x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_, h_, g_] - return [x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_] - - @staticmethod - def build_image_from_1D_box(odd=0, m=0, dc_decomp=True): - n = Helpers.get_input_dim_images_box(odd) - - if dc_decomp: - ( - x_, - y_0, - z_, - u_c_0, - w_u_0, - b_u_0, - l_c_0, - w_l_0, - b_l_0, - h_0, - g_0, - ) = Helpers.get_standard_values_1d_box(m, dc_decomp=dc_decomp) - else: - ( - x_, - y_0, - z_, - u_c_0, - w_u_0, - b_u_0, - l_c_0, - w_l_0, - b_l_0, - ) = Helpers.get_standard_values_1d_box(m, dc_decomp=dc_decomp) - - y_ = np.concatenate([(i + 1) * y_0 for i in range(n * n)], -1).reshape((-1, n, n)) - b_u_ = np.concatenate([(i + 1) * b_u_0 for i in range(n * n)], -1).reshape((-1, n, n)) - b_l_ = np.concatenate([(i + 1) * b_l_0 for i in range(n * n)], -1).reshape((-1, n, n)) - - if dc_decomp: - h_ = np.concatenate([(i + 1) * h_0 for i in range(n * n)], -1).reshape((-1, n, n)) - g_ = np.concatenate([(i + 1) * g_0 for i in range(n * n)], -1).reshape((-1, n, n)) - - u_c_ = np.concatenate([(i + 1) * u_c_0 for i in range(n * n)], -1).reshape((-1, n, n)) - l_c_ = np.concatenate([(i + 1) * l_c_0 for i in range(n * n)], -1).reshape((-1, n, n)) - - w_u_ = np.zeros((len(x_), 1, n * n)) - w_l_ = np.zeros((len(x_), 1, n * n)) - - for i in range(n * n): - w_u_[:, 0, i] = (i + 1) * w_u_0[:, 0, 0] - w_l_[:, 0, i] = (i + 1) * w_l_0[:, 0, 0] - - w_u_ = w_u_.reshape((-1, 1, n, n)) - w_l_ = w_l_.reshape((-1, 1, n, n)) - - if dc_decomp: - return [x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_, h_, g_] - return [x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_] - - @staticmethod - def build_image_from_2D_box(odd=0, m0=0, m1=1, dc_decomp=True): - if dc_decomp: - ( - x_0, - y_0, - z_0, - u_c_0, - w_u_0, - b_u_0, - l_c_0, - w_l_0, - b_l_0, - h_0, - g_0, - ) = Helpers.build_image_from_1D_box(odd, m0, dc_decomp) - ( - x_1, - y_1, - z_1, - u_c_1, - w_u_1, - b_u_1, - l_c_1, - w_l_1, - b_l_1, - h_1, - g_1, - ) = Helpers.build_image_from_1D_box(odd, m1, dc_decomp) + if isinstance(perturbation_domain, BoxDomain): + x = K.repeat(keras_input[:, None], 2, axis=1) else: - ( - x_0, - y_0, - z_0, - u_c_0, - w_u_0, - b_u_0, - l_c_0, - w_l_0, - b_l_0, - ) = Helpers.build_image_from_1D_box(odd, m0, dc_decomp) - ( - x_1, - y_1, - z_1, - u_c_1, - w_u_1, - b_u_1, - l_c_1, - w_l_1, - b_l_1, - ) = Helpers.build_image_from_1D_box(odd, m1, dc_decomp) - - x_ = np.concatenate([x_0, x_1], -1) - z_min_ = np.concatenate([z_0[:, 0], z_1[:, 0]], -1) - z_max_ = np.concatenate([z_0[:, 1], z_1[:, 1]], -1) - z_ = np.concatenate([z_min_[:, None], z_max_[:, None]], 1) - y_ = y_0 + y_1 - b_u_ = b_u_0 + b_u_1 - b_l_ = b_l_0 + b_l_1 - - u_c_ = u_c_0 + u_c_1 - l_c_ = l_c_0 + l_c_1 - - w_u_ = np.concatenate([w_u_0, w_u_1], 1) - w_l_ = np.concatenate([w_l_0, w_l_1], 1) - - if dc_decomp: - h_ = h_0 + h_1 - g_ = g_0 + g_1 + raise NotImplementedError - return [x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_, h_, g_] - return [x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_] - - @staticmethod - def get_standard_values_images_box(data_format="channels_last", odd=0, m0=0, m1=1, dc_decomp=True): - if data_format == "channels_last": - output = Helpers.build_image_from_2D_box(odd, m0, m1, dc_decomp) - if dc_decomp: - x_0, y_0, z_0, u_c_0, w_u_0, b_u_0, l_c_0, w_l_0, b_l_0, h_0, g_0 = output + if affine: + batchsize = keras_input.shape[0] + if propagation == Propagation.FORWARD: + bias_shape = keras_input.shape[1:] else: - x_0, y_0, z_0, u_c_0, w_u_0, b_u_0, l_c_0, w_l_0, b_l_0 = output - - x_ = x_0 - z_ = z_0 - - y_0 = y_0[:, :, :, None] - b_u_0 = b_u_0[:, :, :, None] - b_l_0 = b_l_0[:, :, :, None] - u_c_0 = u_c_0[:, :, :, None] - l_c_0 = l_c_0[:, :, :, None] - w_u_0 = w_u_0[:, :, :, :, None] - w_l_0 = w_l_0[:, :, :, :, None] - y_ = np.concatenate([y_0, y_0], -1) - b_u_ = np.concatenate([b_u_0, b_u_0], -1) - b_l_ = np.concatenate([b_l_0, b_l_0], -1) - u_c_ = np.concatenate([u_c_0, u_c_0], -1) - l_c_ = np.concatenate([l_c_0, l_c_0], -1) - w_u_ = np.concatenate([w_u_0, w_u_0], -1) - w_l_ = np.concatenate([w_l_0, w_l_0], -1) - - if dc_decomp: - h_0 = h_0[:, :, :, None] - g_0 = g_0[:, :, :, None] - h_ = np.concatenate([h_0, h_0], -1) - g_ = np.concatenate([g_0, g_0], -1) - - else: - output = Helpers.get_standard_values_images_box( - data_format="channels_last", odd=odd, m0=m0, m1=m1, dc_decomp=dc_decomp + bias_shape = layer_output_shape + flatten_bias_dim = int(np.prod(bias_shape)) + w_in = K.repeat( + K.reshape(K.eye(flatten_bias_dim), bias_shape + bias_shape)[None], + batchsize, + axis=0, ) - - x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_ = output[:9] - if dc_decomp: - h_, g_ = output[-2:] - h_ = np.transpose(h_, (0, 3, 1, 2)) - g_ = np.transpose(g_, (0, 3, 1, 2)) - y_ = np.transpose(y_, (0, 3, 1, 2)) - u_c_ = np.transpose(u_c_, (0, 3, 1, 2)) - l_c_ = np.transpose(l_c_, (0, 3, 1, 2)) - b_u_ = np.transpose(b_u_, (0, 3, 1, 2)) - b_l_ = np.transpose(b_l_, (0, 3, 1, 2)) - w_u_ = np.transpose(w_u_, (0, 1, 4, 2, 3)) - w_l_ = np.transpose(w_l_, (0, 1, 4, 2, 3)) - - if dc_decomp: - return [x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_, h_, g_] + b_in = K.zeros((batchsize,) + bias_shape) + affine_bounds_to_propagate = [w_in, b_in, w_in, b_in] else: - return [x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_] - - @staticmethod - def get_tensor_decomposition_images_box(data_format, odd, dc_decomp=True): - n = Helpers.get_input_dim_images_box(odd) + affine_bounds_to_propagate = [] - if data_format == "channels_last": - # x, y, z, u, w_u, b_u, l, w_l, b_l - - output = [ - Input((2,), dtype=keras_config.floatx()), - Input((n, n, 2), dtype=keras_config.floatx()), - Input((2, 2), dtype=keras_config.floatx()), - Input((n, n, 2), dtype=keras_config.floatx()), - Input((2, n, n, 2), dtype=keras_config.floatx()), - Input((n, n, 2), dtype=keras_config.floatx()), - Input((n, n, 2), dtype=keras_config.floatx()), - Input((2, n, n, 2), dtype=keras_config.floatx()), - Input((n, n, 2), dtype=keras_config.floatx()), - ] - - if dc_decomp: - output += [Input((n, n, 2), dtype=keras_config.floatx()), Input((n, n, 2), dtype=keras_config.floatx())] + if ibp: + constant_oracle_bounds = [keras_input, keras_input] else: - output = [ - Input((2,), dtype=keras_config.floatx()), - Input((2, n, n), dtype=keras_config.floatx()), - Input((2, 2), dtype=keras_config.floatx()), - Input((2, n, n), dtype=keras_config.floatx()), - Input((2, 2, n, n), dtype=keras_config.floatx()), - Input((2, n, n), dtype=keras_config.floatx()), - Input((2, n, n), dtype=keras_config.floatx()), - Input((2, 2, n, n), dtype=keras_config.floatx()), - Input((2, n, n), dtype=keras_config.floatx()), - ] - if dc_decomp: - output += [Input((2, n, n), dtype=keras_config.floatx()), Input((2, n, n), dtype=keras_config.floatx())] - - return output - - @staticmethod - def assert_output_properties_box(x_, y_, h_, g_, x_min_, x_max_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_, decimal=4): - if y_ is None: - y_ = h_ + g_ - if h_ is not None: - assert_almost_equal(h_ + g_, y_, decimal=decimal, err_msg="decomposition error") - - assert np.min(x_min_ <= x_max_), "x_min >x_max" - - assert_almost_equal(np.clip(x_min_ - x_, 0, np.inf), 0.0, decimal=decimal, err_msg="x_min >x_") - assert_almost_equal(np.clip(x_ - x_max_, 0, np.inf), 0.0, decimal=decimal, err_msg="x_max < x_") - if w_u_ is not None or w_l_ is not None: - x_expand = x_ + np.zeros_like(x_) - n_expand = len(w_u_.shape) - len(x_expand.shape) - for i in range(n_expand): - x_expand = np.expand_dims(x_expand, -1) - - if w_l_ is not None: - lower_ = np.sum(w_l_ * x_expand, 1) + b_l_ - if w_u_ is not None: - upper_ = np.sum(w_u_ * x_expand, 1) + b_u_ - - # check that the functions h_ and g_ remains monotonic - if h_ is not None: - assert_almost_equal( - np.clip(h_[:-1] - h_[1:], 0, np.inf), - np.zeros_like(h_[1:]), - decimal=decimal, - err_msg="h is not increasing", - ) - assert_almost_equal( - np.clip(g_[1:] - g_[:-1], 0, np.inf), - np.zeros_like(g_[1:]), - decimal=decimal, - err_msg="g is not increasing", - ) - - # - if w_u_ is not None: - if keras_config.floatx() == "float32": - assert_almost_equal( - np.clip(y_ - upper_, 0.0, 1e6), - np.zeros_like(y_), - decimal=decimal, - err_msg="upper 0 + ibp = len(decomon_output) > 1 and len(decomon_output[1]) > 0 - @staticmethod - def assert_backward_layer_output_properties_box_linear( - full_inputs, backward_outputs, output_ref=None, upper_constant_bound=None, lower_constant_bound=None, decimal=4 - ): - w_u_, b_u_, w_l_, b_l_ = backward_outputs - x_, y_, z_, u_c_, W_u_, B_u_, l_c_, W_l_, B_l_ = full_inputs + if affine: + w_l, b_l, w_u, b_u = decomon_output[0] + lower_affine = batch_multid_dot(keras_input, w_l) + b_l + upper_affine = batch_multid_dot(keras_input, w_u) + b_u + Helpers.assert_ordered(lower_affine, keras_output, decimal=decimal, err_msg="lower_affine not ok") + Helpers.assert_ordered(keras_output, upper_affine, decimal=decimal, err_msg="upper_affine not ok") - # backward recomposition - w_u_b = np.sum(np.maximum(w_u_, 0) * W_u_ + np.minimum(w_u_, 0) * W_l_, 1)[:, :, None] - b_u_b = ( - b_u_ + np.sum(np.maximum(w_u_, 0) * B_u_[:, :, None], 1) + np.sum(np.minimum(w_u_, 0) * B_l_[:, :, None], 1) - ) - w_l_b = np.sum(np.maximum(w_l_, 0) * W_l_ + np.minimum(w_l_, 0) * W_u_, 1)[:, :, None] - b_l_b = ( - b_l_ + np.sum(np.maximum(w_l_, 0) * B_l_[:, :, None], 1) + np.sum(np.minimum(w_l_, 0) * B_u_[:, :, None], 1) - ) - - Helpers.assert_output_properties_box_linear( - x_, - output_ref, - z_[:, 0], - z_[:, 1], - upper_constant_bound, - w_u_b, - b_u_b, - lower_constant_bound, - w_l_b, - b_l_b, - decimal=decimal, - ) + if ibp: + lower_ibp, upper_ibp = decomon_output[1] + Helpers.assert_ordered(lower_ibp, keras_output, decimal=decimal, err_msg="lower_ibp not ok") + Helpers.assert_ordered(keras_output, upper_ibp, decimal=decimal, err_msg="upper_ibp not ok") @staticmethod - def assert_output_properties_box_linear(x_, y_, x_min_, x_max_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_, decimal=4): - # flatten everyting - n = len(x_) - if y_ is not None: - n = len(y_) - y_ = y_.reshape((n, -1)) - if l_c_ is not None: - u_c_ = u_c_.reshape((n, -1)) - l_c_ = l_c_.reshape((n, -1)) - if w_u_ is not None: - w_u_ = w_u_.reshape((n, w_u_.shape[1], -1)) - w_l_ = w_l_.reshape((n, w_l_.shape[1], -1)) - b_u_ = b_u_.reshape((n, -1)) - b_l_ = b_l_.reshape((n, -1)) - - assert np.min(x_min_ <= x_max_), "x_min >x_max" - - assert_almost_equal(np.clip(x_min_ - x_, 0, np.inf), 0.0, decimal=decimal, err_msg="x_min >x_") - assert_almost_equal(np.clip(x_ - x_max_, 0, np.inf), 0.0, decimal=decimal, err_msg="x_max < x_") - if w_u_ is not None: - x_expand = x_ + np.zeros_like(x_) - n_expand = len(w_u_.shape) - len(x_expand.shape) - for i in range(n_expand): - x_expand = np.expand_dims(x_expand, -1) - - lower_ = np.sum(w_l_ * x_expand, 1) + b_l_ - upper_ = np.sum(w_u_ * x_expand, 1) + b_u_ - - if y_ is not None: - if l_c_ is not None: - assert_almost_equal( - np.clip(l_c_ - y_, 0.0, np.inf), np.zeros_like(y_), decimal=decimal, err_msg="l_c >y" - ) - assert_almost_equal(np.clip(y_ - u_c_, 0.0, 1e6), np.zeros_like(y_), decimal=decimal, err_msg="u_c 0 + ibp = len(decomon_output) > 1 and len(decomon_output[1]) > 0 - lower_ = ( - np.sum(np.maximum(0, w_l_) * x_expand_min, 1) + np.sum(np.minimum(0, w_l_) * x_expand_max, 1) + b_l_ - ) - upper_ = ( - np.sum(np.maximum(0, w_u_) * x_expand_max, 1) + np.sum(np.minimum(0, w_u_) * x_expand_min, 1) + b_u_ - ) + if affine: + w_l, b_l, w_u, b_u = decomon_output[0] + Helpers.assert_almost_equal(w_l, w_u, decimal=decimal) + Helpers.assert_almost_equal(b_l, b_u, decimal=decimal) - if y_ is not None: - assert_almost_equal( - np.clip(lower_ - y_, 0.0, np.inf), np.zeros_like(y_), decimal=decimal, err_msg="l_c >y" - ) - assert_almost_equal( - np.clip(y_ - upper_, 0.0, 1e6), np.zeros_like(y_), decimal=decimal, err_msg="u_c x_max" - - assert_almost_equal(np.clip(x_min_ - x_, 0, np.inf), 0.0, decimal=decimal, err_msg="x_min >x_") - assert_almost_equal(np.clip(x_ - x_max_, 0, np.inf), 0.0, decimal=decimal, err_msg="x_max < x_") - - x_expand = x_ + np.zeros_like(x_) - n_expand = len(w_u_.shape) - len(x_expand.shape) - for i in range(n_expand): - x_expand = np.expand_dims(x_expand, -1) - - lower_ = np.sum(w_l_ * x_expand, 1) + b_l_ - upper_ = np.sum(w_u_ * x_expand, 1) + b_u_ - - # check that the functions h_ and g_ remains monotonic - - assert_almost_equal( - np.clip(l_c_ - y_, 0.0, np.inf), - np.zeros_like(y_), - decimal=decimal, - err_msg="l_c >y", - ) - assert_almost_equal( - np.clip(y_ - u_c_, 0.0, 1e6), - np.zeros_like(y_), - decimal=decimal, - err_msg="u_c Union[np.ndarray, list[np.ndarray]]: + """Make predictions for model directly on small numpy arrays - @staticmethod - def toy_struct_v2_1D(input_dim, archi, sequential, activation, use_bias, merge_op=Add, dtype="float32"): - nnet_0 = Helpers.dense_NN_1D( - input_dim=input_dim, - archi=archi, - sequential=sequential, - activation=activation, - use_bias=use_bias, - dtype=dtype, - ) - nnet_1 = Helpers.dense_NN_1D( - input_dim=input_dim, - archi=archi, - sequential=sequential, - activation=activation, - use_bias=use_bias, - dtype=dtype, - ) - nnet_2 = Dense(archi[-1], use_bias=use_bias, activation="linear", dtype=dtype) + Avoid using `model.predict()` known to be not designed for small arrays, + and leading to memory leaks when used in loops. - x = Input((input_dim,), dtype=dtype) - nnet_0(x) - nnet_1(x) - nnet_1.set_weights([-p for p in nnet_0.get_weights()]) # be sure that the produced output will differ - h_0 = nnet_2(nnet_0(x)) - h_1 = nnet_2(nnet_1(x)) - y = merge_op(dtype=dtype)([h_0, h_1]) + See https://keras.io/api/models/model_training_apis/#predict-method and + https://github.com/tensorflow/tensorflow/issues/44711 - return Model(x, y) + Args: + model: + x: - @staticmethod - def toy_struct_cnn(dtype="float32", image_data_shape=(6, 6, 2)): - input_dim = int(np.prod(image_data_shape)) - layers = [ - Input((input_dim,)), - Reshape(target_shape=image_data_shape), - Conv2D( - 10, - kernel_size=(3, 3), - activation="relu", - data_format="channels_last", - dtype=dtype, - ), - Flatten(dtype=dtype), - Dense(1, dtype=dtype), - ] - return Sequential(layers) + Returns: - @staticmethod - def toy_model(model_name, dtype="float32"): - if model_name == "tutorial": - return Helpers.toy_network_tutorial(dtype=dtype) - elif model_name == "tutorial_activation_embedded": - return Helpers.toy_network_tutorial_with_embedded_activation(dtype=dtype) - elif model_name == "merge_v0": - return Helpers.toy_struct_v0_1D(dtype=dtype, input_dim=1, archi=[2, 3, 2], activation="relu", use_bias=True) - elif model_name == "merge_v1": - return Helpers.toy_struct_v1_1D( - dtype=dtype, input_dim=1, archi=[2, 3, 2], activation="relu", use_bias=True, sequential=False - ) - elif model_name == "merge_v1_seq": - return Helpers.toy_struct_v1_1D( - dtype=dtype, input_dim=1, archi=[2, 3, 2], activation="relu", use_bias=True, sequential=True - ) - elif model_name == "merge_v1_2": - return Helpers.toy_struct_v1_1D( - dtype=dtype, input_dim=2, archi=[2, 3, 2], activation="relu", use_bias=True, sequential=False - ) - elif model_name == "merge_v2": - return Helpers.toy_struct_v2_1D( - dtype=dtype, input_dim=1, archi=[2, 3, 2], activation="relu", use_bias=True, sequential=False - ) - elif model_name == "cnn": - return Helpers.toy_struct_cnn(dtype=dtype) - elif model_name == "embedded_model": - return Helpers.toy_embedded_sequential(dtype=dtype) + """ + output_tensors = model(x) + if isinstance(output_tensors, list): + return [K.convert_to_numpy(output) for output in output_tensors] else: - raise ValueError(f"model_name {model_name} unknown") + return K.convert_to_numpy(output_tensors) @pytest.fixture def helpers(): return Helpers - - -@pytest.fixture( - params=[ - "tutorial", - "tutorial_activation_embedded", - "merge_v0", - "merge_v1", - "merge_v1_seq", - "merge_v1_2", - "merge_v2", - "cnn", - "embedded_model", - ] -) -def toy_model(request, helpers): - model_name = request.param - return helpers.toy_model(model_name, dtype=keras_config.floatx()) - - -@pytest.fixture( - params=[ - "tutorial", - "tutorial_activation_embedded", - "merge_v0", - "merge_v1", - "merge_v1_seq", - "merge_v2", - "embedded_model", - ] -) -def toy_model_1d(request, helpers): - model_name = request.param - return helpers.toy_model(model_name, dtype=keras_config.floatx()) - - -@fixture() -def decomon_inputs_1d(n, mode, dc_decomp, helpers): - # tensors inputs - inputs = helpers.get_tensor_decomposition_1d_box(dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_1d_box(n=n, dc_decomp=dc_decomp) - inputs_for_mode_ = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_, mode=mode, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs_) - - # inputs metadata worth to pass to the layer - inputs_metadata = dict() - - return inputs, inputs_for_mode, input_ref, inputs_, inputs_for_mode_, input_ref_, inputs_metadata - - -@fixture() -def decomon_inputs_multid(odd, mode, dc_decomp, helpers): - # tensors inputs - inputs = helpers.get_tensor_decomposition_multid_box(odd=odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_multid_box(odd=odd, dc_decomp=dc_decomp) - inputs_for_mode_ = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_, mode=mode, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs_) - - # inputs metadata worth to pass to the layer - inputs_metadata = dict() - - return inputs, inputs_for_mode, input_ref, inputs_, inputs_for_mode_, input_ref_, inputs_metadata - - -@fixture() -def decomon_inputs_images(data_format, mode, dc_decomp, helpers): - odd, m_0, m_1 = 0, 0, 1 - - # tensors inputs - inputs = helpers.get_tensor_decomposition_images_box(data_format=data_format, odd=odd, dc_decomp=dc_decomp) - inputs_for_mode = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs, mode=mode, dc_decomp=dc_decomp) - input_ref = helpers.get_input_ref_from_full_inputs(inputs) - - # numpy inputs - inputs_ = helpers.get_standard_values_images_box(data_format=data_format, odd=odd, dc_decomp=dc_decomp) - inputs_for_mode_ = helpers.get_inputs_for_mode_from_full_inputs(inputs=inputs_, mode=mode, dc_decomp=dc_decomp) - input_ref_ = helpers.get_input_ref_from_full_inputs(inputs_) - - # inputs metadata worth to pass to the layer - inputs_metadata = dict(data_format=data_format) - - return inputs, inputs_for_mode, input_ref, inputs_, inputs_for_mode_, input_ref_, inputs_metadata - - -decomon_inputs = fixture_union( - "decomon_inputs", - [decomon_inputs_1d, decomon_inputs_multid, decomon_inputs_images], - unpack_into="inputs, inputs_for_mode, input_ref, inputs_, inputs_for_mode_, input_ref_, inputs_metadata", -) From f8620d5ff7af8a6610b7902f191ac158a83ace5e Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 1 Feb 2024 22:56:27 +0100 Subject: [PATCH 009/101] Add batch_multid_dot() operation on tensors This is a multid equivalent of batch_dot() on tensors. We perform a dot product on tensors by summing over a range of axes instead of a single axis. In the first tensor, we perform it on the last axes, starting from a given one. In the second tensor, we perform it on the first axes (after batch axis), ending to a given one. The option `missing_batchsize` is used to apply batch_multi_dot even when the batch axis is missing for one of the entry. (In that case, we use keras.ops.tensordot under the hood.) This can be useful when combining an affine bounds having a batch size with a layer affine representation without the batch dimension. By default, the number of axes to merge is the number of non-batch axes in the first arg, so that a linear operator represented by tensor `w` operates on an input tensor `x` as `batch_multi_dot(x, w, missing_batchsize=(False, True)` --- src/decomon/keras_utils.py | 72 +++++++++++++++++++++++++++- tests/test_keras_utils.py | 98 +++++++++++++++++++++++++++++++++++++- 2 files changed, 167 insertions(+), 3 deletions(-) diff --git a/src/decomon/keras_utils.py b/src/decomon/keras_utils.py index 681f359c..38942beb 100644 --- a/src/decomon/keras_utils.py +++ b/src/decomon/keras_utils.py @@ -1,9 +1,9 @@ -from typing import Any +from typing import Any, Optional import keras import keras.ops as K import numpy as np -from keras.layers import Layer +from keras.layers import Dot, Layer, Reshape from decomon.types import BackendTensor, Tensor @@ -13,6 +13,74 @@ BACKEND_JAX = "jax" +def batch_multid_dot( + x: Tensor, y: Tensor, nb_merging_axes: Optional[int] = None, missing_batchsize: tuple[bool, bool] = (False, False) +) -> Tensor: + """Dot product of tensors by batch, along multiple axes + + Hypothesis: we sum over last axes of x and first axes (skipping the batch one) of x. + + The 1-dimensional equivalent would be `batch_dot(x,y, axes=(-1, 1))` + or `keras.layers.Dot(axes=(-1, 1))(x,y)` + + Args: + x: + y: + nb_merging_axes: number of axes to be merged. + By default, all (non-batch) axes of x,i.e. + len(x.shape) if missing_batchsize[0] else len(x.shape) - 1 + missing_batchsize: specify if a tensor is missing the batch dimension, for x and y. + In that case, the corresponding tensor is broadcasted accordingly. + + Returns: + + For performance reasons, instead of actually repeating the tensor `batchsize` along a new first axis, + we rather use `keras.ops.tensordot` directly on tensors without broadcasting them. + + Note: + The dimensions of axes along which we perform the dot product + (i.e. x.shape[-nb_merging_axes:] and y.shape[1:1 + nb_merging_axes]) should match. + + """ + missing_batchsize_x, missing_batchsize_y = missing_batchsize + nb_batch_axe_x = 0 if missing_batchsize_x else 1 + nb_batch_axe_y = 0 if missing_batchsize_y else 1 + if nb_merging_axes is None: + nb_merging_axes = len(x.shape) - nb_batch_axe_x + + # check shapes compatibility + x_merging_axes_shape = x.shape[-nb_merging_axes:] + y_merging_axes_shape = y.shape[nb_batch_axe_y : nb_batch_axe_y + nb_merging_axes] + if x_merging_axes_shape != y_merging_axes_shape: + raise ValueError( + "Incompatible input shapes: " + f"Merging axes dimension should match. " + f"Found {x_merging_axes_shape} and {y_merging_axes_shape}. " + f"Full shapes: {x.shape} and {y.shape}." + ) + + # switch on missing batch axe (e.g. with affine layer representation like Dense's kernel) + if missing_batchsize_y: + return K.tensordot(x, y, axes=nb_merging_axes) + elif missing_batchsize_x: + # axes along which summing + merging_axes_x = list(range(-nb_merging_axes, 0)) + merging_axes_y = list(range(nb_batch_axe_y, nb_batch_axe_y + nb_merging_axes)) + # transposition to make to put back batch axe at the beginning + nb_axes_after_merge = len(x.shape) + len(y.shape) - 2 * nb_merging_axes + nb_axes_after_merge_from_x = len(x.shape) - nb_merging_axes + transpose_indices = ( + (nb_axes_after_merge_from_x,) + + tuple(range(nb_axes_after_merge_from_x)) + + tuple(range(nb_axes_after_merge_from_x + 1, nb_axes_after_merge)) + ) + return K.transpose(K.tensordot(x, y, axes=[merging_axes_x, merging_axes_y]), transpose_indices) + else: + new_x_shape = tuple(x.shape[1:-nb_merging_axes]) + (-1,) + new_y_shape = (-1,) + tuple(y.shape[nb_merging_axes + 1 :]) + return Dot(axes=(-1, 1))([Reshape(new_x_shape)(x), Reshape(new_y_shape)(y)]) + + class BatchedIdentityLike(keras.Operation): """Keras Operation creating an identity tensor with shape (including batch_size) based on input. diff --git a/tests/test_keras_utils.py b/tests/test_keras_utils.py index 6fa5aaf3..94cb969f 100644 --- a/tests/test_keras_utils.py +++ b/tests/test_keras_utils.py @@ -5,7 +5,103 @@ from keras.layers import Dense, Input from numpy.testing import assert_almost_equal -from decomon.keras_utils import get_weight_index_from_name, share_layer_all_weights +from decomon.keras_utils import ( + batch_multid_dot, + get_weight_index_from_name, + share_layer_all_weights, +) + + +def test_batch_multid_dot_symbolic_nok(): + with pytest.raises(ValueError): + batch_multid_dot(Input((2, 5, 1)), Input((4, 8, 2, 9)), nb_merging_axes=2) + + +def test_batch_multid_dot_symbolic_ok(): + output = batch_multid_dot(Input((2, 5, 1)), Input((5, 1, 8, 2, 9)), nb_merging_axes=2) + assert output.shape == (None, 2, 8, 2, 9) + + +def test_batch_multid_dot_nok(): + with pytest.raises(ValueError): + batch_multid_dot(K.ones((10, 2, 5, 1)), K.ones((1, 4, 8, 2, 9)), nb_merging_axes=2) + + +def test_batch_multid_dot_ok(): + x = K.repeat(K.reshape(K.eye(5), (1, 5, 5, 1)), 10, axis=0) + y = K.ones((10, 5, 1, 8, 2, 9)) + output = batch_multid_dot(x, y, nb_merging_axes=2) + expected_shape = (10, 5, 8, 2, 9) + assert output.shape == expected_shape + assert K.convert_to_numpy(output == K.ones(expected_shape)).all() + + +def test_batch_multid_dot_missing_y_batch_nok(): + x = K.repeat(K.reshape(K.eye(5), (1, 5, 5, 1)), 10, axis=0) + y = K.ones((10, 5, 1, 8, 2, 9)) + with pytest.raises(ValueError): + batch_multid_dot(x, y, nb_merging_axes=2, missing_batchsize=(False, True)) + + +def test_batch_multid_dot_missing_y_batch_ok(helpers): + batchsize = 10 + x_shape_wo_batch = (4, 5, 2) + y_shape_wo_batch = (5, 2, 3, 7) + nb_merging_axes = 2 + expected_res_shape_wo_batchsize = (4, 3, 7) + x_shape = (batchsize,) + x_shape_wo_batch + + x = K.convert_to_tensor(np.random.random(x_shape)) + y_wo_batch = K.convert_to_tensor(np.random.random(y_shape_wo_batch)) + y_w_batch = K.repeat(y_wo_batch[None], batchsize, axis=0) + + res = batch_multid_dot(x, y_wo_batch, nb_merging_axes=nb_merging_axes, missing_batchsize=(False, True)) + assert tuple(res.shape)[1:] == expected_res_shape_wo_batchsize + res_w_all_batch = batch_multid_dot(x, y_w_batch, nb_merging_axes=nb_merging_axes) + helpers.assert_almost_equal(res, res_w_all_batch) + + +def test_batch_multid_dot_missing_x_batch_ok(helpers): + batchsize = 10 + x_shape_wo_batch = (4, 5, 2) + y_shape_wo_batch = (5, 2, 3, 7) + nb_merging_axes = 2 + expected_res_shape_wo_batchsize = (4, 3, 7) + y_shape = (batchsize,) + y_shape_wo_batch + + y = K.convert_to_tensor(np.random.random(y_shape)) + x_wo_batch = K.convert_to_tensor(np.random.random(x_shape_wo_batch)) + x_w_batch = K.repeat(x_wo_batch[None], batchsize, axis=0) + + res = batch_multid_dot(x_wo_batch, y, nb_merging_axes=nb_merging_axes, missing_batchsize=(True, False)) + assert tuple(res.shape)[1:] == expected_res_shape_wo_batchsize + res_w_all_batch = batch_multid_dot(x_w_batch, y, nb_merging_axes=nb_merging_axes) + helpers.assert_almost_equal(res, res_w_all_batch) + + +@pytest.mark.parametrize("missing_batchsize", [(False, False), (True, False), (False, True)]) +def test_batch_multid_dot_default_nb_merging_axes(missing_batchsize, helpers): + batchsize = 10 + x_shape_wo_batch = (4, 5, 2) + y_shape_wo_batch = (4, 5, 2, 3, 7) + nb_merging_axes = len(x_shape_wo_batch) + expected_res_shape_wo_batchsize = (3, 7) + x_shape = (batchsize,) + x_shape_wo_batch + y_shape = (batchsize,) + y_shape_wo_batch + + x_missing_batchsize, y_missing_batchsize = missing_batchsize + if x_missing_batchsize: + x = K.convert_to_tensor(np.random.random(x_shape_wo_batch)) + else: + x = K.convert_to_tensor(np.random.random(x_shape)) + if y_missing_batchsize: + y = K.convert_to_tensor(np.random.random(y_shape_wo_batch)) + else: + y = K.convert_to_tensor(np.random.random(y_shape)) + + res_default = batch_multid_dot(x, y, missing_batchsize=missing_batchsize) + res = batch_multid_dot(x, y, nb_merging_axes=nb_merging_axes, missing_batchsize=missing_batchsize) + helpers.assert_almost_equal(res, res_default) def test_get_weight_index_from_name_nok_attribute(): From 34ecc3c0507860e5b9ff28c948c51220666a3b42 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 2 Feb 2024 14:26:32 +0100 Subject: [PATCH 010/101] Move is_merge_layer in keras_utils --- src/decomon/keras_utils.py | 4 ++++ src/decomon/models/utils.py | 7 +++++-- tests/test_keras_utils.py | 23 ++++++++++++++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/decomon/keras_utils.py b/src/decomon/keras_utils.py index 38942beb..c1c4e98d 100644 --- a/src/decomon/keras_utils.py +++ b/src/decomon/keras_utils.py @@ -131,6 +131,10 @@ def compute_output_spec(self, x: Tensor) -> keras.KerasTensor: ) +def is_a_merge_layer(layer: Layer) -> bool: + return hasattr(layer, "_merge_function") + + def is_symbolic_tensor(x: Tensor) -> bool: """Check whether the tensor is symbolic or not. diff --git a/src/decomon/models/utils.py b/src/decomon/models/utils.py index f0ecd6aa..60f5bde7 100644 --- a/src/decomon/models/utils.py +++ b/src/decomon/models/utils.py @@ -26,8 +26,11 @@ PerturbationDomain, get_mode, ) -from decomon.keras_utils import BatchedIdentityLike, share_weights_and_build -from decomon.layers.utils import is_a_merge_layer +from decomon.keras_utils import ( + BatchedIdentityLike, + is_a_merge_layer, + share_weights_and_build, +) from decomon.types import BackendTensor diff --git a/tests/test_keras_utils.py b/tests/test_keras_utils.py index 94cb969f..6a6839e7 100644 --- a/tests/test_keras_utils.py +++ b/tests/test_keras_utils.py @@ -2,16 +2,37 @@ import keras.ops as K import numpy as np import pytest -from keras.layers import Dense, Input +from keras.layers import Dense, Input, Layer from numpy.testing import assert_almost_equal from decomon.keras_utils import ( batch_multid_dot, get_weight_index_from_name, + is_a_merge_layer, share_layer_all_weights, ) +class MyLayer(Layer): + """Mock layer unknown from decomon.""" + + ... + + +class MyMerge(Layer): + """Mock merge layer unknown from decomon.""" + + def _merge_function(self, inputs): + return inputs + + +def test_is_merge_layer(): + layer = MyMerge() + assert is_a_merge_layer(layer) + layer = MyLayer() + assert not is_a_merge_layer(layer) + + def test_batch_multid_dot_symbolic_nok(): with pytest.raises(ValueError): batch_multid_dot(Input((2, 5, 1)), Input((4, 8, 2, 9)), nb_merging_axes=2) From e23a6ba2fa5a24ec32fc3b2e1b2f3f07c66464ca Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 1 Feb 2024 23:03:04 +0100 Subject: [PATCH 011/101] Create new DecomonLayer base class to be used for bounds propagation The same layer class is now used for forward and backward propagation. The goal is to simplified the implementation of the decomon version of a custom layer for the user wanting to use decomon algorithms on a model involving such a custom layer. When implementing a new decomon layer it is sufficient to implement - get_affine_bounds(): returns affine relaxation of the keras layer - get_ibp_bounds(): returns constant relaxation of the keras layer In the case of a linear keras layer, one can set the class attribute `linear` to True and then implement get_affine_representation() instead of get_affine_bounds(). Some computations will be simplified in this case. In particular, no oracle bounds are needed to propagate affine bounds. One can also, for performance reasons, override directly forward_affine_propagate() and backward_affine_propagate() instead of implementing get_affine_bounds(). The main attributes of the decomon layers are - layer: the keras underlying layer - perturbation_domain: on which we propagate bounds - ibp: do we propagate constant bounds (forward only) - affine: do we propagate affine bounds (meaningless for backward) - propagation: forward or backward, the direction of bounds propagation ibp and affine booleans replace the previous modes as they seem to be the relevant variables when testing inside decomon layer methods; --- src/decomon/core.py | 7 + src/decomon/layers/layer.py | 674 ++++++++++++++++++++++++++++++++++++ 2 files changed, 681 insertions(+) diff --git a/src/decomon/core.py b/src/decomon/core.py index 74790f4a..41e3574d 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -125,6 +125,13 @@ class ForwardMode(str, Enum): """Propagation of constant and affines bounds from input to output.""" +class Propagation(str, Enum): + """Propagation direction.""" + + FORWARD = "forward" + BACKWARD = "backward" + + def get_mode(ibp: bool = True, affine: bool = True) -> ForwardMode: if ibp: if affine: diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index e69de29b..4bbf7bbc 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -0,0 +1,674 @@ +from inspect import Parameter, signature +from typing import Any, Optional, Union + +import keras +import keras.ops as K +from keras.layers import Layer, Wrapper + +from decomon.core import BoxDomain, PerturbationDomain, Propagation +from decomon.keras_utils import batch_multid_dot +from decomon.types import Tensor + +_keras_base_layer_keyword_parameters = [ + name for name, param in signature(Layer.__init__).parameters.items() if param.kind == Parameter.KEYWORD_ONLY +] + ["input_shape", "input_dim"] + + +class DecomonLayer(Wrapper): + """Base class for decomon layers. + + To enable LiRPA on a custom layer, one should implement a corresponding decomon layer by: + - deriving from DecomonLayer + - override/implement at least: + - linear case: + - set class attribute `linear` to True + - `get_affine_representation()` + - generic case: + - `get_affine_bounds()`: affine bounds on layer output w.r.t. layer input + - `forward_ibp_propagate()`: ibp bounds on layer ouput knowing ibp bounds on layer inpu + + Other possibilities exist like overriding directly + - `forward_affine_propagate()` + - `backward_affine_propagate()` + - `forward_ibp_propagate()` (still needed) + + """ + + linear: bool = False + """Flag telling that the layer is linear. + + Set it to True in child classes to explicit that the corresponding keras layer is linear. + Else will be considered as non-linear. + + When linear is set to True, some computations can be simplified, + even though equivalent with the ones made if linear is set to False. + + """ + + def __init__( + self, + layer: Layer, + perturbation_domain: Optional[PerturbationDomain] = None, + ibp: bool = True, + affine: bool = True, + propagation: Propagation = Propagation.FORWARD, + **kwargs: Any, + ): + """ + Args: + layer: underlying keras layer + perturbation_domain: default to a box domain + ibp: if True, forward propagate constant bounds + affine: if True, forward propagate affine bounds + propagation: direction of bounds propagation + - forward: from input to output + - backward: from output to input + **kwargs: + + """ + # Layer init: + # all options not expected by Layer are removed here + # (`to_decomon` pass options potentially not used by all decomon layers) + keys_to_pop = [k for k in kwargs if k not in _keras_base_layer_keyword_parameters] + for k in keys_to_pop: + kwargs.pop(k) + super().__init__(layer=layer, **kwargs) + + # default args + if perturbation_domain is None: + perturbation_domain = BoxDomain() + + # checks + if not layer.built: + raise ValueError(f"The underlying keras layer {layer.name} is not built.") + if not ibp and not affine: + raise ValueError("ibp and affine cannot be both False.") + + # attributes + self.ibp = ibp + self.affine = affine + self.perturbation_domain = perturbation_domain + self.propagation = propagation + + def get_config(self) -> dict[str, Any]: + config = super().get_config() + config.update( + { + "ibp": self.ibp, + "affine": self.affine, + "perturbation_domain": self.perturbation_domain, + "propagation": self.propagation, + } + ) + return config + + def get_affine_representation(self) -> tuple[Tensor, Tensor]: + """Get affine representation of the layer + + This computes the affine representation of the layer, when this is meaningful, + i.e. `self.linear` is True. + + If implemented, it will be used for backward and forward propagation of affine bounds through the layer. + For non-linear layers, one should implement `get_affine_bounds()` instead. + + Args: + + Returns: + w, b: affine representation of the layer satisfying + + layer(z) = w * z + b + + More precisely, we have + ``` + layer(z) = batch_multid_dot(z, w, missing_batchsize=(False, True)) + b + ``` + + Shapes: !no batchsize! + w ~ self.layer.input.shape[1:] + self.layer.output.shape[1:] + b ~ self.layer.output.shape[1:] + + """ + if not self.linear: + raise RuntimeError("You should not call `get_affine_representation()` when `self.linear` is False.") + else: + raise NotImplementedError( + "`get_affine_representation()` needs to be implemented to get the forward and backward propagation of affine bounds. " + "Alternatively, you can also directly override " + "`forward_ibp_propagate()`, `forward_affine_propagate()` and `backward_affine_propagate()`." + ) + + def get_affine_bounds(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Get affine bounds on layer outputs from layer inputs + + This compute the affine relaxation of the layer, given the oracle constant bounds on the inputs. + + If implemented, it will be used for backward and forward propagation of affine bounds through the layer. + For linear layers, one can implement `get_affine_representation()` instead. + + Args: + lower: lower constant oracle bound on the layer input. + upper: upper constant oracle bound on the layer input. + + Returns: + w_l, b_l, w_u, b_u: affine relaxation of the layer satisfying + + w_l * z + b_l <= layer(z) <= w_u * z + b_u + with lower <= z <= upper + + Shapes: + lower, upper ~ (batchsize,) + self.layer.input.shape[1:] + w_l, w_u ~ (batchsize,) + self.layer.input.shape[1:] + self.layer.output.shape[1:] + b_l, b_u ~ (batchsize,) + self.layer.output.shape[1:] + + Note: + `w * z` means here `batch_multid_dot(z, w)`. + + """ + if self.linear: + w, b = self.get_affine_representation() + batchsize = lower.shape[0] + w_with_batchsize = K.repeat(w[None], batchsize, axis=0) + b_with_batchsize = K.repeat(b[None], batchsize, axis=0) + return w_with_batchsize, b_with_batchsize, w_with_batchsize, b_with_batchsize + else: + raise NotImplementedError( + "`get_affine_bounds()` needs to be implemented to get the forward and backward propagation of affine bounds. " + "Alternatively, you can also directly override `forward_affine_propagate()` and `backward_affine_propagate()`" + ) + + def forward_ibp_propagate(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, Tensor]: + """Propagate ibp bounds through the layer. + + If the underlying keras layer is linear, it will be deduced from its affine representation. + Else, this needs to be implemented to forward propagate ibp (constant) bounds. + + Args: + lower: lower constant oracle bound on the keras layer input. + upper: upper constant oracle bound on the keras layer input. + + Returns: + l_c, u_c: constant relaxation of the layer satisfying + l_c <= layer(z) <= u_c + with lower <= z <= upper + + Shapes: + lower, upper ~ (batchsize,) + self.layer.input.shape[1:] + l_c, u_c ~ (batchsize,) + self.layer.output.shape[1:] + + """ + if self.linear: + w, b = self.get_affine_representation() + + z_value = K.cast(0.0, dtype=w.dtype) + w_pos = K.maximum(w, z_value) + w_neg = K.minimum(w, z_value) + + l_c = ( + batch_multid_dot(lower, w_pos, missing_batchsize=(False, True)) + + batch_multid_dot(upper, w_neg, missing_batchsize=(False, True)) + + b + ) + u_c = ( + batch_multid_dot(upper, w_pos, missing_batchsize=(False, True)) + + batch_multid_dot(lower, w_neg, missing_batchsize=(False, True)) + + b + ) + + return l_c, u_c + else: + raise NotImplementedError( + "`forward_ibp_propagate()` needs to be implemented to get the forward propagation of constant bounds." + ) + + def forward_affine_propagate( + self, input_affine_bounds, input_constant_bounds + ) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Propagate model affine bounds in forward direction. + + By default, this is deduced from `get_affine_bounds()` (or `get_affine_representation()` if `self.linear is True). + But this could be overridden for better performance. See `DecomonConv2D` for an example. + + Args: + input_affine_bounds: [w_l_in, b_l_in, w_u_in, b_u_in] + affine bounds on underlying keras layer input w.r.t. model input + input_constant_bounds: [l_c_in, u_c_in] + constant oracle bounds on underlying keras layer input (already deduced from affine ones if necessary) + + Returns: + w_l, b_l, w_u, b_u: affine bounds on underlying keras layer *output* w.r.t. model input + + If we denote by + - x: keras model input + - z: underlying keras layer input + - h(x) = layer(z): output of the underlying keras layer + + The following inequations are satisfied + w_l * x + b_l <= h(x) <= w_u * x + b_u + l_c_in <= z <= u_c_in + w_l_in * x + b_l_in <= z <= w_u_in * x + b_u_in + + """ + if self.linear: + w, b = self.get_affine_representation() + layer_affine_bounds = [w, b, w, b] + else: + lower, upper = input_constant_bounds + w_l, b_l, w_u, b_u = self.get_affine_bounds(lower=lower, upper=upper) + layer_affine_bounds = [w_l, b_l, w_u, b_u] + + return combine_affine_bounds( + affine_bounds_1=input_affine_bounds, + affine_bounds_2=layer_affine_bounds, + from_linear_layer=(False, self.linear), + ) + + def backward_affine_propagate( + self, output_affine_bounds, input_constant_bounds + ) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Propagate model affine bounds in backward direction. + + By default, this is deduced from `get_affine_bounds()` (or `get_affine_representation()` if `self.linear is True). + But this could be overridden for better performance. See `DecomonConv2D` for an example. + + Args: + output_affine_bounds: [w_l, b_l, w_u, b_u] + partial affine bounds on model output w.r.t underlying keras layer output + input_constant_bounds: [l_c_in, u_c_in] + constant oracle bounds on underlying keras layer input + + Returns: + w_l_new, b_l_new, w_u_new, b_u_new: partial affine bounds on model output w.r.t. underlying keras layer *input* + + If we denote by + - x: keras model input + - m(x): keras model output + - z: underlying keras layer input + - h(x) = layer(z): output of the underlying keras layer + - h_i(x) output of the i-th layer + - w_l_i, b_l_i, w_u_i, b_u_i: current partial linear bounds on model output w.r.t to h_i(x) + + The following inequations are satisfied + + l_c_in <= z <= u_c_in + + Sum_{others layers i}(w_l_i * h_i(x) + b_l_i) + w_l * h(x) + b_l + <= m(x) + <= Sum_{others layers i}(w_u_i * h_i(x) + b_u_i) + w_u * h(x) + b_u + + Sum_{others layers i}(w_l_i * h_i(x) + b_l_i) + w_l_new * z + b_l_new + <= m(x) + <= Sum_{others layers i}(w_u_i * h_i(x) + b_u_i) + w_u_new * z + b_u_new + + """ + if self.linear: + w, b = self.get_affine_representation() + layer_affine_bounds = [w, b, w, b] + else: + lower, upper = input_constant_bounds + w_l, b_l, w_u, b_u = self.get_affine_bounds(lower=lower, upper=upper) + layer_affine_bounds = [w_l, b_l, w_u, b_u] + + return combine_affine_bounds( + affine_bounds_1=layer_affine_bounds, + affine_bounds_2=output_affine_bounds, + from_linear_layer=(self.linear, False), + ) + + def get_forward_oracle( + self, input_affine_bounds: list[Tensor], input_constant_bounds: list[Tensor], x: Tensor + ) -> list[Tensor]: + """Get constant oracle bounds on underlying keras layer input from forward input bounds. + + Args: + input_affine_bounds: affine bounds on keras layer input w.r.t model input . Can be empty if not in affine mode. + input_constant_bounds: ibp constant bounds on keras layer input. Can be empty if not in ibp mode. + x: model input. Necessary only in affine mode. + + Returns: + constant bounds on keras layer input deduced from forward input bounds + + `input_affine_bounds, input_constant_bounds` are the forward bounds to be propagate through the layer. + `input_affine_bounds` (resp. `input_constant_bounds`) will be empty if `self.affine` (resp. `self.ibp) is False. + + In hybrid case (ibp+affine), the constant bounds are assumed to be already tight, which means the previous + forward layer should already have took the tighter constant bounds between the ibp ones and the ones deduced + from the affine bounds given the considered perturbation domain. + + """ + if self.ibp: + # Hyp: in hybrid mode, the constant bounds are already tight + # (affine and ibp mixed in forward layer output to get the tightest constant bounds) + return input_constant_bounds + + elif self.affine: + w_l, b_l, w_u, b_u = input_affine_bounds + l_affine = self.perturbation_domain.get_lower(x, w_l, b_l) + u_affine = self.perturbation_domain.get_upper(x, w_u, b_u) + return [l_affine, u_affine] + + else: + raise RuntimeError("self.ibp and self.affine cannot be both False") + + def call_forward( + self, affine_bounds_to_propagate: list[Tensor], input_bounds_to_propagate: list[Tensor], x: Tensor + ) -> tuple[list[Tensor], list[Tensor]]: + """Propagate forward affine and constant bounds through the layer. + + Args: + affine_bounds_to_propagate: affine bounds on keras layer input w.r.t model input . Can be empty if not in affine mode. + input_bounds_to_propagate: ibp constant bounds on keras layer input. Can be empty if not in ibp mode. + x: model input. Necessary only in affine mode. + + Returns: + output_affine_bounds, output_constant_bounds: affine and constant bounds on the underlying keras layer output + + Note: + In hybrid case (ibp+affine), the constant bounds are assumed to be already tight in input, and we will return + the tighter constant bounds in output. This means that + - for the output: we take the tighter constant bounds between the ibp ones and the ones deduced + from the affine bounds given the considered perturbation domain, on the output. + - for the input: we do not need it, as it should already have been taken care of in the previous layer + + """ + # IBP: interval bounds propragation + if self.ibp: + lower, upper = input_bounds_to_propagate + output_constant_bounds = list(self.forward_ibp_propagate(lower=lower, upper=upper)) + else: + output_constant_bounds = [] + + # Affine bounds propagation + if self.affine: + if not self.linear: + # get oracle input bounds (because input_bounds_to_propagate could be empty at this point) + input_constant_bounds = self.get_forward_oracle( + input_affine_bounds=affine_bounds_to_propagate, input_constant_bounds=input_bounds_to_propagate, x=x + ) + else: + input_constant_bounds = [] + # forward propagation + output_affine_bounds = list( + self.forward_affine_propagate( + input_affine_bounds=affine_bounds_to_propagate, input_constant_bounds=input_constant_bounds + ) + ) + else: + output_affine_bounds = [] + + # Tighten constant bounds in hybrid mode (ibp+affine) + if self.ibp and self.affine: + l_ibp, u_ibp = output_constant_bounds + w_l, b_l, w_u, b_u = output_affine_bounds + l_affine = self.perturbation_domain.get_lower(x, w_l, b_l) + u_affine = self.perturbation_domain.get_upper(x, w_u, b_u) + u = K.minimum(u_ibp, u_affine) + l = K.maximum(l_ibp, l_affine) + output_constant_bounds = [l, u] + + return output_affine_bounds, output_constant_bounds + + def call_backward( + self, affine_bounds_to_propagate: list[Tensor], constant_oracle_bounds: list[Tensor] + ) -> list[Tensor]: + return list( + self.backward_affine_propagate( + output_affine_bounds=affine_bounds_to_propagate, input_constant_bounds=constant_oracle_bounds + ) + ) + + def call( + self, affine_bounds_to_propagate: list[Tensor], constant_oracle_bounds: list[Tensor], x: Tensor + ) -> list[list[Tensor]]: + """Propagate bounds in the specified direction `self.propagation`. + + Args: + affine_bounds_to_propagate: affine bounds to propagate. Can be empty in forward direction if self.affine is False. + constant_oracle_bounds: in forward direction, the ibp bounds (empty if self.ibp is False); in backward direction, the oracle constant bounds on keras inputs + x: the model input. Necessary only in forward direction when self.affine is True. + + Returns: + the propagated bounds. + forward: [affine_bounds_propagated, constant_bounds_propagated], each one being empty if self.affine or self.ibp is False + backward: [affine_bounds_propagated] + + + """ + if self.propagation == Propagation.FORWARD: # forward + affine_bounds_propagated, constant_bounds_propagated = self.call_forward( + affine_bounds_to_propagate=affine_bounds_to_propagate, + input_bounds_to_propagate=constant_oracle_bounds, + x=x, + ) + return [affine_bounds_propagated, constant_bounds_propagated] + + else: # backward + affine_bounds_propagated = self.call_backward( + affine_bounds_to_propagate=affine_bounds_to_propagate, constant_oracle_bounds=constant_oracle_bounds + ) + return [affine_bounds_propagated] + + def build(self, affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, x_shape): + self.built = True + + def compute_output_shape( + self, + affine_bounds_to_propagate_shape: list[tuple[Optional[int], ...]], + constant_oracle_bounds_shape: list[tuple[Optional[int], ...]], + x_shape: tuple[Optional[int], ...], + ): + if self.propagation == Propagation.FORWARD: + if self.ibp: + constant_bounds_propagated_shape = [self.layer.output.shape] * 2 + else: + constant_bounds_propagated_shape = [] + if self.affine: + keras_layer_input_shape_wo_batchsize = self.layer.input.shape[1:] + keras_layer_output_shape_wo_batchsize = self.layer.output.shape[1:] + w_in_shape = affine_bounds_to_propagate_shape[0] + model_input_shape = w_in_shape[: -len(keras_layer_input_shape_wo_batchsize)] + w_out_shape = model_input_shape + keras_layer_output_shape_wo_batchsize + b_out_shape = self.layer.output.shape + affine_bounds_propagated_shape = [w_out_shape, b_out_shape, w_out_shape, b_out_shape] + else: + affine_bounds_propagated_shape = [] + + return [affine_bounds_propagated_shape, constant_bounds_propagated_shape] + + else: # backward + b_shape = affine_bounds_to_propagate_shape[1] + model_output_shape_wo_batchsize = b_shape[1:] + w_shape = self.layer.input.shape + model_output_shape_wo_batchsize + affine_bounds_propagated_shape = [w_shape, b_shape, w_shape, b_shape] + + return [affine_bounds_propagated_shape] + + def compute_output_spec(self, *args: Any, **kwargs: Any) -> list[list[keras.KerasTensor]]: + """Compute output spec from output shape in case of symbolic call.""" + output_spec = Layer.compute_output_spec(self, *args, **kwargs) + + # fix empty list: Layer.compute_output_spec() transform them as empty tensors + def replace_empty_tensor(l: Union[keras.KerasTensor, list[keras.KerasTensor]]): + if isinstance(l, keras.KerasTensor) and len(l.shape) == 0: + return [] + else: + return l + + return [replace_empty_tensor(l) for l in output_spec] + + +def combine_affine_bounds( + affine_bounds_1: list[Tensor], + affine_bounds_2: list[Tensor], + from_linear_layer: tuple[bool, bool] = (False, False), +) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Combine affine bounds + + Args: + affine_bounds_1: [w_l_1, b_l_1, w_u_1, b_u_1] first affine bounds + affine_bounds_2: [w_l_2, b_l_2, w_u_2, b_u_2] second affine bounds + from_linear_layer: specify if affine_bounds_1 or affine_bounds_2 + come from the affine representation of a linear layer + + Returns: + w_l, b_l, w_u, b_u: combined affine bounds + + If x, y, z satisfy + w_l_1 * x + b_l_1 <= y <= w_u_1 * x + b_u_1 + w_l_2 * y + b_l_2 <= z <= w_u_2 * y + b_u_2 + + Then + w_l * x + b_l <= z <= w_u * x + b_u + + + Special case with linear layers: + + If the affine bounds come from the affine representation of a linear layer (e.g. affine_bounds_1), then + - lower and upper bounds are equal: affine_bounds_1 = [w_1, b_1, w_1, b_1] + - the tensors are missing the batch dimension + + In the generic case, tensors in affine_bounds have their first axis corresponding to the batch size. + + """ + if from_linear_layer == (False, False): + return _combine_affine_bounds_generic(affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2) + elif from_linear_layer == (True, False): + return _combine_affine_bounds_left_from_linear(affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2) + elif from_linear_layer == (False, True): + return _combine_affine_bounds_right_from_linear( + affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2 + ) + else: + raise NotImplementedError() + + +def _combine_affine_bounds_generic( + affine_bounds_1: list[Tensor], + affine_bounds_2: list[Tensor], +) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Combine affine bounds + + Args: + affine_bounds_1: [w_l_1, b_l_1, w_u_1, b_u_1] first affine bounds + affine_bounds_2: [w_l_2, b_l_2, w_u_2, b_u_2] second affine bounds + + Returns: + w_l, b_l, w_u, b_u: combined affine bounds + + If x, y, z satisfy + w_l_1 * x + b_l_1 <= y <= w_u_1 * x + b_u_1 + w_l_2 * y + b_l_2 <= z <= w_u_2 * x + b_u_2 + + Then + w_l * x + b_l <= z <= w_u * x + b_u + + """ + w_l_1, b_l_1, w_u_1, b_u_1 = affine_bounds_1 + w_l_2, b_l_2, w_u_2, b_u_2 = affine_bounds_2 + nb_axes_wo_batchsize_y = len(b_l_1.shape) - 1 + + z_value = K.cast(0.0, dtype=w_l_2.dtype) + w_l_2_pos = K.maximum(w_l_2, z_value) + w_u_2_pos = K.maximum(w_u_2, z_value) + w_l_2_neg = K.minimum(w_l_2, z_value) + w_u_2_neg = K.minimum(w_u_2, z_value) + + w_l = batch_multid_dot(w_l_1, w_l_2_pos, nb_merging_axes=nb_axes_wo_batchsize_y) + batch_multid_dot( + w_u_1, w_l_2_neg, nb_merging_axes=nb_axes_wo_batchsize_y + ) + w_u = batch_multid_dot(w_u_1, w_u_2_pos, nb_merging_axes=nb_axes_wo_batchsize_y) + batch_multid_dot( + w_l_1, w_u_2_neg, nb_merging_axes=nb_axes_wo_batchsize_y + ) + b_l = ( + batch_multid_dot(b_l_1, w_l_2_pos, nb_merging_axes=nb_axes_wo_batchsize_y) + + batch_multid_dot(b_u_1, w_l_2_neg, nb_merging_axes=nb_axes_wo_batchsize_y) + + b_l_2 + ) + b_u = ( + batch_multid_dot(b_u_1, w_u_2_pos, nb_merging_axes=nb_axes_wo_batchsize_y) + + batch_multid_dot(b_l_1, w_u_2_neg, nb_merging_axes=nb_axes_wo_batchsize_y) + + b_u_2 + ) + + return w_l, b_l, w_u, b_u + + +def _combine_affine_bounds_right_from_linear( + affine_bounds_1: list[Tensor], + affine_bounds_2: list[Tensor], +) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Combine affine bounds + + Args: + affine_bounds_1: [w_l_1, b_l_1, w_u_1, b_u_1] first affine bounds + affine_bounds_2: [w_2, b_2, w_2, b_2] second affine bounds, with lower=upper + no batchsize + + Returns: + w_l, b_l, w_u, b_u: combined affine bounds + + If x, y, z satisfy + w_l_1 * x + b_l_1 <= y <= w_u_1 * x + b_u_1 + z = w_2 * y + b_2 + + Then + w_l * x + b_l <= z <= w_u * x + b_u + + """ + w_l_1, b_l_1, w_u_1, b_u_1 = affine_bounds_1 + w_2, b_2 = affine_bounds_2[:2] + nb_axes_wo_batchsize_y = len(b_l_1.shape) - 1 + missing_batchsize = (False, True) + kwargs_batch_dot = dict(nb_merging_axes=nb_axes_wo_batchsize_y, missing_batchsize=missing_batchsize) + + z_value = K.cast(0.0, dtype=w_2.dtype) + w_2_pos = K.maximum(w_2, z_value) + w_2_neg = K.minimum(w_2, z_value) + + w_l = batch_multid_dot(w_l_1, w_2_pos, **kwargs_batch_dot) + batch_multid_dot(w_u_1, w_2_neg, **kwargs_batch_dot) + w_u = batch_multid_dot(w_u_1, w_2_pos, **kwargs_batch_dot) + batch_multid_dot(w_l_1, w_2_neg, **kwargs_batch_dot) + b_l = ( + batch_multid_dot(b_l_1, w_2_pos, **kwargs_batch_dot) + + batch_multid_dot(b_u_1, w_2_neg, **kwargs_batch_dot) + + b_2 + ) + b_u = ( + batch_multid_dot(b_u_1, w_2_pos, **kwargs_batch_dot) + + batch_multid_dot(b_l_1, w_2_neg, **kwargs_batch_dot) + + b_2 + ) + + return w_l, b_l, w_u, b_u + + +def _combine_affine_bounds_left_from_linear( + affine_bounds_1: list[Tensor], + affine_bounds_2: list[Tensor], +) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Combine affine bounds + + Args: + affine_bounds_1: [w_1, b_1, w_1, b_1] first affine bounds, with lower=upper + no batchsize + affine_bounds_2: [w_l_2, b_l_2, w_u_2, b_u_2] second affine bounds + + Returns: + w_l, b_l, w_u, b_u: combined affine bounds + + If x, y, z satisfy + y = w_1 * x + b_1 + w_l_2 * y + b_l_2 <= z <= w_u_2 * x + b_u_2 + + Then + w_l * x + b_l <= z <= w_u * x + b_u + + """ + w_1, b_1 = affine_bounds_1[:2] + w_l_2, b_l_2, w_u_2, b_u_2 = affine_bounds_2 + nb_axes_wo_batchsize_y = len(b_1.shape) + missing_batchsize = (True, False) + kwargs_batch_dot = dict(nb_merging_axes=nb_axes_wo_batchsize_y, missing_batchsize=missing_batchsize) + + w_l = batch_multid_dot(w_1, w_l_2, **kwargs_batch_dot) + w_u = batch_multid_dot(w_1, w_u_2, **kwargs_batch_dot) + b_l = batch_multid_dot(b_1, w_l_2, **kwargs_batch_dot) + b_l_2 + b_u = batch_multid_dot(b_1, w_u_2, **kwargs_batch_dot) + b_u_2 + + return w_l, b_l, w_u, b_u From a17131612e0b6e4ab644e2917cdda73310fc2086 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Sat, 3 Feb 2024 00:31:59 +0100 Subject: [PATCH 012/101] Test decomon layer base class --- tests/test_decomon_layer.py | 195 ++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 tests/test_decomon_layer.py diff --git a/tests/test_decomon_layer.py b/tests/test_decomon_layer.py new file mode 100644 index 00000000..6a2409c1 --- /dev/null +++ b/tests/test_decomon_layer.py @@ -0,0 +1,195 @@ +import keras.ops as K +import numpy as np +import pytest +from keras.layers import Dense, Input + +from decomon.core import BoxDomain, Propagation +from decomon.keras_utils import batch_multid_dot +from decomon.layers.layer import DecomonLayer + + +def test_decomon_layer_nok_unbuilt_keras_layer(): + layer = Dense(3) + with pytest.raises(ValueError): + DecomonLayer(layer=layer) + + +def test_decomon_layer_nok_ibp_affine(): + layer = Dense(3) + layer(Input((1,))) + with pytest.raises(ValueError): + DecomonLayer(layer=layer, ibp=False, affine=False) + + +def test_decomon_layer_extra_kwargs(): + layer = Dense(3) + layer(Input((1,))) + DecomonLayer(layer=layer, alpha="jet") + + +class MyLinearDecomonDense1d(DecomonLayer): + linear = True + layer: Dense + + def get_affine_representation(self): + return self.layer.kernel, self.layer.bias + + +class MyNonLinearDecomonDense1d(DecomonLayer): + linear = False + layer: Dense + + def get_affine_bounds(self, lower, upper): + batchsize = lower.shape[0] + extended_kernel = K.repeat(self.layer.kernel[None], batchsize, axis=0) + extended_bias = K.repeat(self.layer.bias[None], batchsize, axis=0) + return extended_kernel, extended_bias, extended_kernel, extended_bias + + def forward_ibp_propagate(self, lower, upper): + z_value = K.cast(0.0, dtype=lower.dtype) + kernel_pos = K.maximum(z_value, self.layer.kernel) + kernel_neg = K.minimum(z_value, self.layer.kernel) + u_c = K.dot(upper, kernel_pos) + K.dot(lower, kernel_neg) + l_c = K.dot(lower, kernel_pos) + K.dot(upper, kernel_neg) + return l_c, u_c + + +@pytest.mark.parametrize("singlelayer_model", [False, True]) +def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helpers): + # input/output shapes + batchsize = 10 + layer_input_dim = 2 + layer_output_dim = 7 + perturbation_domain = BoxDomain() + model_output_shape_if_no_singlelayer_model = (3, 5) + model_input_dim_if_no_singlelayer_model = 9 + + layer_input_shape = (layer_input_dim,) + layer_output_shape = (layer_output_dim,) + + if singlelayer_model: + model_input_dim = layer_input_dim + model_output_shape = layer_output_shape + else: + model_input_dim = model_input_dim_if_no_singlelayer_model + model_output_shape = model_output_shape_if_no_singlelayer_model + + model_input_shape = (model_input_dim,) + + x_shape = perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape) + + # keras layer + layer = Dense(units=layer_output_dim) + layer(Input((layer_input_dim,))) + + # decomon layers + linear_decomon_layer = MyLinearDecomonDense1d( + layer=layer, ibp=ibp, affine=affine, propagation=propagation, perturbation_domain=perturbation_domain + ) + non_linear_decomon_layer = MyNonLinearDecomonDense1d( + layer=layer, ibp=ibp, affine=affine, propagation=propagation, perturbation_domain=perturbation_domain + ) + + # symbolic inputs + decomon_inputs = helpers.get_decomon_symbolic_inputs( + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + layer_input_shape=layer_input_shape, + layer_output_shape=layer_output_shape, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + ) + affine_bounds_to_propagate, constant_oracle_bounds, x = decomon_inputs + + # actual (random) tensors + expected output shapes + x_val = helpers.generate_random_tensor(x.shape[1:], batchsize=batchsize) + + if affine: + if propagation == Propagation.FORWARD: + b_out_shape = layer_output_shape + w_out_shape = model_input_shape + layer_output_shape + else: + b_out_shape = model_output_shape + w_out_shape = layer_input_shape + model_output_shape + affine_bounds_to_propagate_val = [ + helpers.generate_random_tensor(tensor.shape[1:], batchsize=batchsize) + for tensor in affine_bounds_to_propagate + ] + propagated_affine_bounds_expected_shape = [w_out_shape, b_out_shape, w_out_shape, b_out_shape] + else: + propagated_affine_bounds_expected_shape = [] + affine_bounds_to_propagate_val = [] + + if ibp: + constant_oracle_bounds_val = [ + helpers.generate_random_tensor(tensor.shape[1:], batchsize=batchsize) for tensor in constant_oracle_bounds + ] + propagated_ibp_bounds_expected_shape = [layer_output_shape, layer_output_shape] + else: + propagated_ibp_bounds_expected_shape = [] + constant_oracle_bounds_val = [] + + decomon_inputs_val = [affine_bounds_to_propagate_val, constant_oracle_bounds_val, x_val] + + if propagation == Propagation.FORWARD: + decomon_output_expected_shapes = [ + propagated_affine_bounds_expected_shape, + propagated_ibp_bounds_expected_shape, + ] + else: + decomon_output_expected_shapes = [propagated_affine_bounds_expected_shape] + + # symbolic call + linear_decomon_output = linear_decomon_layer(*decomon_inputs) + non_linear_decomon_output = non_linear_decomon_layer(*decomon_inputs) + + # shapes ok ? + linear_decomon_output_shape_from_call = [[tensor.shape[1:] for tensor in l] for l in linear_decomon_output] + assert linear_decomon_output_shape_from_call == decomon_output_expected_shapes + non_linear_decomon_output_shape_from_call = [[tensor.shape[1:] for tensor in l] for l in non_linear_decomon_output] + assert non_linear_decomon_output_shape_from_call == decomon_output_expected_shapes + + # actual call + linear_decomon_output_val = linear_decomon_layer(*decomon_inputs_val) + non_linear_decomon_output_val = non_linear_decomon_layer(*decomon_inputs_val) + + # shapes ok ? + linear_decomon_output_shape_from_call = [[tensor.shape[1:] for tensor in l] for l in linear_decomon_output_val] + assert linear_decomon_output_shape_from_call == decomon_output_expected_shapes + non_linear_decomon_output_shape_from_call = [[tensor.shape[1:] for tensor in l] for l in linear_decomon_output_val] + assert non_linear_decomon_output_shape_from_call == decomon_output_expected_shapes + + # same values ? + helpers.assert_decomon_outputs_equal(linear_decomon_output_val, non_linear_decomon_output_val) + + # inequalities hold ? + # We only check it in case of a single-layer model => model shapes == layer shapes + # And with identity affine bounds + constant bounds = x = [keras_input, keras_input] + if model_input_shape == layer_input_shape and model_output_shape == layer_output_shape: + # keras input + keras_input_val = helpers.generate_random_tensor(layer_input_shape, batchsize=batchsize) + + # new decomon inputs values + decomon_inputs_val = helpers.generate_simple_decomon_layer_inputs_from_keras_input( + keras_input=keras_input_val, + layer_output_shape=layer_output_shape, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + ) + + # decomon call + linear_decomon_output_val = linear_decomon_layer(*decomon_inputs_val) + non_linear_decomon_output_val = non_linear_decomon_layer(*decomon_inputs_val) + + # keras call + keras_output_val = layer(keras_input_val) + + # comparison + helpers.assert_decomon_output_compare_with_keras_input_output_single_layer( + decomon_output=linear_decomon_output_val, keras_output=keras_output_val, keras_input=keras_input_val + ) + helpers.assert_decomon_outputs_equal(linear_decomon_output_val, non_linear_decomon_output_val) From 75c66a16161458f440e50fd454e61c33b6516ef5 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 6 Feb 2024 12:27:06 +0100 Subject: [PATCH 013/101] Update get_x_input_shape_wo_batchsize() to take model_input_shape as arg Was taking model_input_dim under the hypothesis that the model input was flatten. With this, it will allow to drop this hypothesis. --- src/decomon/core.py | 17 +++++++++-------- src/decomon/models/utils.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/decomon/core.py b/src/decomon/core.py index 41e3574d..eb768e67 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -45,15 +45,12 @@ def get_config(self) -> dict[str, Any]: "opt_option": self.opt_option, } - def get_x_input_shape_wo_batchsize(self, original_input_dim: int) -> tuple[int, ...]: + def get_x_input_shape_wo_batchsize(self, original_input_shape: tuple[int, ...]) -> tuple[int, ...]: n_comp_x = self.get_nb_x_components() if n_comp_x == 1: - return (original_input_dim,) + return original_input_shape else: - return ( - n_comp_x, - original_input_dim, - ) + return (n_comp_x,) + original_input_shape class BoxDomain(PerturbationDomain): @@ -238,7 +235,9 @@ def get_fullinputshapes_from_inputshapesformode( elif self.mode == ForwardMode.IBP: u_c_shape, l_c_shape, h_shape, g_shape = inputshapesformode[:nb_tensors] batchsize = u_c_shape[0] - x_shape = (batchsize,) + self.perturbation_domain.get_x_input_shape_wo_batchsize(self.model_input_dim) + x_shape = (batchsize,) + self.perturbation_domain.get_x_input_shape_wo_batchsize( + (self.model_input_dim,) + ) b_shape = tuple(u_c_shape) w_shape = tuple(u_c_shape) + (u_c_shape[-1],) x_shape, w_u_shape, b_u_shape, w_l_shape, b_l_shape = ( @@ -263,7 +262,9 @@ def get_fullinputshapes_from_inputshapesformode( elif self.mode == ForwardMode.IBP: u_c_shape, l_c_shape = inputshapesformode[:nb_tensors] batchsize = u_c_shape[0] - x_shape = (batchsize,) + self.perturbation_domain.get_x_input_shape_wo_batchsize(self.model_input_dim) + x_shape = (batchsize,) + self.perturbation_domain.get_x_input_shape_wo_batchsize( + (self.model_input_dim,) + ) b_shape = tuple(u_c_shape) w_shape = tuple(u_c_shape) + (u_c_shape[-1],) x_shape, w_u_shape, b_u_shape, w_l_shape, b_l_shape = ( diff --git a/src/decomon/models/utils.py b/src/decomon/models/utils.py index 60f5bde7..5c789d55 100644 --- a/src/decomon/models/utils.py +++ b/src/decomon/models/utils.py @@ -100,7 +100,7 @@ def get_input_tensors( inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) empty_tensor = inputs_outputs_spec.get_empty_tensor() - input_shape_x = perturbation_domain.get_x_input_shape_wo_batchsize(input_dim) + input_shape_x = perturbation_domain.get_x_input_shape_wo_batchsize((input_dim,)) z_tensor = Input(shape=input_shape_x, dtype=model.layers[0].dtype) u_c_tensor, l_c_tensor, W, b, h, g = ( empty_tensor, From 639a43957e787947c9b4690ad1a6a1ea80ae9ea7 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 6 Feb 2024 16:42:10 +0100 Subject: [PATCH 014/101] Update get_lower_box() to adapt to tensor-multid model input We update get_lower_box and get_upper_box so that we can use tensor-multid (like images) for x_min and x_max. For now, - we do not update get_lower_ball - we do not treat case where w.shape == b.shape --- src/decomon/core.py | 31 ++++++++----------------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/src/decomon/core.py b/src/decomon/core.py index eb768e67..b5f648f5 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -6,6 +6,7 @@ import numpy as np from keras.config import floatx +from decomon.keras_utils import batch_multid_dot from decomon.types import Tensor @@ -472,7 +473,7 @@ def get_empty_tensor(dtype: Optional[str] = None) -> Tensor: def get_upper_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: - """#compute the max of an affine function + """Compute the max of an affine function within a box (hypercube) defined by its extremal corners Args: @@ -485,22 +486,14 @@ def get_upper_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: max_(x >= x_min, x<=x_max) w*x + b """ - if len(w.shape) == len(b.shape): # identity function - return x_max + if len(w.shape) == len(b.shape): + raise NotImplementedError - # split into positive and negative components z_value = K.cast(0.0, dtype=x_min.dtype) w_pos = K.maximum(w, z_value) w_neg = K.minimum(w, z_value) - x_min_out = x_min + z_value * x_min - x_max_out = x_max + z_value * x_max - - for _ in range(len(w.shape) - len(x_max.shape)): - x_min_out = K.expand_dims(x_min_out, -1) - x_max_out = K.expand_dims(x_max_out, -1) - - return K.sum(w_pos * x_max_out + w_neg * x_min_out, 1) + b + return batch_multid_dot(x_max, w_pos) + batch_multid_dot(x_min, w_neg) + b def get_lower_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: @@ -516,21 +509,13 @@ def get_lower_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: """ if len(w.shape) == len(b.shape): - return x_min + raise NotImplementedError z_value = K.cast(0.0, dtype=x_min.dtype) - w_pos = K.maximum(w, z_value) w_neg = K.minimum(w, z_value) - x_min_out = x_min + z_value * x_min - x_max_out = x_max + z_value * x_max - - for _ in range(len(w.shape) - len(x_max.shape)): - x_min_out = K.expand_dims(x_min_out, -1) - x_max_out = K.expand_dims(x_max_out, -1) - - return K.sum(w_pos * x_min_out + w_neg * x_max_out, 1) + b + return batch_multid_dot(x_min, w_pos) + batch_multid_dot(x_max, w_neg) + b def get_lq_norm(x: Tensor, p: float, axis: int = -1) -> Tensor: @@ -568,7 +553,7 @@ def get_upper_ball(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kw max_(|x - x_0|_p<= eps) w*x + b """ if len(w.shape) == len(b.shape): - return x_0 + eps + raise NotImplementedError if p == np.inf: # compute x_min and x_max according to eps From 1d935bd07114f1f87d8f0f939effb1d69fa04fef Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 6 Feb 2024 16:45:41 +0100 Subject: [PATCH 015/101] Implement a naive version of DecomonDense We only implement get_affine_representation(). We could probably compute faster, for tensor-multid inputs, by avoiding artificially creating a "big" weights representation, and working directly with the kernel itself. --- src/decomon/layers/core/dense.py | 29 ++++++++++++++ tests/test_dense.py | 65 ++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 tests/test_dense.py diff --git a/src/decomon/layers/core/dense.py b/src/decomon/layers/core/dense.py index e69de29b..2249b258 100644 --- a/src/decomon/layers/core/dense.py +++ b/src/decomon/layers/core/dense.py @@ -0,0 +1,29 @@ +import keras.ops as K +from keras.layers import Dense + +from decomon.layers.layer import DecomonLayer + + +class DecomonNaiveDense(DecomonLayer): + layer: Dense + linear = True + + def get_affine_representation(self): + w = self.layer.kernel + b = self.layer.bias if self.layer.use_bias else K.zeros((self.layer.units,)) + + # manage tensor-multid input + for dim in self.layer.input.shape[-2:0:-1]: + # Construct a multid-tensor diagonal by blocks + reshaped_outer_shape = (dim, dim) + w.shape + transposed_outer_axes = ( + (0,) + + tuple(range(2, 2 + len(b.shape))) + + (1,) + + tuple(range(2 + len(b.shape), len(reshaped_outer_shape))) + ) + w = K.transpose(K.reshape(K.outer(K.identity(dim), w), reshaped_outer_shape), transposed_outer_axes) + # repeat bias along first dimensions + b = K.repeat(b[None], dim, axis=0) + + return w, b diff --git a/tests/test_dense.py b/tests/test_dense.py new file mode 100644 index 00000000..2b74f506 --- /dev/null +++ b/tests/test_dense.py @@ -0,0 +1,65 @@ +import keras.ops as K +import numpy as np +import pytest +from keras.layers import Dense, Input + +from decomon.keras_utils import batch_multid_dot +from decomon.layers.core.dense import DecomonNaiveDense + + +@pytest.mark.parametrize("input_shape", [(1,), (3,), (5, 2, 3)], ids=["0d", "1d", "multid"]) +def test_decomon_dense(use_bias, ibp, affine, propagation, input_shape, perturbation_domain, batchsize, helpers): + decimal = 5 + units = 7 + output_shape = input_shape[:-1] + (units,) + keras_symbolic_input = Input(input_shape) + decomon_symbolic_inputs = helpers.get_decomon_symbolic_inputs( + model_input_shape=input_shape, + model_output_shape=output_shape, + layer_input_shape=input_shape, + layer_output_shape=output_shape, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + ) + + layer = Dense(units=units) + layer(keras_symbolic_input) + + decomon_layer = DecomonNaiveDense( + layer=layer, ibp=ibp, affine=affine, propagation=propagation, perturbation_domain=perturbation_domain + ) + decomon_layer(*decomon_symbolic_inputs) + + keras_input = helpers.generate_random_tensor(input_shape, batchsize=batchsize) + decomon_inputs = helpers.generate_simple_decomon_layer_inputs_from_keras_input( + keras_input=keras_input, + layer_output_shape=output_shape, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + ) + + keras_output = layer(keras_input) + + # check affine representation is ok + w, b = decomon_layer.get_affine_representation() + keras_output_2 = batch_multid_dot(keras_input, w, missing_batchsize=(False, True)) + np.testing.assert_almost_equal( + K.convert_to_numpy(keras_output), + K.convert_to_numpy(keras_output_2), + decimal=decimal, + err_msg="wrong affine representation", + ) + + decomon_output = decomon_layer(*decomon_inputs) + + # check ibp and affine bounds well ordered w.r.t. keras output + helpers.assert_decomon_output_compare_with_keras_input_output_single_layer( + decomon_output=decomon_output, keras_output=keras_output, keras_input=keras_input + ) + + # before propagation through linear layer lower == upper => lower == upper after propagation + helpers.assert_decomon_output_lower_equal_upper(decomon_output, decimal=decimal) From f757f50f5c456ae2eaf8155b4b7f07768d1fb2dd Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 8 Feb 2024 12:23:09 +0100 Subject: [PATCH 016/101] Implement DecomonDense Less naive version where we directly override forward_ibp_propagate, forward_affine_propagate, and backward_affine_propagate to avoid repeating artificially the kernel, in case of tensor-multid inputs. --- src/decomon/layers/core/dense.py | 89 ++++++++++++++++++++++++++++++++ tests/test_dense.py | 40 ++++++++++---- 2 files changed, 118 insertions(+), 11 deletions(-) diff --git a/src/decomon/layers/core/dense.py b/src/decomon/layers/core/dense.py index 2249b258..ea7140b2 100644 --- a/src/decomon/layers/core/dense.py +++ b/src/decomon/layers/core/dense.py @@ -1,7 +1,9 @@ import keras.ops as K from keras.layers import Dense +from decomon.keras_utils import batch_multid_dot from decomon.layers.layer import DecomonLayer +from decomon.types import Tensor class DecomonNaiveDense(DecomonLayer): @@ -27,3 +29,90 @@ def get_affine_representation(self): b = K.repeat(b[None], dim, axis=0) return w, b + + +class DecomonDense(DecomonLayer): + layer: Dense + linear = True + + def _get_pseudo_affine_representation(self): + w = self.layer.kernel + b = self.layer.bias if self.layer.use_bias else K.zeros((self.layer.units,)) + return w, b + + def forward_ibp_propagate(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, Tensor]: + w, b = self._get_pseudo_affine_representation() + + z_value = K.cast(0.0, dtype=w.dtype) + w_pos = K.maximum(w, z_value) + w_neg = K.minimum(w, z_value) + + kwargs_batch_dot = dict(nb_merging_axes=1, missing_batchsize=(False, True)) + + l_c = ( + batch_multid_dot(lower, w_pos, **kwargs_batch_dot) + batch_multid_dot(upper, w_neg, **kwargs_batch_dot) + b + ) + u_c = ( + batch_multid_dot(upper, w_pos, **kwargs_batch_dot) + batch_multid_dot(lower, w_neg, **kwargs_batch_dot) + b + ) + + return l_c, u_c + + def forward_affine_propagate( + self, input_affine_bounds, input_constant_bounds + ) -> tuple[Tensor, Tensor, Tensor, Tensor]: + w_l_1, b_l_1, w_u_1, b_u_1 = input_affine_bounds + w_2, b_2 = self._get_pseudo_affine_representation() + + kwargs_batch_dot = dict(nb_merging_axes=1, missing_batchsize=(False, True)) + + z_value = K.cast(0.0, dtype=w_2.dtype) + w_2_pos = K.maximum(w_2, z_value) + w_2_neg = K.minimum(w_2, z_value) + + w_l = batch_multid_dot(w_l_1, w_2_pos, **kwargs_batch_dot) + batch_multid_dot( + w_u_1, w_2_neg, **kwargs_batch_dot + ) + w_u = batch_multid_dot(w_u_1, w_2_pos, **kwargs_batch_dot) + batch_multid_dot( + w_l_1, w_2_neg, **kwargs_batch_dot + ) + b_l = ( + batch_multid_dot(b_l_1, w_2_pos, **kwargs_batch_dot) + + batch_multid_dot(b_u_1, w_2_neg, **kwargs_batch_dot) + + b_2 + ) + b_u = ( + batch_multid_dot(b_u_1, w_2_pos, **kwargs_batch_dot) + + batch_multid_dot(b_l_1, w_2_neg, **kwargs_batch_dot) + + b_2 + ) + + return w_l, b_l, w_u, b_u + + def backward_affine_propagate( + self, output_affine_bounds, input_constant_bounds + ) -> tuple[Tensor, Tensor, Tensor, Tensor]: + w_1, b_1 = self._get_pseudo_affine_representation() + w_l_2, b_l_2, w_u_2, b_u_2 = output_affine_bounds + + nb_nonbatch_axes_keras_input = len(w_l_2.shape) - len(b_l_2.shape) + + # Merge weights on "units" axis and reorder axes + transposed_axes = ( + tuple(range(1, nb_nonbatch_axes_keras_input + 1)) + + (0,) + + tuple(range(nb_nonbatch_axes_keras_input + 1, len(w_l_2.shape))) + ) + w_l = K.transpose(K.tensordot(w_1, w_l_2, axes=[[-1], [nb_nonbatch_axes_keras_input]]), axes=transposed_axes) + w_u = K.transpose(K.tensordot(w_1, w_u_2, axes=[[-1], [nb_nonbatch_axes_keras_input]]), axes=transposed_axes) + + # Merge layer bias with backward weights on "units" axe and reduce on other input axes + reduced_axes = list(range(1, nb_nonbatch_axes_keras_input)) + b_l = K.sum(K.tensordot(b_1, w_l_2, axes=[[-1], [nb_nonbatch_axes_keras_input]]), axis=reduced_axes) + b_u = K.sum(K.tensordot(b_1, w_u_2, axes=[[-1], [nb_nonbatch_axes_keras_input]]), axis=reduced_axes) + + # Add bias from current backward bounds + b_l += b_l_2 + b_u += b_u_2 + + return w_l, b_l, w_u, b_u diff --git a/tests/test_dense.py b/tests/test_dense.py index 2b74f506..a15a6c44 100644 --- a/tests/test_dense.py +++ b/tests/test_dense.py @@ -4,11 +4,23 @@ from keras.layers import Dense, Input from decomon.keras_utils import batch_multid_dot -from decomon.layers.core.dense import DecomonNaiveDense +from decomon.layers.core.dense import DecomonDense, DecomonNaiveDense +@pytest.mark.parametrize("decomon_layer_class", [DecomonNaiveDense, DecomonDense]) @pytest.mark.parametrize("input_shape", [(1,), (3,), (5, 2, 3)], ids=["0d", "1d", "multid"]) -def test_decomon_dense(use_bias, ibp, affine, propagation, input_shape, perturbation_domain, batchsize, helpers): +def test_decomon_dense( + decomon_layer_class, + use_bias, + randomize, + ibp, + affine, + propagation, + input_shape, + perturbation_domain, + batchsize, + helpers, +): decimal = 5 units = 7 output_shape = input_shape[:-1] + (units,) @@ -27,7 +39,12 @@ def test_decomon_dense(use_bias, ibp, affine, propagation, input_shape, perturba layer = Dense(units=units) layer(keras_symbolic_input) - decomon_layer = DecomonNaiveDense( + if randomize: + # randomize weights => non-zero biases + for w in layer.weights: + w.assign(np.random.random(w.shape)) + + decomon_layer = decomon_layer_class( layer=layer, ibp=ibp, affine=affine, propagation=propagation, perturbation_domain=perturbation_domain ) decomon_layer(*decomon_symbolic_inputs) @@ -45,14 +62,15 @@ def test_decomon_dense(use_bias, ibp, affine, propagation, input_shape, perturba keras_output = layer(keras_input) # check affine representation is ok - w, b = decomon_layer.get_affine_representation() - keras_output_2 = batch_multid_dot(keras_input, w, missing_batchsize=(False, True)) - np.testing.assert_almost_equal( - K.convert_to_numpy(keras_output), - K.convert_to_numpy(keras_output_2), - decimal=decimal, - err_msg="wrong affine representation", - ) + if decomon_layer_class == DecomonNaiveDense: + w, b = decomon_layer.get_affine_representation() + keras_output_2 = batch_multid_dot(keras_input, w, missing_batchsize=(False, True)) + b + np.testing.assert_almost_equal( + K.convert_to_numpy(keras_output), + K.convert_to_numpy(keras_output_2), + decimal=decimal, + err_msg="wrong affine representation", + ) decomon_output = decomon_layer(*decomon_inputs) From 06173359f0bc85db85badbfe0d13a8b6fbbd4975 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 9 Feb 2024 14:11:17 +0100 Subject: [PATCH 017/101] Update batch_multid_dot() for diagonal entries "Diagonal" tensors can be represented by their diagonal. In that case, batch_multid_dot simplifies and results in a mere element-wise product, with the correct broadcasting. Diagonal tensors are tensors that can be, batch element by batch element represented as x_full = K.reshape(K.diag(K.ravel(x_diag)), x_diag.shape+x_diag.shape ) x_diag being their "diagonal" representation that can be multid. It will be useful when we represent an affine operator by (w, b) with weights tensor w of the same shape as bias tensor b. --- src/decomon/keras_utils.py | 74 +++++++++++++++++++++---------- tests/test_keras_utils.py | 89 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 23 deletions(-) diff --git a/src/decomon/keras_utils.py b/src/decomon/keras_utils.py index c1c4e98d..9f135a52 100644 --- a/src/decomon/keras_utils.py +++ b/src/decomon/keras_utils.py @@ -14,7 +14,11 @@ def batch_multid_dot( - x: Tensor, y: Tensor, nb_merging_axes: Optional[int] = None, missing_batchsize: tuple[bool, bool] = (False, False) + x: Tensor, + y: Tensor, + nb_merging_axes: Optional[int] = None, + missing_batchsize: tuple[bool, bool] = (False, False), + diagonal: tuple[bool, bool] = (False, False), ) -> Tensor: """Dot product of tensors by batch, along multiple axes @@ -31,17 +35,22 @@ def batch_multid_dot( len(x.shape) if missing_batchsize[0] else len(x.shape) - 1 missing_batchsize: specify if a tensor is missing the batch dimension, for x and y. In that case, the corresponding tensor is broadcasted accordingly. + diagonal: specify is a tensor is only represented by its diagonal. See below for an example. Returns: For performance reasons, instead of actually repeating the tensor `batchsize` along a new first axis, we rather use `keras.ops.tensordot` directly on tensors without broadcasting them. - Note: - The dimensions of axes along which we perform the dot product - (i.e. x.shape[-nb_merging_axes:] and y.shape[1:1 + nb_merging_axes]) should match. + Notes: + - The dimensions of axes along which we perform the dot product + (i.e. x.shape[-nb_merging_axes:] and y.shape[1:1 + nb_merging_axes] when no batch axe is missing) should match. + - diagonal example: if x is diagonal and missing its batch axis, it means that the full tensor x is retrieve with + x_full = K.reshape(K.diag(K.ravel(x)), x.shape + x.shape) + With the batch axis, the above computation should be made batch element by batch element. """ + diag_x, diag_y = diagonal missing_batchsize_x, missing_batchsize_y = missing_batchsize nb_batch_axe_x = 0 if missing_batchsize_x else 1 nb_batch_axe_y = 0 if missing_batchsize_y else 1 @@ -59,26 +68,45 @@ def batch_multid_dot( f"Full shapes: {x.shape} and {y.shape}." ) - # switch on missing batch axe (e.g. with affine layer representation like Dense's kernel) - if missing_batchsize_y: - return K.tensordot(x, y, axes=nb_merging_axes) - elif missing_batchsize_x: - # axes along which summing - merging_axes_x = list(range(-nb_merging_axes, 0)) - merging_axes_y = list(range(nb_batch_axe_y, nb_batch_axe_y + nb_merging_axes)) - # transposition to make to put back batch axe at the beginning - nb_axes_after_merge = len(x.shape) + len(y.shape) - 2 * nb_merging_axes - nb_axes_after_merge_from_x = len(x.shape) - nb_merging_axes - transpose_indices = ( - (nb_axes_after_merge_from_x,) - + tuple(range(nb_axes_after_merge_from_x)) - + tuple(range(nb_axes_after_merge_from_x + 1, nb_axes_after_merge)) - ) - return K.transpose(K.tensordot(x, y, axes=[merging_axes_x, merging_axes_y]), transpose_indices) + # Special cases: diagonal entries (represented only by their diagonal) + if diag_x and diag_y: + # all inputs diagonal: we keep a diagonal output (with batch axis if one input has one) + return x * y + elif diag_x: + # reshape to make broadcast possible + nb_missing_batch_axe_x = 1 - nb_batch_axe_x + nb_missing_axes_x_wo_batch = len(y.shape) - nb_batch_axe_y - len(x.shape) + nb_batch_axe_x + new_shape = nb_missing_batch_axe_x * (1,) + x.shape + (1,) * nb_missing_axes_x_wo_batch + return K.reshape(x, new_shape) * y + elif diag_y: + # reshape necessary for broadcast, only if y has a batch axis + if not missing_batchsize_y: + nb_missing_axes_y_wo_batch = len(x.shape) - nb_batch_axe_x - len(y.shape) + nb_batch_axe_y + new_shape = y.shape[:1] + (1,) * nb_missing_axes_y_wo_batch + y.shape[1:] + return x * K.reshape(y, new_shape) + else: + return x * y else: - new_x_shape = tuple(x.shape[1:-nb_merging_axes]) + (-1,) - new_y_shape = (-1,) + tuple(y.shape[nb_merging_axes + 1 :]) - return Dot(axes=(-1, 1))([Reshape(new_x_shape)(x), Reshape(new_y_shape)(y)]) + # switch on missing batch axe (e.g. with affine layer representation like Dense's kernel) + if missing_batchsize_y: + return K.tensordot(x, y, axes=nb_merging_axes) + elif missing_batchsize_x: + # axes along which summing + merging_axes_x = list(range(-nb_merging_axes, 0)) + merging_axes_y = list(range(nb_batch_axe_y, nb_batch_axe_y + nb_merging_axes)) + # transposition to make to put back batch axe at the beginning + nb_axes_after_merge = len(x.shape) + len(y.shape) - 2 * nb_merging_axes + nb_axes_after_merge_from_x = len(x.shape) - nb_merging_axes + transpose_indices = ( + (nb_axes_after_merge_from_x,) + + tuple(range(nb_axes_after_merge_from_x)) + + tuple(range(nb_axes_after_merge_from_x + 1, nb_axes_after_merge)) + ) + return K.transpose(K.tensordot(x, y, axes=[merging_axes_x, merging_axes_y]), transpose_indices) + else: + new_x_shape = tuple(x.shape[1:-nb_merging_axes]) + (-1,) + new_y_shape = (-1,) + tuple(y.shape[nb_merging_axes + 1 :]) + return Dot(axes=(-1, 1))([Reshape(new_x_shape)(x), Reshape(new_y_shape)(y)]) class BatchedIdentityLike(keras.Operation): diff --git a/tests/test_keras_utils.py b/tests/test_keras_utils.py index 6a6839e7..4ec7727a 100644 --- a/tests/test_keras_utils.py +++ b/tests/test_keras_utils.py @@ -11,6 +11,7 @@ is_a_merge_layer, share_layer_all_weights, ) +from decomon.types import BackendTensor class MyLayer(Layer): @@ -125,6 +126,94 @@ def test_batch_multid_dot_default_nb_merging_axes(missing_batchsize, helpers): helpers.assert_almost_equal(res, res_default) +def generate_tensor_full_n_diag( + batchsize: int, + diag_shape: tuple[int, ...], + other_shape: tuple[int, ...], + diag: bool, + missing_batchsize: bool, + left: bool, +) -> tuple[BackendTensor, BackendTensor]: + batchshape = (batchsize,) + flatten_diag_shape = (int(np.prod(diag_shape)),) + full_shape = diag_shape + diag_shape + + if diag: + if missing_batchsize: + x_diag_flatten = K.convert_to_tensor(np.random.random(flatten_diag_shape), dtype=float) + x_diag = K.reshape(x_diag_flatten, diag_shape) + x_full = K.reshape(K.diag(x_diag_flatten), full_shape) + else: + x_diag_flatten = K.convert_to_tensor(np.random.random(batchshape + flatten_diag_shape), dtype=float) + x_diag = K.reshape(x_diag_flatten, batchshape + diag_shape) + x_full = K.concatenate( + [K.reshape(K.diag(x_diag_flatten[i]), full_shape)[None] for i in range(batchsize)], axis=0 + ) + else: + if left: + x_shape = other_shape + diag_shape + else: + x_shape = diag_shape + other_shape + if not missing_batchsize: + x_shape = batchshape + x_shape + x = K.convert_to_tensor(np.random.random(x_shape), dtype=float) + x_full = x + x_diag = x + + return x_full, x_diag + + +@pytest.mark.parametrize("missing_batchsize", [(False, False), (True, False), (False, True)]) +@pytest.mark.parametrize("diagonal", [(True, True), (True, False), (False, True)]) +def test_batch_multi_dot_diag(missing_batchsize, diagonal, helpers): + batchsize = 10 + diag_shape = (4, 5, 2) + other_shape = (3, 7) + nb_merging_axes = len(diag_shape) + + diag_x, diag_y = diagonal + missing_batchsize_x, missing_batchsize_y = missing_batchsize + + x_full, x_diag = generate_tensor_full_n_diag( + batchsize=batchsize, + diag_shape=diag_shape, + other_shape=other_shape, + diag=diag_x, + missing_batchsize=missing_batchsize_x, + left=True, + ) + y_full, y_diag = generate_tensor_full_n_diag( + batchsize=batchsize, + diag_shape=diag_shape, + other_shape=other_shape, + diag=diag_y, + missing_batchsize=missing_batchsize_y, + left=False, + ) + + res_full = batch_multid_dot(x_full, y_full, nb_merging_axes=nb_merging_axes, missing_batchsize=missing_batchsize) + res_diag = batch_multid_dot( + x_diag, y_diag, nb_merging_axes=nb_merging_axes, missing_batchsize=missing_batchsize, diagonal=diagonal + ) + + if diag_x and diag_y: + # the result stayed diagonal, needs to be reworked to be compared with full result + assert res_diag.shape == (batchsize,) + diag_shape + res_diag = K.concatenate( + [K.reshape(K.diag(K.ravel(res_diag[i])), diag_shape + diag_shape)[None] for i in range(len(res_diag))], + axis=0, + ) + elif diag_x: + assert res_diag.shape == (batchsize,) + diag_shape + other_shape + elif diag_y: + assert res_diag.shape == (batchsize,) + other_shape + diag_shape + + helpers.assert_almost_equal( + res_full, + res_diag, + ) + + def test_get_weight_index_from_name_nok_attribute(): layer = Dense(3) layer(K.zeros((2, 1))) From 13ca5a5c853e2d81230387731b0b4f0dfd026c67 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 9 Feb 2024 14:57:54 +0100 Subject: [PATCH 018/101] Adapt DecomonLayer and DecomonDense to manage diagonal bounds When layer affine bounds or affine bounds to propagate are represented in diagonal mode (ie w.shape == b.shape), we need to specify it to batch_multid_dot. For not naive DecomonDense implementation, this is not a trivial task in backward mode, so not yet implemented. --- src/decomon/layers/core/dense.py | 43 +++++------- src/decomon/layers/layer.py | 112 +++++++++++++++++++------------ 2 files changed, 87 insertions(+), 68 deletions(-) diff --git a/src/decomon/layers/core/dense.py b/src/decomon/layers/core/dense.py index ea7140b2..2a93d626 100644 --- a/src/decomon/layers/core/dense.py +++ b/src/decomon/layers/core/dense.py @@ -47,14 +47,10 @@ def forward_ibp_propagate(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, T w_pos = K.maximum(w, z_value) w_neg = K.minimum(w, z_value) - kwargs_batch_dot = dict(nb_merging_axes=1, missing_batchsize=(False, True)) + kwargs_dot = dict(nb_merging_axes=1, missing_batchsize=(False, True)) - l_c = ( - batch_multid_dot(lower, w_pos, **kwargs_batch_dot) + batch_multid_dot(upper, w_neg, **kwargs_batch_dot) + b - ) - u_c = ( - batch_multid_dot(upper, w_pos, **kwargs_batch_dot) + batch_multid_dot(lower, w_neg, **kwargs_batch_dot) + b - ) + l_c = batch_multid_dot(lower, w_pos, **kwargs_dot) + batch_multid_dot(upper, w_neg, **kwargs_dot) + b + u_c = batch_multid_dot(upper, w_pos, **kwargs_dot) + batch_multid_dot(lower, w_neg, **kwargs_dot) + b return l_c, u_c @@ -63,29 +59,21 @@ def forward_affine_propagate( ) -> tuple[Tensor, Tensor, Tensor, Tensor]: w_l_1, b_l_1, w_u_1, b_u_1 = input_affine_bounds w_2, b_2 = self._get_pseudo_affine_representation() - - kwargs_batch_dot = dict(nb_merging_axes=1, missing_batchsize=(False, True)) + diagonal = ( + w_l_1.shape == b_l_1.shape, + False, + ) + kwargs_dot_w = dict(nb_merging_axes=1, missing_batchsize=(False, True), diagonal=diagonal) + kwargs_dot_b = dict(nb_merging_axes=1, missing_batchsize=(False, True)) z_value = K.cast(0.0, dtype=w_2.dtype) w_2_pos = K.maximum(w_2, z_value) w_2_neg = K.minimum(w_2, z_value) - w_l = batch_multid_dot(w_l_1, w_2_pos, **kwargs_batch_dot) + batch_multid_dot( - w_u_1, w_2_neg, **kwargs_batch_dot - ) - w_u = batch_multid_dot(w_u_1, w_2_pos, **kwargs_batch_dot) + batch_multid_dot( - w_l_1, w_2_neg, **kwargs_batch_dot - ) - b_l = ( - batch_multid_dot(b_l_1, w_2_pos, **kwargs_batch_dot) - + batch_multid_dot(b_u_1, w_2_neg, **kwargs_batch_dot) - + b_2 - ) - b_u = ( - batch_multid_dot(b_u_1, w_2_pos, **kwargs_batch_dot) - + batch_multid_dot(b_l_1, w_2_neg, **kwargs_batch_dot) - + b_2 - ) + w_l = batch_multid_dot(w_l_1, w_2_pos, **kwargs_dot_w) + batch_multid_dot(w_u_1, w_2_neg, **kwargs_dot_w) + w_u = batch_multid_dot(w_u_1, w_2_pos, **kwargs_dot_w) + batch_multid_dot(w_l_1, w_2_neg, **kwargs_dot_w) + b_l = batch_multid_dot(b_l_1, w_2_pos, **kwargs_dot_b) + batch_multid_dot(b_u_1, w_2_neg, **kwargs_dot_b) + b_2 + b_u = batch_multid_dot(b_u_1, w_2_pos, **kwargs_dot_b) + batch_multid_dot(b_l_1, w_2_neg, **kwargs_dot_b) + b_2 return w_l, b_l, w_u, b_u @@ -95,6 +83,11 @@ def backward_affine_propagate( w_1, b_1 = self._get_pseudo_affine_representation() w_l_2, b_l_2, w_u_2, b_u_2 = output_affine_bounds + # affine bounds represented in diagonal mode? + diagonal_bounds = w_l_2.shape == b_l_2.shape + if diagonal_bounds: + raise NotImplementedError + nb_nonbatch_axes_keras_input = len(w_l_2.shape) - len(b_l_2.shape) # Merge weights on "units" axis and reorder axes diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index 4bbf7bbc..30712df7 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -198,21 +198,15 @@ def forward_ibp_propagate(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, T """ if self.linear: w, b = self.get_affine_representation() + is_diag = w.shape == b.shape + kwargs_dot = dict(missing_batchsize=(False, True), diagonal=(False, is_diag)) z_value = K.cast(0.0, dtype=w.dtype) w_pos = K.maximum(w, z_value) w_neg = K.minimum(w, z_value) - l_c = ( - batch_multid_dot(lower, w_pos, missing_batchsize=(False, True)) - + batch_multid_dot(upper, w_neg, missing_batchsize=(False, True)) - + b - ) - u_c = ( - batch_multid_dot(upper, w_pos, missing_batchsize=(False, True)) - + batch_multid_dot(lower, w_neg, missing_batchsize=(False, True)) - + b - ) + l_c = batch_multid_dot(lower, w_pos, **kwargs_dot) + batch_multid_dot(upper, w_neg, **kwargs_dot) + b + u_c = batch_multid_dot(upper, w_pos, **kwargs_dot) + batch_multid_dot(lower, w_neg, **kwargs_dot) + b return l_c, u_c else: @@ -529,13 +523,23 @@ def combine_affine_bounds( In the generic case, tensors in affine_bounds have their first axis corresponding to the batch size. """ + # Are weights in diagonal representation? + + diagonal = ( + affine_bounds_1[0].shape == affine_bounds_1[1].shape, + affine_bounds_2[0].shape == affine_bounds_2[1].shape, + ) if from_linear_layer == (False, False): - return _combine_affine_bounds_generic(affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2) + return _combine_affine_bounds_generic( + affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal + ) elif from_linear_layer == (True, False): - return _combine_affine_bounds_left_from_linear(affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2) + return _combine_affine_bounds_left_from_linear( + affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal + ) elif from_linear_layer == (False, True): return _combine_affine_bounds_right_from_linear( - affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2 + affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal ) else: raise NotImplementedError() @@ -544,12 +548,14 @@ def combine_affine_bounds( def _combine_affine_bounds_generic( affine_bounds_1: list[Tensor], affine_bounds_2: list[Tensor], + diagonal: tuple[bool, bool], ) -> tuple[Tensor, Tensor, Tensor, Tensor]: """Combine affine bounds Args: affine_bounds_1: [w_l_1, b_l_1, w_u_1, b_u_1] first affine bounds affine_bounds_2: [w_l_2, b_l_2, w_u_2, b_u_2] second affine bounds + diagonal: specify if weights of each affine bounds are in diagonal representation or not Returns: w_l, b_l, w_u, b_u: combined affine bounds @@ -566,27 +572,29 @@ def _combine_affine_bounds_generic( w_l_2, b_l_2, w_u_2, b_u_2 = affine_bounds_2 nb_axes_wo_batchsize_y = len(b_l_1.shape) - 1 + #  NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b + kwargs_dot_w = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + diagonal=diagonal, + ) + kwargs_dot_b = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + diagonal=(False, diagonal[1]), + ) + z_value = K.cast(0.0, dtype=w_l_2.dtype) w_l_2_pos = K.maximum(w_l_2, z_value) w_u_2_pos = K.maximum(w_u_2, z_value) w_l_2_neg = K.minimum(w_l_2, z_value) w_u_2_neg = K.minimum(w_u_2, z_value) - w_l = batch_multid_dot(w_l_1, w_l_2_pos, nb_merging_axes=nb_axes_wo_batchsize_y) + batch_multid_dot( - w_u_1, w_l_2_neg, nb_merging_axes=nb_axes_wo_batchsize_y - ) - w_u = batch_multid_dot(w_u_1, w_u_2_pos, nb_merging_axes=nb_axes_wo_batchsize_y) + batch_multid_dot( - w_l_1, w_u_2_neg, nb_merging_axes=nb_axes_wo_batchsize_y - ) + w_l = batch_multid_dot(w_l_1, w_l_2_pos, **kwargs_dot_w) + batch_multid_dot(w_u_1, w_l_2_neg, **kwargs_dot_w) + w_u = batch_multid_dot(w_u_1, w_u_2_pos, **kwargs_dot_w) + batch_multid_dot(w_l_1, w_u_2_neg, **kwargs_dot_w) b_l = ( - batch_multid_dot(b_l_1, w_l_2_pos, nb_merging_axes=nb_axes_wo_batchsize_y) - + batch_multid_dot(b_u_1, w_l_2_neg, nb_merging_axes=nb_axes_wo_batchsize_y) - + b_l_2 + batch_multid_dot(b_l_1, w_l_2_pos, **kwargs_dot_b) + batch_multid_dot(b_u_1, w_l_2_neg, **kwargs_dot_b) + b_l_2 ) b_u = ( - batch_multid_dot(b_u_1, w_u_2_pos, nb_merging_axes=nb_axes_wo_batchsize_y) - + batch_multid_dot(b_l_1, w_u_2_neg, nb_merging_axes=nb_axes_wo_batchsize_y) - + b_u_2 + batch_multid_dot(b_u_1, w_u_2_pos, **kwargs_dot_b) + batch_multid_dot(b_l_1, w_u_2_neg, **kwargs_dot_b) + b_u_2 ) return w_l, b_l, w_u, b_u @@ -595,12 +603,14 @@ def _combine_affine_bounds_generic( def _combine_affine_bounds_right_from_linear( affine_bounds_1: list[Tensor], affine_bounds_2: list[Tensor], + diagonal: tuple[bool, bool], ) -> tuple[Tensor, Tensor, Tensor, Tensor]: """Combine affine bounds Args: affine_bounds_1: [w_l_1, b_l_1, w_u_1, b_u_1] first affine bounds affine_bounds_2: [w_2, b_2, w_2, b_2] second affine bounds, with lower=upper + no batchsize + diagonal: specify if weights of each affine bounds are in diagonal representation or not Returns: w_l, b_l, w_u, b_u: combined affine bounds @@ -617,24 +627,27 @@ def _combine_affine_bounds_right_from_linear( w_2, b_2 = affine_bounds_2[:2] nb_axes_wo_batchsize_y = len(b_l_1.shape) - 1 missing_batchsize = (False, True) - kwargs_batch_dot = dict(nb_merging_axes=nb_axes_wo_batchsize_y, missing_batchsize=missing_batchsize) + + # NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b + kwargs_dot_w = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + missing_batchsize=missing_batchsize, + diagonal=diagonal, + ) + kwargs_dot_b = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + missing_batchsize=missing_batchsize, + diagonal=(False, diagonal[1]), + ) z_value = K.cast(0.0, dtype=w_2.dtype) w_2_pos = K.maximum(w_2, z_value) w_2_neg = K.minimum(w_2, z_value) - w_l = batch_multid_dot(w_l_1, w_2_pos, **kwargs_batch_dot) + batch_multid_dot(w_u_1, w_2_neg, **kwargs_batch_dot) - w_u = batch_multid_dot(w_u_1, w_2_pos, **kwargs_batch_dot) + batch_multid_dot(w_l_1, w_2_neg, **kwargs_batch_dot) - b_l = ( - batch_multid_dot(b_l_1, w_2_pos, **kwargs_batch_dot) - + batch_multid_dot(b_u_1, w_2_neg, **kwargs_batch_dot) - + b_2 - ) - b_u = ( - batch_multid_dot(b_u_1, w_2_pos, **kwargs_batch_dot) - + batch_multid_dot(b_l_1, w_2_neg, **kwargs_batch_dot) - + b_2 - ) + w_l = batch_multid_dot(w_l_1, w_2_pos, **kwargs_dot_w) + batch_multid_dot(w_u_1, w_2_neg, **kwargs_dot_w) + w_u = batch_multid_dot(w_u_1, w_2_pos, **kwargs_dot_w) + batch_multid_dot(w_l_1, w_2_neg, **kwargs_dot_w) + b_l = batch_multid_dot(b_l_1, w_2_pos, **kwargs_dot_b) + batch_multid_dot(b_u_1, w_2_neg, **kwargs_dot_b) + b_2 + b_u = batch_multid_dot(b_u_1, w_2_pos, **kwargs_dot_b) + batch_multid_dot(b_l_1, w_2_neg, **kwargs_dot_b) + b_2 return w_l, b_l, w_u, b_u @@ -642,12 +655,14 @@ def _combine_affine_bounds_right_from_linear( def _combine_affine_bounds_left_from_linear( affine_bounds_1: list[Tensor], affine_bounds_2: list[Tensor], + diagonal: tuple[bool, bool], ) -> tuple[Tensor, Tensor, Tensor, Tensor]: """Combine affine bounds Args: affine_bounds_1: [w_1, b_1, w_1, b_1] first affine bounds, with lower=upper + no batchsize affine_bounds_2: [w_l_2, b_l_2, w_u_2, b_u_2] second affine bounds + diagonal: specify if weights of each affine bounds are in diagonal representation or not Returns: w_l, b_l, w_u, b_u: combined affine bounds @@ -664,11 +679,22 @@ def _combine_affine_bounds_left_from_linear( w_l_2, b_l_2, w_u_2, b_u_2 = affine_bounds_2 nb_axes_wo_batchsize_y = len(b_1.shape) missing_batchsize = (True, False) - kwargs_batch_dot = dict(nb_merging_axes=nb_axes_wo_batchsize_y, missing_batchsize=missing_batchsize) - w_l = batch_multid_dot(w_1, w_l_2, **kwargs_batch_dot) - w_u = batch_multid_dot(w_1, w_u_2, **kwargs_batch_dot) - b_l = batch_multid_dot(b_1, w_l_2, **kwargs_batch_dot) + b_l_2 - b_u = batch_multid_dot(b_1, w_u_2, **kwargs_batch_dot) + b_u_2 + #   NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b + kwargs_dot_w = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + missing_batchsize=missing_batchsize, + diagonal=diagonal, + ) + kwargs_dot_b = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + missing_batchsize=missing_batchsize, + diagonal=(False, diagonal[1]), + ) + + w_l = batch_multid_dot(w_1, w_l_2, **kwargs_dot_w) + w_u = batch_multid_dot(w_1, w_u_2, **kwargs_dot_w) + b_l = batch_multid_dot(b_1, w_l_2, **kwargs_dot_b) + b_l_2 + b_u = batch_multid_dot(b_1, w_u_2, **kwargs_dot_b) + b_u_2 return w_l, b_l, w_u, b_u From 6f30f951d3f1e4bf244949b69c4a11b8be1d277c Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 9 Feb 2024 23:09:16 +0100 Subject: [PATCH 019/101] Manage case with empty affine bounds to merge By convention, empty affine bounds means identity bounds ie w_l=w_u=identity, b_l=b_u=0 thus we return the other bounds unchanged in that case. --- src/decomon/layers/layer.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index 30712df7..55dc2191 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -514,7 +514,9 @@ def combine_affine_bounds( w_l * x + b_l <= z <= w_u * x + b_u - Special case with linear layers: + Special cases + + - with linear layers: If the affine bounds come from the affine representation of a linear layer (e.g. affine_bounds_1), then - lower and upper bounds are equal: affine_bounds_1 = [w_1, b_1, w_1, b_1] @@ -522,9 +524,24 @@ def combine_affine_bounds( In the generic case, tensors in affine_bounds have their first axis corresponding to the batch size. + - diagonal representation: + + If w.shape == b.shape, this means that w is represented by its "diagonal" (potentially a tensor-multid). + + - empty affine bounds: + + when one affine bounds is an empty list, this is actually a convention for identity bounds, i.e. + w = identity, b = 0 + therefore we return the other affine_bounds, unchanged. + """ - # Are weights in diagonal representation? + # special case: empty bounds <=> identity bounds + if len(affine_bounds_1) == 0: + return tuple(affine_bounds_2) + if len(affine_bounds_2) == 0: + return tuple(affine_bounds_1) + # Are weights in diagonal representation? diagonal = ( affine_bounds_1[0].shape == affine_bounds_1[1].shape, affine_bounds_2[0].shape == affine_bounds_2[1].shape, From 6bf8bc697f54e25a86819b6a7e9663b8199d4878 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Sat, 10 Feb 2024 00:37:03 +0100 Subject: [PATCH 020/101] Adapt batch_multid_dot to case where both tensors have no batch axis --- src/decomon/keras_utils.py | 2 +- tests/test_keras_utils.py | 44 ++++++++++++++++++++++++++------------ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/decomon/keras_utils.py b/src/decomon/keras_utils.py index 9f135a52..81201bc6 100644 --- a/src/decomon/keras_utils.py +++ b/src/decomon/keras_utils.py @@ -74,7 +74,7 @@ def batch_multid_dot( return x * y elif diag_x: # reshape to make broadcast possible - nb_missing_batch_axe_x = 1 - nb_batch_axe_x + nb_missing_batch_axe_x = nb_batch_axe_y - nb_batch_axe_x nb_missing_axes_x_wo_batch = len(y.shape) - nb_batch_axe_y - len(x.shape) + nb_batch_axe_x new_shape = nb_missing_batch_axe_x * (1,) + x.shape + (1,) * nb_missing_axes_x_wo_batch return K.reshape(x, new_shape) * y diff --git a/tests/test_keras_utils.py b/tests/test_keras_utils.py index 4ec7727a..fb80ad7a 100644 --- a/tests/test_keras_utils.py +++ b/tests/test_keras_utils.py @@ -157,15 +157,20 @@ def generate_tensor_full_n_diag( if not missing_batchsize: x_shape = batchshape + x_shape x = K.convert_to_tensor(np.random.random(x_shape), dtype=float) + x_full = x x_diag = x + if missing_batchsize: + # reconstruct a batch axis + x_full = K.repeat(x_full[None], batchsize, axis=0) + return x_full, x_diag -@pytest.mark.parametrize("missing_batchsize", [(False, False), (True, False), (False, True)]) -@pytest.mark.parametrize("diagonal", [(True, True), (True, False), (False, True)]) -def test_batch_multi_dot_diag(missing_batchsize, diagonal, helpers): +@pytest.mark.parametrize("missing_batchsize", [(False, False), (True, False), (False, True), (True, True)]) +@pytest.mark.parametrize("diagonal", [(True, True), (True, False), (False, True), (False, False)]) +def test_batch_multi_dot_diag_missing_batchsize(missing_batchsize, diagonal, helpers): batchsize = 10 diag_shape = (4, 5, 2) other_shape = (3, 7) @@ -174,7 +179,7 @@ def test_batch_multi_dot_diag(missing_batchsize, diagonal, helpers): diag_x, diag_y = diagonal missing_batchsize_x, missing_batchsize_y = missing_batchsize - x_full, x_diag = generate_tensor_full_n_diag( + x_full, x_simplified = generate_tensor_full_n_diag( batchsize=batchsize, diag_shape=diag_shape, other_shape=other_shape, @@ -182,7 +187,7 @@ def test_batch_multi_dot_diag(missing_batchsize, diagonal, helpers): missing_batchsize=missing_batchsize_x, left=True, ) - y_full, y_diag = generate_tensor_full_n_diag( + y_full, y_simplified = generate_tensor_full_n_diag( batchsize=batchsize, diag_shape=diag_shape, other_shape=other_shape, @@ -191,26 +196,37 @@ def test_batch_multi_dot_diag(missing_batchsize, diagonal, helpers): left=False, ) - res_full = batch_multid_dot(x_full, y_full, nb_merging_axes=nb_merging_axes, missing_batchsize=missing_batchsize) - res_diag = batch_multid_dot( - x_diag, y_diag, nb_merging_axes=nb_merging_axes, missing_batchsize=missing_batchsize, diagonal=diagonal + res_full = batch_multid_dot(x_full, y_full, nb_merging_axes=nb_merging_axes) + res_simplified = batch_multid_dot( + x_simplified, + y_simplified, + nb_merging_axes=nb_merging_axes, + missing_batchsize=missing_batchsize, + diagonal=diagonal, ) + if missing_batchsize_x and missing_batchsize_y: + # the result stayed w/o batch axis, needs to be added to be compared with full result + res_simplified = K.repeat(res_simplified[None], batchsize, axis=0) + if diag_x and diag_y: # the result stayed diagonal, needs to be reworked to be compared with full result - assert res_diag.shape == (batchsize,) + diag_shape - res_diag = K.concatenate( - [K.reshape(K.diag(K.ravel(res_diag[i])), diag_shape + diag_shape)[None] for i in range(len(res_diag))], + assert res_simplified.shape == (batchsize,) + diag_shape + res_simplified = K.concatenate( + [ + K.reshape(K.diag(K.ravel(res_simplified[i])), diag_shape + diag_shape)[None] + for i in range(len(res_simplified)) + ], axis=0, ) elif diag_x: - assert res_diag.shape == (batchsize,) + diag_shape + other_shape + assert res_simplified.shape == (batchsize,) + diag_shape + other_shape elif diag_y: - assert res_diag.shape == (batchsize,) + other_shape + diag_shape + assert res_simplified.shape == (batchsize,) + other_shape + diag_shape helpers.assert_almost_equal( res_full, - res_diag, + res_simplified, ) From 568cf7165b23d9db9a24b66366e82475afed2d3c Mon Sep 17 00:00:00 2001 From: Nolwen Date: Sat, 10 Feb 2024 00:38:30 +0100 Subject: [PATCH 021/101] Update DecomonLayer.compute_output_shape() to different affine inputs types - empty affine bounds => identity bounds - diagonal bounds - bounds w/o batch --- src/decomon/layers/layer.py | 154 +++++++++++++++++++++++++++++++++--- 1 file changed, 141 insertions(+), 13 deletions(-) diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index 55dc2191..18192291 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -26,6 +26,8 @@ class DecomonLayer(Wrapper): - generic case: - `get_affine_bounds()`: affine bounds on layer output w.r.t. layer input - `forward_ibp_propagate()`: ibp bounds on layer ouput knowing ibp bounds on layer inpu + - set class attribute `diagonal` to True if the affine relaxation is representing w as a diagonal + (see explanation for `diagonal`). Other possibilities exist like overriding directly - `forward_affine_propagate()` @@ -45,6 +47,24 @@ class DecomonLayer(Wrapper): """ + diagonal: bool = False + """Flag telling that the layer affine relaxation can be represented in a diagonal manner. + + This is useful only to compute properly output shapes without making actual computations, during call on symbolic tensors. + During a call on actual tensors, this is not used. + + If diagonal is set to True, this means that if the affine relaxation of the layer is w_l, b_l, w_u, b_u then + weights and bias will all have the same shape, and that the full representation can be retrieved + + - either in linear case with no batch axis by + w_full = K.reshape(K.diag(K.flatten(w))) + - or in generic case with a batch axis by the same computation, batch element by batch element, i.e. + w_full = K.concatenate([K.reshape(K.diag(K.flatten(w[i])), w.shape + w.shape)[None] for i in range(len(w))], axis=0) + + In this case, the computations when merging affine bounds can be simplified. + + """ + def __init__( self, layer: Layer, @@ -52,6 +72,7 @@ def __init__( ibp: bool = True, affine: bool = True, propagation: Propagation = Propagation.FORWARD, + model_output_shape_length: int = 0, **kwargs: Any, ): """ @@ -63,6 +84,10 @@ def __init__( propagation: direction of bounds propagation - forward: from input to output - backward: from output to input + model_output_shape_length: length of the shape of the model output (omitting batch axis) + w.r.t which backward affine bounds will be computed. + It allows determining if the backward bounds are with a bacth axis or not. + This has no meaning in forward propagation. **kwargs: """ @@ -83,12 +108,15 @@ def __init__( raise ValueError(f"The underlying keras layer {layer.name} is not built.") if not ibp and not affine: raise ValueError("ibp and affine cannot be both False.") + if propagation == Propagation.BACKWARD and model_output_shape_length == 0: + raise ValueError("model_output_shape_length must be positive in backward propagation.") # attributes self.ibp = ibp self.affine = affine self.perturbation_domain = perturbation_domain self.propagation = propagation + self.model_output_shape_length = model_output_shape_length def get_config(self) -> dict[str, Any]: config = super().get_config() @@ -98,6 +126,7 @@ def get_config(self) -> dict[str, Any]: "affine": self.affine, "perturbation_domain": self.perturbation_domain, "propagation": self.propagation, + "model_output_shape_length": self.model_output_shape_length, } ) return config @@ -123,9 +152,23 @@ def get_affine_representation(self) -> tuple[Tensor, Tensor]: layer(z) = batch_multid_dot(z, w, missing_batchsize=(False, True)) + b ``` + If w can be represented as a diagonal tensor, which means that the full version of w is retrieved by + w_full = K.reshape(K.diag(K.flatten(w)), w.shape + w.shape) + then + - class attribute `diagonal` should be set to True (in order to have a correct `compute_output_shape()`) + - we got + ``` + layer(z) = batch_multid_dot(z, w, missing_batchsize=(False, True), diagonal=(False, True)) + b + ``` + Shapes: !no batchsize! - w ~ self.layer.input.shape[1:] + self.layer.output.shape[1:] - b ~ self.layer.output.shape[1:] + if diagonal is False: + w ~ self.layer.input.shape[1:] + self.layer.output.shape[1:] + b ~ self.layer.output.shape[1:] + if diagonal is True: + w ~ self.layer.output.shape[1:] + b ~ self.layer.output.shape[1:] + """ if not self.linear: @@ -155,10 +198,19 @@ def get_affine_bounds(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, Tenso w_l * z + b_l <= layer(z) <= w_u * z + b_u with lower <= z <= upper + If w(_l or_u) can be represented as a diagonal tensor, which means that the full version of w is retrieved by + w_full = K.concatenate([K.reshape(K.diag(K.flatten(w[i])), w.shape + w.shape)[None] for i in range(len(w))], axis=0) + then the class attribute `diagonal` should be set to True (in order to have a correct `compute_output_shape()`) + Shapes: - lower, upper ~ (batchsize,) + self.layer.input.shape[1:] - w_l, w_u ~ (batchsize,) + self.layer.input.shape[1:] + self.layer.output.shape[1:] - b_l, b_u ~ (batchsize,) + self.layer.output.shape[1:] + if diagonal is False: + lower, upper ~ (batchsize,) + self.layer.input.shape[1:] + w_l, w_u ~ (batchsize,) + self.layer.input.shape[1:] + self.layer.output.shape[1:] + b_l, b_u ~ (batchsize,) + self.layer.output.shape[1:] + if diagonal is True: + lower, upper ~ (batchsize,) + self.layer.input.shape[1:] + w_l, w_u ~ (batchsize,) + self.layer.output.shape[1:] + b_l, b_u ~ (batchsize,) + self.layer.output.shape[1:] Note: `w * z` means here `batch_multid_dot(z, w)`. @@ -458,10 +510,31 @@ def compute_output_shape( if self.affine: keras_layer_input_shape_wo_batchsize = self.layer.input.shape[1:] keras_layer_output_shape_wo_batchsize = self.layer.output.shape[1:] - w_in_shape = affine_bounds_to_propagate_shape[0] - model_input_shape = w_in_shape[: -len(keras_layer_input_shape_wo_batchsize)] - w_out_shape = model_input_shape + keras_layer_output_shape_wo_batchsize - b_out_shape = self.layer.output.shape + + # inputs are in diagonal representation? without batch axis? + if self.is_diagonal_bounds_shape(affine_bounds_to_propagate_shape): + model_input_shape_wo_batchsize = keras_layer_input_shape_wo_batchsize + else: + w_in_shape = affine_bounds_to_propagate_shape[0] + if self.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): + model_input_shape_wo_batchsize = w_in_shape[: -len(keras_layer_input_shape_wo_batchsize)] + else: + model_input_shape_wo_batchsize = w_in_shape[1 : -len(keras_layer_input_shape_wo_batchsize)] + + # outputs shape depends if layer and inputs are diagonal / linear (w/o batch) + b_out_shape_wo_batchsize = keras_layer_output_shape_wo_batchsize + if self.diagonal and self.is_diagonal_bounds_shape(affine_bounds_to_propagate_shape): + # propagated bounds still diagonal + w_out_shape_wo_batchsize = b_out_shape_wo_batchsize + else: + w_out_shape_wo_batchsize = model_input_shape_wo_batchsize + keras_layer_output_shape_wo_batchsize + if self.linear and self.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): + # no batch in propagated bounds + w_out_shape = w_out_shape_wo_batchsize + b_out_shape = b_out_shape_wo_batchsize + else: + w_out_shape = (None,) + w_out_shape_wo_batchsize + b_out_shape = (None,) + b_out_shape_wo_batchsize affine_bounds_propagated_shape = [w_out_shape, b_out_shape, w_out_shape, b_out_shape] else: affine_bounds_propagated_shape = [] @@ -469,10 +542,29 @@ def compute_output_shape( return [affine_bounds_propagated_shape, constant_bounds_propagated_shape] else: # backward - b_shape = affine_bounds_to_propagate_shape[1] - model_output_shape_wo_batchsize = b_shape[1:] - w_shape = self.layer.input.shape + model_output_shape_wo_batchsize - affine_bounds_propagated_shape = [w_shape, b_shape, w_shape, b_shape] + # find model output shape + if self.is_identity_bounds_shape(affine_bounds_to_propagate_shape): + model_output_shape_wo_batchsize = self.layer.output.shape[1:] + else: + b_shape = affine_bounds_to_propagate_shape[1] + if self.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): + model_output_shape_wo_batchsize = b_shape + else: + model_output_shape_wo_batchsize = b_shape[1:] + # outputs shape depends if layer and inputs are diagonal / linear (w/o batch) + b_shape_wo_batchisze = model_output_shape_wo_batchsize + if self.diagonal and self.is_diagonal_bounds_shape(affine_bounds_to_propagate_shape): + w_shape_wo_batchsize = model_output_shape_wo_batchsize + else: + w_shape_wo_batchsize = self.layer.input.shape[1:] + model_output_shape_wo_batchsize + if self.linear and self.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): + b_new_shape = b_shape_wo_batchisze + w_shape = w_shape_wo_batchsize + else: + b_new_shape = (None,) + b_shape_wo_batchisze + w_shape = (None,) + w_shape_wo_batchsize + + affine_bounds_propagated_shape = [w_shape, b_new_shape, w_shape, b_new_shape] return [affine_bounds_propagated_shape] @@ -489,6 +581,42 @@ def replace_empty_tensor(l: Union[keras.KerasTensor, list[keras.KerasTensor]]): return [replace_empty_tensor(l) for l in output_spec] + def is_identity_bounds(self, affine_bounds: list[Tensor]) -> bool: + return len(affine_bounds) == 0 + + def is_identity_bounds_shape(self, affine_bounds_shape: list[tuple[Optional[int], ...]]) -> bool: + return len(affine_bounds_shape) == 0 + + def is_diagonal_bounds(self, affine_bounds: list[Tensor]) -> bool: + if self.is_identity_bounds(affine_bounds): + return True + w, b = affine_bounds[:2] + return w.shape == b.shape + + def is_diagonal_bounds_shape(self, affine_bounds_shape: list[tuple[Optional[int], ...]]) -> bool: + if self.is_identity_bounds_shape(affine_bounds_shape): + return True + w_shape, b_shape = affine_bounds_shape[:2] + return w_shape == b_shape + + def is_wo_batch_bounds(self, affine_bounds: list[Tensor]) -> bool: + if self.is_identity_bounds(affine_bounds): + return True + b = affine_bounds[1] + if self.propagation == Propagation.FORWARD: + return len(b.shape) == len(self.layer.input.shape) - 1 + else: + return len(b.shape) == self.model_output_shape_length + + def is_wo_batch_bounds_shape(self, affine_bounds_shape: list[tuple[Optional[int], ...]]) -> bool: + if self.is_identity_bounds_shape(affine_bounds_shape): + return True + b_shape = affine_bounds_shape[1] + if self.propagation == Propagation.FORWARD: + return len(b_shape) == len(self.layer.input.shape) - 1 + else: + return len(b_shape) == self.model_output_shape_length + def combine_affine_bounds( affine_bounds_1: list[Tensor], From 36804535b44d8bb6b0bb54bdde3a0da525836c4d Mon Sep 17 00:00:00 2001 From: Nolwen Date: Sat, 10 Feb 2024 01:06:08 +0100 Subject: [PATCH 022/101] Update DecomonDense to adapt to empty, diagonal, and w/o batch inputs --- src/decomon/layers/core/dense.py | 38 +++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/src/decomon/layers/core/dense.py b/src/decomon/layers/core/dense.py index 2a93d626..1bb817c4 100644 --- a/src/decomon/layers/core/dense.py +++ b/src/decomon/layers/core/dense.py @@ -60,11 +60,15 @@ def forward_affine_propagate( w_l_1, b_l_1, w_u_1, b_u_1 = input_affine_bounds w_2, b_2 = self._get_pseudo_affine_representation() diagonal = ( - w_l_1.shape == b_l_1.shape, + self.is_diagonal_bounds(input_affine_bounds), False, ) - kwargs_dot_w = dict(nb_merging_axes=1, missing_batchsize=(False, True), diagonal=diagonal) - kwargs_dot_b = dict(nb_merging_axes=1, missing_batchsize=(False, True)) + missing_batchsize = ( + self.is_wo_batch_bounds(input_affine_bounds), + True, + ) + kwargs_dot_w = dict(nb_merging_axes=1, missing_batchsize=missing_batchsize, diagonal=diagonal) + kwargs_dot_b = dict(nb_merging_axes=1, missing_batchsize=missing_batchsize) z_value = K.cast(0.0, dtype=w_2.dtype) w_2_pos = K.maximum(w_2, z_value) @@ -84,25 +88,37 @@ def backward_affine_propagate( w_l_2, b_l_2, w_u_2, b_u_2 = output_affine_bounds # affine bounds represented in diagonal mode? - diagonal_bounds = w_l_2.shape == b_l_2.shape + diagonal_bounds = self.is_diagonal_bounds(output_affine_bounds) if diagonal_bounds: raise NotImplementedError + # missing batch axis in affine bounds? + nb_batch_axis = 0 if self.is_wo_batch_bounds(output_affine_bounds) else 1 nb_nonbatch_axes_keras_input = len(w_l_2.shape) - len(b_l_2.shape) # Merge weights on "units" axis and reorder axes transposed_axes = ( - tuple(range(1, nb_nonbatch_axes_keras_input + 1)) + tuple(range(1, nb_nonbatch_axes_keras_input + nb_batch_axis)) + (0,) - + tuple(range(nb_nonbatch_axes_keras_input + 1, len(w_l_2.shape))) + + tuple(range(nb_nonbatch_axes_keras_input + nb_batch_axis, len(w_l_2.shape))) + ) + w_l = K.transpose( + K.tensordot(w_1, w_l_2, axes=[[-1], [nb_nonbatch_axes_keras_input - 1 + nb_batch_axis]]), + axes=transposed_axes, + ) + w_u = K.transpose( + K.tensordot(w_1, w_u_2, axes=[[-1], [nb_nonbatch_axes_keras_input - 1 + nb_batch_axis]]), + axes=transposed_axes, ) - w_l = K.transpose(K.tensordot(w_1, w_l_2, axes=[[-1], [nb_nonbatch_axes_keras_input]]), axes=transposed_axes) - w_u = K.transpose(K.tensordot(w_1, w_u_2, axes=[[-1], [nb_nonbatch_axes_keras_input]]), axes=transposed_axes) # Merge layer bias with backward weights on "units" axe and reduce on other input axes - reduced_axes = list(range(1, nb_nonbatch_axes_keras_input)) - b_l = K.sum(K.tensordot(b_1, w_l_2, axes=[[-1], [nb_nonbatch_axes_keras_input]]), axis=reduced_axes) - b_u = K.sum(K.tensordot(b_1, w_u_2, axes=[[-1], [nb_nonbatch_axes_keras_input]]), axis=reduced_axes) + reduced_axes = list(range(1, nb_nonbatch_axes_keras_input - 1 + nb_batch_axis)) + b_l = K.sum( + K.tensordot(b_1, w_l_2, axes=[[-1], [nb_nonbatch_axes_keras_input - 1 + nb_batch_axis]]), axis=reduced_axes + ) + b_u = K.sum( + K.tensordot(b_1, w_u_2, axes=[[-1], [nb_nonbatch_axes_keras_input - 1 + nb_batch_axis]]), axis=reduced_axes + ) # Add bias from current backward bounds b_l += b_l_2 From 9efb6079598e0fb05c57f6006c5c64674dd40c3a Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 12 Feb 2024 09:26:12 +0100 Subject: [PATCH 023/101] Update perturbation domain to manage empty/diagonal/no batch affine bounds For affine bounds empty (ie identity), we add a get_lower_x and get_upper_x method that return lower and upper bounds on the model input x. We implement it in box case. We also take car of diagonal/ w/o batchsize inputs in box case. Ball perturbation domains are to be completed later. --- src/decomon/core.py | 45 +++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/src/decomon/core.py b/src/decomon/core.py index b5f648f5..2bf104c0 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -29,6 +29,14 @@ class PerturbationDomain(ABC): def __init__(self, opt_option: Union[str, Option] = Option.milp): self.opt_option = Option(opt_option) + @abstractmethod + def get_upper_x(self, x: Tensor) -> Tensor: + ... + + @abstractmethod + def get_lower_x(self, x: Tensor) -> Tensor: + ... + @abstractmethod def get_upper(self, x: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: ... @@ -65,6 +73,12 @@ def get_lower(self, x: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: x_max = x[:, 1] return get_lower_box(x_min=x_min, x_max=x_max, w=w, b=b, **kwargs) + def get_upper_x(self, x: Tensor) -> Tensor: + return x[:, 1] + + def get_lower_x(self, x: Tensor) -> Tensor: + return x[:, 0] + def get_nb_x_components(self) -> int: return 2 @@ -484,16 +498,26 @@ def get_upper_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: Returns: max_(x >= x_min, x<=x_max) w*x + b - """ - if len(w.shape) == len(b.shape): - raise NotImplementedError + Note: + We can have w, b in diagonal representation and/or without a batch axis. + We assume that x_min, x_max have always its batch axis. + """ z_value = K.cast(0.0, dtype=x_min.dtype) w_pos = K.maximum(w, z_value) w_neg = K.minimum(w, z_value) - return batch_multid_dot(x_max, w_pos) + batch_multid_dot(x_min, w_neg) + b + is_diag = w.shape == b.shape + is_wo_batch = len(b.shape) < len(x_min.shape) + diagonal = (False, is_diag) + missing_batchsize = (False, is_wo_batch) + + return ( + batch_multid_dot(x_max, w_pos, diagonal=diagonal, missing_batchsize=missing_batchsize) + + batch_multid_dot(x_min, w_neg, diagonal=diagonal, missing_batchsize=missing_batchsize) + + b + ) def get_lower_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: @@ -506,16 +530,13 @@ def get_lower_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: Returns: min_(x >= x_min, x<=x_max) w*x + b - """ - - if len(w.shape) == len(b.shape): - raise NotImplementedError - z_value = K.cast(0.0, dtype=x_min.dtype) - w_pos = K.maximum(w, z_value) - w_neg = K.minimum(w, z_value) + Note: + We can have w, b in diagonal representation and/or without a batch axis. + We assume that x_min, x_max have always its batch axis. - return batch_multid_dot(x_min, w_pos) + batch_multid_dot(x_max, w_neg) + b + """ + return get_upper_box(x_min=x_max, x_max=x_min, w=w, b=b, **kwargs) def get_lq_norm(x: Tensor, p: float, axis: int = -1) -> Tensor: From e563236f20752615bae431d5029eaecd0b330541 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 12 Feb 2024 09:31:33 +0100 Subject: [PATCH 024/101] Adapt DecomonLayer to affine inputs w/o batchsize or empty (identity) --- src/decomon/layers/layer.py | 77 +++++++++++++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 8 deletions(-) diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index 18192291..183ce0a8 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -302,10 +302,12 @@ def forward_affine_propagate( w_l, b_l, w_u, b_u = self.get_affine_bounds(lower=lower, upper=upper) layer_affine_bounds = [w_l, b_l, w_u, b_u] + from_linear_layer = (self.is_wo_batch_bounds(input_affine_bounds), self.linear) + return combine_affine_bounds( affine_bounds_1=input_affine_bounds, affine_bounds_2=layer_affine_bounds, - from_linear_layer=(False, self.linear), + from_linear_layer=from_linear_layer, ) def backward_affine_propagate( @@ -354,10 +356,12 @@ def backward_affine_propagate( w_l, b_l, w_u, b_u = self.get_affine_bounds(lower=lower, upper=upper) layer_affine_bounds = [w_l, b_l, w_u, b_u] + from_linear_layer = (self.linear, self.is_wo_batch_bounds((output_affine_bounds))) + return combine_affine_bounds( affine_bounds_1=layer_affine_bounds, affine_bounds_2=output_affine_bounds, - from_linear_layer=(self.linear, False), + from_linear_layer=from_linear_layer, ) def get_forward_oracle( @@ -387,9 +391,14 @@ def get_forward_oracle( return input_constant_bounds elif self.affine: - w_l, b_l, w_u, b_u = input_affine_bounds - l_affine = self.perturbation_domain.get_lower(x, w_l, b_l) - u_affine = self.perturbation_domain.get_upper(x, w_u, b_u) + if len(input_affine_bounds) == 0: + # special case: empty affine bounds => identity bounds + l_affine = self.perturbation_domain.get_lower_x(x) + u_affine = self.perturbation_domain.get_upper_x(x) + else: + w_l, b_l, w_u, b_u = input_affine_bounds + l_affine = self.perturbation_domain.get_lower(x, w_l, b_l) + u_affine = self.perturbation_domain.get_upper(x, w_u, b_u) return [l_affine, u_affine] else: @@ -401,7 +410,9 @@ def call_forward( """Propagate forward affine and constant bounds through the layer. Args: - affine_bounds_to_propagate: affine bounds on keras layer input w.r.t model input . Can be empty if not in affine mode. + affine_bounds_to_propagate: affine bounds on keras layer input w.r.t model input. + Can be empty if not in affine mode. + Can also be empty in case of identity affine bounds => we simply return layer affine bounds. input_bounds_to_propagate: ibp constant bounds on keras layer input. Can be empty if not in ibp mode. x: model input. Necessary only in affine mode. @@ -468,7 +479,9 @@ def call( """Propagate bounds in the specified direction `self.propagation`. Args: - affine_bounds_to_propagate: affine bounds to propagate. Can be empty in forward direction if self.affine is False. + affine_bounds_to_propagate: affine bounds to propagate. + Can be empty in forward direction if self.affine is False. + Can also be empty in case of identity affine bounds => we simply return layer affine bounds. constant_oracle_bounds: in forward direction, the ibp bounds (empty if self.ibp is False); in backward direction, the oracle constant bounds on keras inputs x: the model input. Necessary only in forward direction when self.affine is True. @@ -687,7 +700,9 @@ def combine_affine_bounds( affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal ) else: - raise NotImplementedError() + return _combine_affine_bounds_both_from_linear( + affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal + ) def _combine_affine_bounds_generic( @@ -843,3 +858,49 @@ def _combine_affine_bounds_left_from_linear( b_u = batch_multid_dot(b_1, w_u_2, **kwargs_dot_b) + b_u_2 return w_l, b_l, w_u, b_u + + +def _combine_affine_bounds_both_from_linear( + affine_bounds_1: list[Tensor], + affine_bounds_2: list[Tensor], + diagonal: tuple[bool, bool], +) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Combine affine bounds + + Args: + affine_bounds_1: [w_1, b_1, w_1, b_1] first affine bounds, with lower=upper + no batchsize + affine_bounds_2: [w_2, b_2, w_2, b_2] second affine bounds, with lower=upper + no batchsize + diagonal: specify if weights of each affine bounds are in diagonal representation or not + + Returns: + w, b, w, b: combined affine bounds + + If x, y, z satisfy + y = w_1 * x + b_1 + z = w_2 * x + b_2 + + Then + z = w * x + b + + """ + w_1, b_1 = affine_bounds_1[:2] + w_2, b_2 = affine_bounds_2[:2] + nb_axes_wo_batchsize_y = len(b_1.shape) + missing_batchsize = (True, True) + + #   NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b + kwargs_dot_w = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + missing_batchsize=missing_batchsize, + diagonal=diagonal, + ) + kwargs_dot_b = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + missing_batchsize=missing_batchsize, + diagonal=(False, diagonal[1]), + ) + + w = batch_multid_dot(w_1, w_2, **kwargs_dot_w) + b = batch_multid_dot(b_1, w_2, **kwargs_dot_b) + b_2 + + return w, b, w, b From 337abc79663191c1333b00bb8b87a71fde53af99 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 8 Feb 2024 17:46:36 +0100 Subject: [PATCH 025/101] Implement DecomonActivation --- src/decomon/layers/activations/activation.py | 165 +++++++++++++++++++ tests/test_activation.py | 62 +++++++ 2 files changed, 227 insertions(+) create mode 100644 tests/test_activation.py diff --git a/src/decomon/layers/activations/activation.py b/src/decomon/layers/activations/activation.py index e69de29b..65b19ed4 100644 --- a/src/decomon/layers/activations/activation.py +++ b/src/decomon/layers/activations/activation.py @@ -0,0 +1,165 @@ +from collections.abc import Callable +from typing import Any, Optional + +from keras import Layer +from keras.activations import linear, relu +from keras.layers import Activation + +from decomon.core import PerturbationDomain, Propagation, Slope +from decomon.keras_utils import BatchedDiagLike +from decomon.layers.layer import DecomonLayer +from decomon.types import Tensor +from decomon.utils import get_linear_hull_relu + + +class DecomonBaseActivation(DecomonLayer): + """Base class for decomon layers corresponding to activation layers.""" + + def __init__( + self, + layer: Layer, + perturbation_domain: Optional[PerturbationDomain] = None, + ibp: bool = True, + affine: bool = True, + propagation: Propagation = Propagation.FORWARD, + slope: Slope = Slope.V_SLOPE, + **kwargs: Any, + ): + super().__init__( + layer=layer, + perturbation_domain=perturbation_domain, + ibp=ibp, + affine=affine, + propagation=propagation, + **kwargs, + ) + self.slope = slope + + def forward_ibp_propagate(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, Tensor]: + """Propagate ibp bounds through the activation layer. + + By default, we simply apply the activation function on bounds. + This is correct when the activation is an increasing function (like relu). + This is not correct when the activation is not monotonic (like gelu). + + Args: + lower: + upper: + + Returns: + + """ + return self.layer.activation(lower), self.layer.activation(upper) + + +class DecomonActivation(DecomonBaseActivation): + """Wrapping class for all decomon activation layer. + + Correspond to keras Activation layer. + Will wrap a more specific activation Layer (DecomonRelu, DecomonLinear, ...) + as it exists also a dedicated Relu layer in keras. + + """ + + layer: Activation + decomon_activation: DecomonBaseActivation + + def __init__( + self, + layer: Layer, + perturbation_domain: Optional[PerturbationDomain] = None, + ibp: bool = True, + affine: bool = True, + propagation: Propagation = Propagation.FORWARD, + slope: Slope = Slope.V_SLOPE, + **kwargs: Any, + ): + super().__init__( + layer=layer, + perturbation_domain=perturbation_domain, + ibp=ibp, + affine=affine, + propagation=propagation, + slope=slope, + **kwargs, + ) + self.slope = slope + decomon_activation_class = get(self.layer.activation) + self.decomon_activation = decomon_activation_class( + layer=layer, + perturbation_domain=perturbation_domain, + ibp=ibp, + affine=affine, + propagation=propagation, + slope=slope, + **kwargs, + ) + + def get_affine_representation(self) -> tuple[Tensor, Tensor]: + return self.decomon_activation.get_affine_representation() + + def get_affine_bounds(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, Tensor, Tensor, Tensor]: + return self.decomon_activation.get_affine_bounds(lower=lower, upper=upper) + + def forward_affine_propagate( + self, input_affine_bounds, input_constant_bounds + ) -> tuple[Tensor, Tensor, Tensor, Tensor]: + return self.decomon_activation.forward_affine_propagate( + input_affine_bounds=input_affine_bounds, input_constant_bounds=input_constant_bounds + ) + + def backward_affine_propagate( + self, output_affine_bounds, input_constant_bounds + ) -> tuple[Tensor, Tensor, Tensor, Tensor]: + return self.decomon_activation.backward_affine_propagate( + output_affine_bounds=output_affine_bounds, input_constant_bounds=input_constant_bounds + ) + + def forward_ibp_propagate(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, Tensor]: + return self.decomon_activation.forward_ibp_propagate(lower=lower, upper=upper) + + def build(self, affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, x_shape): + self.decomon_activation.build( + affine_bounds_to_propagate_shape=affine_bounds_to_propagate_shape, + constant_oracle_bounds_shape=constant_oracle_bounds_shape, + x_shape=x_shape, + ) + + def call( + self, affine_bounds_to_propagate: list[Tensor], constant_oracle_bounds: list[Tensor], x: Tensor + ) -> list[list[Tensor]]: + return self.decomon_activation.call(affine_bounds_to_propagate, constant_oracle_bounds, x) + + +class DecomonLinear(DecomonBaseActivation): + linear = True + + def call( + self, affine_bounds_to_propagate: list[Tensor], constant_oracle_bounds: list[Tensor], x: Tensor + ) -> list[list[Tensor]]: + if self.propagation == Propagation.FORWARD: + return [affine_bounds_to_propagate, constant_oracle_bounds] + else: + return [affine_bounds_to_propagate] + + +class DecomonReLU(DecomonBaseActivation): + diagonal = True + + def get_affine_bounds(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, Tensor, Tensor, Tensor]: + w_u, b_u, w_l, b_l = get_linear_hull_relu(upper=upper, lower=lower, slope=self.slope) + return w_l, b_l, w_u, b_u + + +MAPPING_KERAS_ACTIVATION_TO_DECOMON_ACTIVATION: dict[Callable[[Tensor], Tensor], type[DecomonBaseActivation]] = { + linear: DecomonLinear, + relu: DecomonReLU, +} + + +def get(identifier: Any) -> type[DecomonBaseActivation]: + """Retrieve a decomon activation layer via an identifier.""" + try: + return MAPPING_KERAS_ACTIVATION_TO_DECOMON_ACTIVATION[identifier] + except KeyError: + raise NotImplementedError(f"No decomon layer existing for activation function {identifier}") diff --git a/tests/test_activation.py b/tests/test_activation.py new file mode 100644 index 00000000..e9f9b413 --- /dev/null +++ b/tests/test_activation.py @@ -0,0 +1,62 @@ +import keras.ops as K +import numpy as np +import pytest +from keras.layers import Activation, Dense, Input + +from decomon.keras_utils import batch_multid_dot +from decomon.layers.activations.activation import DecomonActivation + + +@pytest.mark.parametrize("input_shape", [(1,), (3,), (5, 2, 3)], ids=["0d", "1d", "multid"]) +def test_decomon_activation( + activation, + slope, + ibp, + affine, + propagation, + input_shape, + perturbation_domain, + batchsize, + helpers, +): + decimal = 5 + decomon_layer_class = DecomonActivation + + keras_symbolic_input = Input(input_shape) + layer = Activation(activation=activation) + layer(keras_symbolic_input) + output_shape = layer.output.shape[1:] + + decomon_symbolic_inputs = helpers.get_decomon_symbolic_inputs( + model_input_shape=input_shape, + model_output_shape=output_shape, + layer_input_shape=input_shape, + layer_output_shape=output_shape, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + ) + decomon_layer = decomon_layer_class( + layer=layer, ibp=ibp, affine=affine, propagation=propagation, perturbation_domain=perturbation_domain, slope=slope + ) + decomon_layer(*decomon_symbolic_inputs) + + keras_input = helpers.generate_random_tensor(input_shape, batchsize=batchsize) + decomon_inputs = helpers.generate_simple_decomon_layer_inputs_from_keras_input( + keras_input=keras_input, + layer_output_shape=output_shape, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + ) + + keras_output = layer(keras_input) + + decomon_output = decomon_layer(*decomon_inputs) + + # check ibp and affine bounds well ordered w.r.t. keras output + helpers.assert_decomon_output_compare_with_keras_input_output_single_layer( + decomon_output=decomon_output, keras_output=keras_output, keras_input=keras_input + ) From f540d3336b77a3941d7d0a57c0e39aa358cc5a73 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 12 Feb 2024 10:27:57 +0100 Subject: [PATCH 026/101] Set model_output_shape_length in tests for backward decomon layers --- tests/test_activation.py | 9 ++++++++- tests/test_decomon_layer.py | 23 ++++++++++++++++++++--- tests/test_dense.py | 8 +++++++- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/tests/test_activation.py b/tests/test_activation.py index e9f9b413..18480a22 100644 --- a/tests/test_activation.py +++ b/tests/test_activation.py @@ -26,6 +26,7 @@ def test_decomon_activation( layer = Activation(activation=activation) layer(keras_symbolic_input) output_shape = layer.output.shape[1:] + model_output_shape_length = len(output_shape) decomon_symbolic_inputs = helpers.get_decomon_symbolic_inputs( model_input_shape=input_shape, @@ -38,7 +39,13 @@ def test_decomon_activation( perturbation_domain=perturbation_domain, ) decomon_layer = decomon_layer_class( - layer=layer, ibp=ibp, affine=affine, propagation=propagation, perturbation_domain=perturbation_domain, slope=slope + layer=layer, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + model_output_shape_length=model_output_shape_length, + slope=slope, ) decomon_layer(*decomon_symbolic_inputs) diff --git a/tests/test_decomon_layer.py b/tests/test_decomon_layer.py index 6a2409c1..a4156a72 100644 --- a/tests/test_decomon_layer.py +++ b/tests/test_decomon_layer.py @@ -21,6 +21,13 @@ def test_decomon_layer_nok_ibp_affine(): DecomonLayer(layer=layer, ibp=False, affine=False) +def test_decomon_layer_nok_backward_no_model_output_shape_length(): + layer = Dense(3) + layer(Input((1,))) + with pytest.raises(ValueError): + DecomonLayer(layer=layer, propagation=Propagation.BACKWARD) + + def test_decomon_layer_extra_kwargs(): layer = Dense(3) layer(Input((1,))) @@ -75,7 +82,7 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper model_output_shape = model_output_shape_if_no_singlelayer_model model_input_shape = (model_input_dim,) - + model_output_shape_length = len(model_output_shape) x_shape = perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape) # keras layer @@ -84,10 +91,20 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper # decomon layers linear_decomon_layer = MyLinearDecomonDense1d( - layer=layer, ibp=ibp, affine=affine, propagation=propagation, perturbation_domain=perturbation_domain + layer=layer, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + model_output_shape_length=model_output_shape_length, ) non_linear_decomon_layer = MyNonLinearDecomonDense1d( - layer=layer, ibp=ibp, affine=affine, propagation=propagation, perturbation_domain=perturbation_domain + layer=layer, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + model_output_shape_length=model_output_shape_length, ) # symbolic inputs diff --git a/tests/test_dense.py b/tests/test_dense.py index a15a6c44..4eb942ac 100644 --- a/tests/test_dense.py +++ b/tests/test_dense.py @@ -24,6 +24,7 @@ def test_decomon_dense( decimal = 5 units = 7 output_shape = input_shape[:-1] + (units,) + model_output_shape_length = len(output_shape) keras_symbolic_input = Input(input_shape) decomon_symbolic_inputs = helpers.get_decomon_symbolic_inputs( model_input_shape=input_shape, @@ -45,7 +46,12 @@ def test_decomon_dense( w.assign(np.random.random(w.shape)) decomon_layer = decomon_layer_class( - layer=layer, ibp=ibp, affine=affine, propagation=propagation, perturbation_domain=perturbation_domain + layer=layer, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + model_output_shape_length=model_output_shape_length, ) decomon_layer(*decomon_symbolic_inputs) From c445a69a2064183984232a9511aa26d535fd9b51 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 12 Feb 2024 12:38:59 +0100 Subject: [PATCH 027/101] Generate decomon inputs for each case (id, diag, no-batch) --- tests/conftest.py | 55 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fafc9577..c7b66e87 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -126,15 +126,20 @@ def get_decomon_input_shapes( affine, propagation, perturbation_domain, + empty=False, + diag=False, + nobatch=False, ): x_shape = perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape) - if affine: + if affine and not empty: if propagation == Propagation.FORWARD: b_in_shape = layer_input_shape w_in_shape = model_input_shape + layer_input_shape else: b_in_shape = model_output_shape w_in_shape = layer_output_shape + model_output_shape + if diag: + w_in_shape = b_in_shape affine_bounds_to_propagate_shape = [w_in_shape, b_in_shape, w_in_shape, b_in_shape] else: @@ -156,6 +161,9 @@ def get_decomon_symbolic_inputs( affine, propagation, perturbation_domain, + empty=False, + diag=False, + nobatch=False, ): """Generate decomon symbolic inputs for a decomon layer @@ -183,16 +191,30 @@ def get_decomon_symbolic_inputs( affine, propagation, perturbation_domain, + empty=empty, + diag=diag, + nobatch=nobatch, ) affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, x_shape = decomon_input_shape x = Input(x_shape) - affine_bounds_to_propagate = [Input(shape) for shape in affine_bounds_to_propagate_shape] constant_oracle_bounds = [Input(shape) for shape in constant_oracle_bounds_shape] + if nobatch: + affine_bounds_to_propagate = [Input(batch_shape=shape) for shape in affine_bounds_to_propagate_shape] + else: + affine_bounds_to_propagate = [Input(shape=shape) for shape in affine_bounds_to_propagate_shape] return [affine_bounds_to_propagate, constant_oracle_bounds, x] @staticmethod def generate_simple_decomon_layer_inputs_from_keras_input( - keras_input, layer_output_shape, ibp, affine, propagation, perturbation_domain + keras_input, + layer_output_shape, + ibp, + affine, + propagation, + perturbation_domain, + empty=False, + diag=False, + nobatch=False, ): """Generate simple decomon inputs for a layer from the corresponding keras input @@ -210,6 +232,9 @@ def generate_simple_decomon_layer_inputs_from_keras_input( affine: propagation: perturbation_domain: + empty: + diag: + nobatch: Returns: @@ -219,19 +244,29 @@ def generate_simple_decomon_layer_inputs_from_keras_input( else: raise NotImplementedError - if affine: + if affine and not empty: batchsize = keras_input.shape[0] if propagation == Propagation.FORWARD: bias_shape = keras_input.shape[1:] else: bias_shape = layer_output_shape flatten_bias_dim = int(np.prod(bias_shape)) - w_in = K.repeat( - K.reshape(K.eye(flatten_bias_dim), bias_shape + bias_shape)[None], - batchsize, - axis=0, - ) - b_in = K.zeros((batchsize,) + bias_shape) + if diag: + w_in = K.ones(bias_shape) + else: + w_in = K.reshape(K.eye(flatten_bias_dim), bias_shape + bias_shape) + b_in = K.zeros(bias_shape) + if not nobatch: + w_in = K.repeat( + w_in[None], + batchsize, + axis=0, + ) + b_in = K.repeat( + b_in[None], + batchsize, + axis=0, + ) affine_bounds_to_propagate = [w_in, b_in, w_in, b_in] else: affine_bounds_to_propagate = [] From 84546de57a2e4f432b5e14eaed6795f1ee6a4368 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 12 Feb 2024 17:38:04 +0100 Subject: [PATCH 028/101] Add a fixture managing different keras/input inputs --- tests/conftest.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index c7b66e87..2c5564e0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import pytest from keras import KerasTensor, Model from keras.layers import Input -from pytest_cases import param_fixture, param_fixtures +from pytest_cases import fixture, fixture_union, param_fixture, param_fixtures from decomon.core import BoxDomain, Propagation, Slope from decomon.keras_utils import ( @@ -20,6 +20,17 @@ from decomon.models.utils import ConvertMethod from decomon.types import BackendTensor, Tensor +empty, diag, nobatch = param_fixtures( + "empty, diag, nobatch", + [ + (True, True, True), + (False, True, True), + (False, True, False), + (False, False, True), + (False, False, False), + ], + ids=["identity", "diagonal-nobatch", "diagonal", "nobatch", "generic"], +) ibp, affine, propagation = param_fixtures( "ibp, affine, propagation", [ @@ -38,6 +49,7 @@ activation = param_fixture("activation", [None, "relu"]) data_format = param_fixture("data_format", ["channels_last", "channels_first"]) method = param_fixture("method", [m.value for m in ConvertMethod]) +input_shape = param_fixture("input_shape", [(1,), (3,), (5, 2, 3)], ids=["0d", "1d", "multid"]) @pytest.fixture @@ -376,3 +388,47 @@ def predict_on_small_numpy( @pytest.fixture def helpers(): return Helpers + + +@fixture +def simple_layer_input_functions( + ibp, affine, propagation, perturbation_domain, batchsize, input_shape, empty, diag, nobatch, helpers +): + keras_symbolic_input_fn = lambda: Input(input_shape) + + decomon_symbolic_input_fn = lambda output_shape: helpers.get_decomon_symbolic_inputs( + model_input_shape=input_shape, + model_output_shape=output_shape, + layer_input_shape=input_shape, + layer_output_shape=output_shape, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + empty=empty, + diag=diag, + nobatch=nobatch, + ) + + keras_input_fn = lambda: helpers.generate_random_tensor(input_shape, batchsize=batchsize) + + decomon_input_fn = lambda keras_input, output_shape: helpers.generate_simple_decomon_layer_inputs_from_keras_input( + keras_input=keras_input, + layer_output_shape=output_shape, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + empty=empty, + diag=diag, + nobatch=nobatch, + ) + + return keras_symbolic_input_fn, decomon_symbolic_input_fn, keras_input_fn, decomon_input_fn + + +layer_input_functions = fixture_union( + "layer_input_functions", + [simple_layer_input_functions], + unpack_into="keras_symbolic_input_fn, decomon_symbolic_input_fn, keras_input_fn, decomon_input_fn", +) From a0334b9ea1b5915e68a028df7450ecf98e81690a Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 12 Feb 2024 17:38:43 +0100 Subject: [PATCH 029/101] Add a test for methods checking types of inputs (identity/diag/nobatch) --- tests/test_decomon_layer.py | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_decomon_layer.py b/tests/test_decomon_layer.py index a4156a72..87ce3955 100644 --- a/tests/test_decomon_layer.py +++ b/tests/test_decomon_layer.py @@ -210,3 +210,59 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper decomon_output=linear_decomon_output_val, keras_output=keras_output_val, keras_input=keras_input_val ) helpers.assert_decomon_outputs_equal(linear_decomon_output_val, non_linear_decomon_output_val) + + +def test_check_affine_bounds_characteristics( + ibp, + affine, + propagation, + perturbation_domain, + empty, + diag, + nobatch, + keras_symbolic_input_fn, + decomon_symbolic_input_fn, + keras_input_fn, + decomon_input_fn, + helpers, +): + units = 7 + + keras_symbolic_input = keras_symbolic_input_fn() + input_shape = keras_symbolic_input.shape[1:] + output_shape = input_shape[:-1] + (units,) + model_output_shape_length = len(output_shape) + decomon_symbolic_input = decomon_symbolic_input_fn(output_shape=output_shape) + keras_input = keras_input_fn() + decomon_input = decomon_input_fn(keras_input=keras_input, output_shape=output_shape) + + layer = Dense(units=units) + layer(keras_symbolic_input) + + decomon_layer = DecomonLayer( + layer=layer, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + model_output_shape_length=model_output_shape_length, + ) + + if affine: + affine_bounds = decomon_symbolic_input[0] + affine_bounds_shape = [t.shape for t in affine_bounds] + assert decomon_layer.is_identity_bounds(affine_bounds) is empty + assert decomon_layer.is_diagonal_bounds(affine_bounds) is diag + assert decomon_layer.is_wo_batch_bounds(affine_bounds) is nobatch + assert decomon_layer.is_identity_bounds_shape(affine_bounds_shape) is empty + assert decomon_layer.is_diagonal_bounds_shape(affine_bounds_shape) is diag + assert decomon_layer.is_wo_batch_bounds_shape(affine_bounds_shape) is nobatch + + affine_bounds = decomon_input[0] + affine_bounds_shape = [t.shape for t in affine_bounds] + assert decomon_layer.is_identity_bounds(affine_bounds) is empty + assert decomon_layer.is_diagonal_bounds(affine_bounds) is diag + assert decomon_layer.is_wo_batch_bounds(affine_bounds) is nobatch + assert decomon_layer.is_identity_bounds_shape(affine_bounds_shape) is empty + assert decomon_layer.is_diagonal_bounds_shape(affine_bounds_shape) is diag + assert decomon_layer.is_wo_batch_bounds_shape(affine_bounds_shape) is nobatch From 7ea50b49a7229882b9f770763314b162eba66d4a Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 12 Feb 2024 22:03:40 +0100 Subject: [PATCH 030/101] Change (flatten) inputs/outputs format for DecomonLayer.call() To be consistent with DecomonModel that should be seen as special DecomonLayer and that, as functional keras model, can not be defined with inputs and outputs being nested lists of tensors, we choose to flatten inputs and outputs for DecomonLayer.call(). In other methods, that will be used by custom decomon layers developers we keep the split into affine, constant and model inputs. We manage all this split/flatten stuff necessary to go from call() format to call_backward/call_forward format, by a dedicated class InputsOutpusSpec. We update inputs generation utility functions used in tests accordingly. --- src/decomon/core.py | 151 +++++++++++++---- src/decomon/layers/activations/activation.py | 39 +++-- src/decomon/layers/core/dense.py | 8 +- src/decomon/layers/layer.py | 167 ++++++++----------- tests/conftest.py | 118 ++++++++----- tests/test_activation.py | 15 +- tests/test_decomon_layer.py | 92 +++++----- tests/test_dense.py | 18 +- 8 files changed, 374 insertions(+), 234 deletions(-) diff --git a/src/decomon/core.py b/src/decomon/core.py index 2bf104c0..69afc7af 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -171,54 +171,145 @@ def get_affine(mode: Union[str, ForwardMode] = ForwardMode.HYBRID) -> bool: class InputsOutputsSpec: """Storing specifications for inputs and outputs of decomon/backward layer/model.""" + layer_input_shape: tuple[int, ...] + model_input_shape: tuple[int, ...] + model_output_shape: tuple[int, ...] + def __init__( self, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, + ibp: bool = True, + affine: bool = True, + propagation: Propagation = Propagation.FORWARD, perturbation_domain: Optional[PerturbationDomain] = None, - model_input_dim: int = -1, + layer_input_shape: Optional[tuple[int, ...]] = None, + model_input_shape: Optional[tuple[int, ...]] = None, + model_output_shape: Optional[tuple[int, ...]] = None, ): """ Args: - dc_decomp: boolean that indicates whether we return a - difference of convex decomposition of our layer - mode: type of Forward propagation (ibp, affine, or hybrid) - perturbation_domain: type of perturbation domain (box, ball, ...) + perturbation_domain: type of perturbation domain (box, ball, ...). Default to a box domain + ibp: if True, forward propagate constant bounds + affine: if True, forward propagate affine bounds + propagation: direction of bounds propagation + - forward: from input to output + - backward: from output to input + layer_input_shape: shape of the underlying keras layer input (w/o the batch axis) + model_input_shape: shape of the underlying keras model input (w/o the batch axis) + model_output_shape: shape of the underlying keras model output (w/o the batch axis) """ - - self.model_input_dim = model_input_dim - self.mode = ForwardMode(mode) - self.dc_decomp = dc_decomp + # checks + if propagation == Propagation.BACKWARD and model_output_shape is None: + raise ValueError("model_output_shape must be set in backward propagation.") + if propagation == Propagation.FORWARD and layer_input_shape is None: + raise ValueError("layer_input_shape must be set in forward propagation.") + + self.propagation = propagation + self.affine = affine + self.ibp = ibp self.perturbation_domain: PerturbationDomain if perturbation_domain is None: self.perturbation_domain = BoxDomain() else: self.perturbation_domain = perturbation_domain - - @property - def nb_tensors(self) -> int: - if self.mode == ForwardMode.HYBRID: - nb_tensors = 7 - elif self.mode == ForwardMode.IBP: - nb_tensors = 2 - elif self.mode == ForwardMode.AFFINE: - nb_tensors = 5 + if model_output_shape is None: + self.model_output_shape = tuple() + else: + self.model_output_shape = model_output_shape + if model_input_shape is None: + self.model_input_shape = tuple() + else: + self.model_input_shape = model_input_shape + if layer_input_shape is None: + self.layer_input_shape = tuple() + else: + self.layer_input_shape = layer_input_shape + + def split_inputs(self, inputs: list[Tensor]) -> tuple[list[Tensor], list[Tensor], list[Tensor]]: + # Remove keras model input + if self.propagation == Propagation.FORWARD and self.affine: + x = inputs[-1] + inputs = inputs[:-1] + model_inputs = [x] + else: + model_inputs = [] + # Remove constant bounds + if self.propagation == Propagation.BACKWARD or self.ibp: + constant_oracle_bounds = inputs[-2:] + inputs = inputs[:-2] else: - raise NotImplementedError(f"unknown forward mode {self.mode}") + constant_oracle_bounds = [] + # The remaining tensors are affine bounds + # (potentially empty if: not backward or not affine or identity affine bounds) + affine_bounds_to_propagate = inputs - if self.dc_decomp: - nb_tensors += 2 + return affine_bounds_to_propagate, constant_oracle_bounds, model_inputs + + def split_input_shape( + self, input_shape: list[tuple[Optional[int], ...]] + ) -> tuple[list[tuple[Optional[int], ...]], list[tuple[Optional[int], ...]], list[tuple[Optional[int], ...]]]: + return self.split_inputs(inputs=input_shape) # type: ignore - return nb_tensors + def flatten_inputs( + self, affine_bounds_to_propagate: list[Tensor], constant_oracle_bounds: list[Tensor], model_inputs: list[Tensor] + ) -> list[Tensor]: + return affine_bounds_to_propagate + constant_oracle_bounds + model_inputs + + def split_outputs(self, outputs: list[Tensor]) -> tuple[list[Tensor], list[Tensor]]: + # Remove constant bounds + if self.propagation == Propagation.FORWARD and self.ibp: + constant_bounds_propagated = outputs[-2:] + outputs = outputs[:-2] + else: + constant_bounds_propagated = [] + # It remains affine bounds (can be empty if forward + not affine, or identity layer (e.g. DecomonLinear) on identity bounds + affine_bounds_propagated = outputs - @property - def ibp(self) -> bool: - return get_ibp(self.mode) + return affine_bounds_propagated, constant_bounds_propagated - @property - def affine(self) -> bool: - return get_affine(self.mode) + def flatten_outputs( + self, affine_bounds_propagated: list[Tensor], constant_bounds_propagated: Optional[list[Tensor]] = None + ) -> list[Tensor]: + if constant_bounds_propagated is None or self.propagation == Propagation.BACKWARD: + return affine_bounds_propagated + else: + return affine_bounds_propagated + constant_bounds_propagated + + def is_identity_bounds(self, affine_bounds: list[Tensor]) -> bool: + return len(affine_bounds) == 0 + + def is_identity_bounds_shape(self, affine_bounds_shape: list[tuple[Optional[int], ...]]) -> bool: + return len(affine_bounds_shape) == 0 + + def is_diagonal_bounds(self, affine_bounds: list[Tensor]) -> bool: + if self.is_identity_bounds(affine_bounds): + return True + w, b = affine_bounds[:2] + return w.shape == b.shape + + def is_diagonal_bounds_shape(self, affine_bounds_shape: list[tuple[Optional[int], ...]]) -> bool: + if self.is_identity_bounds_shape(affine_bounds_shape): + return True + w_shape, b_shape = affine_bounds_shape[:2] + return w_shape == b_shape + + def is_wo_batch_bounds(self, affine_bounds: list[Tensor]) -> bool: + if self.is_identity_bounds(affine_bounds): + return True + b = affine_bounds[1] + if self.propagation == Propagation.FORWARD: + return len(b.shape) == len(self.layer_input_shape) + else: + return len(b.shape) == len(self.model_output_shape) + + def is_wo_batch_bounds_shape(self, affine_bounds_shape: list[tuple[Optional[int], ...]]) -> bool: + if self.is_identity_bounds_shape(affine_bounds_shape): + return True + b_shape = affine_bounds_shape[1] + if self.propagation == Propagation.FORWARD: + return len(b_shape) == len(self.layer_input_shape) + else: + return len(b_shape) == len(self.model_output_shape) def get_kerasinputshape(self, inputsformode: list[Tensor]) -> tuple[Optional[int], ...]: return inputsformode[-1].shape diff --git a/src/decomon/layers/activations/activation.py b/src/decomon/layers/activations/activation.py index 65b19ed4..c597808e 100644 --- a/src/decomon/layers/activations/activation.py +++ b/src/decomon/layers/activations/activation.py @@ -22,6 +22,8 @@ def __init__( ibp: bool = True, affine: bool = True, propagation: Propagation = Propagation.FORWARD, + model_input_shape: Optional[tuple[int, ...]] = None, + model_output_shape: Optional[tuple[int, ...]] = None, slope: Slope = Slope.V_SLOPE, **kwargs: Any, ): @@ -31,6 +33,8 @@ def __init__( ibp=ibp, affine=affine, propagation=propagation, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, **kwargs, ) self.slope = slope @@ -71,6 +75,8 @@ def __init__( ibp: bool = True, affine: bool = True, propagation: Propagation = Propagation.FORWARD, + model_input_shape: Optional[tuple[int, ...]] = None, + model_output_shape: Optional[tuple[int, ...]] = None, slope: Slope = Slope.V_SLOPE, **kwargs: Any, ): @@ -80,6 +86,8 @@ def __init__( ibp=ibp, affine=affine, propagation=propagation, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, slope=slope, **kwargs, ) @@ -91,6 +99,8 @@ def __init__( ibp=ibp, affine=affine, propagation=propagation, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, slope=slope, **kwargs, ) @@ -118,29 +128,24 @@ def backward_affine_propagate( def forward_ibp_propagate(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, Tensor]: return self.decomon_activation.forward_ibp_propagate(lower=lower, upper=upper) - def build(self, affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, x_shape): - self.decomon_activation.build( - affine_bounds_to_propagate_shape=affine_bounds_to_propagate_shape, - constant_oracle_bounds_shape=constant_oracle_bounds_shape, - x_shape=x_shape, - ) + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: + self.decomon_activation.build(input_shape=input_shape) + super().build(input_shape=input_shape) - def call( - self, affine_bounds_to_propagate: list[Tensor], constant_oracle_bounds: list[Tensor], x: Tensor - ) -> list[list[Tensor]]: - return self.decomon_activation.call(affine_bounds_to_propagate, constant_oracle_bounds, x) + def call(self, inputs: list[Tensor]) -> list[Tensor]: + return self.decomon_activation.call(inputs=inputs) class DecomonLinear(DecomonBaseActivation): linear = True - def call( - self, affine_bounds_to_propagate: list[Tensor], constant_oracle_bounds: list[Tensor], x: Tensor - ) -> list[list[Tensor]]: - if self.propagation == Propagation.FORWARD: - return [affine_bounds_to_propagate, constant_oracle_bounds] - else: - return [affine_bounds_to_propagate] + def call(self, inputs: list[Tensor]) -> list[Tensor]: + affine_bounds_to_propagate, constant_oracle_bounds, model_inputs = self.inputs_outputs_spec.split_inputs( + inputs=inputs + ) + return self.inputs_outputs_spec.flatten_outputs( + affine_bounds_propagated=affine_bounds_to_propagate, constant_bounds_propagated=constant_oracle_bounds + ) class DecomonReLU(DecomonBaseActivation): diff --git a/src/decomon/layers/core/dense.py b/src/decomon/layers/core/dense.py index 1bb817c4..e84a397c 100644 --- a/src/decomon/layers/core/dense.py +++ b/src/decomon/layers/core/dense.py @@ -60,11 +60,11 @@ def forward_affine_propagate( w_l_1, b_l_1, w_u_1, b_u_1 = input_affine_bounds w_2, b_2 = self._get_pseudo_affine_representation() diagonal = ( - self.is_diagonal_bounds(input_affine_bounds), + self.inputs_outputs_spec.is_diagonal_bounds(input_affine_bounds), False, ) missing_batchsize = ( - self.is_wo_batch_bounds(input_affine_bounds), + self.inputs_outputs_spec.is_wo_batch_bounds(input_affine_bounds), True, ) kwargs_dot_w = dict(nb_merging_axes=1, missing_batchsize=missing_batchsize, diagonal=diagonal) @@ -88,11 +88,11 @@ def backward_affine_propagate( w_l_2, b_l_2, w_u_2, b_u_2 = output_affine_bounds # affine bounds represented in diagonal mode? - diagonal_bounds = self.is_diagonal_bounds(output_affine_bounds) + diagonal_bounds = self.inputs_outputs_spec.is_diagonal_bounds(output_affine_bounds) if diagonal_bounds: raise NotImplementedError # missing batch axis in affine bounds? - nb_batch_axis = 0 if self.is_wo_batch_bounds(output_affine_bounds) else 1 + nb_batch_axis = 0 if self.inputs_outputs_spec.is_wo_batch_bounds(output_affine_bounds) else 1 nb_nonbatch_axes_keras_input = len(w_l_2.shape) - len(b_l_2.shape) diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index 183ce0a8..7ac7be90 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -5,7 +5,7 @@ import keras.ops as K from keras.layers import Layer, Wrapper -from decomon.core import BoxDomain, PerturbationDomain, Propagation +from decomon.core import BoxDomain, InputsOutputsSpec, PerturbationDomain, Propagation from decomon.keras_utils import batch_multid_dot from decomon.types import Tensor @@ -72,7 +72,8 @@ def __init__( ibp: bool = True, affine: bool = True, propagation: Propagation = Propagation.FORWARD, - model_output_shape_length: int = 0, + model_input_shape: Optional[tuple[int, ...]] = None, + model_output_shape: Optional[tuple[int, ...]] = None, **kwargs: Any, ): """ @@ -84,10 +85,9 @@ def __init__( propagation: direction of bounds propagation - forward: from input to output - backward: from output to input - model_output_shape_length: length of the shape of the model output (omitting batch axis) - w.r.t which backward affine bounds will be computed. + model_output_shape: shape of the underlying model output (omitting batch axis). It allows determining if the backward bounds are with a bacth axis or not. - This has no meaning in forward propagation. + model_input_shape: shape of the underlying keras model input (omitting batch axis). **kwargs: """ @@ -108,15 +108,23 @@ def __init__( raise ValueError(f"The underlying keras layer {layer.name} is not built.") if not ibp and not affine: raise ValueError("ibp and affine cannot be both False.") - if propagation == Propagation.BACKWARD and model_output_shape_length == 0: - raise ValueError("model_output_shape_length must be positive in backward propagation.") # attributes self.ibp = ibp self.affine = affine self.perturbation_domain = perturbation_domain self.propagation = propagation - self.model_output_shape_length = model_output_shape_length + + # input-output-manager + self.inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + layer_input_shape=layer.input.shape[1:], + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + ) def get_config(self) -> dict[str, Any]: config = super().get_config() @@ -302,7 +310,7 @@ def forward_affine_propagate( w_l, b_l, w_u, b_u = self.get_affine_bounds(lower=lower, upper=upper) layer_affine_bounds = [w_l, b_l, w_u, b_u] - from_linear_layer = (self.is_wo_batch_bounds(input_affine_bounds), self.linear) + from_linear_layer = (self.inputs_outputs_spec.is_wo_batch_bounds(input_affine_bounds), self.linear) return combine_affine_bounds( affine_bounds_1=input_affine_bounds, @@ -356,7 +364,7 @@ def backward_affine_propagate( w_l, b_l, w_u, b_u = self.get_affine_bounds(lower=lower, upper=upper) layer_affine_bounds = [w_l, b_l, w_u, b_u] - from_linear_layer = (self.linear, self.is_wo_batch_bounds((output_affine_bounds))) + from_linear_layer = (self.linear, self.inputs_outputs_spec.is_wo_batch_bounds((output_affine_bounds))) return combine_affine_bounds( affine_bounds_1=layer_affine_bounds, @@ -365,14 +373,14 @@ def backward_affine_propagate( ) def get_forward_oracle( - self, input_affine_bounds: list[Tensor], input_constant_bounds: list[Tensor], x: Tensor + self, input_affine_bounds: list[Tensor], input_constant_bounds: list[Tensor], model_inputs: list[Tensor] ) -> list[Tensor]: """Get constant oracle bounds on underlying keras layer input from forward input bounds. Args: input_affine_bounds: affine bounds on keras layer input w.r.t model input . Can be empty if not in affine mode. input_constant_bounds: ibp constant bounds on keras layer input. Can be empty if not in ibp mode. - x: model input. Necessary only in affine mode. + model_inputs: underlying keras model input, wrapped in a list. Necessary only in affine mode, else empty. Returns: constant bounds on keras layer input deduced from forward input bounds @@ -391,6 +399,9 @@ def get_forward_oracle( return input_constant_bounds elif self.affine: + if len(model_inputs) == 0: + raise RuntimeError("keras model input is necessary for get_forward_oracle() in affine mode.") + x = model_inputs[0] if len(input_affine_bounds) == 0: # special case: empty affine bounds => identity bounds l_affine = self.perturbation_domain.get_lower_x(x) @@ -405,7 +416,10 @@ def get_forward_oracle( raise RuntimeError("self.ibp and self.affine cannot be both False") def call_forward( - self, affine_bounds_to_propagate: list[Tensor], input_bounds_to_propagate: list[Tensor], x: Tensor + self, + affine_bounds_to_propagate: list[Tensor], + input_bounds_to_propagate: list[Tensor], + model_inputs: list[Tensor], ) -> tuple[list[Tensor], list[Tensor]]: """Propagate forward affine and constant bounds through the layer. @@ -414,7 +428,7 @@ def call_forward( Can be empty if not in affine mode. Can also be empty in case of identity affine bounds => we simply return layer affine bounds. input_bounds_to_propagate: ibp constant bounds on keras layer input. Can be empty if not in ibp mode. - x: model input. Necessary only in affine mode. + model_inputs: underlying keras model input, wrapped in a list. Necessary only in affine mode, else empty. Returns: output_affine_bounds, output_constant_bounds: affine and constant bounds on the underlying keras layer output @@ -439,7 +453,9 @@ def call_forward( if not self.linear: # get oracle input bounds (because input_bounds_to_propagate could be empty at this point) input_constant_bounds = self.get_forward_oracle( - input_affine_bounds=affine_bounds_to_propagate, input_constant_bounds=input_bounds_to_propagate, x=x + input_affine_bounds=affine_bounds_to_propagate, + input_constant_bounds=input_bounds_to_propagate, + model_inputs=model_inputs, ) else: input_constant_bounds = [] @@ -454,6 +470,9 @@ def call_forward( # Tighten constant bounds in hybrid mode (ibp+affine) if self.ibp and self.affine: + if len(model_inputs) == 0: + raise RuntimeError("keras model input is necessary for get_forward_oracle() in affine mode.") + x = model_inputs[0] l_ibp, u_ibp = output_constant_bounds w_l, b_l, w_u, b_u = output_affine_bounds l_affine = self.perturbation_domain.get_lower(x, w_l, b_l) @@ -473,48 +492,57 @@ def call_backward( ) ) - def call( - self, affine_bounds_to_propagate: list[Tensor], constant_oracle_bounds: list[Tensor], x: Tensor - ) -> list[list[Tensor]]: + def call(self, inputs: list[Tensor]) -> list[Tensor]: """Propagate bounds in the specified direction `self.propagation`. Args: - affine_bounds_to_propagate: affine bounds to propagate. - Can be empty in forward direction if self.affine is False. - Can also be empty in case of identity affine bounds => we simply return layer affine bounds. - constant_oracle_bounds: in forward direction, the ibp bounds (empty if self.ibp is False); in backward direction, the oracle constant bounds on keras inputs - x: the model input. Necessary only in forward direction when self.affine is True. + inputs: concatenation of affine_bounds_to_propagate + constant_oracle_bounds + keras_model_inputs with + - affine_bounds_to_propagate: affine bounds to propagate. + Can be empty in forward direction if self.affine is False. + Can also be empty in case of identity affine bounds => we simply return layer affine bounds. + - constant_oracle_bounds: + - in forward direction, the ibp bounds (empty if self.ibp is False); + - in backward direction, the oracle constant bounds on keras inputs (never empty) + - keras_model_inputs: the tensors defining the underlying keras model input perturbation. + - in forward direction when self.affine is True: one tensor x whose shape is given by `self.perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape)` + with `model_input_shape=model.input.shape[1:]` if `model` is the underlying keras model to analyse + - else: empty list Returns: the propagated bounds. - forward: [affine_bounds_propagated, constant_bounds_propagated], each one being empty if self.affine or self.ibp is False - backward: [affine_bounds_propagated] - + - in forward direction: affine_bounds_propagated + constant_bounds_propagated, each one being empty if self.affine or self.ibp is False + - in backward direction: affine_bounds_propagated """ + affine_bounds_to_propagate, constant_oracle_bounds, model_inputs = self.inputs_outputs_spec.split_inputs( + inputs=inputs + ) if self.propagation == Propagation.FORWARD: # forward affine_bounds_propagated, constant_bounds_propagated = self.call_forward( affine_bounds_to_propagate=affine_bounds_to_propagate, input_bounds_to_propagate=constant_oracle_bounds, - x=x, + model_inputs=model_inputs, ) - return [affine_bounds_propagated, constant_bounds_propagated] + return self.inputs_outputs_spec.flatten_outputs(affine_bounds_propagated, constant_bounds_propagated) else: # backward affine_bounds_propagated = self.call_backward( affine_bounds_to_propagate=affine_bounds_to_propagate, constant_oracle_bounds=constant_oracle_bounds ) - return [affine_bounds_propagated] + return self.inputs_outputs_spec.flatten_outputs(affine_bounds_propagated) - def build(self, affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, x_shape): + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: self.built = True def compute_output_shape( self, - affine_bounds_to_propagate_shape: list[tuple[Optional[int], ...]], - constant_oracle_bounds_shape: list[tuple[Optional[int], ...]], - x_shape: tuple[Optional[int], ...], - ): + input_shape: list[tuple[Optional[int], ...]], + ) -> list[tuple[Optional[int], ...]]: + ( + affine_bounds_to_propagate_shape, + constant_oracle_bounds_shape, + model_inputs_shape, + ) = self.inputs_outputs_spec.split_input_shape(input_shape=input_shape) if self.propagation == Propagation.FORWARD: if self.ibp: constant_bounds_propagated_shape = [self.layer.output.shape] * 2 @@ -525,23 +553,25 @@ def compute_output_shape( keras_layer_output_shape_wo_batchsize = self.layer.output.shape[1:] # inputs are in diagonal representation? without batch axis? - if self.is_diagonal_bounds_shape(affine_bounds_to_propagate_shape): + if self.inputs_outputs_spec.is_diagonal_bounds_shape(affine_bounds_to_propagate_shape): model_input_shape_wo_batchsize = keras_layer_input_shape_wo_batchsize else: w_in_shape = affine_bounds_to_propagate_shape[0] - if self.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): + if self.inputs_outputs_spec.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): model_input_shape_wo_batchsize = w_in_shape[: -len(keras_layer_input_shape_wo_batchsize)] else: model_input_shape_wo_batchsize = w_in_shape[1 : -len(keras_layer_input_shape_wo_batchsize)] # outputs shape depends if layer and inputs are diagonal / linear (w/o batch) b_out_shape_wo_batchsize = keras_layer_output_shape_wo_batchsize - if self.diagonal and self.is_diagonal_bounds_shape(affine_bounds_to_propagate_shape): + if self.diagonal and self.inputs_outputs_spec.is_diagonal_bounds_shape( + affine_bounds_to_propagate_shape + ): # propagated bounds still diagonal w_out_shape_wo_batchsize = b_out_shape_wo_batchsize else: w_out_shape_wo_batchsize = model_input_shape_wo_batchsize + keras_layer_output_shape_wo_batchsize - if self.linear and self.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): + if self.linear and self.inputs_outputs_spec.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): # no batch in propagated bounds w_out_shape = w_out_shape_wo_batchsize b_out_shape = b_out_shape_wo_batchsize @@ -552,25 +582,25 @@ def compute_output_shape( else: affine_bounds_propagated_shape = [] - return [affine_bounds_propagated_shape, constant_bounds_propagated_shape] + return affine_bounds_propagated_shape + constant_bounds_propagated_shape else: # backward # find model output shape - if self.is_identity_bounds_shape(affine_bounds_to_propagate_shape): + if self.inputs_outputs_spec.is_identity_bounds_shape(affine_bounds_to_propagate_shape): model_output_shape_wo_batchsize = self.layer.output.shape[1:] else: b_shape = affine_bounds_to_propagate_shape[1] - if self.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): + if self.inputs_outputs_spec.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): model_output_shape_wo_batchsize = b_shape else: model_output_shape_wo_batchsize = b_shape[1:] # outputs shape depends if layer and inputs are diagonal / linear (w/o batch) b_shape_wo_batchisze = model_output_shape_wo_batchsize - if self.diagonal and self.is_diagonal_bounds_shape(affine_bounds_to_propagate_shape): + if self.diagonal and self.inputs_outputs_spec.is_diagonal_bounds_shape(affine_bounds_to_propagate_shape): w_shape_wo_batchsize = model_output_shape_wo_batchsize else: w_shape_wo_batchsize = self.layer.input.shape[1:] + model_output_shape_wo_batchsize - if self.linear and self.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): + if self.linear and self.inputs_outputs_spec.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): b_new_shape = b_shape_wo_batchisze w_shape = w_shape_wo_batchsize else: @@ -579,56 +609,7 @@ def compute_output_shape( affine_bounds_propagated_shape = [w_shape, b_new_shape, w_shape, b_new_shape] - return [affine_bounds_propagated_shape] - - def compute_output_spec(self, *args: Any, **kwargs: Any) -> list[list[keras.KerasTensor]]: - """Compute output spec from output shape in case of symbolic call.""" - output_spec = Layer.compute_output_spec(self, *args, **kwargs) - - # fix empty list: Layer.compute_output_spec() transform them as empty tensors - def replace_empty_tensor(l: Union[keras.KerasTensor, list[keras.KerasTensor]]): - if isinstance(l, keras.KerasTensor) and len(l.shape) == 0: - return [] - else: - return l - - return [replace_empty_tensor(l) for l in output_spec] - - def is_identity_bounds(self, affine_bounds: list[Tensor]) -> bool: - return len(affine_bounds) == 0 - - def is_identity_bounds_shape(self, affine_bounds_shape: list[tuple[Optional[int], ...]]) -> bool: - return len(affine_bounds_shape) == 0 - - def is_diagonal_bounds(self, affine_bounds: list[Tensor]) -> bool: - if self.is_identity_bounds(affine_bounds): - return True - w, b = affine_bounds[:2] - return w.shape == b.shape - - def is_diagonal_bounds_shape(self, affine_bounds_shape: list[tuple[Optional[int], ...]]) -> bool: - if self.is_identity_bounds_shape(affine_bounds_shape): - return True - w_shape, b_shape = affine_bounds_shape[:2] - return w_shape == b_shape - - def is_wo_batch_bounds(self, affine_bounds: list[Tensor]) -> bool: - if self.is_identity_bounds(affine_bounds): - return True - b = affine_bounds[1] - if self.propagation == Propagation.FORWARD: - return len(b.shape) == len(self.layer.input.shape) - 1 - else: - return len(b.shape) == self.model_output_shape_length - - def is_wo_batch_bounds_shape(self, affine_bounds_shape: list[tuple[Optional[int], ...]]) -> bool: - if self.is_identity_bounds_shape(affine_bounds_shape): - return True - b_shape = affine_bounds_shape[1] - if self.propagation == Propagation.FORWARD: - return len(b_shape) == len(self.layer.input.shape) - 1 - else: - return len(b_shape) == self.model_output_shape_length + return affine_bounds_propagated_shape def combine_affine_bounds( diff --git a/tests/conftest.py b/tests/conftest.py index 2c5564e0..e4c8463e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ from keras.layers import Input from pytest_cases import fixture, fixture_union, param_fixture, param_fixtures -from decomon.core import BoxDomain, Propagation, Slope +from decomon.core import BoxDomain, InputsOutputsSpec, Propagation, Slope from decomon.keras_utils import ( BACKEND_JAX, BACKEND_NUMPY, @@ -142,7 +142,10 @@ def get_decomon_input_shapes( diag=False, nobatch=False, ): - x_shape = perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape) + if affine and propagation == Propagation.FORWARD: + model_inputs_shape = [perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape)] + else: + model_inputs_shape = [] if affine and not empty: if propagation == Propagation.FORWARD: b_in_shape = layer_input_shape @@ -161,7 +164,7 @@ def get_decomon_input_shapes( else: constant_oracle_bounds_shape = [] - return [affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, x_shape] + return affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, model_inputs_shape @staticmethod def get_decomon_symbolic_inputs( @@ -194,7 +197,11 @@ def get_decomon_symbolic_inputs( Returns: """ - decomon_input_shape = Helpers.get_decomon_input_shapes( + ( + affine_bounds_to_propagate_shape, + constant_oracle_bounds_shape, + model_inputs_shape, + ) = Helpers.get_decomon_input_shapes( model_input_shape, model_output_shape, layer_input_shape, @@ -207,14 +214,26 @@ def get_decomon_symbolic_inputs( diag=diag, nobatch=nobatch, ) - affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, x_shape = decomon_input_shape - x = Input(x_shape) + model_inputs = [Input(shape) for shape in model_inputs_shape] constant_oracle_bounds = [Input(shape) for shape in constant_oracle_bounds_shape] if nobatch: affine_bounds_to_propagate = [Input(batch_shape=shape) for shape in affine_bounds_to_propagate_shape] else: affine_bounds_to_propagate = [Input(shape=shape) for shape in affine_bounds_to_propagate_shape] - return [affine_bounds_to_propagate, constant_oracle_bounds, x] + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + layer_input_shape=layer_input_shape, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + ) + return inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=affine_bounds_to_propagate, + constant_oracle_bounds=constant_oracle_bounds, + model_inputs=model_inputs, + ) @staticmethod def generate_simple_decomon_layer_inputs_from_keras_input( @@ -251,15 +270,22 @@ def generate_simple_decomon_layer_inputs_from_keras_input( Returns: """ - if isinstance(perturbation_domain, BoxDomain): - x = K.repeat(keras_input[:, None], 2, axis=1) + layer_input_shape = keras_input.shape[1:] + model_input_shape = layer_input_shape + model_output_shape = layer_output_shape + if affine and propagation == Propagation.FORWARD: + if isinstance(perturbation_domain, BoxDomain): + x = K.repeat(keras_input[:, None], 2, axis=1) + else: + raise NotImplementedError + model_inputs = [x] else: - raise NotImplementedError + model_inputs = [] if affine and not empty: batchsize = keras_input.shape[0] if propagation == Propagation.FORWARD: - bias_shape = keras_input.shape[1:] + bias_shape = layer_input_shape else: bias_shape = layer_output_shape flatten_bias_dim = int(np.prod(bias_shape)) @@ -288,23 +314,30 @@ def generate_simple_decomon_layer_inputs_from_keras_input( else: constant_oracle_bounds = [] - return [affine_bounds_to_propagate, constant_oracle_bounds, x] + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + layer_input_shape=layer_input_shape, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + ) + return inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=affine_bounds_to_propagate, + constant_oracle_bounds=constant_oracle_bounds, + model_inputs=model_inputs, + ) @staticmethod def assert_decomon_outputs_equal(output_1, output_2, decimal=5): - names = [["w_l", "b_l", "w_u", "b_u"], ["l_c", "u_c"]] assert len(output_1) == len(output_2) for i in range(len(output_1)): - sub_output_1 = output_1[i] - sub_output_2 = output_2[i] - assert len(sub_output_1) == len(sub_output_2) - for j in range(len(sub_output_1)): - Helpers.assert_almost_equal( - sub_output_1[j], - sub_output_2[j], - decimal=decimal, - err_msg=names[i][j], - ) + Helpers.assert_almost_equal( + output_1[i], + output_2[i], + decimal=decimal, + ) @staticmethod def assert_ordered(lower: BackendTensor, upper: BackendTensor, decimal: int = 5, err_msg: str = ""): @@ -328,35 +361,44 @@ def assert_almost_equal(x: BackendTensor, y: BackendTensor, decimal: int = 5, er @staticmethod def assert_decomon_output_compare_with_keras_input_output_single_layer( - decomon_output, keras_output, keras_input, decimal=5 + decomon_output, keras_output, keras_input, ibp, affine, propagation, decimal=5 ): - affine = len(decomon_output[0]) > 0 - ibp = len(decomon_output) > 1 and len(decomon_output[1]) > 0 + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + layer_input_shape=keras_input.shape[1:], + model_output_shape=keras_output.shape[1:], + ) + affine_bounds_propagated, constant_bounds_propagated = inputs_outputs_spec.split_outputs(outputs=decomon_output) - if affine: - w_l, b_l, w_u, b_u = decomon_output[0] + if affine or propagation == Propagation.BACKWARD: + w_l, b_l, w_u, b_u = affine_bounds_propagated lower_affine = batch_multid_dot(keras_input, w_l) + b_l upper_affine = batch_multid_dot(keras_input, w_u) + b_u Helpers.assert_ordered(lower_affine, keras_output, decimal=decimal, err_msg="lower_affine not ok") Helpers.assert_ordered(keras_output, upper_affine, decimal=decimal, err_msg="upper_affine not ok") - if ibp: - lower_ibp, upper_ibp = decomon_output[1] + if ibp and propagation == Propagation.FORWARD: + lower_ibp, upper_ibp = constant_bounds_propagated Helpers.assert_ordered(lower_ibp, keras_output, decimal=decimal, err_msg="lower_ibp not ok") Helpers.assert_ordered(keras_output, upper_ibp, decimal=decimal, err_msg="upper_ibp not ok") @staticmethod - def assert_decomon_output_lower_equal_upper(decomon_output, decimal=5): - affine = len(decomon_output[0]) > 0 - ibp = len(decomon_output) > 1 and len(decomon_output[1]) > 0 - - if affine: - w_l, b_l, w_u, b_u = decomon_output[0] + def assert_decomon_output_lower_equal_upper(decomon_output, ibp, affine, propagation, decimal=5): + inputs_outputs_specs = InputsOutputsSpec( + ibp=ibp, affine=affine, propagation=propagation, layer_input_shape=tuple(), model_output_shape=tuple() + ) + affine_bounds_propagated, constant_bounds_propagated = inputs_outputs_specs.split_outputs( + outputs=decomon_output + ) + if propagation == Propagation.BACKWARD or affine: + w_l, b_l, w_u, b_u = affine_bounds_propagated Helpers.assert_almost_equal(w_l, w_u, decimal=decimal) Helpers.assert_almost_equal(b_l, b_u, decimal=decimal) - if ibp: - lower_ibp, upper_ibp = decomon_output[1] + if propagation == Propagation.FORWARD and ibp: + lower_ibp, upper_ibp = constant_bounds_propagated Helpers.assert_almost_equal(lower_ibp, upper_ibp, decimal=decimal) @staticmethod diff --git a/tests/test_activation.py b/tests/test_activation.py index 18480a22..abf538fc 100644 --- a/tests/test_activation.py +++ b/tests/test_activation.py @@ -19,14 +19,12 @@ def test_decomon_activation( batchsize, helpers, ): - decimal = 5 decomon_layer_class = DecomonActivation keras_symbolic_input = Input(input_shape) layer = Activation(activation=activation) layer(keras_symbolic_input) output_shape = layer.output.shape[1:] - model_output_shape_length = len(output_shape) decomon_symbolic_inputs = helpers.get_decomon_symbolic_inputs( model_input_shape=input_shape, @@ -44,10 +42,10 @@ def test_decomon_activation( affine=affine, propagation=propagation, perturbation_domain=perturbation_domain, - model_output_shape_length=model_output_shape_length, + model_output_shape=output_shape, slope=slope, ) - decomon_layer(*decomon_symbolic_inputs) + decomon_layer(decomon_symbolic_inputs) keras_input = helpers.generate_random_tensor(input_shape, batchsize=batchsize) decomon_inputs = helpers.generate_simple_decomon_layer_inputs_from_keras_input( @@ -61,9 +59,14 @@ def test_decomon_activation( keras_output = layer(keras_input) - decomon_output = decomon_layer(*decomon_inputs) + decomon_output = decomon_layer(decomon_inputs) # check ibp and affine bounds well ordered w.r.t. keras output helpers.assert_decomon_output_compare_with_keras_input_output_single_layer( - decomon_output=decomon_output, keras_output=keras_output, keras_input=keras_input + decomon_output=decomon_output, + keras_output=keras_output, + keras_input=keras_input, + ibp=ibp, + affine=affine, + propagation=propagation, ) diff --git a/tests/test_decomon_layer.py b/tests/test_decomon_layer.py index 87ce3955..9ac2c5fb 100644 --- a/tests/test_decomon_layer.py +++ b/tests/test_decomon_layer.py @@ -21,7 +21,7 @@ def test_decomon_layer_nok_ibp_affine(): DecomonLayer(layer=layer, ibp=False, affine=False) -def test_decomon_layer_nok_backward_no_model_output_shape_length(): +def test_decomon_layer_nok_backward_no_model_output_shape(): layer = Dense(3) layer(Input((1,))) with pytest.raises(ValueError): @@ -82,7 +82,6 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper model_output_shape = model_output_shape_if_no_singlelayer_model model_input_shape = (model_input_dim,) - model_output_shape_length = len(model_output_shape) x_shape = perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape) # keras layer @@ -96,7 +95,7 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper affine=affine, propagation=propagation, perturbation_domain=perturbation_domain, - model_output_shape_length=model_output_shape_length, + model_output_shape=model_output_shape, ) non_linear_decomon_layer = MyNonLinearDecomonDense1d( layer=layer, @@ -104,7 +103,7 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper affine=affine, propagation=propagation, perturbation_domain=perturbation_domain, - model_output_shape_length=model_output_shape_length, + model_output_shape=model_output_shape, ) # symbolic inputs @@ -118,10 +117,14 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper propagation=propagation, perturbation_domain=perturbation_domain, ) - affine_bounds_to_propagate, constant_oracle_bounds, x = decomon_inputs + ( + affine_bounds_to_propagate, + constant_oracle_bounds, + model_inputs, + ) = linear_decomon_layer.inputs_outputs_spec.split_inputs(decomon_inputs) # actual (random) tensors + expected output shapes - x_val = helpers.generate_random_tensor(x.shape[1:], batchsize=batchsize) + model_inputs_val = [helpers.generate_random_tensor(x.shape[1:], batchsize=batchsize) for x in model_inputs] if affine: if propagation == Propagation.FORWARD: @@ -148,34 +151,35 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper propagated_ibp_bounds_expected_shape = [] constant_oracle_bounds_val = [] - decomon_inputs_val = [affine_bounds_to_propagate_val, constant_oracle_bounds_val, x_val] + decomon_inputs_val = linear_decomon_layer.inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=affine_bounds_to_propagate_val, + constant_oracle_bounds=constant_oracle_bounds_val, + model_inputs=model_inputs_val, + ) if propagation == Propagation.FORWARD: - decomon_output_expected_shapes = [ - propagated_affine_bounds_expected_shape, - propagated_ibp_bounds_expected_shape, - ] + decomon_output_expected_shapes = propagated_affine_bounds_expected_shape + propagated_ibp_bounds_expected_shape else: - decomon_output_expected_shapes = [propagated_affine_bounds_expected_shape] + decomon_output_expected_shapes = propagated_affine_bounds_expected_shape # symbolic call - linear_decomon_output = linear_decomon_layer(*decomon_inputs) - non_linear_decomon_output = non_linear_decomon_layer(*decomon_inputs) + linear_decomon_output = linear_decomon_layer(decomon_inputs) + non_linear_decomon_output = non_linear_decomon_layer(decomon_inputs) # shapes ok ? - linear_decomon_output_shape_from_call = [[tensor.shape[1:] for tensor in l] for l in linear_decomon_output] + linear_decomon_output_shape_from_call = [tensor.shape[1:] for tensor in linear_decomon_output] assert linear_decomon_output_shape_from_call == decomon_output_expected_shapes - non_linear_decomon_output_shape_from_call = [[tensor.shape[1:] for tensor in l] for l in non_linear_decomon_output] + non_linear_decomon_output_shape_from_call = [tensor.shape[1:] for tensor in non_linear_decomon_output] assert non_linear_decomon_output_shape_from_call == decomon_output_expected_shapes # actual call - linear_decomon_output_val = linear_decomon_layer(*decomon_inputs_val) - non_linear_decomon_output_val = non_linear_decomon_layer(*decomon_inputs_val) + linear_decomon_output_val = linear_decomon_layer(decomon_inputs_val) + non_linear_decomon_output_val = non_linear_decomon_layer(decomon_inputs_val) # shapes ok ? - linear_decomon_output_shape_from_call = [[tensor.shape[1:] for tensor in l] for l in linear_decomon_output_val] + linear_decomon_output_shape_from_call = [tensor.shape[1:] for tensor in linear_decomon_output_val] assert linear_decomon_output_shape_from_call == decomon_output_expected_shapes - non_linear_decomon_output_shape_from_call = [[tensor.shape[1:] for tensor in l] for l in linear_decomon_output_val] + non_linear_decomon_output_shape_from_call = [tensor.shape[1:] for tensor in linear_decomon_output_val] assert non_linear_decomon_output_shape_from_call == decomon_output_expected_shapes # same values ? @@ -199,15 +203,20 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper ) # decomon call - linear_decomon_output_val = linear_decomon_layer(*decomon_inputs_val) - non_linear_decomon_output_val = non_linear_decomon_layer(*decomon_inputs_val) + linear_decomon_output_val = linear_decomon_layer(decomon_inputs_val) + non_linear_decomon_output_val = non_linear_decomon_layer(decomon_inputs_val) # keras call keras_output_val = layer(keras_input_val) # comparison helpers.assert_decomon_output_compare_with_keras_input_output_single_layer( - decomon_output=linear_decomon_output_val, keras_output=keras_output_val, keras_input=keras_input_val + decomon_output=linear_decomon_output_val, + keras_output=keras_output_val, + keras_input=keras_input_val, + ibp=ibp, + affine=affine, + propagation=propagation, ) helpers.assert_decomon_outputs_equal(linear_decomon_output_val, non_linear_decomon_output_val) @@ -231,7 +240,7 @@ def test_check_affine_bounds_characteristics( keras_symbolic_input = keras_symbolic_input_fn() input_shape = keras_symbolic_input.shape[1:] output_shape = input_shape[:-1] + (units,) - model_output_shape_length = len(output_shape) + model_output_shape = output_shape decomon_symbolic_input = decomon_symbolic_input_fn(output_shape=output_shape) keras_input = keras_input_fn() decomon_input = decomon_input_fn(keras_input=keras_input, output_shape=output_shape) @@ -245,24 +254,27 @@ def test_check_affine_bounds_characteristics( affine=affine, propagation=propagation, perturbation_domain=perturbation_domain, - model_output_shape_length=model_output_shape_length, + model_output_shape=model_output_shape, + ) + affine_bounds_to_propagate, constant_oracle_bounds, model_inputs = decomon_layer.inputs_outputs_spec.split_inputs( + inputs=decomon_symbolic_input ) if affine: - affine_bounds = decomon_symbolic_input[0] + affine_bounds, _, _ = decomon_layer.inputs_outputs_spec.split_inputs(inputs=decomon_symbolic_input) affine_bounds_shape = [t.shape for t in affine_bounds] - assert decomon_layer.is_identity_bounds(affine_bounds) is empty - assert decomon_layer.is_diagonal_bounds(affine_bounds) is diag - assert decomon_layer.is_wo_batch_bounds(affine_bounds) is nobatch - assert decomon_layer.is_identity_bounds_shape(affine_bounds_shape) is empty - assert decomon_layer.is_diagonal_bounds_shape(affine_bounds_shape) is diag - assert decomon_layer.is_wo_batch_bounds_shape(affine_bounds_shape) is nobatch - - affine_bounds = decomon_input[0] + assert decomon_layer.inputs_outputs_spec.is_identity_bounds(affine_bounds) is empty + assert decomon_layer.inputs_outputs_spec.is_diagonal_bounds(affine_bounds) is diag + assert decomon_layer.inputs_outputs_spec.is_wo_batch_bounds(affine_bounds) is nobatch + assert decomon_layer.inputs_outputs_spec.is_identity_bounds_shape(affine_bounds_shape) is empty + assert decomon_layer.inputs_outputs_spec.is_diagonal_bounds_shape(affine_bounds_shape) is diag + assert decomon_layer.inputs_outputs_spec.is_wo_batch_bounds_shape(affine_bounds_shape) is nobatch + + affine_bounds, _, _ = decomon_layer.inputs_outputs_spec.split_inputs(inputs=decomon_input) affine_bounds_shape = [t.shape for t in affine_bounds] - assert decomon_layer.is_identity_bounds(affine_bounds) is empty - assert decomon_layer.is_diagonal_bounds(affine_bounds) is diag - assert decomon_layer.is_wo_batch_bounds(affine_bounds) is nobatch - assert decomon_layer.is_identity_bounds_shape(affine_bounds_shape) is empty - assert decomon_layer.is_diagonal_bounds_shape(affine_bounds_shape) is diag - assert decomon_layer.is_wo_batch_bounds_shape(affine_bounds_shape) is nobatch + assert decomon_layer.inputs_outputs_spec.is_identity_bounds(affine_bounds) is empty + assert decomon_layer.inputs_outputs_spec.is_diagonal_bounds(affine_bounds) is diag + assert decomon_layer.inputs_outputs_spec.is_wo_batch_bounds(affine_bounds) is nobatch + assert decomon_layer.inputs_outputs_spec.is_identity_bounds_shape(affine_bounds_shape) is empty + assert decomon_layer.inputs_outputs_spec.is_diagonal_bounds_shape(affine_bounds_shape) is diag + assert decomon_layer.inputs_outputs_spec.is_wo_batch_bounds_shape(affine_bounds_shape) is nobatch diff --git a/tests/test_dense.py b/tests/test_dense.py index 4eb942ac..0b340454 100644 --- a/tests/test_dense.py +++ b/tests/test_dense.py @@ -24,7 +24,6 @@ def test_decomon_dense( decimal = 5 units = 7 output_shape = input_shape[:-1] + (units,) - model_output_shape_length = len(output_shape) keras_symbolic_input = Input(input_shape) decomon_symbolic_inputs = helpers.get_decomon_symbolic_inputs( model_input_shape=input_shape, @@ -51,9 +50,9 @@ def test_decomon_dense( affine=affine, propagation=propagation, perturbation_domain=perturbation_domain, - model_output_shape_length=model_output_shape_length, + model_output_shape=output_shape, ) - decomon_layer(*decomon_symbolic_inputs) + decomon_layer(decomon_symbolic_inputs) keras_input = helpers.generate_random_tensor(input_shape, batchsize=batchsize) decomon_inputs = helpers.generate_simple_decomon_layer_inputs_from_keras_input( @@ -78,12 +77,19 @@ def test_decomon_dense( err_msg="wrong affine representation", ) - decomon_output = decomon_layer(*decomon_inputs) + decomon_output = decomon_layer(decomon_inputs) # check ibp and affine bounds well ordered w.r.t. keras output helpers.assert_decomon_output_compare_with_keras_input_output_single_layer( - decomon_output=decomon_output, keras_output=keras_output, keras_input=keras_input + decomon_output=decomon_output, + keras_output=keras_output, + keras_input=keras_input, + ibp=ibp, + affine=affine, + propagation=propagation, ) # before propagation through linear layer lower == upper => lower == upper after propagation - helpers.assert_decomon_output_lower_equal_upper(decomon_output, decimal=decimal) + helpers.assert_decomon_output_lower_equal_upper( + decomon_output, ibp=ibp, affine=affine, propagation=propagation, decimal=decimal + ) From b407eaf8c0c5d4cb3790b9202bc2b38c124d6e93 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 12 Feb 2024 22:20:32 +0100 Subject: [PATCH 031/101] Add checks on compute_output_shape() --- tests/conftest.py | 6 +++++- tests/test_activation.py | 7 +++++++ tests/test_dense.py | 7 +++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index e4c8463e..6131c6df 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -from typing import Union +from typing import Optional, Union import keras import keras.config as keras_config @@ -401,6 +401,10 @@ def assert_decomon_output_lower_equal_upper(decomon_output, ibp, affine, propaga lower_ibp, upper_ibp = constant_bounds_propagated Helpers.assert_almost_equal(lower_ibp, upper_ibp, decimal=decimal) + @staticmethod + def replace_none_by_batchsize(shapes: list[tuple[Optional[int], ...]], batchsize: int) -> list[tuple[int]]: + return [tuple(dim if dim is not None else batchsize for dim in shape) for shape in shapes] + @staticmethod def predict_on_small_numpy( model: Model, x: Union[np.ndarray, list[np.ndarray]] diff --git a/tests/test_activation.py b/tests/test_activation.py index abf538fc..16906621 100644 --- a/tests/test_activation.py +++ b/tests/test_activation.py @@ -61,6 +61,13 @@ def test_decomon_activation( decomon_output = decomon_layer(decomon_inputs) + # check output shapes + input_shape = [t.shape for t in decomon_inputs] + output_shape = [t.shape for t in decomon_output] + expected_output_shape = decomon_layer.compute_output_shape(input_shape) + expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) + assert output_shape == expected_output_shape + # check ibp and affine bounds well ordered w.r.t. keras output helpers.assert_decomon_output_compare_with_keras_input_output_single_layer( decomon_output=decomon_output, diff --git a/tests/test_dense.py b/tests/test_dense.py index 0b340454..3ce408ba 100644 --- a/tests/test_dense.py +++ b/tests/test_dense.py @@ -79,6 +79,13 @@ def test_decomon_dense( decomon_output = decomon_layer(decomon_inputs) + # check output shapes + input_shape = [t.shape for t in decomon_inputs] + output_shape = [t.shape for t in decomon_output] + expected_output_shape = decomon_layer.compute_output_shape(input_shape) + expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) + assert output_shape == expected_output_shape + # check ibp and affine bounds well ordered w.r.t. keras output helpers.assert_decomon_output_compare_with_keras_input_output_single_layer( decomon_output=decomon_output, From 32ea8c6b341f87d76306cd5d98e878196091ae3a Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 12 Feb 2024 22:24:52 +0100 Subject: [PATCH 032/101] Test DecomonDense on all kind of inputs --- tests/test_dense.py | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/tests/test_dense.py b/tests/test_dense.py index 3ce408ba..ae23f680 100644 --- a/tests/test_dense.py +++ b/tests/test_dense.py @@ -8,7 +8,6 @@ @pytest.mark.parametrize("decomon_layer_class", [DecomonNaiveDense, DecomonDense]) -@pytest.mark.parametrize("input_shape", [(1,), (3,), (5, 2, 3)], ids=["0d", "1d", "multid"]) def test_decomon_dense( decomon_layer_class, use_bias, @@ -16,25 +15,24 @@ def test_decomon_dense( ibp, affine, propagation, - input_shape, perturbation_domain, batchsize, + keras_symbolic_input_fn, + decomon_symbolic_input_fn, + keras_input_fn, + decomon_input_fn, helpers, ): decimal = 5 units = 7 + + keras_symbolic_input = keras_symbolic_input_fn() + input_shape = keras_symbolic_input.shape[1:] output_shape = input_shape[:-1] + (units,) - keras_symbolic_input = Input(input_shape) - decomon_symbolic_inputs = helpers.get_decomon_symbolic_inputs( - model_input_shape=input_shape, - model_output_shape=output_shape, - layer_input_shape=input_shape, - layer_output_shape=output_shape, - ibp=ibp, - affine=affine, - propagation=propagation, - perturbation_domain=perturbation_domain, - ) + model_output_shape = output_shape + decomon_symbolic_inputs = decomon_symbolic_input_fn(output_shape=output_shape) + keras_input = keras_input_fn() + decomon_inputs = decomon_input_fn(keras_input=keras_input, output_shape=output_shape) layer = Dense(units=units) layer(keras_symbolic_input) @@ -50,20 +48,10 @@ def test_decomon_dense( affine=affine, propagation=propagation, perturbation_domain=perturbation_domain, - model_output_shape=output_shape, + model_output_shape=model_output_shape, ) decomon_layer(decomon_symbolic_inputs) - keras_input = helpers.generate_random_tensor(input_shape, batchsize=batchsize) - decomon_inputs = helpers.generate_simple_decomon_layer_inputs_from_keras_input( - keras_input=keras_input, - layer_output_shape=output_shape, - ibp=ibp, - affine=affine, - propagation=propagation, - perturbation_domain=perturbation_domain, - ) - keras_output = layer(keras_input) # check affine representation is ok From 1b5947bbd30d94e5f20c6dbb683c488b0ce8d30d Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 12 Feb 2024 22:44:23 +0100 Subject: [PATCH 033/101] Keep only "naive" implementation of decomon dense layer --- src/decomon/layers/core/dense.py | 98 +------------------------------- tests/test_dense.py | 22 ++++--- 2 files changed, 11 insertions(+), 109 deletions(-) diff --git a/src/decomon/layers/core/dense.py b/src/decomon/layers/core/dense.py index e84a397c..0638abf0 100644 --- a/src/decomon/layers/core/dense.py +++ b/src/decomon/layers/core/dense.py @@ -6,7 +6,7 @@ from decomon.types import Tensor -class DecomonNaiveDense(DecomonLayer): +class DecomonDense(DecomonLayer): layer: Dense linear = True @@ -29,99 +29,3 @@ def get_affine_representation(self): b = K.repeat(b[None], dim, axis=0) return w, b - - -class DecomonDense(DecomonLayer): - layer: Dense - linear = True - - def _get_pseudo_affine_representation(self): - w = self.layer.kernel - b = self.layer.bias if self.layer.use_bias else K.zeros((self.layer.units,)) - return w, b - - def forward_ibp_propagate(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, Tensor]: - w, b = self._get_pseudo_affine_representation() - - z_value = K.cast(0.0, dtype=w.dtype) - w_pos = K.maximum(w, z_value) - w_neg = K.minimum(w, z_value) - - kwargs_dot = dict(nb_merging_axes=1, missing_batchsize=(False, True)) - - l_c = batch_multid_dot(lower, w_pos, **kwargs_dot) + batch_multid_dot(upper, w_neg, **kwargs_dot) + b - u_c = batch_multid_dot(upper, w_pos, **kwargs_dot) + batch_multid_dot(lower, w_neg, **kwargs_dot) + b - - return l_c, u_c - - def forward_affine_propagate( - self, input_affine_bounds, input_constant_bounds - ) -> tuple[Tensor, Tensor, Tensor, Tensor]: - w_l_1, b_l_1, w_u_1, b_u_1 = input_affine_bounds - w_2, b_2 = self._get_pseudo_affine_representation() - diagonal = ( - self.inputs_outputs_spec.is_diagonal_bounds(input_affine_bounds), - False, - ) - missing_batchsize = ( - self.inputs_outputs_spec.is_wo_batch_bounds(input_affine_bounds), - True, - ) - kwargs_dot_w = dict(nb_merging_axes=1, missing_batchsize=missing_batchsize, diagonal=diagonal) - kwargs_dot_b = dict(nb_merging_axes=1, missing_batchsize=missing_batchsize) - - z_value = K.cast(0.0, dtype=w_2.dtype) - w_2_pos = K.maximum(w_2, z_value) - w_2_neg = K.minimum(w_2, z_value) - - w_l = batch_multid_dot(w_l_1, w_2_pos, **kwargs_dot_w) + batch_multid_dot(w_u_1, w_2_neg, **kwargs_dot_w) - w_u = batch_multid_dot(w_u_1, w_2_pos, **kwargs_dot_w) + batch_multid_dot(w_l_1, w_2_neg, **kwargs_dot_w) - b_l = batch_multid_dot(b_l_1, w_2_pos, **kwargs_dot_b) + batch_multid_dot(b_u_1, w_2_neg, **kwargs_dot_b) + b_2 - b_u = batch_multid_dot(b_u_1, w_2_pos, **kwargs_dot_b) + batch_multid_dot(b_l_1, w_2_neg, **kwargs_dot_b) + b_2 - - return w_l, b_l, w_u, b_u - - def backward_affine_propagate( - self, output_affine_bounds, input_constant_bounds - ) -> tuple[Tensor, Tensor, Tensor, Tensor]: - w_1, b_1 = self._get_pseudo_affine_representation() - w_l_2, b_l_2, w_u_2, b_u_2 = output_affine_bounds - - # affine bounds represented in diagonal mode? - diagonal_bounds = self.inputs_outputs_spec.is_diagonal_bounds(output_affine_bounds) - if diagonal_bounds: - raise NotImplementedError - # missing batch axis in affine bounds? - nb_batch_axis = 0 if self.inputs_outputs_spec.is_wo_batch_bounds(output_affine_bounds) else 1 - - nb_nonbatch_axes_keras_input = len(w_l_2.shape) - len(b_l_2.shape) - - # Merge weights on "units" axis and reorder axes - transposed_axes = ( - tuple(range(1, nb_nonbatch_axes_keras_input + nb_batch_axis)) - + (0,) - + tuple(range(nb_nonbatch_axes_keras_input + nb_batch_axis, len(w_l_2.shape))) - ) - w_l = K.transpose( - K.tensordot(w_1, w_l_2, axes=[[-1], [nb_nonbatch_axes_keras_input - 1 + nb_batch_axis]]), - axes=transposed_axes, - ) - w_u = K.transpose( - K.tensordot(w_1, w_u_2, axes=[[-1], [nb_nonbatch_axes_keras_input - 1 + nb_batch_axis]]), - axes=transposed_axes, - ) - - # Merge layer bias with backward weights on "units" axe and reduce on other input axes - reduced_axes = list(range(1, nb_nonbatch_axes_keras_input - 1 + nb_batch_axis)) - b_l = K.sum( - K.tensordot(b_1, w_l_2, axes=[[-1], [nb_nonbatch_axes_keras_input - 1 + nb_batch_axis]]), axis=reduced_axes - ) - b_u = K.sum( - K.tensordot(b_1, w_u_2, axes=[[-1], [nb_nonbatch_axes_keras_input - 1 + nb_batch_axis]]), axis=reduced_axes - ) - - # Add bias from current backward bounds - b_l += b_l_2 - b_u += b_u_2 - - return w_l, b_l, w_u, b_u diff --git a/tests/test_dense.py b/tests/test_dense.py index ae23f680..2a97a915 100644 --- a/tests/test_dense.py +++ b/tests/test_dense.py @@ -4,12 +4,10 @@ from keras.layers import Dense, Input from decomon.keras_utils import batch_multid_dot -from decomon.layers.core.dense import DecomonDense, DecomonNaiveDense +from decomon.layers.core.dense import DecomonDense -@pytest.mark.parametrize("decomon_layer_class", [DecomonNaiveDense, DecomonDense]) def test_decomon_dense( - decomon_layer_class, use_bias, randomize, ibp, @@ -25,6 +23,7 @@ def test_decomon_dense( ): decimal = 5 units = 7 + decomon_layer_class = DecomonDense keras_symbolic_input = keras_symbolic_input_fn() input_shape = keras_symbolic_input.shape[1:] @@ -55,15 +54,14 @@ def test_decomon_dense( keras_output = layer(keras_input) # check affine representation is ok - if decomon_layer_class == DecomonNaiveDense: - w, b = decomon_layer.get_affine_representation() - keras_output_2 = batch_multid_dot(keras_input, w, missing_batchsize=(False, True)) + b - np.testing.assert_almost_equal( - K.convert_to_numpy(keras_output), - K.convert_to_numpy(keras_output_2), - decimal=decimal, - err_msg="wrong affine representation", - ) + w, b = decomon_layer.get_affine_representation() + keras_output_2 = batch_multid_dot(keras_input, w, missing_batchsize=(False, True)) + b + np.testing.assert_almost_equal( + K.convert_to_numpy(keras_output), + K.convert_to_numpy(keras_output_2), + decimal=decimal, + err_msg="wrong affine representation", + ) decomon_output = decomon_layer(decomon_inputs) From 3f4382c36fa8be1bc02edf1e0ea01692f810a08b Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 12 Feb 2024 23:12:25 +0100 Subject: [PATCH 034/101] Test DecomonActivation on all kind of inputs (diag/nobatch/identity) --- tests/test_activation.py | 41 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/tests/test_activation.py b/tests/test_activation.py index 16906621..688f4af1 100644 --- a/tests/test_activation.py +++ b/tests/test_activation.py @@ -7,58 +7,48 @@ from decomon.layers.activations.activation import DecomonActivation -@pytest.mark.parametrize("input_shape", [(1,), (3,), (5, 2, 3)], ids=["0d", "1d", "multid"]) def test_decomon_activation( activation, slope, ibp, affine, propagation, - input_shape, perturbation_domain, batchsize, + keras_symbolic_input_fn, + decomon_symbolic_input_fn, + keras_input_fn, + decomon_input_fn, helpers, ): + decimal = 5 decomon_layer_class = DecomonActivation - keras_symbolic_input = Input(input_shape) + # init + build keras layer + keras_symbolic_input = keras_symbolic_input_fn() layer = Activation(activation=activation) layer(keras_symbolic_input) - output_shape = layer.output.shape[1:] - decomon_symbolic_inputs = helpers.get_decomon_symbolic_inputs( - model_input_shape=input_shape, - model_output_shape=output_shape, - layer_input_shape=input_shape, - layer_output_shape=output_shape, - ibp=ibp, - affine=affine, - propagation=propagation, - perturbation_domain=perturbation_domain, - ) + # init + build decomon layer + output_shape = layer.output.shape[1:] + model_output_shape = output_shape + decomon_symbolic_inputs = decomon_symbolic_input_fn(output_shape=output_shape) decomon_layer = decomon_layer_class( layer=layer, ibp=ibp, affine=affine, propagation=propagation, perturbation_domain=perturbation_domain, - model_output_shape=output_shape, + model_output_shape=model_output_shape, slope=slope, ) decomon_layer(decomon_symbolic_inputs) - keras_input = helpers.generate_random_tensor(input_shape, batchsize=batchsize) - decomon_inputs = helpers.generate_simple_decomon_layer_inputs_from_keras_input( - keras_input=keras_input, - layer_output_shape=output_shape, - ibp=ibp, - affine=affine, - propagation=propagation, - perturbation_domain=perturbation_domain, - ) + # call on actual inputs + keras_input = keras_input_fn() + decomon_inputs = decomon_input_fn(keras_input=keras_input, output_shape=output_shape) keras_output = layer(keras_input) - decomon_output = decomon_layer(decomon_inputs) # check output shapes @@ -76,4 +66,5 @@ def test_decomon_activation( ibp=ibp, affine=affine, propagation=propagation, + decimal=decimal, ) From 547a281e6d2bae9b479f9375d6c40d1aad220a64 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 12 Feb 2024 23:15:26 +0100 Subject: [PATCH 035/101] Manage case with empty (identity) affine bounds in outputs --- tests/conftest.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6131c6df..3cbb88a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -373,9 +373,20 @@ def assert_decomon_output_compare_with_keras_input_output_single_layer( affine_bounds_propagated, constant_bounds_propagated = inputs_outputs_spec.split_outputs(outputs=decomon_output) if affine or propagation == Propagation.BACKWARD: - w_l, b_l, w_u, b_u = affine_bounds_propagated - lower_affine = batch_multid_dot(keras_input, w_l) + b_l - upper_affine = batch_multid_dot(keras_input, w_u) + b_u + if len(affine_bounds_propagated) == 0: + # identity case + lower_affine = keras_input + upper_affine = keras_input + else: + w_l, b_l, w_u, b_u = affine_bounds_propagated + diagonal = (False, w_l.shape == b_l.shape) + missing_batchsize = (False, len(b_l.shape) < len(keras_output.shape)) + lower_affine = ( + batch_multid_dot(keras_input, w_l, diagonal=diagonal, missing_batchsize=missing_batchsize) + b_l + ) + upper_affine = ( + batch_multid_dot(keras_input, w_u, diagonal=diagonal, missing_batchsize=missing_batchsize) + b_u + ) Helpers.assert_ordered(lower_affine, keras_output, decimal=decimal, err_msg="lower_affine not ok") Helpers.assert_ordered(keras_output, upper_affine, decimal=decimal, err_msg="upper_affine not ok") From 083043f583e06f47640329a535ffb3de71ba2d16 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 12 Feb 2024 23:26:57 +0100 Subject: [PATCH 036/101] Fix DecomonLinear.compute_output_shape() --- src/decomon/core.py | 7 +++++++ src/decomon/layers/activations/activation.py | 21 ++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/decomon/core.py b/src/decomon/core.py index 69afc7af..e96a1fa1 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -275,6 +275,13 @@ def flatten_outputs( else: return affine_bounds_propagated + constant_bounds_propagated + def flatten_outputs_shape( + self, + affine_bounds_propagated_shape: list[tuple[Optional[int], ...]], + constant_bounds_propagated_shape: Optional[list[tuple[Optional[int], ...]]] = None, + ) -> list[tuple[Optional[int], ...]]: + return self.flatten_outputs(affine_bounds_propagated=affine_bounds_propagated_shape, constant_bounds_propagated=constant_bounds_propagated_shape) # type: ignore + def is_identity_bounds(self, affine_bounds: list[Tensor]) -> bool: return len(affine_bounds) == 0 diff --git a/src/decomon/layers/activations/activation.py b/src/decomon/layers/activations/activation.py index c597808e..e72bfecd 100644 --- a/src/decomon/layers/activations/activation.py +++ b/src/decomon/layers/activations/activation.py @@ -1,6 +1,7 @@ from collections.abc import Callable from typing import Any, Optional +import keras from keras import Layer from keras.activations import linear, relu from keras.layers import Activation @@ -132,6 +133,9 @@ def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: self.decomon_activation.build(input_shape=input_shape) super().build(input_shape=input_shape) + def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: + return self.decomon_activation.compute_output_shape(input_shape) + def call(self, inputs: list[Tensor]) -> list[Tensor]: return self.decomon_activation.call(inputs=inputs) @@ -147,6 +151,23 @@ def call(self, inputs: list[Tensor]) -> list[Tensor]: affine_bounds_propagated=affine_bounds_to_propagate, constant_bounds_propagated=constant_oracle_bounds ) + def compute_output_spec(self, inputs: list[keras.KerasTensor]) -> list[keras.KerasTensor]: + return self.call(inputs=inputs) + + def compute_output_shape( + self, + input_shape: list[tuple[Optional[int], ...]], + ) -> list[tuple[Optional[int], ...]]: + ( + affine_bounds_to_propagate_shape, + constant_oracle_bounds_shape, + model_inputs_shape, + ) = self.inputs_outputs_spec.split_input_shape(input_shape=input_shape) + return self.inputs_outputs_spec.flatten_outputs_shape( + affine_bounds_propagated_shape=affine_bounds_to_propagate_shape, + constant_bounds_propagated_shape=constant_oracle_bounds_shape, + ) + class DecomonReLU(DecomonBaseActivation): diagonal = True From 4b61748f6fb254824d005110b25febd15425e88b Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 13 Feb 2024 00:03:11 +0100 Subject: [PATCH 037/101] Fix typing issues --- src/decomon/layers/activations/activation.py | 4 ++-- src/decomon/layers/core/dense.py | 2 +- src/decomon/layers/layer.py | 22 ++++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/decomon/layers/activations/activation.py b/src/decomon/layers/activations/activation.py index e72bfecd..0c42d622 100644 --- a/src/decomon/layers/activations/activation.py +++ b/src/decomon/layers/activations/activation.py @@ -113,14 +113,14 @@ def get_affine_bounds(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, Tenso return self.decomon_activation.get_affine_bounds(lower=lower, upper=upper) def forward_affine_propagate( - self, input_affine_bounds, input_constant_bounds + self, input_affine_bounds: list[Tensor], input_constant_bounds: list[Tensor] ) -> tuple[Tensor, Tensor, Tensor, Tensor]: return self.decomon_activation.forward_affine_propagate( input_affine_bounds=input_affine_bounds, input_constant_bounds=input_constant_bounds ) def backward_affine_propagate( - self, output_affine_bounds, input_constant_bounds + self, output_affine_bounds: list[Tensor], input_constant_bounds: list[Tensor] ) -> tuple[Tensor, Tensor, Tensor, Tensor]: return self.decomon_activation.backward_affine_propagate( output_affine_bounds=output_affine_bounds, input_constant_bounds=input_constant_bounds diff --git a/src/decomon/layers/core/dense.py b/src/decomon/layers/core/dense.py index 0638abf0..27a4baf7 100644 --- a/src/decomon/layers/core/dense.py +++ b/src/decomon/layers/core/dense.py @@ -10,7 +10,7 @@ class DecomonDense(DecomonLayer): layer: Dense linear = True - def get_affine_representation(self): + def get_affine_representation(self) -> tuple[Tensor, Tensor]: w = self.layer.kernel b = self.layer.bias if self.layer.use_bias else K.zeros((self.layer.units,)) diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index 7ac7be90..38e874df 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -259,7 +259,7 @@ def forward_ibp_propagate(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, T if self.linear: w, b = self.get_affine_representation() is_diag = w.shape == b.shape - kwargs_dot = dict(missing_batchsize=(False, True), diagonal=(False, is_diag)) + kwargs_dot: dict[str, Any] = dict(missing_batchsize=(False, True), diagonal=(False, is_diag)) z_value = K.cast(0.0, dtype=w.dtype) w_pos = K.maximum(w, z_value) @@ -275,7 +275,7 @@ def forward_ibp_propagate(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, T ) def forward_affine_propagate( - self, input_affine_bounds, input_constant_bounds + self, input_affine_bounds: list[Tensor], input_constant_bounds: list[Tensor] ) -> tuple[Tensor, Tensor, Tensor, Tensor]: """Propagate model affine bounds in forward direction. @@ -319,7 +319,7 @@ def forward_affine_propagate( ) def backward_affine_propagate( - self, output_affine_bounds, input_constant_bounds + self, output_affine_bounds: list[Tensor], input_constant_bounds: list[Tensor] ) -> tuple[Tensor, Tensor, Tensor, Tensor]: """Propagate model affine bounds in backward direction. @@ -714,11 +714,11 @@ def _combine_affine_bounds_generic( nb_axes_wo_batchsize_y = len(b_l_1.shape) - 1 #  NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b - kwargs_dot_w = dict( + kwargs_dot_w: dict[str, Any] = dict( nb_merging_axes=nb_axes_wo_batchsize_y, diagonal=diagonal, ) - kwargs_dot_b = dict( + kwargs_dot_b: dict[str, Any] = dict( nb_merging_axes=nb_axes_wo_batchsize_y, diagonal=(False, diagonal[1]), ) @@ -770,12 +770,12 @@ def _combine_affine_bounds_right_from_linear( missing_batchsize = (False, True) # NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b - kwargs_dot_w = dict( + kwargs_dot_w: dict[str, Any] = dict( nb_merging_axes=nb_axes_wo_batchsize_y, missing_batchsize=missing_batchsize, diagonal=diagonal, ) - kwargs_dot_b = dict( + kwargs_dot_b: dict[str, Any] = dict( nb_merging_axes=nb_axes_wo_batchsize_y, missing_batchsize=missing_batchsize, diagonal=(False, diagonal[1]), @@ -822,12 +822,12 @@ def _combine_affine_bounds_left_from_linear( missing_batchsize = (True, False) #   NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b - kwargs_dot_w = dict( + kwargs_dot_w: dict[str, Any] = dict( nb_merging_axes=nb_axes_wo_batchsize_y, missing_batchsize=missing_batchsize, diagonal=diagonal, ) - kwargs_dot_b = dict( + kwargs_dot_b: dict[str, Any] = dict( nb_merging_axes=nb_axes_wo_batchsize_y, missing_batchsize=missing_batchsize, diagonal=(False, diagonal[1]), @@ -870,12 +870,12 @@ def _combine_affine_bounds_both_from_linear( missing_batchsize = (True, True) #   NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b - kwargs_dot_w = dict( + kwargs_dot_w: dict[str, Any] = dict( nb_merging_axes=nb_axes_wo_batchsize_y, missing_batchsize=missing_batchsize, diagonal=diagonal, ) - kwargs_dot_b = dict( + kwargs_dot_b: dict[str, Any] = dict( nb_merging_axes=nb_axes_wo_batchsize_y, missing_batchsize=missing_batchsize, diagonal=(False, diagonal[1]), From ff13190b6d71cee0f6edc1b61d3d38a3b9bbac9e Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 13 Feb 2024 10:27:12 +0100 Subject: [PATCH 038/101] Add dtype to decomon inputs --- tests/conftest.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3cbb88a4..87abef39 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -124,7 +124,7 @@ def in_GPU_mode() -> bool: raise NotImplementedError(f"Not implemented for {backend} backend.") @staticmethod - def generate_random_tensor(shape_wo_batchsize, batchsize=10, dtype="float32"): + def generate_random_tensor(shape_wo_batchsize, batchsize=10, dtype=keras_config.floatx()): shape = (batchsize,) + shape_wo_batchsize return K.convert_to_tensor(np.random.random(shape), dtype=dtype) @@ -179,6 +179,7 @@ def get_decomon_symbolic_inputs( empty=False, diag=False, nobatch=False, + dtype=keras_config.floatx(), ): """Generate decomon symbolic inputs for a decomon layer @@ -214,12 +215,14 @@ def get_decomon_symbolic_inputs( diag=diag, nobatch=nobatch, ) - model_inputs = [Input(shape) for shape in model_inputs_shape] - constant_oracle_bounds = [Input(shape) for shape in constant_oracle_bounds_shape] + model_inputs = [Input(shape, dtype=dtype) for shape in model_inputs_shape] + constant_oracle_bounds = [Input(shape, dtype=dtype) for shape in constant_oracle_bounds_shape] if nobatch: - affine_bounds_to_propagate = [Input(batch_shape=shape) for shape in affine_bounds_to_propagate_shape] + affine_bounds_to_propagate = [ + Input(batch_shape=shape, dtype=dtype) for shape in affine_bounds_to_propagate_shape + ] else: - affine_bounds_to_propagate = [Input(shape=shape) for shape in affine_bounds_to_propagate_shape] + affine_bounds_to_propagate = [Input(shape=shape, dtype=dtype) for shape in affine_bounds_to_propagate_shape] inputs_outputs_spec = InputsOutputsSpec( ibp=ibp, affine=affine, @@ -246,6 +249,7 @@ def generate_simple_decomon_layer_inputs_from_keras_input( empty=False, diag=False, nobatch=False, + dtype=keras_config.floatx(), ): """Generate simple decomon inputs for a layer from the corresponding keras input @@ -290,10 +294,10 @@ def generate_simple_decomon_layer_inputs_from_keras_input( bias_shape = layer_output_shape flatten_bias_dim = int(np.prod(bias_shape)) if diag: - w_in = K.ones(bias_shape) + w_in = K.ones(bias_shape, dtype=dtype) else: - w_in = K.reshape(K.eye(flatten_bias_dim), bias_shape + bias_shape) - b_in = K.zeros(bias_shape) + w_in = K.reshape(K.eye(flatten_bias_dim, dtype=dtype), bias_shape + bias_shape) + b_in = K.zeros(bias_shape, dtype=dtype) if not nobatch: w_in = K.repeat( w_in[None], From a6ba40db67b52a6597c9600f59ee4504351c5470 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 13 Feb 2024 13:55:05 +0100 Subject: [PATCH 039/101] Use use_bias in dense layer --- tests/test_dense.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dense.py b/tests/test_dense.py index 2a97a915..1fe10df7 100644 --- a/tests/test_dense.py +++ b/tests/test_dense.py @@ -33,7 +33,7 @@ def test_decomon_dense( keras_input = keras_input_fn() decomon_inputs = decomon_input_fn(keras_input=keras_input, output_shape=output_shape) - layer = Dense(units=units) + layer = Dense(units=units, use_bias=use_bias) layer(keras_symbolic_input) if randomize: From 63bc33737f41a810c1ff78a880ceffafd2794a9c Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 13 Feb 2024 13:54:03 +0100 Subject: [PATCH 040/101] Integrate previous "standard" inputs in tests --- tests/conftest.py | 682 +++++++++++++++++++++++++++++++++++- tests/test_activation.py | 32 +- tests/test_decomon_layer.py | 40 ++- tests/test_dense.py | 58 +-- 4 files changed, 750 insertions(+), 62 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 87abef39..6a96c4b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,13 @@ import pytest from keras import KerasTensor, Model from keras.layers import Input -from pytest_cases import fixture, fixture_union, param_fixture, param_fixtures +from pytest_cases import ( + fixture, + fixture_union, + param_fixture, + param_fixtures, + unpack_fixture, +) from decomon.core import BoxDomain, InputsOutputsSpec, Propagation, Slope from decomon.keras_utils import ( @@ -43,6 +49,7 @@ ) slope = param_fixture("slope", [s.value for s in Slope]) n = param_fixture("n", list(range(10))) +odd = param_fixture("odd", list(range(2))) use_bias = param_fixture("use_bias", [True, False]) randomize = param_fixture("randomize", [True, False]) padding = param_fixture("padding", ["same", "valid"]) @@ -333,6 +340,381 @@ def generate_simple_decomon_layer_inputs_from_keras_input( model_inputs=model_inputs, ) + @staticmethod + def get_standard_values_0d_box(n, batchsize=10): + """A set of functions with their monotonic decomposition for testing the activations""" + w_u_ = np.ones(batchsize) + b_u_ = np.zeros(batchsize) + w_l_ = np.ones(batchsize) + b_l_ = np.zeros(batchsize) + + if n == 0: + # identity + y_ = np.linspace(-2, -1, batchsize) + x_ = np.linspace(-2, -1, batchsize) + + elif n == 1: + y_ = np.linspace(1, 2, batchsize) + x_ = np.linspace(1, 2, batchsize) + + elif n == 2: + y_ = np.linspace(-1, 1, batchsize) + x_ = np.linspace(-1, 1, batchsize) + + elif n == 3: + # identity + y_ = np.linspace(-2, -1, batchsize) + x_ = np.linspace(-2, -1, batchsize) + + elif n == 4: + y_ = np.linspace(1, 2, batchsize) + x_ = np.linspace(1, 2, batchsize) + + elif n == 5: + y_ = np.linspace(-1, 1, batchsize) + x_ = np.linspace(-1, 1, batchsize) + + elif n == 6: + # cosine function + x_ = np.linspace(-np.pi, np.pi, batchsize) + y_ = np.cos(x_) + w_u_ = np.zeros_like(x_) + w_l_ = np.zeros_like(x_) + b_u_ = np.ones_like(x_) + b_l_ = -np.ones_like(x_) + + elif n == 7: + # h and g >0 + h_ = np.linspace(0.5, 2, batchsize) + g_ = np.linspace(1, 2, batchsize)[::-1] + x_ = h_ + g_ + y_ = h_ + g_ + + elif n == 8: + # h <0 and g <0 + # h_max+g_max <=0 + h_ = np.linspace(-2, -1, batchsize) + g_ = np.linspace(-2, -1, batchsize)[::-1] + y_ = h_ + g_ + x_ = h_ + g_ + + elif n == 9: + # h >0 and g <0 + # h_min+g_min >=0 + h_ = np.linspace(4, 5, batchsize) + g_ = np.linspace(-2, -1, batchsize)[::-1] + y_ = h_ + g_ + x_ = h_ + g_ + + else: + raise ValueError("n must be between 0 and 9.") + + x_min_ = x_.min() + np.zeros_like(x_) + x_max_ = x_.max() + np.zeros_like(x_) + + x_0_ = np.concatenate([x_min_[:, None], x_max_[:, None]], 1) + + u_c_ = np.max(y_) * np.ones((batchsize,)) + l_c_ = np.min(y_) * np.ones((batchsize,)) + + output = [ + x_[:, None], + y_[:, None], + x_0_[:, :, None], + u_c_[:, None], + w_u_[:, None, None], + b_u_[:, None], + l_c_[:, None], + w_l_[:, None, None], + b_l_[:, None], + ] + + # cast element + return [e.astype(keras_config.floatx()) for e in output] + + @staticmethod + def get_tensor_decomposition_0d_box(): + return [ + Input((1,), dtype=keras_config.floatx()), + Input((1,), dtype=keras_config.floatx()), + Input((2, 1), dtype=keras_config.floatx()), + Input((1,), dtype=keras_config.floatx()), + Input((1, 1), dtype=keras_config.floatx()), + Input((1,), dtype=keras_config.floatx()), + Input((1,), dtype=keras_config.floatx()), + Input((1, 1), dtype=keras_config.floatx()), + Input((1,), dtype=keras_config.floatx()), + ] + + @staticmethod + def get_standard_values_1d_box(odd=1, batchsize=10): + ( + x_0, + y_0, + z_0, + u_c_0, + w_u_0, + b_u_0, + l_c_0, + w_l_0, + b_l_0, + ) = Helpers.get_standard_values_0d_box(n=0, batchsize=batchsize) + ( + x_1, + y_1, + z_1, + u_c_1, + w_u_1, + b_u_1, + l_c_1, + w_l_1, + b_l_1, + ) = Helpers.get_standard_values_0d_box(n=1, batchsize=batchsize) + + if not odd: + # output (x_0+x_1, x_0+2*x_1) + x_ = np.concatenate([x_0, x_1], -1) + z_min_ = np.concatenate([z_0[:, 0], z_1[:, 0]], -1) + z_max_ = np.concatenate([z_0[:, 1], z_1[:, 1]], -1) + z_ = np.concatenate([z_min_[:, None], z_max_[:, None]], 1) + y_ = np.concatenate([y_0 + y_1, y_0 + 2 * y_1], -1) + b_u_ = np.concatenate([b_u_0 + b_u_1, b_u_0 + 2 * b_u_1], -1) + u_c_ = np.concatenate([u_c_0 + u_c_1, u_c_0 + 2 * u_c_1], -1) + b_l_ = np.concatenate([b_l_0 + b_l_1, b_l_0 + 2 * b_l_1], -1) + l_c_ = np.concatenate([l_c_0 + l_c_1, l_c_0 + 2 * l_c_1], -1) + + w_u_ = np.zeros((len(x_), 2, 2)) + w_u_[:, 0, 0] = w_u_0[:, 0, 0] + w_u_[:, 1, 0] = w_u_1[:, 0, 0] + w_u_[:, 0, 1] = w_u_0[:, 0, 0] + w_u_[:, 1, 1] = 2 * w_u_1[:, 0, 0] + + w_l_ = np.zeros((len(x_), 2, 2)) + w_l_[:, 0, 0] = w_l_0[:, 0, 0] + w_l_[:, 1, 0] = w_l_1[:, 0, 0] + w_l_[:, 0, 1] = w_l_0[:, 0, 0] + w_l_[:, 1, 1] = 2 * w_l_1[:, 0, 0] + + else: + ( + x_2, + y_2, + z_2, + u_c_2, + w_u_2, + b_u_2, + l_c_2, + w_l_2, + b_l_2, + ) = Helpers.get_standard_values_0d_box(n=2, batchsize=batchsize) + + # output (x_0+x_1, x_0+2*x_0, x_2) + x_ = np.concatenate([x_0, x_1, x_2], -1) + z_min_ = np.concatenate([z_0[:, 0], z_1[:, 0], z_2[:, 0]], -1) + z_max_ = np.concatenate([z_0[:, 1], z_1[:, 1], z_2[:, 1]], -1) + z_ = np.concatenate([z_min_[:, None], z_max_[:, None]], 1) + y_ = np.concatenate([y_0 + y_1, y_0 + 2 * y_1, y_2], -1) + b_u_ = np.concatenate([b_u_0 + b_u_1, b_u_0 + 2 * b_u_1, b_u_2], -1) + b_l_ = np.concatenate([b_l_0 + b_l_1, b_l_0 + 2 * b_l_1, b_l_2], -1) + u_c_ = np.concatenate([u_c_0 + u_c_1, u_c_0 + 2 * u_c_1, u_c_2], -1) + l_c_ = np.concatenate([l_c_0 + l_c_1, l_c_0 + 2 * l_c_1, l_c_2], -1) + + w_u_ = np.zeros((len(x_), 3, 3)) + w_u_[:, 0, 0] = w_u_0[:, 0, 0] + w_u_[:, 1, 0] = w_u_1[:, 0, 0] + w_u_[:, 0, 1] = w_u_0[:, 0, 0] + w_u_[:, 1, 1] = 2 * w_u_1[:, 0, 0] + w_u_[:, 2, 2] = w_u_2[:, 0, 0] + + w_l_ = np.zeros((len(x_), 3, 3)) + w_l_[:, 0, 0] = w_l_0[:, 0, 0] + w_l_[:, 1, 0] = w_l_1[:, 0, 0] + w_l_[:, 0, 1] = w_l_0[:, 0, 0] + w_l_[:, 1, 1] = 2 * w_l_1[:, 0, 0] + w_l_[:, 2, 2] = w_l_2[:, 0, 0] + + return [x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_] + + @staticmethod + def get_tensor_decomposition_1d_box(odd=1): + n = Helpers.get_input_dim_1d_box(odd) + + return [ + Input((n,), dtype=keras_config.floatx()), + Input((n,), dtype=keras_config.floatx()), + Input((2, n), dtype=keras_config.floatx()), + Input((n,), dtype=keras_config.floatx()), + Input((n, n), dtype=keras_config.floatx()), + Input((n,), dtype=keras_config.floatx()), + Input((n,), dtype=keras_config.floatx()), + Input((n, n), dtype=keras_config.floatx()), + Input((n,), dtype=keras_config.floatx()), + ] + + @staticmethod + def get_input_dim_1d_box(odd): + if odd: + return 3 + else: + return 2 + + @staticmethod + def get_standard_values_images_box(data_format="channels_last", odd=0, m0=0, m1=1, batchsize=10): + if data_format == "channels_last": + output = Helpers.build_image_from_2D_box(odd=odd, m0=m0, m1=m1, batchsize=batchsize) + x_0, y_0, z_0, u_c_0, w_u_0, b_u_0, l_c_0, w_l_0, b_l_0 = output + + x_ = x_0 + z_ = z_0 + + y_0 = y_0[:, :, :, None] + b_u_0 = b_u_0[:, :, :, None] + b_l_0 = b_l_0[:, :, :, None] + u_c_0 = u_c_0[:, :, :, None] + l_c_0 = l_c_0[:, :, :, None] + w_u_0 = w_u_0[:, :, :, :, None] + w_l_0 = w_l_0[:, :, :, :, None] + y_ = np.concatenate([y_0, y_0], -1) + b_u_ = np.concatenate([b_u_0, b_u_0], -1) + b_l_ = np.concatenate([b_l_0, b_l_0], -1) + u_c_ = np.concatenate([u_c_0, u_c_0], -1) + l_c_ = np.concatenate([l_c_0, l_c_0], -1) + w_u_ = np.concatenate([w_u_0, w_u_0], -1) + w_l_ = np.concatenate([w_l_0, w_l_0], -1) + + else: + output = Helpers.get_standard_values_images_box( + data_format="channels_last", odd=odd, m0=m0, m1=m1, batchsize=batchsize + ) + + x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_ = output + y_ = np.transpose(y_, (0, 3, 1, 2)) + u_c_ = np.transpose(u_c_, (0, 3, 1, 2)) + l_c_ = np.transpose(l_c_, (0, 3, 1, 2)) + b_u_ = np.transpose(b_u_, (0, 3, 1, 2)) + b_l_ = np.transpose(b_l_, (0, 3, 1, 2)) + w_u_ = np.transpose(w_u_, (0, 1, 4, 2, 3)) + w_l_ = np.transpose(w_l_, (0, 1, 4, 2, 3)) + + return [x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_] + + @staticmethod + def get_tensor_decomposition_images_box(data_format, odd): + n = Helpers.get_input_dim_images_box(odd) + + if data_format == "channels_last": + # x, y, z, u, w_u, b_u, l, w_l, b_l + + output = [ + Input((2,), dtype=keras_config.floatx()), + Input((n, n, 2), dtype=keras_config.floatx()), + Input((2, 2), dtype=keras_config.floatx()), + Input((n, n, 2), dtype=keras_config.floatx()), + Input((2, n, n, 2), dtype=keras_config.floatx()), + Input((n, n, 2), dtype=keras_config.floatx()), + Input((n, n, 2), dtype=keras_config.floatx()), + Input((2, n, n, 2), dtype=keras_config.floatx()), + Input((n, n, 2), dtype=keras_config.floatx()), + ] + + else: + output = [ + Input((2,), dtype=keras_config.floatx()), + Input((2, n, n), dtype=keras_config.floatx()), + Input((2, 2), dtype=keras_config.floatx()), + Input((2, n, n), dtype=keras_config.floatx()), + Input((2, 2, n, n), dtype=keras_config.floatx()), + Input((2, n, n), dtype=keras_config.floatx()), + Input((2, n, n), dtype=keras_config.floatx()), + Input((2, 2, n, n), dtype=keras_config.floatx()), + Input((2, n, n), dtype=keras_config.floatx()), + ] + + return output + + @staticmethod + def get_input_dim_images_box(odd): + if odd: + return 7 + else: + return 6 + + @staticmethod + def build_image_from_2D_box(odd=0, m0=0, m1=1, batchsize=10): + ( + x_0, + y_0, + z_0, + u_c_0, + w_u_0, + b_u_0, + l_c_0, + w_l_0, + b_l_0, + ) = Helpers.build_image_from_1D_box(odd=odd, m=m0, batchsize=batchsize) + ( + x_1, + y_1, + z_1, + u_c_1, + w_u_1, + b_u_1, + l_c_1, + w_l_1, + b_l_1, + ) = Helpers.build_image_from_1D_box(odd=odd, m=m1, batchsize=batchsize) + + x_ = np.concatenate([x_0, x_1], -1) + z_min_ = np.concatenate([z_0[:, 0], z_1[:, 0]], -1) + z_max_ = np.concatenate([z_0[:, 1], z_1[:, 1]], -1) + z_ = np.concatenate([z_min_[:, None], z_max_[:, None]], 1) + y_ = y_0 + y_1 + b_u_ = b_u_0 + b_u_1 + b_l_ = b_l_0 + b_l_1 + + u_c_ = u_c_0 + u_c_1 + l_c_ = l_c_0 + l_c_1 + + w_u_ = np.concatenate([w_u_0, w_u_1], 1) + w_l_ = np.concatenate([w_l_0, w_l_1], 1) + + return [x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_] + + @staticmethod + def build_image_from_1D_box(odd=0, m=0, batchsize=10): + n = Helpers.get_input_dim_images_box(odd) + + ( + x_, + y_0, + z_, + u_c_0, + w_u_0, + b_u_0, + l_c_0, + w_l_0, + b_l_0, + ) = Helpers.get_standard_values_0d_box(n=m, batchsize=batchsize) + + y_ = np.concatenate([(i + 1) * y_0 for i in range(n * n)], -1).reshape((-1, n, n)) + b_u_ = np.concatenate([(i + 1) * b_u_0 for i in range(n * n)], -1).reshape((-1, n, n)) + b_l_ = np.concatenate([(i + 1) * b_l_0 for i in range(n * n)], -1).reshape((-1, n, n)) + + u_c_ = np.concatenate([(i + 1) * u_c_0 for i in range(n * n)], -1).reshape((-1, n, n)) + l_c_ = np.concatenate([(i + 1) * l_c_0 for i in range(n * n)], -1).reshape((-1, n, n)) + + w_u_ = np.zeros((len(x_), 1, n * n)) + w_l_ = np.zeros((len(x_), 1, n * n)) + + for i in range(n * n): + w_u_[:, 0, i] = (i + 1) * w_u_0[:, 0, 0] + w_l_[:, 0, i] = (i + 1) * w_l_0[:, 0, 0] + + w_u_ = w_u_.reshape((-1, 1, n, n)) + w_l_ = w_l_.reshape((-1, 1, n, n)) + + return [x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_] + @staticmethod def assert_decomon_outputs_equal(output_1, output_2, decimal=5): assert len(output_1) == len(output_2) @@ -366,16 +748,47 @@ def assert_almost_equal(x: BackendTensor, y: BackendTensor, decimal: int = 5, er @staticmethod def assert_decomon_output_compare_with_keras_input_output_single_layer( decomon_output, keras_output, keras_input, ibp, affine, propagation, decimal=5 + ): + Helpers.assert_decomon_output_compare_with_keras_input_output_layer( + decomon_output, + keras_layer_output=keras_output, + keras_layer_input=keras_input, + keras_model_input=keras_input, + keras_model_output=keras_output, + ibp=ibp, + affine=affine, + propagation=propagation, + decimal=decimal, + ) + + @staticmethod + def assert_decomon_output_compare_with_keras_input_output_layer( + decomon_output, + keras_layer_output, + keras_layer_input, + keras_model_input, + keras_model_output, + ibp, + affine, + propagation, + decimal=5, ): inputs_outputs_spec = InputsOutputsSpec( ibp=ibp, affine=affine, propagation=propagation, - layer_input_shape=keras_input.shape[1:], - model_output_shape=keras_output.shape[1:], + layer_input_shape=keras_layer_input.shape[1:], + model_output_shape=keras_model_output.shape[1:], ) affine_bounds_propagated, constant_bounds_propagated = inputs_outputs_spec.split_outputs(outputs=decomon_output) + if propagation == Propagation.FORWARD: + keras_input = keras_model_input + keras_output = keras_layer_output + else: + keras_input = keras_layer_input + keras_output = keras_model_output + if affine or propagation == Propagation.BACKWARD: if len(affine_bounds_propagated) == 0: # identity case @@ -455,7 +868,8 @@ def helpers(): def simple_layer_input_functions( ibp, affine, propagation, perturbation_domain, batchsize, input_shape, empty, diag, nobatch, helpers ): - keras_symbolic_input_fn = lambda: Input(input_shape) + keras_symbolic_model_input_fn = lambda: Input(input_shape) + keras_symbolic_layer_input_fn = lambda keras_symbolic_model_input: keras_symbolic_model_input decomon_symbolic_input_fn = lambda output_shape: helpers.get_decomon_symbolic_inputs( model_input_shape=input_shape, @@ -471,10 +885,11 @@ def simple_layer_input_functions( nobatch=nobatch, ) - keras_input_fn = lambda: helpers.generate_random_tensor(input_shape, batchsize=batchsize) + keras_model_input_fn = lambda: helpers.generate_random_tensor(input_shape, batchsize=batchsize) + keras_layer_input_fn = lambda keras_model_input: keras_model_input - decomon_input_fn = lambda keras_input, output_shape: helpers.generate_simple_decomon_layer_inputs_from_keras_input( - keras_input=keras_input, + decomon_input_fn = lambda keras_model_input, keras_layer_input, output_shape: helpers.generate_simple_decomon_layer_inputs_from_keras_input( + keras_input=keras_layer_input, layer_output_shape=output_shape, ibp=ibp, affine=affine, @@ -485,11 +900,258 @@ def simple_layer_input_functions( nobatch=nobatch, ) - return keras_symbolic_input_fn, decomon_symbolic_input_fn, keras_input_fn, decomon_input_fn + return ( + keras_symbolic_model_input_fn, + keras_symbolic_layer_input_fn, + decomon_symbolic_input_fn, + keras_model_input_fn, + keras_layer_input_fn, + decomon_input_fn, + True, + ) + + +def convert_standard_input_functions_for_single_layer( + get_tensor_decomposition_fn, get_standard_values_fn, ibp, affine, propagation, perturbation_domain, helpers +): + keras_symbolic_model_input_fn = lambda: get_tensor_decomposition_fn()[0] + keras_symbolic_layer_input_fn = lambda _: get_tensor_decomposition_fn()[1] + keras_model_input_fn = lambda: K.convert_to_tensor(get_standard_values_fn()[0]) + keras_layer_input_fn = lambda _: K.convert_to_tensor(get_standard_values_fn()[1]) + + if propagation == Propagation.FORWARD: + + def decomon_symbolic_input_fn(output_shape): + x, y, z, u_c, w_u, b_u, l_c, w_l, b_l = get_tensor_decomposition_fn() + layer_input_shape = y.shape[1:] + model_input_shape = x.shape[1:] + + affine_bounds_to_propagate = [w_l, b_l, w_u, b_u] + constant_oracle_bounds = [l_c, u_c] + if isinstance(perturbation_domain, BoxDomain): + model_inputs = [z] + else: + raise NotImplementedError + + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + layer_input_shape=layer_input_shape, + model_input_shape=model_input_shape, + model_output_shape=output_shape, + ) + return inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=affine_bounds_to_propagate, + constant_oracle_bounds=constant_oracle_bounds, + model_inputs=model_inputs, + ) + + def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): + x, y, z, u_c, w_u, b_u, l_c, w_l, b_l = get_standard_values_fn() + layer_input_shape = y.shape[1:] + model_input_shape = x.shape[1:] + + if affine: + affine_bounds_to_propagate = [K.convert_to_tensor(a) for a in (w_l, b_l, w_u, b_u)] + else: + affine_bounds_to_propagate = [] + if ibp: + constant_oracle_bounds = [K.convert_to_tensor(a) for a in (l_c, u_c)] + else: + constant_oracle_bounds = [] + if propagation == Propagation.FORWARD and affine: + if isinstance(perturbation_domain, BoxDomain): + model_inputs = [K.convert_to_tensor(z)] + else: + raise NotImplementedError + else: + model_inputs = [] + + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + layer_input_shape=layer_input_shape, + model_input_shape=model_input_shape, + model_output_shape=output_shape, + ) + return inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=affine_bounds_to_propagate, + constant_oracle_bounds=constant_oracle_bounds, + model_inputs=model_inputs, + ) + + else: # backward + + def decomon_symbolic_input_fn(output_shape): + x, y, z, u_c, w_u, b_u, l_c, w_l, b_l = get_tensor_decomposition_fn() + layer_input_shape = y.shape[1:] + model_input_shape = x.shape[1:] + + if ibp: + constant_oracle_bounds = [l_c, u_c] + else: + constant_oracle_bounds = [] + if propagation == Propagation.FORWARD and affine: + if isinstance(perturbation_domain, BoxDomain): + model_inputs = [z] + else: + raise NotImplementedError + else: + model_inputs = [] + + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + layer_input_shape=layer_input_shape, + model_input_shape=model_input_shape, + model_output_shape=output_shape, + ) + + # take identity affine bounds + if affine: + simple_decomon_inputs = helpers.get_decomon_symbolic_inputs( + model_input_shape=model_input_shape, + model_output_shape=output_shape, + layer_input_shape=layer_input_shape, + layer_output_shape=output_shape, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + empty=empty, + diag=diag, + nobatch=nobatch, + ) + affine_bounds_to_propagate, _, _ = inputs_outputs_spec.split_inputs(simple_decomon_inputs) + else: + affine_bounds_to_propagate = [] + + return inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=affine_bounds_to_propagate, + constant_oracle_bounds=constant_oracle_bounds, + model_inputs=model_inputs, + ) + + def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): + x, y, z, u_c, w_u, b_u, l_c, w_l, b_l = get_standard_values_fn() + layer_input_shape = y.shape[1:] + model_input_shape = x.shape[1:] + + if ibp: + constant_oracle_bounds = [K.convert_to_tensor(a) for a in (l_c, u_c)] + else: + constant_oracle_bounds = [] + if propagation == Propagation.FORWARD and affine: + if isinstance(perturbation_domain, BoxDomain): + model_inputs = [K.convert_to_tensor(z)] + else: + raise NotImplementedError + else: + model_inputs = [] + + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + layer_input_shape=layer_input_shape, + model_input_shape=model_input_shape, + model_output_shape=output_shape, + ) + + #  take identity affine bounds + if affine: + simple_decomon_inputs = helpers.generate_simple_decomon_layer_inputs_from_keras_input( + keras_input=keras_layer_input, + layer_output_shape=output_shape, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + empty=empty, + diag=diag, + nobatch=nobatch, + ) + affine_bounds_to_propagate, _, _ = inputs_outputs_spec.split_inputs(simple_decomon_inputs) + else: + affine_bounds_to_propagate = [] + + return inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=affine_bounds_to_propagate, + constant_oracle_bounds=constant_oracle_bounds, + model_inputs=model_inputs, + ) + + return ( + keras_symbolic_model_input_fn, + keras_symbolic_layer_input_fn, + decomon_symbolic_input_fn, + keras_model_input_fn, + keras_layer_input_fn, + decomon_input_fn, + False, + ) + + +@fixture +def standard_layer_input_functions_0d(n, ibp, affine, propagation, batchsize, helpers): + perturbation_domain = BoxDomain() + get_tensor_decomposition_fn = helpers.get_tensor_decomposition_0d_box + get_standard_values_fn = lambda: helpers.get_standard_values_0d_box(n=n, batchsize=batchsize) + return convert_standard_input_functions_for_single_layer( + get_tensor_decomposition_fn, get_standard_values_fn, ibp, affine, propagation, perturbation_domain, helpers + ) + + +@fixture +def standard_layer_input_functions_1d(odd, ibp, affine, propagation, batchsize, helpers): + perturbation_domain = BoxDomain() + get_tensor_decomposition_fn = lambda: helpers.get_tensor_decomposition_1d_box(odd=odd) + get_standard_values_fn = lambda: helpers.get_standard_values_1d_box(odd=odd, batchsize=batchsize) + return convert_standard_input_functions_for_single_layer( + get_tensor_decomposition_fn, get_standard_values_fn, ibp, affine, propagation, perturbation_domain, helpers + ) + + +@fixture +def standard_layer_input_functions_multid(data_format, ibp, affine, propagation, batchsize, helpers): + perturbation_domain = BoxDomain() + odd, m0, m1 = 0, 0, 1 + get_tensor_decomposition_fn = lambda: helpers.get_tensor_decomposition_images_box(data_format=data_format, odd=odd) + get_standard_values_fn = lambda: helpers.get_standard_values_images_box( + data_format=data_format, odd=odd, m0=m0, m1=m1, batchsize=batchsize + ) + return convert_standard_input_functions_for_single_layer( + get_tensor_decomposition_fn, get_standard_values_fn, ibp, affine, propagation, perturbation_domain, helpers + ) layer_input_functions = fixture_union( "layer_input_functions", - [simple_layer_input_functions], - unpack_into="keras_symbolic_input_fn, decomon_symbolic_input_fn, keras_input_fn, decomon_input_fn", + [ + simple_layer_input_functions, + standard_layer_input_functions_0d, + standard_layer_input_functions_1d, + standard_layer_input_functions_multid, + ], + unpack_into="keras_symbolic_model_input_fn, keras_symbolic_layer_input_fn, decomon_symbolic_input_fn, keras_model_input_fn, keras_layer_input_fn, decomon_input_fn, equal_bounds", +) + +( + simple_keras_symbolic_model_input_fn, + simple_keras_symbolic_layer_input_fn, + simple_decomon_symbolic_input_fn, + simple_keras_model_input_fn, + simple_keras_layer_input_fn, + simple_decomon_input_fn, + simple_equal_bounds, +) = unpack_fixture( + "simple_keras_symbolic_model_input_fn, simple_keras_symbolic_layer_input_fn, simple_decomon_symbolic_input_fn, simple_keras_model_input_fn, simple_keras_layer_input_fn, simple_decomon_input_fn, simple_equal_bounds", + simple_layer_input_functions, ) diff --git a/tests/test_activation.py b/tests/test_activation.py index 688f4af1..dabece2a 100644 --- a/tests/test_activation.py +++ b/tests/test_activation.py @@ -15,19 +15,22 @@ def test_decomon_activation( propagation, perturbation_domain, batchsize, - keras_symbolic_input_fn, + keras_symbolic_model_input_fn, + keras_symbolic_layer_input_fn, decomon_symbolic_input_fn, - keras_input_fn, + keras_model_input_fn, + keras_layer_input_fn, decomon_input_fn, helpers, ): - decimal = 5 + decimal = 4 decomon_layer_class = DecomonActivation # init + build keras layer - keras_symbolic_input = keras_symbolic_input_fn() + keras_symbolic_model_input = keras_symbolic_model_input_fn() + keras_symbolic_layer_input = keras_symbolic_layer_input_fn(keras_symbolic_model_input) layer = Activation(activation=activation) - layer(keras_symbolic_input) + layer(keras_symbolic_layer_input) # init + build decomon layer output_shape = layer.output.shape[1:] @@ -45,10 +48,13 @@ def test_decomon_activation( decomon_layer(decomon_symbolic_inputs) # call on actual inputs - keras_input = keras_input_fn() - decomon_inputs = decomon_input_fn(keras_input=keras_input, output_shape=output_shape) + keras_model_input = keras_model_input_fn() + keras_layer_input = keras_layer_input_fn(keras_model_input) + decomon_inputs = decomon_input_fn( + keras_model_input=keras_model_input, keras_layer_input=keras_layer_input, output_shape=output_shape + ) - keras_output = layer(keras_input) + keras_output = layer(keras_layer_input) decomon_output = decomon_layer(decomon_inputs) # check output shapes @@ -58,11 +64,13 @@ def test_decomon_activation( expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) assert output_shape == expected_output_shape - # check ibp and affine bounds well ordered w.r.t. keras output - helpers.assert_decomon_output_compare_with_keras_input_output_single_layer( + # check ibp and affine bounds well ordered w.r.t. keras inputs/outputs + helpers.assert_decomon_output_compare_with_keras_input_output_layer( decomon_output=decomon_output, - keras_output=keras_output, - keras_input=keras_input, + keras_model_input=keras_model_input, + keras_layer_input=keras_layer_input, + keras_model_output=keras_output, + keras_layer_output=keras_output, ibp=ibp, affine=affine, propagation=propagation, diff --git a/tests/test_decomon_layer.py b/tests/test_decomon_layer.py index 9ac2c5fb..6ad1a31e 100644 --- a/tests/test_decomon_layer.py +++ b/tests/test_decomon_layer.py @@ -221,6 +221,7 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper helpers.assert_decomon_outputs_equal(linear_decomon_output_val, non_linear_decomon_output_val) +@pytest.mark.parametrize("affine", [True]) def test_check_affine_bounds_characteristics( ibp, affine, @@ -229,25 +230,26 @@ def test_check_affine_bounds_characteristics( empty, diag, nobatch, - keras_symbolic_input_fn, - decomon_symbolic_input_fn, - keras_input_fn, - decomon_input_fn, + simple_keras_symbolic_model_input_fn, + simple_keras_symbolic_layer_input_fn, + simple_decomon_symbolic_input_fn, + simple_keras_model_input_fn, + simple_keras_layer_input_fn, + simple_decomon_input_fn, helpers, ): units = 7 - keras_symbolic_input = keras_symbolic_input_fn() - input_shape = keras_symbolic_input.shape[1:] - output_shape = input_shape[:-1] + (units,) - model_output_shape = output_shape - decomon_symbolic_input = decomon_symbolic_input_fn(output_shape=output_shape) - keras_input = keras_input_fn() - decomon_input = decomon_input_fn(keras_input=keras_input, output_shape=output_shape) - + # init + build keras layer + keras_symbolic_model_input = simple_keras_symbolic_model_input_fn() + keras_symbolic_layer_input = simple_keras_symbolic_layer_input_fn(keras_symbolic_model_input) layer = Dense(units=units) - layer(keras_symbolic_input) + layer(keras_symbolic_layer_input) + # init + build decomon layer + output_shape = layer.output.shape[1:] + model_output_shape = output_shape + decomon_symbolic_inputs = simple_decomon_symbolic_input_fn(output_shape=output_shape) decomon_layer = DecomonLayer( layer=layer, ibp=ibp, @@ -256,12 +258,16 @@ def test_check_affine_bounds_characteristics( perturbation_domain=perturbation_domain, model_output_shape=model_output_shape, ) - affine_bounds_to_propagate, constant_oracle_bounds, model_inputs = decomon_layer.inputs_outputs_spec.split_inputs( - inputs=decomon_symbolic_input + + # actual inputs + keras_model_input = simple_keras_model_input_fn() + keras_layer_input = simple_keras_layer_input_fn(keras_model_input) + decomon_inputs = simple_decomon_input_fn( + keras_model_input=keras_model_input, keras_layer_input=keras_layer_input, output_shape=output_shape ) if affine: - affine_bounds, _, _ = decomon_layer.inputs_outputs_spec.split_inputs(inputs=decomon_symbolic_input) + affine_bounds, _, _ = decomon_layer.inputs_outputs_spec.split_inputs(inputs=decomon_symbolic_inputs) affine_bounds_shape = [t.shape for t in affine_bounds] assert decomon_layer.inputs_outputs_spec.is_identity_bounds(affine_bounds) is empty assert decomon_layer.inputs_outputs_spec.is_diagonal_bounds(affine_bounds) is diag @@ -270,7 +276,7 @@ def test_check_affine_bounds_characteristics( assert decomon_layer.inputs_outputs_spec.is_diagonal_bounds_shape(affine_bounds_shape) is diag assert decomon_layer.inputs_outputs_spec.is_wo_batch_bounds_shape(affine_bounds_shape) is nobatch - affine_bounds, _, _ = decomon_layer.inputs_outputs_spec.split_inputs(inputs=decomon_input) + affine_bounds, _, _ = decomon_layer.inputs_outputs_spec.split_inputs(inputs=decomon_inputs) affine_bounds_shape = [t.shape for t in affine_bounds] assert decomon_layer.inputs_outputs_spec.is_identity_bounds(affine_bounds) is empty assert decomon_layer.inputs_outputs_spec.is_diagonal_bounds(affine_bounds) is diag diff --git a/tests/test_dense.py b/tests/test_dense.py index 1fe10df7..729ffd6e 100644 --- a/tests/test_dense.py +++ b/tests/test_dense.py @@ -15,32 +15,34 @@ def test_decomon_dense( propagation, perturbation_domain, batchsize, - keras_symbolic_input_fn, + keras_symbolic_model_input_fn, + keras_symbolic_layer_input_fn, decomon_symbolic_input_fn, - keras_input_fn, + keras_model_input_fn, + keras_layer_input_fn, decomon_input_fn, + equal_bounds, helpers, ): - decimal = 5 + decimal = 4 units = 7 decomon_layer_class = DecomonDense - keras_symbolic_input = keras_symbolic_input_fn() - input_shape = keras_symbolic_input.shape[1:] - output_shape = input_shape[:-1] + (units,) - model_output_shape = output_shape - decomon_symbolic_inputs = decomon_symbolic_input_fn(output_shape=output_shape) - keras_input = keras_input_fn() - decomon_inputs = decomon_input_fn(keras_input=keras_input, output_shape=output_shape) - + # init + build keras layer + keras_symbolic_model_input = keras_symbolic_model_input_fn() + keras_symbolic_layer_input = keras_symbolic_layer_input_fn(keras_symbolic_model_input) layer = Dense(units=units, use_bias=use_bias) - layer(keras_symbolic_input) + layer(keras_symbolic_layer_input) if randomize: # randomize weights => non-zero biases for w in layer.weights: w.assign(np.random.random(w.shape)) + # init + build decomon layer + output_shape = layer.output.shape[1:] + model_output_shape = output_shape + decomon_symbolic_inputs = decomon_symbolic_input_fn(output_shape=output_shape) decomon_layer = decomon_layer_class( layer=layer, ibp=ibp, @@ -51,11 +53,19 @@ def test_decomon_dense( ) decomon_layer(decomon_symbolic_inputs) - keras_output = layer(keras_input) + # call on actual inputs + keras_model_input = keras_model_input_fn() + keras_layer_input = keras_layer_input_fn(keras_model_input) + decomon_inputs = decomon_input_fn( + keras_model_input=keras_model_input, keras_layer_input=keras_layer_input, output_shape=output_shape + ) + + keras_output = layer(keras_layer_input) + decomon_output = decomon_layer(decomon_inputs) # check affine representation is ok w, b = decomon_layer.get_affine_representation() - keras_output_2 = batch_multid_dot(keras_input, w, missing_batchsize=(False, True)) + b + keras_output_2 = batch_multid_dot(keras_layer_input, w, missing_batchsize=(False, True)) + b np.testing.assert_almost_equal( K.convert_to_numpy(keras_output), K.convert_to_numpy(keras_output_2), @@ -63,8 +73,6 @@ def test_decomon_dense( err_msg="wrong affine representation", ) - decomon_output = decomon_layer(decomon_inputs) - # check output shapes input_shape = [t.shape for t in decomon_inputs] output_shape = [t.shape for t in decomon_output] @@ -72,17 +80,21 @@ def test_decomon_dense( expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) assert output_shape == expected_output_shape - # check ibp and affine bounds well ordered w.r.t. keras output - helpers.assert_decomon_output_compare_with_keras_input_output_single_layer( + # check ibp and affine bounds well ordered w.r.t. keras inputs/outputs + helpers.assert_decomon_output_compare_with_keras_input_output_layer( decomon_output=decomon_output, - keras_output=keras_output, - keras_input=keras_input, + keras_model_input=keras_model_input, + keras_layer_input=keras_layer_input, + keras_model_output=keras_output, + keras_layer_output=keras_output, ibp=ibp, affine=affine, propagation=propagation, + decimal=decimal, ) # before propagation through linear layer lower == upper => lower == upper after propagation - helpers.assert_decomon_output_lower_equal_upper( - decomon_output, ibp=ibp, affine=affine, propagation=propagation, decimal=decimal - ) + if equal_bounds: + helpers.assert_decomon_output_lower_equal_upper( + decomon_output, ibp=ibp, affine=affine, propagation=propagation, decimal=decimal + ) From 7f4980a40f10e042b39311346bcd9ce99b084abd Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 15 Feb 2024 10:17:46 +0100 Subject: [PATCH 041/101] Add a method needs_keras_model_inputs() for the decomon layer --- src/decomon/core.py | 9 ++- src/decomon/layers/layer.py | 2 +- tests/conftest.py | 129 ++++++++++++++++++++++-------------- 3 files changed, 88 insertions(+), 52 deletions(-) diff --git a/src/decomon/core.py b/src/decomon/core.py index e96a1fa1..0eede1c8 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -225,9 +225,16 @@ def __init__( else: self.layer_input_shape = layer_input_shape + def needs_keras_model_inputs(self) -> bool: + return self.propagation == Propagation.FORWARD and self.affine + + @property + def nb_input_tensors(self): + ... + def split_inputs(self, inputs: list[Tensor]) -> tuple[list[Tensor], list[Tensor], list[Tensor]]: # Remove keras model input - if self.propagation == Propagation.FORWARD and self.affine: + if self.needs_keras_model_inputs(): x = inputs[-1] inputs = inputs[:-1] model_inputs = [x] diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index 38e874df..8864b01d 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -471,7 +471,7 @@ def call_forward( # Tighten constant bounds in hybrid mode (ibp+affine) if self.ibp and self.affine: if len(model_inputs) == 0: - raise RuntimeError("keras model input is necessary for get_forward_oracle() in affine mode.") + raise RuntimeError("keras model input is necessary for call_forward() in affine mode.") x = model_inputs[0] l_ibp, u_ibp = output_constant_bounds w_l, b_l, w_u, b_u = output_affine_bounds diff --git a/tests/conftest.py b/tests/conftest.py index 6a96c4b1..9113c684 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -149,10 +149,20 @@ def get_decomon_input_shapes( diag=False, nobatch=False, ): - if affine and propagation == Propagation.FORWARD: + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + layer_input_shape=layer_input_shape, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + ) + if inputs_outputs_spec.needs_keras_model_inputs(): model_inputs_shape = [perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape)] else: model_inputs_shape = [] + if affine and not empty: if propagation == Propagation.FORWARD: b_in_shape = layer_input_shape @@ -166,6 +176,7 @@ def get_decomon_input_shapes( affine_bounds_to_propagate_shape = [w_in_shape, b_in_shape, w_in_shape, b_in_shape] else: affine_bounds_to_propagate_shape = [] + if ibp: constant_oracle_bounds_shape = [layer_input_shape, layer_input_shape] else: @@ -284,7 +295,17 @@ def generate_simple_decomon_layer_inputs_from_keras_input( layer_input_shape = keras_input.shape[1:] model_input_shape = layer_input_shape model_output_shape = layer_output_shape - if affine and propagation == Propagation.FORWARD: + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + layer_input_shape=layer_input_shape, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + ) + + if inputs_outputs_spec.needs_keras_model_inputs(): if isinstance(perturbation_domain, BoxDomain): x = K.repeat(keras_input[:, None], 2, axis=1) else: @@ -325,15 +346,6 @@ def generate_simple_decomon_layer_inputs_from_keras_input( else: constant_oracle_bounds = [] - inputs_outputs_spec = InputsOutputsSpec( - ibp=ibp, - affine=affine, - propagation=propagation, - perturbation_domain=perturbation_domain, - layer_input_shape=layer_input_shape, - model_input_shape=model_input_shape, - model_output_shape=model_output_shape, - ) return inputs_outputs_spec.flatten_inputs( affine_bounds_to_propagate=affine_bounds_to_propagate, constant_oracle_bounds=constant_oracle_bounds, @@ -926,13 +938,6 @@ def decomon_symbolic_input_fn(output_shape): layer_input_shape = y.shape[1:] model_input_shape = x.shape[1:] - affine_bounds_to_propagate = [w_l, b_l, w_u, b_u] - constant_oracle_bounds = [l_c, u_c] - if isinstance(perturbation_domain, BoxDomain): - model_inputs = [z] - else: - raise NotImplementedError - inputs_outputs_spec = InputsOutputsSpec( ibp=ibp, affine=affine, @@ -942,6 +947,25 @@ def decomon_symbolic_input_fn(output_shape): model_input_shape=model_input_shape, model_output_shape=output_shape, ) + + if affine: + affine_bounds_to_propagate = [w_l, b_l, w_u, b_u] + else: + affine_bounds_to_propagate = [] + + if ibp: + constant_oracle_bounds = [l_c, u_c] + else: + constant_oracle_bounds = [] + + if inputs_outputs_spec.needs_keras_model_inputs(): + if isinstance(perturbation_domain, BoxDomain): + model_inputs = [z] + else: + raise NotImplementedError + else: + model_inputs = [] + return inputs_outputs_spec.flatten_inputs( affine_bounds_to_propagate=affine_bounds_to_propagate, constant_oracle_bounds=constant_oracle_bounds, @@ -953,15 +977,27 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): layer_input_shape = y.shape[1:] model_input_shape = x.shape[1:] + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + layer_input_shape=layer_input_shape, + model_input_shape=model_input_shape, + model_output_shape=output_shape, + ) + if affine: affine_bounds_to_propagate = [K.convert_to_tensor(a) for a in (w_l, b_l, w_u, b_u)] else: affine_bounds_to_propagate = [] + if ibp: constant_oracle_bounds = [K.convert_to_tensor(a) for a in (l_c, u_c)] else: constant_oracle_bounds = [] - if propagation == Propagation.FORWARD and affine: + + if inputs_outputs_spec.needs_keras_model_inputs(): if isinstance(perturbation_domain, BoxDomain): model_inputs = [K.convert_to_tensor(z)] else: @@ -969,15 +1005,6 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): else: model_inputs = [] - inputs_outputs_spec = InputsOutputsSpec( - ibp=ibp, - affine=affine, - propagation=propagation, - perturbation_domain=perturbation_domain, - layer_input_shape=layer_input_shape, - model_input_shape=model_input_shape, - model_output_shape=output_shape, - ) return inputs_outputs_spec.flatten_inputs( affine_bounds_to_propagate=affine_bounds_to_propagate, constant_oracle_bounds=constant_oracle_bounds, @@ -991,11 +1018,22 @@ def decomon_symbolic_input_fn(output_shape): layer_input_shape = y.shape[1:] model_input_shape = x.shape[1:] + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + layer_input_shape=layer_input_shape, + model_input_shape=model_input_shape, + model_output_shape=output_shape, + ) + if ibp: constant_oracle_bounds = [l_c, u_c] else: constant_oracle_bounds = [] - if propagation == Propagation.FORWARD and affine: + + if inputs_outputs_spec.needs_keras_model_inputs(): if isinstance(perturbation_domain, BoxDomain): model_inputs = [z] else: @@ -1003,16 +1041,6 @@ def decomon_symbolic_input_fn(output_shape): else: model_inputs = [] - inputs_outputs_spec = InputsOutputsSpec( - ibp=ibp, - affine=affine, - propagation=propagation, - perturbation_domain=perturbation_domain, - layer_input_shape=layer_input_shape, - model_input_shape=model_input_shape, - model_output_shape=output_shape, - ) - # take identity affine bounds if affine: simple_decomon_inputs = helpers.get_decomon_symbolic_inputs( @@ -1043,11 +1071,22 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): layer_input_shape = y.shape[1:] model_input_shape = x.shape[1:] + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + layer_input_shape=layer_input_shape, + model_input_shape=model_input_shape, + model_output_shape=output_shape, + ) + if ibp: constant_oracle_bounds = [K.convert_to_tensor(a) for a in (l_c, u_c)] else: constant_oracle_bounds = [] - if propagation == Propagation.FORWARD and affine: + + if inputs_outputs_spec.needs_keras_model_inputs(): if isinstance(perturbation_domain, BoxDomain): model_inputs = [K.convert_to_tensor(z)] else: @@ -1055,16 +1094,6 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): else: model_inputs = [] - inputs_outputs_spec = InputsOutputsSpec( - ibp=ibp, - affine=affine, - propagation=propagation, - perturbation_domain=perturbation_domain, - layer_input_shape=layer_input_shape, - model_input_shape=model_input_shape, - model_output_shape=output_shape, - ) - #  take identity affine bounds if affine: simple_decomon_inputs = helpers.generate_simple_decomon_layer_inputs_from_keras_input( From 69ced25dd4907c8bce356e2631783cf554b00579 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 15 Feb 2024 17:44:24 +0100 Subject: [PATCH 042/101] Fix typo --- src/decomon/layers/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index 8864b01d..e0d9d806 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -86,7 +86,7 @@ def __init__( - forward: from input to output - backward: from output to input model_output_shape: shape of the underlying model output (omitting batch axis). - It allows determining if the backward bounds are with a bacth axis or not. + It allows determining if the backward bounds are with a batch axis or not. model_input_shape: shape of the underlying keras model input (omitting batch axis). **kwargs: From cf1aa0f05e2c899710aab64c6cac6b0dc2105655 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 15 Feb 2024 17:40:00 +0100 Subject: [PATCH 043/101] Add DecomonMergeLayer to manage merging layer - Add private class attribute _is_merging to DecomonLayer to avoid repeating common methods like call(), call_forward() - Update InputsOutputsSpec to adapt to merging case - Put diagonal parameter in combine_affine_bounds args to be able to compute it with b=0. instead of a tensor (simplification in DecomonMerge) - Adapt inputs/outputs for DecomonMerge: - keras inputs are given as a list [z[i]]_i - DecomonLayer.call() still applies to a flatten list of tensors but in any other methods - affine bounds on keras inputs are a list of affine input bounds on each keras input (so a list of list of tensors or a list of tuple of tensors) -> either affine_bounds_to_propagate in forward propagation, -> or propagated_affine_bounds in backward propagation - oracle_input_bounds are a list of list of constant bounds on each keras input - lower/upper are each a list of tensors - in affine representation/relaxation: w is a list of weights for each keras input, but b remains a tensor that add to all (layer(z) = Sum_i{w[i] * z[i]} + b) - in forward propagation: w_u_out = Sum_i{w_u_in[i]*w_u_layer[i]_pos + w_l_in[i]*w_u_layer[i]_neg} b_u_out = Sum_i{b_u_in[i]*w_u_layer[i]_pos + b_l_in[i]*w_u_layer[i]_neg} + b_u_layer NB: when summing the different partial bounds, we need to take care of diagonal/from_linear format of each operand so that the sum operates properly. - in backward propagation: w_u_out[i] = w_u_layer[i]*w_u_in_pos + w_l_layer[i]*w_u_in_neg b_u_out[i] = 0 if i > 0 b_u_out[0] = b_u_layer * w_u_in_pos + b_l_layer * w_u_in_neg + b_u_in NB: As all bias will be added at the final step of crown algo, we choose to propagate biases only via the first input. One can see that it is mathematically equivalent to split them between all inputs (if dividing it by the number of keras inputs), but computationally more efficient. - Other notes: - for keras merging layers, layer.input will be a list of KerasTensor (except in the degenerate case when merging a single tensor, in this case layer.input is still a tensor) - a new function in keras_utils jas been added to add tensors that are not in the same representation (one of them diagonal, the other generic, potentially not both with batchsize) --- src/decomon/core.py | 475 ++++++++++++++++++++--- src/decomon/keras_utils.py | 93 +++++ src/decomon/layers/layer.py | 113 ++++-- src/decomon/layers/merging/base_merge.py | 468 ++++++++++++++++++++++ tests/conftest.py | 118 ++++-- tests/test_activation.py | 2 + tests/test_decomon_layer.py | 2 + tests/test_dense.py | 2 + tests/test_keras_utils.py | 72 ++++ 9 files changed, 1226 insertions(+), 119 deletions(-) create mode 100644 src/decomon/layers/merging/base_merge.py diff --git a/src/decomon/core.py b/src/decomon/core.py index 0eede1c8..2911e41d 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import Any, Optional, Union +from typing import Any, Optional, Union, overload import keras.ops as K import numpy as np @@ -171,7 +171,7 @@ def get_affine(mode: Union[str, ForwardMode] = ForwardMode.HYBRID) -> bool: class InputsOutputsSpec: """Storing specifications for inputs and outputs of decomon/backward layer/model.""" - layer_input_shape: tuple[int, ...] + layer_input_shape: Union[tuple[int, ...], list[tuple[int, ...]]] model_input_shape: tuple[int, ...] model_output_shape: tuple[int, ...] @@ -181,9 +181,10 @@ def __init__( affine: bool = True, propagation: Propagation = Propagation.FORWARD, perturbation_domain: Optional[PerturbationDomain] = None, - layer_input_shape: Optional[tuple[int, ...]] = None, + layer_input_shape: Optional[Union[tuple[int, ...], list[tuple[int, ...]]]] = None, model_input_shape: Optional[tuple[int, ...]] = None, model_output_shape: Optional[tuple[int, ...]] = None, + is_merging_layer: bool = False, ): """ Args: @@ -196,17 +197,31 @@ def __init__( layer_input_shape: shape of the underlying keras layer input (w/o the batch axis) model_input_shape: shape of the underlying keras model input (w/o the batch axis) model_output_shape: shape of the underlying keras model output (w/o the batch axis) + is_merging_layer: whether the underlying keras layer is a merging layer (i.e. with several inputs) """ # checks + if not ibp and not affine: + raise ValueError("ibp and affine cannot be both False.") if propagation == Propagation.BACKWARD and model_output_shape is None: raise ValueError("model_output_shape must be set in backward propagation.") - if propagation == Propagation.FORWARD and layer_input_shape is None: - raise ValueError("layer_input_shape must be set in forward propagation.") + if propagation == Propagation.FORWARD or is_merging_layer: + if layer_input_shape is None: + raise ValueError("layer_input_shape must be set in forward propagation or for mergine layer.") + elif is_merging_layer: + if len(layer_input_shape) == 0 or not isinstance(layer_input_shape[0], tuple): + raise ValueError( + "layer_input_shape should be a non-empty list of shapes (tuple of int) for a merging layer." + ) + elif not isinstance(layer_input_shape, tuple) or ( + len(layer_input_shape) > 0 and not isinstance(layer_input_shape[0], int) + ): + raise ValueError("layer_input_shape should be a tuple of int for a unary layer.") self.propagation = propagation self.affine = affine self.ibp = ibp + self.is_merging_layer = is_merging_layer self.perturbation_domain: PerturbationDomain if perturbation_domain is None: self.perturbation_domain = BoxDomain() @@ -221,18 +236,145 @@ def __init__( else: self.model_input_shape = model_input_shape if layer_input_shape is None: - self.layer_input_shape = tuple() + if self.is_merging_layer: + self.layer_input_shape = [tuple()] + else: + self.layer_input_shape = tuple() else: self.layer_input_shape = layer_input_shape def needs_keras_model_inputs(self) -> bool: + """Specify if decomon inputs should integrate keras model inputs.""" return self.propagation == Propagation.FORWARD and self.affine + def cannot_have_empty_affine_inputs(self) -> bool: + """Specify that it is not allowed to have empty affine bounds. + + Indeed, in merging case + forward propagation, it would be impossible to split decomon inputs properly. + + """ + return self.is_merging_layer and self.propagation == Propagation.FORWARD and self.affine + @property - def nb_input_tensors(self): + def nb_keras_inputs(self) -> int: + if self.is_merging_layer: + return len(self.layer_input_shape) + else: + return 1 + + @property + def nb_input_tensors(self) -> int: + nb = 0 + if self.propagation == Propagation.BACKWARD: + # ibp + nb += 2 * self.nb_keras_inputs + # affine + nb += 4 + # model inputs + if self.needs_keras_model_inputs(): + nb += 1 + else: # forward + # ibp + if self.ibp: + nb += 2 * self.nb_keras_inputs + # affine + if self.affine: + nb += 4 * self.nb_keras_inputs + # model inputs + if self.needs_keras_model_inputs(): + nb += 1 + return nb + + @property + def nb_output_tensors(self) -> int: + nb = 0 + if self.propagation == Propagation.BACKWARD: + nb += 4 * self.nb_keras_inputs + else: # forward + if self.ibp: + nb += 2 + if self.affine: + nb += 4 + return nb + + @overload + def split_constant_bounds(self, constant_bounds: list[Tensor]) -> tuple[Tensor, Tensor]: + """Split constant bounds, non-merging layer version.""" ... - def split_inputs(self, inputs: list[Tensor]) -> tuple[list[Tensor], list[Tensor], list[Tensor]]: + @overload + def split_constant_bounds(self, constant_bounds: list[list[Tensor]]) -> tuple[list[Tensor], list[Tensor]]: + """Split constant bounds, merging layer version.""" + ... + + def split_constant_bounds( + self, constant_bounds: Union[list[Tensor], list[list[Tensor]]] + ) -> Union[tuple[Tensor, Tensor], tuple[list[Tensor], list[Tensor]]]: + """Split constant bounds into lower, upper bound. + + Args: + constant_bounds: + if merging layer: list of constant (lower and upper) bounds for each keras layer inputs; + else: list containing lower and upper bounds for the keras layer input. + + Returns: + if merging_layer: 2 lists containing lower and upper bounds for each keras layer inputs; + else: 2 tensors being the lower and upper bounds for the keras layer input. + + """ + if self.is_merging_layer: + lowers, uppers = zip(*constant_bounds) + return list(lowers), list(uppers) + else: + lower, upper = constant_bounds + return lower, upper + + def split_inputs( + self, inputs: list[Tensor] + ) -> Union[ + tuple[list[Tensor], list[Tensor], list[Tensor]], + tuple[list[list[Tensor]], list[list[Tensor]], list[Tensor]], + tuple[list[Tensor], list[list[Tensor]], list[Tensor]], + ]: + """Split decomon inputs. + + Split them according to propagation mode and whether the underlying keras layer is merging or not. + + Args: + inputs: flattened decomon inputs, as seen by `DecomonLayer.call()`. + + Returns: + affine_bounds_to_propagate, constant_oracle_bounds, model_inputs: + each one can be empty if not relevant, and according to propagation mode and merging status, + it will be list of tensors or list of list of tensors. + + More details: + + - non-merging case: + inputs = affine_bounds_to_propagate + constant_oracle_bounds + model_inputs + + - merging case: + - forward: k affine bounds to propagate w.r.t. each keras layer input + k constant bounds + + inputs = ( + affine_bounds_to_propagate_0 + constant_oracle_bounds_0 + ... + + affine_bounds_to_propagate_k + constant_oracle_bounds_k + + model_inputs + ) + + - backward: only 1 affine bounds to propagate w.r.t keras layer output + + k constant bounds w.r.t each keras layer input + + inputs = ( + affine_bounds_to_propagate + + constant_oracle_bounds_0 + ... + constant_oracle_bounds_k + + model_inputs + ) + + Note: in case of merging layer + forward, we should not have empty affine bounds + as it will be impossible to split properly the inputs. + + """ # Remove keras model input if self.needs_keras_model_inputs(): x = inputs[-1] @@ -240,29 +382,161 @@ def split_inputs(self, inputs: list[Tensor]) -> tuple[list[Tensor], list[Tensor] model_inputs = [x] else: model_inputs = [] - # Remove constant bounds - if self.propagation == Propagation.BACKWARD or self.ibp: - constant_oracle_bounds = inputs[-2:] - inputs = inputs[:-2] + if self.is_merging_layer: + if self.propagation == Propagation.BACKWARD: + # expected number of constant bounds + nb_constant_bounds_by_keras_input = 2 + nb_constant_bounds = self.nb_keras_inputs * nb_constant_bounds_by_keras_input + # remove affine bounds (could be empty to express identity bounds) + affine_bounds_to_propagate = inputs[: len(inputs) - nb_constant_bounds] + inputs = inputs[len(inputs) - nb_constant_bounds :] + # split constant bounds by keras input + constant_oracle_bounds = [ + [inputs[i], inputs[i + 1]] for i in range(0, len(inputs), nb_constant_bounds_by_keras_input) + ] + else: # forward + # split bounds by keras input + nb_affine_bounds_by_keras_input = 4 if self.affine else 0 + nb_constant_bounds_by_keras_input = 2 if self.ibp else 0 + nb_bounds_by_keras_input = nb_affine_bounds_by_keras_input + nb_constant_bounds_by_keras_input + affine_bounds_to_propagate = [ + [inputs[start_input + j_bound] for j_bound in range(nb_affine_bounds_by_keras_input)] + for start_input in range(0, len(inputs), nb_bounds_by_keras_input) + ] + constant_oracle_bounds = [ + [ + inputs[start_input + nb_affine_bounds_by_keras_input + j_bound] + for j_bound in range(nb_constant_bounds_by_keras_input) + ] + for start_input in range(0, len(inputs), nb_bounds_by_keras_input) + ] else: - constant_oracle_bounds = [] - # The remaining tensors are affine bounds - # (potentially empty if: not backward or not affine or identity affine bounds) - affine_bounds_to_propagate = inputs + # Remove constant bounds + if self.propagation == Propagation.BACKWARD or self.ibp: + constant_oracle_bounds = inputs[-2:] + inputs = inputs[:-2] + else: + constant_oracle_bounds = [] + # The remaining tensors are affine bounds + # (potentially empty if: not backward or not affine or identity affine bounds) + affine_bounds_to_propagate = inputs return affine_bounds_to_propagate, constant_oracle_bounds, model_inputs def split_input_shape( self, input_shape: list[tuple[Optional[int], ...]] - ) -> tuple[list[tuple[Optional[int], ...]], list[tuple[Optional[int], ...]], list[tuple[Optional[int], ...]]]: + ) -> Union[ + tuple[list[tuple[Optional[int], ...]], list[tuple[Optional[int], ...]], list[tuple[Optional[int], ...]]], + tuple[ + list[list[tuple[Optional[int], ...]]], + list[list[tuple[Optional[int], ...]]], + list[tuple[Optional[int], ...]], + ], + tuple[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]], list[tuple[Optional[int], ...]]], + ]: + """Split decomon inputs. + + Split them according to propagation mode and whether the underlying keras layer is merging or not. + + Args: + input_shape: flattened decomon inputs, as seen by `DecomonLayer.call()`. + + Returns: + affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, model_inputs_shape: + each one can be empty if not relevant, and according to propagation mode and merging status, + it will be list of shapes or list of list of shapes. + + """ return self.split_inputs(inputs=input_shape) # type: ignore def flatten_inputs( - self, affine_bounds_to_propagate: list[Tensor], constant_oracle_bounds: list[Tensor], model_inputs: list[Tensor] + self, + affine_bounds_to_propagate: Union[list[Tensor], list[list[Tensor]]], + constant_oracle_bounds: Union[list[Tensor], list[list[Tensor]]], + model_inputs: list[Tensor], ) -> list[Tensor]: - return affine_bounds_to_propagate + constant_oracle_bounds + model_inputs + """Flatten decomon inputs. + + Reverse `self.split_inputs()`. + + Args: + affine_bounds_to_propagate: + - forward + affine: affine bounds on keras layer outputs w.r.t. model input + - backward: affine bounds on model output w.r.t each keras layer input + -> list of list of tensor in merging case; + -> list of tensor else. + - else: empty + constant_bounds_propagated: + - forward + ibp: ibp bounds on keras layer outputs + - else: empty or None + + Returns: + flattened inputs + - non-merging case: + inputs = affine_bounds_to_propagate + constant_oracle_bounds + model_inputs + + - merging case: + - forward: k affine bounds to propagate w.r.t. each keras layer input + k constant bounds + + inputs = ( + affine_bounds_to_propagate_0 + constant_oracle_bounds_0 + ... + + affine_bounds_to_propagate_k + constant_oracle_bounds_k + + model_inputs + ) + + - backward: only 1 affine bounds to propagate w.r.t keras layer output + + k constant bounds w.r.t each keras layer input + + inputs = ( + affine_bounds_to_propagate + + constant_oracle_bounds_0 + ... + constant_oracle_bounds_k + + model_inputs + ) + + """ + if self.is_merging_layer: + if self.propagation == Propagation.BACKWARD: + flattened_constant_oracle_bounds = [ + t for constant_oracle_bounds_i in constant_oracle_bounds for t in constant_oracle_bounds_i + ] + return affine_bounds_to_propagate + flattened_constant_oracle_bounds + model_inputs + else: # forward + bounds_by_keras_input = [ + affine_bounds_to_propagate_i + constant_oracle_bounds_i + for affine_bounds_to_propagate_i, constant_oracle_bounds_i in zip( + affine_bounds_to_propagate, constant_oracle_bounds + ) + ] + flattened_bounds_by_keras_input = [ + t for bounds_by_keras_input_i in bounds_by_keras_input for t in bounds_by_keras_input_i + ] + return flattened_bounds_by_keras_input + model_inputs + else: + return affine_bounds_to_propagate + constant_oracle_bounds + model_inputs + + def split_outputs(self, outputs: list[Tensor]) -> tuple[Union[list[Tensor], list[list[Tensor]]], list[Tensor]]: + """Split decomon inputs. + + Reverse operation of `self.flatten_outputs()` - def split_outputs(self, outputs: list[Tensor]) -> tuple[list[Tensor], list[Tensor]]: + Args: + outputs: flattened decomon outputs, as returned by `DecomonLayer.call()`. + + Returns: + affine_bounds_propagated, constant_bounds_propagated: + each one can be empty if not relevant and can be list of tensors or a list of list of tensors + according to propagation and merging status. + + More details: + + - forward: affine_bounds_propagated, constant_bounds_propagated: both simple lists of tensors corresponding to + affine and constant bounds on keras layer output. + - backward: constant_bounds_propagated is empty (not relevant) and + - merging layer: affine_bounds_propagated is a list of list of tensors corresponding + to partial affine bounds on model output w.r.t each keras input + - else: affine_bounds_propagated is a simple list of tensors + + """ # Remove constant bounds if self.propagation == Propagation.FORWARD and self.ibp: constant_bounds_propagated = outputs[-2:] @@ -271,59 +545,148 @@ def split_outputs(self, outputs: list[Tensor]) -> tuple[list[Tensor], list[Tenso constant_bounds_propagated = [] # It remains affine bounds (can be empty if forward + not affine, or identity layer (e.g. DecomonLinear) on identity bounds affine_bounds_propagated = outputs + if self.propagation == Propagation.BACKWARD and self.is_merging_layer: + nb_affine_bounds_by_keras_input = 4 + affine_bounds_propagated = [ + affine_bounds_propagated[i : i + nb_affine_bounds_by_keras_input] + for i in range(0, len(affine_bounds_propagated), nb_affine_bounds_by_keras_input) + ] return affine_bounds_propagated, constant_bounds_propagated def flatten_outputs( - self, affine_bounds_propagated: list[Tensor], constant_bounds_propagated: Optional[list[Tensor]] = None + self, + affine_bounds_propagated: Union[list[Tensor], list[list[Tensor]]], + constant_bounds_propagated: Optional[list[Tensor]] = None, ) -> list[Tensor]: + """Flatten decomon outputs. + + Args: + affine_bounds_propagated: + - forward + affine: affine bounds on keras layer outputs w.r.t. model input + - backward: affine bounds on model output w.r.t each keras layer input + -> list of list of tensor in merging case; + -> list of tensor else. + - else: empty + constant_bounds_propagated: + - forward + ibp: ibp bounds on keras layer outputs + - else: empty or None + + Returns: + flattened outputs + - forward: affine_bounds_propagated + constant_bounds_propagated + - backward: + - merging layer (k keras layer inputs): affine_bounds_propagated_0 + ... + affine_bounds_propagated_k + - else: affine_bounds_propagated + + """ if constant_bounds_propagated is None or self.propagation == Propagation.BACKWARD: - return affine_bounds_propagated + if self.is_merging_layer and self.propagation == Propagation.BACKWARD: + return [ + t for affine_bounds_propagated_i in affine_bounds_propagated for t in affine_bounds_propagated_i + ] + else: + return affine_bounds_propagated else: return affine_bounds_propagated + constant_bounds_propagated def flatten_outputs_shape( self, - affine_bounds_propagated_shape: list[tuple[Optional[int], ...]], + affine_bounds_propagated_shape: Union[ + list[tuple[Optional[int], ...]], + list[list[tuple[Optional[int], ...]]], + ], constant_bounds_propagated_shape: Optional[list[tuple[Optional[int], ...]]] = None, ) -> list[tuple[Optional[int], ...]]: + """Flatten decomon outputs shape.""" return self.flatten_outputs(affine_bounds_propagated=affine_bounds_propagated_shape, constant_bounds_propagated=constant_bounds_propagated_shape) # type: ignore - def is_identity_bounds(self, affine_bounds: list[Tensor]) -> bool: - return len(affine_bounds) == 0 - - def is_identity_bounds_shape(self, affine_bounds_shape: list[tuple[Optional[int], ...]]) -> bool: - return len(affine_bounds_shape) == 0 - - def is_diagonal_bounds(self, affine_bounds: list[Tensor]) -> bool: - if self.is_identity_bounds(affine_bounds): - return True - w, b = affine_bounds[:2] - return w.shape == b.shape - - def is_diagonal_bounds_shape(self, affine_bounds_shape: list[tuple[Optional[int], ...]]) -> bool: - if self.is_identity_bounds_shape(affine_bounds_shape): - return True - w_shape, b_shape = affine_bounds_shape[:2] - return w_shape == b_shape - - def is_wo_batch_bounds(self, affine_bounds: list[Tensor]) -> bool: - if self.is_identity_bounds(affine_bounds): - return True - b = affine_bounds[1] - if self.propagation == Propagation.FORWARD: - return len(b.shape) == len(self.layer_input_shape) + def has_multiple_affine_inputs(self) -> bool: + return self.propagation == Propagation.FORWARD and self.affine and self.is_merging_layer + + @overload + def extract_shapes_from_affine_bounds(self, affine_bounds: list[Tensor]) -> list[tuple[Optional[int], ...]]: + ... + + @overload + def extract_shapes_from_affine_bounds( + self, affine_bounds: list[list[Tensor]] + ) -> list[list[tuple[Optional[int], ...]]]: + ... + + def extract_shapes_from_affine_bounds( + self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], i: int = -1 + ) -> Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]]: + if self.has_multiple_affine_inputs() and i == -1: + return [[t.shape for t in sub_bounds] for sub_bounds in affine_bounds] + else: + return [t.shape for t in affine_bounds] # type: ignore + + def is_identity_bounds(self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], i: int = -1) -> bool: + return self.is_identity_bounds_shape( + affine_bounds_shape=self.extract_shapes_from_affine_bounds(affine_bounds=affine_bounds, i=i), i=i + ) + + def is_identity_bounds_shape( + self, + affine_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], + i: int = -1, + ) -> bool: + if self.has_multiple_affine_inputs() and i == -1: + return all( + self.is_identity_bounds_shape(affine_bounds_shape=affine_bounds_shape[i], i=i) # type: ignore + for i in range(self.nb_keras_inputs) + ) + else: + return len(affine_bounds_shape) == 0 + + def is_diagonal_bounds(self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], i: int = -1) -> bool: + return self.is_diagonal_bounds_shape( + affine_bounds_shape=self.extract_shapes_from_affine_bounds(affine_bounds=affine_bounds, i=i), i=i + ) + + def is_diagonal_bounds_shape( + self, + affine_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], + i: int = -1, + ) -> bool: + if self.has_multiple_affine_inputs() and i == -1: + return all( + self.is_diagonal_bounds_shape(affine_bounds_shape=affine_bounds_shape[i], i=i) # type: ignore + for i in range(self.nb_keras_inputs) + ) else: - return len(b.shape) == len(self.model_output_shape) - - def is_wo_batch_bounds_shape(self, affine_bounds_shape: list[tuple[Optional[int], ...]]) -> bool: - if self.is_identity_bounds_shape(affine_bounds_shape): - return True - b_shape = affine_bounds_shape[1] - if self.propagation == Propagation.FORWARD: - return len(b_shape) == len(self.layer_input_shape) + if self.is_identity_bounds_shape(affine_bounds_shape): + return True + w_shape, b_shape = affine_bounds_shape[:2] + return w_shape == b_shape + + def is_wo_batch_bounds(self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], i: int = -1) -> bool: + return self.is_wo_batch_bounds_shape( + affine_bounds_shape=self.extract_shapes_from_affine_bounds(affine_bounds=affine_bounds, i=i), i=i + ) + + def is_wo_batch_bounds_shape( + self, + affine_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], + i: int = -1, + ) -> bool: + if self.has_multiple_affine_inputs() and i == -1: + return all( + self.is_wo_batch_bounds_shape(affine_bounds_shape=affine_bounds_shape[i], i=i) # type: ignore + for i in range(self.nb_keras_inputs) + ) else: - return len(b_shape) == len(self.model_output_shape) + if self.is_identity_bounds_shape(affine_bounds_shape): + return True + b_shape = affine_bounds_shape[1] + if self.propagation == Propagation.FORWARD: + if i > -1: + return len(b_shape) == len(self.layer_input_shape[i]) + else: + return len(b_shape) == len(self.layer_input_shape) + else: + return len(b_shape) == len(self.model_output_shape) def get_kerasinputshape(self, inputsformode: list[Tensor]) -> tuple[Optional[int], ...]: return inputsformode[-1].shape diff --git a/src/decomon/keras_utils.py b/src/decomon/keras_utils.py index 81201bc6..452f9579 100644 --- a/src/decomon/keras_utils.py +++ b/src/decomon/keras_utils.py @@ -109,6 +109,99 @@ def batch_multid_dot( return Dot(axes=(-1, 1))([Reshape(new_x_shape)(x), Reshape(new_y_shape)(y)]) +def add_tensors( + x: Tensor, + y: Tensor, + missing_batchsize: tuple[bool, bool] = (False, False), + diagonal: tuple[bool, bool] = (False, False), +) -> Tensor: + """Sum tensors in a compatible way. + + Generate broadcastable versions of the tensors before summing them, + depending on 2 characteristics: + + - missing batchsize? + - diagonal representation? + + We only have to modify the tensors if a characteristic differ between them. + More precisely: + - missing batchsize: we add a batch axis (of dimension 1) + - diagonal representation: we make a full representation of the tensor + + Args: + x: + y: + missing_batchsize: + diagonal: + + Returns: + + """ + # get broadcastable version of the tensors + x_broadcastable, y_broadcastable = _convert_to_broacastable_tensors( + x=x, y=y, missing_batchsize=missing_batchsize, diagonal=diagonal + ) + # operate on broadcastable tensors + return x_broadcastable + y_broadcastable + + +def _convert_to_broacastable_tensors( + x: Tensor, + y: Tensor, + missing_batchsize: tuple[bool, bool], + diagonal: tuple[bool, bool], +) -> tuple[Tensor, Tensor]: + x_broadcastable = x + y_broadcastable = y + x_full_shape = x.shape + y_full_shape = y.shape + if missing_batchsize == (True, False): + batchsize = y_full_shape[0] + x_full_shape = (batchsize,) + x_full_shape + x_broadcastable = x_broadcastable[None] + elif missing_batchsize == (False, True): + batchsize = x_full_shape[0] + y_full_shape = (batchsize,) + y_full_shape + y_broadcastable = y_broadcastable[None] + if diagonal == (True, False): + x_broadcastable, x_full_shape = _convert_from_diag_to_generic( + x_broadcastable=x_broadcastable, x_full_shape=x_full_shape, missing_batchsize=all(missing_batchsize) + ) + elif diagonal == (False, True): + y_broadcastable, y_full_shape = _convert_from_diag_to_generic( + x_broadcastable=y_broadcastable, x_full_shape=y_full_shape, missing_batchsize=all(missing_batchsize) + ) + # check shapes + if x_full_shape != y_full_shape: + raise ValueError( + f"Incompatible shapes: {x.shape} and {y.shape}, " + f"with missing_batchsize={missing_batchsize} and diagonal={diagonal}." + ) + + return x_broadcastable, y_broadcastable + + +def _convert_from_diag_to_generic( + x_broadcastable: Tensor, x_full_shape: tuple[int, ...], missing_batchsize: bool = False +) -> tuple[Tensor, tuple[int, ...]]: + if missing_batchsize: + x_full_shape = x_full_shape + x_full_shape + new_shape = x_broadcastable.shape + x_broadcastable.shape + x_broadcastable = K.reshape(K.diag(K.ravel(x_broadcastable)), new_shape) + else: + x_full_shape = x_full_shape[:1] + x_full_shape[1:] + x_full_shape[1:] + partial_new_shape = x_broadcastable.shape[1:] + x_broadcastable.shape[1:] + x_broadcastable = K.concatenate( + [ + K.reshape(K.diag(K.ravel(x_broadcastable[i])), partial_new_shape)[None] + for i in range(len(x_broadcastable)) + ], + axis=0, + ) + + return x_broadcastable, x_full_shape + + class BatchedIdentityLike(keras.Operation): """Keras Operation creating an identity tensor with shape (including batch_size) based on input. diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index e0d9d806..ceb967ee 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -65,6 +65,8 @@ class DecomonLayer(Wrapper): """ + _is_merging: bool = False # set to True in child class DecomonMerge + def __init__( self, layer: Layer, @@ -116,14 +118,23 @@ def __init__( self.propagation = propagation # input-output-manager + if self._is_merging: + if isinstance(layer.input, keras.KerasTensor): + # special case: merging a single input -> self.layer.input is already flattened + layer_input_shape = [layer.input.shape[1:]] + else: + layer_input_shape = [t.shape[1:] for t in layer.input] + else: + layer_input_shape = layer.input.shape[1:] self.inputs_outputs_spec = InputsOutputsSpec( ibp=ibp, affine=affine, propagation=propagation, perturbation_domain=perturbation_domain, - layer_input_shape=layer.input.shape[1:], + layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=model_output_shape, + is_merging_layer=self._is_merging, ) def get_config(self) -> dict[str, Any]: @@ -306,16 +317,20 @@ def forward_affine_propagate( w, b = self.get_affine_representation() layer_affine_bounds = [w, b, w, b] else: - lower, upper = input_constant_bounds + lower, upper = self.inputs_outputs_spec.split_constant_bounds(constant_bounds=input_constant_bounds) w_l, b_l, w_u, b_u = self.get_affine_bounds(lower=lower, upper=upper) layer_affine_bounds = [w_l, b_l, w_u, b_u] from_linear_layer = (self.inputs_outputs_spec.is_wo_batch_bounds(input_affine_bounds), self.linear) - + diagonal = ( + self.inputs_outputs_spec.is_diagonal_bounds(input_affine_bounds), + self.inputs_outputs_spec.is_diagonal_bounds(layer_affine_bounds), + ) return combine_affine_bounds( affine_bounds_1=input_affine_bounds, affine_bounds_2=layer_affine_bounds, from_linear_layer=from_linear_layer, + diagonal=diagonal, ) def backward_affine_propagate( @@ -360,16 +375,20 @@ def backward_affine_propagate( w, b = self.get_affine_representation() layer_affine_bounds = [w, b, w, b] else: - lower, upper = input_constant_bounds + lower, upper = self.inputs_outputs_spec.split_constant_bounds(constant_bounds=input_constant_bounds) w_l, b_l, w_u, b_u = self.get_affine_bounds(lower=lower, upper=upper) layer_affine_bounds = [w_l, b_l, w_u, b_u] from_linear_layer = (self.linear, self.inputs_outputs_spec.is_wo_batch_bounds((output_affine_bounds))) - + diagonal = ( + self.inputs_outputs_spec.is_diagonal_bounds(layer_affine_bounds), + self.inputs_outputs_spec.is_diagonal_bounds(output_affine_bounds), + ) return combine_affine_bounds( affine_bounds_1=layer_affine_bounds, affine_bounds_2=output_affine_bounds, from_linear_layer=from_linear_layer, + diagonal=diagonal, ) def get_forward_oracle( @@ -443,7 +462,7 @@ def call_forward( """ # IBP: interval bounds propragation if self.ibp: - lower, upper = input_bounds_to_propagate + lower, upper = self.inputs_outputs_spec.split_constant_bounds(constant_bounds=input_bounds_to_propagate) output_constant_bounds = list(self.forward_ibp_propagate(lower=lower, upper=upper)) else: output_constant_bounds = [] @@ -549,20 +568,14 @@ def compute_output_shape( else: constant_bounds_propagated_shape = [] if self.affine: - keras_layer_input_shape_wo_batchsize = self.layer.input.shape[1:] + # layer output shape keras_layer_output_shape_wo_batchsize = self.layer.output.shape[1:] + # model input shape + model_input_shape_wo_batchsize = ( + self.inputs_outputs_spec.model_input_shape + ) # should be set to get accurate compute_output_shape() - # inputs are in diagonal representation? without batch axis? - if self.inputs_outputs_spec.is_diagonal_bounds_shape(affine_bounds_to_propagate_shape): - model_input_shape_wo_batchsize = keras_layer_input_shape_wo_batchsize - else: - w_in_shape = affine_bounds_to_propagate_shape[0] - if self.inputs_outputs_spec.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): - model_input_shape_wo_batchsize = w_in_shape[: -len(keras_layer_input_shape_wo_batchsize)] - else: - model_input_shape_wo_batchsize = w_in_shape[1 : -len(keras_layer_input_shape_wo_batchsize)] - - # outputs shape depends if layer and inputs are diagonal / linear (w/o batch) + # outputs shape depends on layer and inputs being diagonal / linear (w/o batch) b_out_shape_wo_batchsize = keras_layer_output_shape_wo_batchsize if self.diagonal and self.inputs_outputs_spec.is_diagonal_bounds_shape( affine_bounds_to_propagate_shape @@ -582,40 +595,61 @@ def compute_output_shape( else: affine_bounds_propagated_shape = [] - return affine_bounds_propagated_shape + constant_bounds_propagated_shape + return self.inputs_outputs_spec.flatten_outputs_shape( + affine_bounds_propagated_shape=affine_bounds_propagated_shape, + constant_bounds_propagated_shape=constant_bounds_propagated_shape, + ) else: # backward - # find model output shape - if self.inputs_outputs_spec.is_identity_bounds_shape(affine_bounds_to_propagate_shape): - model_output_shape_wo_batchsize = self.layer.output.shape[1:] - else: - b_shape = affine_bounds_to_propagate_shape[1] - if self.inputs_outputs_spec.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): - model_output_shape_wo_batchsize = b_shape - else: - model_output_shape_wo_batchsize = b_shape[1:] + # model output shape + model_output_shape_wo_batchsize = self.inputs_outputs_spec.model_output_shape # outputs shape depends if layer and inputs are diagonal / linear (w/o batch) b_shape_wo_batchisze = model_output_shape_wo_batchsize if self.diagonal and self.inputs_outputs_spec.is_diagonal_bounds_shape(affine_bounds_to_propagate_shape): - w_shape_wo_batchsize = model_output_shape_wo_batchsize + if self._is_merging: + w_shape_wo_batchsize = [model_output_shape_wo_batchsize] * self.inputs_outputs_spec.nb_keras_inputs + else: + w_shape_wo_batchsize = model_output_shape_wo_batchsize else: - w_shape_wo_batchsize = self.layer.input.shape[1:] + model_output_shape_wo_batchsize + if self._is_merging: + w_shape_wo_batchsize = [ + self.layer.input[i].shape[1:] + model_output_shape_wo_batchsize + for i in range(self.inputs_outputs_spec.nb_keras_inputs) + ] + else: + w_shape_wo_batchsize = self.layer.input.shape[1:] + model_output_shape_wo_batchsize if self.linear and self.inputs_outputs_spec.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): - b_new_shape = b_shape_wo_batchisze + b_shape = b_shape_wo_batchisze w_shape = w_shape_wo_batchsize else: - b_new_shape = (None,) + b_shape_wo_batchisze - w_shape = (None,) + w_shape_wo_batchsize - - affine_bounds_propagated_shape = [w_shape, b_new_shape, w_shape, b_new_shape] + b_shape = (None,) + b_shape_wo_batchisze + if self._is_merging: + w_shape = [(None,) + sub_w_shape_wo_batchsize for sub_w_shape_wo_batchsize in w_shape_wo_batchsize] + else: + w_shape = (None,) + w_shape_wo_batchsize + if self._is_merging: + affine_bounds_propagated_shape = [ + [ + w_shape_i, + b_shape, + w_shape_i, + b_shape, + ] + for w_shape_i in w_shape + ] + else: + affine_bounds_propagated_shape = [w_shape, b_shape, w_shape, b_shape] - return affine_bounds_propagated_shape + return self.inputs_outputs_spec.flatten_outputs_shape( + affine_bounds_propagated_shape=affine_bounds_propagated_shape + ) def combine_affine_bounds( affine_bounds_1: list[Tensor], affine_bounds_2: list[Tensor], from_linear_layer: tuple[bool, bool] = (False, False), + diagonal: tuple[bool, bool] = (False, False), ) -> tuple[Tensor, Tensor, Tensor, Tensor]: """Combine affine bounds @@ -624,6 +658,8 @@ def combine_affine_bounds( affine_bounds_2: [w_l_2, b_l_2, w_u_2, b_u_2] second affine bounds from_linear_layer: specify if affine_bounds_1 or affine_bounds_2 come from the affine representation of a linear layer + diagonal: specify if affine_bounds_1 or affine_bounds_2 + are in diagonal representation Returns: w_l, b_l, w_u, b_u: combined affine bounds @@ -663,11 +699,6 @@ def combine_affine_bounds( if len(affine_bounds_2) == 0: return tuple(affine_bounds_1) - # Are weights in diagonal representation? - diagonal = ( - affine_bounds_1[0].shape == affine_bounds_1[1].shape, - affine_bounds_2[0].shape == affine_bounds_2[1].shape, - ) if from_linear_layer == (False, False): return _combine_affine_bounds_generic( affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal diff --git a/src/decomon/layers/merging/base_merge.py b/src/decomon/layers/merging/base_merge.py new file mode 100644 index 00000000..c1e560d3 --- /dev/null +++ b/src/decomon/layers/merging/base_merge.py @@ -0,0 +1,468 @@ +from typing import Any + +import keras.ops as K + +from decomon.keras_utils import add_tensors, batch_multid_dot +from decomon.layers.layer import DecomonLayer, combine_affine_bounds +from decomon.types import Tensor + + +class DecomonMerge(DecomonLayer): + _is_merging = True + + @property + def keras_layer_input(self): + """self.layer.input returned as a list. + + In the degenerate case where only 1 input is merged, self.layer.input is a single keras tensor. + We want here to have always a list, even with a single element, for consistency purpose. + + """ + if isinstance(self.layer.input, list): + return self.layer.input + else: + return [self.layer.input] + + @property + def nb_keras_inputs(self): + """Number of inputs merged by the underlying layer.""" + return len(self.keras_layer_input) + + def get_affine_representation(self) -> tuple[list[Tensor], Tensor]: + """Get affine representation of the layer + + This computes the affine representation of the layer, when this is meaningful, + i.e. `self.linear` is True. + + If implemented, it will be used for backward and forward propagation of affine bounds through the layer. + For non-linear layers, one should implement `get_affine_bounds()` instead. + + Args: + + Returns: + [w_i]_i, b: affine representation of the layer satisfying + + layer(z) = Sum_i{w_i * z_i} + b + + More precisely, if the keras layer is merging inputs represented by z = (z_i)_i, we have + ``` + layer(z) = sum(batch_multid_dot(z[i], w[i], missing_batchsize=(False, True)) for i in range(len(z))) + b + ``` + + If w can be represented as a diagonal tensor, which means that the full version of w is retrieved by + w_full = K.reshape(K.diag(K.flatten(w)), w.shape + w.shape) + then + - class attribute `diagonal` should be set to True (in order to have a correct `compute_output_shape()`) + - we got + ``` + layer(z) = sum(batch_multid_dot(z[i], w[i], missing_batchsize=(False, True), diagonal=(False, True)) for i in range(len(z)))+ b + ``` + + Shapes: !no batchsize! + if diagonal is False: + w[i] ~ self.layer.input[i].shape[1:] + self.layer.output.shape[1:] + b ~ self.layer.output.shape[1:] + if diagonal is True: + w[i] ~ self.layer.output.shape[1:] + b ~ self.layer.output.shape[1:] + + + """ + if not self.linear: + raise RuntimeError("You should not call `get_affine_representation()` when `self.linear` is False.") + else: + raise NotImplementedError( + "`get_affine_representation()` needs to be implemented to get the forward and backward propagation of affine bounds. " + "Alternatively, you can also directly override " + "`forward_ibp_propagate()`, `forward_affine_propagate()` and `backward_affine_propagate()`." + ) + + def get_affine_bounds( + self, lower: list[Tensor], upper: list[Tensor] + ) -> tuple[list[Tensor], Tensor, list[Tensor], Tensor]: + """Get affine bounds on layer outputs from layer inputs + + This compute the affine relaxation of the layer, given the oracle constant bounds on the inputs. + + If implemented, it will be used for backward and forward propagation of affine bounds through the layer. + For linear layers, one can implement `get_affine_representation()` instead. + + Args: + lower: lower constant oracle bounds on the layer inputs. + upper: upper constant oracle bounds on the layer inputs. + + Returns: + w_l, b_l, w_u, b_u: affine relaxation of the layer satisfying + + Sum_i{w_l[i] * z[i]} + b_l <= layer(z) <= Sum_i{w_u[i] * z[i]} + b_u + with lower[i] <= z[i] <= upper[i] for all i + + If w(_l or_u)[i] can be represented as a diagonal tensor, which means that the full version of w is retrieved by + w_full = K.concatenate([K.reshape(K.diag(K.flatten(w[i])), w.shape + w.shape)[None] for i in range(len(w))], axis=0) + then the class attribute `diagonal` should be set to True (in order to have a correct `compute_output_shape()`) + + Shapes: + if diagonal is False: + lowers[i], uppers[i] ~ (batchsize,) + self.layer.input[i].shape[1:] + w_l[i], w_u[i] ~ (batchsize,) + self.layer.input[i].shape[1:] + self.layer.output.shape[1:] + b_l, b_u ~ (batchsize,) + self.layer.output.shape[1:] + if diagonal is True: + lower[i], upper[i] ~ (batchsize,) + self.layer.input[i].shape[1:] + w_l[i], w_u[i] ~ (batchsize,) + self.layer.output.shape[1:] + b_l, b_u ~ (batchsize,) + self.layer.output.shape[1:] + + Note: + `w * z` means here `batch_multid_dot(z, w)`. + + """ + if self.linear: + w, b = self.get_affine_representation() + batchsize = lower[0].shape[0] + w_with_batchsize = [K.repeat(w_i[None], batchsize, axis=0) for w_i in w] + b_with_batchsize = K.repeat(b[None], batchsize, axis=0) + return w_with_batchsize, b_with_batchsize, w_with_batchsize, b_with_batchsize + else: + raise NotImplementedError( + "`get_affine_bounds()` needs to be implemented to get the forward and backward propagation of affine bounds. " + "Alternatively, you can also directly override `forward_affine_propagate()` and `backward_affine_propagate()`" + ) + + def forward_ibp_propagate(self, lower: list[Tensor], upper: list[Tensor]) -> tuple[Tensor, Tensor]: + """Propagate ibp bounds through the layer. + + If the underlying keras layer is linear, it will be deduced from its affine representation. + Else, this needs to be implemented to forward propagate ibp (constant) bounds. + + Args: + lower: lower constant oracle bounds on the keras layer inputs. + upper: upper constant oracle bounds on the keras layer inputs. + + Returns: + l_c, u_c: constant relaxation of the layer satisfying + l_c <= layer(z) <= u_c + with lower[i] <= z[i] <= upper[i] for all i + + Shapes: + lower[i], upper[i] ~ (batchsize,) + self.layer.input[i].shape[1:] + l_c, u_c ~ (batchsize,) + self.layer.output.shape[1:] + + """ + if self.linear: + w, b = self.get_affine_representation() + z_value = K.cast(0.0, dtype=b.dtype) + + l_c = b + u_c = b + + for w_i, lower_i, upper_i in zip(w, lower, upper): + is_diag = w_i.shape == b.shape + kwargs_dot: dict[str, Any] = dict(missing_batchsize=(False, True), diagonal=(False, is_diag)) + + w_i_pos = K.maximum(w_i, z_value) + w_i_neg = K.minimum(w_i, z_value) + + l_c += batch_multid_dot(lower_i, w_i_pos, **kwargs_dot) + batch_multid_dot( + upper_i, w_i_neg, **kwargs_dot + ) + u_c += batch_multid_dot(upper_i, w_i_pos, **kwargs_dot) + batch_multid_dot( + lower_i, w_i_neg, **kwargs_dot + ) + + return l_c, u_c + else: + raise NotImplementedError( + "`forward_ibp_propagate()` needs to be implemented to get the forward propagation of constant bounds." + ) + + def forward_affine_propagate( + self, input_affine_bounds: list[list[Tensor]], input_constant_bounds: list[list[Tensor]] + ) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Propagate model affine bounds in forward direction. + + By default, this is deduced from `get_affine_bounds()` (or `get_affine_representation()` if `self.linear is True). + But this could be overridden for better performance. See `DecomonConv2D` for an example. + + Args: + input_affine_bounds[i]: [w_l_in[i], b_l_in[i], w_u_in[i], b_u_in[i]] + affine bounds on underlying keras layer i-th input w.r.t. model input + input_constant_bounds[i]: [l_c_in[i], u_c_in[i]] + constant oracle bounds on underlying keras layer i-th input (already deduced from affine ones if necessary) + + Returns: + w_l_new, b_l_new, w_u_new, b_u_new: affine bounds on underlying keras layer *output* w.r.t. model input + + If we denote by + - x: keras model input + - z[i]: underlying keras layer i-th input + - h(x) = layer(z): output of the underlying keras layer + + The following inequations are satisfied + w_l * x + b_l <= h(x) <= w_u * x + b_u + l_c_in[i] <= z[i] <= u_c_in[i] for all i + w_l_in[i] * x + b_l_in[i] <= z[i] <= w_u_in[i] * x + b_u_in[i] for all i + + """ + if self.linear: + w, b = self.get_affine_representation() + w_l, b_l, w_u, b_u = w, b, w, b + else: + lower, upper = self.inputs_outputs_spec.split_constant_bounds(constant_bounds=input_constant_bounds) + w_l, b_l, w_u, b_u = self.get_affine_bounds(lower=lower, upper=upper) + + b_l_new, b_u_new = b_l, b_u + from_linear_layer_new = self.linear + diagonal_w_new = True + first_iteration = True + for i in range(len(input_affine_bounds)): + input_affine_bounds_i = input_affine_bounds[i] + partial_layer_affine_bounds_i = [w_l[i], 0.0, w_u[i], 0.0] + partial_layer_affine_bounds_true_shape_i = [w_l[i].shape, b_l.shape, w_u[i].shape, b_u.shape] + from_linear_layer_combine = ( + self.inputs_outputs_spec.is_wo_batch_bounds(input_affine_bounds_i, i=i), + self.linear, + ) + diagonal_combine = ( + self.inputs_outputs_spec.is_diagonal_bounds(input_affine_bounds_i, i=i), + self.inputs_outputs_spec.is_diagonal_bounds_shape(partial_layer_affine_bounds_true_shape_i, i=i), + ) + delta_w_l_new, delta_b_l_new, delta_w_u_new, delta_b_u_new = combine_affine_bounds( + affine_bounds_1=input_affine_bounds_i, + affine_bounds_2=partial_layer_affine_bounds_i, + from_linear_layer=from_linear_layer_combine, + diagonal=diagonal_combine, + ) + # add delta_b and delta_w, taking into account format (diagonal & from_linear_layer) + from_linear_layer_delta = all(from_linear_layer_combine) + diagonal_delta = all(diagonal_combine) + from_linear_add = (from_linear_layer_new, from_linear_layer_delta) + if first_iteration: + w_l_new = delta_w_l_new + w_u_new = delta_w_u_new + first_iteration = False + diagonal_w_new = diagonal_delta + else: + diagonal_add_w = (diagonal_w_new, diagonal_delta) + w_l_new = add_tensors( + w_l_new, + delta_w_l_new, + missing_batchsize=from_linear_add, + diagonal=diagonal_add_w, + ) + w_u_new = add_tensors( + w_u_new, + delta_w_u_new, + missing_batchsize=from_linear_add, + diagonal=diagonal_add_w, + ) + diagonal_w_new = all(diagonal_add_w) + b_l_new = add_tensors( + b_l_new, + delta_b_l_new, + missing_batchsize=from_linear_add, + ) + b_u_new = add_tensors( + b_u_new, + delta_b_u_new, + missing_batchsize=from_linear_add, + ) + from_linear_layer_new = all(from_linear_add) + return w_l_new, b_l_new, w_u_new, b_u_new + + def backward_affine_propagate( + self, output_affine_bounds: list[Tensor], input_constant_bounds: list[list[Tensor]] + ) -> list[tuple[Tensor, Tensor, Tensor, Tensor]]: + """Propagate model affine bounds in backward direction. + + By default, this is deduced from `get_affine_bounds()` (or `get_affine_representation()` if `self.linear is True). + But this could be overridden for better performance. See `DecomonConv2D` for an example. + + Args: + output_affine_bounds: [w_l, b_l, w_u, b_u] + partial affine bounds on model output w.r.t underlying keras layer output + input_constant_bounds: [[l_c_in[i], u_c_in[i]]]_i + constant oracle bounds on underlying keras layer inputs + + Returns: + [w_l_new[i], b_l_new[i], w_u_new[i], b_u_new[i]]_i: partial affine bounds on model output w.r.t. underlying keras layer *inputs* + + If we denote by + - x: keras model input + - m(x): keras model output + - z[i]: underlying keras layer i-th input + - h(x) = layer(z): output of the underlying keras layer + - h_j(x) output of the j-th layer + - w_l_j, b_l_j, w_u_j, b_u_j: current partial linear bounds on model output w.r.t to h_j(x) + + The following inequations are satisfied + + l_c_in[i] <= z[i] <= u_c_in[i] for all i + + Sum_{others layers j}{w_l_j * h_j(x) + b_l_j} + w_l * h(x) + b_l + <= m(x) + <= Sum_{others layers j}{w_u_j * h_j(x) + b_u_j} + w_u * h(x) + b_u + + Sum_{others layers j}{w_l_j * h_j(x) + b_l_j} + Sum_i{w_l_new[i] * z[i] + b_l_new[i]} + <= m(x) + <= Sum_{others layers j}{w_u_j * h_j(x) + b_u_j} + Sum_i[w_u_new[i] * z[i] + b_u_new[i]} + + """ + if self.linear: + w, b = self.get_affine_representation() + w_l_layer, b_l_layer, w_u_layer, b_u_layer = w, b, w, b + else: + lower, upper = self.inputs_outputs_spec.split_constant_bounds(constant_bounds=input_constant_bounds) + w_l_layer, b_l_layer, w_u_layer, b_u_layer = self.get_affine_bounds(lower=lower, upper=upper) + + from_linear_output_affine_bounds = self.inputs_outputs_spec.is_wo_batch_bounds(output_affine_bounds) + diagonal_output_affine_bounds = self.inputs_outputs_spec.is_diagonal_bounds(output_affine_bounds) + + first_iteration = True + propagated_affine_bounds = [] + output_affine_bounds_for_i = list(output_affine_bounds) + for i in range(len(w_l_layer)): + w_l_layer_i = w_l_layer[i] + w_u_layer_i = w_u_layer[i] + if first_iteration: + # we mix the biases for first input + b_l_layer_i = b_l_layer + b_u_layer_i = b_u_layer + else: + # we skip the biases for the other inputs as already taken into account + b_l_layer_i = K.zeros_like(b_l_layer) + b_u_layer_i = K.zeros_like(b_u_layer) + output_affine_bounds_for_i[1] = 0.0 + output_affine_bounds_for_i[3] = 0.0 + partial_layer_affine_bounds_i = [w_l_layer_i, b_l_layer_i, w_u_layer_i, b_u_layer_i] + from_linear_layer = (self.linear, from_linear_output_affine_bounds) + diagonal = ( + self.inputs_outputs_spec.is_diagonal_bounds(partial_layer_affine_bounds_i), + diagonal_output_affine_bounds, + ) + propagated_affine_bounds.append( + combine_affine_bounds( + affine_bounds_1=partial_layer_affine_bounds_i, + affine_bounds_2=output_affine_bounds_for_i, + from_linear_layer=from_linear_layer, + diagonal=diagonal, + ) + ) + return propagated_affine_bounds + + def get_forward_oracle( + self, + input_affine_bounds: list[list[Tensor]], + input_constant_bounds: list[list[Tensor]], + model_inputs: list[Tensor], + ) -> list[list[Tensor]]: + """Get constant oracle bounds on underlying keras layer input from forward input bounds. + + Args: + input_affine_bounds: affine bounds on each keras layer input w.r.t model input . Can be empty if not in affine mode. + input_constant_bounds: ibp constant bounds on each keras layer input. Can be empty if not in ibp mode. + model_inputs: underlying keras model input, wrapped in a list. Necessary only in affine mode, else empty. + + Returns: + constant bounds on each keras layer input deduced from forward input bounds + + `input_affine_bounds, input_constant_bounds` are the forward bounds to be propagate through the layer. + `input_affine_bounds` (resp. `input_constant_bounds`) will be empty if `self.affine` (resp. `self.ibp) is False. + + In hybrid case (ibp+affine), the constant bounds are assumed to be already tight, which means the previous + forward layer should already have took the tighter constant bounds between the ibp ones and the ones deduced + from the affine bounds given the considered perturbation domain. + + """ + if self.ibp: + # Hyp: in hybrid mode, the constant bounds are already tight + # (affine and ibp mixed in forward layer output to get the tightest constant bounds) + return input_constant_bounds + + elif self.affine: + if len(model_inputs) == 0: + raise RuntimeError("keras model input is necessary for get_forward_oracle() in affine mode.") + x = model_inputs[0] + constant_bounds = [] + for input_affine_bounds_i in input_affine_bounds: + if len(input_affine_bounds_i) == 0: + # special case: empty affine bounds => identity bounds + l_affine = self.perturbation_domain.get_lower_x(x) + u_affine = self.perturbation_domain.get_upper_x(x) + else: + w_l, b_l, w_u, b_u = input_affine_bounds_i + l_affine = self.perturbation_domain.get_lower(x, w_l, b_l) + u_affine = self.perturbation_domain.get_upper(x, w_u, b_u) + constant_bounds.append([l_affine, u_affine]) + return constant_bounds + + else: + raise RuntimeError("self.ibp and self.affine cannot be both False") + + def call_forward( + self, + affine_bounds_to_propagate: list[list[Tensor]], + input_bounds_to_propagate: list[list[Tensor]], + perturbation_domain_inputs: list[Tensor], + ) -> tuple[list[Tensor], list[Tensor]]: + """Propagate forward affine and constant bounds through the layer. + + Args: + affine_bounds_to_propagate: affine bounds on each keras layer input w.r.t model input. + Can be empty if not in affine mode. + input_bounds_to_propagate: ibp constant bounds on each keras layer input. Can be empty if not in ibp mode. + perturbation_domain_inputs: perturbation domain input, wrapped in a list. Necessary only in affine mode, else empty. + + Returns: + output_affine_bounds, output_constant_bounds: affine and constant bounds on the underlying keras layer output + + """ + return super().call_forward(affine_bounds_to_propagate, input_bounds_to_propagate, perturbation_domain_inputs) + + def call_backward( + self, affine_bounds_to_propagate: list[Tensor], constant_oracle_bounds: list[list[Tensor]] + ) -> list[list[Tensor]]: + return [ + list(partial_affine_bounds) + for partial_affine_bounds in self.backward_affine_propagate( + output_affine_bounds=affine_bounds_to_propagate, input_constant_bounds=constant_oracle_bounds + ) + ] + + def call(self, inputs: list[Tensor]) -> list[Tensor]: + """Propagate bounds in the specified direction `self.propagation`. + + Args: + inputs: flattened decomon inputs + - forward propagation: k affine bounds to propagate w.r.t. each keras layer input + k constant bounds + + inputs = ( + affine_bounds_to_propagate_0 + constant_oracle_bounds_0 + ... + + affine_bounds_to_propagate_k + constant_oracle_bounds_k + + model_inputs + ) + + with + - affine_bounds_to_propagate_* empty when affine is False; + (never to express identity bounds, as it would be impossible to separate bounds in flattened inputs) + - constant_oracle_bounds_* empty when ibp is False; + - model_inputs: + - if affine: the tensor defining the underlying keras model input perturbation, wrapped in a list; + - else: empty + + - backward propagation: only 1 affine bounds to propagate w.r.t keras layer output + + k constant bounds w.r.t each keras layer input + + inputs = ( + affine_bounds_to_propagate + + constant_oracle_bounds_0 + ... + constant_oracle_bounds_k + ) + + with affine_bounds_to_propagate potentially empty to express identity affine bounds. + + Returns: + the propagated bounds, in a flattened list. + + - in forward direction: affine_bounds_propagated + constant_bounds_propagated, each one being empty if self.affine or self.ibp is False + - in backward direction: affine_bounds_propagated_0 + ... + affine_bounds_propagated_k, affine bounds w.r.t to each keras layer input + + """ + return super().call(inputs=inputs) diff --git a/tests/conftest.py b/tests/conftest.py index 9113c684..4b329ae6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -292,7 +292,7 @@ def generate_simple_decomon_layer_inputs_from_keras_input( Returns: """ - layer_input_shape = keras_input.shape[1:] + layer_input_shape = tuple(keras_input.shape[1:]) model_input_shape = layer_input_shape model_output_shape = layer_output_shape inputs_outputs_spec = InputsOutputsSpec( @@ -352,6 +352,47 @@ def generate_simple_decomon_layer_inputs_from_keras_input( model_inputs=model_inputs, ) + @staticmethod + def generate_merging_decomon_input_from_single_decomon_inputs( + decomon_inputs: list[list[Tensor]], ibp: bool, affine: bool, propagation: Propagation + ) -> list[Tensor]: + inputs_outputs_spec_single = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + layer_input_shape=tuple(), + model_input_shape=tuple(), + model_output_shape=tuple(), + ) + affine_bounds_to_propagate, constant_oracle_bounds, model_inputs = [], [], [] + for decomon_input in decomon_inputs: + ( + affine_bounds_to_propagate_i, + constant_oracle_bounds_i, + model_inputs_i, + ) = inputs_outputs_spec_single.split_inputs(decomon_input) + model_inputs = model_inputs_i + if propagation == Propagation.FORWARD: + affine_bounds_to_propagate.append(affine_bounds_to_propagate_i) + else: + affine_bounds_to_propagate = affine_bounds_to_propagate_i + constant_oracle_bounds.append(constant_oracle_bounds_i) + + inputs_outputs_spec_merging = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + layer_input_shape=[tuple()], + model_input_shape=tuple(), + model_output_shape=tuple(), + is_merging_layer=True, + ) + return inputs_outputs_spec_merging.flatten_inputs( + affine_bounds_to_propagate=affine_bounds_to_propagate, + constant_oracle_bounds=constant_oracle_bounds, + model_inputs=model_inputs, + ) + @staticmethod def get_standard_values_0d_box(n, batchsize=10): """A set of functions with their monotonic decomposition for testing the activations""" @@ -784,13 +825,20 @@ def assert_decomon_output_compare_with_keras_input_output_layer( affine, propagation, decimal=5, + is_merging_layer=False, ): + if is_merging_layer and not isinstance(keras_layer_input, KerasTensor): + # merging layer, except degenerated case with a single input + layer_input_shape = [tuple(t.shape[1:]) for t in keras_layer_input] + else: + layer_input_shape = tuple(keras_layer_input.shape[1:]) inputs_outputs_spec = InputsOutputsSpec( ibp=ibp, affine=affine, propagation=propagation, - layer_input_shape=keras_layer_input.shape[1:], - model_output_shape=keras_model_output.shape[1:], + layer_input_shape=layer_input_shape, + model_output_shape=tuple(keras_model_output.shape[1:]), + is_merging_layer=is_merging_layer, ) affine_bounds_propagated, constant_bounds_propagated = inputs_outputs_spec.split_outputs(outputs=decomon_output) @@ -802,22 +850,48 @@ def assert_decomon_output_compare_with_keras_input_output_layer( keras_output = keras_model_output if affine or propagation == Propagation.BACKWARD: - if len(affine_bounds_propagated) == 0: - # identity case - lower_affine = keras_input - upper_affine = keras_input + if is_merging_layer and propagation == Propagation.BACKWARD: + # one list of affine bounds by keras (layer) input + lower_affine = 0.0 + upper_affine = 0.0 + for keras_input_i, affine_bounds_propagated_i in zip(keras_input, affine_bounds_propagated): + if len(affine_bounds_propagated_i) == 0: + # identity case + lower_affine += keras_input_i + upper_affine += keras_input_i + else: + w_l, b_l, w_u, b_u = affine_bounds_propagated_i + diagonal = (False, w_l.shape == b_l.shape) + missing_batchsize = (False, len(b_l.shape) < len(keras_output.shape)) + lower_affine += ( + batch_multid_dot(keras_input_i, w_l, diagonal=diagonal, missing_batchsize=missing_batchsize) + + b_l + ) + upper_affine += ( + batch_multid_dot(keras_input_i, w_u, diagonal=diagonal, missing_batchsize=missing_batchsize) + + b_u + ) + Helpers.assert_ordered(lower_affine, keras_output, decimal=decimal, err_msg="lower_affine not ok") + Helpers.assert_ordered(keras_output, upper_affine, decimal=decimal, err_msg="upper_affine not ok") + else: - w_l, b_l, w_u, b_u = affine_bounds_propagated - diagonal = (False, w_l.shape == b_l.shape) - missing_batchsize = (False, len(b_l.shape) < len(keras_output.shape)) - lower_affine = ( - batch_multid_dot(keras_input, w_l, diagonal=diagonal, missing_batchsize=missing_batchsize) + b_l - ) - upper_affine = ( - batch_multid_dot(keras_input, w_u, diagonal=diagonal, missing_batchsize=missing_batchsize) + b_u - ) - Helpers.assert_ordered(lower_affine, keras_output, decimal=decimal, err_msg="lower_affine not ok") - Helpers.assert_ordered(keras_output, upper_affine, decimal=decimal, err_msg="upper_affine not ok") + # generic case: one single list of affine bounds and single one keras input (layer or model input according to propagation) + if len(affine_bounds_propagated) == 0: + # identity case + lower_affine = keras_input + upper_affine = keras_input + else: + w_l, b_l, w_u, b_u = affine_bounds_propagated + diagonal = (False, w_l.shape == b_l.shape) + missing_batchsize = (False, len(b_l.shape) < len(keras_output.shape)) + lower_affine = ( + batch_multid_dot(keras_input, w_l, diagonal=diagonal, missing_batchsize=missing_batchsize) + b_l + ) + upper_affine = ( + batch_multid_dot(keras_input, w_u, diagonal=diagonal, missing_batchsize=missing_batchsize) + b_u + ) + Helpers.assert_ordered(lower_affine, keras_output, decimal=decimal, err_msg="lower_affine not ok") + Helpers.assert_ordered(keras_output, upper_affine, decimal=decimal, err_msg="upper_affine not ok") if ibp and propagation == Propagation.FORWARD: lower_ibp, upper_ibp = constant_bounds_propagated @@ -974,8 +1048,8 @@ def decomon_symbolic_input_fn(output_shape): def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): x, y, z, u_c, w_u, b_u, l_c, w_l, b_l = get_standard_values_fn() - layer_input_shape = y.shape[1:] - model_input_shape = x.shape[1:] + layer_input_shape = tuple(y.shape[1:]) + model_input_shape = tuple(x.shape[1:]) inputs_outputs_spec = InputsOutputsSpec( ibp=ibp, @@ -1068,8 +1142,8 @@ def decomon_symbolic_input_fn(output_shape): def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): x, y, z, u_c, w_u, b_u, l_c, w_l, b_l = get_standard_values_fn() - layer_input_shape = y.shape[1:] - model_input_shape = x.shape[1:] + layer_input_shape = tuple(y.shape[1:]) + model_input_shape = tuple(x.shape[1:]) inputs_outputs_spec = InputsOutputsSpec( ibp=ibp, diff --git a/tests/test_activation.py b/tests/test_activation.py index dabece2a..8d91c95f 100644 --- a/tests/test_activation.py +++ b/tests/test_activation.py @@ -35,6 +35,7 @@ def test_decomon_activation( # init + build decomon layer output_shape = layer.output.shape[1:] model_output_shape = output_shape + model_input_shape = keras_symbolic_model_input.shape[1:] decomon_symbolic_inputs = decomon_symbolic_input_fn(output_shape=output_shape) decomon_layer = decomon_layer_class( layer=layer, @@ -43,6 +44,7 @@ def test_decomon_activation( propagation=propagation, perturbation_domain=perturbation_domain, model_output_shape=model_output_shape, + model_input_shape=model_input_shape, slope=slope, ) decomon_layer(decomon_symbolic_inputs) diff --git a/tests/test_decomon_layer.py b/tests/test_decomon_layer.py index 6ad1a31e..cc43c7aa 100644 --- a/tests/test_decomon_layer.py +++ b/tests/test_decomon_layer.py @@ -96,6 +96,7 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper propagation=propagation, perturbation_domain=perturbation_domain, model_output_shape=model_output_shape, + model_input_shape=model_input_shape, ) non_linear_decomon_layer = MyNonLinearDecomonDense1d( layer=layer, @@ -104,6 +105,7 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper propagation=propagation, perturbation_domain=perturbation_domain, model_output_shape=model_output_shape, + model_input_shape=model_input_shape, ) # symbolic inputs diff --git a/tests/test_dense.py b/tests/test_dense.py index 729ffd6e..ac4e351b 100644 --- a/tests/test_dense.py +++ b/tests/test_dense.py @@ -42,6 +42,7 @@ def test_decomon_dense( # init + build decomon layer output_shape = layer.output.shape[1:] model_output_shape = output_shape + model_input_shape = keras_symbolic_model_input.shape[1:] decomon_symbolic_inputs = decomon_symbolic_input_fn(output_shape=output_shape) decomon_layer = decomon_layer_class( layer=layer, @@ -50,6 +51,7 @@ def test_decomon_dense( propagation=propagation, perturbation_domain=perturbation_domain, model_output_shape=model_output_shape, + model_input_shape=model_input_shape, ) decomon_layer(decomon_symbolic_inputs) diff --git a/tests/test_keras_utils.py b/tests/test_keras_utils.py index fb80ad7a..eff9bcaa 100644 --- a/tests/test_keras_utils.py +++ b/tests/test_keras_utils.py @@ -6,6 +6,7 @@ from numpy.testing import assert_almost_equal from decomon.keras_utils import ( + add_tensors, batch_multid_dot, get_weight_index_from_name, is_a_merge_layer, @@ -230,6 +231,77 @@ def test_batch_multi_dot_diag_missing_batchsize(missing_batchsize, diagonal, hel ) +@pytest.mark.parametrize( + "x_shape, y_shape, missing_batchsize, diagonal", + [ + ((4, 6), (6,), (True, True), (False, False)), + ((4, 6), (6,), (False, False), (False, False)), + ((4, 6), (6, 6), (False, False), (True, False)), + ], +) +def test_add_tensors_nok_incompatible_shapes(x_shape, y_shape, missing_batchsize, diagonal): + x = K.ones(x_shape) + y = K.ones(y_shape) + with pytest.raises(ValueError): + add_tensors(x, y, missing_batchsize=missing_batchsize, diagonal=diagonal) + + +@pytest.mark.parametrize("missing_batchsize", [(False, False), (True, False), (False, True), (True, True)]) +@pytest.mark.parametrize("diagonal", [(True, True), (True, False), (False, True), (False, False)]) +def test_add_tensors_ok(missing_batchsize, diagonal, helpers): + batchsize = 10 + diag_shape = (4, 5, 2) + other_shape = diag_shape + + diag_x, diag_y = diagonal + missing_batchsize_x, missing_batchsize_y = missing_batchsize + + x_full, x_simplified = generate_tensor_full_n_diag( + batchsize=batchsize, + diag_shape=diag_shape, + other_shape=other_shape, + diag=diag_x, + missing_batchsize=missing_batchsize_x, + left=True, + ) + y_full, y_simplified = generate_tensor_full_n_diag( + batchsize=batchsize, + diag_shape=diag_shape, + other_shape=other_shape, + diag=diag_y, + missing_batchsize=missing_batchsize_y, + left=True, + ) + + res_full = x_full + y_full + res_simplified = add_tensors( + x_simplified, + y_simplified, + missing_batchsize=missing_batchsize, + diagonal=diagonal, + ) + + if missing_batchsize_x and missing_batchsize_y: + # the result stayed w/o batch axis, needs to be added to be compared with full result + res_simplified = K.repeat(res_simplified[None], batchsize, axis=0) + + if diag_x and diag_y: + # the result stayed diagonal, needs to be reworked to be compared with full result + assert res_simplified.shape == (batchsize,) + diag_shape + res_simplified = K.concatenate( + [ + K.reshape(K.diag(K.ravel(res_simplified[i])), diag_shape + diag_shape)[None] + for i in range(len(res_simplified)) + ], + axis=0, + ) + + helpers.assert_almost_equal( + res_full, + res_simplified, + ) + + def test_get_weight_index_from_name_nok_attribute(): layer = Dense(3) layer(K.zeros((2, 1))) From 2a4dc9745bf2b1cfdb1b06356e5c8ec4017a0672 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 16 Feb 2024 10:42:04 +0100 Subject: [PATCH 044/101] Implement and test DecomonAdd Test also non-diag, non-linear, or both, versions of DecomonAdd to test each kind of DecomonMerge layer. --- src/decomon/layers/merging/add.py | 17 +++ tests/conftest.py | 34 ++++- tests/test_merge_layers.py | 210 ++++++++++++++++++++++++++++++ 3 files changed, 256 insertions(+), 5 deletions(-) create mode 100644 tests/test_merge_layers.py diff --git a/src/decomon/layers/merging/add.py b/src/decomon/layers/merging/add.py index e69de29b..7811f87f 100644 --- a/src/decomon/layers/merging/add.py +++ b/src/decomon/layers/merging/add.py @@ -0,0 +1,17 @@ +import keras.ops as K +from keras.layers import Add + +from decomon.layers.merging.base_merge import DecomonMerge +from decomon.types import Tensor + + +class DecomonAdd(DecomonMerge): + layer: Add + linear = True + diagonal = True + + def get_affine_representation(self) -> tuple[list[Tensor], Tensor]: + w = [K.ones(input_i.shape[1:]) for input_i in self.keras_layer_input] + b = K.zeros(self.layer.output.shape[1:]) + + return w, b diff --git a/tests/conftest.py b/tests/conftest.py index 4b329ae6..9dbd2ef1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -899,17 +899,41 @@ def assert_decomon_output_compare_with_keras_input_output_layer( Helpers.assert_ordered(keras_output, upper_ibp, decimal=decimal, err_msg="upper_ibp not ok") @staticmethod - def assert_decomon_output_lower_equal_upper(decomon_output, ibp, affine, propagation, decimal=5): + def assert_decomon_output_lower_equal_upper( + decomon_output, + ibp, + affine, + propagation, + decimal=5, + is_merging_layer=False, + ): + if is_merging_layer: + layer_input_shape = [tuple()] + else: + layer_input_shape = tuple() inputs_outputs_specs = InputsOutputsSpec( - ibp=ibp, affine=affine, propagation=propagation, layer_input_shape=tuple(), model_output_shape=tuple() + ibp=ibp, + affine=affine, + propagation=propagation, + layer_input_shape=layer_input_shape, + model_output_shape=tuple(), + is_merging_layer=is_merging_layer, ) affine_bounds_propagated, constant_bounds_propagated = inputs_outputs_specs.split_outputs( outputs=decomon_output ) if propagation == Propagation.BACKWARD or affine: - w_l, b_l, w_u, b_u = affine_bounds_propagated - Helpers.assert_almost_equal(w_l, w_u, decimal=decimal) - Helpers.assert_almost_equal(b_l, b_u, decimal=decimal) + if is_merging_layer and propagation == Propagation.BACKWARD: + # one list of affine bounds by keras layer input + for affine_bounds_propagated_i in affine_bounds_propagated: + w_l, b_l, w_u, b_u = affine_bounds_propagated_i + Helpers.assert_almost_equal(w_l, w_u, decimal=decimal) + Helpers.assert_almost_equal(b_l, b_u, decimal=decimal) + else: + # generic case: one single list of affine bounds + w_l, b_l, w_u, b_u = affine_bounds_propagated + Helpers.assert_almost_equal(w_l, w_u, decimal=decimal) + Helpers.assert_almost_equal(b_l, b_u, decimal=decimal) if propagation == Propagation.FORWARD and ibp: lower_ibp, upper_ibp = constant_bounds_propagated diff --git a/tests/test_merge_layers.py b/tests/test_merge_layers.py new file mode 100644 index 00000000..9bdd5dc0 --- /dev/null +++ b/tests/test_merge_layers.py @@ -0,0 +1,210 @@ +from typing import TypeVar + +import keras.ops as K +import numpy as np +import pytest +from keras.layers import Add + +from decomon.keras_utils import batch_multid_dot +from decomon.layers.merging.add import DecomonAdd, DecomonMerge +from decomon.types import Tensor + + +# Defining non-linear and/or non-diagonal versions of DecomonAdd +class DecomonNonDiagAdd(DecomonMerge): + layer: Add + linear = True + diagonal = False + + def get_affine_representation(self) -> tuple[list[Tensor], Tensor]: + w = [] + for input_i in self.keras_layer_input: + diag_shape = input_i.shape[1:] + full_shape = diag_shape + diag_shape + flattened_diag_shape = (int(np.prod(diag_shape)),) + w.append(K.reshape(K.diag(K.ones(flattened_diag_shape)), full_shape)) + + b = K.zeros(self.layer.output.shape[1:]) + + return w, b + + +class DecomonNonLinearAdd(DecomonMerge): + layer: Add + linear = False + diagonal = True + + def get_affine_bounds( + self, lower: list[Tensor], upper: list[Tensor] + ) -> tuple[list[Tensor], Tensor, list[Tensor], Tensor]: + w = [] + batchsize = lower[0].shape[0] + for input_i in self.keras_layer_input: + diag_shape = input_i.shape[1:] + w.append(K.repeat(K.ones(diag_shape)[None], batchsize, axis=0)) + + b = K.zeros_like(lower[0]) + + return w, b, w, b + + def forward_ibp_propagate(self, lower: list[Tensor], upper: list[Tensor]) -> tuple[Tensor, Tensor]: + return sum(lower), sum(upper) + + +class DecomonNonLinearNonDiagAdd(DecomonMerge): + layer: Add + linear = False + diagonal = False + + def get_affine_bounds( + self, lower: list[Tensor], upper: list[Tensor] + ) -> tuple[list[Tensor], Tensor, list[Tensor], Tensor]: + w = [] + batchsize = lower[0].shape[0] + for input_i in self.keras_layer_input: + diag_shape = input_i.shape[1:] + full_shape = diag_shape + diag_shape + flattened_diag_shape = (int(np.prod(diag_shape)),) + w.append(K.repeat(K.reshape(K.diag(K.ones(flattened_diag_shape)), full_shape)[None], batchsize, axis=0)) + + b = K.zeros_like(lower[0]) + + return w, b, w, b + + def forward_ibp_propagate(self, lower: list[Tensor], upper: list[Tensor]) -> tuple[Tensor, Tensor]: + return sum(lower), sum(upper) + + +T = TypeVar("T") + + +def double_input(input: T) -> list[T]: + return [input] * 2 + + +@pytest.mark.parametrize( + "decomon_layer_class, decomon_layer_kwargs, keras_layer_class, keras_layer_kwargs, is_actually_linear", + [ + (DecomonAdd, {}, Add, {}, True), + (DecomonNonDiagAdd, {}, Add, {}, True), + (DecomonNonLinearAdd, {}, Add, {}, True), + (DecomonNonLinearNonDiagAdd, {}, Add, {}, True), + ], +) +def test_decomon_merge( + decomon_layer_class, + decomon_layer_kwargs, + keras_layer_class, + keras_layer_kwargs, + is_actually_linear, + ibp, + affine, + propagation, + perturbation_domain, + batchsize, + keras_symbolic_model_input_fn, + keras_symbolic_layer_input_fn, + decomon_symbolic_input_fn, + keras_model_input_fn, + keras_layer_input_fn, + decomon_input_fn, + equal_bounds, + helpers, +): + decimal = 4 + if is_actually_linear is None: + is_actually_linear = decomon_layer_class.linear + + # init + build keras layer + keras_symbolic_model_input = keras_symbolic_model_input_fn() + keras_symbolic_layer_input_0 = keras_symbolic_layer_input_fn(keras_symbolic_model_input) + # we merge twice the same input + keras_symbolic_layer_input = double_input(keras_symbolic_layer_input_0) + layer = keras_layer_class(**keras_layer_kwargs) + layer(keras_symbolic_layer_input) + + # init + build decomon layer + output_shape = layer.output.shape[1:] + model_output_shape = output_shape + model_input_shape = keras_symbolic_model_input.shape[1:] + decomon_symbolic_inputs_0 = decomon_symbolic_input_fn(output_shape=output_shape) + decomon_symbolic_inputs = helpers.generate_merging_decomon_input_from_single_decomon_inputs( + decomon_inputs=double_input(decomon_symbolic_inputs_0), ibp=ibp, affine=affine, propagation=propagation + ) + + decomon_layer = decomon_layer_class( + layer=layer, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + model_output_shape=model_output_shape, + model_input_shape=model_input_shape, + **decomon_layer_kwargs, + ) + # skip if empty affine bounds in forward propagation (it would generate issues to split inputs) + if ( + decomon_layer.inputs_outputs_spec.cannot_have_empty_affine_inputs() + and decomon_layer.inputs_outputs_spec.nb_input_tensors > len(decomon_symbolic_inputs) + ): + pytest.skip("Skip cases with empty (meaning identity) affine inputs that cannot be properly split") + + decomon_layer(decomon_symbolic_inputs) + + # call on actual inputs + keras_model_input = keras_model_input_fn() + keras_layer_input_0 = keras_layer_input_fn(keras_model_input) + decomon_inputs_0 = decomon_input_fn( + keras_model_input=keras_model_input, keras_layer_input=keras_layer_input_0, output_shape=output_shape + ) + keras_layer_input = double_input(keras_layer_input_0) + decomon_inputs = helpers.generate_merging_decomon_input_from_single_decomon_inputs( + decomon_inputs=double_input(decomon_inputs_0), ibp=ibp, affine=affine, propagation=propagation + ) + + keras_output = layer(keras_layer_input) + decomon_output = decomon_layer(decomon_inputs) + + # check affine representation is ok + if decomon_layer.linear: + w, b = decomon_layer.get_affine_representation() + keras_output_2 = b + for w_i, keras_layer_input_i in zip(w, keras_layer_input): + diagonal = (False, w_i.shape == b.shape) + missing_batchsize = (False, True) + keras_output_2 += batch_multid_dot( + keras_layer_input_i, w_i, missing_batchsize=missing_batchsize, diagonal=diagonal + ) + np.testing.assert_almost_equal( + K.convert_to_numpy(keras_output), + K.convert_to_numpy(keras_output_2), + decimal=decimal, + err_msg="wrong affine representation", + ) + + # check output shapes + input_shape = [t.shape for t in decomon_inputs] + output_shape = [t.shape for t in decomon_output] + expected_output_shape = decomon_layer.compute_output_shape(input_shape) + expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) + assert output_shape == expected_output_shape + + # check ibp and affine bounds well ordered w.r.t. keras inputs/outputs + helpers.assert_decomon_output_compare_with_keras_input_output_layer( + decomon_output=decomon_output, + keras_model_input=keras_model_input, + keras_layer_input=keras_layer_input, + keras_model_output=keras_output, + keras_layer_output=keras_output, + ibp=ibp, + affine=affine, + propagation=propagation, + decimal=decimal, + is_merging_layer=True, + ) + + # before propagation through linear layer lower == upper => lower == upper after propagation + if equal_bounds and is_actually_linear: + helpers.assert_decomon_output_lower_equal_upper( + decomon_output, ibp=ibp, affine=affine, propagation=propagation, decimal=decimal, is_merging_layer=True + ) From f19f8b3173b368dfbcce8ba185f998ff682fe003 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 16 Feb 2024 17:53:30 +0100 Subject: [PATCH 045/101] Allow decomon layers import from decomon.layers --- src/decomon/layers/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/decomon/layers/__init__.py b/src/decomon/layers/__init__.py index e69de29b..17d47221 100644 --- a/src/decomon/layers/__init__.py +++ b/src/decomon/layers/__init__.py @@ -0,0 +1,4 @@ +from .activations.activation import DecomonActivation, DecomonReLU +from .core.dense import DecomonDense +from .layer import DecomonLayer +from .merging.add import DecomonAdd From a26ac576e299843f298da7fe2a5f37e7a5056e4a Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 16 Feb 2024 18:35:30 +0100 Subject: [PATCH 046/101] Merge tests for unary layers in a single test --- tests/test_activation.py | 80 ------------------- tests/{test_dense.py => test_unary_layers.py} | 74 ++++++++++++----- 2 files changed, 52 insertions(+), 102 deletions(-) delete mode 100644 tests/test_activation.py rename tests/{test_dense.py => test_unary_layers.py} (60%) diff --git a/tests/test_activation.py b/tests/test_activation.py deleted file mode 100644 index 8d91c95f..00000000 --- a/tests/test_activation.py +++ /dev/null @@ -1,80 +0,0 @@ -import keras.ops as K -import numpy as np -import pytest -from keras.layers import Activation, Dense, Input - -from decomon.keras_utils import batch_multid_dot -from decomon.layers.activations.activation import DecomonActivation - - -def test_decomon_activation( - activation, - slope, - ibp, - affine, - propagation, - perturbation_domain, - batchsize, - keras_symbolic_model_input_fn, - keras_symbolic_layer_input_fn, - decomon_symbolic_input_fn, - keras_model_input_fn, - keras_layer_input_fn, - decomon_input_fn, - helpers, -): - decimal = 4 - decomon_layer_class = DecomonActivation - - # init + build keras layer - keras_symbolic_model_input = keras_symbolic_model_input_fn() - keras_symbolic_layer_input = keras_symbolic_layer_input_fn(keras_symbolic_model_input) - layer = Activation(activation=activation) - layer(keras_symbolic_layer_input) - - # init + build decomon layer - output_shape = layer.output.shape[1:] - model_output_shape = output_shape - model_input_shape = keras_symbolic_model_input.shape[1:] - decomon_symbolic_inputs = decomon_symbolic_input_fn(output_shape=output_shape) - decomon_layer = decomon_layer_class( - layer=layer, - ibp=ibp, - affine=affine, - propagation=propagation, - perturbation_domain=perturbation_domain, - model_output_shape=model_output_shape, - model_input_shape=model_input_shape, - slope=slope, - ) - decomon_layer(decomon_symbolic_inputs) - - # call on actual inputs - keras_model_input = keras_model_input_fn() - keras_layer_input = keras_layer_input_fn(keras_model_input) - decomon_inputs = decomon_input_fn( - keras_model_input=keras_model_input, keras_layer_input=keras_layer_input, output_shape=output_shape - ) - - keras_output = layer(keras_layer_input) - decomon_output = decomon_layer(decomon_inputs) - - # check output shapes - input_shape = [t.shape for t in decomon_inputs] - output_shape = [t.shape for t in decomon_output] - expected_output_shape = decomon_layer.compute_output_shape(input_shape) - expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) - assert output_shape == expected_output_shape - - # check ibp and affine bounds well ordered w.r.t. keras inputs/outputs - helpers.assert_decomon_output_compare_with_keras_input_output_layer( - decomon_output=decomon_output, - keras_model_input=keras_model_input, - keras_layer_input=keras_layer_input, - keras_model_output=keras_output, - keras_layer_output=keras_output, - ibp=ibp, - affine=affine, - propagation=propagation, - decimal=decimal, - ) diff --git a/tests/test_dense.py b/tests/test_unary_layers.py similarity index 60% rename from tests/test_dense.py rename to tests/test_unary_layers.py index ac4e351b..8c8ad010 100644 --- a/tests/test_dense.py +++ b/tests/test_unary_layers.py @@ -1,15 +1,40 @@ import keras.ops as K import numpy as np -import pytest -from keras.layers import Dense, Input +from keras.layers import Activation, Dense +from pytest_cases import fixture, fixture_ref, parametrize from decomon.keras_utils import batch_multid_dot -from decomon.layers.core.dense import DecomonDense +from decomon.layers import DecomonActivation, DecomonDense -def test_decomon_dense( - use_bias, - randomize, +@fixture +def keras_dense_kwargs(use_bias): + return dict(units=7, use_bias=use_bias) + + +@fixture +def keras_activation_kwargs(activation): + return dict(activation=activation) + + +@fixture +def decomon_activation_kwargs(slope): + return dict(slope=slope) + + +@parametrize( + "decomon_layer_class, decomon_layer_kwargs, keras_layer_class, keras_layer_kwargs, is_actually_linear", + [ + (DecomonDense, {}, Dense, keras_dense_kwargs, None), + (DecomonActivation, decomon_activation_kwargs, Activation, keras_activation_kwargs, None), + ], +) +def test_decomon_unary_layer( + decomon_layer_class, + decomon_layer_kwargs, + keras_layer_class, + keras_layer_kwargs, + is_actually_linear, ibp, affine, propagation, @@ -25,19 +50,18 @@ def test_decomon_dense( helpers, ): decimal = 4 - units = 7 - decomon_layer_class = DecomonDense + if is_actually_linear is None: + is_actually_linear = decomon_layer_class.linear # init + build keras layer keras_symbolic_model_input = keras_symbolic_model_input_fn() keras_symbolic_layer_input = keras_symbolic_layer_input_fn(keras_symbolic_model_input) - layer = Dense(units=units, use_bias=use_bias) + layer = keras_layer_class(**keras_layer_kwargs) layer(keras_symbolic_layer_input) - if randomize: - # randomize weights => non-zero biases - for w in layer.weights: - w.assign(np.random.random(w.shape)) + # randomize weights => non-zero biases + for w in layer.weights: + w.assign(np.random.random(w.shape)) # init + build decomon layer output_shape = layer.output.shape[1:] @@ -52,6 +76,7 @@ def test_decomon_dense( perturbation_domain=perturbation_domain, model_output_shape=model_output_shape, model_input_shape=model_input_shape, + **decomon_layer_kwargs, ) decomon_layer(decomon_symbolic_inputs) @@ -66,14 +91,19 @@ def test_decomon_dense( decomon_output = decomon_layer(decomon_inputs) # check affine representation is ok - w, b = decomon_layer.get_affine_representation() - keras_output_2 = batch_multid_dot(keras_layer_input, w, missing_batchsize=(False, True)) + b - np.testing.assert_almost_equal( - K.convert_to_numpy(keras_output), - K.convert_to_numpy(keras_output_2), - decimal=decimal, - err_msg="wrong affine representation", - ) + if decomon_layer.linear: + w, b = decomon_layer.get_affine_representation() + diagonal = (False, w.shape == b.shape) + missing_batchsize = (False, True) + keras_output_2 = ( + batch_multid_dot(keras_layer_input, w, missing_batchsize=missing_batchsize, diagonal=diagonal) + b + ) + np.testing.assert_almost_equal( + K.convert_to_numpy(keras_output), + K.convert_to_numpy(keras_output_2), + decimal=decimal, + err_msg="wrong affine representation", + ) # check output shapes input_shape = [t.shape for t in decomon_inputs] @@ -96,7 +126,7 @@ def test_decomon_dense( ) # before propagation through linear layer lower == upper => lower == upper after propagation - if equal_bounds: + if equal_bounds and is_actually_linear: helpers.assert_decomon_output_lower_equal_upper( decomon_output, ibp=ibp, affine=affine, propagation=propagation, decimal=decimal ) From d8031f4d85ac420b627d588e6c2b8435b12335c4 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 19 Feb 2024 11:37:00 +0100 Subject: [PATCH 047/101] Add model_input_shape and model_output_shape properties to DecomonLayer As it is in __init__ signature, it is better to be able to retrive them directly from DecomonLayer. --- src/decomon/layers/layer.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index ceb967ee..b0ab2d1a 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -137,6 +137,14 @@ def __init__( is_merging_layer=self._is_merging, ) + @property + def model_input_shape(self) -> tuple[int, ...]: + return self.inputs_outputs_spec.model_input_shape + + @property + def model_output_shape(self) -> tuple[int, ...]: + return self.inputs_outputs_spec.model_output_shape + def get_config(self) -> dict[str, Any]: config = super().get_config() config.update( From 78f5f70488b130ddf8f3397ab0b0f5fcb34f7b5c Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 19 Feb 2024 12:05:10 +0100 Subject: [PATCH 048/101] Fix merge layers with pytorch backend the operator += does not accept to change shapes with pytorch backend. So we replace u_c += ... by u_c = u_c + ... --- src/decomon/layers/merging/base_merge.py | 13 +++++++++---- tests/test_merge_layers.py | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/decomon/layers/merging/base_merge.py b/src/decomon/layers/merging/base_merge.py index c1e560d3..78329fcf 100644 --- a/src/decomon/layers/merging/base_merge.py +++ b/src/decomon/layers/merging/base_merge.py @@ -161,11 +161,16 @@ def forward_ibp_propagate(self, lower: list[Tensor], upper: list[Tensor]) -> tup w_i_pos = K.maximum(w_i, z_value) w_i_neg = K.minimum(w_i, z_value) - l_c += batch_multid_dot(lower_i, w_i_pos, **kwargs_dot) + batch_multid_dot( - upper_i, w_i_neg, **kwargs_dot + # NB: += does not work well with broadcasting on pytorch backend => we use l_c = l_c + ... + l_c = ( + l_c + + batch_multid_dot(lower_i, w_i_pos, **kwargs_dot) + + batch_multid_dot(upper_i, w_i_neg, **kwargs_dot) ) - u_c += batch_multid_dot(upper_i, w_i_pos, **kwargs_dot) + batch_multid_dot( - lower_i, w_i_neg, **kwargs_dot + u_c = ( + u_c + + batch_multid_dot(upper_i, w_i_pos, **kwargs_dot) + + batch_multid_dot(lower_i, w_i_neg, **kwargs_dot) ) return l_c, u_c diff --git a/tests/test_merge_layers.py b/tests/test_merge_layers.py index 9bdd5dc0..85151ea7 100644 --- a/tests/test_merge_layers.py +++ b/tests/test_merge_layers.py @@ -172,9 +172,9 @@ def test_decomon_merge( for w_i, keras_layer_input_i in zip(w, keras_layer_input): diagonal = (False, w_i.shape == b.shape) missing_batchsize = (False, True) - keras_output_2 += batch_multid_dot( + keras_output_2 = keras_output_2 + batch_multid_dot( keras_layer_input_i, w_i, missing_batchsize=missing_batchsize, diagonal=diagonal - ) + ) # += does not work well with broadcasting on pytorch backend np.testing.assert_almost_equal( K.convert_to_numpy(keras_output), K.convert_to_numpy(keras_output_2), From 0893f226b4067a8b831edb4665c12d2ec519a7e7 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 19 Feb 2024 11:13:33 +0100 Subject: [PATCH 049/101] Implement to_decomon() conversion method We deduce the DecomonLayer class to use, by (higher to lower priority): - user-defined mapping - default mapping defined in same module - using keras layer name + adding Decomon prefix + looking in decomon.layers namespace --- src/decomon/layers/convert.py | 100 ++++++++++++++++++ tests/test_to_decomon.py | 184 ++++++++++++++++++++++++++++++++++ 2 files changed, 284 insertions(+) create mode 100644 tests/test_to_decomon.py diff --git a/src/decomon/layers/convert.py b/src/decomon/layers/convert.py index e69de29b..ac6dbff5 100644 --- a/src/decomon/layers/convert.py +++ b/src/decomon/layers/convert.py @@ -0,0 +1,100 @@ +import logging +from typing import Any, Optional + +from keras.layers import Activation, Add, Dense, Layer + +import decomon.layers +from decomon.core import PerturbationDomain, Propagation, Slope +from decomon.layers import DecomonActivation, DecomonAdd, DecomonDense, DecomonLayer + +logger = logging.getLogger(__name__) + + +DECOMON_PREFIX = "Decomon" + +default_mapping_keras2decomon_classes: dict[type[Layer], type[DecomonLayer]] = { + Add: DecomonAdd, + Dense: DecomonDense, + Activation: DecomonActivation, +} +"""Default mapping between keras layers and decomon layers.""" + +default_mapping_kerasname2decomonclass: dict[str, type[DecomonLayer]] = { + k[len(DECOMON_PREFIX) :]: v + for k, v in vars(decomon.layers).items() + if k.startswith(DECOMON_PREFIX) and issubclass(v, DecomonLayer) +} +"""Default mapping from a keras class name to a decomon layer class. + +This mapping is generated automatically from `decomon.layers` namespace. +It is used only when `default_mapping_keras2decomon_classes` does not contain +the desired keras layer class. + +""" + + +def to_decomon( + layer: Layer, + perturbation_domain: Optional[PerturbationDomain] = None, + ibp: bool = True, + affine: bool = True, + propagation: Propagation = Propagation.FORWARD, + model_input_shape: Optional[tuple[int, ...]] = None, + model_output_shape: Optional[tuple[int, ...]] = None, + slope: Slope = Slope.V_SLOPE, + mapping_keras2decomon_classes: Optional[dict[type[Layer], type[DecomonLayer]]] = None, + **kwargs: Any, +) -> DecomonLayer: + """Convert a keras layer into the corresponding decomon layer. + + Args: + layer: keras layer to convert + perturbation_domain: perturbation domain. Default to a box domain + ibp: if True, forward propagate constant bounds + affine: if True, forward propagate affine bounds + propagation: direction of bounds propagation + - forward: from input to output + - backward: from output to input + model_output_shape: shape of the underlying model output (omitting batch axis). + It allows determining if the backward bounds are with a batch axis or not. + model_input_shape: shape of the underlying keras model input (omitting batch axis). + slope: slope used for the affine relaxation of activation layers + mapping_keras2decomon_classes: user-defined mapping between keras and decomon layers classes + **kwargs: keyword-args passed to decomon layers + + Returns: + the converted decomon layer + + The decomon class is chosen as followed (from higher to lower priority): + - user-defined `mapping_keras2decomon_classes`, + - default mapping `default_mapping_keras2decomon_classes`, + - using keras class name, by adding a "Decomon" prefix, thanks to `default_mapping_kerasname2decomonclass`. + + """ + # Choose the corresponding decomon class. User mapping -> default mapping -> name. + keras_class = type(layer) + decomon_class = None + if mapping_keras2decomon_classes is not None: + decomon_class = mapping_keras2decomon_classes.get(keras_class, None) + if decomon_class is None: + decomon_class = default_mapping_keras2decomon_classes.get(keras_class, None) + if decomon_class is None: + logger.warning( + f"Keras layer {layer} not in user-defined nor default mapping: " + f"using class name to deduce the proper decomon class to use." + ) + decomon_class = default_mapping_kerasname2decomonclass.get(keras_class.__name__, None) + if decomon_class is None: + raise NotImplementedError(f"The decomon version of {keras_class} is not yet implemented.") + + return decomon_class( + layer=layer, + perturbation_domain=perturbation_domain, + ibp=ibp, + affine=affine, + propagation=propagation, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + slope=slope, + **kwargs, + ) diff --git a/tests/test_to_decomon.py b/tests/test_to_decomon.py new file mode 100644 index 00000000..744fd477 --- /dev/null +++ b/tests/test_to_decomon.py @@ -0,0 +1,184 @@ +import logging +from importlib import reload +from typing import Any, Optional + +import pytest +from keras.layers import Activation, Dense, Input, Layer + +import decomon.layers +import decomon.layers.convert +from decomon.core import BoxDomain, PerturbationDomain, Propagation, Slope +from decomon.layers import DecomonActivation, DecomonDense, DecomonLayer +from decomon.layers.convert import to_decomon + +# Add a new class in decomon.layers namespace for testing conversion by name +# We must do it *before* importing decomon.layers.convert +decomon.layers.DecomonToto = DecomonDense +reload(decomon.layers.convert) +from decomon.layers.convert import to_decomon + + +class Toto(Dense): + ... + + +class MyBoxDomain(BoxDomain): + ... + + +class MyDenseDecomonLayer(DecomonLayer): + def __init__( + self, + layer: Layer, + perturbation_domain: Optional[PerturbationDomain] = None, + ibp: bool = True, + affine: bool = True, + propagation: Propagation = Propagation.FORWARD, + model_input_shape: Optional[tuple[int, ...]] = None, + model_output_shape: Optional[tuple[int, ...]] = None, + my_super_attribute: float = 0.0, + **kwargs: Any, + ): + super().__init__( + layer, perturbation_domain, ibp, affine, propagation, model_input_shape, model_output_shape, **kwargs + ) + self.my_super_attribute = my_super_attribute + + +class MyKerasDenseLayer(Dense): + ... + + +ibp = True +affine = False +propagation = Propagation.BACKWARD +perturbation_domain = MyBoxDomain() +slope = Slope.Z_SLOPE +model_input_shape = (1,) +model_output_shape = (1,) + + +def test_to_decomon_userdefined(): + mymapping = {Dense: MyDenseDecomonLayer} + my_super_attribute = 123.0 + layer = Dense(3) + layer(Input((1,))) + decomon_layer = to_decomon( + layer=layer, + perturbation_domain=perturbation_domain, + ib=ibp, + affine=affine, + propagation=propagation, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + slope=slope, + my_super_attribute=my_super_attribute, + mapping_keras2decomon_classes=mymapping, + ) + assert isinstance(decomon_layer, MyDenseDecomonLayer) + + assert decomon_layer.ibp == ibp + assert decomon_layer.affine == affine + assert decomon_layer.propagation == propagation + assert decomon_layer.perturbation_domain == perturbation_domain + assert decomon_layer.model_input_shape == model_input_shape + assert decomon_layer.model_output_shape == model_output_shape + + assert decomon_layer.my_super_attribute == my_super_attribute + + +def test_to_decomon_default(caplog): + layer = Dense(3) + layer(Input((1,))) + + with caplog.at_level(logging.WARNING): + decomon_layer = to_decomon( + layer=layer, + perturbation_domain=perturbation_domain, + ib=ibp, + affine=affine, + propagation=propagation, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + slope=slope, + ) + + assert isinstance(decomon_layer, DecomonDense) + + assert decomon_layer.ibp == ibp + assert decomon_layer.affine == affine + assert decomon_layer.propagation == propagation + assert decomon_layer.perturbation_domain == perturbation_domain + assert decomon_layer.model_input_shape == model_input_shape + assert decomon_layer.model_output_shape == model_output_shape + + assert "using class name" not in caplog.text + + +def test_to_decomon_slope(): + layer = Activation(activation=None) + layer(Input((1,))) + decomon_layer = to_decomon( + layer=layer, + perturbation_domain=perturbation_domain, + ib=ibp, + affine=affine, + propagation=propagation, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + slope=slope, + ) + assert isinstance(decomon_layer, DecomonActivation) + + assert decomon_layer.ibp == ibp + assert decomon_layer.affine == affine + assert decomon_layer.propagation == propagation + assert decomon_layer.perturbation_domain == perturbation_domain + assert decomon_layer.model_input_shape == model_input_shape + assert decomon_layer.model_output_shape == model_output_shape + + assert decomon_layer.slope == slope + + +def test_to_decomon_by_name(caplog): + layer = Toto(3) + layer(Input((1,))) + + with caplog.at_level(logging.WARNING): + decomon_layer = to_decomon( + layer=layer, + perturbation_domain=perturbation_domain, + ib=ibp, + affine=affine, + propagation=propagation, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + slope=slope, + ) + + assert isinstance(decomon_layer, DecomonDense) + + assert decomon_layer.ibp == ibp + assert decomon_layer.affine == affine + assert decomon_layer.propagation == propagation + assert decomon_layer.perturbation_domain == perturbation_domain + assert decomon_layer.model_input_shape == model_input_shape + assert decomon_layer.model_output_shape == model_output_shape + + assert "using class name" in caplog.text + + +def test_to_decomon_nok(): + layer = MyKerasDenseLayer(3) + layer(Input((1,))) + with pytest.raises(NotImplementedError): + to_decomon( + layer=layer, + perturbation_domain=perturbation_domain, + ib=ibp, + affine=affine, + propagation=propagation, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + slope=slope, + ) From b7472bcc3780245d340e079a61f91cec522e3044 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 19 Feb 2024 14:52:13 +0100 Subject: [PATCH 050/101] Rename `model_inputs` into `perturbation_domain_inputs` Indeed this is not directly the keras model inputs but rather the tensor(s) defining the perturbation on the keras model inputs. --- src/decomon/core.py | 38 ++++++------ src/decomon/layers/activations/activation.py | 10 ++-- src/decomon/layers/layer.py | 39 ++++++------ src/decomon/layers/merging/base_merge.py | 12 ++-- tests/conftest.py | 62 ++++++++++---------- tests/test_decomon_layer.py | 8 ++- 6 files changed, 89 insertions(+), 80 deletions(-) diff --git a/src/decomon/core.py b/src/decomon/core.py index 2911e41d..fbb59f1b 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -243,7 +243,7 @@ def __init__( else: self.layer_input_shape = layer_input_shape - def needs_keras_model_inputs(self) -> bool: + def needs_perturbation_domain_inputs(self) -> bool: """Specify if decomon inputs should integrate keras model inputs.""" return self.propagation == Propagation.FORWARD and self.affine @@ -271,7 +271,7 @@ def nb_input_tensors(self) -> int: # affine nb += 4 # model inputs - if self.needs_keras_model_inputs(): + if self.needs_perturbation_domain_inputs(): nb += 1 else: # forward # ibp @@ -281,7 +281,7 @@ def nb_input_tensors(self) -> int: if self.affine: nb += 4 * self.nb_keras_inputs # model inputs - if self.needs_keras_model_inputs(): + if self.needs_perturbation_domain_inputs(): nb += 1 return nb @@ -344,14 +344,14 @@ def split_inputs( inputs: flattened decomon inputs, as seen by `DecomonLayer.call()`. Returns: - affine_bounds_to_propagate, constant_oracle_bounds, model_inputs: + affine_bounds_to_propagate, constant_oracle_bounds, perturbation_domain_inputs: each one can be empty if not relevant, and according to propagation mode and merging status, it will be list of tensors or list of list of tensors. More details: - non-merging case: - inputs = affine_bounds_to_propagate + constant_oracle_bounds + model_inputs + inputs = affine_bounds_to_propagate + constant_oracle_bounds + perturbation_domain_inputs - merging case: - forward: k affine bounds to propagate w.r.t. each keras layer input + k constant bounds @@ -359,7 +359,7 @@ def split_inputs( inputs = ( affine_bounds_to_propagate_0 + constant_oracle_bounds_0 + ... + affine_bounds_to_propagate_k + constant_oracle_bounds_k - + model_inputs + + perturbation_domain_inputs ) - backward: only 1 affine bounds to propagate w.r.t keras layer output @@ -368,7 +368,7 @@ def split_inputs( inputs = ( affine_bounds_to_propagate + constant_oracle_bounds_0 + ... + constant_oracle_bounds_k - + model_inputs + + perturbation_domain_inputs ) Note: in case of merging layer + forward, we should not have empty affine bounds @@ -376,12 +376,12 @@ def split_inputs( """ # Remove keras model input - if self.needs_keras_model_inputs(): + if self.needs_perturbation_domain_inputs(): x = inputs[-1] inputs = inputs[:-1] - model_inputs = [x] + perturbation_domain_inputs = [x] else: - model_inputs = [] + perturbation_domain_inputs = [] if self.is_merging_layer: if self.propagation == Propagation.BACKWARD: # expected number of constant bounds @@ -421,7 +421,7 @@ def split_inputs( # (potentially empty if: not backward or not affine or identity affine bounds) affine_bounds_to_propagate = inputs - return affine_bounds_to_propagate, constant_oracle_bounds, model_inputs + return affine_bounds_to_propagate, constant_oracle_bounds, perturbation_domain_inputs def split_input_shape( self, input_shape: list[tuple[Optional[int], ...]] @@ -442,7 +442,7 @@ def split_input_shape( input_shape: flattened decomon inputs, as seen by `DecomonLayer.call()`. Returns: - affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, model_inputs_shape: + affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, perturbation_domain_inputs_shape: each one can be empty if not relevant, and according to propagation mode and merging status, it will be list of shapes or list of list of shapes. @@ -453,7 +453,7 @@ def flatten_inputs( self, affine_bounds_to_propagate: Union[list[Tensor], list[list[Tensor]]], constant_oracle_bounds: Union[list[Tensor], list[list[Tensor]]], - model_inputs: list[Tensor], + perturbation_domain_inputs: list[Tensor], ) -> list[Tensor]: """Flatten decomon inputs. @@ -473,7 +473,7 @@ def flatten_inputs( Returns: flattened inputs - non-merging case: - inputs = affine_bounds_to_propagate + constant_oracle_bounds + model_inputs + inputs = affine_bounds_to_propagate + constant_oracle_bounds + perturbation_domain_inputs - merging case: - forward: k affine bounds to propagate w.r.t. each keras layer input + k constant bounds @@ -481,7 +481,7 @@ def flatten_inputs( inputs = ( affine_bounds_to_propagate_0 + constant_oracle_bounds_0 + ... + affine_bounds_to_propagate_k + constant_oracle_bounds_k - + model_inputs + + perturbation_domain_inputs ) - backward: only 1 affine bounds to propagate w.r.t keras layer output @@ -490,7 +490,7 @@ def flatten_inputs( inputs = ( affine_bounds_to_propagate + constant_oracle_bounds_0 + ... + constant_oracle_bounds_k - + model_inputs + + perturbation_domain_inputs ) """ @@ -499,7 +499,7 @@ def flatten_inputs( flattened_constant_oracle_bounds = [ t for constant_oracle_bounds_i in constant_oracle_bounds for t in constant_oracle_bounds_i ] - return affine_bounds_to_propagate + flattened_constant_oracle_bounds + model_inputs + return affine_bounds_to_propagate + flattened_constant_oracle_bounds + perturbation_domain_inputs else: # forward bounds_by_keras_input = [ affine_bounds_to_propagate_i + constant_oracle_bounds_i @@ -510,9 +510,9 @@ def flatten_inputs( flattened_bounds_by_keras_input = [ t for bounds_by_keras_input_i in bounds_by_keras_input for t in bounds_by_keras_input_i ] - return flattened_bounds_by_keras_input + model_inputs + return flattened_bounds_by_keras_input + perturbation_domain_inputs else: - return affine_bounds_to_propagate + constant_oracle_bounds + model_inputs + return affine_bounds_to_propagate + constant_oracle_bounds + perturbation_domain_inputs def split_outputs(self, outputs: list[Tensor]) -> tuple[Union[list[Tensor], list[list[Tensor]]], list[Tensor]]: """Split decomon inputs. diff --git a/src/decomon/layers/activations/activation.py b/src/decomon/layers/activations/activation.py index 0c42d622..cd8e7e88 100644 --- a/src/decomon/layers/activations/activation.py +++ b/src/decomon/layers/activations/activation.py @@ -144,9 +144,11 @@ class DecomonLinear(DecomonBaseActivation): linear = True def call(self, inputs: list[Tensor]) -> list[Tensor]: - affine_bounds_to_propagate, constant_oracle_bounds, model_inputs = self.inputs_outputs_spec.split_inputs( - inputs=inputs - ) + ( + affine_bounds_to_propagate, + constant_oracle_bounds, + perturbation_domain_inputs, + ) = self.inputs_outputs_spec.split_inputs(inputs=inputs) return self.inputs_outputs_spec.flatten_outputs( affine_bounds_propagated=affine_bounds_to_propagate, constant_bounds_propagated=constant_oracle_bounds ) @@ -161,7 +163,7 @@ def compute_output_shape( ( affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, - model_inputs_shape, + perturbation_domain_inputs_shape, ) = self.inputs_outputs_spec.split_input_shape(input_shape=input_shape) return self.inputs_outputs_spec.flatten_outputs_shape( affine_bounds_propagated_shape=affine_bounds_to_propagate_shape, diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index b0ab2d1a..ccae7732 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -400,14 +400,17 @@ def backward_affine_propagate( ) def get_forward_oracle( - self, input_affine_bounds: list[Tensor], input_constant_bounds: list[Tensor], model_inputs: list[Tensor] + self, + input_affine_bounds: list[Tensor], + input_constant_bounds: list[Tensor], + perturbation_domain_inputs: list[Tensor], ) -> list[Tensor]: """Get constant oracle bounds on underlying keras layer input from forward input bounds. Args: input_affine_bounds: affine bounds on keras layer input w.r.t model input . Can be empty if not in affine mode. input_constant_bounds: ibp constant bounds on keras layer input. Can be empty if not in ibp mode. - model_inputs: underlying keras model input, wrapped in a list. Necessary only in affine mode, else empty. + perturbation_domain_inputs: perturbation domain input, wrapped in a list. Necessary only in affine mode, else empty. Returns: constant bounds on keras layer input deduced from forward input bounds @@ -426,9 +429,9 @@ def get_forward_oracle( return input_constant_bounds elif self.affine: - if len(model_inputs) == 0: + if len(perturbation_domain_inputs) == 0: raise RuntimeError("keras model input is necessary for get_forward_oracle() in affine mode.") - x = model_inputs[0] + x = perturbation_domain_inputs[0] if len(input_affine_bounds) == 0: # special case: empty affine bounds => identity bounds l_affine = self.perturbation_domain.get_lower_x(x) @@ -446,7 +449,7 @@ def call_forward( self, affine_bounds_to_propagate: list[Tensor], input_bounds_to_propagate: list[Tensor], - model_inputs: list[Tensor], + perturbation_domain_inputs: list[Tensor], ) -> tuple[list[Tensor], list[Tensor]]: """Propagate forward affine and constant bounds through the layer. @@ -455,7 +458,7 @@ def call_forward( Can be empty if not in affine mode. Can also be empty in case of identity affine bounds => we simply return layer affine bounds. input_bounds_to_propagate: ibp constant bounds on keras layer input. Can be empty if not in ibp mode. - model_inputs: underlying keras model input, wrapped in a list. Necessary only in affine mode, else empty. + perturbation_domain_inputs: perturbation domain input, wrapped in a list. Necessary only in affine mode, else empty. Returns: output_affine_bounds, output_constant_bounds: affine and constant bounds on the underlying keras layer output @@ -482,7 +485,7 @@ def call_forward( input_constant_bounds = self.get_forward_oracle( input_affine_bounds=affine_bounds_to_propagate, input_constant_bounds=input_bounds_to_propagate, - model_inputs=model_inputs, + perturbation_domain_inputs=perturbation_domain_inputs, ) else: input_constant_bounds = [] @@ -497,9 +500,9 @@ def call_forward( # Tighten constant bounds in hybrid mode (ibp+affine) if self.ibp and self.affine: - if len(model_inputs) == 0: + if len(perturbation_domain_inputs) == 0: raise RuntimeError("keras model input is necessary for call_forward() in affine mode.") - x = model_inputs[0] + x = perturbation_domain_inputs[0] l_ibp, u_ibp = output_constant_bounds w_l, b_l, w_u, b_u = output_affine_bounds l_affine = self.perturbation_domain.get_lower(x, w_l, b_l) @@ -523,15 +526,15 @@ def call(self, inputs: list[Tensor]) -> list[Tensor]: """Propagate bounds in the specified direction `self.propagation`. Args: - inputs: concatenation of affine_bounds_to_propagate + constant_oracle_bounds + keras_model_inputs with + inputs: concatenation of affine_bounds_to_propagate + constant_oracle_bounds + perturbation_domain_inputs with - affine_bounds_to_propagate: affine bounds to propagate. Can be empty in forward direction if self.affine is False. Can also be empty in case of identity affine bounds => we simply return layer affine bounds. - constant_oracle_bounds: - in forward direction, the ibp bounds (empty if self.ibp is False); - in backward direction, the oracle constant bounds on keras inputs (never empty) - - keras_model_inputs: the tensors defining the underlying keras model input perturbation. - - in forward direction when self.affine is True: one tensor x whose shape is given by `self.perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape)` + - perturbation_domain_inputs: the tensor defining the underlying keras model input perturbation, wrapped in a list. + - in forward direction when self.affine is True: a list with a single tensor x whose shape is given by `self.perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape)` with `model_input_shape=model.input.shape[1:]` if `model` is the underlying keras model to analyse - else: empty list @@ -541,14 +544,16 @@ def call(self, inputs: list[Tensor]) -> list[Tensor]: - in backward direction: affine_bounds_propagated """ - affine_bounds_to_propagate, constant_oracle_bounds, model_inputs = self.inputs_outputs_spec.split_inputs( - inputs=inputs - ) + ( + affine_bounds_to_propagate, + constant_oracle_bounds, + perturbation_domain_inputs, + ) = self.inputs_outputs_spec.split_inputs(inputs=inputs) if self.propagation == Propagation.FORWARD: # forward affine_bounds_propagated, constant_bounds_propagated = self.call_forward( affine_bounds_to_propagate=affine_bounds_to_propagate, input_bounds_to_propagate=constant_oracle_bounds, - model_inputs=model_inputs, + perturbation_domain_inputs=perturbation_domain_inputs, ) return self.inputs_outputs_spec.flatten_outputs(affine_bounds_propagated, constant_bounds_propagated) @@ -568,7 +573,7 @@ def compute_output_shape( ( affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, - model_inputs_shape, + perturbation_domain_inputs_shape, ) = self.inputs_outputs_spec.split_input_shape(input_shape=input_shape) if self.propagation == Propagation.FORWARD: if self.ibp: diff --git a/src/decomon/layers/merging/base_merge.py b/src/decomon/layers/merging/base_merge.py index 78329fcf..93654fde 100644 --- a/src/decomon/layers/merging/base_merge.py +++ b/src/decomon/layers/merging/base_merge.py @@ -357,14 +357,14 @@ def get_forward_oracle( self, input_affine_bounds: list[list[Tensor]], input_constant_bounds: list[list[Tensor]], - model_inputs: list[Tensor], + perturbation_domain_inputs: list[Tensor], ) -> list[list[Tensor]]: """Get constant oracle bounds on underlying keras layer input from forward input bounds. Args: input_affine_bounds: affine bounds on each keras layer input w.r.t model input . Can be empty if not in affine mode. input_constant_bounds: ibp constant bounds on each keras layer input. Can be empty if not in ibp mode. - model_inputs: underlying keras model input, wrapped in a list. Necessary only in affine mode, else empty. + perturbation_domain_inputs: perturbation domain input, wrapped in a list. Necessary only in affine mode, else empty. Returns: constant bounds on each keras layer input deduced from forward input bounds @@ -383,9 +383,9 @@ def get_forward_oracle( return input_constant_bounds elif self.affine: - if len(model_inputs) == 0: + if len(perturbation_domain_inputs) == 0: raise RuntimeError("keras model input is necessary for get_forward_oracle() in affine mode.") - x = model_inputs[0] + x = perturbation_domain_inputs[0] constant_bounds = [] for input_affine_bounds_i in input_affine_bounds: if len(input_affine_bounds_i) == 0: @@ -442,14 +442,14 @@ def call(self, inputs: list[Tensor]) -> list[Tensor]: inputs = ( affine_bounds_to_propagate_0 + constant_oracle_bounds_0 + ... + affine_bounds_to_propagate_k + constant_oracle_bounds_k - + model_inputs + + perturbation_domain_inputs ) with - affine_bounds_to_propagate_* empty when affine is False; (never to express identity bounds, as it would be impossible to separate bounds in flattened inputs) - constant_oracle_bounds_* empty when ibp is False; - - model_inputs: + - perturbation_domain_inputs: - if affine: the tensor defining the underlying keras model input perturbation, wrapped in a list; - else: empty diff --git a/tests/conftest.py b/tests/conftest.py index 9dbd2ef1..9d570126 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -158,10 +158,10 @@ def get_decomon_input_shapes( model_input_shape=model_input_shape, model_output_shape=model_output_shape, ) - if inputs_outputs_spec.needs_keras_model_inputs(): - model_inputs_shape = [perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape)] + if inputs_outputs_spec.needs_perturbation_domain_inputs(): + perturbation_domain_inputs_shape = [perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape)] else: - model_inputs_shape = [] + perturbation_domain_inputs_shape = [] if affine and not empty: if propagation == Propagation.FORWARD: @@ -182,7 +182,7 @@ def get_decomon_input_shapes( else: constant_oracle_bounds_shape = [] - return affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, model_inputs_shape + return affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, perturbation_domain_inputs_shape @staticmethod def get_decomon_symbolic_inputs( @@ -219,7 +219,7 @@ def get_decomon_symbolic_inputs( ( affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, - model_inputs_shape, + perturbation_domain_inputs_shape, ) = Helpers.get_decomon_input_shapes( model_input_shape, model_output_shape, @@ -233,7 +233,7 @@ def get_decomon_symbolic_inputs( diag=diag, nobatch=nobatch, ) - model_inputs = [Input(shape, dtype=dtype) for shape in model_inputs_shape] + perturbation_domain_inputs = [Input(shape, dtype=dtype) for shape in perturbation_domain_inputs_shape] constant_oracle_bounds = [Input(shape, dtype=dtype) for shape in constant_oracle_bounds_shape] if nobatch: affine_bounds_to_propagate = [ @@ -253,7 +253,7 @@ def get_decomon_symbolic_inputs( return inputs_outputs_spec.flatten_inputs( affine_bounds_to_propagate=affine_bounds_to_propagate, constant_oracle_bounds=constant_oracle_bounds, - model_inputs=model_inputs, + perturbation_domain_inputs=perturbation_domain_inputs, ) @staticmethod @@ -305,14 +305,14 @@ def generate_simple_decomon_layer_inputs_from_keras_input( model_output_shape=model_output_shape, ) - if inputs_outputs_spec.needs_keras_model_inputs(): + if inputs_outputs_spec.needs_perturbation_domain_inputs(): if isinstance(perturbation_domain, BoxDomain): x = K.repeat(keras_input[:, None], 2, axis=1) else: raise NotImplementedError - model_inputs = [x] + perturbation_domain_inputs = [x] else: - model_inputs = [] + perturbation_domain_inputs = [] if affine and not empty: batchsize = keras_input.shape[0] @@ -349,7 +349,7 @@ def generate_simple_decomon_layer_inputs_from_keras_input( return inputs_outputs_spec.flatten_inputs( affine_bounds_to_propagate=affine_bounds_to_propagate, constant_oracle_bounds=constant_oracle_bounds, - model_inputs=model_inputs, + perturbation_domain_inputs=perturbation_domain_inputs, ) @staticmethod @@ -364,14 +364,14 @@ def generate_merging_decomon_input_from_single_decomon_inputs( model_input_shape=tuple(), model_output_shape=tuple(), ) - affine_bounds_to_propagate, constant_oracle_bounds, model_inputs = [], [], [] + affine_bounds_to_propagate, constant_oracle_bounds, perturbation_domain_inputs = [], [], [] for decomon_input in decomon_inputs: ( affine_bounds_to_propagate_i, constant_oracle_bounds_i, - model_inputs_i, + perturbation_domain_inputs_i, ) = inputs_outputs_spec_single.split_inputs(decomon_input) - model_inputs = model_inputs_i + perturbation_domain_inputs = perturbation_domain_inputs_i if propagation == Propagation.FORWARD: affine_bounds_to_propagate.append(affine_bounds_to_propagate_i) else: @@ -390,7 +390,7 @@ def generate_merging_decomon_input_from_single_decomon_inputs( return inputs_outputs_spec_merging.flatten_inputs( affine_bounds_to_propagate=affine_bounds_to_propagate, constant_oracle_bounds=constant_oracle_bounds, - model_inputs=model_inputs, + perturbation_domain_inputs=perturbation_domain_inputs, ) @staticmethod @@ -1056,18 +1056,18 @@ def decomon_symbolic_input_fn(output_shape): else: constant_oracle_bounds = [] - if inputs_outputs_spec.needs_keras_model_inputs(): + if inputs_outputs_spec.needs_perturbation_domain_inputs(): if isinstance(perturbation_domain, BoxDomain): - model_inputs = [z] + perturbation_domain_inputs = [z] else: raise NotImplementedError else: - model_inputs = [] + perturbation_domain_inputs = [] return inputs_outputs_spec.flatten_inputs( affine_bounds_to_propagate=affine_bounds_to_propagate, constant_oracle_bounds=constant_oracle_bounds, - model_inputs=model_inputs, + perturbation_domain_inputs=perturbation_domain_inputs, ) def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): @@ -1095,18 +1095,18 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): else: constant_oracle_bounds = [] - if inputs_outputs_spec.needs_keras_model_inputs(): + if inputs_outputs_spec.needs_perturbation_domain_inputs(): if isinstance(perturbation_domain, BoxDomain): - model_inputs = [K.convert_to_tensor(z)] + perturbation_domain_inputs = [K.convert_to_tensor(z)] else: raise NotImplementedError else: - model_inputs = [] + perturbation_domain_inputs = [] return inputs_outputs_spec.flatten_inputs( affine_bounds_to_propagate=affine_bounds_to_propagate, constant_oracle_bounds=constant_oracle_bounds, - model_inputs=model_inputs, + perturbation_domain_inputs=perturbation_domain_inputs, ) else: # backward @@ -1131,13 +1131,13 @@ def decomon_symbolic_input_fn(output_shape): else: constant_oracle_bounds = [] - if inputs_outputs_spec.needs_keras_model_inputs(): + if inputs_outputs_spec.needs_perturbation_domain_inputs(): if isinstance(perturbation_domain, BoxDomain): - model_inputs = [z] + perturbation_domain_inputs = [z] else: raise NotImplementedError else: - model_inputs = [] + perturbation_domain_inputs = [] # take identity affine bounds if affine: @@ -1161,7 +1161,7 @@ def decomon_symbolic_input_fn(output_shape): return inputs_outputs_spec.flatten_inputs( affine_bounds_to_propagate=affine_bounds_to_propagate, constant_oracle_bounds=constant_oracle_bounds, - model_inputs=model_inputs, + perturbation_domain_inputs=perturbation_domain_inputs, ) def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): @@ -1184,13 +1184,13 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): else: constant_oracle_bounds = [] - if inputs_outputs_spec.needs_keras_model_inputs(): + if inputs_outputs_spec.needs_perturbation_domain_inputs(): if isinstance(perturbation_domain, BoxDomain): - model_inputs = [K.convert_to_tensor(z)] + perturbation_domain_inputs = [K.convert_to_tensor(z)] else: raise NotImplementedError else: - model_inputs = [] + perturbation_domain_inputs = [] #  take identity affine bounds if affine: @@ -1212,7 +1212,7 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): return inputs_outputs_spec.flatten_inputs( affine_bounds_to_propagate=affine_bounds_to_propagate, constant_oracle_bounds=constant_oracle_bounds, - model_inputs=model_inputs, + perturbation_domain_inputs=perturbation_domain_inputs, ) return ( diff --git a/tests/test_decomon_layer.py b/tests/test_decomon_layer.py index cc43c7aa..f141acd8 100644 --- a/tests/test_decomon_layer.py +++ b/tests/test_decomon_layer.py @@ -122,11 +122,13 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper ( affine_bounds_to_propagate, constant_oracle_bounds, - model_inputs, + perturbation_domain_inputs, ) = linear_decomon_layer.inputs_outputs_spec.split_inputs(decomon_inputs) # actual (random) tensors + expected output shapes - model_inputs_val = [helpers.generate_random_tensor(x.shape[1:], batchsize=batchsize) for x in model_inputs] + perturbation_domain_inputs_val = [ + helpers.generate_random_tensor(x.shape[1:], batchsize=batchsize) for x in perturbation_domain_inputs + ] if affine: if propagation == Propagation.FORWARD: @@ -156,7 +158,7 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper decomon_inputs_val = linear_decomon_layer.inputs_outputs_spec.flatten_inputs( affine_bounds_to_propagate=affine_bounds_to_propagate_val, constant_oracle_bounds=constant_oracle_bounds_val, - model_inputs=model_inputs_val, + perturbation_domain_inputs=perturbation_domain_inputs_val, ) if propagation == Propagation.FORWARD: From 3f33f5423f62b037ee4e5c26bd25c83a618d80ed Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 19 Feb 2024 15:58:47 +0100 Subject: [PATCH 051/101] Derive a tensor of keras model input shape from x Add a new method in PerturbationDomain that generated a tensor of the same shape the input of the underlying keras model from the perturabtion domain input. --- src/decomon/core.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/decomon/core.py b/src/decomon/core.py index fbb59f1b..6fcc712d 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -54,6 +54,21 @@ def get_config(self) -> dict[str, Any]: "opt_option": self.opt_option, } + def get_kerasinputlike_from_x(self, x: Tensor) -> Tensor: + """Get tensor of same shape as keras model input, from perturbation domain input x + + Args: + x: perturbation domain input + + Returns: + tensor of same shape as keras model input + + """ + if self.get_nb_x_components() == 1: + return x + else: + return x[:, 0] + def get_x_input_shape_wo_batchsize(self, original_input_shape: tuple[int, ...]) -> tuple[int, ...]: n_comp_x = self.get_nb_x_components() if n_comp_x == 1: From a1e2177d52e043dc149ba34ebafc55b7a05386f8 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 22 Feb 2024 23:52:42 +0100 Subject: [PATCH 052/101] Add fixture to generate toy models (from a given input_shape x dtype) --- tests/conftest.py | 342 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 340 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9d570126..9eb906f7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,8 +5,8 @@ import keras.ops as K import numpy as np import pytest -from keras import KerasTensor, Model -from keras.layers import Input +from keras import KerasTensor, Model, Sequential +from keras.layers import Activation, Add, Conv2D, Dense, Flatten, Input from pytest_cases import ( fixture, fixture_union, @@ -968,6 +968,282 @@ def predict_on_small_numpy( else: return K.convert_to_numpy(output_tensors) + @staticmethod + def toy_network_tutorial( + input_shape: tuple[int, ...] = (1,), dtype: Optional[str] = None, activation: Optional[str] = "relu" + ) -> Model: + if dtype is None: + dtype = keras_config.floatx() + layers = [] + layers.append(Input(input_shape, dtype=dtype)) + layers.append(Dense(100, dtype=dtype)) + if activation is not None: + layers.append(Activation(activation, dtype=dtype)) + layers.append(Dense(100, dtype=dtype)) + layers.append(Dense(1, activation="linear", dtype=dtype)) + model = Sequential(layers) + return model + + @staticmethod + def toy_network_submodel( + input_shape: tuple[int, ...] = (1,), dtype: Optional[str] = None, activation: Optional[str] = "relu" + ) -> Model: + if dtype is None: + dtype = keras_config.floatx() + submodel_input_shape = input_shape[:-1] + (100,) + layers = [] + layers.append(Input(input_shape, dtype=dtype)) + layers.append(Dense(100, dtype=dtype)) + if activation is not None: + layers.append(Activation(activation, dtype=dtype)) + layers.append(Helpers.toy_network_tutorial(submodel_input_shape, dtype=dtype, activation=activation)) + layers.append(Dense(100, dtype=dtype)) + layers.append(Dense(1, activation="linear", dtype=dtype)) + model = Sequential(layers) + return model + + @staticmethod + def toy_network_add( + input_shape: tuple[int, ...] = (1,), dtype: Optional[str] = None, activation: Optional[str] = "relu" + ) -> Model: + if dtype is None: + dtype = keras_config.floatx() + input_tensor = Input(input_shape, dtype=dtype) + output = Dense(100, dtype=dtype)(input_tensor) + if activation is not None: + output = Activation(activation, dtype=dtype)(output) + output = Add()([output, output]) + output = Dense(100, dtype=dtype)(output) + if activation is not None: + output = Activation(activation, dtype=dtype)(output) + model = Model(inputs=input_tensor, outputs=output) + return model + + @staticmethod + def toy_network_add_monolayer( + input_shape: tuple[int, ...] = (1,), dtype: Optional[str] = None, activation: Optional[str] = "relu" + ) -> Model: + if dtype is None: + dtype = keras_config.floatx() + input_tensor = Input(input_shape, dtype=dtype) + output = Dense(100, dtype=dtype)(input_tensor) + if activation is not None: + output = Activation(activation, dtype=dtype)(output) + output = Add()([output]) + output = Dense(100, dtype=dtype)(output) + if activation is not None: + output = Activation(activation, dtype=dtype)(output) + model = Model(inputs=input_tensor, outputs=output) + return model + + @staticmethod + def toy_network_tutorial_with_embedded_activation(input_shape: tuple[int, ...] = (1,), dtype: Optional[str] = None): + if dtype is None: + dtype = keras_config.floatx() + layers = [] + layers.append(Input(input_shape, dtype=dtype)) + layers.append(Dense(100, activation="relu", dtype=dtype)) + layers.append(Dense(100, dtype=dtype)) + layers.append(Dense(1, activation="linear", dtype=dtype)) + model = Sequential(layers) + return model + + @staticmethod + def toy_embedded_sequential(input_shape: tuple[int, ...] = (1,), dtype: Optional[str] = None): + if dtype is None: + dtype = keras_config.floatx() + layers = [] + units = 10 + submodel_input_shape = input_shape[:-1] + (units,) + layers.append(Input(input_shape, dtype=dtype)) + layers.append(Dense(units, activation="relu", dtype=dtype)) + layers.append( + Helpers.dense_NN( + dtype=dtype, + archi=[2, 3, 2], + sequential=True, + input_shape=submodel_input_shape, + activation="relu", + use_bias=False, + ) + ) + layers.append(Dense(1, activation="linear", dtype=dtype)) + model = Sequential(layers) + return model + + @staticmethod + def dense_NN( + archi, sequential, activation, use_bias, input_shape: tuple[int, ...] = (1,), dtype: Optional[str] = None + ): + if dtype is None: + dtype = keras_config.floatx() + layers = [Input(input_shape, dtype=dtype)] + layers += [Dense(n_i, use_bias=use_bias, activation=activation, dtype=dtype) for n_i in archi] + + if sequential: + return Sequential(layers) + else: + input = layers[0] + output = input + for layer_ in layers[1:]: + output = layer_(output) + return Model(input, output) + + @staticmethod + def toy_struct_v0( + archi, activation, use_bias, merge_op=Add, input_shape: tuple[int, ...] = (1,), dtype: Optional[str] = None + ): + if dtype is None: + dtype = keras_config.floatx() + nnet_0 = Helpers.dense_NN( + input_shape=input_shape, + archi=archi, + sequential=False, + activation=activation, + use_bias=use_bias, + dtype=dtype, + ) + nnet_1 = Dense(archi[-1], use_bias=use_bias, activation="linear", name="toto", dtype=dtype) + + x = Input(input_shape, dtype=dtype) + h_0 = nnet_0(x) + h_1 = nnet_1(x) + + y = merge_op(dtype=dtype)([h_0, h_1]) + + return Model(x, y) + + @staticmethod + def toy_struct_v1( + archi, + sequential, + activation, + use_bias, + merge_op=Add, + input_shape: tuple[int, ...] = (1,), + dtype: Optional[str] = None, + ): + if dtype is None: + dtype = keras_config.floatx() + nnet_0 = Helpers.dense_NN( + input_shape=input_shape, + archi=archi, + sequential=sequential, + activation=activation, + use_bias=use_bias, + dtype=dtype, + ) + + x = Input(input_shape, dtype=dtype) + h_0 = nnet_0(x) + h_1 = nnet_0(x) + y = merge_op(dtype=dtype)([h_0, h_1]) + + return Model(x, y) + + @staticmethod + def toy_struct_v2( + archi, + sequential, + activation, + use_bias, + merge_op=Add, + input_shape: tuple[int, ...] = (1,), + dtype: Optional[str] = None, + ): + if dtype is None: + dtype = keras_config.floatx() + nnet_0 = Helpers.dense_NN( + input_shape=input_shape, + archi=archi, + sequential=sequential, + activation=activation, + use_bias=use_bias, + dtype=dtype, + ) + nnet_1 = Helpers.dense_NN( + input_shape=input_shape, + archi=archi, + sequential=sequential, + activation=activation, + use_bias=use_bias, + dtype=dtype, + ) + nnet_2 = Dense(archi[-1], use_bias=use_bias, activation="linear", dtype=dtype) + + x = Input(input_shape, dtype=dtype) + nnet_0(x) + nnet_1(x) + nnet_1.set_weights([-p for p in nnet_0.get_weights()]) # be sure that the produced output will differ + h_0 = nnet_2(nnet_0(x)) + h_1 = nnet_2(nnet_1(x)) + y = merge_op(dtype=dtype)([h_0, h_1]) + + return Model(x, y) + + @staticmethod + def toy_struct_cnn(input_shape: tuple[int, ...] = (6, 6, 2), dtype: Optional[str] = None): + if dtype is None: + dtype = keras_config.floatx() + layers = [ + Input(input_shape), + Conv2D( + 10, + kernel_size=(3, 3), + activation="relu", + data_format="channels_last", + dtype=dtype, + ), + Flatten(dtype=dtype), + Dense(1, dtype=dtype), + ] + return Sequential(layers) + + @staticmethod + def toy_model(model_name, input_shape: tuple[int, ...] = (1,), dtype: Optional[str] = None): + if dtype is None: + dtype = keras_config.floatx() + if model_name == "tutorial": + return Helpers.toy_network_tutorial(input_shape=input_shape, dtype=dtype) + elif model_name == "tutorial_activation_embedded": + return Helpers.toy_network_tutorial_with_embedded_activation(input_shape=input_shape, dtype=dtype) + elif model_name == "add": + return Helpers.toy_network_add(input_shape=input_shape, dtype=dtype) + elif model_name == "merge_v0": + return Helpers.toy_struct_v0( + dtype=dtype, input_shape=input_shape, archi=[2, 3, 2], activation="relu", use_bias=True + ) + elif model_name == "merge_v1": + return Helpers.toy_struct_v1( + dtype=dtype, + input_shape=input_shape, + archi=[2, 3, 2], + activation="relu", + use_bias=True, + sequential=False, + ) + elif model_name == "merge_v1_seq": + return Helpers.toy_struct_v1( + dtype=dtype, input_shape=input_shape, archi=[2, 3, 2], activation="relu", use_bias=True, sequential=True + ) + elif model_name == "merge_v2": + return Helpers.toy_struct_v2( + dtype=dtype, + input_shape=input_shape, + archi=[2, 3, 2], + activation="relu", + use_bias=True, + sequential=False, + ) + elif model_name == "cnn": + return Helpers.toy_struct_cnn(input_shape=input_shape, dtype=dtype) + elif model_name == "embedded_model_v1": + return Helpers.toy_embedded_sequential(input_shape=input_shape, dtype=dtype) + elif model_name == "embedded_model_v2": + return Helpers.toy_network_submodel(input_shape=input_shape, dtype=dtype) + else: + raise ValueError(f"model_name {model_name} unknown") + @pytest.fixture def helpers(): @@ -1282,3 +1558,65 @@ def standard_layer_input_functions_multid(data_format, ibp, affine, propagation, "simple_keras_symbolic_model_input_fn, simple_keras_symbolic_layer_input_fn, simple_decomon_symbolic_input_fn, simple_keras_model_input_fn, simple_keras_layer_input_fn, simple_decomon_input_fn, simple_equal_bounds", simple_layer_input_functions, ) + +# keras toy models +toy_model_name = param_fixture( + "toy_model_name", + [ + "tutorial", + "tutorial_linear", + "tutorial_activation_embedded", + "add", + "add_linear", + "merge_v0", + "merge_v1", + "merge_v1_seq", + "merge_v2", + "cnn", + "embedded_model_v1", + "embedded_model_v2", + ], +) + + +@fixture +def toy_model_fn(toy_model_name, helpers): + """Return a function generating a keras model from (input_shape, dtype).""" + if toy_model_name == "tutorial": + return Helpers.toy_network_tutorial + elif toy_model_name == "tutorial_linear": + return lambda input_shape, dtype=None: Helpers.toy_network_tutorial( + input_shape=input_shape, dtype=dtype, activation=None + ) + elif toy_model_name == "tutorial_activation_embedded": + return Helpers.toy_network_tutorial_with_embedded_activation + elif toy_model_name == "add": + return Helpers.toy_network_add + elif toy_model_name == "add_linear": + return lambda input_shape, dtype=None: Helpers.toy_network_add( + input_shape=input_shape, dtype=dtype, activation=None + ) + elif toy_model_name == "merge_v0": + return lambda input_shape, dtype=None: Helpers.toy_struct_v0( + dtype=dtype, input_shape=input_shape, archi=[2, 3, 2], activation="relu", use_bias=True + ) + elif toy_model_name == "merge_v1": + return lambda input_shape, dtype=None: Helpers.toy_struct_v1( + dtype=dtype, input_shape=input_shape, archi=[2, 3, 2], activation="relu", use_bias=True, sequential=False + ) + elif toy_model_name == "merge_v1_seq": + return lambda input_shape, dtype=None: Helpers.toy_struct_v1( + dtype=dtype, input_shape=input_shape, archi=[2, 3, 2], activation="relu", use_bias=True, sequential=True + ) + elif toy_model_name == "merge_v2": + return lambda input_shape, dtype=None: Helpers.toy_struct_v2( + dtype=dtype, input_shape=input_shape, archi=[2, 3, 2], activation="relu", use_bias=True, sequential=False + ) + elif toy_model_name == "cnn": + return Helpers.toy_struct_cnn + elif toy_model_name == "embedded_model_v1": + return Helpers.toy_embedded_sequential + elif toy_model_name == "embedded_model_v2": + return Helpers.toy_network_submodel + else: + raise ValueError(f"model_name {toy_model_name} unknown") From 7670794e4ca8cf007ee4a6fb3262421a76363b3b Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 19 Feb 2024 16:40:24 +0100 Subject: [PATCH 053/101] Update forward conversion and test it --- src/decomon/models/forward_cloning.py | 334 ++++++++++++++++---------- src/decomon/models/utils.py | 35 +++ tests/conftest.py | 92 ++++++- tests/test_convert_forward.py | 110 +++++++++ 4 files changed, 443 insertions(+), 128 deletions(-) create mode 100644 tests/test_convert_forward.py diff --git a/src/decomon/models/forward_cloning.py b/src/decomon/models/forward_cloning.py index 2c12946d..3d6c32bb 100644 --- a/src/decomon/models/forward_cloning.py +++ b/src/decomon/models/forward_cloning.py @@ -3,108 +3,102 @@ It inherits from keras Sequential class. """ -import inspect from collections.abc import Callable -from copy import deepcopy from typing import Any, Optional, Union import keras +import keras.ops as K from keras.layers import InputLayer, Layer from keras.models import Model -from keras.src.utils.python_utils import to_list -from decomon.core import BoxDomain, PerturbationDomain, Slope +from decomon.core import ( + BoxDomain, + InputsOutputsSpec, + PerturbationDomain, + Propagation, + Slope, +) +from decomon.layers import DecomonLayer from decomon.layers.convert import to_decomon -from decomon.layers.core import DecomonLayer -from decomon.layers.utils import softmax_to_linear as softmax_2_linear from decomon.models.utils import ( ensure_functional_model, get_depth_dict, - get_inner_layers, - get_input_dim, + get_output_nodes, prepare_inputs_for_layer, wrap_outputs_from_layer_in_list, ) -OutputMapKey = Union[str, int] -OutputMapVal = Union[list[keras.KerasTensor], "OutputMapDict"] -OutputMapDict = dict[OutputMapKey, OutputMapVal] - -LayerMapVal = Union[list[DecomonLayer], "LayerMapDict"] -LayerMapDict = dict[int, LayerMapVal] - -def include_dim_layer_fn( - layer_fn: Callable[..., Layer], - input_dim: int, - slope: Union[str, Slope] = Slope.V_SLOPE, - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - ibp: bool = True, - affine: bool = True, - finetune: bool = False, - shared: bool = True, -) -> Callable[[Layer], list[Layer]]: - """include external parameters inside the translation of a layer to its decomon counterpart +def get_ibp_inputs(x: keras.KerasTensor, perturbation_domain: PerturbationDomain) -> list[keras.KerasTensor]: + """Get ibp inputs from perturbation domain input Args: - layer_fn - input_dim - dc_decomp - perturbation_domain - finetune + x: perturbation domain input + perturbation_domain: perturbation domain type Returns: + [l_c, u_c] constant bounds on keras model input """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() + return [perturbation_domain.get_lower_x(x=x), perturbation_domain.get_upper_x(x=x)] - if not callable(layer_fn): - raise ValueError("Expected `layer_fn` argument to be a callable.") - - layer_fn_copy = deepcopy(layer_fn) - - if "input_dim" in inspect.signature(layer_fn).parameters: - - def func(layer: Layer) -> list[Layer]: - return [ - layer_fn_copy( - layer, - input_dim=input_dim, - slope=slope, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - ibp=ibp, - affine=affine, - finetune=finetune, - shared=shared, - ) - ] - else: +def get_affine_inputs(x: keras.KerasTensor, perturbation_domain: PerturbationDomain) -> list[keras.KerasTensor]: + """Get affine inputs from perturbation domain input - def func(layer: Layer) -> list[Layer]: - return [layer_fn_copy(layer)] + Args: + x: perturbation domain input + perturbation_domain: perturbation domain type - return func + Returns: + [w_l, b_l, w_u, b_u] identity affine bounds on keras model input + + We start with identity bounds: w_l = w_u = Identity, b_l = b_u = 0 + We need the perturbation domain input to get the proper shape, and to construct the inputs + from the future decomon model input x. + + """ + keras_input_like_tensor_wo_batchsize = perturbation_domain.get_kerasinputlike_from_x(x=x)[0] + # identity: diag representation + w/o batchisze + w = K.ones_like(keras_input_like_tensor_wo_batchsize) # identity in diag + b = K.zeros_like(keras_input_like_tensor_wo_batchsize) + return [w, b, w, b] def convert_forward( model: Model, - input_tensors: list[keras.KerasTensor], - layer_fn: Callable[..., Layer] = to_decomon, - slope: Union[str, Slope] = Slope.V_SLOPE, - input_dim: int = -1, - dc_decomp: bool = False, + perturbation_domain_input: keras.KerasTensor, + layer_fn: Callable[..., DecomonLayer] = to_decomon, + slope: Slope = Slope.V_SLOPE, perturbation_domain: Optional[PerturbationDomain] = None, ibp: bool = True, affine: bool = True, - finetune: bool = False, - shared: bool = True, - softmax_to_linear: bool = True, **kwargs: Any, -) -> tuple[list[keras.KerasTensor], list[keras.KerasTensor], LayerMapDict, OutputMapDict]: +) -> tuple[list[keras.KerasTensor], dict[int, list[keras.KerasTensor]], dict[int, DecomonLayer]]: + """Convert keras model via forward propagation. + + Prepare layer_fn by freezing all args except layer. + Ensure that model is functional (transform sequential ones to functional equivalent ones). + + Args: + model: keras model to convert + perturbation_domain_input: perturbation domain input (input to the future decomon model). + Used to convert affine bounds into constant ones. + layer_fn: conversion function on layers. Default to `to_decomon()`. + slope: slope used by decomon activation layers + perturbation_domain: perturbation domain type for keras input + ibp: specify if constant bounds are propagated + affine: specify if affine bounds are propagated + **kwargs: keyword arguments to pass to layer_fn + + Returns: + output, output_map, layer_map: + - output: propagated bounds (concatenated), see `DecomonLayer.call()` for the format. + output of the future decomon model + - output_map: output of each converted node. Can be used to feed oracle bounds used by backward conversion. + - layer_map: converted layer corresponding to each node. Can be used to transform output_map into oracle bounds + + """ if perturbation_domain is None: perturbation_domain = BoxDomain() @@ -112,43 +106,106 @@ def convert_forward( raise ValueError("Expected `model` argument " "to be a `Model` instance, got ", model) model = ensure_functional_model(model) + model_input_shape = model.inputs[0].shape[1:] - if input_dim == -1: - input_dim = get_input_dim(model) + propagation = Propagation.FORWARD + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + model_input_shape=model_input_shape, + layer_input_shape=model_input_shape, + ) - layer_fn_to_list = include_dim_layer_fn( + if inputs_outputs_spec.needs_perturbation_domain_inputs(): + perturbation_domain_inputs = [perturbation_domain_input] + else: + perturbation_domain_inputs = [] + + layer_fn_to_list = include_kwargs_layer_fn( layer_fn, - input_dim=input_dim, + model_input_shape=model_input_shape, slope=slope, perturbation_domain=perturbation_domain, ibp=ibp, affine=affine, - finetune=finetune, - shared=shared, - ) # return a list of Decomon layers + propagation=propagation, + **kwargs, + ) + + # generate input tensors + if affine: + affine_bounds_to_propagate = get_affine_inputs( + x=perturbation_domain_input, perturbation_domain=perturbation_domain + ) + else: + affine_bounds_to_propagate = [] + if ibp: + constant_oracle_bounds = get_ibp_inputs(x=perturbation_domain_input, perturbation_domain=perturbation_domain) + else: + constant_oracle_bounds = [] + input_tensors_wo_pertubation_domain_inputs = inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=affine_bounds_to_propagate, + constant_oracle_bounds=constant_oracle_bounds, + perturbation_domain_inputs=[], + ) - input_tensors, output, layer_map, output_map, _ = convert_forward_functional_model( + output_map: dict[int, list[keras.KerasTensor]] = {} + layer_list_map: dict[int, list[DecomonLayer]] = {} + output = convert_forward_functional_model( model=model, - input_tensors=input_tensors, + input_tensors=input_tensors_wo_pertubation_domain_inputs, layer_fn=layer_fn_to_list, - softmax_to_linear=softmax_to_linear, + common_inputs_part=perturbation_domain_inputs, + output_map=output_map, + layer_map=layer_list_map, ) + layer_map: dict[int, DecomonLayer] = {k: v[0] for k, v in layer_list_map.items()} - return input_tensors, output, layer_map, output_map + return output, output_map, layer_map def convert_forward_functional_model( model: Model, layer_fn: Callable[[Layer], list[Layer]], input_tensors: list[keras.KerasTensor], - softmax_to_linear: bool = True, - count: int = 0, - output_map: Optional[OutputMapDict] = None, - layer_map: Optional[LayerMapDict] = None, - layer2layer_map: Optional[dict[int, list[Layer]]] = None, -) -> tuple[list[keras.KerasTensor], list[keras.KerasTensor], LayerMapDict, OutputMapDict, dict[int, list[Layer]]]: - if softmax_to_linear: - model, has_softmax = softmax_2_linear(model) + common_inputs_part: Optional[list[keras.KerasTensor]] = None, + output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, + layer_map: Optional[dict[int, list[Layer]]] = None, + submodel: bool = False, +) -> list[keras.KerasTensor]: + """Convert a functional keras model via forward propagation. + + Used + - for decomon conversion in forward mode with layer_fn=lambda l: [to_decomon(l)] + - also for preprocessing keras models (e.g. splitting activation layers) with layer_fn=preprocess_layer + + Hypothesis: potential embedded submodels have only one input and one output. + + Args: + model: keras model to convert + layer_fn: callable converting a layer into a list of converted layers + input_tensors: input tensors used by the converted (list of) layer(s) corresponding to the keras model input nodes + common_inputs_part: inputs part common to all converted layers. + The inputs of a converted layers is + the concatenation of the outputs of the converted layers corresponding to its inputs nodes + + this common_inputs_part (only once even for merging layers) + This allows to pass perturbation_domain_inputs only once when creating the decomon forward version of a keras model. + Default to an empty list. + output_map: output of each converted node (to be used when called recursively on submodels) + When a layer is converted into several layers, we store the propagated output of the last one. + This map is updated during the conversion. + layer_map: map between node and converted layers (to be used when called recursively on submodels) + This map is updated during the conversion. + submodel: specify if called from within another conversion to propagate through an embedded submodel + + Returns: + concatenated outputs of the converted layers corresponding to the output nodes of the keras model + + """ + if common_inputs_part is None: + common_inputs_part = [] # ensure the model is functional (to be able to use get_depth_dict), convert sequential ones into functional ones model = ensure_functional_model(model) @@ -162,8 +219,6 @@ def convert_forward_functional_model( output_map = {} if layer_map is None: layer_map = {} - if layer2layer_map is None: - layer2layer_map = {} output: list[keras.KerasTensor] = input_tensors for depth in keys: nodes = dico_nodes[depth] @@ -178,51 +233,82 @@ def convert_forward_functional_model( output += output_map[id(parent)] if isinstance(layer, InputLayer): - # no conversion, no transformation for output (instead of trying to pass in identity layer) - layer_map[id(node)] = layer + # no conversion, propagate output unchanged + pass elif isinstance(layer, Model): - ( - _, - output, - layer_map_submodel, - output_map_submodel, - layer2layer_map_submodel, - ) = convert_forward_functional_model( + output = convert_forward_functional_model( model=layer, input_tensors=output, + common_inputs_part=common_inputs_part, layer_fn=layer_fn, - softmax_to_linear=softmax_to_linear, - count=count, - layer2layer_map=layer2layer_map, + output_map=output_map, + layer_map=layer_map, ) - count = count + get_inner_layers(layer) - layer_map.update(layer_map_submodel) - output_map.update(output_map_submodel) - layer2layer_map.update(layer2layer_map_submodel) - layer_map[id(node)] = layer_map_submodel else: - if id(layer) in layer2layer_map: + if id(node) in layer_map: # avoid converting twice layers that are shared by several nodes - converted_layers = layer2layer_map[id(layer)] + converted_layers = layer_map[id(node)] else: converted_layers = layer_fn(layer) - layer2layer_map[id(layer)] = converted_layers + layer_map[id(node)] = converted_layers for converted_layer in converted_layers: - converted_layer.name = f"{converted_layer.name}_{count}" - count += 1 - output = converted_layer(prepare_inputs_for_layer(output)) - if len(converted_layers) > 1: - output_map[f"{id(node)}_{converted_layer.name}"] = wrap_outputs_from_layer_in_list(output) - layer_map[id(node)] = converted_layers + output = wrap_outputs_from_layer_in_list( + converted_layer(prepare_inputs_for_layer(output + common_inputs_part)) + ) output_map[id(node)] = wrap_outputs_from_layer_in_list(output) output = [] - output_nodes = dico_nodes[0] - # the ordering may change - output_names = [tensor._keras_history.operation.name for tensor in to_list(model.output)] - for output_name in output_names: - for node in output_nodes: - if node.operation.name == output_name: - output += output_map[id(node)] - - return input_tensors, output, layer_map, output_map, layer2layer_map + output_nodes = get_output_nodes(model) + if submodel and len(output_nodes) > 1: + raise NotImplementedError( + "convert_forward_functional_model() not yet implemented for model " + "whose embedded submodels have multiple outputs." + ) + for node in output_nodes: + output += output_map[id(node)] + + return output + + +def include_kwargs_layer_fn( + layer_fn: Callable[..., Layer], + model_input_shape: tuple[int, ...], + perturbation_domain: PerturbationDomain, + ibp: bool, + affine: bool, + propagation: Propagation, + slope: Slope, + **kwargs: Any, +) -> Callable[[Layer], list[Layer]]: + """Include external parameters in the function converting layers + + In particular, include propagation=Propagation.FORWARD. + + Args: + layer_fn: + model_input_shape: + perturbation_domain: + ibp: + affine: + slope: + **kwargs: + + Returns: + + """ + + def func(layer: Layer) -> list[Layer]: + return [ + layer_fn( + layer, + model_input_shape=model_input_shape, + slope=slope, + perturbation_domain=perturbation_domain, + ibp=ibp, + affine=affine, + propagation=propagation, + **kwargs, + ) + ] + + return func diff --git a/src/decomon/models/utils.py b/src/decomon/models/utils.py index 5c789d55..662428e7 100644 --- a/src/decomon/models/utils.py +++ b/src/decomon/models/utils.py @@ -88,6 +88,17 @@ def check_model2convert_inputs(model: Model) -> None: raise ValueError("The model must have a flattened input to be converted.") +def generate_perturbation_domain_input( + model: Model, + perturbation_domain: PerturbationDomain, +) -> keras.KerasTensor: + model_input_shape = model.input.shape[1:] + dtype = model.input.dtype + + input_shape_x = perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape) + return Input(shape=input_shape_x, dtype=dtype) + + def get_input_tensors( model: Model, perturbation_domain: PerturbationDomain, @@ -178,6 +189,17 @@ def prepare_inputs_for_layer( def wrap_outputs_from_layer_in_list( outputs: Union[tuple[keras.KerasTensor, ...], list[keras.KerasTensor], keras.KerasTensor] ) -> list[keras.KerasTensor]: + """Normalizes a list/tensor into a list. + + If a tensor is passed, we return + a list of size 1 containing the tensor. + + Args: + x: target object to be normalized. + + Returns: + A list. + """ if not isinstance(outputs, list): if isinstance(outputs, tuple): return list(outputs) @@ -229,6 +251,19 @@ def is_input_node(node: Node) -> bool: return len(node.input_tensors) == 0 +def get_output_nodes(model: Model) -> list[Node]: + """Get list of output nodes ordered as model.outputs + + Args: + model: + + Returns: + + """ + nodes_by_operation = {n.operation: n for subnodes in model._nodes_by_depth.values() for n in subnodes} + return [nodes_by_operation[output._keras_history.operation] for output in model.outputs] + + def get_depth_dict(model: Model) -> dict[int, list[Node]]: depth_keys = list(model._nodes_by_depth.keys()) depth_keys.sort(reverse=True) diff --git a/tests/conftest.py b/tests/conftest.py index 9eb906f7..f5294115 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -306,10 +306,9 @@ def generate_simple_decomon_layer_inputs_from_keras_input( ) if inputs_outputs_spec.needs_perturbation_domain_inputs(): - if isinstance(perturbation_domain, BoxDomain): - x = K.repeat(keras_input[:, None], 2, axis=1) - else: - raise NotImplementedError + x = Helpers.generate_simple_perturbation_domain_inputs_from_keras_input( + keras_input=keras_input, perturbation_domain=perturbation_domain + ) perturbation_domain_inputs = [x] else: perturbation_domain_inputs = [] @@ -352,6 +351,15 @@ def generate_simple_decomon_layer_inputs_from_keras_input( perturbation_domain_inputs=perturbation_domain_inputs, ) + @staticmethod + def generate_simple_perturbation_domain_inputs_from_keras_input(keras_input, perturbation_domain): + if isinstance(perturbation_domain, BoxDomain): + return K.concatenate( + [keras_input[:, None] - keras.config.epsilon(), keras_input[:, None] + keras.config.epsilon()], axis=1 + ) + else: + raise NotImplementedError + @staticmethod def generate_merging_decomon_input_from_single_decomon_inputs( decomon_inputs: list[list[Tensor]], ibp: bool, affine: bool, propagation: Propagation @@ -898,6 +906,51 @@ def assert_decomon_output_compare_with_keras_input_output_layer( Helpers.assert_ordered(lower_ibp, keras_output, decimal=decimal, err_msg="lower_ibp not ok") Helpers.assert_ordered(keras_output, upper_ibp, decimal=decimal, err_msg="upper_ibp not ok") + @staticmethod + def assert_decomon_output_compare_with_keras_input_output_model( + decomon_output, + keras_input, + keras_output, + ibp, + affine, + propagation, + decimal=5, + ): + keras_input_shape = tuple(keras_input.shape[1:]) + keras_output_shape = tuple(keras_output.shape[1:]) + inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=propagation, + layer_input_shape=keras_input_shape, + model_input_shape=keras_input_shape, + model_output_shape=keras_output_shape, + ) + affine_bounds_propagated, constant_bounds_propagated = inputs_outputs_spec.split_outputs(outputs=decomon_output) + + if affine or propagation == Propagation.BACKWARD: + if len(affine_bounds_propagated) == 0: + # identity case + lower_affine = keras_input + upper_affine = keras_input + else: + w_l, b_l, w_u, b_u = affine_bounds_propagated + diagonal = (False, w_l.shape == b_l.shape) + missing_batchsize = (False, len(b_l.shape) < len(keras_output.shape)) + lower_affine = ( + batch_multid_dot(keras_input, w_l, diagonal=diagonal, missing_batchsize=missing_batchsize) + b_l + ) + upper_affine = ( + batch_multid_dot(keras_input, w_u, diagonal=diagonal, missing_batchsize=missing_batchsize) + b_u + ) + Helpers.assert_ordered(lower_affine, keras_output, decimal=decimal, err_msg="lower_affine not ok") + Helpers.assert_ordered(keras_output, upper_affine, decimal=decimal, err_msg="upper_affine not ok") + + if ibp and propagation == Propagation.FORWARD: + lower_ibp, upper_ibp = constant_bounds_propagated + Helpers.assert_ordered(lower_ibp, keras_output, decimal=decimal, err_msg="lower_ibp not ok") + Helpers.assert_ordered(keras_output, upper_ibp, decimal=decimal, err_msg="upper_ibp not ok") + @staticmethod def assert_decomon_output_lower_equal_upper( decomon_output, @@ -1559,6 +1612,37 @@ def standard_layer_input_functions_multid(data_format, ibp, affine, propagation, simple_layer_input_functions, ) + +# keras/decomon model inputs +@fixture +def simple_model_input_functions(perturbation_domain, batchsize, helpers): + decomon_symbolic_input_fn = lambda keras_symbolic_input: Input( + perturbation_domain.get_x_input_shape_wo_batchsize(keras_symbolic_input.shape[1:]) + ) + keras_input_fn = lambda keras_symbolic_input: helpers.generate_random_tensor( + keras_symbolic_input.shape[1:], batchsize=batchsize + ) + decomon_input_fn = lambda keras_input: helpers.generate_simple_perturbation_domain_inputs_from_keras_input( + keras_input=keras_input, perturbation_domain=perturbation_domain + ) + + return ( + decomon_symbolic_input_fn, + keras_input_fn, + decomon_input_fn, + ) + + +( + simple_model_decomon_symbolic_input_fn, + simple_model_keras_input_fn, + simple_model_decomon_input_fn, +) = unpack_fixture( + "simple_model_decomon_symbolic_input_fn, simple_model_keras_input_fn, simple_model_decomon_input_fn", + simple_model_input_functions, +) + + # keras toy models toy_model_name = param_fixture( "toy_model_name", diff --git a/tests/test_convert_forward.py b/tests/test_convert_forward.py new file mode 100644 index 00000000..9a9e45b3 --- /dev/null +++ b/tests/test_convert_forward.py @@ -0,0 +1,110 @@ +import pytest +from keras.models import Model +from pytest_cases import fixture, parametrize + +from decomon.core import Propagation, Slope +from decomon.layers.activations.activation import DecomonBaseActivation +from decomon.models.forward_cloning import convert_forward + + +@fixture +@parametrize("name", ["tutorial", "tutorial_linear", "submodel", "submodel_linear", "add", "add_linear", "add_1layer"]) +def keras_model_fn(name, helpers): + if name == "tutorial": + return helpers.toy_network_tutorial + elif name == "tutorial_linear": + return lambda input_shape, dtype=None: helpers.toy_network_tutorial( + input_shape=input_shape, dtype=dtype, activation=None + ) + elif name == "add": + return helpers.toy_network_add + elif name == "add_linear": + return lambda input_shape, dtype=None: helpers.toy_network_add( + input_shape=input_shape, dtype=dtype, activation=None + ) + elif name == "add_1layer": + return helpers.toy_network_add_monolayer + elif name == "submodel": + return helpers.toy_network_submodel + elif name == "submodel_linear": + return lambda input_shape, dtype=None: helpers.toy_network_submodel( + input_shape=input_shape, dtype=dtype, activation=None + ) + else: + raise ValueError() + + +def test_convert_forward( + ibp, + affine, + propagation, + perturbation_domain, + input_shape, + keras_model_fn, + simple_model_decomon_symbolic_input_fn, + simple_model_keras_input_fn, + simple_model_decomon_input_fn, + helpers, +): + if propagation == Propagation.BACKWARD: + pytest.skip("backward propagation meaningless for convert_forward()") + + slope = Slope.Z_SLOPE + decimal = 4 + + # keras model to convert + keras_model = keras_model_fn(input_shape=input_shape) + + # symbolic inputs + keras_symbolic_input = keras_model.inputs[0] + decomon_symbolic_input = simple_model_decomon_symbolic_input_fn(keras_symbolic_input) + + # actual inputs + keras_input = simple_model_keras_input_fn(keras_symbolic_input) + decomon_input = simple_model_decomon_input_fn(keras_input) + + # keras output + keras_output = keras_model(keras_input) + + # decomon conversion + decomon_symbolic_output, output_map, layer_map = convert_forward( + keras_model, + ibp=ibp, + affine=affine, + perturbation_domain_input=decomon_symbolic_input, + perturbation_domain=perturbation_domain, + slope=slope, + ) + + # check output map + def check_maps(output_map, layer_map, keras_model): + for layer in keras_model.layers: + for node in layer._inbound_nodes: + if isinstance(layer, Model): + # submodel, iterate on its layers + check_maps(output_map, layer_map, layer) + else: + assert id(node) in output_map + if len(node.parent_nodes) > 0: # not an input node + assert id(node) in layer_map + decomon_layer = layer_map[id(node)] + assert decomon_layer.layer is layer + if isinstance(decomon_layer, DecomonBaseActivation): + assert decomon_layer.slope == slope + + check_maps(output_map, layer_map, keras_model) + + #  decomon outputs + decomon_model = Model(inputs=decomon_symbolic_input, outputs=decomon_symbolic_output) + decomon_output = decomon_model(decomon_input) + + # check ibp and affine bounds well ordered w.r.t. keras inputs/outputs + helpers.assert_decomon_output_compare_with_keras_input_output_model( + decomon_output=decomon_output, + keras_input=keras_input, + keras_output=keras_output, + ibp=ibp, + affine=affine, + propagation=propagation, + decimal=decimal, + ) From 87336dc54da12ad01b8382dcb57c130c192c099b Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 22 Feb 2024 15:08:27 +0100 Subject: [PATCH 054/101] Test keras model preprocessing (split activation) --- tests/test_preprocess_keras_layer.py | 170 +++++++++++++++++++++++++++ tests/test_preprocess_keras_model.py | 84 +++++++++++++ 2 files changed, 254 insertions(+) create mode 100644 tests/test_preprocess_keras_layer.py create mode 100644 tests/test_preprocess_keras_model.py diff --git a/tests/test_preprocess_keras_layer.py b/tests/test_preprocess_keras_layer.py new file mode 100644 index 00000000..d15c344c --- /dev/null +++ b/tests/test_preprocess_keras_layer.py @@ -0,0 +1,170 @@ +import keras +import keras.ops as K +import numpy as np +import pytest +from keras import Input +from keras.layers import Activation, Conv2D, Dense, Layer, Permute, PReLU +from numpy.testing import assert_almost_equal + +from decomon.keras_utils import get_weight_index_from_name +from decomon.models.utils import preprocess_layer, split_activation + + +@pytest.mark.parametrize( + "layer_class, layer_kwargs, input_shape_wo_batchsize, embedded_activation_layer_class", + [ + (Dense, {"units": 3, "activation": "relu"}, (1,), None), + (Dense, {"units": 3}, (1,), PReLU), + (Conv2D, {"filters": 2, "kernel_size": (3, 3), "activation": "relu"}, (64, 64, 3), None), + ], +) +def test_split_activation_do_split( + layer_class, layer_kwargs, input_shape_wo_batchsize, embedded_activation_layer_class, use_bias +): + # init layer + if embedded_activation_layer_class is not None: + layer = layer_class(use_bias=use_bias, activation=embedded_activation_layer_class(), **layer_kwargs) + else: + layer = layer_class(use_bias=use_bias, **layer_kwargs) + # init input_shape and weights + input_shape = input_shape_wo_batchsize + input_tensor = Input(input_shape) + layer(input_tensor) + # split + layers = split_activation(layer) + # check layer split + assert len(layers) == 2 + layer_wo_activation, activation_layer = layers + assert isinstance(layer_wo_activation, layer.__class__) + assert layer_wo_activation.get_config()["activation"] == "linear" + if isinstance(layer.activation, Layer): + assert activation_layer == layer.activation + else: + assert isinstance(activation_layer, Activation) + assert activation_layer.get_config()["activation"] == layer_kwargs["activation"] + # check names starts with original name + "_" + assert layer_wo_activation.name.startswith(f"{layer.name}_") + assert activation_layer.name.startswith(f"{layer.name}_") + # check already built + assert layer_wo_activation.built + assert activation_layer.built + # check same outputs + input_shape_with_batch_size = (5,) + input_shape_wo_batchsize + flatten_dim = np.prod(input_shape_with_batch_size) + inputs_np = np.linspace(-1, 1, flatten_dim).reshape(input_shape_with_batch_size) + output_np_ref = K.convert_to_numpy(layer(inputs_np)) + output_np_new = K.convert_to_numpy(activation_layer(layer_wo_activation(inputs_np))) + assert_almost_equal(output_np_new, output_np_ref) + # check same weights (really same objects) + for i in range(len(layer_wo_activation.weights)): + assert layer.weights[i] is layer_wo_activation.weights[i] + + +@pytest.mark.parametrize( + "layer_class, layer_kwargs", + [ + (Dense, {"units": 3}), + (Activation, {"activation": "relu"}), + (Permute, {"dims": (1, 2, 3)}), + ], +) +def test_split_activation_do_nothing(layer_class, layer_kwargs): + layer = layer_class(**layer_kwargs) + layers = split_activation(layer) + assert len(layers) == 1 + assert layers[0] == layer + + +def test_split_activation_uninitialized_layer_ko(): + layer = Dense(3, activation="relu") + with pytest.raises(ValueError): + layers = split_activation(layer) + + +@pytest.mark.parametrize( + "layer_class_name, layer_kwargs, input_shape_wo_batchsize", + [ + ("Dense", {"units": 3}, (1,)), + ("Activation", {"activation": "relu"}, (1,)), + ("Permute", {"dims": (1, 2, 3)}, (1, 1, 1)), + ], +) +def test_preprocess_layer_no_nonlinear_activation(layer_class_name, layer_kwargs, input_shape_wo_batchsize): + layer_class = globals()[layer_class_name] + layer = layer_class(**layer_kwargs) + # build layer + input_tensor = Input(input_shape_wo_batchsize) + layer(input_tensor) + # preprocess + layers = preprocess_layer(layer) + # check resulting layers + assert len(layers) == 1 + keras_layer = layers[0] + # check values + assert keras_layer is layer + # try to call the resulting layers 3 times + input_tensor = K.ones((5,) + input_shape_wo_batchsize) + for _ in range(3): + keras_layer(input_tensor) + + +@pytest.mark.parametrize( + "layer_class_name, " + "layer_kwargs, " + "input_shape_wo_batchsize, " + "embedded_activation_layer_class_name, " + "embedded_activation_layer_class_kwargs, ", + [ + ("Dense", {"units": 3, "activation": "relu"}, (1,), None, None), + ("Dense", {"units": 3}, (1,), "PReLU", {}), + ("Conv2D", {"filters": 2, "kernel_size": (3, 3), "activation": "relu"}, (64, 64, 3), None, None), + ], +) +def test_preprocess_layer_nonlinear_activation( + layer_class_name, + layer_kwargs, + input_shape_wo_batchsize, + embedded_activation_layer_class_name, + embedded_activation_layer_class_kwargs, + use_bias, +): + # init layer + layer_class = globals()[layer_class_name] + if embedded_activation_layer_class_name is not None: + embedded_activation_layer_class = globals()[embedded_activation_layer_class_name] + embedded_activation_layer = embedded_activation_layer_class(**embedded_activation_layer_class_kwargs) + layer = layer_class(use_bias=use_bias, activation=embedded_activation_layer, **layer_kwargs) + else: + layer = layer_class(use_bias=use_bias, **layer_kwargs) + # init input_shape and weights + input_shape = input_shape_wo_batchsize + input_tensor = Input(input_shape) + layer(input_tensor) + # split + layers = preprocess_layer(layer) + # check layer split + assert len(layers) == 2 + layer_wo_activation, activation_layer = layers + assert isinstance(layer_wo_activation, layer.__class__) + assert layer_wo_activation.get_config()["activation"] == "linear" + if isinstance(layer.activation, Layer): + assert activation_layer == layer.activation + else: + assert isinstance(activation_layer, Activation) + assert activation_layer.get_config()["activation"] == layer_kwargs["activation"] + # check names starts with with original name + "_" + assert layer_wo_activation.name.startswith(f"{layer.name}_") + assert activation_layer.name.startswith(f"{layer.name}_") + # check already built + assert layer_wo_activation.built + assert activation_layer.built + # check same outputs + input_shape_with_batch_size = (5,) + input_shape_wo_batchsize + flatten_dim = np.prod(input_shape_with_batch_size) + inputs_np = np.linspace(-1, 1, flatten_dim).reshape(input_shape_with_batch_size) + output_np_ref = K.convert_to_numpy(layer(inputs_np)) + output_np_new = K.convert_to_numpy(activation_layer(layer_wo_activation(inputs_np))) + assert_almost_equal(output_np_new, output_np_ref) + # check same weights (really same objects) + for i in range(len(layer_wo_activation.weights)): + assert layer.weights[i] is layer_wo_activation.weights[i] diff --git a/tests/test_preprocess_keras_model.py b/tests/test_preprocess_keras_model.py new file mode 100644 index 00000000..de4394e6 --- /dev/null +++ b/tests/test_preprocess_keras_model.py @@ -0,0 +1,84 @@ +import keras.ops as K +import numpy as np +import pytest +from keras.layers import Activation, Conv2D, Dense, Flatten, Input, PReLU +from keras.models import Model, Sequential +from numpy.testing import assert_almost_equal + +from decomon.models.convert import ( + preprocess_keras_model, + split_activations_in_keras_model, +) + + +def test_split_activations_in_keras_model(toy_model_fn, input_shape, toy_model_name): + # skip cnn on 0d or 1d input_shape + if toy_model_name == "cnn" and len(input_shape) == 1: + pytest.skip("cnn not possible on 0d or 1d input.") + + toy_model = toy_model_fn(input_shape=input_shape) + converted_model = split_activations_in_keras_model(toy_model) + assert isinstance(converted_model, Model) + # check no more activation functions in non-activation layers + for layer in converted_model.layers: + activation = layer.get_config().get("activation", None) + assert isinstance(layer, Activation) or activation is None or activation == "linear" + # check same outputs + input_shape_wo_batchsize = toy_model.input_shape[1:] + input_shape_with_batch_size = (5,) + input_shape_wo_batchsize + flatten_dim = np.prod(input_shape_with_batch_size) + inputs_np = np.linspace(-1, 1, flatten_dim).reshape(input_shape_with_batch_size) + output_np_ref = K.convert_to_numpy(toy_model(inputs_np)) + output_np_new = K.convert_to_numpy(converted_model(inputs_np)) + assert_almost_equal(output_np_new, output_np_ref, decimal=4) + + +@pytest.mark.parametrize( + "layer_class_name, " + "layer_kwargs, " + "input_shape_wo_batchsize, " + "embedded_activation_layer_class_name, " + "embedded_activation_layer_class_kwargs, ", + [ + ("Dense", {"units": 3, "activation": "relu"}, (1,), None, None), + ("Dense", {"units": 3}, (1,), "PReLU", {}), + ("Conv2D", {"filters": 2, "kernel_size": (3, 3), "activation": "relu"}, (64, 64, 3), None, None), + ], +) +def test_preprocess( + layer_class_name, + layer_kwargs, + input_shape_wo_batchsize, + embedded_activation_layer_class_name, + embedded_activation_layer_class_kwargs, + use_bias, +): + # init hidden layer + layer_class = globals()[layer_class_name] + if embedded_activation_layer_class_name is not None: + embedded_activation_layer_class = globals()[embedded_activation_layer_class_name] + embedded_activation_layer = embedded_activation_layer_class(**embedded_activation_layer_class_kwargs) + hidden_layer = layer_class(use_bias=use_bias, activation=embedded_activation_layer, **layer_kwargs) + else: + hidden_layer = layer_class(use_bias=use_bias, **layer_kwargs) + layers = [ + Input(shape=input_shape_wo_batchsize), + hidden_layer, + Dense(1), + ] + model = Sequential(layers) + converted_model = preprocess_keras_model(model) + # check no more embedded activation + for layer in converted_model.layers: + activation = layer.get_config().get("activation", None) + assert isinstance(layer, Activation) or activation is None or activation == "linear" + # check number of layers + assert len(converted_model.layers) == 4 + # check same outputs + input_shape_wo_batchsize = model.input_shape[1:] + input_shape_with_batch_size = (5,) + input_shape_wo_batchsize + flatten_dim = np.prod(input_shape_with_batch_size) + inputs_np = np.linspace(-1, 1, flatten_dim).reshape(input_shape_with_batch_size) + output_np_ref = K.convert_to_numpy(model(inputs_np)) + output_np_new = K.convert_to_numpy(converted_model(inputs_np)) + assert_almost_equal(output_np_new, output_np_ref, decimal=4) From c5e024c853b2049c3254dec70b0cfdc65ccc1dc2 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Wed, 21 Feb 2024 10:23:27 +0100 Subject: [PATCH 055/101] Add a call_oracle() method on decomon layers It allows to get (forward or crown) oracle bounds from - affine bounds computed previously on keras layer inputs w.r.t keras model input, either from forward conversion or sub-crown - the perturbation domain input - all these given as a single flattened list of tensors --- src/decomon/layers/layer.py | 88 +++++++++++++++++++++++- src/decomon/layers/merging/base_merge.py | 33 ++++++++- 2 files changed, 115 insertions(+), 6 deletions(-) diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index ccae7732..08a3298b 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -118,6 +118,26 @@ def __init__( self.propagation = propagation # input-output-manager + self.inputs_outputs_spec = self.create_inputs_outputs_spec( + layer=layer, + perturbation_domain=perturbation_domain, + ibp=ibp, + affine=affine, + propagation=propagation, + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + ) + + def create_inputs_outputs_spec( + self, + layer: Layer, + perturbation_domain: PerturbationDomain, + ibp: bool, + affine: bool, + propagation: Propagation, + model_input_shape: Optional[tuple[int, ...]], + model_output_shape: Optional[tuple[int, ...]], + ) -> InputsOutputsSpec: if self._is_merging: if isinstance(layer.input, keras.KerasTensor): # special case: merging a single input -> self.layer.input is already flattened @@ -126,7 +146,7 @@ def __init__( layer_input_shape = [t.shape[1:] for t in layer.input] else: layer_input_shape = layer.input.shape[1:] - self.inputs_outputs_spec = InputsOutputsSpec( + return InputsOutputsSpec( ibp=ibp, affine=affine, propagation=propagation, @@ -404,6 +424,8 @@ def get_forward_oracle( input_affine_bounds: list[Tensor], input_constant_bounds: list[Tensor], perturbation_domain_inputs: list[Tensor], + ibp: Optional[bool] = None, + affine: Optional[bool] = None, ) -> list[Tensor]: """Get constant oracle bounds on underlying keras layer input from forward input bounds. @@ -411,6 +433,8 @@ def get_forward_oracle( input_affine_bounds: affine bounds on keras layer input w.r.t model input . Can be empty if not in affine mode. input_constant_bounds: ibp constant bounds on keras layer input. Can be empty if not in ibp mode. perturbation_domain_inputs: perturbation domain input, wrapped in a list. Necessary only in affine mode, else empty. + ibp: if set, overrides temporarily `self.ibp` (used by `call_oracle()`) + affine: if set, overrides temporarily `self.affine` (used by `call_oracle()`) Returns: constant bounds on keras layer input deduced from forward input bounds @@ -423,12 +447,17 @@ def get_forward_oracle( from the affine bounds given the considered perturbation domain. """ - if self.ibp: + if ibp is None: + ibp = self.ibp + if affine is None: + affine = self.affine + + if ibp: # Hyp: in hybrid mode, the constant bounds are already tight # (affine and ibp mixed in forward layer output to get the tightest constant bounds) return input_constant_bounds - elif self.affine: + elif affine: if len(perturbation_domain_inputs) == 0: raise RuntimeError("keras model input is necessary for get_forward_oracle() in affine mode.") x = perturbation_domain_inputs[0] @@ -445,6 +474,59 @@ def get_forward_oracle( else: raise RuntimeError("self.ibp and self.affine cannot be both False") + def call_oracle(self, inputs: list[Tensor]) -> list[Tensor]: + """Compute oracle constant bounds on keras inputs from flatten decomon inputs. + + - forward: this is `self.get_forward_oracle()` after a split of inputs + - backward: this is a crown oracle. + The inputs are then equivalent to inputs for forward propagation + ibp=False + affine=True, + i.e. affine bounds (a priori coming from crowns) on each keras input + perturbation_domain_inputs + The computation is also equivalent to the one done in `self.get_forward_oracle()` with ibp=False, affine=True + + Args: + inputs: affine bounds on keras layer input + perturbation_domain_inputs + + Returns: + constant bounds on keras layer input deduced from affine bounds + + """ + ( + affine_bounds_to_propagate, + constant_oracle_bounds, + perturbation_domain_inputs, + ) = self.inputs_outputs_spec.split_inputs(inputs=inputs) + if self.propagation == Propagation.FORWARD: # forward + return self.get_forward_oracle( + input_affine_bounds=affine_bounds_to_propagate, + input_constant_bounds=constant_oracle_bounds, + perturbation_domain_inputs=perturbation_domain_inputs, + ) + else: # backward + ibp = False + affine = True + propagation = Propagation.FORWARD + inputs_outputs_spec = self.create_inputs_outputs_spec( + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=self.perturbation_domain, + layer=self.layer, + model_input_shape=self.model_input_shape, + model_output_shape=self.model_output_shape, + ) + ( + affine_bounds_to_propagate, + constant_oracle_bounds, + perturbation_domain_inputs, + ) = inputs_outputs_spec.split_inputs(inputs=inputs) + return self.get_forward_oracle( + input_affine_bounds=affine_bounds_to_propagate, + input_constant_bounds=constant_oracle_bounds, + perturbation_domain_inputs=perturbation_domain_inputs, + ibp=ibp, + affine=affine, + ) + def call_forward( self, affine_bounds_to_propagate: list[Tensor], diff --git a/src/decomon/layers/merging/base_merge.py b/src/decomon/layers/merging/base_merge.py index 93654fde..7e8fbae7 100644 --- a/src/decomon/layers/merging/base_merge.py +++ b/src/decomon/layers/merging/base_merge.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, Optional import keras.ops as K @@ -358,6 +358,8 @@ def get_forward_oracle( input_affine_bounds: list[list[Tensor]], input_constant_bounds: list[list[Tensor]], perturbation_domain_inputs: list[Tensor], + ibp: Optional[bool] = None, + affine: Optional[bool] = None, ) -> list[list[Tensor]]: """Get constant oracle bounds on underlying keras layer input from forward input bounds. @@ -365,6 +367,8 @@ def get_forward_oracle( input_affine_bounds: affine bounds on each keras layer input w.r.t model input . Can be empty if not in affine mode. input_constant_bounds: ibp constant bounds on each keras layer input. Can be empty if not in ibp mode. perturbation_domain_inputs: perturbation domain input, wrapped in a list. Necessary only in affine mode, else empty. + ibp: if set, overrides temporarily `self.ibp` (used by `call_oracle()`) + affine: if set, overrides temporarily `self.affine` (used by `call_oracle()`) Returns: constant bounds on each keras layer input deduced from forward input bounds @@ -377,12 +381,17 @@ def get_forward_oracle( from the affine bounds given the considered perturbation domain. """ - if self.ibp: + if ibp is None: + ibp = self.ibp + if affine is None: + affine = self.affine + + if ibp: # Hyp: in hybrid mode, the constant bounds are already tight # (affine and ibp mixed in forward layer output to get the tightest constant bounds) return input_constant_bounds - elif self.affine: + elif affine: if len(perturbation_domain_inputs) == 0: raise RuntimeError("keras model input is necessary for get_forward_oracle() in affine mode.") x = perturbation_domain_inputs[0] @@ -402,6 +411,24 @@ def get_forward_oracle( else: raise RuntimeError("self.ibp and self.affine cannot be both False") + def call_oracle(self, inputs: list[Tensor]) -> list[list[Tensor]]: + """Compute oracle constant bounds on keras inputs from flatten decomon inputs. + + - forward: this is `self.get_forward_oracle()` after a split of inputs + - backward: this is a crown oracle. + The inputs are then equivalent to inputs for forward propagation + ibp=False + affine=True, + i.e. affine bounds (a priori coming from crowns) on each keras input. + The computation is also equivalent to the one done in `self.get_forward_oracle()` with ibp=False, affine=True + + Args: + inputs: concatenated affine bounds on each keras layer input + perturbation_domain_inputs + + Returns: + constant bounds on each keras layer input deduced from affine bounds + + """ + return super().call_oracle(inputs) + def call_forward( self, affine_bounds_to_propagate: list[list[Tensor]], From 789af4382c0b18b53d104f3a3bf4bb75d04f5c0e Mon Sep 17 00:00:00 2001 From: Nolwen Date: Wed, 21 Feb 2024 15:16:26 +0100 Subject: [PATCH 056/101] Avoid passing constant oracle bounds to backward *linear* layers To manage this, we add methods to know if constant bounds are needed in decomon inputs (and the same for affine bounds, ...): InputsOutputsSpec.needs_constant_bounds_inputs() This implies - passing `linear` to inputs_outputs_spec - update split_inputs() and flatten_inputs() - fixing DecomonActivation so that linear status + inputs/outputs format is taken from wrapped decomon_activation (DecomonLinear or DecomonReLU for instance) - passing linear arg to functions generating decomon inputs in tests --- src/decomon/core.py | 97 +++++++++++++------- src/decomon/layers/activations/activation.py | 5 +- src/decomon/layers/layer.py | 1 + tests/conftest.py | 51 ++++++---- tests/test_decomon_layer.py | 83 ++++++++++++----- tests/test_merge_layers.py | 25 +++-- tests/test_unary_layers.py | 17 +++- 7 files changed, 194 insertions(+), 85 deletions(-) diff --git a/src/decomon/core.py b/src/decomon/core.py index 6fcc712d..c8024eef 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -200,6 +200,7 @@ def __init__( model_input_shape: Optional[tuple[int, ...]] = None, model_output_shape: Optional[tuple[int, ...]] = None, is_merging_layer: bool = False, + linear: bool = False, ): """ Args: @@ -213,6 +214,7 @@ def __init__( model_input_shape: shape of the underlying keras model input (w/o the batch axis) model_output_shape: shape of the underlying keras model output (w/o the batch axis) is_merging_layer: whether the underlying keras layer is a merging layer (i.e. with several inputs) + linear: whether the underlying keras layer is linear (thus do not need oracle bounds for instance) """ # checks @@ -237,6 +239,7 @@ def __init__( self.affine = affine self.ibp = ibp self.is_merging_layer = is_merging_layer + self.linear = linear self.perturbation_domain: PerturbationDomain if perturbation_domain is None: self.perturbation_domain = BoxDomain() @@ -262,6 +265,20 @@ def needs_perturbation_domain_inputs(self) -> bool: """Specify if decomon inputs should integrate keras model inputs.""" return self.propagation == Propagation.FORWARD and self.affine + def needs_oracle_bounds(self) -> bool: + """Specify if decomon layer needs oracle bounds on keras layer inputs.""" + return not self.linear and (self.propagation == Propagation.BACKWARD or self.affine) + + def needs_constant_bounds_inputs(self) -> bool: + """Specify if decomon inputs should integrate constant bounds.""" + return (self.propagation == Propagation.FORWARD and self.ibp) or ( + self.propagation == Propagation.BACKWARD and self.needs_oracle_bounds() + ) + + def needs_affine_bounds_inputs(self) -> bool: + """Specify if decomon inputs should integrate affine bounds.""" + return (self.propagation == Propagation.FORWARD and self.affine) or (self.propagation == Propagation.BACKWARD) + def cannot_have_empty_affine_inputs(self) -> bool: """Specify that it is not allowed to have empty affine bounds. @@ -281,8 +298,9 @@ def nb_keras_inputs(self) -> int: def nb_input_tensors(self) -> int: nb = 0 if self.propagation == Propagation.BACKWARD: - # ibp - nb += 2 * self.nb_keras_inputs + # oracle bounds + if self.needs_oracle_bounds(): + nb += 2 * self.nb_keras_inputs # affine nb += 4 # model inputs @@ -360,8 +378,9 @@ def split_inputs( Returns: affine_bounds_to_propagate, constant_oracle_bounds, perturbation_domain_inputs: - each one can be empty if not relevant, and according to propagation mode and merging status, - it will be list of tensors or list of list of tensors. + each one can be empty if not relevant, + moreover, according to propagation mode and merging status, + it will be list of tensors or list of lists of tensors. More details: @@ -378,14 +397,13 @@ def split_inputs( ) - backward: only 1 affine bounds to propagate w.r.t keras layer output - + k constant bounds w.r.t each keras layer input - - inputs = ( - affine_bounds_to_propagate - + constant_oracle_bounds_0 + ... + constant_oracle_bounds_k - + perturbation_domain_inputs - ) + + k constant bounds w.r.t each keras layer input (empty if layer not linear) + inputs = ( + affine_bounds_to_propagate + + constant_oracle_bounds_0 + ... + constant_oracle_bounds_k + + perturbation_domain_inputs + ) Note: in case of merging layer + forward, we should not have empty affine bounds as it will be impossible to split properly the inputs. @@ -400,15 +418,18 @@ def split_inputs( if self.is_merging_layer: if self.propagation == Propagation.BACKWARD: # expected number of constant bounds - nb_constant_bounds_by_keras_input = 2 + nb_constant_bounds_by_keras_input = 2 if self.needs_oracle_bounds() else 0 nb_constant_bounds = self.nb_keras_inputs * nb_constant_bounds_by_keras_input # remove affine bounds (could be empty to express identity bounds) affine_bounds_to_propagate = inputs[: len(inputs) - nb_constant_bounds] inputs = inputs[len(inputs) - nb_constant_bounds :] # split constant bounds by keras input - constant_oracle_bounds = [ - [inputs[i], inputs[i + 1]] for i in range(0, len(inputs), nb_constant_bounds_by_keras_input) - ] + if nb_constant_bounds > 0: + constant_oracle_bounds = [ + [inputs[i], inputs[i + 1]] for i in range(0, len(inputs), nb_constant_bounds_by_keras_input) + ] + else: + constant_oracle_bounds = [] else: # forward # split bounds by keras input nb_affine_bounds_by_keras_input = 4 if self.affine else 0 @@ -427,7 +448,7 @@ def split_inputs( ] else: # Remove constant bounds - if self.propagation == Propagation.BACKWARD or self.ibp: + if self.needs_constant_bounds_inputs(): constant_oracle_bounds = inputs[-2:] inputs = inputs[:-2] else: @@ -459,7 +480,7 @@ def split_input_shape( Returns: affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, perturbation_domain_inputs_shape: each one can be empty if not relevant, and according to propagation mode and merging status, - it will be list of shapes or list of list of shapes. + it will be list of shapes or list of lists of shapes. """ return self.split_inputs(inputs=input_shape) # type: ignore @@ -476,14 +497,19 @@ def flatten_inputs( Args: affine_bounds_to_propagate: - - forward + affine: affine bounds on keras layer outputs w.r.t. model input - - backward: affine bounds on model output w.r.t each keras layer input - -> list of list of tensor in merging case; - -> list of tensor else. + - forward + affine: affine bounds on each keras layer input w.r.t. model input + -> list of lists of tensors in merging case; + -> list of tensors else. + - backward: affine bounds on model output w.r.t keras layer output + -> list of tensors + - else: empty + constant_oracle_bounds: + - forward + ibp: ibp bounds on keras layer inputs + - backward + not linear: oracle bounds on keras layer inputs + - else: empty + perturbation_domain_inputs: + - forward + affine: perturbation domain input wrapped in a list - else: empty - constant_bounds_propagated: - - forward + ibp: ibp bounds on keras layer outputs - - else: empty or None Returns: flattened inputs @@ -500,7 +526,7 @@ def flatten_inputs( ) - backward: only 1 affine bounds to propagate w.r.t keras layer output - + k constant bounds w.r.t each keras layer input + + k constant bounds w.r.t each keras layer input (empty of linear layer) inputs = ( affine_bounds_to_propagate @@ -511,9 +537,12 @@ def flatten_inputs( """ if self.is_merging_layer: if self.propagation == Propagation.BACKWARD: - flattened_constant_oracle_bounds = [ - t for constant_oracle_bounds_i in constant_oracle_bounds for t in constant_oracle_bounds_i - ] + if self.needs_oracle_bounds(): + flattened_constant_oracle_bounds = [ + t for constant_oracle_bounds_i in constant_oracle_bounds for t in constant_oracle_bounds_i + ] + else: + flattened_constant_oracle_bounds = [] return affine_bounds_to_propagate + flattened_constant_oracle_bounds + perturbation_domain_inputs else: # forward bounds_by_keras_input = [ @@ -539,7 +568,7 @@ def split_outputs(self, outputs: list[Tensor]) -> tuple[Union[list[Tensor], list Returns: affine_bounds_propagated, constant_bounds_propagated: - each one can be empty if not relevant and can be list of tensors or a list of list of tensors + each one can be empty if not relevant and can be list of tensors or a list of lists of tensors according to propagation and merging status. More details: @@ -547,7 +576,7 @@ def split_outputs(self, outputs: list[Tensor]) -> tuple[Union[list[Tensor], list - forward: affine_bounds_propagated, constant_bounds_propagated: both simple lists of tensors corresponding to affine and constant bounds on keras layer output. - backward: constant_bounds_propagated is empty (not relevant) and - - merging layer: affine_bounds_propagated is a list of list of tensors corresponding + - merging layer: affine_bounds_propagated is a list of lists of tensors corresponding to partial affine bounds on model output w.r.t each keras input - else: affine_bounds_propagated is a simple list of tensors @@ -578,13 +607,13 @@ def flatten_outputs( Args: affine_bounds_propagated: - - forward + affine: affine bounds on keras layer outputs w.r.t. model input + - forward + affine: affine bounds on keras layer output w.r.t. model input - backward: affine bounds on model output w.r.t each keras layer input - -> list of list of tensor in merging case; - -> list of tensor else. + -> list of lists of tensors in merging case; + -> list of tensors else. - else: empty constant_bounds_propagated: - - forward + ibp: ibp bounds on keras layer outputs + - forward + ibp: ibp bounds on keras layer output - else: empty or None Returns: diff --git a/src/decomon/layers/activations/activation.py b/src/decomon/layers/activations/activation.py index cd8e7e88..54bbee2e 100644 --- a/src/decomon/layers/activations/activation.py +++ b/src/decomon/layers/activations/activation.py @@ -92,7 +92,6 @@ def __init__( slope=slope, **kwargs, ) - self.slope = slope decomon_activation_class = get(self.layer.activation) self.decomon_activation = decomon_activation_class( layer=layer, @@ -105,6 +104,10 @@ def __init__( slope=slope, **kwargs, ) + # linearity of the wrapping activation layer is decided by the wrapped activation layer + self.linear = self.decomon_activation.linear + # so do the inputs/outputs format + self.inputs_outputs_spec = self.decomon_activation.inputs_outputs_spec def get_affine_representation(self) -> tuple[Tensor, Tensor]: return self.decomon_activation.get_affine_representation() diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index 08a3298b..960619d5 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -155,6 +155,7 @@ def create_inputs_outputs_spec( model_input_shape=model_input_shape, model_output_shape=model_output_shape, is_merging_layer=self._is_merging, + linear=self.linear, ) @property diff --git a/tests/conftest.py b/tests/conftest.py index f5294115..0396e809 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -148,6 +148,7 @@ def get_decomon_input_shapes( empty=False, diag=False, nobatch=False, + for_linear_layer=False, ): inputs_outputs_spec = InputsOutputsSpec( ibp=ibp, @@ -157,13 +158,14 @@ def get_decomon_input_shapes( layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=model_output_shape, + linear=for_linear_layer, ) if inputs_outputs_spec.needs_perturbation_domain_inputs(): perturbation_domain_inputs_shape = [perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape)] else: perturbation_domain_inputs_shape = [] - if affine and not empty: + if inputs_outputs_spec.needs_affine_bounds_inputs() and not empty: if propagation == Propagation.FORWARD: b_in_shape = layer_input_shape w_in_shape = model_input_shape + layer_input_shape @@ -177,7 +179,7 @@ def get_decomon_input_shapes( else: affine_bounds_to_propagate_shape = [] - if ibp: + if inputs_outputs_spec.needs_constant_bounds_inputs(): constant_oracle_bounds_shape = [layer_input_shape, layer_input_shape] else: constant_oracle_bounds_shape = [] @@ -197,6 +199,7 @@ def get_decomon_symbolic_inputs( empty=False, diag=False, nobatch=False, + for_linear_layer=False, dtype=keras_config.floatx(), ): """Generate decomon symbolic inputs for a decomon layer @@ -232,6 +235,7 @@ def get_decomon_symbolic_inputs( empty=empty, diag=diag, nobatch=nobatch, + for_linear_layer=for_linear_layer, ) perturbation_domain_inputs = [Input(shape, dtype=dtype) for shape in perturbation_domain_inputs_shape] constant_oracle_bounds = [Input(shape, dtype=dtype) for shape in constant_oracle_bounds_shape] @@ -267,6 +271,7 @@ def generate_simple_decomon_layer_inputs_from_keras_input( empty=False, diag=False, nobatch=False, + for_linear_layer=False, dtype=keras_config.floatx(), ): """Generate simple decomon inputs for a layer from the corresponding keras input @@ -303,6 +308,7 @@ def generate_simple_decomon_layer_inputs_from_keras_input( layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=model_output_shape, + linear=for_linear_layer, ) if inputs_outputs_spec.needs_perturbation_domain_inputs(): @@ -313,7 +319,7 @@ def generate_simple_decomon_layer_inputs_from_keras_input( else: perturbation_domain_inputs = [] - if affine and not empty: + if inputs_outputs_spec.needs_affine_bounds_inputs() and not empty: batchsize = keras_input.shape[0] if propagation == Propagation.FORWARD: bias_shape = layer_input_shape @@ -340,7 +346,7 @@ def generate_simple_decomon_layer_inputs_from_keras_input( else: affine_bounds_to_propagate = [] - if ibp: + if inputs_outputs_spec.needs_constant_bounds_inputs(): constant_oracle_bounds = [keras_input, keras_input] else: constant_oracle_bounds = [] @@ -362,7 +368,7 @@ def generate_simple_perturbation_domain_inputs_from_keras_input(keras_input, per @staticmethod def generate_merging_decomon_input_from_single_decomon_inputs( - decomon_inputs: list[list[Tensor]], ibp: bool, affine: bool, propagation: Propagation + decomon_inputs: list[list[Tensor]], ibp: bool, affine: bool, propagation: Propagation, linear: bool ) -> list[Tensor]: inputs_outputs_spec_single = InputsOutputsSpec( ibp=ibp, @@ -371,6 +377,7 @@ def generate_merging_decomon_input_from_single_decomon_inputs( layer_input_shape=tuple(), model_input_shape=tuple(), model_output_shape=tuple(), + linear=linear, ) affine_bounds_to_propagate, constant_oracle_bounds, perturbation_domain_inputs = [], [], [] for decomon_input in decomon_inputs: @@ -394,6 +401,7 @@ def generate_merging_decomon_input_from_single_decomon_inputs( model_input_shape=tuple(), model_output_shape=tuple(), is_merging_layer=True, + linear=linear, ) return inputs_outputs_spec_merging.flatten_inputs( affine_bounds_to_propagate=affine_bounds_to_propagate, @@ -964,7 +972,7 @@ def assert_decomon_output_lower_equal_upper( layer_input_shape = [tuple()] else: layer_input_shape = tuple() - inputs_outputs_specs = InputsOutputsSpec( + inputs_outputs_spec = InputsOutputsSpec( ibp=ibp, affine=affine, propagation=propagation, @@ -972,9 +980,7 @@ def assert_decomon_output_lower_equal_upper( model_output_shape=tuple(), is_merging_layer=is_merging_layer, ) - affine_bounds_propagated, constant_bounds_propagated = inputs_outputs_specs.split_outputs( - outputs=decomon_output - ) + affine_bounds_propagated, constant_bounds_propagated = inputs_outputs_spec.split_outputs(outputs=decomon_output) if propagation == Propagation.BACKWARD or affine: if is_merging_layer and propagation == Propagation.BACKWARD: # one list of affine bounds by keras layer input @@ -1310,7 +1316,7 @@ def simple_layer_input_functions( keras_symbolic_model_input_fn = lambda: Input(input_shape) keras_symbolic_layer_input_fn = lambda keras_symbolic_model_input: keras_symbolic_model_input - decomon_symbolic_input_fn = lambda output_shape: helpers.get_decomon_symbolic_inputs( + decomon_symbolic_input_fn = lambda output_shape, linear: helpers.get_decomon_symbolic_inputs( model_input_shape=input_shape, model_output_shape=output_shape, layer_input_shape=input_shape, @@ -1322,12 +1328,13 @@ def simple_layer_input_functions( empty=empty, diag=diag, nobatch=nobatch, + for_linear_layer=linear, ) keras_model_input_fn = lambda: helpers.generate_random_tensor(input_shape, batchsize=batchsize) keras_layer_input_fn = lambda keras_model_input: keras_model_input - decomon_input_fn = lambda keras_model_input, keras_layer_input, output_shape: helpers.generate_simple_decomon_layer_inputs_from_keras_input( + decomon_input_fn = lambda keras_model_input, keras_layer_input, output_shape, linear: helpers.generate_simple_decomon_layer_inputs_from_keras_input( keras_input=keras_layer_input, layer_output_shape=output_shape, ibp=ibp, @@ -1337,6 +1344,7 @@ def simple_layer_input_functions( empty=empty, diag=diag, nobatch=nobatch, + for_linear_layer=linear, ) return ( @@ -1360,7 +1368,7 @@ def convert_standard_input_functions_for_single_layer( if propagation == Propagation.FORWARD: - def decomon_symbolic_input_fn(output_shape): + def decomon_symbolic_input_fn(output_shape, linear): x, y, z, u_c, w_u, b_u, l_c, w_l, b_l = get_tensor_decomposition_fn() layer_input_shape = y.shape[1:] model_input_shape = x.shape[1:] @@ -1373,6 +1381,7 @@ def decomon_symbolic_input_fn(output_shape): layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=output_shape, + linear=linear, ) if affine: @@ -1399,7 +1408,7 @@ def decomon_symbolic_input_fn(output_shape): perturbation_domain_inputs=perturbation_domain_inputs, ) - def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): + def decomon_input_fn(keras_model_input, keras_layer_input, output_shape, linear): x, y, z, u_c, w_u, b_u, l_c, w_l, b_l = get_standard_values_fn() layer_input_shape = tuple(y.shape[1:]) model_input_shape = tuple(x.shape[1:]) @@ -1412,6 +1421,7 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=output_shape, + linear=linear, ) if affine: @@ -1440,7 +1450,7 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): else: # backward - def decomon_symbolic_input_fn(output_shape): + def decomon_symbolic_input_fn(output_shape, linear): x, y, z, u_c, w_u, b_u, l_c, w_l, b_l = get_tensor_decomposition_fn() layer_input_shape = y.shape[1:] model_input_shape = x.shape[1:] @@ -1453,9 +1463,10 @@ def decomon_symbolic_input_fn(output_shape): layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=output_shape, + linear=linear, ) - if ibp: + if inputs_outputs_spec.needs_constant_bounds_inputs(): constant_oracle_bounds = [l_c, u_c] else: constant_oracle_bounds = [] @@ -1469,7 +1480,7 @@ def decomon_symbolic_input_fn(output_shape): perturbation_domain_inputs = [] # take identity affine bounds - if affine: + if inputs_outputs_spec.needs_affine_bounds_inputs(): simple_decomon_inputs = helpers.get_decomon_symbolic_inputs( model_input_shape=model_input_shape, model_output_shape=output_shape, @@ -1493,7 +1504,7 @@ def decomon_symbolic_input_fn(output_shape): perturbation_domain_inputs=perturbation_domain_inputs, ) - def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): + def decomon_input_fn(keras_model_input, keras_layer_input, output_shape, linear): x, y, z, u_c, w_u, b_u, l_c, w_l, b_l = get_standard_values_fn() layer_input_shape = tuple(y.shape[1:]) model_input_shape = tuple(x.shape[1:]) @@ -1506,9 +1517,10 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=output_shape, + linear=linear, ) - if ibp: + if inputs_outputs_spec.needs_constant_bounds_inputs(): constant_oracle_bounds = [K.convert_to_tensor(a) for a in (l_c, u_c)] else: constant_oracle_bounds = [] @@ -1522,7 +1534,7 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): perturbation_domain_inputs = [] #  take identity affine bounds - if affine: + if inputs_outputs_spec.needs_affine_bounds_inputs(): simple_decomon_inputs = helpers.generate_simple_decomon_layer_inputs_from_keras_input( keras_input=keras_layer_input, layer_output_shape=output_shape, @@ -1533,6 +1545,7 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape): empty=empty, diag=diag, nobatch=nobatch, + for_linear_layer=linear, ) affine_bounds_to_propagate, _, _ = inputs_outputs_spec.split_inputs(simple_decomon_inputs) else: diff --git a/tests/test_decomon_layer.py b/tests/test_decomon_layer.py index f141acd8..a2de2cdd 100644 --- a/tests/test_decomon_layer.py +++ b/tests/test_decomon_layer.py @@ -82,7 +82,6 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper model_output_shape = model_output_shape_if_no_singlelayer_model model_input_shape = (model_input_dim,) - x_shape = perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape) # keras layer layer = Dense(units=layer_output_dim) @@ -98,7 +97,7 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper model_output_shape=model_output_shape, model_input_shape=model_input_shape, ) - non_linear_decomon_layer = MyNonLinearDecomonDense1d( + nonlinear_decomon_layer = MyNonLinearDecomonDense1d( layer=layer, ibp=ibp, affine=affine, @@ -109,7 +108,7 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper ) # symbolic inputs - decomon_inputs = helpers.get_decomon_symbolic_inputs( + decomon_inputs_linear = helpers.get_decomon_symbolic_inputs( model_input_shape=model_input_shape, model_output_shape=model_output_shape, layer_input_shape=layer_input_shape, @@ -118,14 +117,26 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper affine=affine, propagation=propagation, perturbation_domain=perturbation_domain, + for_linear_layer=True, ) + decomon_inputs_nonlinear = helpers.get_decomon_symbolic_inputs( + model_input_shape=model_input_shape, + model_output_shape=model_output_shape, + layer_input_shape=layer_input_shape, + layer_output_shape=layer_output_shape, + ibp=ibp, + affine=affine, + propagation=propagation, + perturbation_domain=perturbation_domain, + for_linear_layer=False, + ) + + # actual (random) tensors + expected output shapes ( affine_bounds_to_propagate, constant_oracle_bounds, perturbation_domain_inputs, - ) = linear_decomon_layer.inputs_outputs_spec.split_inputs(decomon_inputs) - - # actual (random) tensors + expected output shapes + ) = nonlinear_decomon_layer.inputs_outputs_spec.split_inputs(decomon_inputs_nonlinear) perturbation_domain_inputs_val = [ helpers.generate_random_tensor(x.shape[1:], batchsize=batchsize) for x in perturbation_domain_inputs ] @@ -155,7 +166,18 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper propagated_ibp_bounds_expected_shape = [] constant_oracle_bounds_val = [] - decomon_inputs_val = linear_decomon_layer.inputs_outputs_spec.flatten_inputs( + # linear case only: no constant bounds in backward + if linear_decomon_layer.inputs_outputs_spec.needs_constant_bounds_inputs(): + constant_oracle_bounds_val_linear = constant_oracle_bounds_val + else: + constant_oracle_bounds_val_linear = [] + + decomon_inputs_val_linear = linear_decomon_layer.inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=affine_bounds_to_propagate_val, + constant_oracle_bounds=constant_oracle_bounds_val_linear, + perturbation_domain_inputs=perturbation_domain_inputs_val, + ) + decomon_inputs_val_nonlinear = nonlinear_decomon_layer.inputs_outputs_spec.flatten_inputs( affine_bounds_to_propagate=affine_bounds_to_propagate_val, constant_oracle_bounds=constant_oracle_bounds_val, perturbation_domain_inputs=perturbation_domain_inputs_val, @@ -167,27 +189,27 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper decomon_output_expected_shapes = propagated_affine_bounds_expected_shape # symbolic call - linear_decomon_output = linear_decomon_layer(decomon_inputs) - non_linear_decomon_output = non_linear_decomon_layer(decomon_inputs) + linear_decomon_output = linear_decomon_layer(decomon_inputs_linear) + nonlinear_decomon_output = nonlinear_decomon_layer(decomon_inputs_nonlinear) # shapes ok ? linear_decomon_output_shape_from_call = [tensor.shape[1:] for tensor in linear_decomon_output] assert linear_decomon_output_shape_from_call == decomon_output_expected_shapes - non_linear_decomon_output_shape_from_call = [tensor.shape[1:] for tensor in non_linear_decomon_output] - assert non_linear_decomon_output_shape_from_call == decomon_output_expected_shapes + nonlinear_decomon_output_shape_from_call = [tensor.shape[1:] for tensor in nonlinear_decomon_output] + assert nonlinear_decomon_output_shape_from_call == decomon_output_expected_shapes # actual call - linear_decomon_output_val = linear_decomon_layer(decomon_inputs_val) - non_linear_decomon_output_val = non_linear_decomon_layer(decomon_inputs_val) + linear_decomon_output_val = linear_decomon_layer(decomon_inputs_val_linear) + nonlinear_decomon_output_val = nonlinear_decomon_layer(decomon_inputs_val_nonlinear) # shapes ok ? linear_decomon_output_shape_from_call = [tensor.shape[1:] for tensor in linear_decomon_output_val] assert linear_decomon_output_shape_from_call == decomon_output_expected_shapes - non_linear_decomon_output_shape_from_call = [tensor.shape[1:] for tensor in linear_decomon_output_val] - assert non_linear_decomon_output_shape_from_call == decomon_output_expected_shapes + nonlinear_decomon_output_shape_from_call = [tensor.shape[1:] for tensor in linear_decomon_output_val] + assert nonlinear_decomon_output_shape_from_call == decomon_output_expected_shapes # same values ? - helpers.assert_decomon_outputs_equal(linear_decomon_output_val, non_linear_decomon_output_val) + helpers.assert_decomon_outputs_equal(linear_decomon_output_val, nonlinear_decomon_output_val) # inequalities hold ? # We only check it in case of a single-layer model => model shapes == layer shapes @@ -204,11 +226,27 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper affine=affine, propagation=propagation, perturbation_domain=perturbation_domain, + for_linear_layer=False, ) + # linear case only: no constant bounds in backward + if linear_decomon_layer.inputs_outputs_spec.needs_constant_bounds_inputs(): + decomon_inputs_val_linear = decomon_inputs_val + else: + ( + affine_bounds_to_propagate_val, + constant_oracle_bounds_val, + perturbation_domain_inputs_val, + ) = nonlinear_decomon_layer.inputs_outputs_spec.split_inputs(decomon_inputs_val) + decomon_inputs_val_linear = linear_decomon_layer.inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=affine_bounds_to_propagate_val, + constant_oracle_bounds=[], + perturbation_domain_inputs=perturbation_domain_inputs_val, + ) + # decomon call - linear_decomon_output_val = linear_decomon_layer(decomon_inputs_val) - non_linear_decomon_output_val = non_linear_decomon_layer(decomon_inputs_val) + linear_decomon_output_val = linear_decomon_layer(decomon_inputs_val_linear) + nonlinear_decomon_output_val = nonlinear_decomon_layer(decomon_inputs_val) # keras call keras_output_val = layer(keras_input_val) @@ -222,7 +260,7 @@ def test_my_decomon_dense_1d(singlelayer_model, ibp, affine, propagation, helper affine=affine, propagation=propagation, ) - helpers.assert_decomon_outputs_equal(linear_decomon_output_val, non_linear_decomon_output_val) + helpers.assert_decomon_outputs_equal(linear_decomon_output_val, nonlinear_decomon_output_val) @pytest.mark.parametrize("affine", [True]) @@ -253,7 +291,7 @@ def test_check_affine_bounds_characteristics( # init + build decomon layer output_shape = layer.output.shape[1:] model_output_shape = output_shape - decomon_symbolic_inputs = simple_decomon_symbolic_input_fn(output_shape=output_shape) + decomon_symbolic_inputs = simple_decomon_symbolic_input_fn(output_shape=output_shape, linear=False) decomon_layer = DecomonLayer( layer=layer, ibp=ibp, @@ -267,7 +305,10 @@ def test_check_affine_bounds_characteristics( keras_model_input = simple_keras_model_input_fn() keras_layer_input = simple_keras_layer_input_fn(keras_model_input) decomon_inputs = simple_decomon_input_fn( - keras_model_input=keras_model_input, keras_layer_input=keras_layer_input, output_shape=output_shape + keras_model_input=keras_model_input, + keras_layer_input=keras_layer_input, + output_shape=output_shape, + linear=False, ) if affine: diff --git a/tests/test_merge_layers.py b/tests/test_merge_layers.py index 85151ea7..0d6b6514 100644 --- a/tests/test_merge_layers.py +++ b/tests/test_merge_layers.py @@ -127,10 +127,6 @@ def test_decomon_merge( output_shape = layer.output.shape[1:] model_output_shape = output_shape model_input_shape = keras_symbolic_model_input.shape[1:] - decomon_symbolic_inputs_0 = decomon_symbolic_input_fn(output_shape=output_shape) - decomon_symbolic_inputs = helpers.generate_merging_decomon_input_from_single_decomon_inputs( - decomon_inputs=double_input(decomon_symbolic_inputs_0), ibp=ibp, affine=affine, propagation=propagation - ) decomon_layer = decomon_layer_class( layer=layer, @@ -142,6 +138,16 @@ def test_decomon_merge( model_input_shape=model_input_shape, **decomon_layer_kwargs, ) + + decomon_symbolic_inputs_0 = decomon_symbolic_input_fn(output_shape=output_shape, linear=decomon_layer.linear) + decomon_symbolic_inputs = helpers.generate_merging_decomon_input_from_single_decomon_inputs( + decomon_inputs=double_input(decomon_symbolic_inputs_0), + ibp=ibp, + affine=affine, + propagation=propagation, + linear=decomon_layer.linear, + ) + # skip if empty affine bounds in forward propagation (it would generate issues to split inputs) if ( decomon_layer.inputs_outputs_spec.cannot_have_empty_affine_inputs() @@ -155,11 +161,18 @@ def test_decomon_merge( keras_model_input = keras_model_input_fn() keras_layer_input_0 = keras_layer_input_fn(keras_model_input) decomon_inputs_0 = decomon_input_fn( - keras_model_input=keras_model_input, keras_layer_input=keras_layer_input_0, output_shape=output_shape + keras_model_input=keras_model_input, + keras_layer_input=keras_layer_input_0, + output_shape=output_shape, + linear=decomon_layer.linear, ) keras_layer_input = double_input(keras_layer_input_0) decomon_inputs = helpers.generate_merging_decomon_input_from_single_decomon_inputs( - decomon_inputs=double_input(decomon_inputs_0), ibp=ibp, affine=affine, propagation=propagation + decomon_inputs=double_input(decomon_inputs_0), + ibp=ibp, + affine=affine, + propagation=propagation, + linear=decomon_layer.linear, ) keras_output = layer(keras_layer_input) diff --git a/tests/test_unary_layers.py b/tests/test_unary_layers.py index 8c8ad010..3a5d4cd7 100644 --- a/tests/test_unary_layers.py +++ b/tests/test_unary_layers.py @@ -5,6 +5,7 @@ from decomon.keras_utils import batch_multid_dot from decomon.layers import DecomonActivation, DecomonDense +from decomon.layers.activations.activation import DecomonLinear @fixture @@ -67,7 +68,7 @@ def test_decomon_unary_layer( output_shape = layer.output.shape[1:] model_output_shape = output_shape model_input_shape = keras_symbolic_model_input.shape[1:] - decomon_symbolic_inputs = decomon_symbolic_input_fn(output_shape=output_shape) + decomon_layer = decomon_layer_class( layer=layer, ibp=ibp, @@ -78,20 +79,28 @@ def test_decomon_unary_layer( model_input_shape=model_input_shape, **decomon_layer_kwargs, ) + + decomon_symbolic_inputs = decomon_symbolic_input_fn(output_shape=output_shape, linear=decomon_layer.linear) decomon_layer(decomon_symbolic_inputs) # call on actual inputs keras_model_input = keras_model_input_fn() keras_layer_input = keras_layer_input_fn(keras_model_input) decomon_inputs = decomon_input_fn( - keras_model_input=keras_model_input, keras_layer_input=keras_layer_input, output_shape=output_shape + keras_model_input=keras_model_input, + keras_layer_input=keras_layer_input, + output_shape=output_shape, + linear=decomon_layer.linear, ) keras_output = layer(keras_layer_input) decomon_output = decomon_layer(decomon_inputs) - # check affine representation is ok - if decomon_layer.linear: + # check affine representation is ok (except for linear activation, undefined) + if decomon_layer.linear and not ( + (isinstance(decomon_layer, DecomonActivation) and isinstance(decomon_layer.decomon_activation, DecomonLinear)) + or isinstance(decomon_layer, DecomonLinear) + ): w, b = decomon_layer.get_affine_representation() diagonal = (False, w.shape == b.shape) missing_batchsize = (False, True) From 61680624cf00abc61839a7136db6c5c0bc035c0a Mon Sep 17 00:00:00 2001 From: Nolwen Date: Wed, 21 Feb 2024 22:04:45 +0100 Subject: [PATCH 057/101] Update backward conversion and test it Major changes: - oracle bounds are computed "on demand": we do not ask for oracle bounds for node that do not need them (with linear layers) to avoid in particular some sub-crowns when in full crown mode - oracle bounds are computed via `call_oracle()` of corresponding forward layer for forwad oracle or corresponding backward layer for crown oracle - we store backward layers by sub-crown as layer_fn depends on model_output_shape in use (shape of current output node considered to start the crown) Still wip: - merging layers - submodels --- src/decomon/core.py | 44 +- src/decomon/models/backward_cloning.py | 803 +++++++++++++------------ tests/test_convert_backward.py | 102 ++++ 3 files changed, 559 insertions(+), 390 deletions(-) create mode 100644 tests/test_convert_backward.py diff --git a/src/decomon/core.py b/src/decomon/core.py index c8024eef..f7a07bfa 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -6,7 +6,7 @@ import numpy as np from keras.config import floatx -from decomon.keras_utils import batch_multid_dot +from decomon.keras_utils import add_tensors, batch_multid_dot from decomon.types import Tensor @@ -732,6 +732,48 @@ def is_wo_batch_bounds_shape( else: return len(b_shape) == len(self.model_output_shape) + def sum_backward_bounds(self, affine_bounds_list: list[list[Tensor]]): + """Reduce a list of partial affine bounds on model output w.r.t a same model input by summing them. + + The complication come from the fact that some bounds can be empty, diag or w/o batchsize. + + Args: + affine_bounds_list: + + Returns: + + """ + if self.propagation != Propagation.BACKWARD: + raise NotImplementedError() + if len(affine_bounds_list) == 0: + raise ValueError("affine_bounds_list should not be empty") + + affine_bounds = affine_bounds_list[0] + for affine_bounds_i in affine_bounds_list[1:]: + # identity: put on diag + w/o batchsize form to be able to sum + missing_batchsize = (self.is_wo_batch_bounds(affine_bounds), self.is_wo_batch_bounds(affine_bounds_i)) + diagonal = (self.is_diagonal_bounds(affine_bounds), self.is_diagonal_bounds(affine_bounds_i)) + if len(affine_bounds) == 0: + w = K.ones(self.model_output_shape) + b = K.zeros(self.model_output_shape) + w_l, b_l, w_u, b_u = w, b, w, b + else: + w_l, b_l, w_u, b_u = affine_bounds + if len(affine_bounds_i) == 0: + w = K.ones(self.model_output_shape) + b = K.zeros(self.model_output_shape) + w_l_i, b_l_i, w_u_i, b_u_i = w, b, w, b + else: + w_l_i, b_l_i, w_u_i, b_u_i = affine_bounds_i + w_l = add_tensors(w_l, w_l_i, missing_batchsize=missing_batchsize, diagonal=diagonal) + w_u = add_tensors(w_u, w_u_i, missing_batchsize=missing_batchsize, diagonal=diagonal) + b_l = add_tensors(b_l, b_l_i, missing_batchsize=missing_batchsize) + b_u = add_tensors(b_u, b_u_i, missing_batchsize=missing_batchsize) + + affine_bounds = w_l, b_l, w_u, b_u + + return affine_bounds + def get_kerasinputshape(self, inputsformode: list[Tensor]) -> tuple[Optional[int], ...]: return inputsformode[-1].shape diff --git a/src/decomon/models/backward_cloning.py b/src/decomon/models/backward_cloning.py index 89137c77..36664bbb 100644 --- a/src/decomon/models/backward_cloning.py +++ b/src/decomon/models/backward_cloning.py @@ -1,463 +1,488 @@ -from collections.abc import Callable from copy import deepcopy +from collections.abc import Callable from typing import Any, Optional, Union import keras import keras.ops as K from keras.config import floatx -from keras.layers import Concatenate, Lambda, Layer +from keras.layers import Lambda, Layer from keras.models import Model from keras.src.ops.node import Node from keras.src.utils.python_utils import to_list -from decomon.backward_layers.backward_merge import BackwardMerge -from decomon.backward_layers.convert import to_backward -from decomon.backward_layers.core import BackwardLayer from decomon.core import ( BoxDomain, ForwardMode, InputsOutputsSpec, PerturbationDomain, + Propagation, Slope, get_affine, get_mode, ) -from decomon.keras_utils import BatchedIdentityLike -from decomon.layers.utils import softmax_to_linear as softmax_2_linear +from decomon.layers import DecomonLayer +from decomon.layers.convert import to_decomon +from decomon.layers.merging.base_merge import DecomonMerge from decomon.models.crown import Convert2BackwardMode, Fuse, MergeWithPrevious -from decomon.models.forward_cloning import OutputMapDict from decomon.models.utils import ( - Convert2Mode, ensure_functional_model, get_depth_dict, - get_input_dim, + get_output_nodes, ) -from decomon.types import BackendTensor, Tensor - +from decomon.types import Tensor -def get_disconnected_input( - mode: Union[str, ForwardMode], - perturbation_domain: PerturbationDomain, - dtype: Optional[str] = None, -) -> Layer: - mode = ForwardMode(mode) - dc_decomp = False - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - affine = get_affine(mode) - if dtype is None: - dtype = floatx() - - def disco_priv(inputs: list[Tensor]) -> list[Tensor]: - x, u_c, w_f_u, b_f_u, l_c, w_f_l, b_f_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - dtype = x.dtype - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - if affine: - x = K.concatenate([K.expand_dims(l_c, 1), K.expand_dims(u_c, 1)], 1) - w_u = BatchedIdentityLike()(u_c) - b_u = K.zeros_like(u_c) - else: - w_u, b_u = empty_tensor, empty_tensor - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs([x, u_c, w_u, b_u, l_c, w_u, b_u]) - - return Lambda(disco_priv, dtype=dtype) - - -def retrieve_layer( +def crown( node: Node, - layer_fn: Callable[[Layer], BackwardLayer], - backward_map: dict[int, BackwardLayer], - joint: bool = True, -) -> BackwardLayer: - if id(node) in backward_map: - backward_layer = backward_map[id(node)] - else: - backward_layer = layer_fn(node.operation) - if joint: - backward_map[id(node)] = backward_layer - return backward_layer - - -def crown_( - node: Node, - ibp: bool, - affine: bool, - perturbation_domain: PerturbationDomain, - input_map: dict[int, list[keras.KerasTensor]], - layer_fn: Callable[[Layer], BackwardLayer], + layer_fn: Callable[[Layer, tuple[int, ...]], DecomonLayer], + model_output_shape: tuple[int, ...], backward_bounds: list[keras.KerasTensor], - backward_map: Optional[dict[int, BackwardLayer]] = None, - joint: bool = True, - fuse: bool = True, - output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, - merge_layers: Optional[Layer] = None, - fuse_layer: Optional[Layer] = None, - **kwargs: Any, -) -> tuple[list[keras.KerasTensor], Optional[Layer]]: + backward_map: dict[int, DecomonLayer], + oracle_map: dict[int, Union[list[keras.KerasTensor], list[list[keras.KerasTensor]]]], + forward_output_map: dict[int, list[keras.KerasTensor]], + forward_layer_map: dict[int, DecomonLayer], + crown_output_map: dict[int, list[keras.KerasTensor]], + perturbation_domain_input: keras.KerasTensor, + perturbation_domain: PerturbationDomain, +) -> list[keras.KerasTensor]: """ + Args: + node: node of the model until which the backward bounds have been propagated + layer_fn: function converting a keras layer into its backward version, + according to the proper (sub)model output shape + Will be passed to `get_oracle()` for crown oracle deriving from sub-crowns + model_output_shape: model_output_shape of the current crown, + to be passed to `crown_model()` on embedded submodels. + backward_bounds: backward bounds to propagate + oracle_map: oracle bounds on inputs of each keras layer, stored by layers id + oracle_map: already registered oracle bounds per node + To be used by `get_oracle()`. + forward_output_map: forward outputs per node from a previously performed forward conversion. + To be used by `get_oracle()`. + forward_layer_map: forward decomon layer per node from a previously performed forward conversion. + To be used by `get_oracle()`. + crown_output_map: output of subcrowns per output node. + Avoids relaunching a crown if several nodes share parents. + To be used by `get_oracle()`. + backward_map: stores converted layer by node for the current crown + (should depend on the proper model output and thus change for each sub-crown) + + Returns: + propagated backward bounds until model input for the proper output node - :param node: - :param ibp: - :param affine: - :param input_map: - :param layer_fn: - :param backward_bounds: - :param backward_map: - :param joint: - :param fuse: - :return: list of 4 tensors affine upper and lower bounds """ - if backward_map is None: - backward_map = {} - - if output_map is None: - output_map = {} - - inputs = input_map[id(node)] - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - if isinstance(node.operation, Model): - inputs_tensors = get_disconnected_input(get_mode(ibp, affine), perturbation_domain, dtype=inputs[0].dtype)( - inputs - ) - _, backward_bounds, _, _ = crown_model( - model=node.operation, - input_tensors=inputs_tensors, - backward_bounds=backward_bounds, - ibp=ibp, - affine=affine, - perturbation_domain=None, - finetune=False, - joint=joint, - fuse=False, - **kwargs, - ) - + parents = node.parent_nodes + if len(parents) == 0: + # Input layer => no conversion, propagate output unchanged + return backward_bounds else: - backward_layer = retrieve_layer(node=node, layer_fn=layer_fn, backward_map=backward_map, joint=joint) - - if id(node) not in output_map: - backward_bounds_new = backward_layer(inputs) - output_map[id(node)] = backward_bounds_new + if isinstance(node.operation, Model): + # TO CHECK perturbation input submodel ? + # forward oracle: + # - ibp (+-affine): not needed + # - affine (w/o ibp): forward_output give affine bounds on outer model input => perturbation_domain + # crown oracle: + # - see where stop subcrowns, maybe to submodel inputs, + # in which case we need to construct perturbation_domain_input_submodel from get_oracle(node) + perturbation_domain_input_submodel = perturbation_domain_input + backward_bounds = crown_model( + model=node.operation, + layer_fn=layer_fn, + backward_bounds=[backward_bounds], + perturbation_domain_input=perturbation_domain_input_submodel, + perturbation_domain=perturbation_domain, + oracle_map=oracle_map, + forward_output_map=forward_output_map, + forward_layer_map=forward_layer_map, + crown_output_map=crown_output_map, + is_submodel=True, + backward_map=backward_map, + model_output_shape=model_output_shape, + ) else: - backward_bounds_new = output_map[id(node)] + if id(node) in backward_map: + backward_layer = backward_map[id(node)] + else: + backward_layer = layer_fn(node.operation, model_output_shape) + backward_map[id(node)] = backward_layer - # import pdb; pdb.set_trace() - if len(backward_bounds): - if merge_layers is None: - merge_layers = MergeWithPrevious(backward_bounds_new[0].shape, backward_bounds[0].shape) - backward_bounds = merge_layers(backward_bounds_new + backward_bounds) - else: - backward_bounds = backward_bounds_new + # get oracle bounds if needed + if backward_layer.inputs_outputs_spec.needs_oracle_bounds(): + constant_oracle_bounds = get_oracle( + node=node, + perturbation_domain_input=perturbation_domain_input, + perturbation_domain=perturbation_domain, + oracle_map=oracle_map, + forward_output_map=forward_output_map, + forward_layer_map=forward_layer_map, + backward_layer=backward_layer, + crown_output_map=crown_output_map, + layer_fn=layer_fn, + ) - parents = node.parent_nodes + else: + constant_oracle_bounds = [] - if len(parents): - if len(parents) > 1: - if isinstance(backward_layer, BackwardMerge): - raise NotImplementedError() - crown_bound_list = [] - for backward_bound, parent in zip(backward_bounds, parents): - crown_bound_i, _ = crown_( + # propagate backward bounds through the decomon layer + backward_layer_inputs = backward_layer.inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=backward_bounds, + constant_oracle_bounds=constant_oracle_bounds, + perturbation_domain_inputs=[], + ) + backward_layer_outputs = backward_layer(backward_layer_inputs) + backward_bounds, _ = backward_layer.inputs_outputs_spec.split_outputs(backward_layer_outputs) + + # Call crown recursively on parent nodes + if isinstance(backward_layer, DecomonMerge): + # merging layer + crown_bounds_list: list[list[Tensor]] = [] + for backward_bounds_i, parent in zip(backward_bounds, parents): + crown_bounds_list.append( + crown( node=parent, - ibp=ibp, - affine=affine, - perturbation_domain=perturbation_domain, - input_map=input_map, layer_fn=layer_fn, - backward_bounds=backward_bound, + model_output_shape=model_output_shape, + backward_bounds=backward_bounds_i, backward_map=backward_map, - joint=joint, - fuse=fuse, + oracle_map=oracle_map, + forward_output_map=forward_output_map, + forward_layer_map=forward_layer_map, + crown_output_map=crown_output_map, + perturbation_domain_input=perturbation_domain_input, + perturbation_domain=perturbation_domain, ) + ) + # reduce by summing all bounds together + # (indeed all bounds are partial affine bounds on model output w.r.t the same model input + # under the hypotheses of a single model input) + crown_bounds = backward_layer.inputs_outputs_spec.sum_backward_bounds(crown_bounds_list) - crown_bound_list.append(crown_bound_i) - - # avg_layer = Average(dtype=node.outbound_layer.dtype) - # crown_bound = [avg_layer([e[i] for e in crown_bound_list]) for i in range(4)] - crown_bound = crown_bound_list[0] - - else: - raise NotImplementedError() + elif len(parents) > 1: + raise RuntimeError("Node with multiple parents should have been converted to a DecomonMerge layer.") else: - crown_bound, fuse_layer_new = crown_( + # unary layer + crown_bounds = crown( node=parents[0], - ibp=ibp, - affine=affine, - perturbation_domain=perturbation_domain, - input_map=input_map, layer_fn=layer_fn, + model_output_shape=model_output_shape, backward_bounds=backward_bounds, backward_map=backward_map, - joint=joint, - fuse=fuse, - output_map=output_map, - merge_layers=None, # AKA merge_layers - fuse_layer=fuse_layer, + oracle_map=oracle_map, + forward_output_map=forward_output_map, + forward_layer_map=forward_layer_map, + crown_output_map=crown_output_map, + perturbation_domain_input=perturbation_domain_input, + perturbation_domain=perturbation_domain, ) - if fuse_layer is None: - fuse_layer = fuse_layer_new - return crown_bound, fuse_layer - else: - # do something - if fuse: - if fuse_layer is None: - fuse_layer = Fuse(get_mode(ibp=ibp, affine=affine)) - result = fuse_layer(inputs + backward_bounds) + return crown_bounds - return result, fuse_layer - else: - return backward_bounds, fuse_layer - - -def get_input_nodes( - model: Model, - dico_nodes: dict[int, list[Node]], - ibp: bool, - affine: bool, - input_tensors: list[keras.KerasTensor], - output_map: OutputMapDict, - layer_fn: Callable[[Layer], BackwardLayer], - joint: bool, - set_mode_layer: Layer, - perturbation_domain: Optional[PerturbationDomain] = None, - **kwargs: Any, -) -> tuple[dict[int, list[keras.KerasTensor]], dict[int, BackwardLayer], dict[int, list[keras.KerasTensor]]]: - keys = [e for e in dico_nodes.keys()] - keys.sort(reverse=True) - fuse_layer = None - input_map: dict[int, list[keras.KerasTensor]] = {} - backward_map: dict[int, BackwardLayer] = {} - if perturbation_domain is None: - perturbation_domain = BoxDomain() - crown_map: dict[int, list[keras.KerasTensor]] = {} - for depth in keys: - nodes = dico_nodes[depth] - for node in nodes: - layer = node.operation - - parents = node.parent_nodes - if not len(parents): - # if 'debug' in kwargs.keys(): - # import pdb; pdb.set_trace() - input_map[id(node)] = input_tensors - else: - output: list[keras.KerasTensor] = [] - for parent in parents: - # do something - if id(parent) in output_map.keys(): - output += output_map[id(parent)] - else: - output_crown, fuse_layer_tmp = crown_( - node=parent, - ibp=ibp, - affine=affine, - input_map=input_map, - layer_fn=layer_fn, - backward_bounds=[], - backward_map=backward_map, - joint=joint, - fuse=True, - perturbation_domain=perturbation_domain, - output_map=crown_map, - merge_layers=None, # AKA merge_layers - fuse_layer=fuse_layer, - ) - if fuse_layer is None: - fuse_layer = fuse_layer_tmp - - # convert output_crown in the right mode - if set_mode_layer is None: - set_mode_layer = Convert2BackwardMode(get_mode(ibp, affine), perturbation_domain) - output_crown = set_mode_layer(input_tensors + output_crown) - output += to_list(output_crown) - # crown_map[id(parent)]=output_crown_ - - input_map[id(node)] = output - return input_map, backward_map, crown_map +def get_oracle( + node: Node, + perturbation_domain_input: keras.KerasTensor, + perturbation_domain: PerturbationDomain, + oracle_map: dict[int, Union[list[keras.KerasTensor], list[list[keras.KerasTensor]]]], + forward_output_map: dict[int, list[keras.KerasTensor]], + forward_layer_map: dict[int, DecomonLayer], + backward_layer: DecomonLayer, + crown_output_map: dict[int, list[keras.KerasTensor]], + layer_fn: Callable[[Layer, tuple[int, ...]], DecomonLayer], +) -> Union[list[keras.KerasTensor], list[list[keras.KerasTensor]]]: + """Get oracle bounds "on demand". + + When needed by a node, get oracle constant bounds on keras layer inputs either: + - from `oracle_map`, if already computed + - from forward oracle, when a first forward conversion has been done + - from crown oracle, by launching sub-crowns on parent nodes + + Args: + node: considered node whose operation (keras layer) needs oracle bounds + perturbation_domain_input: perturbation domain input + perturbation_domain: perturbation domain type on keras model input + oracle_map: already registered oracle bounds per node + forward_output_map: forward outputs per node from a previously performed forward conversion. + To be used for forward oracle. + forward_layer_map: forward decomon layer per node from a previously performed forward conversion. + To be used for forward oracle. + backward_layer: converted backward decomon layer for this node. + To be used for crown oracle. + crown_output_map: output of subcrowns per output node. + Avoids relaunching a crown if several nodes share parents. + To be used for crown oracle. + layer_fn: callable converting a layer and a model_output_shape into a (backward) decomon layer. + To be used for crown oracle. + + Returns: + oracle bounds on node inputs + """ + # Do not recompute if already existing + if id(node) in oracle_map: + return oracle_map[id(node)] -def crown_model( - model: Model, - input_tensors: list[keras.KerasTensor], - back_bounds: Optional[list[keras.KerasTensor]] = None, - slope: Union[str, Slope] = Slope.V_SLOPE, - ibp: bool = True, - affine: bool = True, - perturbation_domain: Optional[PerturbationDomain] = None, - finetune: bool = False, - forward_map: Optional[OutputMapDict] = None, - softmax_to_linear: bool = True, - joint: bool = True, - layer_fn: Callable[..., BackwardLayer] = to_backward, - fuse: bool = True, - **kwargs: Any, -) -> tuple[list[keras.KerasTensor], list[keras.KerasTensor], dict[int, BackwardLayer], None]: - if back_bounds is None: - back_bounds = [] - if forward_map is None: - forward_map = {} - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if not isinstance(model, Model): - raise ValueError() - # import time - # zero_time = time.process_time() - has_softmax = False - if softmax_to_linear: - model, has_softmax = softmax_2_linear(model) # do better because you modify the model eventually - - # layer_fn - ########## - has_iter = False - if layer_fn is not None and len(layer_fn.__code__.co_varnames) == 1 and "layer" in layer_fn.__code__.co_varnames: - has_iter = True - - if not has_iter: - layer_fn_copy = deepcopy(layer_fn) - - def func(layer: Layer) -> Layer: - return layer_fn_copy( - layer, - mode=get_mode(ibp, affine), - finetune=finetune, - perturbation_domain=perturbation_domain, - slope=slope, - ) + parents = node.parent_nodes + if not len(parents): + # input node: can be deduced directly from perturbation domain + oracle_bounds = [ + perturbation_domain.get_lower_x(x=perturbation_domain_input), + perturbation_domain.get_upper_x(x=perturbation_domain_input), + ] + else: + if id(node) in forward_layer_map: + # forward oracle + forward_layer = forward_layer_map[id(node)] + forward_input: list[keras.KerasTensor] = [] + for parent in parents: + forward_input += forward_output_map[id(parent)] + if forward_layer.inputs_outputs_spec.needs_perturbation_domain_inputs(): + forward_input += [perturbation_domain_input] + oracle_bounds = forward_layer.call_oracle(forward_input) + else: + # crown oracle + crown_bounds = [] + for parent in parents: + if id(parent) in crown_output_map: + # already computed sub-crown? + crown_bounds_parent = crown_output_map[id(parent)] + else: + submodel_output_shape = get_model_output_shape(node=parent, backward_bounds_node=[]) + crown_bounds_parent = crown( + node=parent, + layer_fn=layer_fn, + model_output_shape=submodel_output_shape, + backward_bounds=[], + backward_map={}, # new output node, thus new backward_map + oracle_map=oracle_map, + forward_output_map=forward_output_map, + forward_layer_map=forward_layer_map, + crown_output_map=crown_output_map, + perturbation_domain_input=perturbation_domain_input, + perturbation_domain=perturbation_domain, + ) + # store sub-crown output + crown_output_map[id(parent)] = crown_bounds_parent + crown_bounds += crown_bounds_parent - layer_fn = func + # deduce oracle bounds from affine bounds on keras layer inputs + backward_oracle_inputs = crown_bounds + [perturbation_domain_input] + oracle_bounds = backward_layer.call_oracle(backward_oracle_inputs) - if not callable(layer_fn): - raise ValueError("Expected `layer_fn` argument to be a callable.") - ############### + # store oracle + oracle_map[id(node)] = oracle_bounds - if len(back_bounds) and len(to_list(model.output)) > 1: - raise NotImplementedError() + return oracle_bounds - # sort nodes from input to output - dico_nodes = get_depth_dict(model) - keys = [e for e in dico_nodes.keys()] - keys.sort(reverse=True) - # generate input_map - if not finetune: - joint = True - set_mode_layer = Convert2BackwardMode(get_mode(ibp, affine), perturbation_domain) +def crown_model( + model: Model, + layer_fn: Callable[[Layer, tuple[int, ...]], DecomonLayer], + backward_bounds: list[list[keras.KerasTensor]], + perturbation_domain_input: keras.KerasTensor, + perturbation_domain: PerturbationDomain, + oracle_map: Optional[dict[int, Union[list[keras.KerasTensor], list[list[keras.KerasTensor]]]]] = None, + forward_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, + forward_layer_map: Optional[dict[int, DecomonLayer]] = None, + crown_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, + is_submodel: bool = False, + backward_map: Optional[dict[int, DecomonLayer]] = None, + model_output_shape: Optional[tuple[int, ...]] = None, +) -> list[keras.KerasTensor]: + """Convert a functional keras model via crown algorithm (backward propagation) + + Hypothesis: potential embedded submodels have only one input and one output. + + Args: + model: keras model to convert + layer_fn: callable converting a layer and a model_output_shape into a (backward) decomon layer + perturbation_domain_input: perturbation domain input + perturbation_domain: perturbation domain type on keras model input + backward_bounds: should be of the same size as the number of model outputs + (each sublist potentially empty for starting with identity bounds) + oracle_map: already registered oracle bounds per node + forward_output_map: forward outputs per node from a previously performed forward conversion. + To be used for forward oracle. + forward_layer_map: forward decomon layer per node from a previously performed forward conversion. + To be used for forward oracle. + crown_output_map: output of subcrowns per output node. + Avoids relaunching a crown if several nodes share parents. + To be used for crown oracle. + is_submodel: specify if called from within a crown to propagate through an embedded submodel + backward_map: stores converted layer by node for the current crown + Should be set only if is_submodel is True. + model_output_shape: if submodel is True, must be set to the output_shape used in the current crown + + Returns: + concatenated propagated backward bounds corresponding to each output node of the keras model - input_map, backward_map, crown_map = get_input_nodes( - model=model, - dico_nodes=dico_nodes, - ibp=ibp, - affine=affine, - input_tensors=input_tensors, - output_map=forward_map, - layer_fn=layer_fn, - joint=joint, - perturbation_domain=perturbation_domain, - set_mode_layer=set_mode_layer, - **kwargs, - ) - # time_1 = time.process_time() - # print('step1', time_1-zero_time) - # retrieve output nodes + """ + if oracle_map is None: + oracle_map = {} + if forward_layer_map is None: + forward_layer_map = {} + if forward_output_map is None: + forward_output_map = {} + if crown_output_map is None: + crown_output_map = {} + + # Retrieve output nodes in same order as model.outputs + output_nodes = get_output_nodes(model) + if is_submodel and len(output_nodes) > 1: + raise NotImplementedError( + "crown_model() not yet implemented for model " "whose embedded submodels have multiple outputs." + ) + # Apply crown on each output, with the appropriate backward_bounds and model_output_shape output = [] - output_nodes = dico_nodes[0] - # the ordering may change - output_names = [tensor._keras_history.operation.name for tensor in to_list(model.output)] - fuse_layer = None - for output_name in output_names: - for node in output_nodes: - if node.operation.name == output_name: - # compute with crown - output_crown, fuse_layer = crown_( - node=node, - ibp=ibp, - affine=affine, - input_map=input_map, - layer_fn=layer_fn, - backward_bounds=back_bounds, - backward_map=backward_map, - joint=joint, - fuse=fuse, - perturbation_domain=perturbation_domain, - output_map=crown_map, - fuse_layer=fuse_layer, - ) - # time_2 = time.process_time() - # print('step2', time_2-time_1) - if fuse: - # import pdb; pdb.set_trace() - output += to_list(set_mode_layer(input_tensors + output_crown)) - else: - output = output_crown + for node, backward_bounds_node in zip(output_nodes, backward_bounds): + if is_submodel: + # for embedded submodel, pass the frozen model_output_shape fixed and the current backward_map + backward_map_node = backward_map + if model_output_shape is None: + raise RuntimeError("`submodel_output_shape` must be set if `submodel` is True.") + else: + # new backward_map and new model_output_shape for each output node + model_output_shape = get_model_output_shape(node=node, backward_bounds_node=backward_bounds_node) + backward_map_node = {} + + output_crown = crown( + node=node, + layer_fn=layer_fn, + model_output_shape=model_output_shape, + backward_bounds=backward_bounds_node, + backward_map=backward_map_node, + oracle_map=oracle_map, + forward_output_map=forward_output_map, + forward_layer_map=forward_layer_map, + crown_output_map=crown_output_map, + perturbation_domain_input=perturbation_domain_input, + perturbation_domain=perturbation_domain, + ) + output += output_crown - return input_tensors, output, backward_map, None + return output def convert_backward( model: Model, - input_tensors: list[keras.KerasTensor], - back_bounds: Optional[list[keras.KerasTensor]] = None, - slope: Union[str, Slope] = Slope.V_SLOPE, - ibp: bool = True, - affine: bool = True, + perturbation_domain_input: keras.KerasTensor, perturbation_domain: Optional[PerturbationDomain] = None, - finetune: bool = False, - forward_map: Optional[OutputMapDict] = None, - softmax_to_linear: bool = True, - joint: bool = True, - layer_fn: Callable[..., BackwardLayer] = to_backward, - final_ibp: bool = True, - final_affine: bool = False, - input_dim: int = -1, + layer_fn: Callable[..., DecomonLayer] = to_decomon, + backward_bounds: Optional[list[list[keras.KerasTensor]]] = None, + slope: Union[str, Slope] = Slope.V_SLOPE, + forward_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, + forward_layer_map: Optional[dict[int, DecomonLayer]] = None, **kwargs: Any, -) -> tuple[list[keras.KerasTensor], list[keras.KerasTensor], dict[int, BackwardLayer], None]: - model = ensure_functional_model(model) - if input_dim == -1: - input_dim = get_input_dim(model) - if back_bounds is None: - back_bounds = [] - if forward_map is None: - forward_map = {} +) -> list[keras.KerasTensor]: + """Convert keras model via backward propagation. + + Prepare layer_fn by freezing all args except layer and model_output_shape. + Ensure that model is functional (transform sequential ones to functional equivalent ones). + + Args: + model: keras model to convert + perturbation_domain_input: perturbation domain input + perturbation_domain: perturbation domain type on keras model input + layer_fn: callable converting a layer and a model_output_shape into a (backward) decomon layer + backward_bounds: if set, should be of the same size as the number of model outputs + (each sublist potentially empty for starting with identity bounds) + forward_output_map: forward outputs per node from a previously performed forward conversion. + To be used for forward oracle if not empty. + forward_layer_map: forward decomon layer per node from a previously performed forward conversion. + To be used for forward oracle if not empty. + slope: slope used by decomon activation layers + **kwargs: keyword arguments to pass to layer_fn + + Returns: + propagated affine bounds (concatenated), output of the future decomon model + + """ if perturbation_domain is None: perturbation_domain = BoxDomain() - if len(back_bounds): - if len(back_bounds) == 1: - C = back_bounds[0] - bias = K.zeros_like(C[:, 0]) - back_bounds = [C, bias] * 2 - result = crown_model( - model=model, - input_tensors=input_tensors, - back_bounds=back_bounds, + if backward_bounds is None: + backward_bounds = [[]] * len(model.outputs) + + model = ensure_functional_model(model) + propagation = Propagation.BACKWARD + + layer_fn = include_kwargs_layer_fn( + layer_fn, slope=slope, - ibp=ibp, - affine=affine, perturbation_domain=perturbation_domain, - finetune=finetune, - forward_map=forward_map, - softmax_to_linear=softmax_to_linear, - joint=joint, - layer_fn=layer_fn, - fuse=True, + propagation=propagation, **kwargs, ) - input_tensors, output, backward_map, toto = result - mode_from = get_mode(ibp, affine) - mode_to = get_mode(final_ibp, final_affine) - output = Convert2Mode( - mode_from=mode_from, - mode_to=mode_to, + output = crown_model( + model=model, + layer_fn=layer_fn, + backward_bounds=backward_bounds, + perturbation_domain_input=perturbation_domain_input, perturbation_domain=perturbation_domain, - input_dim=input_dim, - )(output) - if mode_to != mode_from and mode_from == ForwardMode.IBP: - f_input = Lambda(lambda z: Concatenate(1)([z[0][:, None], z[1][:, None]])) - output[0] = f_input([input_tensors[1], input_tensors[0]]) - return input_tensors, output, backward_map, toto + forward_output_map=forward_output_map, + forward_layer_map=forward_layer_map, + ) + + return output + + +def get_model_output_shape(node: Node, backward_bounds_node: list[Tensor]): + """Get outer model output shape w/o batchsize. + + If any backward bounds are passed, we deduce the outer keras model output shape from it. + We assume for that: + - backward_bounds = [w_l, b_l, w_u, b_u] + - we can have w_l, w_u in diagonal representation (w_l.shape == b_l.shape) + - we have the batchsize included in the backward_bounds + + => model_output_shape = backward_bounds[1].shape[1:] + + If no backward bounds are given, we fall back to the output shape of the given output node. + + Args: + node: current output node of the (potentially inner) keras model to convert + backward_bounds_node: backward bounds specified for this node + + Returns: + outer keras model output shape, excluding batchsize + + """ + if len(backward_bounds_node) == 0: + return node.outputs[0].shape[1:] + else: + _, b, _, _ = backward_bounds_node + return b.shape[1:] + + +def include_kwargs_layer_fn( + layer_fn: Callable[..., DecomonLayer], + perturbation_domain: PerturbationDomain, + propagation: Propagation, + slope: Slope, + **kwargs: Any, +) -> Callable[[Layer, tuple[int, ...]], DecomonLayer]: + """Include external parameters in the function converting layers + + In particular, include propagation=Propagation.BACKWARD. + + Args: + layer_fn: + perturbation_domain: + propagation: + slope: + **kwargs: + + Returns: + + """ + + def func(layer: Layer, model_output_shape: tuple[int, ...]) -> DecomonLayer: + return layer_fn( + layer, + model_output_shape=model_output_shape, + slope=slope, + perturbation_domain=perturbation_domain, + propagation=propagation, + **kwargs, + ) + + return func diff --git a/tests/test_convert_backward.py b/tests/test_convert_backward.py new file mode 100644 index 00000000..2522697f --- /dev/null +++ b/tests/test_convert_backward.py @@ -0,0 +1,102 @@ +from keras.models import Model +from pytest_cases import fixture, parametrize + +from decomon.core import Propagation, Slope +from decomon.models.backward_cloning import convert_backward +from decomon.models.forward_cloning import convert_forward + + +@fixture +@parametrize("name", ["tutorial", "tutorial_linear", "submodel", "submodel_linear", "add", "add_linear", "add_1layer"]) +def keras_model_fn(name, helpers): + if name == "tutorial": + return helpers.toy_network_tutorial + elif name == "tutorial_linear": + return lambda input_shape, dtype=None: helpers.toy_network_tutorial( + input_shape=input_shape, dtype=dtype, activation=None + ) + elif name == "tutorial_activation_embedded": + return helpers.toy_network_tutorial_with_embedded_activation + elif name == "add": + return helpers.toy_network_add + elif name == "add_linear": + return lambda input_shape, dtype=None: helpers.toy_network_add( + input_shape=input_shape, dtype=dtype, activation=None + ) + elif name == "add_1layer": + return helpers.toy_network_add_monolayer + elif name == "submodel": + return helpers.toy_network_submodel + elif name == "submodel_linear": + return lambda input_shape, dtype=None: helpers.toy_network_submodel( + input_shape=input_shape, dtype=dtype, activation=None + ) + else: + raise ValueError() + + +def test_convert_backward( + ibp, + affine, + propagation, + perturbation_domain, + input_shape, + keras_model_fn, + simple_model_decomon_symbolic_input_fn, + simple_model_keras_input_fn, + simple_model_decomon_input_fn, + helpers, +): + slope = Slope.Z_SLOPE + decimal = 4 + + # keras model to convert + keras_model = keras_model_fn(input_shape=input_shape) + + # symbolic inputs + keras_symbolic_input = keras_model.inputs[0] + decomon_symbolic_input = simple_model_decomon_symbolic_input_fn(keras_symbolic_input) + + # actual inputs + keras_input = simple_model_keras_input_fn(keras_symbolic_input) + decomon_input = simple_model_decomon_input_fn(keras_input) + + # keras output + keras_output = keras_model(keras_input) + + # forward conversion for forward oracle + if propagation == Propagation.FORWARD: + _, forward_output_map, forward_layer_map = convert_forward( + keras_model, ibp=ibp, affine=affine, perturbation_domain_input=decomon_symbolic_input, slope=slope + ) + else: + forward_output_map, forward_layer_map = None, None + + # backward conversion + decomon_symbolic_output = convert_backward( + keras_model, + perturbation_domain_input=decomon_symbolic_input, + perturbation_domain=perturbation_domain, + slope=slope, + forward_output_map=forward_output_map, + forward_layer_map=forward_layer_map, + ) + + #  decomon outputs + if None in decomon_symbolic_output[0].shape: + decomon_model = Model(inputs=decomon_symbolic_input, outputs=decomon_symbolic_output) + decomon_output = decomon_model(decomon_input) + else: + # special case for pure linear keras model => bounds not depending on batch, already computed. + decomon_output = decomon_symbolic_output + + # check ibp and affine bounds well ordered w.r.t. keras inputs/outputs + helpers.assert_decomon_output_compare_with_keras_input_output_model( + decomon_output=decomon_output, + keras_input=keras_input, + keras_output=keras_output, + propagation=Propagation.BACKWARD, + decimal=decimal, + ibp=ibp, + affine=affine, + ) From b3dab6dc6b3093cf52a0cf460ff6a6a90f24730d Mon Sep 17 00:00:00 2001 From: Nolwen Date: Wed, 21 Feb 2024 23:39:31 +0100 Subject: [PATCH 058/101] Fix convert_backward with embedded submodels + full crown In full crown mode, to get oracle bounds inside an embedded submodel, we need to reconstruct the proper perturbation domain inputs for this submodel. We do it dynamically when necessary (to avoid it if the submodel has only linear layers for instance). We deduce it from constant bounds on submodel input, i.e. oracle bounds for the node corresponding to the submodel. To reconstruct from them the perturbation domain input, we need a new method PerturbationDomain.get_input_from_constant_bounds() which generates x from lower and upper. We reuse the formula used in previous version of the code which was valid only for BoxDomain. Not yet implemented for BallDomain. --- src/decomon/core.py | 17 ++++ src/decomon/models/backward_cloning.py | 104 +++++++++++++++++++++---- tests/test_convert_backward.py | 2 - 3 files changed, 108 insertions(+), 15 deletions(-) diff --git a/src/decomon/core.py b/src/decomon/core.py index f7a07bfa..e9a804c6 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -49,6 +49,19 @@ def get_lower(self, x: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: def get_nb_x_components(self) -> int: ... + @abstractmethod + def get_input_from_constant_bounds(self, constant_bounds: list[Tensor]) -> Tensor: + """Construct perturbation domain input x from constant bounds on keras model input + + Args: + constant_bounds: lower and upper constant bounds on keras model input + + Returns: + x: perturbation domain input + + """ + ... + def get_config(self) -> dict[str, Any]: return { "opt_option": self.opt_option, @@ -97,6 +110,10 @@ def get_lower_x(self, x: Tensor) -> Tensor: def get_nb_x_components(self) -> int: return 2 + def get_input_from_constant_bounds(self, constant_bounds: list[Tensor]) -> Tensor: + lower, upper = constant_bounds + return K.concatenate([lower[:, None], upper[:, None]], axis=1) + class GridDomain(PerturbationDomain): pass diff --git a/src/decomon/models/backward_cloning.py b/src/decomon/models/backward_cloning.py index 36664bbb..84e66717 100644 --- a/src/decomon/models/backward_cloning.py +++ b/src/decomon/models/backward_cloning.py @@ -42,6 +42,8 @@ def crown( forward_output_map: dict[int, list[keras.KerasTensor]], forward_layer_map: dict[int, DecomonLayer], crown_output_map: dict[int, list[keras.KerasTensor]], + submodels_stack: list[Node], + submodel_perturbation_domain_input_map: dict[int, keras.KerasTensor], perturbation_domain_input: keras.KerasTensor, perturbation_domain: PerturbationDomain, ) -> list[keras.KerasTensor]: @@ -65,6 +67,14 @@ def crown( crown_output_map: output of subcrowns per output node. Avoids relaunching a crown if several nodes share parents. To be used by `get_oracle()`. + submodels_stack: not empty only if in a submodel. + A list of nodes corresponding to the successive embedded submodels, + from the outerest to the innerest submodel, the last one being the current submodel. + Will be used to get perturbation_domain_input for this submodel, to be used only by crown oracle. + (Forward oracle being precomputed from the full model, the original perturbation_domain_input is used for it) + To be used by `get_oracle()`. + submodel_perturbation_domain_input_map: stores already computed perturbation_domain_input for submodels. + To be used by `get_oracle()`. backward_map: stores converted layer by node for the current crown (should depend on the proper model output and thus change for each sub-crown) @@ -78,25 +88,23 @@ def crown( return backward_bounds else: if isinstance(node.operation, Model): - # TO CHECK perturbation input submodel ? - # forward oracle: - # - ibp (+-affine): not needed - # - affine (w/o ibp): forward_output give affine bounds on outer model input => perturbation_domain - # crown oracle: - # - see where stop subcrowns, maybe to submodel inputs, - # in which case we need to construct perturbation_domain_input_submodel from get_oracle(node) - perturbation_domain_input_submodel = perturbation_domain_input + if len(parents) > 1: + raise NotImplementedError( + "crown_model() not yet implemented for model whose embedded submodels have multiple inputs." + ) backward_bounds = crown_model( model=node.operation, layer_fn=layer_fn, backward_bounds=[backward_bounds], - perturbation_domain_input=perturbation_domain_input_submodel, + perturbation_domain_input=perturbation_domain_input, perturbation_domain=perturbation_domain, oracle_map=oracle_map, forward_output_map=forward_output_map, forward_layer_map=forward_layer_map, crown_output_map=crown_output_map, is_submodel=True, + submodels_stack=submodels_stack + [node], + submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, backward_map=backward_map, model_output_shape=model_output_shape, ) @@ -118,6 +126,8 @@ def crown( forward_layer_map=forward_layer_map, backward_layer=backward_layer, crown_output_map=crown_output_map, + submodels_stack=submodels_stack, + submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, layer_fn=layer_fn, ) @@ -134,7 +144,7 @@ def crown( backward_bounds, _ = backward_layer.inputs_outputs_spec.split_outputs(backward_layer_outputs) # Call crown recursively on parent nodes - if isinstance(backward_layer, DecomonMerge): + if not isinstance(node.operation, Model) and isinstance(backward_layer, DecomonMerge): # merging layer crown_bounds_list: list[list[Tensor]] = [] for backward_bounds_i, parent in zip(backward_bounds, parents): @@ -149,6 +159,8 @@ def crown( forward_output_map=forward_output_map, forward_layer_map=forward_layer_map, crown_output_map=crown_output_map, + submodels_stack=submodels_stack, + submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, perturbation_domain_input=perturbation_domain_input, perturbation_domain=perturbation_domain, ) @@ -172,6 +184,8 @@ def crown( forward_output_map=forward_output_map, forward_layer_map=forward_layer_map, crown_output_map=crown_output_map, + submodels_stack=submodels_stack, + submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, perturbation_domain_input=perturbation_domain_input, perturbation_domain=perturbation_domain, ) @@ -187,6 +201,8 @@ def get_oracle( forward_layer_map: dict[int, DecomonLayer], backward_layer: DecomonLayer, crown_output_map: dict[int, list[keras.KerasTensor]], + submodels_stack: list[Node], + submodel_perturbation_domain_input_map: dict[int, keras.KerasTensor], layer_fn: Callable[[Layer, tuple[int, ...]], DecomonLayer], ) -> Union[list[keras.KerasTensor], list[list[keras.KerasTensor]]]: """Get oracle bounds "on demand". @@ -210,6 +226,14 @@ def get_oracle( crown_output_map: output of subcrowns per output node. Avoids relaunching a crown if several nodes share parents. To be used for crown oracle. + submodels_stack: not empty only if in a submodel. + A list of nodes corresponding to the successive embedded submodels, + from the outerest to the innerest submodel, the last one being the current submodel. + Will be used to get perturbation_domain_input for this submodel, to be used only by crown oracle. + (Forward oracle being precomputed from the full model, the original perturbation_domain_input is used for it) + To be used for crown oracle. + submodel_perturbation_domain_input_map: stores already computed perturbation_domain_input for submodels. + To be used for crown oracle. layer_fn: callable converting a layer and a model_output_shape into a (backward) decomon layer. To be used for crown oracle. @@ -240,6 +264,39 @@ def get_oracle( oracle_bounds = forward_layer.call_oracle(forward_input) else: # crown oracle + + # perturbation domain input for the (sub)model? + if len(submodels_stack) == 0: + # in outer model: we already got the proper pertubation domain input + perturbation_domain_input_submodel = perturbation_domain_input + else: + submodel_node = submodels_stack[-1] + if id(submodel_node) in submodel_perturbation_domain_input_map: + # already computed for this submodel + perturbation_domain_input_submodel = submodel_perturbation_domain_input_map[id(submodel_node)] + else: + # reconstruct perturbation domain input if in a new submodel + # 1. get oracle bounds for the current submodel node + # 2. use perturbation domain to reconstruct its input from it for the submodel + oracle_bounds_on_submodel_input = get_oracle( + node=submodel_node, + perturbation_domain_input=perturbation_domain_input, + perturbation_domain=perturbation_domain, + oracle_map=oracle_map, + forward_output_map=forward_output_map, + forward_layer_map=forward_layer_map, + backward_layer=backward_layer, # to replace by an "oracle" layer + crown_output_map=crown_output_map, + submodels_stack=submodels_stack[:-1], # remaining submodels above the current one (if any) + submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, + layer_fn=layer_fn, + ) + perturbation_domain_input_submodel = perturbation_domain.get_input_from_constant_bounds( + oracle_bounds_on_submodel_input + ) + submodel_perturbation_domain_input_map[id(submodel_node)] = perturbation_domain_input_submodel + + # affine bounds on parents from sub-crowns crown_bounds = [] for parent in parents: if id(parent) in crown_output_map: @@ -257,6 +314,8 @@ def get_oracle( forward_output_map=forward_output_map, forward_layer_map=forward_layer_map, crown_output_map=crown_output_map, + submodels_stack=submodels_stack, + submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, perturbation_domain_input=perturbation_domain_input, perturbation_domain=perturbation_domain, ) @@ -264,8 +323,8 @@ def get_oracle( crown_output_map[id(parent)] = crown_bounds_parent crown_bounds += crown_bounds_parent - # deduce oracle bounds from affine bounds on keras layer inputs - backward_oracle_inputs = crown_bounds + [perturbation_domain_input] + # deduce oracle bounds from affine bounds on keras (sub)model inputs and corresponding perturbation domain input + backward_oracle_inputs = crown_bounds + [perturbation_domain_input_submodel] oracle_bounds = backward_layer.call_oracle(backward_oracle_inputs) # store oracle @@ -285,6 +344,8 @@ def crown_model( forward_layer_map: Optional[dict[int, DecomonLayer]] = None, crown_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, is_submodel: bool = False, + submodels_stack: Optional[list[Node]] = None, + submodel_perturbation_domain_input_map: Optional[dict[int, keras.KerasTensor]] = None, backward_map: Optional[dict[int, DecomonLayer]] = None, model_output_shape: Optional[tuple[int, ...]] = None, ) -> list[keras.KerasTensor]: @@ -308,6 +369,14 @@ def crown_model( Avoids relaunching a crown if several nodes share parents. To be used for crown oracle. is_submodel: specify if called from within a crown to propagate through an embedded submodel + submodels_stack: Not empty only if in a submodel. + A list of nodes corresponding to the successive embedded submodels, + from the outerest to the innerest submodel, the last one being the current submodel. + Will be used to get perturbation_domain_input for this submodel, to be used only by crown oracle. + (Forward oracle being precomputed from the full model, the original perturbation_domain_input is used for it) + To be used for crown oracle. + submodel_perturbation_domain_input_map: stores already computed perturbation_domain_input for submodels. + To be used for crown oracle. backward_map: stores converted layer by node for the current crown Should be set only if is_submodel is True. model_output_shape: if submodel is True, must be set to the output_shape used in the current crown @@ -324,12 +393,19 @@ def crown_model( forward_output_map = {} if crown_output_map is None: crown_output_map = {} + if submodels_stack is None: + submodels_stack = [] + if submodel_perturbation_domain_input_map is None: + submodel_perturbation_domain_input_map = {} + + # ensure (sub)model is functional + model = ensure_functional_model(model) # Retrieve output nodes in same order as model.outputs output_nodes = get_output_nodes(model) if is_submodel and len(output_nodes) > 1: raise NotImplementedError( - "crown_model() not yet implemented for model " "whose embedded submodels have multiple outputs." + "crown_model() not yet implemented for model whose embedded submodels have multiple outputs." ) # Apply crown on each output, with the appropriate backward_bounds and model_output_shape output = [] @@ -354,6 +430,8 @@ def crown_model( forward_output_map=forward_output_map, forward_layer_map=forward_layer_map, crown_output_map=crown_output_map, + submodels_stack=submodels_stack, + submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, perturbation_domain_input=perturbation_domain_input, perturbation_domain=perturbation_domain, ) diff --git a/tests/test_convert_backward.py b/tests/test_convert_backward.py index 2522697f..6b9e9438 100644 --- a/tests/test_convert_backward.py +++ b/tests/test_convert_backward.py @@ -15,8 +15,6 @@ def keras_model_fn(name, helpers): return lambda input_shape, dtype=None: helpers.toy_network_tutorial( input_shape=input_shape, dtype=dtype, activation=None ) - elif name == "tutorial_activation_embedded": - return helpers.toy_network_tutorial_with_embedded_activation elif name == "add": return helpers.toy_network_add elif name == "add_linear": From 51327777fe4fbe752def3ea91bc3cac77d882e23 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 22 Feb 2024 22:34:29 +0100 Subject: [PATCH 059/101] Add plot_model() utility We tweak keras.utils.plot_model() to be able to represent some decomon_layer attributes. By default we add the propagation and the underlying keras layer name We can add custom attributes thanks to the argument show_layer_attributes. An example: show_layer_attributes = { "propagation": { "forward": { "bgcolor": "#b3e6ff", }, "backward": { "bgcolor": "#ffccdd", }, None: {"color": "black"}, }, "layer.name": {}, } We list the attribute (even nested ones) to show, and the style (bgcolor and color) to apply for them. We define a default style by adding a None entry into the style dict None: "color": "black" -> all propagation will be written in black We define a style for a specific value by adding an entry with the value as a key and the style dict as value. "forward": { "bgcolor": "#b3e6ff", }, --- src/decomon/visualization/__init__.py | 1 + .../visualization/model_visualization.py | 406 ++++++++++++++++++ 2 files changed, 407 insertions(+) create mode 100644 src/decomon/visualization/__init__.py create mode 100644 src/decomon/visualization/model_visualization.py diff --git a/src/decomon/visualization/__init__.py b/src/decomon/visualization/__init__.py new file mode 100644 index 00000000..8ffb3f4b --- /dev/null +++ b/src/decomon/visualization/__init__.py @@ -0,0 +1 @@ +from .model_visualization import plot_model diff --git a/src/decomon/visualization/model_visualization.py b/src/decomon/visualization/model_visualization.py new file mode 100644 index 00000000..45ce0868 --- /dev/null +++ b/src/decomon/visualization/model_visualization.py @@ -0,0 +1,406 @@ +from keras.src.utils.model_visualization import * + + +def get_layer_attribute(layer, name): + subnames = name.split(".") + attr = layer + for subname in subnames: + attr = getattr(attr, subname, None) + if attr is None: + return None + return attr + + +def make_layer_label(layer, **kwargs): + class_name = layer.__class__.__name__ + + show_layer_names = kwargs.pop("show_layer_names") + show_layer_activations = kwargs.pop("show_layer_activations") + show_layer_attributes = kwargs.pop("show_layer_attributes", []) + show_dtype = kwargs.pop("show_dtype") + show_shapes = kwargs.pop("show_shapes") + show_trainable = kwargs.pop("show_trainable") + if kwargs: + raise ValueError(f"Invalid kwargs: {kwargs}") + + table = '<' + + colspan = max(1, sum(int(x) for x in (show_dtype, show_shapes, show_trainable))) + + if show_layer_names: + table += ( + f'" + ) + else: + table += ( + f'" + ) + if show_layer_activations and hasattr(layer, "activation") and layer.activation is not None: + table += ( + f'" + ) + for attr_name, attr_style in show_layer_attributes.items(): + attribute = get_layer_attribute(layer, attr_name) + if attribute is not None: + default_style = {"bgcolor": "white", "color": "black"} + default_style.update(attr_style.get(None, {})) + style_to_apply = dict(default_style) + if attribute in attr_style: + style_to_apply.update(attr_style[attribute]) + table += ( + f'" + ) + + cols = [] + if show_shapes: + shape = None + try: + shape = layer.output.shape + except ValueError: + pass + cols.append( + ('") + ) + if show_dtype: + dtype = None + try: + dtype = layer.output.dtype + except ValueError: + pass + cols.append( + ('") + ) + if show_trainable and hasattr(layer, "trainable") and layer.weights: + if layer.trainable: + cols.append( + ('") + ) + else: + cols.append( + ('") + ) + if cols: + colspan = len(cols) + else: + colspan = 1 + + if cols: + table += "" + "".join(cols) + "" + table += "
' + '' + f"{layer.name} ({class_name})" + "
' + '' + f"{class_name}" + "
' + '' + f"Activation: {get_layer_activation_name(layer)}" + "
' + f'' + f"{attr_name.capitalize()}: {attribute}" + "
' f'Output shape: {shape or "?"}' "' f'Output dtype: {dtype or "?"}' "' '' "Trainable' '' "Non-trainable
>" + return table + + +def make_node(layer, **kwargs): + node = pydot.Node(str(id(layer)), label=make_layer_label(layer, **kwargs)) + node.set("fontname", "Helvetica") + node.set("border", "0") + node.set("margin", "0") + return node + + +def model_to_dot( + model, + show_shapes=False, + show_dtype=False, + show_layer_names=True, + rankdir="TB", + expand_nested=False, + dpi=200, + subgraph=False, + show_layer_activations=False, + show_trainable=False, + show_layer_attributes=None, + **kwargs, +): + """Convert a Keras model to dot format. + + Args: + model: A Keras model instance. + show_shapes: whether to display shape information. + show_dtype: whether to display layer dtypes. + show_layer_names: whether to display layer names. + rankdir: `rankdir` argument passed to PyDot, + a string specifying the format of the plot: `"TB"` + creates a vertical plot; `"LR"` creates a horizontal plot. + expand_nested: whether to expand nested Functional models + into clusters. + dpi: Image resolution in dots per inch. + subgraph: whether to return a `pydot.Cluster` instance. + show_layer_activations: Display layer activations (only for layers that + have an `activation` property). + show_trainable: whether to display if a layer is trainable. + + Returns: + A `pydot.Dot` instance representing the Keras model or + a `pydot.Cluster` instance representing nested model if + `subgraph=True`. + """ + from keras.src.ops.function import make_node_key + + if not model.built: + raise ValueError( + "This model has not yet been built. " + "Build the model first by calling `build()` or by calling " + "the model on a batch of data." + ) + + from keras.src.models import functional, sequential + + # from keras.src.layers import Wrapper + + if not check_pydot(): + raise ImportError("You must install pydot (`pip install pydot`) for " "model_to_dot to work.") + + if subgraph: + dot = pydot.Cluster(style="dashed", graph_name=model.name) + dot.set("label", model.name) + dot.set("labeljust", "l") + else: + dot = pydot.Dot() + dot.set("rankdir", rankdir) + dot.set("concentrate", True) + dot.set("dpi", dpi) + dot.set("splines", "ortho") + dot.set_node_defaults(shape="record") + + if kwargs.pop("layer_range", None) is not None: + raise ValueError("Argument `layer_range` is no longer supported.") + if kwargs: + raise ValueError(f"Unrecognized keyword arguments: {kwargs}") + + kwargs = { + "show_layer_names": show_layer_names, + "show_layer_activations": show_layer_activations, + "show_dtype": show_dtype, + "show_shapes": show_shapes, + "show_trainable": show_trainable, + "show_layer_attributes": show_layer_attributes, + } + + if isinstance(model, sequential.Sequential): + # TODO + layers = model.layers + elif not isinstance(model, functional.Functional): + # We treat subclassed models as a single node. + node = make_node(model, **kwargs) + dot.add_node(node) + return dot + else: + layers = model._operations + + # Create graph nodes. + sub_n_first_node = {} + sub_n_last_node = {} + for i, layer in enumerate(layers): + # Process nested functional models. + if expand_nested and isinstance(layer, functional.Functional): + submodel = model_to_dot( + layer, + show_shapes, + show_dtype, + show_layer_names, + rankdir, + expand_nested, + subgraph=True, + show_layer_activations=show_layer_activations, + show_trainable=show_trainable, + show_layer_attributes=show_layer_attributes, + ) + # sub_n : submodel + sub_n_nodes = submodel.get_nodes() + sub_n_first_node[layer.name] = sub_n_nodes[0] + sub_n_last_node[layer.name] = sub_n_nodes[-1] + dot.add_subgraph(submodel) + + else: + node = make_node(layer, **kwargs) + dot.add_node(node) + + # Connect nodes with edges. + # Sequential case. + if isinstance(model, sequential.Sequential): + for i in range(len(layers) - 1): + inbound_layer_id = str(id(layers[i])) + layer_id = str(id(layers[i + 1])) + add_edge(dot, inbound_layer_id, layer_id) + return dot + + # Functional case. + for i, layer in enumerate(layers): + layer_id = str(id(layer)) + for i, node in enumerate(layer._inbound_nodes): + node_key = make_node_key(layer, i) + if node_key in model._nodes: + for parent_node in node.parent_nodes: + inbound_layer = parent_node.operation + inbound_layer_id = str(id(inbound_layer)) + if not expand_nested: + assert dot.get_node(inbound_layer_id) + assert dot.get_node(layer_id) + add_edge(dot, inbound_layer_id, layer_id) + else: + # if inbound_layer is not Functional + if not isinstance(inbound_layer, functional.Functional): + # if current layer is not Functional + if not isinstance(layer, functional.Functional): + assert dot.get_node(inbound_layer_id) + assert dot.get_node(layer_id) + add_edge(dot, inbound_layer_id, layer_id) + # if current layer is Functional + elif isinstance(layer, functional.Functional): + add_edge( + dot, + inbound_layer_id, + sub_n_first_node[layer.name].get_name(), + ) + # if inbound_layer is Functional + elif isinstance(inbound_layer, functional.Functional): + name = sub_n_last_node[inbound_layer.name].get_name() + if isinstance(layer, functional.Functional): + output_name = sub_n_first_node[layer.name].get_name() + add_edge(dot, name, output_name) + else: + add_edge(dot, name, layer_id) + return dot + + +def plot_model( + model, + to_file="model.png", + show_shapes=False, + show_dtype=False, + show_layer_names=True, + rankdir="TB", + expand_nested=False, + dpi=64, + show_layer_activations=False, + show_trainable=False, + show_layer_attributes=None, + **kwargs, +): + """Converts a Keras model to dot format and save to a file. + + Example: + + ```python + inputs = ... + outputs = ... + model = keras.Model(inputs=inputs, outputs=outputs) + + dot_img_file = '/tmp/model_1.png' + keras.utils.plot_model(model, to_file=dot_img_file, show_shapes=True) + ``` + + Args: + model: A Keras model instance + to_file: File name of the plot image. + show_shapes: whether to display shape information. + show_dtype: whether to display layer dtypes. + show_layer_names: whether to display layer names. + rankdir: `rankdir` argument passed to PyDot, + a string specifying the format of the plot: `"TB"` + creates a vertical plot; `"LR"` creates a horizontal plot. + expand_nested: whether to expand nested Functional models + into clusters. + dpi: Image resolution in dots per inch. + show_layer_activations: Display layer activations (only for layers that + have an `activation` property). + show_trainable: whether to display if a layer is trainable. + + Returns: + A Jupyter notebook Image object if Jupyter is installed. + This enables in-line display of the model plots in notebooks. + """ + + if not model.built: + raise ValueError( + "This model has not yet been built. " + "Build the model first by calling `build()` or by calling " + "the model on a batch of data." + ) + if not check_pydot(): + message = "You must install pydot (`pip install pydot`) " "for `plot_model` to work." + if "IPython.core.magics.namespace" in sys.modules: + # We don't raise an exception here in order to avoid crashing + # notebook tests where graphviz is not available. + io_utils.print_msg(message) + return + else: + raise ImportError(message) + if not check_graphviz(): + message = ( + "You must install graphviz " + "(see instructions at https://graphviz.gitlab.io/download/) " + "for `plot_model` to work." + ) + if "IPython.core.magics.namespace" in sys.modules: + # We don't raise an exception here in order to avoid crashing + # notebook tests where graphviz is not available. + io_utils.print_msg(message) + return + else: + raise ImportError(message) + + if kwargs.pop("layer_range", None) is not None: + raise ValueError("Argument `layer_range` is no longer supported.") + if kwargs: + raise ValueError(f"Unrecognized keyword arguments: {kwargs}") + + if show_layer_attributes is None: + show_layer_attributes = { + "propagation": { + "forward": { + "bgcolor": "#b3e6ff", + }, + "backward": { + "bgcolor": "#ffccdd", + }, + None: {"color": "black"}, + }, + "layer.name": {}, + } + + dot = model_to_dot( + model, + show_shapes=show_shapes, + show_dtype=show_dtype, + show_layer_names=show_layer_names, + rankdir=rankdir, + expand_nested=expand_nested, + dpi=dpi, + show_layer_activations=show_layer_activations, + show_trainable=show_trainable, + show_layer_attributes=show_layer_attributes, + ) + to_file = str(to_file) + if dot is None: + return + _, extension = os.path.splitext(to_file) + if not extension: + extension = "png" + else: + extension = extension[1:] + # Save image to disk. + dot.write(to_file, format=extension) + # Return the image as a Jupyter Image object, to be displayed in-line. + # Note that we cannot easily detect whether the code is running in a + # notebook, and thus we always return the Image if Jupyter is available. + if extension != "pdf": + try: + from IPython import display + + return display.Image(filename=to_file) + except ImportError: + pass From b109cd20dd653b876770786ae05c28eba9c2be7b Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 23 Feb 2024 00:09:50 +0100 Subject: [PATCH 060/101] Fix input_shape for cnn toy model --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0396e809..5d3e0922 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,7 +56,7 @@ activation = param_fixture("activation", [None, "relu"]) data_format = param_fixture("data_format", ["channels_last", "channels_first"]) method = param_fixture("method", [m.value for m in ConvertMethod]) -input_shape = param_fixture("input_shape", [(1,), (3,), (5, 2, 3)], ids=["0d", "1d", "multid"]) +input_shape = param_fixture("input_shape", [(1,), (3,), (5, 6, 2)], ids=["0d", "1d", "multid"]) @pytest.fixture From 0a69431ab97506ac56d9f01c5ca70eef48c5ea3f Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 26 Feb 2024 14:37:10 +0100 Subject: [PATCH 061/101] Tighten (fully recursive) crown for model embedding submodels When hitting a node whose operation is a submodel during crown, instead of *before*: - simulating a backward layer by a crown_model() call on the submodel to propagate affine bounds through the submodel only - calling then a crown on parent nodes to carry on the propagation we rather *after*: - call a crown on the output node of the submodel - carry on the propagation by not stopping at input node of the submodel but rather calling the crown on submodel node parents then This is tighter in the case of non-linear layers in the submodel which needs the call to an oracle to get constant bounds on its keras inputs: - *before*: we launch a subcrown which ends with the submodel input and then combine the resulting affine bounds with a BoxDomain on submodel input coming from an ibp estimation (so a degradation of crown bounds obtained by another crown on submodel node) - *after*: we launch a subcrown which does not end with the submodel input but directly on outer model input with original perturbation domain. In case of successive embedded model, the improvement should be even more significative. (Benchmark to be done to assess this.) Hypotheses: submodel must have only 1 input node and 1 output node --- src/decomon/models/backward_cloning.py | 318 +++++++++++-------------- 1 file changed, 138 insertions(+), 180 deletions(-) diff --git a/src/decomon/models/backward_cloning.py b/src/decomon/models/backward_cloning.py index 84e66717..803ef7dc 100644 --- a/src/decomon/models/backward_cloning.py +++ b/src/decomon/models/backward_cloning.py @@ -5,7 +5,7 @@ import keras import keras.ops as K from keras.config import floatx -from keras.layers import Lambda, Layer +from keras.layers import InputLayer, Lambda, Layer from keras.models import Model from keras.src.ops.node import Node from keras.src.utils.python_utils import to_list @@ -43,7 +43,6 @@ def crown( forward_layer_map: dict[int, DecomonLayer], crown_output_map: dict[int, list[keras.KerasTensor]], submodels_stack: list[Node], - submodel_perturbation_domain_input_map: dict[int, keras.KerasTensor], perturbation_domain_input: keras.KerasTensor, perturbation_domain: PerturbationDomain, ) -> list[keras.KerasTensor]: @@ -53,9 +52,9 @@ def crown( node: node of the model until which the backward bounds have been propagated layer_fn: function converting a keras layer into its backward version, according to the proper (sub)model output shape - Will be passed to `get_oracle()` for crown oracle deriving from sub-crowns - model_output_shape: model_output_shape of the current crown, - to be passed to `crown_model()` on embedded submodels. + Will be also passed to `get_oracle()` for crown oracle deriving from sub-crowns + model_output_shape: model_output_shape of the current (sub)crown, + to be passed to `layer_fn`. backward_bounds: backward bounds to propagate oracle_map: oracle bounds on inputs of each keras layer, stored by layers id oracle_map: already registered oracle bounds per node @@ -70,126 +69,148 @@ def crown( submodels_stack: not empty only if in a submodel. A list of nodes corresponding to the successive embedded submodels, from the outerest to the innerest submodel, the last one being the current submodel. - Will be used to get perturbation_domain_input for this submodel, to be used only by crown oracle. - (Forward oracle being precomputed from the full model, the original perturbation_domain_input is used for it) - To be used by `get_oracle()`. - submodel_perturbation_domain_input_map: stores already computed perturbation_domain_input for submodels. - To be used by `get_oracle()`. + Used to carry on the propagation through the outer model when reaching the input of a submodel. backward_map: stores converted layer by node for the current crown - (should depend on the proper model output and thus change for each sub-crown) + (should depend on the proper model output and thus change for each sub-crown) Returns: propagated backward bounds until model input for the proper output node """ + + ## Special case: node == embedded submodel => crown on its output node + if isinstance(node.operation, Model): + submodel = node.operation + submodel = ensure_functional_model(submodel) + output_nodes = get_output_nodes(submodel) + if len(output_nodes) > 1: + raise NotImplementedError( + "Backward propagation not yet implemented for model whose embedded submodels have multiple outputs." + ) + output_node = output_nodes[0] + return crown( + node=output_node, + layer_fn=layer_fn, + model_output_shape=model_output_shape, + backward_bounds=backward_bounds, + backward_map=backward_map, + oracle_map=oracle_map, + forward_output_map=forward_output_map, + forward_layer_map=forward_layer_map, + crown_output_map=crown_output_map, + submodels_stack=submodels_stack + [node], + perturbation_domain_input=perturbation_domain_input, + perturbation_domain=perturbation_domain, + ) + parents = node.parent_nodes + is_merging_node = False + + ## 1. Propagation through the current node if len(parents) == 0: - # Input layer => no conversion, propagate output unchanged - return backward_bounds - else: - if isinstance(node.operation, Model): - if len(parents) > 1: - raise NotImplementedError( - "crown_model() not yet implemented for model whose embedded submodels have multiple inputs." - ) - backward_bounds = crown_model( - model=node.operation, - layer_fn=layer_fn, - backward_bounds=[backward_bounds], + # input layer: no conversion, propagate output unchanged + ... + + else: # generic case: a node with parents whose operation is a keras.Layer + # conversion to a backward DecomonLayer + if id(node) in backward_map: + backward_layer = backward_map[id(node)] + else: + backward_layer = layer_fn(node.operation, model_output_shape) + backward_map[id(node)] = backward_layer + + # get oracle bounds if needed + if backward_layer.inputs_outputs_spec.needs_oracle_bounds(): + constant_oracle_bounds = get_oracle( + node=node, perturbation_domain_input=perturbation_domain_input, perturbation_domain=perturbation_domain, oracle_map=oracle_map, forward_output_map=forward_output_map, forward_layer_map=forward_layer_map, + backward_layer=backward_layer, crown_output_map=crown_output_map, - is_submodel=True, - submodels_stack=submodels_stack + [node], - submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, - backward_map=backward_map, - model_output_shape=model_output_shape, + submodels_stack=submodels_stack, + layer_fn=layer_fn, ) else: - if id(node) in backward_map: - backward_layer = backward_map[id(node)] - else: - backward_layer = layer_fn(node.operation, model_output_shape) - backward_map[id(node)] = backward_layer - - # get oracle bounds if needed - if backward_layer.inputs_outputs_spec.needs_oracle_bounds(): - constant_oracle_bounds = get_oracle( - node=node, - perturbation_domain_input=perturbation_domain_input, - perturbation_domain=perturbation_domain, + constant_oracle_bounds = [] + + # propagate backward bounds through the decomon layer + backward_layer_inputs = backward_layer.inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=backward_bounds, + constant_oracle_bounds=constant_oracle_bounds, + perturbation_domain_inputs=[], + ) + backward_layer_outputs = backward_layer(backward_layer_inputs) + backward_bounds, _ = backward_layer.inputs_outputs_spec.split_outputs(backward_layer_outputs) + + # merging layer? (to known later how to handle propagated backward_bounds) + is_merging_node = isinstance(backward_layer, DecomonMerge) + + ## 2. Stopping criteria: is this a leaf node of the outer model? + if len(parents) == 0: + if len(submodels_stack) == 0: + # Input layer of outer model => we are done + return backward_bounds + else: + # Input layer of an embedded model => we propagate through the sumodel node parents + node = submodels_stack[-1] + submodels_stack = submodels_stack[:-1] + parents = node.parent_nodes + # check on parents + if len(parents) == 0: + raise RuntimeError("Submodel node should have parents.") + elif len(parents) > 1: + raise NotImplementedError( + "crown_model() not yet implemented for model whose embedded submodels have multiple inputs." + ) + + ## 3. Call crown recursively on parent nodes + if is_merging_node: + # merging layer (NB: could be merging only 1 parent, this is possible for Add layers) + crown_bounds_list: list[list[Tensor]] = [] + for backward_bounds_i, parent in zip(backward_bounds, parents): + crown_bounds_list.append( + crown( + node=parent, + layer_fn=layer_fn, + model_output_shape=model_output_shape, + backward_bounds=backward_bounds_i, + backward_map=backward_map, oracle_map=oracle_map, forward_output_map=forward_output_map, forward_layer_map=forward_layer_map, - backward_layer=backward_layer, crown_output_map=crown_output_map, submodels_stack=submodels_stack, - submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, - layer_fn=layer_fn, + perturbation_domain_input=perturbation_domain_input, + perturbation_domain=perturbation_domain, ) - - else: - constant_oracle_bounds = [] - - # propagate backward bounds through the decomon layer - backward_layer_inputs = backward_layer.inputs_outputs_spec.flatten_inputs( - affine_bounds_to_propagate=backward_bounds, - constant_oracle_bounds=constant_oracle_bounds, - perturbation_domain_inputs=[], ) - backward_layer_outputs = backward_layer(backward_layer_inputs) - backward_bounds, _ = backward_layer.inputs_outputs_spec.split_outputs(backward_layer_outputs) - - # Call crown recursively on parent nodes - if not isinstance(node.operation, Model) and isinstance(backward_layer, DecomonMerge): - # merging layer - crown_bounds_list: list[list[Tensor]] = [] - for backward_bounds_i, parent in zip(backward_bounds, parents): - crown_bounds_list.append( - crown( - node=parent, - layer_fn=layer_fn, - model_output_shape=model_output_shape, - backward_bounds=backward_bounds_i, - backward_map=backward_map, - oracle_map=oracle_map, - forward_output_map=forward_output_map, - forward_layer_map=forward_layer_map, - crown_output_map=crown_output_map, - submodels_stack=submodels_stack, - submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, - perturbation_domain_input=perturbation_domain_input, - perturbation_domain=perturbation_domain, - ) - ) - # reduce by summing all bounds together - # (indeed all bounds are partial affine bounds on model output w.r.t the same model input - # under the hypotheses of a single model input) - crown_bounds = backward_layer.inputs_outputs_spec.sum_backward_bounds(crown_bounds_list) + # reduce by summing all bounds together + # (indeed all bounds are partial affine bounds on model output w.r.t the same model input + # under the hypotheses of a single model input) + crown_bounds = backward_layer.inputs_outputs_spec.sum_backward_bounds(crown_bounds_list) - elif len(parents) > 1: - raise RuntimeError("Node with multiple parents should have been converted to a DecomonMerge layer.") - else: - # unary layer - crown_bounds = crown( - node=parents[0], - layer_fn=layer_fn, - model_output_shape=model_output_shape, - backward_bounds=backward_bounds, - backward_map=backward_map, - oracle_map=oracle_map, - forward_output_map=forward_output_map, - forward_layer_map=forward_layer_map, - crown_output_map=crown_output_map, - submodels_stack=submodels_stack, - submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, - perturbation_domain_input=perturbation_domain_input, - perturbation_domain=perturbation_domain, - ) - return crown_bounds + elif len(parents) > 1: + raise RuntimeError("Node with multiple parents should have been converted to a DecomonMerge layer.") + else: + # unary layer + crown_bounds = crown( + node=parents[0], + layer_fn=layer_fn, + model_output_shape=model_output_shape, + backward_bounds=backward_bounds, + backward_map=backward_map, + oracle_map=oracle_map, + forward_output_map=forward_output_map, + forward_layer_map=forward_layer_map, + crown_output_map=crown_output_map, + submodels_stack=submodels_stack, + perturbation_domain_input=perturbation_domain_input, + perturbation_domain=perturbation_domain, + ) + return crown_bounds def get_oracle( @@ -202,7 +223,6 @@ def get_oracle( backward_layer: DecomonLayer, crown_output_map: dict[int, list[keras.KerasTensor]], submodels_stack: list[Node], - submodel_perturbation_domain_input_map: dict[int, keras.KerasTensor], layer_fn: Callable[[Layer, tuple[int, ...]], DecomonLayer], ) -> Union[list[keras.KerasTensor], list[list[keras.KerasTensor]]]: """Get oracle bounds "on demand". @@ -232,8 +252,6 @@ def get_oracle( Will be used to get perturbation_domain_input for this submodel, to be used only by crown oracle. (Forward oracle being precomputed from the full model, the original perturbation_domain_input is used for it) To be used for crown oracle. - submodel_perturbation_domain_input_map: stores already computed perturbation_domain_input for submodels. - To be used for crown oracle. layer_fn: callable converting a layer and a model_output_shape into a (backward) decomon layer. To be used for crown oracle. @@ -246,7 +264,7 @@ def get_oracle( return oracle_map[id(node)] parents = node.parent_nodes - if not len(parents): + if not len(parents): # NB: get_oracle() is never called on node w/o parents (i.e. InputLayer) # input node: can be deduced directly from perturbation domain oracle_bounds = [ perturbation_domain.get_lower_x(x=perturbation_domain_input), @@ -265,37 +283,6 @@ def get_oracle( else: # crown oracle - # perturbation domain input for the (sub)model? - if len(submodels_stack) == 0: - # in outer model: we already got the proper pertubation domain input - perturbation_domain_input_submodel = perturbation_domain_input - else: - submodel_node = submodels_stack[-1] - if id(submodel_node) in submodel_perturbation_domain_input_map: - # already computed for this submodel - perturbation_domain_input_submodel = submodel_perturbation_domain_input_map[id(submodel_node)] - else: - # reconstruct perturbation domain input if in a new submodel - # 1. get oracle bounds for the current submodel node - # 2. use perturbation domain to reconstruct its input from it for the submodel - oracle_bounds_on_submodel_input = get_oracle( - node=submodel_node, - perturbation_domain_input=perturbation_domain_input, - perturbation_domain=perturbation_domain, - oracle_map=oracle_map, - forward_output_map=forward_output_map, - forward_layer_map=forward_layer_map, - backward_layer=backward_layer, # to replace by an "oracle" layer - crown_output_map=crown_output_map, - submodels_stack=submodels_stack[:-1], # remaining submodels above the current one (if any) - submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, - layer_fn=layer_fn, - ) - perturbation_domain_input_submodel = perturbation_domain.get_input_from_constant_bounds( - oracle_bounds_on_submodel_input - ) - submodel_perturbation_domain_input_map[id(submodel_node)] = perturbation_domain_input_submodel - # affine bounds on parents from sub-crowns crown_bounds = [] for parent in parents: @@ -303,11 +290,11 @@ def get_oracle( # already computed sub-crown? crown_bounds_parent = crown_output_map[id(parent)] else: - submodel_output_shape = get_model_output_shape(node=parent, backward_bounds_node=[]) + subcrown_output_shape = get_model_output_shape(node=parent, backward_bounds=[]) crown_bounds_parent = crown( node=parent, layer_fn=layer_fn, - model_output_shape=submodel_output_shape, + model_output_shape=subcrown_output_shape, backward_bounds=[], backward_map={}, # new output node, thus new backward_map oracle_map=oracle_map, @@ -315,7 +302,6 @@ def get_oracle( forward_layer_map=forward_layer_map, crown_output_map=crown_output_map, submodels_stack=submodels_stack, - submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, perturbation_domain_input=perturbation_domain_input, perturbation_domain=perturbation_domain, ) @@ -323,8 +309,10 @@ def get_oracle( crown_output_map[id(parent)] = crown_bounds_parent crown_bounds += crown_bounds_parent - # deduce oracle bounds from affine bounds on keras (sub)model inputs and corresponding perturbation domain input - backward_oracle_inputs = crown_bounds + [perturbation_domain_input_submodel] + # deduce oracle bounds from + # - affine bounds on keras (sub)model inputs and + # - corresponding perturbation domain input + backward_oracle_inputs = crown_bounds + [perturbation_domain_input] oracle_bounds = backward_layer.call_oracle(backward_oracle_inputs) # store oracle @@ -343,11 +331,6 @@ def crown_model( forward_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, forward_layer_map: Optional[dict[int, DecomonLayer]] = None, crown_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, - is_submodel: bool = False, - submodels_stack: Optional[list[Node]] = None, - submodel_perturbation_domain_input_map: Optional[dict[int, keras.KerasTensor]] = None, - backward_map: Optional[dict[int, DecomonLayer]] = None, - model_output_shape: Optional[tuple[int, ...]] = None, ) -> list[keras.KerasTensor]: """Convert a functional keras model via crown algorithm (backward propagation) @@ -368,17 +351,6 @@ def crown_model( crown_output_map: output of subcrowns per output node. Avoids relaunching a crown if several nodes share parents. To be used for crown oracle. - is_submodel: specify if called from within a crown to propagate through an embedded submodel - submodels_stack: Not empty only if in a submodel. - A list of nodes corresponding to the successive embedded submodels, - from the outerest to the innerest submodel, the last one being the current submodel. - Will be used to get perturbation_domain_input for this submodel, to be used only by crown oracle. - (Forward oracle being precomputed from the full model, the original perturbation_domain_input is used for it) - To be used for crown oracle. - submodel_perturbation_domain_input_map: stores already computed perturbation_domain_input for submodels. - To be used for crown oracle. - backward_map: stores converted layer by node for the current crown - Should be set only if is_submodel is True. model_output_shape: if submodel is True, must be set to the output_shape used in the current crown Returns: @@ -393,32 +365,19 @@ def crown_model( forward_output_map = {} if crown_output_map is None: crown_output_map = {} - if submodels_stack is None: - submodels_stack = [] - if submodel_perturbation_domain_input_map is None: - submodel_perturbation_domain_input_map = {} # ensure (sub)model is functional model = ensure_functional_model(model) # Retrieve output nodes in same order as model.outputs output_nodes = get_output_nodes(model) - if is_submodel and len(output_nodes) > 1: - raise NotImplementedError( - "crown_model() not yet implemented for model whose embedded submodels have multiple outputs." - ) + # Apply crown on each output, with the appropriate backward_bounds and model_output_shape output = [] for node, backward_bounds_node in zip(output_nodes, backward_bounds): - if is_submodel: - # for embedded submodel, pass the frozen model_output_shape fixed and the current backward_map - backward_map_node = backward_map - if model_output_shape is None: - raise RuntimeError("`submodel_output_shape` must be set if `submodel` is True.") - else: - # new backward_map and new model_output_shape for each output node - model_output_shape = get_model_output_shape(node=node, backward_bounds_node=backward_bounds_node) - backward_map_node = {} + # new backward_map and new model_output_shape for each output node + model_output_shape = get_model_output_shape(node=node, backward_bounds=backward_bounds_node) + backward_map_node = {} output_crown = crown( node=node, @@ -430,8 +389,7 @@ def crown_model( forward_output_map=forward_output_map, forward_layer_map=forward_layer_map, crown_output_map=crown_output_map, - submodels_stack=submodels_stack, - submodel_perturbation_domain_input_map=submodel_perturbation_domain_input_map, + submodels_stack=[], # main model, not in any submodel perturbation_domain_input=perturbation_domain_input, perturbation_domain=perturbation_domain, ) @@ -503,7 +461,7 @@ def convert_backward( return output -def get_model_output_shape(node: Node, backward_bounds_node: list[Tensor]): +def get_model_output_shape(node: Node, backward_bounds: list[Tensor]): """Get outer model output shape w/o batchsize. If any backward bounds are passed, we deduce the outer keras model output shape from it. @@ -518,16 +476,16 @@ def get_model_output_shape(node: Node, backward_bounds_node: list[Tensor]): Args: node: current output node of the (potentially inner) keras model to convert - backward_bounds_node: backward bounds specified for this node + backward_bounds: backward bounds specified for this node Returns: outer keras model output shape, excluding batchsize """ - if len(backward_bounds_node) == 0: + if len(backward_bounds) == 0: return node.outputs[0].shape[1:] else: - _, b, _, _ = backward_bounds_node + _, b, _, _ = backward_bounds return b.shape[1:] From fa95b9f931916df4f21544be85cfdcc952344b20 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 19 Feb 2024 13:24:55 +0100 Subject: [PATCH 062/101] Remove old backward layers --- src/decomon/backward_layers/__init__.py | 0 src/decomon/backward_layers/activations.py | 485 ------------- .../backward_layers/backward_layers.py | 629 ---------------- .../backward_layers/backward_maxpooling.py | 204 ------ src/decomon/backward_layers/backward_merge.py | 609 ---------------- .../backward_layers/backward_reshape.py | 0 src/decomon/backward_layers/convert.py | 45 -- src/decomon/backward_layers/core.py | 144 ---- src/decomon/backward_layers/utils.py | 669 ------------------ src/decomon/backward_layers/utils_conv.py | 159 ----- 10 files changed, 2944 deletions(-) delete mode 100644 src/decomon/backward_layers/__init__.py delete mode 100644 src/decomon/backward_layers/activations.py delete mode 100644 src/decomon/backward_layers/backward_layers.py delete mode 100644 src/decomon/backward_layers/backward_maxpooling.py delete mode 100644 src/decomon/backward_layers/backward_merge.py delete mode 100644 src/decomon/backward_layers/backward_reshape.py delete mode 100644 src/decomon/backward_layers/convert.py delete mode 100644 src/decomon/backward_layers/core.py delete mode 100644 src/decomon/backward_layers/utils.py delete mode 100644 src/decomon/backward_layers/utils_conv.py diff --git a/src/decomon/backward_layers/__init__.py b/src/decomon/backward_layers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/decomon/backward_layers/activations.py b/src/decomon/backward_layers/activations.py deleted file mode 100644 index 153f1cd9..00000000 --- a/src/decomon/backward_layers/activations.py +++ /dev/null @@ -1,485 +0,0 @@ -import warnings -from collections.abc import Callable -from typing import Any, Optional, Union - -import keras.ops as K -import numpy as np -from keras.layers import Layer - -from decomon.core import ( - BoxDomain, - ForwardMode, - InputsOutputsSpec, - PerturbationDomain, - Slope, -) -from decomon.types import Tensor -from decomon.utils import ( - get_linear_hull_relu, - get_linear_hull_s_shape, - get_linear_hull_sigmoid, - get_linear_hull_tanh, - get_linear_softplus_hull, - sigmoid_prime, - softsign_prime, - tanh_prime, -) - -ELU = "elu" -SELU = "selu" -SOFTPLUS = "softplus" -SOFTSIGN = "softsign" -SOFTMAX = "softmax" -RELU_ = "relu_" -RELU = "relu" -SIGMOID = "sigmoid" -TANH = "tanh" -EXPONENTIAL = "exponential" -HARD_SIGMOID = "hard_sigmoid" -LINEAR = "linear" - - -def backward_relu( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - alpha: float = 0.0, - max_value: Optional[float] = None, - threshold: float = 0.0, - slope: Union[str, Slope] = Slope.V_SLOPE, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - **kwargs: Any, -) -> list[Tensor]: - """Backward LiRPA of relu - - Args: - inputs - dc_decomp - perturbation_domain - alpha - max_value - threshold - slope - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - if dc_decomp: - raise NotImplementedError() - - if threshold != 0: - raise NotImplementedError() - - if not alpha and max_value is None: - # default values: return relu_(x) = max(x, 0) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - input_shape = inputs_outputs_spec.get_kerasinputshape(inputs) - bounds = get_linear_hull_relu(upper=u_c, lower=l_c, slope=slope, **kwargs) - dim = int(np.prod(input_shape[1:])) # type: ignore - return [K.reshape(elem, (-1, dim)) for elem in bounds] - - raise NotImplementedError() - - -def backward_sigmoid( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - slope: Union[str, Slope] = Slope.V_SLOPE, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - **kwargs: Any, -) -> list[Tensor]: - """Backward LiRPA of sigmoid - - Args: - inputs - dc_decomp - perturbation_domain - slope - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if dc_decomp: - raise NotImplementedError() - mode = ForwardMode(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - return get_linear_hull_sigmoid(u_c, l_c, slope=slope, **kwargs) - - -def backward_tanh( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - slope: Union[str, Slope] = Slope.V_SLOPE, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - **kwargs: Any, -) -> list[Tensor]: - """Backward LiRPA of tanh - - Args: - inputs - dc_decomp - perturbation_domain - slope - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if dc_decomp: - raise NotImplementedError() - mode = ForwardMode(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - return get_linear_hull_tanh(u_c, l_c, slope=slope, **kwargs) - - -def backward_hard_sigmoid( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - slope: Union[str, Slope] = Slope.V_SLOPE, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - **kwargs: Any, -) -> list[Tensor]: - """Backward LiRPA of hard sigmoid - - Args: - inputs - dc_decomp - perturbation_domain - slope - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if dc_decomp: - raise NotImplementedError() - mode = ForwardMode(mode) - raise NotImplementedError() - - -def backward_elu( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - slope: Union[str, Slope] = Slope.V_SLOPE, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - **kwargs: Any, -) -> list[Tensor]: - """Backward LiRPA of Exponential Linear Unit - - Args: - inputs - dc_decomp - perturbation_domain - slope - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if dc_decomp: - raise NotImplementedError() - mode = ForwardMode(mode) - slope = Slope(slope) - raise NotImplementedError() - - -def backward_selu( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - slope: Union[str, Slope] = Slope.V_SLOPE, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - **kwargs: Any, -) -> list[Tensor]: - """Backward LiRPA of Scaled Exponential Linear Unit (SELU) - - Args: - inputs - dc_decomp - perturbation_domain - slope - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if dc_decomp: - raise NotImplementedError() - mode = ForwardMode(mode) - slope = Slope(slope) - raise NotImplementedError() - - -def backward_linear( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - slope: Union[str, Slope] = Slope.V_SLOPE, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - **kwargs: Any, -) -> list[Tensor]: - """Backward LiRPA of linear - - Args: - inputs - dc_decomp - perturbation_domain - slope - mode - - Returns: - - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - slope = Slope(slope) - raise NotImplementedError() - - -def backward_exponential( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - slope: Union[str, Slope] = Slope.V_SLOPE, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - **kwargs: Any, -) -> list[Tensor]: - """Backward LiRPAof exponential - - Args: - inputs - dc_decomp - perturbation_domain - slope - mode - - Returns: - - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if dc_decomp: - raise NotImplementedError() - mode = ForwardMode(mode) - slope = Slope(slope) - raise NotImplementedError() - - -def backward_softplus( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - slope: Union[str, Slope] = Slope.V_SLOPE, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - **kwargs: Any, -) -> list[Tensor]: - """Backward LiRPA of softplus - - Args: - inputs - dc_decomp - perturbation_domain - slope - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if dc_decomp: - raise NotImplementedError() - mode = ForwardMode(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - return get_linear_softplus_hull(u_c, l_c, slope=slope, **kwargs) - - -def backward_softsign( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - slope: Union[str, Slope] = Slope.V_SLOPE, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - **kwargs: Any, -) -> list[Tensor]: - """Backward LiRPA of softsign - - Args: - inputs - w_u_out - b_u_out - w_l_out - b_l_out - perturbation_domain - slope: backward slope - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - bounds = get_linear_hull_s_shape( - inputs, func=K.softsign, f_prime=softsign_prime, perturbation_domain=perturbation_domain, mode=mode - ) - shape = int(np.prod(inputs[-1].shape[1:])) - return [K.reshape(elem, (-1, shape)) for elem in bounds] - - -def backward_softsign_( - inputs: list[Tensor], - w_u_out: Tensor, - b_u_out: Tensor, - w_l_out: Tensor, - b_l_out: Tensor, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - if perturbation_domain is None: - perturbation_domain = BoxDomain() - w_u_0, b_u_0, w_l_0, b_l_0 = get_linear_hull_s_shape( - inputs, func=K.softsign, f_prime=softsign_prime, perturbation_domain=perturbation_domain, mode=mode, slope=slope - ) - - w_u_0 = K.expand_dims(w_u_0, -1) - w_l_0 = K.expand_dims(w_l_0, -1) - - b_u_0 = K.expand_dims(b_u_0, -1) - b_l_0 = K.expand_dims(b_l_0, -1) - - z_value = K.cast(0.0, dtype=inputs[0].dtype) - b_u_out = K.sum(K.maximum(z_value, w_u_out) * b_u_0 + K.minimum(z_value, w_u_out) * b_l_0, 2) + b_u_out - b_l_out = K.sum(K.maximum(z_value, w_l_out) * b_l_0 + K.minimum(z_value, w_l_out) * b_u_0, 2) + b_l_out - w_u_out = K.maximum(z_value, w_u_out) * w_u_0 + K.minimum(z_value, w_u_out) * w_l_0 - w_l_out = K.maximum(z_value, w_l_out) * w_l_0 + K.minimum(z_value, w_l_out) * w_u_0 - - return [w_u_out, b_u_out, w_l_out, b_l_out] - - -def backward_softmax( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - slope: Union[str, Slope] = Slope.V_SLOPE, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - axis: int = -1, - **kwargs: Any, -) -> list[Tensor]: - """Backward LiRPA of softmax - - Args: - inputs - dc_decomp - perturbation_domain - slope - mode - axis - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - if dc_decomp: - raise NotImplementedError() - mode = ForwardMode(mode) - slope = Slope(slope) - raise NotImplementedError() - - -def deserialize(name: str) -> Callable[..., list[Tensor]]: - """Get the activation from name. - - Args: - name: name of the method. - among the implemented Keras activation function. - - Returns: - - """ - name = name.lower() - - if name == SOFTMAX: - return backward_softmax - if name == ELU: - return backward_elu - if name == SELU: - return backward_selu - if name == SOFTPLUS: - return backward_softplus - if name == SOFTSIGN: - return backward_softsign - if name == SIGMOID: - return backward_sigmoid - if name == TANH: - return backward_tanh - if name in [RELU, RELU_]: - return backward_relu - if name == EXPONENTIAL: - return backward_exponential - if name == LINEAR: - return backward_linear - raise ValueError("Could not interpret " "activation function identifier:", name) - - -def get(identifier: Any) -> Callable[..., list[Tensor]]: - """Get the `identifier` activation function. - - Args: - identifier: None or str, name of the function. - - Returns: - The activation function, `linear` if `identifier` is None. - - """ - if identifier is None: - return backward_linear - if isinstance(identifier, str): - return deserialize(identifier) - elif callable(identifier): - if isinstance(identifier, Layer): - warnings.warn( - "Do not pass a layer instance (such as {identifier}) as the " - "activation argument of another layer. Instead, advanced " - "activation layers should be used just like any other " - "layer in a model.".format(identifier=identifier.__class__.__name__) - ) - return identifier - else: - raise ValueError("Could not interpret " "activation function identifier:", identifier) diff --git a/src/decomon/backward_layers/backward_layers.py b/src/decomon/backward_layers/backward_layers.py deleted file mode 100644 index 76619e64..00000000 --- a/src/decomon/backward_layers/backward_layers.py +++ /dev/null @@ -1,629 +0,0 @@ -from typing import Any, Optional, Union - -import keras -import keras.ops as K -import numpy as np -from keras.layers import Flatten, Layer - -from decomon.backward_layers.activations import get -from decomon.backward_layers.core import BackwardLayer -from decomon.backward_layers.utils import get_identity_lirpa -from decomon.backward_layers.utils_conv import get_toeplitz -from decomon.core import ( - ForwardMode, - GridDomain, - Option, - PerturbationDomain, - Slope, - get_affine, - get_ibp, -) -from decomon.keras_utils import BatchedDiagLike, BatchedIdentityLike -from decomon.layers.convert import to_decomon -from decomon.layers.core import DecomonLayer -from decomon.layers.decomon_layers import DecomonBatchNormalization -from decomon.layers.utils import ClipAlpha, NonNeg, NonPos -from decomon.models.utils import get_input_dim -from decomon.types import BackendTensor - - -class BackwardDense(BackwardLayer): - """Backward LiRPA of Dense""" - - def __init__( - self, - layer: Layer, - input_dim: int = -1, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - self.use_bias = self.layer.use_bias - if not isinstance(self.layer, DecomonLayer): - if input_dim < 0: - input_dim = get_input_dim(self.layer) - self.layer = to_decomon( - layer=layer, - input_dim=input_dim, - dc_decomp=False, - perturbation_domain=self.perturbation_domain, - finetune=False, - ibp=get_ibp(self.mode), - affine=get_affine(self.mode), - shared=True, - fast=False, - ) - self.frozen_weights = False - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - if len(inputs) == 0: - inputs = self.layer.input - - y = inputs[-1] - flatten_inputdim_wo_last_dim = int(np.prod(y.shape[1:-1])) - flatten_outputdim = flatten_inputdim_wo_last_dim * self.layer.units - batchsize = y.shape[0] - - # kernel reshaped: diagonal by blocks, with original kernel on the diagonal, `flatten_inputdim_wo_last_dim` times - # repeated batchsize times - zero_block = K.zeros_like(self.kernel) - kernel_diag_by_block = K.concatenate( - [ - K.concatenate( - [self.kernel if i == j else zero_block for i in range(flatten_inputdim_wo_last_dim)], axis=-1 - ) - for j in range(flatten_inputdim_wo_last_dim) - ], - axis=-2, - ) - w = K.repeat(kernel_diag_by_block[None], batchsize, axis=0) - if self.use_bias: - b = K.repeat( - K.reshape(K.repeat(self.bias[None], flatten_inputdim_wo_last_dim, axis=0), (1, -1)), batchsize, axis=0 - ) - else: - b = K.zeros((batchsize, flatten_outputdim), dtype=self.dtype) - - return [w, b, w, b] - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - """ - Args: - input_shape: list of input shape - - Returns: - - """ - if self.layer.kernel is None: - raise RuntimeError("self.layer.kernel cannot be None when calling self.build()") - self.kernel = self.layer.kernel - self._trainable_weights = [self.kernel] - if self.use_bias: - if self.layer.bias is None: - raise RuntimeError("self.layer.bias cannot be None when calling self.build()") - self.bias = self.layer.bias - self._trainable_weights.append(self.bias) - self.built = True - - def freeze_weights(self) -> None: - if not self.frozen_weights: - self._trainable_weights = [] - if getattr(self.layer, "freeze_weights"): - self.layer.freeze_weights() - self.frozen_weights = True - - def unfreeze_weights(self) -> None: - if self.frozen_weights: - if getattr(self.layer, "unfreeze_weights"): - self.layer.unfreeze_weights() - self.frozen_weights = False - - -class BackwardConv2D(BackwardLayer): - """Backward LiRPA of Conv2D""" - - def __init__( - self, - layer: Layer, - input_dim: int = -1, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - if not isinstance(self.layer, DecomonLayer): - if input_dim < 0: - input_dim = get_input_dim(self.layer) - self.layer = to_decomon( - layer=layer, - input_dim=input_dim, - dc_decomp=False, - perturbation_domain=self.perturbation_domain, - finetune=False, - ibp=get_ibp(self.mode), - affine=get_affine(self.mode), - shared=True, - fast=False, - ) - self.frozen_weights = False - - def get_affine_components(self, inputs: list[BackendTensor]) -> tuple[BackendTensor, BackendTensor]: - """Express the implicit affine matrix of the convolution layer. - - Conv is a linear operator but its affine component is implicit - we use im2col and extract_patches to express the affine matrix - Note that this matrix is Toeplitz - - Args: - inputs: list of input tensors - Returns: - the affine operators W, b : conv(inputs)= W.inputs + b - """ - - w_out_u_ = get_toeplitz(self.layer, True) - output = self.layer.output - if isinstance(output, keras.KerasTensor): - output_shape = output.shape - else: # list of outputs - output_shape = output[-1].shape - output_shape = output_shape[1:] - if self.layer.data_format == "channels_last": - b_out_u_ = K.reshape(K.zeros(output_shape, dtype=self.layer.dtype), (-1, output_shape[-1])) - else: - b_out_u_ = K.transpose( - K.reshape(K.zeros(output_shape, dtype=self.layer.dtype), (-1, output_shape[0])), (1, 0) - ) - - if self.layer.use_bias: - bias_ = K.cast(self.layer.bias, self.layer.dtype) - if self.layer.data_format == "channels_last": - b_out_u_ = b_out_u_ + bias_[None] - else: - b_out_u_ = b_out_u_ + bias_[:, None] - b_out_u_ = K.ravel(b_out_u_) - - z_value = K.cast(0.0, self.dtype) - y_ = inputs[-1] - shape = int(np.prod(y_.shape[1:])) - y_flatten = K.reshape(z_value * y_, (-1, int(np.prod(shape)))) # (None, n_in) - w_out_ = K.sum(y_flatten, -1)[:, None, None] + w_out_u_ - b_out_ = K.sum(y_flatten, -1)[:, None] + b_out_u_ - - return w_out_, b_out_ - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - weight_, bias_ = self.get_affine_components(inputs) - return [weight_, bias_] * 2 - - def freeze_weights(self) -> None: - if not self.frozen_weights: - self._trainable_weights = [] - if getattr(self.layer, "freeze_weights"): - self.layer.freeze_weights() - self.frozen_weights = True - - def unfreeze_weights(self) -> None: - if self.frozen_weights: - if getattr(self.layer, "unfreeze_weights"): - self.layer.unfreeze_weights() - self.frozen_weights = False - - -class BackwardActivation(BackwardLayer): - def __init__( - self, - layer: Layer, - slope: Union[str, Slope] = Slope.V_SLOPE, - finetune: bool = False, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - self.activation = get(layer.get_config()["activation"]) - self.activation_name = layer.get_config()["activation"] - self.slope = Slope(slope) - self.finetune = finetune - self.finetune_param: list[keras.Variable] = [] - if self.finetune: - self.frozen_alpha = False - self.grid_finetune: list[keras.Variable] = [] - self.frozen_grid = False - - def get_config(self) -> dict[str, Any]: - config = super().get_config() - config.update( - { - "slope": self.slope, - "finetune": self.finetune, - } - ) - return config - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - """ - Args: - input_shape: list of input shape - - Returns: - - """ - input_dim = int(np.prod(input_shape[-1][1:])) # type: ignore - - if self.finetune and self.activation_name != "linear": - if isinstance(self.perturbation_domain, GridDomain): - if self.activation_name[:4] == "relu": - self.alpha_b_l = self.add_weight( - shape=( - 3, - input_dim, - ), - initializer="ones", - name="alpha_l_b_0", - regularizer=None, - constraint=ClipAlpha(), - ) - alpha_b_l = np.zeros((3, input_dim)) - alpha_b_l[0] = 1 - self.alpha_b_l.assign(alpha_b_l) - self.finetune_param.append(self.alpha_b_l) - - else: - self.alpha_b_l = self.add_weight( - shape=(input_dim,), - initializer="ones", - name="alpha_l_b", - regularizer=None, - constraint=ClipAlpha(), - ) - alpha_b_l = np.ones((input_dim,)) - # alpha_b_l[0] = 1 - self.alpha_b_l.assign(alpha_b_l) - - if self.activation_name[:4] != "relu": - self.alpha_b_u = self.add_weight( - shape=(input_dim,), - initializer="ones", - name="alpha_u_b", - regularizer=None, - constraint=ClipAlpha(), - ) - self.finetune_param.append(self.alpha_b_u) - - self.finetune_param.append(self.alpha_b_l) - if len(self.finetune_param) == 1: - self.finetune_param = self.finetune_param[0] - - # grid domain - if self.activation_name[:4] == "relu": - if ( - isinstance(self.perturbation_domain, GridDomain) - and self.perturbation_domain.opt_option == Option.lagrangian - and self.mode != ForwardMode.IBP - ): - finetune_grid_pos = self.add_weight( - shape=(input_dim,), - initializer="zeros", - name="lambda_grid_neg", - regularizer=None, - constraint=NonNeg(), - ) - - finetune_grid_neg = self.add_weight( - shape=(input_dim,), - initializer="zeros", - name="lambda_grid_pos", - regularizer=None, - constraint=NonPos(), - ) - - self.grid_finetune = [finetune_grid_neg, finetune_grid_pos] - - if ( - isinstance(self.perturbation_domain, GridDomain) - and self.perturbation_domain.opt_option == Option.milp - and self.mode != ForwardMode.IBP - ): - finetune_grid_A = self.add_weight( - shape=(input_dim,), - initializer="zeros", - name=f"A_{self.layer.name}_{self.rec}", - regularizer=None, - trainable=False, - ) # constraint=NonPos() - finetune_grid_B = self.add_weight( - shape=(input_dim,), - initializer="zeros", - name=f"B_{self.layer.name}_{self.rec}", - regularizer=None, - trainable=False, - ) # constraint=NonNeg() - - self.grid_finetune = [finetune_grid_A, finetune_grid_B] - - self.built = True - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - # infer the output dimension - if self.activation_name != "linear": - if self.finetune: - w_u_out, b_u_out, w_l_out, b_l_out = self.activation( - inputs, - perturbation_domain=self.perturbation_domain, - slope=self.slope, - mode=self.mode, - finetune=self.finetune_param, - finetune_grid=self.grid_finetune, - ) - else: - w_u_out, b_u_out, w_l_out, b_l_out = self.activation( - inputs, - perturbation_domain=self.perturbation_domain, - slope=self.slope, - mode=self.mode, - finetune_grid=self.grid_finetune, - ) - else: - w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) - - if len(w_u_out.shape) == 2: - w_u_out = BatchedDiagLike()(w_u_out) - if len(w_l_out.shape) == 2: - w_l_out = BatchedDiagLike()(w_l_out) - - return [w_u_out, b_u_out, w_l_out, b_l_out] - - def freeze_alpha(self) -> None: - if not self.frozen_alpha: - if self.finetune and self.mode in [ForwardMode.AFFINE, ForwardMode.HYBRID]: - if len(self.grid_finetune): - self._trainable_weights = self._trainable_weights[:2] - else: - self._trainable_weights = [] - self.frozen_alpha = True - - def unfreeze_alpha(self) -> None: - if self.frozen_alpha: - if self.finetune and self.mode in [ForwardMode.AFFINE, ForwardMode.HYBRID]: - if self.activation_name != "linear": - if self.activation_name[:4] != "relu": - self._trainable_weights += [self.alpha_b_u, self.alpha_b_l] - else: - self._trainable_weights += [self.alpha_b_l] - self.frozen_alpha = False - - def freeze_grid(self) -> None: - if len(self.grid_finetune) and not self.frozen_grid: - self._trainable_weights = self._trainable_weights[2:] - self.frozen_grid = True - - def unfreeze_grid(self) -> None: - if len(self.grid_finetune) and self.frozen_grid: - self._trainable_weights = self.grid_finetune + self._trainable_weights - self.frozen_grid = False - - -class BackwardFlatten(BackwardLayer): - """Backward LiRPA of Flatten""" - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - return get_identity_lirpa(inputs) - - -class BackwardReshape(BackwardLayer): - """Backward LiRPA of Reshape""" - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - return get_identity_lirpa(inputs) - - -class BackwardPermute(BackwardLayer): - """Backward LiRPA of Permute""" - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - self.dims = layer.dims - self.op = layer.call - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) - - # w_u_out (None, n_in, n_out) - y = inputs[-1] - n_dim = w_u_out.shape[1] - n_out = w_u_out.shape[-1] - shape = list(y.shape[1:]) - - w_u_out = K.reshape(w_u_out, [-1] + shape + [n_out]) - w_l_out = K.reshape(w_l_out, [-1] + shape + [n_out]) - - dims = [0] + list(self.dims) + [len(y.shape)] - dims = list(np.argsort(dims)) - w_u_out = K.reshape(K.transpose(w_u_out, dims), (-1, n_dim, n_out)) - w_l_out = K.reshape(K.transpose(w_l_out, dims), (-1, n_dim, n_out)) - - return [w_u_out, b_u_out, w_l_out, b_l_out] - - -class BackwardDropout(BackwardLayer): - """Backward LiRPA of Dropout""" - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - return get_identity_lirpa(inputs) - - -class BackwardBatchNormalization(BackwardLayer): - """Backward LiRPA of Batch Normalization""" - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - - self.axis = self.layer.axis - self.op_flat = Flatten() - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - y = inputs[-1] - n_out = int(np.prod(y.shape[1:])) - - n_dim = len(y.shape) - shape = [1] * n_dim - shape[self.axis] = self.layer.moving_mean.shape[0] - - if not hasattr(self.layer, "gamma") or self.layer.gamma is None: # scale = False - gamma = K.ones_like(self.layer.moving_variance) - else: # scale = True - gamma = self.layer.gamma - if not hasattr(self.layer, "beta") or self.layer.beta is None: # center = False - beta = K.zeros_like(self.layer.moving_mean) - else: # center = True - beta = self.layer.beta - - w = gamma / K.sqrt(self.layer.moving_variance + self.layer.epsilon) - b = beta - w * self.layer.moving_mean - - # reshape w - w_b = K.reshape( - K.reshape(BatchedIdentityLike()(K.reshape(y, (-1, n_out))), tuple(y.shape) + (-1,)) - * K.reshape(w, shape + [1]), - (-1, n_out, n_out), - ) - - # reshape b - b_b = K.reshape(K.ones_like(y) * K.reshape(b, shape), (-1, n_out)) - - return [w_b, b_b, w_b, b_b] - - -class BackwardInputLayer(BackwardLayer): - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - return get_identity_lirpa(inputs) diff --git a/src/decomon/backward_layers/backward_maxpooling.py b/src/decomon/backward_layers/backward_maxpooling.py deleted file mode 100644 index e1e5f567..00000000 --- a/src/decomon/backward_layers/backward_maxpooling.py +++ /dev/null @@ -1,204 +0,0 @@ -from typing import Any, Optional, Union - -import keras.ops as K -import numpy as np -from keras.layers import Flatten, Layer - -from decomon.backward_layers.core import BackwardLayer -from decomon.backward_layers.utils import backward_max_, get_identity_lirpa -from decomon.core import ForwardMode, PerturbationDomain -from decomon.keras_utils import BatchedIdentityLike -from decomon.types import BackendTensor - - -class BackwardMaxPooling2D(BackwardLayer): - """Backward LiRPA of MaxPooling2D""" - - pool_size: tuple[int, int] - strides: tuple[int, int] - padding: str - data_format: str - fast: bool - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - raise NotImplementedError() - - def _pooling_function_fast( - self, - inputs: list[BackendTensor], - w_u_out: BackendTensor, - b_u_out: BackendTensor, - w_l_out: BackendTensor, - b_l_out: BackendTensor, - ) -> list[BackendTensor]: - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - - op_flat = Flatten() - - b_u_pooled = K.max_pool(u_c, self.pool_size, self.strides, self.padding, self.data_format) - b_l_pooled = K.max_pool(l_c, self.pool_size, self.strides, self.padding, self.data_format) - - b_u_pooled = K.expand_dims(K.expand_dims(op_flat(b_u_pooled), 1), -1) - b_l_pooled = K.expand_dims(K.expand_dims(op_flat(b_l_pooled), 1), -1) - - y = inputs[-1] - n_out = w_u_out.shape[-1] - - w_u_out_new = K.concatenate([K.expand_dims(K.expand_dims(0 * (op_flat(y)), 1), -1)] * n_out, -1) - w_l_out_new = w_u_out_new - - z_value = K.cast(0.0, dtype=w_u_out.dtype) - b_u_out_new = ( - K.sum(K.maximum(w_u_out, z_value) * b_u_pooled, 2) - + K.sum(K.minimum(w_u_out, z_value) * b_l_pooled, 2) - + b_u_out - ) - b_l_out_new = ( - K.sum(K.maximum(w_l_out, z_value) * b_l_pooled, 2) - + K.sum(K.minimum(w_l_out, z_value) * b_u_pooled, 2) - + b_l_out - ) - - return [w_u_out_new, b_u_out_new, w_l_out_new, b_l_out_new] - - def _pooling_function_not_fast( - self, - inputs: list[BackendTensor], - w_u_out: BackendTensor, - b_u_out: BackendTensor, - w_l_out: BackendTensor, - b_l_out: BackendTensor, - ) -> list[BackendTensor]: - """ - Args: - inputs - pool_size - strides - padding - data_format - - Returns: - - """ - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - dtype = x.dtype - empty_tensor = self.inputs_outputs_spec.get_empty_tensor(dtype=dtype) - y = inputs[-1] - input_shape = y.shape - - if self.data_format in [None, "channels_last"]: - axis = -1 - else: - axis = 1 - - # initialize vars - u_c_tmp, w_u_tmp, b_u_tmp, l_c_tmp, w_l_tmp, b_l_tmp = ( - empty_tensor, - empty_tensor, - empty_tensor, - empty_tensor, - empty_tensor, - empty_tensor, - ) - - if self.ibp: - u_c_tmp = K.concatenate([self.internal_op(elem) for elem in K.split(u_c, input_shape[-1], -1)], -2) - l_c_tmp = K.concatenate([self.internal_op(elem) for elem in K.split(l_c, input_shape[-1], -1)], -2) - - if self.affine: - b_u_tmp = K.concatenate([self.internal_op(elem) for elem in K.split(b_u, input_shape[-1], -1)], -2) - b_l_tmp = K.concatenate([self.internal_op(elem) for elem in K.split(b_l, input_shape[-1], -1)], -2) - w_u_tmp = K.concatenate([self.internal_op(elem) for elem in K.split(w_u, input_shape[-1], -1)], -2) - w_l_tmp = K.concatenate([self.internal_op(elem) for elem in K.split(w_l, input_shape[-1], -1)], -2) - - if self.dc_decomp: - raise NotImplementedError() - else: - h_tmp, g_tmp = empty_tensor, empty_tensor - - outputs_tmp = self.inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_tmp, w_u_tmp, b_u_tmp, l_c_tmp, w_l_tmp, b_l_tmp, h_tmp, g_tmp] - ) - - w_u_out, b_u_out, w_l_out, b_l_out = backward_max_( - outputs_tmp, - w_u_out, - b_u_out, - w_l_out, - b_l_out, - perturbation_domain=self.perturbation_domain, - mode=self.mode, - dc_decomp=self.dc_decomp, - axis=-1, - ) - - # invert the convolution - op_flat = Flatten() - - # do not do the activation so far - # get input shape - input_shape_channelreduced = list(inputs[0].shape[1:]) - n_axis = input_shape_channelreduced[axis] - input_shape_channelreduced[axis] = 1 - n_dim = int(np.prod(input_shape_channelreduced)) - - # create diagonal matrix - id_list = [BatchedIdentityLike()(op_flat(elem[0][None])) for elem in K.split(y, input_shape[axis], axis)] - - id_list = [K.reshape(identity_mat, [-1] + input_shape_channelreduced) for identity_mat in id_list] - w_list = [self.internal_op(identity_mat) for identity_mat in id_list] - - # flatten - weights = [K.reshape(op_flat(weights), (n_dim, -1, int(np.prod(self.pool_size)))) for weights in w_list] - - n_0 = weights[0].shape[1] - n_1 = weights[0].shape[2] - - w_u_out = K.reshape(w_u_out, (-1, 1, n_0, input_shape[axis], w_u_out.shape[-2], n_1)) - w_l_out = K.reshape(w_l_out, (-1, 1, n_0, input_shape[axis], w_l_out.shape[-2], n_1)) - - weights = K.expand_dims(K.concatenate([K.expand_dims(K.expand_dims(w, -2), -2) for w in weights], 2), 0) - - w_u_out = K.reshape( - K.sum(K.expand_dims(w_u_out, 1) * weights, (3, -1)), (-1, 1, n_dim * n_axis, w_u_out.shape[-2]) - ) - w_l_out = K.reshape( - K.sum(K.expand_dims(w_l_out, 1) * weights, (3, -1)), (-1, 1, n_dim * n_axis, w_l_out.shape[-2]) - ) - - return [w_u_out, b_u_out, w_l_out, b_l_out] - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) - if self.fast: - return self._pooling_function_fast( - inputs=inputs, - w_u_out=w_u_out, - b_u_out=b_u_out, - w_l_out=w_l_out, - b_l_out=b_l_out, - ) - else: - return self._pooling_function_not_fast( - inputs=inputs, - w_u_out=w_u_out, - b_u_out=b_u_out, - w_l_out=w_l_out, - b_l_out=b_l_out, - ) diff --git a/src/decomon/backward_layers/backward_merge.py b/src/decomon/backward_layers/backward_merge.py deleted file mode 100644 index 0a9028a7..00000000 --- a/src/decomon/backward_layers/backward_merge.py +++ /dev/null @@ -1,609 +0,0 @@ -from abc import ABC, abstractmethod -from itertools import chain -from typing import Any, Optional, Union - -import keras -import keras.ops as K -import numpy as np -from keras.layers import Layer, Wrapper - -from decomon.backward_layers.utils import ( - backward_add, - backward_maximum, - backward_minimum, - backward_multiply, - backward_subtract, - get_identity_lirpa, -) -from decomon.core import ( - BoxDomain, - ForwardMode, - InputsOutputsSpec, - PerturbationDomain, - get_affine, - get_ibp, -) -from decomon.layers.core import DecomonLayer -from decomon.layers.decomon_merge_layers import ( - DecomonAdd, - DecomonConcatenate, - DecomonDot, - DecomonMaximum, - DecomonMinimum, - DecomonMultiply, - DecomonSubtract, -) -from decomon.layers.utils import broadcast, multiply, permute_dimensions, split -from decomon.types import BackendTensor - - -class BackwardMerge(ABC, Wrapper): - layer: Layer - _trainable_weights: list[keras.Variable] - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - kwargs.pop("slope", None) - kwargs.pop("finetune", None) - super().__init__(layer, **kwargs) - self.rec = rec - if isinstance(self.layer, DecomonLayer): - self.mode = self.layer.mode - self.perturbation_domain = self.layer.perturbation_domain - self.dc_decomp = self.layer.dc_decomp - else: - self.mode = ForwardMode(mode) - if perturbation_domain is None: - self.perturbation_domain = BoxDomain() - else: - self.perturbation_domain = perturbation_domain - self.dc_decomp = dc_decomp - self.inputs_outputs_spec = InputsOutputsSpec( - dc_decomp=self.dc_decomp, mode=self.mode, perturbation_domain=self.perturbation_domain - ) - - @property - def ibp(self) -> bool: - return get_ibp(self.mode) - - @property - def affine(self) -> bool: - return get_affine(self.mode) - - def get_config(self) -> dict[str, Any]: - config = super().get_config() - config.update( - { - "rec": self.rec, - "mode": self.mode, - "perturbation_domain": self.perturbation_domain, - "dc_decomp": self.dc_decomp, - } - ) - return config - - @abstractmethod - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: - """ - Args: - inputs - - Returns: - - """ - pass - - def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: - """Compute expected output shape according to input shape - - Will be called by symbolic calls on Keras Tensors. - - Args: - input_shape - - Returns: - - """ - raise NotImplementedError() - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - """ - Args: - input_shape - - Returns: - - """ - # generic case: nothing to do before call - pass - - def freeze_weights(self) -> None: - pass - - def unfreeze_weights(self) -> None: - pass - - def freeze_alpha(self) -> None: - pass - - def unfreeze_alpha(self) -> None: - pass - - def reset_finetuning(self) -> None: - pass - - -class BackwardAdd(BackwardMerge): - """Backward LiRPA of Add""" - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - self.op = DecomonAdd( - mode=self.mode, perturbation_domain=self.perturbation_domain, dc_decomp=self.dc_decomp - ).call - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: - w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) - - inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) - #  check number of inputs - if len(inputs_list) == 1: # nothing to merge - return [[w_u_out, b_u_out, w_l_out, b_l_out]] - elif len(inputs_list) == 2: - bounds_0, bounds_1 = backward_add( - inputs_list[0], - inputs_list[1], - w_u_out, - b_u_out, - w_l_out, - b_l_out, - perturbation_domain=self.perturbation_domain, - mode=self.mode, - dc_decomp=self.dc_decomp, - ) - return [bounds_0, bounds_1] - - else: - raise NotImplementedError("This layer is intended to merge only 2 layers.") - - -class BackwardAverage(BackwardMerge): - """Backward LiRPA of Average""" - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - self.op = DecomonAdd(mode=self.mode, perturbation_domain=self.perturbation_domain, dc_decomp=False).call - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: - w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) - - inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) - n_elem = len(inputs_list) - #  check number of inputs - if n_elem == 1: # nothing to merge - return [[w_u_out, b_u_out, w_l_out, b_l_out]] - else: - bounds: list[list[BackendTensor]] = [] - input_bounds: list[list[BackendTensor]] = [] - - for j in range(n_elem - 1, 0, -1): - inputs_1 = inputs_list[j] - if j == 1: - inputs_0 = inputs_list[0] - else: - inputs_0 = self.op(list(chain(*inputs_list[: (j - 1)]))) # merge (j-1) first inputs - bounds_0, bounds_1 = backward_add( - inputs_0, - inputs_1, - w_u_out, - b_u_out, - w_l_out, - b_l_out, - perturbation_domain=self.perturbation_domain, - mode=self.mode, - dc_decomp=self.dc_decomp, - ) - input_bounds.append(bounds_1) - bounds.append(bounds_0) - if j == 1: - input_bounds.append(bounds_0) - # update bounds to use for next iteration - w_u_out, b_u_out, w_l_out, b_l_out = bounds_0 - - input_bounds = input_bounds[::-1] - return [[1.0 / n_elem * elem_i for elem_i in elem] for elem in input_bounds] - - -class BackwardSubtract(BackwardMerge): - """Backward LiRPA of Subtract""" - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - if not isinstance(layer, DecomonSubtract): - raise KeyError() - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: - w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) - - inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) - n_elem = len(inputs_list) - - if n_elem != 2: - raise NotImplementedError("This layer is intended to merge only 2 layers.") - - return backward_subtract( - inputs_list[0], - inputs_list[1], - w_u_out, - b_u_out, - w_l_out, - b_l_out, - perturbation_domain=self.layer.perturbation_domain, - mode=self.mode, - dc_decomp=self.dc_decomp, - ) - - -class BackwardMaximum(BackwardMerge): - """Backward LiRPA of Maximum""" - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - if not isinstance(layer, DecomonMaximum): - raise KeyError() - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: - w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) - - inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) - n_elem = len(inputs_list) - - if n_elem != 2: - raise NotImplementedError("This layer is intended to merge only 2 layers.") - - return backward_maximum( - inputs_list[0], - inputs_list[1], - w_u_out, - b_u_out, - w_l_out, - b_l_out, - perturbation_domain=self.layer.perturbation_domain, - mode=self.mode, - dc_decomp=self.dc_decomp, - ) - - -class BackwardMinimum(BackwardMerge): - """Backward LiRPA of Minimum""" - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - if not isinstance(layer, DecomonMinimum): - raise KeyError() - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: - w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) - - inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) - n_elem = len(inputs_list) - - if n_elem != 2: - raise NotImplementedError("This layer is intended to merge only 2 layers.") - - return backward_minimum( - inputs_list[0], - inputs_list[1], - w_u_out, - b_u_out, - w_l_out, - b_l_out, - perturbation_domain=self.layer.perturbation_domain, - mode=self.mode, - dc_decomp=self.dc_decomp, - ) - - -class BackwardConcatenate(BackwardMerge): - """Backward LiRPA of Concatenate""" - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - if not isinstance(layer, DecomonConcatenate): - raise KeyError() - - self.axis = self.layer.axis - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: - w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) - - inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) - n_elem = len(inputs_list) - n_list = [self.inputs_outputs_spec.get_kerasinputshape(subinputs)[self.axis] for subinputs in inputs_list] - indices_split = np.cumsum(n_list[:-1]) - axis_w = self.axis - if axis_w != -1: - axis_w += 1 - w_u_out_list = K.split(w_u_out, indices_split, axis_w) - w_l_out_list = K.split(w_l_out, indices_split, axis_w) - b_u_out_list = K.split(b_u_out, indices_split, self.axis) - b_l_out_list = K.split(b_l_out, indices_split, self.axis) - - bounds = [[w_u_out_list[i], b_u_out_list[i], w_l_out_list[i], b_l_out_list[i]] for i in range(n_elem)] - - return bounds - - -class BackwardMultiply(BackwardMerge): - """Backward LiRPA of Multiply""" - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - if not isinstance(layer, DecomonMultiply): - raise KeyError() - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: - w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) - - inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) - n_elem = len(inputs_list) - - if n_elem != 2: - raise NotImplementedError("This layer is intended to merge only 2 layers.") - - return backward_multiply( - inputs_list[0], - inputs_list[1], - w_u_out, - b_u_out, - w_l_out, - b_l_out, - perturbation_domain=self.layer.perturbation_domain, - mode=self.mode, - dc_decomp=self.dc_decomp, - ) - - -class BackwardDot(BackwardMerge): - """Backward LiRPA of Dot""" - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - super().__init__( - layer=layer, - rec=rec, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - **kwargs, - ) - if not isinstance(layer, DecomonDot): - raise KeyError() - - self.axes = [i for i in self.layer.axes] - self.op = BackwardAdd(self.layer) - - raise NotImplementedError() - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[list[BackendTensor]]: - w_u_out, b_u_out, w_l_out, b_l_out = get_identity_lirpa(inputs) - - inputs_list = self.inputs_outputs_spec.split_inputsformode_to_merge(inputs) - n_elem = len(inputs_list) - - if n_elem != 2: - raise NotImplementedError("This layer is intended to merge only 2 layers.") - - # permute dimensions and reshape - inputs_0 = inputs_list[0] - inputs_1 = inputs_list[1] - - n_0 = len(inputs_0[0].shape) - 2 - n_1 = len(inputs_1[0].shape) - 2 - - input_0_permuted = permute_dimensions(inputs_0, self.axes[0], mode=self.mode, dc_decomp=self.dc_decomp) - input_1_permuted = permute_dimensions(inputs_1, self.axes[1], mode=self.mode, dc_decomp=self.dc_decomp) - - inputs_0_broadcasted = broadcast(input_0_permuted, n_1, -1, mode=self.mode, dc_decomp=self.dc_decomp) - inputs_1_broadcasted = broadcast(input_1_permuted, n_0, 2, mode=self.mode, dc_decomp=self.dc_decomp) - - inputs_multiplied = multiply( - inputs_0_broadcasted, - inputs_1_broadcasted, - dc_decomp=self.layer.dc_decomp, - perturbation_domain=self.perturbation_domain, - mode=self.mode, - ) - - inputs_add = [] - for elem in split(inputs_multiplied, axis=self.axes[0], mode=self.mode): - inputs_add += elem - - bounds = self.op.call(inputs_add) - n = len(inputs_add) - bounds = [[1.0 / n * elem for elem in bounds_i] for bounds_i in bounds] - - # concatenate - shape = [-1, 1] + list(inputs_multiplied[0].shape[1:]) + list(w_u_out.shape[3:]) - - bounds_reshaped = [[K.reshape(elem[0], shape), elem[1], K.reshape(elem[2], shape), elem[3]] for elem in bounds] - w_u = K.concatenate([elem[0] for elem in bounds_reshaped], 1) - b_u = sum([elem[1] for elem in bounds_reshaped], 1) - w_l = K.concatenate([elem[2] for elem in bounds_reshaped], 1) - b_l = sum([elem[3] for elem in bounds_reshaped]) - - bounds_m_0, bounds_m_1 = backward_multiply( - inputs_0_broadcasted, - inputs_1_broadcasted, - w_u, - b_u, - w_l, - b_l, - perturbation_domain=self.perturbation_domain, - mode=self.mode, - dc_decomp=self.dc_decomp, - ) - - shape_0 = [-1, 1] + list(input_0_permuted[0].shape[1:]) + list(w_u.shape[3:]) - shape_1 = [-1, 1] + list(input_1_permuted[0].shape[1:]) + list(w_u.shape[3:]) - - bounds_m_0 = [ - K.reshape(bounds_m_0[0], shape_0), - bounds_m_0[1], - K.reshape(bounds_m_0[2], shape_0), - bounds_m_0[3], - ] - bounds_m_1 = [ - K.reshape(bounds_m_1[0], shape_1), - bounds_m_1[1], - K.reshape(bounds_m_1[2], shape_1), - bounds_m_1[3], - ] - - axes = [i for i in self.axes] - if axes[0] == -1: - axes[0] = len(inputs_0[0].shape) - if axes[1] == -1: - axes[1] = len(inputs_1[0].shape) - - index_0 = np.arange(len(shape_0)) - index_0[2] = axes[0] + 1 - index_0[axes[0] + 1] = 2 - - index_1 = np.arange(len(shape_1)) - index_1[2] = axes[1] + 1 - index_1[axes[1] + 1] = 2 - - bounds_m_0 = [ - K.transpose(bounds_m_0[0], index_0), - bounds_m_0[1], - K.transpose(bounds_m_0[2], index_0), - bounds_m_0[3], - ] - bounds_m_1 = [ - K.transpose(bounds_m_1[0], index_1), - bounds_m_1[1], - K.transpose(bounds_m_1[2], index_1), - bounds_m_1[3], - ] - - return [bounds_m_0, bounds_m_1] diff --git a/src/decomon/backward_layers/backward_reshape.py b/src/decomon/backward_layers/backward_reshape.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/decomon/backward_layers/convert.py b/src/decomon/backward_layers/convert.py deleted file mode 100644 index f1f6d08b..00000000 --- a/src/decomon/backward_layers/convert.py +++ /dev/null @@ -1,45 +0,0 @@ -from typing import Any, Optional, Union - -from keras.layers import Layer - -import decomon.backward_layers.backward_layers -import decomon.backward_layers.backward_maxpooling -import decomon.backward_layers.backward_merge -from decomon.backward_layers.core import BackwardLayer -from decomon.core import BoxDomain, ForwardMode, PerturbationDomain, Slope - -_mapping_name2class: dict[str, Any] = vars(decomon.backward_layers.backward_layers) -_mapping_name2class.update(vars(decomon.backward_layers.backward_merge)) -_mapping_name2class.update(vars(decomon.backward_layers.backward_maxpooling)) - - -def to_backward( - layer: Layer, - slope: Union[str, Slope] = Slope.V_SLOPE, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - finetune: bool = False, - **kwargs: Any, -) -> BackwardLayer: - if perturbation_domain is None: - perturbation_domain = BoxDomain() - class_name = layer.__class__.__name__ - if class_name.startswith("Decomon"): - class_name = "".join(layer.__class__.__name__.split("Decomon")[1:]) - - backward_class_name = f"Backward{class_name}" - try: - class_ = _mapping_name2class[backward_class_name] - except KeyError: - raise NotImplementedError(f"The backward version of {class_name} is not yet implemented.") - backward_layer_name = f"{layer.name}_backward" - return class_( - layer, - slope=slope, - mode=mode, - perturbation_domain=perturbation_domain, - finetune=finetune, - dtype=layer.dtype, - name=backward_layer_name, - **kwargs, - ) diff --git a/src/decomon/backward_layers/core.py b/src/decomon/backward_layers/core.py deleted file mode 100644 index 99baf276..00000000 --- a/src/decomon/backward_layers/core.py +++ /dev/null @@ -1,144 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, Optional, Union - -import keras -import numpy as np -from keras.layers import InputLayer, Layer, Wrapper - -from decomon.backward_layers.utils import get_identity_lirpa_shapes -from decomon.core import ( - BoxDomain, - ForwardMode, - InputsOutputsSpec, - PerturbationDomain, - get_affine, - get_ibp, -) -from decomon.layers.core import DecomonLayer -from decomon.types import BackendTensor - - -class BackwardLayer(ABC, Wrapper): - layer: Layer - _trainable_weights: list[keras.Variable] - - def __init__( - self, - layer: Layer, - rec: int = 1, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - **kwargs: Any, - ): - kwargs.pop("slope", None) - kwargs.pop("finetune", None) - super().__init__(layer, **kwargs) - self.rec = rec - if isinstance(self.layer, DecomonLayer): - self.mode = self.layer.mode - self.perturbation_domain = self.layer.perturbation_domain - self.dc_decomp = self.layer.dc_decomp - else: - self.mode = ForwardMode(mode) - if perturbation_domain is None: - self.perturbation_domain = BoxDomain() - else: - self.perturbation_domain = perturbation_domain - self.dc_decomp = dc_decomp - self.inputs_outputs_spec = InputsOutputsSpec( - dc_decomp=self.dc_decomp, mode=self.mode, perturbation_domain=self.perturbation_domain - ) - - @property - def ibp(self) -> bool: - return get_ibp(self.mode) - - @property - def affine(self) -> bool: - return get_affine(self.mode) - - def get_config(self) -> dict[str, Any]: - config = super().get_config() - config.update( - { - "rec": self.rec, - "mode": self.mode, - "perturbation_domain": self.perturbation_domain, - "dc_decomp": self.dc_decomp, - } - ) - return config - - @abstractmethod - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - """ - Args: - inputs - - Returns: - - """ - pass - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - """ - Args: - input_shape - - Returns: - - """ - # generic case: nothing to do before call - pass - - def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: - """Compute expected output shape according to input shape - - Will be called by symbolic calls on Keras Tensors. - - Args: - input_shape - - Returns: - - """ - if isinstance(self.layer, DecomonLayer): - decomon_input_shape = [inp.shape for inp in self.layer.input] - decomon_output_shape = [out.shape for out in self.layer.output] - keras_output_shape = self.inputs_outputs_spec.get_kerasinputshape_from_inputshapesformode( - decomon_output_shape - ) - keras_input_shape = self.inputs_outputs_spec.get_kerasinputshape_from_inputshapesformode( - decomon_input_shape - ) - else: # Keras layer - keras_output_shape = self.layer.output.shape - if isinstance(self.layer, InputLayer): - keras_input_shape = keras_output_shape - else: - keras_input_shape = self.layer.input.shape - - batch_size = keras_input_shape[0] - flattened_keras_output_shape = int(np.prod(keras_output_shape[1:])) # type: ignore - flattened_keras_input_shape = int(np.prod(keras_input_shape[1:])) # type: ignore - - b_shape = batch_size, flattened_keras_output_shape - w_shape = batch_size, flattened_keras_input_shape, flattened_keras_output_shape - - return [w_shape, b_shape, w_shape, b_shape] - - def freeze_weights(self) -> None: - pass - - def unfreeze_weights(self) -> None: - pass - - def freeze_alpha(self) -> None: - pass - - def unfreeze_alpha(self) -> None: - pass - - def reset_finetuning(self) -> None: - pass diff --git a/src/decomon/backward_layers/utils.py b/src/decomon/backward_layers/utils.py deleted file mode 100644 index 1b6094e5..00000000 --- a/src/decomon/backward_layers/utils.py +++ /dev/null @@ -1,669 +0,0 @@ -from typing import Any, Optional, Union - -import keras.ops as K -import numpy as np -from keras.config import floatx -from keras.layers import Flatten - -from decomon.core import ( - BoxDomain, - ForwardMode, - InputsOutputsSpec, - PerturbationDomain, - get_affine, - get_ibp, - get_lower_box, - get_upper_box, -) -from decomon.keras_utils import BatchedIdentityLike -from decomon.layers.utils import sort -from decomon.types import Tensor -from decomon.utils import maximum, minus, relu_, subtract - - -def backward_add( - inputs_0: list[Tensor], - inputs_1: list[Tensor], - w_u_out: Tensor, - b_u_out: Tensor, - w_l_out: Tensor, - b_l_out: Tensor, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - dc_decomp: bool = False, -) -> list[list[Tensor]]: - """Backward LiRPA of inputs_0+inputs_1 - - Args: - inputs_0 - inputs_1 - w_u_out - b_u_out - w_l_out - b_l_out - perturbation_domain - mode - - Returns: - - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - op_flat = Flatten(dtype=floatx()) # pas terrible a revoir - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - x_0, u_c_0, w_u_0, b_u_0, l_c_0, w_l_0, b_l_0, h_0, g_0 = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs_0 - ) - x_1, u_c_1, w_u_1, b_u_1, l_c_1, w_l_1, b_l_1, h_1, g_1 = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs_1 - ) - - u_c_0 = op_flat(u_c_0) - u_c_1 = op_flat(u_c_1) - l_c_0 = op_flat(l_c_0) - l_c_1 = op_flat(l_c_1) - - upper_0 = get_upper_box(l_c_0, u_c_0, w_u_out, b_u_out) - upper_1 = get_upper_box(l_c_1, u_c_1, w_u_out, b_u_out) - lower_0 = get_lower_box(l_c_0, u_c_0, w_l_out, b_l_out) - lower_1 = get_lower_box(l_c_1, u_c_1, w_l_out, b_l_out) - - w_u_out_0 = w_u_out - b_u_out_0 = upper_1 - w_l_out_0 = w_l_out - b_l_out_0 = lower_1 - - w_u_out_1 = w_u_out - b_u_out_1 = upper_0 - w_l_out_1 = w_l_out - b_l_out_1 = lower_0 - - return [[w_u_out_0, b_u_out_0, w_l_out_0, b_l_out_0], [w_u_out_1, b_u_out_1, w_l_out_1, b_l_out_1]] - - -def backward_linear_prod( - x_0: Tensor, - bounds_x: list[Tensor], - back_bounds: list[Tensor], - perturbation_domain: Optional[PerturbationDomain] = None, -) -> list[Tensor]: - """Backward LiRPA of a subroutine prod - - Args: - bounds_x - back_bounds - - Returns: - - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - z_value = K.cast(0.0, x_0.dtype) - o_value = K.cast(1.0, x_0.dtype) - - w_u_i, b_u_i, w_l_i, b_l_i = bounds_x - w_u, b_u, w_l, b_l = back_bounds - - if len(w_u_i.shape) > 3: - n_dim = w_u_i.shape[1] - w_u_i = K.reshape(w_u_i, (-1, n_dim, n_dim)) - w_l_i = K.reshape(w_l_i, (-1, n_dim, n_dim)) - b_u_i = K.reshape(b_u_i, (-1, n_dim)) - b_l_i = K.reshape(b_l_i, (-1, n_dim)) - - x_max = perturbation_domain.get_upper(x_0, w_u_i - w_l_i, b_u_i - b_l_i) - mask_b = o_value - K.sign(x_max) - mask_a = o_value - mask_b - - w_u_i_expanded = K.expand_dims(K.expand_dims(w_u_i, 1), -1) - w_l_i_expanded = K.expand_dims(K.expand_dims(w_l_i, 1), -1) - b_u_i_expanded = K.expand_dims(K.expand_dims(b_u_i, 1), -1) - b_l_i_expanded = K.expand_dims(K.expand_dims(b_l_i, 1), -1) - mask_a = K.expand_dims(K.expand_dims(mask_a, 1), -1) - mask_b = K.expand_dims(K.expand_dims(mask_b, 1), -1) - - w_u_pos = K.maximum(w_u, z_value) - w_u_neg = K.minimum(w_u, z_value) - w_l_pos = K.maximum(w_l, z_value) - w_l_neg = K.minimum(w_l, z_value) - - w_u_pos_expanded = K.expand_dims(w_u_pos, 2) - w_u_neg_expanded = K.expand_dims(w_u_neg, 2) - w_l_pos_expanded = K.expand_dims(w_l_pos, 2) - w_l_neg_expanded = K.expand_dims(w_l_neg, 2) - mask_a_expanded = K.expand_dims(mask_a, 2) - mask_b_expanded = K.expand_dims(mask_b, 2) - - w_u_out = K.sum( - mask_a_expanded * (w_u_pos_expanded * w_u_i_expanded + w_u_neg_expanded * w_l_i_expanded), 3 - ) + K.sum(K.expand_dims(w_u, 2) * mask_b_expanded * w_u_i_expanded, 3) - w_l_out = K.sum( - mask_a_expanded * (w_l_pos_expanded * w_l_i_expanded + w_l_neg_expanded * w_u_i_expanded), 3 - ) + K.sum(K.expand_dims(w_l, 2) * mask_b_expanded * w_l_i_expanded, 3) - - b_u_out = ( - K.sum(mask_a * (w_u_pos * b_u_i_expanded + w_u_neg * b_l_i_expanded), 2) - + K.sum(mask_b * (w_u * b_u_i_expanded), 2) - + b_u - ) - b_l_out = ( - K.sum(mask_a * (w_l_pos * b_l_i_expanded + w_l_neg * b_u_i_expanded), 2) - + K.sum(mask_b * (w_l * b_l_i_expanded), 2) - + b_l - ) - - return [w_u_out, b_u_out, w_l_out, b_l_out] - - -def backward_maximum( - inputs_0: list[Tensor], - inputs_1: list[Tensor], - w_u_out: Tensor, - b_u_out: Tensor, - w_l_out: Tensor, - b_l_out: Tensor, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - dc_decomp: bool = False, - **kwargs: Any, -) -> list[list[Tensor]]: - """Backward LiRPA of maximum(inputs_0, inputs_1) - - Args: - inputs_0 - inputs_1 - w_u_out - b_u_out - w_l_out - b_l_out - perturbation_domain - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - input_step_a_0 = subtract( - inputs_0, inputs_1, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode - ) - input_step_0 = relu_( - input_step_a_0, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode, **kwargs - ) - _, bounds_1 = backward_add( - input_step_0, - inputs_1, - w_u_out, - b_u_out, - w_l_out, - b_l_out, - perturbation_domain=perturbation_domain, - mode=mode, - dc_decomp=dc_decomp, - ) - - input_step_a_1 = subtract( - inputs_1, inputs_0, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode - ) - input_step_1 = relu_( - input_step_a_1, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode, **kwargs - ) - _, bounds_0 = backward_add( - input_step_1, - inputs_0, - w_u_out, - b_u_out, - w_l_out, - b_l_out, - perturbation_domain=perturbation_domain, - mode=mode, - dc_decomp=dc_decomp, - ) - - return [bounds_0, bounds_1] - - -# convex hull of the maximum between two functions -def backward_max_( - inputs: list[Tensor], - w_u_out: Tensor, - b_u_out: Tensor, - w_l_out: Tensor, - b_l_out: Tensor, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - axis: int = -1, - dc_decomp: bool = False, - **kwargs: Any, -) -> list[Tensor]: - """Backward LiRPA of max - - Args: - inputs: list of tensors - dc_decomp: boolean that indicates - grad_bounds: boolean that indicates whether - perturbation_domain: the type of perturbation domain - axis: axis to perform the maximum - whether we return a difference of convex decomposition of our layer - we propagate upper and lower bounds on the values of the gradient - - Returns: - max operation along an axis - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, tight=False, compute_ibp_from_affine=False - ) - dtype = x.dtype - input_shape = inputs_outputs_spec.get_kerasinputshape(inputs) - max_dim = input_shape[axis] - if max_dim is None: - raise ValueError(f"Dimension {axis} corresponding to `axis` cannot be None") - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - empty_tensor_list = [empty_tensor] * max_dim - z_value = K.cast(0.0, dtype=dtype) - - # do some transpose so that the last axis is also at the end - if ibp: - u_c_list = K.split(u_c, max_dim, axis) - l_c_list = K.split(l_c, max_dim, axis) - u_c_tmp = u_c_list[0] + z_value * (u_c_list[0]) - l_c_tmp = l_c_list[0] + z_value * (l_c_list[0]) - else: - u_c_list, l_c_list = empty_tensor_list, empty_tensor_list - u_c_tmp, l_c_tmp = empty_tensor, empty_tensor - - if affine: - b_u_list = K.split(b_u, max_dim, axis) - b_l_list = K.split(b_l, max_dim, axis) - b_u_tmp = b_u_list[0] + z_value * (b_u_list[0]) - b_l_tmp = b_l_list[0] + z_value * (b_l_list[0]) - - if axis == -1: - w_u_list = K.split(w_u, max_dim, axis) - w_l_list = K.split(w_l, max_dim, axis) - else: - w_u_list = K.split(w_u, max_dim, axis + 1) - w_l_list = K.split(w_l, max_dim, axis + 1) - w_u_tmp = w_u_list[0] + z_value * (w_u_list[0]) - w_l_tmp = w_l_list[0] + z_value * (w_l_list[0]) - else: - b_u_list, b_l_list, w_u_list, w_l_list = ( - empty_tensor_list, - empty_tensor_list, - empty_tensor_list, - empty_tensor_list, - ) - b_u_tmp, b_l_tmp, w_u_tmp, w_l_tmp = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - if dc_decomp: - raise NotImplementedError() - else: - h_list, g_list = empty_tensor_list, empty_tensor_list - h_tmp, g_tmp = empty_tensor, empty_tensor - - outputs = [] - output_tmp = inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_tmp, w_u_tmp, b_u_tmp, l_c_tmp, w_l_tmp, b_l_tmp, h_tmp, g_tmp] - ) - for i in range(1, max_dim): - output_i = inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_list[i], w_u_list[i], b_u_list[i], l_c_list[i], w_l_list[i], b_l_list[i], h_list[i], g_list[i]] - ) - outputs.append([[elem for elem in output_tmp], output_i]) - output_tmp = maximum( - output_tmp, output_i, dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain - ) - - outputs = outputs[::-1] - bounds = [] - if len(outputs) > 0: - for input_0, input_1 in outputs: - bounds_0, bounds_1 = backward_maximum( - input_0, input_1, w_u_out, b_u_out, w_l_out, b_l_out, mode=mode, dc_decomp=dc_decomp, **kwargs - ) - bounds.append(bounds_1) - w_u_out, b_u_out, w_l_out, b_l_out = bounds_0 - bounds.append(bounds_0) - bounds = bounds[::-1] - - if axis < 0: - w_u_out = K.concatenate([b[0] for b in bounds], axis - 1) - w_l_out = K.concatenate([b[2] for b in bounds], axis - 1) - b_u_out = K.sum(K.concatenate([K.expand_dims(b[1], axis - 1) for b in bounds], axis - 1), axis - 1) - b_l_out = K.sum(K.concatenate([K.expand_dims(b[3], axis - 1) for b in bounds], axis - 1), axis - 1) - else: - w_u_out = K.concatenate([b[0] for b in bounds], axis) - w_l_out = K.concatenate([b[2] for b in bounds], axis) - b_u_out = K.sum(K.concatenate([K.expand_dims(b[1], axis) for b in bounds], axis), axis) - b_l_out = K.sum(K.concatenate([K.expand_dims(b[3], axis) for b in bounds], axis), axis) - - return [w_u_out, b_u_out, w_l_out, b_l_out] - - -def backward_minimum( - inputs_0: list[Tensor], - inputs_1: list[Tensor], - w_u_out: Tensor, - b_u_out: Tensor, - w_l_out: Tensor, - b_l_out: Tensor, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - dc_decomp: bool = False, - **kwargs: Any, -) -> list[list[Tensor]]: - """Backward LiRPA of minimum(inputs_0, inputs_1) - - Args: - inputs_0 - inputs_1 - w_u_out - b_u_out - w_l_out - b_l_out - perturbation_domain - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - w_u_out, b_u_out, w_l_out, b_l_out = backward_minus(w_u_out, b_u_out, w_l_out, b_l_out) - bounds_0, bounds_1 = backward_maximum( - inputs_0, - inputs_1, - w_u_out, - b_u_out, - w_l_out, - b_l_out, - perturbation_domain=perturbation_domain, - mode=mode, - dc_decomp=dc_decomp, - **kwargs, - ) - - w_u_out_0, b_u_out_0, w_l_out_0, b_l_out_0 = bounds_0 - w_u_out_1, b_u_out_1, w_l_out_1, b_l_out_1 = bounds_1 - - bounds_0 = backward_minus(w_u_out_0, b_u_out_0, w_l_out_0, b_l_out_0) - bounds_1 = backward_minus(w_u_out_1, b_u_out_1, w_l_out_1, b_l_out_1) - - return [bounds_0, bounds_1] - - -def backward_minus( - w_u_out: Tensor, - b_u_out: Tensor, - w_l_out: Tensor, - b_l_out: Tensor, -) -> list[Tensor]: - """Backward LiRPA of -x - - Args: - w_u_out - b_u_out - w_l_out - b_l_out - perturbation_domain - mode - - Returns: - - """ - return [-w_l_out, -b_l_out, -w_u_out, -b_u_out] - - -def backward_scale( - scale_factor: float, - w_u_out: Tensor, - b_u_out: Tensor, - w_l_out: Tensor, - b_l_out: Tensor, -) -> list[Tensor]: - """Backward LiRPA of scale_factor*x - - Args: - scale_factor - w_u_out - b_u_out - w_l_out - b_l_out - - Returns: - - """ - - if scale_factor >= 0: - output = [scale_factor * w_u_out, b_u_out, scale_factor * w_l_out, b_l_out] - else: - output = [scale_factor * w_l_out, b_l_out, scale_factor * w_u_out, b_u_out] - - return output - - -def backward_subtract( - inputs_0: list[Tensor], - inputs_1: list[Tensor], - w_u_out: Tensor, - b_u_out: Tensor, - w_l_out: Tensor, - b_l_out: Tensor, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - dc_decomp: bool = False, -) -> list[list[Tensor]]: - """Backward LiRPA of inputs_0 - inputs_1 - - Args: - inputs_0 - inputs_1 - w_u_out - b_u_out - w_l_out - b_l_out - perturbation_domain - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - inputs_1 = minus(inputs_1, mode=mode, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain) - bounds_0, bounds_1 = backward_add( - inputs_0, - inputs_1, - w_u_out, - b_u_out, - w_l_out, - b_l_out, - perturbation_domain=perturbation_domain, - mode=mode, - dc_decomp=dc_decomp, - ) - - bounds_1 = [-bounds_1[0], bounds_1[1], -bounds_1[2], bounds_1[3]] - return [bounds_0, bounds_1] - - -def backward_multiply( - inputs_0: list[Tensor], - inputs_1: list[Tensor], - w_u_out: Tensor, - b_u_out: Tensor, - w_l_out: Tensor, - b_l_out: Tensor, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - dc_decomp: bool = False, -) -> list[list[Tensor]]: - """Backward LiRPA of element-wise multiply inputs_0*inputs_1 - - Args: - inputs_0 - inputs_1 - w_u_out - b_u_out - w_l_out - b_l_out - perturbation_domain - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - x_0, u_c_0, w_u_0, b_u_0, l_c_0, w_l_0, b_l_0, h_0, g_0 = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs_0 - ) - x_1, u_c_1, w_u_1, b_u_1, l_c_1, w_l_1, b_l_1, h_1, g_1 = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs_1 - ) - - z_value = K.cast(0.0, u_c_0.dtype) - - n = int(np.prod(u_c_0.shape[1:])) - n_shape = [-1, n] - # broadcast dimensions if needed - n_out = len(w_u_out.shape[1:]) - for _ in range(n_out): - n_shape += [1] - a_u_0 = K.reshape(u_c_1, n_shape) - a_u_1 = K.reshape(u_c_0, n_shape) - - b_u_0 = K.reshape((K.maximum(l_c_0, z_value) * u_c_1 + K.minimum(l_c_0, z_value) * l_c_1 - u_c_1 * l_c_0), n_shape) - b_u_1 = K.reshape((K.maximum(l_c_1, z_value) * u_c_0 + K.minimum(l_c_1, z_value) * l_c_0 - u_c_0 * l_c_1), n_shape) - - a_l_0 = K.reshape(l_c_1, n_shape) - a_l_1 = K.reshape(l_c_0, n_shape) - b_l_0 = K.reshape((K.maximum(l_c_0, z_value) * l_c_1 + K.minimum(l_c_0, z_value) * u_c_1 - l_c_1 * l_c_0), n_shape) - b_l_1 = K.reshape((K.maximum(l_c_1, z_value) * l_c_0 + K.minimum(l_c_1, z_value) * u_c_0 - l_c_0 * l_c_1), n_shape) - - # upper - w_u_out_max = K.maximum(w_u_out, z_value) - w_u_out_min = K.minimum(w_u_out, z_value) - - w_u_out_0 = w_u_out_max * a_u_0 + w_u_out_min * a_l_0 - w_u_out_1 = w_u_out_max * a_u_1 + w_u_out_min * a_l_1 - - b_u_out_0 = K.sum(w_u_out_max * b_u_0, 1) + K.sum(w_u_out_min * b_l_0, 1) + b_u_out - b_u_out_1 = K.sum(w_u_out_max * b_u_1, 1) + K.sum(w_u_out_min * b_l_1, 1) + b_u_out - - # lower - w_l_out_max = K.maximum(w_l_out, z_value) - w_l_out_min = K.minimum(w_l_out, z_value) - - w_l_out_0 = w_l_out_max * a_l_0 + w_l_out_min * a_u_0 - w_l_out_1 = w_l_out_max * a_l_1 + w_l_out_min * a_u_1 - - b_l_out_0 = K.sum(w_l_out_max * b_l_0, 1) + K.sum(w_l_out_min * b_u_0, 1) + b_l_out - b_l_out_1 = K.sum(w_l_out_max * b_l_1, 1) + K.sum(w_l_out_min * b_u_1, 1) + b_l_out - - bounds_0 = [w_u_out_0, b_u_out_0, w_l_out_0, b_l_out_0] - bounds_1 = [w_u_out_1, b_u_out_1, w_l_out_1, b_l_out_1] - - return [bounds_0, bounds_1] - - -def backward_sort( - inputs: list[Tensor], - w_u_out: Tensor, - b_u_out: Tensor, - w_l_out: Tensor, - b_l_out: Tensor, - axis: int = -1, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - dc_decomp: bool = False, -) -> list[Tensor]: - """Backward LiRPA of sort - - Args: - inputs - w_u_out - b_u_out - w_l_out - b_l_out - axis - perturbation_domain - mode - - Returns: - - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - affine = get_affine(mode) - z_value = K.cast(0.0, w_u_out.dtype) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - - # build fake inputs with no linearity - n_dim = int(np.prod(u_c.shape[1:])) - w_tmp = z_value * K.concatenate([u_c[:, None] * n_dim], 1) - - inputs_tmp = [x, u_c, w_tmp, u_c, l_c, w_tmp, l_c] - outputs_tmp = sort( - inputs_tmp, axis=axis, perturbation_domain=perturbation_domain, mode=ForwardMode.HYBRID, dc_decomp=False - ) - _, _, w_u_tmp, b_u_tmp, _, w_l_tmp, b_l_tmp = outputs_tmp - - # w_u_tmp (None, n_dim, y.shape[1:) - # w_u_out (None, 1, n_dim, n_out) - w_u_tmp = K.reshape(w_u_tmp, [-1, n_dim, n_dim]) # (None, n_dim, n_dim) - b_u_tmp = K.reshape(b_u_tmp, [-1, n_dim]) # (None, n_dim) - w_l_tmp = K.reshape(w_l_tmp, [-1, n_dim, n_dim]) - b_l_tmp = K.reshape(b_l_tmp, [-1, n_dim]) - - # combine with backward bounds - w_u_out_pos = K.maximum(w_u_out, z_value) # (None, 1, n_dim, n_out) - w_u_out_neg = K.minimum(w_u_out, z_value) - w_l_out_pos = K.maximum(w_l_out, z_value) - w_l_out_neg = K.minimum(w_l_out, z_value) - - w_u_out = K.sum(w_u_out_pos * K.expand_dims(w_u_tmp, -1) + w_u_out_pos * K.expand_dims(w_l_tmp, -1), 1) - w_l_out = K.sum(w_u_out_pos * K.expand_dims(w_u_tmp, -1) + w_u_out_pos * K.expand_dims(w_l_tmp, -1), 1) - b_u_out = b_u_out + K.sum(w_u_out_pos * b_u_tmp, 1) + K.sum(w_u_out_neg * b_l_tmp, 1) - b_l_out = b_l_out + K.sum(w_l_out_pos * b_l_tmp, 1) + K.sum(w_l_out_neg * b_u_tmp, 1) - - return [w_u_out, b_u_out, w_l_out, b_l_out] - - -def get_identity_lirpa(inputs: list[Tensor]) -> list[Tensor]: - y = inputs[-1] - shape = int(np.prod(y.shape[1:])) - - y_flat = K.reshape(y, [-1, shape]) - - w_u_out, w_l_out = [BatchedIdentityLike()(y_flat)] * 2 - b_u_out, b_l_out = [K.zeros_like(y_flat)] * 2 - - return [w_u_out, b_u_out, w_l_out, b_l_out] - - -def get_identity_lirpa_shapes(input_shapes: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: - y_shape = input_shapes[-1] - batch_size = y_shape[0] - flatten_dim = int(np.prod(y_shape[1:])) # type: ignore - - b_shape = batch_size, flatten_dim - w_shape = batch_size, flatten_dim, flatten_dim - - return [w_shape, b_shape, w_shape, b_shape] diff --git a/src/decomon/backward_layers/utils_conv.py b/src/decomon/backward_layers/utils_conv.py deleted file mode 100644 index ae0faf87..00000000 --- a/src/decomon/backward_layers/utils_conv.py +++ /dev/null @@ -1,159 +0,0 @@ -import warnings - -import keras -import keras.ops as K -import numpy as np -from keras.layers import Conv2D, Input -from keras.ops.image import extract_patches - -from decomon.types import BackendTensor - - -def get_toeplitz(conv_layer: Conv2D, flatten: bool = True) -> BackendTensor: - """Express formally the affine component of the convolution - Conv is a linear operator but its affine component is implicit - we use im2col and extract_patches to express the affine matrix - Note that this matrix is Toeplitz - - Args: - conv_layer: Keras Conv2D layer or Decomon Conv2D layer - flatten (optional): convert the affine component as a 2D matrix (n_in, n_out). Defaults to True. - - Returns: - the affine operator W: conv(x)= Wx + bias - """ - if conv_layer.dtype == "float16": - warnings.warn("Loss of precision for float16 !") - if conv_layer.data_format == "channels_last": - return get_toeplitz_channels_last(conv_layer, flatten) - else: - return get_toeplitz_channels_first(conv_layer, flatten) - - -def get_toeplitz_channels_last(conv_layer: Conv2D, flatten: bool = True) -> BackendTensor: - """Express formally the affine component of the convolution for data_format=channels_last - Conv is a linear operator but its affine component is implicit - we use im2col and extract_patches to express the affine matrix - Note that this matrix is Toeplitz - - Args: - conv_layer: Keras Conv2D layer or Decomon Conv2D layer - flatten (optional): convert the affine component as a 2D matrix (n_in, n_out). Defaults to True. - - Returns: - the affine operator W: conv(x)= Wx + bias - """ - - input = conv_layer.input - if isinstance(input, keras.KerasTensor): - input_shape = input.shape - else: # list of inputs - input_shape = input[-1].shape - _, w_in, h_in, c_in = input_shape - output = conv_layer.output - if isinstance(output, keras.KerasTensor): - output_shape = output.shape - else: # list of outputs - output_shape = output[-1].shape - _, w_out, h_out, c_out = output_shape - - kernel_filter = conv_layer.kernel - filter_size = kernel_filter.shape[0] - - diag = K.reshape(K.identity(w_in * h_in * c_in), (w_in * h_in * c_in, w_in, h_in, c_in)) - - diag_patches = extract_patches( - diag, - size=[filter_size, filter_size], - strides=list(conv_layer.strides), - dilation_rate=list(conv_layer.dilation_rate), - padding=conv_layer.padding, - ) - - diag_patches_ = K.reshape(diag_patches, (w_in, h_in, c_in, w_out, h_out, filter_size**2, c_in)) - - shape = list(range(len(diag_patches_.shape))) - shape[-1] -= 1 - shape[-2] += 1 - diag_patches_ = K.transpose(diag_patches_, shape) - kernel = conv_layer.kernel # (filter_size, filter_size, c_in, c_out) - - kernel = K.transpose(K.reshape(kernel, (filter_size**2, c_in, c_out)), (1, 2, 0))[ - None, None, None, None, None - ] # (1,1,c_in, c_out, filter_size^2) - - # element-wise multiplication is only compatible with float32 - diag_patches_ = K.cast(diag_patches_, "float32") - kernel = K.cast(kernel, "float32") - - w = K.sum(K.expand_dims(diag_patches_, -2) * kernel, (5, 7)) - # kernel shape: (1,1,1,1,1,c_in, c_out, filter_size^2) - # diag_patches shape: (w_in, h_in, c_in, w_out, h_out, c_in, 1, filter_size^2) - # (w_in, h_in, c_in, w_out, h_out, c_in, c_out, filter^2) - - w = K.cast(w, conv_layer.dtype) - # cast w for dtype - - if flatten: - return K.reshape(w, (w_in * h_in * c_in, w_out * h_out * c_out)) - else: - return w - - -def get_toeplitz_channels_first(conv_layer: Conv2D, flatten: bool = True) -> BackendTensor: - """Express formally the affine component of the convolution for data_format=channels_first - Conv is a linear operator but its affine component is implicit - we use im2col and extract_patches to express the affine matrix - Note that this matrix is Toeplitz - - Args: - conv_layer: Keras Conv2D layer or Decomon Conv2D layer - flatten (optional): convert the affine component as a 2D matrix (n_in, n_out). Defaults to True. - - Returns: - the affine operator W: conv(x)= Wx + bias - """ - - input = conv_layer.input - if isinstance(input, keras.KerasTensor): - input_shape = input.shape - else: # list of inputs - input_shape = input[-1].shape - _, c_in, w_in, h_in = input_shape - output = conv_layer.output - if isinstance(output, keras.KerasTensor): - output_shape = output.shape - else: # list of outputs - output_shape = output[-1].shape - _, c_out, w_out, h_out = output_shape - kernel_filter = conv_layer.kernel - filter_size = kernel_filter.shape[0] - - diag = K.reshape(K.identity(w_in * h_in * c_in), (w_in * h_in * c_in, w_in, h_in, c_in)) - - diag_patches = extract_patches(diag, [filter_size, filter_size], [1, 1], [1, 1], padding=conv_layer.padding) - - diag_patches_ = K.reshape(diag_patches, (w_in, h_in, c_in, w_out, h_out, filter_size**2, c_in)) - - shape = list(range(len(diag_patches_.shape))) - shape[-1] -= 1 - shape[-2] += 1 - diag_patches_ = K.transpose(diag_patches_, shape) - kernel = conv_layer.kernel # (filter_size, filter_size, c_in, c_out) - - kernel = K.transpose(K.reshape(kernel, (filter_size**2, c_in, c_out)), (1, 2, 0))[ - None, None, None, None, None - ] # (1,1,c_in, c_out, filter_size^2) - - w = K.sum(K.expand_dims(diag_patches_, -2) * kernel, (5, 7)) - # kernel shape: (1, 1, 1, 1, 1, c_in, c_out, filter_size^2) - # diag_patches shape: (w_in, h_in, c_in, w_out, h_out, c_in, 1, filter_size^2) - # (w_in, h_in, c_in, w_out, h_out, c_in, c_out, filter^2) - - w = K.cast(w, conv_layer.dtype) - # cast w for dtype - - if flatten: - return K.reshape(w, (w_in * h_in * c_in, w_out * h_out * c_out)) - else: - return w From 494922fd17c9834052ca0cca844c075c689793c0 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 29 Feb 2024 11:29:19 +0100 Subject: [PATCH 063/101] Update clone/convert with new api Main changes: - Remove input_tensors and extra_inputs of convert inputs (-> not needed anymore as we treat now embedded model in a more optimal way) - Generate only perturbation domain input - Preprocess backward_bounds in clone() to get a list of list of tensors (one by keras model output). Some conventions are taken to manage case where the user wants to specify less backward_bounds: see preprocess_backward_bounds() doctring. - remove finetune, shared, dc_decomp - Add inputs for testing clone, random ones of different shapes and the previous "standard" ones - Add toy models to test with clone - Main entry point for end-user is now clone() rather than convert() (called by clone()) - Put ConvertMethod in decomon.core --- src/decomon/core.py | 26 ++ src/decomon/models/convert.py | 250 ++++++++--------- src/decomon/models/models.py | 91 +------ src/decomon/models/utils.py | 251 ++++++++---------- src/decomon/wrapper.py | 3 +- tests/conftest.py | 80 +++++- .../lirpa_comparison/test_comparison_lirpa.py | 2 +- tests/test_clone.py | 87 ++++++ 8 files changed, 427 insertions(+), 363 deletions(-) create mode 100644 tests/test_clone.py diff --git a/src/decomon/core.py b/src/decomon/core.py index e9a804c6..3fd24233 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -1305,3 +1305,29 @@ def get_upper_ball_finetune(x_0: Tensor, eps: float, p: float, w: Tensor, b: Ten return score_box + score_ball return get_upper_ball(x_0, eps, p, w, b) + + +class ConvertMethod(str, Enum): + CROWN = "crown" + """Crown fully recursive: backward propagation using crown oracle. + + (spawning subcrowns for each non-linear layer) + + """ + CROWN_FORWARD_IBP = "crown-forward-ibp" + """Crown + forward ibp: backward propagation using a forward-ibp oracle.""" + CROWN_FORWARD_AFFINE = "crown-forward-affine" + """Crown + forward ibp: backward propagation using a forward-affine oracle.""" + CROWN_FORWARD_HYBRID = "crown-forward-hybrid" + """Crown + forward ibp: backward propagation using a forward-hybrid oracle.""" + FORWARD_IBP = "forward-ibp" + """Forward propagation of constant bounds.""" + FORWARD_AFFINE = "forward-affine" + """Forward propagation of affine bounds.""" + FORWARD_HYBRID = "forward-hybrid" + """Forward propagation of constant+affine bounds. + + After each layer, the tightest constant bounds is keep between the ibp one + and the affine one combined with perturbation domain input. + + """ diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index 2fc66445..fd62bb75 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -2,31 +2,32 @@ from typing import Any, Optional, Union import keras -from keras.layers import InputLayer, Layer +from keras.layers import Layer from keras.models import Model -from decomon.backward_layers.core import BackwardLayer -from decomon.core import BoxDomain, PerturbationDomain, Slope, get_mode +from decomon.core import ( + BoxDomain, + ConvertMethod, + PerturbationDomain, + Propagation, + Slope, +) +from decomon.layers import DecomonLayer from decomon.layers.convert import to_decomon from decomon.models.backward_cloning import convert_backward from decomon.models.forward_cloning import ( - LayerMapDict, - OutputMapDict, convert_forward, convert_forward_functional_model, ) from decomon.models.models import DecomonModel from decomon.models.utils import ( - Convert2Mode, - ConvertMethod, - FeedDirection, - check_model2convert_inputs, ensure_functional_model, - get_direction, + generate_perturbation_domain_input, + get_final_ibp_affine_from_method, get_ibp_affine_from_method, - get_input_dim, - get_input_tensors, is_input_node, + method2propagation, + preprocess_backward_bounds, preprocess_layer, split_activation, ) @@ -42,21 +43,17 @@ def _clone_keras_model(model: Model, layer_fn: Callable[[Layer], list[Layer]]) - # initialize output_map and layer_map to avoid # - recreating input layers # - and converting input layers and have a cycle around them - output_map: OutputMapDict = {} - layer_map: LayerMapDict = {} + output_map: dict[int, list[keras.KerasTensor]] = {} for depth, nodes in model._nodes_by_depth.items(): for node in nodes: if is_input_node(node): output_map[id(node)] = node.output_tensors - layer_map[id(node)] = node.operation - _, output, _, _, _ = convert_forward_functional_model( + output = convert_forward_functional_model( model=model, input_tensors=model.inputs, - softmax_to_linear=False, layer_fn=layer_fn, output_map=output_map, - layer_map=layer_map, ) return Model( @@ -80,179 +77,164 @@ def preprocess_keras_model( # create status def convert( model: Model, - input_tensors: list[keras.KerasTensor], - method: Union[str, ConvertMethod] = ConvertMethod.CROWN, - ibp: bool = False, - affine: bool = False, - back_bounds: Optional[list[keras.KerasTensor]] = None, - layer_fn: Callable[..., Layer] = to_decomon, - slope: Union[str, Slope] = Slope.V_SLOPE, - input_dim: int = -1, - perturbation_domain: Optional[PerturbationDomain] = None, - finetune: bool = False, - forward_map: Optional[OutputMapDict] = None, - shared: bool = True, - softmax_to_linear: bool = True, - finetune_forward: bool = False, - finetune_backward: bool = False, + perturbation_domain_input: keras.KerasTensor, + perturbation_domain: PerturbationDomain, + method: ConvertMethod = ConvertMethod.CROWN, + backward_bounds: Optional[list[list[keras.KerasTensor]]] = None, + layer_fn: Callable[..., DecomonLayer] = to_decomon, + slope: Slope = Slope.V_SLOPE, + forward_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, + forward_layer_map: Optional[dict[int, DecomonLayer]] = None, final_ibp: bool = False, - final_affine: bool = False, + final_affine: bool = True, **kwargs: Any, -) -> tuple[ - list[keras.KerasTensor], - list[keras.KerasTensor], - Union[LayerMapDict, dict[int, BackwardLayer]], - Optional[OutputMapDict], -]: - if back_bounds is None: - back_bounds = [] +) -> list[keras.KerasTensor]: + """ + + Args: + model: keras model to convert + perturbation_domain_input: perturbation domain input + perturbation_domain: perturbation domain type on keras model input + method: method used to convert the model to a decomon model. See `ConvertMethod`. + backward_bounds: backward bounds to propagate, see `preprocess_backward_bounds()` for conventions + layer_fn: callable converting a layer and a model_output_shape into a decomon layer + slope: slope used by decomon activation layers + forward_output_map: forward outputs per node from a previously performed forward conversion. + To be used for forward oracle if not empty. + To be recomputed if empty and needed by the method. + forward_layer_map: forward decomon layer per node from a previously performed forward conversion. + To be used for forward oracle if not empty. + To be recomputed if empty and needed by the method. + final_ibp: specify if final outputs should include constant bounds. + final_affine: specify if final outputs should include affine bounds. + **kwargs: keyword arguments to pass to layer_fn + + Returns: + propagated bounds (concatenated), output of the future decomon model + + """ if perturbation_domain is None: perturbation_domain = BoxDomain() - if finetune: - finetune_forward = True - finetune_backward = True - if input_dim == -1: - input_dim = get_input_dim(model) - - if isinstance(method, str): - method = ConvertMethod(method.lower()) # prepare the Keras Model: split non-linear activation functions into separate Activation layers model = preprocess_keras_model(model) - layer_map: Union[LayerMapDict, dict[int, BackwardLayer]] + # loop over propagations needed + propagations = method2propagation(method) + ibp, affine = get_ibp_affine_from_method(method) + output: list[keras.KerasTensor] = [] - if method != ConvertMethod.CROWN: - input_tensors, output, layer_map, forward_map = convert_forward( + if Propagation.FORWARD in propagations: + output, forward_output_map, forward_layer_map = convert_forward( model=model, - input_tensors=input_tensors, + perturbation_domain_input=perturbation_domain_input, layer_fn=layer_fn, slope=slope, - input_dim=input_dim, - dc_decomp=False, perturbation_domain=perturbation_domain, ibp=ibp, affine=affine, - finetune=finetune_forward, - shared=shared, - softmax_to_linear=softmax_to_linear, - back_bounds=back_bounds, + **kwargs, ) - if get_direction(method) == FeedDirection.BACKWARD: - input_tensors, output, layer_map, forward_map = convert_backward( + if Propagation.BACKWARD in propagations: + output = convert_backward( model=model, - input_tensors=input_tensors, - back_bounds=back_bounds, - slope=slope, + perturbation_domain_input=perturbation_domain_input, perturbation_domain=perturbation_domain, - ibp=ibp, - affine=affine, - finetune=finetune_backward, - forward_map=forward_map, - final_ibp=final_ibp, - final_affine=final_affine, - input_dim=input_dim, + layer_fn=layer_fn, + backward_bounds=backward_bounds, + slope=slope, + forward_output_map=forward_output_map, + forward_layer_map=forward_layer_map, **kwargs, ) - else: - # check final_ibp and final_affine - mode_from = get_mode(ibp, affine) - mode_to = get_mode(final_ibp, final_affine) - output = Convert2Mode( - mode_from=mode_from, mode_to=mode_to, perturbation_domain=perturbation_domain, input_dim=input_dim - )(output) + + # Update output for final_ibp and final_affine + ... # build decomon model - return input_tensors, output, layer_map, forward_map + return output def clone( model: Model, - layer_fn: Callable[..., Layer] = to_decomon, slope: Union[str, Slope] = Slope.V_SLOPE, perturbation_domain: Optional[PerturbationDomain] = None, method: Union[str, ConvertMethod] = ConvertMethod.CROWN, - back_bounds: Optional[list[keras.KerasTensor]] = None, - finetune: bool = False, - shared: bool = True, - finetune_forward: bool = False, - finetune_backward: bool = False, - extra_inputs: Optional[list[keras.KerasTensor]] = None, - to_keras: bool = True, + backward_bounds: Optional[Union[keras.KerasTensor, list[keras.KerasTensor], list[list[keras.KerasTensor]]]] = None, final_ibp: Optional[bool] = None, final_affine: Optional[bool] = None, + layer_fn: Callable[..., DecomonLayer] = to_decomon, + forward_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, + forward_layer_map: Optional[dict[int, DecomonLayer]] = None, **kwargs: Any, ) -> DecomonModel: + """ + + Args: + model: keras model to convert + slope: slope used by decomon activation layers + perturbation_domain: perturbation domain type on keras model input + method: method used to convert the model to a decomon model. See `ConvertMethod`. + backward_bounds: backward bounds to propagate, see `preprocess_backward_bounds()` for conventions + final_ibp: specify if final outputs should include constant bounds. + Default to False except for forward-ibp and forward-hybrid. + final_affine: specify if final outputs should include affine bounds. + Default to True all methods except forward-ibp. + layer_fn: callable converting a layer and a model_output_shape into a decomon layer + forward_output_map: forward outputs per node from a previously performed forward conversion. + To be used for forward oracle if not empty. + To be recomputed if empty and needed by the method. + forward_layer_map: forward decomon layer per node from a previously performed forward conversion. + To be used for forward oracle if not empty. + To be recomputed if empty and needed by the method. + **kwargs: keyword arguments to pass to layer_fn + + Returns: + + """ + # Check hypotheses: functional model + 1 flattened input + model = ensure_functional_model(model) + if len(model.inputs) > 1: + raise ValueError("The model must have only 1 input to be converted.") + + # Args preprocessing if perturbation_domain is None: perturbation_domain = BoxDomain() - if back_bounds is None: - back_bounds = [] - if extra_inputs is None: - extra_inputs = [] - if not isinstance(model, Model): - raise ValueError("Expected `model` argument " "to be a `Model` instance, got ", model) - ibp, affine = get_ibp_affine_from_method(method) + default_final_ibp, default_final_affine = get_final_ibp_affine_from_method(method) if final_ibp is None: - final_ibp = ibp + final_ibp = default_final_ibp if final_affine is None: - final_affine = affine + final_affine = default_final_affine if isinstance(method, str): method = ConvertMethod(method.lower()) - if not to_keras: - raise NotImplementedError("Only convert to Keras for now.") + backward_bounds = preprocess_backward_bounds(backward_bounds=backward_bounds, nb_model_outputs=len(model.outputs)) - if finetune: - finetune_forward = True - finetune_backward = True + perturbation_domain_input = generate_perturbation_domain_input(model=model, perturbation_domain=perturbation_domain) - # Check hypotheses: functional model + 1 flattened input - model = ensure_functional_model(model) - check_model2convert_inputs(model) - - z_tensor, input_tensors = get_input_tensors( + output = convert( model=model, + perturbation_domain_input=perturbation_domain_input, perturbation_domain=perturbation_domain, - ibp=ibp, - affine=affine, - ) - - _, output, _, _ = convert( - model, + method=method, + backward_bounds=backward_bounds, layer_fn=layer_fn, slope=slope, - input_tensors=input_tensors, - back_bounds=back_bounds, - method=method, - ibp=ibp, - affine=affine, - input_dim=-1, - perturbation_domain=perturbation_domain, - finetune=finetune, - shared=shared, - softmax_to_linear=True, - layer_map={}, - forward_map={}, - finetune_forward=finetune_forward, - finetune_backward=finetune_backward, + forward_output_map=forward_output_map, + forward_layer_map=forward_layer_map, final_ibp=final_ibp, final_affine=final_affine, + **kwargs, ) - back_bounds_from_inputs = [elem for elem in back_bounds if isinstance(elem._keras_history.operation, InputLayer)] - return DecomonModel( - inputs=[z_tensor] + back_bounds_from_inputs + extra_inputs, + inputs=[perturbation_domain_input], outputs=output, perturbation_domain=perturbation_domain, - dc_decomp=False, method=method, ibp=final_ibp, affine=final_affine, - finetune=finetune, - shared=shared, - backward_bounds=(len(back_bounds) > 0), ) diff --git a/src/decomon/models/models.py b/src/decomon/models/models.py index 572ac072..a7c8ea18 100644 --- a/src/decomon/models/models.py +++ b/src/decomon/models/models.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, Union +from typing import Any, Union import keras import keras.ops as K @@ -6,14 +6,7 @@ from keras import Model from keras.utils import serialize_keras_object -from decomon.core import ( - BoxDomain, - GridDomain, - InputsOutputsSpec, - Option, - PerturbationDomain, -) -from decomon.models.utils import ConvertMethod +from decomon.core import ConvertMethod, PerturbationDomain class DecomonModel(keras.Model): @@ -21,27 +14,17 @@ def __init__( self, inputs: Union[keras.KerasTensor, list[keras.KerasTensor]], outputs: Union[keras.KerasTensor, list[keras.KerasTensor]], - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - method: Union[str, ConvertMethod] = ConvertMethod.FORWARD_AFFINE, - ibp: bool = True, - affine: bool = True, - finetune: bool = False, - shared: bool = True, - backward_bounds: bool = False, + perturbation_domain: PerturbationDomain, + method: ConvertMethod, + ibp: bool, + affine: bool, **kwargs: Any, ): super().__init__(inputs, outputs, **kwargs) - if perturbation_domain is None: - perturbation_domain = BoxDomain() self.perturbation_domain = perturbation_domain - self.dc_decomp = dc_decomp - self.method = ConvertMethod(method) + self.method = method self.ibp = ibp self.affine = affine - self.finetune = finetune - self.backward_bounds = backward_bounds - self.shared = shared def get_config(self) -> dict[str, Any]: # force having functional config which is skipped by default @@ -70,31 +53,6 @@ def set_domain(self, perturbation_domain: PerturbationDomain) -> None: if hasattr(layer, "perturbation_domain"): layer.perturbation_domain = self.perturbation_domain - def freeze_weights(self) -> None: - for layer in self.layers: - if hasattr(layer, "freeze_weights"): - layer.freeze_weights() - - def unfreeze_weights(self) -> None: - for layer in self.layers: - if hasattr(layer, "unfreeze_weights"): - layer.unfreeze_weights() - - def freeze_alpha(self) -> None: - for layer in self.layers: - if hasattr(layer, "freeze_alpha"): - layer.freeze_alpha() - - def unfreeze_alpha(self) -> None: - for layer in self.layers: - if hasattr(layer, "unfreeze_alpha"): - layer.unfreeze_alpha() - - def reset_finetuning(self) -> None: - for layer in self.layers: - if hasattr(layer, "reset_finetuning"): - layer.reset_finetuning() - def predict_on_single_batch_np( self, inputs: Union[np.ndarray, list[np.ndarray]] ) -> Union[np.ndarray, list[np.ndarray]]: @@ -126,38 +84,3 @@ def _check_domain( raise NotImplementedError("We can only change the parameters of the perturbation domain, not its type.") return perturbation_domain - - -def get_AB(model: DecomonModel) -> dict[str, list[keras.Variable]]: - dico_AB: dict[str, list[keras.Variable]] = {} - perturbation_domain = model.perturbation_domain - if not (isinstance(perturbation_domain, GridDomain) and perturbation_domain.opt_option == Option.milp): - return dico_AB - - for layer in model.layers: - name = layer.name - sub_names = name.split("backward_activation") - if len(sub_names) > 1: - key = f"{layer.layer.name}_{layer.rec}" - if key not in dico_AB: - dico_AB[key] = layer.grid_finetune - return dico_AB - - -def get_AB_finetune(model: DecomonModel) -> dict[str, keras.Variable]: - dico_AB: dict[str, keras.Variable] = {} - perturbation_domain = model.perturbation_domain - if not (isinstance(perturbation_domain, GridDomain) and perturbation_domain.opt_option == Option.milp): - return dico_AB - - if not model.finetune: - return dico_AB - - for layer in model.layers: - name = layer.name - sub_names = name.split("backward_activation") - if len(sub_names) > 1: - key = f"{layer.layer.name}_{layer.rec}" - if key not in dico_AB: - dico_AB[key] = layer.alpha_b_l - return dico_AB diff --git a/src/decomon/models/utils.py b/src/decomon/models/utils.py index 662428e7..91b0b743 100644 --- a/src/decomon/models/utils.py +++ b/src/decomon/models/utils.py @@ -1,29 +1,22 @@ -from enum import Enum from typing import Any, Optional, Union import keras import keras.ops as K import numpy as np from keras import Model, Sequential -from keras.layers import ( - Activation, - Concatenate, - Flatten, - Input, - Lambda, - Layer, - Maximum, - Minimum, -) +from keras import ops as K +from keras.layers import Activation, Input, Lambda, Layer from keras.src import Functional from keras.src.ops.node import Node from decomon.core import ( BallDomain, BoxDomain, + ConvertMethod, ForwardMode, InputsOutputsSpec, PerturbationDomain, + Propagation, get_mode, ) from decomon.keras_utils import ( @@ -34,143 +27,17 @@ from decomon.types import BackendTensor -class ConvertMethod(str, Enum): - CROWN = "crown" - CROWN_FORWARD_IBP = "crown-forward-ibp" - CROWN_FORWARD_AFFINE = "crown-forward-affine" - CROWN_FORWARD_HYBRID = "crown-forward-hybrid" - FORWARD_IBP = "forward-ibp" - FORWARD_AFFINE = "forward-affine" - FORWARD_HYBRID = "forward-hybrid" - - -def get_ibp_affine_from_method(method: Union[str, ConvertMethod]) -> tuple[bool, bool]: - method = ConvertMethod(method) - if method in [ConvertMethod.FORWARD_IBP, ConvertMethod.CROWN_FORWARD_IBP]: - return True, False - if method in [ConvertMethod.FORWARD_AFFINE, ConvertMethod.CROWN_FORWARD_AFFINE]: - return False, True - if method in [ConvertMethod.FORWARD_HYBRID, ConvertMethod.CROWN_FORWARD_HYBRID]: - return True, True - if method == ConvertMethod.CROWN: - return True, False - return True, True - - -class FeedDirection(str, Enum): - FORWARD = "feed_forward" - BACKWARD = "feed_backward" - - -def get_direction(method: Union[str, ConvertMethod]) -> FeedDirection: - if ConvertMethod(method) in [ConvertMethod.FORWARD_IBP, ConvertMethod.FORWARD_AFFINE, ConvertMethod.FORWARD_HYBRID]: - return FeedDirection.FORWARD - else: - return FeedDirection.BACKWARD - - -def has_merge_layers(model: Model) -> bool: - return any(is_a_merge_layer(layer) for layer in model.layers) - - -def check_model2convert_inputs(model: Model) -> None: - """Check that the model to convert satisfy the hypotheses for decomon on inputs. - - Which means: - - - only one input - - the input must be flattened: only batchsize + another dimension - - """ - if len(model.inputs) > 1: - raise ValueError("The model must have only 1 input to be converted.") - if len(model.inputs[0].shape) > 2: - raise ValueError("The model must have a flattened input to be converted.") - - def generate_perturbation_domain_input( model: Model, perturbation_domain: PerturbationDomain, ) -> keras.KerasTensor: - model_input_shape = model.input.shape[1:] - dtype = model.input.dtype + model_input_shape = model.inputs[0].shape[1:] + dtype = model.inputs[0].dtype input_shape_x = perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape) return Input(shape=input_shape_x, dtype=dtype) -def get_input_tensors( - model: Model, - perturbation_domain: PerturbationDomain, - ibp: bool = True, - affine: bool = True, -) -> tuple[keras.KerasTensor, list[keras.KerasTensor]]: - input_dim = get_input_dim(model) - mode = get_mode(ibp=ibp, affine=affine) - dc_decomp = False - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - empty_tensor = inputs_outputs_spec.get_empty_tensor() - - input_shape_x = perturbation_domain.get_x_input_shape_wo_batchsize((input_dim,)) - z_tensor = Input(shape=input_shape_x, dtype=model.layers[0].dtype) - u_c_tensor, l_c_tensor, W, b, h, g = ( - empty_tensor, - empty_tensor, - empty_tensor, - empty_tensor, - empty_tensor, - empty_tensor, - ) - - if isinstance(perturbation_domain, BoxDomain): - if ibp: - u_c_tensor = Lambda(lambda z: z[:, 1], dtype=z_tensor.dtype)(z_tensor) - l_c_tensor = Lambda(lambda z: z[:, 0], dtype=z_tensor.dtype)(z_tensor) - - if affine: - W = BatchedIdentityLike()(z_tensor[:, 0]) - b = K.zeros_like(z_tensor[:, 0]) - - elif isinstance(perturbation_domain, BallDomain): - if perturbation_domain.p == np.inf: - radius = perturbation_domain.eps - u_c_tensor = Lambda( - lambda var: var + K.cast(radius, dtype=model.layers[0].dtype), dtype=model.layers[0].dtype - )(z_tensor) - if ibp: - l_c_tensor = Lambda( - lambda var: var - K.cast(radius, dtype=model.layers[0].dtype), dtype=model.layers[0].dtype - )(z_tensor) - if affine: - W = BatchedIdentityLike()(u_c_tensor) - b = K.zeros_like(u_c_tensor) - - else: - W = BatchedIdentityLike()(z_tensor) - b = K.zeros_like(z_tensor) - if ibp: - u_c_tensor = perturbation_domain.get_upper(z_tensor, W, b) - l_c_tensor = perturbation_domain.get_lower(z_tensor, W, b) - - else: - raise NotImplementedError(f"Not implemented for perturbation domain type {type(perturbation_domain)}") - - input_tensors = inputs_outputs_spec.extract_inputsformode_from_fullinputs( - [z_tensor, u_c_tensor, W, b, l_c_tensor, W, b, h, g] - ) - - return z_tensor, input_tensors - - -def get_input_dim(layer: Layer) -> int: - if isinstance(layer.input, list): - if len(layer.input) == 0: - return 0 - return int(np.prod(layer.input[0].shape[1:])) - else: - return int(np.prod(layer.input.shape[1:])) - - def prepare_inputs_for_layer( inputs: Union[tuple[keras.KerasTensor, ...], list[keras.KerasTensor], keras.KerasTensor] ) -> Union[tuple[keras.KerasTensor, ...], list[keras.KerasTensor], keras.KerasTensor]: @@ -402,3 +269,109 @@ def ensure_functional_model(model: Model) -> Functional: return model else: raise NotImplementedError("Decomon model available only for functional or sequential models.") + + +def preprocess_backward_bounds( + backward_bounds: Optional[Union[keras.KerasTensor, list[keras.KerasTensor], list[list[keras.KerasTensor]]]], + nb_model_outputs: int, +) -> Optional[list[list[keras.KerasTensor]]]: + """Preprocess backward bounds to be used by `convert()`. + + Args: + backward_bounds: backward bounds to propagate + nb_model_outputs: number of outputs of the keras model to convert + + Returns: + formatted backward bounds + + Backward bounds can be given as + - None or empty list => backward bounds to propagate will be identity + - a single list (potentially partially filled) => same backward bounds on all model outputs (assuming that all outputs have same shape) + - a list of list : different backward bounds for each model output + + Which leads to the following formatting: + - None, [], or [[]] -> None + - single keras tensor w -> [[w, 0, w, 0]] * nb_model_outputs + - [w] -> idem + - [w, b] -> [[w, b, w, b]] * nb_model_outputs + - [w_l, b_l, w_u, b_u] -> [[w_l, b_l, w_u, b_u]] * nb_model_outputs + - [[w_l, b_l, w_u, b_u]] -> [[w_l, b_l, w_u, b_u]] * nb_model_outputs + - list of lists of tensors [w_l[i], b_l[i], w_u[i], b_u[i]]_i -> we enforce each sublist to have 4 elements + + """ + if backward_bounds is None: + # None + return None + if isinstance(backward_bounds, keras.KerasTensor): + # single tensor w + w = backward_bounds + b = K.zeros_like(w[:, 0]) + backward_bounds = [w, b, w, b] + if len(backward_bounds) == 0: + return None + else: + if isinstance(backward_bounds[0], keras.KerasTensor): + # list of tensors + if len(backward_bounds) == 1: + # single tensor w + return preprocess_backward_bounds(backward_bounds=backward_bounds[0], nb_model_outputs=nb_model_outputs) + elif len(backward_bounds) == 2: + # w, b + w, b = backward_bounds + return preprocess_backward_bounds(backward_bounds=[w, b, w, b], nb_model_outputs=nb_model_outputs) + elif len(backward_bounds) == 4: + return [backward_bounds] * nb_model_outputs + else: + raise ValueError( + "If backward_bounds is given as a list of tensors, it should have 1, 2, or 4 elements." + ) + else: + # list of list of tensors + if len(backward_bounds) == 1: + return [backward_bounds[0]] * nb_model_outputs + elif len(backward_bounds) != nb_model_outputs: + raise ValueError( + "If backward_bounds is given as a list of tensors, it should have nb_model_ouptputs elements." + ) + elif not all([len(backward_bounds_i) == 4 for backward_bounds_i in backward_bounds]): + raise ValueError( + "If backward_bounds is given as a list of tensors, each sublist should have 4 elements (w_l_, b_l, w_u, b_u)." + ) + else: + return backward_bounds + + +def get_ibp_affine_from_method(method: ConvertMethod) -> tuple[bool, bool]: + method = ConvertMethod(method) + if method in [ConvertMethod.FORWARD_IBP, ConvertMethod.CROWN_FORWARD_IBP]: + return True, False + if method in [ConvertMethod.FORWARD_AFFINE, ConvertMethod.CROWN_FORWARD_AFFINE]: + return False, True + if method in [ConvertMethod.FORWARD_HYBRID, ConvertMethod.CROWN_FORWARD_HYBRID]: + return True, True + if method == ConvertMethod.CROWN: + return False, True + return True, True + + +def get_final_ibp_affine_from_method(method: ConvertMethod) -> tuple[bool, bool]: + method = ConvertMethod(method) + if method in [ConvertMethod.FORWARD_IBP, ConvertMethod.FORWARD_HYBRID]: + final_ibp = True + else: + final_ibp = False + if method == ConvertMethod.FORWARD_IBP: + final_affine = False + else: + final_affine = True + + return final_ibp, final_affine + + +def method2propagation(method: ConvertMethod) -> list[Propagation]: + if method == ConvertMethod.CROWN: + return [Propagation.BACKWARD] + elif method in [ConvertMethod.FORWARD_IBP, ConvertMethod.FORWARD_AFFINE, ConvertMethod.FORWARD_HYBRID]: + return [Propagation.FORWARD] + else: + return [Propagation.FORWARD, Propagation.BACKWARD] diff --git a/src/decomon/wrapper.py b/src/decomon/wrapper.py index 2a5129b8..8d26c9d8 100644 --- a/src/decomon/wrapper.py +++ b/src/decomon/wrapper.py @@ -5,10 +5,9 @@ import numpy as np import numpy.typing as npt -from decomon.core import BallDomain, BoxDomain, GridDomain +from decomon.core import BallDomain, BoxDomain, ConvertMethod, GridDomain from decomon.models.convert import clone from decomon.models.models import DecomonModel -from decomon.models.utils import ConvertMethod IntegerType = Union[int, np.int_] """Alias for integers types.""" diff --git a/tests/conftest.py b/tests/conftest.py index 5d3e0922..839f9e87 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,7 +15,7 @@ unpack_fixture, ) -from decomon.core import BoxDomain, InputsOutputsSpec, Propagation, Slope +from decomon.core import BoxDomain, ConvertMethod, InputsOutputsSpec, Propagation, Slope from decomon.keras_utils import ( BACKEND_JAX, BACKEND_NUMPY, @@ -23,7 +23,6 @@ BACKEND_TENSORFLOW, batch_multid_dot, ) -from decomon.models.utils import ConvertMethod from decomon.types import BackendTensor, Tensor empty, diag, nobatch = param_fixtures( @@ -921,7 +920,7 @@ def assert_decomon_output_compare_with_keras_input_output_model( keras_output, ibp, affine, - propagation, + propagation=Propagation.FORWARD, decimal=5, ): keras_input_shape = tuple(keras_input.shape[1:]) @@ -1656,6 +1655,81 @@ def simple_model_input_functions(perturbation_domain, batchsize, helpers): ) +@fixture +def simple_model_inputs(simple_model_input_functions, input_shape): + ( + decomon_symbolic_input_fn, + keras_input_fn, + decomon_input_fn, + ) = simple_model_input_functions + + keras_symbolic_input = Input(input_shape) + decomon_symbolic_input = decomon_symbolic_input_fn(keras_symbolic_input) + keras_input = keras_input_fn(keras_symbolic_input) + decomon_input = decomon_input_fn(keras_input) + + return keras_symbolic_input, decomon_symbolic_input, keras_input, decomon_input + + +( + simple_model_keras_symbolic_input, + simple_model_decomon_symbolic_input, + simple_model_keras_input, + simple_model_decomon_input, +) = unpack_fixture( + "simple_model_keras_symbolic_input, simple_model_decomon_symbolic_input, simple_model_keras_input, simple_model_decomon_input", + simple_model_inputs, +) + + +def convert_standard_inputs_for_model(get_tensor_decomposition_fn, get_standard_values_fn): + x, y, z, u_c, w_u, b_u, l_c, w_l, b_l = get_tensor_decomposition_fn() + x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_ = get_standard_values_fn() + + keras_symbolic_input = y + keras_input = K.convert_to_tensor(y_) + decomon_symbolic_input = K.concatenate([l_c[:, None], u_c[:, None]], axis=1) + decomon_input = K.convert_to_tensor(np.concatenate((l_c_[:, None], u_c_[:, None]), axis=1)) + + return keras_symbolic_input, decomon_symbolic_input, keras_input, decomon_input + + +@fixture +def standard_model_inputs_0d(n, batchsize, helpers): + get_tensor_decomposition_fn = helpers.get_tensor_decomposition_0d_box + get_standard_values_fn = lambda: helpers.get_standard_values_0d_box(n=n, batchsize=batchsize) + return convert_standard_inputs_for_model(get_tensor_decomposition_fn, get_standard_values_fn) + + +@fixture +def standard_model_inputs_1d(odd, batchsize, helpers): + get_tensor_decomposition_fn = lambda: helpers.get_tensor_decomposition_1d_box(odd=odd) + get_standard_values_fn = lambda: helpers.get_standard_values_1d_box(odd=odd, batchsize=batchsize) + return convert_standard_inputs_for_model(get_tensor_decomposition_fn, get_standard_values_fn) + + +@fixture +def standard_model_inputs_multid(data_format, batchsize, helpers): + odd, m0, m1 = 0, 0, 1 + get_tensor_decomposition_fn = lambda: helpers.get_tensor_decomposition_images_box(data_format=data_format, odd=odd) + get_standard_values_fn = lambda: helpers.get_standard_values_images_box( + data_format=data_format, odd=odd, m0=m0, m1=m1, batchsize=batchsize + ) + return convert_standard_inputs_for_model(get_tensor_decomposition_fn, get_standard_values_fn) + + +model_inputs = fixture_union( + "model_inputs", + [ + simple_model_inputs, + standard_model_inputs_0d, + standard_model_inputs_1d, + standard_model_inputs_multid, + ], + unpack_into="model_keras_symbolic_input, model_decomon_symbolic_input, model_keras_input, model_decomon_input", +) + + # keras toy models toy_model_name = param_fixture( "toy_model_name", diff --git a/tests/lirpa_comparison/test_comparison_lirpa.py b/tests/lirpa_comparison/test_comparison_lirpa.py index c8780b8b..cc3fdbcd 100644 --- a/tests/lirpa_comparison/test_comparison_lirpa.py +++ b/tests/lirpa_comparison/test_comparison_lirpa.py @@ -9,8 +9,8 @@ from onnx2keras import onnx_to_keras from onnx2torch import convert +from decomon.core import ConvertMethod from decomon.models.convert import clone -from decomon.models.utils import ConvertMethod @pytest.mark.parametrize( diff --git a/tests/test_clone.py b/tests/test_clone.py new file mode 100644 index 00000000..e8f1e487 --- /dev/null +++ b/tests/test_clone.py @@ -0,0 +1,87 @@ +import pytest +from keras.layers import Dense, Input +from keras.models import Model +from pytest_cases import fixture, parametrize + +from decomon.core import ConvertMethod, Propagation, Slope +from decomon.models.convert import clone + + +def test_clone_nok_several_inputs(): + a = Input((1,)) + b = Input((2,)) + model = Model([a, b], a) + + with pytest.raises(ValueError, match="only 1 input"): + clone(model) + + +@parametrize( + "toy_model_name", + [ + "tutorial", + # "tutorial_linear", + "tutorial_activation_embedded", + "add", + # "add_linear", + "merge_v0", + "merge_v1", + "merge_v1_seq", + "merge_v2", + # "cnn", # DecomonConv2D not yet implemented + "embedded_model_v1", + "embedded_model_v2", + ], +) +def test_clone( + toy_model_name, + toy_model_fn, + method, + perturbation_domain, + model_keras_symbolic_input, + model_keras_input, + model_decomon_input, + helpers, +): + # input shape? + input_shape = model_keras_symbolic_input.shape[1:] + + # skip cnn on 0d or 1d input_shape + if toy_model_name == "cnn" and len(input_shape) == 1: + pytest.skip("cnn not possible on 0d or 1d input.") + + slope = Slope.Z_SLOPE + decimal = 4 + + # keras model to convert + keras_model = toy_model_fn(input_shape=input_shape) + + # conversion + decomon_model = clone(model=keras_model, slope=slope, perturbation_domain=perturbation_domain, method=method) + + # call on actual outputs + keras_output = keras_model(model_keras_input) + decomon_output = decomon_model(model_decomon_input) + + ibp = decomon_model.ibp + affine = decomon_model.affine + + if method in (ConvertMethod.FORWARD_IBP, ConvertMethod.FORWARD_HYBRID): + assert ibp + else: + assert not ibp + + if method == ConvertMethod.FORWARD_IBP: + assert not affine + else: + assert affine + + # check ibp and affine bounds well ordered w.r.t. keras inputs/outputs + helpers.assert_decomon_output_compare_with_keras_input_output_model( + decomon_output=decomon_output, + keras_input=model_keras_input, + keras_output=keras_output, + decimal=decimal, + ibp=ibp, + affine=affine, + ) From 48db81eeb977c99979ef36c9ae9ad48cfe88331b Mon Sep 17 00:00:00 2001 From: Nolwen Date: Wed, 28 Feb 2024 21:27:01 +0100 Subject: [PATCH 064/101] Add simple inputs with lower < keras_input < upper --- tests/conftest.py | 46 +++++++++++++++++++++++++++----------- tests/test_merge_layers.py | 14 +++++++++--- tests/test_unary_layers.py | 13 ++++++++--- 3 files changed, 54 insertions(+), 19 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 839f9e87..71e4dbfb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,6 +56,7 @@ data_format = param_fixture("data_format", ["channels_last", "channels_first"]) method = param_fixture("method", [m.value for m in ConvertMethod]) input_shape = param_fixture("input_shape", [(1,), (3,), (5, 6, 2)], ids=["0d", "1d", "multid"]) +equal_ibp = param_fixture("equal_ibp", [True, False]) @pytest.fixture @@ -272,6 +273,7 @@ def generate_simple_decomon_layer_inputs_from_keras_input( nobatch=False, for_linear_layer=False, dtype=keras_config.floatx(), + equal_ibp=True, ): """Generate simple decomon inputs for a layer from the corresponding keras input @@ -346,7 +348,13 @@ def generate_simple_decomon_layer_inputs_from_keras_input( affine_bounds_to_propagate = [] if inputs_outputs_spec.needs_constant_bounds_inputs(): - constant_oracle_bounds = [keras_input, keras_input] + if equal_ibp: + constant_oracle_bounds = [keras_input, keras_input] + else: + batchsize = keras_input.shape[0] + lower = K.repeat(K.min(keras_input, axis=0, keepdims=True), batchsize, axis=0) + upper = K.repeat(K.max(keras_input, axis=0, keepdims=True), batchsize, axis=0) + constant_oracle_bounds = [lower, upper] else: constant_oracle_bounds = [] @@ -357,11 +365,17 @@ def generate_simple_decomon_layer_inputs_from_keras_input( ) @staticmethod - def generate_simple_perturbation_domain_inputs_from_keras_input(keras_input, perturbation_domain): + def generate_simple_perturbation_domain_inputs_from_keras_input( + keras_input, perturbation_domain, equal_bounds=True + ): if isinstance(perturbation_domain, BoxDomain): - return K.concatenate( - [keras_input[:, None] - keras.config.epsilon(), keras_input[:, None] + keras.config.epsilon()], axis=1 - ) + if equal_bounds: + return K.concatenate([keras_input[:, None], keras_input[:, None]], axis=1) + else: + batchsize = keras_input.shape[0] + lower = K.repeat(K.min(keras_input, axis=0, keepdims=True), batchsize, axis=0) + upper = K.repeat(K.max(keras_input, axis=0, keepdims=True), batchsize, axis=0) + return K.concatenate([lower[:, None], upper[:, None]], axis=1) else: raise NotImplementedError @@ -966,6 +980,8 @@ def assert_decomon_output_lower_equal_upper( propagation, decimal=5, is_merging_layer=False, + check_ibp=True, + check_affine=True, ): if is_merging_layer: layer_input_shape = [tuple()] @@ -980,7 +996,7 @@ def assert_decomon_output_lower_equal_upper( is_merging_layer=is_merging_layer, ) affine_bounds_propagated, constant_bounds_propagated = inputs_outputs_spec.split_outputs(outputs=decomon_output) - if propagation == Propagation.BACKWARD or affine: + if check_affine and (propagation == Propagation.BACKWARD or affine): if is_merging_layer and propagation == Propagation.BACKWARD: # one list of affine bounds by keras layer input for affine_bounds_propagated_i in affine_bounds_propagated: @@ -993,7 +1009,7 @@ def assert_decomon_output_lower_equal_upper( Helpers.assert_almost_equal(w_l, w_u, decimal=decimal) Helpers.assert_almost_equal(b_l, b_u, decimal=decimal) - if propagation == Propagation.FORWARD and ibp: + if check_ibp and propagation == Propagation.FORWARD and ibp: lower_ibp, upper_ibp = constant_bounds_propagated Helpers.assert_almost_equal(lower_ibp, upper_ibp, decimal=decimal) @@ -1310,7 +1326,7 @@ def helpers(): @fixture def simple_layer_input_functions( - ibp, affine, propagation, perturbation_domain, batchsize, input_shape, empty, diag, nobatch, helpers + ibp, affine, propagation, perturbation_domain, batchsize, input_shape, equal_ibp, empty, diag, nobatch, helpers ): keras_symbolic_model_input_fn = lambda: Input(input_shape) keras_symbolic_layer_input_fn = lambda keras_symbolic_model_input: keras_symbolic_model_input @@ -1344,6 +1360,7 @@ def simple_layer_input_functions( diag=diag, nobatch=nobatch, for_linear_layer=linear, + equal_ibp=equal_ibp, ) return ( @@ -1353,6 +1370,7 @@ def simple_layer_input_functions( keras_model_input_fn, keras_layer_input_fn, decomon_input_fn, + equal_ibp, True, ) @@ -1564,6 +1582,7 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape, linear) keras_layer_input_fn, decomon_input_fn, False, + False, ) @@ -1608,7 +1627,7 @@ def standard_layer_input_functions_multid(data_format, ibp, affine, propagation, standard_layer_input_functions_1d, standard_layer_input_functions_multid, ], - unpack_into="keras_symbolic_model_input_fn, keras_symbolic_layer_input_fn, decomon_symbolic_input_fn, keras_model_input_fn, keras_layer_input_fn, decomon_input_fn, equal_bounds", + unpack_into="keras_symbolic_model_input_fn, keras_symbolic_layer_input_fn, decomon_symbolic_input_fn, keras_model_input_fn, keras_layer_input_fn, decomon_input_fn, equal_ibp_bounds, equal_affine_bounds", ) ( @@ -1618,16 +1637,17 @@ def standard_layer_input_functions_multid(data_format, ibp, affine, propagation, simple_keras_model_input_fn, simple_keras_layer_input_fn, simple_decomon_input_fn, - simple_equal_bounds, + simple_equal_ibp_bounds, + simple_equal_affine_bounds, ) = unpack_fixture( - "simple_keras_symbolic_model_input_fn, simple_keras_symbolic_layer_input_fn, simple_decomon_symbolic_input_fn, simple_keras_model_input_fn, simple_keras_layer_input_fn, simple_decomon_input_fn, simple_equal_bounds", + "simple_keras_symbolic_model_input_fn, simple_keras_symbolic_layer_input_fn, simple_decomon_symbolic_input_fn, simple_keras_model_input_fn, simple_keras_layer_input_fn, simple_decomon_input_fn, simple_equal_ibp_bounds, simple_equal_affine_bounds", simple_layer_input_functions, ) # keras/decomon model inputs @fixture -def simple_model_input_functions(perturbation_domain, batchsize, helpers): +def simple_model_input_functions(perturbation_domain, equal_ibp, batchsize, helpers): decomon_symbolic_input_fn = lambda keras_symbolic_input: Input( perturbation_domain.get_x_input_shape_wo_batchsize(keras_symbolic_input.shape[1:]) ) @@ -1635,7 +1655,7 @@ def simple_model_input_functions(perturbation_domain, batchsize, helpers): keras_symbolic_input.shape[1:], batchsize=batchsize ) decomon_input_fn = lambda keras_input: helpers.generate_simple_perturbation_domain_inputs_from_keras_input( - keras_input=keras_input, perturbation_domain=perturbation_domain + keras_input=keras_input, perturbation_domain=perturbation_domain, equal_bounds=equal_ibp ) return ( diff --git a/tests/test_merge_layers.py b/tests/test_merge_layers.py index 0d6b6514..06bfd8a3 100644 --- a/tests/test_merge_layers.py +++ b/tests/test_merge_layers.py @@ -108,7 +108,8 @@ def test_decomon_merge( keras_model_input_fn, keras_layer_input_fn, decomon_input_fn, - equal_bounds, + equal_ibp_bounds, + equal_affine_bounds, helpers, ): decimal = 4 @@ -217,7 +218,14 @@ def test_decomon_merge( ) # before propagation through linear layer lower == upper => lower == upper after propagation - if equal_bounds and is_actually_linear: + if is_actually_linear: helpers.assert_decomon_output_lower_equal_upper( - decomon_output, ibp=ibp, affine=affine, propagation=propagation, decimal=decimal, is_merging_layer=True + decomon_output, + ibp=ibp, + affine=affine, + propagation=propagation, + decimal=decimal, + check_ibp=equal_ibp_bounds, + check_affine=equal_affine_bounds, + is_merging_layer=True, ) diff --git a/tests/test_unary_layers.py b/tests/test_unary_layers.py index 3a5d4cd7..8fe12ad8 100644 --- a/tests/test_unary_layers.py +++ b/tests/test_unary_layers.py @@ -47,7 +47,8 @@ def test_decomon_unary_layer( keras_model_input_fn, keras_layer_input_fn, decomon_input_fn, - equal_bounds, + equal_ibp_bounds, + equal_affine_bounds, helpers, ): decimal = 4 @@ -135,7 +136,13 @@ def test_decomon_unary_layer( ) # before propagation through linear layer lower == upper => lower == upper after propagation - if equal_bounds and is_actually_linear: + if is_actually_linear: helpers.assert_decomon_output_lower_equal_upper( - decomon_output, ibp=ibp, affine=affine, propagation=propagation, decimal=decimal + decomon_output, + ibp=ibp, + affine=affine, + propagation=propagation, + decimal=decimal, + check_ibp=equal_ibp_bounds, + check_affine=equal_affine_bounds, ) From 4f6f8ff58c22487f4209ccd5f070730646338627 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 29 Feb 2024 15:01:21 +0100 Subject: [PATCH 065/101] Choose random values between -1 and 1 (instead of 0 and 1) --- tests/conftest.py | 2 +- tests/test_unary_layers.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 71e4dbfb..418e2f1b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -133,7 +133,7 @@ def in_GPU_mode() -> bool: @staticmethod def generate_random_tensor(shape_wo_batchsize, batchsize=10, dtype=keras_config.floatx()): shape = (batchsize,) + shape_wo_batchsize - return K.convert_to_tensor(np.random.random(shape), dtype=dtype) + return K.convert_to_tensor(2.0 * np.random.random(shape) - 1.0, dtype=dtype) @staticmethod def get_decomon_input_shapes( diff --git a/tests/test_unary_layers.py b/tests/test_unary_layers.py index 8fe12ad8..9a365a61 100644 --- a/tests/test_unary_layers.py +++ b/tests/test_unary_layers.py @@ -61,9 +61,9 @@ def test_decomon_unary_layer( layer = keras_layer_class(**keras_layer_kwargs) layer(keras_symbolic_layer_input) - # randomize weights => non-zero biases + # randomize weights between -1 and 1 => non-zero biases for w in layer.weights: - w.assign(np.random.random(w.shape)) + w.assign(2.0 * np.random.random(w.shape) - 1.0) # init + build decomon layer output_shape = layer.output.shape[1:] From 67ca34d7d012db00f12a5a8bddb5c4bbc3697550 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 29 Feb 2024 15:57:59 +0100 Subject: [PATCH 066/101] Add LinkToPerturbationDomainInput layer to use in clone() for fully linear keras models When a keras model is fully linear, its affine bounds will be independent from sample, and in backward prapagation with no first backward_bounds, they will even be eagerly computed as backend tensors. It is not possible to instantiate a decomon model in that case since some outputs are not depending on the perturbation domain input. So we artificially make them depend on the perturbation domain input by adding a dedicated layer at the end of the conversion. --- src/decomon/layers/utils/__init__.py | 1 + src/decomon/layers/utils/batchsize.py | 53 +++++++++++++++++++++++++++ src/decomon/layers/utils/symbolify.py | 29 +++++++++++++++ src/decomon/models/convert.py | 14 +++++++ tests/test_clone.py | 11 +++++- 5 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 src/decomon/layers/utils/__init__.py create mode 100644 src/decomon/layers/utils/batchsize.py create mode 100644 src/decomon/layers/utils/symbolify.py diff --git a/src/decomon/layers/utils/__init__.py b/src/decomon/layers/utils/__init__.py new file mode 100644 index 00000000..576238d8 --- /dev/null +++ b/src/decomon/layers/utils/__init__.py @@ -0,0 +1 @@ +"""Utilitary layers for decomon.""" diff --git a/src/decomon/layers/utils/batchsize.py b/src/decomon/layers/utils/batchsize.py new file mode 100644 index 00000000..7bbf1b6a --- /dev/null +++ b/src/decomon/layers/utils/batchsize.py @@ -0,0 +1,53 @@ +"""Adding batchsize to batch-independent outputs.""" + + +from typing import Any, Optional + +import keras +import keras.ops as K +from keras.layers import Layer + +from decomon.types import BackendTensor + + +class InsertBatchAxis(Layer): + """Layer adding a batch axis to its inputs. + + It accepts a list of tensors as its inputs. The convention is that: + - The first tensor contains the batch axis (this we can deduce from it the batchsize) + - The following tensors may miss or not this batch axis + + It returns the tensors, except the first ones, + with a new first axis repeated to the proper batchsize when it was missing. + + """ + + def call(self, inputs: list[BackendTensor]) -> list[BackendTensor]: + batchsize = inputs[0].shape[0] + + outputs = [ + K.repeat(input_i[None], batchsize, axis=0) if missing_batchsize_i else input_i + for input_i, missing_batchsize_i in zip(inputs[1:], self.missing_batchaxis[1:]) + ] + + return outputs + + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: + # called on list? + if not isinstance(input_shape[0], (tuple, list)): + raise ValueError( + "An InsertBatchAxis layer should be called on a list of inputs. " + f"Received: input_shape={input_shape} (not a list of shapes)" + ) + # check first input has a batch axis + if None not in input_shape[0]: + raise ValueError("The first tensor in InsertBatchAxis inputs is supposed to have a batch axis.") + # store the input indices missing the batchaxis + self.missing_batchaxis = [None not in input_shape_i for input_shape_i in input_shape] + self.built = True + + def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: + return [ + (None,) + input_shape_i if missing_bacthaxis_i else input_shape_i + for input_shape_i, missing_bacthaxis_i in zip(input_shape[1:], self.missing_batchaxis[1:]) + ] diff --git a/src/decomon/layers/utils/symbolify.py b/src/decomon/layers/utils/symbolify.py new file mode 100644 index 00000000..f2eb3d0b --- /dev/null +++ b/src/decomon/layers/utils/symbolify.py @@ -0,0 +1,29 @@ +"""Converting backend tensors to symbolic tensors.""" + +from typing import Any, Optional + +import keras +from keras.layers import Layer + +from decomon.types import BackendTensor + + +class LinkToPerturbationDomainInput(Layer): + """Layer making its inputs artificially depend on the first one. + + It accepts a list of tensors as its inputs. The convention is that: + - The first tensor is the tensor we want the other to depend on, will be dropped in the output + - The following tensors will be returned as is + + It returns the tensors, except the first one. + + The usecase is to be able to create a DecomonModel even when the propagated bounds does not really + depend on the perturbation input. + + """ + + def call(self, inputs: list[BackendTensor]) -> list[BackendTensor]: + return inputs[1:] + + def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: + return input_shape[1:] diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index fd62bb75..123d5401 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -1,3 +1,4 @@ +import logging from collections.abc import Callable from typing import Any, Optional, Union @@ -14,6 +15,7 @@ ) from decomon.layers import DecomonLayer from decomon.layers.convert import to_decomon +from decomon.layers.utils.symbolify import LinkToPerturbationDomainInput from decomon.models.backward_cloning import convert_backward from decomon.models.forward_cloning import ( convert_forward, @@ -32,6 +34,8 @@ split_activation, ) +logger = logging.getLogger(__name__) + def _clone_keras_model(model: Model, layer_fn: Callable[[Layer], list[Layer]]) -> Model: if model.inputs is None: @@ -230,6 +234,16 @@ def clone( **kwargs, ) + # full linear model? => batch independent output + if any([not isinstance(o, keras.KerasTensor) for o in output]): + logger.warning( + "Some propagated bounds have been eagerly computed, being independent from batch. " + "This should only be possible if the keras model is fully affine. -" + "We will make them artificially depend on perturbation input in order to create the DecomonModel." + ) + # Insert batch axis and repeat it to get the correct batchsize + output = LinkToPerturbationDomainInput()([perturbation_domain_input] + output) + return DecomonModel( inputs=[perturbation_domain_input], outputs=output, diff --git a/tests/test_clone.py b/tests/test_clone.py index e8f1e487..5144b54b 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -4,6 +4,7 @@ from pytest_cases import fixture, parametrize from decomon.core import ConvertMethod, Propagation, Slope +from decomon.layers.utils.symbolify import LinkToPerturbationDomainInput from decomon.models.convert import clone @@ -20,10 +21,10 @@ def test_clone_nok_several_inputs(): "toy_model_name", [ "tutorial", - # "tutorial_linear", + "tutorial_linear", "tutorial_activation_embedded", "add", - # "add_linear", + "add_linear", "merge_v0", "merge_v1", "merge_v1_seq", @@ -85,3 +86,9 @@ def test_clone( ibp=ibp, affine=affine, ) + + # check that we added a layer to insert batch axis + if toy_model_name.endswith("_linear") and str(method).lower().startswith("crown"): + assert isinstance(decomon_model.layers[-1], LinkToPerturbationDomainInput) + else: + assert not isinstance(decomon_model.layers[-1], LinkToPerturbationDomainInput) From 51b9602fbdd5b07f84227ce3a0116854b106787f Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 29 Feb 2024 22:25:41 +0100 Subject: [PATCH 067/101] Create a ForwardInput layer for the first forward layer This layer deduces forward (ibp and affine) inputs from the perturbation domain input. This simplifies the graph representation of a forward decomon model. --- src/decomon/core.py | 31 ++++++++ src/decomon/layers/input.py | 105 ++++++++++++++++++++++++++ src/decomon/models/forward_cloning.py | 54 +------------ tests/test_inputs.py | 29 +++++++ 4 files changed, 168 insertions(+), 51 deletions(-) create mode 100644 src/decomon/layers/input.py create mode 100644 tests/test_inputs.py diff --git a/src/decomon/core.py b/src/decomon/core.py index 3fd24233..b93d83d3 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -89,6 +89,13 @@ def get_x_input_shape_wo_batchsize(self, original_input_shape: tuple[int, ...]) else: return (n_comp_x,) + original_input_shape + def get_keras_input_shape_wo_batchsize(self, x_shape: Tuple[int, ...]) -> Tuple[int, ...]: + n_comp_x = self.get_nb_x_components() + if n_comp_x == 1: + return x_shape + else: + return x_shape[1:] + class BoxDomain(PerturbationDomain): def get_upper(self, x: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: @@ -575,6 +582,30 @@ def flatten_inputs( else: return affine_bounds_to_propagate + constant_oracle_bounds + perturbation_domain_inputs + def flatten_inputs_shape( + self, + affine_bounds_to_propagate_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], + constant_oracle_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], + perturbation_domain_inputs_shape: list[tuple[Optional[int], ...]], + ) -> list[tuple[Optional[int], ...]]: + """Flatten inputs shape + + Same operation as `flatten_inputs` but on tensor shapes. + + Args: + affine_bounds_to_propagate_shape: + constant_oracle_bounds_shape: + perturbation_domain_inputs_shape: + + Returns: + + """ + return self.flatten_inputs( # type: ignore + affine_bounds_to_propagate=affine_bounds_to_propagate_shape, + constant_oracle_bounds=constant_oracle_bounds_shape, + perturbation_domain_inputs=perturbation_domain_inputs_shape, + ) # type: ignore + def split_outputs(self, outputs: list[Tensor]) -> tuple[Union[list[Tensor], list[list[Tensor]]], list[Tensor]]: """Split decomon inputs. diff --git a/src/decomon/layers/input.py b/src/decomon/layers/input.py new file mode 100644 index 00000000..3e5bfcc3 --- /dev/null +++ b/src/decomon/layers/input.py @@ -0,0 +1,105 @@ +"""Generate decomon inputs from perturbation domain input.""" + + +from typing import Any, Optional + +import keras +import keras.ops as K +from keras.layers import Layer + +from decomon.core import InputsOutputsSpec, PerturbationDomain, Propagation +from decomon.types import BackendTensor + + +class ForwardInput(Layer): + """Layer generating the input of the first forward layer of a decomon layer.""" + + def __init__( + self, + perturbation_domain: PerturbationDomain, + ibp: bool, + affine: bool, + **kwargs: Any, + ): + """ + Args: + perturbation_domain: default to a box domain + ibp: if True, forward propagate constant bounds + affine: if True, forward propagate affine bounds + **kwargs: + + """ + super().__init__(**kwargs) + + self.perturbation_domain = perturbation_domain + self.ibp = ibp + self.affine = affine + self.inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + propagation=Propagation.FORWARD, + perturbation_domain=perturbation_domain, + model_input_shape=tuple(), + layer_input_shape=tuple(), + ) + + def call(self, inputs: BackendTensor) -> list[BackendTensor]: + """Generate ibp and affine bounds to propagate by the first forward layer. + + Args: + inputs: the perturbation domain input + + Returns: + affine_bounds + constant_bounds: the affine and constant bounds concatenated + + - affine_bounds: [w, 0, w, 0], with w the identity tensor of the proper shape + (without batchsize, in diagonal representation) + - constant_bounds: deduced from perturbation domain type and input + + """ + if self.affine: + keras_input_like_tensor_wo_batchsize = self.perturbation_domain.get_kerasinputlike_from_x(x=inputs)[0] + # identity: diag representation + w/o batchisze + w = K.ones_like(keras_input_like_tensor_wo_batchsize) + b = K.zeros_like(keras_input_like_tensor_wo_batchsize) + affine_bounds = [w, b, w, b] + else: + affine_bounds = [] + if self.ibp: + constant_bounds = [ + self.perturbation_domain.get_lower_x(x=inputs), + self.perturbation_domain.get_upper_x(x=inputs), + ] + else: + constant_bounds = [] + return self.inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=affine_bounds, + constant_oracle_bounds=constant_bounds, + perturbation_domain_inputs=[], + ) + + def compute_output_shape( + self, + input_shape: tuple[Optional[int], ...], + ) -> list[tuple[Optional[int], ...]]: + perturbation_domain_input_shape_wo_batchsize = input_shape[1:] + keras_input_shape_wo_batchsize = self.perturbation_domain.get_keras_input_shape_wo_batchsize( + x_shape=perturbation_domain_input_shape_wo_batchsize + ) + + if self.affine: + w_shape = keras_input_shape_wo_batchsize + b_shape = keras_input_shape_wo_batchsize + affine_bounds_shape = [w_shape, b_shape, w_shape, b_shape] + else: + affine_bounds_shape = [] + if self.ibp: + lower_shape = (None,) + keras_input_shape_wo_batchsize + constant_bounds_shape = [lower_shape, lower_shape] + else: + constant_bounds_shape = [] + return self.inputs_outputs_spec.flatten_inputs_shape( + affine_bounds_to_propagate_shape=affine_bounds_shape, + constant_oracle_bounds_shape=constant_bounds_shape, + perturbation_domain_inputs_shape=[], + ) diff --git a/src/decomon/models/forward_cloning.py b/src/decomon/models/forward_cloning.py index 3d6c32bb..596092ad 100644 --- a/src/decomon/models/forward_cloning.py +++ b/src/decomon/models/forward_cloning.py @@ -20,6 +20,7 @@ ) from decomon.layers import DecomonLayer from decomon.layers.convert import to_decomon +from decomon.layers.input import ForwardInput from decomon.models.utils import ( ensure_functional_model, get_depth_dict, @@ -29,42 +30,6 @@ ) -def get_ibp_inputs(x: keras.KerasTensor, perturbation_domain: PerturbationDomain) -> list[keras.KerasTensor]: - """Get ibp inputs from perturbation domain input - - Args: - x: perturbation domain input - perturbation_domain: perturbation domain type - - Returns: - [l_c, u_c] constant bounds on keras model input - - """ - return [perturbation_domain.get_lower_x(x=x), perturbation_domain.get_upper_x(x=x)] - - -def get_affine_inputs(x: keras.KerasTensor, perturbation_domain: PerturbationDomain) -> list[keras.KerasTensor]: - """Get affine inputs from perturbation domain input - - Args: - x: perturbation domain input - perturbation_domain: perturbation domain type - - Returns: - [w_l, b_l, w_u, b_u] identity affine bounds on keras model input - - We start with identity bounds: w_l = w_u = Identity, b_l = b_u = 0 - We need the perturbation domain input to get the proper shape, and to construct the inputs - from the future decomon model input x. - - """ - keras_input_like_tensor_wo_batchsize = perturbation_domain.get_kerasinputlike_from_x(x=x)[0] - # identity: diag representation + w/o batchisze - w = K.ones_like(keras_input_like_tensor_wo_batchsize) # identity in diag - b = K.zeros_like(keras_input_like_tensor_wo_batchsize) - return [w, b, w, b] - - def convert_forward( model: Model, perturbation_domain_input: keras.KerasTensor, @@ -135,21 +100,8 @@ def convert_forward( ) # generate input tensors - if affine: - affine_bounds_to_propagate = get_affine_inputs( - x=perturbation_domain_input, perturbation_domain=perturbation_domain - ) - else: - affine_bounds_to_propagate = [] - if ibp: - constant_oracle_bounds = get_ibp_inputs(x=perturbation_domain_input, perturbation_domain=perturbation_domain) - else: - constant_oracle_bounds = [] - input_tensors_wo_pertubation_domain_inputs = inputs_outputs_spec.flatten_inputs( - affine_bounds_to_propagate=affine_bounds_to_propagate, - constant_oracle_bounds=constant_oracle_bounds, - perturbation_domain_inputs=[], - ) + forward_input_layer = ForwardInput(perturbation_domain=perturbation_domain, ibp=ibp, affine=affine) + input_tensors_wo_pertubation_domain_inputs = forward_input_layer(perturbation_domain_input) output_map: dict[int, list[keras.KerasTensor]] = {} layer_list_map: dict[int, list[DecomonLayer]] = {} diff --git a/tests/test_inputs.py b/tests/test_inputs.py new file mode 100644 index 00000000..45b71431 --- /dev/null +++ b/tests/test_inputs.py @@ -0,0 +1,29 @@ +import pytest +from pytest_cases import parametrize + +from decomon.core import Propagation +from decomon.layers.input import ForwardInput + + +def test_forward_input( + ibp, + affine, + propagation, + perturbation_domain, + simple_model_decomon_symbolic_input, + simple_model_decomon_input, + batchsize, + helpers, +): + if propagation == Propagation.BACKWARD: + pytest.skip("backward propagation meaningless for ForwardInput") + + layer = ForwardInput(perturbation_domain=perturbation_domain, ibp=ibp, affine=affine) + symbolic_output = layer(simple_model_decomon_symbolic_input) + output = layer(simple_model_decomon_input) + + # check shapes + output_shape = [t.shape for t in output] + expected_output_shape = [t.shape for t in symbolic_output] + expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) + assert output_shape == expected_output_shape From 5fd6eca09bd6895f3774833ec7c783ec50fb7d1d Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 1 Mar 2024 15:18:18 +0100 Subject: [PATCH 068/101] Introduce oracle layers to manage "decomon outputs -> oracle" This simplifies the graph of a crown model fully recursive or in forward pure affine mode, by gathering the affine bounds + perturbation domain input combination. --- src/decomon/layers/layer.py | 116 +++--------- src/decomon/layers/merging/base_merge.py | 56 +----- src/decomon/layers/oracle.py | 228 +++++++++++++++++++++++ src/decomon/models/backward_cloning.py | 32 +++- tests/test_oracle.py | 92 +++++++++ 5 files changed, 371 insertions(+), 153 deletions(-) create mode 100644 src/decomon/layers/oracle.py create mode 100644 tests/test_oracle.py diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index 960619d5..3031ec8a 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -7,6 +7,7 @@ from decomon.core import BoxDomain, InputsOutputsSpec, PerturbationDomain, Propagation from decomon.keras_utils import batch_multid_dot +from decomon.layers.oracle import get_forward_oracle from decomon.types import Tensor _keras_base_layer_keyword_parameters = [ @@ -65,7 +66,7 @@ class DecomonLayer(Wrapper): """ - _is_merging: bool = False # set to True in child class DecomonMerge + _is_merging_layer: bool = False # set to True in child class DecomonMerge def __init__( self, @@ -138,7 +139,7 @@ def create_inputs_outputs_spec( model_input_shape: Optional[tuple[int, ...]], model_output_shape: Optional[tuple[int, ...]], ) -> InputsOutputsSpec: - if self._is_merging: + if self._is_merging_layer: if isinstance(layer.input, keras.KerasTensor): # special case: merging a single input -> self.layer.input is already flattened layer_input_shape = [layer.input.shape[1:]] @@ -154,10 +155,19 @@ def create_inputs_outputs_spec( layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=model_output_shape, - is_merging_layer=self._is_merging, + is_merging_layer=self._is_merging_layer, linear=self.linear, ) + @property + def is_merging_layer(self) -> bool: + """Flag telling if the underlying keras layer is a merging layer or not (i.e. ~ with several inputs).""" + return self._is_merging_layer + + @property + def layer_input_shape(self) -> tuple[int, ...]: + return self.inputs_outputs_spec.layer_input_shape + @property def model_input_shape(self) -> tuple[int, ...]: return self.inputs_outputs_spec.model_input_shape @@ -425,8 +435,6 @@ def get_forward_oracle( input_affine_bounds: list[Tensor], input_constant_bounds: list[Tensor], perturbation_domain_inputs: list[Tensor], - ibp: Optional[bool] = None, - affine: Optional[bool] = None, ) -> list[Tensor]: """Get constant oracle bounds on underlying keras layer input from forward input bounds. @@ -434,8 +442,6 @@ def get_forward_oracle( input_affine_bounds: affine bounds on keras layer input w.r.t model input . Can be empty if not in affine mode. input_constant_bounds: ibp constant bounds on keras layer input. Can be empty if not in ibp mode. perturbation_domain_inputs: perturbation domain input, wrapped in a list. Necessary only in affine mode, else empty. - ibp: if set, overrides temporarily `self.ibp` (used by `call_oracle()`) - affine: if set, overrides temporarily `self.affine` (used by `call_oracle()`) Returns: constant bounds on keras layer input deduced from forward input bounds @@ -448,85 +454,15 @@ def get_forward_oracle( from the affine bounds given the considered perturbation domain. """ - if ibp is None: - ibp = self.ibp - if affine is None: - affine = self.affine - - if ibp: - # Hyp: in hybrid mode, the constant bounds are already tight - # (affine and ibp mixed in forward layer output to get the tightest constant bounds) - return input_constant_bounds - - elif affine: - if len(perturbation_domain_inputs) == 0: - raise RuntimeError("keras model input is necessary for get_forward_oracle() in affine mode.") - x = perturbation_domain_inputs[0] - if len(input_affine_bounds) == 0: - # special case: empty affine bounds => identity bounds - l_affine = self.perturbation_domain.get_lower_x(x) - u_affine = self.perturbation_domain.get_upper_x(x) - else: - w_l, b_l, w_u, b_u = input_affine_bounds - l_affine = self.perturbation_domain.get_lower(x, w_l, b_l) - u_affine = self.perturbation_domain.get_upper(x, w_u, b_u) - return [l_affine, u_affine] - - else: - raise RuntimeError("self.ibp and self.affine cannot be both False") - - def call_oracle(self, inputs: list[Tensor]) -> list[Tensor]: - """Compute oracle constant bounds on keras inputs from flatten decomon inputs. - - - forward: this is `self.get_forward_oracle()` after a split of inputs - - backward: this is a crown oracle. - The inputs are then equivalent to inputs for forward propagation + ibp=False + affine=True, - i.e. affine bounds (a priori coming from crowns) on each keras input + perturbation_domain_inputs - The computation is also equivalent to the one done in `self.get_forward_oracle()` with ibp=False, affine=True - - Args: - inputs: affine bounds on keras layer input + perturbation_domain_inputs - - Returns: - constant bounds on keras layer input deduced from affine bounds - - """ - ( - affine_bounds_to_propagate, - constant_oracle_bounds, - perturbation_domain_inputs, - ) = self.inputs_outputs_spec.split_inputs(inputs=inputs) - if self.propagation == Propagation.FORWARD: # forward - return self.get_forward_oracle( - input_affine_bounds=affine_bounds_to_propagate, - input_constant_bounds=constant_oracle_bounds, - perturbation_domain_inputs=perturbation_domain_inputs, - ) - else: # backward - ibp = False - affine = True - propagation = Propagation.FORWARD - inputs_outputs_spec = self.create_inputs_outputs_spec( - ibp=ibp, - affine=affine, - propagation=propagation, - perturbation_domain=self.perturbation_domain, - layer=self.layer, - model_input_shape=self.model_input_shape, - model_output_shape=self.model_output_shape, - ) - ( - affine_bounds_to_propagate, - constant_oracle_bounds, - perturbation_domain_inputs, - ) = inputs_outputs_spec.split_inputs(inputs=inputs) - return self.get_forward_oracle( - input_affine_bounds=affine_bounds_to_propagate, - input_constant_bounds=constant_oracle_bounds, - perturbation_domain_inputs=perturbation_domain_inputs, - ibp=ibp, - affine=affine, - ) + return get_forward_oracle( + affine_bounds=input_affine_bounds, + ibp_bounds=input_constant_bounds, + perturbation_domain_inputs=perturbation_domain_inputs, + perturbation_domain=self.perturbation_domain, + ibp=self.ibp, + affine=self.affine, + is_merging_layer=self.is_merging_layer, + ) def call_forward( self, @@ -702,12 +638,12 @@ def compute_output_shape( # outputs shape depends if layer and inputs are diagonal / linear (w/o batch) b_shape_wo_batchisze = model_output_shape_wo_batchsize if self.diagonal and self.inputs_outputs_spec.is_diagonal_bounds_shape(affine_bounds_to_propagate_shape): - if self._is_merging: + if self._is_merging_layer: w_shape_wo_batchsize = [model_output_shape_wo_batchsize] * self.inputs_outputs_spec.nb_keras_inputs else: w_shape_wo_batchsize = model_output_shape_wo_batchsize else: - if self._is_merging: + if self._is_merging_layer: w_shape_wo_batchsize = [ self.layer.input[i].shape[1:] + model_output_shape_wo_batchsize for i in range(self.inputs_outputs_spec.nb_keras_inputs) @@ -719,11 +655,11 @@ def compute_output_shape( w_shape = w_shape_wo_batchsize else: b_shape = (None,) + b_shape_wo_batchisze - if self._is_merging: + if self._is_merging_layer: w_shape = [(None,) + sub_w_shape_wo_batchsize for sub_w_shape_wo_batchsize in w_shape_wo_batchsize] else: w_shape = (None,) + w_shape_wo_batchsize - if self._is_merging: + if self._is_merging_layer: affine_bounds_propagated_shape = [ [ w_shape_i, diff --git a/src/decomon/layers/merging/base_merge.py b/src/decomon/layers/merging/base_merge.py index 7e8fbae7..150caa81 100644 --- a/src/decomon/layers/merging/base_merge.py +++ b/src/decomon/layers/merging/base_merge.py @@ -8,7 +8,7 @@ class DecomonMerge(DecomonLayer): - _is_merging = True + _is_merging_layer = True @property def keras_layer_input(self): @@ -358,17 +358,13 @@ def get_forward_oracle( input_affine_bounds: list[list[Tensor]], input_constant_bounds: list[list[Tensor]], perturbation_domain_inputs: list[Tensor], - ibp: Optional[bool] = None, - affine: Optional[bool] = None, - ) -> list[list[Tensor]]: + ) -> list[list[Tensor]]: # type: ignore """Get constant oracle bounds on underlying keras layer input from forward input bounds. Args: input_affine_bounds: affine bounds on each keras layer input w.r.t model input . Can be empty if not in affine mode. input_constant_bounds: ibp constant bounds on each keras layer input. Can be empty if not in ibp mode. perturbation_domain_inputs: perturbation domain input, wrapped in a list. Necessary only in affine mode, else empty. - ibp: if set, overrides temporarily `self.ibp` (used by `call_oracle()`) - affine: if set, overrides temporarily `self.affine` (used by `call_oracle()`) Returns: constant bounds on each keras layer input deduced from forward input bounds @@ -381,53 +377,7 @@ def get_forward_oracle( from the affine bounds given the considered perturbation domain. """ - if ibp is None: - ibp = self.ibp - if affine is None: - affine = self.affine - - if ibp: - # Hyp: in hybrid mode, the constant bounds are already tight - # (affine and ibp mixed in forward layer output to get the tightest constant bounds) - return input_constant_bounds - - elif affine: - if len(perturbation_domain_inputs) == 0: - raise RuntimeError("keras model input is necessary for get_forward_oracle() in affine mode.") - x = perturbation_domain_inputs[0] - constant_bounds = [] - for input_affine_bounds_i in input_affine_bounds: - if len(input_affine_bounds_i) == 0: - # special case: empty affine bounds => identity bounds - l_affine = self.perturbation_domain.get_lower_x(x) - u_affine = self.perturbation_domain.get_upper_x(x) - else: - w_l, b_l, w_u, b_u = input_affine_bounds_i - l_affine = self.perturbation_domain.get_lower(x, w_l, b_l) - u_affine = self.perturbation_domain.get_upper(x, w_u, b_u) - constant_bounds.append([l_affine, u_affine]) - return constant_bounds - - else: - raise RuntimeError("self.ibp and self.affine cannot be both False") - - def call_oracle(self, inputs: list[Tensor]) -> list[list[Tensor]]: - """Compute oracle constant bounds on keras inputs from flatten decomon inputs. - - - forward: this is `self.get_forward_oracle()` after a split of inputs - - backward: this is a crown oracle. - The inputs are then equivalent to inputs for forward propagation + ibp=False + affine=True, - i.e. affine bounds (a priori coming from crowns) on each keras input. - The computation is also equivalent to the one done in `self.get_forward_oracle()` with ibp=False, affine=True - - Args: - inputs: concatenated affine bounds on each keras layer input + perturbation_domain_inputs - - Returns: - constant bounds on each keras layer input deduced from affine bounds - - """ - return super().call_oracle(inputs) + return super().get_forward_oracle(input_affine_bounds=input_affine_bounds, input_constant_bounds=input_constant_bounds, perturbation_domain_inputs=perturbation_domain_inputs) # type: ignore def call_forward( self, diff --git a/src/decomon/layers/oracle.py b/src/decomon/layers/oracle.py new file mode 100644 index 00000000..76846d77 --- /dev/null +++ b/src/decomon/layers/oracle.py @@ -0,0 +1,228 @@ +"""Layers specifying constant oracle bounds on keras layer input.""" + + +from typing import Any, Optional, Union, overload + +import keras +from keras.layers import Layer + +from decomon.core import InputsOutputsSpec, PerturbationDomain +from decomon.types import BackendTensor + + +class BaseOracle(Layer): + """Base class for oracle layers.""" + + ... + + +class DecomonOracle(BaseOracle): + """Layer deducing oracle bounds from decomon model outputs (forward or crown). + + The decomon model is supposed to give bounds for the inputs of the keras layer of interest. + + - In case of forward ibp or hybrid decomon model, + the ibp bounds in decomon outputs are already the oracle bounds. + - In case of forward pure affine or crown decomon model, + the outputs are affine bounds on the inputs of the keras layer of interest w.r.t the keras model input. + So we combine them with the perturbation domain input. + + Merging keras layer case: if the keras layer needing oracle is a merging layer, + - the decomon model outputs are the concatenation of bounds on each keras layer input, + - the oracle will be the concatenation of oracle bounds on each keras layer input + + """ + + def __init__( + self, + perturbation_domain: PerturbationDomain, + ibp: bool, + affine: bool, + layer_input_shape: Union[tuple[int, ...], list[tuple[int, ...]]], + is_merging_layer: bool, + **kwargs: Any, + ): + """ + Args: + perturbation_domain: default to a box domain + ibp: True for forward ibp or forward hybrid decomon model. + In that case, the ibp bounds in the outputs are already the oracle bounds + affine: True for forward affine (and hybrid) model and crown model. + In that case (except for hybrid case, see above), the oracle bounds are deduce from the affine bounds. + layer_input_shape: input shape of the underlying keras layer, without batchsize + is_merging_layer: whether the underlying keras layer is a merging layer (i.e. with several inputs) + **kwargs: + + """ + super().__init__(**kwargs) + + self.perturbation_domain = perturbation_domain + self.ibp = ibp + self.affine = affine + self.is_merging_layer = is_merging_layer + self.layer_input_shape = layer_input_shape + self.inputs_outputs_spec = InputsOutputsSpec( + ibp=ibp, + affine=affine, + perturbation_domain=perturbation_domain, + layer_input_shape=layer_input_shape, + is_merging_layer=is_merging_layer, + ) + + def call(self, inputs: list[BackendTensor]) -> Union[list[BackendTensor], list[list[BackendTensor]]]: + """Deduce ibp and affine bounds to propagate by the first forward layer. + + Args: + inputs: the outputs of a decomon model + perturbation domain input if necessary + + Returns: + concatenation of constant bounds on each keras layer input deduced from ibp and/or affine bounds + + According to the sense of propagation of the decomon model, we have + + - forward: inputs = affine_bounds + ibp_bounds + perturbation_domain_inputs with + - affine_bounds: empty if self.affine = False + - ibp_bounds: empty if self.ibp = False, already tight in hyb rid case (self.ibp=true and self.affine=True) + - perturbation_domain_inputs: if self.ibp=False this is the pertubation domain input wrapped in a list, else empty. + + outputs = ibp_bounds when available, else combine affine_bounds with perturbation_domain_inputs + + - backward: inputs = crown_bounds + perturbation_domain_inputs with + - affine_bounds: empty if self.affine = False + - ibp_bounds: empty if self.ibp = False, already tight in hyb rid case (self.ibp=true and self.affine=True) + - perturbation_domain_inputs: if self.ibp=False this is the pertubation domain input wrapped in a list, else empty. + + outputs = crown_bounds (affine) combined with perturbation domain input + + """ + ( + affine_bounds, + ibp_bounds, + perturbation_domain_inputs, + ) = self.inputs_outputs_spec.split_inputs(inputs=inputs) + + return get_forward_oracle( + affine_bounds=affine_bounds, + ibp_bounds=ibp_bounds, + perturbation_domain_inputs=perturbation_domain_inputs, + perturbation_domain=self.perturbation_domain, + ibp=self.ibp, + affine=self.affine, + is_merging_layer=self.is_merging_layer, + ) + + def compute_output_shape( + self, + input_shape: tuple[Optional[int], ...], + ) -> Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]]: + """Compute output shape in case of symbolic call.""" + if self.is_merging_layer: + output_shape = [] + for layer_input_shape_i in self.layer_input_shape: + layer_input_shape_w_batchsize_i = (None,) + layer_input_shape_i + output_shape.append([layer_input_shape_w_batchsize_i, layer_input_shape_w_batchsize_i]) + return output_shape + else: + layer_input_shape_w_batchsize = (None,) + self.layer_input_shape + return [layer_input_shape_w_batchsize, layer_input_shape_w_batchsize] + + +@overload +def get_forward_oracle( + affine_bounds: list[BackendTensor], + ibp_bounds: list[BackendTensor], + perturbation_domain_inputs: list[BackendTensor], + perturbation_domain: PerturbationDomain, + ibp: bool, + affine: bool, + is_merging_layer: bool, +) -> list[BackendTensor]: + """Get constant oracle bounds on keras layer inputs from forward input bounds. + + Non-merging layer version. + + """ + ... + + +@overload +def get_forward_oracle( + affine_bounds: list[list[BackendTensor]], + ibp_bounds: list[list[BackendTensor]], + perturbation_domain_inputs: list[BackendTensor], + perturbation_domain: PerturbationDomain, + ibp: bool, + affine: bool, + is_merging_layer: bool, +) -> list[list[BackendTensor]]: + """Get constant oracle bounds on keras layer inputs from forward input bounds. + + Merging layer version. + + """ + ... + + +def get_forward_oracle( + affine_bounds: Union[list[BackendTensor], list[list[BackendTensor]]], + ibp_bounds: Union[list[BackendTensor], list[list[BackendTensor]]], + perturbation_domain_inputs: list[BackendTensor], + perturbation_domain: PerturbationDomain, + ibp: bool, + affine: bool, + is_merging_layer: bool, +) -> Union[list[BackendTensor], list[list[BackendTensor]]]: + """Get constant oracle bounds on keras layer inputs from forward input bounds. + + Args: + affine_bounds: affine bounds on keras layer input w.r.t model input . Can be empty if not in affine mode. + ibp_bounds: ibp constant bounds on keras layer input. Can be empty if not in ibp mode. + perturbation_domain_inputs: perturbation domain input, wrapped in a list. Necessary only in affine mode, else empty. + perturbation_domain: perturbation domain spec. + ibp: ibp bounds exist? + affine: affine bounds exist? + is_merging_layer: keras layer is a merging layer? + + Returns: + constant bounds on keras layer input deduced from forward layer input bounds or crown output + perturbation_domain_input + + In hybrid case (ibp+affine), the constant bounds are assumed to be already tight, which means the previous + forward layer should already have took the tighter constant bounds between the ibp ones and the ones deduced + from the affine bounds given the considered perturbation domain. + + """ + if ibp: + # Hyp: in hybrid mode, the constant bounds are already tight + # (affine and ibp mixed in forward layer output to get the tightest constant bounds) + return ibp_bounds + + elif affine: + if len(perturbation_domain_inputs) == 0: + raise RuntimeError("Perturbation domain input is necessary for get_forward_oracle() in affine mode.") + x = perturbation_domain_inputs[0] + if is_merging_layer: + constant_bounds = [] + for affine_bounds_i in affine_bounds: + if len(affine_bounds_i) == 0: + # special case: empty affine bounds => identity bounds + l_affine = perturbation_domain.get_lower_x(x) + u_affine = perturbation_domain.get_upper_x(x) + else: + w_l, b_l, w_u, b_u = affine_bounds_i + l_affine = perturbation_domain.get_lower(x, w_l, b_l) + u_affine = perturbation_domain.get_upper(x, w_u, b_u) + constant_bounds.append([l_affine, u_affine]) + return constant_bounds + else: + if len(affine_bounds) == 0: + # special case: empty affine bounds => identity bounds + l_affine = perturbation_domain.get_lower_x(x) + u_affine = perturbation_domain.get_upper_x(x) + else: + w_l, b_l, w_u, b_u = affine_bounds + l_affine = perturbation_domain.get_lower(x, w_l, b_l) + u_affine = perturbation_domain.get_upper(x, w_u, b_u) + return [l_affine, u_affine] + + else: + raise RuntimeError("ibp and affine cannot be both False") diff --git a/src/decomon/models/backward_cloning.py b/src/decomon/models/backward_cloning.py index 803ef7dc..47115bb2 100644 --- a/src/decomon/models/backward_cloning.py +++ b/src/decomon/models/backward_cloning.py @@ -23,6 +23,7 @@ from decomon.layers import DecomonLayer from decomon.layers.convert import to_decomon from decomon.layers.merging.base_merge import DecomonMerge +from decomon.layers.oracle import DecomonOracle from decomon.models.crown import Convert2BackwardMode, Fuse, MergeWithPrevious from decomon.models.utils import ( ensure_functional_model, @@ -274,12 +275,19 @@ def get_oracle( if id(node) in forward_layer_map: # forward oracle forward_layer = forward_layer_map[id(node)] - forward_input: list[keras.KerasTensor] = [] + oracle_layer = DecomonOracle( + perturbation_domain=forward_layer.perturbation_domain, + ibp=forward_layer.ibp, + affine=forward_layer.affine, + layer_input_shape=forward_layer.layer_input_shape, + is_merging_layer=forward_layer.is_merging_layer, + ) + oracle_input: list[keras.KerasTensor] = [] for parent in parents: - forward_input += forward_output_map[id(parent)] - if forward_layer.inputs_outputs_spec.needs_perturbation_domain_inputs(): - forward_input += [perturbation_domain_input] - oracle_bounds = forward_layer.call_oracle(forward_input) + oracle_input += forward_output_map[id(parent)] + if oracle_layer.inputs_outputs_spec.needs_perturbation_domain_inputs(): + oracle_input += [perturbation_domain_input] + oracle_bounds = oracle_layer(oracle_input) else: # crown oracle @@ -309,11 +317,15 @@ def get_oracle( crown_output_map[id(parent)] = crown_bounds_parent crown_bounds += crown_bounds_parent - # deduce oracle bounds from - # - affine bounds on keras (sub)model inputs and - # - corresponding perturbation domain input - backward_oracle_inputs = crown_bounds + [perturbation_domain_input] - oracle_bounds = backward_layer.call_oracle(backward_oracle_inputs) + oracle_input = crown_bounds + [perturbation_domain_input] + oracle_layer = DecomonOracle( + perturbation_domain=backward_layer.perturbation_domain, + ibp=False, + affine=True, + layer_input_shape=backward_layer.layer_input_shape, + is_merging_layer=backward_layer.is_merging_layer, + ) # crown bounds contains only affine bounds => ibp=False, affine=True + oracle_bounds = oracle_layer(oracle_input) # store oracle oracle_map[id(node)] = oracle_bounds diff --git a/tests/test_oracle.py b/tests/test_oracle.py new file mode 100644 index 00000000..78f65031 --- /dev/null +++ b/tests/test_oracle.py @@ -0,0 +1,92 @@ +from typing import TypeVar + +import pytest +from pytest_cases import parametrize + +from decomon.core import Propagation +from decomon.layers.oracle import DecomonOracle + +T = TypeVar("T") + + +def double_input(input: T) -> list[T]: + return [input] * 2 + + +@parametrize("is_merging_layer", [False, True]) +def test_decomon_oracle( + ibp, + affine, + propagation, + perturbation_domain, + is_merging_layer, + input_shape, + empty, + batchsize, + simple_decomon_symbolic_input_fn, + simple_decomon_input_fn, + simple_keras_model_input_fn, + simple_keras_layer_input_fn, + helpers, +): + if propagation == Propagation.BACKWARD: + pytest.skip( + "We skip backward propagation as we need same inputs for DecomonOracle as for a forward layer, even for crown." + ) + if empty: + pytest.skip("Empty inputs without meaning for DecomonOracle.") + + output_shape = None + linear = False + + decomon_symbolic_input = simple_decomon_symbolic_input_fn(output_shape, linear) + keras_model_input = simple_keras_model_input_fn() + keras_layer_input = simple_keras_layer_input_fn(keras_model_input) + decomon_input = simple_decomon_input_fn( + keras_model_input=keras_model_input, + keras_layer_input=keras_layer_input, + output_shape=output_shape, + linear=linear, + ) + + if is_merging_layer: + decomon_symbolic_input = helpers.generate_merging_decomon_input_from_single_decomon_inputs( + decomon_inputs=double_input(decomon_symbolic_input), + ibp=ibp, + affine=affine, + propagation=propagation, + linear=linear, + ) + decomon_input = helpers.generate_merging_decomon_input_from_single_decomon_inputs( + decomon_inputs=double_input(decomon_input), + ibp=ibp, + affine=affine, + propagation=propagation, + linear=linear, + ) + input_shape = double_input(input_shape) + + layer = DecomonOracle( + perturbation_domain=perturbation_domain, + ibp=ibp, + affine=affine, + is_merging_layer=is_merging_layer, + layer_input_shape=input_shape, + ) + symbolic_output = layer(decomon_symbolic_input) + output = layer(decomon_input) + + # check shapes + if is_merging_layer: + output_shape = [[t.shape for t in output_i] for output_i in output] + expected_output_shape = [ + helpers.replace_none_by_batchsize(shapes=[t.shape for t in symbolic_output_i], batchsize=batchsize) + for symbolic_output_i in symbolic_output + ] + assert output_shape == expected_output_shape + + else: + output_shape = [t.shape for t in output] + expected_output_shape = [t.shape for t in symbolic_output] + expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) + assert output_shape == expected_output_shape From a206727949ae1ceeed7015801d6370a7ecaa196e Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 1 Mar 2024 15:30:10 +0100 Subject: [PATCH 069/101] Add the name "perturbation_domain_input" to the input generated during clone() This clarifies the resulting graph. --- src/decomon/models/convert.py | 7 ++++++- src/decomon/models/utils.py | 5 ++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index 123d5401..30cbc1de 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -197,6 +197,9 @@ def clone( Returns: """ + # Store model name (before converting to functional) + model_name = model.name + # Check hypotheses: functional model + 1 flattened input model = ensure_functional_model(model) if len(model.inputs) > 1: @@ -217,7 +220,9 @@ def clone( backward_bounds = preprocess_backward_bounds(backward_bounds=backward_bounds, nb_model_outputs=len(model.outputs)) - perturbation_domain_input = generate_perturbation_domain_input(model=model, perturbation_domain=perturbation_domain) + perturbation_domain_input = generate_perturbation_domain_input( + model=model, perturbation_domain=perturbation_domain, name=f"perturbation_domain_input_{model_name}" + ) output = convert( model=model, diff --git a/src/decomon/models/utils.py b/src/decomon/models/utils.py index 91b0b743..06472caa 100644 --- a/src/decomon/models/utils.py +++ b/src/decomon/models/utils.py @@ -28,14 +28,13 @@ def generate_perturbation_domain_input( - model: Model, - perturbation_domain: PerturbationDomain, + model: Model, perturbation_domain: PerturbationDomain, name: str = "perturbation_domain_input" ) -> keras.KerasTensor: model_input_shape = model.inputs[0].shape[1:] dtype = model.inputs[0].dtype input_shape_x = perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape) - return Input(shape=input_shape_x, dtype=dtype) + return Input(shape=input_shape_x, dtype=dtype, name=name) def prepare_inputs_for_layer( From d43169273ae0e36538b8e8583b1edaa87001915b Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 1 Mar 2024 21:57:39 +0100 Subject: [PATCH 070/101] Add a layer to reduce crown bounds coming from parent nodes of a merging layer --- src/decomon/core.py | 42 ---------- src/decomon/layers/crown.py | 105 +++++++++++++++++++++++++ src/decomon/layers/oracle.py | 2 +- src/decomon/models/backward_cloning.py | 4 +- 4 files changed, 109 insertions(+), 44 deletions(-) create mode 100644 src/decomon/layers/crown.py diff --git a/src/decomon/core.py b/src/decomon/core.py index b93d83d3..708ed7e1 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -780,48 +780,6 @@ def is_wo_batch_bounds_shape( else: return len(b_shape) == len(self.model_output_shape) - def sum_backward_bounds(self, affine_bounds_list: list[list[Tensor]]): - """Reduce a list of partial affine bounds on model output w.r.t a same model input by summing them. - - The complication come from the fact that some bounds can be empty, diag or w/o batchsize. - - Args: - affine_bounds_list: - - Returns: - - """ - if self.propagation != Propagation.BACKWARD: - raise NotImplementedError() - if len(affine_bounds_list) == 0: - raise ValueError("affine_bounds_list should not be empty") - - affine_bounds = affine_bounds_list[0] - for affine_bounds_i in affine_bounds_list[1:]: - # identity: put on diag + w/o batchsize form to be able to sum - missing_batchsize = (self.is_wo_batch_bounds(affine_bounds), self.is_wo_batch_bounds(affine_bounds_i)) - diagonal = (self.is_diagonal_bounds(affine_bounds), self.is_diagonal_bounds(affine_bounds_i)) - if len(affine_bounds) == 0: - w = K.ones(self.model_output_shape) - b = K.zeros(self.model_output_shape) - w_l, b_l, w_u, b_u = w, b, w, b - else: - w_l, b_l, w_u, b_u = affine_bounds - if len(affine_bounds_i) == 0: - w = K.ones(self.model_output_shape) - b = K.zeros(self.model_output_shape) - w_l_i, b_l_i, w_u_i, b_u_i = w, b, w, b - else: - w_l_i, b_l_i, w_u_i, b_u_i = affine_bounds_i - w_l = add_tensors(w_l, w_l_i, missing_batchsize=missing_batchsize, diagonal=diagonal) - w_u = add_tensors(w_u, w_u_i, missing_batchsize=missing_batchsize, diagonal=diagonal) - b_l = add_tensors(b_l, b_l_i, missing_batchsize=missing_batchsize) - b_u = add_tensors(b_u, b_u_i, missing_batchsize=missing_batchsize) - - affine_bounds = w_l, b_l, w_u, b_u - - return affine_bounds - def get_kerasinputshape(self, inputsformode: list[Tensor]) -> tuple[Optional[int], ...]: return inputsformode[-1].shape diff --git a/src/decomon/layers/crown.py b/src/decomon/layers/crown.py new file mode 100644 index 00000000..4258d1c8 --- /dev/null +++ b/src/decomon/layers/crown.py @@ -0,0 +1,105 @@ +"""Layers needed by crown algorithm.""" + + +from typing import Any, Optional, Union, overload + +import keras +import keras.ops as K +from keras.layers import Layer + +from decomon.core import InputsOutputsSpec, PerturbationDomain, Propagation +from decomon.keras_utils import add_tensors +from decomon.types import BackendTensor + + +class ReduceCrownBounds(Layer): + """Layer reducing crown bounds computed from parent nodes of a merging layer. + + When encoutering a merging layer, crown algorithm propagates crown bounds for each parent. + As keras models are supposed to have a single input, all resulting crown bounds are affine bounds + on the same keras model output w.r.t this same single keras input. + Thus we can reduce them to a single crown bound by simply summing each component (lower/upper weight/bias). + + NB: We need to take into account the fact that some bounds can be empty, diagonal, and/or w/o batchsize. + + """ + + def __init__( + self, + model_output_shape: tuple[int, ...], + **kwargs: Any, + ): + """ + Args: + model_output_shape: shape of the model output for which the crown bounds have been computed + **kwargs: + + """ + super().__init__(**kwargs) + self.model_output_shape = model_output_shape + + def call(self, inputs: list[list[BackendTensor]]) -> list[BackendTensor]: + """Reduce the list of crown bounds to a single one by summation. + + Args: + inputs: list of crown bounds of the form [[w_l[i], b_l[i], w_u[i], b_u[i]]_i] + + Returns: + [w_l_tot, b_l_tot, w_u_tot, b_u_tot]: a single crown bound + + """ + if len(inputs) == 0: + raise ValueError("inputs should not be empty") + + affine_bounds = inputs[0] + for i in range(1, len(inputs)): + affine_bounds_i = inputs[i] + # identity: put on diag + w/o batchsize form to be able to sum + missing_batchsize = ( + self.inputs_outputs_spec.is_wo_batch_bounds(affine_bounds, 0), + self.inputs_outputs_spec.is_wo_batch_bounds(affine_bounds_i, i), + ) + diagonal = ( + self.inputs_outputs_spec.is_diagonal_bounds(affine_bounds, 0), + self.inputs_outputs_spec.is_diagonal_bounds(affine_bounds_i, i), + ) + if len(affine_bounds) == 0: + w = K.ones(self.model_output_shape) + b = K.zeros(self.model_output_shape) + w_l, b_l, w_u, b_u = w, b, w, b + else: + w_l, b_l, w_u, b_u = affine_bounds + if len(affine_bounds_i) == 0: + w = K.ones(self.model_output_shape) + b = K.zeros(self.model_output_shape) + w_l_i, b_l_i, w_u_i, b_u_i = w, b, w, b + else: + w_l_i, b_l_i, w_u_i, b_u_i = affine_bounds_i + w_l = add_tensors(w_l, w_l_i, missing_batchsize=missing_batchsize, diagonal=diagonal) + w_u = add_tensors(w_u, w_u_i, missing_batchsize=missing_batchsize, diagonal=diagonal) + b_l = add_tensors(b_l, b_l_i, missing_batchsize=missing_batchsize) + b_u = add_tensors(b_u, b_u_i, missing_batchsize=missing_batchsize) + + affine_bounds = w_l, b_l, w_u, b_u + return affine_bounds + + def build(self, input_shape: list[list[tuple[Optional[int], ...]]]) -> None: + self.nb_keras_inputs = len(input_shape) + self.inputs_outputs_spec = InputsOutputsSpec( + ibp=False, + affine=True, + propagation=Propagation.BACKWARD, + is_merging_layer=True, + model_output_shape=self.model_output_shape, + layer_input_shape=[tuple()] * self.nb_keras_inputs, + ) + self.built = True + + def compute_output_shape( + self, + input_shape: list[list[tuple[Optional[int], ...]]], + ) -> list[tuple[Optional[int], ...]]: + """Compute output shape in case of symbolic call.""" + if len(input_shape) == 0: + raise ValueError("inputs should not be empty") + return input_shape[0] diff --git a/src/decomon/layers/oracle.py b/src/decomon/layers/oracle.py index 76846d77..b1df38ff 100644 --- a/src/decomon/layers/oracle.py +++ b/src/decomon/layers/oracle.py @@ -113,7 +113,7 @@ def call(self, inputs: list[BackendTensor]) -> Union[list[BackendTensor], list[l def compute_output_shape( self, - input_shape: tuple[Optional[int], ...], + input_shape: list[tuple[Optional[int], ...]], ) -> Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]]: """Compute output shape in case of symbolic call.""" if self.is_merging_layer: diff --git a/src/decomon/models/backward_cloning.py b/src/decomon/models/backward_cloning.py index 47115bb2..7fe8823c 100644 --- a/src/decomon/models/backward_cloning.py +++ b/src/decomon/models/backward_cloning.py @@ -22,6 +22,7 @@ ) from decomon.layers import DecomonLayer from decomon.layers.convert import to_decomon +from decomon.layers.crown import ReduceCrownBounds from decomon.layers.merging.base_merge import DecomonMerge from decomon.layers.oracle import DecomonOracle from decomon.models.crown import Convert2BackwardMode, Fuse, MergeWithPrevious @@ -191,7 +192,8 @@ def crown( # reduce by summing all bounds together # (indeed all bounds are partial affine bounds on model output w.r.t the same model input # under the hypotheses of a single model input) - crown_bounds = backward_layer.inputs_outputs_spec.sum_backward_bounds(crown_bounds_list) + reducing_layer = ReduceCrownBounds(model_output_shape=model_output_shape) + crown_bounds = reducing_layer(crown_bounds_list) elif len(parents) > 1: raise RuntimeError("Node with multiple parents should have been converted to a DecomonMerge layer.") From cf1b96191a9602315b5897abb0b377407d754dab Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 4 Mar 2024 14:48:36 +0100 Subject: [PATCH 071/101] Update BallDomain - update get_lower_ball() / get_upper_ball() for unflattened x - add BallDomain.get_lower_x() and get_upper_x() NB: not tested for now... --- src/decomon/core.py | 64 ++++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/src/decomon/core.py b/src/decomon/core.py index 708ed7e1..97a3a723 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -162,6 +162,12 @@ def get_upper(self, x: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: def get_nb_x_components(self) -> int: return 1 + def get_lower_x(self, x: Tensor) -> Tensor: + return x - self.eps + + def get_upper_x(self, x: Tensor) -> Tensor: + return x + self.eps + class ForwardMode(str, Enum): """The different forward (from input to output) linear based relaxation perturbation analysis.""" @@ -1099,7 +1105,7 @@ def get_lower_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: return get_upper_box(x_min=x_max, x_max=x_min, w=w, b=b, **kwargs) -def get_lq_norm(x: Tensor, p: float, axis: int = -1) -> Tensor: +def get_lq_norm(x: Tensor, p: float, axis: Union[int, list[int]] = -1) -> Tensor: """compute Lp norm (p=1 or 2) Args: @@ -1133,9 +1139,6 @@ def get_upper_ball(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kw Returns: max_(|x - x_0|_p<= eps) w*x + b """ - if len(w.shape) == len(b.shape): - raise NotImplementedError - if p == np.inf: # compute x_min and x_max according to eps x_min = x_0 - eps @@ -1143,18 +1146,29 @@ def get_upper_ball(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kw return get_upper_box(x_min, x_max, w, b) else: - # use Holder's inequality p+q=1 - # ||w||_q*eps + w*x_0 + b - if len(kwargs): return get_upper_ball_finetune(x_0, eps, p, w, b, **kwargs) - upper = eps * get_lq_norm(w, p, axis=1) + b + # use Holder's inequality p+q=1 + # ||w||_q*eps + w*x_0 + b - for _ in range(len(w.shape) - len(x_0.shape)): - x_0 = K.expand_dims(x_0, -1) + is_diag = w.shape == b.shape + is_wo_batch = len(b.shape) < len(x_0.shape) - return K.sum(w * x_0, 1) + upper + # lq-norm of w + if is_diag: + w_q = K.abs(w) + else: + nb_axes_wo_batchsize_x = len(x_0.shape) - 1 + if is_wo_batch: + reduced_axes = list(range(nb_axes_wo_batchsize_x)) + else: + reduced_axes = list(range(1, 1 + nb_axes_wo_batchsize_x)) + w_q = get_lq_norm(w, p, axis=reduced_axes) + + diagonal = (False, is_diag) + missing_batchsize = (False, is_wo_batch) + return batch_multid_dot(x_0, w, diagonal=diagonal, missing_batchsize=missing_batchsize) + b + w_q * eps def get_lower_ball(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: @@ -1170,9 +1184,6 @@ def get_lower_ball(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kw Returns: min_(|x - x_0|_p<= eps) w*x + b """ - if len(w.shape) == len(b.shape): - return x_0 - eps - if p == np.inf: # compute x_min and x_max according to eps x_min = x_0 - eps @@ -1180,18 +1191,29 @@ def get_lower_ball(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kw return get_lower_box(x_min, x_max, w, b) else: - # use Holder's inequality p+q=1 - # ||w||_q*eps + w*x_0 + b - if len(kwargs): return get_lower_ball_finetune(x_0, eps, p, w, b, **kwargs) - lower = -eps * get_lq_norm(w, p, axis=1) + b + # use Holder's inequality p+q=1 + # - ||w||_q*eps + w*x_0 + b + + is_diag = w.shape == b.shape + is_wo_batch = len(b.shape) < len(x_0.shape) - for _ in range(len(w.shape) - len(x_0.shape)): - x_0 = K.expand_dims(x_0, -1) + # lq-norm of w + if is_diag: + w_q = K.abs(w) + else: + nb_axes_wo_batchsize_x = len(x_0.shape) - 1 + if is_wo_batch: + reduced_axes = list(range(nb_axes_wo_batchsize_x)) + else: + reduced_axes = list(range(1, 1 + nb_axes_wo_batchsize_x)) + w_q = get_lq_norm(w, p, axis=reduced_axes) - return K.sum(w * x_0, 1) + lower + diagonal = (False, is_diag) + missing_batchsize = (False, is_wo_batch) + return batch_multid_dot(x_0, w, diagonal=diagonal, missing_batchsize=missing_batchsize) + b - w_q * eps def get_lower_ball_finetune(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: From 7ce1f565d6b75c764aaed7ce63dd42a8fbaa11d5 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 8 Mar 2024 14:49:17 +0100 Subject: [PATCH 072/101] Fix PerturbationDomain for affine bounds w/o batchsize The way we checked for affine bounds w/o batchsize was wrong (we compared b and x shape although b is of the output shape and x of the input shape). We need actually a specific arg to specify whether w and b are missing batchsize or not. We also add doctrings to methods for perturbation domains. --- src/decomon/core.py | 143 +++++++++++++++++++++++++++++++++----------- 1 file changed, 109 insertions(+), 34 deletions(-) diff --git a/src/decomon/core.py b/src/decomon/core.py index 97a3a723..9744cad5 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -31,22 +31,55 @@ def __init__(self, opt_option: Union[str, Option] = Option.milp): @abstractmethod def get_upper_x(self, x: Tensor) -> Tensor: + """Get upper constant bound on perturbation domain input.""" ... @abstractmethod def get_lower_x(self, x: Tensor) -> Tensor: + """Get lower constant bound on perturbation domain input.""" ... @abstractmethod - def get_upper(self, x: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: + def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + """Merge upper affine bounds with perturbation domain input to get upper constant bound. + + Args: + x: perturbation domain input + w: weights of the affine bound + b: bias of the affine bound + missing_batchsize: whether w and b are missing batchsize + **kwargs: + + Returns: + + """ ... @abstractmethod - def get_lower(self, x: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: + def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + """Merge lower affine bounds with perturbation domain input to get lower constant bound. + + Args: + x: perturbation domain input + w: weights of the affine bound + b: bias of the affine bound + missing_batchsize: whether w and b are missing batchsize + **kwargs: + + Returns: + + """ ... @abstractmethod def get_nb_x_components(self) -> int: + """Get the number of components in perturabation domain input. + + For instance: + - box domain: each corner of the box -> 2 components + - ball domain: center of the ball -> 1 component + + """ ... @abstractmethod @@ -83,13 +116,15 @@ def get_kerasinputlike_from_x(self, x: Tensor) -> Tensor: return x[:, 0] def get_x_input_shape_wo_batchsize(self, original_input_shape: tuple[int, ...]) -> tuple[int, ...]: + """Get expected perturbation domain input shape, excepting the batch axis.""" n_comp_x = self.get_nb_x_components() if n_comp_x == 1: return original_input_shape else: return (n_comp_x,) + original_input_shape - def get_keras_input_shape_wo_batchsize(self, x_shape: Tuple[int, ...]) -> Tuple[int, ...]: + def get_keras_input_shape_wo_batchsize(self, x_shape: tuple[int, ...]) -> tuple[int, ...]: + """Deduce keras model input shape from perturbation domain input shape.""" n_comp_x = self.get_nb_x_components() if n_comp_x == 1: return x_shape @@ -98,15 +133,15 @@ def get_keras_input_shape_wo_batchsize(self, x_shape: Tuple[int, ...]) -> Tuple[ class BoxDomain(PerturbationDomain): - def get_upper(self, x: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: + def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: x_min = x[:, 0] x_max = x[:, 1] - return get_upper_box(x_min=x_min, x_max=x_max, w=w, b=b, **kwargs) + return get_upper_box(x_min=x_min, x_max=x_max, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) - def get_lower(self, x: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: + def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: x_min = x[:, 0] x_max = x[:, 1] - return get_lower_box(x_min=x_min, x_max=x_max, w=w, b=b, **kwargs) + return get_lower_box(x_min=x_min, x_max=x_max, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) def get_upper_x(self, x: Tensor) -> Tensor: return x[:, 1] @@ -153,11 +188,11 @@ def get_config(self) -> dict[str, Any]: ) return config - def get_lower(self, x: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: - return get_lower_ball(x_0=x, eps=self.eps, p=self.p, w=w, b=b, **kwargs) + def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + return get_lower_ball(x_0=x, eps=self.eps, p=self.p, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) - def get_upper(self, x: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: - return get_upper_ball(x_0=x, eps=self.eps, p=self.p, w=w, b=b, **kwargs) + def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + return get_upper_ball(x_0=x, eps=self.eps, p=self.p, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) def get_nb_x_components(self) -> int: return 1 @@ -703,12 +738,14 @@ def has_multiple_affine_inputs(self) -> bool: return self.propagation == Propagation.FORWARD and self.affine and self.is_merging_layer @overload - def extract_shapes_from_affine_bounds(self, affine_bounds: list[Tensor]) -> list[tuple[Optional[int], ...]]: + def extract_shapes_from_affine_bounds( + self, affine_bounds: list[Tensor], i: int = -1 + ) -> list[tuple[Optional[int], ...]]: ... @overload def extract_shapes_from_affine_bounds( - self, affine_bounds: list[list[Tensor]] + self, affine_bounds: list[list[Tensor]], i: int = -1 ) -> list[list[tuple[Optional[int], ...]]]: ... @@ -786,6 +823,29 @@ def is_wo_batch_bounds_shape( else: return len(b_shape) == len(self.model_output_shape) + @overload + def is_wo_batch_bounds_by_keras_input( + self, + affine_bounds: list[Tensor], + ) -> bool: + ... + + @overload + def is_wo_batch_bounds_by_keras_input( + self, + affine_bounds: list[list[Tensor]], + ) -> list[bool]: + ... + + def is_wo_batch_bounds_by_keras_input( + self, + affine_bounds: Union[list[Tensor], list[list[Tensor]]], + ) -> Union[bool, list[bool]]: + if self.has_multiple_affine_inputs(): + return [self.is_wo_batch_bounds(affine_bounds_i, i=i) for i, affine_bounds_i in enumerate(affine_bounds)] + else: + return self.is_wo_batch_bounds(affine_bounds) + def get_kerasinputshape(self, inputsformode: list[Tensor]) -> tuple[Optional[int], ...]: return inputsformode[-1].shape @@ -1052,7 +1112,7 @@ def get_empty_tensor(dtype: Optional[str] = None) -> Tensor: return K.convert_to_tensor([], dtype=dtype) -def get_upper_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: +def get_upper_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: """Compute the max of an affine function within a box (hypercube) defined by its extremal corners @@ -1061,6 +1121,7 @@ def get_upper_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: x_max: upper bound of the box domain w: weights of the affine function b: bias of the affine function + missing_batchsize: whether w and b are missing the batchsize Returns: max_(x >= x_min, x<=x_max) w*x + b @@ -1075,9 +1136,8 @@ def get_upper_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: w_neg = K.minimum(w, z_value) is_diag = w.shape == b.shape - is_wo_batch = len(b.shape) < len(x_min.shape) diagonal = (False, is_diag) - missing_batchsize = (False, is_wo_batch) + missing_batchsize = (False, missing_batchsize) return ( batch_multid_dot(x_max, w_pos, diagonal=diagonal, missing_batchsize=missing_batchsize) @@ -1086,13 +1146,14 @@ def get_upper_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: ) -def get_lower_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: +def get_lower_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: """ Args: x_min: lower bound of the box domain x_max: upper bound of the box domain - w_l: weights of the affine lower bound - b_l: bias of the affine lower bound + w: weights of the affine lower bound + b: bias of the affine lower bound + missing_batchsize: whether w and b are missing the batchsize Returns: min_(x >= x_min, x<=x_max) w*x + b @@ -1102,7 +1163,7 @@ def get_lower_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, **kwargs: We assume that x_min, x_max have always its batch axis. """ - return get_upper_box(x_min=x_max, x_max=x_min, w=w, b=b, **kwargs) + return get_upper_box(x_min=x_max, x_max=x_min, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) def get_lq_norm(x: Tensor, p: float, axis: Union[int, list[int]] = -1) -> Tensor: @@ -1126,7 +1187,9 @@ def get_lq_norm(x: Tensor, p: float, axis: Union[int, list[int]] = -1) -> Tensor return x_q -def get_upper_ball(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: +def get_upper_ball( + x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any +) -> Tensor: """max of an affine function over an Lp ball Args: @@ -1135,6 +1198,7 @@ def get_upper_ball(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kw p: the type of Lp norm considered w: weights of the affine function b: bias of the affine function + missing_batchsize: whether w and b are missing the batchsize Returns: max_(|x - x_0|_p<= eps) w*x + b @@ -1143,35 +1207,36 @@ def get_upper_ball(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kw # compute x_min and x_max according to eps x_min = x_0 - eps x_max = x_0 + eps - return get_upper_box(x_min, x_max, w, b) + return get_upper_box(x_min, x_max, w, b, missing_batchsize=missing_batchsize) else: if len(kwargs): - return get_upper_ball_finetune(x_0, eps, p, w, b, **kwargs) + return get_upper_ball_finetune(x_0, eps, p, w, b, missing_batchsize=missing_batchsize, **kwargs) # use Holder's inequality p+q=1 # ||w||_q*eps + w*x_0 + b is_diag = w.shape == b.shape - is_wo_batch = len(b.shape) < len(x_0.shape) # lq-norm of w if is_diag: w_q = K.abs(w) else: nb_axes_wo_batchsize_x = len(x_0.shape) - 1 - if is_wo_batch: + if missing_batchsize: reduced_axes = list(range(nb_axes_wo_batchsize_x)) else: reduced_axes = list(range(1, 1 + nb_axes_wo_batchsize_x)) w_q = get_lq_norm(w, p, axis=reduced_axes) diagonal = (False, is_diag) - missing_batchsize = (False, is_wo_batch) + missing_batchsize = (False, missing_batchsize) return batch_multid_dot(x_0, w, diagonal=diagonal, missing_batchsize=missing_batchsize) + b + w_q * eps -def get_lower_ball(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: +def get_lower_ball( + x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any +) -> Tensor: """min of an affine fucntion over an Lp ball Args: @@ -1180,6 +1245,7 @@ def get_lower_ball(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kw p: the type of Lp norm considered w: weights of the affine function b: bias of the affine function + missing_batchsize: whether w and b are missing the batchsize Returns: min_(|x - x_0|_p<= eps) w*x + b @@ -1188,35 +1254,39 @@ def get_lower_ball(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kw # compute x_min and x_max according to eps x_min = x_0 - eps x_max = x_0 + eps - return get_lower_box(x_min, x_max, w, b) + return get_lower_box(x_min, x_max, w, b, missing_batchsize=missing_batchsize) else: if len(kwargs): - return get_lower_ball_finetune(x_0, eps, p, w, b, **kwargs) + return get_lower_ball_finetune(x_0, eps, p, w, b, missing_batchsize=missing_batchsize, **kwargs) # use Holder's inequality p+q=1 # - ||w||_q*eps + w*x_0 + b is_diag = w.shape == b.shape - is_wo_batch = len(b.shape) < len(x_0.shape) # lq-norm of w if is_diag: w_q = K.abs(w) else: nb_axes_wo_batchsize_x = len(x_0.shape) - 1 - if is_wo_batch: + if missing_batchsize: reduced_axes = list(range(nb_axes_wo_batchsize_x)) else: reduced_axes = list(range(1, 1 + nb_axes_wo_batchsize_x)) w_q = get_lq_norm(w, p, axis=reduced_axes) diagonal = (False, is_diag) - missing_batchsize = (False, is_wo_batch) + missing_batchsize = (False, missing_batchsize) return batch_multid_dot(x_0, w, diagonal=diagonal, missing_batchsize=missing_batchsize) + b - w_q * eps -def get_lower_ball_finetune(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: +def get_lower_ball_finetune( + x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any +) -> Tensor: + if missing_batchsize: + raise NotImplementedError() + if "finetune_lower" in kwargs and "upper" in kwargs or "lower" in kwargs: alpha = kwargs["finetune_lower"] # assume alpha is the same shape as w, minus the batch dimension @@ -1267,7 +1337,12 @@ def get_lower_ball_finetune(x_0: Tensor, eps: float, p: float, w: Tensor, b: Ten return get_lower_ball(x_0, eps, p, w, b) -def get_upper_ball_finetune(x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, **kwargs: Any) -> Tensor: +def get_upper_ball_finetune( + x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any +) -> Tensor: + if missing_batchsize: + raise NotImplementedError() + if "finetune_upper" in kwargs and "upper" in kwargs or "lower" in kwargs: alpha = kwargs["finetune_upper"] # assume alpha is the same shape as w, minus the batch dimension From 1885d2540b21cbdc7763e1600919e485f2e0eca4 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 8 Mar 2024 20:07:56 +0100 Subject: [PATCH 073/101] Fix checks for identity, diag and nobatch in inputs_outputs_specs --- src/decomon/core.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/decomon/core.py b/src/decomon/core.py index 9744cad5..21667394 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -734,8 +734,8 @@ def flatten_outputs_shape( """Flatten decomon outputs shape.""" return self.flatten_outputs(affine_bounds_propagated=affine_bounds_propagated_shape, constant_bounds_propagated=constant_bounds_propagated_shape) # type: ignore - def has_multiple_affine_inputs(self) -> bool: - return self.propagation == Propagation.FORWARD and self.affine and self.is_merging_layer + def has_multiple_bounds_inputs(self) -> bool: + return self.propagation == Propagation.FORWARD and self.is_merging_layer @overload def extract_shapes_from_affine_bounds( @@ -752,7 +752,7 @@ def extract_shapes_from_affine_bounds( def extract_shapes_from_affine_bounds( self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], i: int = -1 ) -> Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]]: - if self.has_multiple_affine_inputs() and i == -1: + if self.has_multiple_bounds_inputs() and i == -1: return [[t.shape for t in sub_bounds] for sub_bounds in affine_bounds] else: return [t.shape for t in affine_bounds] # type: ignore @@ -767,7 +767,7 @@ def is_identity_bounds_shape( affine_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], i: int = -1, ) -> bool: - if self.has_multiple_affine_inputs() and i == -1: + if self.has_multiple_bounds_inputs() and i == -1: return all( self.is_identity_bounds_shape(affine_bounds_shape=affine_bounds_shape[i], i=i) # type: ignore for i in range(self.nb_keras_inputs) @@ -785,13 +785,13 @@ def is_diagonal_bounds_shape( affine_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], i: int = -1, ) -> bool: - if self.has_multiple_affine_inputs() and i == -1: + if self.has_multiple_bounds_inputs() and i == -1: return all( self.is_diagonal_bounds_shape(affine_bounds_shape=affine_bounds_shape[i], i=i) # type: ignore for i in range(self.nb_keras_inputs) ) else: - if self.is_identity_bounds_shape(affine_bounds_shape): + if self.is_identity_bounds_shape(affine_bounds_shape, i=i): return True w_shape, b_shape = affine_bounds_shape[:2] return w_shape == b_shape @@ -806,13 +806,13 @@ def is_wo_batch_bounds_shape( affine_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], i: int = -1, ) -> bool: - if self.has_multiple_affine_inputs() and i == -1: + if self.has_multiple_bounds_inputs() and i == -1: return all( self.is_wo_batch_bounds_shape(affine_bounds_shape=affine_bounds_shape[i], i=i) # type: ignore for i in range(self.nb_keras_inputs) ) else: - if self.is_identity_bounds_shape(affine_bounds_shape): + if self.is_identity_bounds_shape(affine_bounds_shape, i=i): return True b_shape = affine_bounds_shape[1] if self.propagation == Propagation.FORWARD: @@ -841,7 +841,7 @@ def is_wo_batch_bounds_by_keras_input( self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], ) -> Union[bool, list[bool]]: - if self.has_multiple_affine_inputs(): + if self.has_multiple_bounds_inputs(): return [self.is_wo_batch_bounds(affine_bounds_i, i=i) for i, affine_bounds_i in enumerate(affine_bounds)] else: return self.is_wo_batch_bounds(affine_bounds) From 3708beb797cb4e4a4bd4f5773d858314aeb868eb Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 4 Mar 2024 17:08:42 +0100 Subject: [PATCH 074/101] Add a Fuse layer to combine affine (and ibp) bounds on successive models - Put combine_affine_bounds in Fuse layer module - Create combine_affine_bound_with_ibp_bound to be used by fuse layer but also for forward ibp propagation. - fix get_forward_oracle to specify if bounds are missing batchsize or not - tests on various types of inputs --- src/decomon/core.py | 8 + src/decomon/layers/fuse.py | 579 +++++++++++++++++++++++ src/decomon/layers/layer.py | 304 +----------- src/decomon/layers/merging/base_merge.py | 3 +- src/decomon/layers/oracle.py | 19 +- tests/conftest.py | 8 +- tests/test_fuse.py | 298 ++++++++++++ 7 files changed, 922 insertions(+), 297 deletions(-) create mode 100644 src/decomon/layers/fuse.py create mode 100644 tests/test_fuse.py diff --git a/src/decomon/core.py b/src/decomon/core.py index 21667394..c9c33959 100644 --- a/src/decomon/core.py +++ b/src/decomon/core.py @@ -687,6 +687,14 @@ def split_outputs(self, outputs: list[Tensor]) -> tuple[Union[list[Tensor], list return affine_bounds_propagated, constant_bounds_propagated + def split_output_shape( + self, output_shape: list[tuple[Optional[int], ...]] + ) -> tuple[ + Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], list[tuple[Optional[int], ...]] + ]: + """Split decomon output shape.""" + return self.split_outputs(outputs=output_shape) # type: ignore + def flatten_outputs( self, affine_bounds_propagated: Union[list[Tensor], list[list[Tensor]]], diff --git a/src/decomon/layers/fuse.py b/src/decomon/layers/fuse.py new file mode 100644 index 00000000..0b166e06 --- /dev/null +++ b/src/decomon/layers/fuse.py @@ -0,0 +1,579 @@ +"""Layers specifying constant oracle bounds on keras layer input.""" + + +from typing import Any, Optional, Union, overload + +import keras +from keras import ops as K +from keras.layers import Layer + +from decomon.core import ( + BoxDomain, + InputsOutputsSpec, + PerturbationDomain, + Propagation, + get_lower_box, + get_upper_box, +) +from decomon.keras_utils import add_tensors, batch_multid_dot +from decomon.types import BackendTensor, Tensor + + +class Fuse(Layer): + """Layer combining bounds on successive models. + + We merge (ibp and/or affine) bounds on 2 models which are supposed to be chained. + Both models can or not have both types of bounds + + We have, for any x in the given perturbation domain: + + m(x) = m2(m1(x)) + + w_l_1 * x + b_l_1 <= m1(x) <= w_u_1 * x + b_u_1 + l_c_1 <= m1(x) <= u_c_1 + + w_l_2 * m1(x) + b_l_2 <= m2(m1(x)) <= w_u_2 * m1(x) + b_u_2 + l_c_2 <= m2(m1(x)) <= u_c_2 + + and we will deduce [w_l, b_l, w_u, b_u, u_c, l_c] such that + + w_l * x + b_l <= m(x) <= w_u * x + b_u + l_c <= m(x) <= u_c + + In the case of multiple outputs for the first model, we assume that each output of m1 is branche to a model + with a single output. That is to say, we have something like this: + + m(x)[i] = m2[i](m1(x)[i]) + + i.e. i-th output of m(x) is the i-th output of m1(x) chained with a model m2[i] with single output. + + Hypothesis: the first model has only a single input. + + """ + + def __init__( + self, + ibp_1: bool, + affine_1: bool, + ibp_2: bool, + affine_2: bool, + m1_input_shape: tuple[int, ...], + m_1_output_shapes: list[tuple[int, ...]], + from_linear_2: list[bool], + **kwargs, + ): + """ + + Args: + ibp_1: specifying if first model constant bounds have been computed + affine_1: specifying if first model affine bounds have been computed + ibp_2: specifying if second model constant bounds have been computed + affine_2: specifying if second model affine bounds have been computed + m1_input_shape: input shape of the first model (w/o batchsize) + m_1_output_shapes: shape of each output of the first model (w/o batchsize) + from_linear_2: specifying if affine bounds for second model are from linear layers + i.e. no bacthsize and w_l == w_u and b_l == b_u + **kwargs: passed to Layer.__init__() + + """ + if not ibp_1 and not affine_1: + raise ValueError("ibp_1 and affine_1 cannot be both False.") + if not ibp_2 and not affine_2: + raise ValueError("ibp_2 and affine_2 cannot be both False.") + if len(m_1_output_shapes) == 0: + raise ValueError("m_1_output_shapes cannot be empty") + if not isinstance(m_1_output_shapes[0], tuple): + raise ValueError("m_1_output_shapes must be a list of shapes (tuple of integers)") + + super().__init__(**kwargs) + + self.m_1_output_shapes = m_1_output_shapes + self.m1_input_shape = m1_input_shape + self.ibp_1 = ibp_1 + self.affine_1 = affine_1 + self.ibp_2 = ibp_2 + self.affine_2 = affine_2 + self.from_linear_2 = from_linear_2 + + self.nb_outputs_first_model = len(m_1_output_shapes) + + self.ibp_fused = self.ibp_2 or (self.ibp_1 and self.affine_2) + self.affine_fused = self.affine_1 and self.affine_2 + + self.inputs_outputs_spec_1 = InputsOutputsSpec(ibp=ibp_1, affine=affine_1, layer_input_shape=m1_input_shape) + self.inputs_outputs_spec_2 = [ + InputsOutputsSpec(ibp=ibp_2, affine=affine_2, layer_input_shape=m2_input_shape) + for m2_input_shape in m_1_output_shapes + ] + + def build(self, input_shape: tuple[list[tuple[Optional[int], ...]], list[tuple[Optional[int], ...]]]) -> None: + input_shape_1, input_shape_2 = input_shape + + # check number of inputs + if len(input_shape_1) != self.inputs_outputs_spec_1.nb_output_tensors * self.nb_outputs_first_model: + raise ValueError( + f"The first input should be a list whose length is the product of " + f"{self.inputs_outputs_spec_1.nb_output_tensors} (number of tensors per bound) " + f"by {self.nb_outputs_first_model} (number of outputs for the first model)." + ) + + if len(input_shape_2) != self.inputs_outputs_spec_2[0].nb_output_tensors * self.nb_outputs_first_model: + raise ValueError( + f"The second input should be a list whose length is the product of" + f"{self.inputs_outputs_spec_2[0].nb_output_tensors} (number of tensors per bound) " + f"by {self.nb_outputs_first_model} (number of outputs for the first model)." + ) + + self.built = True + + def _is_from_linear_m1_ith_affine_bounds(self, affine_bounds: list[Tensor], i: int) -> bool: + return len(affine_bounds) == 0 or affine_bounds[1].shape == self.m_1_output_shapes[i] + + def _is_from_linear_m1_ith_affine_bounds_shape( + self, affine_bounds_shape: list[tuple[Optional[int]]], i: int + ) -> bool: + return len(affine_bounds_shape) == 0 or affine_bounds_shape[1] == self.m_1_output_shapes[i] + + def call(self, inputs: tuple[list[BackendTensor], list[BackendTensor]]) -> list[BackendTensor]: + """Fuse affine bounds. + + Args: + inputs: (sum_{i} affine_bounds_1[i] + constant_bounds_1[i], sum_{i} affine_bounds_2[i] + constant_bounds2[i]) + being the affine and constant bounds on first and second model for each output of the first model, with + - i: the indice of the *first* model output considered + - sum_{i}: the concatenation of subsequent lists over i + - affine_bounds_1[i]: empty if `self.affine_1` is False + - constant_bounds_1[i]: empty if `self.ibp_1` is False + - affine_bounds_2[i]: empty if `self.affine_2` is False + - constant_bounds_2[i]: empty if `self.ibp_2` is False + + Returns: + sum_{i} affine_bounds_fused[i] + constant_bounds_fused[i]: fused affine and constant bounds for each output of the first model + + """ + bounds_1, bounds_2 = inputs + + bounds_fused: list[BackendTensor] = [] + for i in range(self.nb_outputs_first_model): + bounds_1_i = bounds_1[ + i + * self.inputs_outputs_spec_1.nb_output_tensors : (i + 1) + * self.inputs_outputs_spec_1.nb_output_tensors + ] + affine_bounds_1, constant_bounds_1 = self.inputs_outputs_spec_1.split_outputs(bounds_1_i) + + bounds_2_i = bounds_2[ + i + * self.inputs_outputs_spec_2[0].nb_output_tensors : (i + 1) + * self.inputs_outputs_spec_2[0].nb_output_tensors + ] + affine_bounds_2, constant_bounds_2 = self.inputs_outputs_spec_2[0].split_outputs(bounds_2_i) + + # constant bounds + if self.ibp_2: + # ibp bounds already computed in second model + constant_bounds_fused = constant_bounds_2 + elif self.ibp_1 and self.affine_2: + # combine constant bounds on first model with affine bounds on second model + lower, upper = constant_bounds_1 + constant_bounds_fused = list( + combine_affine_bound_with_constant_bound( + lower=lower, + upper=upper, + affine_bounds=affine_bounds_2, + missing_batchsize=self.from_linear_2[i], + ) + ) + else: + constant_bounds_fused = [] + + # affine bounds + if self.affine_1 and self.affine_2: + diagonal = ( + self.inputs_outputs_spec_1.is_diagonal_bounds(affine_bounds_1), + self.inputs_outputs_spec_2[i].is_diagonal_bounds(affine_bounds_2), + ) + from_linear_layer = ( + self._is_from_linear_m1_ith_affine_bounds(affine_bounds=affine_bounds_1, i=i), + self.from_linear_2[i], + ) + affine_bounds_fused = list( + combine_affine_bounds( + affine_bounds_1=affine_bounds_1, + affine_bounds_2=affine_bounds_2, + diagonal=diagonal, + from_linear_layer=from_linear_layer, + ) + ) + else: + affine_bounds_fused = [] + + # concatenate bounds + bounds_fused += affine_bounds_fused + constant_bounds_fused + + return bounds_fused + + def compute_output_shape( + self, input_shape: tuple[list[tuple[Optional[int], ...]], list[tuple[Optional[int], ...]]] + ) -> list[tuple[Optional[int], ...]]: + bounds_1_shape, bounds_2_shape = input_shape + + bounds_fused_shape: list[tuple[int, ...]] = [] + for i in range(self.nb_outputs_first_model): + bounds_1_i_shape = bounds_1_shape[ + i + * self.inputs_outputs_spec_1.nb_output_tensors : (i + 1) + * self.inputs_outputs_spec_1.nb_output_tensors + ] + affine_bounds_1_shape, constant_bounds_1_shape = self.inputs_outputs_spec_1.split_output_shape( + bounds_1_i_shape + ) + + bounds_2_i_shape = bounds_2_shape[ + i + * self.inputs_outputs_spec_2[0].nb_output_tensors : (i + 1) + * self.inputs_outputs_spec_2[0].nb_output_tensors + ] + affine_bounds_2_shape, constant_bounds_2_shape = self.inputs_outputs_spec_2[0].split_output_shape( + bounds_2_i_shape + ) + + # constant bounds + if self.ibp_2: + # ibp bounds already computed in second model + constant_bounds_fused_shape = constant_bounds_2_shape + elif self.ibp_1 and self.affine_2: + # combine constant bounds on first model with affine bounds on second model + _, b2_shape, _, _ = affine_bounds_2_shape + if self.from_linear_2[i]: + lower_fused_shape = (None,) + b2_shape + else: + lower_fused_shape = b2_shape + constant_bounds_fused_shape = [lower_fused_shape, lower_fused_shape] + else: + constant_bounds_fused_shape = [] + + # affine bounds + if self.affine_1 and self.affine_2: + _, b2_shape, _, _ = affine_bounds_2_shape + if self.from_linear_2[i]: + model_2_output_shape_wo_batchisze = b2_shape + else: + model_2_output_shape_wo_batchisze = b2_shape[1:] + + diagonal = self.inputs_outputs_spec_1.is_diagonal_bounds_shape( + affine_bounds_1_shape + ) and self.inputs_outputs_spec_2[i].is_diagonal_bounds_shape(affine_bounds_2_shape) + if diagonal: + w_fused_shape_wo_batchsize = self.m1_input_shape + else: + w_fused_shape_wo_batchsize = self.m1_input_shape + model_2_output_shape_wo_batchisze + + from_linear_layer = ( + self._is_from_linear_m1_ith_affine_bounds_shape(affine_bounds_shape=affine_bounds_1_shape, i=i) + and self.from_linear_2[i] + ) + if from_linear_layer: + w_fused_shape = w_fused_shape_wo_batchsize + b_fused_shape = model_2_output_shape_wo_batchisze + else: + w_fused_shape = (None,) + w_fused_shape_wo_batchsize + b_fused_shape = (None,) + model_2_output_shape_wo_batchisze + + affine_bounds_fused_shape = [w_fused_shape, b_fused_shape, w_fused_shape, b_fused_shape] + else: + affine_bounds_fused_shape = [] + + # concatenate bounds + bounds_fused_shape += affine_bounds_fused_shape + constant_bounds_fused_shape + + return bounds_fused_shape + + +def combine_affine_bounds( + affine_bounds_1: list[Tensor], + affine_bounds_2: list[Tensor], + from_linear_layer: tuple[bool, bool] = (False, False), + diagonal: tuple[bool, bool] = (False, False), +) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Combine affine bounds + + Args: + affine_bounds_1: [w_l_1, b_l_1, w_u_1, b_u_1] first affine bounds + affine_bounds_2: [w_l_2, b_l_2, w_u_2, b_u_2] second affine bounds + from_linear_layer: specify if affine_bounds_1 or affine_bounds_2 + come from the affine representation of a linear layer + diagonal: specify if affine_bounds_1 or affine_bounds_2 + are in diagonal representation + + Returns: + w_l, b_l, w_u, b_u: combined affine bounds + + If x, y, z satisfy + w_l_1 * x + b_l_1 <= y <= w_u_1 * x + b_u_1 + w_l_2 * y + b_l_2 <= z <= w_u_2 * y + b_u_2 + + Then + w_l * x + b_l <= z <= w_u * x + b_u + + + Special cases + + - with linear layers: + + If the affine bounds come from the affine representation of a linear layer (e.g. affine_bounds_1), then + - lower and upper bounds are equal: affine_bounds_1 = [w_1, b_1, w_1, b_1] + - the tensors are missing the batch dimension + + In the generic case, tensors in affine_bounds have their first axis corresponding to the batch size. + + - diagonal representation: + + If w.shape == b.shape, this means that w is represented by its "diagonal" (potentially a tensor-multid). + + - empty affine bounds: + + when one affine bounds is an empty list, this is actually a convention for identity bounds, i.e. + w = identity, b = 0 + therefore we return the other affine_bounds, unchanged. + + """ + # special case: empty bounds <=> identity bounds + if len(affine_bounds_1) == 0: + return tuple(affine_bounds_2) + if len(affine_bounds_2) == 0: + return tuple(affine_bounds_1) + + if from_linear_layer == (False, False): + return _combine_affine_bounds_generic( + affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal + ) + elif from_linear_layer == (True, False): + return _combine_affine_bounds_left_from_linear( + affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal + ) + elif from_linear_layer == (False, True): + return _combine_affine_bounds_right_from_linear( + affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal + ) + else: + return _combine_affine_bounds_both_from_linear( + affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal + ) + + +def _combine_affine_bounds_generic( + affine_bounds_1: list[Tensor], + affine_bounds_2: list[Tensor], + diagonal: tuple[bool, bool], +) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Combine affine bounds + + Args: + affine_bounds_1: [w_l_1, b_l_1, w_u_1, b_u_1] first affine bounds + affine_bounds_2: [w_l_2, b_l_2, w_u_2, b_u_2] second affine bounds + diagonal: specify if weights of each affine bounds are in diagonal representation or not + + Returns: + w_l, b_l, w_u, b_u: combined affine bounds + + If x, y, z satisfy + w_l_1 * x + b_l_1 <= y <= w_u_1 * x + b_u_1 + w_l_2 * y + b_l_2 <= z <= w_u_2 * x + b_u_2 + + Then + w_l * x + b_l <= z <= w_u * x + b_u + + """ + w_l_1, b_l_1, w_u_1, b_u_1 = affine_bounds_1 + w_l_2, b_l_2, w_u_2, b_u_2 = affine_bounds_2 + nb_axes_wo_batchsize_y = len(b_l_1.shape) - 1 + + #  NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b + kwargs_dot_w: dict[str, Any] = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + diagonal=diagonal, + ) + kwargs_dot_b: dict[str, Any] = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + diagonal=(False, diagonal[1]), + ) + + z_value = K.cast(0.0, dtype=w_l_2.dtype) + w_l_2_pos = K.maximum(w_l_2, z_value) + w_u_2_pos = K.maximum(w_u_2, z_value) + w_l_2_neg = K.minimum(w_l_2, z_value) + w_u_2_neg = K.minimum(w_u_2, z_value) + + w_l = batch_multid_dot(w_l_1, w_l_2_pos, **kwargs_dot_w) + batch_multid_dot(w_u_1, w_l_2_neg, **kwargs_dot_w) + w_u = batch_multid_dot(w_u_1, w_u_2_pos, **kwargs_dot_w) + batch_multid_dot(w_l_1, w_u_2_neg, **kwargs_dot_w) + b_l = ( + batch_multid_dot(b_l_1, w_l_2_pos, **kwargs_dot_b) + batch_multid_dot(b_u_1, w_l_2_neg, **kwargs_dot_b) + b_l_2 + ) + b_u = ( + batch_multid_dot(b_u_1, w_u_2_pos, **kwargs_dot_b) + batch_multid_dot(b_l_1, w_u_2_neg, **kwargs_dot_b) + b_u_2 + ) + + return w_l, b_l, w_u, b_u + + +def _combine_affine_bounds_right_from_linear( + affine_bounds_1: list[Tensor], + affine_bounds_2: list[Tensor], + diagonal: tuple[bool, bool], +) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Combine affine bounds + + Args: + affine_bounds_1: [w_l_1, b_l_1, w_u_1, b_u_1] first affine bounds + affine_bounds_2: [w_2, b_2, w_2, b_2] second affine bounds, with lower=upper + no batchsize + diagonal: specify if weights of each affine bounds are in diagonal representation or not + + Returns: + w_l, b_l, w_u, b_u: combined affine bounds + + If x, y, z satisfy + w_l_1 * x + b_l_1 <= y <= w_u_1 * x + b_u_1 + z = w_2 * y + b_2 + + Then + w_l * x + b_l <= z <= w_u * x + b_u + + """ + w_l_1, b_l_1, w_u_1, b_u_1 = affine_bounds_1 + w_2, b_2 = affine_bounds_2[:2] + nb_axes_wo_batchsize_y = len(b_l_1.shape) - 1 + missing_batchsize = (False, True) + + # NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b + kwargs_dot_w: dict[str, Any] = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + missing_batchsize=missing_batchsize, + diagonal=diagonal, + ) + kwargs_dot_b: dict[str, Any] = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + missing_batchsize=missing_batchsize, + diagonal=(False, diagonal[1]), + ) + + z_value = K.cast(0.0, dtype=w_2.dtype) + w_2_pos = K.maximum(w_2, z_value) + w_2_neg = K.minimum(w_2, z_value) + + w_l = batch_multid_dot(w_l_1, w_2_pos, **kwargs_dot_w) + batch_multid_dot(w_u_1, w_2_neg, **kwargs_dot_w) + w_u = batch_multid_dot(w_u_1, w_2_pos, **kwargs_dot_w) + batch_multid_dot(w_l_1, w_2_neg, **kwargs_dot_w) + b_l = batch_multid_dot(b_l_1, w_2_pos, **kwargs_dot_b) + batch_multid_dot(b_u_1, w_2_neg, **kwargs_dot_b) + b_2 + b_u = batch_multid_dot(b_u_1, w_2_pos, **kwargs_dot_b) + batch_multid_dot(b_l_1, w_2_neg, **kwargs_dot_b) + b_2 + + return w_l, b_l, w_u, b_u + + +def _combine_affine_bounds_left_from_linear( + affine_bounds_1: list[Tensor], + affine_bounds_2: list[Tensor], + diagonal: tuple[bool, bool], +) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Combine affine bounds + + Args: + affine_bounds_1: [w_1, b_1, w_1, b_1] first affine bounds, with lower=upper + no batchsize + affine_bounds_2: [w_l_2, b_l_2, w_u_2, b_u_2] second affine bounds + diagonal: specify if weights of each affine bounds are in diagonal representation or not + + Returns: + w_l, b_l, w_u, b_u: combined affine bounds + + If x, y, z satisfy + y = w_1 * x + b_1 + w_l_2 * y + b_l_2 <= z <= w_u_2 * x + b_u_2 + + Then + w_l * x + b_l <= z <= w_u * x + b_u + + """ + w_1, b_1 = affine_bounds_1[:2] + w_l_2, b_l_2, w_u_2, b_u_2 = affine_bounds_2 + nb_axes_wo_batchsize_y = len(b_1.shape) + missing_batchsize = (True, False) + + #   NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b + kwargs_dot_w: dict[str, Any] = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + missing_batchsize=missing_batchsize, + diagonal=diagonal, + ) + kwargs_dot_b: dict[str, Any] = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + missing_batchsize=missing_batchsize, + diagonal=(False, diagonal[1]), + ) + + w_l = batch_multid_dot(w_1, w_l_2, **kwargs_dot_w) + w_u = batch_multid_dot(w_1, w_u_2, **kwargs_dot_w) + b_l = batch_multid_dot(b_1, w_l_2, **kwargs_dot_b) + b_l_2 + b_u = batch_multid_dot(b_1, w_u_2, **kwargs_dot_b) + b_u_2 + + return w_l, b_l, w_u, b_u + + +def _combine_affine_bounds_both_from_linear( + affine_bounds_1: list[Tensor], + affine_bounds_2: list[Tensor], + diagonal: tuple[bool, bool], +) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Combine affine bounds + + Args: + affine_bounds_1: [w_1, b_1, w_1, b_1] first affine bounds, with lower=upper + no batchsize + affine_bounds_2: [w_2, b_2, w_2, b_2] second affine bounds, with lower=upper + no batchsize + diagonal: specify if weights of each affine bounds are in diagonal representation or not + + Returns: + w, b, w, b: combined affine bounds + + If x, y, z satisfy + y = w_1 * x + b_1 + z = w_2 * x + b_2 + + Then + z = w * x + b + + """ + w_1, b_1 = affine_bounds_1[:2] + w_2, b_2 = affine_bounds_2[:2] + nb_axes_wo_batchsize_y = len(b_1.shape) + missing_batchsize = (True, True) + + #   NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b + kwargs_dot_w: dict[str, Any] = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + missing_batchsize=missing_batchsize, + diagonal=diagonal, + ) + kwargs_dot_b: dict[str, Any] = dict( + nb_merging_axes=nb_axes_wo_batchsize_y, + missing_batchsize=missing_batchsize, + diagonal=(False, diagonal[1]), + ) + + w = batch_multid_dot(w_1, w_2, **kwargs_dot_w) + b = batch_multid_dot(b_1, w_2, **kwargs_dot_b) + b_2 + + return w, b, w, b + + +def combine_affine_bound_with_constant_bound( + lower: Tensor, + upper: Tensor, + affine_bounds: list[Tensor], + missing_batchsize: bool = False, +) -> tuple[Tensor, Tensor]: + if len(affine_bounds) == 0: + # identity affine bounds + return lower, upper + + w_l, b_l, w_u, b_u = affine_bounds + lower_fused = get_lower_box(x_min=lower, x_max=upper, w=w_l, b=b_l, missing_batchsize=missing_batchsize) + upper_fused = get_upper_box(x_min=lower, x_max=upper, w=w_u, b=b_u, missing_batchsize=missing_batchsize) + return lower_fused, upper_fused diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index 3031ec8a..89fc70b6 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -1,12 +1,15 @@ from inspect import Parameter, signature -from typing import Any, Optional, Union +from typing import Any, Optional import keras import keras.ops as K from keras.layers import Layer, Wrapper from decomon.core import BoxDomain, InputsOutputsSpec, PerturbationDomain, Propagation -from decomon.keras_utils import batch_multid_dot +from decomon.layers.fuse import ( + combine_affine_bound_with_constant_bound, + combine_affine_bounds, +) from decomon.layers.oracle import get_forward_oracle from decomon.types import Tensor @@ -308,17 +311,10 @@ def forward_ibp_propagate(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, T """ if self.linear: w, b = self.get_affine_representation() - is_diag = w.shape == b.shape - kwargs_dot: dict[str, Any] = dict(missing_batchsize=(False, True), diagonal=(False, is_diag)) - - z_value = K.cast(0.0, dtype=w.dtype) - w_pos = K.maximum(w, z_value) - w_neg = K.minimum(w, z_value) - - l_c = batch_multid_dot(lower, w_pos, **kwargs_dot) + batch_multid_dot(upper, w_neg, **kwargs_dot) + b - u_c = batch_multid_dot(upper, w_pos, **kwargs_dot) + batch_multid_dot(lower, w_neg, **kwargs_dot) + b - - return l_c, u_c + affine_bounds = [w, b, w, b] + return combine_affine_bound_with_constant_bound( + lower=lower, upper=upper, affine_bounds=affine_bounds, missing_batchsize=self.linear + ) else: raise NotImplementedError( "`forward_ibp_propagate()` needs to be implemented to get the forward propagation of constant bounds." @@ -454,6 +450,7 @@ def get_forward_oracle( from the affine bounds given the considered perturbation domain. """ + from_linear = self.inputs_outputs_spec.is_wo_batch_bounds_by_keras_input(input_affine_bounds) return get_forward_oracle( affine_bounds=input_affine_bounds, ibp_bounds=input_constant_bounds, @@ -462,6 +459,7 @@ def get_forward_oracle( ibp=self.ibp, affine=self.affine, is_merging_layer=self.is_merging_layer, + from_linear=from_linear, ) def call_forward( @@ -524,8 +522,11 @@ def call_forward( x = perturbation_domain_inputs[0] l_ibp, u_ibp = output_constant_bounds w_l, b_l, w_u, b_u = output_affine_bounds - l_affine = self.perturbation_domain.get_lower(x, w_l, b_l) - u_affine = self.perturbation_domain.get_upper(x, w_u, b_u) + from_linear = self.linear and self.inputs_outputs_spec.is_wo_batch_bounds( + affine_bounds=affine_bounds_to_propagate + ) + l_affine = self.perturbation_domain.get_lower(x, w_l, b_l, missing_batchsize=from_linear) + u_affine = self.perturbation_domain.get_upper(x, w_u, b_u, missing_batchsize=from_linear) u = K.minimum(u_ibp, u_affine) l = K.maximum(l_ibp, l_affine) output_constant_bounds = [l, u] @@ -675,276 +676,3 @@ def compute_output_shape( return self.inputs_outputs_spec.flatten_outputs_shape( affine_bounds_propagated_shape=affine_bounds_propagated_shape ) - - -def combine_affine_bounds( - affine_bounds_1: list[Tensor], - affine_bounds_2: list[Tensor], - from_linear_layer: tuple[bool, bool] = (False, False), - diagonal: tuple[bool, bool] = (False, False), -) -> tuple[Tensor, Tensor, Tensor, Tensor]: - """Combine affine bounds - - Args: - affine_bounds_1: [w_l_1, b_l_1, w_u_1, b_u_1] first affine bounds - affine_bounds_2: [w_l_2, b_l_2, w_u_2, b_u_2] second affine bounds - from_linear_layer: specify if affine_bounds_1 or affine_bounds_2 - come from the affine representation of a linear layer - diagonal: specify if affine_bounds_1 or affine_bounds_2 - are in diagonal representation - - Returns: - w_l, b_l, w_u, b_u: combined affine bounds - - If x, y, z satisfy - w_l_1 * x + b_l_1 <= y <= w_u_1 * x + b_u_1 - w_l_2 * y + b_l_2 <= z <= w_u_2 * y + b_u_2 - - Then - w_l * x + b_l <= z <= w_u * x + b_u - - - Special cases - - - with linear layers: - - If the affine bounds come from the affine representation of a linear layer (e.g. affine_bounds_1), then - - lower and upper bounds are equal: affine_bounds_1 = [w_1, b_1, w_1, b_1] - - the tensors are missing the batch dimension - - In the generic case, tensors in affine_bounds have their first axis corresponding to the batch size. - - - diagonal representation: - - If w.shape == b.shape, this means that w is represented by its "diagonal" (potentially a tensor-multid). - - - empty affine bounds: - - when one affine bounds is an empty list, this is actually a convention for identity bounds, i.e. - w = identity, b = 0 - therefore we return the other affine_bounds, unchanged. - - """ - # special case: empty bounds <=> identity bounds - if len(affine_bounds_1) == 0: - return tuple(affine_bounds_2) - if len(affine_bounds_2) == 0: - return tuple(affine_bounds_1) - - if from_linear_layer == (False, False): - return _combine_affine_bounds_generic( - affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal - ) - elif from_linear_layer == (True, False): - return _combine_affine_bounds_left_from_linear( - affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal - ) - elif from_linear_layer == (False, True): - return _combine_affine_bounds_right_from_linear( - affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal - ) - else: - return _combine_affine_bounds_both_from_linear( - affine_bounds_1=affine_bounds_1, affine_bounds_2=affine_bounds_2, diagonal=diagonal - ) - - -def _combine_affine_bounds_generic( - affine_bounds_1: list[Tensor], - affine_bounds_2: list[Tensor], - diagonal: tuple[bool, bool], -) -> tuple[Tensor, Tensor, Tensor, Tensor]: - """Combine affine bounds - - Args: - affine_bounds_1: [w_l_1, b_l_1, w_u_1, b_u_1] first affine bounds - affine_bounds_2: [w_l_2, b_l_2, w_u_2, b_u_2] second affine bounds - diagonal: specify if weights of each affine bounds are in diagonal representation or not - - Returns: - w_l, b_l, w_u, b_u: combined affine bounds - - If x, y, z satisfy - w_l_1 * x + b_l_1 <= y <= w_u_1 * x + b_u_1 - w_l_2 * y + b_l_2 <= z <= w_u_2 * x + b_u_2 - - Then - w_l * x + b_l <= z <= w_u * x + b_u - - """ - w_l_1, b_l_1, w_u_1, b_u_1 = affine_bounds_1 - w_l_2, b_l_2, w_u_2, b_u_2 = affine_bounds_2 - nb_axes_wo_batchsize_y = len(b_l_1.shape) - 1 - - #  NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b - kwargs_dot_w: dict[str, Any] = dict( - nb_merging_axes=nb_axes_wo_batchsize_y, - diagonal=diagonal, - ) - kwargs_dot_b: dict[str, Any] = dict( - nb_merging_axes=nb_axes_wo_batchsize_y, - diagonal=(False, diagonal[1]), - ) - - z_value = K.cast(0.0, dtype=w_l_2.dtype) - w_l_2_pos = K.maximum(w_l_2, z_value) - w_u_2_pos = K.maximum(w_u_2, z_value) - w_l_2_neg = K.minimum(w_l_2, z_value) - w_u_2_neg = K.minimum(w_u_2, z_value) - - w_l = batch_multid_dot(w_l_1, w_l_2_pos, **kwargs_dot_w) + batch_multid_dot(w_u_1, w_l_2_neg, **kwargs_dot_w) - w_u = batch_multid_dot(w_u_1, w_u_2_pos, **kwargs_dot_w) + batch_multid_dot(w_l_1, w_u_2_neg, **kwargs_dot_w) - b_l = ( - batch_multid_dot(b_l_1, w_l_2_pos, **kwargs_dot_b) + batch_multid_dot(b_u_1, w_l_2_neg, **kwargs_dot_b) + b_l_2 - ) - b_u = ( - batch_multid_dot(b_u_1, w_u_2_pos, **kwargs_dot_b) + batch_multid_dot(b_l_1, w_u_2_neg, **kwargs_dot_b) + b_u_2 - ) - - return w_l, b_l, w_u, b_u - - -def _combine_affine_bounds_right_from_linear( - affine_bounds_1: list[Tensor], - affine_bounds_2: list[Tensor], - diagonal: tuple[bool, bool], -) -> tuple[Tensor, Tensor, Tensor, Tensor]: - """Combine affine bounds - - Args: - affine_bounds_1: [w_l_1, b_l_1, w_u_1, b_u_1] first affine bounds - affine_bounds_2: [w_2, b_2, w_2, b_2] second affine bounds, with lower=upper + no batchsize - diagonal: specify if weights of each affine bounds are in diagonal representation or not - - Returns: - w_l, b_l, w_u, b_u: combined affine bounds - - If x, y, z satisfy - w_l_1 * x + b_l_1 <= y <= w_u_1 * x + b_u_1 - z = w_2 * y + b_2 - - Then - w_l * x + b_l <= z <= w_u * x + b_u - - """ - w_l_1, b_l_1, w_u_1, b_u_1 = affine_bounds_1 - w_2, b_2 = affine_bounds_2[:2] - nb_axes_wo_batchsize_y = len(b_l_1.shape) - 1 - missing_batchsize = (False, True) - - # NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b - kwargs_dot_w: dict[str, Any] = dict( - nb_merging_axes=nb_axes_wo_batchsize_y, - missing_batchsize=missing_batchsize, - diagonal=diagonal, - ) - kwargs_dot_b: dict[str, Any] = dict( - nb_merging_axes=nb_axes_wo_batchsize_y, - missing_batchsize=missing_batchsize, - diagonal=(False, diagonal[1]), - ) - - z_value = K.cast(0.0, dtype=w_2.dtype) - w_2_pos = K.maximum(w_2, z_value) - w_2_neg = K.minimum(w_2, z_value) - - w_l = batch_multid_dot(w_l_1, w_2_pos, **kwargs_dot_w) + batch_multid_dot(w_u_1, w_2_neg, **kwargs_dot_w) - w_u = batch_multid_dot(w_u_1, w_2_pos, **kwargs_dot_w) + batch_multid_dot(w_l_1, w_2_neg, **kwargs_dot_w) - b_l = batch_multid_dot(b_l_1, w_2_pos, **kwargs_dot_b) + batch_multid_dot(b_u_1, w_2_neg, **kwargs_dot_b) + b_2 - b_u = batch_multid_dot(b_u_1, w_2_pos, **kwargs_dot_b) + batch_multid_dot(b_l_1, w_2_neg, **kwargs_dot_b) + b_2 - - return w_l, b_l, w_u, b_u - - -def _combine_affine_bounds_left_from_linear( - affine_bounds_1: list[Tensor], - affine_bounds_2: list[Tensor], - diagonal: tuple[bool, bool], -) -> tuple[Tensor, Tensor, Tensor, Tensor]: - """Combine affine bounds - - Args: - affine_bounds_1: [w_1, b_1, w_1, b_1] first affine bounds, with lower=upper + no batchsize - affine_bounds_2: [w_l_2, b_l_2, w_u_2, b_u_2] second affine bounds - diagonal: specify if weights of each affine bounds are in diagonal representation or not - - Returns: - w_l, b_l, w_u, b_u: combined affine bounds - - If x, y, z satisfy - y = w_1 * x + b_1 - w_l_2 * y + b_l_2 <= z <= w_u_2 * x + b_u_2 - - Then - w_l * x + b_l <= z <= w_u * x + b_u - - """ - w_1, b_1 = affine_bounds_1[:2] - w_l_2, b_l_2, w_u_2, b_u_2 = affine_bounds_2 - nb_axes_wo_batchsize_y = len(b_1.shape) - missing_batchsize = (True, False) - - #   NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b - kwargs_dot_w: dict[str, Any] = dict( - nb_merging_axes=nb_axes_wo_batchsize_y, - missing_batchsize=missing_batchsize, - diagonal=diagonal, - ) - kwargs_dot_b: dict[str, Any] = dict( - nb_merging_axes=nb_axes_wo_batchsize_y, - missing_batchsize=missing_batchsize, - diagonal=(False, diagonal[1]), - ) - - w_l = batch_multid_dot(w_1, w_l_2, **kwargs_dot_w) - w_u = batch_multid_dot(w_1, w_u_2, **kwargs_dot_w) - b_l = batch_multid_dot(b_1, w_l_2, **kwargs_dot_b) + b_l_2 - b_u = batch_multid_dot(b_1, w_u_2, **kwargs_dot_b) + b_u_2 - - return w_l, b_l, w_u, b_u - - -def _combine_affine_bounds_both_from_linear( - affine_bounds_1: list[Tensor], - affine_bounds_2: list[Tensor], - diagonal: tuple[bool, bool], -) -> tuple[Tensor, Tensor, Tensor, Tensor]: - """Combine affine bounds - - Args: - affine_bounds_1: [w_1, b_1, w_1, b_1] first affine bounds, with lower=upper + no batchsize - affine_bounds_2: [w_2, b_2, w_2, b_2] second affine bounds, with lower=upper + no batchsize - diagonal: specify if weights of each affine bounds are in diagonal representation or not - - Returns: - w, b, w, b: combined affine bounds - - If x, y, z satisfy - y = w_1 * x + b_1 - z = w_2 * x + b_2 - - Then - z = w * x + b - - """ - w_1, b_1 = affine_bounds_1[:2] - w_2, b_2 = affine_bounds_2[:2] - nb_axes_wo_batchsize_y = len(b_1.shape) - missing_batchsize = (True, True) - - #   NB: bias is never a diagonal representation! => we split kwargs_dot_w and kwargs_dot_b - kwargs_dot_w: dict[str, Any] = dict( - nb_merging_axes=nb_axes_wo_batchsize_y, - missing_batchsize=missing_batchsize, - diagonal=diagonal, - ) - kwargs_dot_b: dict[str, Any] = dict( - nb_merging_axes=nb_axes_wo_batchsize_y, - missing_batchsize=missing_batchsize, - diagonal=(False, diagonal[1]), - ) - - w = batch_multid_dot(w_1, w_2, **kwargs_dot_w) - b = batch_multid_dot(b_1, w_2, **kwargs_dot_b) + b_2 - - return w, b, w, b diff --git a/src/decomon/layers/merging/base_merge.py b/src/decomon/layers/merging/base_merge.py index 150caa81..9a121bd3 100644 --- a/src/decomon/layers/merging/base_merge.py +++ b/src/decomon/layers/merging/base_merge.py @@ -3,7 +3,8 @@ import keras.ops as K from decomon.keras_utils import add_tensors, batch_multid_dot -from decomon.layers.layer import DecomonLayer, combine_affine_bounds +from decomon.layers.fuse import combine_affine_bounds +from decomon.layers.layer import DecomonLayer from decomon.types import Tensor diff --git a/src/decomon/layers/oracle.py b/src/decomon/layers/oracle.py index b1df38ff..f8c4f4f3 100644 --- a/src/decomon/layers/oracle.py +++ b/src/decomon/layers/oracle.py @@ -101,6 +101,8 @@ def call(self, inputs: list[BackendTensor]) -> Union[list[BackendTensor], list[l perturbation_domain_inputs, ) = self.inputs_outputs_spec.split_inputs(inputs=inputs) + from_linear = self.inputs_outputs_spec.is_wo_batch_bounds_by_keras_input(affine_bounds) + return get_forward_oracle( affine_bounds=affine_bounds, ibp_bounds=ibp_bounds, @@ -109,6 +111,7 @@ def call(self, inputs: list[BackendTensor]) -> Union[list[BackendTensor], list[l ibp=self.ibp, affine=self.affine, is_merging_layer=self.is_merging_layer, + from_linear=from_linear, ) def compute_output_shape( @@ -136,6 +139,7 @@ def get_forward_oracle( ibp: bool, affine: bool, is_merging_layer: bool, + from_linear: bool, ) -> list[BackendTensor]: """Get constant oracle bounds on keras layer inputs from forward input bounds. @@ -154,6 +158,7 @@ def get_forward_oracle( ibp: bool, affine: bool, is_merging_layer: bool, + from_linear: list[bool], ) -> list[list[BackendTensor]]: """Get constant oracle bounds on keras layer inputs from forward input bounds. @@ -171,17 +176,19 @@ def get_forward_oracle( ibp: bool, affine: bool, is_merging_layer: bool, + from_linear: Union[bool, list[bool]], ) -> Union[list[BackendTensor], list[list[BackendTensor]]]: """Get constant oracle bounds on keras layer inputs from forward input bounds. Args: - affine_bounds: affine bounds on keras layer input w.r.t model input . Can be empty if not in affine mode. + affine_bounds: affine bounds on keras layer inputs w.r.t model input . Can be empty if not in affine mode. ibp_bounds: ibp constant bounds on keras layer input. Can be empty if not in ibp mode. perturbation_domain_inputs: perturbation domain input, wrapped in a list. Necessary only in affine mode, else empty. perturbation_domain: perturbation domain spec. ibp: ibp bounds exist? affine: affine bounds exist? is_merging_layer: keras layer is a merging layer? + from_linear: affine bounds from linear layer/model? (ie no batchsize + upper affine bound==lower affine bound) Returns: constant bounds on keras layer input deduced from forward layer input bounds or crown output + perturbation_domain_input @@ -202,15 +209,15 @@ def get_forward_oracle( x = perturbation_domain_inputs[0] if is_merging_layer: constant_bounds = [] - for affine_bounds_i in affine_bounds: + for affine_bounds_i, from_linear_i in zip(affine_bounds, from_linear): if len(affine_bounds_i) == 0: # special case: empty affine bounds => identity bounds l_affine = perturbation_domain.get_lower_x(x) u_affine = perturbation_domain.get_upper_x(x) else: w_l, b_l, w_u, b_u = affine_bounds_i - l_affine = perturbation_domain.get_lower(x, w_l, b_l) - u_affine = perturbation_domain.get_upper(x, w_u, b_u) + l_affine = perturbation_domain.get_lower(x, w_l, b_l, missing_batchsize=from_linear_i) + u_affine = perturbation_domain.get_upper(x, w_u, b_u, missing_batchsize=from_linear_i) constant_bounds.append([l_affine, u_affine]) return constant_bounds else: @@ -220,8 +227,8 @@ def get_forward_oracle( u_affine = perturbation_domain.get_upper_x(x) else: w_l, b_l, w_u, b_u = affine_bounds - l_affine = perturbation_domain.get_lower(x, w_l, b_l) - u_affine = perturbation_domain.get_upper(x, w_u, b_u) + l_affine = perturbation_domain.get_lower(x, w_l, b_l, missing_batchsize=from_linear) + u_affine = perturbation_domain.get_upper(x, w_u, b_u, missing_batchsize=from_linear) return [l_affine, u_affine] else: diff --git a/tests/conftest.py b/tests/conftest.py index 418e2f1b..eb7d0fc4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -149,6 +149,7 @@ def get_decomon_input_shapes( diag=False, nobatch=False, for_linear_layer=False, + remove_perturbation_domain_inputs=False, ): inputs_outputs_spec = InputsOutputsSpec( ibp=ibp, @@ -160,7 +161,7 @@ def get_decomon_input_shapes( model_output_shape=model_output_shape, linear=for_linear_layer, ) - if inputs_outputs_spec.needs_perturbation_domain_inputs(): + if inputs_outputs_spec.needs_perturbation_domain_inputs() and not remove_perturbation_domain_inputs: perturbation_domain_inputs_shape = [perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape)] else: perturbation_domain_inputs_shape = [] @@ -200,6 +201,7 @@ def get_decomon_symbolic_inputs( diag=False, nobatch=False, for_linear_layer=False, + remove_perturbation_domain_inputs=False, dtype=keras_config.floatx(), ): """Generate decomon symbolic inputs for a decomon layer @@ -236,6 +238,7 @@ def get_decomon_symbolic_inputs( diag=diag, nobatch=nobatch, for_linear_layer=for_linear_layer, + remove_perturbation_domain_inputs=remove_perturbation_domain_inputs, ) perturbation_domain_inputs = [Input(shape, dtype=dtype) for shape in perturbation_domain_inputs_shape] constant_oracle_bounds = [Input(shape, dtype=dtype) for shape in constant_oracle_bounds_shape] @@ -274,6 +277,7 @@ def generate_simple_decomon_layer_inputs_from_keras_input( for_linear_layer=False, dtype=keras_config.floatx(), equal_ibp=True, + remove_perturbation_domain_inputs=False, ): """Generate simple decomon inputs for a layer from the corresponding keras input @@ -312,7 +316,7 @@ def generate_simple_decomon_layer_inputs_from_keras_input( linear=for_linear_layer, ) - if inputs_outputs_spec.needs_perturbation_domain_inputs(): + if inputs_outputs_spec.needs_perturbation_domain_inputs() and not remove_perturbation_domain_inputs: x = Helpers.generate_simple_perturbation_domain_inputs_from_keras_input( keras_input=keras_input, perturbation_domain=perturbation_domain ) diff --git a/tests/test_fuse.py b/tests/test_fuse.py new file mode 100644 index 00000000..efab7793 --- /dev/null +++ b/tests/test_fuse.py @@ -0,0 +1,298 @@ +import keras.ops as K +import numpy as np +import pytest +from pytest_cases import parametrize + +from decomon.core import InputsOutputsSpec, Propagation +from decomon.layers.fuse import Fuse + + +def generate_simple_inputs(ibp, affine, input_shape, output_shape, batchsize, diag, nobatch): + if ibp: + x = K.convert_to_tensor(2.0 * np.random.random((batchsize,) + output_shape) - 1.0) + lower = x - 0.5 + upper = x + 0.5 + constant_bounds = [lower, upper] + else: + constant_bounds = [] + if affine: + if diag: + w = K.ones(input_shape) + else: + w = K.reshape(K.eye(int(np.prod(input_shape)), int(np.prod(output_shape))), input_shape + output_shape) + b = 0.1 * K.ones(output_shape) + if not nobatch: + w = K.repeat(w[None], batchsize, axis=0) + b = K.repeat(b[None], batchsize, axis=0) + affine_bounds = [w, -b, w, b] + else: + affine_bounds = [] + + return affine_bounds + constant_bounds + + +@parametrize("ibp1, affine1", [(True, False), (False, True), (True, True)], ids=["ibp", "affine", "hybrid"]) +@parametrize("ibp2, affine2", [(True, False), (False, True), (True, True)], ids=["ibp", "affine", "hybrid"]) +@parametrize( + "diag1, nobatch1", + [ + (True, True), + (True, False), + (False, True), + (False, False), + ], + ids=["diagonal-nobatch", "diagonal", "nobatch", "generic"], +) +@parametrize( + "diag2, nobatch2", + [ + (True, True), + (True, False), + (False, True), + (False, False), + ], + ids=["diagonal-nobatch", "diagonal", "nobatch", "generic"], +) +@parametrize("input_shape1", [(3,), (5, 6, 2)], ids=["1d", "multid"]) +@parametrize("output_shape1", [(2,), (4, 5, 1)], ids=["1d", "multid"]) +@parametrize("output_shape2", [(4,), (3, 4, 3)], ids=["1d", "multid"]) +def test_fuse_1output( + ibp1, + affine1, + ibp2, + affine2, + diag1, + nobatch1, + diag2, + nobatch2, + input_shape1, + output_shape1, + output_shape2, + batchsize, + helpers, +): + if not affine1: + if diag1 or nobatch1: + pytest.skip("diag and nobatch have no meaning when affine is False") + if not affine2: + if diag2 or nobatch2: + pytest.skip("diag and nobatch have no meaning when affine is False") + if diag1: + if len(input_shape1) != len(output_shape1): + pytest.skip("we need input_shape==output_shape if diag") + else: + output_shape1 = input_shape1 + if diag2: + if len(output_shape1) != len(output_shape2): + pytest.skip("we need input_shape==output_shape if diag") + else: + output_shape2 = output_shape1 + + input_shape2 = output_shape1 + + symbolic_input_1 = helpers.get_decomon_symbolic_inputs( + model_input_shape=input_shape1, + model_output_shape=None, + layer_input_shape=output_shape1, + layer_output_shape=None, + ibp=ibp1, + affine=affine1, + propagation=Propagation.FORWARD, + perturbation_domain=None, + diag=diag1, + nobatch=nobatch1, + remove_perturbation_domain_inputs=True, + ) + symbolic_input_2 = helpers.get_decomon_symbolic_inputs( + model_input_shape=input_shape2, + model_output_shape=None, + layer_input_shape=output_shape2, + layer_output_shape=None, + ibp=ibp2, + affine=affine2, + propagation=Propagation.FORWARD, + perturbation_domain=None, + diag=diag2, + nobatch=nobatch2, + remove_perturbation_domain_inputs=True, + ) + + input_1 = generate_simple_inputs( + ibp=ibp1, + affine=affine1, + input_shape=input_shape1, + output_shape=output_shape1, + batchsize=batchsize, + diag=diag1, + nobatch=nobatch1, + ) + input_2 = generate_simple_inputs( + ibp=ibp2, + affine=affine2, + input_shape=input_shape2, + output_shape=output_shape2, + batchsize=batchsize, + diag=diag2, + nobatch=nobatch2, + ) + + layer = Fuse( + ibp_1=ibp1, + affine_1=affine1, + ibp_2=ibp2, + affine_2=affine2, + m1_input_shape=input_shape1, + m_1_output_shapes=[output_shape1], + from_linear_2=[nobatch2], + ) + + symbolic_output = layer((symbolic_input_1, symbolic_input_2)) + output = layer((input_1, input_2)) + + # check shapes + output_shape = [t.shape for t in output] + expected_output_shape = [t.shape for t in symbolic_output] + expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) + assert output_shape == expected_output_shape + + inputs_outputs_spec_fused = InputsOutputsSpec( + ibp=layer.ibp_fused, affine=layer.affine_fused, layer_input_shape=input_shape1 + ) + assert len(output_shape) == inputs_outputs_spec_fused.nb_output_tensors + + +@parametrize( + "diag1_1, nobatch1_1", + [ + (True, True), + (True, False), + (False, True), + (False, False), + ], + ids=["diagonal-nobatch", "diagonal", "nobatch", "generic"], +) +@parametrize( + "diag2_2, nobatch2_2", + [ + (True, True), + (True, False), + (False, True), + (False, False), + ], + ids=["diagonal-nobatch", "diagonal", "nobatch", "generic"], +) +@parametrize("input_shape1", [(3,), (5, 6, 2)], ids=["1d", "multid"]) +@parametrize("output_shape1_1", [(2,), (4, 5, 1)], ids=["1d", "multid"]) +@parametrize("output_shape1_2", [(3,), (2, 5, 1)], ids=["1d", "multid"]) +@parametrize("output_shape2", [(4,), (3, 4, 3)], ids=["1d", "multid"]) +def test_fuse_2outputs( + diag1_1, + nobatch1_1, + diag2_2, + nobatch2_2, + input_shape1, + output_shape1_1, + output_shape1_2, + output_shape2, + batchsize, + helpers, +): + if diag1_1: + if len(input_shape1) != len(output_shape1_1): + pytest.skip("we need input_shape==output_shape if diag") + else: + output_shape1_1 = input_shape1 + if diag2_2: + if len(output_shape1_2) != len(output_shape2): + pytest.skip("we need input_shape==output_shape if diag") + else: + output_shape2 = output_shape1_2 + + ibp1, affine1, ibp2, affine2 = True, True, False, True + diag1_2, nobatch1_2 = False, False + diag2_1, nobatch2_1 = False, False + diag1 = [diag1_1, diag1_2] + diag2 = [diag2_1, diag2_2] + nobatch1 = [nobatch1_1, nobatch1_2] + nobatch2 = [nobatch2_1, nobatch2_2] + output_shape1 = [output_shape1_1, output_shape1_2] + input_shape2 = output_shape1 + nb_m1_outputs = len(output_shape1) + + symbolic_input_1 = [] + for output_shape1_i, diag1_i, nobatch1_i in zip(output_shape1, diag1, nobatch1): + symbolic_input_1 += helpers.get_decomon_symbolic_inputs( + model_input_shape=input_shape1, + model_output_shape=None, + layer_input_shape=output_shape1_i, + layer_output_shape=None, + ibp=ibp1, + affine=affine1, + propagation=Propagation.FORWARD, + perturbation_domain=None, + diag=diag1_i, + nobatch=nobatch1_i, + remove_perturbation_domain_inputs=True, + ) + symbolic_input_2 = [] + for input_shape2_i, diag2_i, nobatch2_i in zip(input_shape2, diag2, nobatch2): + symbolic_input_2 += helpers.get_decomon_symbolic_inputs( + model_input_shape=input_shape2_i, + model_output_shape=None, + layer_input_shape=output_shape2, + layer_output_shape=None, + ibp=ibp2, + affine=affine2, + propagation=Propagation.FORWARD, + perturbation_domain=None, + diag=diag2_i, + nobatch=nobatch2_i, + remove_perturbation_domain_inputs=True, + ) + + input_1 = [] + for output_shape1_i, diag1_i, nobatch1_i in zip(output_shape1, diag1, nobatch1): + input_1 += generate_simple_inputs( + ibp=ibp1, + affine=affine1, + input_shape=input_shape1, + output_shape=output_shape1_i, + batchsize=batchsize, + diag=diag1_i, + nobatch=nobatch1_i, + ) + input_2 = [] + for input_shape2_i, diag2_i, nobatch2_i in zip(input_shape2, diag2, nobatch2): + input_2 += generate_simple_inputs( + ibp=ibp2, + affine=affine2, + input_shape=input_shape2_i, + output_shape=output_shape2, + batchsize=batchsize, + diag=diag2_i, + nobatch=nobatch2_i, + ) + + layer = Fuse( + ibp_1=ibp1, + affine_1=affine1, + ibp_2=ibp2, + affine_2=affine2, + m1_input_shape=input_shape1, + m_1_output_shapes=output_shape1, + from_linear_2=nobatch2, + ) + + symbolic_output = layer((symbolic_input_1, symbolic_input_2)) + output = layer((input_1, input_2)) + + # check shapes + output_shape = [t.shape for t in output] + expected_output_shape = [t.shape for t in symbolic_output] + expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) + assert output_shape == expected_output_shape + + inputs_outputs_spec_fused = InputsOutputsSpec( + ibp=layer.ibp_fused, affine=layer.affine_fused, layer_input_shape=input_shape1 + ) + assert len(output_shape) == inputs_outputs_spec_fused.nb_output_tensors * nb_m1_outputs From 1a2bb466c4068f1fdef80d3c17ffd4c6b222c3a6 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 11 Mar 2024 09:55:48 +0100 Subject: [PATCH 075/101] Fuse forward outputs with specified backward_bounds if not using a crown --- src/decomon/models/convert.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index 30cbc1de..b006d992 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -15,6 +15,7 @@ ) from decomon.layers import DecomonLayer from decomon.layers.convert import to_decomon +from decomon.layers.fuse import Fuse from decomon.layers.utils.symbolify import LinkToPerturbationDomainInput from decomon.models.backward_cloning import convert_backward from decomon.models.forward_cloning import ( @@ -91,6 +92,7 @@ def convert( forward_layer_map: Optional[dict[int, DecomonLayer]] = None, final_ibp: bool = False, final_affine: bool = True, + from_linear_backward_bounds: Union[bool, list[bool]] = False, **kwargs: Any, ) -> list[keras.KerasTensor]: """ @@ -153,6 +155,22 @@ def convert( **kwargs, ) + elif backward_bounds is not None: + # Fuse backward_bounds with forward bounds if method not using backward propagation + if isinstance(from_linear_backward_bounds, bool): + from_linear_backward_bounds = [from_linear_backward_bounds] * len(model.outputs) + backward_bounds_flatten = [t for backward_bound in backward_bounds for t in backward_bound] + fuse_layer = Fuse( + ibp_1=ibp, + affine_1=affine, + ibp_2=False, + affine_2=True, + m1_input_shape=model.inputs[0].shape[1:], + m_1_output_shapes=[t.shape[1:] for t in model.outputs], + from_linear_2=from_linear_backward_bounds, + ) + output = fuse_layer((output, backward_bounds_flatten)) + # Update output for final_ibp and final_affine ... From c90d7a4f624b40d978a9b0d82347b652b13652db Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 11 Mar 2024 14:13:52 +0100 Subject: [PATCH 076/101] Add ConvertOutput to change final ibp, affine setting for decomon output --- src/decomon/layers/output.py | 180 ++++++++++++++++++++++++ src/decomon/models/convert.py | 23 +++- tests/conftest.py | 15 +- tests/test_clone.py | 65 ++++++++- tests/test_output.py | 252 ++++++++++++++++++++++++++++++++++ 5 files changed, 529 insertions(+), 6 deletions(-) create mode 100644 src/decomon/layers/output.py create mode 100644 tests/test_output.py diff --git a/src/decomon/layers/output.py b/src/decomon/layers/output.py new file mode 100644 index 00000000..f78fa222 --- /dev/null +++ b/src/decomon/layers/output.py @@ -0,0 +1,180 @@ +"""Convert decomon outputs to the specified format.""" + + +from typing import Any, Optional, Union + +import keras +import keras.ops as K +from keras.layers import Layer + +from decomon.core import InputsOutputsSpec, PerturbationDomain, Propagation +from decomon.layers.oracle import get_forward_oracle +from decomon.types import BackendTensor, Tensor + + +class ConvertOutput(Layer): + """Layer converting output of decomon model to desired final format.""" + + def __init__( + self, + perturbation_domain: PerturbationDomain, + ibp_from: bool, + affine_from: bool, + ibp_to: bool, + affine_to: bool, + model_output_shapes: list[tuple[int, ...]], + **kwargs: Any, + ): + """ + Args: + perturbation_domain: perturbation domain considered on keras model input + ibp_from: ibp bounds present in current outputs? + affine_from: affine bounds present in current outputs? + ibp_to: ibp bounds to be present in new outputs? + affine_to: affine bounds to be present in new outputs? + model_output_shapes: shape of each output of the keras model (w/o batchsize) + **kwargs: passed to Layer.__init__() + + """ + super().__init__(**kwargs) + + self.model_output_shapes = model_output_shapes + self.affine_to = affine_to + self.ibp_to = ibp_to + self.affine_from = affine_from + self.ibp_from = ibp_from + self.perturbation_domain = perturbation_domain + self.nb_outputs_keras_model = len(model_output_shapes) + self.inputs_outputs_spec = InputsOutputsSpecForConvertOutput( + ibp=ibp_from, + affine=affine_from, + ibp_to=ibp_to, + affine_to=affine_to, + model_output_shapes=model_output_shapes, + ) + + def needs_perturbation_domain_inputs(self) -> bool: + return self.inputs_outputs_spec.needs_perturbation_domain_inputs() + + def call(self, inputs: BackendTensor) -> list[BackendTensor]: + """Compute ibp and affine bounds according to desired format from current decomon outputs. + + Args: + inputs: sum_{i} (affine_bounds_from[i] + constant_bounds_from[i]) + perturbation_domain_inputs + being the affine and constant bounds for each output of the keras model, with + - i: the indice of the model output considered + - sum_{i}: the concatenation of subsequent lists over i + - affine_bounds_from[i]: empty if `self.affine_from` is False + - constant_bounds_from[i]: empty if `self.ibp_from` is False + - perturbation_domain_inputs: perturbation domain input wrapped in a list if self.ibp_from is False + and self.ibp_to is True, else empty + + Returns: + sum_{i} (affine_bounds_to[i] + constant_bounds_to[i]): the affine and constant bounds computed + - affine_bounds_to[i]: empty if `self.affine_to` is False + - constant_bounds_to[i]: empty if `self.ibp_to` is False + + Computation: + - ibp: if ibp_to is True and ibp_from is False, we use `decomon.layers.oracle.get_forward_oracle()` + - affine: if affine_to is True and affine_from is False, we construct trivial bounds from ibp bounds + w_l = w_u = 0 and b_l=lower, b_u=upper + + """ + affine_bounds_from, constant_bounds_from, perturbation_domain_inputs = self.inputs_outputs_spec.split_inputs( + inputs + ) + + if self.ibp_to: + if self.ibp_from: + constant_bounds_to = constant_bounds_from + else: + from_linear = self.inputs_outputs_spec.is_wo_batch_bounds_by_keras_input( + affine_bounds=affine_bounds_from + ) + constant_bounds_to = get_forward_oracle( + affine_bounds=affine_bounds_from, + ibp_bounds=constant_bounds_from, + perturbation_domain_inputs=perturbation_domain_inputs, + perturbation_domain=self.perturbation_domain, + ibp=self.ibp_from, + affine=self.affine_from, + is_merging_layer=True, + from_linear=from_linear, + ) + else: + constant_bounds_to = [[]] * self.nb_outputs_keras_model + + if self.affine_to: + if self.affine_from: + affine_bounds_to = affine_bounds_from + else: + x = perturbation_domain_inputs[0] + keras_input_shape_with_batchsize = self.perturbation_domain.get_kerasinputlike_from_x(x).shape + affine_bounds_to = [] + for constant_bounds_from_i in constant_bounds_from: + lower, upper = constant_bounds_from_i + w = K.zeros(keras_input_shape_with_batchsize + lower.shape[1:], dtype=x.dtype) + affine_bounds_to.append([w, lower, w, upper]) + else: + affine_bounds_to = [[]] * self.nb_outputs_keras_model + + return self.inputs_outputs_spec.flatten_inputs( + affine_bounds_to_propagate=affine_bounds_to, + constant_oracle_bounds=constant_bounds_to, + perturbation_domain_inputs=[], + ) + + def compute_output_shape( + self, + input_shape: list[tuple[Optional[int], ...]], + ) -> list[tuple[Optional[int], ...]]: + ( + affine_bounds_from_shape, + constant_bounds_from_shape, + perturbation_domain_inputs_shape, + ) = self.inputs_outputs_spec.split_input_shape(input_shape) + constant_bounds_to_shape: list[list[tuple[Optional[int], ...]]] + affine_bounds_to_shape: list[list[tuple[Optional[int], ...]]] + + if self.ibp_to: + if self.ibp_from: + constant_bounds_to_shape = constant_bounds_from_shape + else: + constant_bounds_to_shape = [] + for model_output_shape in self.model_output_shapes: + lower_shape = (None,) + model_output_shape + constant_bounds_to_shape.append([lower_shape, lower_shape]) + else: + constant_bounds_to_shape = [[]] * self.nb_outputs_keras_model + + if self.affine_to: + if self.affine_from: + affine_bounds_to_shape = affine_bounds_from_shape + else: + x_shape = perturbation_domain_inputs_shape[0] + keras_input_shape = self.perturbation_domain.get_keras_input_shape_wo_batchsize(x_shape[1:]) + affine_bounds_to_shape = [] + for model_output_shape in self.model_output_shapes: + b_shape = (None,) + model_output_shape + w_shape = (None,) + keras_input_shape + model_output_shape + affine_bounds_to_shape.append([w_shape, b_shape, w_shape, b_shape]) + else: + affine_bounds_to_shape = [[]] * self.nb_outputs_keras_model + + return self.inputs_outputs_spec.flatten_inputs_shape( + affine_bounds_to_propagate_shape=affine_bounds_to_shape, + constant_oracle_bounds_shape=constant_bounds_to_shape, + perturbation_domain_inputs_shape=[], + ) + + +class InputsOutputsSpecForConvertOutput(InputsOutputsSpec): + def __init__( + self, ibp: bool, affine: bool, ibp_to: bool, affine_to: bool, model_output_shapes: list[tuple[int, ...]] + ): + super().__init__(ibp=ibp, affine=affine, layer_input_shape=model_output_shapes, is_merging_layer=True) + self.affine_to = affine_to + self.ibp_to = ibp_to + + def needs_perturbation_domain_inputs(self) -> bool: + return (self.ibp_to and not self.ibp) or (self.affine_to and not self.affine) diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index b006d992..c6492055 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -16,6 +16,7 @@ from decomon.layers import DecomonLayer from decomon.layers.convert import to_decomon from decomon.layers.fuse import Fuse +from decomon.layers.output import ConvertOutput from decomon.layers.utils.symbolify import LinkToPerturbationDomainInput from decomon.models.backward_cloning import convert_backward from decomon.models.forward_cloning import ( @@ -122,6 +123,9 @@ def convert( if perturbation_domain is None: perturbation_domain = BoxDomain() + if not final_ibp and not final_affine: + raise ValueError("One of final_ibp and final_affine must be True.") + # prepare the Keras Model: split non-linear activation functions into separate Activation layers model = preprocess_keras_model(model) @@ -154,6 +158,9 @@ def convert( forward_layer_map=forward_layer_map, **kwargs, ) + # output updated mode + affine = True + ibp = False elif backward_bounds is not None: # Fuse backward_bounds with forward bounds if method not using backward propagation @@ -170,9 +177,23 @@ def convert( from_linear_2=from_linear_backward_bounds, ) output = fuse_layer((output, backward_bounds_flatten)) + # output updated mode + affine = fuse_layer.affine_fused + ibp = fuse_layer.ibp_fused # Update output for final_ibp and final_affine - ... + if final_ibp != ibp or final_affine != affine: + convert_layer = ConvertOutput( + ibp_from=ibp, + affine_from=affine, + ibp_to=final_ibp, + affine_to=final_affine, + perturbation_domain=perturbation_domain, + model_output_shapes=[t.shape[1:] for t in model.outputs], + ) + if convert_layer.needs_perturbation_domain_inputs(): + output.append(perturbation_domain_input) + output = convert_layer(output) # build decomon model return output diff --git a/tests/conftest.py b/tests/conftest.py index eb7d0fc4..a3bcd75d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,6 +46,9 @@ ], ids=["forward-hybrid", "forward-affine", "forward-ibp", "backward"], ) +final_ibp, final_affine = param_fixtures( + "final_ibp, final_affine", [(True, False), (False, True), (True, True)], ids=["ibp", "affine", "hybrid"] +) slope = param_fixture("slope", [s.value for s in Slope]) n = param_fixture("n", list(range(10))) odd = param_fixture("odd", list(range(2))) @@ -150,6 +153,7 @@ def get_decomon_input_shapes( nobatch=False, for_linear_layer=False, remove_perturbation_domain_inputs=False, + add_perturbation_domain_inputs=False, ): inputs_outputs_spec = InputsOutputsSpec( ibp=ibp, @@ -161,7 +165,9 @@ def get_decomon_input_shapes( model_output_shape=model_output_shape, linear=for_linear_layer, ) - if inputs_outputs_spec.needs_perturbation_domain_inputs() and not remove_perturbation_domain_inputs: + if ( + inputs_outputs_spec.needs_perturbation_domain_inputs() and not remove_perturbation_domain_inputs + ) or add_perturbation_domain_inputs: perturbation_domain_inputs_shape = [perturbation_domain.get_x_input_shape_wo_batchsize(model_input_shape)] else: perturbation_domain_inputs_shape = [] @@ -202,6 +208,7 @@ def get_decomon_symbolic_inputs( nobatch=False, for_linear_layer=False, remove_perturbation_domain_inputs=False, + add_perturbation_domain_inputs=False, dtype=keras_config.floatx(), ): """Generate decomon symbolic inputs for a decomon layer @@ -239,6 +246,7 @@ def get_decomon_symbolic_inputs( nobatch=nobatch, for_linear_layer=for_linear_layer, remove_perturbation_domain_inputs=remove_perturbation_domain_inputs, + add_perturbation_domain_inputs=add_perturbation_domain_inputs, ) perturbation_domain_inputs = [Input(shape, dtype=dtype) for shape in perturbation_domain_inputs_shape] constant_oracle_bounds = [Input(shape, dtype=dtype) for shape in constant_oracle_bounds_shape] @@ -278,6 +286,7 @@ def generate_simple_decomon_layer_inputs_from_keras_input( dtype=keras_config.floatx(), equal_ibp=True, remove_perturbation_domain_inputs=False, + add_perturbation_domain_inputs=False, ): """Generate simple decomon inputs for a layer from the corresponding keras input @@ -316,7 +325,9 @@ def generate_simple_decomon_layer_inputs_from_keras_input( linear=for_linear_layer, ) - if inputs_outputs_spec.needs_perturbation_domain_inputs() and not remove_perturbation_domain_inputs: + if ( + inputs_outputs_spec.needs_perturbation_domain_inputs() and not remove_perturbation_domain_inputs + ) or add_perturbation_domain_inputs: x = Helpers.generate_simple_perturbation_domain_inputs_from_keras_input( keras_input=keras_input, perturbation_domain=perturbation_domain ) diff --git a/tests/test_clone.py b/tests/test_clone.py index 5144b54b..2550759a 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -1,9 +1,9 @@ import pytest -from keras.layers import Dense, Input +from keras.layers import Input from keras.models import Model -from pytest_cases import fixture, parametrize +from pytest_cases import parametrize -from decomon.core import ConvertMethod, Propagation, Slope +from decomon.core import ConvertMethod, Slope from decomon.layers.utils.symbolify import LinkToPerturbationDomainInput from decomon.models.convert import clone @@ -92,3 +92,62 @@ def test_clone( assert isinstance(decomon_model.layers[-1], LinkToPerturbationDomainInput) else: assert not isinstance(decomon_model.layers[-1], LinkToPerturbationDomainInput) + + +@parametrize( + "toy_model_name", + [ + "tutorial", + ], +) +def test_clone_final_mode( + toy_model_name, + toy_model_fn, + method, + final_ibp, + final_affine, + perturbation_domain, + simple_model_keras_symbolic_input, + simple_model_keras_input, + simple_model_decomon_input, + helpers, +): + # input shape? + input_shape = simple_model_keras_symbolic_input.shape[1:] + + # skip cnn on 0d or 1d input_shape + if toy_model_name == "cnn" and len(input_shape) == 1: + pytest.skip("cnn not possible on 0d or 1d input.") + + slope = Slope.Z_SLOPE + decimal = 4 + + # keras model to convert + keras_model = toy_model_fn(input_shape=input_shape) + + # conversion + decomon_model = clone( + model=keras_model, + slope=slope, + perturbation_domain=perturbation_domain, + method=method, + final_ibp=final_ibp, + final_affine=final_affine, + ) + + # call on actual outputs + keras_output = keras_model(simple_model_keras_input) + decomon_output = decomon_model(simple_model_decomon_input) + + assert final_ibp == decomon_model.ibp + assert final_affine == decomon_model.affine + + # check ibp and affine bounds well ordered w.r.t. keras inputs/outputs + helpers.assert_decomon_output_compare_with_keras_input_output_model( + decomon_output=decomon_output, + keras_input=simple_model_keras_input, + keras_output=keras_output, + decimal=decimal, + ibp=final_ibp, + affine=final_affine, + ) diff --git a/tests/test_output.py b/tests/test_output.py new file mode 100644 index 00000000..2275653d --- /dev/null +++ b/tests/test_output.py @@ -0,0 +1,252 @@ +import keras.ops as K +import numpy as np +import pytest +from pytest_cases import parametrize + +from decomon.core import BoxDomain, InputsOutputsSpec, Propagation +from decomon.layers.output import ConvertOutput + + +def generate_simple_inputs( + ibp, + affine, + input_shape, + output_shape, + batchsize, + diag, + nobatch, + perturbation_domain, + needs_perturbation_domain_inputs=False, +): + if ibp: + x = K.convert_to_tensor(2.0 * np.random.random((batchsize,) + output_shape) - 1.0) + lower = x - 0.5 + upper = x + 0.5 + constant_bounds = [lower, upper] + else: + constant_bounds = [] + if affine: + if diag: + w = K.ones(input_shape) + else: + w = K.reshape(K.eye(int(np.prod(input_shape)), int(np.prod(output_shape))), input_shape + output_shape) + b = 0.1 * K.ones(output_shape) + if not nobatch: + w = K.repeat(w[None], batchsize, axis=0) + b = K.repeat(b[None], batchsize, axis=0) + affine_bounds = [w, -b, w, b] + else: + affine_bounds = [] + + if needs_perturbation_domain_inputs: + if isinstance(perturbation_domain, BoxDomain): + keras_input = K.convert_to_tensor(2.0 * np.random.random((batchsize,) + input_shape) - 1.0) + perturbation_domain_inputs = [ + K.concatenate([keras_input[:, None] - 0.1, keras_input[:, None] + 0.1], axis=1) + ] + else: + raise NotImplementedError + else: + perturbation_domain_inputs = [] + + return affine_bounds + constant_bounds + perturbation_domain_inputs + + +@parametrize("ibp_from, affine_from", [(True, False), (False, True), (True, True)], ids=["ibp", "affine", "hybrid"]) +@parametrize("ibp_to, affine_to", [(True, False), (False, True), (True, True)], ids=["ibp", "affine", "hybrid"]) +@parametrize( + "diag, nobatch", + [ + (True, True), + (True, False), + (False, True), + (False, False), + ], + ids=["diagonal-nobatch", "diagonal", "nobatch", "generic"], +) +@parametrize("input_shape", [(3,), (5, 6, 2)], ids=["1d", "multid"]) +@parametrize("output_shape", [(2,), (4, 5, 1)], ids=["1d", "multid"]) +def test_convert_1output( + ibp_from, + affine_from, + ibp_to, + affine_to, + diag, + nobatch, + input_shape, + output_shape, + batchsize, + perturbation_domain, + helpers, +): + if not affine_from: + if diag or nobatch: + pytest.skip("diag and nobatch have no meaning when affine is False") + if diag: + if len(input_shape) != len(output_shape): + pytest.skip("we need input_shape==output_shape if diag") + else: + output_shape = input_shape + + layer = ConvertOutput( + perturbation_domain=perturbation_domain, + ibp_from=ibp_from, + affine_from=affine_from, + ibp_to=ibp_to, + affine_to=affine_to, + model_output_shapes=[output_shape], + ) + + symbolic_input = helpers.get_decomon_symbolic_inputs( + model_input_shape=input_shape, + model_output_shape=None, + layer_input_shape=output_shape, + layer_output_shape=None, + ibp=ibp_from, + affine=affine_from, + propagation=Propagation.FORWARD, + perturbation_domain=perturbation_domain, + diag=diag, + nobatch=nobatch, + remove_perturbation_domain_inputs=True, + add_perturbation_domain_inputs=layer.needs_perturbation_domain_inputs(), + ) + + input = generate_simple_inputs( + ibp=ibp_from, + affine=affine_from, + input_shape=input_shape, + output_shape=output_shape, + batchsize=batchsize, + diag=diag, + nobatch=nobatch, + perturbation_domain=perturbation_domain, + needs_perturbation_domain_inputs=layer.needs_perturbation_domain_inputs(), + ) + + symbolic_output = layer(symbolic_input) + output = layer(input) + + # check shapes + output_shape = [t.shape for t in output] + expected_output_shape = [t.shape for t in symbolic_output] + expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) + assert output_shape == expected_output_shape + + inputs_outputs_spec_fused = InputsOutputsSpec( + ibp=ibp_to, affine=layer.affine_to, layer_input_shape=layer.model_output_shapes, is_merging_layer=True + ) + assert len(output_shape) == inputs_outputs_spec_fused.nb_output_tensors * layer.nb_outputs_keras_model + + +@parametrize("ibp_from, affine_from", [(True, False), (False, True), (True, True)], ids=["ibp", "affine", "hybrid"]) +@parametrize("ibp_to, affine_to", [(True, False), (False, True), (True, True)], ids=["ibp", "affine", "hybrid"]) +@parametrize( + "diag_1, nobatch_1", + [ + (True, True), + (True, False), + (False, True), + (False, False), + ], + ids=["diagonal-nobatch", "diagonal", "nobatch", "generic"], +) +@parametrize("input_shape", [(3,), (5, 6, 2)], ids=["1d", "multid"]) +@parametrize("output_shape_1", [(2,), (4, 5, 1)], ids=["1d", "multid"]) +@parametrize("output_shape_2", [(3,), (2, 5, 1)], ids=["1d", "multid"]) +def test_convert_2outputs( + ibp_from, + affine_from, + ibp_to, + affine_to, + diag_1, + nobatch_1, + input_shape, + output_shape_1, + output_shape_2, + batchsize, + perturbation_domain, + helpers, +): + if not affine_from: + if diag_1 or nobatch_1: + pytest.skip("diag and nobatch have no meaning when affine is False") + if diag_1: + if len(input_shape) != len(output_shape_1): + pytest.skip("we need input_shape==output_shape if diag") + else: + output_shape_1 = input_shape + + model_output_shapes = [output_shape_1, output_shape_2] + layer = ConvertOutput( + perturbation_domain=perturbation_domain, + ibp_from=ibp_from, + affine_from=affine_from, + ibp_to=ibp_to, + affine_to=affine_to, + model_output_shapes=model_output_shapes, + ) + + symbolic_input_1 = helpers.get_decomon_symbolic_inputs( + model_input_shape=input_shape, + model_output_shape=None, + layer_input_shape=output_shape_1, + layer_output_shape=None, + ibp=ibp_from, + affine=affine_from, + propagation=Propagation.FORWARD, + perturbation_domain=perturbation_domain, + diag=diag_1, + nobatch=nobatch_1, + remove_perturbation_domain_inputs=True, + ) + symbolic_input_2 = helpers.get_decomon_symbolic_inputs( + model_input_shape=input_shape, + model_output_shape=None, + layer_input_shape=output_shape_2, + layer_output_shape=None, + ibp=ibp_from, + affine=affine_from, + propagation=Propagation.FORWARD, + perturbation_domain=perturbation_domain, + remove_perturbation_domain_inputs=True, + add_perturbation_domain_inputs=layer.needs_perturbation_domain_inputs(), + ) + symbolic_input = symbolic_input_1 + symbolic_input_2 + + input_1 = generate_simple_inputs( + ibp=ibp_from, + affine=affine_from, + input_shape=input_shape, + output_shape=output_shape_1, + batchsize=batchsize, + diag=diag_1, + nobatch=nobatch_1, + perturbation_domain=perturbation_domain, + ) + input_2 = generate_simple_inputs( + ibp=ibp_from, + affine=affine_from, + input_shape=input_shape, + output_shape=output_shape_2, + batchsize=batchsize, + perturbation_domain=perturbation_domain, + needs_perturbation_domain_inputs=layer.needs_perturbation_domain_inputs(), + diag=False, + nobatch=False, + ) + input = input_1 + input_2 + + symbolic_output = layer(symbolic_input) + output = layer(input) + + # check shapes + output_shape = [t.shape for t in output] + expected_output_shape = [t.shape for t in symbolic_output] + expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) + assert output_shape == expected_output_shape + + inputs_outputs_spec_fused = InputsOutputsSpec( + ibp=ibp_to, affine=layer.affine_to, layer_input_shape=layer.model_output_shapes, is_merging_layer=True + ) + assert len(output_shape) == inputs_outputs_spec_fused.nb_output_tensors * layer.nb_outputs_keras_model From 07f8aaa9edcfeff635da87f79f7182838cfb8cb8 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 8 Mar 2024 22:19:08 +0100 Subject: [PATCH 077/101] Improve backward_bounds management - Create a dedicated layer to preprocess backward_bounds - Change preferred format for backward_bounds to a flatten list which is the concatenation of backward_bounds for each model output. The reason being that this is how a crown model will return backward bounds and thus such crown models can be directly used to generate backward bounds. - If None, or empty or list tof empty list we set backward_bounds to None to be used by convert. - Else we preprocess them with BackwardInput to ensure having a list of 4 * nb_model_outputs. (See conventions taken in BackwardInput docstring) - convert() get a flatten list to be used by Fuse layer, but we need to split it in convert_backward to feed crown_model with the expected list of list of tensors. - The flatten list of backward_bounds *before* preprocessing is used as inputs for DecomonModel (together with perturbation domain input). - We manage possibility to have backward_bounds without batchsize (if coming from a linear model) with from_linear_backward_bounds parameter. We use it for: - Fuse layer - BackwardInput layer - convert_backward()/crown_model() in order to compute correctly the model output shape to consider (derived from the backward bounds if not empty) --- src/decomon/layers/input.py | 202 ++++++++++++++++++++++++- src/decomon/models/backward_cloning.py | 37 +++-- src/decomon/models/convert.py | 77 ++++++++-- src/decomon/models/utils.py | 6 +- tests/conftest.py | 7 +- tests/test_clone.py | 98 +++++++++++- tests/test_inputs.py | 88 ++++++++++- 7 files changed, 487 insertions(+), 28 deletions(-) diff --git a/src/decomon/layers/input.py b/src/decomon/layers/input.py index 3e5bfcc3..0cead2a7 100644 --- a/src/decomon/layers/input.py +++ b/src/decomon/layers/input.py @@ -1,7 +1,7 @@ """Generate decomon inputs from perturbation domain input.""" -from typing import Any, Optional +from typing import Any, Optional, Union import keras import keras.ops as K @@ -12,7 +12,7 @@ class ForwardInput(Layer): - """Layer generating the input of the first forward layer of a decomon layer.""" + """Layer generating the input of the first forward layer of a decomon model.""" def __init__( self, @@ -103,3 +103,201 @@ def compute_output_shape( constant_oracle_bounds_shape=constant_bounds_shape, perturbation_domain_inputs_shape=[], ) + + +class BackwardInput(Layer): + """Layer preprocessing backward bounds to be used as input of the first backward layer of a decomon model. + + The backward bounds are supposed to be already flattened via `flatten_backward_bounds()`. + This layer ensure having 4* nb_model_outputs tensors in backward_bounds so that it is the concatenation of + backward bounds to be propagated for each keras model output. + + If this is already the case, this layer does nothing. Else there are 3 cases: + + - single tensor w: same bounds for each output, given by [w, 0, w, 0] + - 2 tensors w, b: same bounds for each output, lower==upper, given by [w, b, w, b] + - 4 tensors: same bounds for each output + + we concatenate the same bounds nb_model_outputs times. + + """ + + single_tensor = False + """A single tensor is given as entry.""" + lower_equal_upper = False + """Lower and upper bounds are equal.""" + same_bounds_per_output = False + """The same bounds are used for each keras model output.""" + + def __init__( + self, + model_output_shapes: list[tuple[int, ...]], + from_linear: list[bool], + **kwargs: Any, + ): + """ + Args: + model_output_shapes: shape of each output of the keras model (w/o batchsize) + from_linear: specify if each backward bound corresponding to each keras model output is from a linear model + (i.e. would be w/o batchsize and with lower == upper) + **kwargs: passed to Layer.__init__() + + """ + super().__init__(**kwargs) + + self.model_output_shapes = model_output_shapes + self.nb_model_outputs = len(model_output_shapes) + self.from_linear = from_linear + + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: + # list of tensors + if len(input_shape) == 1: + # single tensor + self.single_tensor = True + self.lower_equal_upper = True + self.same_bounds_per_output = True + elif len(input_shape) == 2: + # upper == lower, same for all model outputs + self.lower_equal_upper = True + self.same_bounds_per_output = True + elif len(input_shape) == 4: + # same bounds for all model outputs + self.same_bounds_per_output = True + elif len(input_shape) != 4 * self.nb_model_outputs: + raise ValueError("backward_bounds should be given as a list of 1, 2, 4, or 4*nb_model_outputs tensors.") + if self.same_bounds_per_output: + for model_output_shape in self.model_output_shapes[1:]: + if model_output_shape != self.model_output_shapes[0]: + raise ValueError( + "backward_bounds should be given for all model outputs " + "(i.e. as a list of 4*nb_model_outputs tensors) " + "if model outputs does not share the same shape" + ) + + self.built = True + + def call(self, inputs: BackendTensor) -> list[BackendTensor]: + """Reconstruct backward bounds at format needed by decomon conversion. + + Args: + inputs: the flattened backward bounds given to `clone` + + Returns: + backward_bounds as a list of 4 * nb_model_outputs tensors, + concatenation of backward bounds on each keras model output + + """ + if self.same_bounds_per_output: + if self.lower_equal_upper: + if self.single_tensor: + w = inputs[0] + model_output_shape = self.model_output_shapes[0] + from_linear = self.from_linear[0] + if from_linear: + w_shape_wo_batchsize = w.shape + else: + w_shape_wo_batchsize = w.shape[1:] + is_diag = w_shape_wo_batchsize == model_output_shape + if is_diag: + m2_output_shape = model_output_shape + else: + m2_output_shape = w_shape_wo_batchsize[len(model_output_shape) :] + b_shape_wo_batchsize = m2_output_shape + if from_linear: + b_shape = b_shape_wo_batchsize + else: + batchsize = w.shape[0] + b_shape = (batchsize,) + b_shape_wo_batchsize + b = K.zeros(b_shape) + else: + w, b = inputs + bounds_per_output = [w, b, w, b] + else: + bounds_per_output = inputs + return bounds_per_output * self.nb_model_outputs + else: + return inputs + + def compute_output_shape( + self, + input_shape: list[tuple[Optional[int], ...]], + ) -> list[tuple[Optional[int], ...]]: + if self.same_bounds_per_output: + if self.lower_equal_upper: + if self.single_tensor: + w_shape = input_shape[0] + model_output_shape = self.model_output_shapes[0] + from_linear = self.from_linear[0] + if from_linear: + w_shape_wo_batchsize = w_shape + else: + w_shape_wo_batchsize = w_shape[1:] + is_diag = w_shape_wo_batchsize == model_output_shape + if is_diag: + m2_output_shape = model_output_shape + else: + m2_output_shape = w_shape_wo_batchsize[len(model_output_shape) :] + b_shape_wo_batchsize = m2_output_shape + if from_linear: + b_shape = b_shape_wo_batchsize + else: + batchsize = w_shape[0] + b_shape = (batchsize,) + b_shape_wo_batchsize + else: + w_shape, b_shape = input_shape + bounds_per_output_shape = [w_shape, b_shape, w_shape, b_shape] + else: + bounds_per_output_shape = input_shape + return bounds_per_output_shape * self.nb_model_outputs + else: + return input_shape + + +def _is_keras_tensor_shape(shape): + return len(shape) > 0 and (shape[0] is None or isinstance(shape[0], int)) + + +def flatten_backward_bounds( + backward_bounds: Union[keras.KerasTensor, list[keras.KerasTensor], list[list[keras.KerasTensor]]] +) -> list[keras.KerasTensor]: + """Flatten backward bounds given to `clone` + + Args: + backward_bounds: + + Returns: + backward_bounds_flattened, computed from backward_bounds as follows: + - single tensor -> [backward_bounds] + - list of tensors -> backward_bounds + - list of list of tensors -> flatten: [t for sublist in backward_bounds for t in sublist] + + """ + if isinstance(backward_bounds, keras.KerasTensor): + return [backward_bounds] + elif len(backward_bounds) == 0 or isinstance(backward_bounds[0], keras.KerasTensor): + return backward_bounds + else: + return [t for sublist in backward_bounds for t in sublist] + + +def has_no_backward_bounds( + backward_bounds: Optional[Union[keras.KerasTensor, list[keras.KerasTensor], list[list[keras.KerasTensor]]]], +) -> bool: + """Check whether some backward bounds are to be propagated or not. + + Args: + backward_bounds: + + Returns: + + """ + return backward_bounds is None or ( + not isinstance(backward_bounds, keras.KerasTensor) + and ( + len(backward_bounds) == 0 + or all( + not isinstance(backward_bounds_i, keras.KerasTensor) and len(backward_bounds_i) == 0 + for backward_bounds_i in backward_bounds + ) + ) + ) diff --git a/src/decomon/models/backward_cloning.py b/src/decomon/models/backward_cloning.py index 7fe8823c..b0c7ea52 100644 --- a/src/decomon/models/backward_cloning.py +++ b/src/decomon/models/backward_cloning.py @@ -339,6 +339,7 @@ def crown_model( model: Model, layer_fn: Callable[[Layer, tuple[int, ...]], DecomonLayer], backward_bounds: list[list[keras.KerasTensor]], + from_linear_backward_bounds: list[bool], perturbation_domain_input: keras.KerasTensor, perturbation_domain: PerturbationDomain, oracle_map: Optional[dict[int, Union[list[keras.KerasTensor], list[list[keras.KerasTensor]]]]] = None, @@ -357,6 +358,7 @@ def crown_model( perturbation_domain: perturbation domain type on keras model input backward_bounds: should be of the same size as the number of model outputs (each sublist potentially empty for starting with identity bounds) + from_linear_backward_bounds: specify if the backward_bounds come from a linear model (=> no batchsize + upper == lower) oracle_map: already registered oracle bounds per node forward_output_map: forward outputs per node from a previously performed forward conversion. To be used for forward oracle. @@ -388,9 +390,11 @@ def crown_model( # Apply crown on each output, with the appropriate backward_bounds and model_output_shape output = [] - for node, backward_bounds_node in zip(output_nodes, backward_bounds): + for node, backward_bounds_node, from_linear in zip(output_nodes, backward_bounds, from_linear_backward_bounds): # new backward_map and new model_output_shape for each output node - model_output_shape = get_model_output_shape(node=node, backward_bounds=backward_bounds_node) + model_output_shape = get_model_output_shape( + node=node, backward_bounds=backward_bounds_node, from_linear=from_linear + ) backward_map_node = {} output_crown = crown( @@ -417,7 +421,8 @@ def convert_backward( perturbation_domain_input: keras.KerasTensor, perturbation_domain: Optional[PerturbationDomain] = None, layer_fn: Callable[..., DecomonLayer] = to_decomon, - backward_bounds: Optional[list[list[keras.KerasTensor]]] = None, + backward_bounds: Optional[list[keras.KerasTensor]] = None, + from_linear_backward_bounds: Union[bool, list[bool]] = False, slope: Union[str, Slope] = Slope.V_SLOPE, forward_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, forward_layer_map: Optional[dict[int, DecomonLayer]] = None, @@ -433,8 +438,10 @@ def convert_backward( perturbation_domain_input: perturbation domain input perturbation_domain: perturbation domain type on keras model input layer_fn: callable converting a layer and a model_output_shape into a (backward) decomon layer - backward_bounds: if set, should be of the same size as the number of model outputs - (each sublist potentially empty for starting with identity bounds) + backward_bounds: if set, should be of the same size as the number of model outputs times 4, + being the concatenation of backward bounds for each keras model output + from_linear_backward_bounds: specify if backward_bounds come from a linear model (=> no batchsize + upper == lower) + if a boolean, flag for each backward bound, else a list of boolean, one per keras model output. forward_output_map: forward outputs per node from a previously performed forward conversion. To be used for forward oracle if not empty. forward_layer_map: forward decomon layer per node from a previously performed forward conversion. @@ -449,7 +456,12 @@ def convert_backward( if perturbation_domain is None: perturbation_domain = BoxDomain() if backward_bounds is None: - backward_bounds = [[]] * len(model.outputs) + backward_bounds_for_crown_model = [[]] * len(model.outputs) + else: + # split backward bounds per model output + backward_bounds_for_crown_model = [backward_bounds[i : i + 4] for i in range(0, len(backward_bounds), 4)] + if isinstance(from_linear_backward_bounds, bool): + from_linear_backward_bounds = [from_linear_backward_bounds] * len(model.outputs) model = ensure_functional_model(model) propagation = Propagation.BACKWARD @@ -465,7 +477,8 @@ def convert_backward( output = crown_model( model=model, layer_fn=layer_fn, - backward_bounds=backward_bounds, + backward_bounds=backward_bounds_for_crown_model, + from_linear_backward_bounds=from_linear_backward_bounds, perturbation_domain_input=perturbation_domain_input, perturbation_domain=perturbation_domain, forward_output_map=forward_output_map, @@ -475,14 +488,14 @@ def convert_backward( return output -def get_model_output_shape(node: Node, backward_bounds: list[Tensor]): +def get_model_output_shape(node: Node, backward_bounds: list[Tensor], from_linear: bool = False): """Get outer model output shape w/o batchsize. If any backward bounds are passed, we deduce the outer keras model output shape from it. We assume for that: - backward_bounds = [w_l, b_l, w_u, b_u] - we can have w_l, w_u in diagonal representation (w_l.shape == b_l.shape) - - we have the batchsize included in the backward_bounds + - we have the batchsize included in the backward_bounds, except if from_linear is True => model_output_shape = backward_bounds[1].shape[1:] @@ -491,6 +504,7 @@ def get_model_output_shape(node: Node, backward_bounds: list[Tensor]): Args: node: current output node of the (potentially inner) keras model to convert backward_bounds: backward bounds specified for this node + from_linear: flag telling if backward_bounds are from a linear model (and thus w/o batchsize + lower==upper) Returns: outer keras model output shape, excluding batchsize @@ -500,7 +514,10 @@ def get_model_output_shape(node: Node, backward_bounds: list[Tensor]): return node.outputs[0].shape[1:] else: _, b, _, _ = backward_bounds - return b.shape[1:] + if from_linear: + return b.shape + else: + return b.shape[1:] def include_kwargs_layer_fn( diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index c6492055..c22e7dbb 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -16,6 +16,11 @@ from decomon.layers import DecomonLayer from decomon.layers.convert import to_decomon from decomon.layers.fuse import Fuse +from decomon.layers.input import ( + BackwardInput, + flatten_backward_bounds, + has_no_backward_bounds, +) from decomon.layers.output import ConvertOutput from decomon.layers.utils.symbolify import LinkToPerturbationDomainInput from decomon.models.backward_cloning import convert_backward @@ -86,14 +91,14 @@ def convert( perturbation_domain_input: keras.KerasTensor, perturbation_domain: PerturbationDomain, method: ConvertMethod = ConvertMethod.CROWN, - backward_bounds: Optional[list[list[keras.KerasTensor]]] = None, + backward_bounds: Optional[list[keras.KerasTensor]] = None, + from_linear_backward_bounds: Union[bool, list[bool]] = False, layer_fn: Callable[..., DecomonLayer] = to_decomon, slope: Slope = Slope.V_SLOPE, forward_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, forward_layer_map: Optional[dict[int, DecomonLayer]] = None, final_ibp: bool = False, final_affine: bool = True, - from_linear_backward_bounds: Union[bool, list[bool]] = False, **kwargs: Any, ) -> list[keras.KerasTensor]: """ @@ -103,7 +108,9 @@ def convert( perturbation_domain_input: perturbation domain input perturbation_domain: perturbation domain type on keras model input method: method used to convert the model to a decomon model. See `ConvertMethod`. - backward_bounds: backward bounds to propagate, see `preprocess_backward_bounds()` for conventions + backward_bounds: backward bounds to propagate, concatenation of backward bounds for each keras model output + from_linear_backward_bounds: specify if backward_bounds come from a linear model (=> no batchsize + upper == lower) + if a boolean, flag for each backward bound, else a list of boolean, one per keras model output. layer_fn: callable converting a layer and a model_output_shape into a decomon layer slope: slope used by decomon activation layers forward_output_map: forward outputs per node from a previously performed forward conversion. @@ -126,6 +133,9 @@ def convert( if not final_ibp and not final_affine: raise ValueError("One of final_ibp and final_affine must be True.") + if isinstance(from_linear_backward_bounds, bool): + from_linear_backward_bounds = [from_linear_backward_bounds] * len(model.outputs) + # prepare the Keras Model: split non-linear activation functions into separate Activation layers model = preprocess_keras_model(model) @@ -153,6 +163,7 @@ def convert( perturbation_domain=perturbation_domain, layer_fn=layer_fn, backward_bounds=backward_bounds, + from_linear_backward_bounds=from_linear_backward_bounds, slope=slope, forward_output_map=forward_output_map, forward_layer_map=forward_layer_map, @@ -164,9 +175,6 @@ def convert( elif backward_bounds is not None: # Fuse backward_bounds with forward bounds if method not using backward propagation - if isinstance(from_linear_backward_bounds, bool): - from_linear_backward_bounds = [from_linear_backward_bounds] * len(model.outputs) - backward_bounds_flatten = [t for backward_bound in backward_bounds for t in backward_bound] fuse_layer = Fuse( ibp_1=ibp, affine_1=affine, @@ -176,7 +184,7 @@ def convert( m_1_output_shapes=[t.shape[1:] for t in model.outputs], from_linear_2=from_linear_backward_bounds, ) - output = fuse_layer((output, backward_bounds_flatten)) + output = fuse_layer((output, backward_bounds)) # output updated mode affine = fuse_layer.affine_fused ibp = fuse_layer.ibp_fused @@ -205,6 +213,7 @@ def clone( perturbation_domain: Optional[PerturbationDomain] = None, method: Union[str, ConvertMethod] = ConvertMethod.CROWN, backward_bounds: Optional[Union[keras.KerasTensor, list[keras.KerasTensor], list[list[keras.KerasTensor]]]] = None, + from_linear_backward_bounds: Union[bool, list[bool]] = False, final_ibp: Optional[bool] = None, final_affine: Optional[bool] = None, layer_fn: Callable[..., DecomonLayer] = to_decomon, @@ -219,7 +228,10 @@ def clone( slope: slope used by decomon activation layers perturbation_domain: perturbation domain type on keras model input method: method used to convert the model to a decomon model. See `ConvertMethod`. - backward_bounds: backward bounds to propagate, see `preprocess_backward_bounds()` for conventions + backward_bounds: backward bounds to propagate, see `BackwardInput` for conventions + to be fused with ibp + affine bounds computed on the keras model outputs + from_linear_backward_bounds: specify if backward_bounds come from a linear model (=> no batchsize + upper == lower) + if a boolean, flag for each backward bound, else a list of boolean, one per keras model output. final_ibp: specify if final outputs should include constant bounds. Default to False except for forward-ibp and forward-hybrid. final_affine: specify if final outputs should include affine bounds. @@ -234,6 +246,30 @@ def clone( **kwargs: keyword arguments to pass to layer_fn Returns: + decomon model mapping perturbation domain input and backward bounds + to ibp + affine bounds (according to final_ibp and final_affine) on model outputs + fused with the backward bounds + + The resulting DecomonModel have flatten inputs and outputs: + - inputs: [perturbation_domain_input] + backward_bounds_flattened + where backward_bounds_flattened is computed from backward_bounds as follows: + - None -> [] + - single tensor -> [backward_bounds] + - list of tensors -> backward_bounds + - list of list of tensors -> flatten: [t for sublist in backward_bounds for t in sublist] + + - single tensor -> [backward_bounds, 0, backward_bounds, 0] * nb_model_outputs + - list of 2 tensors (upper = lower bounds, for all model outputs) -> backward_bounds * 2 * nb_model_outputs + - list of 4 tensors -> backward_bounds * nb_model_outputs + - list of 4 * nb_model_outputs tensors -> backward_bounds + - list of list of tensors -> ensure having nb_model_outputs sublists -> flatten: [t for sublist in backward_bounds for t in sublist] + + - outputs: sum_{i} (affine_bounds_from[i] + constant_bounds_from[i]) + being the affine and constant bounds for each output of the keras model, with + - i: the indice of the model output considered + - sum_{i}: the concatenation of subsequent lists over i + - affine_bounds_from[i]: empty if `final_affine` is False + - constant_bounds_from[i]: empty if `final_ibp` is False """ # Store model name (before converting to functional) @@ -257,7 +293,21 @@ def clone( if isinstance(method, str): method = ConvertMethod(method.lower()) - backward_bounds = preprocess_backward_bounds(backward_bounds=backward_bounds, nb_model_outputs=len(model.outputs)) + # preprocess backward_bounds + backward_bounds_flattened: Optional[list[keras.KerasTensor]] + backward_bounds_for_convert: Optional[list[keras.KerasTensor]] + if has_no_backward_bounds(backward_bounds): + backward_bounds_flattened = None + backward_bounds_for_convert = None + else: + if isinstance(from_linear_backward_bounds, bool): + from_linear_backward_bounds = [from_linear_backward_bounds] * len(model.outputs) + # flatten backward bounds + backward_bounds_flattened = flatten_backward_bounds(backward_bounds) + # prepare for convert: ensure having 4 * nb_model_outputs tensors + backward_bounds_for_convert = BackwardInput( + model_output_shapes=[t.shape[1:] for t in model.outputs], from_linear=from_linear_backward_bounds + )(backward_bounds_flattened) perturbation_domain_input = generate_perturbation_domain_input( model=model, perturbation_domain=perturbation_domain, name=f"perturbation_domain_input_{model_name}" @@ -268,7 +318,8 @@ def clone( perturbation_domain_input=perturbation_domain_input, perturbation_domain=perturbation_domain, method=method, - backward_bounds=backward_bounds, + backward_bounds=backward_bounds_for_convert, + from_linear_backward_bounds=from_linear_backward_bounds, layer_fn=layer_fn, slope=slope, forward_output_map=forward_output_map, @@ -288,8 +339,12 @@ def clone( # Insert batch axis and repeat it to get the correct batchsize output = LinkToPerturbationDomainInput()([perturbation_domain_input] + output) + decomon_inputs = [perturbation_domain_input] + if backward_bounds_flattened is not None: + decomon_inputs += backward_bounds_flattened + return DecomonModel( - inputs=[perturbation_domain_input], + inputs=decomon_inputs, outputs=output, perturbation_domain=perturbation_domain, method=method, diff --git a/src/decomon/models/utils.py b/src/decomon/models/utils.py index 06472caa..1f1b57cf 100644 --- a/src/decomon/models/utils.py +++ b/src/decomon/models/utils.py @@ -327,7 +327,11 @@ def preprocess_backward_bounds( else: # list of list of tensors if len(backward_bounds) == 1: - return [backward_bounds[0]] * nb_model_outputs + if len(backward_bounds[0]) == 0: + # [[]] + return None + else: + return [backward_bounds[0]] * nb_model_outputs elif len(backward_bounds) != nb_model_outputs: raise ValueError( "If backward_bounds is given as a list of tensors, it should have nb_model_ouptputs elements." diff --git a/tests/conftest.py b/tests/conftest.py index a3bcd75d..36db26a9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -134,8 +134,11 @@ def in_GPU_mode() -> bool: raise NotImplementedError(f"Not implemented for {backend} backend.") @staticmethod - def generate_random_tensor(shape_wo_batchsize, batchsize=10, dtype=keras_config.floatx()): - shape = (batchsize,) + shape_wo_batchsize + def generate_random_tensor(shape_wo_batchsize, batchsize=10, dtype=keras_config.floatx(), nobatch=False): + if nobatch: + shape = shape_wo_batchsize + else: + shape = (batchsize,) + shape_wo_batchsize return K.convert_to_tensor(2.0 * np.random.random(shape) - 1.0, dtype=dtype) @staticmethod diff --git a/tests/test_clone.py b/tests/test_clone.py index 2550759a..d8c38072 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -1,9 +1,10 @@ +import keras.ops as K import pytest from keras.layers import Input from keras.models import Model from pytest_cases import parametrize -from decomon.core import ConvertMethod, Slope +from decomon.core import BoxDomain, ConvertMethod, Slope from decomon.layers.utils.symbolify import LinkToPerturbationDomainInput from decomon.models.convert import clone @@ -151,3 +152,98 @@ def test_clone_final_mode( ibp=final_ibp, affine=final_affine, ) + + +@parametrize( + "toy_model_name", + [ + "tutorial", + ], +) +@parametrize("equal_ibp, input_shape", [(False, (5, 6, 2))], ids=["multid"]) # fix some parameters of inputs +def test_clone_w_backwardbounds( + toy_model_name, + toy_model_fn, + method, + perturbation_domain, + equal_ibp, + input_shape, + simple_model_keras_symbolic_input, + simple_model_keras_input, + simple_model_decomon_input, + helpers, +): + # input shape? + input_shape = simple_model_keras_symbolic_input.shape[1:] + + slope = Slope.Z_SLOPE + decimal = 4 + + # keras model to convert: chaining 2 models + keras_model_1 = toy_model_fn(input_shape=input_shape) + output_shape_1 = keras_model_1.outputs[0].shape # only 1 output + + keras_model_2 = toy_model_fn(input_shape=output_shape_1[1:]) + + input_tot = keras_model_1.inputs[0] + output_tot = keras_model_2(keras_model_1(input_tot)) + keras_model_tot = Model(input_tot, output_tot) + + # perturbation domain for 2nd model: computed by foward conversion of first model + forward_model_1 = clone( + model=keras_model_1, + slope=slope, + perturbation_domain=perturbation_domain, + method=ConvertMethod.FORWARD_HYBRID, + ) + decomon_input_1 = simple_model_decomon_input + decomon_output_1 = forward_model_1(decomon_input_1) + _, _, _, _, lower_ibp, upper_ibp = decomon_output_1 + decomon_input_2 = K.concatenate([lower_ibp[:, None], upper_ibp[:, None]], axis=1) + + # backward_bounds: crown on 2nd model + crown_model_2 = clone( + model=keras_model_2, + slope=slope, + perturbation_domain=BoxDomain(), + method=ConvertMethod.CROWN, + ) + symbolic_backward_bounds = crown_model_2.outputs + backward_bounds = crown_model_2(decomon_input_2) + + # conversion of first model with backward_bounds + decomon_model = clone( + model=keras_model_1, + slope=slope, + perturbation_domain=perturbation_domain, + method=method, + backward_bounds=symbolic_backward_bounds, + ) + + # call on actual outputs + keras_output = keras_model_tot(simple_model_keras_input) + decomon_output = decomon_model([simple_model_decomon_input] + backward_bounds) + + # check output mode + ibp = decomon_model.ibp + affine = decomon_model.affine + + if method in (ConvertMethod.FORWARD_IBP, ConvertMethod.FORWARD_HYBRID): + assert ibp + else: + assert not ibp + + if method == ConvertMethod.FORWARD_IBP: + assert not affine + else: + assert affine + + # check ibp and affine bounds well ordered w.r.t. keras inputs/outputs + helpers.assert_decomon_output_compare_with_keras_input_output_model( + decomon_output=decomon_output, + keras_input=simple_model_keras_input, + keras_output=keras_output, + decimal=decimal, + ibp=ibp, + affine=affine, + ) diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 45b71431..d3fe6dff 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -2,7 +2,7 @@ from pytest_cases import parametrize from decomon.core import Propagation -from decomon.layers.input import ForwardInput +from decomon.layers.input import BackwardInput, ForwardInput def test_forward_input( @@ -27,3 +27,89 @@ def test_forward_input( expected_output_shape = [t.shape for t in symbolic_output] expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) assert output_shape == expected_output_shape + + +@parametrize("m1_output_shape", [(2,), (4, 5, 1)], ids=["1d", "multid"]) +@parametrize("m2_output_shape", [(4,), (3, 4, 3)], ids=["1d", "multid"]) +@parametrize("bb_nb_tensors", [1, 2, 3, 4, 8]) +@parametrize("nb_outputs", [1, 2]) +def test_backward_input( + empty, diag, nobatch, bb_nb_tensors, m1_output_shape, m2_output_shape, nb_outputs, batchsize, helpers +): + if empty: + pytest.skip("backward bounds cannot be empty for BackwardInput") + if diag: + if len(m1_output_shape) != len(m2_output_shape): + pytest.skip("we need output_shape1==output_shape2 if diag") + else: + m2_output_shape = m1_output_shape + if bb_nb_tensors > 4 and nb_outputs == 1: + pytest.skip("no meaning for more than 4 tensors if only 1 output to keras model") + + # for other outputs after the first + if bb_nb_tensors <= 4: + # same output shapes + m1_output_shape_2 = m1_output_shape + m2_output_shape_2 = m2_output_shape + else: + # can be different + m1_output_shape_2 = (2, 3) + m2_output_shape_2 = (3,) + + from_linear = [nobatch] + (nb_outputs - 1) * [False] + model_output_shapes = [m1_output_shape] + (nb_outputs - 1) * [m1_output_shape_2] + layer = BackwardInput(model_output_shapes=model_output_shapes, from_linear=from_linear) + + symbolic_input = helpers.get_decomon_symbolic_inputs( + model_input_shape=m1_output_shape, + model_output_shape=m2_output_shape, + layer_input_shape=m2_output_shape, + layer_output_shape=m2_output_shape, + perturbation_domain=None, + ibp=False, + affine=True, + propagation=Propagation.FORWARD, + empty=empty, + diag=diag, + nobatch=nobatch, + remove_perturbation_domain_inputs=True, + ) + for model_output_shape in model_output_shapes[1:]: + symbolic_input += helpers.get_decomon_symbolic_inputs( + model_input_shape=model_output_shape, + model_output_shape=m2_output_shape_2, + layer_input_shape=m2_output_shape_2, + layer_output_shape=m2_output_shape_2, + perturbation_domain=None, + ibp=False, + affine=True, + propagation=Propagation.FORWARD, + remove_perturbation_domain_inputs=True, + ) + symbolic_input = symbolic_input[:bb_nb_tensors] + if nobatch: + # nobatch only for 4 first tensors + random_input = [ + helpers.generate_random_tensor(t.shape, batchsize=batchsize, nobatch=nobatch) for t in symbolic_input[:4] + ] + # add batchsize for following tensors + random_input += [ + helpers.generate_random_tensor(t.shape[1:], batchsize=batchsize, nobatch=False) for t in symbolic_input[4:] + ] + else: + random_input = [ + helpers.generate_random_tensor(t.shape[1:], batchsize=batchsize, nobatch=nobatch) for t in symbolic_input + ] + + if bb_nb_tensors == 3: + with pytest.raises(ValueError): + symbolic_output = layer(symbolic_input) + else: + symbolic_output = layer(symbolic_input) + output = layer(random_input) + + # check shapes + output_shape = [t.shape for t in output] + expected_output_shape = [t.shape for t in symbolic_output] + expected_output_shape = helpers.replace_none_by_batchsize(shapes=expected_output_shape, batchsize=batchsize) + assert output_shape == expected_output_shape From 4783f75d7b7f26720344ceb33bed67a190a1cf74 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 12 Mar 2024 11:24:51 +0100 Subject: [PATCH 078/101] Testing multiple outputs --- tests/conftest.py | 20 +++++ tests/test_clone.py | 177 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 36db26a9..2623d1ea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1076,6 +1076,26 @@ def toy_network_tutorial( model = Sequential(layers) return model + @staticmethod + def toy_network_2outputs( + input_shape: tuple[int, ...] = (1,), + dtype: Optional[str] = None, + activation: Optional[str] = "relu", + same_output_shape=False, + ) -> Model: + if dtype is None: + dtype = keras_config.floatx() + if same_output_shape: + n1, n2, n3 = 10, 10, 10 + else: + n1, n2, n3 = 10, 11, 12 + input_tensor = Input(input_shape, dtype=dtype) + output_tensor = Dense(n1, dtype=dtype, activation=activation)(input_tensor) + output_tensor_1 = Dense(n2, dtype=dtype, activation=activation)(output_tensor) + output_tensor_2 = Dense(n3, dtype=dtype, activation=activation)(output_tensor) + model = Model(input_tensor, [output_tensor_1, output_tensor_2]) + return model + @staticmethod def toy_network_submodel( input_shape: tuple[int, ...] = (1,), dtype: Optional[str] = None, activation: Optional[str] = "relu" diff --git a/tests/test_clone.py b/tests/test_clone.py index d8c38072..9b3ffa95 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -1,4 +1,5 @@ import keras.ops as K +import numpy as np import pytest from keras.layers import Input from keras.models import Model @@ -247,3 +248,179 @@ def test_clone_w_backwardbounds( ibp=ibp, affine=affine, ) + + +def test_clone_2outputs( + method, + final_ibp, + final_affine, + perturbation_domain, + simple_model_keras_symbolic_input, + simple_model_keras_input, + simple_model_decomon_input, + helpers, +): + # input shape? + input_shape = simple_model_keras_symbolic_input.shape[1:] + + slope = Slope.Z_SLOPE + decimal = 4 + + # keras model to convert + keras_model = helpers.toy_network_2outputs(input_shape=input_shape) + + # conversion + decomon_model = clone( + model=keras_model, + slope=slope, + perturbation_domain=perturbation_domain, + method=method, + final_ibp=final_ibp, + final_affine=final_affine, + ) + + assert final_ibp == decomon_model.ibp + assert final_affine == decomon_model.affine + + nb_decomon_output_tensor_per_keras_output = 0 + if decomon_model.ibp: + nb_decomon_output_tensor_per_keras_output += 2 + if decomon_model.affine: + nb_decomon_output_tensor_per_keras_output += 4 + assert len(decomon_model.outputs) == len(keras_model.outputs) * nb_decomon_output_tensor_per_keras_output + + # call on actual inputs + keras_output = keras_model(simple_model_keras_input) + decomon_output = decomon_model(simple_model_decomon_input) + + # check ibp and affine bounds well ordered w.r.t. keras inputs/outputs + for i in range(len(keras_model.outputs)): + keras_output_i = keras_output[i] + decomon_output_i = decomon_output[ + i * nb_decomon_output_tensor_per_keras_output : (i + 1) * nb_decomon_output_tensor_per_keras_output + ] + helpers.assert_decomon_output_compare_with_keras_input_output_model( + decomon_output=decomon_output_i, + keras_input=simple_model_keras_input, + keras_output=keras_output_i, + decimal=decimal, + ibp=final_ibp, + affine=final_affine, + ) + + +@parametrize( + "toy_model_name", + [ + "tutorial", + ], +) +@parametrize("equal_ibp, input_shape", [(False, (5, 6, 2))], ids=["multid"]) # fix some parameters of inputs +def test_clone_w_backwardbounds_2outputs( + toy_model_name, + toy_model_fn, + method, + perturbation_domain, + equal_ibp, + input_shape, + simple_model_keras_symbolic_input, + simple_model_keras_input, + simple_model_decomon_input, + helpers, +): + # input shape? + input_shape = simple_model_keras_symbolic_input.shape[1:] + + slope = Slope.Z_SLOPE + decimal = 4 + + # keras model to convert: chaining 2 models (second one can be different for each output of the first one) + keras_model_1 = helpers.toy_network_2outputs(input_shape=input_shape) + keras_models_2 = [toy_model_fn(input_shape=output.shape[1:]) for output in keras_model_1.outputs] + + input_tot = keras_model_1.inputs[0] + outputs_tmp = keras_model_1(input_tot) + output_tot = [keras_models_2[i](outputs_tmp[i]) for i in range(len(outputs_tmp))] + keras_model_tot = Model(input_tot, output_tot) + + # perturbation domain for 2nd model: computed by forward conversion of first model + forward_model_1 = clone( + model=keras_model_1, + slope=slope, + perturbation_domain=perturbation_domain, + method=ConvertMethod.FORWARD_HYBRID, + ) + decomon_input_1 = simple_model_decomon_input + decomon_output_1 = forward_model_1(decomon_input_1) + _, _, _, _, lower_ibp_1, upper_ibp_1, _, _, _, _, lower_ibp_2, upper_ibp_2 = decomon_output_1 + decomon_inputs_2 = [ + K.concatenate([lower_ibp_1[:, None], upper_ibp_1[:, None]], axis=1), + K.concatenate([lower_ibp_2[:, None], upper_ibp_2[:, None]], axis=1), + ] + + # backward_bounds: crown on 2nd model + crown_models_2 = [ + clone( + model=keras_model_2, + slope=slope, + perturbation_domain=BoxDomain(), + method=ConvertMethod.CROWN_FORWARD_IBP, + ) + for keras_model_2 in keras_models_2 + ] + symbolic_backward_bounds_flattened = [t for crown_model_2 in crown_models_2 for t in crown_model_2.outputs] + backward_bounds_flattened = [ + t + for crown_model_2, decomon_input_2 in zip(crown_models_2, decomon_inputs_2) + for t in crown_model_2(decomon_input_2) + ] + + # conversion of first model with backward_bounds + decomon_model = clone( + model=keras_model_1, + slope=slope, + perturbation_domain=perturbation_domain, + method=method, + backward_bounds=symbolic_backward_bounds_flattened, + ) + + # check output mode + ibp = decomon_model.ibp + affine = decomon_model.affine + + if method in (ConvertMethod.FORWARD_IBP, ConvertMethod.FORWARD_HYBRID): + assert ibp + else: + assert not ibp + + if method == ConvertMethod.FORWARD_IBP: + assert not affine + else: + assert affine + + # check number of outputs + nb_decomon_output_tensor_per_keras_output = 0 + if decomon_model.ibp: + nb_decomon_output_tensor_per_keras_output += 2 + if decomon_model.affine: + nb_decomon_output_tensor_per_keras_output += 4 + assert len(decomon_model.outputs) == len(keras_model_1.outputs) * nb_decomon_output_tensor_per_keras_output + + # call on actual inputs + keras_output = keras_model_tot(simple_model_keras_input) + decomon_output = decomon_model([simple_model_decomon_input] + backward_bounds_flattened) + + # check ibp and affine bounds well ordered w.r.t. keras inputs/outputs + for i in range(len(keras_model_1.outputs)): + keras_output_i = keras_output[i] + decomon_output_i = decomon_output[ + i * nb_decomon_output_tensor_per_keras_output : (i + 1) * nb_decomon_output_tensor_per_keras_output + ] + helpers.assert_decomon_output_compare_with_keras_input_output_model( + decomon_output=decomon_output_i, + keras_input=simple_model_keras_input, + keras_output=keras_output_i, + decimal=decimal, + ibp=ibp, + affine=affine, + ) From d94dbb5887f431205be2e186f7c14e1bbeb8a7d7 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 12 Mar 2024 15:11:30 +0100 Subject: [PATCH 079/101] Testing single tensor backward bound for adversarial robustness study --- tests/test_clone.py | 59 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/test_clone.py b/tests/test_clone.py index 9b3ffa95..e9b123db 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -424,3 +424,62 @@ def test_clone_w_backwardbounds_2outputs( ibp=ibp, affine=affine, ) + + +@parametrize("equal_ibp, input_shape", [(False, (5,))], ids=["1d"]) # fix some parameters of inputs +def test_clone_2outputs_with_backwardbounds_for_adv_box( + method, + final_ibp, + final_affine, + perturbation_domain, + equal_ibp, + input_shape, + simple_model_keras_symbolic_input, + simple_model_keras_input, + simple_model_decomon_input, + helpers, +): + # input shape? + input_shape = simple_model_keras_symbolic_input.shape[1:] + + slope = Slope.Z_SLOPE + decimal = 4 + + # keras model to convert + keras_model = helpers.toy_network_2outputs(input_shape=input_shape, same_output_shape=True) + output_shape = keras_model.outputs[0].shape[1:] + output_dim = int(np.prod(output_shape)) + + # create C for adversarial robustnedd + symbolic_C = Input(output_shape + output_shape) + batchsize = simple_model_decomon_input.shape[0] + C = K.reshape( + K.eye(output_dim)[None] - K.eye(batchsize, output_dim)[:, :, None], (-1,) + output_shape + output_shape + ) + + # conversion + decomon_model = clone( + model=keras_model, + slope=slope, + perturbation_domain=perturbation_domain, + method=method, + final_ibp=final_ibp, + final_affine=final_affine, + backward_bounds=symbolic_C, + ) + + assert final_ibp == decomon_model.ibp + assert final_affine == decomon_model.affine + + nb_decomon_output_tensor_per_keras_output = 0 + if decomon_model.ibp: + nb_decomon_output_tensor_per_keras_output += 2 + if decomon_model.affine: + nb_decomon_output_tensor_per_keras_output += 4 + assert len(decomon_model.outputs) == len(keras_model.outputs) * nb_decomon_output_tensor_per_keras_output + + # call on actual inputs + keras_output = keras_model(simple_model_keras_input) + decomon_output = decomon_model([simple_model_decomon_input, C]) + + # todo: check to perform on bounds? From a064ea397184a1c25864222cdb9a90cb4deb5911 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 12 Mar 2024 16:18:53 +0100 Subject: [PATCH 080/101] Manage special case of crown + identity model --- src/decomon/layers/input.py | 47 ++++++++++++++++++++++++ src/decomon/models/convert.py | 5 +++ tests/test_clone.py | 69 ++++++++++++++++++++++++++++++++++- 3 files changed, 119 insertions(+), 2 deletions(-) diff --git a/src/decomon/layers/input.py b/src/decomon/layers/input.py index 0cead2a7..302e0306 100644 --- a/src/decomon/layers/input.py +++ b/src/decomon/layers/input.py @@ -105,6 +105,53 @@ def compute_output_shape( ) +class IdentityInput(Layer): + """Layer generating identity affine bounds.""" + + def __init__( + self, + perturbation_domain: PerturbationDomain, + **kwargs: Any, + ): + """ + Args: + perturbation_domain: + **kwargs: + + """ + super().__init__(**kwargs) + + self.perturbation_domain = perturbation_domain + + def call(self, inputs: BackendTensor) -> list[BackendTensor]: + """Generate ibp and affine bounds to propagate by the first forward layer. + + Args: + inputs: the perturbation domain input + + Returns: + affine_bounds: [w, 0, w, 0], with w the identity tensor of the proper shape + (without batchsize, in diagonal representation) + + """ + keras_input_like_tensor_wo_batchsize = self.perturbation_domain.get_kerasinputlike_from_x(x=inputs)[0] + w = K.ones_like(keras_input_like_tensor_wo_batchsize) + b = K.zeros_like(keras_input_like_tensor_wo_batchsize) + return [w, b, w, b] + + def compute_output_shape( + self, + input_shape: tuple[Optional[int], ...], + ) -> list[tuple[Optional[int], ...]]: + perturbation_domain_input_shape_wo_batchsize = input_shape[1:] + keras_input_shape_wo_batchsize = self.perturbation_domain.get_keras_input_shape_wo_batchsize( + x_shape=perturbation_domain_input_shape_wo_batchsize + ) + w_shape = keras_input_shape_wo_batchsize + b_shape = keras_input_shape_wo_batchsize + return [w_shape, b_shape, w_shape, b_shape] + + class BackwardInput(Layer): """Layer preprocessing backward bounds to be used as input of the first backward layer of a decomon model. diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index c22e7dbb..acd06eac 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -18,6 +18,7 @@ from decomon.layers.fuse import Fuse from decomon.layers.input import ( BackwardInput, + IdentityInput, flatten_backward_bounds, has_no_backward_bounds, ) @@ -329,6 +330,10 @@ def clone( **kwargs, ) + # model ~ identity: in backward propagation, output can still be empty => diag representation of identity + if len(output) == 0: # identity bounds propagated as is (only if successive identity layers) + output = IdentityInput(perturbation_domain=perturbation_domain)(perturbation_domain_input) + # full linear model? => batch independent output if any([not isinstance(o, keras.KerasTensor) for o in output]): logger.warning( diff --git a/tests/test_clone.py b/tests/test_clone.py index e9b123db..8ac67d23 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -1,11 +1,12 @@ import keras.ops as K import numpy as np import pytest -from keras.layers import Input +from keras.layers import Activation, Input from keras.models import Model from pytest_cases import parametrize -from decomon.core import BoxDomain, ConvertMethod, Slope +from decomon.core import BoxDomain, ConvertMethod, Propagation, Slope +from decomon.layers.input import IdentityInput from decomon.layers.utils.symbolify import LinkToPerturbationDomainInput from decomon.models.convert import clone @@ -483,3 +484,67 @@ def test_clone_2outputs_with_backwardbounds_for_adv_box( decomon_output = decomon_model([simple_model_decomon_input, C]) # todo: check to perform on bounds? + + +def test_clone_identity_model( + method, + perturbation_domain, + model_keras_symbolic_input, + model_keras_input, + model_decomon_input, + helpers, +): + slope = Slope.Z_SLOPE + decimal = 4 + + # identity model + output_tensor = Activation(activation=None)(model_keras_symbolic_input) + keras_model = Model(model_keras_symbolic_input, output_tensor) + + # conversion + decomon_model = clone(model=keras_model, slope=slope, perturbation_domain=perturbation_domain, method=method) + + # call on actual outputs + keras_output = keras_model(model_keras_input) + decomon_output = decomon_model(model_decomon_input) + + ibp = decomon_model.ibp + affine = decomon_model.affine + + if method in (ConvertMethod.FORWARD_IBP, ConvertMethod.FORWARD_HYBRID): + assert ibp + else: + assert not ibp + + if method == ConvertMethod.FORWARD_IBP: + assert not affine + else: + assert affine + + # check ibp and affine bounds well ordered w.r.t. keras inputs/outputs + helpers.assert_decomon_output_compare_with_keras_input_output_model( + decomon_output=decomon_output, + keras_input=model_keras_input, + keras_output=keras_output, + decimal=decimal, + ibp=ibp, + affine=affine, + ) + + # check exact bounds + if affine: + # identity + w_l, b_l, w_u, b_u = decomon_output[:4] + helpers.assert_almost_equal(b_l, 0.0) + helpers.assert_almost_equal(b_u, 0.0) + helpers.assert_almost_equal(w_l, 1.0) + helpers.assert_almost_equal(w_u, 1.0) + if ibp: + # perturbation domain bounds + lower, upper = decomon_output[-2:] + helpers.assert_almost_equal(lower, perturbation_domain.get_lower_x(model_decomon_input)) + helpers.assert_almost_equal(upper, perturbation_domain.get_upper_x(model_decomon_input)) + + # check that we added a layer to insert batch axis + if method.lower().startswith("crown"): + assert isinstance(decomon_model.layers[-1], IdentityInput) From ba23702becdd1f8d6d7e2f2e4c25155cfc0cedcd Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 12 Mar 2024 17:01:39 +0100 Subject: [PATCH 081/101] Xfail test_clone with crown + add model + standard-multid inputs --- tests/conftest.py | 21 +++++++++++++-------- tests/test_clone.py | 9 +++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2623d1ea..0647b383 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1725,8 +1725,9 @@ def simple_model_inputs(simple_model_input_functions, input_shape): decomon_symbolic_input = decomon_symbolic_input_fn(keras_symbolic_input) keras_input = keras_input_fn(keras_symbolic_input) decomon_input = decomon_input_fn(keras_input) + metadata = dict(name="simple") - return keras_symbolic_input, decomon_symbolic_input, keras_input, decomon_input + return keras_symbolic_input, decomon_symbolic_input, keras_input, decomon_input, metadata ( @@ -1734,13 +1735,14 @@ def simple_model_inputs(simple_model_input_functions, input_shape): simple_model_decomon_symbolic_input, simple_model_keras_input, simple_model_decomon_input, + simple_model_decomon_input_metadata, ) = unpack_fixture( - "simple_model_keras_symbolic_input, simple_model_decomon_symbolic_input, simple_model_keras_input, simple_model_decomon_input", + "simple_model_keras_symbolic_input, simple_model_decomon_symbolic_input, simple_model_keras_input, simple_model_decomon_input, simple_model_decomon_input_metadata", simple_model_inputs, ) -def convert_standard_inputs_for_model(get_tensor_decomposition_fn, get_standard_values_fn): +def convert_standard_inputs_for_model(get_tensor_decomposition_fn, get_standard_values_fn, metadata): x, y, z, u_c, w_u, b_u, l_c, w_l, b_l = get_tensor_decomposition_fn() x_, y_, z_, u_c_, w_u_, b_u_, l_c_, w_l_, b_l_ = get_standard_values_fn() @@ -1749,21 +1751,23 @@ def convert_standard_inputs_for_model(get_tensor_decomposition_fn, get_standard_ decomon_symbolic_input = K.concatenate([l_c[:, None], u_c[:, None]], axis=1) decomon_input = K.convert_to_tensor(np.concatenate((l_c_[:, None], u_c_[:, None]), axis=1)) - return keras_symbolic_input, decomon_symbolic_input, keras_input, decomon_input + return keras_symbolic_input, decomon_symbolic_input, keras_input, decomon_input, metadata @fixture def standard_model_inputs_0d(n, batchsize, helpers): get_tensor_decomposition_fn = helpers.get_tensor_decomposition_0d_box get_standard_values_fn = lambda: helpers.get_standard_values_0d_box(n=n, batchsize=batchsize) - return convert_standard_inputs_for_model(get_tensor_decomposition_fn, get_standard_values_fn) + metadata = dict(name="standard-0d", n=n) + return convert_standard_inputs_for_model(get_tensor_decomposition_fn, get_standard_values_fn, metadata) @fixture def standard_model_inputs_1d(odd, batchsize, helpers): get_tensor_decomposition_fn = lambda: helpers.get_tensor_decomposition_1d_box(odd=odd) get_standard_values_fn = lambda: helpers.get_standard_values_1d_box(odd=odd, batchsize=batchsize) - return convert_standard_inputs_for_model(get_tensor_decomposition_fn, get_standard_values_fn) + metadata = dict(name="standard-1d", odd=odd) + return convert_standard_inputs_for_model(get_tensor_decomposition_fn, get_standard_values_fn, metadata) @fixture @@ -1773,7 +1777,8 @@ def standard_model_inputs_multid(data_format, batchsize, helpers): get_standard_values_fn = lambda: helpers.get_standard_values_images_box( data_format=data_format, odd=odd, m0=m0, m1=m1, batchsize=batchsize ) - return convert_standard_inputs_for_model(get_tensor_decomposition_fn, get_standard_values_fn) + metadata = dict(name="standard-multid", data_format=data_format) + return convert_standard_inputs_for_model(get_tensor_decomposition_fn, get_standard_values_fn, metadata) model_inputs = fixture_union( @@ -1784,7 +1789,7 @@ def standard_model_inputs_multid(data_format, batchsize, helpers): standard_model_inputs_1d, standard_model_inputs_multid, ], - unpack_into="model_keras_symbolic_input, model_decomon_symbolic_input, model_keras_input, model_decomon_input", + unpack_into="model_keras_symbolic_input, model_decomon_symbolic_input, model_keras_input, model_decomon_input, model_decomon_input_metadata", ) diff --git a/tests/test_clone.py b/tests/test_clone.py index 8ac67d23..6acd3623 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -45,6 +45,7 @@ def test_clone( model_keras_symbolic_input, model_keras_input, model_decomon_input, + model_decomon_input_metadata, helpers, ): # input shape? @@ -54,6 +55,14 @@ def test_clone( if toy_model_name == "cnn" and len(input_shape) == 1: pytest.skip("cnn not possible on 0d or 1d input.") + # xfail add model with standard multid input for now (memory issues to be fixed) + if ( + model_decomon_input_metadata["name"] == "standard-multid" + and toy_model_name == "add" + and method.lower().startswith("crown") + ): + pytest.xfail("crown on 'add' toy model crashed sometimes with standard-multid, to be investigated.") + slope = Slope.Z_SLOPE decimal = 4 From a9ef3e1dc023b41ca35594daad04bb39da125737 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 12 Mar 2024 17:22:35 +0100 Subject: [PATCH 082/101] Change color of decomon specific layers in plot_model Add also a notebook showing graphs of several decomon models. With an example of customization to see more attributes and change color of a specific layer. --- .../visualization/model_visualization.py | 69 +- tests/visu-decomon.ipynb | 678 ++++++++++++++++++ 2 files changed, 730 insertions(+), 17 deletions(-) create mode 100644 tests/visu-decomon.ipynb diff --git a/src/decomon/visualization/model_visualization.py b/src/decomon/visualization/model_visualization.py index 45ce0868..8c401942 100644 --- a/src/decomon/visualization/model_visualization.py +++ b/src/decomon/visualization/model_visualization.py @@ -1,5 +1,39 @@ from keras.src.utils.model_visualization import * +from decomon.layers.crown import ReduceCrownBounds +from decomon.layers.fuse import Fuse +from decomon.layers.input import BackwardInput, ForwardInput, IdentityInput +from decomon.layers.oracle import DecomonOracle +from decomon.layers.output import ConvertOutput +from decomon.layers.utils.batchsize import InsertBatchAxis +from decomon.layers.utils.symbolify import LinkToPerturbationDomainInput + +decomon_utilitary_layers_default_style = dict(bgcolor="#82E0AA", color="black") +decomon_utilitary_layers = [ + LinkToPerturbationDomainInput, + InsertBatchAxis, + ReduceCrownBounds, + Fuse, + BackwardInput, + IdentityInput, + ForwardInput, + ConvertOutput, + DecomonOracle, +] +decomon_utilitary_layers_styles = {l: decomon_utilitary_layers_default_style for l in decomon_utilitary_layers} +show_decomon_layer_attributes = { + "propagation": { + "forward": { + "bgcolor": "#b3e6ff", + }, + "backward": { + "bgcolor": "#ffccdd", + }, + None: {"color": "black"}, + }, + "layer.name": {}, +} + def get_layer_attribute(layer, name): subnames = name.split(".") @@ -16,10 +50,11 @@ def make_layer_label(layer, **kwargs): show_layer_names = kwargs.pop("show_layer_names") show_layer_activations = kwargs.pop("show_layer_activations") - show_layer_attributes = kwargs.pop("show_layer_attributes", []) + show_layer_attributes = kwargs.pop("show_layer_attributes", {}) show_dtype = kwargs.pop("show_dtype") show_shapes = kwargs.pop("show_shapes") show_trainable = kwargs.pop("show_trainable") + layer_styles = kwargs.pop("layer_styles", {}) if kwargs: raise ValueError(f"Invalid kwargs: {kwargs}") @@ -27,17 +62,21 @@ def make_layer_label(layer, **kwargs): colspan = max(1, sum(int(x) for x in (show_dtype, show_shapes, show_trainable))) + # style for layer node, according to its class, as specified in layer_styles + layer_style_to_apply = {"color": "white", "bgcolor": "black"} # default style + layer_style_to_apply.update(layer_styles.get(type(layer), {})) + if show_layer_names: table += ( - f'' - '' + f'' + f'' f"{layer.name} ({class_name})" "" ) else: table += ( - f'' - '' + f'' + f'' f"{class_name}" "" ) @@ -122,6 +161,7 @@ def model_to_dot( show_layer_activations=False, show_trainable=False, show_layer_attributes=None, + layer_styles=None, **kwargs, ): """Convert a Keras model to dot format. @@ -187,6 +227,7 @@ def model_to_dot( "show_shapes": show_shapes, "show_trainable": show_trainable, "show_layer_attributes": show_layer_attributes, + "layer_styles": layer_styles, } if isinstance(model, sequential.Sequential): @@ -217,6 +258,7 @@ def model_to_dot( show_layer_activations=show_layer_activations, show_trainable=show_trainable, show_layer_attributes=show_layer_attributes, + layer_styles=layer_styles, ) # sub_n : submodel sub_n_nodes = submodel.get_nodes() @@ -288,6 +330,7 @@ def plot_model( show_layer_activations=False, show_trainable=False, show_layer_attributes=None, + layer_styles=None, **kwargs, ): """Converts a Keras model to dot format and save to a file. @@ -359,18 +402,9 @@ def plot_model( raise ValueError(f"Unrecognized keyword arguments: {kwargs}") if show_layer_attributes is None: - show_layer_attributes = { - "propagation": { - "forward": { - "bgcolor": "#b3e6ff", - }, - "backward": { - "bgcolor": "#ffccdd", - }, - None: {"color": "black"}, - }, - "layer.name": {}, - } + show_layer_attributes = show_decomon_layer_attributes + if layer_styles is None: + layer_styles = decomon_utilitary_layers_styles dot = model_to_dot( model, @@ -383,6 +417,7 @@ def plot_model( show_layer_activations=show_layer_activations, show_trainable=show_trainable, show_layer_attributes=show_layer_attributes, + layer_styles=layer_styles, ) to_file = str(to_file) if dot is None: diff --git a/tests/visu-decomon.ipynb b/tests/visu-decomon.ipynb new file mode 100644 index 00000000..5f4b51e7 --- /dev/null +++ b/tests/visu-decomon.ipynb @@ -0,0 +1,678 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "08f5910b-8cbc-4094-84f4-e066cd28b1be", + "metadata": {}, + "source": [ + "# Visualization of Decomon Models" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "caa9eb31-b384-4351-9f73-c73d5ffd39ef", + "metadata": {}, + "outputs": [], + "source": [ + "from conftest import *\n", + "\n", + "helpers = Helpers()\n", + "from keras import Model, Sequential\n", + "from keras.activations import linear, serialize\n", + "from keras.layers import Activation, Add, Dense, Identity, Input, Layer, Subtract\n", + "\n", + "from decomon.layers.core.dense import DecomonDense\n", + "from decomon.layers.utils.batchsize import InsertBatchAxis\n", + "from decomon.models.convert import clone\n", + "from decomon.visualization.model_visualization import plot_model" + ] + }, + { + "cell_type": "markdown", + "id": "1d87858c-3624-4e6e-a952-ac2af71b8879", + "metadata": { + "jp-MarkdownHeadingCollapsed": true + }, + "source": [ + "## Linear model\n", + "\n", + "All activations are linear." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4eb3df47-8aca-48ba-8027-31f4a829d309", + "metadata": {}, + "outputs": [], + "source": [ + "input_shape = (5, 2, 3)\n", + "dtype = \"float32\"\n", + "toy_model = Helpers.toy_network_tutorial(input_shape=input_shape, dtype=dtype, activation=None)\n", + "plot_model(toy_model)" + ] + }, + { + "cell_type": "markdown", + "id": "4c259a7d-e898-4fef-a95b-a52a9dbcf1a5", + "metadata": {}, + "source": [ + "### forward ibp" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11bce920-f435-41eb-873b-bc5fee101584", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.FORWARD_IBP\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "8e642dc6-6736-4ab0-9757-3acacbfe72b6", + "metadata": {}, + "source": [ + "### forward affine" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b20dc58-febb-4463-ad03-d22921978d0c", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.FORWARD_AFFINE\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "69c34829-9aa8-41eb-a7a5-4cc8d7506585", + "metadata": {}, + "source": [ + "### forward hybrid" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e264918-53e7-45ee-89ac-13c8b1fb2e34", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.FORWARD_HYBRID\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "2a3feddf-d25f-45fc-9f74-182a785bb5db", + "metadata": {}, + "source": [ + "### CROWN - forward hybrid" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "829c119f-d7bd-42ec-8f1a-a5d4354758d1", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.CROWN_FORWARD_HYBRID\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "075d5534-805c-4904-ab3f-78d63eabd649", + "metadata": {}, + "source": [ + "### CROWN - full recursive" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "32113da8-efba-4c37-b9e2-05fe19267bd3", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.CROWN\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "313de968-53e4-4eed-a364-22451072f76c", + "metadata": {}, + "source": [ + "## Tutorial model\n", + "\n", + "Some activations are `relu`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c2cfb12-fded-41e9-83f6-2e3432b006b8", + "metadata": {}, + "outputs": [], + "source": [ + "input_shape = (5, 2, 3)\n", + "dtype = \"float32\"\n", + "toy_model = Helpers.toy_network_tutorial(input_shape=input_shape, dtype=dtype, activation=\"relu\")\n", + "plot_model(toy_model)" + ] + }, + { + "cell_type": "markdown", + "id": "359f85d7-e80f-4dab-91f7-f67f3ffbe8b3", + "metadata": {}, + "source": [ + "### forward ibp" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35b26ba6-7856-400e-8d6d-15bbbb2849f3", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.FORWARD_IBP\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "6be26887-31af-4ef6-b303-03e243f960a5", + "metadata": {}, + "source": [ + "### forward affine" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fb8e8c3f-cd32-4e03-b8da-d955d1412b92", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.FORWARD_AFFINE\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "1557848d-2359-4c75-9f45-eb319b9042ad", + "metadata": {}, + "source": [ + "### forward hybrid" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce38468a-03e2-4430-8101-b73e5386aab6", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.FORWARD_HYBRID\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "a05cf391-9229-4bf5-8571-c3a2fdf45d91", + "metadata": {}, + "source": [ + "### CROWN - forward affine" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72724491-545b-4d81-89e2-5cb6e7ed65db", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.CROWN_FORWARD_AFFINE\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "5bb7d6e2-3a40-4da8-abf7-e0232680ab03", + "metadata": {}, + "source": [ + "### CROWN - forward hybrid" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26f89451-8c5e-485c-beb7-903584d95f98", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.CROWN_FORWARD_HYBRID\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "13c6da86-5f49-4657-9444-b248fd0c63c2", + "metadata": {}, + "source": [ + "### CROWN - full recursive" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fb70b268-7124-492f-87d2-3f78d5a0ba4a", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.CROWN\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "86fa0adb-bafd-4f30-ab8e-5c39a1e27ac6", + "metadata": { + "jp-MarkdownHeadingCollapsed": true + }, + "source": [ + "## Merge v0 model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d9c020e6-9f88-41e2-bd6d-fd628fee66d2", + "metadata": {}, + "outputs": [], + "source": [ + "input_shape = (5, 2, 3)\n", + "dtype = \"float32\"\n", + "toy_model = Helpers.toy_struct_v0(\n", + " input_shape=input_shape, dtype=dtype, activation=\"relu\", archi=[2, 3, 2], use_bias=True\n", + ")\n", + "plot_model(toy_model)" + ] + }, + { + "cell_type": "markdown", + "id": "84d8e914-bb62-42ff-92ea-504614f41526", + "metadata": {}, + "source": [ + "### forward ibp" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d41346eb-c043-446c-b31a-996234ab5239", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.FORWARD_IBP\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "1d9b03ab-08e7-4eb6-9f92-538afe74d41d", + "metadata": {}, + "source": [ + "### forward affine" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5cac93d-9cc4-4d8b-a783-83da3ff425ad", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.FORWARD_AFFINE\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "c69afa79-e788-4ec7-8a0a-f3f04fec19c7", + "metadata": {}, + "source": [ + "### forward hybrid" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "756ce062-9ad8-4dd5-9898-11a328b7a254", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.FORWARD_HYBRID\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "03c4fe9e-102b-4b2a-bec5-e5a88a2a37af", + "metadata": {}, + "source": [ + "### CROWN - forward affine" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "78f8b820-654b-4708-9eca-60d9e7dcb276", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.CROWN_FORWARD_AFFINE\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "14b5631b-3010-4066-a71c-c15fa6015141", + "metadata": {}, + "source": [ + "### CROWN - forward hybrid" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "83e938db-e051-4450-9f85-4e08f7200ad3", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.CROWN_FORWARD_HYBRID\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "2ff11f24-11bd-404a-9b59-d4a3ee098965", + "metadata": {}, + "source": [ + "### CROWN - full recursive" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9004b065-1b20-4d07-b704-527d39b9fbc2", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.CROWN\n", + "decomon_model = clone(toy_model, method=method)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "10c352a9-a534-48d0-9299-cb97cd597ac5", + "metadata": {}, + "source": [ + "## Merge v0 model + precomputed backward_bounds" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e9c6482-0793-4e49-98bb-79833e32a387", + "metadata": {}, + "outputs": [], + "source": [ + "input_shape = (5, 2, 3)\n", + "dtype = \"float32\"\n", + "toy_model = Helpers.toy_struct_v0(\n", + " input_shape=input_shape, dtype=dtype, activation=\"relu\", archi=[2, 3, 2], use_bias=True\n", + ")\n", + "plot_model(toy_model)\n", + "\n", + "# backward_bounds: single tensor, diagonal representation of the weights (same for lower and upper bound, no bias)\n", + "backward_bounds = Input(toy_model.outputs[0].shape[1:], name=\"backward_bounds\")" + ] + }, + { + "cell_type": "markdown", + "id": "4dfe5ac2-5a5b-4873-bb70-6e26d734c028", + "metadata": {}, + "source": [ + "### forward ibp" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86600707-b827-4622-a1a4-a37007ac355a", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.FORWARD_IBP\n", + "decomon_model = clone(toy_model, method=method, backward_bounds=backward_bounds)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "c30bf36a-8896-414b-bd5b-545e4bc49112", + "metadata": {}, + "source": [ + "### forward affine" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4965005-7283-439b-9b59-25da706566cd", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.FORWARD_AFFINE\n", + "decomon_model = clone(toy_model, method=method, backward_bounds=backward_bounds)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "e71307af-3542-42a6-95be-5b31fb24cc40", + "metadata": {}, + "source": [ + "### forward hybrid" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b790f7c9-c7bb-4b3f-bfa3-b26eeae9b826", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.FORWARD_HYBRID\n", + "decomon_model = clone(toy_model, method=method, backward_bounds=backward_bounds)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "34fbb661-71ad-44fe-b5a3-ee34a81608d4", + "metadata": {}, + "source": [ + "### CROWN - forward affine" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ecb862c9-0229-4adb-8a13-d2e4c26a10af", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.CROWN_FORWARD_AFFINE\n", + "decomon_model = clone(toy_model, method=method, backward_bounds=backward_bounds)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "00fa9db4-1599-4c46-94d5-e9027126d63d", + "metadata": {}, + "source": [ + "### CROWN - forward hybrid" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "607d010d-45a9-4562-8016-bb7ea231d3e1", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.CROWN_FORWARD_HYBRID\n", + "decomon_model = clone(toy_model, method=method, backward_bounds=backward_bounds)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "1d5930b4-22a8-4617-ab0a-53e0edc15a5f", + "metadata": {}, + "source": [ + "### CROWN - full recursive" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e40c9b81-a8c6-439d-b3fe-0b04ba134a57", + "metadata": {}, + "outputs": [], + "source": [ + "method = ConvertMethod.CROWN\n", + "decomon_model = clone(toy_model, method=method, backward_bounds=backward_bounds)\n", + "plot_model(decomon_model)" + ] + }, + { + "cell_type": "markdown", + "id": "9ddd1d2d-6a0f-4483-9ef7-463d7bf3963f", + "metadata": {}, + "source": [ + "## Example of customization of `plot_model()`\n", + "\n", + "NB: colors can be set as in html code. See for instance https://htmlcolorcodes.com/ to pick colors.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "87995d1c-d88d-42ae-809f-d1c00c31a033", + "metadata": {}, + "outputs": [], + "source": [ + "input_shape = (5, 2, 3)\n", + "dtype = \"float32\"\n", + "toy_model = Helpers.toy_struct_v0(\n", + " input_shape=input_shape, dtype=dtype, activation=\"relu\", archi=[2, 3, 2], use_bias=True\n", + ")\n", + "method = ConvertMethod.CROWN_FORWARD_IBP\n", + "decomon_model = clone(toy_model, method=method)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0f7d946-4a21-4c39-8d5a-018a6e2c97ee", + "metadata": {}, + "outputs": [], + "source": [ + "from keras.activations import relu\n", + "\n", + "from decomon.layers.oracle import DecomonOracle\n", + "from decomon.visualization.model_visualization import (\n", + " decomon_utilitary_layers_styles,\n", + " show_decomon_layer_attributes,\n", + ")\n", + "\n", + "# layer styles\n", + "layer_styles = dict(decomon_utilitary_layers_styles) # start from default styles for decomon layers\n", + "layer_styles[DecomonOracle] = dict(bgcolor=\"#A569BD \", color=\"white\") # show DecomonOracle in purple/white\n", + "\n", + "# extra attributes to show\n", + "show_layer_attributes = dict(\n", + " show_decomon_layer_attributes\n", + ") # start with default decomon attributes (propagation, keras layer name)\n", + "show_layer_attributes[\"slope\"] = {None: {}} # display slope with default style\n", + "show_layer_attributes[\"layer.activation.__name__\"] = { # display keras layer activation name\n", + " None: dict(bgcolor=\"white\", color=\"black\"), # default: white/black\n", + " \"relu\": dict(bgcolor=\"#F5B7B1\", color=\"black\"), # relu: red/black\n", + "}\n", + "\n", + "plot_model(decomon_model, layer_styles=layer_styles, show_layer_attributes=show_layer_attributes)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b9bc27f6-1a42-4e0f-a383-61b530820903", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 12bafb8b8e62f01fdb79a1de26b81af2066212e8 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 14 Mar 2024 10:59:02 +0100 Subject: [PATCH 083/101] Clean old code - InputsOutputsSpecs: - remove all methods from previous api, - remove perturbation_domain attribute - move into decomon.layers.inputs_outputs_specs - ForwardMode: removed, replaced by ibp + affine booleans - PerturbationDomain and related stuff -> decomon.perturbation_domain - decomon.core -> decomon.constants: contains only enumerations - decomon.keras_utils: remove unused operations like BatchDiag and BatchIdentity - decomon.utils: keep only activation relaxations, moved into decomon.layers.activations.utils. Remove get_linear_hull_s_shape() that relies on old inputs_outputs_specs api (and thus need ForwardMode => cannot be imported) - decomon.metrics: removed as rely on previous api and will fail to import (need ForwardMode) - decomon.wrappers: kept but untested. Need to be adapted - decomon.wrappers_with_tuning: removed as need decomon.metrics - decomon.models.crown: removed. Specific layers for conversion (like ReduceCrown, Fuse, ...) are now in dedicated modules within decomon.layers --- src/decomon/constants.py | 42 + src/decomon/core.py | 1427 ----------------- src/decomon/keras_utils.py | 177 -- src/decomon/layers/activations/activation.py | 6 +- src/decomon/layers/activations/utils.py | 194 +++ src/decomon/layers/convert.py | 3 +- src/decomon/layers/crown.py | 4 +- src/decomon/layers/fuse.py | 8 +- src/decomon/layers/input.py | 5 +- src/decomon/layers/inputs_outputs_specs.py | 605 +++++++ src/decomon/layers/layer.py | 7 +- src/decomon/layers/oracle.py | 4 +- src/decomon/layers/output.py | 4 +- src/decomon/metrics/__init__.py | 10 - src/decomon/metrics/complexity.py | 19 - src/decomon/metrics/loss.py | 639 -------- src/decomon/metrics/metric.py | 454 ------ src/decomon/metrics/utils.py | 33 - src/decomon/models/backward_cloning.py | 27 +- src/decomon/models/convert.py | 10 +- src/decomon/models/crown.py | 186 --- src/decomon/models/forward_cloning.py | 14 +- src/decomon/models/models.py | 3 +- src/decomon/models/utils.py | 180 +-- src/decomon/perturbation_domain.py | 476 ++++++ src/decomon/utils.py | 722 --------- src/decomon/wrapper.py | 3 +- src/decomon/wrapper_with_tuning.py | 183 --- tests/conftest.py | 11 +- .../lirpa_comparison/test_comparison_lirpa.py | 2 +- tests/test_clone.py | 3 +- tests/test_convert_backward.py | 2 +- tests/test_convert_forward.py | 2 +- tests/test_decomon_layer.py | 3 +- tests/test_fuse.py | 3 +- tests/test_inputs.py | 2 +- tests/test_keras_utils.py | 59 +- tests/test_oracle.py | 2 +- tests/test_output.py | 4 +- tests/test_to_decomon.py | 3 +- tests/visu-decomon.ipynb | 2 +- 41 files changed, 1380 insertions(+), 4163 deletions(-) create mode 100644 src/decomon/constants.py delete mode 100644 src/decomon/core.py create mode 100644 src/decomon/layers/activations/utils.py create mode 100644 src/decomon/layers/inputs_outputs_specs.py delete mode 100644 src/decomon/metrics/__init__.py delete mode 100644 src/decomon/metrics/complexity.py delete mode 100644 src/decomon/metrics/loss.py delete mode 100644 src/decomon/metrics/metric.py delete mode 100644 src/decomon/metrics/utils.py delete mode 100644 src/decomon/models/crown.py create mode 100644 src/decomon/perturbation_domain.py delete mode 100644 src/decomon/utils.py delete mode 100644 src/decomon/wrapper_with_tuning.py diff --git a/src/decomon/constants.py b/src/decomon/constants.py new file mode 100644 index 00000000..4bbc291c --- /dev/null +++ b/src/decomon/constants.py @@ -0,0 +1,42 @@ +from enum import Enum + + +class Slope(str, Enum): + V_SLOPE = "volume-slope" + A_SLOPE = "adaptative-slope" + S_SLOPE = "same-slope" + Z_SLOPE = "zero-lb" + O_SLOPE = "one-lb" + + +class Propagation(str, Enum): + """Propagation direction.""" + + FORWARD = "forward" + BACKWARD = "backward" + + +class ConvertMethod(str, Enum): + CROWN = "crown" + """Crown fully recursive: backward propagation using crown oracle. + + (spawning subcrowns for each non-linear layer) + + """ + CROWN_FORWARD_IBP = "crown-forward-ibp" + """Crown + forward ibp: backward propagation using a forward-ibp oracle.""" + CROWN_FORWARD_AFFINE = "crown-forward-affine" + """Crown + forward ibp: backward propagation using a forward-affine oracle.""" + CROWN_FORWARD_HYBRID = "crown-forward-hybrid" + """Crown + forward ibp: backward propagation using a forward-hybrid oracle.""" + FORWARD_IBP = "forward-ibp" + """Forward propagation of constant bounds.""" + FORWARD_AFFINE = "forward-affine" + """Forward propagation of affine bounds.""" + FORWARD_HYBRID = "forward-hybrid" + """Forward propagation of constant+affine bounds. + + After each layer, the tightest constant bounds is keep between the ibp one + and the affine one combined with perturbation domain input. + + """ diff --git a/src/decomon/core.py b/src/decomon/core.py deleted file mode 100644 index c9c33959..00000000 --- a/src/decomon/core.py +++ /dev/null @@ -1,1427 +0,0 @@ -from abc import ABC, abstractmethod -from enum import Enum -from typing import Any, Optional, Union, overload - -import keras.ops as K -import numpy as np -from keras.config import floatx - -from decomon.keras_utils import add_tensors, batch_multid_dot -from decomon.types import Tensor - - -class Option(str, Enum): - lagrangian = "lagrangian" - milp = "milp" - - -class Slope(str, Enum): - V_SLOPE = "volume-slope" - A_SLOPE = "adaptative-slope" - S_SLOPE = "same-slope" - Z_SLOPE = "zero-lb" - O_SLOPE = "one-lb" - - -class PerturbationDomain(ABC): - opt_option: Option - - def __init__(self, opt_option: Union[str, Option] = Option.milp): - self.opt_option = Option(opt_option) - - @abstractmethod - def get_upper_x(self, x: Tensor) -> Tensor: - """Get upper constant bound on perturbation domain input.""" - ... - - @abstractmethod - def get_lower_x(self, x: Tensor) -> Tensor: - """Get lower constant bound on perturbation domain input.""" - ... - - @abstractmethod - def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: - """Merge upper affine bounds with perturbation domain input to get upper constant bound. - - Args: - x: perturbation domain input - w: weights of the affine bound - b: bias of the affine bound - missing_batchsize: whether w and b are missing batchsize - **kwargs: - - Returns: - - """ - ... - - @abstractmethod - def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: - """Merge lower affine bounds with perturbation domain input to get lower constant bound. - - Args: - x: perturbation domain input - w: weights of the affine bound - b: bias of the affine bound - missing_batchsize: whether w and b are missing batchsize - **kwargs: - - Returns: - - """ - ... - - @abstractmethod - def get_nb_x_components(self) -> int: - """Get the number of components in perturabation domain input. - - For instance: - - box domain: each corner of the box -> 2 components - - ball domain: center of the ball -> 1 component - - """ - ... - - @abstractmethod - def get_input_from_constant_bounds(self, constant_bounds: list[Tensor]) -> Tensor: - """Construct perturbation domain input x from constant bounds on keras model input - - Args: - constant_bounds: lower and upper constant bounds on keras model input - - Returns: - x: perturbation domain input - - """ - ... - - def get_config(self) -> dict[str, Any]: - return { - "opt_option": self.opt_option, - } - - def get_kerasinputlike_from_x(self, x: Tensor) -> Tensor: - """Get tensor of same shape as keras model input, from perturbation domain input x - - Args: - x: perturbation domain input - - Returns: - tensor of same shape as keras model input - - """ - if self.get_nb_x_components() == 1: - return x - else: - return x[:, 0] - - def get_x_input_shape_wo_batchsize(self, original_input_shape: tuple[int, ...]) -> tuple[int, ...]: - """Get expected perturbation domain input shape, excepting the batch axis.""" - n_comp_x = self.get_nb_x_components() - if n_comp_x == 1: - return original_input_shape - else: - return (n_comp_x,) + original_input_shape - - def get_keras_input_shape_wo_batchsize(self, x_shape: tuple[int, ...]) -> tuple[int, ...]: - """Deduce keras model input shape from perturbation domain input shape.""" - n_comp_x = self.get_nb_x_components() - if n_comp_x == 1: - return x_shape - else: - return x_shape[1:] - - -class BoxDomain(PerturbationDomain): - def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: - x_min = x[:, 0] - x_max = x[:, 1] - return get_upper_box(x_min=x_min, x_max=x_max, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) - - def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: - x_min = x[:, 0] - x_max = x[:, 1] - return get_lower_box(x_min=x_min, x_max=x_max, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) - - def get_upper_x(self, x: Tensor) -> Tensor: - return x[:, 1] - - def get_lower_x(self, x: Tensor) -> Tensor: - return x[:, 0] - - def get_nb_x_components(self) -> int: - return 2 - - def get_input_from_constant_bounds(self, constant_bounds: list[Tensor]) -> Tensor: - lower, upper = constant_bounds - return K.concatenate([lower[:, None], upper[:, None]], axis=1) - - -class GridDomain(PerturbationDomain): - pass - - -class VertexDomain(PerturbationDomain): - pass - - -class BallDomain(PerturbationDomain): - def __init__(self, eps: float, p: float = 2, opt_option: Option = Option.milp): - super().__init__(opt_option=opt_option) - self.eps = eps - # check on p - p_error_msg = "p must be a positive integer or np.inf" - try: - if p != np.inf and (int(p) != p or p <= 0): - raise ValueError(p_error_msg) - except: - raise ValueError(p_error_msg) - self.p = p - - def get_config(self) -> dict[str, Any]: - config = super().get_config() - config.update( - { - "eps": self.eps, - "p": self.p, - } - ) - return config - - def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: - return get_lower_ball(x_0=x, eps=self.eps, p=self.p, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) - - def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: - return get_upper_ball(x_0=x, eps=self.eps, p=self.p, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) - - def get_nb_x_components(self) -> int: - return 1 - - def get_lower_x(self, x: Tensor) -> Tensor: - return x - self.eps - - def get_upper_x(self, x: Tensor) -> Tensor: - return x + self.eps - - -class ForwardMode(str, Enum): - """The different forward (from input to output) linear based relaxation perturbation analysis.""" - - IBP = "ibp" - """Propagation of constant bounds from input to output.""" - - AFFINE = "affine" - """Propagation of affine bounds from input to output.""" - - HYBRID = "hybrid" - """Propagation of constant and affines bounds from input to output.""" - - -class Propagation(str, Enum): - """Propagation direction.""" - - FORWARD = "forward" - BACKWARD = "backward" - - -def get_mode(ibp: bool = True, affine: bool = True) -> ForwardMode: - if ibp: - if affine: - return ForwardMode.HYBRID - else: - return ForwardMode.IBP - else: - return ForwardMode.AFFINE - - -def get_ibp(mode: Union[str, ForwardMode] = ForwardMode.HYBRID) -> bool: - mode = ForwardMode(mode) - if mode in [ForwardMode.HYBRID, ForwardMode.IBP]: - return True - return False - - -def get_affine(mode: Union[str, ForwardMode] = ForwardMode.HYBRID) -> bool: - mode = ForwardMode(mode) - if mode in [ForwardMode.HYBRID, ForwardMode.AFFINE]: - return True - return False - - -class InputsOutputsSpec: - """Storing specifications for inputs and outputs of decomon/backward layer/model.""" - - layer_input_shape: Union[tuple[int, ...], list[tuple[int, ...]]] - model_input_shape: tuple[int, ...] - model_output_shape: tuple[int, ...] - - def __init__( - self, - ibp: bool = True, - affine: bool = True, - propagation: Propagation = Propagation.FORWARD, - perturbation_domain: Optional[PerturbationDomain] = None, - layer_input_shape: Optional[Union[tuple[int, ...], list[tuple[int, ...]]]] = None, - model_input_shape: Optional[tuple[int, ...]] = None, - model_output_shape: Optional[tuple[int, ...]] = None, - is_merging_layer: bool = False, - linear: bool = False, - ): - """ - Args: - perturbation_domain: type of perturbation domain (box, ball, ...). Default to a box domain - ibp: if True, forward propagate constant bounds - affine: if True, forward propagate affine bounds - propagation: direction of bounds propagation - - forward: from input to output - - backward: from output to input - layer_input_shape: shape of the underlying keras layer input (w/o the batch axis) - model_input_shape: shape of the underlying keras model input (w/o the batch axis) - model_output_shape: shape of the underlying keras model output (w/o the batch axis) - is_merging_layer: whether the underlying keras layer is a merging layer (i.e. with several inputs) - linear: whether the underlying keras layer is linear (thus do not need oracle bounds for instance) - - """ - # checks - if not ibp and not affine: - raise ValueError("ibp and affine cannot be both False.") - if propagation == Propagation.BACKWARD and model_output_shape is None: - raise ValueError("model_output_shape must be set in backward propagation.") - if propagation == Propagation.FORWARD or is_merging_layer: - if layer_input_shape is None: - raise ValueError("layer_input_shape must be set in forward propagation or for mergine layer.") - elif is_merging_layer: - if len(layer_input_shape) == 0 or not isinstance(layer_input_shape[0], tuple): - raise ValueError( - "layer_input_shape should be a non-empty list of shapes (tuple of int) for a merging layer." - ) - elif not isinstance(layer_input_shape, tuple) or ( - len(layer_input_shape) > 0 and not isinstance(layer_input_shape[0], int) - ): - raise ValueError("layer_input_shape should be a tuple of int for a unary layer.") - - self.propagation = propagation - self.affine = affine - self.ibp = ibp - self.is_merging_layer = is_merging_layer - self.linear = linear - self.perturbation_domain: PerturbationDomain - if perturbation_domain is None: - self.perturbation_domain = BoxDomain() - else: - self.perturbation_domain = perturbation_domain - if model_output_shape is None: - self.model_output_shape = tuple() - else: - self.model_output_shape = model_output_shape - if model_input_shape is None: - self.model_input_shape = tuple() - else: - self.model_input_shape = model_input_shape - if layer_input_shape is None: - if self.is_merging_layer: - self.layer_input_shape = [tuple()] - else: - self.layer_input_shape = tuple() - else: - self.layer_input_shape = layer_input_shape - - def needs_perturbation_domain_inputs(self) -> bool: - """Specify if decomon inputs should integrate keras model inputs.""" - return self.propagation == Propagation.FORWARD and self.affine - - def needs_oracle_bounds(self) -> bool: - """Specify if decomon layer needs oracle bounds on keras layer inputs.""" - return not self.linear and (self.propagation == Propagation.BACKWARD or self.affine) - - def needs_constant_bounds_inputs(self) -> bool: - """Specify if decomon inputs should integrate constant bounds.""" - return (self.propagation == Propagation.FORWARD and self.ibp) or ( - self.propagation == Propagation.BACKWARD and self.needs_oracle_bounds() - ) - - def needs_affine_bounds_inputs(self) -> bool: - """Specify if decomon inputs should integrate affine bounds.""" - return (self.propagation == Propagation.FORWARD and self.affine) or (self.propagation == Propagation.BACKWARD) - - def cannot_have_empty_affine_inputs(self) -> bool: - """Specify that it is not allowed to have empty affine bounds. - - Indeed, in merging case + forward propagation, it would be impossible to split decomon inputs properly. - - """ - return self.is_merging_layer and self.propagation == Propagation.FORWARD and self.affine - - @property - def nb_keras_inputs(self) -> int: - if self.is_merging_layer: - return len(self.layer_input_shape) - else: - return 1 - - @property - def nb_input_tensors(self) -> int: - nb = 0 - if self.propagation == Propagation.BACKWARD: - # oracle bounds - if self.needs_oracle_bounds(): - nb += 2 * self.nb_keras_inputs - # affine - nb += 4 - # model inputs - if self.needs_perturbation_domain_inputs(): - nb += 1 - else: # forward - # ibp - if self.ibp: - nb += 2 * self.nb_keras_inputs - # affine - if self.affine: - nb += 4 * self.nb_keras_inputs - # model inputs - if self.needs_perturbation_domain_inputs(): - nb += 1 - return nb - - @property - def nb_output_tensors(self) -> int: - nb = 0 - if self.propagation == Propagation.BACKWARD: - nb += 4 * self.nb_keras_inputs - else: # forward - if self.ibp: - nb += 2 - if self.affine: - nb += 4 - return nb - - @overload - def split_constant_bounds(self, constant_bounds: list[Tensor]) -> tuple[Tensor, Tensor]: - """Split constant bounds, non-merging layer version.""" - ... - - @overload - def split_constant_bounds(self, constant_bounds: list[list[Tensor]]) -> tuple[list[Tensor], list[Tensor]]: - """Split constant bounds, merging layer version.""" - ... - - def split_constant_bounds( - self, constant_bounds: Union[list[Tensor], list[list[Tensor]]] - ) -> Union[tuple[Tensor, Tensor], tuple[list[Tensor], list[Tensor]]]: - """Split constant bounds into lower, upper bound. - - Args: - constant_bounds: - if merging layer: list of constant (lower and upper) bounds for each keras layer inputs; - else: list containing lower and upper bounds for the keras layer input. - - Returns: - if merging_layer: 2 lists containing lower and upper bounds for each keras layer inputs; - else: 2 tensors being the lower and upper bounds for the keras layer input. - - """ - if self.is_merging_layer: - lowers, uppers = zip(*constant_bounds) - return list(lowers), list(uppers) - else: - lower, upper = constant_bounds - return lower, upper - - def split_inputs( - self, inputs: list[Tensor] - ) -> Union[ - tuple[list[Tensor], list[Tensor], list[Tensor]], - tuple[list[list[Tensor]], list[list[Tensor]], list[Tensor]], - tuple[list[Tensor], list[list[Tensor]], list[Tensor]], - ]: - """Split decomon inputs. - - Split them according to propagation mode and whether the underlying keras layer is merging or not. - - Args: - inputs: flattened decomon inputs, as seen by `DecomonLayer.call()`. - - Returns: - affine_bounds_to_propagate, constant_oracle_bounds, perturbation_domain_inputs: - each one can be empty if not relevant, - moreover, according to propagation mode and merging status, - it will be list of tensors or list of lists of tensors. - - More details: - - - non-merging case: - inputs = affine_bounds_to_propagate + constant_oracle_bounds + perturbation_domain_inputs - - - merging case: - - forward: k affine bounds to propagate w.r.t. each keras layer input + k constant bounds - - inputs = ( - affine_bounds_to_propagate_0 + constant_oracle_bounds_0 + ... - + affine_bounds_to_propagate_k + constant_oracle_bounds_k - + perturbation_domain_inputs - ) - - - backward: only 1 affine bounds to propagate w.r.t keras layer output - + k constant bounds w.r.t each keras layer input (empty if layer not linear) - - inputs = ( - affine_bounds_to_propagate - + constant_oracle_bounds_0 + ... + constant_oracle_bounds_k - + perturbation_domain_inputs - ) - Note: in case of merging layer + forward, we should not have empty affine bounds - as it will be impossible to split properly the inputs. - - """ - # Remove keras model input - if self.needs_perturbation_domain_inputs(): - x = inputs[-1] - inputs = inputs[:-1] - perturbation_domain_inputs = [x] - else: - perturbation_domain_inputs = [] - if self.is_merging_layer: - if self.propagation == Propagation.BACKWARD: - # expected number of constant bounds - nb_constant_bounds_by_keras_input = 2 if self.needs_oracle_bounds() else 0 - nb_constant_bounds = self.nb_keras_inputs * nb_constant_bounds_by_keras_input - # remove affine bounds (could be empty to express identity bounds) - affine_bounds_to_propagate = inputs[: len(inputs) - nb_constant_bounds] - inputs = inputs[len(inputs) - nb_constant_bounds :] - # split constant bounds by keras input - if nb_constant_bounds > 0: - constant_oracle_bounds = [ - [inputs[i], inputs[i + 1]] for i in range(0, len(inputs), nb_constant_bounds_by_keras_input) - ] - else: - constant_oracle_bounds = [] - else: # forward - # split bounds by keras input - nb_affine_bounds_by_keras_input = 4 if self.affine else 0 - nb_constant_bounds_by_keras_input = 2 if self.ibp else 0 - nb_bounds_by_keras_input = nb_affine_bounds_by_keras_input + nb_constant_bounds_by_keras_input - affine_bounds_to_propagate = [ - [inputs[start_input + j_bound] for j_bound in range(nb_affine_bounds_by_keras_input)] - for start_input in range(0, len(inputs), nb_bounds_by_keras_input) - ] - constant_oracle_bounds = [ - [ - inputs[start_input + nb_affine_bounds_by_keras_input + j_bound] - for j_bound in range(nb_constant_bounds_by_keras_input) - ] - for start_input in range(0, len(inputs), nb_bounds_by_keras_input) - ] - else: - # Remove constant bounds - if self.needs_constant_bounds_inputs(): - constant_oracle_bounds = inputs[-2:] - inputs = inputs[:-2] - else: - constant_oracle_bounds = [] - # The remaining tensors are affine bounds - # (potentially empty if: not backward or not affine or identity affine bounds) - affine_bounds_to_propagate = inputs - - return affine_bounds_to_propagate, constant_oracle_bounds, perturbation_domain_inputs - - def split_input_shape( - self, input_shape: list[tuple[Optional[int], ...]] - ) -> Union[ - tuple[list[tuple[Optional[int], ...]], list[tuple[Optional[int], ...]], list[tuple[Optional[int], ...]]], - tuple[ - list[list[tuple[Optional[int], ...]]], - list[list[tuple[Optional[int], ...]]], - list[tuple[Optional[int], ...]], - ], - tuple[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]], list[tuple[Optional[int], ...]]], - ]: - """Split decomon inputs. - - Split them according to propagation mode and whether the underlying keras layer is merging or not. - - Args: - input_shape: flattened decomon inputs, as seen by `DecomonLayer.call()`. - - Returns: - affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, perturbation_domain_inputs_shape: - each one can be empty if not relevant, and according to propagation mode and merging status, - it will be list of shapes or list of lists of shapes. - - """ - return self.split_inputs(inputs=input_shape) # type: ignore - - def flatten_inputs( - self, - affine_bounds_to_propagate: Union[list[Tensor], list[list[Tensor]]], - constant_oracle_bounds: Union[list[Tensor], list[list[Tensor]]], - perturbation_domain_inputs: list[Tensor], - ) -> list[Tensor]: - """Flatten decomon inputs. - - Reverse `self.split_inputs()`. - - Args: - affine_bounds_to_propagate: - - forward + affine: affine bounds on each keras layer input w.r.t. model input - -> list of lists of tensors in merging case; - -> list of tensors else. - - backward: affine bounds on model output w.r.t keras layer output - -> list of tensors - - else: empty - constant_oracle_bounds: - - forward + ibp: ibp bounds on keras layer inputs - - backward + not linear: oracle bounds on keras layer inputs - - else: empty - perturbation_domain_inputs: - - forward + affine: perturbation domain input wrapped in a list - - else: empty - - Returns: - flattened inputs - - non-merging case: - inputs = affine_bounds_to_propagate + constant_oracle_bounds + perturbation_domain_inputs - - - merging case: - - forward: k affine bounds to propagate w.r.t. each keras layer input + k constant bounds - - inputs = ( - affine_bounds_to_propagate_0 + constant_oracle_bounds_0 + ... - + affine_bounds_to_propagate_k + constant_oracle_bounds_k - + perturbation_domain_inputs - ) - - - backward: only 1 affine bounds to propagate w.r.t keras layer output - + k constant bounds w.r.t each keras layer input (empty of linear layer) - - inputs = ( - affine_bounds_to_propagate - + constant_oracle_bounds_0 + ... + constant_oracle_bounds_k - + perturbation_domain_inputs - ) - - """ - if self.is_merging_layer: - if self.propagation == Propagation.BACKWARD: - if self.needs_oracle_bounds(): - flattened_constant_oracle_bounds = [ - t for constant_oracle_bounds_i in constant_oracle_bounds for t in constant_oracle_bounds_i - ] - else: - flattened_constant_oracle_bounds = [] - return affine_bounds_to_propagate + flattened_constant_oracle_bounds + perturbation_domain_inputs - else: # forward - bounds_by_keras_input = [ - affine_bounds_to_propagate_i + constant_oracle_bounds_i - for affine_bounds_to_propagate_i, constant_oracle_bounds_i in zip( - affine_bounds_to_propagate, constant_oracle_bounds - ) - ] - flattened_bounds_by_keras_input = [ - t for bounds_by_keras_input_i in bounds_by_keras_input for t in bounds_by_keras_input_i - ] - return flattened_bounds_by_keras_input + perturbation_domain_inputs - else: - return affine_bounds_to_propagate + constant_oracle_bounds + perturbation_domain_inputs - - def flatten_inputs_shape( - self, - affine_bounds_to_propagate_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], - constant_oracle_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], - perturbation_domain_inputs_shape: list[tuple[Optional[int], ...]], - ) -> list[tuple[Optional[int], ...]]: - """Flatten inputs shape - - Same operation as `flatten_inputs` but on tensor shapes. - - Args: - affine_bounds_to_propagate_shape: - constant_oracle_bounds_shape: - perturbation_domain_inputs_shape: - - Returns: - - """ - return self.flatten_inputs( # type: ignore - affine_bounds_to_propagate=affine_bounds_to_propagate_shape, - constant_oracle_bounds=constant_oracle_bounds_shape, - perturbation_domain_inputs=perturbation_domain_inputs_shape, - ) # type: ignore - - def split_outputs(self, outputs: list[Tensor]) -> tuple[Union[list[Tensor], list[list[Tensor]]], list[Tensor]]: - """Split decomon inputs. - - Reverse operation of `self.flatten_outputs()` - - Args: - outputs: flattened decomon outputs, as returned by `DecomonLayer.call()`. - - Returns: - affine_bounds_propagated, constant_bounds_propagated: - each one can be empty if not relevant and can be list of tensors or a list of lists of tensors - according to propagation and merging status. - - More details: - - - forward: affine_bounds_propagated, constant_bounds_propagated: both simple lists of tensors corresponding to - affine and constant bounds on keras layer output. - - backward: constant_bounds_propagated is empty (not relevant) and - - merging layer: affine_bounds_propagated is a list of lists of tensors corresponding - to partial affine bounds on model output w.r.t each keras input - - else: affine_bounds_propagated is a simple list of tensors - - """ - # Remove constant bounds - if self.propagation == Propagation.FORWARD and self.ibp: - constant_bounds_propagated = outputs[-2:] - outputs = outputs[:-2] - else: - constant_bounds_propagated = [] - # It remains affine bounds (can be empty if forward + not affine, or identity layer (e.g. DecomonLinear) on identity bounds - affine_bounds_propagated = outputs - if self.propagation == Propagation.BACKWARD and self.is_merging_layer: - nb_affine_bounds_by_keras_input = 4 - affine_bounds_propagated = [ - affine_bounds_propagated[i : i + nb_affine_bounds_by_keras_input] - for i in range(0, len(affine_bounds_propagated), nb_affine_bounds_by_keras_input) - ] - - return affine_bounds_propagated, constant_bounds_propagated - - def split_output_shape( - self, output_shape: list[tuple[Optional[int], ...]] - ) -> tuple[ - Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], list[tuple[Optional[int], ...]] - ]: - """Split decomon output shape.""" - return self.split_outputs(outputs=output_shape) # type: ignore - - def flatten_outputs( - self, - affine_bounds_propagated: Union[list[Tensor], list[list[Tensor]]], - constant_bounds_propagated: Optional[list[Tensor]] = None, - ) -> list[Tensor]: - """Flatten decomon outputs. - - Args: - affine_bounds_propagated: - - forward + affine: affine bounds on keras layer output w.r.t. model input - - backward: affine bounds on model output w.r.t each keras layer input - -> list of lists of tensors in merging case; - -> list of tensors else. - - else: empty - constant_bounds_propagated: - - forward + ibp: ibp bounds on keras layer output - - else: empty or None - - Returns: - flattened outputs - - forward: affine_bounds_propagated + constant_bounds_propagated - - backward: - - merging layer (k keras layer inputs): affine_bounds_propagated_0 + ... + affine_bounds_propagated_k - - else: affine_bounds_propagated - - """ - if constant_bounds_propagated is None or self.propagation == Propagation.BACKWARD: - if self.is_merging_layer and self.propagation == Propagation.BACKWARD: - return [ - t for affine_bounds_propagated_i in affine_bounds_propagated for t in affine_bounds_propagated_i - ] - else: - return affine_bounds_propagated - else: - return affine_bounds_propagated + constant_bounds_propagated - - def flatten_outputs_shape( - self, - affine_bounds_propagated_shape: Union[ - list[tuple[Optional[int], ...]], - list[list[tuple[Optional[int], ...]]], - ], - constant_bounds_propagated_shape: Optional[list[tuple[Optional[int], ...]]] = None, - ) -> list[tuple[Optional[int], ...]]: - """Flatten decomon outputs shape.""" - return self.flatten_outputs(affine_bounds_propagated=affine_bounds_propagated_shape, constant_bounds_propagated=constant_bounds_propagated_shape) # type: ignore - - def has_multiple_bounds_inputs(self) -> bool: - return self.propagation == Propagation.FORWARD and self.is_merging_layer - - @overload - def extract_shapes_from_affine_bounds( - self, affine_bounds: list[Tensor], i: int = -1 - ) -> list[tuple[Optional[int], ...]]: - ... - - @overload - def extract_shapes_from_affine_bounds( - self, affine_bounds: list[list[Tensor]], i: int = -1 - ) -> list[list[tuple[Optional[int], ...]]]: - ... - - def extract_shapes_from_affine_bounds( - self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], i: int = -1 - ) -> Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]]: - if self.has_multiple_bounds_inputs() and i == -1: - return [[t.shape for t in sub_bounds] for sub_bounds in affine_bounds] - else: - return [t.shape for t in affine_bounds] # type: ignore - - def is_identity_bounds(self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], i: int = -1) -> bool: - return self.is_identity_bounds_shape( - affine_bounds_shape=self.extract_shapes_from_affine_bounds(affine_bounds=affine_bounds, i=i), i=i - ) - - def is_identity_bounds_shape( - self, - affine_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], - i: int = -1, - ) -> bool: - if self.has_multiple_bounds_inputs() and i == -1: - return all( - self.is_identity_bounds_shape(affine_bounds_shape=affine_bounds_shape[i], i=i) # type: ignore - for i in range(self.nb_keras_inputs) - ) - else: - return len(affine_bounds_shape) == 0 - - def is_diagonal_bounds(self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], i: int = -1) -> bool: - return self.is_diagonal_bounds_shape( - affine_bounds_shape=self.extract_shapes_from_affine_bounds(affine_bounds=affine_bounds, i=i), i=i - ) - - def is_diagonal_bounds_shape( - self, - affine_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], - i: int = -1, - ) -> bool: - if self.has_multiple_bounds_inputs() and i == -1: - return all( - self.is_diagonal_bounds_shape(affine_bounds_shape=affine_bounds_shape[i], i=i) # type: ignore - for i in range(self.nb_keras_inputs) - ) - else: - if self.is_identity_bounds_shape(affine_bounds_shape, i=i): - return True - w_shape, b_shape = affine_bounds_shape[:2] - return w_shape == b_shape - - def is_wo_batch_bounds(self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], i: int = -1) -> bool: - return self.is_wo_batch_bounds_shape( - affine_bounds_shape=self.extract_shapes_from_affine_bounds(affine_bounds=affine_bounds, i=i), i=i - ) - - def is_wo_batch_bounds_shape( - self, - affine_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], - i: int = -1, - ) -> bool: - if self.has_multiple_bounds_inputs() and i == -1: - return all( - self.is_wo_batch_bounds_shape(affine_bounds_shape=affine_bounds_shape[i], i=i) # type: ignore - for i in range(self.nb_keras_inputs) - ) - else: - if self.is_identity_bounds_shape(affine_bounds_shape, i=i): - return True - b_shape = affine_bounds_shape[1] - if self.propagation == Propagation.FORWARD: - if i > -1: - return len(b_shape) == len(self.layer_input_shape[i]) - else: - return len(b_shape) == len(self.layer_input_shape) - else: - return len(b_shape) == len(self.model_output_shape) - - @overload - def is_wo_batch_bounds_by_keras_input( - self, - affine_bounds: list[Tensor], - ) -> bool: - ... - - @overload - def is_wo_batch_bounds_by_keras_input( - self, - affine_bounds: list[list[Tensor]], - ) -> list[bool]: - ... - - def is_wo_batch_bounds_by_keras_input( - self, - affine_bounds: Union[list[Tensor], list[list[Tensor]]], - ) -> Union[bool, list[bool]]: - if self.has_multiple_bounds_inputs(): - return [self.is_wo_batch_bounds(affine_bounds_i, i=i) for i, affine_bounds_i in enumerate(affine_bounds)] - else: - return self.is_wo_batch_bounds(affine_bounds) - - def get_kerasinputshape(self, inputsformode: list[Tensor]) -> tuple[Optional[int], ...]: - return inputsformode[-1].shape - - def get_kerasinputshape_from_inputshapesformode( - self, inputshapesformode: list[tuple[Optional[int], ...]] - ) -> tuple[Optional[int], ...]: - return inputshapesformode[-1] - - def get_fullinputshapes_from_inputshapesformode( - self, - inputshapesformode: list[tuple[Optional[int], ...]], - ) -> list[tuple[Optional[int], ...]]: - nb_tensors = self.nb_tensors - empty_shape: tuple[Optional[int], ...] = tuple() - if self.dc_decomp: - if self.mode == ForwardMode.HYBRID: - ( - x_shape, - u_c_shape, - w_u_shape, - b_u_shape, - l_c_shape, - w_l_shape, - b_l_shape, - h_shape, - g_shape, - ) = inputshapesformode[:nb_tensors] - elif self.mode == ForwardMode.IBP: - u_c_shape, l_c_shape, h_shape, g_shape = inputshapesformode[:nb_tensors] - batchsize = u_c_shape[0] - x_shape = (batchsize,) + self.perturbation_domain.get_x_input_shape_wo_batchsize( - (self.model_input_dim,) - ) - b_shape = tuple(u_c_shape) - w_shape = tuple(u_c_shape) + (u_c_shape[-1],) - x_shape, w_u_shape, b_u_shape, w_l_shape, b_l_shape = ( - x_shape, - w_shape, - b_shape, - w_shape, - b_shape, - ) - elif self.mode == ForwardMode.AFFINE: - x_shape, w_u_shape, b_u_shape, w_l_shape, b_l_shape, h_shape, g_shape = inputshapesformode[:nb_tensors] - u_l_shape = tuple(b_u_shape) - u_c_shape, l_c_shape = u_l_shape, u_l_shape - else: - raise ValueError(f"Unknown mode {self.mode}") - else: - h_shape, g_shape = empty_shape, empty_shape - if self.mode == ForwardMode.HYBRID: - x_shape, u_c_shape, w_u_shape, b_u_shape, l_c_shape, w_l_shape, b_l_shape = inputshapesformode[ - :nb_tensors - ] - elif self.mode == ForwardMode.IBP: - u_c_shape, l_c_shape = inputshapesformode[:nb_tensors] - batchsize = u_c_shape[0] - x_shape = (batchsize,) + self.perturbation_domain.get_x_input_shape_wo_batchsize( - (self.model_input_dim,) - ) - b_shape = tuple(u_c_shape) - w_shape = tuple(u_c_shape) + (u_c_shape[-1],) - x_shape, w_u_shape, b_u_shape, w_l_shape, b_l_shape = ( - x_shape, - w_shape, - b_shape, - w_shape, - b_shape, - ) - elif self.mode == ForwardMode.AFFINE: - x_shape, w_u_shape, b_u_shape, w_l_shape, b_l_shape = inputshapesformode[:nb_tensors] - u_l_shape = tuple(b_u_shape) - u_c_shape, l_c_shape = u_l_shape, u_l_shape - else: - raise ValueError(f"Unknown mode {self.mode}") - - return [x_shape, u_c_shape, w_u_shape, b_u_shape, l_c_shape, w_l_shape, b_l_shape, h_shape, g_shape] - - def get_fullinputs_from_inputsformode( - self, inputsformode: list[Tensor], compute_ibp_from_affine: bool = True, tight: bool = True - ) -> list[Tensor]: - """ - - Args: - inputsformode: - compute_ibp_from_affine: if True and mode == affine, compute ibp bounds from affine ones - with get_upper/get_lower - tight: if True and mode==hybrid, compute tight ibp bounds, i.e. take tighter bound between - - the ones from inputs, and - - the ones computed from affine bounds with get_upper/get_lower - - Returns: - - """ - dtype = inputsformode[0].dtype - nb_tensors = self.nb_tensors - nonelike_tensor = self.get_empty_tensor(dtype=dtype) - if self.dc_decomp: - if self.mode == ForwardMode.HYBRID: - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputsformode[:nb_tensors] - elif self.mode == ForwardMode.IBP: - u_c, l_c, h, g = inputsformode[:nb_tensors] - x, w_u, b_u, w_l, b_l = ( - nonelike_tensor, - nonelike_tensor, - nonelike_tensor, - nonelike_tensor, - nonelike_tensor, - ) - elif self.mode == ForwardMode.AFFINE: - x, w_u, b_u, w_l, b_l, h, g = inputsformode[:nb_tensors] - u_c, l_c = nonelike_tensor, nonelike_tensor - else: - raise ValueError(f"Unknown mode {self.mode}") - else: - h, g = nonelike_tensor, nonelike_tensor - if self.mode == ForwardMode.HYBRID: - x, u_c, w_u, b_u, l_c, w_l, b_l = inputsformode[:nb_tensors] - elif self.mode == ForwardMode.IBP: - u_c, l_c = inputsformode[:nb_tensors] - x, w_u, b_u, w_l, b_l = ( - nonelike_tensor, - nonelike_tensor, - nonelike_tensor, - nonelike_tensor, - nonelike_tensor, - ) - elif self.mode == ForwardMode.AFFINE: - x, w_u, b_u, w_l, b_l = inputsformode[:nb_tensors] - u_c, l_c = nonelike_tensor, nonelike_tensor - else: - raise ValueError(f"Unknown mode {self.mode}") - - compute_ibp_from_affine = (compute_ibp_from_affine and self.mode == ForwardMode.AFFINE) or ( - tight and self.mode == ForwardMode.HYBRID - ) - - if compute_ibp_from_affine: - u_c_affine = self.perturbation_domain.get_upper(x, w_u, b_u) - l_c_affine = self.perturbation_domain.get_lower(x, w_l, b_l) - if self.mode == ForwardMode.AFFINE: - u_c = u_c_affine - l_c = l_c_affine - else: - u_c = K.minimum(u_c, u_c_affine) - l_c = K.maximum(l_c, l_c_affine) - - return [x, u_c, w_u, b_u, l_c, w_l, b_l, h, g] - - def get_fullinputs_by_type_from_inputsformode_to_merge( - self, inputsformode: list[Tensor], compute_ibp_from_affine: bool = False, tight: bool = True - ) -> list[list[Tensor]]: - """ - - Args: - inputsformode: - compute_ibp_from_affine: if True and mode == affine, compute ibp bounds from affine ones - with get_upper/get_lower - tight: if True and mode==hybrid, compute tight ibp bounds, i.e. take tighter bound between - - the ones from inputs, and - - the ones computed from affine bounds with get_upper/get_lower - - Returns: - - """ - dtype = inputsformode[0].dtype - nb_tensors_by_input = self.nb_tensors - nb_inputs = len(inputsformode) // nb_tensors_by_input - nonelike_tensor = self.get_empty_tensor(dtype=dtype) - nonelike_tensor_list = [nonelike_tensor] * nb_inputs - if self.mode == ForwardMode.HYBRID: - inputs_x = inputsformode[0::nb_tensors_by_input] - inputs_u_c = inputsformode[1::nb_tensors_by_input] - inputs_w_u = inputsformode[2::nb_tensors_by_input] - inputs_b_u = inputsformode[3::nb_tensors_by_input] - inputs_l_c = inputsformode[4::nb_tensors_by_input] - inputs_w_l = inputsformode[5::nb_tensors_by_input] - inputs_b_l = inputsformode[6::nb_tensors_by_input] - elif self.mode == ForwardMode.IBP: - inputs_u_c = inputsformode[0::nb_tensors_by_input] - inputs_l_c = inputsformode[1::nb_tensors_by_input] - inputs_x, inputs_w_u, inputs_b_u, inputs_w_l, inputs_b_l = ( - nonelike_tensor_list, - nonelike_tensor_list, - nonelike_tensor_list, - nonelike_tensor_list, - nonelike_tensor_list, - ) - elif self.mode == ForwardMode.AFFINE: - inputs_x = inputsformode[0::nb_tensors_by_input] - inputs_w_u = inputsformode[1::nb_tensors_by_input] - inputs_b_u = inputsformode[2::nb_tensors_by_input] - inputs_w_l = inputsformode[3::nb_tensors_by_input] - inputs_b_l = inputsformode[4::nb_tensors_by_input] - inputs_u_c, inputs_l_c = nonelike_tensor_list, nonelike_tensor_list - else: - raise ValueError(f"Unknown mode {self.mode}") - - if self.dc_decomp: - inputs_h = inputsformode[nb_tensors_by_input - 2 :: nb_tensors_by_input] - inputs_g = inputsformode[nb_tensors_by_input - 1 :: nb_tensors_by_input] - else: - inputs_h, inputs_g = nonelike_tensor_list, nonelike_tensor_list - - # compute ibp bounds from affine bounds - compute_ibp_from_affine = (compute_ibp_from_affine and self.mode == ForwardMode.AFFINE) or ( - tight and self.mode == ForwardMode.HYBRID - ) - - if compute_ibp_from_affine: - for i in range(len(inputs_x)): - u_c_affine = self.perturbation_domain.get_upper(inputs_x[i], inputs_w_u[i], inputs_b_u[i]) - l_c_affine = self.perturbation_domain.get_lower(inputs_x[i], inputs_w_l[i], inputs_b_l[i]) - if self.mode == ForwardMode.AFFINE: - inputs_u_c[i] = u_c_affine - inputs_l_c[i] = l_c_affine - else: - inputs_u_c[i] = K.minimum(inputs_u_c[i], u_c_affine) - inputs_l_c[i] = K.maximum(inputs_l_c[i], l_c_affine) - - return [inputs_x, inputs_u_c, inputs_w_u, inputs_b_u, inputs_l_c, inputs_w_l, inputs_b_l, inputs_h, inputs_g] - - def split_inputsformode_to_merge(self, inputsformode: list[Any]) -> list[list[Any]]: - n_comp = self.nb_tensors - return [inputsformode[n_comp * i : n_comp * (i + 1)] for i in range(len(inputsformode) // n_comp)] - - def extract_inputsformode_from_fullinputs(self, inputs: list[Tensor]) -> list[Tensor]: - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs - if self.mode == ForwardMode.HYBRID: - inputsformode = [x, u_c, w_u, b_u, l_c, w_l, b_l] - elif self.mode == ForwardMode.IBP: - inputsformode = [u_c, l_c] - elif self.mode == ForwardMode.AFFINE: - inputsformode = [x, w_u, b_u, w_l, b_l] - else: - raise ValueError(f"Unknown mode {self.mode}") - if self.dc_decomp: - inputsformode += [h, g] - return inputsformode - - def extract_inputshapesformode_from_fullinputshapes( - self, inputshapes: list[tuple[Optional[int], ...]] - ) -> list[tuple[Optional[int], ...]]: - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputshapes - if self.mode == ForwardMode.HYBRID: - inputshapesformode = [x, u_c, w_u, b_u, l_c, w_l, b_l] - elif self.mode == ForwardMode.IBP: - inputshapesformode = [u_c, l_c] - elif self.mode == ForwardMode.AFFINE: - inputshapesformode = [x, w_u, b_u, w_l, b_l] - else: - raise ValueError(f"Unknown mode {self.mode}") - if self.dc_decomp: - inputshapesformode += [h, g] - return inputshapesformode - - def extract_outputsformode_from_fulloutputs(self, outputs: list[Tensor]) -> list[Tensor]: - return self.extract_inputsformode_from_fullinputs(outputs) - - @staticmethod - def get_empty_tensor(dtype: Optional[str] = None) -> Tensor: - if dtype is None: - dtype = floatx() - return K.convert_to_tensor([], dtype=dtype) - - -def get_upper_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: - """Compute the max of an affine function - within a box (hypercube) defined by its extremal corners - - Args: - x_min: lower bound of the box domain - x_max: upper bound of the box domain - w: weights of the affine function - b: bias of the affine function - missing_batchsize: whether w and b are missing the batchsize - - Returns: - max_(x >= x_min, x<=x_max) w*x + b - - Note: - We can have w, b in diagonal representation and/or without a batch axis. - We assume that x_min, x_max have always its batch axis. - - """ - z_value = K.cast(0.0, dtype=x_min.dtype) - w_pos = K.maximum(w, z_value) - w_neg = K.minimum(w, z_value) - - is_diag = w.shape == b.shape - diagonal = (False, is_diag) - missing_batchsize = (False, missing_batchsize) - - return ( - batch_multid_dot(x_max, w_pos, diagonal=diagonal, missing_batchsize=missing_batchsize) - + batch_multid_dot(x_min, w_neg, diagonal=diagonal, missing_batchsize=missing_batchsize) - + b - ) - - -def get_lower_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: - """ - Args: - x_min: lower bound of the box domain - x_max: upper bound of the box domain - w: weights of the affine lower bound - b: bias of the affine lower bound - missing_batchsize: whether w and b are missing the batchsize - - Returns: - min_(x >= x_min, x<=x_max) w*x + b - - Note: - We can have w, b in diagonal representation and/or without a batch axis. - We assume that x_min, x_max have always its batch axis. - - """ - return get_upper_box(x_min=x_max, x_max=x_min, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) - - -def get_lq_norm(x: Tensor, p: float, axis: Union[int, list[int]] = -1) -> Tensor: - """compute Lp norm (p=1 or 2) - - Args: - x: tensor - p: the power must be an integer in (1, 2) - axis: the axis on which we compute the norm - - Returns: - ||w||^p - """ - if p == 1: - x_q = K.max(K.abs(x), axis) - elif p == 2: - x_q = K.sqrt(K.sum(K.power(x, p), axis)) - else: - raise NotImplementedError("p must be equal to 1 or 2") - - return x_q - - -def get_upper_ball( - x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any -) -> Tensor: - """max of an affine function over an Lp ball - - Args: - x_0: the center of the ball - eps: the radius - p: the type of Lp norm considered - w: weights of the affine function - b: bias of the affine function - missing_batchsize: whether w and b are missing the batchsize - - Returns: - max_(|x - x_0|_p<= eps) w*x + b - """ - if p == np.inf: - # compute x_min and x_max according to eps - x_min = x_0 - eps - x_max = x_0 + eps - return get_upper_box(x_min, x_max, w, b, missing_batchsize=missing_batchsize) - - else: - if len(kwargs): - return get_upper_ball_finetune(x_0, eps, p, w, b, missing_batchsize=missing_batchsize, **kwargs) - - # use Holder's inequality p+q=1 - # ||w||_q*eps + w*x_0 + b - - is_diag = w.shape == b.shape - - # lq-norm of w - if is_diag: - w_q = K.abs(w) - else: - nb_axes_wo_batchsize_x = len(x_0.shape) - 1 - if missing_batchsize: - reduced_axes = list(range(nb_axes_wo_batchsize_x)) - else: - reduced_axes = list(range(1, 1 + nb_axes_wo_batchsize_x)) - w_q = get_lq_norm(w, p, axis=reduced_axes) - - diagonal = (False, is_diag) - missing_batchsize = (False, missing_batchsize) - return batch_multid_dot(x_0, w, diagonal=diagonal, missing_batchsize=missing_batchsize) + b + w_q * eps - - -def get_lower_ball( - x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any -) -> Tensor: - """min of an affine fucntion over an Lp ball - - Args: - x_0: the center of the ball - eps: the radius - p: the type of Lp norm considered - w: weights of the affine function - b: bias of the affine function - missing_batchsize: whether w and b are missing the batchsize - - Returns: - min_(|x - x_0|_p<= eps) w*x + b - """ - if p == np.inf: - # compute x_min and x_max according to eps - x_min = x_0 - eps - x_max = x_0 + eps - return get_lower_box(x_min, x_max, w, b, missing_batchsize=missing_batchsize) - - else: - if len(kwargs): - return get_lower_ball_finetune(x_0, eps, p, w, b, missing_batchsize=missing_batchsize, **kwargs) - - # use Holder's inequality p+q=1 - # - ||w||_q*eps + w*x_0 + b - - is_diag = w.shape == b.shape - - # lq-norm of w - if is_diag: - w_q = K.abs(w) - else: - nb_axes_wo_batchsize_x = len(x_0.shape) - 1 - if missing_batchsize: - reduced_axes = list(range(nb_axes_wo_batchsize_x)) - else: - reduced_axes = list(range(1, 1 + nb_axes_wo_batchsize_x)) - w_q = get_lq_norm(w, p, axis=reduced_axes) - - diagonal = (False, is_diag) - missing_batchsize = (False, missing_batchsize) - return batch_multid_dot(x_0, w, diagonal=diagonal, missing_batchsize=missing_batchsize) + b - w_q * eps - - -def get_lower_ball_finetune( - x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any -) -> Tensor: - if missing_batchsize: - raise NotImplementedError() - - if "finetune_lower" in kwargs and "upper" in kwargs or "lower" in kwargs: - alpha = kwargs["finetune_lower"] - # assume alpha is the same shape as w, minus the batch dimension - n_shape = len(w.shape) - 2 - z_value = K.cast(0.0, dtype=w.dtype) - - if "upper" and "lower" in kwargs: - upper = kwargs["upper"] # flatten vector - lower = kwargs["lower"] # flatten vector - - upper_reshaped = np.reshape(upper, [1, -1] + [1] * n_shape) - lower_reshaped = np.reshape(lower, [1, -1] + [1] * n_shape) - - w_alpha = w * alpha[None] - w_alpha_bar = w * (1 - alpha) - - score_box = K.sum(K.maximum(z_value, w_alpha_bar) * lower_reshaped, 1) + K.sum( - K.minimum(z_value, w_alpha_bar) * upper_reshaped, 1 - ) - score_ball = get_lower_ball(x_0, eps, p, w_alpha, b) - - return score_box + score_ball - - if "upper" in kwargs: - upper = kwargs["upper"] # flatten vector - upper_reshaped = np.reshape(upper, [1, -1] + [1] * n_shape) - - w_alpha = K.minimum(z_value, w) * alpha[None] + K.maximum(z_value, w) - w_alpha_bar = K.minimum(z_value, w) * (1 - alpha[None]) - - score_box = K.sum(K.minimum(z_value, w_alpha_bar) * upper_reshaped, 1) - score_ball = get_lower_ball(x_0, eps, p, w_alpha, b) - - return score_box + score_ball - - if "lower" in kwargs: - lower = kwargs["lower"] # flatten vector - lower_reshaped = np.reshape(lower, [1, -1] + [1] * n_shape) - - w_alpha = K.maximum(z_value, w) * alpha[None] + K.minimum(z_value, w) - w_alpha_bar = K.maximum(z_value, w) * (1 - alpha[None]) - - score_box = K.sum(K.maximum(z_value, w_alpha_bar) * lower_reshaped, 1) - score_ball = get_lower_ball(x_0, eps, p, w_alpha, b) - - return score_box + score_ball - - return get_lower_ball(x_0, eps, p, w, b) - - -def get_upper_ball_finetune( - x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any -) -> Tensor: - if missing_batchsize: - raise NotImplementedError() - - if "finetune_upper" in kwargs and "upper" in kwargs or "lower" in kwargs: - alpha = kwargs["finetune_upper"] - # assume alpha is the same shape as w, minus the batch dimension - n_shape = len(w.shape) - 2 - z_value = K.cast(0.0, dtype=w.dtype) - - if "upper" and "lower" in kwargs: - upper = kwargs["upper"] # flatten vector - lower = kwargs["lower"] # flatten vector - - upper_reshaped = np.reshape(upper, [1, -1] + [1] * n_shape) - lower_reshaped = np.reshape(lower, [1, -1] + [1] * n_shape) - - w_alpha = w * alpha[None] - w_alpha_bar = w * (1 - alpha) - - score_box = K.sum(K.maximum(z_value, w_alpha_bar) * upper_reshaped, 1) + K.sum( - K.minimum(z_value, w_alpha_bar) * lower_reshaped, 1 - ) - score_ball = get_lower_ball(x_0, eps, p, w_alpha, b) - - return score_box + score_ball - - if "upper" in kwargs: - upper = kwargs["upper"] # flatten vector - upper_reshaped = np.reshape(upper, [1, -1] + [1] * n_shape) - - w_alpha = K.minimum(z_value, w) * alpha[None] + K.maximum(z_value, w) - w_alpha_bar = K.minimum(z_value, w) * (1 - alpha[None]) - - score_box = K.sum(K.maximum(z_value, w_alpha_bar) * upper_reshaped, 1) - score_ball = get_lower_ball(x_0, eps, p, w_alpha, b) - - return score_box + score_ball - - if "lower" in kwargs: - lower = kwargs["lower"] # flatten vector - lower_reshaped = np.reshape(lower, [1, -1] + [1] * n_shape) - - w_alpha = K.maximum(z_value, w) * alpha[None] + K.minimum(z_value, w) - w_alpha_bar = K.maximum(z_value, w) * (1 - alpha[None]) - - score_box = K.sum(K.minimum(z_value, w_alpha_bar) * lower_reshaped, 1) - score_ball = get_lower_ball(x_0, eps, p, w_alpha, b) - - return score_box + score_ball - - return get_upper_ball(x_0, eps, p, w, b) - - -class ConvertMethod(str, Enum): - CROWN = "crown" - """Crown fully recursive: backward propagation using crown oracle. - - (spawning subcrowns for each non-linear layer) - - """ - CROWN_FORWARD_IBP = "crown-forward-ibp" - """Crown + forward ibp: backward propagation using a forward-ibp oracle.""" - CROWN_FORWARD_AFFINE = "crown-forward-affine" - """Crown + forward ibp: backward propagation using a forward-affine oracle.""" - CROWN_FORWARD_HYBRID = "crown-forward-hybrid" - """Crown + forward ibp: backward propagation using a forward-hybrid oracle.""" - FORWARD_IBP = "forward-ibp" - """Forward propagation of constant bounds.""" - FORWARD_AFFINE = "forward-affine" - """Forward propagation of affine bounds.""" - FORWARD_HYBRID = "forward-hybrid" - """Forward propagation of constant+affine bounds. - - After each layer, the tightest constant bounds is keep between the ibp one - and the affine one combined with perturbation domain input. - - """ diff --git a/src/decomon/keras_utils.py b/src/decomon/keras_utils.py index 452f9579..daa56eb1 100644 --- a/src/decomon/keras_utils.py +++ b/src/decomon/keras_utils.py @@ -202,169 +202,10 @@ def _convert_from_diag_to_generic( return x_broadcastable, x_full_shape -class BatchedIdentityLike(keras.Operation): - """Keras Operation creating an identity tensor with shape (including batch_size) based on input. - - The output shape is tuple(x.shape) + (x.shape[-1],), the tensor being the identity - along the 2 last dimensions. - - """ - - def call(self, x: BackendTensor) -> Tensor: - input_shape = x.shape - identity_tensor = K.identity(input_shape[-1], dtype=x.dtype) - n_repeat = int(np.prod(input_shape[:-1])) - return K.reshape(K.repeat(identity_tensor[None], n_repeat, axis=0), tuple(input_shape) + (-1,)) - - def compute_output_spec(self, x: Tensor) -> keras.KerasTensor: - x_shape = x.shape - x_type = getattr(x, "dtype", type(x)) - x_sparse = getattr(x, "sparse", False) - return keras.KerasTensor( - shape=tuple(x_shape) + (x_shape[-1],), - dtype=x_type, - sparse=x_sparse, - ) - - -class BatchedDiagLike(keras.Operation): - """Keras Operation transforming last dimension into a diagonal tensor. - - The output shape is tuple(x.shape) + (x.shape[-1],). - When fixing all but 2 last dimensions, the output tensor is a square tensor - whose main diagonal is the input tensor with same first dimensions fixed, and 0 elsewhere. - - This is a replacement for tensorflow.linalg.diag(). - - """ - - def call(self, x: BackendTensor) -> Tensor: - return K.concatenate([K.diag(K.ravel(w_part))[None] for w_part in K.split(x, len(x), axis=0)], axis=0) - - def compute_output_spec(self, x: Tensor) -> keras.KerasTensor: - x_shape = x.shape - x_type = getattr(x, "dtype", type(x)) - x_sparse = getattr(x, "sparse", False) - return keras.KerasTensor( - shape=tuple(x_shape) + (x_shape[-1],), - dtype=x_type, - sparse=x_sparse, - ) - - def is_a_merge_layer(layer: Layer) -> bool: return hasattr(layer, "_merge_function") -def is_symbolic_tensor(x: Tensor) -> bool: - """Check whether the tensor is symbolic or not. - - Works even during backend calls made by layers without actual compute_output_shape(). - In this case, x is not KerasTensor anymore but a backend Tensor with None in its shape. - - """ - return None in x.shape - - -def get_weight_index(layer: Layer, weight: keras.Variable) -> int: - """Get weight index among layer tracked weights - - Args: - layer: layer we are looking - weight: weight supposed to be part of tracked weights by the layer - - Returns: - the index of the weight in `layer.weights` list - - Raises: - IndexError: if `weight` is not part of `layer.weights` - - """ - indexes = [i for i, w in enumerate(layer.weights) if w is weight] - try: - return indexes[0] - except IndexError: - raise IndexError(f"The weight {weight} is not tracked by the layer {layer}.") - - -def get_weight_index_from_name(layer: Layer, weight_name: str) -> int: - """Get weight index among layer tracked weights - - Args: - layer: layer we are looking - weight_name: name of the weight supposed to be part of tracked weights by the layer - - Returns: - the index of the weight in `layer.weights` list - - Raises: - AttributeError: if `weight_name` is not the name of an attribute of `layer` - IndexError: if the corresponding layer attribute is not part of `layer.weights` - - """ - weight = getattr(layer, weight_name) - try: - return get_weight_index(layer=layer, weight=weight) - except IndexError: - raise IndexError(f"The weight {weight_name} is not tracked by the layer {layer}.") - - -def reset_layer(new_layer: Layer, original_layer: Layer, weight_names: list[str]) -> None: - """Reset some weights of a layer by using the weights of another layer. - - Args: - new_layer: the decomon layer whose weights will be updated - original_layer: the layer used to update the weights - weight_names: the names of the weights to update - - Returns: - - """ - if not original_layer.built: - raise ValueError(f"the layer {original_layer.name} has not been built yet") - if not new_layer.built: - raise ValueError(f"the layer {new_layer.name} has not been built yet") - else: - new_params = new_layer.get_weights() - original_params = original_layer.get_weights() - for weight_name in weight_names: - new_params[get_weight_index_from_name(new_layer, weight_name)] = original_params[ - get_weight_index_from_name(original_layer, weight_name) - ] - new_layer.set_weights(new_params) - - -def reset_layer_all_weights(new_layer: Layer, original_layer: Layer) -> None: - """Reset all the weights of a layer by using the weights of another layer. - - Args: - new_layer: the decomon layer whose weights will be updated - original_layer: the layer used to update the weights - - Returns: - - """ - reset_layer(new_layer=new_layer, original_layer=original_layer, weight_names=[w.name for w in new_layer.weights]) - - -def share_layer_all_weights( - original_layer: Layer, - new_layer: Layer, -) -> None: - """Share all the weights of an already built layer to another unbuilt layer. - - Args: - original_layer: the layer used to share the weights - new_layer: the new layer which will be buit and will share the weights of the original layer - - Returns: - - """ - share_weights_and_build( - new_layer=new_layer, original_layer=original_layer, weight_names=[w.name for w in original_layer.weights] - ) - - def share_weights_and_build(original_layer: Layer, new_layer: Layer, weight_names: list[str]) -> None: """Share the weights specidifed by names of an already built layer to another unbuilt layer. @@ -413,21 +254,3 @@ def share_weights_and_build(original_layer: Layer, new_layer: Layer, weight_name setattr(new_layer, w_name, w) # untrack the not used anymore weight new_layer._tracker.untrack(w_to_drop) - - -def check_if_single_shape(shape: Any) -> bool: - """ - - Args: - input_shape: - - Returns: - - """ - if isinstance(shape, list) and shape and isinstance(shape[0], (int, type(None))): - return True - - if not isinstance(shape, (list, tuple, dict)): - shape = tuple(shape) - - return isinstance(shape, tuple) and len(shape) > 0 and isinstance(shape[0], (int, type(None))) diff --git a/src/decomon/layers/activations/activation.py b/src/decomon/layers/activations/activation.py index 54bbee2e..a65c25b9 100644 --- a/src/decomon/layers/activations/activation.py +++ b/src/decomon/layers/activations/activation.py @@ -6,11 +6,11 @@ from keras.activations import linear, relu from keras.layers import Activation -from decomon.core import PerturbationDomain, Propagation, Slope -from decomon.keras_utils import BatchedDiagLike +from decomon.constants import Propagation, Slope +from decomon.layers.activations.utils import get_linear_hull_relu from decomon.layers.layer import DecomonLayer +from decomon.perturbation_domain import PerturbationDomain from decomon.types import Tensor -from decomon.utils import get_linear_hull_relu class DecomonBaseActivation(DecomonLayer): diff --git a/src/decomon/layers/activations/utils.py b/src/decomon/layers/activations/utils.py new file mode 100644 index 00000000..747b7e5f --- /dev/null +++ b/src/decomon/layers/activations/utils.py @@ -0,0 +1,194 @@ +from collections.abc import Callable +from typing import Any, Union + +from keras import ops as K +from keras.src.backend import epsilon + +from decomon.constants import Slope +from decomon.types import Tensor + +TensorFunction = Callable[[Tensor], Tensor] + + +def sigmoid_prime(x: Tensor) -> Tensor: + """Derivative of sigmoid + + Args: + x + + Returns: + + """ + + s_x = K.sigmoid(x) + return s_x * (K.cast(1, dtype=x.dtype) - s_x) + + +def tanh_prime(x: Tensor) -> Tensor: + """Derivative of tanh + + Args: + x + + Returns: + + """ + + s_x = K.tanh(x) + return K.cast(1, dtype=x.dtype) - K.power(s_x, K.cast(2, dtype=x.dtype)) + + +def get_linear_hull_relu( + upper: Tensor, + lower: Tensor, + slope: Union[str, Slope], + upper_g: float = 0.0, + lower_g: float = 0.0, + **kwargs: Any, +) -> list[Tensor]: + slope = Slope(slope) + # in case upper=lower, this cases are + # considered with index_dead and index_linear + alpha = (K.relu(upper) - K.relu(lower)) / K.maximum(K.cast(epsilon(), dtype=upper.dtype), upper - lower) + + # scaling factor for the upper bound on the relu + # see README + + w_u = alpha + b_u = K.relu(lower) - alpha * lower + z_value = K.cast(0.0, dtype=upper.dtype) + o_value = K.cast(1.0, dtype=upper.dtype) + + if slope == Slope.V_SLOPE: + # 1 if upper<=-lower else 0 + index_a = -K.clip(K.sign(upper + lower) - o_value, -o_value, z_value) + + # 1 if upper>-lower else 0 + index_b = o_value - index_a + w_l = index_b + b_l = z_value * b_u + + elif slope == Slope.A_SLOPE: + w_l = K.clip(K.sign(w_u - 0.5), 0, 1) + b_l = z_value * b_u + + elif slope == Slope.Z_SLOPE: + w_l = z_value * w_u + b_l = z_value * b_u + + elif slope == Slope.O_SLOPE: + w_l = z_value * w_u + o_value + b_l = z_value * b_u + + elif slope == Slope.S_SLOPE: + w_l = w_u + b_l = z_value * b_u + + else: + raise NotImplementedError(f"Not implemented for slope {slope}") + + if "upper_grid" in kwargs: + raise NotImplementedError() + + gamma = o_value + if "finetune" in kwargs: + # retrieve variables to optimize the slopes + gamma = kwargs["finetune"][None] + + w_l = gamma * w_l + (o_value - gamma) * (o_value - w_l) + + # check inactive relu state: u<=0 + index_dead = -K.clip(K.sign(upper) - o_value, -o_value, z_value) # =1 if inactive state + index_linear = K.clip(K.sign(lower) + o_value, z_value, o_value) # 1 if linear state + + w_u = (o_value - index_dead) * w_u + w_l = (o_value - index_dead) * w_l + b_u = (o_value - index_dead) * b_u + b_l = (o_value - index_dead) * b_l + + w_u = (o_value - index_linear) * w_u + index_linear + w_l = (o_value - index_linear) * w_l + index_linear + b_u = (o_value - index_linear) * b_u + b_l = (o_value - index_linear) * b_l + + return [w_u, b_u, w_l, b_l] + + +def get_linear_softplus_hull(upper: Tensor, lower: Tensor, slope: Union[str, Slope], **kwargs: Any) -> list[Tensor]: + slope = Slope(slope) + # in case upper=lower, this cases are + # considered with index_dead and index_linear + u_c = K.softsign(upper) + l_c = K.softsign(lower) + alpha = (u_c - l_c) / K.maximum(K.cast(epsilon(), dtype=upper.dtype), (upper - lower)) + w_u = alpha + b_u = -alpha * lower + l_c + + z_value = K.cast(0.0, dtype=upper.dtype) + o_value = K.cast(1.0, dtype=upper.dtype) + + if slope == Slope.V_SLOPE: + # 1 if upper<=-lower else 0 + index_a = -K.clip(K.sign(upper + lower) - o_value, -o_value, z_value) + # 1 if upper>-lower else 0 + index_b = o_value - index_a + w_l = index_b + b_l = z_value * b_u + elif slope == Slope.Z_SLOPE: + w_l = z_value * w_u + b_l = z_value * b_u + elif slope == Slope.O_SLOPE: + w_l = z_value * w_u + o_value + b_l = z_value * b_u + elif slope == Slope.S_SLOPE: + w_l = w_u + b_l = z_value * b_u + else: + raise ValueError(f"Unknown slope {slope}") + + index_dead = -K.clip(K.sign(upper) - o_value, -o_value, z_value) + + w_u = (o_value - index_dead) * w_u + w_l = (o_value - index_dead) * w_l + b_u = (o_value - index_dead) * b_u + b_l = (o_value - index_dead) * b_l + + if "finetune" in kwargs: + # weighted linear combination + alpha_u, alpha_l = kwargs["finetune"] + alpha_u = alpha_u[None] + alpha_l = alpha_l[None] + + w_u = alpha_u * w_u + b_u = alpha_u * b_u + (o_value - alpha_u) * K.maximum(upper, z_value) + + w_l = alpha_l * w_l + b_l = alpha_l * b_l + (o_value - alpha_l) * K.maximum(lower, z_value) + + return [w_u, b_u, w_l, b_l] + + +def relu_prime(x: Tensor) -> Tensor: + """Derivative of relu + + Args: + x + + Returns: + + """ + + return K.clip(K.sign(x), K.cast(0, dtype=x.dtype), K.cast(1, dtype=x.dtype)) + + +def softsign_prime(x: Tensor) -> Tensor: + """Derivative of softsign + + Args: + x + + Returns: + + """ + + return K.cast(1.0, dtype=x.dtype) / K.power(K.cast(1.0, dtype=x.dtype) + K.abs(x), K.cast(2, dtype=x.dtype)) diff --git a/src/decomon/layers/convert.py b/src/decomon/layers/convert.py index ac6dbff5..071d0ffb 100644 --- a/src/decomon/layers/convert.py +++ b/src/decomon/layers/convert.py @@ -4,8 +4,9 @@ from keras.layers import Activation, Add, Dense, Layer import decomon.layers -from decomon.core import PerturbationDomain, Propagation, Slope +from decomon.constants import Propagation, Slope from decomon.layers import DecomonActivation, DecomonAdd, DecomonDense, DecomonLayer +from decomon.perturbation_domain import PerturbationDomain logger = logging.getLogger(__name__) diff --git a/src/decomon/layers/crown.py b/src/decomon/layers/crown.py index 4258d1c8..8e5370d9 100644 --- a/src/decomon/layers/crown.py +++ b/src/decomon/layers/crown.py @@ -7,8 +7,10 @@ import keras.ops as K from keras.layers import Layer -from decomon.core import InputsOutputsSpec, PerturbationDomain, Propagation +from decomon.constants import Propagation from decomon.keras_utils import add_tensors +from decomon.layers.inputs_outputs_specs import InputsOutputsSpec +from decomon.perturbation_domain import PerturbationDomain from decomon.types import BackendTensor diff --git a/src/decomon/layers/fuse.py b/src/decomon/layers/fuse.py index 0b166e06..8e689929 100644 --- a/src/decomon/layers/fuse.py +++ b/src/decomon/layers/fuse.py @@ -7,15 +7,15 @@ from keras import ops as K from keras.layers import Layer -from decomon.core import ( +from decomon.constants import Propagation +from decomon.keras_utils import add_tensors, batch_multid_dot +from decomon.layers.inputs_outputs_specs import InputsOutputsSpec +from decomon.perturbation_domain import ( BoxDomain, - InputsOutputsSpec, PerturbationDomain, - Propagation, get_lower_box, get_upper_box, ) -from decomon.keras_utils import add_tensors, batch_multid_dot from decomon.types import BackendTensor, Tensor diff --git a/src/decomon/layers/input.py b/src/decomon/layers/input.py index 302e0306..560b0eae 100644 --- a/src/decomon/layers/input.py +++ b/src/decomon/layers/input.py @@ -7,7 +7,9 @@ import keras.ops as K from keras.layers import Layer -from decomon.core import InputsOutputsSpec, PerturbationDomain, Propagation +from decomon.constants import Propagation +from decomon.layers.inputs_outputs_specs import InputsOutputsSpec +from decomon.perturbation_domain import PerturbationDomain from decomon.types import BackendTensor @@ -38,7 +40,6 @@ def __init__( ibp=ibp, affine=affine, propagation=Propagation.FORWARD, - perturbation_domain=perturbation_domain, model_input_shape=tuple(), layer_input_shape=tuple(), ) diff --git a/src/decomon/layers/inputs_outputs_specs.py b/src/decomon/layers/inputs_outputs_specs.py new file mode 100644 index 00000000..0d215a0f --- /dev/null +++ b/src/decomon/layers/inputs_outputs_specs.py @@ -0,0 +1,605 @@ +from typing import Optional, Union, overload + +from decomon.constants import Propagation +from decomon.types import Tensor + + +class InputsOutputsSpec: + """Storing specifications for inputs and outputs of decomon/backward layer/model.""" + + layer_input_shape: Union[tuple[int, ...], list[tuple[int, ...]]] + model_input_shape: tuple[int, ...] + model_output_shape: tuple[int, ...] + + def __init__( + self, + ibp: bool = True, + affine: bool = True, + propagation: Propagation = Propagation.FORWARD, + layer_input_shape: Optional[Union[tuple[int, ...], list[tuple[int, ...]]]] = None, + model_input_shape: Optional[tuple[int, ...]] = None, + model_output_shape: Optional[tuple[int, ...]] = None, + is_merging_layer: bool = False, + linear: bool = False, + ): + """ + Args: + ibp: if True, forward propagate constant bounds + affine: if True, forward propagate affine bounds + propagation: direction of bounds propagation + - forward: from input to output + - backward: from output to input + layer_input_shape: shape of the underlying keras layer input (w/o the batch axis) + model_input_shape: shape of the underlying keras model input (w/o the batch axis) + model_output_shape: shape of the underlying keras model output (w/o the batch axis) + is_merging_layer: whether the underlying keras layer is a merging layer (i.e. with several inputs) + linear: whether the underlying keras layer is linear (thus do not need oracle bounds for instance) + + """ + # checks + if not ibp and not affine: + raise ValueError("ibp and affine cannot be both False.") + if propagation == Propagation.BACKWARD and model_output_shape is None: + raise ValueError("model_output_shape must be set in backward propagation.") + if propagation == Propagation.FORWARD or is_merging_layer: + if layer_input_shape is None: + raise ValueError("layer_input_shape must be set in forward propagation or for mergine layer.") + elif is_merging_layer: + if len(layer_input_shape) == 0 or not isinstance(layer_input_shape[0], tuple): + raise ValueError( + "layer_input_shape should be a non-empty list of shapes (tuple of int) for a merging layer." + ) + elif not isinstance(layer_input_shape, tuple) or ( + len(layer_input_shape) > 0 and not isinstance(layer_input_shape[0], int) + ): + raise ValueError("layer_input_shape should be a tuple of int for a unary layer.") + + self.propagation = propagation + self.affine = affine + self.ibp = ibp + self.is_merging_layer = is_merging_layer + self.linear = linear + + if model_output_shape is None: + self.model_output_shape = tuple() + else: + self.model_output_shape = model_output_shape + if model_input_shape is None: + self.model_input_shape = tuple() + else: + self.model_input_shape = model_input_shape + if layer_input_shape is None: + if self.is_merging_layer: + self.layer_input_shape = [tuple()] + else: + self.layer_input_shape = tuple() + else: + self.layer_input_shape = layer_input_shape + + def needs_perturbation_domain_inputs(self) -> bool: + """Specify if decomon inputs should integrate keras model inputs.""" + return self.propagation == Propagation.FORWARD and self.affine + + def needs_oracle_bounds(self) -> bool: + """Specify if decomon layer needs oracle bounds on keras layer inputs.""" + return not self.linear and (self.propagation == Propagation.BACKWARD or self.affine) + + def needs_constant_bounds_inputs(self) -> bool: + """Specify if decomon inputs should integrate constant bounds.""" + return (self.propagation == Propagation.FORWARD and self.ibp) or ( + self.propagation == Propagation.BACKWARD and self.needs_oracle_bounds() + ) + + def needs_affine_bounds_inputs(self) -> bool: + """Specify if decomon inputs should integrate affine bounds.""" + return (self.propagation == Propagation.FORWARD and self.affine) or (self.propagation == Propagation.BACKWARD) + + def cannot_have_empty_affine_inputs(self) -> bool: + """Specify that it is not allowed to have empty affine bounds. + + Indeed, in merging case + forward propagation, it would be impossible to split decomon inputs properly. + + """ + return self.is_merging_layer and self.propagation == Propagation.FORWARD and self.affine + + @property + def nb_keras_inputs(self) -> int: + if self.is_merging_layer: + return len(self.layer_input_shape) + else: + return 1 + + @property + def nb_input_tensors(self) -> int: + nb = 0 + if self.propagation == Propagation.BACKWARD: + # oracle bounds + if self.needs_oracle_bounds(): + nb += 2 * self.nb_keras_inputs + # affine + nb += 4 + # model inputs + if self.needs_perturbation_domain_inputs(): + nb += 1 + else: # forward + # ibp + if self.ibp: + nb += 2 * self.nb_keras_inputs + # affine + if self.affine: + nb += 4 * self.nb_keras_inputs + # model inputs + if self.needs_perturbation_domain_inputs(): + nb += 1 + return nb + + @property + def nb_output_tensors(self) -> int: + nb = 0 + if self.propagation == Propagation.BACKWARD: + nb += 4 * self.nb_keras_inputs + else: # forward + if self.ibp: + nb += 2 + if self.affine: + nb += 4 + return nb + + @overload + def split_constant_bounds(self, constant_bounds: list[Tensor]) -> tuple[Tensor, Tensor]: + """Split constant bounds, non-merging layer version.""" + ... + + @overload + def split_constant_bounds(self, constant_bounds: list[list[Tensor]]) -> tuple[list[Tensor], list[Tensor]]: + """Split constant bounds, merging layer version.""" + ... + + def split_constant_bounds( + self, constant_bounds: Union[list[Tensor], list[list[Tensor]]] + ) -> Union[tuple[Tensor, Tensor], tuple[list[Tensor], list[Tensor]]]: + """Split constant bounds into lower, upper bound. + + Args: + constant_bounds: + if merging layer: list of constant (lower and upper) bounds for each keras layer inputs; + else: list containing lower and upper bounds for the keras layer input. + + Returns: + if merging_layer: 2 lists containing lower and upper bounds for each keras layer inputs; + else: 2 tensors being the lower and upper bounds for the keras layer input. + + """ + if self.is_merging_layer: + lowers, uppers = zip(*constant_bounds) + return list(lowers), list(uppers) + else: + lower, upper = constant_bounds + return lower, upper + + def split_inputs( + self, inputs: list[Tensor] + ) -> Union[ + tuple[list[Tensor], list[Tensor], list[Tensor]], + tuple[list[list[Tensor]], list[list[Tensor]], list[Tensor]], + tuple[list[Tensor], list[list[Tensor]], list[Tensor]], + ]: + """Split decomon inputs. + + Split them according to propagation mode and whether the underlying keras layer is merging or not. + + Args: + inputs: flattened decomon inputs, as seen by `DecomonLayer.call()`. + + Returns: + affine_bounds_to_propagate, constant_oracle_bounds, perturbation_domain_inputs: + each one can be empty if not relevant, + moreover, according to propagation mode and merging status, + it will be list of tensors or list of lists of tensors. + + More details: + + - non-merging case: + inputs = affine_bounds_to_propagate + constant_oracle_bounds + perturbation_domain_inputs + + - merging case: + - forward: k affine bounds to propagate w.r.t. each keras layer input + k constant bounds + + inputs = ( + affine_bounds_to_propagate_0 + constant_oracle_bounds_0 + ... + + affine_bounds_to_propagate_k + constant_oracle_bounds_k + + perturbation_domain_inputs + ) + + - backward: only 1 affine bounds to propagate w.r.t keras layer output + + k constant bounds w.r.t each keras layer input (empty if layer not linear) + + inputs = ( + affine_bounds_to_propagate + + constant_oracle_bounds_0 + ... + constant_oracle_bounds_k + + perturbation_domain_inputs + ) + Note: in case of merging layer + forward, we should not have empty affine bounds + as it will be impossible to split properly the inputs. + + """ + # Remove keras model input + if self.needs_perturbation_domain_inputs(): + x = inputs[-1] + inputs = inputs[:-1] + perturbation_domain_inputs = [x] + else: + perturbation_domain_inputs = [] + if self.is_merging_layer: + if self.propagation == Propagation.BACKWARD: + # expected number of constant bounds + nb_constant_bounds_by_keras_input = 2 if self.needs_oracle_bounds() else 0 + nb_constant_bounds = self.nb_keras_inputs * nb_constant_bounds_by_keras_input + # remove affine bounds (could be empty to express identity bounds) + affine_bounds_to_propagate = inputs[: len(inputs) - nb_constant_bounds] + inputs = inputs[len(inputs) - nb_constant_bounds :] + # split constant bounds by keras input + if nb_constant_bounds > 0: + constant_oracle_bounds = [ + [inputs[i], inputs[i + 1]] for i in range(0, len(inputs), nb_constant_bounds_by_keras_input) + ] + else: + constant_oracle_bounds = [] + else: # forward + # split bounds by keras input + nb_affine_bounds_by_keras_input = 4 if self.affine else 0 + nb_constant_bounds_by_keras_input = 2 if self.ibp else 0 + nb_bounds_by_keras_input = nb_affine_bounds_by_keras_input + nb_constant_bounds_by_keras_input + affine_bounds_to_propagate = [ + [inputs[start_input + j_bound] for j_bound in range(nb_affine_bounds_by_keras_input)] + for start_input in range(0, len(inputs), nb_bounds_by_keras_input) + ] + constant_oracle_bounds = [ + [ + inputs[start_input + nb_affine_bounds_by_keras_input + j_bound] + for j_bound in range(nb_constant_bounds_by_keras_input) + ] + for start_input in range(0, len(inputs), nb_bounds_by_keras_input) + ] + else: + # Remove constant bounds + if self.needs_constant_bounds_inputs(): + constant_oracle_bounds = inputs[-2:] + inputs = inputs[:-2] + else: + constant_oracle_bounds = [] + # The remaining tensors are affine bounds + # (potentially empty if: not backward or not affine or identity affine bounds) + affine_bounds_to_propagate = inputs + + return affine_bounds_to_propagate, constant_oracle_bounds, perturbation_domain_inputs + + def split_input_shape( + self, input_shape: list[tuple[Optional[int], ...]] + ) -> Union[ + tuple[list[tuple[Optional[int], ...]], list[tuple[Optional[int], ...]], list[tuple[Optional[int], ...]]], + tuple[ + list[list[tuple[Optional[int], ...]]], + list[list[tuple[Optional[int], ...]]], + list[tuple[Optional[int], ...]], + ], + tuple[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]], list[tuple[Optional[int], ...]]], + ]: + """Split decomon inputs. + + Split them according to propagation mode and whether the underlying keras layer is merging or not. + + Args: + input_shape: flattened decomon inputs, as seen by `DecomonLayer.call()`. + + Returns: + affine_bounds_to_propagate_shape, constant_oracle_bounds_shape, perturbation_domain_inputs_shape: + each one can be empty if not relevant, and according to propagation mode and merging status, + it will be list of shapes or list of lists of shapes. + + """ + return self.split_inputs(inputs=input_shape) # type: ignore + + def flatten_inputs( + self, + affine_bounds_to_propagate: Union[list[Tensor], list[list[Tensor]]], + constant_oracle_bounds: Union[list[Tensor], list[list[Tensor]]], + perturbation_domain_inputs: list[Tensor], + ) -> list[Tensor]: + """Flatten decomon inputs. + + Reverse `self.split_inputs()`. + + Args: + affine_bounds_to_propagate: + - forward + affine: affine bounds on each keras layer input w.r.t. model input + -> list of lists of tensors in merging case; + -> list of tensors else. + - backward: affine bounds on model output w.r.t keras layer output + -> list of tensors + - else: empty + constant_oracle_bounds: + - forward + ibp: ibp bounds on keras layer inputs + - backward + not linear: oracle bounds on keras layer inputs + - else: empty + perturbation_domain_inputs: + - forward + affine: perturbation domain input wrapped in a list + - else: empty + + Returns: + flattened inputs + - non-merging case: + inputs = affine_bounds_to_propagate + constant_oracle_bounds + perturbation_domain_inputs + + - merging case: + - forward: k affine bounds to propagate w.r.t. each keras layer input + k constant bounds + + inputs = ( + affine_bounds_to_propagate_0 + constant_oracle_bounds_0 + ... + + affine_bounds_to_propagate_k + constant_oracle_bounds_k + + perturbation_domain_inputs + ) + + - backward: only 1 affine bounds to propagate w.r.t keras layer output + + k constant bounds w.r.t each keras layer input (empty of linear layer) + + inputs = ( + affine_bounds_to_propagate + + constant_oracle_bounds_0 + ... + constant_oracle_bounds_k + + perturbation_domain_inputs + ) + + """ + if self.is_merging_layer: + if self.propagation == Propagation.BACKWARD: + if self.needs_oracle_bounds(): + flattened_constant_oracle_bounds = [ + t for constant_oracle_bounds_i in constant_oracle_bounds for t in constant_oracle_bounds_i + ] + else: + flattened_constant_oracle_bounds = [] + return affine_bounds_to_propagate + flattened_constant_oracle_bounds + perturbation_domain_inputs + else: # forward + bounds_by_keras_input = [ + affine_bounds_to_propagate_i + constant_oracle_bounds_i + for affine_bounds_to_propagate_i, constant_oracle_bounds_i in zip( + affine_bounds_to_propagate, constant_oracle_bounds + ) + ] + flattened_bounds_by_keras_input = [ + t for bounds_by_keras_input_i in bounds_by_keras_input for t in bounds_by_keras_input_i + ] + return flattened_bounds_by_keras_input + perturbation_domain_inputs + else: + return affine_bounds_to_propagate + constant_oracle_bounds + perturbation_domain_inputs + + def flatten_inputs_shape( + self, + affine_bounds_to_propagate_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], + constant_oracle_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], + perturbation_domain_inputs_shape: list[tuple[Optional[int], ...]], + ) -> list[tuple[Optional[int], ...]]: + """Flatten inputs shape + + Same operation as `flatten_inputs` but on tensor shapes. + + Args: + affine_bounds_to_propagate_shape: + constant_oracle_bounds_shape: + perturbation_domain_inputs_shape: + + Returns: + + """ + return self.flatten_inputs( # type: ignore + affine_bounds_to_propagate=affine_bounds_to_propagate_shape, + constant_oracle_bounds=constant_oracle_bounds_shape, + perturbation_domain_inputs=perturbation_domain_inputs_shape, + ) # type: ignore + + def split_outputs(self, outputs: list[Tensor]) -> tuple[Union[list[Tensor], list[list[Tensor]]], list[Tensor]]: + """Split decomon inputs. + + Reverse operation of `self.flatten_outputs()` + + Args: + outputs: flattened decomon outputs, as returned by `DecomonLayer.call()`. + + Returns: + affine_bounds_propagated, constant_bounds_propagated: + each one can be empty if not relevant and can be list of tensors or a list of lists of tensors + according to propagation and merging status. + + More details: + + - forward: affine_bounds_propagated, constant_bounds_propagated: both simple lists of tensors corresponding to + affine and constant bounds on keras layer output. + - backward: constant_bounds_propagated is empty (not relevant) and + - merging layer: affine_bounds_propagated is a list of lists of tensors corresponding + to partial affine bounds on model output w.r.t each keras input + - else: affine_bounds_propagated is a simple list of tensors + + """ + # Remove constant bounds + if self.propagation == Propagation.FORWARD and self.ibp: + constant_bounds_propagated = outputs[-2:] + outputs = outputs[:-2] + else: + constant_bounds_propagated = [] + # It remains affine bounds (can be empty if forward + not affine, or identity layer (e.g. DecomonLinear) on identity bounds + affine_bounds_propagated = outputs + if self.propagation == Propagation.BACKWARD and self.is_merging_layer: + nb_affine_bounds_by_keras_input = 4 + affine_bounds_propagated = [ + affine_bounds_propagated[i : i + nb_affine_bounds_by_keras_input] + for i in range(0, len(affine_bounds_propagated), nb_affine_bounds_by_keras_input) + ] + + return affine_bounds_propagated, constant_bounds_propagated + + def split_output_shape( + self, output_shape: list[tuple[Optional[int], ...]] + ) -> tuple[ + Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], list[tuple[Optional[int], ...]] + ]: + """Split decomon output shape.""" + return self.split_outputs(outputs=output_shape) # type: ignore + + def flatten_outputs( + self, + affine_bounds_propagated: Union[list[Tensor], list[list[Tensor]]], + constant_bounds_propagated: Optional[list[Tensor]] = None, + ) -> list[Tensor]: + """Flatten decomon outputs. + + Args: + affine_bounds_propagated: + - forward + affine: affine bounds on keras layer output w.r.t. model input + - backward: affine bounds on model output w.r.t each keras layer input + -> list of lists of tensors in merging case; + -> list of tensors else. + - else: empty + constant_bounds_propagated: + - forward + ibp: ibp bounds on keras layer output + - else: empty or None + + Returns: + flattened outputs + - forward: affine_bounds_propagated + constant_bounds_propagated + - backward: + - merging layer (k keras layer inputs): affine_bounds_propagated_0 + ... + affine_bounds_propagated_k + - else: affine_bounds_propagated + + """ + if constant_bounds_propagated is None or self.propagation == Propagation.BACKWARD: + if self.is_merging_layer and self.propagation == Propagation.BACKWARD: + return [ + t for affine_bounds_propagated_i in affine_bounds_propagated for t in affine_bounds_propagated_i + ] + else: + return affine_bounds_propagated + else: + return affine_bounds_propagated + constant_bounds_propagated + + def flatten_outputs_shape( + self, + affine_bounds_propagated_shape: Union[ + list[tuple[Optional[int], ...]], + list[list[tuple[Optional[int], ...]]], + ], + constant_bounds_propagated_shape: Optional[list[tuple[Optional[int], ...]]] = None, + ) -> list[tuple[Optional[int], ...]]: + """Flatten decomon outputs shape.""" + return self.flatten_outputs(affine_bounds_propagated=affine_bounds_propagated_shape, constant_bounds_propagated=constant_bounds_propagated_shape) # type: ignore + + def has_multiple_bounds_inputs(self) -> bool: + return self.propagation == Propagation.FORWARD and self.is_merging_layer + + @overload + def extract_shapes_from_affine_bounds( + self, affine_bounds: list[Tensor], i: int = -1 + ) -> list[tuple[Optional[int], ...]]: + ... + + @overload + def extract_shapes_from_affine_bounds( + self, affine_bounds: list[list[Tensor]], i: int = -1 + ) -> list[list[tuple[Optional[int], ...]]]: + ... + + def extract_shapes_from_affine_bounds( + self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], i: int = -1 + ) -> Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]]: + if self.has_multiple_bounds_inputs() and i == -1: + return [[t.shape for t in sub_bounds] for sub_bounds in affine_bounds] + else: + return [t.shape for t in affine_bounds] # type: ignore + + def is_identity_bounds(self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], i: int = -1) -> bool: + return self.is_identity_bounds_shape( + affine_bounds_shape=self.extract_shapes_from_affine_bounds(affine_bounds=affine_bounds, i=i), i=i + ) + + def is_identity_bounds_shape( + self, + affine_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], + i: int = -1, + ) -> bool: + if self.has_multiple_bounds_inputs() and i == -1: + return all( + self.is_identity_bounds_shape(affine_bounds_shape=affine_bounds_shape[i], i=i) # type: ignore + for i in range(self.nb_keras_inputs) + ) + else: + return len(affine_bounds_shape) == 0 + + def is_diagonal_bounds(self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], i: int = -1) -> bool: + return self.is_diagonal_bounds_shape( + affine_bounds_shape=self.extract_shapes_from_affine_bounds(affine_bounds=affine_bounds, i=i), i=i + ) + + def is_diagonal_bounds_shape( + self, + affine_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], + i: int = -1, + ) -> bool: + if self.has_multiple_bounds_inputs() and i == -1: + return all( + self.is_diagonal_bounds_shape(affine_bounds_shape=affine_bounds_shape[i], i=i) # type: ignore + for i in range(self.nb_keras_inputs) + ) + else: + if self.is_identity_bounds_shape(affine_bounds_shape, i=i): + return True + w_shape, b_shape = affine_bounds_shape[:2] + return w_shape == b_shape + + def is_wo_batch_bounds(self, affine_bounds: Union[list[Tensor], list[list[Tensor]]], i: int = -1) -> bool: + return self.is_wo_batch_bounds_shape( + affine_bounds_shape=self.extract_shapes_from_affine_bounds(affine_bounds=affine_bounds, i=i), i=i + ) + + def is_wo_batch_bounds_shape( + self, + affine_bounds_shape: Union[list[tuple[Optional[int], ...]], list[list[tuple[Optional[int], ...]]]], + i: int = -1, + ) -> bool: + if self.has_multiple_bounds_inputs() and i == -1: + return all( + self.is_wo_batch_bounds_shape(affine_bounds_shape=affine_bounds_shape[i], i=i) # type: ignore + for i in range(self.nb_keras_inputs) + ) + else: + if self.is_identity_bounds_shape(affine_bounds_shape, i=i): + return True + b_shape = affine_bounds_shape[1] + if self.propagation == Propagation.FORWARD: + if i > -1: + return len(b_shape) == len(self.layer_input_shape[i]) + else: + return len(b_shape) == len(self.layer_input_shape) + else: + return len(b_shape) == len(self.model_output_shape) + + @overload + def is_wo_batch_bounds_by_keras_input( + self, + affine_bounds: list[Tensor], + ) -> bool: + ... + + @overload + def is_wo_batch_bounds_by_keras_input( + self, + affine_bounds: list[list[Tensor]], + ) -> list[bool]: + ... + + def is_wo_batch_bounds_by_keras_input( + self, + affine_bounds: Union[list[Tensor], list[list[Tensor]]], + ) -> Union[bool, list[bool]]: + if self.has_multiple_bounds_inputs(): + return [self.is_wo_batch_bounds(affine_bounds_i, i=i) for i, affine_bounds_i in enumerate(affine_bounds)] + else: + return self.is_wo_batch_bounds(affine_bounds) diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index 89fc70b6..9a362815 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -5,12 +5,14 @@ import keras.ops as K from keras.layers import Layer, Wrapper -from decomon.core import BoxDomain, InputsOutputsSpec, PerturbationDomain, Propagation +from decomon.constants import Propagation from decomon.layers.fuse import ( combine_affine_bound_with_constant_bound, combine_affine_bounds, ) +from decomon.layers.inputs_outputs_specs import InputsOutputsSpec from decomon.layers.oracle import get_forward_oracle +from decomon.perturbation_domain import BoxDomain, PerturbationDomain from decomon.types import Tensor _keras_base_layer_keyword_parameters = [ @@ -124,7 +126,6 @@ def __init__( # input-output-manager self.inputs_outputs_spec = self.create_inputs_outputs_spec( layer=layer, - perturbation_domain=perturbation_domain, ibp=ibp, affine=affine, propagation=propagation, @@ -135,7 +136,6 @@ def __init__( def create_inputs_outputs_spec( self, layer: Layer, - perturbation_domain: PerturbationDomain, ibp: bool, affine: bool, propagation: Propagation, @@ -154,7 +154,6 @@ def create_inputs_outputs_spec( ibp=ibp, affine=affine, propagation=propagation, - perturbation_domain=perturbation_domain, layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=model_output_shape, diff --git a/src/decomon/layers/oracle.py b/src/decomon/layers/oracle.py index f8c4f4f3..30ae9a32 100644 --- a/src/decomon/layers/oracle.py +++ b/src/decomon/layers/oracle.py @@ -6,7 +6,8 @@ import keras from keras.layers import Layer -from decomon.core import InputsOutputsSpec, PerturbationDomain +from decomon.layers.inputs_outputs_specs import InputsOutputsSpec +from decomon.perturbation_domain import PerturbationDomain from decomon.types import BackendTensor @@ -64,7 +65,6 @@ def __init__( self.inputs_outputs_spec = InputsOutputsSpec( ibp=ibp, affine=affine, - perturbation_domain=perturbation_domain, layer_input_shape=layer_input_shape, is_merging_layer=is_merging_layer, ) diff --git a/src/decomon/layers/output.py b/src/decomon/layers/output.py index f78fa222..d33eceeb 100644 --- a/src/decomon/layers/output.py +++ b/src/decomon/layers/output.py @@ -7,8 +7,10 @@ import keras.ops as K from keras.layers import Layer -from decomon.core import InputsOutputsSpec, PerturbationDomain, Propagation +from decomon.constants import Propagation +from decomon.layers.inputs_outputs_specs import InputsOutputsSpec from decomon.layers.oracle import get_forward_oracle +from decomon.perturbation_domain import PerturbationDomain from decomon.types import BackendTensor, Tensor diff --git a/src/decomon/metrics/__init__.py b/src/decomon/metrics/__init__.py deleted file mode 100644 index 4ad5389d..00000000 --- a/src/decomon/metrics/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from .loss import ( - build_asymptotic_crossentropy_model, - build_crossentropy_model, - build_radius_robust_model, -) -from .metric import ( - build_formal_adv_check_model, - build_formal_adv_model, - build_formal_upper_model, -) diff --git a/src/decomon/metrics/complexity.py b/src/decomon/metrics/complexity.py deleted file mode 100644 index fcdeaf2d..00000000 --- a/src/decomon/metrics/complexity.py +++ /dev/null @@ -1,19 +0,0 @@ -import keras -from keras.layers import InputLayer, Lambda - - -def get_graph_complexity(model: keras.Model) -> int: - # do not consider Input nodes or Lambda nodes - # enumerate the number of - nb_nodes = 0 - depth_keys = list(model._nodes_by_depth.keys()) - - for depth in depth_keys: - nodes = model._nodes_by_depth[depth] - for node in nodes: - if isinstance(node.operation, (InputLayer, Lambda)): - continue - - nb_nodes += 1 - - return nb_nodes diff --git a/src/decomon/metrics/loss.py b/src/decomon/metrics/loss.py deleted file mode 100644 index fce6797d..00000000 --- a/src/decomon/metrics/loss.py +++ /dev/null @@ -1,639 +0,0 @@ -from collections.abc import Callable -from typing import Any, Optional, Union - -import keras -import keras.ops as K -import numpy as np -from keras.config import epsilon -from keras.layers import Lambda, Layer - -from decomon.core import ( - BallDomain, - BoxDomain, - ForwardMode, - PerturbationDomain, - get_mode, -) -from decomon.layers.activations import softmax as softmax_ -from decomon.layers.core import DecomonLayer -from decomon.models.models import DecomonModel -from decomon.models.utils import Convert2Mode -from decomon.types import BackendTensor, Tensor - - -def get_model(model: DecomonModel) -> DecomonModel: - ibp = model.ibp - affine = model.affine - - mode = get_mode(ibp, affine) - perturbation_domain = model.perturbation_domain - - inputs = model.inputs - outputs = model.outputs - new_output: keras.KerasTensor - - if mode == ForwardMode.IBP: - - def func(outputs: list[Tensor]) -> Tensor: - u_c, l_c = outputs - return K.concatenate([K.expand_dims(u_c, -1), K.expand_dims(l_c, -1)], -1) - - new_output = Lambda(func)(outputs) - - elif mode == ForwardMode.AFFINE: - - def func(outputs: list[Tensor]) -> Tensor: - x_0, w_u, b_u, w_l, b_l = outputs - if len(x_0.shape) == 2: - x_0_reshaped = x_0[:, :, None] - else: - x_0_reshaped = K.transpose(x_0, (0, 2, 1)) - x_fake = K.sum(x_0_reshaped, 1)[:, None] - x_0_reshaped = K.concatenate([x_0_reshaped, x_fake], 1) # (None, n_in+1, n_comp) - - w_b_u = K.concatenate([w_u, b_u[:, None]], 1) # (None, n_in+1, n_out) - w_b_l = K.concatenate([w_l, b_l[:, None]], 1) - - w_b = K.concatenate([w_b_u, w_b_l], -1) # (None, n_in+1, 2*n_out) - return K.concatenate([x_0_reshaped, w_b], -1) # (None, n_in+1, n_comp+2*n_out) - - new_output = Lambda(func)(outputs) - - elif mode == ForwardMode.HYBRID: - - def func(outputs: list[Tensor]) -> Tensor: - x_0, u_c, w_u, b_u, l_c, w_l, b_l = outputs - - if len(x_0.shape) == 2: - x_0_reshaped = x_0[:, :, None] - else: - x_0_reshaped = K.transpose(x_0, (0, 2, 1)) - x_fake = K.sum(x_0_reshaped, 1)[:, None] - x_0_reshaped = K.concatenate([x_0_reshaped, x_fake, x_fake], 1) # (None, n_in+2, n_comp) - - w_b_u = K.concatenate([w_u, b_u[:, None]], 1) # (None, n_in+1, n_out) - w_b_l = K.concatenate([w_l, b_l[:, None]], 1) - - w_b = K.concatenate([w_b_u, w_b_l], -1) # (None, n_in+1, 2*n_out) - - u_l = K.concatenate([u_c, l_c], 1)[:, None] # (None, 1, 2*n_out) - - w_b_u_l = K.concatenate([w_b, u_l], 1) # (None, n_in+2, 2*n_out) - - return K.concatenate([x_0_reshaped, w_b_u_l], -1) # (None, n_in+1, n_comp+2*n_out) - - new_output = Lambda(func)(outputs) - - else: - raise ValueError(f"Unknown mode {mode}") - - return DecomonModel( - inputs=inputs, - outputs=new_output, - perturbation_domain=model.perturbation_domain, - ibp=ibp, - affine=affine, - finetune=model.finetune, - ) - - -def get_upper_loss(model: DecomonModel) -> Callable[[Tensor, Tensor], Tensor]: - ibp = model.ibp - affine = model.affine - - mode = get_mode(ibp, affine) - perturbation_domain = model.perturbation_domain - - n_comp = perturbation_domain.get_nb_x_components() - n_out = np.prod(model.output[-1].shape[1:]) - - def upper_ibp(u_c: Tensor, u_ref: Tensor) -> Tensor: - # minimize the upper bound compared to the reference - return K.max(u_c - u_ref, -1) - - def upper_affine(x: Tensor, w_u: Tensor, b_u: Tensor, u_ref: Tensor) -> Tensor: - upper = perturbation_domain.get_upper(x, w_u, b_u) - - return K.max(upper - u_ref, -1) - - def loss_upper(y_true: Tensor, y_pred: Tensor) -> Tensor: - if mode == ForwardMode.IBP: - u_c = y_pred[:, :, 0] - - elif mode == ForwardMode.AFFINE: - if len(y_pred.shape) == 3: - x_0 = K.transpose(y_pred[:, :-1, :n_comp], (0, 2, 1)) - else: - x_0 = y_pred[:, :-1, 0] - - w_u = y_pred[:, :-1, n_comp : n_comp + n_out] - b_u = y_pred[:, -1, n_comp : n_comp + n_out] - - elif mode == ForwardMode.HYBRID: - if len(y_pred.shape) == 3: - x_0 = K.transpose(y_pred[:, :-2, :n_comp], (0, 2, 1)) - else: - x_0 = y_pred[:, :-2, 0] - - w_u = y_pred[:, :-2, n_comp : n_comp + n_out] - b_u = y_pred[:, -2, n_comp : n_comp + n_out] - u_c = y_pred[:, -1, n_comp : n_comp + n_out] - - else: - raise ValueError(f"Unknown mode {mode}") - - if ibp: - score_ibp = upper_ibp(u_c, y_true) - if affine: - score_affine = upper_affine(x_0, w_u, b_u, y_true) - - if mode == ForwardMode.IBP: - return K.mean(score_ibp) - elif mode == ForwardMode.AFFINE: - return K.mean(score_affine) - elif mode == ForwardMode.HYBRID: - return K.mean(K.minimum(score_ibp, score_affine)) - - raise NotImplementedError() - - return loss_upper - - -def get_lower_loss(model: DecomonModel) -> Callable[[Tensor, Tensor], Tensor]: - ibp = model.ibp - affine = model.affine - - mode = get_mode(ibp, affine) - perturbation_domain = model.perturbation_domain - - n_comp = perturbation_domain.get_nb_x_components() - n_out = np.prod(model.output[-1].shape[1:]) - - def lower_ibp(l_c: Tensor, l_ref: Tensor) -> Tensor: - # minimize the upper bound compared to the reference - return K.max(l_ref - l_c, -1) - - def lower_affine(x: Tensor, w_l: Tensor, b_l: Tensor, l_ref: Tensor) -> Tensor: - lower = perturbation_domain.get_lower(x, w_l, b_l) - - return K.max(l_ref - lower, -1) - - def loss_lower(y_true: Tensor, y_pred: Tensor) -> Tensor: - if mode == ForwardMode.IBP: - l_c = y_pred[:, :, 1] - - elif mode == ForwardMode.AFFINE: - if len(y_pred.shape) == 3: - x_0 = K.transpose(y_pred[:, :-1, :n_comp], (0, 2, 1)) - else: - x_0 = y_pred[:, :-1, 0] - - w_l = y_pred[:, :-1, n_comp + n_out :] - b_l = y_pred[:, -1, n_comp + n_out :] - - elif mode == ForwardMode.HYBRID: - if len(y_pred.shape) == 3: - x_0 = K.transpose(y_pred[:, :-2, :n_comp], (0, 2, 1)) - else: - x_0 = y_pred[:, :-2, 0] - - w_l = y_pred[:, :-2, n_comp + n_out :] - b_l = y_pred[:, -2, n_comp + n_out :] - l_c = y_pred[:, -1, n_comp + n_out :] - - else: - raise ValueError(f"Unknown mode {mode}") - - if ibp: - score_ibp = lower_ibp(l_c, y_true) - if affine: - score_affine = lower_affine(x_0, w_l, b_l, y_true) - - if mode == ForwardMode.IBP: - return K.mean(score_ibp) - elif mode == ForwardMode.AFFINE: - return K.mean(score_affine) - elif mode == ForwardMode.HYBRID: - return K.mean(K.minimum(score_ibp, score_affine)) - - raise NotImplementedError() - - return loss_lower - - -def get_adv_loss( - model: DecomonModel, sigmoid: bool = False, clip_value: Optional[float] = None, softmax: bool = False -) -> Callable[[Tensor, Tensor], Tensor]: - ibp = model.ibp - affine = model.affine - - mode = get_mode(ibp, affine) - perturbation_domain = model.perturbation_domain - - n_comp = perturbation_domain.get_nb_x_components() - n_out = np.prod(model.output[-1].shape[1:]) - - def adv_ibp(u_c: Tensor, l_c: Tensor, y_tensor: Tensor) -> Tensor: - t_tensor: Tensor = 1 - y_tensor - s_tensor = y_tensor - - t_tensor = t_tensor[:, :, None] - s_tensor = s_tensor[:, None, :] - M = t_tensor * s_tensor - upper = K.expand_dims(u_c, -1) - K.expand_dims(l_c, 1) - const = (K.max(upper, (-1, -2)) - K.min(upper, (-1, -2)))[:, None, None] - upper = upper - (const + K.cast(1, const.dtype)) * (1 - M) - return K.max(upper, (-1, -2)) - - def adv_affine( - x: Tensor, - w_u: Tensor, - b_u: Tensor, - w_l: Tensor, - b_l: Tensor, - y_tensor: Tensor, - ) -> Tensor: - w_u_reshaped = K.expand_dims(w_u, -1) - w_l_reshaped = K.expand_dims(w_l, -2) - - w_adv = w_u_reshaped - w_l_reshaped - b_adv = K.expand_dims(b_u, -1) - K.expand_dims(b_l, 1) - - upper = perturbation_domain.get_upper(x, w_adv, b_adv) - - t_tensor: Tensor = 1 - y_tensor - s_tensor = y_tensor - - t_tensor = t_tensor[:, :, None] - s_tensor = s_tensor[:, None, :] - M = t_tensor * s_tensor - - const = (K.max(upper, (-1, -2)) - K.min(upper, (-1, -2)))[:, None, None] - upper = upper - (const + K.cast(1, const.dtype)) * (1 - M) - return K.max(upper, (-1, -2)) - - def loss_adv(y_true: Tensor, y_pred: Tensor) -> Tensor: - if mode == ForwardMode.IBP: - u_c = y_pred[:, :, 0] - l_c = y_pred[:, :, 1] - - if softmax: - u_c, l_c = softmax_([u_c, l_c], mode=mode, perturbation_domain=perturbation_domain, clip=False) - - elif mode == ForwardMode.AFFINE: - if len(y_pred.shape) == 3: - x_0 = K.transpose(y_pred[:, :-1, :n_comp], (0, 2, 1)) - else: - x_0 = y_pred[:, :-1, 0] - - w_u = y_pred[:, :-1, n_comp : n_comp + n_out] - b_u = y_pred[:, -1, n_comp : n_comp + n_out] - w_l = y_pred[:, :-1, n_comp + n_out :] - b_l = y_pred[:, -1, n_comp + n_out :] - - if softmax: - _, w_u, b_u, w_l, b_l = softmax_( - [x_0, w_u, b_u, w_l, b_l], mode=mode, perturbation_domain=perturbation_domain, clip=False - ) - - elif mode == ForwardMode.HYBRID: - if len(y_pred.shape) == 3: - x_0 = K.transpose(y_pred[:, :-2, :n_comp], (0, 2, 1)) - else: - x_0 = y_pred[:, :-2, 0] - - w_u = y_pred[:, :-2, n_comp : n_comp + n_out] - b_u = y_pred[:, -2, n_comp : n_comp + n_out] - w_l = y_pred[:, :-2, n_comp + n_out :] - b_l = y_pred[:, -2, n_comp + n_out :] - u_c = y_pred[:, -1, n_comp : n_comp + n_out] - l_c = y_pred[:, -1, n_comp + n_out :] - - _, u_c, w_u, b_u, l_c, w_l, b_l = softmax_( - [x_0, u_c, w_u, b_u, l_c, w_l, b_l], mode=mode, perturbation_domain=perturbation_domain, clip=False - ) - - else: - raise ValueError(f"Unknown mode {mode}") - - if ibp: - score_ibp = adv_ibp(u_c, l_c, y_true) - if clip_value is not None: - score_ibp = K.maximum(score_ibp, K.cast(clip_value, dtype=score_ibp.dtype)) - if affine: - score_affine = adv_affine(x_0, w_u, b_u, w_l, b_l, y_true) - if clip_value is not None: - score_affine = K.maximum(score_affine, K.cast(clip_value, dtype=score_affine.dtype)) - - if mode == ForwardMode.IBP: - if sigmoid: - return K.mean(K.sigmoid(score_ibp)) - else: - return K.mean(score_ibp) - elif mode == ForwardMode.AFFINE: - if sigmoid: - return K.mean(K.sigmoid(score_affine)) - else: - return K.mean(score_affine) - elif mode == ForwardMode.HYBRID: - if sigmoid: - return K.mean(K.sigmoid(K.minimum(score_ibp, score_affine))) - else: - return K.mean(K.minimum(score_ibp, score_affine)) - - raise NotImplementedError() - - return loss_adv - - -def _create_identity_tensor_like(x: Tensor) -> BackendTensor: - identity_tensor = K.identity(x.shape[-1]) - n_repeat = int(np.prod(x.shape[:-1])) - return K.reshape(K.repeat(identity_tensor[None], n_repeat, axis=0), tuple(x.shape) + (-1,)) - - -# create a layer -class DecomonLossFusion(DecomonLayer): - original_keras_layer_class = Layer - - def __init__( - self, - asymptotic: bool = False, - backward: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - self.convert2mode_layer = Convert2Mode( - mode_from=mode, - mode_to=ForwardMode.IBP, - perturbation_domain=self.perturbation_domain, - ) - self.asymptotic = asymptotic - self.backward = backward - - def get_config(self) -> dict[str, Any]: - config = super().get_config() - config.update( - { - "asymptotic": self.asymptotic, - "backward": self.backward, - } - ) - return config - - def call_no_backward(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: - if not self.asymptotic: - u_c, l_c = self.convert2mode_layer(inputs) - - return -l_c + K.log(K.sum(K.exp(u_c - K.max(u_c, -1)[:, None]), -1))[:, None] + K.max(u_c, -1)[:, None] - - else: - u_c, l_c = self.convert2mode_layer(inputs) - shape = u_c.shape[-1] - - def adv_ibp(u_c: BackendTensor, l_c: BackendTensor, y_tensor: BackendTensor) -> BackendTensor: - t_tensor = 1 - y_tensor - s_tensor = y_tensor - - t_tensor = t_tensor[:, :, None] - s_tensor = s_tensor[:, None, :] - M = t_tensor * s_tensor - upper = K.expand_dims(u_c, -1) - K.expand_dims(l_c, 1) - const = (K.max(upper, (-1, -2)) - K.min(upper, (-1, -2)))[:, None, None] - upper = upper - (const + K.cast(1, const.dtype)) * (1 - M) - return K.max(upper, (-1, -2)) - - source_tensor = _create_identity_tensor_like(l_c) - - score = K.concatenate([adv_ibp(u_c, l_c, source_tensor[:, i])[:, None] for i in range(shape)], -1) - return K.maximum( - score, K.cast(-1, dtype=score.dtype) - ) # + 1e-3*K.maximum(K.max(K.abs(u_c), -1)[:,None], K.abs(l_c)) - - def call_backward(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: - if not self.asymptotic: - u_c, l_c = self.convert2mode_layer(inputs) - return K.softmax(u_c) - - else: - raise NotImplementedError() - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: - if self.backward: - return self.call_backward(inputs, **kwargs) - else: - return self.call_no_backward(inputs, **kwargs) - - def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> tuple[Optional[int], ...]: # type: ignore - return input_shape[-1] - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - return None - - -# new layer for new loss functions -class DecomonRadiusRobust(DecomonLayer): - original_keras_layer_class = Layer - - def __init__( - self, - backward: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - shared: bool = False, - fast: bool = True, - **kwargs: Any, - ): - super().__init__( - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - mode=mode, - finetune=finetune, - shared=shared, - fast=fast, - **kwargs, - ) - - if self.mode == ForwardMode.IBP: - raise NotImplementedError - - if not isinstance(self.perturbation_domain, BoxDomain): - raise NotImplementedError() - - self.backward = backward - - def get_config(self) -> dict[str, Any]: - config = super().get_config() - config.update( - { - "backward": self.backward, - } - ) - return config - - def call_no_backward(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: - if self.mode == ForwardMode.HYBRID: - x, _, w_u, b_u, _, w_l, b_l = inputs - else: - x, w_u, b_u, w_l, b_l = inputs - - # compute center - x_0 = (x[:, 0] + x[:, 1]) / 2.0 - radius = K.maximum((x[:, 1] - x[:, 0]) / 2.0, K.cast(epsilon(), dtype=x.dtype)) - source_tensor = _create_identity_tensor_like(b_l) - - shape = b_l.shape[-1] - - def radius_label(y_tensor: BackendTensor, backward: bool = False) -> BackendTensor: - t_tensor = 1 - y_tensor - s_tensor = y_tensor - - W_adv = ( - K.sum(-w_l * (s_tensor[:, None]), -1, keepdims=True) + w_u * t_tensor[:, None] + w_l * y_tensor[:, None] - ) # (None, n_in, n_out) - b_adv = K.sum(-b_l * s_tensor, -1, keepdims=True) + b_u * t_tensor + (b_l - 1e6) * y_tensor # (None, n_out) - - score = K.sum(W_adv * x_0[:, :, None], 1) + b_adv # (None, n_out) - - denum = K.maximum( - K.sum(K.abs(W_adv * radius[:, :, None]), 1), K.cast(epsilon(), dtype=W_adv.dtype) - ) # (None, n_out) - - eps_adv = K.minimum(-score / denum + y_tensor, K.cast(2.0, dtype=score.dtype)) - - adv_volume = 1.0 - eps_adv - - return K.max(adv_volume, -1, keepdims=True) - - return K.concatenate([radius_label(source_tensor[:, i]) for i in range(shape)], -1) - - def call_backward(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: - if self.mode == ForwardMode.HYBRID: - x, _, w_u, b_u, _, w_l, b_l = inputs - else: - x, w_u, b_u, w_l, b_l = inputs - - # compute center - x_0 = (x[:, 0] + x[:, 1]) / 2.0 - radius = K.maximum((x[:, 1] - x[:, 0]) / 2.0, K.cast(epsilon(), dtype=x.dtype)) - source_tensor = _create_identity_tensor_like(b_l) - - shape = b_l.shape[-1] - - def radius_label(y_tensor: BackendTensor) -> BackendTensor: - W_adv = w_u - b_adv = b_u - 1e6 * y_tensor - - score = K.sum(W_adv * x_0[:, :, None], 1) + b_adv # (None, n_out) - denum = K.maximum( - K.sum(K.abs(W_adv * radius[:, :, None]), 1), K.cast(epsilon(), dtype=W_adv.dtype) - ) # (None, n_out) - - eps_adv = K.minimum(-score / denum + y_tensor, K.cast(2.0, dtype=score.dtype)) - - adv_volume = 1.0 - eps_adv - - return K.max(adv_volume, -1, keepdims=True) - - return K.concatenate([radius_label(source_tensor[:, i]) for i in range(shape)], -1) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: - if self.backward: - return self.call_backward(inputs, **kwargs) - else: - return self.call_no_backward(inputs, **kwargs) - - def compute_output_shape(self, input_shape: Union[list[tuple[Optional[int], ...]],]) -> tuple[Optional[int], ...]: # type: ignore - return input_shape[-1] - - def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: - return None - - -def build_radius_robust_model(model: DecomonModel) -> DecomonModel: - ibp = model.ibp - affine = model.affine - - mode = get_mode(ibp, affine) - perturbation_domain = model.perturbation_domain - - inputs = model.input - output = model.output - - layer_robust = DecomonRadiusRobust( - mode=mode, perturbation_domain=perturbation_domain, backward=model.backward_bounds - ) - output_robust = layer_robust(output) - - return DecomonModel( - inputs=inputs, - outputs=output_robust, - perturbation_domain=model.perturbation_domain, - ibp=ibp, - affine=affine, - finetune=model.finetune, - ) - - -##### DESIGN LOSS FUNCTIONS -def build_crossentropy_model(model: DecomonModel) -> DecomonModel: - ibp = model.ibp - affine = model.affine - - mode = get_mode(ibp, affine) - perturbation_domain = model.perturbation_domain - - inputs = model.input - output = model.output - - layer_fusion = DecomonLossFusion(mode=mode, backward=model.backward_bounds) - output_fusion = layer_fusion(output, mode=mode) - - return DecomonModel( - inputs=inputs, - outputs=output_fusion, - perturbation_domain=model.perturbation_domain, - ibp=ibp, - affine=affine, - finetune=model.finetune, - ) - - -##### DESIGN LOSS FUNCTIONS -def build_asymptotic_crossentropy_model(model: DecomonModel) -> DecomonModel: - ibp = model.ibp - affine = model.affine - - mode = get_mode(ibp, affine) - perturbation_domain = model.perturbation_domain - - inputs = model.input - output = model.output - - layer_fusion = DecomonLossFusion(mode=mode, asymptotic=True) - output_fusion = layer_fusion(output, mode=mode) - - return DecomonModel( - inputs=inputs, - outputs=output_fusion, - perturbation_domain=model.perturbation_domain, - ibp=ibp, - affine=affine, - finetune=model.finetune, - ) diff --git a/src/decomon/metrics/metric.py b/src/decomon/metrics/metric.py deleted file mode 100644 index 45b600d7..00000000 --- a/src/decomon/metrics/metric.py +++ /dev/null @@ -1,454 +0,0 @@ -from abc import ABC, abstractmethod -from enum import Enum -from typing import Any, Optional, Union - -import keras -import keras.ops as K -import numpy as np -from keras.layers import Input, Layer -from keras.models import Model - -from decomon.core import BoxDomain, PerturbationDomain -from decomon.models.models import DecomonModel -from decomon.types import BackendTensor, Tensor - - -class MetricMode(str, Enum): - FORWARD = "forward" - BACKWARD = "backward" - - -class MetricLayer(ABC, Layer): - def __init__( - self, - ibp: bool, - affine: bool, - mode: Union[str, MetricMode], - perturbation_domain: Optional[PerturbationDomain], - **kwargs: Any, - ): - """ - Args: - ibp: boolean that indicates whether we propagate constant - bounds - affine: boolean that indicates whether we propagate affine - bounds - mode: str: 'backward' or 'forward' whether we doforward or - backward linear relaxation - perturbation_domain: the type of input perturbation domain for the - linear relaxation - **kwargs - """ - super().__init__(**kwargs) - self.ibp = ibp - self.affine = affine - self.mode = MetricMode(mode) - self.perturbation_domain: PerturbationDomain - if perturbation_domain is None: - self.perturbation_domain = BoxDomain() - else: - self.perturbation_domain = perturbation_domain - - def get_config(self) -> dict[str, Any]: - config = super().get_config() - config.update( - { - "ibp": self.ibp, - "affine": self.affine, - "mode": self.mode, - "perturbation_domain": self.perturbation_domain, - } - ) - return config - - @abstractmethod - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: - """ - Args: - inputs - - Returns: - - """ - pass - - -class AdversarialCheck(MetricLayer): - """Training with symbolic LiRPA bounds for promoting adversarial robustness""" - - def __init__( - self, - ibp: bool, - affine: bool, - mode: Union[str, MetricMode], - perturbation_domain: Optional[PerturbationDomain], - **kwargs: Any, - ): - """ - Args: - ibp: boolean that indicates whether we propagate constant - bounds - affine: boolean that indicates whether we propagate affine - bounds - mode: str: 'backward' or 'forward' whether we doforward or - backward linear relaxation - perturbation_domain: the type of input perturbation domain for the - linear relaxation - **kwargs - """ - super().__init__(ibp=ibp, affine=affine, mode=mode, perturbation_domain=perturbation_domain, **kwargs) - - def linear_adv( - self, - z_tensor: Tensor, - y_tensor: Tensor, - w_u: Tensor, - b_u: Tensor, - w_l: Tensor, - b_l: Tensor, - ) -> Tensor: - w_upper = w_u * (1 - y_tensor[:, None]) - K.expand_dims(K.sum(w_l * y_tensor[:, None], -1), -1) - b_upper = b_u * (1 - y_tensor) - b_l * y_tensor - - adv_score = self.perturbation_domain.get_upper(z_tensor, w_upper, b_upper) - 1e6 * y_tensor - - return K.max(adv_score, -1) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: - """ - Args: - inputs - - Returns: - adv_score <0 if the predictionis robust on the input convex - domain - """ - - y_tensor = inputs[-1] - - if self.ibp and self.affine: - _, z, u_c, w_u_f, b_u_f, l_c, w_l_f, b_l_f = inputs[:8] - elif not self.ibp and self.affine: - _, z, w_u_f, b_u_f, w_l_f, b_l_f = inputs[:6] - elif self.ibp and not self.affine: - _, z, u_c, l_c = inputs[:4] - else: - raise NotImplementedError("not ibp and not affine not implemented") - - if self.ibp: - adv_ibp = _get_ibp_score(u_c, l_c, y_tensor) - if self.affine: - adv_f = _get_affine_score( - z, w_u_f, b_u_f, w_l_f, b_l_f, y_tensor, perturbation_domain=self.perturbation_domain - ) - - if self.ibp and not self.affine: - adv_score = adv_ibp - elif self.ibp and self.affine: - adv_score = K.minimum(adv_ibp, adv_f) - elif not self.ibp and self.affine: - adv_score = adv_f - else: - raise NotImplementedError("not ibp and not affine not implemented") - - if self.mode == MetricMode.BACKWARD: - w_u_b, b_u_b, w_l_b, b_l_b, _ = inputs[-5:] - adv_b = _get_backward_score( - z, w_u_b, b_u_b, w_l_b, b_l_b, y_tensor, perturbation_domain=self.perturbation_domain - ) - adv_score = K.minimum(adv_score, adv_b) - - return adv_score - - -class AdversarialScore(AdversarialCheck): - """Training with symbolic LiRPA bounds for promoting adversarial robustness""" - - def __init__( - self, - ibp: bool, - affine: bool, - mode: Union[str, MetricMode], - perturbation_domain: Optional[PerturbationDomain], - **kwargs: Any, - ): - """ - Args: - ibp: boolean that indicates whether we propagate constant - bounds - affine: boolean that indicates whether we propagate affine - bounds - mode: str: 'backward' or 'forward' whether we doforward or - backward linear relaxation - perturbation_domain: the type of input perturbation domain for the - linear relaxation - **kwargs - """ - super().__init__(ibp=ibp, affine=affine, mode=mode, perturbation_domain=perturbation_domain, **kwargs) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: - """ - Args: - inputs - - Returns: - adv_score <0 if the predictionis robust on the input convex - domain - """ - - y_tensor = inputs[-1] - - if self.ibp and self.affine: - _, z, u_c, w_u_f, b_u_f, l_c, w_l_f, b_l_f = inputs[:8] - elif not self.ibp and self.affine: - _, z, w_u_f, b_u_f, w_l_f, b_l_f = inputs[:6] - elif self.ibp and not self.affine: - _, z, u_c, l_c = inputs[:4] - else: - raise NotImplementedError("not ibp and not affine not implemented") - - if self.ibp: - adv_ibp = _get_ibp_score(u_c=l_c, l_c=u_c, source_tensor=y_tensor) - if self.affine: - adv_f = _get_affine_score( - z, - w_u=w_l_f, - b_u=b_l_f, - w_l=w_u_f, - b_l=b_u_f, - source_tensor=y_tensor, - perturbation_domain=self.perturbation_domain, - ) - - if self.ibp and not self.affine: - adv_score = adv_ibp - elif self.ibp and self.affine: - adv_score = K.minimum(adv_ibp, adv_f) - elif not self.ibp and self.affine: - adv_score = adv_f - else: - raise NotImplementedError("not ibp and not affine not implemented") - - if self.mode == MetricMode.BACKWARD: - w_u_b, b_u_b, w_l_b, b_l_b, _ = inputs[-5:] - adv_b = _get_backward_score( - z, w_u_b, b_u_b, w_l_b, b_l_b, y_tensor, perturbation_domain=self.perturbation_domain - ) - adv_score = K.minimum(adv_score, adv_b) - - return adv_score - - -def build_formal_adv_check_model(decomon_model: DecomonModel) -> keras.Model: - """automatic design on a Keras model which predicts a certificate of adversarial robustness - - Args: - decomon_model - - Returns: - - """ - # check type and that backward pass is available - - perturbation_domain = decomon_model.perturbation_domain - layer = AdversarialCheck(decomon_model.ibp, decomon_model.affine, decomon_model.mode, perturbation_domain) - output = decomon_model.output - input = decomon_model.input - n_out = decomon_model.output[0].shape[1:] - y_out = Input(n_out) - - adv_score = layer(output + [y_out]) - adv_model = Model(input + [y_out], adv_score) - return adv_model - - -def build_formal_adv_model(decomon_model: DecomonModel) -> keras.Model: - """automatic design on a Keras model which predicts a certificate of adversarial robustness - - Args: - decomon_model - - Returns: - - """ - # check type and that backward pass is available - - perturbation_domain = decomon_model.perturbation_domain - layer = AdversarialScore(decomon_model.ibp, decomon_model.affine, decomon_model.mode, perturbation_domain) - output = decomon_model.output - input = decomon_model.input - n_out = decomon_model.output[0].shape[1:] - y_out = Input(n_out) - - adv_score = layer(output + [y_out]) - adv_model = Model(input + [y_out], adv_score) - return adv_model - - -class UpperScore(MetricLayer): - """Training with symbolic LiRPA bounds for limiting the local maximum of a neural network""" - - def __init__( - self, - ibp: bool, - affine: bool, - mode: Union[str, MetricMode], - perturbation_domain: Optional[PerturbationDomain], - **kwargs: Any, - ): - """ - Args: - ibp: boolean that indicates whether we propagate constant - bounds - affine: boolean that indicates whether we propagate affine - bounds - mode: str: 'backward' or 'forward' whether we doforward or - backward linear relaxation - perturbation_domain: the type of input perturbation domain for the - linear relaxation - **kwargs - """ - super().__init__(ibp=ibp, affine=affine, mode=mode, perturbation_domain=perturbation_domain, **kwargs) - - def linear_upper(self, z_tensor: Tensor, y_tensor: Tensor, w_u: Tensor, b_u: Tensor) -> Tensor: - w_upper = w_u * y_tensor[:, None] - b_upper = b_u * y_tensor - - upper_score = self.perturbation_domain.get_upper(z_tensor, w_upper, b_upper) - - return K.sum(upper_score, -1) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> BackendTensor: - """ - Args: - inputs - - Returns: - upper_score <=0 if the maximum of the neural network is - lower than the target - """ - - y_tensor = inputs[-1] - z_tensor = inputs[1] - - if self.ibp and self.affine: - _, _, u_c, w_u_f, b_u_f, _, _, _ = inputs[:8] - - upper_ibp = K.sum(u_c * y_tensor, -1) - upper_affine = self.linear_upper(z_tensor, y_tensor, w_u_f, b_u_f) - upper_score = K.minimum(upper_ibp, upper_affine) - elif not self.ibp and self.affine: - _, _, w_u_f, b_u_f = inputs[:6] - upper_score = self.linear_upper(z_tensor, y_tensor, w_u_f, b_u_f) - elif self.ibp and not self.affine: - _, _, u_c, l_c = inputs[:4] - upper_score = K.sum(u_c * y_tensor, -1) - else: - raise NotImplementedError("not ibp and not affine not implemented") - - if self.mode == MetricMode.BACKWARD: - w_u_b, b_u_b, _, _, _ = inputs[-5:] - upper_backward = self.linear_upper(z_tensor, y_tensor, w_u_b[:, 0], b_u_b[:, 0]) - upper_score = K.minimum(upper_score, upper_backward) - - return upper_score - - -def build_formal_upper_model(decomon_model: DecomonModel) -> keras.Model: - """automatic design on a Keras model which predicts a certificate on the local upper bound - - Args: - decomon_model - - Returns: - - """ - # check type and that backward pass is available - - perturbation_domain = decomon_model.perturbation_domain - layer = UpperScore(decomon_model.ibp, decomon_model.affine, decomon_model.mode, perturbation_domain) - output = decomon_model.output - input = decomon_model.input - n_out = decomon_model.output[0].shape[1:] - y_out = Input(n_out) - - upper_score = layer(output + [y_out]) - upper_model = Model(input + [y_out], upper_score) - return upper_model - - -def _get_ibp_score( - u_c: Tensor, - l_c: Tensor, - source_tensor: Tensor, - target_tensor: Optional[Tensor] = None, -) -> Tensor: - if target_tensor is None: - target_tensor = 1.0 - source_tensor - - shape = int(np.prod(u_c.shape[1:])) - u_c_reshaped = K.reshape(u_c, (-1, shape)) - l_c_reshaped = K.reshape(l_c, (-1, shape)) - - score_u = ( - l_c_reshaped * target_tensor - - K.expand_dims(K.min(u_c_reshaped * source_tensor, -1), -1) - - 1e6 * (1 - target_tensor) - ) - - return K.max(score_u, -1) - - -def _get_affine_score( - z_tensor: Tensor, - w_u: Tensor, - b_u: Tensor, - w_l: Tensor, - b_l: Tensor, - source_tensor: Tensor, - perturbation_domain: PerturbationDomain, - target_tensor: Optional[Tensor] = None, -) -> Tensor: - if target_tensor is None: - target_tensor = 1 - source_tensor - - n_dim = w_u.shape[1] - shape = int(np.prod(b_u.shape[1:])) - w_u_reshaped = K.reshape(w_u, (-1, n_dim, shape, 1)) - w_l_reshaped = K.reshape(w_l, (-1, n_dim, 1, shape)) - b_u_reshaped = K.reshape(b_u, (-1, shape, 1)) - b_l_reshaped = K.reshape(b_l, (-1, 1, shape)) - - w_u_f = w_l_reshaped - w_u_reshaped - b_u_f = b_l_reshaped - b_u_reshaped - - # add penalties on biases - oneminus_target_tensor: Tensor = 1 - target_tensor # for mypy - oneminus_source_tensor: Tensor = 1 - source_tensor # for mypy - b_u_f = b_u_f - 1e6 * oneminus_target_tensor[:, None, :] - b_u_f = b_u_f - 1e6 * oneminus_source_tensor[:, :, None] - - upper = perturbation_domain.get_upper(z_tensor, w_u_f, b_u_f) - return K.max(upper, (-1, -2)) - - -def _get_backward_score( - z_tensor: Tensor, - w_u: Tensor, - b_u: Tensor, - w_l: Tensor, - b_l: Tensor, - source_tensor: Tensor, - perturbation_domain: PerturbationDomain, - target_tensor: Optional[Tensor] = None, -) -> Tensor: - return _get_affine_score( - z_tensor, - w_u[:, 0], - b_u[:, 0], - w_l[:, 0], - b_l[:, 0], - source_tensor, - perturbation_domain=perturbation_domain, - target_tensor=target_tensor, - ) diff --git a/src/decomon/metrics/utils.py b/src/decomon/metrics/utils.py deleted file mode 100644 index 3cc9a7e5..00000000 --- a/src/decomon/metrics/utils.py +++ /dev/null @@ -1,33 +0,0 @@ -from typing import Optional, Union - -from decomon.core import BoxDomain, ForwardMode, PerturbationDomain -from decomon.layers.utils import exp, expand_dims, log, sum -from decomon.types import Tensor -from decomon.utils import add, minus - -# compute categorical cross entropy - - -def categorical_cross_entropy( - inputs: list[Tensor], - dc_decomp: bool = False, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - perturbation_domain: Optional[PerturbationDomain] = None, -) -> list[Tensor]: - # step 1: exponential - if perturbation_domain is None: - perturbation_domain = BoxDomain() - outputs = exp(inputs, mode=mode, perturbation_domain=perturbation_domain, dc_decomp=dc_decomp) - # step 2: sum - outputs = sum(outputs, axis=-1, mode=mode) - # step 3: log - outputs = log(outputs, dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - outputs = expand_dims(outputs, dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain, axis=-1) - # step 4: - inputs - return add( - minus(inputs, mode=mode, perturbation_domain=perturbation_domain, dc_decomp=dc_decomp), - outputs, - mode=mode, - perturbation_domain=perturbation_domain, - dc_decomp=dc_decomp, - ) diff --git a/src/decomon/models/backward_cloning.py b/src/decomon/models/backward_cloning.py index b0c7ea52..39d3a531 100644 --- a/src/decomon/models/backward_cloning.py +++ b/src/decomon/models/backward_cloning.py @@ -1,36 +1,19 @@ -from copy import deepcopy from collections.abc import Callable from typing import Any, Optional, Union import keras -import keras.ops as K -from keras.config import floatx -from keras.layers import InputLayer, Lambda, Layer +from keras.layers import Layer from keras.models import Model from keras.src.ops.node import Node -from keras.src.utils.python_utils import to_list - -from decomon.core import ( - BoxDomain, - ForwardMode, - InputsOutputsSpec, - PerturbationDomain, - Propagation, - Slope, - get_affine, - get_mode, -) + +from decomon.constants import Propagation, Slope from decomon.layers import DecomonLayer from decomon.layers.convert import to_decomon from decomon.layers.crown import ReduceCrownBounds from decomon.layers.merging.base_merge import DecomonMerge from decomon.layers.oracle import DecomonOracle -from decomon.models.crown import Convert2BackwardMode, Fuse, MergeWithPrevious -from decomon.models.utils import ( - ensure_functional_model, - get_depth_dict, - get_output_nodes, -) +from decomon.models.utils import ensure_functional_model, get_output_nodes +from decomon.perturbation_domain import BoxDomain, PerturbationDomain from decomon.types import Tensor diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index acd06eac..46c8b792 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -6,13 +6,7 @@ from keras.layers import Layer from keras.models import Model -from decomon.core import ( - BoxDomain, - ConvertMethod, - PerturbationDomain, - Propagation, - Slope, -) +from decomon.constants import ConvertMethod, Propagation, Slope from decomon.layers import DecomonLayer from decomon.layers.convert import to_decomon from decomon.layers.fuse import Fuse @@ -37,10 +31,10 @@ get_ibp_affine_from_method, is_input_node, method2propagation, - preprocess_backward_bounds, preprocess_layer, split_activation, ) +from decomon.perturbation_domain import BoxDomain, PerturbationDomain logger = logging.getLogger(__name__) diff --git a/src/decomon/models/crown.py b/src/decomon/models/crown.py deleted file mode 100644 index 6abbd7df..00000000 --- a/src/decomon/models/crown.py +++ /dev/null @@ -1,186 +0,0 @@ -# extra layers necessary for backward LiRPA -from typing import Any, Optional, Union - -import keras.ops as K -from keras.layers import InputSpec, Layer -from keras.src.layers.merging.dot import batch_dot - -from decomon.core import ForwardMode, PerturbationDomain -from decomon.keras_utils import BatchedDiagLike -from decomon.types import BackendTensor - - -class Fuse(Layer): - def __init__(self, mode: Union[str, ForwardMode], **kwargs: Any): - super().__init__(**kwargs) - self.mode = ForwardMode(mode) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - inputs_wo_backward_bounds = inputs[:-4] - backward_bounds = inputs[-4:] - - if self.mode == ForwardMode.AFFINE: - x_0, w_f_u, b_f_u, w_f_l, b_f_l = inputs_wo_backward_bounds - elif self.mode == ForwardMode.HYBRID: - x_0, u_c, w_f_u, b_f_u, l_c, w_f_l, b_f_l = inputs_wo_backward_bounds - else: - return backward_bounds - - return merge_with_previous([w_f_u, b_f_u, w_f_l, b_f_l] + backward_bounds) - - def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: - inputs_wo_backward_bounds_shapes = input_shape[:-4] - backward_bounds_shapes = input_shape[-4:] - - if self.mode == ForwardMode.AFFINE: - x_0_shape, w_f_u_shape, b_f_u_shape, w_f_l_shape, b_f_l_shape = inputs_wo_backward_bounds_shapes - elif self.mode == ForwardMode.HYBRID: - ( - x_0_shape, - u_c_shape, - w_f_u_shape, - b_f_u_shape, - l_c_shape, - w_f_l_shape, - b_f_l_shape, - ) = inputs_wo_backward_bounds_shapes - else: - return backward_bounds_shapes - - return merge_with_previous_compute_output_shape( - [w_f_u_shape, b_f_u_shape, w_f_l_shape, b_f_l_shape] + backward_bounds_shapes - ) - - def get_config(self) -> dict[str, Any]: - config = super().get_config() - config.update({"mode": self.mode}) - return config - - -class Convert2BackwardMode(Layer): - def __init__(self, mode: Union[str, ForwardMode], perturbation_domain: PerturbationDomain, **kwargs: Any): - super().__init__(**kwargs) - self.mode = ForwardMode(mode) - self.perturbation_domain = perturbation_domain - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - inputs_wo_backward_bounds = inputs[:-4] - backward_bounds = inputs[-4:] - w_u_out, b_u_out, w_l_out, b_l_out = backward_bounds - empty_tensor = K.convert_to_tensor([]) - - if self.mode in [ForwardMode.AFFINE, ForwardMode.HYBRID]: - x_0 = inputs_wo_backward_bounds[0] - else: - u_c, l_c = inputs_wo_backward_bounds - x_0 = K.concatenate([K.expand_dims(l_c, 1), K.expand_dims(u_c, 1)], 1) - - if self.mode in [ForwardMode.IBP, ForwardMode.HYBRID]: - u_c_out = self.perturbation_domain.get_upper(x_0, w_u_out, b_u_out) - l_c_out = self.perturbation_domain.get_lower(x_0, w_l_out, b_l_out) - else: - u_c_out, l_c_out = empty_tensor, empty_tensor - - if self.mode == ForwardMode.AFFINE: - return [x_0] + backward_bounds - elif self.mode == ForwardMode.IBP: - return [u_c_out, l_c_out] - elif self.mode == ForwardMode.HYBRID: - return [x_0, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out] - - else: - raise ValueError(f"Unknwon mode {self.mode}") - - def get_config(self) -> dict[str, Any]: - config = super().get_config() - config.update({"mode": self.mode, "perturbation_domain": self.perturbation_domain}) - return config - - -class MergeWithPrevious(Layer): - def __init__( - self, - input_shape_layer: Optional[tuple[int, ...]] = None, - backward_shape_layer: Optional[tuple[int, ...]] = None, - **kwargs: Any, - ): - super().__init__(**kwargs) - self.input_shape_layer = input_shape_layer - self.backward_shape_layer = backward_shape_layer - if not (input_shape_layer is None) and not (backward_shape_layer is None): - _, n_in, n_h = input_shape_layer - _, n_h, n_out = backward_shape_layer - - w_out_spec = InputSpec(ndim=3, axes={-1: n_h, -2: n_in}) - b_out_spec = InputSpec(ndim=2, axes={-1: n_h}) - w_b_spec = InputSpec(ndim=3, axes={-1: n_out, -2: n_h}) - b_b_spec = InputSpec(ndim=2, axes={-1: n_out}) - self.input_spec = [w_out_spec, b_out_spec] * 2 + [w_b_spec, b_b_spec] * 2 # - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - return merge_with_previous(inputs) - - def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: - return merge_with_previous_compute_output_shape(input_shape) - - def get_config(self) -> dict[str, Any]: - config = super().get_config() - config.update( - { - "input_shape_layer": self.input_shape_layer, - "backward_shape_layer": self.backward_shape_layer, - } - ) - return config - - -def merge_with_previous_compute_output_shape( - input_shapes: list[tuple[Optional[int], ...]] -) -> list[tuple[Optional[int], ...]]: - w_b_in_shape, b_b_in_shape = input_shapes[-2:] - w_out_in_shape, b_out_in_shape = input_shapes[:2] - batch_size, flattened_keras_input_shape, flattened_keras_output_shape = w_out_in_shape - _, _, flattened_model_output_shape = w_b_in_shape - b_b_out_shape = b_b_in_shape - w_b_out_shape = batch_size, flattened_keras_input_shape, flattened_model_output_shape - return [w_b_out_shape, b_b_out_shape] * 2 - - -def merge_with_previous(inputs: list[BackendTensor]) -> list[BackendTensor]: - w_u_out, b_u_out, w_l_out, b_l_out, w_b_u, b_b_u, w_b_l, b_b_l = inputs - - # w_u_out (None, n_h_in, n_h_out) - # w_b_u (None, n_h_out, n_out) - - # w_u_out_ (None, n_h_in, n_h_out, 1) - # w_b_u_ (None, 1, n_h_out, n_out) - # w_u_out_*w_b_u_ (None, n_h_in, n_h_out, n_out) - - # result (None, n_h_in, n_out) - - if len(w_u_out.shape) == 2: - w_u_out = BatchedDiagLike()(w_u_out) - - if len(w_l_out.shape) == 2: - w_l_out = BatchedDiagLike()(w_l_out) - - if len(w_b_u.shape) == 2: - w_b_u = BatchedDiagLike()(w_b_u) - - if len(w_b_l.shape) == 2: - w_b_l = BatchedDiagLike()(w_b_l) - - # import pdb; pdb.set_trace() - - z_value = K.cast(0.0, dtype=w_u_out.dtype) - w_b_u_pos = K.maximum(w_b_u, z_value) - w_b_u_neg = K.minimum(w_b_u, z_value) - w_b_l_pos = K.maximum(w_b_l, z_value) - w_b_l_neg = K.minimum(w_b_l, z_value) - - w_u = batch_dot(w_u_out, w_b_u_pos, (-1, -2)) + batch_dot(w_l_out, w_b_u_neg, (-1, -2)) - w_l = batch_dot(w_l_out, w_b_l_pos, (-1, -2)) + batch_dot(w_u_out, w_b_l_neg, (-1, -2)) - b_u = batch_dot(b_u_out, w_b_u_pos, (-1, -2)) + batch_dot(b_l_out, w_b_u_neg, (-1, -2)) + b_b_u - b_l = batch_dot(b_l_out, w_b_l_pos, (-1, -2)) + batch_dot(b_u_out, w_b_l_neg, (-1, -2)) + b_b_l - - return [w_u, b_u, w_l, b_l] diff --git a/src/decomon/models/forward_cloning.py b/src/decomon/models/forward_cloning.py index 596092ad..d40a3928 100644 --- a/src/decomon/models/forward_cloning.py +++ b/src/decomon/models/forward_cloning.py @@ -4,23 +4,17 @@ """ from collections.abc import Callable -from typing import Any, Optional, Union +from typing import Any, Optional import keras -import keras.ops as K from keras.layers import InputLayer, Layer from keras.models import Model -from decomon.core import ( - BoxDomain, - InputsOutputsSpec, - PerturbationDomain, - Propagation, - Slope, -) +from decomon.constants import Propagation, Slope from decomon.layers import DecomonLayer from decomon.layers.convert import to_decomon from decomon.layers.input import ForwardInput +from decomon.layers.inputs_outputs_specs import InputsOutputsSpec from decomon.models.utils import ( ensure_functional_model, get_depth_dict, @@ -28,6 +22,7 @@ prepare_inputs_for_layer, wrap_outputs_from_layer_in_list, ) +from decomon.perturbation_domain import BoxDomain, PerturbationDomain def convert_forward( @@ -78,7 +73,6 @@ def convert_forward( ibp=ibp, affine=affine, propagation=propagation, - perturbation_domain=perturbation_domain, model_input_shape=model_input_shape, layer_input_shape=model_input_shape, ) diff --git a/src/decomon/models/models.py b/src/decomon/models/models.py index a7c8ea18..03ac92af 100644 --- a/src/decomon/models/models.py +++ b/src/decomon/models/models.py @@ -6,7 +6,8 @@ from keras import Model from keras.utils import serialize_keras_object -from decomon.core import ConvertMethod, PerturbationDomain +from decomon.constants import ConvertMethod +from decomon.perturbation_domain import PerturbationDomain class DecomonModel(keras.Model): diff --git a/src/decomon/models/utils.py b/src/decomon/models/utils.py index 1f1b57cf..bd557754 100644 --- a/src/decomon/models/utils.py +++ b/src/decomon/models/utils.py @@ -1,30 +1,15 @@ from typing import Any, Optional, Union import keras -import keras.ops as K import numpy as np from keras import Model, Sequential -from keras import ops as K -from keras.layers import Activation, Input, Lambda, Layer +from keras.layers import Activation, Input, Layer from keras.src import Functional from keras.src.ops.node import Node -from decomon.core import ( - BallDomain, - BoxDomain, - ConvertMethod, - ForwardMode, - InputsOutputsSpec, - PerturbationDomain, - Propagation, - get_mode, -) -from decomon.keras_utils import ( - BatchedIdentityLike, - is_a_merge_layer, - share_weights_and_build, -) -from decomon.types import BackendTensor +from decomon.constants import ConvertMethod, Propagation +from decomon.keras_utils import share_weights_and_build +from decomon.perturbation_domain import PerturbationDomain def generate_perturbation_domain_input( @@ -176,89 +161,6 @@ def fill_dico(node: Node, dico_depth: Optional[dict[int, int]] = None) -> dict[i return dico_nodes -def get_inner_layers(model: Model) -> int: - count = 0 - for layer in model.layers: - if isinstance(layer, Model): - count += get_inner_layers(layer) - else: - count += 1 - return count - - -class Convert2Mode(Layer): - def __init__( - self, - mode_from: Union[str, ForwardMode], - mode_to: Union[str, ForwardMode], - perturbation_domain: PerturbationDomain, - input_dim: int = -1, - **kwargs: Any, - ): - super().__init__(**kwargs) - self.mode_from = ForwardMode(mode_from) - self.mode_to = ForwardMode(mode_to) - self.perturbation_domain = perturbation_domain - self.input_dim = input_dim - dc_decomp = False - self.dc_decomp = dc_decomp - self.inputs_outputs_spec_from = InputsOutputsSpec( - dc_decomp=dc_decomp, - mode=mode_from, - perturbation_domain=perturbation_domain, - model_input_dim=self.input_dim, - ) - self.inputs_outputs_spec_to = InputsOutputsSpec( - dc_decomp=dc_decomp, - mode=mode_to, - perturbation_domain=perturbation_domain, - model_input_dim=self.input_dim, - ) - - def call(self, inputs: list[BackendTensor], **kwargs: Any) -> list[BackendTensor]: - compute_ibp_from_affine = self.mode_from == ForwardMode.AFFINE and self.mode_to != ForwardMode.AFFINE - tight = self.mode_from == ForwardMode.HYBRID and self.mode_to != ForwardMode.AFFINE - compute_dummy_affine = self.mode_from == ForwardMode.IBP and self.mode_to != ForwardMode.IBP - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = self.inputs_outputs_spec_from.get_fullinputs_from_inputsformode( - inputs, compute_ibp_from_affine=compute_ibp_from_affine, tight=tight - ) - - if compute_dummy_affine: - x = K.concatenate([K.expand_dims(l_c, 1), K.expand_dims(u_c, 1)], 1) - w = K.zeros_like(BatchedIdentityLike()(l_c)) - w_u = w - b_u = u_c - w_l = w - b_l = l_c - - return self.inputs_outputs_spec_to.extract_outputsformode_from_fulloutputs( - [x, u_c, w_u, b_u, l_c, w_l, b_l, h, g] - ) - - def compute_output_shape(self, input_shape: list[tuple[Optional[int], ...]]) -> list[tuple[Optional[int], ...]]: - ( - x_shape, - u_c_shape, - w_u_shape, - b_u_shape, - l_c_shape, - w_l_shape, - b_l_shape, - h_shape, - g_shape, - ) = self.inputs_outputs_spec_from.get_fullinputshapes_from_inputshapesformode(input_shape) - return self.inputs_outputs_spec_to.extract_inputshapesformode_from_fullinputshapes( - [x_shape, u_c_shape, w_u_shape, b_u_shape, l_c_shape, w_l_shape, b_l_shape, h_shape, g_shape] - ) - - def get_config(self) -> dict[str, Any]: - config = super().get_config() - config.update( - {"mode_from": self.mode_from, "mode_to": self.mode_to, "perturbation_domain": self.perturbation_domain} - ) - return config - - def ensure_functional_model(model: Model) -> Functional: if isinstance(model, Functional): return model @@ -270,80 +172,6 @@ def ensure_functional_model(model: Model) -> Functional: raise NotImplementedError("Decomon model available only for functional or sequential models.") -def preprocess_backward_bounds( - backward_bounds: Optional[Union[keras.KerasTensor, list[keras.KerasTensor], list[list[keras.KerasTensor]]]], - nb_model_outputs: int, -) -> Optional[list[list[keras.KerasTensor]]]: - """Preprocess backward bounds to be used by `convert()`. - - Args: - backward_bounds: backward bounds to propagate - nb_model_outputs: number of outputs of the keras model to convert - - Returns: - formatted backward bounds - - Backward bounds can be given as - - None or empty list => backward bounds to propagate will be identity - - a single list (potentially partially filled) => same backward bounds on all model outputs (assuming that all outputs have same shape) - - a list of list : different backward bounds for each model output - - Which leads to the following formatting: - - None, [], or [[]] -> None - - single keras tensor w -> [[w, 0, w, 0]] * nb_model_outputs - - [w] -> idem - - [w, b] -> [[w, b, w, b]] * nb_model_outputs - - [w_l, b_l, w_u, b_u] -> [[w_l, b_l, w_u, b_u]] * nb_model_outputs - - [[w_l, b_l, w_u, b_u]] -> [[w_l, b_l, w_u, b_u]] * nb_model_outputs - - list of lists of tensors [w_l[i], b_l[i], w_u[i], b_u[i]]_i -> we enforce each sublist to have 4 elements - - """ - if backward_bounds is None: - # None - return None - if isinstance(backward_bounds, keras.KerasTensor): - # single tensor w - w = backward_bounds - b = K.zeros_like(w[:, 0]) - backward_bounds = [w, b, w, b] - if len(backward_bounds) == 0: - return None - else: - if isinstance(backward_bounds[0], keras.KerasTensor): - # list of tensors - if len(backward_bounds) == 1: - # single tensor w - return preprocess_backward_bounds(backward_bounds=backward_bounds[0], nb_model_outputs=nb_model_outputs) - elif len(backward_bounds) == 2: - # w, b - w, b = backward_bounds - return preprocess_backward_bounds(backward_bounds=[w, b, w, b], nb_model_outputs=nb_model_outputs) - elif len(backward_bounds) == 4: - return [backward_bounds] * nb_model_outputs - else: - raise ValueError( - "If backward_bounds is given as a list of tensors, it should have 1, 2, or 4 elements." - ) - else: - # list of list of tensors - if len(backward_bounds) == 1: - if len(backward_bounds[0]) == 0: - # [[]] - return None - else: - return [backward_bounds[0]] * nb_model_outputs - elif len(backward_bounds) != nb_model_outputs: - raise ValueError( - "If backward_bounds is given as a list of tensors, it should have nb_model_ouptputs elements." - ) - elif not all([len(backward_bounds_i) == 4 for backward_bounds_i in backward_bounds]): - raise ValueError( - "If backward_bounds is given as a list of tensors, each sublist should have 4 elements (w_l_, b_l, w_u, b_u)." - ) - else: - return backward_bounds - - def get_ibp_affine_from_method(method: ConvertMethod) -> tuple[bool, bool]: method = ConvertMethod(method) if method in [ConvertMethod.FORWARD_IBP, ConvertMethod.CROWN_FORWARD_IBP]: diff --git a/src/decomon/perturbation_domain.py b/src/decomon/perturbation_domain.py new file mode 100644 index 00000000..a8f45ed8 --- /dev/null +++ b/src/decomon/perturbation_domain.py @@ -0,0 +1,476 @@ +from abc import ABC, abstractmethod +from enum import Enum +from typing import Any, Union + +import numpy as np +from keras import ops as K + +from decomon.keras_utils import batch_multid_dot +from decomon.types import Tensor + + +class Option(str, Enum): + lagrangian = "lagrangian" + milp = "milp" + + +class PerturbationDomain(ABC): + opt_option: Option + + def __init__(self, opt_option: Union[str, Option] = Option.milp): + self.opt_option = Option(opt_option) + + @abstractmethod + def get_upper_x(self, x: Tensor) -> Tensor: + """Get upper constant bound on perturbation domain input.""" + ... + + @abstractmethod + def get_lower_x(self, x: Tensor) -> Tensor: + """Get lower constant bound on perturbation domain input.""" + ... + + @abstractmethod + def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + """Merge upper affine bounds with perturbation domain input to get upper constant bound. + + Args: + x: perturbation domain input + w: weights of the affine bound + b: bias of the affine bound + missing_batchsize: whether w and b are missing batchsize + **kwargs: + + Returns: + + """ + ... + + @abstractmethod + def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + """Merge lower affine bounds with perturbation domain input to get lower constant bound. + + Args: + x: perturbation domain input + w: weights of the affine bound + b: bias of the affine bound + missing_batchsize: whether w and b are missing batchsize + **kwargs: + + Returns: + + """ + ... + + @abstractmethod + def get_nb_x_components(self) -> int: + """Get the number of components in perturabation domain input. + + For instance: + - box domain: each corner of the box -> 2 components + - ball domain: center of the ball -> 1 component + + """ + ... + + @abstractmethod + def get_input_from_constant_bounds(self, constant_bounds: list[Tensor]) -> Tensor: + """Construct perturbation domain input x from constant bounds on keras model input + + Args: + constant_bounds: lower and upper constant bounds on keras model input + + Returns: + x: perturbation domain input + + """ + ... + + def get_config(self) -> dict[str, Any]: + return { + "opt_option": self.opt_option, + } + + def get_kerasinputlike_from_x(self, x: Tensor) -> Tensor: + """Get tensor of same shape as keras model input, from perturbation domain input x + + Args: + x: perturbation domain input + + Returns: + tensor of same shape as keras model input + + """ + if self.get_nb_x_components() == 1: + return x + else: + return x[:, 0] + + def get_x_input_shape_wo_batchsize(self, original_input_shape: tuple[int, ...]) -> tuple[int, ...]: + """Get expected perturbation domain input shape, excepting the batch axis.""" + n_comp_x = self.get_nb_x_components() + if n_comp_x == 1: + return original_input_shape + else: + return (n_comp_x,) + original_input_shape + + def get_keras_input_shape_wo_batchsize(self, x_shape: tuple[int, ...]) -> tuple[int, ...]: + """Deduce keras model input shape from perturbation domain input shape.""" + n_comp_x = self.get_nb_x_components() + if n_comp_x == 1: + return x_shape + else: + return x_shape[1:] + + +class BoxDomain(PerturbationDomain): + def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + x_min = x[:, 0] + x_max = x[:, 1] + return get_upper_box(x_min=x_min, x_max=x_max, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) + + def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + x_min = x[:, 0] + x_max = x[:, 1] + return get_lower_box(x_min=x_min, x_max=x_max, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) + + def get_upper_x(self, x: Tensor) -> Tensor: + return x[:, 1] + + def get_lower_x(self, x: Tensor) -> Tensor: + return x[:, 0] + + def get_nb_x_components(self) -> int: + return 2 + + def get_input_from_constant_bounds(self, constant_bounds: list[Tensor]) -> Tensor: + lower, upper = constant_bounds + return K.concatenate([lower[:, None], upper[:, None]], axis=1) + + +class GridDomain(PerturbationDomain): + pass + + +class VertexDomain(PerturbationDomain): + pass + + +class BallDomain(PerturbationDomain): + def __init__(self, eps: float, p: float = 2, opt_option: Option = Option.milp): + super().__init__(opt_option=opt_option) + self.eps = eps + # check on p + p_error_msg = "p must be a positive integer or np.inf" + try: + if p != np.inf and (int(p) != p or p <= 0): + raise ValueError(p_error_msg) + except: + raise ValueError(p_error_msg) + self.p = p + + def get_config(self) -> dict[str, Any]: + config = super().get_config() + config.update( + { + "eps": self.eps, + "p": self.p, + } + ) + return config + + def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + return get_lower_ball(x_0=x, eps=self.eps, p=self.p, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) + + def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + return get_upper_ball(x_0=x, eps=self.eps, p=self.p, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) + + def get_nb_x_components(self) -> int: + return 1 + + def get_lower_x(self, x: Tensor) -> Tensor: + return x - self.eps + + def get_upper_x(self, x: Tensor) -> Tensor: + return x + self.eps + + +def get_upper_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + """Compute the max of an affine function + within a box (hypercube) defined by its extremal corners + + Args: + x_min: lower bound of the box domain + x_max: upper bound of the box domain + w: weights of the affine function + b: bias of the affine function + missing_batchsize: whether w and b are missing the batchsize + + Returns: + max_(x >= x_min, x<=x_max) w*x + b + + Note: + We can have w, b in diagonal representation and/or without a batch axis. + We assume that x_min, x_max have always its batch axis. + + """ + z_value = K.cast(0.0, dtype=x_min.dtype) + w_pos = K.maximum(w, z_value) + w_neg = K.minimum(w, z_value) + + is_diag = w.shape == b.shape + diagonal = (False, is_diag) + missing_batchsize = (False, missing_batchsize) + + return ( + batch_multid_dot(x_max, w_pos, diagonal=diagonal, missing_batchsize=missing_batchsize) + + batch_multid_dot(x_min, w_neg, diagonal=diagonal, missing_batchsize=missing_batchsize) + + b + ) + + +def get_lower_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + """ + Args: + x_min: lower bound of the box domain + x_max: upper bound of the box domain + w: weights of the affine lower bound + b: bias of the affine lower bound + missing_batchsize: whether w and b are missing the batchsize + + Returns: + min_(x >= x_min, x<=x_max) w*x + b + + Note: + We can have w, b in diagonal representation and/or without a batch axis. + We assume that x_min, x_max have always its batch axis. + + """ + return get_upper_box(x_min=x_max, x_max=x_min, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) + + +def get_lq_norm(x: Tensor, p: float, axis: Union[int, list[int]] = -1) -> Tensor: + """compute Lp norm (p=1 or 2) + + Args: + x: tensor + p: the power must be an integer in (1, 2) + axis: the axis on which we compute the norm + + Returns: + ||w||^p + """ + if p == 1: + x_q = K.max(K.abs(x), axis) + elif p == 2: + x_q = K.sqrt(K.sum(K.power(x, p), axis)) + else: + raise NotImplementedError("p must be equal to 1 or 2") + + return x_q + + +def get_upper_ball( + x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any +) -> Tensor: + """max of an affine function over an Lp ball + + Args: + x_0: the center of the ball + eps: the radius + p: the type of Lp norm considered + w: weights of the affine function + b: bias of the affine function + missing_batchsize: whether w and b are missing the batchsize + + Returns: + max_(|x - x_0|_p<= eps) w*x + b + """ + if p == np.inf: + # compute x_min and x_max according to eps + x_min = x_0 - eps + x_max = x_0 + eps + return get_upper_box(x_min, x_max, w, b, missing_batchsize=missing_batchsize) + + else: + if len(kwargs): + return get_upper_ball_finetune(x_0, eps, p, w, b, missing_batchsize=missing_batchsize, **kwargs) + + # use Holder's inequality p+q=1 + # ||w||_q*eps + w*x_0 + b + + is_diag = w.shape == b.shape + + # lq-norm of w + if is_diag: + w_q = K.abs(w) + else: + nb_axes_wo_batchsize_x = len(x_0.shape) - 1 + if missing_batchsize: + reduced_axes = list(range(nb_axes_wo_batchsize_x)) + else: + reduced_axes = list(range(1, 1 + nb_axes_wo_batchsize_x)) + w_q = get_lq_norm(w, p, axis=reduced_axes) + + diagonal = (False, is_diag) + missing_batchsize = (False, missing_batchsize) + return batch_multid_dot(x_0, w, diagonal=diagonal, missing_batchsize=missing_batchsize) + b + w_q * eps + + +def get_lower_ball( + x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any +) -> Tensor: + """min of an affine fucntion over an Lp ball + + Args: + x_0: the center of the ball + eps: the radius + p: the type of Lp norm considered + w: weights of the affine function + b: bias of the affine function + missing_batchsize: whether w and b are missing the batchsize + + Returns: + min_(|x - x_0|_p<= eps) w*x + b + """ + if p == np.inf: + # compute x_min and x_max according to eps + x_min = x_0 - eps + x_max = x_0 + eps + return get_lower_box(x_min, x_max, w, b, missing_batchsize=missing_batchsize) + + else: + if len(kwargs): + return get_lower_ball_finetune(x_0, eps, p, w, b, missing_batchsize=missing_batchsize, **kwargs) + + # use Holder's inequality p+q=1 + # - ||w||_q*eps + w*x_0 + b + + is_diag = w.shape == b.shape + + # lq-norm of w + if is_diag: + w_q = K.abs(w) + else: + nb_axes_wo_batchsize_x = len(x_0.shape) - 1 + if missing_batchsize: + reduced_axes = list(range(nb_axes_wo_batchsize_x)) + else: + reduced_axes = list(range(1, 1 + nb_axes_wo_batchsize_x)) + w_q = get_lq_norm(w, p, axis=reduced_axes) + + diagonal = (False, is_diag) + missing_batchsize = (False, missing_batchsize) + return batch_multid_dot(x_0, w, diagonal=diagonal, missing_batchsize=missing_batchsize) + b - w_q * eps + + +def get_lower_ball_finetune( + x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any +) -> Tensor: + if missing_batchsize: + raise NotImplementedError() + + if "finetune_lower" in kwargs and "upper" in kwargs or "lower" in kwargs: + alpha = kwargs["finetune_lower"] + # assume alpha is the same shape as w, minus the batch dimension + n_shape = len(w.shape) - 2 + z_value = K.cast(0.0, dtype=w.dtype) + + if "upper" and "lower" in kwargs: + upper = kwargs["upper"] # flatten vector + lower = kwargs["lower"] # flatten vector + + upper_reshaped = np.reshape(upper, [1, -1] + [1] * n_shape) + lower_reshaped = np.reshape(lower, [1, -1] + [1] * n_shape) + + w_alpha = w * alpha[None] + w_alpha_bar = w * (1 - alpha) + + score_box = K.sum(K.maximum(z_value, w_alpha_bar) * lower_reshaped, 1) + K.sum( + K.minimum(z_value, w_alpha_bar) * upper_reshaped, 1 + ) + score_ball = get_lower_ball(x_0, eps, p, w_alpha, b) + + return score_box + score_ball + + if "upper" in kwargs: + upper = kwargs["upper"] # flatten vector + upper_reshaped = np.reshape(upper, [1, -1] + [1] * n_shape) + + w_alpha = K.minimum(z_value, w) * alpha[None] + K.maximum(z_value, w) + w_alpha_bar = K.minimum(z_value, w) * (1 - alpha[None]) + + score_box = K.sum(K.minimum(z_value, w_alpha_bar) * upper_reshaped, 1) + score_ball = get_lower_ball(x_0, eps, p, w_alpha, b) + + return score_box + score_ball + + if "lower" in kwargs: + lower = kwargs["lower"] # flatten vector + lower_reshaped = np.reshape(lower, [1, -1] + [1] * n_shape) + + w_alpha = K.maximum(z_value, w) * alpha[None] + K.minimum(z_value, w) + w_alpha_bar = K.maximum(z_value, w) * (1 - alpha[None]) + + score_box = K.sum(K.maximum(z_value, w_alpha_bar) * lower_reshaped, 1) + score_ball = get_lower_ball(x_0, eps, p, w_alpha, b) + + return score_box + score_ball + + return get_lower_ball(x_0, eps, p, w, b) + + +def get_upper_ball_finetune( + x_0: Tensor, eps: float, p: float, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any +) -> Tensor: + if missing_batchsize: + raise NotImplementedError() + + if "finetune_upper" in kwargs and "upper" in kwargs or "lower" in kwargs: + alpha = kwargs["finetune_upper"] + # assume alpha is the same shape as w, minus the batch dimension + n_shape = len(w.shape) - 2 + z_value = K.cast(0.0, dtype=w.dtype) + + if "upper" and "lower" in kwargs: + upper = kwargs["upper"] # flatten vector + lower = kwargs["lower"] # flatten vector + + upper_reshaped = np.reshape(upper, [1, -1] + [1] * n_shape) + lower_reshaped = np.reshape(lower, [1, -1] + [1] * n_shape) + + w_alpha = w * alpha[None] + w_alpha_bar = w * (1 - alpha) + + score_box = K.sum(K.maximum(z_value, w_alpha_bar) * upper_reshaped, 1) + K.sum( + K.minimum(z_value, w_alpha_bar) * lower_reshaped, 1 + ) + score_ball = get_lower_ball(x_0, eps, p, w_alpha, b) + + return score_box + score_ball + + if "upper" in kwargs: + upper = kwargs["upper"] # flatten vector + upper_reshaped = np.reshape(upper, [1, -1] + [1] * n_shape) + + w_alpha = K.minimum(z_value, w) * alpha[None] + K.maximum(z_value, w) + w_alpha_bar = K.minimum(z_value, w) * (1 - alpha[None]) + + score_box = K.sum(K.maximum(z_value, w_alpha_bar) * upper_reshaped, 1) + score_ball = get_lower_ball(x_0, eps, p, w_alpha, b) + + return score_box + score_ball + + if "lower" in kwargs: + lower = kwargs["lower"] # flatten vector + lower_reshaped = np.reshape(lower, [1, -1] + [1] * n_shape) + + w_alpha = K.maximum(z_value, w) * alpha[None] + K.minimum(z_value, w) + w_alpha_bar = K.maximum(z_value, w) * (1 - alpha[None]) + + score_box = K.sum(K.minimum(z_value, w_alpha_bar) * lower_reshaped, 1) + score_ball = get_lower_ball(x_0, eps, p, w_alpha, b) + + return score_box + score_ball + + return get_upper_ball(x_0, eps, p, w, b) diff --git a/src/decomon/utils.py b/src/decomon/utils.py deleted file mode 100644 index a4d79448..00000000 --- a/src/decomon/utils.py +++ /dev/null @@ -1,722 +0,0 @@ -from collections.abc import Callable -from typing import Any, Optional, Union - -import keras.ops as K -import numpy as np -from keras.config import epsilon - -from decomon.core import ( - BoxDomain, - ForwardMode, - GridDomain, - InputsOutputsSpec, - PerturbationDomain, - Slope, - get_affine, - get_ibp, -) -from decomon.types import Tensor - -TensorFunction = Callable[[Tensor], Tensor] - - -# linear hull for activation function -def relu_prime(x: Tensor) -> Tensor: - """Derivative of relu - - Args: - x - - Returns: - - """ - - return K.clip(K.sign(x), K.cast(0, dtype=x.dtype), K.cast(1, dtype=x.dtype)) - - -def sigmoid_prime(x: Tensor) -> Tensor: - """Derivative of sigmoid - - Args: - x - - Returns: - - """ - - s_x = K.sigmoid(x) - return s_x * (K.cast(1, dtype=x.dtype) - s_x) - - -def tanh_prime(x: Tensor) -> Tensor: - """Derivative of tanh - - Args: - x - - Returns: - - """ - - s_x = K.tanh(x) - return K.cast(1, dtype=x.dtype) - K.power(s_x, K.cast(2, dtype=x.dtype)) - - -def softsign_prime(x: Tensor) -> Tensor: - """Derivative of softsign - - Args: - x - - Returns: - - """ - - return K.cast(1.0, dtype=x.dtype) / K.power(K.cast(1.0, dtype=x.dtype) + K.abs(x), K.cast(2, dtype=x.dtype)) - - -##### corners ###### -def get_lower_bound_grid(x: Tensor, W: Tensor, b: Tensor, n: int) -> Tensor: - A, B = convert_lower_search_2_subset_sum(x, W, b, n) - return subset_sum_lower(A, B, repeat=n) - - -def get_upper_bound_grid(x: Tensor, W: Tensor, b: Tensor, n: int) -> Tensor: - return -get_lower_bound_grid(x, -W, -b, n) - - -def get_bound_grid( - x: Tensor, - W_u: Tensor, - b_u: Tensor, - W_l: Tensor, - b_l: Tensor, - n: int, -) -> tuple[Tensor, Tensor]: - upper = get_upper_bound_grid(x, W_u, b_u, n) - lower = get_lower_bound_grid(x, W_l, b_l, n) - - return upper, lower - - -# convert max Wx +b s.t Wx+b<=0 into a subset-sum problem with positive values -def convert_lower_search_2_subset_sum(x: Tensor, W: Tensor, b: Tensor, n: int) -> tuple[Tensor, Tensor]: - x_min = x[:, 0] - x_max = x[:, 1] - - if len(W.shape) > 3: - W = K.reshape(W, (-1, W.shape[1], int(np.prod(W.shape[2:])))) - b = K.reshape(b, (-1, int(np.prod(b.shape[1:])))) - - const = BoxDomain().get_lower(x, W, b) - - weights = K.abs(W) * K.expand_dims((x_max - x_min) / n, -1) - return weights, const - - -def subset_sum_lower(W: Tensor, b: Tensor, repeat: int = 1) -> Tensor: - B = K.sort(W, axis=1) - C = K.repeat(B, repeats=repeat, axis=1) - C_reduced = K.cumsum(C, axis=1) - D = K.minimum(K.sign(K.expand_dims(-b, 1) - C_reduced) + 1, K.cast(1.0, dtype=b.dtype)) - - score = K.minimum(K.sum(D * C, 1) + b, K.cast(0.0, dtype=b.dtype)) - return score - - -# define routines to get linear relaxations useful both for forward and backward -def get_linear_hull_relu( - upper: Tensor, - lower: Tensor, - slope: Union[str, Slope], - upper_g: float = 0.0, - lower_g: float = 0.0, - **kwargs: Any, -) -> list[Tensor]: - slope = Slope(slope) - # in case upper=lower, this cases are - # considered with index_dead and index_linear - alpha = (K.relu(upper) - K.relu(lower)) / K.maximum(K.cast(epsilon(), dtype=upper.dtype), upper - lower) - - # scaling factor for the upper bound on the relu - # see README - - w_u = alpha - b_u = K.relu(lower) - alpha * lower - z_value = K.cast(0.0, dtype=upper.dtype) - o_value = K.cast(1.0, dtype=upper.dtype) - - if slope == Slope.V_SLOPE: - # 1 if upper<=-lower else 0 - index_a = -K.clip(K.sign(upper + lower) - o_value, -o_value, z_value) - - # 1 if upper>-lower else 0 - index_b = o_value - index_a - w_l = index_b - b_l = z_value * b_u - - elif slope == Slope.A_SLOPE: - w_l = K.clip(K.sign(w_u - 0.5), 0, 1) - b_l = z_value * b_u - - elif slope == Slope.Z_SLOPE: - w_l = z_value * w_u - b_l = z_value * b_u - - elif slope == Slope.O_SLOPE: - w_l = z_value * w_u + o_value - b_l = z_value * b_u - - elif slope == Slope.S_SLOPE: - w_l = w_u - b_l = z_value * b_u - - else: - raise NotImplementedError(f"Not implemented for slope {slope}") - - if "upper_grid" in kwargs: - raise NotImplementedError() - - gamma = o_value - if "finetune" in kwargs: - # retrieve variables to optimize the slopes - gamma = kwargs["finetune"][None] - - w_l = gamma * w_l + (o_value - gamma) * (o_value - w_l) - - # check inactive relu state: u<=0 - index_dead = -K.clip(K.sign(upper) - o_value, -o_value, z_value) # =1 if inactive state - index_linear = K.clip(K.sign(lower) + o_value, z_value, o_value) # 1 if linear state - - w_u = (o_value - index_dead) * w_u - w_l = (o_value - index_dead) * w_l - b_u = (o_value - index_dead) * b_u - b_l = (o_value - index_dead) * b_l - - w_u = (o_value - index_linear) * w_u + index_linear - w_l = (o_value - index_linear) * w_l + index_linear - b_u = (o_value - index_linear) * b_u - b_l = (o_value - index_linear) * b_l - - return [w_u, b_u, w_l, b_l] - - -def get_linear_hull_sigmoid(upper: Tensor, lower: Tensor, slope: Union[str, Slope], **kwargs: Any) -> list[Tensor]: - x = [upper, lower] - return get_linear_hull_s_shape(x, func=K.sigmoid, f_prime=sigmoid_prime, mode=ForwardMode.IBP, **kwargs) - - -def get_linear_hull_tanh(upper: Tensor, lower: Tensor, slope: Union[str, Slope], **kwargs: Any) -> list[Tensor]: - x = [upper, lower] - return get_linear_hull_s_shape(x, func=K.tanh, f_prime=tanh_prime, mode=ForwardMode.IBP, **kwargs) - - -def get_linear_softplus_hull(upper: Tensor, lower: Tensor, slope: Union[str, Slope], **kwargs: Any) -> list[Tensor]: - slope = Slope(slope) - # in case upper=lower, this cases are - # considered with index_dead and index_linear - u_c = K.softsign(upper) - l_c = K.softsign(lower) - alpha = (u_c - l_c) / K.maximum(K.cast(epsilon(), dtype=upper.dtype), (upper - lower)) - w_u = alpha - b_u = -alpha * lower + l_c - - z_value = K.cast(0.0, dtype=upper.dtype) - o_value = K.cast(1.0, dtype=upper.dtype) - - if slope == Slope.V_SLOPE: - # 1 if upper<=-lower else 0 - index_a = -K.clip(K.sign(upper + lower) - o_value, -o_value, z_value) - # 1 if upper>-lower else 0 - index_b = o_value - index_a - w_l = index_b - b_l = z_value * b_u - elif slope == Slope.Z_SLOPE: - w_l = z_value * w_u - b_l = z_value * b_u - elif slope == Slope.O_SLOPE: - w_l = z_value * w_u + o_value - b_l = z_value * b_u - elif slope == Slope.S_SLOPE: - w_l = w_u - b_l = z_value * b_u - else: - raise ValueError(f"Unknown slope {slope}") - - index_dead = -K.clip(K.sign(upper) - o_value, -o_value, z_value) - - w_u = (o_value - index_dead) * w_u - w_l = (o_value - index_dead) * w_l - b_u = (o_value - index_dead) * b_u - b_l = (o_value - index_dead) * b_l - - if "finetune" in kwargs: - # weighted linear combination - alpha_u, alpha_l = kwargs["finetune"] - alpha_u = alpha_u[None] - alpha_l = alpha_l[None] - - w_u = alpha_u * w_u - b_u = alpha_u * b_u + (o_value - alpha_u) * K.maximum(upper, z_value) - - w_l = alpha_l * w_l - b_l = alpha_l * b_l + (o_value - alpha_l) * K.maximum(lower, z_value) - - return [w_u, b_u, w_l, b_l] - - -def subtract( - inputs_0: list[Tensor], - inputs_1: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> list[Tensor]: - """LiRPA implementation of inputs_0-inputs_1 - - Args: - inputs_0: tensor - inputs_1: tensor - dc_decomp: boolean that indicates - perturbation_domain: the type of perturbation domain - whether we return a difference of convex decomposition of our layer - - Returns: - inputs_0 - inputs_1 - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - inputs_1 = minus(inputs_1, mode=mode, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain) - output = add(inputs_0, inputs_1, dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - return output - - -def add( - inputs_0: list[Tensor], - inputs_1: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, -) -> list[Tensor]: - """LiRPA implementation of inputs_0+inputs_1 - - Args: - inputs_0: tensor - inputs_1: tensor - dc_decomp: boolean that indicates - perturbation_domain: the type of perturbation domain - whether we return a difference of convex decomposition of our layer - - Returns: - inputs_0 + inputs_1 - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - mode = ForwardMode(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x_0, u_c_0, w_u_0, b_u_0, l_c_0, w_l_0, b_l_0, h_0, g_0 = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs_0 - ) - _, u_c_1, w_u_1, b_u_1, l_c_1, w_l_1, b_l_1, h_1, g_1 = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs_1 - ) - - x = x_0 - h = h_0 + h_1 - g = g_0 + g_1 - u_c = u_c_0 + u_c_1 - l_c = l_c_0 + l_c_1 - w_u = w_u_0 + w_u_1 - w_l = w_l_0 + w_l_1 - b_u = b_u_0 + b_u_1 - b_l = b_l_0 + b_l_1 - - fulloutputs = [x, u_c, w_u, b_u, l_c, w_l, b_l, h, g] - outputs = inputs_outputs_spec.extract_outputsformode_from_fulloutputs(fulloutputs) - - return outputs - - -def relu_( - inputs: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - **kwargs: Any, -) -> list[Tensor]: - if perturbation_domain is None: - perturbation_domain = BoxDomain() - - mode = ForwardMode(mode) - affine = get_affine(mode) - ibp = get_ibp(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - dtype = x.dtype - - z_value = K.cast(0.0, dtype=dtype) - o_value = K.cast(1.0, dtype=dtype) - empty_tensor = inputs_outputs_spec.get_empty_tensor(dtype=dtype) - - if dc_decomp: - h_out = K.maximum(h, -g) - g_out = g - index_dead = -K.clip(K.sign(u_c) - o_value, -o_value, z_value) # =1 if inactive state - index_linear = K.clip(K.sign(l_c) + o_value, z_value, o_value) # 1 if linear state - - h_out = (o_value - index_dead) * h_out - g_out = (o_value - index_dead) * g_out - h_out = (o_value - index_linear) * h_out + index_linear * h - g_out = (o_value - index_linear) * g_out + index_linear * g - else: - h_out, g_out = empty_tensor, empty_tensor - - if ibp: - u_c_out = K.relu(u_c) - l_c_out = K.relu(l_c) - else: - u_c_out = empty_tensor - l_c_out = empty_tensor - - if affine: - if isinstance(perturbation_domain, GridDomain): - upper_g, lower_g = get_bound_grid(x, w_u, b_u, w_l, b_l, 1) - kwargs.update({"upper_grid": upper_g, "lower_grid": lower_g}) - - w_u_out, b_u_out, w_l_out, b_l_out = get_linear_hull_relu(u_c, l_c, slope, **kwargs) - b_u_out = w_u_out * b_u + b_u_out - b_l_out = w_l_out * b_l + b_l_out - w_u_out = K.expand_dims(w_u_out, 1) * w_u - w_l_out = K.expand_dims(w_l_out, 1) * w_l - else: - w_u_out, b_u_out, w_l_out, b_l_out = empty_tensor, empty_tensor, empty_tensor, empty_tensor - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -def minus( - inputs: list[Tensor], - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA implementation of minus(x)=-x. - - Args: - inputs - mode - - Returns: - - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode( - inputs, tight=False, compute_ibp_from_affine=False - ) - - u_c_out = -l_c - l_c_out = -u_c - w_u_out = -w_l - b_u_out = -b_l - w_l_out = -w_u - b_l_out = -b_u - h_out = -g - g_out = -h - - return inputs_outputs_spec.extract_outputsformode_from_fulloutputs( - [x, u_c_out, w_u_out, b_u_out, l_c_out, w_l_out, b_l_out, h_out, g_out] - ) - - -def maximum( - inputs_0: list[Tensor], - inputs_1: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA implementation of element-wise max - - Args: - inputs_0: list of tensors - inputs_1: list of tensors - dc_decomp: boolean that indicates - perturbation_domain: the type of perturbation domain - whether we return a difference of convex decomposition of our layer - - Returns: - maximum(inputs_0, inputs_1) - """ - if perturbation_domain is None: - perturbation_domain = BoxDomain() - output_0 = subtract(inputs_1, inputs_0, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode) - if finetune: - finetune = kwargs["finetune_params"] - output_1 = relu_( - output_0, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode, finetune=finetune - ) - else: - output_1 = relu_(output_0, dc_decomp=dc_decomp, perturbation_domain=perturbation_domain, mode=mode) - - return add( - output_1, - inputs_0, - dc_decomp=dc_decomp, - perturbation_domain=perturbation_domain, - mode=mode, - ) - - -def minimum( - inputs_0: list[Tensor], - inputs_1: list[Tensor], - dc_decomp: bool = False, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - finetune: bool = False, - **kwargs: Any, -) -> list[Tensor]: - """LiRPA implementation of element-wise min - - Args: - inputs_0 - inputs_1 - dc_decomp - perturbation_domain - mode - - Returns: - - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - return minus( - maximum( - minus(inputs_0, dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain), - minus(inputs_1, dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain), - dc_decomp=dc_decomp, - perturbation_domain=perturbation_domain, - mode=mode, - finetune=finetune, - **kwargs, - ), - dc_decomp=dc_decomp, - mode=mode, - perturbation_domain=perturbation_domain, - ) - - -def get_linear_hull_s_shape( - inputs: list[Tensor], - func: TensorFunction = K.sigmoid, - f_prime: TensorFunction = sigmoid_prime, - perturbation_domain: Optional[PerturbationDomain] = None, - mode: Union[str, ForwardMode] = ForwardMode.HYBRID, - slope: Union[str, Slope] = Slope.V_SLOPE, - dc_decomp: bool = False, - **kwargs: Any, -) -> list[Tensor]: - """Computing the linear hull of shape functions given the pre activation neurons - - Args: - inputs: list of input tensors - func: the function (sigmoid, tanh, softsign...) - f_prime: the derivative of the function (sigmoid_prime...) - perturbation_domain: the type of convex input domain - mode: type of Forward propagation (ibp, affine, or hybrid) - - Returns: - the updated list of tensors - """ - - if perturbation_domain is None: - perturbation_domain = BoxDomain() - mode = ForwardMode(mode) - inputs_outputs_spec = InputsOutputsSpec(dc_decomp=dc_decomp, mode=mode, perturbation_domain=perturbation_domain) - - x, u_c, w_u, b_u, l_c, w_l, b_l, h, g = inputs_outputs_spec.get_fullinputs_from_inputsformode(inputs) - dtype = x.dtype - - z_value = K.cast(0.0, dtype=dtype) - o_value = K.cast(1.0, dtype=dtype) - t_value = K.cast(2.0, dtype=dtype) - - # flatten - shape = list(u_c.shape[1:]) - u_c_flat = K.reshape(u_c, (-1, int(np.prod(shape)))) # (None, n) - l_c_flat = K.reshape(l_c, (-1, int(np.prod(shape)))) # (None, n) - - # upper bound - # derivative - s_u_prime = f_prime(u_c_flat) # (None, n) - s_l_prime = f_prime(l_c_flat) # (None, n) - s_u = func(u_c_flat) # (None, n) - s_l = func(l_c_flat) # (None, n) - - # case 0: - coeff = (s_u - s_l) / K.maximum(K.cast(epsilon(), dtype=dtype), u_c_flat - l_c_flat) - alpha_u_0 = K.where( - K.greater_equal(s_u_prime, coeff), o_value + z_value * u_c_flat, z_value * u_c_flat - ) # (None, n) - alpha_u_1 = (o_value - alpha_u_0) * ((K.sign(l_c_flat) + o_value) / t_value) - - w_u_0 = coeff - b_u_0 = -w_u_0 * l_c_flat + s_l - - w_u_1 = z_value * u_c_flat - b_u_1 = s_u - - w_u_2, b_u_2 = get_t_upper(u_c_flat, l_c_flat, s_l, func=func, f_prime=f_prime) - - w_u_out = K.reshape(alpha_u_0 * w_u_0 + alpha_u_1 * w_u_1 + (o_value - alpha_u_0 - alpha_u_1) * w_u_2, [-1] + shape) - b_u_out = K.reshape(alpha_u_0 * b_u_0 + alpha_u_1 * b_u_1 + (o_value - alpha_u_0 - alpha_u_1) * b_u_2, [-1] + shape) - - # linear hull - # case 0: - alpha_l_0 = K.where( - K.greater_equal(s_l_prime, coeff), o_value + z_value * l_c_flat, z_value * l_c_flat - ) # (None, n) - alpha_l_1 = (o_value - alpha_l_0) * ((K.sign(-u_c_flat) + o_value) / t_value) - - w_l_0 = coeff - b_l_0 = -w_l_0 * u_c_flat + s_u - - w_l_1 = z_value * u_c_flat - b_l_1 = s_l - - w_l_2, b_l_2 = get_t_lower(u_c_flat, l_c_flat, s_u, func=func, f_prime=f_prime) - - w_l_out = K.reshape(alpha_l_0 * w_l_0 + alpha_l_1 * w_l_1 + (o_value - alpha_l_0 - alpha_l_1) * w_l_2, [-1] + shape) - b_l_out = K.reshape(alpha_l_0 * b_l_0 + alpha_l_1 * b_l_1 + (o_value - alpha_l_0 - alpha_l_1) * b_l_2, [-1] + shape) - - return [w_u_out, b_u_out, w_l_out, b_l_out] - - -def get_t_upper( - u_c_flat: Tensor, - l_c_flat: Tensor, - s_l: Tensor, - func: TensorFunction = K.sigmoid, - f_prime: TensorFunction = sigmoid_prime, -) -> list[Tensor]: - """linear interpolation between lower and upper bounds on the function func to have a symbolic approximation of the best - coefficient for the affine upper bound - - Args: - u_c_flat: flatten tensor of constant upper bound - l_c_flat: flatten tensor of constant lower bound - s_l: lowest value of the function func on the domain - func: the function (sigmoid, tanh, softsign) - f_prime: the derivative of the function - - Returns: - the upper affine bounds in this subcase - """ - - o_value = K.cast(1.0, dtype=u_c_flat.dtype) - z_value = K.cast(0.0, dtype=u_c_flat.dtype) - - # step1: find t - u_c_reshaped = K.expand_dims(u_c_flat, -1) # (None, n , 1) - l_c_reshaped = K.expand_dims(l_c_flat, -1) # (None, n, 1) - t = K.cast(np.linspace(0, 1, 100)[None, None, :], dtype=u_c_flat.dtype) * u_c_reshaped # (None, n , 100) - - s_p_t = f_prime(t) # (None, n, 100) - s_t = func(t) # (None, n, 100) - - score = K.abs(s_p_t - (s_t - K.expand_dims(s_l, -1)) / (t - l_c_reshaped)) # (None, n, 100) - index = K.argmin(score, -1) # (None, n) - threshold = K.min(score, -1) # (None, n) - - index_t = K.cast( - K.where(K.greater(threshold, z_value * threshold), index, K.clip(index - 1, 0, 100)), dtype=u_c_flat.dtype - ) # (None, n) - t_value = K.sum( - K.where( - K.equal( - o_value * K.cast(np.arange(0, 100)[None, None, :], dtype=u_c_flat.dtype) + z_value * u_c_reshaped, - K.expand_dims(index_t, -1) + z_value * u_c_reshaped, - ), - t, - z_value * t, - ), - -1, - ) # (None, n) - - s_t = func(t_value) # (None, n) - w_u = (s_t - s_l) / K.maximum(K.cast(epsilon(), dtype=u_c_flat.dtype), t_value - l_c_flat) # (None, n) - b_u = -w_u * l_c_flat + s_l # + func(l_c_flat) - - return [w_u, b_u] - - -def get_t_lower( - u_c_flat: Tensor, - l_c_flat: Tensor, - s_u: Tensor, - func: TensorFunction = K.sigmoid, - f_prime: TensorFunction = sigmoid_prime, -) -> list[Tensor]: - """linear interpolation between lower and upper bounds on the function func to have a symbolic approximation of the best - coefficient for the affine lower bound - - Args: - u_c_flat: flatten tensor of constant upper bound - l_c_flat: flatten tensor of constant lower bound - s_u: highest value of the function func on the domain - func: the function (sigmoid, tanh, softsign) - f_prime: the derivative of the function - - Returns: - the lower affine bounds in this subcase - """ - z_value = K.cast(0.0, dtype=u_c_flat.dtype) - o_value = K.cast(1.0, dtype=u_c_flat.dtype) - - # step1: find t - u_c_reshaped = K.expand_dims(u_c_flat, -1) # (None, n , 1) - l_c_reshaped = K.expand_dims(l_c_flat, -1) # (None, n, 1) - t = K.cast(np.linspace(0, 1.0, 100)[None, None, :], dtype=u_c_flat.dtype) * l_c_reshaped # (None, n , 100) - - s_p_t = f_prime(t) # (None, n, 100) - s_t = func(t) # (None, n, 100) - - score = K.abs(s_p_t - (K.expand_dims(s_u, -1) - s_t) / (u_c_reshaped - t)) # (None, n, 100) - index = K.argmin(score, -1) # (None, n) - - threshold = K.min(score, -1) - index_t = K.cast( - K.where(K.greater(threshold, z_value * threshold), index, K.clip(index + 1, 0, 100)), dtype=u_c_flat.dtype - ) # (None, n) - t_value = K.sum( - K.where( - K.equal( - o_value * K.cast(np.arange(0, 100)[None, None, :], dtype=u_c_flat.dtype) + z_value * u_c_reshaped, - K.expand_dims(index_t, -1) + z_value * u_c_reshaped, - ), - t, - z_value * t, - ), - -1, - ) - - s_t = func(t_value) # (None, n) - w_l = (s_u - s_t) / K.maximum(K.cast(epsilon(), dtype=u_c_flat.dtype), u_c_flat - t_value) # (None, n) - b_l = -w_l * u_c_flat + s_u # func(u_c_flat) - - return [w_l, b_l] diff --git a/src/decomon/wrapper.py b/src/decomon/wrapper.py index 8d26c9d8..deb396ed 100644 --- a/src/decomon/wrapper.py +++ b/src/decomon/wrapper.py @@ -5,9 +5,10 @@ import numpy as np import numpy.typing as npt -from decomon.core import BallDomain, BoxDomain, ConvertMethod, GridDomain +from decomon.constants import ConvertMethod from decomon.models.convert import clone from decomon.models.models import DecomonModel +from decomon.perturbation_domain import BallDomain, BoxDomain, GridDomain IntegerType = Union[int, np.int_] """Alias for integers types.""" diff --git a/src/decomon/wrapper_with_tuning.py b/src/decomon/wrapper_with_tuning.py deleted file mode 100644 index c7af68a6..00000000 --- a/src/decomon/wrapper_with_tuning.py +++ /dev/null @@ -1,183 +0,0 @@ -from typing import Union - -import keras -import numpy as np -import numpy.typing as npt -from keras.optimizers import Adam - -from decomon.metrics.loss import get_upper_loss -from decomon.models.models import DecomonModel -from decomon.wrapper import get_lower_box, get_upper_box, refine_boxes - - -#### FORMAL BOUNDS ###### -def get_upper_box_tuning( - model: Union[keras.Model, DecomonModel], - decomon_model_concat: keras.Model, - x_min: npt.NDArray[np.float_], - x_max: npt.NDArray[np.float_], - batch_size: int = 1, - n_sub_boxes: int = 1, - lr: float = 0.1, - epochs: int = 100, -) -> npt.NDArray[np.float_]: - """upper bound the maximum of a model in a given box - - Args: - model: either a Keras model or a Decomon model - x_min: numpy array for the extremal lower corner of the boxes - x_max: numpy array for the extremal upper corner of the boxes - batch_size: for computational efficiency, one can split the - calls to minibatches - fast: useful in the forward-backward or in the hybrid-backward - mode to optimize the scores - - Returns: - numpy array, vector with upper bounds for adversarial attacks - """ - - if np.min(x_max - x_min) < 0: - raise UserWarning("Inconsistency Error: x_max < x_min") - - # check that the model is a DecomonModel, else do the conversion - decomon_model = model - - baseline_upper = get_upper_box(model, x_min, x_max, batch_size=batch_size, n_sub_boxes=n_sub_boxes) - - if n_sub_boxes > 1: - x_min, x_max = refine_boxes(x_min, x_max, n_sub_boxes) - # reshape - shape = list(x_min.shape[2:]) - x_min = np.reshape(x_min, [-1] + shape) - x_max = np.reshape(x_max, [-1] + shape) - - # reshape x_mmin, x_max - input_shape = list(decomon_model.input_shape[2:]) - input_dim = np.prod(input_shape) - x_reshaped = x_min + 0 * x_min - x_reshaped = x_reshaped.reshape([-1] + input_shape) - x_min = x_min.reshape((-1, 1, input_dim)) - x_max = x_max.reshape((-1, 1, input_dim)) - - z = np.concatenate([x_min, x_max], 1) - - if batch_size > 1: - # split - r = 0 - if len(x_reshaped) % batch_size > 0: - r += 1 - x_min_list = [x_min[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r)] - x_max_list = [x_max[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r)] - - results = [ - get_upper_box_tuning( - decomon_model, decomon_model_concat, x_min_list[i], x_max_list[i], -1, lr=lr, epochs=epochs - ) - for i in range(len(x_min_list)) - ] - - return np.concatenate(results) - - else: - # freeze_weights - model.freeze_weights() - - # create loss - loss_upper = get_upper_loss(model) - decomon_model_concat.compile(Adam(lr=lr), loss_upper) - - decomon_model_concat.fit(z, baseline_upper, epochs=epochs, verbose=0) - - upper = get_upper_box(model, x_min, x_max, batch_size=batch_size, n_sub_boxes=n_sub_boxes) - - # reset alpha - model.reset_finetuning() - - return np.minimum(baseline_upper, upper) - - -#### FORMAL BOUNDS ###### -def get_lower_box_tuning( - model: Union[keras.Model, DecomonModel], - decomon_model_concat: keras.Model, - x_min: npt.NDArray[np.float_], - x_max: npt.NDArray[np.float_], - batch_size: int = 1, - n_sub_boxes: int = 1, - lr: float = 0.1, - epochs: int = 100, -) -> npt.NDArray[np.float_]: - """upper bound the maximum of a model in a given box - - Args: - model: either a Keras model or a Decomon model - x_min: numpy array for the extremal lower corner of the boxes - x_max: numpy array for the extremal upper corner of the boxes - batch_size: for computational efficiency, one can split the - calls to minibatches - fast: useful in the forward-backward or in the hybrid-backward - mode to optimize the scores - - Returns: - numpy array, vector with upper bounds for adversarial attacks - """ - - if np.min(x_max - x_min) < 0: - raise UserWarning("Inconsistency Error: x_max < x_min") - - # check that the model is a DecomonModel, else do the conversion - decomon_model = model - - baseline_upper = get_lower_box(model, x_min, x_max, batch_size=batch_size, n_sub_boxes=n_sub_boxes) - - if n_sub_boxes > 1: - x_min, x_max = refine_boxes(x_min, x_max, n_sub_boxes) - # reshape - shape = list(x_min.shape[2:]) - x_min = np.reshape(x_min, [-1] + shape) - x_max = np.reshape(x_max, [-1] + shape) - - # reshape x_mmin, x_max - input_shape = list(decomon_model.input_shape[2:]) - input_dim = np.prod(input_shape) - x_reshaped = x_min + 0 * x_min - x_reshaped = x_reshaped.reshape([-1] + input_shape) - x_min = x_min.reshape((-1, 1, input_dim)) - x_max = x_max.reshape((-1, 1, input_dim)) - - z = np.concatenate([x_min, x_max], 1) - - if batch_size > 0 and batch_size != len(x_min): - # split - r = 0 - if len(x_reshaped) % batch_size > 0: - r += 1 - x_min_list = [x_min[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r)] - x_max_list = [x_max[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r)] - - results = [ - get_lower_box_tuning( - decomon_model, decomon_model_concat, x_min_list[i], x_max_list[i], -1, lr=lr, epochs=epochs - ) - for i in range(len(x_min_list)) - ] - - model.reset_finetuning() - - return np.concatenate(results) - else: - # freeze_weights - model.freeze_weights() - - # create loss - loss_upper = get_upper_loss(model) - decomon_model_concat.compile(Adam(lr=lr), loss_upper) - - decomon_model_concat.fit(z, baseline_upper, epochs=epochs, verbose=0) - - upper = get_lower_box(model, x_min, x_max, batch_size=batch_size, n_sub_boxes=n_sub_boxes) - - # reset alpha - model.reset_finetuning() - - return np.minimum(baseline_upper, upper) diff --git a/tests/conftest.py b/tests/conftest.py index 0647b383..b69c2879 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,7 +15,7 @@ unpack_fixture, ) -from decomon.core import BoxDomain, ConvertMethod, InputsOutputsSpec, Propagation, Slope +from decomon.constants import ConvertMethod, Propagation, Slope from decomon.keras_utils import ( BACKEND_JAX, BACKEND_NUMPY, @@ -23,6 +23,8 @@ BACKEND_TENSORFLOW, batch_multid_dot, ) +from decomon.layers.inputs_outputs_specs import InputsOutputsSpec +from decomon.perturbation_domain import BoxDomain from decomon.types import BackendTensor, Tensor empty, diag, nobatch = param_fixtures( @@ -162,7 +164,6 @@ def get_decomon_input_shapes( ibp=ibp, affine=affine, propagation=propagation, - perturbation_domain=perturbation_domain, layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=model_output_shape, @@ -263,7 +264,6 @@ def get_decomon_symbolic_inputs( ibp=ibp, affine=affine, propagation=propagation, - perturbation_domain=perturbation_domain, layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=model_output_shape, @@ -321,7 +321,6 @@ def generate_simple_decomon_layer_inputs_from_keras_input( ibp=ibp, affine=affine, propagation=propagation, - perturbation_domain=perturbation_domain, layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=model_output_shape, @@ -1432,7 +1431,6 @@ def decomon_symbolic_input_fn(output_shape, linear): ibp=ibp, affine=affine, propagation=propagation, - perturbation_domain=perturbation_domain, layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=output_shape, @@ -1472,7 +1470,6 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape, linear) ibp=ibp, affine=affine, propagation=propagation, - perturbation_domain=perturbation_domain, layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=output_shape, @@ -1514,7 +1511,6 @@ def decomon_symbolic_input_fn(output_shape, linear): ibp=ibp, affine=affine, propagation=propagation, - perturbation_domain=perturbation_domain, layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=output_shape, @@ -1568,7 +1564,6 @@ def decomon_input_fn(keras_model_input, keras_layer_input, output_shape, linear) ibp=ibp, affine=affine, propagation=propagation, - perturbation_domain=perturbation_domain, layer_input_shape=layer_input_shape, model_input_shape=model_input_shape, model_output_shape=output_shape, diff --git a/tests/lirpa_comparison/test_comparison_lirpa.py b/tests/lirpa_comparison/test_comparison_lirpa.py index cc3fdbcd..8bfad380 100644 --- a/tests/lirpa_comparison/test_comparison_lirpa.py +++ b/tests/lirpa_comparison/test_comparison_lirpa.py @@ -9,7 +9,7 @@ from onnx2keras import onnx_to_keras from onnx2torch import convert -from decomon.core import ConvertMethod +from decomon.constants import ConvertMethod from decomon.models.convert import clone diff --git a/tests/test_clone.py b/tests/test_clone.py index 6acd3623..e83e7902 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -5,10 +5,11 @@ from keras.models import Model from pytest_cases import parametrize -from decomon.core import BoxDomain, ConvertMethod, Propagation, Slope +from decomon.constants import ConvertMethod, Propagation, Slope from decomon.layers.input import IdentityInput from decomon.layers.utils.symbolify import LinkToPerturbationDomainInput from decomon.models.convert import clone +from decomon.perturbation_domain import BoxDomain def test_clone_nok_several_inputs(): diff --git a/tests/test_convert_backward.py b/tests/test_convert_backward.py index 6b9e9438..0079d6b2 100644 --- a/tests/test_convert_backward.py +++ b/tests/test_convert_backward.py @@ -1,7 +1,7 @@ from keras.models import Model from pytest_cases import fixture, parametrize -from decomon.core import Propagation, Slope +from decomon.constants import Propagation, Slope from decomon.models.backward_cloning import convert_backward from decomon.models.forward_cloning import convert_forward diff --git a/tests/test_convert_forward.py b/tests/test_convert_forward.py index 9a9e45b3..358b33fe 100644 --- a/tests/test_convert_forward.py +++ b/tests/test_convert_forward.py @@ -2,7 +2,7 @@ from keras.models import Model from pytest_cases import fixture, parametrize -from decomon.core import Propagation, Slope +from decomon.constants import Propagation, Slope from decomon.layers.activations.activation import DecomonBaseActivation from decomon.models.forward_cloning import convert_forward diff --git a/tests/test_decomon_layer.py b/tests/test_decomon_layer.py index a2de2cdd..d533f800 100644 --- a/tests/test_decomon_layer.py +++ b/tests/test_decomon_layer.py @@ -3,9 +3,10 @@ import pytest from keras.layers import Dense, Input -from decomon.core import BoxDomain, Propagation +from decomon.constants import Propagation from decomon.keras_utils import batch_multid_dot from decomon.layers.layer import DecomonLayer +from decomon.perturbation_domain import BoxDomain def test_decomon_layer_nok_unbuilt_keras_layer(): diff --git a/tests/test_fuse.py b/tests/test_fuse.py index efab7793..e97a0e06 100644 --- a/tests/test_fuse.py +++ b/tests/test_fuse.py @@ -3,8 +3,9 @@ import pytest from pytest_cases import parametrize -from decomon.core import InputsOutputsSpec, Propagation +from decomon.constants import Propagation from decomon.layers.fuse import Fuse +from decomon.layers.inputs_outputs_specs import InputsOutputsSpec def generate_simple_inputs(ibp, affine, input_shape, output_shape, batchsize, diag, nobatch): diff --git a/tests/test_inputs.py b/tests/test_inputs.py index d3fe6dff..4cdfe655 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -1,7 +1,7 @@ import pytest from pytest_cases import parametrize -from decomon.core import Propagation +from decomon.constants import Propagation from decomon.layers.input import BackwardInput, ForwardInput diff --git a/tests/test_keras_utils.py b/tests/test_keras_utils.py index eff9bcaa..d0818125 100644 --- a/tests/test_keras_utils.py +++ b/tests/test_keras_utils.py @@ -5,13 +5,7 @@ from keras.layers import Dense, Input, Layer from numpy.testing import assert_almost_equal -from decomon.keras_utils import ( - add_tensors, - batch_multid_dot, - get_weight_index_from_name, - is_a_merge_layer, - share_layer_all_weights, -) +from decomon.keras_utils import add_tensors, batch_multid_dot, is_a_merge_layer from decomon.types import BackendTensor @@ -300,54 +294,3 @@ def test_add_tensors_ok(missing_batchsize, diagonal, helpers): res_full, res_simplified, ) - - -def test_get_weight_index_from_name_nok_attribute(): - layer = Dense(3) - layer(K.zeros((2, 1))) - with pytest.raises(AttributeError): - get_weight_index_from_name(layer=layer, weight_name="toto") - - -def test_get_weight_index_from_name_nok_index(): - layer = Dense(3, use_bias=False) - layer(K.zeros((2, 1))) - with pytest.raises(IndexError): - get_weight_index_from_name(layer=layer, weight_name="bias") - - -def test_get_weight_index_from_name_ok(): - layer = Dense(3) - layer(K.zeros((2, 1))) - assert get_weight_index_from_name(layer=layer, weight_name="bias") in [0, 1] - - -def test_share_layer_all_weights_nok_original_layer_unbuilt(): - original_layer = Dense(3) - new_layer = original_layer.__class__.from_config(original_layer.get_config()) - with pytest.raises(ValueError): - share_layer_all_weights(original_layer=original_layer, new_layer=new_layer) - - -def test_share_layer_all_weights_nok_new_layer_built(): - original_layer = Dense(3) - inp = Input((1,)) - original_layer(inp) - new_layer = original_layer.__class__.from_config(original_layer.get_config()) - new_layer(inp) - with pytest.raises(ValueError): - share_layer_all_weights(original_layer=original_layer, new_layer=new_layer) - - -def test_share_layer_all_weights_ok(): - original_layer = Dense(3) - inp = Input((1,)) - original_layer(inp) - new_layer = original_layer.__class__.from_config(original_layer.get_config()) - share_layer_all_weights(original_layer=original_layer, new_layer=new_layer) - - # check same weights - assert len(original_layer.weights) == len(new_layer.weights) - for w in original_layer.weights: - new_w = [ww for ww in new_layer.weights if ww.name == w.name][0] - assert w is new_w diff --git a/tests/test_oracle.py b/tests/test_oracle.py index 78f65031..7c208d33 100644 --- a/tests/test_oracle.py +++ b/tests/test_oracle.py @@ -3,7 +3,7 @@ import pytest from pytest_cases import parametrize -from decomon.core import Propagation +from decomon.constants import Propagation from decomon.layers.oracle import DecomonOracle T = TypeVar("T") diff --git a/tests/test_output.py b/tests/test_output.py index 2275653d..2323eef1 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -3,8 +3,10 @@ import pytest from pytest_cases import parametrize -from decomon.core import BoxDomain, InputsOutputsSpec, Propagation +from decomon.constants import Propagation +from decomon.layers.inputs_outputs_specs import InputsOutputsSpec from decomon.layers.output import ConvertOutput +from decomon.perturbation_domain import BoxDomain def generate_simple_inputs( diff --git a/tests/test_to_decomon.py b/tests/test_to_decomon.py index 744fd477..ef19c263 100644 --- a/tests/test_to_decomon.py +++ b/tests/test_to_decomon.py @@ -7,9 +7,10 @@ import decomon.layers import decomon.layers.convert -from decomon.core import BoxDomain, PerturbationDomain, Propagation, Slope +from decomon.constants import Propagation, Slope from decomon.layers import DecomonActivation, DecomonDense, DecomonLayer from decomon.layers.convert import to_decomon +from decomon.perturbation_domain import BoxDomain, PerturbationDomain # Add a new class in decomon.layers namespace for testing conversion by name # We must do it *before* importing decomon.layers.convert diff --git a/tests/visu-decomon.ipynb b/tests/visu-decomon.ipynb index 5f4b51e7..75912136 100644 --- a/tests/visu-decomon.ipynb +++ b/tests/visu-decomon.ipynb @@ -670,7 +670,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.9.18" } }, "nbformat": 4, From c0856fcd732ca550610402d8a68d23242afb88e9 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Thu, 14 Mar 2024 15:57:03 +0100 Subject: [PATCH 084/101] Remove unused imports --- src/decomon/keras_utils.py | 3 +-- src/decomon/layers/core/dense.py | 1 - src/decomon/layers/crown.py | 4 +--- src/decomon/layers/fuse.py | 13 +++---------- src/decomon/layers/merging/base_merge.py | 2 +- src/decomon/layers/oracle.py | 1 - src/decomon/layers/output.py | 6 ++---- src/decomon/layers/utils/batchsize.py | 3 +-- src/decomon/layers/utils/symbolify.py | 3 +-- src/decomon/models/utils.py | 2 +- tests/test_clone.py | 2 +- tests/test_decomon_layer.py | 2 -- tests/test_keras_utils.py | 2 +- tests/test_preprocess_keras_layer.py | 2 -- tests/test_preprocess_keras_model.py | 2 +- tests/test_unary_layers.py | 2 +- 16 files changed, 15 insertions(+), 35 deletions(-) diff --git a/src/decomon/keras_utils.py b/src/decomon/keras_utils.py index daa56eb1..38af5247 100644 --- a/src/decomon/keras_utils.py +++ b/src/decomon/keras_utils.py @@ -1,8 +1,7 @@ -from typing import Any, Optional +from typing import Optional import keras import keras.ops as K -import numpy as np from keras.layers import Dot, Layer, Reshape from decomon.types import BackendTensor, Tensor diff --git a/src/decomon/layers/core/dense.py b/src/decomon/layers/core/dense.py index 27a4baf7..057cd76a 100644 --- a/src/decomon/layers/core/dense.py +++ b/src/decomon/layers/core/dense.py @@ -1,7 +1,6 @@ import keras.ops as K from keras.layers import Dense -from decomon.keras_utils import batch_multid_dot from decomon.layers.layer import DecomonLayer from decomon.types import Tensor diff --git a/src/decomon/layers/crown.py b/src/decomon/layers/crown.py index 8e5370d9..0210b077 100644 --- a/src/decomon/layers/crown.py +++ b/src/decomon/layers/crown.py @@ -1,16 +1,14 @@ """Layers needed by crown algorithm.""" -from typing import Any, Optional, Union, overload +from typing import Any, Optional -import keras import keras.ops as K from keras.layers import Layer from decomon.constants import Propagation from decomon.keras_utils import add_tensors from decomon.layers.inputs_outputs_specs import InputsOutputsSpec -from decomon.perturbation_domain import PerturbationDomain from decomon.types import BackendTensor diff --git a/src/decomon/layers/fuse.py b/src/decomon/layers/fuse.py index 8e689929..6e5f8351 100644 --- a/src/decomon/layers/fuse.py +++ b/src/decomon/layers/fuse.py @@ -1,21 +1,14 @@ """Layers specifying constant oracle bounds on keras layer input.""" -from typing import Any, Optional, Union, overload +from typing import Any, Optional -import keras from keras import ops as K from keras.layers import Layer -from decomon.constants import Propagation -from decomon.keras_utils import add_tensors, batch_multid_dot +from decomon.keras_utils import batch_multid_dot from decomon.layers.inputs_outputs_specs import InputsOutputsSpec -from decomon.perturbation_domain import ( - BoxDomain, - PerturbationDomain, - get_lower_box, - get_upper_box, -) +from decomon.perturbation_domain import get_lower_box, get_upper_box from decomon.types import BackendTensor, Tensor diff --git a/src/decomon/layers/merging/base_merge.py b/src/decomon/layers/merging/base_merge.py index 9a121bd3..c73d0afa 100644 --- a/src/decomon/layers/merging/base_merge.py +++ b/src/decomon/layers/merging/base_merge.py @@ -1,4 +1,4 @@ -from typing import Any, Optional +from typing import Any import keras.ops as K diff --git a/src/decomon/layers/oracle.py b/src/decomon/layers/oracle.py index 30ae9a32..54d4e0e4 100644 --- a/src/decomon/layers/oracle.py +++ b/src/decomon/layers/oracle.py @@ -3,7 +3,6 @@ from typing import Any, Optional, Union, overload -import keras from keras.layers import Layer from decomon.layers.inputs_outputs_specs import InputsOutputsSpec diff --git a/src/decomon/layers/output.py b/src/decomon/layers/output.py index d33eceeb..c6d622ec 100644 --- a/src/decomon/layers/output.py +++ b/src/decomon/layers/output.py @@ -1,17 +1,15 @@ """Convert decomon outputs to the specified format.""" -from typing import Any, Optional, Union +from typing import Any, Optional -import keras import keras.ops as K from keras.layers import Layer -from decomon.constants import Propagation from decomon.layers.inputs_outputs_specs import InputsOutputsSpec from decomon.layers.oracle import get_forward_oracle from decomon.perturbation_domain import PerturbationDomain -from decomon.types import BackendTensor, Tensor +from decomon.types import BackendTensor class ConvertOutput(Layer): diff --git a/src/decomon/layers/utils/batchsize.py b/src/decomon/layers/utils/batchsize.py index 7bbf1b6a..1ab645f8 100644 --- a/src/decomon/layers/utils/batchsize.py +++ b/src/decomon/layers/utils/batchsize.py @@ -1,9 +1,8 @@ """Adding batchsize to batch-independent outputs.""" -from typing import Any, Optional +from typing import Optional -import keras import keras.ops as K from keras.layers import Layer diff --git a/src/decomon/layers/utils/symbolify.py b/src/decomon/layers/utils/symbolify.py index f2eb3d0b..b33dda4b 100644 --- a/src/decomon/layers/utils/symbolify.py +++ b/src/decomon/layers/utils/symbolify.py @@ -1,8 +1,7 @@ """Converting backend tensors to symbolic tensors.""" -from typing import Any, Optional +from typing import Optional -import keras from keras.layers import Layer from decomon.types import BackendTensor diff --git a/src/decomon/models/utils.py b/src/decomon/models/utils.py index bd557754..a01301fd 100644 --- a/src/decomon/models/utils.py +++ b/src/decomon/models/utils.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, Union +from typing import Optional, Union import keras import numpy as np diff --git a/tests/test_clone.py b/tests/test_clone.py index e83e7902..8270056e 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -5,7 +5,7 @@ from keras.models import Model from pytest_cases import parametrize -from decomon.constants import ConvertMethod, Propagation, Slope +from decomon.constants import ConvertMethod, Slope from decomon.layers.input import IdentityInput from decomon.layers.utils.symbolify import LinkToPerturbationDomainInput from decomon.models.convert import clone diff --git a/tests/test_decomon_layer.py b/tests/test_decomon_layer.py index d533f800..f43191e4 100644 --- a/tests/test_decomon_layer.py +++ b/tests/test_decomon_layer.py @@ -1,10 +1,8 @@ import keras.ops as K -import numpy as np import pytest from keras.layers import Dense, Input from decomon.constants import Propagation -from decomon.keras_utils import batch_multid_dot from decomon.layers.layer import DecomonLayer from decomon.perturbation_domain import BoxDomain diff --git a/tests/test_keras_utils.py b/tests/test_keras_utils.py index d0818125..8a705263 100644 --- a/tests/test_keras_utils.py +++ b/tests/test_keras_utils.py @@ -2,7 +2,7 @@ import keras.ops as K import numpy as np import pytest -from keras.layers import Dense, Input, Layer +from keras.layers import Input, Layer from numpy.testing import assert_almost_equal from decomon.keras_utils import add_tensors, batch_multid_dot, is_a_merge_layer diff --git a/tests/test_preprocess_keras_layer.py b/tests/test_preprocess_keras_layer.py index d15c344c..a2efedb3 100644 --- a/tests/test_preprocess_keras_layer.py +++ b/tests/test_preprocess_keras_layer.py @@ -1,4 +1,3 @@ -import keras import keras.ops as K import numpy as np import pytest @@ -6,7 +5,6 @@ from keras.layers import Activation, Conv2D, Dense, Layer, Permute, PReLU from numpy.testing import assert_almost_equal -from decomon.keras_utils import get_weight_index_from_name from decomon.models.utils import preprocess_layer, split_activation diff --git a/tests/test_preprocess_keras_model.py b/tests/test_preprocess_keras_model.py index de4394e6..61affb5c 100644 --- a/tests/test_preprocess_keras_model.py +++ b/tests/test_preprocess_keras_model.py @@ -1,7 +1,7 @@ import keras.ops as K import numpy as np import pytest -from keras.layers import Activation, Conv2D, Dense, Flatten, Input, PReLU +from keras.layers import Activation, Conv2D, Dense, Input, PReLU from keras.models import Model, Sequential from numpy.testing import assert_almost_equal diff --git a/tests/test_unary_layers.py b/tests/test_unary_layers.py index 9a365a61..adcf56ca 100644 --- a/tests/test_unary_layers.py +++ b/tests/test_unary_layers.py @@ -1,7 +1,7 @@ import keras.ops as K import numpy as np from keras.layers import Activation, Dense -from pytest_cases import fixture, fixture_ref, parametrize +from pytest_cases import fixture, parametrize from decomon.keras_utils import batch_multid_dot from decomon.layers import DecomonActivation, DecomonDense From 7b72cf5f9921679b04a6a85ff22535de059e5e18 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 15 Mar 2024 15:41:06 +0100 Subject: [PATCH 085/101] Import clone at decomon and decomon.models levels --- src/decomon/__init__.py | 3 ++- src/decomon/models/__init__.py | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/decomon/__init__.py b/src/decomon/__init__.py index 2debee7b..dd913f39 100644 --- a/src/decomon/__init__.py +++ b/src/decomon/__init__.py @@ -8,7 +8,8 @@ import keras -# from . import layers, models +from .models import clone + # from .metrics.loss import get_adv_loss, get_lower_loss, get_model, get_upper_loss # from .models.models import get_AB as get_grid_params # from .models.models import get_AB_finetune as get_grid_slope diff --git a/src/decomon/models/__init__.py b/src/decomon/models/__init__.py index d2ae1fe9..bd4607de 100644 --- a/src/decomon/models/__init__.py +++ b/src/decomon/models/__init__.py @@ -1,4 +1,2 @@ -# from .convert import clone -# from .models import DecomonModel - -# from .decomon_sequential import DecomonModel +from .convert import clone +from .models import DecomonModel From dd1638ba3ca7437ead13997ff4097381d649653f Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 15 Mar 2024 15:45:53 +0100 Subject: [PATCH 086/101] Convert slope as Slope in clone() if passed as a string --- src/decomon/models/convert.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index 46c8b792..ff472acb 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -288,6 +288,9 @@ def clone( if isinstance(method, str): method = ConvertMethod(method.lower()) + if isinstance(slope, str): + slope = Slope(slope.lower()) + # preprocess backward_bounds backward_bounds_flattened: Optional[list[keras.KerasTensor]] backward_bounds_for_convert: Optional[list[keras.KerasTensor]] From 94e6e434607d0b26065ba67ece67ba596341ed86 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 15 Mar 2024 15:53:23 +0100 Subject: [PATCH 087/101] Update example in "getting started" --- docs/source/getting_started.md | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/source/getting_started.md b/docs/source/getting_started.md index e71905c2..285e7d9f 100644 --- a/docs/source/getting_started.md +++ b/docs/source/getting_started.md @@ -8,29 +8,34 @@ within a box domain. ````python # Imports +import keras import numpy as np -from keras.layers import Dense +from keras.layers import Dense, Input from keras.models import Sequential -from decomon import get_lower_box, get_upper_box -from decomon.models import clone +# from decomon import get_lower_box, get_upper_box +from decomon import clone # Toy example with a Keras model: -model = Sequential([Dense(10, activation="relu", input_dim=2)]) +model = Sequential([Input((2,)), Dense(10, activation="relu")]) + +# Convert into a decomon neural network +decomon_model = clone( + model, + final_ibp=True, # keep constant bounds + final_affine=False # drop affine bounds +) # Create a fake box with the right shape x_min = np.zeros((1, 2)) x_max = np.ones((1, 2)) - -# Convert into a decomon neural network -decomon_model = clone(model) +x_box = np.concatenate([x_min[:, None], x_max[:, None]], axis=1) # Get lower and upper bounds -lower_bound = get_lower_box(decomon_model, x_min, x_max) -upper_bound = get_upper_box(decomon_model, x_min, x_max) +lower_bound, upper_bound = decomon_model.predict_on_single_batch_np(x_box) # more efficient than predict on very small batch -print(lower_bound) -print(upper_bound) +print(f"lower bound: {lower_bound}") +print(f"upper bound: {upper_bound}") ```` Other types of domains are possible and illustrated From 72ac89770609c300f8f6fc1d0edbd06ecbbb6f71 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 15 Mar 2024 18:03:33 +0100 Subject: [PATCH 088/101] Add mapping between custom (keras->decomon) layers in clone --- src/decomon/models/backward_cloning.py | 6 ++++++ src/decomon/models/convert.py | 7 +++++++ src/decomon/models/forward_cloning.py | 6 ++++++ 3 files changed, 19 insertions(+) diff --git a/src/decomon/models/backward_cloning.py b/src/decomon/models/backward_cloning.py index 39d3a531..80a8a24e 100644 --- a/src/decomon/models/backward_cloning.py +++ b/src/decomon/models/backward_cloning.py @@ -409,6 +409,7 @@ def convert_backward( slope: Union[str, Slope] = Slope.V_SLOPE, forward_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, forward_layer_map: Optional[dict[int, DecomonLayer]] = None, + mapping_keras2decomon_classes: Optional[dict[type[Layer], type[DecomonLayer]]] = None, **kwargs: Any, ) -> list[keras.KerasTensor]: """Convert keras model via backward propagation. @@ -421,6 +422,7 @@ def convert_backward( perturbation_domain_input: perturbation domain input perturbation_domain: perturbation domain type on keras model input layer_fn: callable converting a layer and a model_output_shape into a (backward) decomon layer + mapping_keras2decomon_classes: user-defined mapping between keras and decomon layers classes, to be passed to `layer_fn` backward_bounds: if set, should be of the same size as the number of model outputs times 4, being the concatenation of backward bounds for each keras model output from_linear_backward_bounds: specify if backward_bounds come from a linear model (=> no batchsize + upper == lower) @@ -454,6 +456,7 @@ def convert_backward( slope=slope, perturbation_domain=perturbation_domain, propagation=propagation, + mapping_keras2decomon_classes=mapping_keras2decomon_classes, **kwargs, ) @@ -508,6 +511,7 @@ def include_kwargs_layer_fn( perturbation_domain: PerturbationDomain, propagation: Propagation, slope: Slope, + mapping_keras2decomon_classes: Optional[dict[type[Layer], type[DecomonLayer]]], **kwargs: Any, ) -> Callable[[Layer, tuple[int, ...]], DecomonLayer]: """Include external parameters in the function converting layers @@ -519,6 +523,7 @@ def include_kwargs_layer_fn( perturbation_domain: propagation: slope: + mapping_keras2decomon_classes: **kwargs: Returns: @@ -532,6 +537,7 @@ def func(layer: Layer, model_output_shape: tuple[int, ...]) -> DecomonLayer: slope=slope, perturbation_domain=perturbation_domain, propagation=propagation, + mapping_keras2decomon_classes=mapping_keras2decomon_classes, **kwargs, ) diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index ff472acb..d319fe10 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -94,6 +94,7 @@ def convert( forward_layer_map: Optional[dict[int, DecomonLayer]] = None, final_ibp: bool = False, final_affine: bool = True, + mapping_keras2decomon_classes: Optional[dict[type[Layer], type[DecomonLayer]]] = None, **kwargs: Any, ) -> list[keras.KerasTensor]: """ @@ -107,6 +108,7 @@ def convert( from_linear_backward_bounds: specify if backward_bounds come from a linear model (=> no batchsize + upper == lower) if a boolean, flag for each backward bound, else a list of boolean, one per keras model output. layer_fn: callable converting a layer and a model_output_shape into a decomon layer + mapping_keras2decomon_classes: user-defined mapping between keras and decomon layers classes, to be passed to `layer_fn` slope: slope used by decomon activation layers forward_output_map: forward outputs per node from a previously performed forward conversion. To be used for forward oracle if not empty. @@ -148,6 +150,7 @@ def convert( perturbation_domain=perturbation_domain, ibp=ibp, affine=affine, + mapping_keras2decomon_classes=mapping_keras2decomon_classes, **kwargs, ) @@ -162,6 +165,7 @@ def convert( slope=slope, forward_output_map=forward_output_map, forward_layer_map=forward_layer_map, + mapping_keras2decomon_classes=mapping_keras2decomon_classes, **kwargs, ) # output updated mode @@ -214,6 +218,7 @@ def clone( layer_fn: Callable[..., DecomonLayer] = to_decomon, forward_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, forward_layer_map: Optional[dict[int, DecomonLayer]] = None, + mapping_keras2decomon_classes: Optional[dict[type[Layer], type[DecomonLayer]]] = None, **kwargs: Any, ) -> DecomonModel: """ @@ -232,6 +237,7 @@ def clone( final_affine: specify if final outputs should include affine bounds. Default to True all methods except forward-ibp. layer_fn: callable converting a layer and a model_output_shape into a decomon layer + mapping_keras2decomon_classes: user-defined mapping between keras and decomon layers classes, to be passed to `layer_fn` forward_output_map: forward outputs per node from a previously performed forward conversion. To be used for forward oracle if not empty. To be recomputed if empty and needed by the method. @@ -324,6 +330,7 @@ def clone( forward_layer_map=forward_layer_map, final_ibp=final_ibp, final_affine=final_affine, + mapping_keras2decomon_classes=mapping_keras2decomon_classes, **kwargs, ) diff --git a/src/decomon/models/forward_cloning.py b/src/decomon/models/forward_cloning.py index d40a3928..0cb0ffce 100644 --- a/src/decomon/models/forward_cloning.py +++ b/src/decomon/models/forward_cloning.py @@ -33,6 +33,7 @@ def convert_forward( perturbation_domain: Optional[PerturbationDomain] = None, ibp: bool = True, affine: bool = True, + mapping_keras2decomon_classes: Optional[dict[type[Layer], type[DecomonLayer]]] = None, **kwargs: Any, ) -> tuple[list[keras.KerasTensor], dict[int, list[keras.KerasTensor]], dict[int, DecomonLayer]]: """Convert keras model via forward propagation. @@ -45,6 +46,7 @@ def convert_forward( perturbation_domain_input: perturbation domain input (input to the future decomon model). Used to convert affine bounds into constant ones. layer_fn: conversion function on layers. Default to `to_decomon()`. + mapping_keras2decomon_classes: user-defined mapping between keras and decomon layers classes, to be passed to `layer_fn` slope: slope used by decomon activation layers perturbation_domain: perturbation domain type for keras input ibp: specify if constant bounds are propagated @@ -90,6 +92,7 @@ def convert_forward( ibp=ibp, affine=affine, propagation=propagation, + mapping_keras2decomon_classes=mapping_keras2decomon_classes, **kwargs, ) @@ -224,6 +227,7 @@ def include_kwargs_layer_fn( affine: bool, propagation: Propagation, slope: Slope, + mapping_keras2decomon_classes: Optional[dict[type[Layer], type[DecomonLayer]]], **kwargs: Any, ) -> Callable[[Layer], list[Layer]]: """Include external parameters in the function converting layers @@ -237,6 +241,7 @@ def include_kwargs_layer_fn( ibp: affine: slope: + mapping_keras2decomon_classes: **kwargs: Returns: @@ -253,6 +258,7 @@ def func(layer: Layer) -> list[Layer]: ibp=ibp, affine=affine, propagation=propagation, + mapping_keras2decomon_classes=mapping_keras2decomon_classes, **kwargs, ) ] From ac7957a36e5e21a06e9d87f39d33162198994c16 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 15 Mar 2024 18:14:16 +0100 Subject: [PATCH 089/101] Add a tutorial for custom layers --- tutorials/z_Advanced/custom_layer.ipynb | 502 ++++++++++++++++++++++++ 1 file changed, 502 insertions(+) create mode 100644 tutorials/z_Advanced/custom_layer.ipynb diff --git a/tutorials/z_Advanced/custom_layer.ipynb b/tutorials/z_Advanced/custom_layer.ipynb new file mode 100644 index 00000000..86406797 --- /dev/null +++ b/tutorials/z_Advanced/custom_layer.ipynb @@ -0,0 +1,502 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ed5852cf-7c70-440a-b8ae-558f9e792ec5", + "metadata": {}, + "source": [ + "# How to implement its own custom decomon layer?\n", + "\n", + "\n", + "

\n", + " \"Decomon!\"\n", + "

\n", + "\n", + "\n", + "\n", + "- 📚 Documentation \n", + "- Github \n", + "- Tutorials \n", + " \n", + "_Author: [Melanie DUCOFFE](https://fr.linkedin.com/in/m%C3%A9lanie-ducoffe-bbb53165)_\n", + "\n", + "\n", + "When using decomon on customized Keras layers (or not already implemented in decomon), one has to implement their decomon counterpart. \n", + "The easiest way is to simply implement their constant and affine relaxation, as explained in this notebook." + ] + }, + { + "cell_type": "markdown", + "id": "d5478238-ebf0-4760-af96-60d78c4f1f38", + "metadata": {}, + "source": [ + "## Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4ae44949-26f2-4eb8-bcfd-2bd28c16f2dc", + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import keras\n", + "import keras.ops as K\n", + "import numpy as np\n", + "from keras.layers import Dense, Input, Layer\n", + "from keras.models import Sequential\n", + "\n", + "# from decomon import get_lower_box, get_upper_box\n", + "from decomon import clone\n", + "from decomon.keras_utils import batch_multid_dot\n", + "from decomon.layers import DecomonLayer" + ] + }, + { + "cell_type": "markdown", + "id": "6a695903-0d16-4fa8-be3a-d868b9884b1a", + "metadata": {}, + "source": [ + "## Custom keras layer and keras model" + ] + }, + { + "cell_type": "markdown", + "id": "73d5523e-23a3-4c36-82ca-ef65a20b1e03", + "metadata": {}, + "source": [ + "We implement here 2 keras layers:\n", + "- a linear layer: simply doubling its input\n", + "- a non-linear layer: squaring its input" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8ba665b-20d0-48c4-8a69-5723136aa0d9", + "metadata": {}, + "outputs": [], + "source": [ + "class Double(Layer):\n", + " \"\"\"Doubling layer.\"\"\"\n", + "\n", + " def call(self, inputs):\n", + " \"\"\"Take double.\"\"\"\n", + " return inputs * 2\n", + "\n", + "\n", + "class Square(Layer):\n", + " \"\"\"Square layer.\"\"\"\n", + "\n", + " def call(self, inputs):\n", + " \"\"\"Take square.\"\"\"\n", + " return inputs**2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc24f31a-bfba-432f-8e51-93e196096d96", + "metadata": {}, + "outputs": [], + "source": [ + "model = Sequential([Input((2,)), Double(), Dense(10), Square()])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6c18d606-ae77-4616-b914-f52fddb7994e", + "metadata": {}, + "outputs": [], + "source": [ + "model.layers[2].input.shape" + ] + }, + { + "cell_type": "markdown", + "id": "fa4811e9-cd61-4e37-aed5-608188a7b067", + "metadata": {}, + "source": [ + "## Decomon custom layer implementation\n", + "\n", + "We need to derive from `DecomonLayer` and implement some methods." + ] + }, + { + "cell_type": "markdown", + "id": "fe939f13-8f8c-4a4e-916e-383d0ae641cf", + "metadata": {}, + "source": [ + "### Linear layer\n", + "\n", + "For the linear layer, we only have to give the proper affine representation of the layer (with proper shape), once we have specified that it is a linear layer. \n", + "As we have a linear (or affine) layer, this representation is independent of the batch and will be given as such. \n", + "\n", + "More precisely, we need to return weights and bias `w` and `b` such that\n", + "\n", + " layer(x) = x * w + b\n", + "\n", + "where the multiplication is actually `keras.ops.tensordot` on all non-batch axes of `x` and the the first non-batch ones of `w`, same number as the number of non-batch axes in `x`. \n", + "\n", + "This can performed via `batch_multi_dot()` from decomon (same function will be used for non-linear case).\n", + "\n", + "In the generic case, with 1 output, the shapes are:\n", + " \n", + " - x ~ (batchsize,) + layer.input.shape[1:]\n", + " - b ~ layer.output.shape[1:]\n", + " - w ~ layer.input.shape[1:] + layer.output.shape[1:]\n", + "\n", + "We can also use *diagonal representation*: this means that `w` is represented only by its diagonal. (This only possible if input and output are of same shape). The shapes are:\n", + "\n", + " - x ~ (batchsize,) + layer.input.shape[1:]\n", + " - b ~ layer.output.shape[1:]\n", + " - w ~ layer.output.shape[1:]\n", + "\n", + " - layer.input.shape[1:] == layer.output.shape[1:]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0c519f84-de7b-40e6-8ee2-8240e4a3a0bf", + "metadata": {}, + "outputs": [], + "source": [ + "class DecomonDouble(DecomonLayer):\n", + " linear = True # specifying that this is a linear layer\n", + " diagonal = True # specifying `w` is represented by its diagonal\n", + "\n", + " def get_affine_representation(self):\n", + " bias_shape = self.layer.input.shape[\n", + " 1:\n", + " ] # a decomon layer has always an attribute `layer` which is the corresponding keras layer for which it is the decomon counterpart.\n", + " w = 2 * keras.ops.ones(bias_shape)\n", + " b = keras.ops.zeros(bias_shape)\n", + " return w, b" + ] + }, + { + "cell_type": "markdown", + "id": "10c7bfe2-4ff4-471e-a7f3-007d9309b9ba", + "metadata": {}, + "source": [ + "#### Verification\n", + "\n", + "Let us check the affine representation by comparing it with actual output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e4735f8-1f6f-46b7-ace6-c4ef69287bbd", + "metadata": {}, + "outputs": [], + "source": [ + "# Instantiate the keras layer and build it\n", + "layer = Double()\n", + "layer(Input((2,)))\n", + "\n", + "# Instantiate the corresponding decomon layer\n", + "decomon_layer = DecomonDouble(layer=layer)\n", + "\n", + "# Keras input/output\n", + "x = K.convert_to_tensor(np.random.random((3, 2)), dtype=keras.config.floatx()) # ensure using same precision as default\n", + "layer_output_np = K.convert_to_numpy(layer(x))\n", + "\n", + "# Recompute with affine representation\n", + "w, b = decomon_layer.get_affine_representation()\n", + "missing_batchsize = (False, True) # specify that `w` is missing batchsize (but not x)\n", + "diagonal = (False, True) # specify that `w` is represented by its diagonal\n", + "recomputed_layer_output = batch_multid_dot(x, w, missing_batchsize=missing_batchsize, diagonal=diagonal) + b\n", + "recomputed_layer_output_np = K.convert_to_numpy(recomputed_layer_output)\n", + "\n", + "# Compare\n", + "np.testing.assert_almost_equal(recomputed_layer_output_np, layer_output_np)\n", + "\n", + "print(\"Perfect!\")" + ] + }, + { + "cell_type": "markdown", + "id": "b2bb4cb1-7548-4344-a007-90091301608e", + "metadata": {}, + "source": [ + "### Non-linear layer\n", + "\n", + "For the non-linear layer, we need to give the constant and affine relaxation of the layer (with proper shape)\n", + "\n", + "#### Constant relaxation\n", + "\n", + "Given lower and upper bounds on layer input, we give lower and upper constant bounds on layer output (with a batchsize).\n", + "\n", + "#### Affine relaxation\n", + ". \n", + "Given lower and upper bounds on layer input, we need to return weights and biases `w_l`, `b_l`, `w_u`, and `b_u` such that\n", + "\n", + " x * w_l + b_l <= layer(x) <= x * w_u + b_u\n", + "\n", + "where the multiplication is batch-wise, and on multiple axes (the first non-batch ones of `w`, same number as the number of non-batch axes in `x`). This is performed via `batch_multi_dot()` from decomon.\n", + "\n", + "In the generic case, with 1 output, the shape are:\n", + " \n", + " - x ~ (batchsize,) + layer.input.shape[1:]\n", + " - b_l, b_l ~ (batchsize,) + layer.output.shape[1:]\n", + " - w_l, w_u ~ (batchsize,) + layer.input.shape[1:] + layer.output.shape[1:]\n", + "\n", + "\n", + "We can also use *diagonal representation*: as before, this means that `w_l` and `w_u` will be represented by their diagonal, so of the same shape as `b_l` and `b_u`. Only possible if input and output of the layer share the same shape.\n", + "\n", + " - x ~ (batchsize,) + layer.input.shape[1:]\n", + " - b_l, b_l ~ (batchsize,) + layer.output.shape[1:]\n", + " - w_l, w_u ~ (batchsize,) + layer.output.shape[1:]\n", + "\n", + " - layer.input.shape[1:] == layer.output.shape[1:]\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a4c9c28-712b-424d-9a7e-238890dca400", + "metadata": {}, + "outputs": [], + "source": [ + "class DecomonSquare(DecomonLayer):\n", + " diagonal = True # specifying `w_l` and `w_u` are represented by their diagonal\n", + "\n", + " def forward_ibp_propagate(self, lower, upper):\n", + " # image of bounds\n", + " f_lower = lower**2\n", + " f_upper = upper**2\n", + "\n", + " # lower bound: if same sign, the minimum between 2 images, if opposite signs, 0.\n", + " lower_out = K.where(K.sign(lower * upper) > 0, K.minimum(f_lower, f_upper), 0.0)\n", + " # upper_bound: the maximum between 2 images\n", + " upper_out = K.maximum(f_lower, f_upper)\n", + "\n", + " return lower_out, upper_out\n", + "\n", + " def get_affine_bounds(self, lower, upper):\n", + " # image of bounds\n", + " f_lower = lower**2\n", + " f_upper = upper**2\n", + "\n", + " # lower bound:\n", + " # - opposite signs: 0 hyperplan\n", + " # - same signs: tangent hyperplan at minimum point\n", + " w_l = K.where(\n", + " K.sign(lower * upper) > 0,\n", + " K.where(\n", + " lower < 0,\n", + " 2 * upper,\n", + " 2 * lower,\n", + " ),\n", + " 0.0,\n", + " )\n", + " b_l = K.where(\n", + " K.sign(lower * upper) > 0,\n", + " K.where(\n", + " lower < 0,\n", + " -(upper**2),\n", + " -(lower**2),\n", + " ),\n", + " 0.0,\n", + " )\n", + "\n", + " # upper bound: by convexity, hyperplan between lower and upper\n", + " w_u = (f_upper - f_lower) / K.maximum(\n", + " K.cast(keras.config.epsilon(), dtype=upper.dtype), upper - lower\n", + " ) # avoid dividing by 0. -> replace by epsilon.\n", + " b_u = f_lower - w_u * lower\n", + "\n", + " return w_l, b_l, w_u, b_u" + ] + }, + { + "cell_type": "markdown", + "id": "702a789d-2f83-44c3-b1bf-d11b71902a11", + "metadata": {}, + "source": [ + "#### Verification\n", + "\n", + "Let us check the relaxations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24a7b648-426b-4481-9dcb-d6512d1e8d5f", + "metadata": {}, + "outputs": [], + "source": [ + "# Instantiate the keras layer and build it\n", + "layer = Square()\n", + "layer(Input((2,)))\n", + "\n", + "# Instantiate the corresponding decomon layer\n", + "decomon_layer = DecomonSquare(layer=layer)\n", + "\n", + "# Keras bounds/input/output\n", + "lowers = [-2, -1, 1]\n", + "uppers = [-1, 1, 2]\n", + "\n", + "x_np = np.concatenate([np.random.random((1, 2)) * (u - l) + l for l, u in zip(lowers, uppers)], axis=0)\n", + "lower_np = np.concatenate([np.reshape([l, l], (1, 2)) for l in lowers], axis=0)\n", + "upper_np = np.concatenate([np.reshape([u, u], (1, 2)) for u in uppers], axis=0)\n", + "\n", + "x = K.convert_to_tensor(x_np, dtype=keras.config.floatx()) # ensure using same precision as default\n", + "lower = K.convert_to_tensor(lower_np, dtype=keras.config.floatx())\n", + "upper = K.convert_to_tensor(upper_np, dtype=keras.config.floatx())\n", + "\n", + "layer_output_np = K.convert_to_numpy(layer(x))\n", + "\n", + "\n", + "# constant bounds\n", + "lower_ibp, upper_ibp = decomon_layer.forward_ibp_propagate(lower, upper)\n", + "\n", + "# affine bounds => computed constant bounds\n", + "w_l, b_l, w_u, b_u = decomon_layer.get_affine_bounds(lower, upper)\n", + "diagonal = (False, True) # specify that `w` is represented by its diagonal\n", + "lower_affine = batch_multid_dot(x, w_l, diagonal=diagonal) + b_l\n", + "upper_affine = batch_multid_dot(x, w_u, diagonal=diagonal) + b_u\n", + "\n", + "\n", + "lower_affine_np = K.convert_to_numpy(lower_affine)\n", + "upper_affine_np = K.convert_to_numpy(upper_affine)\n", + "lower_ibp_np = K.convert_to_numpy(lower_ibp)\n", + "upper_ibp_np = K.convert_to_numpy(upper_ibp)\n", + "\n", + "\n", + "# comparison\n", + "assert (lower_affine_np <= layer_output_np).all()\n", + "assert (upper_affine_np >= layer_output_np).all()\n", + "assert (lower_ibp_np <= layer_output_np).all()\n", + "assert (upper_ibp_np >= layer_output_np).all()\n", + "\n", + "print(\"Perfect!\")" + ] + }, + { + "cell_type": "markdown", + "id": "73f5b684-34d9-40e2-a5e3-718410e0e65d", + "metadata": {}, + "source": [ + "## Convert to decomon model\n", + "\n", + "We need to specify the mapping between keras and decomon custom layers." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9cd201a5-a066-4690-9068-9c435da76a73", + "metadata": {}, + "outputs": [], + "source": [ + "decomon_model = clone(\n", + " model,\n", + " mapping_keras2decomon_classes={Square: DecomonSquare, Double: DecomonDouble}, # custom layers mapping\n", + " final_ibp=True, # keep final constant bounds\n", + " final_affine=False, # drop final affine bounds\n", + ")\n", + "\n", + "decomon_model.summary()" + ] + }, + { + "cell_type": "markdown", + "id": "fa9f79b2-cc58-4f28-9da2-a741b736a7be", + "metadata": {}, + "source": [ + "Get formal lower and upper bounds on a box domain [0,1] for inputs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8970932a-fafb-4ef0-8dcf-516ac220639f", + "metadata": {}, + "outputs": [], + "source": [ + "# Create a fake box with the right shape\n", + "x_min = np.zeros((1, 2))\n", + "x_max = np.ones((1, 2))\n", + "x_box = np.concatenate([x_min[:, None], x_max[:, None]], axis=1)\n", + "\n", + "# Get lower and upper bounds\n", + "lower_bound, upper_bound = decomon_model.predict_on_single_batch_np(\n", + " x_box\n", + ") # more efficient than predict on very small batch\n", + "\n", + "print(f\"lower bound: {lower_bound}\")\n", + "print(f\"upper bound: {upper_bound}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e26f590c-408a-42df-b104-a76e64a82c6b", + "metadata": {}, + "source": [ + "Compare with empirical bounds\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70079724-8050-4d06-9c98-b8d32778ec6f", + "metadata": {}, + "outputs": [], + "source": [ + "keras_input = K.convert_to_tensor(np.random.random((100, 2)))\n", + "keras_output = K.convert_to_numpy(model(keras_input))\n", + "lower_empirical = np.min(keras_output, axis=0)\n", + "upper_empirical = np.max(keras_output, axis=0)\n", + "\n", + "print(f\"empirical lower bound: {lower_empirical}\")\n", + "print(f\"empirical upper bound: {upper_empirical}\")" + ] + }, + { + "cell_type": "markdown", + "id": "67b72d24-8b6f-4c2c-8d74-e42d971af06c", + "metadata": {}, + "source": [ + "We should have (and the tightest, the best the approximation):\n", + "\n", + " lower_bounds <= lower_empirical , upper_empirical <= upper_bound\n", + " " + ] + }, + { + "cell_type": "markdown", + "id": "5326dd7e-62fd-4311-81a2-ff9ab714345c", + "metadata": {}, + "source": [ + "That's all folks!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From d86e1d8f939d75d1e5895cffcc0871cfac5b4550 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Fri, 15 Mar 2024 20:38:15 +0100 Subject: [PATCH 090/101] Add test for clone with custom layers --- tests/test_clone.py | 55 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/tests/test_clone.py b/tests/test_clone.py index 8270056e..587853e4 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -1,11 +1,12 @@ import keras.ops as K import numpy as np import pytest -from keras.layers import Activation, Input +from keras.layers import Activation, Dense, Input from keras.models import Model from pytest_cases import parametrize -from decomon.constants import ConvertMethod, Slope +from decomon.constants import ConvertMethod, Propagation, Slope +from decomon.layers import DecomonDense, DecomonLayer from decomon.layers.input import IdentityInput from decomon.layers.utils.symbolify import LinkToPerturbationDomainInput from decomon.models.convert import clone @@ -558,3 +559,53 @@ def test_clone_identity_model( # check that we added a layer to insert batch axis if method.lower().startswith("crown"): assert isinstance(decomon_model.layers[-1], IdentityInput) + + +class MyDenseDecomonLayer(DecomonLayer): + def __init__( + self, + layer, + perturbation_domain=None, + ibp: bool = True, + affine: bool = True, + propagation=Propagation.FORWARD, + model_input_shape=None, + model_output_shape=None, + my_super_attribute=0.0, + **kwargs, + ): + super().__init__( + layer, perturbation_domain, ibp, affine, propagation, model_input_shape, model_output_shape, **kwargs + ) + self.my_super_attribute = my_super_attribute + + +def test_clone_custom_layer( + method, + perturbation_domain, + helpers, +): + decimal = 4 + + input_shape = (5,) + + mapping_keras2decomon_classes = {Dense: MyDenseDecomonLayer} + + # keras model + keras_model = helpers.toy_network_tutorial(input_shape=input_shape) + + # conversion + decomon_model = clone( + model=keras_model, + perturbation_domain=perturbation_domain, + method=method, + mapping_keras2decomon_classes=mapping_keras2decomon_classes, + my_super_attribute=12.5, + ) + + # check layers + assert any([isinstance(l, MyDenseDecomonLayer) for l in decomon_model.layers]) + assert not any([isinstance(l, DecomonDense) for l in decomon_model.layers]) + for l in decomon_model.layers: + if isinstance(l, MyDenseDecomonLayer): + assert l.my_super_attribute == 12.5 From 65b5dbc1512231690a9de1dc78b256b66124e1b9 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 18 Mar 2024 15:12:22 +0100 Subject: [PATCH 091/101] Add method to ease final constant bounds computation for a decomon model --- src/decomon/models/convert.py | 1 + src/decomon/models/models.py | 80 +++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index d319fe10..e70f57ac 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -359,4 +359,5 @@ def clone( method=method, ibp=final_ibp, affine=final_affine, + model=model, ) diff --git a/src/decomon/models/models.py b/src/decomon/models/models.py index 03ac92af..6fff46ee 100644 --- a/src/decomon/models/models.py +++ b/src/decomon/models/models.py @@ -7,10 +7,38 @@ from keras.utils import serialize_keras_object from decomon.constants import ConvertMethod +from decomon.layers.output import ConvertOutput from decomon.perturbation_domain import PerturbationDomain class DecomonModel(keras.Model): + """Decomon models computing constant and/or affine bounds on outputs of a keras model. + + This is the model generated by `decomon.clone()`. + + Inputs: perturbation_domain_inputs + backward_bounds with + - perturbation_domain_inputs: peturbation domain input wrapped in a list + - backward_bounds: the precomputed backward_bounds if the decomon model has been generated with it, empty else. + + Outputs: sum_{i} (affine_bounds[i] + constant_bounds[i]): the affine and constant bounds computed + - affine_bounds_to[i]: affine bounds on i-th output of the keras model; + empty if `self.affine` is False, else: w_l[i], b_l[i], w_u[i], b_u[i] so that + + x * w_l[i] + b_l[i] <= keras_model(x)[i] <= x * w_u[i] + b_u[i] + + - constant_bounds_to[i]: constant bounds on i-th output of the keras model; + empty if `self.ibp` is False, else lower[i], upper[i] so that + + lower[i] <= keras_model(x)[i] <= upper[i] + + See `decomon.clone()` doc for more details on format. + + Args: + + + + """ + def __init__( self, inputs: Union[keras.KerasTensor, list[keras.KerasTensor]], @@ -19,9 +47,24 @@ def __init__( method: ConvertMethod, ibp: bool, affine: bool, + model: Model, **kwargs: Any, ): + """ + + Args: + inputs: inputs of the decomon model (perturbation domain input + optionally backward_bounds on a consecutive model) + outputs: outputs of the decomon model + perturbation_domain: perturbation domain type considered on keras model inputs + method: conversion method + ibp: outputing constant bounds? + affine: outputing affine bounds? + model: original keras model to bound + **kwargs: passed to `keras.Model` constructor + + """ super().__init__(inputs, outputs, **kwargs) + self.model = model self.perturbation_domain = perturbation_domain self.method = method self.ibp = ibp @@ -77,6 +120,43 @@ def predict_on_single_batch_np( else: return K.convert_to_numpy(output_tensors) + def compute_constant_bounds_np(self, inputs: Union[np.ndarray, list[np.ndarray]]) -> list[np.ndarray]: + """ + + Args: + inputs: perturbation_domain_inputs + backward_bounds, same format as `self.inputs` + + Returns: + constant bounds: sum_{i} constant_bounds[i], constant bounds for each keras model output, concatenated + with constant_bounds[i] = [lower[i] + upper[i]] + + Notes: + Constant bounds are + - either directly computed as is (forward-ibp method) + - or deduced from affine bounds (forward-affine or crown-* methods) + - or chosen as tightest between both ibp and affine (forward-hybrid) + + """ + output_tensors = self(inputs) + if not (self.ibp and not self.affine): + convert_layer = ConvertOutput( + ibp_from=self.ibp, + affine_from=self.affine, + ibp_to=True, + affine_to=False, + perturbation_domain=self.perturbation_domain, + model_output_shapes=[t.shape[1:] for t in self.model.outputs], + ) + if convert_layer.needs_perturbation_domain_inputs(): + if isinstance(inputs, np.ndarray): + perturbation_domain_input = inputs + else: + perturbation_domain_input = inputs[0] + output_tensors.append(perturbation_domain_input) + # convert actual outputs + output_tensors = convert_layer(output_tensors) + return [K.convert_to_numpy(output) for output in output_tensors] + def _check_domain( perturbation_domain_prev: PerturbationDomain, perturbation_domain: PerturbationDomain From 5bdfb9c7742b556f6843671cdc90823f43590348 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 18 Mar 2024 15:13:03 +0100 Subject: [PATCH 092/101] Update get_range_box(), get_range_noise(), and get_adv_box() for the new decomon api --- src/decomon/__init__.py | 23 ++- src/decomon/models/models.py | 38 +++- src/decomon/wrapper.py | 347 +++++++++++++---------------------- tests/test_wrapper.py | 114 ++++++++++++ 4 files changed, 290 insertions(+), 232 deletions(-) create mode 100644 tests/test_wrapper.py diff --git a/src/decomon/__init__.py b/src/decomon/__init__.py index dd913f39..225e82d2 100644 --- a/src/decomon/__init__.py +++ b/src/decomon/__init__.py @@ -13,18 +13,17 @@ # from .metrics.loss import get_adv_loss, get_lower_loss, get_model, get_upper_loss # from .models.models import get_AB as get_grid_params # from .models.models import get_AB_finetune as get_grid_slope -# from .wrapper import ( -# check_adv_box, -# get_adv_box, -# get_adv_noise, -# get_lower_box, -# get_lower_noise, -# get_range_box, -# get_range_noise, -# get_upper_box, -# get_upper_noise, -# refine_box, -# ) +from .wrapper import ( # check_adv_box,; refine_box, + get_adv_box, + get_adv_noise, + get_lower_box, + get_lower_noise, + get_range_box, + get_range_noise, + get_upper_box, + get_upper_noise, +) + # from .wrapper_with_tuning import get_lower_box_tuning, get_upper_box_tuning try: diff --git a/src/decomon/models/models.py b/src/decomon/models/models.py index 6fff46ee..91681e8c 100644 --- a/src/decomon/models/models.py +++ b/src/decomon/models/models.py @@ -128,7 +128,7 @@ def compute_constant_bounds_np(self, inputs: Union[np.ndarray, list[np.ndarray]] Returns: constant bounds: sum_{i} constant_bounds[i], constant bounds for each keras model output, concatenated - with constant_bounds[i] = [lower[i] + upper[i]] + with constant_bounds[i] = [lower[i], upper[i]] Notes: Constant bounds are @@ -157,6 +157,42 @@ def compute_constant_bounds_np(self, inputs: Union[np.ndarray, list[np.ndarray]] output_tensors = convert_layer(output_tensors) return [K.convert_to_numpy(output) for output in output_tensors] + def compute_affine_bounds_np(self, inputs: Union[np.ndarray, list[np.ndarray]]) -> list[np.ndarray]: + """ + + Args: + inputs: perturbation_domain_inputs + backward_bounds, same format as `self.inputs` + + Returns: + affine bounds: sum_{i} affine_bounds[i], affine bounds for each keras model output, concatenated + with affine_bounds[i] = [w_l[i], b_l[i], w_u[i], b_u[i]] + + Notes: + Affine bounds are + - either directly computed as is (forward-affine or crown-* methods) + - or deduced from constant bounds (forward-ibp method) + + """ + output_tensors = self(inputs) + if not (self.affine and not self.ibp): + convert_layer = ConvertOutput( + ibp_from=self.ibp, + affine_from=self.affine, + ibp_to=False, + affine_to=True, + perturbation_domain=self.perturbation_domain, + model_output_shapes=[t.shape[1:] for t in self.model.outputs], + ) + if convert_layer.needs_perturbation_domain_inputs(): + if isinstance(inputs, np.ndarray): + perturbation_domain_input = inputs + else: + perturbation_domain_input = inputs[0] + output_tensors.append(perturbation_domain_input) + # convert actual outputs + output_tensors = convert_layer(output_tensors) + return [K.convert_to_numpy(output) for output in output_tensors] + def _check_domain( perturbation_domain_prev: PerturbationDomain, perturbation_domain: PerturbationDomain diff --git a/src/decomon/wrapper.py b/src/decomon/wrapper.py index deb396ed..34c3c248 100644 --- a/src/decomon/wrapper.py +++ b/src/decomon/wrapper.py @@ -62,8 +62,12 @@ def get_adv_box( batch_size: for computational efficiency, one can split the calls to minibatches n_sub_boxes: + Returns: numpy array, vector with upper bounds for adversarial attacks + + Hypothesis: the underlying keras model has only 1 output + """ if np.min(x_max - x_min) < 0: raise UserWarning("Inconsistency Error: x_max < x_min") @@ -76,6 +80,11 @@ def get_adv_box( assert isinstance(model.perturbation_domain, (BoxDomain, GridDomain)) decomon_model = model + # if missing batch axis, add it + input_shape = decomon_model.model.inputs[0].shape[1:] + x_min = np.reshape(x_min, (-1,) + input_shape) + x_max = np.reshape(x_max, (-1,) + input_shape) + n_split = 1 n_batch = len(x_min) @@ -87,24 +96,12 @@ def get_adv_box( x_min = np.reshape(x_min, [-1] + shape) x_max = np.reshape(x_max, [-1] + shape) - # reshape x_mmin, x_max - if decomon_model.backward_bounds: - input_shape = list(decomon_model.input_shape[0][2:]) - else: - input_shape = list(decomon_model.input_shape[2:]) - input_dim = np.prod(input_shape) - x_reshaped = x_min + 0 * x_min - x_reshaped = x_reshaped.reshape([-1] + input_shape) - - x_min = x_min.reshape((-1, 1, input_dim)) - x_max = x_max.reshape((-1, 1, input_dim)) - - z = np.concatenate([x_min, x_max], 1) + # construct perturbation_domain_input + perturbation_domain_input = np.concatenate([x_min[:, None], x_max[:, None]], 1) source_labels = prepare_labels(source_labels, n_batch) if target_labels is not None: target_labels = prepare_labels(target_labels, n_batch) - if n_split > 1: shape = list(source_labels.shape[1:]) source_labels = np.reshape(np.concatenate([source_labels[:, None]] * n_split, 1), [-1] + shape) @@ -114,16 +111,24 @@ def get_adv_box( if batch_size > 0: # split r = 0 - if len(x_reshaped) % batch_size > 0: + if len(perturbation_domain_input) % batch_size > 0: r += 1 - x_min_list = [x_min[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r)] - x_max_list = [x_max[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r)] + x_min_list = [ + x_min[batch_size * i : batch_size * (i + 1)] + for i in range(len(perturbation_domain_input) // batch_size + r) + ] + x_max_list = [ + x_max[batch_size * i : batch_size * (i + 1)] + for i in range(len(perturbation_domain_input) // batch_size + r) + ] source_labels_list = [ - source_labels[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r) + source_labels[batch_size * i : batch_size * (i + 1)] + for i in range(len(perturbation_domain_input) // batch_size + r) ] if target_labels is not None: target_labels_list = [ - target_labels[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r) + target_labels[batch_size * i : batch_size * (i + 1)] + for i in range(len(perturbation_domain_input) // batch_size + r) ] adv_score = np.concatenate( [ @@ -148,11 +153,12 @@ def get_adv_box( # two possitible cases: the model improves the bound based on the knowledge of the labels output: list[npt.NDArray[np.float_]] - if decomon_model.backward_bounds: + needs_backward_bounds = len(decomon_model.inputs) > 1 + if needs_backward_bounds: # backward bounds needed C = np.diag([1] * n_label)[None] - source_labels[:, :, None] - output = decomon_model.predict_on_single_batch_np([z, C]) # type: ignore + output = decomon_model.predict_on_single_batch_np([perturbation_domain_input, C]) # type: ignore else: - output = decomon_model.predict_on_single_batch_np(z) # type: ignore + output = decomon_model.predict_on_single_batch_np(perturbation_domain_input) # type: ignore def get_ibp_score( u_c: npt.NDArray[np.float_], @@ -230,19 +236,26 @@ def get_affine_score( return np.max(np.max(upper, -2), -1) if ibp and affine: - z, u_c, w_u_f, b_u_f, l_c, w_l_f, b_l_f = output[:7] + w_l_f, b_l_f, w_u_f, b_u_f, l_c, u_c = output elif not ibp and affine: - z, w_u_f, b_u_f, w_l_f, b_l_f = output[:5] + w_l_f, b_l_f, w_u_f, b_u_f = output elif ibp and not affine: - u_c, l_c = output[:2] + l_c, u_c = output else: raise NotImplementedError("not ibp and not affine not implemented") if ibp: - adv_ibp = get_ibp_score(u_c, l_c, source_labels, target_labels, backward=decomon_model.backward_bounds) + adv_ibp = get_ibp_score(u_c, l_c, source_labels, target_labels, backward=needs_backward_bounds) if affine: adv_f = get_affine_score( - z, w_u_f, b_u_f, w_l_f, b_l_f, source_labels, target_labels, backward=decomon_model.backward_bounds + perturbation_domain_input, + w_u_f, + b_u_f, + w_l_f, + b_l_f, + source_labels, + target_labels, + backward=needs_backward_bounds, ) if ibp and not affine: @@ -296,21 +309,17 @@ def check_adv_box( assert isinstance(model.perturbation_domain, (BoxDomain, GridDomain)) decomon_model = model - ibp = decomon_model.ibp - affine = decomon_model.affine + # if missing batch axis, add it + input_shape = decomon_model.model.inputs[0].shape[1:] + x_min = np.reshape(x_min, (-1,) + input_shape) + x_max = np.reshape(x_max, (-1,) + input_shape) n_split = 1 n_batch = len(x_min) - # reshape x_mmin, x_max - input_shape = list(decomon_model.input_shape[2:]) - input_dim = np.prod(input_shape) - x_reshaped = x_min + 0 * x_min - x_reshaped = x_reshaped.reshape([-1] + input_shape) - x_min = x_min.reshape((-1, 1, input_dim)) - x_max = x_max.reshape((-1, 1, input_dim)) + # construct perturbation_domain_input + perturbation_domain_input = np.concatenate([x_min[:, None], x_max[:, None]], 1) - z = np.concatenate([x_min, x_max], 1) if isinstance(source_labels, (int, np.int_)): source_labels = np.zeros((n_batch, 1)) + source_labels @@ -333,12 +342,19 @@ def check_adv_box( if batch_size > 0: # split r = 0 - if len(x_reshaped) % batch_size > 0: + if len(perturbation_domain_input) % batch_size > 0: r += 1 - x_min_list = [x_min[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r)] - x_max_list = [x_max[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r)] + x_min_list = [ + x_min[batch_size * i : batch_size * (i + 1)] + for i in range(len(perturbation_domain_input) // batch_size + r) + ] + x_max_list = [ + x_max[batch_size * i : batch_size * (i + 1)] + for i in range(len(perturbation_domain_input) // batch_size + r) + ] source_labels_list = [ - source_labels[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r) + source_labels[batch_size * i : batch_size * (i + 1)] + for i in range(len(perturbation_domain_input) // batch_size + r) ] target_labels_list: list[Optional[npt.NDArray[np.int_]]] if ( @@ -347,10 +363,11 @@ def check_adv_box( and (str(target_labels.dtype)[:3] != "int") ): target_labels_list = [ - target_labels[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r) + target_labels[batch_size * i : batch_size * (i + 1)] + for i in range(len(perturbation_domain_input) // batch_size + r) ] else: - target_labels_list = [target_labels] * (len(x_reshaped) // batch_size + r) + target_labels_list = [target_labels] * (len(perturbation_domain_input) // batch_size + r) return np.concatenate( [ @@ -362,15 +379,7 @@ def check_adv_box( ) else: - output = decomon_model.predict_on_single_batch_np(z) - - if not affine: - # translate into affine information - u_c = output[0] - w_u = 0 * u_c[:, None] + np.zeros((1, input_dim, 1)) - output = [z, w_u, u_c, w_u, output[-1]] - ibp = False - affine = True + w_l_f, b_l_f, w_u_f, b_u_f = decomon_model.compute_affine_bounds_np([perturbation_domain_input]) def get_affine_sample( z_tensor: npt.NDArray[np.float_], @@ -410,12 +419,7 @@ def get_affine_sample( return np.max(np.max(upper, -2), -1) - if ibp: - z, u_c, w_u_f, b_u_f, l_c, w_l_f, b_l_f = output[:7] - else: - z, w_u_f, b_u_f, w_l_f, b_l_f = output[:5] - - return get_affine_sample(z, w_l_f, b_l_f, w_u_f, b_u_f, source_labels) + return get_affine_sample(perturbation_domain_input, w_l_f, b_l_f, w_u_f, b_u_f, source_labels) #### FORMAL BOUNDS ###### @@ -472,19 +476,23 @@ def get_range_box( batch_size: int = -1, n_sub_boxes: int = 1, ) -> tuple[npt.NDArray[np.float_], npt.NDArray[np.float_]]: - """bounding the outputs of a model in a given box - if the constant is negative, then it is a formal guarantee that there is no adversarial examples + """Bound the outputs of a model in a given box Args: model: either a Keras model or a Decomon model x_min: numpy array for the extremal lower corner of the boxes x_max: numpy array for the extremal upper corner of the boxes batch_size: for computational efficiency, one can split the - calls to minibatches + calls into minibatches Returns: 2 numpy array, vector with upper bounds and vector with lower bounds + + Hypotheses: + - the underlying keras model has only 1 output; + - if already cloned as a decomon model, non precomputed backward_bounds have been given. + """ if np.min(x_max - x_min) < 0: raise UserWarning("Inconsistency Error: x_max < x_min") @@ -496,8 +504,10 @@ def get_range_box( assert isinstance(model.perturbation_domain, (BoxDomain, GridDomain)) decomon_model = model - n_split = 1 - n_batch = len(x_min) + # if missing batch axis, add it + input_shape = decomon_model.model.inputs[0].shape[1:] + x_min = np.reshape(x_min, (-1,) + input_shape) + x_max = np.reshape(x_max, (-1,) + input_shape) if n_sub_boxes > 1: x_min, x_max = refine_boxes(x_min, x_max, n_sub_boxes) @@ -506,87 +516,33 @@ def get_range_box( shape = list(x_min.shape[2:]) x_min = np.reshape(x_min, [-1] + shape) x_max = np.reshape(x_max, [-1] + shape) + else: + n_split = 1 - # reshape x_mmin, x_max - input_shape = list(decomon_model.input_shape[2:]) - input_dim = np.prod(input_shape) - x_reshaped = x_min + 0 * x_min - x_reshaped = x_reshaped.reshape([-1] + input_shape) - x_min = x_min.reshape((-1, 1, input_dim)) - x_max = x_max.reshape((-1, 1, input_dim)) - - z = np.concatenate([x_min, x_max], 1) + # construct perturbation_domain_input + perturbation_domain_input = np.concatenate([x_min[:, None], x_max[:, None]], 1) if batch_size > 0: - # split + # call by minibatch r = 0 - if len(x_reshaped) % batch_size > 0: + if len(perturbation_domain_input) % batch_size > 0: r += 1 - x_min_list = [x_min[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r)] - x_max_list = [x_max[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r)] + x_min_list = [ + x_min[batch_size * i : batch_size * (i + 1)] + for i in range(len(perturbation_domain_input) // batch_size + r) + ] + x_max_list = [ + x_max[batch_size * i : batch_size * (i + 1)] + for i in range(len(perturbation_domain_input) // batch_size + r) + ] results = [get_range_box(decomon_model, x_min_list[i], x_max_list[i], -1) for i in range(len(x_min_list))] u_out = np.concatenate([r[0] for r in results]) l_out = np.concatenate([r[1] for r in results]) else: - ibp = decomon_model.ibp - affine = decomon_model.affine - - output = decomon_model.predict_on_single_batch_np(z) - shape = list(output[-1].shape[1:]) - output_dim = np.prod(shape) - - if affine: - if ibp: - _, u_i, w_u_f, b_u_f, l_i, w_l_f, b_l_f = output[:7] - if len(u_i.shape) > 2: - u_i = np.reshape(u_i, (-1, output_dim)) - l_i = np.reshape(l_i, (-1, output_dim)) - else: - _, w_u_f, b_u_f, w_l_f, b_l_f = output[:5] - - # reshape if necessary - if len(w_u_f.shape) > 3: - w_u_f = np.reshape(w_u_f, (-1, input_dim, output_dim)) - w_l_f = np.reshape(w_l_f, (-1, input_dim, output_dim)) - b_u_f = np.reshape(b_u_f, (-1, output_dim)) - b_l_f = np.reshape(b_l_f, (-1, output_dim)) - - u_f = ( - np.sum(np.maximum(w_u_f, 0) * x_max[:, 0, :, None], 1) - + np.sum(np.minimum(w_u_f, 0) * x_min[:, 0, :, None], 1) - + b_u_f - ) - l_f = ( - np.sum(np.maximum(w_l_f, 0) * x_min[:, 0, :, None], 1) - + np.sum(np.minimum(w_l_f, 0) * x_max[:, 0, :, None], 1) - + b_l_f - ) - - else: - u_i = output[0] - l_i = output[1] - if len(u_i.shape) > 2: - u_i = np.reshape(u_i, (-1, output_dim)) - l_i = np.reshape(l_i, (-1, output_dim)) - - if ibp and affine: - u_out = np.minimum(u_i, u_f) - l_out = np.maximum(l_i, l_f) - elif ibp and not affine: - u_out = u_i - l_out = l_i - elif not ibp and affine: - u_out = u_f - l_out = l_f - else: - raise NotImplementedError("not ibp and not affine not implemented") - - ##### - if len(shape) > 1: - u_out = np.reshape(u_out, [-1] + shape) - l_out = np.reshape(l_out, [-1] + shape) + # call on all data at once + l_out, u_out = decomon_model.compute_constant_bounds_np([perturbation_domain_input]) if n_split > 1: u_out = np.max(np.reshape(u_out, (-1, n_split)), -1) @@ -654,7 +610,7 @@ def get_range_noise( p: float = np.inf, batch_size: int = -1, ) -> tuple[npt.NDArray[np.float_], npt.NDArray[np.float_]]: - """Bounds the output of a model in an Lp Ball + """Bound the output of a model on an Lp Ball Args: model: either a Keras model or a Decomon model @@ -662,11 +618,15 @@ def get_range_noise( eps: the radius of the ball p: the type of Lp norm (p=2, 1, np.inf) batch_size: for computational efficiency, one can split the - calls to minibatches + calls into minibatches Returns: - 2 numpy arrays, vector with upper andlower bounds - of the range of values taken by the model inside the ball + 2 numpy arrays, vector with upper and lower bounds of the range of values taken by the model inside the ball + + Hypotheses: + - the underlying keras model has only 1 output; + - if already cloned as a decomon model, non precomputed backward_bounds have been given. + """ # check that the model is a DecomonModel, else do the conversion @@ -683,76 +643,29 @@ def get_range_noise( if eps >= 0: decomon_model.set_domain(perturbation_domain) - # reshape x_mmin, x_max - input_shape = list(decomon_model.input_shape[1:]) - input_dim = np.prod(input_shape) - x_reshaped = x + 0 * x - x_reshaped = x_reshaped.reshape([-1] + input_shape) + # if missing batch axis, add it + input_shape = decomon_model.model.inputs[0].shape[1:] + x = np.reshape(x, (-1,) + input_shape) + + # perturbation_domain_input : x itself + perturbation_domain_input = x if batch_size > 0: - # split + # call by minibatch r = 0 - if len(x_reshaped) % batch_size > 0: + if len(perturbation_domain_input) % batch_size > 0: r += 1 - x_list = [x_reshaped[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r)] + x_list = [ + x[batch_size * i : batch_size * (i + 1)] for i in range(len(perturbation_domain_input) // batch_size + r) + ] results = [get_range_noise(decomon_model, x_list[i], eps=eps, p=p, batch_size=-1) for i in range(len(x_list))] return np.concatenate([r[0] for r in results]), np.concatenate([r[1] for r in results]) - ibp = decomon_model.ibp - affine = decomon_model.affine - - output = decomon_model.predict_on_single_batch_np(x_reshaped) - shape = list(output[-1].shape[1:]) - output_dim = np.prod(shape) - - x_reshaped = x_reshaped.reshape((len(x_reshaped), -1)) - ord = _get_dual_ord(p) - - if affine: - if ibp: - _, u_i, w_u_f, b_u_f, l_i, w_l_f, b_l_f = output[:7] - if len(u_i.shape) > 2: - u_i = np.reshape(u_i, (-1, output_dim)) - l_i = np.reshape(l_i, (-1, output_dim)) - else: - _, w_u_f, b_u_f, w_l_f, b_l_f = output[:5] - - # reshape if necessary - if len(w_u_f.shape) > 3: - w_u_f = np.reshape(w_u_f, (-1, input_dim, output_dim)) - b_u_f = np.reshape(b_u_f, (-1, output_dim)) - w_l_f = np.reshape(w_l_f, (-1, input_dim, output_dim)) - b_l_f = np.reshape(b_l_f, (-1, output_dim)) - - u_f = eps * np.linalg.norm(w_u_f, ord=ord, axis=1) + np.sum(w_u_f * x_reshaped[:, :, None], 1) + b_u_f - l_f = -eps * np.linalg.norm(w_l_f, ord=ord, axis=1) + np.sum(w_l_f * x_reshaped[:, :, None], 1) + b_l_f - - else: - u_i = output[0] - l_i = output[1] - if len(u_i.shape) > 2: - u_i = np.reshape(u_i, (-1, output_dim)) - l_i = np.reshape(l_i, (-1, output_dim)) - ###### - - if ibp and affine: - u_out = np.minimum(u_i, u_f) - l_out = np.maximum(l_i, l_f) - elif ibp and not affine: - u_out = u_i - l_out = l_i - elif not ibp and affine: - u_out = u_f - l_out = l_f else: - raise NotImplementedError("not ibp and not affine not implemented") - - if len(shape) > 1: - u_out = np.reshape(u_out, [-1] + shape) - l_out = np.reshape(l_out, [-1] + shape) - - return u_out, l_out + # call on all data at once + l_out, u_out = decomon_model.compute_constant_bounds_np([perturbation_domain_input]) + return u_out, l_out def refine_boxes( @@ -949,16 +862,12 @@ def get_adv_noise( eps = model.perturbation_domain.eps - # reshape x_mmin, x_max - input_shape = list(decomon_model.input_shape[1:]) - x_reshaped = x + 0 * x - x_reshaped = x_reshaped.reshape([-1] + input_shape) - n_split = 1 - n_batch = len(x_reshaped) + # if missing batch axis, add it + input_shape = decomon_model.model.inputs[0].shape[1:] + x = np.reshape(x, (-1,) + input_shape) - input_shape = list(decomon_model.input_shape[1:]) - x_reshaped = x + 0 * x - x_reshaped = x_reshaped.reshape([-1] + input_shape) + n_split = 1 + n_batch = len(x) if isinstance(source_labels, (int, np.int_)): source_labels = np.zeros((n_batch, 1), dtype=np.int_) + source_labels @@ -972,11 +881,11 @@ def get_adv_noise( if batch_size > 0: # split r = 0 - if len(x_reshaped) % batch_size > 0: + if len(x) % batch_size > 0: r += 1 - x_list = [x_reshaped[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r)] + x_list = [x[batch_size * i : batch_size * (i + 1)] for i in range(len(x) // batch_size + r)] source_labels_list = [ - source_labels[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r) + source_labels[batch_size * i : batch_size * (i + 1)] for i in range(len(x) // batch_size + r) ] target_labels_list: list[Optional[npt.NDArray[np.int_]]] if ( @@ -985,10 +894,10 @@ def get_adv_noise( and (str(target_labels.dtype)[:3] != "int") ): target_labels_list = [ - target_labels[batch_size * i : batch_size * (i + 1)] for i in range(len(x_reshaped) // batch_size + r) + target_labels[batch_size * i : batch_size * (i + 1)] for i in range(len(x) // batch_size + r) ] else: - target_labels_list = [target_labels] * (len(x_reshaped) // batch_size + r) + target_labels_list = [target_labels] * (len(x) // batch_size + r) results = [ get_adv_noise( @@ -1007,7 +916,7 @@ def get_adv_noise( else: ibp = decomon_model.ibp affine = decomon_model.affine - output = decomon_model.predict_on_single_batch_np(x_reshaped) + output = decomon_model.predict_on_single_batch_np(x) def get_ibp_score( u_c: npt.NDArray[np.float_], @@ -1084,18 +993,18 @@ def get_affine_score( return np.max(np.max(upper, -2), -1) if ibp and affine: - z, u_c, w_u_f, b_u_f, l_c, w_l_f, b_l_f = output[:7] - if not ibp and affine: - z, w_u_f, b_u_f, w_l_f, b_l_f = output[:5] - if ibp and not affine: - u_c, l_c = output[:2] + w_l_f, b_l_f, w_u_f, b_u_f, l_c, u_c = output + elif not ibp and affine: + w_l_f, b_l_f, w_u_f, b_u_f = output + elif ibp and not affine: + l_c, u_c = output else: raise NotImplementedError("not ibp and not affine not implemented") if ibp: adv_ibp = get_ibp_score(u_c, l_c, source_labels, target_labels) if affine: - adv_f = get_affine_score(z, w_u_f, b_u_f, w_l_f, b_l_f, source_labels, target_labels) + adv_f = get_affine_score(x, w_u_f, b_u_f, w_l_f, b_l_f, source_labels, target_labels) if ibp and not affine: adv_score = adv_ibp diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py new file mode 100644 index 00000000..f0d38cd4 --- /dev/null +++ b/tests/test_wrapper.py @@ -0,0 +1,114 @@ +import keras.ops as K +import numpy as np +import pytest +from keras.layers import Activation, Dense, Input +from keras.models import Sequential +from numpy.testing import assert_almost_equal + +from decomon import get_adv_box, get_lower_box, get_range_box, get_upper_box +from decomon.models import clone +from decomon.perturbation_domain import BallDomain +from decomon.wrapper import check_adv_box, get_range_noise + + +@pytest.fixture() +def toy_model_0d(): + sequential = Sequential() + sequential.add(Input((1,))) + sequential.add(Dense(1, activation="linear")) + sequential.add(Activation("relu")) + sequential.add(Dense(1, activation="linear")) + return sequential + + +@pytest.fixture() +def toy_model_1d(odd, helpers): + input_dim = helpers.get_input_dim_1d_box(odd) + sequential = Sequential() + sequential.add(Input((input_dim,))) + sequential.add(Dense(1, activation="linear")) + sequential.add(Activation("relu")) + sequential.add(Dense(1, activation="linear")) + return sequential + + +def test_get_adv_box_0d(toy_model_0d, helpers): + inputs_ = helpers.get_standard_values_0d_box(n=0) + x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ + + score = get_adv_box(toy_model_0d, z[:, 0], z[:, 1], source_labels=0) + + +def test_check_adv_box_0d(toy_model_0d, helpers): + inputs_ = helpers.get_standard_values_0d_box(n=0) + x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ + + score = check_adv_box(toy_model_0d, z[:, 0], z[:, 1], source_labels=0) + + +def test_get_upper_0d_box(toy_model_0d, n, method, final_ibp, final_affine, helpers): + inputs_ = helpers.get_standard_values_0d_box(n) + x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ + + decomon_model = clone(toy_model_0d, method=method, final_ibp=final_ibp, final_affine=final_affine) + upper = get_upper_box(decomon_model, z[:, 0], z[:, 1]) + y_ref = helpers.predict_on_small_numpy(toy_model_0d, x) + + assert (upper - y_ref).min() + 1e-6 >= 0.0 + + +def test_get_lower_0d_box(toy_model_0d, n, method, final_ibp, final_affine, helpers): + inputs_ = helpers.get_standard_values_0d_box(n) + x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ + + decomon_model = clone(toy_model_0d, method=method, final_ibp=final_ibp, final_affine=final_affine) + lower = get_lower_box(decomon_model, z[:, 0], z[:, 1]) + y_ref = helpers.predict_on_small_numpy(toy_model_0d, x) + + assert (y_ref - lower).min() + 1e-6 >= 0.0 + + +def test_get_range_0d_box(toy_model_0d, n, method, final_ibp, final_affine, helpers): + inputs_ = helpers.get_standard_values_0d_box(n) + x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ + + decomon_model = clone(toy_model_0d, method=method, final_ibp=final_ibp, final_affine=final_affine) + upper, lower = get_range_box(decomon_model, z[:, 0], z[:, 1]) + y_ref = helpers.predict_on_small_numpy(toy_model_0d, x) + + assert (upper - y_ref).min() + 1e-6 >= 0.0 + assert (y_ref - lower).min() + 1e-6 >= 0.0 + + +def test_get_upper_1d_box(toy_model_1d, odd, method, final_ibp, final_affine, helpers): + inputs_ = helpers.get_standard_values_1d_box(odd) + x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ + + decomon_model = clone(toy_model_1d, method=method, final_ibp=final_ibp, final_affine=final_affine) + upper = get_upper_box(decomon_model, z[:, 0], z[:, 1]) + y_ref = helpers.predict_on_small_numpy(toy_model_1d, x) + + assert (upper - y_ref).min() + 1e-6 >= 0.0 + + +def test_get_lower_1d_box(toy_model_1d, odd, method, final_ibp, final_affine, helpers): + inputs_ = helpers.get_standard_values_1d_box(odd) + x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ + + decomon_model = clone(toy_model_1d, method=method, final_ibp=final_ibp, final_affine=final_affine) + lower = get_lower_box(decomon_model, z[:, 0], z[:, 1]) + y_ref = helpers.predict_on_small_numpy(toy_model_1d, x) + + assert (y_ref - lower).min() + 1e-6 >= 0.0 + + +def test_get_range_1d_box(toy_model_1d, odd, method, final_ibp, final_affine, helpers): + inputs_ = helpers.get_standard_values_1d_box(odd) + x, y, z, u_c, W_u, b_u, l_c, W_l, b_l = inputs_ + + decomon_model = clone(toy_model_1d, method=method, final_ibp=final_ibp, final_affine=final_affine) + upper, lower = get_range_box(decomon_model, z[:, 0], z[:, 1]) + y_ref = helpers.predict_on_small_numpy(toy_model_1d, x) + + assert (upper - y_ref).min() + 1e-6 >= 0.0 + assert (y_ref - lower).min() + 1e-6 >= 0.0 From 143731e27aa914ec967ff4b36fe74c49eff015d2 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 18 Mar 2024 16:35:22 +0100 Subject: [PATCH 093/101] Update explanation texts in tutorials and fix imports --- tutorials/tutorial1_sinus-interactive.ipynb | 8 +++++--- tutorials/tutorial2_noise_sensor.ipynb | 8 ++++---- tutorials/z_Advanced/tensorboard-and-decomon.ipynb | 7 ++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tutorials/tutorial1_sinus-interactive.ipynb b/tutorials/tutorial1_sinus-interactive.ipynb index fbc533b4..de531bd0 100644 --- a/tutorials/tutorial1_sinus-interactive.ipynb +++ b/tutorials/tutorial1_sinus-interactive.ipynb @@ -252,9 +252,11 @@ "\n", "Depending on the inner recipes, those methods will output different informations:\n", "\n", - "- **forward-ibp, crown-forward-ibp, crown**: constant upper and lower bounds that outer-approximate every output neurons\n", - "- **forward-affine, crown-forward-affine**: affine upper bound and affine lower bound given the input of the network that outer-approximate every output neurons\n", - "- **forward-hybrid, crown-forward-hybrid**: both types of bounds\n", + "- **forward-ibp**: constant upper and lower bounds that outer-approximate every output neurons\n", + "- **forward-affine, crown-forward-affine, crown-forward-ibp, crown**: affine upper bound and affine lower bound given the input of the network that outer-approximate every output neurons\n", + "- **forward-hybrid**: both types of bounds\n", + "\n", + "We can also override this default output format by setting `final_ibp` (for constant bounds) and `final_affine` (for affine bounds).\n", " \n", "By default, the convert method converts the Keras model in **crown**." ] diff --git a/tutorials/tutorial2_noise_sensor.ipynb b/tutorials/tutorial2_noise_sensor.ipynb index 0f0fbf88..5ca3c377 100644 --- a/tutorials/tutorial2_noise_sensor.ipynb +++ b/tutorials/tutorial2_noise_sensor.ipynb @@ -82,7 +82,7 @@ "+ i_q: Current q-component\n", "\n", "\n", - "The recorded temperature refers to the Permanent Magnet surface temperature (pm) representing the rotor temperature. This was measured with an infrared with 140 hrs recordings. Distinctive sessions are identified with \"profile_id\". You will find additional information in the [official data repository](https://www.kaggle.com/wkirgsn/electric-motor-temperature)" + "The recorded temperature refers to the Permanent Magnet surface temperature (pm) representing the rotor temperature. This was measured with an infrared with 140 hrs recordings. Distinctive sessions are identified with \"profile_id\". You will find additional information in the [official data repository](https://www.kaggle.com/datasets/wkirgsn/electric-motor-temperature)" ] }, { @@ -91,7 +91,7 @@ "source": [ "### Download the dataset locally\n", "\n", - "To download the data you need a [Kaggle](https://www.kaggle.com) account. Then you can download the dataset by clicking on the \"download\" button on the [official data repository](https://www.kaggle.com/wkirgsn/electric-motor-temperature). Unzip the file in the same directory as this notebook.\n", + "To download the data you need a [Kaggle](https://www.kaggle.com) account. Then you can download the dataset by clicking on the \"download\" button on the [official data repository](https://www.kaggle.com/datasets/wkirgsn/electric-motor-temperature). Unzip the file in the same directory as this notebook.\n", "\n", "\n", "You can also use the method described for Binder and Colab below." @@ -271,8 +271,8 @@ "outputs": [], "source": [ "from decomon import get_lower_noise, get_range_noise, get_upper_noise\n", - "from decomon.core import BallDomain\n", - "from decomon.models import clone" + "from decomon.models import clone\n", + "from decomon.perturbation_domain import BallDomain" ] }, { diff --git a/tutorials/z_Advanced/tensorboard-and-decomon.ipynb b/tutorials/z_Advanced/tensorboard-and-decomon.ipynb index fcb6cd8f..31d4c1bb 100644 --- a/tutorials/z_Advanced/tensorboard-and-decomon.ipynb +++ b/tutorials/z_Advanced/tensorboard-and-decomon.ipynb @@ -91,7 +91,7 @@ "import keras\n", "import numpy as np\n", "import tensorboard\n", - "from keras.layers import Activation, Dense\n", + "from keras.layers import Activation, Dense, Input\n", "from keras.models import Sequential\n", "\n", "from decomon.models import clone\n", @@ -141,7 +141,8 @@ "outputs": [], "source": [ "layers = []\n", - "layers.append(Dense(100, activation=\"linear\", input_dim=1, name=\"dense1\")) # specify the dimension of the input space\n", + "layers.append(Input((1,)))\n", + "layers.append(Dense(100, activation=\"linear\", name=\"dense1\"))\n", "layers.append(Activation(\"relu\", name=\"relu1\"))\n", "layers.append(Dense(100, activation=\"linear\", name=\"dense2\"))\n", "layers.append(Activation(\"relu\", name=\"relu2\"))\n", @@ -196,7 +197,7 @@ "outputs": [], "source": [ "# convert our model into a decomon model:\n", - "decomon_model = clone(model, method=\"crown\") # method is optionnal" + "decomon_model = clone(model, method=\"forward-ibp\") # method is optionnal" ] }, { From f361a991b2f51ecc2821f48b34107ffaf36f33e6 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 18 Mar 2024 17:07:48 +0100 Subject: [PATCH 094/101] Remove unused method get_input_from_constant_bounds() (no meaning outside BoxDomain) --- src/decomon/perturbation_domain.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/decomon/perturbation_domain.py b/src/decomon/perturbation_domain.py index a8f45ed8..4202251d 100644 --- a/src/decomon/perturbation_domain.py +++ b/src/decomon/perturbation_domain.py @@ -73,19 +73,6 @@ def get_nb_x_components(self) -> int: """ ... - @abstractmethod - def get_input_from_constant_bounds(self, constant_bounds: list[Tensor]) -> Tensor: - """Construct perturbation domain input x from constant bounds on keras model input - - Args: - constant_bounds: lower and upper constant bounds on keras model input - - Returns: - x: perturbation domain input - - """ - ... - def get_config(self) -> dict[str, Any]: return { "opt_option": self.opt_option, @@ -143,10 +130,6 @@ def get_lower_x(self, x: Tensor) -> Tensor: def get_nb_x_components(self) -> int: return 2 - def get_input_from_constant_bounds(self, constant_bounds: list[Tensor]) -> Tensor: - lower, upper = constant_bounds - return K.concatenate([lower[:, None], upper[:, None]], axis=1) - class GridDomain(PerturbationDomain): pass From d228e7b689f11454ddf60cba72636ae895cd2691 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 18 Mar 2024 18:16:28 +0100 Subject: [PATCH 095/101] Remove last softmax layer during preprocessing --- src/decomon/models/convert.py | 31 ++++++++++++++++++++++-- src/decomon/models/utils.py | 35 ++++++++++++++++++++++++++++ tests/test_preprocess_keras_model.py | 18 ++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index e70f57ac..efd55abb 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -32,6 +32,7 @@ is_input_node, method2propagation, preprocess_layer, + remove_last_softmax_layers, split_activation, ) from decomon.perturbation_domain import BoxDomain, PerturbationDomain @@ -76,8 +77,29 @@ def split_activations_in_keras_model( def preprocess_keras_model( model: Model, + rm_last_softmax: bool = False, ) -> Model: - return _clone_keras_model(model=model, layer_fn=preprocess_layer) + """Preprocess keras model before decomon conversion + + - split activations into separate layers + - if `rm_last_softmax` is true: remove last layer for each output, + if it is a softmax activation layer. + + Args: + model: model to preprocess + rm_last_softmax: flag to enable last softmax removing + + Returns: + + """ + # split activations + model = _clone_keras_model(model=model, layer_fn=preprocess_layer) + + # remove softmax last activation layers (for each output) + if rm_last_softmax: + model = remove_last_softmax_layers(model) + + return model # create status @@ -94,6 +116,7 @@ def convert( forward_layer_map: Optional[dict[int, DecomonLayer]] = None, final_ibp: bool = False, final_affine: bool = True, + rm_last_softmax: bool = True, mapping_keras2decomon_classes: Optional[dict[type[Layer], type[DecomonLayer]]] = None, **kwargs: Any, ) -> list[keras.KerasTensor]: @@ -118,6 +141,7 @@ def convert( To be recomputed if empty and needed by the method. final_ibp: specify if final outputs should include constant bounds. final_affine: specify if final outputs should include affine bounds. + rm_last_softmax: specify if last softmax layer (for each output) are removed during preprocessing **kwargs: keyword arguments to pass to layer_fn Returns: @@ -134,7 +158,7 @@ def convert( from_linear_backward_bounds = [from_linear_backward_bounds] * len(model.outputs) # prepare the Keras Model: split non-linear activation functions into separate Activation layers - model = preprocess_keras_model(model) + model = preprocess_keras_model(model=model, rm_last_softmax=rm_last_softmax) # loop over propagations needed propagations = method2propagation(method) @@ -219,6 +243,7 @@ def clone( forward_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, forward_layer_map: Optional[dict[int, DecomonLayer]] = None, mapping_keras2decomon_classes: Optional[dict[type[Layer], type[DecomonLayer]]] = None, + rm_last_softmax: bool = True, **kwargs: Any, ) -> DecomonModel: """ @@ -244,6 +269,7 @@ def clone( forward_layer_map: forward decomon layer per node from a previously performed forward conversion. To be used for forward oracle if not empty. To be recomputed if empty and needed by the method. + rm_last_softmax: specify if last softmax layer (for each output) are removed during preprocessing **kwargs: keyword arguments to pass to layer_fn Returns: @@ -331,6 +357,7 @@ def clone( final_ibp=final_ibp, final_affine=final_affine, mapping_keras2decomon_classes=mapping_keras2decomon_classes, + rm_last_softmax=rm_last_softmax, **kwargs, ) diff --git a/src/decomon/models/utils.py b/src/decomon/models/utils.py index a01301fd..2a5d49f5 100644 --- a/src/decomon/models/utils.py +++ b/src/decomon/models/utils.py @@ -98,6 +98,41 @@ def preprocess_layer(layer: Layer) -> list[Layer]: return split_activation(layer) +def remove_last_softmax_layers(model: Model) -> Model: + """Remove for each output the last layer if it is a softmax activation + + NB: this should be applied after the split of activations, so that we only need + to check for Activation layers. + + Args: + model: original keras model + + Returns: + model without the last softmax layers + + It will return the same model if no such softmax have to be removed, else construct a new functional model. + + """ + output_nodes = get_output_nodes(model) + new_outputs = [] + has_changed = False + for output_node in output_nodes: + layer = output_node.operation + # NB: activations have been already split => we need only to check for Activation layers + if isinstance(layer, Activation) and layer.get_config()["activation"] == "softmax": + # softmax: take parent nodes outputs instead + for parent in output_node.parent_nodes: + new_outputs += parent.outputs + has_changed = True + else: + # no softmax: keep same outputs + new_outputs += output_node.outputs + if has_changed: + return Model(model.inputs, new_outputs) + else: + return model + + def is_input_node(node: Node) -> bool: return len(node.input_tensors) == 0 diff --git a/tests/test_preprocess_keras_model.py b/tests/test_preprocess_keras_model.py index 61affb5c..bed2c226 100644 --- a/tests/test_preprocess_keras_model.py +++ b/tests/test_preprocess_keras_model.py @@ -1,6 +1,7 @@ import keras.ops as K import numpy as np import pytest +from keras.activations import softmax from keras.layers import Activation, Conv2D, Dense, Input, PReLU from keras.models import Model, Sequential from numpy.testing import assert_almost_equal @@ -82,3 +83,20 @@ def test_preprocess( output_np_ref = K.convert_to_numpy(model(inputs_np)) output_np_new = K.convert_to_numpy(converted_model(inputs_np)) assert_almost_equal(output_np_new, output_np_ref, decimal=4) + + +def test_preprocess_keras_model_with_softmax(): + input_tensor = Input((1,)) + + output_tensor = Dense(3, activation="relu")(input_tensor) + + output_tensor1 = Dense(2)(output_tensor) + output_tensor2 = Activation(activation="softmax")(Dense(3)(output_tensor)) + output_tensor3 = Dense(4, activation="softmax")(output_tensor) + + model = Model(input_tensor, [output_tensor1, output_tensor2, output_tensor3]) + + new_model = preprocess_keras_model(model=model, rm_last_softmax=True) + + assert len(new_model.layers) == 6 + assert not any([(isinstance(layer, Activation) and layer.activation == softmax) for layer in new_model.layers]) From 4bc82e3385360123d0698f54b67625171c91aafe Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 18 Mar 2024 21:32:15 +0100 Subject: [PATCH 096/101] Implement softsign activation decomon layer --- src/decomon/layers/activations/activation.py | 56 +++++- src/decomon/layers/activations/utils.py | 193 +++++++++++++++++++ tests/conftest.py | 2 +- tests/test_unary_layers.py | 35 +++- 4 files changed, 275 insertions(+), 11 deletions(-) diff --git a/src/decomon/layers/activations/activation.py b/src/decomon/layers/activations/activation.py index a65c25b9..78a7262e 100644 --- a/src/decomon/layers/activations/activation.py +++ b/src/decomon/layers/activations/activation.py @@ -2,12 +2,18 @@ from typing import Any, Optional import keras +import keras.ops as K from keras import Layer -from keras.activations import linear, relu +from keras.activations import linear, relu, softsign +from keras.config import epsilon from keras.layers import Activation from decomon.constants import Propagation, Slope -from decomon.layers.activations.utils import get_linear_hull_relu +from decomon.layers.activations.utils import ( + get_linear_hull_relu, + get_linear_hull_s_shape, + softsign_prime, +) from decomon.layers.layer import DecomonLayer from decomon.perturbation_domain import PerturbationDomain from decomon.types import Tensor @@ -182,9 +188,55 @@ def get_affine_bounds(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, Tenso return w_l, b_l, w_u, b_u +class DecomonSoftSign(DecomonBaseActivation): + diagonal = True + + def get_affine_bounds(self, lower: Tensor, upper: Tensor) -> tuple[Tensor, Tensor, Tensor, Tensor]: + func = softsign + func_prime = softsign_prime + + # chord + w_chord = (func(upper) - func(lower)) / K.maximum(K.cast(epsilon(), dtype=upper.dtype), upper - lower) + b_chord = func(lower) - w_chord * lower + + # tangent at upper + w_tangent_upper = func_prime(upper) + b_tangent_upper = func(upper) - w_tangent_upper * upper + + # tangent at lower + w_tangent_lower = func_prime(lower) + b_tangent_lower = func(lower) - w_tangent_lower * lower + + # compare slopes to choose between chord and tangent + w_l = K.where( + w_chord <= w_tangent_lower, + w_chord, + w_tangent_lower, + ) + b_l = K.where( + w_chord <= w_tangent_lower, + b_chord, + b_tangent_lower, + ) + + w_u = K.where( + w_chord <= w_tangent_upper, + w_chord, + w_tangent_upper, + ) + b_u = K.where( + w_chord <= w_tangent_upper, + b_chord, + b_tangent_upper, + ) + + return w_l, b_l, w_u, b_u + + MAPPING_KERAS_ACTIVATION_TO_DECOMON_ACTIVATION: dict[Callable[[Tensor], Tensor], type[DecomonBaseActivation]] = { linear: DecomonLinear, relu: DecomonReLU, + softsign: DecomonSoftSign, } diff --git a/src/decomon/layers/activations/utils.py b/src/decomon/layers/activations/utils.py index 747b7e5f..0621813f 100644 --- a/src/decomon/layers/activations/utils.py +++ b/src/decomon/layers/activations/utils.py @@ -1,6 +1,7 @@ from collections.abc import Callable from typing import Any, Union +import numpy as np from keras import ops as K from keras.src.backend import epsilon @@ -192,3 +193,195 @@ def softsign_prime(x: Tensor) -> Tensor: """ return K.cast(1.0, dtype=x.dtype) / K.power(K.cast(1.0, dtype=x.dtype) + K.abs(x), K.cast(2, dtype=x.dtype)) + + +def get_linear_hull_s_shape( + lower: Tensor, + upper: Tensor, + func: TensorFunction = K.sigmoid, + f_prime: TensorFunction = sigmoid_prime, +) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Computing the linear hull of shape functions given the pre activation neurons + + Args: + lower: lower bound on keras input + upper: upper bound on keras input + func: the function (sigmoid, tanh, softsign...) + f_prime: the derivative of the function (sigmoid_prime...) + perturbation_domain: the type of convex input domain + mode: type of Forward propagation (ibp, affine, or hybrid) + + Returns: + w_l, b_l, w_u, b_u: affine bounds on activation layer + """ + + dtype = lower.dtype + + z_value = K.cast(0.0, dtype=dtype) + o_value = K.cast(1.0, dtype=dtype) + t_value = K.cast(2.0, dtype=dtype) + + # flatten + shape = list(lower.shape[1:]) + upper_flat = K.reshape(upper, (-1, int(np.prod(shape)))) # (None, n) + lower_flat = K.reshape(lower, (-1, int(np.prod(shape)))) # (None, n) + + # upper bound + # derivative + s_u_prime = f_prime(upper_flat) # (None, n) + s_l_prime = f_prime(lower_flat) # (None, n) + s_u = func(upper_flat) # (None, n) + s_l = func(lower_flat) # (None, n) + + # case 0: + coeff = (s_u - s_l) / K.maximum(K.cast(epsilon(), dtype=dtype), upper_flat - lower_flat) + alpha_u_0 = K.where( + K.greater_equal(s_u_prime, coeff), o_value + z_value * upper_flat, z_value * upper_flat + ) # (None, n) + alpha_u_1 = (o_value - alpha_u_0) * ((K.sign(lower_flat) + o_value) / t_value) + + w_u_0 = coeff + b_u_0 = -w_u_0 * lower_flat + s_l + + w_u_1 = z_value * upper_flat + b_u_1 = s_u + + w_u_2, b_u_2 = get_t_upper(upper_flat, lower_flat, s_l, func=func, f_prime=f_prime) + + w_u_out = K.reshape(alpha_u_0 * w_u_0 + alpha_u_1 * w_u_1 + (o_value - alpha_u_0 - alpha_u_1) * w_u_2, [-1] + shape) + b_u_out = K.reshape(alpha_u_0 * b_u_0 + alpha_u_1 * b_u_1 + (o_value - alpha_u_0 - alpha_u_1) * b_u_2, [-1] + shape) + + # linear hull + # case 0: + alpha_l_0 = K.where( + K.greater_equal(s_l_prime, coeff), o_value + z_value * lower_flat, z_value * lower_flat + ) # (None, n) + alpha_l_1 = (o_value - alpha_l_0) * ((K.sign(-upper_flat) + o_value) / t_value) + + w_l_0 = coeff + b_l_0 = -w_l_0 * upper_flat + s_u + + w_l_1 = z_value * upper_flat + b_l_1 = s_l + + w_l_2, b_l_2 = get_t_lower(upper_flat, lower_flat, s_u, func=func, f_prime=f_prime) + + w_l_out = K.reshape(alpha_l_0 * w_l_0 + alpha_l_1 * w_l_1 + (o_value - alpha_l_0 - alpha_l_1) * w_l_2, [-1] + shape) + b_l_out = K.reshape(alpha_l_0 * b_l_0 + alpha_l_1 * b_l_1 + (o_value - alpha_l_0 - alpha_l_1) * b_l_2, [-1] + shape) + + return w_l_out, b_l_out, w_u_out, b_u_out + + +def get_t_upper( + u_c_flat: Tensor, + l_c_flat: Tensor, + s_l: Tensor, + func: TensorFunction = K.sigmoid, + f_prime: TensorFunction = sigmoid_prime, +) -> tuple[Tensor, Tensor]: + """linear interpolation between lower and upper bounds on the function func to have a symbolic approximation of the best + coefficient for the affine upper bound + + Args: + u_c_flat: flatten tensor of constant upper bound + l_c_flat: flatten tensor of constant lower bound + s_l: lowest value of the function func on the domain + func: the function (sigmoid, tanh, softsign) + f_prime: the derivative of the function + + Returns: + the upper affine bounds in this subcase + """ + + o_value = K.cast(1.0, dtype=u_c_flat.dtype) + z_value = K.cast(0.0, dtype=u_c_flat.dtype) + + # step1: find t + u_c_reshaped = K.expand_dims(u_c_flat, -1) # (None, n , 1) + l_c_reshaped = K.expand_dims(l_c_flat, -1) # (None, n, 1) + t = K.cast(np.linspace(0, 1, 100)[None, None, :], dtype=u_c_flat.dtype) * u_c_reshaped # (None, n , 100) + + s_p_t = f_prime(t) # (None, n, 100) + s_t = func(t) # (None, n, 100) + + score = K.abs(s_p_t - (s_t - K.expand_dims(s_l, -1)) / (t - l_c_reshaped)) # (None, n, 100) + index = K.argmin(score, -1) # (None, n) + threshold = K.min(score, -1) # (None, n) + + index_t = K.cast( + K.where(K.greater(threshold, z_value * threshold), index, K.clip(index - 1, 0, 100)), dtype=u_c_flat.dtype + ) # (None, n) + t_value = K.sum( + K.where( + K.equal( + o_value * K.cast(np.arange(0, 100)[None, None, :], dtype=u_c_flat.dtype) + z_value * u_c_reshaped, + K.expand_dims(index_t, -1) + z_value * u_c_reshaped, + ), + t, + z_value * t, + ), + -1, + ) # (None, n) + + s_t = func(t_value) # (None, n) + w_u = (s_t - s_l) / K.maximum(K.cast(epsilon(), dtype=u_c_flat.dtype), t_value - l_c_flat) # (None, n) + b_u = -w_u * l_c_flat + s_l # + func(l_c_flat) + + return w_u, b_u + + +def get_t_lower( + u_c_flat: Tensor, + l_c_flat: Tensor, + s_u: Tensor, + func: TensorFunction = K.sigmoid, + f_prime: TensorFunction = sigmoid_prime, +) -> tuple[Tensor, Tensor]: + """linear interpolation between lower and upper bounds on the function func to have a symbolic approximation of the best + coefficient for the affine lower bound + + Args: + u_c_flat: flatten tensor of constant upper bound + l_c_flat: flatten tensor of constant lower bound + s_u: highest value of the function func on the domain + func: the function (sigmoid, tanh, softsign) + f_prime: the derivative of the function + + Returns: + the lower affine bounds in this subcase + """ + z_value = K.cast(0.0, dtype=u_c_flat.dtype) + o_value = K.cast(1.0, dtype=u_c_flat.dtype) + + # step1: find t + u_c_reshaped = K.expand_dims(u_c_flat, -1) # (None, n , 1) + l_c_reshaped = K.expand_dims(l_c_flat, -1) # (None, n, 1) + t = K.cast(np.linspace(0, 1.0, 100)[None, None, :], dtype=u_c_flat.dtype) * l_c_reshaped # (None, n , 100) + + s_p_t = f_prime(t) # (None, n, 100) + s_t = func(t) # (None, n, 100) + + score = K.abs(s_p_t - (K.expand_dims(s_u, -1) - s_t) / (u_c_reshaped - t)) # (None, n, 100) + index = K.argmin(score, -1) # (None, n) + + threshold = K.min(score, -1) + index_t = K.cast( + K.where(K.greater(threshold, z_value * threshold), index, K.clip(index + 1, 0, 100)), dtype=u_c_flat.dtype + ) # (None, n) + t_value = K.sum( + K.where( + K.equal( + o_value * K.cast(np.arange(0, 100)[None, None, :], dtype=u_c_flat.dtype) + z_value * u_c_reshaped, + K.expand_dims(index_t, -1) + z_value * u_c_reshaped, + ), + t, + z_value * t, + ), + -1, + ) + + s_t = func(t_value) # (None, n) + w_l = (s_u - s_t) / K.maximum(K.cast(epsilon(), dtype=u_c_flat.dtype), u_c_flat - t_value) # (None, n) + b_l = -w_l * u_c_flat + s_u # func(u_c_flat) + + return w_l, b_l diff --git a/tests/conftest.py b/tests/conftest.py index b69c2879..82e2574b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,7 +57,7 @@ use_bias = param_fixture("use_bias", [True, False]) randomize = param_fixture("randomize", [True, False]) padding = param_fixture("padding", ["same", "valid"]) -activation = param_fixture("activation", [None, "relu"]) +activation = param_fixture("activation", [None, "relu", "softsign"]) data_format = param_fixture("data_format", ["channels_last", "channels_first"]) method = param_fixture("method", [m.value for m in ConvertMethod]) input_shape = param_fixture("input_shape", [(1,), (3,), (5, 6, 2)], ids=["0d", "1d", "multid"]) diff --git a/tests/test_unary_layers.py b/tests/test_unary_layers.py index adcf56ca..ecbedce7 100644 --- a/tests/test_unary_layers.py +++ b/tests/test_unary_layers.py @@ -1,7 +1,7 @@ import keras.ops as K import numpy as np from keras.layers import Activation, Dense -from pytest_cases import fixture, parametrize +from pytest_cases import fixture, fixture_union, parametrize, unpack_fixture from decomon.keras_utils import batch_multid_dot from decomon.layers import DecomonActivation, DecomonDense @@ -9,25 +9,44 @@ @fixture -def keras_dense_kwargs(use_bias): +def dense_keras_kwargs(use_bias): return dict(units=7, use_bias=use_bias) +def _activation_kwargs(activation, slope=None): + keras_kwargs = dict(activation=activation) + if activation == "relu": + decomon_kwargs = dict(slope=slope) + else: + decomon_kwargs = {} + return keras_kwargs, decomon_kwargs + + @fixture -def keras_activation_kwargs(activation): - return dict(activation=activation) +@parametrize("activation", [None, "softsign"]) +def non_relu_activation_kwargs(activation): + return _activation_kwargs(activation) @fixture -def decomon_activation_kwargs(slope): - return dict(slope=slope) +def relu_activation_kwargs(slope): + return _activation_kwargs(activation="relu", slope=slope) + + +activation_kwargs = fixture_union( + "activation_kwargs", + [non_relu_activation_kwargs, relu_activation_kwargs], +) +activation_keras_kwargs, activation_decomon_kwargs = unpack_fixture( + "activation_keras_kwargs, activation_decomon_kwargs", activation_kwargs +) @parametrize( "decomon_layer_class, decomon_layer_kwargs, keras_layer_class, keras_layer_kwargs, is_actually_linear", [ - (DecomonDense, {}, Dense, keras_dense_kwargs, None), - (DecomonActivation, decomon_activation_kwargs, Activation, keras_activation_kwargs, None), + (DecomonDense, {}, Dense, dense_keras_kwargs, None), + (DecomonActivation, activation_decomon_kwargs, Activation, activation_keras_kwargs, None), ], ) def test_decomon_unary_layer( From ae63d93794fbb8a525a80ee2ab6ff21efc313395 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Mon, 18 Mar 2024 23:04:39 +0100 Subject: [PATCH 097/101] Update and add get_config() methods to decomon objects --- src/decomon/layers/crown.py | 9 +++++++++ src/decomon/layers/fuse.py | 15 +++++++++++++++ src/decomon/layers/input.py | 31 +++++++++++++++++++++++++++++++ src/decomon/layers/layer.py | 6 ++++-- src/decomon/layers/oracle.py | 14 ++++++++++++++ src/decomon/layers/output.py | 15 +++++++++++++++ src/decomon/models/models.py | 5 +---- 7 files changed, 89 insertions(+), 6 deletions(-) diff --git a/src/decomon/layers/crown.py b/src/decomon/layers/crown.py index 0210b077..142023bb 100644 --- a/src/decomon/layers/crown.py +++ b/src/decomon/layers/crown.py @@ -38,6 +38,15 @@ def __init__( super().__init__(**kwargs) self.model_output_shape = model_output_shape + def get_config(self) -> dict[str, Any]: + config = super().get_config() + config.update( + { + "model_output_shape": self.model_output_shape, + } + ) + return config + def call(self, inputs: list[list[BackendTensor]]) -> list[BackendTensor]: """Reduce the list of crown bounds to a single one by summation. diff --git a/src/decomon/layers/fuse.py b/src/decomon/layers/fuse.py index 6e5f8351..f9447f09 100644 --- a/src/decomon/layers/fuse.py +++ b/src/decomon/layers/fuse.py @@ -99,6 +99,21 @@ def __init__( for m2_input_shape in m_1_output_shapes ] + def get_config(self) -> dict[str, Any]: + config = super().get_config() + config.update( + { + "ibp_1": self.ibp_1, + "affine_1": self.affine_1, + "ibp_2": self.ibp_2, + "affine_2": self.affine_2, + "m1_input_shape": self.m1_input_shape, + "m_1_output_shapes": self.m_1_output_shapes, + "from_linear_2": self.from_linear_2, + } + ) + return config + def build(self, input_shape: tuple[list[tuple[Optional[int], ...]], list[tuple[Optional[int], ...]]]) -> None: input_shape_1, input_shape_2 = input_shape diff --git a/src/decomon/layers/input.py b/src/decomon/layers/input.py index 560b0eae..607f8fc2 100644 --- a/src/decomon/layers/input.py +++ b/src/decomon/layers/input.py @@ -6,6 +6,7 @@ import keras import keras.ops as K from keras.layers import Layer +from keras.utils import serialize_keras_object from decomon.constants import Propagation from decomon.layers.inputs_outputs_specs import InputsOutputsSpec @@ -44,6 +45,17 @@ def __init__( layer_input_shape=tuple(), ) + def get_config(self) -> dict[str, Any]: + config = super().get_config() + config.update( + { + "ibp": self.ibp, + "affine": self.affine, + "perturbation_domain": serialize_keras_object(self.perturbation_domain), + } + ) + return config + def call(self, inputs: BackendTensor) -> list[BackendTensor]: """Generate ibp and affine bounds to propagate by the first forward layer. @@ -124,6 +136,15 @@ def __init__( self.perturbation_domain = perturbation_domain + def get_config(self) -> dict[str, Any]: + config = super().get_config() + config.update( + { + "perturbation_domain": serialize_keras_object(self.perturbation_domain), + } + ) + return config + def call(self, inputs: BackendTensor) -> list[BackendTensor]: """Generate ibp and affine bounds to propagate by the first forward layer. @@ -197,6 +218,16 @@ def __init__( self.nb_model_outputs = len(model_output_shapes) self.from_linear = from_linear + def get_config(self) -> dict[str, Any]: + config = super().get_config() + config.update( + { + "model_output_shapes": self.model_output_shapes, + "from_linear": self.from_linear, + } + ) + return config + def build(self, input_shape: list[tuple[Optional[int], ...]]) -> None: # list of tensors if len(input_shape) == 1: diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index 9a362815..a37a504b 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -4,6 +4,7 @@ import keras import keras.ops as K from keras.layers import Layer, Wrapper +from keras.utils import serialize_keras_object from decomon.constants import Propagation from decomon.layers.fuse import ( @@ -184,9 +185,10 @@ def get_config(self) -> dict[str, Any]: { "ibp": self.ibp, "affine": self.affine, - "perturbation_domain": self.perturbation_domain, + "perturbation_domain": serialize_keras_object(self.perturbation_domain), "propagation": self.propagation, - "model_output_shape_length": self.model_output_shape_length, + "model_input_shape": self.model_input_shape, + "model_output_shape": self.model_output_shape, } ) return config diff --git a/src/decomon/layers/oracle.py b/src/decomon/layers/oracle.py index 54d4e0e4..773048ca 100644 --- a/src/decomon/layers/oracle.py +++ b/src/decomon/layers/oracle.py @@ -4,6 +4,7 @@ from typing import Any, Optional, Union, overload from keras.layers import Layer +from keras.utils import serialize_keras_object from decomon.layers.inputs_outputs_specs import InputsOutputsSpec from decomon.perturbation_domain import PerturbationDomain @@ -68,6 +69,19 @@ def __init__( is_merging_layer=is_merging_layer, ) + def get_config(self) -> dict[str, Any]: + config = super().get_config() + config.update( + { + "ibp": self.ibp, + "affine": self.affine, + "perturbation_domain": serialize_keras_object(self.perturbation_domain), + "layer_input_shape": self.layer_input_shape, + "is_merging_layer": self.is_merging_layer, + } + ) + return config + def call(self, inputs: list[BackendTensor]) -> Union[list[BackendTensor], list[list[BackendTensor]]]: """Deduce ibp and affine bounds to propagate by the first forward layer. diff --git a/src/decomon/layers/output.py b/src/decomon/layers/output.py index c6d622ec..398f5f07 100644 --- a/src/decomon/layers/output.py +++ b/src/decomon/layers/output.py @@ -5,6 +5,7 @@ import keras.ops as K from keras.layers import Layer +from keras.utils import serialize_keras_object from decomon.layers.inputs_outputs_specs import InputsOutputsSpec from decomon.layers.oracle import get_forward_oracle @@ -53,6 +54,20 @@ def __init__( model_output_shapes=model_output_shapes, ) + def get_config(self) -> dict[str, Any]: + config = super().get_config() + config.update( + { + "ibp_from": self.ibp_from, + "affine_from": self.affine_from, + "ibp_to": self.ibp_to, + "affine_to": self.affine_to, + "model_output_shapes": self.model_output_shapes, + "perturbation_domain": serialize_keras_object(self.perturbation_domain), + } + ) + return config + def needs_perturbation_domain_inputs(self) -> bool: return self.inputs_outputs_spec.needs_perturbation_domain_inputs() diff --git a/src/decomon/models/models.py b/src/decomon/models/models.py index 91681e8c..1c61c536 100644 --- a/src/decomon/models/models.py +++ b/src/decomon/models/models.py @@ -79,13 +79,10 @@ def get_config(self) -> dict[str, Any]: dict( name=self.name, perturbation_domain=serialize_keras_object(self.perturbation_domain), - dc_decomp=self.dc_decomp, method=self.method, ibp=self.ibp, affine=self.affine, - finetune=self.finetune, - shared=self.shared, - backward_bounds=self.backward_bounds, + model=serialize_keras_object(self.model), ) ) return config From e06e4d9118bf3cfda5068287fd7f55f84ecaa0cc Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 19 Mar 2024 16:14:25 +0100 Subject: [PATCH 098/101] Reduce size of toy network to pass on github runners Remove xfail for add + crown + multid --- tests/conftest.py | 22 +++++++++++----------- tests/test_clone.py | 8 -------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 82e2574b..eae8aa17 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1067,10 +1067,10 @@ def toy_network_tutorial( dtype = keras_config.floatx() layers = [] layers.append(Input(input_shape, dtype=dtype)) - layers.append(Dense(100, dtype=dtype)) + layers.append(Dense(10, dtype=dtype)) if activation is not None: layers.append(Activation(activation, dtype=dtype)) - layers.append(Dense(100, dtype=dtype)) + layers.append(Dense(10, dtype=dtype)) layers.append(Dense(1, activation="linear", dtype=dtype)) model = Sequential(layers) return model @@ -1101,14 +1101,14 @@ def toy_network_submodel( ) -> Model: if dtype is None: dtype = keras_config.floatx() - submodel_input_shape = input_shape[:-1] + (100,) + submodel_input_shape = input_shape[:-1] + (10,) layers = [] layers.append(Input(input_shape, dtype=dtype)) - layers.append(Dense(100, dtype=dtype)) + layers.append(Dense(10, dtype=dtype)) if activation is not None: layers.append(Activation(activation, dtype=dtype)) layers.append(Helpers.toy_network_tutorial(submodel_input_shape, dtype=dtype, activation=activation)) - layers.append(Dense(100, dtype=dtype)) + layers.append(Dense(10, dtype=dtype)) layers.append(Dense(1, activation="linear", dtype=dtype)) model = Sequential(layers) return model @@ -1120,11 +1120,11 @@ def toy_network_add( if dtype is None: dtype = keras_config.floatx() input_tensor = Input(input_shape, dtype=dtype) - output = Dense(100, dtype=dtype)(input_tensor) + output = Dense(10, dtype=dtype)(input_tensor) if activation is not None: output = Activation(activation, dtype=dtype)(output) output = Add()([output, output]) - output = Dense(100, dtype=dtype)(output) + output = Dense(10, dtype=dtype)(output) if activation is not None: output = Activation(activation, dtype=dtype)(output) model = Model(inputs=input_tensor, outputs=output) @@ -1137,11 +1137,11 @@ def toy_network_add_monolayer( if dtype is None: dtype = keras_config.floatx() input_tensor = Input(input_shape, dtype=dtype) - output = Dense(100, dtype=dtype)(input_tensor) + output = Dense(10, dtype=dtype)(input_tensor) if activation is not None: output = Activation(activation, dtype=dtype)(output) output = Add()([output]) - output = Dense(100, dtype=dtype)(output) + output = Dense(10, dtype=dtype)(output) if activation is not None: output = Activation(activation, dtype=dtype)(output) model = Model(inputs=input_tensor, outputs=output) @@ -1153,8 +1153,8 @@ def toy_network_tutorial_with_embedded_activation(input_shape: tuple[int, ...] = dtype = keras_config.floatx() layers = [] layers.append(Input(input_shape, dtype=dtype)) - layers.append(Dense(100, activation="relu", dtype=dtype)) - layers.append(Dense(100, dtype=dtype)) + layers.append(Dense(10, activation="relu", dtype=dtype)) + layers.append(Dense(10, dtype=dtype)) layers.append(Dense(1, activation="linear", dtype=dtype)) model = Sequential(layers) return model diff --git a/tests/test_clone.py b/tests/test_clone.py index 587853e4..55943990 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -57,14 +57,6 @@ def test_clone( if toy_model_name == "cnn" and len(input_shape) == 1: pytest.skip("cnn not possible on 0d or 1d input.") - # xfail add model with standard multid input for now (memory issues to be fixed) - if ( - model_decomon_input_metadata["name"] == "standard-multid" - and toy_model_name == "add" - and method.lower().startswith("crown") - ): - pytest.xfail("crown on 'add' toy model crashed sometimes with standard-multid, to be investigated.") - slope = Slope.Z_SLOPE decimal = 4 From e59b7ddfa7aa5a51d87d261f6c58f4ea7cc57816 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 19 Mar 2024 16:18:09 +0100 Subject: [PATCH 099/101] Prepare test_clone for cnn model (using data_format metadata) --- tests/conftest.py | 6 ++++-- tests/test_clone.py | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index eae8aa17..99b82805 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1293,7 +1293,9 @@ def toy_struct_v2( return Model(x, y) @staticmethod - def toy_struct_cnn(input_shape: tuple[int, ...] = (6, 6, 2), dtype: Optional[str] = None): + def toy_struct_cnn( + input_shape: tuple[int, ...] = (6, 6, 2), dtype: Optional[str] = None, data_format="channels_last" + ): if dtype is None: dtype = keras_config.floatx() layers = [ @@ -1302,7 +1304,7 @@ def toy_struct_cnn(input_shape: tuple[int, ...] = (6, 6, 2), dtype: Optional[str 10, kernel_size=(3, 3), activation="relu", - data_format="channels_last", + data_format=data_format, dtype=dtype, ), Flatten(dtype=dtype), diff --git a/tests/test_clone.py b/tests/test_clone.py index 55943990..b46415c5 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -61,7 +61,11 @@ def test_clone( decimal = 4 # keras model to convert - keras_model = toy_model_fn(input_shape=input_shape) + if toy_model_name == "cnn": + kwargs_toy_model = dict(data_format=model_decomon_input_metadata["data_format"]) + else: + kwargs_toy_model = {} + keras_model = toy_model_fn(input_shape=input_shape, **kwargs_toy_model) # conversion decomon_model = clone(model=keras_model, slope=slope, perturbation_domain=perturbation_domain, method=method) From 537148d0716537d6c30c9bdace30864f16d1803f Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 19 Mar 2024 16:32:44 +0100 Subject: [PATCH 100/101] Make mypy ignore model_visualization.py --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b7e8ef89..5e4329c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,3 +79,7 @@ module = [ "torch.*" ] ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "decomon.visualization.model_visualization" +ignore_errors = true From 1c857776ac42986c9e9f6645c2714614c915e482 Mon Sep 17 00:00:00 2001 From: Nolwen Date: Tue, 19 Mar 2024 17:24:55 +0100 Subject: [PATCH 101/101] Fix mypy errors --- src/decomon/layers/activations/activation.py | 2 +- src/decomon/layers/crown.py | 2 +- src/decomon/layers/fuse.py | 28 ++++++++++------ src/decomon/layers/input.py | 12 +++---- src/decomon/layers/inputs_outputs_specs.py | 6 ++-- src/decomon/layers/layer.py | 13 +++++--- src/decomon/layers/merging/base_merge.py | 7 ++-- src/decomon/layers/oracle.py | 9 ++++-- src/decomon/layers/output.py | 10 ++++-- src/decomon/models/backward_cloning.py | 7 ++-- src/decomon/models/convert.py | 12 +++---- src/decomon/perturbation_domain.py | 34 +++++++++++--------- 12 files changed, 84 insertions(+), 58 deletions(-) diff --git a/src/decomon/layers/activations/activation.py b/src/decomon/layers/activations/activation.py index 78a7262e..6390d5dc 100644 --- a/src/decomon/layers/activations/activation.py +++ b/src/decomon/layers/activations/activation.py @@ -176,7 +176,7 @@ def compute_output_shape( ) = self.inputs_outputs_spec.split_input_shape(input_shape=input_shape) return self.inputs_outputs_spec.flatten_outputs_shape( affine_bounds_propagated_shape=affine_bounds_to_propagate_shape, - constant_bounds_propagated_shape=constant_oracle_bounds_shape, + constant_bounds_propagated_shape=constant_oracle_bounds_shape, # type: ignore ) diff --git a/src/decomon/layers/crown.py b/src/decomon/layers/crown.py index 142023bb..52e5ad76 100644 --- a/src/decomon/layers/crown.py +++ b/src/decomon/layers/crown.py @@ -89,7 +89,7 @@ def call(self, inputs: list[list[BackendTensor]]) -> list[BackendTensor]: b_l = add_tensors(b_l, b_l_i, missing_batchsize=missing_batchsize) b_u = add_tensors(b_u, b_u_i, missing_batchsize=missing_batchsize) - affine_bounds = w_l, b_l, w_u, b_u + affine_bounds = [w_l, b_l, w_u, b_u] return affine_bounds def build(self, input_shape: list[list[tuple[Optional[int], ...]]]) -> None: diff --git a/src/decomon/layers/fuse.py b/src/decomon/layers/fuse.py index f9447f09..5886133c 100644 --- a/src/decomon/layers/fuse.py +++ b/src/decomon/layers/fuse.py @@ -53,7 +53,7 @@ def __init__( m1_input_shape: tuple[int, ...], m_1_output_shapes: list[tuple[int, ...]], from_linear_2: list[bool], - **kwargs, + **kwargs: Any, ): """ @@ -138,7 +138,7 @@ def _is_from_linear_m1_ith_affine_bounds(self, affine_bounds: list[Tensor], i: i return len(affine_bounds) == 0 or affine_bounds[1].shape == self.m_1_output_shapes[i] def _is_from_linear_m1_ith_affine_bounds_shape( - self, affine_bounds_shape: list[tuple[Optional[int]]], i: int + self, affine_bounds_shape: list[tuple[Optional[int], ...]], i: int ) -> bool: return len(affine_bounds_shape) == 0 or affine_bounds_shape[1] == self.m_1_output_shapes[i] @@ -226,14 +226,16 @@ def compute_output_shape( ) -> list[tuple[Optional[int], ...]]: bounds_1_shape, bounds_2_shape = input_shape - bounds_fused_shape: list[tuple[int, ...]] = [] + bounds_fused_shape: list[tuple[Optional[int], ...]] = [] for i in range(self.nb_outputs_first_model): bounds_1_i_shape = bounds_1_shape[ i * self.inputs_outputs_spec_1.nb_output_tensors : (i + 1) * self.inputs_outputs_spec_1.nb_output_tensors ] - affine_bounds_1_shape, constant_bounds_1_shape = self.inputs_outputs_spec_1.split_output_shape( + affine_bounds_1_shape: list[tuple[Optional[int], ...]] + constant_bounds_1_shape: list[tuple[Optional[int], ...]] + affine_bounds_1_shape, constant_bounds_1_shape = self.inputs_outputs_spec_1.split_output_shape( # type: ignore bounds_1_i_shape ) @@ -242,9 +244,14 @@ def compute_output_shape( * self.inputs_outputs_spec_2[0].nb_output_tensors : (i + 1) * self.inputs_outputs_spec_2[0].nb_output_tensors ] - affine_bounds_2_shape, constant_bounds_2_shape = self.inputs_outputs_spec_2[0].split_output_shape( - bounds_2_i_shape - ) + affine_bounds_2_shape: list[tuple[Optional[int], ...]] + constant_bounds_2_shape: list[tuple[Optional[int], ...]] + ( + affine_bounds_2_shape, + constant_bounds_2_shape, + ) = self.inputs_outputs_spec_2[ # type:ignore + 0 + ].split_output_shape(bounds_2_i_shape) # constant bounds if self.ibp_2: @@ -264,10 +271,11 @@ def compute_output_shape( # affine bounds if self.affine_1 and self.affine_2: _, b2_shape, _, _ = affine_bounds_2_shape + model_2_output_shape_wo_batchisze: tuple[int, ...] if self.from_linear_2[i]: - model_2_output_shape_wo_batchisze = b2_shape + model_2_output_shape_wo_batchisze = b2_shape # type: ignore else: - model_2_output_shape_wo_batchisze = b2_shape[1:] + model_2_output_shape_wo_batchisze = b2_shape[1:] # type: ignore diagonal = self.inputs_outputs_spec_1.is_diagonal_bounds_shape( affine_bounds_1_shape @@ -281,6 +289,8 @@ def compute_output_shape( self._is_from_linear_m1_ith_affine_bounds_shape(affine_bounds_shape=affine_bounds_1_shape, i=i) and self.from_linear_2[i] ) + w_fused_shape: tuple[Optional[int], ...] + b_fused_shape: tuple[Optional[int], ...] if from_linear_layer: w_fused_shape = w_fused_shape_wo_batchsize b_fused_shape = model_2_output_shape_wo_batchisze diff --git a/src/decomon/layers/input.py b/src/decomon/layers/input.py index 607f8fc2..a2e79921 100644 --- a/src/decomon/layers/input.py +++ b/src/decomon/layers/input.py @@ -95,7 +95,7 @@ def compute_output_shape( self, input_shape: tuple[Optional[int], ...], ) -> list[tuple[Optional[int], ...]]: - perturbation_domain_input_shape_wo_batchsize = input_shape[1:] + perturbation_domain_input_shape_wo_batchsize: tuple[int, ...] = input_shape[1:] # type: ignore keras_input_shape_wo_batchsize = self.perturbation_domain.get_keras_input_shape_wo_batchsize( x_shape=perturbation_domain_input_shape_wo_batchsize ) @@ -112,7 +112,7 @@ def compute_output_shape( else: constant_bounds_shape = [] return self.inputs_outputs_spec.flatten_inputs_shape( - affine_bounds_to_propagate_shape=affine_bounds_shape, + affine_bounds_to_propagate_shape=affine_bounds_shape, # type: ignore constant_oracle_bounds_shape=constant_bounds_shape, perturbation_domain_inputs_shape=[], ) @@ -165,7 +165,7 @@ def compute_output_shape( self, input_shape: tuple[Optional[int], ...], ) -> list[tuple[Optional[int], ...]]: - perturbation_domain_input_shape_wo_batchsize = input_shape[1:] + perturbation_domain_input_shape_wo_batchsize: tuple[int, ...] = input_shape[1:] # type: ignore keras_input_shape_wo_batchsize = self.perturbation_domain.get_keras_input_shape_wo_batchsize( x_shape=perturbation_domain_input_shape_wo_batchsize ) @@ -312,11 +312,13 @@ def compute_output_shape( else: w_shape_wo_batchsize = w_shape[1:] is_diag = w_shape_wo_batchsize == model_output_shape + m2_output_shape: tuple[Optional[int], ...] if is_diag: m2_output_shape = model_output_shape else: m2_output_shape = w_shape_wo_batchsize[len(model_output_shape) :] b_shape_wo_batchsize = m2_output_shape + b_shape: tuple[Optional[int], ...] if from_linear: b_shape = b_shape_wo_batchsize else: @@ -332,10 +334,6 @@ def compute_output_shape( return input_shape -def _is_keras_tensor_shape(shape): - return len(shape) > 0 and (shape[0] is None or isinstance(shape[0], int)) - - def flatten_backward_bounds( backward_bounds: Union[keras.KerasTensor, list[keras.KerasTensor], list[list[keras.KerasTensor]]] ) -> list[keras.KerasTensor]: diff --git a/src/decomon/layers/inputs_outputs_specs.py b/src/decomon/layers/inputs_outputs_specs.py index 0d215a0f..3c6cde63 100644 --- a/src/decomon/layers/inputs_outputs_specs.py +++ b/src/decomon/layers/inputs_outputs_specs.py @@ -496,7 +496,7 @@ def has_multiple_bounds_inputs(self) -> bool: return self.propagation == Propagation.FORWARD and self.is_merging_layer @overload - def extract_shapes_from_affine_bounds( + def extract_shapes_from_affine_bounds( # type:ignore self, affine_bounds: list[Tensor], i: int = -1 ) -> list[tuple[Optional[int], ...]]: ... @@ -575,14 +575,14 @@ def is_wo_batch_bounds_shape( b_shape = affine_bounds_shape[1] if self.propagation == Propagation.FORWARD: if i > -1: - return len(b_shape) == len(self.layer_input_shape[i]) + return len(b_shape) == len(self.layer_input_shape[i]) # type: ignore else: return len(b_shape) == len(self.layer_input_shape) else: return len(b_shape) == len(self.model_output_shape) @overload - def is_wo_batch_bounds_by_keras_input( + def is_wo_batch_bounds_by_keras_input( # type: ignore self, affine_bounds: list[Tensor], ) -> bool: diff --git a/src/decomon/layers/layer.py b/src/decomon/layers/layer.py index a37a504b..cbfcfc9b 100644 --- a/src/decomon/layers/layer.py +++ b/src/decomon/layers/layer.py @@ -1,5 +1,5 @@ from inspect import Parameter, signature -from typing import Any, Optional +from typing import Any, Optional, Union import keras import keras.ops as K @@ -169,7 +169,7 @@ def is_merging_layer(self) -> bool: @property def layer_input_shape(self) -> tuple[int, ...]: - return self.inputs_outputs_spec.layer_input_shape + return self.inputs_outputs_spec.layer_input_shape # type: ignore @property def model_input_shape(self) -> tuple[int, ...]: @@ -640,6 +640,7 @@ def compute_output_shape( # outputs shape depends if layer and inputs are diagonal / linear (w/o batch) b_shape_wo_batchisze = model_output_shape_wo_batchsize if self.diagonal and self.inputs_outputs_spec.is_diagonal_bounds_shape(affine_bounds_to_propagate_shape): + w_shape_wo_batchsize: Union[tuple[int, ...], list[tuple[int, ...]]] if self._is_merging_layer: w_shape_wo_batchsize = [model_output_shape_wo_batchsize] * self.inputs_outputs_spec.nb_keras_inputs else: @@ -652,15 +653,17 @@ def compute_output_shape( ] else: w_shape_wo_batchsize = self.layer.input.shape[1:] + model_output_shape_wo_batchsize + b_shape: tuple[Optional[int], ...] + w_shape: Union[tuple[Optional[int], ...], list[tuple[Optional[int], ...]]] if self.linear and self.inputs_outputs_spec.is_wo_batch_bounds_shape(affine_bounds_to_propagate_shape): b_shape = b_shape_wo_batchisze - w_shape = w_shape_wo_batchsize + w_shape = w_shape_wo_batchsize # type: ignore else: b_shape = (None,) + b_shape_wo_batchisze if self._is_merging_layer: - w_shape = [(None,) + sub_w_shape_wo_batchsize for sub_w_shape_wo_batchsize in w_shape_wo_batchsize] + w_shape = [(None,) + sub_w_shape_wo_batchsize for sub_w_shape_wo_batchsize in w_shape_wo_batchsize] # type: ignore else: - w_shape = (None,) + w_shape_wo_batchsize + w_shape = (None,) + w_shape_wo_batchsize # type: ignore if self._is_merging_layer: affine_bounds_propagated_shape = [ [ diff --git a/src/decomon/layers/merging/base_merge.py b/src/decomon/layers/merging/base_merge.py index c73d0afa..fd966e66 100644 --- a/src/decomon/layers/merging/base_merge.py +++ b/src/decomon/layers/merging/base_merge.py @@ -1,5 +1,6 @@ from typing import Any +import keras import keras.ops as K from decomon.keras_utils import add_tensors, batch_multid_dot @@ -12,7 +13,7 @@ class DecomonMerge(DecomonLayer): _is_merging_layer = True @property - def keras_layer_input(self): + def keras_layer_input(self) -> list[keras.KerasTensor]: """self.layer.input returned as a list. In the degenerate case where only 1 input is merged, self.layer.input is a single keras tensor. @@ -25,7 +26,7 @@ def keras_layer_input(self): return [self.layer.input] @property - def nb_keras_inputs(self): + def nb_keras_inputs(self) -> int: """Number of inputs merged by the underlying layer.""" return len(self.keras_layer_input) @@ -274,7 +275,7 @@ def forward_affine_propagate( from_linear_layer_new = all(from_linear_add) return w_l_new, b_l_new, w_u_new, b_u_new - def backward_affine_propagate( + def backward_affine_propagate( # type: ignore self, output_affine_bounds: list[Tensor], input_constant_bounds: list[list[Tensor]] ) -> list[tuple[Tensor, Tensor, Tensor, Tensor]]: """Propagate model affine bounds in backward direction. diff --git a/src/decomon/layers/oracle.py b/src/decomon/layers/oracle.py index 773048ca..a2f211d7 100644 --- a/src/decomon/layers/oracle.py +++ b/src/decomon/layers/oracle.py @@ -134,12 +134,13 @@ def compute_output_shape( """Compute output shape in case of symbolic call.""" if self.is_merging_layer: output_shape = [] - for layer_input_shape_i in self.layer_input_shape: + layer_input_shape_i: tuple[int, ...] + for layer_input_shape_i in self.layer_input_shape: # type: ignore layer_input_shape_w_batchsize_i = (None,) + layer_input_shape_i output_shape.append([layer_input_shape_w_batchsize_i, layer_input_shape_w_batchsize_i]) return output_shape else: - layer_input_shape_w_batchsize = (None,) + self.layer_input_shape + layer_input_shape_w_batchsize = (None,) + self.layer_input_shape # type: ignore return [layer_input_shape_w_batchsize, layer_input_shape_w_batchsize] @@ -222,7 +223,7 @@ def get_forward_oracle( x = perturbation_domain_inputs[0] if is_merging_layer: constant_bounds = [] - for affine_bounds_i, from_linear_i in zip(affine_bounds, from_linear): + for affine_bounds_i, from_linear_i in zip(affine_bounds, from_linear): # type: ignore if len(affine_bounds_i) == 0: # special case: empty affine bounds => identity bounds l_affine = perturbation_domain.get_lower_x(x) @@ -239,6 +240,8 @@ def get_forward_oracle( l_affine = perturbation_domain.get_lower_x(x) u_affine = perturbation_domain.get_upper_x(x) else: + if not isinstance(from_linear, bool): + raise ValueError("from_linear must be a boolean for a non-merging layer") w_l, b_l, w_u, b_u = affine_bounds l_affine = perturbation_domain.get_lower(x, w_l, b_l, missing_batchsize=from_linear) u_affine = perturbation_domain.get_upper(x, w_u, b_u, missing_batchsize=from_linear) diff --git a/src/decomon/layers/output.py b/src/decomon/layers/output.py index 398f5f07..74e1147d 100644 --- a/src/decomon/layers/output.py +++ b/src/decomon/layers/output.py @@ -143,11 +143,16 @@ def compute_output_shape( self, input_shape: list[tuple[Optional[int], ...]], ) -> list[tuple[Optional[int], ...]]: + affine_bounds_from_shape: list[list[tuple[Optional[int], ...]]] + constant_bounds_from_shape: list[list[tuple[Optional[int], ...]]] + perturbation_domain_inputs_shape: list[tuple[Optional[int], ...]] ( affine_bounds_from_shape, constant_bounds_from_shape, perturbation_domain_inputs_shape, - ) = self.inputs_outputs_spec.split_input_shape(input_shape) + ) = self.inputs_outputs_spec.split_input_shape( # type: ignore + input_shape + ) constant_bounds_to_shape: list[list[tuple[Optional[int], ...]]] affine_bounds_to_shape: list[list[tuple[Optional[int], ...]]] @@ -167,7 +172,8 @@ def compute_output_shape( affine_bounds_to_shape = affine_bounds_from_shape else: x_shape = perturbation_domain_inputs_shape[0] - keras_input_shape = self.perturbation_domain.get_keras_input_shape_wo_batchsize(x_shape[1:]) + x_shape_wo_batchsize: tuple[int, ...] = x_shape[1:] # type: ignore + keras_input_shape = self.perturbation_domain.get_keras_input_shape_wo_batchsize(x_shape_wo_batchsize) affine_bounds_to_shape = [] for model_output_shape in self.model_output_shapes: b_shape = (None,) + model_output_shape diff --git a/src/decomon/models/backward_cloning.py b/src/decomon/models/backward_cloning.py index 80a8a24e..1e3e81c4 100644 --- a/src/decomon/models/backward_cloning.py +++ b/src/decomon/models/backward_cloning.py @@ -378,7 +378,7 @@ def crown_model( model_output_shape = get_model_output_shape( node=node, backward_bounds=backward_bounds_node, from_linear=from_linear ) - backward_map_node = {} + backward_map_node: dict[int, DecomonLayer] = {} output_crown = crown( node=node, @@ -406,7 +406,7 @@ def convert_backward( layer_fn: Callable[..., DecomonLayer] = to_decomon, backward_bounds: Optional[list[keras.KerasTensor]] = None, from_linear_backward_bounds: Union[bool, list[bool]] = False, - slope: Union[str, Slope] = Slope.V_SLOPE, + slope: Slope = Slope.V_SLOPE, forward_output_map: Optional[dict[int, list[keras.KerasTensor]]] = None, forward_layer_map: Optional[dict[int, DecomonLayer]] = None, mapping_keras2decomon_classes: Optional[dict[type[Layer], type[DecomonLayer]]] = None, @@ -440,6 +440,7 @@ def convert_backward( """ if perturbation_domain is None: perturbation_domain = BoxDomain() + backward_bounds_for_crown_model: list[list[keras.KerasTensor]] if backward_bounds is None: backward_bounds_for_crown_model = [[]] * len(model.outputs) else: @@ -474,7 +475,7 @@ def convert_backward( return output -def get_model_output_shape(node: Node, backward_bounds: list[Tensor], from_linear: bool = False): +def get_model_output_shape(node: Node, backward_bounds: list[Tensor], from_linear: bool = False) -> tuple[int, ...]: """Get outer model output shape w/o batchsize. If any backward bounds are passed, we deduce the outer keras model output shape from it. diff --git a/src/decomon/models/convert.py b/src/decomon/models/convert.py index efd55abb..c8233be5 100644 --- a/src/decomon/models/convert.py +++ b/src/decomon/models/convert.py @@ -311,18 +311,18 @@ def clone( if perturbation_domain is None: perturbation_domain = BoxDomain() - default_final_ibp, default_final_affine = get_final_ibp_affine_from_method(method) - if final_ibp is None: - final_ibp = default_final_ibp - if final_affine is None: - final_affine = default_final_affine - if isinstance(method, str): method = ConvertMethod(method.lower()) if isinstance(slope, str): slope = Slope(slope.lower()) + default_final_ibp, default_final_affine = get_final_ibp_affine_from_method(method) + if final_ibp is None: + final_ibp = default_final_ibp + if final_affine is None: + final_affine = default_final_affine + # preprocess backward_bounds backward_bounds_flattened: Optional[list[keras.KerasTensor]] backward_bounds_for_convert: Optional[list[keras.KerasTensor]] diff --git a/src/decomon/perturbation_domain.py b/src/decomon/perturbation_domain.py index 4202251d..dbfbe1d6 100644 --- a/src/decomon/perturbation_domain.py +++ b/src/decomon/perturbation_domain.py @@ -31,7 +31,7 @@ def get_lower_x(self, x: Tensor) -> Tensor: ... @abstractmethod - def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any) -> Tensor: """Merge upper affine bounds with perturbation domain input to get upper constant bound. Args: @@ -47,7 +47,7 @@ def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, ** ... @abstractmethod - def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any) -> Tensor: """Merge lower affine bounds with perturbation domain input to get lower constant bound. Args: @@ -111,12 +111,12 @@ def get_keras_input_shape_wo_batchsize(self, x_shape: tuple[int, ...]) -> tuple[ class BoxDomain(PerturbationDomain): - def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any) -> Tensor: x_min = x[:, 0] x_max = x[:, 1] return get_upper_box(x_min=x_min, x_max=x_max, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) - def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any) -> Tensor: x_min = x[:, 0] x_max = x[:, 1] return get_lower_box(x_min=x_min, x_max=x_max, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) @@ -162,10 +162,10 @@ def get_config(self) -> dict[str, Any]: ) return config - def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + def get_lower(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any) -> Tensor: return get_lower_ball(x_0=x, eps=self.eps, p=self.p, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) - def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: + def get_upper(self, x: Tensor, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any) -> Tensor: return get_upper_ball(x_0=x, eps=self.eps, p=self.p, w=w, b=b, missing_batchsize=missing_batchsize, **kwargs) def get_nb_x_components(self) -> int: @@ -178,7 +178,9 @@ def get_upper_x(self, x: Tensor) -> Tensor: return x + self.eps -def get_upper_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: +def get_upper_box( + x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any +) -> Tensor: """Compute the max of an affine function within a box (hypercube) defined by its extremal corners @@ -203,16 +205,18 @@ def get_upper_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, missing_ba is_diag = w.shape == b.shape diagonal = (False, is_diag) - missing_batchsize = (False, missing_batchsize) + missing_batchsize_dot = (False, missing_batchsize) return ( - batch_multid_dot(x_max, w_pos, diagonal=diagonal, missing_batchsize=missing_batchsize) - + batch_multid_dot(x_min, w_neg, diagonal=diagonal, missing_batchsize=missing_batchsize) + batch_multid_dot(x_max, w_pos, diagonal=diagonal, missing_batchsize=missing_batchsize_dot) + + batch_multid_dot(x_min, w_neg, diagonal=diagonal, missing_batchsize=missing_batchsize_dot) + b ) -def get_lower_box(x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, missing_batchsize=False, **kwargs: Any) -> Tensor: +def get_lower_box( + x_min: Tensor, x_max: Tensor, w: Tensor, b: Tensor, missing_batchsize: bool = False, **kwargs: Any +) -> Tensor: """ Args: x_min: lower bound of the box domain @@ -296,8 +300,8 @@ def get_upper_ball( w_q = get_lq_norm(w, p, axis=reduced_axes) diagonal = (False, is_diag) - missing_batchsize = (False, missing_batchsize) - return batch_multid_dot(x_0, w, diagonal=diagonal, missing_batchsize=missing_batchsize) + b + w_q * eps + missing_batchsize_dot = (False, missing_batchsize) + return batch_multid_dot(x_0, w, diagonal=diagonal, missing_batchsize=missing_batchsize_dot) + b + w_q * eps def get_lower_ball( @@ -343,8 +347,8 @@ def get_lower_ball( w_q = get_lq_norm(w, p, axis=reduced_axes) diagonal = (False, is_diag) - missing_batchsize = (False, missing_batchsize) - return batch_multid_dot(x_0, w, diagonal=diagonal, missing_batchsize=missing_batchsize) + b - w_q * eps + missing_batchsize_dot = (False, missing_batchsize) + return batch_multid_dot(x_0, w, diagonal=diagonal, missing_batchsize=missing_batchsize_dot) + b - w_q * eps def get_lower_ball_finetune(