Merge branch 'experiment/ri_evn' into 'main'

Directional random walk and Bug Fixes and Experiments

See merge request es/ai/hannah/hannah!374

experiments/constrained_space_comparison/augmentation/cifar_augment.yaml

@@ -0,0 +1,20 @@
+batch_augment:
+  pipeline: null
+  transforms:
+    #RandomVerticalFlip:
+    #  p: 0.5
+    RandomHorizontalFlip:
+      p: 0.5
+    RandomAffine:
+      degrees: [-15, 15]
+      translate: [0.1, 0.1]
+      scale: [0.9, 1.1]
+      shear: [-5, 5]
+      p: 0.5
+    RandomCrop:
+      size: [32,32]
+      padding: 4
+    RandomErasing:
+      p: 0.5
+      #scale: [0.!, 0.3]
+      #value: [0.4914, 0.4822, 0.4465]

experiments/constrained_space_comparison/config.yaml

@@ -23,7 +23,7 @@ defaults:
     - override features: identity        # Feature extractor configuration name (use identity for vision datasets)
     #- override model: timm_mobilenetv3_small_075      # Neural network name (for now timm_resnet50 or timm_efficientnet_lite1)
     - override scheduler: 1cycle         # learning rate scheduler config name
-    - override optimizer: adamw          # Optimizer config name
+    - override optimizer: sgd          # Optimizer config name
     - override normalizer: null          # Feature normalizer (used for quantized neural networks)
     - override module: image_classifier  # Lightning module config for the training loop (image classifier for image classification tasks)
     - _self_
@@ -32,12 +32,7 @@ defaults:
 dataset:
   data_folder: ${oc.env:HANNAH_DATA_FOLDER,${hydra:runtime.cwd}/../../datasets/}
 
-module:
-  batch_size: 128
-  num_workers: 8
-
 trainer:
-  max_epochs: 10
+  max_epochs: 100
 
-scheduler:
-  max_lr: 0.001
+fx_mac_summary: True

experiments/constrained_space_comparison/eval.yaml

@@ -0,0 +1,77 @@
+##
+## Copyright (c) 2022 University of Tübingen.
+##
+## This file is part of hannah.
+## See https://atreus.informatik.uni-tuebingen.de/ties/ai/hannah/hannah for further info.
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+data:
+  # baseline: trained_models/ae_nas_cifar10_baseline/embedded_vision_net
+  # weights: trained_models/ae_nas_cifar10_weight/embedded_vision_net
+  # weights_and_macs: trained_models/ae_nas_cifar10_weight_and_macs/embedded_vision_net
+  # macs: trained_models/ae_nas_cifar10_macs/embedded_vision_net
+  # nopred: trained_models/ae_nas_cifar10_weight_and_macs_no_pred/embedded_vision_net
+  # sortbymacs: trained_models/ae_nas_cifar10_weight_and_macs_sortbymacs/embedded_vision_net
+  250k: trained_models/ae_nas_cifar10_weight_250k/embedded_vision_net
+  250k_20e: trained_models/ae_nas_cifar10_weight_250k_20epochs/embedded_vision_net
+  250k_50m: trained_models/ae_nas_cifar10_weight_250k_macs_50m/embedded_vision_net
+
+
+metrics:
+  total_act:
+    name: Activations
+  total_weights:
+    name: Weights
+  weights_m:
+    name: Weights [M]
+    derived: data["total_weights"] / 1000 / 1000
+  val_accuracy:
+    name: Accuracy [%]
+    derived: (1.0 - data["val_error"]) * 100.0
+  act_k:
+    name: Activations [k]
+    derived: data["total_act"] / 1000
+  macs_m:
+    name: MACS [M]
+    derived: data["total_macs"] / 1000 / 1000
+
+plots:
+  # Comparison plots 2-3 metrics using y, x and size as visualization points
+  - type: comparison
+    name: accuracy_memory
+    metrics:
+      - val_accuracy
+      - weights_m
+      - act_k
+
+  - type: comparison
+    name: accuracy_macs
+    metrics:
+      - val_accuracy
+      - macs_m
+
+extract:
+  random_nas_cifar10:
+    bounds:
+        val_error: 0.20
+        total_macs: 100000000
+        total_weights: 1000000
+
+
+experiment: presampling
+force: false
+
+hydra:
+  run:
+    dir: ./nas_results/${experiment}

experiments/constrained_space_comparison/experiment/ae_nas_cifar10_weights_150-250k_macs_50m.yaml

@@ -0,0 +1,23 @@
+# @package _global_
+defaults:
+  - override /nas: aging_evolution_nas
+  - override /model: embedded_vision_net
+  - override /dataset: cifar10
+  - override /nas/constraint_model: random_walk
+
+model:
+  num_classes: 10
+  constraints:
+    - name: weights
+      upper: 250000
+      # lower: 150000
+    - name: macs
+      lower: 50000000
+
+nas:
+  budget: 600
+  n_jobs: 8
+
+seed: [1234]
+
+experiment_id: "ae_nas_cifar10_weight_250k_macs_50m"

experiments/constrained_space_comparison/experiment/ae_nas_cifar10_weights_250k.yaml

@@ -0,0 +1,20 @@
+# @package _global_
+defaults:
+  - override /nas: aging_evolution_nas
+  - override /model: embedded_vision_net
+  - override /dataset: cifar10
+  - override /nas/constraint_model: random_walk
+
+model:
+  num_classes: 10
+  constraints:
+    - name: weights
+      upper: 250000
+
+nas:
+  budget: 600
+  n_jobs: 8
+
+seed: [1234]
+
+experiment_id: "ae_nas_cifar10_weight_250k"

experiments/constrained_space_comparison/experiment/ae_nas_cifar10_weights_250k_20epochs.yaml

@@ -0,0 +1,29 @@
+# @package _global_
+defaults:
+  - override /nas: aging_evolution_nas
+  - override /model: embedded_vision_net
+  - override /dataset: cifar10
+  - override /nas/constraint_model: random_walk
+
+model:
+  num_classes: 10
+  constraints:
+    - name: weights
+      upper: 250000
+
+nas:
+  budget: 1000
+  n_jobs: 2
+  total_candidates: 100
+  num_selected_candidates: 25
+  bounds:
+    val_error: 0.1
+    total_macs: 200000000
+    total_weights: 250000
+
+trainer:
+  max_epochs: 20
+
+seed: [1234]
+
+experiment_id: "ae_nas_cifar10_weight_250k_20epochs"

experiments/constrained_space_comparison/experiment/train_best_model.yaml

@@ -0,0 +1,23 @@
+# @package _global_
+defaults:
+  - override /model: embedded_vision_net_model
+  - override /augmentation: cifar_augment
+
+model:
+  param_path: /local/reiber/hannah/experiments/constrained_space_comparison/parameters.pkl
+  task_name: 250k_50m
+  index: 524
+  input_shape: [1, 3, 32, 32]
+  labels: 10
+
+module:
+  batch_size: 32
+  num_workers: 32
+
+trainer:
+  max_epochs: 100
+
+scheduler:
+  max_lr: 0.001
+
+experiment_id: best_model_augment_adam0001_b32_100e

experiments/ri_evn/config.yaml

@@ -0,0 +1,45 @@
+##
+## Copyright (c) 2022 University of Tübingen.
+##
+## This file is part of hannah.
+## See https://atreus.informatik.uni-tuebingen.de/ties/ai/hannah/hannah for further info.
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+defaults:
+    - base_config
+    - experiment: optional
+    - override dataset: ri_capsule   # Dataset configuration name
+    - override features: identity        # Feature extractor configuration name (use identity for vision datasets)
+    # - override model: embedded_vision_net_model      # Neural network name (for now timm_resnet50 or timm_efficientnet_lite1)
+    - override scheduler: 1cycle         # learning rate scheduler config name
+    - override optimizer: adamw          # Optimizer config name
+    - override normalizer: null          # Feature normalizer (used for quantized neural networks)
+    - override module: image_classifier  # Lightning module config for the training loop (image classifier for image classification tasks)
+    - _self_
+
+
+dataset:
+  data_folder: /data
+
+module:
+  batch_size: 32
+  num_workers: 8
+
+trainer:
+  max_epochs: 20
+
+scheduler:
+  max_lr: 0.001
+
+fx_mac_summary: True

experiments/ri_evn/eval.yaml

@@ -0,0 +1,70 @@
+##
+## Copyright (c) 2022 University of Tübingen.
+##
+## This file is part of hannah.
+## See https://atreus.informatik.uni-tuebingen.de/ties/ai/hannah/hannah for further info.
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+data:
+  250k: trained_models/250k_weights/embedded_vision_net
+
+
+
+metrics:
+  total_act:
+    name: Activations
+  total_weights:
+    name: Weights
+  weights_m:
+    name: Weights [M]
+    derived: data["total_weights"] / 1000 / 1000
+  val_accuracy:
+    name: Accuracy [%]
+    derived: (1.0 - data["val_error"]) * 100.0
+  act_k:
+    name: Activations [k]
+    derived: data["total_act"] / 1000
+  macs_m:
+    name: MACS [M]
+    derived: data["total_macs"] / 1000 / 1000
+
+plots:
+  # Comparison plots 2-3 metrics using y, x and size as visualization points
+  - type: comparison
+    name: accuracy_memory
+    metrics:
+      - val_accuracy
+      - weights_m
+      - act_k
+
+  - type: comparison
+    name: accuracy_macs
+    metrics:
+      - val_accuracy
+      - macs_m
+
+extract:
+  random_nas_cifar10:
+    bounds:
+        val_error: 0.20
+        total_macs: 100000000
+        total_weights: 1000000
+
+
+experiment: presampling
+force: false
+
+hydra:
+  run:
+    dir: ./nas_results/${experiment}

experiments/ri_evn/experiment/250k_weights.yaml

@@ -0,0 +1,27 @@
+# @package _global_
+defaults:
+  - override /nas: aging_evolution_nas
+  - override /model: embedded_vision_net
+  - override /nas/constraint_model: random_walk
+
+model:
+  num_classes: 4
+  constraints:
+    - name: weights
+      upper: 250000
+    # - name: macs
+    #   upper: 128000000
+
+nas:
+  budget: 600
+  n_jobs: 1
+  num_selected_candidates: 20
+  total_candidates: 50
+  bounds:
+    val_error: 0.1
+    total_macs: 128000000
+    total_weights: 250000
+
+seed: [1234]
+
+experiment_id: 250k_weights

experiments/ri_evn/experiment/train_model.yaml

@@ -0,0 +1,17 @@
+#  @package _global_
+defaults:
+  - override /model: embedded_vision_net_model
+
+model:
+  param_path: /local/reiber/hannah/experiments/ri_evn/parameters.pkl
+  task_name: 250k
+  index: 5
+  input_shape: (1, 3, 320, 320)
+  labels: 4
+
+seed: [1234]
+
+trainer:
+  max_epochs: 50
+
+experiment_id: train_best_model