Merge branch 'f/merge-extended-search-space-and-constraints' into 'main'

F/merge extended search space and constraints

See merge request es/ai/hannah/hannah!369

.vscode/launch.json

@@ -15,11 +15,11 @@
             "justMyCode": false,
             "args": [
                 "nas=random_nas",
-                "model=lazy_convnet",
+                "model=embedded_vision_net",
                 "module=image_classifier",
-                "dataset=cifar10",
+                "dataset=kvasir_capsule",
                 "features=identity",
-                "module.batch_size=32",
+                "module.batch_size=2",
                 "trainer.max_epochs=15",
                 "nas.budget=3",
                 "module.num_workers=8",
@@ -36,18 +36,82 @@
             "cwd": "${workspaceFolder}/experiments/",
             "args": [
                 "nas=random_nas",
-                "model=capsule_net_v2",
+                "nas.presample=False",
+                "model=embedded_vision_net_model",
                 "module=image_classifier",
                 "dataset=cifar10",
                 "features=identity",
                 "module.batch_size=128",
-                "trainer.max_epochs=2",
-                "trainer.overfit_batches=20",
-                "nas.budget=100",
+                "trainer.max_epochs=100",
+                // "trainer.overfit_batches=1",
+                "nas.budget=1",
+                "module.num_workers=1",
+                "nas.n_jobs=1",
+                "experiment_id=more_training",
+                "fx_mac_summary=True",
+                "nas.predictor.model.input_feature_size=31",
+                // "nas.presampler=single_range_checker",
+                // "~nas.predictor",
+                // "~nas.constraint_model",
+                "~normalizer"
+            ]
+        },
+        {
+            "name": "mobilenet",
+            "type": "python",
+            "request": "launch",
+            "module": "hannah.tools.train",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "cwd": "${workspaceFolder}/experiments/",
+            "args": [
+                "nas=random_nas",
+                "model=mobilenetv2_functional",
+                "module=image_classifier",
+                "dataset=cifar10",
+                "features=identity",
+                "module.batch_size=128",
+                "trainer.max_epochs=25",
+                // "trainer.overfit_batches=1",
+                "nas.budget=8",
+                "module.num_workers=1",
+                "nas.n_jobs=1",
+                "experiment_id=mobilenet",
+                "fx_mac_summary=True",
+                // "nas.predictor.model.input_feature_size=30",
+                "nas.total_candidates=1",
+                "nas.num_selected_candidates=1",
+                "nas.presample=False",
+                "~nas.predictor",
+                // "~nas.constraint_model",
+                "~normalizer",
+            ]
+        },
+        {
+            "name": "transformer",
+            "type": "python",
+            "request": "launch",
+            "module": "hannah.tools.train",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "cwd": "${workspaceFolder}/experiments/",
+            "args": [
+                "nas=random_nas",
+                "nas.presample=False",
+                "model=transformer_classifier",
+                "module=stream_classifier",
+                "dataset=kws",
+                "features=identity",
+                "module.batch_size=128",
+                "trainer.max_epochs=5",
+                "trainer.overfit_batches=6",
+                "nas.budget=600",
                 "module.num_workers=1",
                 "nas.n_jobs=1",
-                "experiment_id=functional_ops_test",
+                "experiment_id=transformer_fx_error",
                 "fx_mac_summary=True",
+                "nas.predictor.model.input_feature_size=37",
+                // "nas.presampler=single_range_checker",
                 // "~nas.predictor",
                 // "~nas.constraint_model",
                 "~normalizer"
@@ -70,9 +134,13 @@
             "args": [
                 "trainer.limit_train_batches=1.0",
                 "trainer.max_epochs=2",
-                "model=conv-net-trax",
-                "normalizer=fixedpoint",
-                "compression=pruning_only"
+                "model=embedded_vision_net_model",
+                "dataset=cifar10",
+                "module=image_classifier",
+                "features=identity",
+                "fx_mac_summary=True",
+                "~normalizer",
+                // "compression=pruning_only"
             ]
         },
 
@@ -85,19 +153,64 @@
             "cwd": "${workspaceFolder}/experiments/",
             "args": [
                 "nas=aging_evolution_nas",
-                "model=capsule_net_v2",
+                "model=embedded_vision_net",
                 "module=image_classifier",
                 "dataset=cifar10",
                 "features=identity",
-                "module.batch_size=4",
+                "module.batch_size=128",
                 "trainer.max_epochs=1",
-                "trainer.overfit_batches=20",
+                "trainer.overfit_batches=1",
                 "nas.n_jobs=1",
                 "nas.budget=100",
-                "nas.total_candidates=10",
+                "nas.total_candidates=50",
+                "nas.num_selected_candidates=20",
+                "nas.sampler.population_size=20",
+                // "nas.predictor.model.input_feature_size=31",
                 "module.num_workers=8",
-                "experiment_id=ae_debug",
-                "~nas.predictor",
+                "experiment_id=test_constraint_config",
+                "fx_mac_summary=True",
+                // "~nas.predictor",
+                "~normalizer"
+            ]
+        },
+        {
+            "name": "AgingEvolutionRestricted",
+            "type": "python",
+            "request": "launch",
+            "module": "hannah.tools.train",
+            "justMyCode": false,
+            "cwd": "${workspaceFolder}/experiments/constrained_space_comparison/",
+            "args": [
+                "trainer.gpus=1",
+                "experiment=ae_nas_cifar10_weight_and_macs_sortbymacs_finetune",
+                "nas.n_jobs=1",
+                "fx_mac_summary=True",
+                "~normalizer"
+            ]
+        },
+        {
+            "name": "DefinedSpace",
+            "type": "python",
+            "request": "launch",
+            "module": "hannah.tools.train",
+            "justMyCode": false,
+            "cwd": "${workspaceFolder}/experiments/",
+            "args": [
+                "nas=defined_space_exploration",
+                "model=embedded_vision_net",
+                "module=image_classifier",
+                "dataset=cifar10",
+                "features=identity",
+                "module.batch_size=128",
+                "trainer.max_epochs=1",
+                "trainer.overfit_batches=1",
+                "nas.n_jobs=1",
+                "nas.budget=1000",
+                // "nas.predictor.model.input_feature_size=31",
+                "module.num_workers=8",
+                "experiment_id=defined_space",
+                "fx_mac_summary=True",
+                // "~nas.predictor",
                 "~normalizer"
             ]
         },

experiments/constrained_space_comparison/config.yaml

@@ -0,0 +1,43 @@
+##
+## Copyright (c) 2022 University of Tübingen.
+##
+## This file is part of hannah.
+## See https://atreus.informatik.uni-tuebingen.de/ties/ai/hannah/hannah for further info.
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+defaults:
+    - base_config
+    - experiment: optional
+    - override dataset: cifar10   # Dataset configuration name
+    - override features: identity        # Feature extractor configuration name (use identity for vision datasets)
+    #- override model: timm_mobilenetv3_small_075      # Neural network name (for now timm_resnet50 or timm_efficientnet_lite1)
+    - override scheduler: 1cycle         # learning rate scheduler config name
+    - override optimizer: adamw          # Optimizer config name
+    - override normalizer: null          # Feature normalizer (used for quantized neural networks)
+    - override module: image_classifier  # Lightning module config for the training loop (image classifier for image classification tasks)
+    - _self_
+
+
+dataset:
+  data_folder: ${oc.env:HANNAH_DATA_FOLDER,${hydra:runtime.cwd}/../../datasets/}
+
+module:
+  batch_size: 128
+  num_workers: 8
+
+trainer:
+  max_epochs: 10
+
+scheduler:
+  max_lr: 0.001

experiments/constrained_space_comparison/experiment/ae_nas_cifar10_baseline.yaml

@@ -0,0 +1,17 @@
+# @package _global_
+defaults:
+  - override /nas: aging_evolution_nas
+  - override /model: embedded_vision_net
+  - override /dataset: cifar10
+  - override /nas/constraint_model: random_walk
+
+model:
+  num_classes: 10
+
+nas:
+  budget: 600
+  n_jobs: 8
+
+seed: [1234]
+
+experiment_id: "ae_nas_cifar10_baseline"

experiments/constrained_space_comparison/experiment/ae_nas_cifar10_macs.yaml

@@ -0,0 +1,20 @@
+# @package _global_
+defaults:
+  - override /nas: aging_evolution_nas
+  - override /model: embedded_vision_net
+  - override /dataset: cifar10
+  - override /nas/constraint_model: random_walk
+
+model:
+  num_classes: 10
+  constraints:
+    - name: macs
+      upper: 128000000
+
+nas:
+  budget: 600
+  n_jobs: 8
+
+seed: [1234]
+
+experiment_id: "ae_nas_cifar10_macs"

experiments/constrained_space_comparison/experiment/ae_nas_cifar10_weight.yaml

@@ -0,0 +1,20 @@
+# @package _global_
+defaults:
+  - override /nas: aging_evolution_nas
+  - override /model: embedded_vision_net
+  - override /dataset: cifar10
+  - override /nas/constraint_model: random_walk
+
+model:
+  num_classes: 10
+  constraints:
+    - name: weights
+      upper: 550000
+
+nas:
+  budget: 600
+  n_jobs: 8
+
+seed: [1234]
+
+experiment_id: "ae_nas_cifar10_weight"

experiments/constrained_space_comparison/experiment/ae_nas_cifar10_weight_and_macs.yaml

@@ -0,0 +1,22 @@
+# @package _global_
+defaults:
+  - override /nas: aging_evolution_nas
+  - override /model: embedded_vision_net
+  - override /dataset: cifar10
+  - override /nas/constraint_model: random_walk
+
+model:
+  num_classes: 10
+  constraints:
+    - name: weights
+      upper: 550000
+    - name: macs
+      upper: 128000000
+
+nas:
+  budget: 600
+  n_jobs: 8
+
+seed: [1234]
+
+experiment_id: "ae_nas_cifar10_weight_and_macs"

experiments/constrained_space_comparison/experiment/ae_nas_cifar10_weight_and_macs_no_pred.yaml

@@ -0,0 +1,24 @@
+# @package _global_
+defaults:
+  - override /nas: aging_evolution_nas
+  - override /model: embedded_vision_net
+  - override /dataset: cifar10
+  - override /nas/constraint_model: random_walk
+
+model:
+  num_classes: 10
+  constraints:
+    - name: weights
+      upper: 550000
+    - name: macs
+      upper: 128000000
+
+nas:
+  budget: 600
+  n_jobs: 1
+  num_selected_candidates: 20
+  total_candidates: 20
+
+seed: [1234]
+
+experiment_id: "ae_nas_cifar10_weight_and_macs_no_pred"

experiments/constrained_space_comparison/experiment/ae_nas_cifar10_weight_and_macs_sortbymacs.yaml

@@ -0,0 +1,24 @@
+# @package _global_
+defaults:
+  - override /nas: aging_evolution_nas
+  - override /model: embedded_vision_net
+  - override /dataset: cifar10
+  - override /nas/constraint_model: random_walk
+
+model:
+  num_classes: 10
+  constraints:
+    - name: weights
+      upper: 550000
+    - name: macs
+      upper: 128000000
+
+nas:
+  budget: 600
+  n_jobs: 1
+  num_selected_candidates: 20
+  total_candidates: 50
+
+seed: [1234]
+
+experiment_id: "ae_nas_cifar10_weight_and_macs_sortbymacs"