diff --git a/experiments/embedded_vision_net_ri/config.yaml b/experiments/embedded_vision_net_ri/config.yaml new file mode 100644 index 00000000..5f7f022c --- /dev/null +++ b/experiments/embedded_vision_net_ri/config.yaml @@ -0,0 +1,43 @@ +## +## Copyright (c) 2022 University of Tübingen. +## +## This file is part of hannah. +## See https://atreus.informatik.uni-tuebingen.de/ties/ai/hannah/hannah for further info. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +defaults: + - base_config + - experiment: optional + - override dataset: ri_capsule # Dataset configuration name + - override features: identity # Feature extractor configuration name (use identity for vision datasets) + #- override model: timm_mobilenetv3_small_075 # Neural network name (for now timm_resnet50 or timm_efficientnet_lite1) + - override scheduler: 1cycle # learning rate scheduler config name + - override optimizer: adamw # Optimizer config name + - override normalizer: null # Feature normalizer (used for quantized neural networks) + - override module: image_classifier # Lightning module config for the training loop (image classifier for image classification tasks) + - _self_ + + +dataset: + data_folder: ${oc.env:HANNAH_DATA_FOLDER,${hydra:runtime.cwd}/../../datasets/} + +module: + batch_size: 16 + num_workers: 8 + +trainer: + max_epochs: 10 + +scheduler: + max_lr: 0.001 diff --git a/experiments/embedded_vision_net_ri/eval.yaml b/experiments/embedded_vision_net_ri/eval.yaml new file mode 100644 index 00000000..ff931120 --- /dev/null +++ b/experiments/embedded_vision_net_ri/eval.yaml @@ -0,0 +1,69 @@ +## +## Copyright (c) 2022 University of Tübingen. +## +## This file is part of hannah. +## See https://atreus.informatik.uni-tuebingen.de/ties/ai/hannah/hannah for further info. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +data: + AE: trained_models/ae_nas_cifar10/embedded_vision_net + RANDOM: trained_models/random_nas_cifar10_workingbn/embedded_vision_net + +metrics: + total_act: + name: Activations + total_weights: + name: Weights + weights_m: + name: Weights [M] + derived: data["total_weights"] / 1000 / 1000 + val_accuracy: + name: Accuracy [%] + derived: (1.0 - data["val_error"]) * 100.0 + act_k: + name: Activations [k] + derived: data["total_act"] / 1000 + macs_m: + name: MACS [M] + derived: data["total_macs"] / 1000 / 1000 + +plots: + # Comparison plots 2-3 metrics using y, x and size as visualization points + - type: comparison + name: accuracy_memory + metrics: + - val_accuracy + - weights_m + - act_k + + - type: comparison + name: accuracy_macs + metrics: + - val_accuracy + - macs_m + +extract: + AE: + bounds: + val_error: 0.20 + total_macs: 100000000 + total_weights: 1000000 + + +experiment: embedded_vision_net_ri +force: false + +hydra: + run: + dir: ./nas_results/${experiment} diff --git a/experiments/embedded_vision_net_ri/experiment/ae_nas_cifar10.yaml b/experiments/embedded_vision_net_ri/experiment/ae_nas_cifar10.yaml new file mode 100644 index 00000000..3b31f914 --- /dev/null +++ b/experiments/embedded_vision_net_ri/experiment/ae_nas_cifar10.yaml @@ -0,0 +1,21 @@ +# @package _global_ +defaults: + - override /nas: aging_evolution_nas + - override /model: embedded_vision_nas + - override /dataset: cifar10 + +model: + num_classes: 10 +module: + batch_size: 128 +nas: + budget: 300 + n_jobs: 8 + + +trainer: + max_epochs: 10 + +seed: [1234] + +experiment_id: "ae_nas_cifar10" diff --git a/experiments/embedded_vision_net_ri/experiment/ae_nas_cifar10_v2.yaml b/experiments/embedded_vision_net_ri/experiment/ae_nas_cifar10_v2.yaml new file mode 100644 index 00000000..ac5ec9fa --- /dev/null +++ b/experiments/embedded_vision_net_ri/experiment/ae_nas_cifar10_v2.yaml @@ -0,0 +1,21 @@ +# @package _global_ +defaults: + - override /nas: aging_evolution_nas + - override /model: embedded_vision_nas + - override /dataset: cifar10 + +model: + num_classes: 10 +module: + batch_size: 128 +nas: + budget: 600 + n_jobs: 8 + + +trainer: + max_epochs: 10 + +seed: [1234] + +experiment_id: "ae_nas_cifar10_v2" diff --git a/experiments/embedded_vision_net_ri/experiment/ae_nas_ri.yaml b/experiments/embedded_vision_net_ri/experiment/ae_nas_ri.yaml new file mode 100644 index 00000000..d3f40413 --- /dev/null +++ b/experiments/embedded_vision_net_ri/experiment/ae_nas_ri.yaml @@ -0,0 +1,18 @@ +# @package _global_ +defaults: + - override /nas: aging_evolution_nas + - override /model: embedded_vision_net + + +nas: + budget: 300 + n_jobs: 8 + presample: False + + +trainer: + max_epochs: 10 + +seed: [1234] + +experiment_id: "ae_nas_ri" diff --git a/experiments/embedded_vision_net_ri/experiment/random_nas.yaml b/experiments/embedded_vision_net_ri/experiment/random_nas.yaml new file mode 100644 index 00000000..a1fd0b08 --- /dev/null +++ b/experiments/embedded_vision_net_ri/experiment/random_nas.yaml @@ -0,0 +1,17 @@ +# @package _global_ +defaults: + - override /nas: random_nas + - override /model: embedded_vision_net + + +nas: + budget: 300 + n_jobs: 8 + + +trainer: + max_epochs: 10 + +seed: [1234] + +oxperiment_id: "random_nas" diff --git a/experiments/embedded_vision_net/embedded_vision_net_ri/experiment/random_nas.yaml b/experiments/embedded_vision_net_ri/experiment/random_nas_cifar10.yaml similarity index 53% rename from experiments/embedded_vision_net/embedded_vision_net_ri/experiment/random_nas.yaml rename to experiments/embedded_vision_net_ri/experiment/random_nas_cifar10.yaml index f7fca962..ef33c5b6 100644 --- a/experiments/embedded_vision_net/embedded_vision_net_ri/experiment/random_nas.yaml +++ b/experiments/embedded_vision_net_ri/experiment/random_nas_cifar10.yaml @@ -2,12 +2,15 @@ defaults: - override /nas: random_nas - override /model: embedded_vision_nas + - override /dataset: cifar10 - +model: + num_classes: 10 +module: + batch_size: 128 nas: - budget: 100 + budget: 300 n_jobs: 8 - presample: False trainer: @@ -15,4 +18,4 @@ trainer: seed: [1234] -experiment_id: "random_nas_log_saving" +experiment_id: "random_nas_cifar10_workingbn" diff --git a/experiments/embedded_vision_net_ri/experiment/random_nas_cifar10_nopool.yaml b/experiments/embedded_vision_net_ri/experiment/random_nas_cifar10_nopool.yaml new file mode 100644 index 00000000..3d642938 --- /dev/null +++ b/experiments/embedded_vision_net_ri/experiment/random_nas_cifar10_nopool.yaml @@ -0,0 +1,23 @@ +# @package _global_ +defaults: + - override /nas: random_nas + - override /model: embedded_vision_nas + - override /dataset: cifar10 + +model: + num_classes: 10 +module: + batch_size: 128 +nas: + budget: 300 + n_jobs: 8 + predictor: + model: + input_feature_size: 30 + +trainer: + max_epochs: 10 + +seed: [1234] + +experiment_id: "random_nas_cifar10_nopool" diff --git a/experiments/embedded_vision_net_ri/hydra/launcher/ml_cloud_4gpu.yaml b/experiments/embedded_vision_net_ri/hydra/launcher/ml_cloud_4gpu.yaml new file mode 100644 index 00000000..940c9d6b --- /dev/null +++ b/experiments/embedded_vision_net_ri/hydra/launcher/ml_cloud_4gpu.yaml @@ -0,0 +1,26 @@ +## +## Copyright (c) 2022 University of Tübingen. +## +## This file is part of hannah. +## See https://atreus.informatik.uni-tuebingen.de/ties/ai/hannah/hannah for further info. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +defaults: + - submitit_slurm + - _self_ + +timeout_min: 3600 +gpus_per_task: 4 +cpus_per_gpu: 8 +partition: gpu-2080ti diff --git a/experiments/embedded_vision_net_ri/result_exploration.ipynb b/experiments/embedded_vision_net_ri/result_exploration.ipynb new file mode 100644 index 00000000..b29acd67 --- /dev/null +++ b/experiments/embedded_vision_net_ri/result_exploration.ipynb @@ -0,0 +1,104 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import pickle as pkl\n", + "from pathlib import Path\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "metrics = pd.read_pickle(Path(\"metrics.pkl\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "def minseries(series):\n", + " s = []\n", + " for i in series:\n", + " if s:\n", + " s.append(min(s[-1], i))\n", + " else:\n", + " s.append(i)\n", + " return s" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "metrics_ae = metrics[metrics['Task'] == \"AE\"]\n", + "metrics_rn = metrics[metrics['Task'] == \"RANDOM\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjQAAAGdCAYAAAAFcOm4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/OQEPoAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA7yUlEQVR4nO3dfXxU9Z3//fc5k5ncEEhAICEYCQjeoAiUu41ciluygNX+pKWKistNLd0q2ZXGsiU+Vqjdqw0qpbjKD1oL4l61im0XbelKq7FArVEKGG9A8Y6CEhJAhUBCMpOZc/2RzISQTGYSZuacJK/n45EHJzMnZ745Ds6b7+d7Y1iWZQkAAKALM+1uAAAAwPki0AAAgC6PQAMAALo8Ag0AAOjyCDQAAKDLI9AAAIAuj0ADAAC6PAINAADo8pLsbkCsBAIBVVRUqHfv3jIMw+7mAACAKFiWpVOnTiknJ0em2fl+lm4TaCoqKpSbm2t3MwAAQCd88sknuvDCCzv9890m0PTu3VtS4w3p06ePza05f16vVz/5yU8kSffee688Ho/NLQIAIPaqq6uVm5sb+hzvrG4TaIJlpj59+nSbQJOSkiKp8Xci0AAAurPzHS7CoGAAANDldZsemu7GNE2NHj06dAwAAMIj0DhUUlKSZs6caXczAADoEgg0AADEmN/vl8/ns7sZjuByuZSUlBT3JVUINA5lWVboL4Pb7WZtHQDoIk6fPq1PP/1UlmXZ3RTHSEtL06BBg+I6wYVA41A+n08lJSWSpOLiYmY5AUAX4Pf79emnnyotLU0DBgzo8f8YtSxLXq9Xx44d04EDBzRixIi4jQsl0AAAECM+n0+WZWnAgAFKTU21uzmOkJqaKrfbrYMHD7ZYkiTWmD4DAECM9fSemXMlYrYugQYAAHR5BBoAANDlEWgAAECXR6ABAKCHmz9/vgzDkGEYcrvdGjp0qP793/9ddXV1oXMMw1BKSooOHjzY4mdnzpyp+fPnJ7jFrRFoIlj1p/36we/2qvJkXeSTY8g0TY0cOVIjR45k6wMAQNzNmDFDR44c0ccff6yf/vSn+tnPfqbly5e3OMcwDC1btsymFraPadsRPPO3T3T0VL1uGZ+r7Iz4TDVrS1JSkm6++eaEvR4AIPYsy9IZn9+W1051uzo02yo5OVnZ2dmSpNzcXBUUFOjFF1/Ugw8+GDqnsLBQq1at0pIlS3TllVfGvM3ng0ATgdn0Zgiw4iMAoIPO+PwaueyPtrz2vh9OV5qncx/z77zzjl599VUNGTKkxeOTJ0/W+++/r6VLl2rLli2xaGbMEGgiMJvCLXkGANCdbdmyRenp6WpoaFB9fb1M09Rjjz3W6rySkhJdddVV+stf/qJrrrnGhpa2jUATgWFTD43X62XrAwDo4lLdLu374XTbXrsj/vEf/1Fr165VTU2NfvrTnyopKUmzZs1qdd7IkSM1d+5cLV26VH/9619j1dzzRqCJIDge108XDQCggwzD6HTZJ9F69eql4cOHS5I2bNig0aNHa/369brzzjtbnfvAAw/okksu0XPPPZfgVobH9JkIgmNo2DUVANBTmKap++67T//xH/+hM2fOtHo+NzdXhYWFuu++++T32zPo+VwEmghcoZKTzQ0BACCBbr75ZrlcLq1Zs6bN54uLi1VRUaGXXnopwS1rG4EmguCMtwCJBgDQgyQlJamwsFAPPfSQampqWj3fr18/ff/732+x+J6dukZhz0YmPTQAgG5u48aNbT6+dOlSLV26VFLbQy+Ki4tVXFwcz6ZFjR6aCBhDAwCA89FDE0Gw5JToWU6maWrEiBGhYwAAEB6BJgK7Sk5JSUm6/fbbE/uiAAB0UfzTPwKXydYHAAA4HYEmguatDwg0AAA4FSWnCEJbHwQS+7per1crV66UJH3ve99j6wMAANpBoIkg2ENjR8nJ5/Ml/DUBAOiKOlVyWrNmjfLy8pSSkqJJkyZp586dYc/du3evZs2apby8PBmGodWrV7c6x+/36/7779fQoUOVmpqqiy++WP/5n//piDIP69AAAOB8HQ40mzZtUlFRkZYvX649e/Zo9OjRmj59uo4ePdrm+bW1tRo2bJhWrFih7OzsNs958MEHtXbtWj322GN699139eCDD+qhhx7So48+2tHmxZxp027bAAAgeh0ONKtWrdLChQu1YMECjRw5UuvWrVNaWpo2bNjQ5vkTJkzQww8/rFtvvVXJycltnvPqq6/qpptu0g033KC8vDx94xvf0LRp09rt+UmU4BIwBBoAQHdXVlYml8ulG264ocXjf//732UYRptfr732mk2tbalDgcbr9Wr37t0qKChovoBpqqCgQGVlZZ1uxNVXX63S0lK9//77kqQ333xTr7zyiq6//vqwP1NfX6/q6uoWX/FAyQkA0FOsX79e//qv/6odO3aooqKi1fMvvfSSjhw50uJr3LhxNrS0tQ4NCj5+/Lj8fr+ysrJaPJ6VlaX33nuv041YunSpqqurddlll8nlcsnv9+tHP/qR5syZE/ZnSkpK9MADD3T6NaPF1gcAgJ7g9OnT2rRpk3bt2qXKykpt3LhR9913X4tzLrjggrDDR+zmiFlOzz77rJ566in96le/0hVXXKHy8nItXrxYOTk5mjdvXps/U1xcrKKiotD31dXVys3NjXnbDJtmORmGoSFDhoSOAQBdkGVJvlp7Xtud1vwhFoVnn31Wl112mS699FLdcccdWrx4sYqLi7vMZ1CHAk3//v3lcrlUVVXV4vGqqqrzSmxLlizR0qVLdeutt0qSRo0apYMHD6qkpCRsoElOTg47JieWTJvWoXG73Zo/f35iXxQAEFu+WunHOfa89n0VkqdX1KevX79ed9xxhyRpxowZOnnypLZv367rrrsudM7VV1/dan/B06dPx6S556tDY2g8Ho/GjRun0tLS0GOBQEClpaXKz8/vdCNqa2tb3SCXy6VAolNEG75S/ayWJD2jpLpjdjcFAIC42L9/v3bu3KnbbrtNUuN+grNnz9b69etbnLdp0yaVl5e3+HKKDpecioqKNG/ePI0fP14TJ07U6tWrVVNTowULFkiS5s6dq8GDB6ukpERS40Diffv2hY4PHz6s8vJypaena/jw4ZKkr371q/rRj36kiy66SFdccYXeeOMNrVq1St/85jdj9Xt22vRTv1Vm0uf6Y91cu5sCAOhq3GmNPSV2vXaU1q9fr4aGBuXkNPcmWZal5ORkPfbYY6HHcnNzQ5/dTtPhQDN79mwdO3ZMy5YtU2VlpcaMGaOtW7eGBgofOnSoRW9LRUWFxo4dG/p+5cqVWrlypaZMmaJt27ZJkh599FHdf//9uvvuu3X06FHl5OToX/7lX7Rs2bLz/PXOX4PRuOWA4a9P6Ot6vV498sgjkqR77rmHrQ8AoCsyjA6VfezQ0NCg//7v/9ZPfvITTZs2rcVzM2fO1NNPP60ZM2bY1LrodWpQcGFhoQoLC9t8LhhSgvLy8iLOEOrdu7dWr17d5irCdvMFA01DYgON1FiKAwAgnrZs2aIvvvhCd955pzIyMlo8N2vWLK1fvz4UaD777DNVVla2OCczM1MpKSkJa2847LYdQaiHJuC1uSUAAMTe+vXrVVBQ0CrMSI2BZteuXaG13goKCjRo0KAWX88991yCW9w2R0zbdjKf2RhozASXnAAASITf//73YZ+bOHFiqMri9PXY6KGJINRD01Bnc0sAAEA4BJoIGkI9NJScAABwKgJNBMEeGjNAyQkAAKdiDE0EfpvG0BiGEVoPoKssOw0AgF0INBEES06uBAcat9uthQsXJvQ1AQDoqig5ReCn5AQA6CCnzwhKtETcDwJNBA1m4waYDAoGAETicrkkNa72jmbBhWLdbnfcXoOSUwShklOCe2h8Pp/WrFkjSVq0aFFc3wQAgNhISkpSWlqajh07Jrfb3Wrj5Z7GsizV1tbq6NGjyszMDAW+eCDQRBCwaVCwZVk6efJk6BgA4HyGYWjQoEE6cOCADh48aHdzHCMzM1PZ2dlxfQ0CTQT+ppKTK+CzuSUAgK7A4/FoxIgRlJ2auN3uuPbMBBFoIvC7goGGQcEAgOiYpumIDRt7kp5d3IuCXWNoAABA9Ag0ETSXnOg6BADAqQg0EQTMxtlFBBoAAJyLMTQRBGwaQ2MYhgYMGBA6BgAA4RFoIgg0lZySEtxD43a7dffddyf0NQEA6KooOUXgdwUHBVNyAgDAqQg0EQRLTknMcgIAwLEoOUXgNxvXEUiyEttD4/P59Pjjj0uSFi5cyNYHAAC0g0ATQaCp5JToMTSWZenYsWOhYwAAEB4lpwis4Cwni60PAABwKgJNBJbLnllOAAAgegSaCELr0Mgv+Rtsbg0AAGgLgSaCYKCRJDXU2dcQAAAQFoEmAqtpULAkqYGp2wAAOBGznCIwTZe8lksew5/QHhrDMJSRkRE6BgAA4RFoIjAMQ/XyyKMzCQ00brdbixcvTtjrAQDQlVFyisBlSPVqWtSOkhMAAI5EoInANI2zAg2DggEAcCJKThEYhqF6yy0ZSmgPjc/n08aNGyVJ8+fPZ+sDAADaQaCJwDy75ORPXKCxLEsVFRWhYwAAEB4lpwhMw2AMDQAADkegiaCxh6ZpLRrG0AAA4EgEmgjM4BgaiR4aAAAcikATQcuSEz00AAA4EYEmAtOUvMGx0/TQAADgSMxyisBsWilYUsJ7aNLS0hL6egAAdFUEmgiMFmNoEhdoPB6PlixZkrDXAwCgK6PkFIHJ1gcAADgegSYCF4OCAQBwPEpOERg2Lazn8/n01FNPSZLmzJnD1gcAALSDQBOBaUj1VnBQcGK3Pjh48GDoGAAAhEfJKQK2PgAAwPkINBGYphhDAwCAwxFoImgxhqb+lFT7uXTmC3sbBQAAWmAMTQSmYcgbXIfmo1LpoaGNx2PukGausa9hAAAghB6aCFyGod3WCH1hZLZ84sAOW9oDAABaI9BEYBrSJ1aWbuv9pHT/Z9K3tzU+EfDF/bXdbjfTtQEAiAIlpwgMw5Ak+WVIriQpKaXxCX98A43H49F9990X19cAAKC7oIcmArMxzygQXAvG1bQmTZwDDQAAiB6BJgKzKdEEgmvbmU2dWgkoOQEAgOhQcoqgdQ9N05gWvzeur9vQ0KBnn31WknTLLbcoKYn/VAAAhMOnZASmEeyhOafkFGiQLEtqej7WAoGAPvjgg9AxAAAIj5JTBKFAE8wU5lkZMNCQ+AYBAIBWCDQRBAONdW7JSYp72QkAAESHQBOBERpD0/RAsOQkMdMJAACHINBEEOyh8Qd7aCg5AQDgOASaCMymOxQqORlGc6ih5AQAgCMQaCJwGeesQyOxuB4AAA7DtO0IjHOnbUuS2TQwOI4lJ4/Ho+XLl8ft+gAAdCf00EQQWljv7C4aFyUnAACchEATQfO07bMepOQEAICjUHKKoNVKwVJzySmOgaahoUGbN2+WJH3ta19j6wMAANpBD00EwXVo/GcHmuDienHcoDIQCGjfvn3at28fWx8AABABgSaCVrttS2dtUEnJCQAAJ+hUoFmzZo3y8vKUkpKiSZMmaefOnWHP3bt3r2bNmqW8vDwZhqHVq1e3ed7hw4d1xx136IILLlBqaqpGjRqlXbt2daZ5MeU6d+sD6aySE4OCAQBwgg4Hmk2bNqmoqEjLly/Xnj17NHr0aE2fPl1Hjx5t8/za2loNGzZMK1asUHZ2dpvnfPHFF5o8ebLcbrdeeOEF7du3Tz/5yU/Ut2/fjjYv5sxztz6Qzio5sVIwAABO0OGRpqtWrdLChQu1YMECSdK6dev0hz/8QRs2bNDSpUtbnT9hwgRNmDBBktp8XpIefPBB5ebm6oknngg9NnTo0I42LS7aXIeGkhMAAI7SoR4ar9er3bt3q6CgoPkCpqmCggKVlZV1uhG/+93vNH78eN18880aOHCgxo4dq8cff7zdn6mvr1d1dXWLr3gI9tBY1lllJ0pOAAA4SocCzfHjx+X3+5WVldXi8aysLFVWVna6ER9//LHWrl2rESNG6I9//KPuuusu/du//ZuefPLJsD9TUlKijIyM0Fdubm6nX789wWnb0tk7blNyAgDASRyxuEkgEND48eP14x//WJI0duxYvfPOO1q3bp3mzZvX5s8UFxerqKgo9H11dXVcQk3LQGPJJSMhJSe3263i4uLQMQAACK9DgaZ///5yuVyqqqpq8XhVVVXYAb/RGDRokEaOHNniscsvv1y//e1vw/5McnKykpOTO/2a0TLP6sMKJLDkZBiGPB5P3K4PAEB30qGSk8fj0bhx41RaWhp6LBAIqLS0VPn5+Z1uxOTJk7V///4Wj73//vsaMmRIp68ZK2f30FiUnAAAcKQOl5yKioo0b948jR8/XhMnTtTq1atVU1MTmvU0d+5cDR48WCUlJZIaBxLv27cvdHz48GGVl5crPT1dw4cPlyR997vf1dVXX60f//jHuuWWW7Rz5079/Oc/189//vNY/Z6ddm7JSdJZJaf49dA0NDRoy5YtkqQbb7yRrQ8AAGhHhz8lZ8+erWPHjmnZsmWqrKzUmDFjtHXr1tBA4UOHDsk8q05TUVGhsWPHhr5fuXKlVq5cqSlTpmjbtm2SGqd2b968WcXFxfrhD3+ooUOHavXq1ZozZ855/nrn76w8c9ag4PhvThkIBPTmm29Kkr7yla/E7XUAAOgOOvXP/sLCQhUWFrb5XDCkBOXl5bVcZTeMG2+8UTfeeGNnmhNXZ/fQ+IOJxmy6bXHcywkAAESPvZwiMM/qobFalZwINAAAOAGBJgKX2dY6NPEvOQEAgOgRaCIw2hoUTMkJAABHIdBEoXmDSkpOAAA4EYEmCsGBwRYlJwAAHInFTaLQGGishM5ycrvd+t73vhc6BgAA4RFoomDYUHIyDEO9evWK2/UBAOhOKDlFITjTiZITAADORA9NFIJjaFrNcorz1gd//OMfJUnTp09n6wMAANpBD00UmktOTQ8Ee2jiuDllIBDQrl27tGvXLgUCgbi9DgAA3QGBJgqtemiYtg0AgKMQaKIQXIcmtPWBGf/dtgEAQPQINFEI9tD4g5WfYA9NHEtOAAAgegSaKBiUnAAAcDQCTRRcTXcpQMkJAABHItBEofXWB5ScAABwEhY3iYIds5zcbrfuueee0DEAAAiPQBOFVuvQJKDkZBiGMjMz43Z9AAC6E0pOUWie5XRODw0lJwAAHIEemii0WofGFf8eGr/fr9LSUknS1KlT5XK54vZaAAB0dfTQRME0g2Nomh5IwOaUfr9fZWVlKisrk9/vj9vrAADQHRBoohB2c0pKTgAAOAKBJgpmaFBw4kpOAAAgegSaKLRehyb+JScAABA9Ak0UjHNnOQWnbVt+KRAI81MAACBRCDRRaF1yOmtyWIBeGgAA7EagiYLLDFNykig7AQDgAKxDE4VWu22bZ21FEKceGrfbrbvuuit0DAAAwiPQRMFstfXBWYvcxamHxjAMDRw4MC7XBgCgu6HkFIVW69AYBjOdAABwEHpoohDqoQl10aix7OT3xm0tGr/fr7/85S+SpGuuuYatDwAAaAeBJgrNY2jOetDllnyK22rBfr9f27dvlyRdffXVBBoAANpBySkKraZtS2etFkzJCQAAuxFoouAyzxlDIzXPdGL7AwAAbEegiUKrrQ+k5h4aNqgEAMB2BJootFqHRqLkBACAgxBoohAcQ+M/d5aTRMkJAAAHINBEof2SEz00AADYjWnbUWh/llN8xtAkJSXpW9/6VugYAACExydlFMy21qGJc8nJNE0NHjw4LtcGAKC7oeQUhVZbH0iUnAAAcBB6aKJgNsU+K4ElJ7/fr9dee02S9A//8A+sFAwAQDsINFFoe+uD4OaU8dvL6aWXXpIkTZgwgUADAEA7KDlFIVhyajltuykLvrVJ2r/VhlYBAIAgAk0U2pzllNav8c+//0X67Z2JbxQAAAgh0ETB1dY6NFOWStf+e+Ox9zQrBgMAYCMCTRTa3PogY7B0zb3N3/vOJLhVAAAgiEATheaS0zlPJCU3HzfUJaw9AACgJQJNFNpch0aSDENKSmk8JtAAAGAbpm1HIbgOTaBVF40aA01DneSLbaBJSkrSvHnzQscAACA8Pimj0OY6NEHuVKnuhNQQ2zE0pmkqLy8vptcEAKC7ouQUBVe4kpPUPI4mxj00AAAgevTQRCE4KNhqM9CkNv4Z4zE0fr9fu3fvliSNGzeOlYIBAGgHgSYK7Zec4jMo2O/364UXXpAkjRkzhkADAEA7KDlFIewsJ6m5h4Z1aAAAsA2BJgrBkpO/vTE0TNsGAMA2BJoomGYbWx8EueMzhgYAAESPQBMFI7hScLh1aCRmOQEAYCMCTRRckdahkWK+Dg0AAIgegSYK7Q8KZh0aAADsxrTtKNixDk1SUpJuu+220DEAAAiPT8ooBNehaXOWU5zWoTFNU5dccklMrwkAQHdFySkKZntjaEKDghlDAwCAXeihiUL7JadgD019TF/T7/fr7bffliSNGjWKlYIBAGgHgSYKwXVoAoE2nozTLCe/36/nn39ekjRy5EgCDQAA7aDkFIX2ZzmxDg0AAHbrVKBZs2aN8vLylJKSokmTJmnnzp1hz927d69mzZqlvLw8GYah1atXt3vtFStWyDAMLV68uDNNi4tgyandMTSsQwMAgG06HGg2bdqkoqIiLV++XHv27NHo0aM1ffp0HT16tM3za2trNWzYMK1YsULZ2dntXvtvf/ubfvazn+mqq67qaLPiKthD0+YYGnd8xtAAAIDodTjQrFq1SgsXLtSCBQs0cuRIrVu3TmlpadqwYUOb50+YMEEPP/ywbr31ViUnJ4e97unTpzVnzhw9/vjj6tu3b0ebFVdGu5tTsts2AAB261Cg8Xq92r17twoKCpovYJoqKChQWVnZeTVk0aJFuuGGG1pcuz319fWqrq5u8RUv7U7bjtM6NAAAIHodCjTHjx+X3+9XVlZWi8ezsrJUWVnZ6UY888wz2rNnj0pKSqL+mZKSEmVkZIS+cnNzO/36kbhMBgUDAOBktk/b/uSTT3TPPffoxRdfVEpKStQ/V1xcrKKiotD31dXVcQs10a1DE/utD77xjW+EjgEAQHgd+qTs37+/XC6XqqqqWjxeVVUVccBvOLt379bRo0f1pS99KfSY3+/Xjh079Nhjj6m+vr7NNViSk5PbHZMTS8GtD9pfhyb2Wx9cccUVMb0mAADdVYdKTh6PR+PGjVNpaWnosUAgoNLSUuXn53eqAVOnTtXbb7+t8vLy0Nf48eM1Z84clZeXO2JBuejWoWFQMAAAdulwLaOoqEjz5s3T+PHjNXHiRK1evVo1NTVasGCBJGnu3LkaPHhwaDyM1+vVvn37QseHDx9WeXm50tPTNXz4cPXu3VtXXnlli9fo1auXLrjgglaP26V5HZp2Ak3AJwX8khmbABYIBPTuu+9Kki6//HKZJmsgAgAQTocDzezZs3Xs2DEtW7ZMlZWVGjNmjLZu3RoaKHzo0KEWH74VFRUaO3Zs6PuVK1dq5cqVmjJlirZt23b+v0ECRDXLSWosO3l6xeQ1Gxoa9Jvf/EZS43ghj8cTk+sCANAddWq0aWFhoQoLC9t87tyQkpeX1/Zg2nY4LegY0fTQSI0znWIUaAAAQPSoY0Shedp2G0+aLsl0Nx6z/QEAALYg0ESh3a0PpOaZTqxFAwCALVjgJArBktPrBz7X5BUvt3r+j/4kpUusFgwAgE0INFG4eEC6DEPyNgR0+ETrstIXHpfSTRFoAACwCYEmClcOztBfv/9lHT/dekftm9eVqV5NY2hYiwYAAFsQaKKUk5mqnMzUVo9nZ6So7lTTlOqG1oGns1wul2666abQMQAACI9Ac56y+pwdaGLXQ+NyuTRmzJiYXQ8AgO6MWU7nKbtPiuqsYMmJMTQAANiBHprzlJ2RonrFvocmEAjoww8/lCQNHz6crQ8AAGgHn5LnKatPiuqCg4JjOIamoaFBTz/9tJ5++mk1NDTE7LoAAHRHBJrzlN0nRXXBHhpmOQEAYAsCzXnKzkhRvRUsOTGGBgAAOxBozlPjGJrGkpPlpYcGAAA7EGjO08DeyTqjZElSXV2Nza0BAKBnItCcJ7fLlOFJkySdqTllc2sAAOiZCDQx4ElpDDT1Z07b3BIAAHom1qGJgeTUdKlW8sWw5ORyuXT99deHjgEAQHgEmhhISestfSb562tjdk2Xy6WJEyfG7HoAAHRnlJxiwJPaS5JkxHClYAAAED16aGLAbBoU7IrhOjSBQECHDh2SJF100UVsfQAAQDv4lIwBV3JjD01SIHY9NA0NDXryySf15JNPsvUBAAAREGhiICm5sYcmKRC7vZwAAED0CDQx4E5Jb/wzwNYHAADYgUATA8FBwckWPTQAANiBQBMD7pTGQOOx6iXLsrk1AAD0PASaGEhO6y1Jcikg+b02twYAgJ6HQBMDKU0lJ0mSlw0qAQBINNahiYG01BR5LZc8hl/yxWbqtsvlUkFBQegYAACER6CJgTRPkuqULI9q5ffWKhbxw+VyafLkyTG4EgAA3R8lpxhI87h0Rh5JUl3tKZtbAwBAz0MPTQwkJ5k6o2RJUv2ZGvWKcH40AoGAjhw5IkkaNGgQWx8AANAOPiVjwDAM1TcFGu+Z0zG5ZkNDg37xi1/oF7/4BVsfAAAQAYEmRrxmMNAwywkAgEQj0MSIz0iRJDXUE2gAAEg0Ak2M+Jp6aHx1BBoAABKNQBMjDa7GHhp/fa3NLQEAoOch0MSI35UqSQqwUjAAAAlHoIkRfxKBBgAAu7AOTYxYTYFG3thtfTBlypTQMQAACI9AEyNWUlrjnzHcy+m6666LybUAAOjuKDnFiruxh8bwMSgYAIBEo4cmRgxPU6BpiE0PjWVZOnbsmCRpwIABMgwjJtcFAKA7oocmRgxPY8nJ1VAXk+v5fD6tXbtWa9eulc/ni8k1AQDorgg0MeLyNG5J6fLHpocGAABEj0ATI2ZyUw+NPzY9NAAAIHoEmhhJago0SQECDQAAiUagiRF3SmPJyROot7klAAD0PASaGElKSZckeSx6aAAASDQCTYykpDb10Fj00AAAkGisQxMjnrTGHppUxaaHxuVyKT8/P3QMAADCI9DESHJab0mSW37J75Nc7vO6nsvl0rRp02LRNAAAuj0CTYykpqaHjv0HXpErOb2ds9vQb6jUq3+MWwUAQM9AoImR1NQ0+S1DLsOS65czO34BT7pU9K6U0kdS49YHJ0+elCRlZGSw9QEAAO0g0MRIstulDdYN+ifr9RaPD8pIldsVIYycOCR5T0unjoQCjc/n0yOPPCJJKi4ulsfjiUu7AQDoDgg0MWIYhvp//SGteu+oJOnl946quq5Bv/w/k/T/jIhQSnp4uFRzTAo0JKClAAB0PwSaGLppzGDdNGawJGnmmr+q/JMTqvVGEVLMpgHEfjahBACgM1iHJk7SPI1Trc/4/JFPNptyZSCKcwEAQCsEmjgJBRpvFCHFFQw09NAAANAZBJo4SXE3BpraaAJNqIeGMTQAAHQGgSZOOlZyYgwNAADng0HBcZLmaby1UZWczKatDc7qoTFNU+PHjw8dAwCA8Ag0cZLq6UDJKbhNwlmBJikpSTfccEM8mgYAQLfDP/3jJNUdLDlFM227KVdScgIAoFPooYmTDs1yMlv30FiWpdra2sZrpaWx9QEAAO2ghyZOOlRyamMMjc/n08qVK7Vy5Ur5fPTcAADQHgJNnDSXnDo3hgYAAESvU4FmzZo1ysvLU0pKiiZNmqSdO3eGPXfv3r2aNWuW8vLyZBiGVq9e3eqckpISTZgwQb1799bAgQM1c+ZM7d+/vzNNc4yOlZwYQwMAwPnocKDZtGmTioqKtHz5cu3Zs0ejR4/W9OnTdfTo0TbPr62t1bBhw7RixQplZ2e3ec727du1aNEivfbaa3rxxRfl8/k0bdo01dTUdLR5jpHaNG2bhfUAAIi/Dg8KXrVqlRYuXKgFCxZIktatW6c//OEP2rBhg5YuXdrq/AkTJmjChAmS1ObzkrR169YW32/cuFEDBw7U7t27de2113a0iY7QoZITgQYAgPPSoR4ar9er3bt3q6CgoPkCpqmCggKVlZXFrFEnT56UJPXr1y9m10y0tNCg4ChCCmNoAAA4Lx3qoTl+/Lj8fr+ysrJaPJ6VlaX33nsvJg0KBAJavHixJk+erCuvvDLsefX19aqvrw99X11dHZPXj5VUxtAAAJAwjluHZtGiRXrnnXf0yiuvtHteSUmJHnjggQS1quM6tpdT65KTaZoaPXp06BgAAITXoUDTv39/uVwuVVVVtXi8qqoq7IDfjigsLNSWLVu0Y8cOXXjhhe2eW1xcrKKiotD31dXVys3NPe82xEpwDI3Pb8nnD8jtaieUtBFokpKSNHPmzDi2EACA7qND//T3eDwaN26cSktLQ48FAgGVlpYqPz+/042wLEuFhYXavHmzXn75ZQ0dOjTizyQnJ6tPnz4tvpwkWHKSouilYQwNAADnpcMlp6KiIs2bN0/jx4/XxIkTtXr1atXU1IRmPc2dO1eDBw9WSUmJpMaBxPv27QsdHz58WOXl5UpPT9fw4cMlNZaZfvWrX+n5559X7969VVlZKUnKyMhQampqTH7RRPO4TLlMQ/6ApTNev/qkuMOf3MYYGsuyQisEu91utj4AAKAdHQ40s2fP1rFjx7Rs2TJVVlZqzJgx2rp1a2ig8KFDh1qM+aioqNDYsWND3weX858yZYq2bdsmSVq7dq0k6brrrmvxWk888YTmz5/f0SY6gmEYSnW7dLq+IfJaNG2UnHw+XygUFhcXy+PxxKupAAB0eZ0aFFxYWKjCwsI2nwuGlKC8vDxZltXu9SI931WlehoDTcSZTqxDAwDAeWH6TBw1z3SKEFQYQwMAwHkh0MRRcKZT5JJTU6BhHRoAADqFQBNHqZ5oA03TjKhAFGvWAACAVgg0cRQsOdVFPW2bHhoAADqDQBNHqe4od9xmUDAAAOfFcVsfdCfRl5xar0NjmqZGjhwZOgYAAOERaOIozR1lySnUQ9N8XlJSkm6++eZ4NQ0AgG6Ff/rHUXMPTbTTthlDAwBAZxBo4qjDJSfG0AAA0CmUnOKowyWns8bQeL1etj4AACBK9NDEUcd7aFiHBgCAziDQxFGaJ8pp24yhAQDgvFByiqNUT2NePHaqXu8cPhn2vPQv6pUn6UxdnT5qOq/B1xxu9lVU67LBfZXSVMICAAAtEWjiKNhDU/7JCd346Cthz7vG3K//zyMdOFodOi9Jfv1zauPz31j3qi4Z1Ff/e881cW8zAABdEYEmjiYN7aexF2XqyIm6ds/rE0iVGqQUV0DZfVIkSS7LL5016WnfkWqdrm9QejL/yQAAOBefjnGUmebR5rsnRz7xYKr0hDSsX7Je+9epkoKznMokSRmpbn12JqBPPq/V5YP6xLPJAAB0SQwKdoIwWx+MGDFCI0aMUG6/NEnSoc9r7WgdAACORw+NE4TZ+uD222+XJL36qz0qP3xahz4j0AAA0BZ6aJwgFGjanrZ9ET00AAC0i0DjBKF1aNre+oBAAwBA+yg5OUFoDE1zoPF6vVq5cqUk6ZpZCyRJnxBoAABoE4HGCcJsTulrWlzvwr6NC9J88kWt/AFLLtNIaPMAAHA6Sk5OEGEMTXafVLldhnx+S5XV7a9pAwBAT0SgcYIIY2hcpqEL+zaNo2GmEwAArVBycoJgD40VkAIByWydM3P7penA8Rr9bMdHeundqtDjef176Y5JF8kwKEMBAHouAo0TmGf9Zwg0SKan1SnDB6Rrx/vHtG1/49fZhvRL07WXDIh3KwEAcCwCjRO0CDQ+Sa0DzV3XXazMNLfO+JoX33vj0Bd67ePP9dTrBwk0AIAejUDjBMExNFJoHI1hGBoyZEjoeEBvj/5t6ogWP/Z+1SlN++kOvfTuUVVV1ymraWNLAAB6GgKNE5zdQ9O0Fo3b7db8+fPb/bFLsnpr/JC+2nXwC337v3dpUEZqHBsJpxiRla75V+fpgvRku5sCAI5BoHEC0yXJkGSFnekUzj/nD9Gug1/ozU9P6s1PT8aleXCWrXuln+34WJmp7sgnJ9i1lwzQw9+4ikHqABKOQOMULrfk94Zdiyac/zM6R26Xqc9qvHFqGJykwR/Q/+w5rLcPn9TRU/V2N6eV3+z+VMXXX0bvEYCEI9A4hZnUFGgae2i8Xq8eeeQRSdI999wjj6f1QGGpcXzNV0YNSlgzYb/5V+fp4+M1qvcF7G5KC9968m+qOFmnj47VEGgAJByBxinMpvLBWfs51dayiB5aMwxDFw9It7sZrQzP6q2Kk3X6+NhpTRzaz+7mAOhhWCnYKUxX458dHEMDOMXFA3pJkj46dtrmlgDoiQg0ThHa/qBjY2gApxjW1Gv08bEam1sCoCci0DhFmB23ga6CHhoAdiLQOEUw0PgJNOiaguN6PvnijOob/BHOBoDYItA4BT006OIG9k5WenKS/AGLXeEBJByBxinOGUNjGIZycnKUk5PDImXoEhpnX1F2AmAPpm07xTk9NG63WwsXLrSxQUDHDRuQrjc/Pakf/G6fHin9MPT4Bb08WjA5T1df3F9Oy+dulymX6bBGAegwAo1TMIYG3cC4IX21+Y3DqqyuU2V1XYvnXvnwuE2tal/v5CT9c/4QTR7eX06MNYZhaHRuhtI8/O8aaA9/Q5wi1EPDtG10XbdPvEiXD+qjmvrmYG5JevWj4/pl2UHVeJ03WPhUfYP+77aP9H+3fWR3U8KacskAPfnNiXY3A3A0Ao1ThMbQNH4Q+Hw+rVmzRpK0aNEiud3O24gQOJdpGho3pG+rx6dcMkBLpl2q+gZnbddgSSr76DNteOWAPnfgfmg+f0AfH6/R24fZeBaIhEDjFKGSU2MPjWVZOnnyZOgY6OqSXKaSXM6bh/BPI7P0TyOz7G5Gm06e8Wn0A3/S5zVenfH6lepx2d0kwLGc93+XnipUcnJelzwAe/RJSVJ6cuP/Gw6fOGNzawBnI9A4BWNoAJzDMAzlZKZIkioINEC7CDROcc4YGgCQpMGZqZIINEAkBBqnCO627aeHBkCznKZAQ8kJaB+BxinMYA8NY2gANCPQANFhlpNTnDOGxjAMDRgwIHQMoGe6sC8lJyAaBBqnOGcMjdvt1t13321jgwA4AT00QHQoOTlFaAwNg4IBNAsGmsqTdfIHWJMKCIdA4xQms5wAtJbVO1ku05DPb+n46Xq7mwM4FiUnpzhnDI3P59Pjjz8uSVq4cCFbHwA9VJLLVHafFB0+cUbvV51SSlL3WS042W0qxd19fh/Yi0DjFOeMobEsS8eOHQsdA+i5cjIbA80/r99pd1NiKjnJ1C+/NUkT8vrZ3RR0A5ScnIIxNADC+MqoQXKZ3W+2Y31DQDsPfG53M9BN0EPjFIyhARDGgslDNTc/r1v11pa88J7Wv3JAJ8+wmChig0DjFKGSE3+5AbTW2EPTfXpp+vXySJJO1Hptbgm6C0pOThEsOdFDA6AHyEht/EccPTSIFQKNUwRLToyhAdADBAPNiVoCDWKDkpNThKZtNwYawzCUkZEROgaA7iQzjR4axBaBxinOGUPjdru1ePFi+9oDAHGUmRocQ0OgQWxQcnIKxtAA6EHooUGs0UPjFMExNH9/RXriK/a2BQDibFAgoE2eLyRJgQ2PyKS03jXM/qWU5syFEAk0TpFxYeOfZ76QDv5VPiVpo26RJM3Xs3KLnhsA3UeSpEnBGsEhO1uCDvE7t0eNQOMUF39ZWvCCdPqoJMlq8KviuXcaj2f+XOpG+7cAgCQt+c2bOl3v1/03Xq6cjFS7m4NopGTY3YKwCDROYRjSkKubv/d6paZAo8u/Knk89rQLAOJk1wuZOnCmRt/MyVcO+znhPHVqUPCaNWuUl5enlJQUTZo0STt3ht8wbe/evZo1a5by8vJkGIZWr1593tcEAHR9rEWDWOpwoNm0aZOKioq0fPly7dmzR6NHj9b06dN19OjRNs+vra3VsGHDtGLFCmVnZ8fkmgCArq850LD9Ac5fhwPNqlWrtHDhQi1YsEAjR47UunXrlJaWpg0bNrR5/oQJE/Twww/r1ltvVXJyckyuCQDo+pi6jVjqUKDxer3avXu3CgoKmi9gmiooKFBZWVmnGtDZa9bX16u6urrFFwCg68hkPyfEUIcCzfHjx+X3+5WVldXi8aysLFVWVnaqAZ29ZklJiTIyMkJfubm5nXp9J0tLS1NaWprdzQCAuMhIY7VgxE6XneVUXFysoqKi0PfV1dXdKtR4PB4tWbLE7mYAQNwEe2hO0EODGOhQoOnfv79cLpeqqqpaPF5VVRV2wG+8rpmcnBx2TA4AwPkYFIxY6lDJyePxaNy4cSotLQ09FggEVFpaqvz8/E41IB7XBAA4X3BQcDU9NIiBDpecioqKNG/ePI0fP14TJ07U6tWrVVNTowULFkiS5s6dq8GDB6ukpERS46Dfffv2hY4PHz6s8vJypaena/jw4VFdsyfy+Xx66qmnJElz5syR2+22uUUAEFvBQEPJCbHQ4UAze/ZsHTt2TMuWLVNlZaXGjBmjrVu3hgb1Hjp0SKbZ3PFTUVGhsWPHhr5fuXKlVq5cqSlTpmjbtm1RXbMnsixLBw8eDB0DQHeTkdo4KLiquk4P/H6vza2BJN028SJdktXb7mZ0imF1k0/L6upqZWRk6OTJk+rTp4/dzTlvXq831MtVXFwsD1sfAOhmTtb69KX/90X5A93iY6hb2DB/vL58WWI7E2L1+d1lZzkBALq2jDS31s75kt789ITdTUGTIRf0srsJnUagAQDYZtoV2Zp2RedmyQJn69TmlAAAAE5CoAEAAF0eJScHY6o2AADRYZYTAACwTaw+vyk5AQCALo9AAwAAujzG0DhUQ0ODnn32WUnSLbfcoqQk/lMBABAOn5IOFQgE9MEHH4SOAQBAeJScAABAl0egAQAAXR6BBgAAdHkEGgAA0OURaAAAQJfXbWY5BRc8rq6utrklseH1elVXVyep8XfyeDw2twgAgNgLfm6f78YF3Wbrg08//VS5ubl2NwMAAHTCJ598ogsvvLDTP99tAk0gEFBFRYV69+4twzBidt3q6mrl5ubqk08+YY+oDuC+dQ73rXO4b53Dfesc7lvnhLtvlmXp1KlTysnJkWl2fiRMtyk5maZ5Xskukj59+vDG7QTuW+dw3zqH+9Y53LfO4b51Tlv3LSMj47yvy6BgAADQ5RFoAABAl0egiSA5OVnLly9XcnKy3U3pUrhvncN96xzuW+dw3zqH+9Y58b5v3WZQMAAA6LnooQEAAF0egQYAAHR5BBoAANDlEWgAAECXR6CJYM2aNcrLy1NKSoomTZqknTt32t0kx/jBD34gwzBafF122WWh5+vq6rRo0SJdcMEFSk9P16xZs1RVVWVji+2zY8cOffWrX1VOTo4Mw9Bzzz3X4nnLsrRs2TINGjRIqampKigo0AcffNDinM8//1xz5sxRnz59lJmZqTvvvFOnT59O4G+ReJHu2/z581u9B2fMmNHinJ5230pKSjRhwgT17t1bAwcO1MyZM7V///4W50Tzd/PQoUO64YYblJaWpoEDB2rJkiVqaGhI5K+SUNHct+uuu67V++073/lOi3N62n1bu3atrrrqqtBiefn5+XrhhRdCzyfyvUagacemTZtUVFSk5cuXa8+ePRo9erSmT5+uo0eP2t00x7jiiit05MiR0Ncrr7wSeu673/2ufv/73+vXv/61tm/froqKCn3961+3sbX2qamp0ejRo7VmzZo2n3/ooYf0X//1X1q3bp1ef/119erVS9OnTw9tUCpJc+bM0d69e/Xiiy9qy5Yt2rFjh7797W8n6lewRaT7JkkzZsxo8R58+umnWzzf0+7b9u3btWjRIr322mt68cUX5fP5NG3aNNXU1ITOifR30+/364YbbpDX69Wrr76qJ598Uhs3btSyZcvs+JUSIpr7JkkLFy5s8X576KGHQs/1xPt24YUXasWKFdq9e7d27dqlL3/5y7rpppu0d+9eSQl+r1kIa+LEidaiRYtC3/v9fisnJ8cqKSmxsVXOsXz5cmv06NFtPnfixAnL7XZbv/71r0OPvfvuu5Ykq6ysLEEtdCZJ1ubNm0PfBwIBKzs723r44YdDj504ccJKTk62nn76acuyLGvfvn2WJOtvf/tb6JwXXnjBMgzDOnz4cMLabqdz75tlWda8efOsm266KezPcN8s6+jRo5Yka/v27ZZlRfd383//938t0zStysrK0Dlr1661+vTpY9XX1yf2F7DJuffNsixrypQp1j333BP2Z7hvjfr27Wv94he/SPh7jR6aMLxer3bv3q2CgoLQY6ZpqqCgQGVlZTa2zFk++OAD5eTkaNiwYZozZ44OHTokSdq9e7d8Pl+L+3fZZZfpoosu4v6d48CBA6qsrGxxrzIyMjRp0qTQvSorK1NmZqbGjx8fOqegoECmaer1119PeJudZNu2bRo4cKAuvfRS3XXXXfrss89Cz3HfpJMnT0qS+vXrJym6v5tlZWUaNWqUsrKyQudMnz5d1dXVoX95d3fn3regp556Sv3799eVV16p4uJi1dbWhp7r6ffN7/frmWeeUU1NjfLz8xP+Xus2m1PG2vHjx+X3+1vcZEnKysrSe++9Z1OrnGXSpEnauHGjLr30Uh05ckQPPPCArrnmGr3zzjuqrKyUx+NRZmZmi5/JyspSZWWlPQ12qOD9aOu9FnyusrJSAwcObPF8UlKS+vXr16Pv54wZM/T1r39dQ4cO1UcffaT77rtP119/vcrKyuRyuXr8fQsEAlq8eLEmT56sK6+8UpKi+rtZWVnZ5vsx+Fx319Z9k6Tbb79dQ4YMUU5Ojt566y19//vf1/79+/U///M/knrufXv77beVn5+vuro6paena/PmzRo5cqTKy8sT+l4j0KDTrr/++tDxVVddpUmTJmnIkCF69tlnlZqaamPL0FPceuutoeNRo0bpqquu0sUXX6xt27Zp6tSpNrbMGRYtWqR33nmnxdg2RBbuvp099mrUqFEaNGiQpk6dqo8++kgXX3xxopvpGJdeeqnKy8t18uRJ/eY3v9G8efO0ffv2hLeDklMY/fv3l8vlajUau6qqStnZ2Ta1ytkyMzN1ySWX6MMPP1R2dra8Xq9OnDjR4hzuX2vB+9Heey07O7vVYPSGhgZ9/vnn3M+zDBs2TP3799eHH34oqWfft8LCQm3ZskV//vOfdeGFF4Yej+bvZnZ2dpvvx+Bz3Vm4+9aWSZMmSVKL91tPvG8ej0fDhw/XuHHjVFJSotGjR+uRRx5J+HuNQBOGx+PRuHHjVFpaGnosEAiotLRU+fn5NrbMuU6fPq2PPvpIgwYN0rhx4+R2u1vcv/379+vQoUPcv3MMHTpU2dnZLe5VdXW1Xn/99dC9ys/P14kTJ7R79+7QOS+//LICgUDof6qQPv30U3322WcaNGiQpJ553yzLUmFhoTZv3qyXX35ZQ4cObfF8NH838/Pz9fbbb7cIgy+++KL69OmjkSNHJuYXSbBI960t5eXlktTi/dbT7ltbAoGA6uvrE/9ei8WI5u7qmWeesZKTk62NGzda+/bts7797W9bmZmZLUZj92T33nuvtW3bNuvAgQPWX//6V6ugoMDq37+/dfToUcuyLOs73/mOddFFF1kvv/yytWvXLis/P9/Kz8+3udX2OHXqlPXGG29Yb7zxhiXJWrVqlfXGG29YBw8etCzLslasWGFlZmZazz//vPXWW29ZN910kzV06FDrzJkzoWvMmDHDGjt2rPX6669br7zyijVixAjrtttus+tXSoj27tupU6es733ve1ZZWZl14MAB66WXXrK+9KUvWSNGjLDq6upC1+hp9+2uu+6yMjIyrG3btllHjhwJfdXW1obOifR3s6GhwbryyiutadOmWeXl5dbWrVutAQMGWMXFxXb8SgkR6b59+OGH1g9/+ENr165d1oEDB6znn3/eGjZsmHXttdeGrtET79vSpUut7du3WwcOHLDeeusta+nSpZZhGNaf/vQny7IS+14j0ETw6KOPWhdddJHl8XisiRMnWq+99prdTXKM2bNnW4MGDbI8Ho81ePBga/bs2daHH34Yev7MmTPW3XffbfXt29dKS0uzvva1r1lHjhyxscX2+fOf/2xJavU1b948y7Iap27ff//9VlZWlpWcnGxNnTrV2r9/f4trfPbZZ9Ztt91mpaenW3369LEWLFhgnTp1yobfJnHau2+1tbXWtGnTrAEDBlhut9saMmSItXDhwlb/4Ohp962t+yXJeuKJJ0LnRPN38+9//7t1/fXXW6mpqVb//v2te++91/L5fAn+bRIn0n07dOiQde2111r9+vWzkpOTreHDh1tLliyxTp482eI6Pe2+ffOb37SGDBlieTwea8CAAdbUqVNDYcayEvteMyzLsjrWpwMAAOAsjKEBAABdHoEGAAB0eQQaAADQ5RFoAABAl0egAQAAXR6BBgAAdHkEGgAA0OURaAAAQJdHoAEAAF0egQYAAHR5BBoAANDlEWgAAECX9/8DOWbvXFwCHHkAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(minseries(metrics_rn[\"val_error\"]), label=\"RN\")\n", + "plt.plot(minseries(metrics_ae[\"val_error\"]), label=\"AE\")\n", + "plt.axvline(20, color=\"grey\", linestyle=\"--\")\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "hannah-rvIoGOA8-py3.10", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/experiments/embedded_vision_net_ri/scripts/experiment_slurm.sh b/experiments/embedded_vision_net_ri/scripts/experiment_slurm.sh new file mode 100755 index 00000000..b4bd63e9 --- /dev/null +++ b/experiments/embedded_vision_net_ri/scripts/experiment_slurm.sh @@ -0,0 +1,65 @@ +#!/bin/bash +## +## Copyright (c) 2023 Hannah contributors. +## +## This file is part of hannah. +## See https://github.com/ekut-es/hannah for further info. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + + +#SBATCH --job-name=ri_random_nas + +#resources: + +#SBATCH --partition=gpu-2080ti +# the slurm partition the job is queued to. +# FIXME: test if preemptable is avallable + +#SBATCH --nodes=1 +# requests that the cores are all on one node + +#SBATCH --gres=gpu:rtx2080ti:2 +#the job can use and see 4 GPUs (8 GPUs are available in total on one node) + +#SBATCH --time=4320 +# the maximum time the scripts needs to run (720 minutes = 12 hours) + +#SBATCH --error=jobs/%j.err +# write the error output to job.*jobID*.err + +#SBATCH --output=jobs/%j.out +# write the standard output to your home directory job.*jobID*.out + +#SBATCH --mail-type=ALL +#write a mail if a job begins, ends, fails, gets requeued or stages out + +#SBATCH --mail-user=christoph.gerum@uni-tuebingen.de +# your mail address + + +#Script +echo "Job information" +scontrol show job $SLURM_JOB_ID + + +export HANNAH_DATA_FOLDER=/mnt/qb/datasets/STAGING/bringmann/datasets/ + +GPUS=2 +BATCH_SIZE=32 + +# trainer=sharded + +hannah-train +experiment=$1 model=embedded_vision_net trainer.gpus=${gpus} module.batch_size=${BATCH_SIZE} +module.num_workers=16 diff --git a/experiments/embedded_vision_net_ri/scripts/run-ae_nas-slurm-cifar10.sh b/experiments/embedded_vision_net_ri/scripts/run-ae_nas-slurm-cifar10.sh new file mode 100755 index 00000000..f8c72ced --- /dev/null +++ b/experiments/embedded_vision_net_ri/scripts/run-ae_nas-slurm-cifar10.sh @@ -0,0 +1,62 @@ +#!/bin/bash +## +## Copyright (c) 2023 Hannah contributors. +## +## This file is part of hannah. +## See https://github.com/ekut-es/hannah for further info. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + + +#SBATCH --job-name=run-random_nas + +#resources: + +#SBATCH --partition=gpu-2080ti +# the slurm partition the job is queued to. +# FIXME: test if preemptable is avallable + +#SBATCH --nodes=1 +# requests that the cores are all on one node + +#SBATCH --gres=gpu:rtx2080ti:8 +#the job can use and see 4 GPUs (8 GPUs are available in total on one node) + +#SBATCH --time=4320 +# the maximum time the scripts needs to run (720 minutes = 12 hours) + +#SBATCH --error=jobs/%j.err +# write the error output to job.*jobID*.err + +#SBATCH --output=jobs/%j.out +# write the standard output to your home directory job.*jobID*.out + +#SBATCH --mail-type=ALL +#write a mail if a job begins, ends, fails, gets requeued or stages out + +#SBATCH --mail-user=moritz.reiber@uni-tuebingen.de +# your mail address + + +#Script +echo "Job information" +scontrol show job $SLURM_JOB_ID + + + +# export HANNAH_DATA_FOLDER=/mnt/qb/datasets/STAGING/bringmann/datasets/ +conda activate hannah + + +hannah-train trainer.gpus=8 experiment=ae_nas_cifar10_v2 model=embedded_vision_net dataset=cifar10 model.num_classes=10 nas.n_jobs=8 fx_mac_summary=True ~normalizer diff --git a/experiments/embedded_vision_net_ri/scripts/run-ae_nas-slurm-ri.sh b/experiments/embedded_vision_net_ri/scripts/run-ae_nas-slurm-ri.sh new file mode 100755 index 00000000..32786e1e --- /dev/null +++ b/experiments/embedded_vision_net_ri/scripts/run-ae_nas-slurm-ri.sh @@ -0,0 +1,62 @@ +#!/bin/bash +## +## Copyright (c) 2023 Hannah contributors. +## +## This file is part of hannah. +## See https://github.com/ekut-es/hannah for further info. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + + +#SBATCH --job-name=run-random_nas + +#resources: + +#SBATCH --partition=gpu-2080ti +# the slurm partition the job is queued to. +# FIXME: test if preemptable is avallable + +#SBATCH --nodes=1 +# requests that the cores are all on one node + +#SBATCH --gres=gpu:rtx2080ti:8 +#the job can use and see 4 GPUs (8 GPUs are available in total on one node) + +#SBATCH --time=4320 +# the maximum time the scripts needs to run (720 minutes = 12 hours) + +#SBATCH --error=jobs/%j.err +# write the error output to job.*jobID*.err + +#SBATCH --output=jobs/%j.out +# write the standard output to your home directory job.*jobID*.out + +#SBATCH --mail-type=ALL +#write a mail if a job begins, ends, fails, gets requeued or stages out + +#SBATCH --mail-user=moritz.reiber@uni-tuebingen.de +# your mail address + + +#Script +echo "Job information" +scontrol show job $SLURM_JOB_ID + + + +export HANNAH_DATA_FOLDER=/mnt/qb/datasets/STAGING/bringmann/datasets/ +conda activate hannah + + +hannah-train trainer.gpus=8 experiment=ae_nas_ri model=embedded_vision_net model.num_classes=4 nas.n_jobs=8 fx_mac_summary=True ~normalizer diff --git a/experiments/embedded_vision_net_ri/scripts/run-random_nas-slurm-cifar10.sh b/experiments/embedded_vision_net_ri/scripts/run-random_nas-slurm-cifar10.sh new file mode 100755 index 00000000..e8008d65 --- /dev/null +++ b/experiments/embedded_vision_net_ri/scripts/run-random_nas-slurm-cifar10.sh @@ -0,0 +1,62 @@ +#!/bin/bash +## +## Copyright (c) 2023 Hannah contributors. +## +## This file is part of hannah. +## See https://github.com/ekut-es/hannah for further info. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + + +#SBATCH --job-name=run-random_nas + +#resources: + +#SBATCH --partition=gpu-2080ti +# the slurm partition the job is queued to. +# FIXME: test if preemptable is avallable + +#SBATCH --nodes=1 +# requests that the cores are all on one node + +#SBATCH --gres=gpu:rtx2080ti:8 +#the job can use and see 4 GPUs (8 GPUs are available in total on one node) + +#SBATCH --time=4320 +# the maximum time the scripts needs to run (720 minutes = 12 hours) + +#SBATCH --error=jobs/%j.err +# write the error output to job.*jobID*.err + +#SBATCH --output=jobs/%j.out +# write the standard output to your home directory job.*jobID*.out + +#SBATCH --mail-type=ALL +#write a mail if a job begins, ends, fails, gets requeued or stages out + +#SBATCH --mail-user=moritz.reiber@uni-tuebingen.de +# your mail address + + +#Script +echo "Job information" +scontrol show job $SLURM_JOB_ID + + + +# export HANNAH_DATA_FOLDER=/mnt/qb/datasets/STAGING/bringmann/datasets/ +conda activate hannah + + +hannah-train trainer.gpus=8 experiment=random_nas_cifar10 model=embedded_vision_net dataset=cifar10 model.num_classes=10 nas.n_jobs=8 fx_mac_summary=True ~normalizer diff --git a/experiments/embedded_vision_net_ri/scripts/run-random_nas-slurm.sh b/experiments/embedded_vision_net_ri/scripts/run-random_nas-slurm.sh new file mode 100755 index 00000000..87a98b8f --- /dev/null +++ b/experiments/embedded_vision_net_ri/scripts/run-random_nas-slurm.sh @@ -0,0 +1,62 @@ +#!/bin/bash +## +## Copyright (c) 2023 Hannah contributors. +## +## This file is part of hannah. +## See https://github.com/ekut-es/hannah for further info. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + + +#SBATCH --job-name=run-random_nas + +#resources: + +#SBATCH --partition=gpu-2080ti +# the slurm partition the job is queued to. +# FIXME: test if preemptable is avallable + +#SBATCH --nodes=1 +# requests that the cores are all on one node + +#SBATCH --gres=gpu:rtx2080ti:8 +#the job can use and see 4 GPUs (8 GPUs are available in total on one node) + +#SBATCH --time=4320 +# the maximum time the scripts needs to run (720 minutes = 12 hours) + +#SBATCH --error=jobs/%j.err +# write the error output to job.*jobID*.err + +#SBATCH --output=jobs/%j.out +# write the standard output to your home directory job.*jobID*.out + +#SBATCH --mail-type=ALL +#write a mail if a job begins, ends, fails, gets requeued or stages out + +#SBATCH --mail-user=christoph.gerum@uni-tuebingen.de +# your mail address + + +#Script +echo "Job information" +scontrol show job $SLURM_JOB_ID + + + +export HANNAH_DATA_FOLDER=/mnt/qb/datasets/STAGING/bringmann/datasets/ +conda activate hannah + + +hannah-train trainer.gpus=8 module.batch_size=16 experiment=random_nas model=embedded_vision_net nas.n_jobs=8 fx_mac_summary=True ~normalizer diff --git a/experiments/embedded_vision_net_ri/scripts/train_all_baselines.sh b/experiments/embedded_vision_net_ri/scripts/train_all_baselines.sh new file mode 100755 index 00000000..19153763 --- /dev/null +++ b/experiments/embedded_vision_net_ri/scripts/train_all_baselines.sh @@ -0,0 +1,31 @@ +#!/bin/bash +## +## Copyright (c) 2023 Hannah contributors. +## +## This file is part of hannah. +## See https://github.com/ekut-es/hannah for further info. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +EXPERIMENT="baseline" +MODELS="timm_resnet18 timm_resnet50 timm_resnet152 timm_efficientnet_lite1 timm_mobilenetv3_small_100 timm_mobilenetv3_small_075 timm_mobilenetv3_large_100 timm_focalnet_base_srf" + +export HANNAH_DATA_FOLDER=/mnt/qb/datasets/STAGING/bringmann/datasets/ + +SBATCH="" +#SBATCH="sbatch --job-name=baseline" + +for MODEL in ${MODELS}; do + sbatch --job-name=$MODEL scripts/train_baselines_slurm.sh -m $MODEL +done diff --git a/experiments/embedded_vision_net_ri/scripts/train_all_baselines_submitit.sh b/experiments/embedded_vision_net_ri/scripts/train_all_baselines_submitit.sh new file mode 100755 index 00000000..14f566be --- /dev/null +++ b/experiments/embedded_vision_net_ri/scripts/train_all_baselines_submitit.sh @@ -0,0 +1,29 @@ +#!/bin/bash +## +## Copyright (c) 2023 Hannah contributors. +## +## This file is part of hannah. +## See https://github.com/ekut-es/hannah for further info. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +EXPERIMENT="baseline" +MODEL="timm_resnet18,timm_resnet50,timm_resnet152,timm_efficientnet_lite1,timm_mobilenetv3_small_100,timm_mobilenetv3_small_075,timm_mobilenetv3_large_100" + +export HANNAH_DATA_FOLDER=/mnt/qb/datasets/STAGING/bringmann/datasets/ + +hannah-train experiment_id=$EXPERIMENT model=$MODEL hydra/launcher=ml_cloud_4gpu \ + hydra.sweep.dir='${output_dir}/${experiment_id}/' hydra.sweep.subdir='${model.name}' \ + module.num_workers=8 module.batch_size=16 trainer.gpus=4 trainer=sharded \ + -m diff --git a/hannah/conf/nas/aging_evolution_nas.yaml b/hannah/conf/nas/aging_evolution_nas.yaml index 897bd889..e71d9844 100644 --- a/hannah/conf/nas/aging_evolution_nas.yaml +++ b/hannah/conf/nas/aging_evolution_nas.yaml @@ -25,10 +25,10 @@ defaults: _target_: hannah.nas.search.search.DirectNAS budget: 2000 n_jobs: 10 -presample: False +presample: True total_candidates: 100 -num_selected_candidates: 10 +num_selected_candidates: 20 bounds: - val_error: 0.06 + val_error: 0.12 total_macs: 128000000 total_weights: 500000 diff --git a/hannah/conf/nas/sampler/aging_evolution.yaml b/hannah/conf/nas/sampler/aging_evolution.yaml index f0cacb21..761b04e4 100644 --- a/hannah/conf/nas/sampler/aging_evolution.yaml +++ b/hannah/conf/nas/sampler/aging_evolution.yaml @@ -1,3 +1,3 @@ _target_: hannah.nas.search.sampler.aging_evolution.AgingEvolutionSampler bounds: {} -population_size: 100 \ No newline at end of file +population_size: 20 \ No newline at end of file diff --git a/hannah/models/embedded_vision_net/models.py b/hannah/models/embedded_vision_net/models.py index d628b371..f6d1b761 100644 --- a/hannah/models/embedded_vision_net/models.py +++ b/hannah/models/embedded_vision_net/models.py @@ -114,15 +114,14 @@ def stem(input, kernel_size, stride, out_channels): @scope def classifier_head(input, num_classes): - out = adaptive_avg_pooling(input) - # out = input + out = choice(input, adaptive_avg_pooling) out = linear(out, num_classes) return out def search_space(name, input, num_classes=10): - out_channels = IntScalarParameter(4, 64, name='out_channels') - kernel_size = CategoricalParameter([1, 3, 5], name='kernel_size') + out_channels = IntScalarParameter(32, 256, step_size=4, name='out_channels') + kernel_size = CategoricalParameter([3, 5, 7, 9], name='kernel_size') stride = CategoricalParameter([1, 2], name='stride') expand_ratio = IntScalarParameter(1, 6, name='expand_ratio') reduce_ratio = IntScalarParameter(1, 6, name='reduce_ratio') @@ -139,7 +138,7 @@ def search_space(name, input, num_classes=10): exits.append(out) out = dynamic_depth(*exits, switch=num_blocks) - out = classifier_head(out, num_classes=num_classes) # FIXME: Configure num_classes automatically + out = classifier_head(out, num_classes=num_classes) strides = [v for k, v in out.parametrization(flatten=True).items() if k.split('.')[-1] == 'stride'] total_stride = expr_product(strides) diff --git a/hannah/nas/eval/__main__.py b/hannah/nas/eval/__main__.py index f9d78e47..33631931 100644 --- a/hannah/nas/eval/__main__.py +++ b/hannah/nas/eval/__main__.py @@ -36,7 +36,7 @@ def main(config): logger.info("Current working directory %s", os.getcwd()) result_metrics, parameters = prepare_summary( config.data, - base_path=hydra.utils.get_original_cwd(), + base_dir=hydra.utils.get_original_cwd(), force=config.get("force", False), ) diff --git a/hannah/nas/eval/extract.py b/hannah/nas/eval/extract.py index 2b8ae3ca..d6d9884f 100644 --- a/hannah/nas/eval/extract.py +++ b/hannah/nas/eval/extract.py @@ -22,7 +22,7 @@ import pandas as pd import yaml from genericpath import exists -from hannah_optimizer.utils import is_pareto +from hannah.nas.utils import is_pareto logger = logging.getLogger("nas_eval.extract") diff --git a/hannah/nas/eval/prepare.py b/hannah/nas/eval/prepare.py index f5b4a80c..ff55461b 100644 --- a/hannah/nas/eval/prepare.py +++ b/hannah/nas/eval/prepare.py @@ -65,7 +65,7 @@ def prepare_summary( parameters_all = {} for name, source in data.items(): logger.info(" Extracting design points for task: %s", name) - history_path = base_path / source / "history.pkl" + history_path = base_path / source / "history.yml" if history_path.suffix == ".yml": with history_path.open("r") as yaml_file: diff --git a/hannah/nas/search/sampler/aging_evolution.py b/hannah/nas/search/sampler/aging_evolution.py index 7cf10314..1096e515 100644 --- a/hannah/nas/search/sampler/aging_evolution.py +++ b/hannah/nas/search/sampler/aging_evolution.py @@ -28,6 +28,7 @@ from hannah.nas.parameters.parameters import CategoricalParameter, FloatScalarParameter, IntScalarParameter from hannah.nas.parameters.parametrize import set_parametrization from hannah.nas.search.sampler.mutator import ParameterMutator +from hannah.nas.search.utils import np_to_primitive from ...parametrization import SearchSpace from ...utils import is_pareto @@ -122,7 +123,7 @@ def next_parameters(self): def tell_result(self, parameters, metrics): "Tell the result of a task" - + parameters = np_to_primitive(parameters) result = SearchResult(len(self.history), parameters, metrics) self.history.append(result)