Generic train generate build dev

.dockerignore

@@ -0,0 +1,8 @@
+*?Dockerfile
+README.md
+.vscode
+.git
+venv.local
+tutorials
+__pycache__
+/common/docker/*/sample-inputs

.gitignore

@@ -0,0 +1,8 @@
+# CPython cache directory
+__pycache__
+
+# Development virtual environment
+venv.local
+
+# CubeIDE installation 
+en.st-stm32cubeide_1.14.0_19471_20231121_1200_amd64.deb_bundle.sh.zip

.python-version

@@ -0,0 +1 @@
+3.10

common/docker/build/README.md

@@ -0,0 +1,14 @@
+# Generic build workload meant to be run from jobcontrol-api
+
+## Runtime interface
+
+### Input
+
+
+### Output
+
+
+## Build
+
+
+## jobcontrol-api

common/docker/build/api.http

@@ -0,0 +1,42 @@
+@base_url = https://dev.stm-vespucci.com
+@jobcontrol_api_base_path = /svc/jobcontrol/v1alpha1
+@token = dummy
+###
+# @name create
+POST {{base_url}}{{jobcontrol_api_base_path}}/jobs
+Authorization: Bearer {{token}}
+Accept: application/json
+Content-Type: multipart/form-data; boundary=------------------------327ad9cec9ffd168
+
+--------------------------327ad9cec9ffd168
+Content-Disposition: form-data; name="uploadedFile"; filename="cubeai_output"
+Content-Type: application/octet-stream
+
+< ./sample-inputs/object_detection/cubeai-output.zip
+--------------------------327ad9cec9ffd168
+Content-Disposition: form-data; name="uploadedFile"; filename="model"
+Content-Type: application/octet-stream
+
+< ./sample-inputs/object_detection/model.tflite
+--------------------------327ad9cec9ffd168
+Content-Disposition: form-data; name="templateId"
+
+01HMBW4VR0P1YQXJZEDW04YHJY
+--------------------------327ad9cec9ffd168
+Content-Disposition: form-data; name="runtimeInput"
+
+{
+    "job_config": {
+        "root": "object_detection/scripts/deployment"
+    },
+    "build_config": {}
+}
+
+--------------------------327ad9cec9ffd168--
+###
+GET {{base_url}}{{jobcontrol_api_base_path}}/jobs/{{create.response.body.$.id}}
+Authorization: Bearer {{token}}
+Accept: application/json
+###
+DELETE {{base_url}}{{jobcontrol_api_base_path}}/jobs/{{create.response.body.$.id}}
+Authorization: Bearer {{token}}

common/docker/build/build.Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.10-bullseye as prepare
+
+ARG cubeide_filename="en.st-stm32cubeide_1.14.0_19471_20231121_1200_amd64.deb_bundle.sh.zip"
+
+RUN apt-get update
+
+COPY ./common/docker/build/${cubeide_filename} /run/cubeide.zip
+RUN cd /run && unzip cubeide.zip && LICENSE_ALREADY_ACCEPTED=1 sh /run/*cubeide*.sh
+RUN rm /run/*cubeide*
+
+FROM scratch as run
+
+COPY --from=prepare / /
+
+RUN useradd --create-home --user-group --uid 9001 --shell /bin/bash stm32ai
+USER 9001:9001
+
+RUN mkdir -p /tmp/inputs /tmp/outputs
+
+WORKDIR /stm32ai-modelzoo
+COPY --chown=9001:9001 ./requirements.txt .
+
+RUN pip install --no-cache-dir --user --requirement ./requirements.txt colorama
+COPY --chown=9001:9001 . .
+
+ENV HOME=/home/stm32ai
+
+ENTRYPOINT [ "/stm32ai-modelzoo/common/docker/build/build.entrypoint.py" ]

common/docker/build/build.entrypoint.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+import yaml
+import sys
+from os import path, makedirs
+import logging
+import shutil
+import subprocess as sp
+import zipfile as zf
+
+logging.basicConfig(level=logging.DEBUG)
+
+log = logging.getLogger(__name__)
+
+# In place dict merge (a is the receiver)
+def merge(a: dict, b: dict, path=[]):
+    for key in b:
+        if key in a:
+            if isinstance(a[key], dict) and isinstance(b[key], dict):
+                merge(a[key], b[key], path + [str(key)])
+            # elif a[key] != b[key]:
+            #     raise Exception('Conflict at ' + '.'.join(path + [str(key)]))
+            else:
+                a[key] = b[key]
+        else:
+            a[key] = b[key]  
+    return a
+
+CUBEAI_CODEGEN_ZIP_PATH = "/tmp/inputs/cubeai-output.zip"
+
+def run():
+    log.info("Starting up build job...")
+
+    config = {}
+
+    with open("/tmp/inputs/job.json", "rt") as file:
+        job_config = yaml.safe_load(file)
+
+    logging.debug("Job config: %r", job_config)
+
+    assert "root" in job_config
+    root = job_config["root"]
+    assert isinstance(root, str)
+
+    root = path.abspath(root)
+    assert path.isdir(root), "Configured script root does not exist"
+
+    assert path.isfile(CUBEAI_CODEGEN_ZIP_PATH), "Missing CubeAI codegen archive"
+
+    outputs_path = path.join(root, "outputs")
+    stm32ai_files_path = path.join(outputs_path, "stm32ai_files")
+
+    makedirs(stm32ai_files_path, exist_ok=True)
+
+    with zf.ZipFile(CUBEAI_CODEGEN_ZIP_PATH, "r") as archive:
+        archive.extractall(stm32ai_files_path)
+
+    user_config_path = path.join(root, "user_config.yaml")
+
+    with open(user_config_path, "rt") as file:
+        base_config = yaml.safe_load(file)
+
+    _ = merge(config, base_config)
+
+    with open("/tmp/inputs/config.json", "rt") as file:
+        override_config = yaml.safe_load(file)
+
+    _ = merge(config, override_config)
+
+    config["hydra"] = { "run": { "dir": "outputs" } }
+    config["model"]["model_path"] = "/tmp/inputs/model.tflite"
+
+    logging.debug("Final config: %r", config)
+
+    with open(user_config_path, "wt") as file:
+        yaml.safe_dump(config, file)
+
+    _spawned = sp.run(
+        "python ./deploy.py",
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+        shell=True,
+        cwd=root,
+        check=True
+    )
+
+    project_folder = path.join(path.dirname(path.dirname(root)), "getting_started")
+    assert path.isdir(project_folder), "Missing project folder"
+
+    shutil.make_archive("/tmp/outputs/getting_started", "zip", root_dir=project_folder)
+
+if __name__ == "__main__":
+    run()

common/docker/build/sample-inputs/object_detection/config.json

@@ -0,0 +1 @@
+{}

common/docker/build/sample-inputs/object_detection/job.json

@@ -0,0 +1,3 @@
+{
+    "root": "object_detection/scripts/deployment"
+}

common/docker/train/README.md

@@ -0,0 +1,31 @@
+# Generic train workload meant to be run from jobcontrol-api
+
+## Runtime interface
+
+### Input
+
+The job expects the following files as input:
+
+- A json-formatted file at `/tmp/inputs/config.json` representing overrides for the options provided in `user_config.yaml`. The options in this file are merged together with the defaults in the training script folder
+- A json-formatted file at `/tmp/inputs/job.json` containing job-specific input values. Most importantly, we expect a property under the key `root` whose value must be a string representing the path to the directory containing the target script. This is meant to be the mechanism enabling generality over all modelzoo examples. We'll henceforth refer to this path as `<script-root>`
+- The training dataset as a zip archive located at `/tmp/inputs/train.zip`
+- The (optional) validation dataset as a zip archive located at `/tmp/inputs/valid.zip`
+- The (optional) test dataset as a zip archive located at `/tmp/inputs/test.zip` 
+
+The dataset archives must contain the dataset folder at the top level. These archives will each be extracted in a target folder, the path to which will be respectively set as value to the options of `dataset.training_path`, `dataset.validation_path` and `dataset.test_path`.
+
+### Output
+
+After completing the training, we expect the folder `outputs` to be created in `<script-root>`. This folder is archived at `/tmp/outputs/outputs.zip`. 
+
+## Build
+
+The build context must be set at the root of the repository. from the folder where this file is located, the correct relative path to the build context is `../../..`. 
+
+Assuming the repo root is the current directory, build with
+
+```sh
+docker build --file common/docker/train/train.Dockerfile .
+```
+
+## jobcontrol-api

common/docker/train/api.http

@@ -0,0 +1,69 @@
+@base_url = https://dev.stm-vespucci.com
+@jobcontrol_api_base_path = /svc/jobcontrol/v1alpha1
+@token = dummy
+###
+# @name create
+POST {{base_url}}{{jobcontrol_api_base_path}}/jobs
+Authorization: Bearer {{token}}
+Accept: application/json
+Content-Type: multipart/form-data; boundary=------------------------327ad9cec9ffd168
+
+--------------------------327ad9cec9ffd168
+Content-Disposition: form-data; name="uploadedFile"; filename="train_data"
+Content-Type: application/octet-stream
+
+< ./sample-inputs/object_detection/train.zip
+--------------------------327ad9cec9ffd168
+Content-Disposition: form-data; name="uploadedFile2"; filename="valid_data"
+Content-Type: application/octet-stream
+
+< ./sample-inputs/object_detection/valid.zip
+--------------------------327ad9cec9ffd168
+Content-Disposition: form-data; name="templateId"
+
+01HM2EE60T3HJ4G5MPYK3GVT5D
+--------------------------327ad9cec9ffd168
+Content-Disposition: form-data; name="runtimeInput"
+
+{
+    "job_config": {
+        "root": "object_detection/scripts/training"
+    },
+    "training_config": {}
+}
+
+--------------------------327ad9cec9ffd168--
+###
+# @name create
+POST {{base_url}}{{jobcontrol_api_base_path}}/jobs
+Authorization: Bearer {{token}}
+Accept: application/json
+Content-Type: multipart/form-data; boundary=------------------------327ad9cec9ffd168
+
+--------------------------327ad9cec9ffd168
+Content-Disposition: form-data; name="uploadedFile"; filename="train_data"
+Content-Type: application/octet-stream
+
+< ./sample-inputs/hand_posture/train.zip
+--------------------------327ad9cec9ffd168
+Content-Disposition: form-data; name="templateId"
+
+01HM2EE60T3HJ4G5MPYK3GVT5D
+--------------------------327ad9cec9ffd168
+Content-Disposition: form-data; name="runtimeInput"
+
+{
+    "job_config": {
+        "root": "hand_posture/scripts/training"
+    },
+    "training_config": {}
+}
+
+--------------------------327ad9cec9ffd168--
+###
+GET {{base_url}}{{jobcontrol_api_base_path}}/jobs/{{create.response.body.$.id}}
+Authorization: Bearer {{token}}
+Accept: application/json
+###
+DELETE {{base_url}}{{jobcontrol_api_base_path}}/jobs/{{create.response.body.$.id}}
+Authorization: Bearer {{token}}

common/docker/train/train.Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.10
+
+RUN apt-get update
+RUN apt-get install --yes libgl1
+
+RUN useradd --create-home --user-group --uid 9001 --shell /bin/bash stm32ai
+
+USER 9001:9001
+
+RUN mkdir -p /tmp/inputs /tmp/outputs \
+    /tmp/datasets/train /tmp/datasets/valid /tmp/datasets/test
+
+WORKDIR /stm32ai-modelzoo
+
+COPY --chown=9001:9001 ./requirements.txt .
+RUN pip install --user --requirement ./requirements.txt
+
+COPY --chown=9001:9001 . .
+
+ENV HOME=/home/stm32ai
+
+ENTRYPOINT [ "/stm32ai-modelzoo/common/docker/train/train.entrypoint.py" ]