From f4001d41443fe738d363c17dd0816fa76e0dfe21 Mon Sep 17 00:00:00 2001 From: Jon Nordby Date: Sat, 30 Mar 2024 01:09:34 +0100 Subject: [PATCH 1/8] examples: Fix exceptions due to np.int Will cause exceptions on recent numpy versions, such as AttributeError: module 'numpy' has no attribute 'int'. At least on numpy 1.24.3 Must be replaced with either np.int32/np.int64 --- examples/auto_test/README.md | 2 +- examples/auto_test/main.py | 2 +- examples/uci-har-rnn/uci_rnn.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/auto_test/README.md b/examples/auto_test/README.md index 70d44099..7fa4e706 100644 --- a/examples/auto_test/README.md +++ b/examples/auto_test/README.md @@ -109,7 +109,7 @@ Data format in a testing binary file os.system(cmd) try: # get NNoM results - result = np.genfromtxt('result.csv', delimiter=',', dtype=np.int, skip_header=1) + result = np.genfromtxt('result.csv', delimiter=',', dtype=np.int32, skip_header=1) result = result[:,0] # the first column is the label, the second is the probability label = y_test_original[:len(y_test)].flatten() # use the original numerical label acc = np.sum(result == label).astype('float32')/len(result) diff --git a/examples/auto_test/main.py b/examples/auto_test/main.py index 5a4780ad..76803ee7 100644 --- a/examples/auto_test/main.py +++ b/examples/auto_test/main.py @@ -145,7 +145,7 @@ def main(): os.system(cmd) try: # get NNoM results - result = np.genfromtxt('result.csv', delimiter=',', dtype=np.int, skip_header=1) + result = np.genfromtxt('result.csv', delimiter=',', dtype=np.int32, skip_header=1) result = result[:,0] # the first column is the label, the second is the probability label = y_test_original[:len(y_test)].flatten() # use the original numerical label acc = np.sum(result == label).astype('float32')/len(result) diff --git a/examples/uci-har-rnn/uci_rnn.py b/examples/uci-har-rnn/uci_rnn.py index e4d50adf..fbb05dad 100644 --- a/examples/uci-har-rnn/uci_rnn.py +++ b/examples/uci-har-rnn/uci_rnn.py @@ -238,7 +238,7 @@ def main(): os.system(cmd) try: # get NNoM results - result = np.genfromtxt('result.csv', delimiter=',', dtype=np.int, skip_header=1) + result = np.genfromtxt('result.csv', delimiter=',', dtype=np.int32, skip_header=1) result = result[:,0] # the first column is the label, the second is the probability label = y_test_original[:len(y_test)].flatten() # use the original numerical label acc = np.sum(result == label).astype('float32')/len(result) From 89db9aa4c609787357fb0ee1c526f1caad56fce2 Mon Sep 17 00:00:00 2001 From: Jon Nordby Date: Sat, 30 Mar 2024 03:07:46 +0100 Subject: [PATCH 2/8] scripts: Use package relative imports Needed when installed --- scripts/__init__.py | 4 ++++ scripts/nnom.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/__init__.py b/scripts/__init__.py index 5bb534f7..a8f5ef50 100644 --- a/scripts/__init__.py +++ b/scripts/__init__.py @@ -1 +1,5 @@ + # package + +from .nnom import * + diff --git a/scripts/nnom.py b/scripts/nnom.py index 45e6b30a..9f10efd3 100644 --- a/scripts/nnom.py +++ b/scripts/nnom.py @@ -16,8 +16,8 @@ import tensorflow.keras.backend as K from tensorflow.keras import * from tensorflow.keras.layers import * -from fully_connected_opt_weight_generation import * -from gen_config import * +from .fully_connected_opt_weight_generation import * +from .gen_config import * import scipy.stats import time import warnings From 8b2274e58221981c722984f7c56e67f47502d4bd Mon Sep 17 00:00:00 2001 From: Jon Nordby Date: Sat, 30 Mar 2024 03:08:03 +0100 Subject: [PATCH 3/8] packaging: Add initial pyproject.toml --- pyproject.toml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..de53045f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,33 @@ + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["scripts", "src"] + +[tool.hatch.build.targets.wheel.sources] +"scripts" = "nnom" +"src" = "nnom_core" + +[project] +name = "nnom" +readme = "README.md" +requires-python = ">= 3.8" +keywords = ["machine learning", "deep learning", "microcontroller", "embedded"] +version = "0.5.0a1" + +dependencies = [ + "keras<3.0.0", +] + +classifiers = [ + "License :: OSI Approved :: MIT License", +] + + +[project.urls] +Homepage = "https://github.com/majianjia/nnom" +Documentation = "https://majianjia.github.io/nnom/" +Repository = "https://github.com/majianjia/nnom" + From 7bf397ea28b9bd8f3e243508ef1499ed1f7f20c5 Mon Sep 17 00:00:00 2001 From: Jon Nordby Date: Sat, 30 Mar 2024 03:16:57 +0100 Subject: [PATCH 4/8] examples: Do not modify sys.path The nnom package is expected to be installed --- docs/api_nnom.md | 3 +-- examples/auto_test/main.py | 2 -- examples/keyword_spotting/kws-rnn.py | 2 -- examples/keyword_spotting/kws.py | 2 -- examples/mnist-densenet/mnist_densenet.py | 2 -- examples/mnist-simple/mnist_simple.py | 2 -- examples/octave-conv/mnist_octaveconv.py | 2 -- examples/rnn-denoise/main.py | 4 ---- examples/uci-har-rnn/uci_rnn.py | 2 -- examples/uci-inception/uci_test.py | 2 -- 10 files changed, 1 insertion(+), 22 deletions(-) diff --git a/docs/api_nnom.md b/docs/api_nnom.md index 12120425..c37c7946 100644 --- a/docs/api_nnom.md +++ b/docs/api_nnom.md @@ -2,7 +2,7 @@ # NNoM Utils - Structured API What makes NNoM easy to use is the models can be deployed to MCU automatically or manually with the help of NNoM utils. -These functions are located in `scripts/nnom.py` +These functions are located in the `nnom` Python package. This tools, will generate NNoM model using **Structured API** instead of 'layer API' in the previous script. @@ -11,7 +11,6 @@ Usage in short, in your python file, add the scripts the directory to your envir ~~~ import sys import os -sys.path.append(os.path.abspath("../../scripts")) from nnom import * ~~~ diff --git a/examples/auto_test/main.py b/examples/auto_test/main.py index 76803ee7..20a03c2b 100644 --- a/examples/auto_test/main.py +++ b/examples/auto_test/main.py @@ -10,8 +10,6 @@ import sys import os -sys.path.append(os.path.abspath("../../scripts")) -print(sys.path) from tensorflow.keras import * from tensorflow.keras.datasets import mnist diff --git a/examples/keyword_spotting/kws-rnn.py b/examples/keyword_spotting/kws-rnn.py index f2a214e6..3c6ae3f4 100644 --- a/examples/keyword_spotting/kws-rnn.py +++ b/examples/keyword_spotting/kws-rnn.py @@ -11,8 +11,6 @@ import matplotlib.pyplot as plt import os -nnscript = os.path.abspath('../../scripts') -os.sys.path.append(nnscript) from tensorflow.keras import * from tensorflow.keras.datasets import mnist diff --git a/examples/keyword_spotting/kws.py b/examples/keyword_spotting/kws.py index d278c294..f7d8e6c8 100644 --- a/examples/keyword_spotting/kws.py +++ b/examples/keyword_spotting/kws.py @@ -11,8 +11,6 @@ import matplotlib.pyplot as plt import os -nnscript = os.path.abspath('../../scripts') -os.sys.path.append(nnscript) from tensorflow.keras import * from tensorflow.keras.datasets import mnist diff --git a/examples/mnist-densenet/mnist_densenet.py b/examples/mnist-densenet/mnist_densenet.py index 80565d0a..7dafa0a1 100644 --- a/examples/mnist-densenet/mnist_densenet.py +++ b/examples/mnist-densenet/mnist_densenet.py @@ -11,8 +11,6 @@ import matplotlib.pyplot as plt import os -nnscript = os.path.abspath('../../scripts') -os.sys.path.append(nnscript) from tensorflow.keras import * from tensorflow.keras.datasets import mnist diff --git a/examples/mnist-simple/mnist_simple.py b/examples/mnist-simple/mnist_simple.py index 47ac424a..3a6fe73e 100644 --- a/examples/mnist-simple/mnist_simple.py +++ b/examples/mnist-simple/mnist_simple.py @@ -11,8 +11,6 @@ import matplotlib.pyplot as plt import os -nnscript = os.path.abspath('../../scripts') -os.sys.path.append(nnscript) from tensorflow.keras import * from tensorflow.keras.datasets import mnist diff --git a/examples/octave-conv/mnist_octaveconv.py b/examples/octave-conv/mnist_octaveconv.py index fd0fb815..0cbc8844 100644 --- a/examples/octave-conv/mnist_octaveconv.py +++ b/examples/octave-conv/mnist_octaveconv.py @@ -11,8 +11,6 @@ import matplotlib.pyplot as plt import os -nnscript = os.path.abspath('../../scripts') -os.sys.path.append(nnscript) from tensorflow.keras import * from tensorflow.keras.datasets import mnist diff --git a/examples/rnn-denoise/main.py b/examples/rnn-denoise/main.py index ca732f64..dd990140 100644 --- a/examples/rnn-denoise/main.py +++ b/examples/rnn-denoise/main.py @@ -10,10 +10,6 @@ import tensorflow as tf import numpy as np -import sys -sys.path.append(os.path.abspath("../../scripts")) -print(sys.path) - from gen_dataset import * from nnom import * diff --git a/examples/uci-har-rnn/uci_rnn.py b/examples/uci-har-rnn/uci_rnn.py index fbb05dad..9c6f3be5 100644 --- a/examples/uci-har-rnn/uci_rnn.py +++ b/examples/uci-har-rnn/uci_rnn.py @@ -11,8 +11,6 @@ import matplotlib.pyplot as plt import os -nnscript = os.path.abspath('../../scripts') -os.sys.path.append(nnscript) from tensorflow.keras import * from tensorflow.keras.datasets import mnist diff --git a/examples/uci-inception/uci_test.py b/examples/uci-inception/uci_test.py index 327e571f..c98336ac 100644 --- a/examples/uci-inception/uci_test.py +++ b/examples/uci-inception/uci_test.py @@ -11,8 +11,6 @@ import matplotlib.pyplot as plt import os -nnscript = os.path.abspath('../../scripts') -os.sys.path.append(nnscript) from tensorflow.keras import * from tensorflow.keras.datasets import mnist From e636197d95a1c72c7ec9e9017c8618742e90e2b5 Mon Sep 17 00:00:00 2001 From: Jon Nordby Date: Sat, 30 Mar 2024 03:47:58 +0100 Subject: [PATCH 5/8] packaging: Also include C code from port/ and inc/ directories --- pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index de53045f..b5a13f76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,11 +4,13 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] -packages = ["scripts", "src"] +packages = ["scripts", "src", "inc", "port"] [tool.hatch.build.targets.wheel.sources] "scripts" = "nnom" -"src" = "nnom_core" +"src" = "nnom_core/src" +"inc" = "nnom_core/inc" +"port" = "nnom_core/port" [project] name = "nnom" From 1c516bce3ba42133933ffbbbf6009c9e5ab0646a Mon Sep 17 00:00:00 2001 From: Jon Nordby Date: Sat, 30 Mar 2024 03:57:42 +0100 Subject: [PATCH 6/8] Add requirements.dev.txt What is needed to run examples etc --- requirements.dev.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 requirements.dev.txt diff --git a/requirements.dev.txt b/requirements.dev.txt new file mode 100644 index 00000000..ab6b541e --- /dev/null +++ b/requirements.dev.txt @@ -0,0 +1,4 @@ +tensorflow-cpu==2.14.1 +matplotlib +scikit-learn +numpy From 73b4461edb54ad983c0abd34097454432466e8b8 Mon Sep 17 00:00:00 2001 From: Jon Nordby Date: Sat, 30 Mar 2024 03:58:20 +0100 Subject: [PATCH 7/8] CI: Add initial Github Actions configuration --- .github/workflows/tests.yaml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/tests.yaml diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 00000000..39443286 --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,36 @@ +name: Python package + +on: [push, pull_request] + +jobs: + test: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10"] + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install OS dependencies + run: | + sudo apt-get install -yqq python-tk scons git + - name: Install Python dependencies + run: | + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements.dev.txt ]; then pip install -r requirements.dev.txt; fi + - name: Install as pip package + run: | + pip install ./ -v + - name: Test installed package + working-directory: examples/auto_test + run: | + python main.py - + From ad83485dce2f326a07cfe09a930f990f2a59a376 Mon Sep 17 00:00:00 2001 From: Jon Nordby Date: Sat, 30 Mar 2024 03:58:38 +0100 Subject: [PATCH 8/8] CI: Delete Travis CI configuration Travis is removing support for open source projects Has not ran any jobs for many months --- .travis.yml | 55 ----------------------------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 92165757..00000000 --- a/.travis.yml +++ /dev/null @@ -1,55 +0,0 @@ -language: python -python: - - "3.7" - -os: linux -dist: focal - -notifications: - email: false - -before_install: - - "sudo apt-get update || true" - -install: - # We do this conditionally because it saves us some downloading if the - # version is the same. - - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; - else - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - fi - - bash miniconda.sh -b -p $HOME/miniconda - - source "$HOME/miniconda/etc/profile.d/conda.sh" - - hash -r - - conda config --set always_yes yes --set changeps1 no - #- conda update -q conda - # Useful for debugging any issues with conda - - conda info -a - - # Replace dep1 dep2 ... with your dependencies - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION - - conda activate test-environment - - # dependencies - - "sudo apt-get install -qq python-tk scons git || true" - - "conda install numpy matplotlib tensorflow scikit-learn || true" - - "conda list" - -before_script: - - python --version - - python3 --version - - python2 --version - -script: - - export PYTHONPATH=`pwd`/scripts - - echo ====== New auto-test ====== - - cd examples/auto_test && python main.py - - #- mkdir -p examples/mnist-densenet/tmp - #- echo ====== local NNOM test ====== - #- cd examples/mnist-densenet && python model/mnist_densenet.py && cp -v weights.h mcu && scons && NNOM_TEST_ON_CI=YES python model/mnist_densenet.py && cd - - #- echo ====== CMSIS NN NNOM test ====== - #- cd examples/mnist-densenet && USE_CMSIS_NN=YES scons && NNOM_TEST_ON_CI=YES python model/mnist_densenet.py && cd - - -env: -- NNOM_ON_CI=YES