fixing tests

.github/workflows/test-cm-script-features.yml

@@ -33,6 +33,6 @@ jobs:
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test CM Script Features
       run: |
-        python tests/script/test_install.py
-        python tests/script/test_docker.py
-        python tests/script/test_features.py
+        python script/test-cm-core/src/script/test_install.py
+        python script/test-cm-core/src/script/test_docker.py
+        python script/test-cm-core/src/script/test_features.py

.github/workflows/test-cm-scripts.yml

@@ -33,4 +33,4 @@ jobs:
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test CM Script Automation
       run: |
-        python tests/script/test_deps.py
+        python script/test-cm-core/src/script/test_deps.py

.github/workflows/test-cm-tutorial-retinanet.yml

@@ -32,4 +32,4 @@ jobs:
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test CM Tutorial Retinanet
       run: |
-        python tests/tutorials/test_tutorial_retinanet.py
+        python script/test-cm-core/src/tutorials/test_tutorial_retinanet.py

.github/workflows/test-cm-tutorial-tvm-pip.yml

@@ -32,7 +32,7 @@ jobs:
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test CM Tutorial TVM pip install with VirtualMachine Runtime
       run: |
-        python tests/tutorials/test_tutorial_tvm_pip_vm.py
+        python script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_vm.py
 
   test_ge_runtime:
     runs-on: ubuntu-latest
@@ -54,4 +54,4 @@ jobs:
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test CM Tutorial TVM pip install with GraphExecutor Runtime
       run: |
-        python tests/tutorials/test_tutorial_tvm_pip_ge.py
+        python script/test-cm-core/src/tutorials/test_tutorial_tvm_pip_ge.py

.github/workflows/test-cm-tutorial-tvm.yml

@@ -33,4 +33,4 @@ jobs:
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test CM Tutorial TVM
       run: |
-        python tests/tutorials/test_tutorial_tvm.py
+        python script/test-cm-core/src/tutorials/test_tutorial_tvm.py

automation/script/module.py

@@ -4387,7 +4387,7 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
             utils.merge_dicts({'dict1':env, 'dict2':updated_env, 'append_lists':True, 'append_unique':True})
 
 
-    if postprocess != '' and customize_code is not None:
+    if postprocess != '' and customize_code is not None and postprocess in dir(customize_code):
         print (recursion_spaces+'       ! call "{}" from {}'.format(postprocess, customize_code.__file__))
 
     if len(posthook_deps)>0 and (postprocess == "postprocess"):

script/test-cm-core/README-extra.md

@@ -0,0 +1 @@
+# CM script

script/test-cm-core/_cm.yaml

@@ -0,0 +1,12 @@
+alias: test-cm-core
+uid: 2c2fb9d20dc64cf3
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+tags:
+- test
+- cm
+- core
+
+cache: false

script/test-cm-core/customize.py

@@ -0,0 +1,16 @@
+from cmind import utils
+import os
+
+def preprocess(i):
+
+    os_info = i['os_info']
+
+    env = i['env']
+
+    meta = i['meta']
+
+    automation = i['automation']
+
+    quiet = (env.get('CM_QUIET', False) == 'yes')
+
+    return {'return':0}

script/test-cm-core/run.bat

@@ -0,0 +1,3 @@
+rem native script
+
+echo "TBD"

script/test-cm-core/run.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+#CM Script location: ${CM_TMP_CURRENT_SCRIPT_PATH}
+
+#To export any variable
+#echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out
+
+#${CM_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency
+
+echo "TBD"
+
+#Add your run commands here...
+# run "$CM_RUN_CMD"

script/test-cm-core/src/script/check.py

@@ -0,0 +1,14 @@
+def check_return(r):
+    if 'return' not in r:
+        raise Exception('CM access function should always return key \'return\'!')
+    if 'error' in r:
+        raise Exception(r['error'])
+
+def check_list(r, string, found=True):
+    check_return(r)
+    if 'list' not in r:
+        raise Exception('CM search should return a list!')
+    if len(r['list']) < 1 and found:
+        raise Exception('CM search returned an empty list for ' + string)
+    if len(r['list']) > 0 and not found:
+        raise Exception('CM search returned at lease one entry for ' + string)

script/test-cm-core/src/script/process_dockerfile.py

@@ -0,0 +1,30 @@
+import sys
+import os
+import cmind as cm
+import check as checks
+import json
+import yaml
+
+files=sys.argv[1:]
+
+for file in files:
+    if not os.path.isfile(file):
+        continue
+    if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"):
+        continue
+    if not file.startswith(os.path.join("script")):
+        continue
+
+    script_path = os.path.dirname(file)
+
+    f = open(file)
+
+    if file.endswith(".json"):
+        data = json.load(f)
+    elif file.endswith(".yaml"):
+        data = yaml.safe_load(f)
+
+    uid = data['uid']
+
+    r = cm.access({'action':'dockerfile', 'automation':'script', 'artifact': uid, 'quiet': 'yes'})
+    checks.check_return(r)

script/test-cm-core/src/script/process_readme.py

@@ -0,0 +1,26 @@
+import sys
+import os
+import cmind as cm
+import check as checks
+import json
+import yaml
+
+files=sys.argv[1:]
+
+for file in files:
+    if not os.path.isfile(file):
+        continue
+    if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"):
+        continue
+    if not file.startswith(os.path.join("script")):
+        continue
+    script_path = os.path.dirname(file)
+    f = open(file)
+    if file.endswith(".json"):
+        data = json.load(f)
+    elif file.endswith(".yaml"):
+        data = yaml.safe_load(f)
+    uid = data['uid']
+
+    r = cm.access({'action':'doc', 'automation':'script', 'artifact': uid, 'quiet': 'yes'})
+    checks.check_return(r)

script/test-cm-core/src/script/test_deps.py

@@ -0,0 +1,23 @@
+# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps
+
+import cmind as cm
+import check as checks
+
+# MLPerf v3.0 inference is now very outdated and we are testing inference in separate tests
+
+#r = cm.access({'action':'run', 'automation':'script', 'tags': 'generate-run-cmds,mlperf', 'adr':
+#    {'loadgen': {'version': 'r3.0'}, 'compiler': {'tags': "gcc"}}, 'env': {'CM_MODEL': 'resnet50',
+#        'CM_DEVICE': 'cpu', 'CM_BACKEND': 'onnxruntime'}, 'quiet': 'yes'})
+#checks.check_return(r)
+#
+#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'loadgen,version-r3.0,deps-python-non-virtual'})
+#checks.check_list(r, "loadgen,version-r3.0,deps-python-non-virtual")
+#
+#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'inference,src,version-r3.0'})
+#checks.check_list(r, "inference,src,version-r3.0")
+#
+#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'})
+#checks.check_return(r)
+#
+#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_tf,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'})
+#checks.check_return(r)

script/test-cm-core/src/script/test_docker.py

@@ -0,0 +1,21 @@
+# This test covers version, variation, compilation from src, add_deps_recursive, post_deps
+
+import cmind as cm
+import check as checks
+
+r = cm.access({'action':'run', 
+               'automation':'script', 
+               'tags': 'run,docker,container', 
+               'add_deps_recursive': {
+                  'compiler': {'tags': "gcc"}
+               }, 
+               'image_name':'cm-script-app-image-classification-onnx-py',
+               'env': {
+                 'CM_DOCKER_RUN_SCRIPT_TAGS': 'app,image-classification,onnx,python',
+                 'CM_MLOPS_REPO': 'ctuning@mlcommons-ck', 
+                 'CM_DOCKER_IMAGE_BASE': 'ubuntu:22.04'
+               }, 
+               'quiet': 'yes'
+              })
+
+checks.check_return(r)

script/test-cm-core/src/script/test_features.py

@@ -0,0 +1,21 @@
+# This test covers
+# 1. python-virtual-env and update_deps inside customize.py
+# 2. cache search using "-" prefix
+
+import cmind as cm
+import check as checks
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'name':'test', 'quiet': 'yes'})
+checks.check_return(r)
+
+r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,python,virtual,name-test'})
+checks.check_list(r, "get,python-venv")
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'get,dataset,preprocessed,imagenet,_NHWC', 'quiet': 'yes'})
+checks.check_return(r)
+
+r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NCHW'})
+checks.check_list(r, "_NHWC")
+
+r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NHWC'})
+checks.check_list(r, "_NHWC", False)

script/test-cm-core/src/script/test_install.py

@@ -0,0 +1,10 @@
+# This test covers script installation, version, shared library install
+
+import cmind as cm
+import check as checks
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'python,src,install,_shared', 'version': '3.9.10', 'quiet': 'true'})
+checks.check_return(r)
+
+r = cm.access({'action':'search', 'automation':'cache', 'tags': 'python,src,install,_shared,version-3.9.10'})
+checks.check_list(r, "python,src,install,_shared,version-3.9.10")

script/test-cm-core/src/test_cm.py

@@ -0,0 +1,14 @@
+try:
+    import cmind as cm
+
+    r = cm.access(['test', 'script'])
+    if 'return' not in r:
+        raise Exception('CM access function should always return key \'return\'!')
+    exit(0)
+
+except ImportError as e:
+    from sys import stderr
+    from subprocess import call
+    print('WARNING: CM module for python is not installed & jupyter notebooks will not be supported', file=stderr)
+    retcode = call(['cm', 'test', 'script'])
+    exit(retcode)

script/test-cm-core/src/test_search_speed.py

@@ -0,0 +1,25 @@
+import cmind as cm
+import time
+
+times = []
+
+steps = 10
+
+print ('Running search with tags {} times ...'.format(steps))
+
+for step in range(steps):
+
+    start = time.time()
+    r = cm.access({'action':'search',
+                   'automation':'script',
+                   'tags':'detect,os'})
+    timer = time.time() - start
+
+    if r['return']>0: cm.error(r)
+
+    times.append(timer)
+
+step = 0
+for t in times:
+    step += 1
+    print ("{}) {:0.3f} sec.".format(step, t))