Merge pull request #118 from mlcommons/dev

Sync Dev
mlcommons · Jan 8, 2025 · a7fc2c6 · a7fc2c6
2 parents 5bcde4c + 166e8e5
commit a7fc2c6
Show file tree

Hide file tree

Showing 41 changed files with 989 additions and 421 deletions.
diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml
@@ -32,6 +32,6 @@ jobs:
         for file in ${{ steps.getfile.outputs.files }}; do
           echo $file
         done
-        python3 -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
         DOCKER_CM_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_CM_REPO_BRANCH=${{ github.event.pull_request.head.ref }} TEST_INPUT_INDEX=${{ matrix.test-input-index }} python3 script/test-cm-core/src/script/process_tests.py ${{ steps.getfile.outputs.files }}
diff --git a/.github/workflows/test-cm-based-submission-generation.yml b/.github/workflows/test-cm-based-submission-generation.yml
@@ -48,9 +48,16 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install cm4mlops on Windows
+      if: matrix.os == 'windows-latest'
+      run: |
+        $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops
+    - name: Install dependencies on Unix Platforms
+      if: matrix.os != 'windows-latest'
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        python3 -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
     - name: Pull repo where test cases are uploaded
       run: |

diff --git a/.github/workflows/test-cm-script-features.yml b/.github/workflows/test-cm-script-features.yml
@@ -30,9 +30,16 @@ jobs:
       if: matrix.os == 'windows-latest'
       run: |
         git config --system core.longpaths true
-    - name: Install dependencies
+    - name: Install cm4mlops on Windows
+      if: matrix.os == 'windows-latest'
+      run: |
+        $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops
+    - name: Install dependencies on Unix Platforms
+      if: matrix.os != 'windows-latest'
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        python -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Run test_docker on linux

diff --git a/.github/workflows/test-cm-tutorial-retinanet.yml b/.github/workflows/test-cm-tutorial-retinanet.yml
@@ -26,9 +26,11 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install dependencies on Unix Platforms
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        python -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test CM Tutorial Retinanet

diff --git a/.github/workflows/test-cm-tutorial-tvm-pip.yml b/.github/workflows/test-cm-tutorial-tvm-pip.yml
@@ -27,9 +27,11 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install dependencies on Unix Platforms
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        python -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test CM Tutorial TVM pip install with VirtualMachine Runtime
@@ -49,9 +51,11 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install dependencies on Unix Platforms
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        python -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test CM Tutorial TVM pip install with GraphExecutor Runtime

diff --git a/.github/workflows/test-cm-tutorial-tvm.yml b/.github/workflows/test-cm-tutorial-tvm.yml
@@ -28,9 +28,11 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install dependencies on Unix Platforms
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        python -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test CM Tutorial TVM

diff --git a/.github/workflows/test-image-classification-onnx.yml b/.github/workflows/test-image-classification-onnx.yml
@@ -30,9 +30,16 @@ jobs:
       if: matrix.os == 'windows-latest'
       run: |
         git config --system core.longpaths true
-    - name: Install dependencies
+    - name: Install cm4mlops on Windows
+      if: matrix.os == 'windows-latest'
+      run: |
+        $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops
+    - name: Install dependencies on Unix Platforms
+      if: matrix.os != 'windows-latest'
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        python3 -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test image classification with ONNX

diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml
@@ -51,9 +51,16 @@ jobs:
       if: matrix.os == 'windows-latest'
       run: |
         git config --system core.longpaths true
-    - name: Install dependencies
+    - name: Install cm4mlops on Windows
+      if: matrix.os == 'windows-latest'
+      run: |
+        $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops
+    - name: Install dependencies on Unix Platforms
+      if: matrix.os != 'windows-latest'
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
         #cm pull repo mlcommons@cm4abtf --branch=poc
   

diff --git a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml
@@ -35,9 +35,16 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install cm4mlops on Windows
+      if: matrix.os == 'windows-latest'
+      run: |
+        $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops
+    - name: Install dependencies on Unix Platforms
+      if: matrix.os != 'windows-latest'
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        python3 -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
     - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }}
       if: matrix.os == 'windows-latest'

diff --git a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml
@@ -32,16 +32,23 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install cm4mlops on Windows
+      if: matrix.os == 'windows-latest'
+      run: |
+        $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops
+    - name: Install dependencies on Unix Platforms
+      if: matrix.os != 'windows-latest'
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        python3 -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
         cm run script --quiet --tags=get,sys-utils-cm
         cm run script --quiet --tags=install,prebuilt,llvm --version=${{ matrix.llvm-version }}
     - name: Test MLPerf Inference MLCommons C++ ResNet50 on ${{ matrix.os }}
       if: matrix.os == 'windows-latest'
       run: |
-        cmr "app mlperf inference mlcommons cpp" --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --adr.loadgen.tags=_from-pip --pip_loadgen=yes -v --quiet
+        cm run script --tags=app,mlperf,inference,mlcommons,cpp --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --adr.loadgen.tags=_from-pip --pip_loadgen=yes -v --quiet
     - name: Test MLPerf Inference MLCommons C++ ResNet50 on ${{ matrix.os }}
       if: matrix.os != 'windows-latest'
       run: |

diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml
@@ -69,7 +69,7 @@ jobs:
           ubwkjh-Ii8UJDpG2EoU6GQ/field/Access Token > env:PAT  # Fetch PAT and store in environment variable
 
     - name: Push Results
-      if: github.repository_owner == 'mlcommons'
+      if: github.repository_owner == 'mlcommons_off'
       env:
           GITHUB_TOKEN: ${{ env.PAT }}
       run: |

diff --git a/.github/workflows/test-mlperf-inference-rgat.yml b/.github/workflows/test-mlperf-inference-rgat.yml
@@ -25,9 +25,16 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install cm4mlops on Windows
+      if: matrix.os == 'windows-latest'
+      run: |
+        $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops
+    - name: Install dependencies on Unix Platforms
+      if: matrix.os != 'windows-latest'
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        python3 -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
     - name: Test MLPerf Inference R-GAT using ${{ matrix.backend }} on ${{ matrix.os }}
       run: |

diff --git a/.github/workflows/test-mlperf-inference-rnnt.yml b/.github/workflows/test-mlperf-inference-rnnt.yml
@@ -28,9 +28,11 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install dependencies on Unix Platforms
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        python3 -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test MLPerf Inference RNNT

diff --git a/.github/workflows/test-mlperf-inference-tvm.yml b/.github/workflows/test-mlperf-inference-tvm.yml
@@ -28,9 +28,11 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install dependencies on Unix Platforms
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        python3 -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
         cm run script --quiet --tags=get,sys-utils-cm
     - name: MLPerf Inference ResNet50 using TVM

diff --git a/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml b/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml
@@ -26,9 +26,11 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install dependencies on Unix Platforms
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repository
       run: |
-        python3 -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
     - name: Test MLPerf loadgen with HuggingFace bert onnx fp32 squad model
       run: |

diff --git a/.github/workflows/test-qaic-compute-sdk-build.yml b/.github/workflows/test-qaic-compute-sdk-build.yml
@@ -26,8 +26,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python3 -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
-        cm pull repo mlcommons@ck
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
         cm run script --tags=get,sys-utils-cm --quiet
   
     - name: Test QAIC Compute SDK for compilation

diff --git a/.github/workflows/test-qaic-software-kit.yml b/.github/workflows/test-qaic-software-kit.yml
@@ -29,10 +29,9 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Pull MLOps repository
       run: |
-        python3 -m pip install "cmind @ git+https://[email protected]/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
-        cm pull repo mlcommons@ck
+        pip install cm4mlops
         cm run script --tags=get,sys-utils-cm --quiet
   
     - name: Test Software Kit for compilation on Ubuntu 20.04

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
@@ -0,0 +1,43 @@
+# Contributors
+
+Thank you for your interest in contributing to **MLPerf Automations**! We welcome contributions that help improve the project and expand its functionality.  
+
+---
+
+## How to Become a Contributor
+
+We value all contributions, whether they are code, documentation, bug reports, or feature suggestions. If you contribute **more than 50 lines of code** (including tests and documentation), you will be officially recognized as a project contributor.  
+
+**Note:** Trivial contributions, such as minor typo fixes or small formatting changes, will not count toward the 50-line threshold.  
+
+To contribute:  
+1. **Fork** the repository.  
+2. **Create** a new branch for your feature or bug fix.  
+3. **Submit** a pull request (PR) describing your changes.
+Please see [here](CONTRIBUTING.md) for further guidelines for official contribution to any MLCommons repository.
+
+---
+
+## Contributor Recognition
+
+Once your contribution exceeds 50 lines of code (in total), we will:  
+- Add your name to this `CONTRIBUTORS.md` file.  
+- Highlight your contribution in the next release notes.  
+- Grant you access to suggest and vote on new features.  
+
+---
+
+## Current Contributors
+
+- **Grigori Fursin** - *Initial Development, CLI workflow support via CMind, Added core automation features*
+- **Arjun Suresh** - *Initial Development, Added core automation features* 
+- **Anandhu Sooraj** - *Added multiple CM scripts for MLPerf Inference*
+- **Thomaz Zhu** - *Added CPP implementation for MLPerf Inference Onnxruntime*
+- **Sahil Avaran** - *Adding logging support in MLPerf script automation* 
+- **[Your Name Here]** - This could be you! 🎉  
+
+---
+
+We believe in collaborative growth, and every contribution makes a difference. Feel free to reach out by opening an issue if you have any questions or ideas.  
+
+Happy Coding! 🚀
diff --git a/README.md b/README.md
@@ -1,13 +1,65 @@
 # MLPerf Automations and Scripts
 
-This repository contains the automations and scripts used to run MLPerf benchmarks, primarily focusing on MLPerf inference benchmarks. The automations used here are largely based on and extended from the [Collective Mind script automations](https://github.com/mlcommons/cm4mlops/tree/main/automation/script).
+[![License](https://img.shields.io/badge/License-Apache%202.0-green)](LICENSE.md)
+[![Downloads](https://static.pepy.tech/badge/cm4mlops)](https://pepy.tech/project/cm4mlops)
+[![CM Script Automation Test](https://github.com/mlcommons/mlperf-automations/actions/workflows/test-cm-script-features.yml/badge.svg)](https://github.com/mlcommons/mlperf-automations/actions/workflows/test-cm-script-features.yml)
+[![MLPerf Inference ABTF POC Test](https://github.com/mlcommons/mlperf-automations/actions/workflows/test-mlperf-inference-abtf-poc.yml/badge.svg)](https://github.com/mlcommons/mlperf-automations/actions/workflows/test-mlperf-inference-abtf-poc.yml)
 
+Welcome to the **MLPerf Automations and Scripts** repository! This repository provides tools, automations, and scripts to facilitate running MLPerf benchmarks, with a primary focus on **MLPerf Inference benchmarks**.
 
-## Collective Mind (CM) Automations
+The automations build upon and extend the powerful [Collective Mind (CM) script automations](https://github.com/mlcommons/cm4mlops/tree/main/automation/script) to streamline benchmarking and workflow processes.
 
-**CM (Collective Mind)** is a Python package with a CLI and API designed to create and manage automations. Two key automations developed using CM are **Script** and **Cache**, which streamline ML workflows, including managing Docker runs.
+---
 
+## 🚀 Key Features
+- **Automated Benchmarking** – Simplifies running MLPerf Inference benchmarks with minimal manual intervention.
+- **Modular and Extensible** – Easily extend the scripts to support additional benchmarks and configurations.
+- **Seamless Integration** – Compatible with Docker, cloud environments, and local machines.
+- **Collective Mind (CM) Integration** – Utilizes the CM framework to enhance reproducibility and automation.
 
-## License
+---
 
-[Apache 2.0](LICENSE.md)
+## 🧰 Collective Mind (CM) Automations
+
+The **Collective Mind (CM)** framework is a Python-based package offering both CLI and API support for creating and managing automations. CM automations enhance ML workflows by simplifying complex tasks such as Docker container management and caching.
+
+### Core Automations
+- **Script Automation** – Automates script execution across different environments.
+- **Cache Management** – Manages reusable cached results to accelerate workflow processes.
+
+Learn more about CM in the [CM4MLOps documentation](https://github.com/mlcommons/cm4mlops).
+
+---
+
+## 🤝 Contributing
+We welcome contributions from the community! To contribute:
+1. Submit pull requests (PRs) to the **`dev`** branch.
+2. Review our [CONTRIBUTORS.md](here) for guidelines and best practices.
+3. Explore more about MLPerf Inference automation in the official [MLPerf Inference Documentation](https://docs.mlcommons.org/inference/).
+
+Your contributions help drive the project forward!
+
+---
+
+## 📰 News
+Stay tuned for upcoming updates and announcements.
+
+---
+
+## 📄 License
+This project is licensed under the [Apache 2.0 License](LICENSE.md).
+
+---
+
+## 💡 Acknowledgments and Funding
+This project is made possible through the generous support of:
+- [OctoML](https://octoml.ai)
+- [cKnowledge.org](https://cKnowledge.org)
+- [cTuning Foundation](https://cTuning.org)
+- [MLCommons](https://mlcommons.org)
+
+We appreciate their contributions and sponsorship!
+
+---
+
+Thank you for your interest and support in MLPerf Automations and Scripts!