diff --git a/.github/action.yml b/.github/action.yml index 48d5c29b..79ae8f1a 100644 --- a/.github/action.yml +++ b/.github/action.yml @@ -69,7 +69,7 @@ runs: REGISTRY=${{ inputs.registry }} \ REPO=${{ inputs.repo }} \ COMPOSE_PROJECT_NAME=${{ env.random-number }} \ - ${{ inputs.env_overrides }} docker compose -p ${{ env.random-number }} images --format json | jq '.[] | "\(.Repository):\(.Tag)"' >> $GITHUB_STEP_SUMMARY + ${{ inputs.env_overrides }} docker compose -p ${{ env.random-number }} images --format json | jq --arg registry "${{ secrets.registry }}" '.[] | select(.Repository | contains($registry)) | "\(.Repository):\(.Tag)"' | jq -s '.' >> $GITHUB_STEP_SUMMARY working-directory: ${{ inputs.group_dir }} - name: Un-Tag Containers if: ${{ always() }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 94fbb80c..094c89d3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,11 +19,15 @@ version: 2 updates: - - package-ecosystem: "pip" # See documentation for possible values - directory: "test-runner/" # Location of package manifests + - package-ecosystem: pip + directory: /test-runner schedule: - interval: "weekly" - - package-ecosystem: "github-actions" # See documentation for possible values - directory: ".github/workflows" # Location of package manifests + interval: weekly + - package-ecosystem: github-actions + directory: / schedule: - interval: "weekly" + interval: weekly + - package-ecosystem: pip + directory: /pytorch/serving + schedule: + interval: weekly diff --git a/.github/workflows/bandit.yaml b/.github/workflows/bandit.yaml deleted file mode 100644 index cb740854..00000000 --- a/.github/workflows/bandit.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -name: Bandit Scan -on: - pull_request_review: - types: [submitted] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -permissions: read-all -jobs: - scan-bandit: - if: github.event.review.state == 'approved' - uses: intel-innersource/frameworks.ai.infrastructure.code-scan-tools/.github/workflows/Scanner_Bandit.yml@one-ci-cd - with: - org: ${{ github.repository_owner }} - repo: ${{ github.event.repository.name }} - refs: ${{ github.event.pull_request.base.ref }} - runners: '[''k8-runners'']' - secrets: - token: ${{ secrets.ACTION_TOKEN }} diff --git a/.github/workflows/container-pipeline-tester.yaml b/.github/workflows/container-pipeline-tester.yaml index db143c6e..4b7864c8 100644 --- a/.github/workflows/container-pipeline-tester.yaml +++ b/.github/workflows/container-pipeline-tester.yaml @@ -57,6 +57,9 @@ jobs: matrix: ${{ steps.matrix.outputs.matrix }} runs-on: [ k8-runners ] steps: + - uses: step-security/harden-runner@v2 + with: + egress-policy: audit - uses: actions/checkout@v4 - name: Set Matrix id: matrix @@ -83,11 +86,14 @@ jobs: matrix: ${{ fromJson(needs.setup-build-matrix.outputs.matrix) }} fail-fast: false steps: + - uses: step-security/harden-runner@v2 + with: + egress-policy: audit - uses: actions/checkout@v4 if: ${{ !inputs.no-build }} - uses: docker/login-action@v3 with: - registry: ${{ vars.REGISTRY }} + registry: ${{ secrets.REGISTRY }} username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_TOKEN }} if: ${{ !inputs.no-build }} @@ -99,8 +105,8 @@ jobs: with: group_dir: ${{ inputs.group_dir }} env_overrides: ${{ inputs.env_overrides }} - registry: ${{ vars.REGISTRY }} - repo: ${{ vars.REPO }} + registry: ${{ secrets.REGISTRY }} + repo: ${{ secrets.REPO }} no-push: false setup-test: needs: [ build-containers ] @@ -108,6 +114,9 @@ jobs: outputs: recipes: ${{ steps.recipes.outputs.RECIPES }} steps: + - uses: step-security/harden-runner@v2 + with: + egress-policy: audit - uses: actions/checkout@v4 - name: Get Recipes id: recipes @@ -122,18 +131,21 @@ jobs: experimental: [true] fail-fast: false steps: + - uses: step-security/harden-runner@v2 + with: + egress-policy: audit - uses: actions/checkout@v4 - uses: docker/login-action@v3 with: - registry: ${{ vars.REGISTRY }} + registry: ${{ secrets.REGISTRY }} username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_TOKEN }} - name: Test Container Group uses: ./test-runner with: - mlops_repo: ${{ vars.MLOPS_REPO }} + mlops_repo: ${{ secrets.MLOPS_REPO }} mlops_ref: ${{ github.ref }} recipe_dir: ${{ inputs.group_dir }} - registry: ${{ vars.REGISTRY }} + registry: ${{ secrets.REGISTRY }} test_dir: ${{ matrix.recipe }} token: ${{ github.token }} diff --git a/.github/workflows/dependency-review.yaml b/.github/workflows/dependency-review.yaml new file mode 100644 index 00000000..c03f9ad9 --- /dev/null +++ b/.github/workflows/dependency-review.yaml @@ -0,0 +1,40 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dependency Review Action +# +# This Action will scan dependency manifest files that change as part of a Pull Request, +# surfacing known-vulnerable versions of the packages declared or updated in the PR. +# Once installed, if the workflow run is marked as required, +# PRs introducing known-vulnerable packages will be blocked from merging. +# +# Source repository: https://github.com/actions/dependency-review-action +name: 'Dependency Review' +on: [pull_request] +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: 'Checkout Repository' + uses: actions/checkout@v4 + - name: 'Dependency Review' + uses: actions/dependency-review-action@v4 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 9b449566..bdc36487 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -27,13 +27,11 @@ concurrency: jobs: build: name: Lint - runs-on: [k8-runners] - permissions: - contents: read - packages: read - # To report GitHub Actions status checks - statuses: write + runs-on: k8-runners # ubuntu-latest steps: + - uses: step-security/harden-runner@v2 + with: + egress-policy: audit - name: Checkout code uses: actions/checkout@v4 with: diff --git a/.github/workflows/ossf.yaml b/.github/workflows/ossf.yaml deleted file mode 100644 index 09392a52..00000000 --- a/.github/workflows/ossf.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This workflow uses actions that are not certified by GitHub. They are provided -# by a third-party and are governed by separate terms of service, privacy -# policy, and support documentation. - -name: Scorecard supply-chain security -on: - # For Branch-Protection check. Only the default branch is supported. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection - branch_protection_rule: - # To guarantee Maintained check is occasionally updated. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained - schedule: - - cron: '29 17 * * 4' - push: - branches: [ "main" ] - -# Declare default permissions as read only. -permissions: read-all - -jobs: - analysis: - name: Scorecard analysis - runs-on: ubuntu-latest - permissions: - # Needed to upload the results to code-scanning dashboard. - security-events: write - # Needed to publish results and get a badge (see publish_results below). - id-token: write - # Uncomment the permissions below if installing in a private repository. - contents: read - actions: read - - steps: - - name: "Checkout code" - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - persist-credentials: false - - - name: "Run analysis" - uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86 # v2.1.2 - with: - results_file: results.sarif - results_format: sarif - # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: - # - you want to enable the Branch-Protection check on a *public* repository, or - # - you are installing Scorecard on a *private* repository - # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. - repo_token: ${{ secrets.ACTION_TOKEN }} - - # Public repositories: - # - Publish results to OpenSSF REST API for easy access by consumers - # - Allows the repository to include the Scorecard badge. - # - See https://github.com/ossf/scorecard-action#publishing-results. - # For private repositories: - # - `publish_results` will always be set to `false`, regardless - # of the value entered here. - publish_results: true - - # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF - # format to the repository Actions tab. - - name: "Upload artifact" - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0 - with: - name: SARIF file - path: results.sarif - retention-days: 5 - - # Upload the results to GitHub's code scanning dashboard. - - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@17573ee1cc1b9d061760f3a006fc4aac4f944fd5 # v2.2.4 - with: - sarif_file: results.sarif diff --git a/.github/workflows/review-trigger.yml b/.github/workflows/review-trigger.yml index 52b501bf..f429ca95 100644 --- a/.github/workflows/review-trigger.yml +++ b/.github/workflows/review-trigger.yml @@ -31,6 +31,9 @@ jobs: runner_label: ${{ steps.comment-inputs.outputs.runner_label }} no-build: ${{ steps.comment-inputs.outputs.no-build || false }} steps: + - uses: step-security/harden-runner@v2 + with: + egress-policy: audit - uses: actions/checkout@v4 - name: Get Inputs id: comment-inputs @@ -89,6 +92,9 @@ jobs: pull-requests: write runs-on: [ k8-runners ] steps: + - uses: step-security/harden-runner@v2 + with: + egress-policy: audit - name: Unlock if: ${{ needs.check-review.result != 'skipped' }} uses: actions/github-script@v7 diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml new file mode 100644 index 00000000..c2e8b310 --- /dev/null +++ b/.github/workflows/scorecard.yaml @@ -0,0 +1,57 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow uses actions that are not certified by GitHub. They are provided +# by a third-party and are governed by separate terms of service, privacy +# policy, and support documentation. + +name: Scorecard supply-chain security +on: + branch_protection_rule: + schedule: + - cron: '0 6 * * 0' + push: + branches: [ "main" ] + workflow_dispatch: +permissions: read-all +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + security-events: write + id-token: write + contents: read + actions: read + steps: + - uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: ossf/scorecard-action@v2.3.1 + with: + results_file: results.sarif + results_format: sarif + repo_token: ${{ secrets.ACTION_TOKEN }} + publish_results: true + - uses: actions/upload-artifact@v3 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + - uses: github/codeql-action/upload-sarif@v3.24.10 + with: + sarif_file: results.sarif diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 510d0e83..7c34bf37 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -15,23 +15,28 @@ --- name: Test Runner Unit Tests on: - pull_request: + push: null permissions: read-all concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: unit-test: - runs-on: [ test-runner ] + runs-on: [ k8-runners ] steps: + - uses: step-security/harden-runner@v2 + with: + egress-policy: audit - uses: actions/checkout@v4 - uses: docker/login-action@v3 with: - registry: ${{ vars.REGISTRY }} + registry: ${{ secrets.REGISTRY }} username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_TOKEN }} - name: Install requirements run: | + sudo apt-get update -y + sudo apt-get install python3.8-venv -y python -m venv venv source venv/bin/activate venv/bin/python -m pip install pytest hypothesis @@ -41,4 +46,4 @@ jobs: run: venv/bin/python -m pytest tests/utest.py -W ignore::UserWarning working-directory: ${{ github.workspace }}/test-runner env: - REGISTRY: ${{ vars.REGISTRY }} + REGISTRY: ${{ secrets.REGISTRY }} diff --git a/.gitignore b/.gitignore index be0bf130..36231154 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ **.ipynb_checkpoints** logs/ output/ +venv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 57dacd94..a84c3e8e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,20 @@ --- +ci: + autofix_commit_msg: "[pre-commit.ci] auto fixes from pre-commit.com hooks" + autofix_prs: true + autoupdate_commit_msg: "[pre-commit.ci] pre-commit autoupdate" + autoupdate_schedule: "weekly" + skip: + [ + pylint, + pytest-test-runner, + shellcheck, + markdownlint, + hadolint-docker, + shfmt + ] repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 - hooks: + - hooks: - id: check-added-large-files - id: check-ast - id: check-merge-conflict @@ -11,53 +23,59 @@ repos: - id: forbid-submodules - id: sort-simple-yaml - id: trailing-whitespace - - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.5 - hooks: - - id: insert-license - files: | - (?x)^( - .*(py|yaml|yml|sh|Dockerfile)| - )$ - args: + repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + - hooks: + - id: gitleaks + repo: https://github.com/gitleaks/gitleaks + rev: v8.16.3 + - hooks: + - args: [ --license-filepath=.github/license_template.txt, --use-current-year, --detect-license-in-X-top-lines=40, - --skip-license-insertion-comment=Copyright, + --skip-license-insertion-comment=Copyright ] - - repo: https://github.com/hadolint/hadolint + files: | + (?x)^( + .*(py|yaml|yml|sh|Dockerfile)| + )$ + id: insert-license + repo: https://github.com/Lucas-C/pre-commit-hooks + rev: v1.5.5 + - hooks: + - args: [ "--config", ".github/linters/.hadolint.yaml" ] + id: hadolint-docker + repo: https://github.com/hadolint/hadolint rev: v2.12.1-beta - hooks: - - id: hadolint-docker - args: ["--config", ".github/linters/.hadolint.yaml"] - - repo: https://github.com/igorshubovych/markdownlint-cli + - hooks: + - args: [ "--config", ".github/linters/.markdown-lint.yaml" ] + id: markdownlint + repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.38.0 - hooks: - - id: markdownlint - args: ["--config", ".github/linters/.markdown-lint.yaml"] - - repo: https://github.com/koalaman/shellcheck-precommit - rev: v0.9.0 - hooks: + - hooks: - id: shellcheck - - repo: https://github.com/cisagov/pre-commit-shfmt - rev: v0.0.2 - hooks: + repo: https://github.com/koalaman/shellcheck-precommit + rev: v0.9.0 + - hooks: - id: shfmt - - repo: https://github.com/ambv/black - rev: 23.12.1 - hooks: + repo: https://github.com/cisagov/pre-commit-shfmt + rev: v0.0.2 + - hooks: - id: black - - repo: local - hooks: - - id: pylint - name: pylint + repo: https://github.com/ambv/black + rev: 23.12.1 + - hooks: + - args: [ "--rcfile=.github/linters/.python-lint" ] entry: pylint + id: pylint language: system - types: [python] - args: ["--rcfile=.github/linters/.python-lint"] - - id: pytest-test-runner - name: pytest-test-runner + name: pylint + types: [ python ] + - entry: bash -c 'cd test-runner && pytest tests/utest.py -W ignore::UserWarning' + id: pytest-test-runner language: system - types: [python] - entry: bash -c 'cd test-runner && pytest tests/utest.py -W ignore::UserWarning' + name: pytest-test-runner + types: [ python ] + repo: local diff --git a/README.md b/README.md index eafe948c..01384595 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # Intel® AI Containers [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8270/badge)](https://www.bestpractices.dev/projects/8270) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/intel/ai-containers/badge)](https://securityscorecards.dev/viewer/?uri=github.com/intel/ai-containers) +[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fintel%2Fai-containers.svg?type=shield&issueType=license)](https://app.fossa.com/projects/git%2Bgithub.com%2Fintel%2Fai-containers?ref=badge_shield&issueType=license) +[![CodeQL](https://github.com/intel/ai-containers/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/intel/ai-containers/actions/workflows/github-code-scanning/codeql) +[![Unit Tests](https://github.com/intel/ai-containers/actions/workflows/unit-test.yaml/badge.svg?branch=main)](https://github.com/intel/ai-containers/actions/workflows/unit-test.yaml) This repository contains Dockerfiles, scripts, yaml files, Helm charts, etc. used to scale out AI containers with versions of TensorFlow and PyTorch that have been optimized for Intel platforms. Scaling is done with python, Docker, kubernetes, kubeflow, cnvrg.io, Helm, and other container orchestration frameworks for use in the cloud and on-premise. diff --git a/test-runner/README.md b/test-runner/README.md index 43d9c7f3..717bb8a3 100644 --- a/test-runner/README.md +++ b/test-runner/README.md @@ -190,14 +190,14 @@ test-containers: - uses: actions/checkout@v4 - uses: docker/login-action@v3 with: - registry: ${{ vars.REGISTRY }} + registry: ${{ secrets.REGISTRY }} username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_TOKEN }} - name: Test Container Group uses: intel/ai-containers/test-runner@main with: mlops_repo: intel/ai-containers - registry: ${{ vars.REGISTRY }} + registry: ${{ secrets.REGISTRY }} test_dir: /path/to/test/dir token: ${{ github.token }} ```