Impl media crc

.ci/docker-compose.yml

@@ -4,6 +4,9 @@ services:
       context: ./../src
       dockerfile: Dockerfile.test
       target: test
+      args:
+        - "UID=${UID:-1000}"
+        - "GID=${GID:-1000}"
 #    env_file: ./../src/test.env
     command: bash -c "python -m unittest discover -s ./tests/core/models -p "test_*.py";
       python -m unittest core.operators.test_vid_vec_rep_resnet;

.github/dependabot.yml

@@ -0,0 +1,33 @@
+version: 2
+updates:
+  # Enable version updates for npm
+  - package-ecosystem: "npm"
+    # Look for `package.json` and `lock` files in the `root` directory
+    directory: "/docs"
+    # Check the npm registry for updates every day (weekdays)
+    schedule:
+      interval: "weekly"
+
+  # Enable version updates for Docker
+  - package-ecosystem: "docker"
+    # Look for a `Dockerfile` in the `root` directory
+    directory: "/src"
+    # Check for updates once a week
+    schedule:
+      interval: "weekly"
+
+  # Enable version updates for Pip
+  - package-ecosystem: "pip"
+    # Look for a `Dockerfile` in the `root` directory
+    directory: "/src"
+    # Check for updates once a week
+    schedule:
+      interval: "weekly"
+
+  # Enable version updates for github-actions
+  - package-ecosystem: "github-actions"
+    # Look for a `Dockerfile` in the `root` directory
+    directory: ".github/workflows"
+    # Check for updates once a week
+    schedule:
+      interval: "weekly"

.github/workflows/bandit.yml

@@ -11,6 +11,10 @@
 # https://pypi.org/project/bandit/ is Apache v2.0 licensed, by PyCQA
 
 name: Bandit
+
+permissions:
+  contents: read # for actions/checkout to fetch code
+
 on:
   push:
     branches: [ "main" ]
@@ -22,15 +26,16 @@ jobs:
     permissions:
       contents: read # for actions/checkout to fetch code
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
-      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
+      # actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
 
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           ref: main
+
       - name: Bandit Scan
-        uses: shundor/python-bandit-scan@9cc5aa4a006482b8a7f91134412df6772dbda22c
+        uses: shundor/python-bandit-scan@9cc5aa4a006482b8a7f91134412df6772dbda22c # v1.0
         with: # optional arguments
           # exit with 0, even with results found
           exit_zero: true # optional, default is DEFAULT
@@ -48,4 +53,3 @@ jobs:
           # skips: # optional, default is DEFAULT
           # path to a .bandit file that supplies command line arguments
           # ini_path: # optional, default is DEFAULT
-

.github/workflows/docker-push-audiovec-worker-staging.yml

@@ -1,33 +1,46 @@
 name: Publish Audio Worker to Dockerhub for Staging
 
+permissions:
+  contents: read
+
 on: workflow_dispatch
 
 jobs:
-  api:
+  docker:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v.3.1.0
         with:
-          ref: development
-      - name: "Setup Node version"
-        uses: actions/setup-node@v4
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push amd64
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
         with:
-          node-version: 20.9.0
-      - name: Declare some variables
-        id: vars
-        shell: bash
-        run: |
-          echo "setting variables"
-          echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          context: "{{defaultContext}}:src/"
+          file: worker/audiovec/Dockerfile.audio_worker
+          platforms: linux/amd64
+          build-args: |
+            "UID=1000"
+            "GID=1000"
+          push: true
+          tags: tattletech/feluda-operator-audiovec:worker-amd64-latest
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - uses: elgohr/Publish-Docker-Github-Action@main
+      - name: Build and push arm64
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
         with:
-          username: tattletech
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          name: tattletech/feluda-operator-audiovec
-          workdir: src/
-          dockerfile: worker/audiovec/Dockerfile.audio_worker
-          tags: worker-amd64-latest
-          platforms: linux/amd64
+          context: "{{defaultContext}}:src/"
+          file: worker/audiovec/Dockerfile.audio_worker_graviton
+          platforms: linux/arm64
+          build-args: |
+            "UID=1000"
+            "GID=1000"
+          push: true
+          tags: tattletech/feluda-operator-audiovec:worker-arm64-latest

.github/workflows/docker-push-hash-worker-staging.yml

@@ -1,33 +1,46 @@
 name: Publish Hash Worker to Dockerhub for Staging
 
+permissions:
+  contents: read
+
 on: workflow_dispatch
 
 jobs:
-  api:
+  docker:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v.3.1.0
         with:
-          ref: development
-      - name: "Setup Node version"
-        uses: actions/setup-node@v4
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push amd64
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
         with:
-          node-version: 20.9.0
-      - name: Declare some variables
-        id: vars
-        shell: bash
-        run: |
-          echo "setting variables"
-          echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          context: "{{defaultContext}}:src/"
+          file: worker/hash/Dockerfile.hash_worker
+          platforms: linux/amd64
+          build-args: |
+            "UID=1000"
+            "GID=1000"
+          push: true
+          tags: tattletech/feluda-operator-hash:worker-amd64-latest
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - uses: elgohr/Publish-Docker-Github-Action@main
+      - name: Build and push arm64
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
         with:
-          username: tattletech
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          name: tattletech/feluda-operator-hash
-          workdir: src/
-          dockerfile: worker/hash/Dockerfile.hash_worker
-          tags: worker-amd64-latest
-          platforms: linux/amd64
+          context: "{{defaultContext}}:src/"
+          file: worker/hash/Dockerfile.hash_worker_graviton
+          platforms: linux/arm64
+          build-args: |
+            "UID=1000"
+            "GID=1000"
+          push: true
+          tags: tattletech/feluda-operator-hash:worker-arm64-latest

.github/workflows/docker-push-vidvec-benchmark-test.yml

@@ -1,42 +1,65 @@
-name: Publish Video Operator to Dockerhub
+name: Test Publish Video Operator Benchmark to Dockerhub
+
+permissions:
+  contents: read
 
 on: workflow_dispatch
 
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: main
-
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
+
+      - name: Cache Docker layers
+        uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v.3.1.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build and push amd64
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
         with:
-          context: ./src/
+          context: "{{defaultContext}}:src/"
           file: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet
           platforms: linux/amd64
+          build-args: |
+            "UID=1000"
+            "GID=1000"
           push: true
           tags: tattletech/feluda-operator-vidvec:benchmark-amd64-latest-test
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
 
       - name: Build and push arm64
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
         with:
-          context: ./src/
+          context: "{{defaultContext}}:src/"
           file: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton
           platforms: linux/arm64
+          build-args: |
+            "UID=1000"
+            "GID=1000"
           push: true
           tags: tattletech/feluda-operator-vidvec:benchmark-arm64-latest-test
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+      - # Temp fix - only do this after the last docker build step to have cache for all platforms
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/docker-push-vidvec-benchmark.yml

@@ -1,54 +1,46 @@
 name: Publish Video Operator to Dockerhub
 
+permissions:
+  contents: read
+
 on: workflow_dispatch
 
 jobs:
-  api:
+  docker:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: master
-      - name: "Setup Node version"
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20.9.0
-      - name: Declare some variables
-        id: vars
-        shell: bash
-        run: |
-          echo "setting variables"
-          echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - uses: elgohr/Publish-Docker-Github-Action@main
+        uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v.3.1.0
         with:
-          username: tattletech
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          name: tattletech/feluda-operator-vidvec
-          workdir: src/
-          dockerfile: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet
-          tags: benchmark-amd64-latest
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push amd64
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+        with:
+          context: "{{defaultContext}}:src/"
+          file: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet
           platforms: linux/amd64
-      - uses: elgohr/Publish-Docker-Github-Action@main
+          build-args: |
+            "UID=1000"
+            "GID=1000"
+          push: true
+          tags: tattletech/feluda-operator-vidvec:benchmark-amd64-latest
+
+      - name: Build and push arm64
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
         with:
-          username: tattletech
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          name: tattletech/feluda-operator-vidvec
-          workdir: src/
-          dockerfile: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton
-          tags: benchmark-arm64-latest
+          context: "{{defaultContext}}:src/"
+          file: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton
           platforms: linux/arm64
-
-      # - name: deploy to cluster
-      #   uses: steebchen/[email protected]
-      #   with: # defaults to latest kubectl binary version
-      #     config: ${{ secrets.PROD_KUBE_CONFIG_DATA }}
-      #     command: set image --record deployment/feluda-server feluda-server=tattletech/feluda-server:${{ steps.vars.outputs.sha_short }}
-      # - name: verify deployment
-      #   uses: steebchen/[email protected]
-      #   with:
-      #     config: ${{ secrets.PROD_KUBE_CONFIG_DATA }}
-      #     version: v1.21.0 # specify kubectl binary version explicitly
-      #     command: rollout status deployment/feluda-server
+          build-args: |
+            "UID=1000"
+            "GID=1000"
+          push: true
+          tags: tattletech/feluda-operator-vidvec:benchmark-arm64-latest