Use hosted CI runners for building docker images (#1915)

* New technique * needs * explicit all * Volume prune not going * Skip volume * versions * Avoid checkout perhaps? * Working dir * Don't include dot-slash? * Accelerate prefix? * Working directory? * Context? * other workingdir * Faster iteration * Right tag * Full * Release * GPU
huggingface · Aug 31, 2023 · 1b998f1 · 1b998f1
1 parent 7befe58
commit 1b998f1
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 32 deletions.
diff --git a/.github/workflows/build-docker-images-release.yml b/.github/workflows/build-docker-images-release.yml
@@ -21,40 +21,40 @@ jobs:
 
   version-cpu:
     name: "Latest Accelerate CPU [version]"
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
     needs: get-version
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Check out code
-        uses: actions/checkout@v2
+        uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Build and Push CPU
-        run: |
-          docker build -t huggingface/accelerate-cpu -f ./docker/accelerate-cpu/Dockerfile .
-          docker push huggingface/accelerate-cpu:${{needs.get-version.outputs.version}}
+        uses: docker/build-push-action@v4
+        with:
+          file: docker/accelerate-cpu/Dockerfile
+          push: true
+          tags: huggingface/accelerate-cpu:${{needs.get-version.outputs.version}}
 
   version-cuda:
     name: "Latest Accelerate GPU [version]"
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
     needs: get-version
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Check out code
-        uses: actions/checkout@v2
+        uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Build and Push GPU
-        run: |
-          docker build -t huggingface/accelerate-gpu -f ./docker/accelerate-gpu/Dockerfile .
-          docker push huggingface/accelerate-gpu:${{needs.get-version.outputs.version}}
+        uses: docker/build-push-action@v4
+        with:
+          file: docker/accelerate-gpu/Dockerfile
+          push: true
+          tags: huggingface/accelerate-gpu:${{needs.get-version.outputs.version}}
diff --git a/.github/workflows/build_docker_images.yml b/.github/workflows/build_docker_images.yml
@@ -11,39 +11,50 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  clean-storage:
+    name: "Clean docker image storage"
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    steps:
+      - name: Clean storage
+        run: |
+          docker image prune --all -f --filter "until=48h"
+          docker system prune --all -f --filter "until=48h"
+
   latest-cpu:
     name: "Latest Accelerate CPU [dev]"
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    needs: clean-storage
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Check out code
-        uses: actions/checkout@v2
+        uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
       - name: Build and Push CPU
-        run: |
-          docker build -t huggingface/accelerate-cpu -f ./docker/accelerate-cpu/Dockerfile .
-          docker push huggingface/accelerate-cpu:latest
+        uses: docker/build-push-action@v4
+        with:
+          file: docker/accelerate-cpu/Dockerfile          
+          push: true
+          tags: huggingface/accelerate-cpu
 
   latest-cuda:
     name: "Latest Accelerate GPU [dev]"
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    needs: clean-storage
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Check out code
-        uses: actions/checkout@v2
+        uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Build and Push GPU
-        run: |
-          docker build -t huggingface/accelerate-gpu -f ./docker/accelerate-gpu/Dockerfile .
-          docker push huggingface/accelerate-gpu:latest
+        uses: docker/build-push-action@v4
+        with:
+          file: docker/accelerate-gpu/Dockerfile          
+          push: true
+          tags: huggingface/accelerate-gpu