Cache pip

ml6team · Jun 27, 2023 · c7a283e · c7a283e
1 parent 882193f
commit c7a283e
Show file tree

Hide file tree

Showing 16 changed files with 73 additions and 39 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -12,6 +12,12 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+          cache: pip
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -17,7 +17,7 @@ jobs:
     - uses: actions/checkout@master
 
     - name: Set up Python 3.9
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
         python-version: 3.9
 

diff --git a/components/caption_images/Dockerfile b/components/caption_images/Dockerfile
@@ -7,12 +7,14 @@ RUN apt-get update && \
 
 # Install requirements
 COPY requirements.txt ./
-RUN pip3 install --no-cache-dir -r requirements.txt
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

diff --git a/components/download_images/Dockerfile b/components/download_images/Dockerfile
@@ -6,13 +6,15 @@ RUN apt-get update && \
     apt-get install git -y
 
 # Install requirements
-COPY requirements.txt /
-RUN pip3 install --no-cache-dir -r requirements.txt
+COPY requirements.txt ./
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

diff --git a/components/embedding_based_laion_retrieval/Dockerfile b/components/embedding_based_laion_retrieval/Dockerfile
@@ -6,13 +6,15 @@ RUN apt-get update && \
     apt-get install git -y
 
 # Install requirements
-COPY requirements.txt /
-RUN pip3 install --no-cache-dir -r requirements.txt
+COPY requirements.txt ./
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

diff --git a/components/filter_comments/Dockerfile b/components/filter_comments/Dockerfile
@@ -7,12 +7,14 @@ RUN apt-get update && \
 
 # Install requirements
 COPY requirements.txt ./
-RUN pip3 install --no-cache-dir -r requirements.txt
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

diff --git a/components/filter_line_length/Dockerfile b/components/filter_line_length/Dockerfile
@@ -7,12 +7,14 @@ RUN apt-get update && \
 
 # Install requirements
 COPY requirements.txt ./
-RUN pip3 install --no-cache-dir -r requirements.txt
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

diff --git a/components/image_cropping/Dockerfile b/components/image_cropping/Dockerfile
@@ -6,13 +6,15 @@ RUN apt-get update && \
     apt-get install git -y
 
 # Install requirements
-COPY requirements.txt /
-RUN pip3 install --no-cache-dir -r requirements.txt
+COPY requirements.txt ./
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

diff --git a/components/image_embedding/Dockerfile b/components/image_embedding/Dockerfile
@@ -6,13 +6,15 @@ RUN apt-get update && \
     apt-get install git -y
 
 # Install requirements
-COPY requirements.txt /
-RUN pip3 install --no-cache-dir -r requirements.txt
+COPY requirements.txt ./
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

diff --git a/components/image_resolution_extraction/Dockerfile b/components/image_resolution_extraction/Dockerfile
@@ -6,13 +6,15 @@ RUN apt-get update && \
     apt-get install git -y
 
 # Install requirements
-COPY requirements.txt /
-RUN pip3 install --no-cache-dir -r requirements.txt
+COPY requirements.txt ./
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

diff --git a/components/image_resolution_filtering/Dockerfile b/components/image_resolution_filtering/Dockerfile
@@ -6,13 +6,15 @@ RUN apt-get update && \
     apt-get install git -y
 
 # Install requirements
-COPY requirements.txt /
-RUN pip3 install --no-cache-dir -r requirements.txt
+COPY requirements.txt ./
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

diff --git a/components/load_from_hf_hub/Dockerfile b/components/load_from_hf_hub/Dockerfile
@@ -6,13 +6,15 @@ RUN apt-get update && \
     apt-get install git -y
 
 # Install requirements
-COPY requirements.txt /
-RUN pip3 install --no-cache-dir -r requirements.txt
+COPY requirements.txt ./
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

diff --git a/components/pii_redaction/Dockerfile b/components/pii_redaction/Dockerfile
@@ -6,13 +6,15 @@ RUN apt-get update && \
     apt-get install git -y
 
 # Install requirements
-COPY requirements.txt /
-RUN pip3 install --no-cache-dir -r requirements.txt
+COPY requirements.txt ./
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

diff --git a/components/prompt_based_laion_retrieval/Dockerfile b/components/prompt_based_laion_retrieval/Dockerfile
@@ -6,13 +6,15 @@ RUN apt-get update && \
     apt-get install git -y
 
 # Install requirements
-COPY requirements.txt /
-RUN pip3 install --no-cache-dir -r requirements.txt
+COPY requirements.txt ./
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

diff --git a/components/segment_images/Dockerfile b/components/segment_images/Dockerfile
@@ -7,12 +7,14 @@ RUN apt-get update && \
 
 # Install requirements
 COPY requirements.txt ./
-RUN pip3 install --no-cache-dir -r requirements.txt
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

diff --git a/components/write_to_hf_hub/Dockerfile b/components/write_to_hf_hub/Dockerfile
@@ -6,13 +6,15 @@ RUN apt-get update && \
     apt-get install git -y
 
 # Install requirements
-COPY requirements.txt /
-RUN pip3 install --no-cache-dir -r requirements.txt
+COPY requirements.txt ./
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=dev
-RUN pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN --mount=type=cache,target=~/.cache/pip \
+    pip3 install git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src