Merge pull request #239 from tattle-made/hotfix

Hotfix

README.md

@@ -39,7 +39,7 @@ Please create a new Discussion [here](https://github.com/tattle-made/tattle-api/
   ```
   # Install locally in venv
   $ cd src/api/
-  $ pip install -r requirements.txt
+  $ pip install  --require-hashes --no-deps -r requirements.txt
   ```
 
 
@@ -67,8 +67,8 @@ Please create a new Discussion [here](https://github.com/tattle-made/tattle-api/
   ```
   # Install locally in venv
   $ cd src/api/core/operators/
-  $ pip install -r image_vec_rep_resnet_requirements.txt
-  $ pip install -r vid_vec_rep_resnet_requirements.txt
+  $ pip install  --require-hashes --no-deps -r image_vec_rep_resnet_requirements.txt
+  $ pip install  --require-hashes --no-deps -r vid_vec_rep_resnet_requirements.txt
 ..
 # Create the docker containers
   $ cd src/api/
@@ -127,13 +127,13 @@ Note:
 ```bash
 $ cd src/
 $ pip install --upgrade pip-tools
-$ TMPDIR=<temp_dir> pip-compile --verbose --generate-hashes --emit-index-url --emit-find-links requirements.in
+$ TMPDIR=<temp_dir> pip-compile --verbose --allow-unsafe --generate-hashes --emit-index-url --emit-find-links requirements.in
 
 # Updating operators
 $ cd src/core/operators/
 # The link for torch is required since PyPi only hosts the GPU version of torch packages.
-$ TMPDIR=<temp_dir> pip-compile --verbose --generate-hashes --emit-index-url --emit-find-links --find-links https://download.pytorch.org/whl/torch_stable.html vid_vec_rep_resnet_requirements.in
-$ TMPDIR=<temp_dir> pip-compile --verbose --generate-hashes --emit-index-url --emit-find-links --find-links https://download.pytorch.org/whl/torch_stable.html audio_vec_embedding_requirements.in
+$ TMPDIR=<temp_dir> pip-compile --verbose --allow-unsafe --generate-hashes --emit-index-url --emit-find-links --find-links https://download.pytorch.org/whl/torch_stable.html vid_vec_rep_resnet_requirements.in
+$ TMPDIR=<temp_dir> pip-compile --verbose --allow-unsafe --generate-hashes --emit-index-url --emit-find-links --find-links https://download.pytorch.org/whl/torch_stable.html audio_vec_embedding_requirements.in
 ```
 
 #### Modify generated `requirements.txt` for platform specific torch packages
@@ -199,7 +199,7 @@ torchvision==0.17.0+cpu; platform_machine=='x86_64' \
 This is useful to update dependencies e.g. when using `pip-audit` 
 
 ```bash
-$ TMPDIR=<temp_dir> pip-compile --verbose --generate-hashes --find-links https://download.pytorch.org/whl/torch_stable.html --upgrade-package <package>==<version> --upgrade-package <package>
+$ TMPDIR=<temp_dir> pip-compile --verbose --allow-unsafe --generate-hashes --find-links https://download.pytorch.org/whl/torch_stable.html --upgrade-package <package>==<version> --upgrade-package <package>
 
 ```
 

src/Dockerfile

@@ -26,7 +26,10 @@ WORKDIR /usr/app
 RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv
 ENV PATH="/usr/app/venv/bin:$PATH"
 
-RUN pip install --no-cache-dir --upgrade pip
+# base requirements file
+COPY --chown=python:python base_requirements.txt /usr/app/base_requirements.txt
+RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/base_requirements.txt
+
 RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends vim curl
 # RUN apt-get install -y ffmpeg
 # RUN apt-get update && \

src/Dockerfile.test

@@ -24,13 +24,14 @@ RUN python -m venv /home/python/app/venv \
 # Set venv in path
 ENV PATH="/home/python/app/venv/bin:$PATH"
 
-# Copy core and operator requirements
+# Copy base, core and operator requirements
+COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt
 COPY --chown=python:python requirements.txt /home/python/app/requirements.txt
 COPY --chown=python:python ./core/operators/vid_vec_rep_resnet_requirements.txt /home/python/app/core/operators/vid_vec_rep_resnet_requirements.txt
 COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /home/python/app/core/operators/audio_vec_embedding_requirements.txt
 
 # Run pip install
-RUN pip install --no-cache-dir --upgrade pip \
+RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \
     && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt \
     && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/core/operators/vid_vec_rep_resnet_requirements.txt \
     && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/core/operators/audio_vec_embedding_requirements.txt

src/base_requirements.in

@@ -0,0 +1 @@
+pip==24.0

src/base_requirements.txt

@@ -0,0 +1,12 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --allow-unsafe --generate-hashes base_requirements.in
+#
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==24.0 \
+    --hash=sha256:ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc \
+    --hash=sha256:ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2
+    # via -r base_requirements.in

src/benchmark/audiovec/Dockerfile.audio_vec_embedding

@@ -20,7 +20,9 @@ WORKDIR /usr/app
 RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv
 ENV PATH="/usr/app/venv/bin:$PATH"
 
-RUN pip install --no-cache-dir --upgrade pip
+# base requirements file
+COPY --chown=python:python base_requirements.txt /usr/app/base_requirements.txt
+RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/base_requirements.txt
 
 # audio requirments file
 COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /usr/app/core/operators/audio_vec_embedding_requirements.txt

src/benchmark/audiovec/Dockerfile.audio_vec_embedding_graviton

@@ -20,7 +20,9 @@ WORKDIR /usr/app
 RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv
 ENV PATH="/usr/app/venv/bin:$PATH"
 
-RUN pip install --no-cache-dir --upgrade pip
+# base requirements file
+COPY --chown=python:python base_requirements.txt /usr/app/base_requirements.txt
+RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/base_requirements.txt
 
 # audio requirments file
 COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /usr/app/core/operators/audio_vec_embedding_requirements.txt

src/benchmark/imgvec/Dockerfile.image_vec_rep_resnet

@@ -23,7 +23,10 @@ WORKDIR /usr/app
 RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv
 ENV PATH="/usr/app/venv/bin:$PATH"
 
-RUN pip install --no-cache-dir --upgrade pip
+# base requirements file
+COPY --chown=python:python base_requirements.txt /usr/app/base_requirements.txt
+RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/base_requirements.txt
+
 COPY --chown=python:python ./core/operators/image_vec_rep_resnet_requirements.txt /usr/app/core/operators/image_vec_rep_resnet_requirements.txt
 RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/core/operators/image_vec_rep_resnet_requirements.txt
 COPY --chown=python:python ./core/operators/image_vec_rep_resnet.py /usr/app/core/operators/image_vec_rep_resnet.py

src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet

@@ -20,7 +20,10 @@ WORKDIR /usr/app
 RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv
 ENV PATH="/usr/app/venv/bin:$PATH"
 
-RUN pip install --no-cache-dir --upgrade pip
+# base requirements file
+COPY --chown=python:python base_requirements.txt /usr/app/base_requirements.txt
+RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/base_requirements.txt
+
 # video requirments file
 COPY --chown=python:python ./core/operators/vid_vec_rep_resnet_requirements.txt /usr/app/core/operators/vid_vec_rep_resnet_requirements.txt
 RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/core/operators/vid_vec_rep_resnet_requirements.txt

src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton

@@ -20,7 +20,9 @@ WORKDIR /usr/app
 RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv
 ENV PATH="/usr/app/venv/bin:$PATH"
 
-RUN pip install --no-cache-dir --upgrade pip
+# base requirements file
+COPY --chown=python:python base_requirements.txt /usr/app/base_requirements.txt
+RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/base_requirements.txt
 
 ### AWS Graviton Optimization ###
 

src/requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --generate-hashes requirements.in
+#    pip-compile --allow-unsafe --generate-hashes requirements.in
 #
 blinker==1.7.0 \
     --hash=sha256:c3f865d4d54db7abc53758a01601cf343fe55b84c1de4e3fa910e420b438d5b9 \
@@ -1203,7 +1203,10 @@ zope-interface==6.1 \
     --hash=sha256:fddbab55a2473f1d3b8833ec6b7ac31e8211b0aa608df5ab09ce07f3727326de
     # via gevent
 
-# WARNING: The following packages were not pinned, but pip requires them to be
-# pinned when the requirements file includes hashes and the requirement is not
-# satisfied by a package already installed. Consider using the --allow-unsafe flag.
-# setuptools
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==69.2.0 \
+    --hash=sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e \
+    --hash=sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c
+    # via
+    #   zope-event
+    #   zope-interface

src/worker/audiovec/Dockerfile.audio_worker

@@ -25,12 +25,13 @@ RUN python -m venv /home/python/app/venv \
 # Set venv in path
 ENV PATH="/home/python/app/venv/bin:$PATH"
 
-# Copy core and operator requirements
+# Copy base, core and operator requirements
+COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt
 COPY --chown=python:python requirements.txt /home/python/app/requirements.txt
 COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /home/python/app/core/operators/audio_vec_embedding_requirements.txt
 
 # Run pip install
-RUN pip install --no-cache-dir --upgrade pip \
+RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \
     && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt \
     && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/core/operators/audio_vec_embedding_requirements.txt
 

src/worker/audiovec/Dockerfile.audio_worker_graviton

@@ -25,12 +25,13 @@ RUN python -m venv /home/python/app/venv \
 # Set venv in path
 ENV PATH="/home/python/app/venv/bin:$PATH"
 
-# Copy core and operator requirements
+# Copy base, core and operator requirements
+COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt
 COPY --chown=python:python requirements.txt /home/python/app/requirements.txt
 COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /home/python/app/core/operators/audio_vec_embedding_requirements.txt
 
 # Run pip install
-RUN pip install --no-cache-dir --upgrade pip \
+RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \
     && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt \
     && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/core/operators/audio_vec_embedding_requirements.txt
 

src/worker/hash/Dockerfile.hash_worker

@@ -25,11 +25,12 @@ RUN python -m venv /home/python/app/venv \
 # Set venv in path
 ENV PATH="/home/python/app/venv/bin:$PATH"
 
-# Copy core and operator requirements
+# Copy base, core and operator requirements
+COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt
 COPY --chown=python:python requirements.txt /home/python/app/requirements.txt
 
 # Run pip install
-RUN pip install --no-cache-dir --upgrade pip \
+RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \
     && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt
 
 #####################################

src/worker/hash/Dockerfile.hash_worker_graviton

@@ -25,11 +25,12 @@ RUN python -m venv /home/python/app/venv \
 # Set venv in path
 ENV PATH="/home/python/app/venv/bin:$PATH"
 
-# Copy core and operator requirements
+# Copy base, core and operator requirements
+COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt
 COPY --chown=python:python requirements.txt /home/python/app/requirements.txt
 
 # Run pip install
-RUN pip install --no-cache-dir --upgrade pip \
+RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \
     && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt
 
 #####################################

src/worker/vidvec/Dockerfile.video_worker

@@ -25,12 +25,13 @@ RUN python -m venv /home/python/app/venv \
 # Set venv in path
 ENV PATH="/home/python/app/venv/bin:$PATH"
 
-# Copy core and operator requirements
+# Copy base, core and operator requirements
+COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt
 COPY --chown=python:python requirements.txt /home/python/app/requirements.txt
 COPY --chown=python:python ./core/operators/vid_vec_rep_resnet_requirements.txt /home/python/app/core/operators/vid_vec_rep_resnet_requirements.txt
 
 # Run pip install
-RUN pip install --no-cache-dir --upgrade pip \
+RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \
     && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt \
     && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/core/operators/vid_vec_rep_resnet_requirements.txt
 

src/worker/vidvec/Dockerfile.video_worker_graviton

@@ -25,12 +25,13 @@ RUN python -m venv /home/python/app/venv \
 # Set venv in path
 ENV PATH="/home/python/app/venv/bin:$PATH"
 
-# Copy core and operator requirements
+# Copy base, core and operator requirements
+COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt
 COPY --chown=python:python requirements.txt /home/python/app/requirements.txt
 COPY --chown=python:python ./core/operators/vid_vec_rep_resnet_requirements.txt /home/python/app/core/operators/vid_vec_rep_resnet_requirements.txt
 
 # Run pip install
-RUN pip install --no-cache-dir --upgrade pip \
+RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \
     && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt \
     && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/core/operators/vid_vec_rep_resnet_requirements.txt