Switch to Poetry for dependency and build management

.github/workflows/ci.yml

@@ -30,27 +30,23 @@ jobs:
       uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Set up Poetry
+      uses: snok/install-poetry@v1
     - name: Install dependencies
       run: |
-        python -m venv venv
-        . venv/bin/activate
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install tox
+        poetry install
 
     - name: Lint with flake8
       run: |
-        pip install flake8
         # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=W503
+        poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=W503
     - name: Run tox
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
-        . venv/bin/activate
-        tox -e ${{ matrix.tox-py }}
+        poetry run tox -e ${{ matrix.tox-py }}
 
     - uses: codecov/codecov-action@v1
       name: Report code coverage  
@@ -173,23 +169,13 @@ jobs:
       with:
         java-version: '8.0.252'
         architecture: x64
+    - name: Setup Poetry
+      uses: snok/install-poetry@v1
     - name: Install dependencies
       run: |
-        python -m venv venv
-        . venv/bin/activate
-        sudo apt-get remove azure-cli
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install -r requirements-test.txt
+        poetry install
+        sudo apt-get remove azure-cli || true
         pip install ccm
-        case '${{ matrix.it-backend }}' in
-          'azure'|'azure-hierarchical')
-            echo "No extra requirements for now."
-            ;;
-          'ibm'|'minio'|'s3')
-            echo "No extra requirements for now."
-            ;;
-        esac
 
     - name: Check if integration tests can run
       env:
@@ -216,7 +202,6 @@ jobs:
         AWS_SECRET_ACCESS_KEY: ${{ secrets.BUCKET_SECRET }}
       run: |
         set -e
-        . venv/bin/activate
         if [[ "${{ matrix.it-backend }}" == "ibm" ]];
         then
           # Prevent awscli from using AWS secrets in case we're running against IBM cloud
@@ -292,7 +277,8 @@ jobs:
 
         # Move and convert the coverage analysis file to XML
         mv tests/integration/.coverage .
-        coverage xml
+        poetry run coverage xml
+
     - uses: codecov/codecov-action@v1
       name: Report code coverage
 

debian/cassandra-medusa.links

@@ -1,2 +1,2 @@
 usr/share/cassandra-medusa/bin/medusa usr/bin/medusa
-usr/share/cassandra-medusa/bin/medusa-wrapper usr/bin/medusa-wrapper
+usr/share/cassandra-medusa/bin/medusa-wrapper usr/bin/medusa-wrapper

debian/rules

@@ -21,6 +21,7 @@ PACKAGEVERSION = $(VERSION)-0~$(DISTRIBUTION)0
 PY3VER = $(shell py3versions -d)
 SSH2_LIBS_SUFFIX = debian/cassandra-medusa/usr/share/cassandra-medusa/lib/$(PY3VER)/site-packages/ssh2_python.libs/
 SSH_LIBS_SUFFIX = debian/cassandra-medusa/usr/share/cassandra-medusa/lib/$(PY3VER)/site-packages/ssh_python.libs/
+POETRY_VIRTUALENVS_IN_PROJECT = true
 
 export DH_ALWAYS_EXCLUDE = .git
 export DH_VIRTUALENV_INSTALL_ROOT = /usr/share
@@ -29,8 +30,8 @@ export DH_VIRTUALENV_INSTALL_ROOT = /usr/share
 
 override_dh_virtualenv:
 	dh_virtualenv \
-	  --extra-pip-arg "--no-cache-dir" \
-	  --python /usr/bin/python3 --preinstall=setuptools==40.3.0 --preinstall=pip==21.3.1 --preinstall=wheel --builtin-venv
+	--python /usr/bin/python3 --preinstall=setuptools==40.3.0 --preinstall=pip==21.3.1 --preinstall=wheel --builtin-venv \
+	--preinstall=poetry==1.6.1 --preinstall=dh-poetry==0.2.0 --pip-tool=dh-poetry
 
 override_dh_strip:
 	dh_strip --no-automatic-dbgsym -X libssh2 -X libssh -X libgssapi_krb5 -X libcrypto -X libkrb5 -X libk5crypto
@@ -40,3 +41,7 @@ override_dh_shlibdeps:
 
 override_dh_gencontrol:
 	dh_gencontrol -- -v$(PACKAGEVERSION)
+
+override_dh_auto_configure:
+	# overriding with a no-op to avoid dh to look for setup.py
+	true

k8s/Dockerfile

@@ -27,21 +27,18 @@ ENV PATH=/root/.local/bin:$PATH
 COPY . /build/
 
 # General requirements
-RUN python3 -m pip install -U pip && pip3 install --ignore-installed --user \
-    -r /build/requirements.txt \
-    -r /build/requirements-grpc-runtime.txt
+ENV POETRY_VIRTUALENVS_IN_PROJECT=true
+RUN python3 -m pip install -U pip && pip3 install --ignore-installed --user poetry
 
 # Build medusa itself so we can add the executables in the final image
-RUN pip3 install --ignore-installed --user /build
+RUN cd /build && poetry install
 
 # Could be python:slim, but we have a .sh entrypoint
 FROM ubuntu:22.04
 
 ## add user
 RUN groupadd -r cassandra --gid=999 && useradd -r -g cassandra --uid=999 --create-home cassandra
 
-COPY --from=base /install /usr/local
-
 # wget could happen in the build-phase
 RUN apt-get update && apt-get install -y python3 python3-setuptools wget \
     && rm -rf /var/lib/apt/lists/*
@@ -59,6 +56,8 @@ ENV PATH=/home/cassandra/.local/bin:/home/cassandra/google-cloud-sdk/bin:$PATH
 
 COPY --from=base --chown=cassandra:cassandra /root/.local /home/cassandra/.local
 
+COPY --from=base --chown=cassandra:cassandra /build/.venv /home/cassandra/.venv
+COPY --from=base --chown=cassandra:cassandra /build/pyproject.toml /home/cassandra/pyproject.toml
 COPY --chown=cassandra:cassandra medusa /home/cassandra/medusa
 COPY --chown=cassandra:cassandra k8s/docker-entrypoint.sh /home/cassandra
 

k8s/docker-entrypoint.sh

@@ -41,14 +41,14 @@ restore() {
         echo "Skipping restore operation"    
     else
         echo "Restoring backup $BACKUP_NAME"
-        python3 -m medusa.service.grpc.restore -- "/etc/medusa/medusa.ini" $RESTORE_KEY
+        poetry run python -m medusa.service.grpc.restore -- "/etc/medusa/medusa.ini" $RESTORE_KEY
         echo $RESTORE_KEY > $last_restore_file
     fi
 }
 
 grpc() {
     echo "Starting Medusa gRPC service"
-    exec python3 -m medusa.service.grpc.server server.py
+    exec poetry run python -m medusa.service.grpc.server server.py
 }
 
 echo "sleeping for $DEBUG_SLEEP sec"

medusa/service/grpc/restore.py

@@ -104,9 +104,9 @@ def restore_backup(in_place, config):
 
 
 if __name__ == '__main__':
-    if len(sys.argv) > 3:
-        config_file_path = sys.argv[2]
-        restore_key = sys.argv[3]
+    if len(sys.argv) > 2:
+        config_file_path = sys.argv[1]
+        restore_key = sys.argv[2]
     else:
         logging.error("Usage: {} <config_file_path> <restore_key>".format(sys.argv[0]))
         sys.exit(1)

packaging/docker-build/Dockerfile

@@ -33,8 +33,8 @@ RUN apt-get update && \
 # install dependencies
 RUN apt-get update \
     && apt-get install -y \
-        libffi-dev \   
-        libssl-dev \  
+        libffi-dev \
+        libssl-dev \
         debhelper \
         gdebi-core \
         gcc \
@@ -58,13 +58,14 @@ RUN apt-get update \
 RUN pip3 install --upgrade pip
 
 RUN cd /tmp && \
-    wget http://mirrors.kernel.org/ubuntu/pool/universe/d/dh-virtualenv/dh-virtualenv_1.0-1_all.deb && \
+    wget http://mirrors.kernel.org/ubuntu/pool/universe/d/dh-virtualenv/dh-virtualenv_1.2.2-1.4_all.deb && \
     gdebi -n dh-virtualenv*.deb && \
     rm dh-virtualenv_*.deb
 
 
 RUN pip3 install greenlet
 RUN pip3 install gevent
+RUN pip3 install poetry
 
 # Add entrypoint script
 COPY packaging/docker-build/docker-entrypoint.sh ${WORKDIR}