Use Python 3.6 in AM/SS RPM packages

* Install dh-virtualenv 1.2 from author's PPA * Use Python 3.6 and new requirements file * Update PYTHONPATH of MCPClient * Remove workarounds for Python 2.x * Remove workarounds for old releases * Remove quilt (no patches needed) * Remove unused build variables * Introduce archivematica placeholder package (deb) with virtualenv * Stop configuring packages.archivematica.org
artefactual-labs · May 31, 2021 · e3c3989 · e3c3989
1 parent b7a19de
commit e3c3989
Show file tree

Hide file tree

Showing 39 changed files with 292 additions and 505 deletions.
diff --git a/debs/bionic/archivematica-storage-service/Dockerfile b/debs/bionic/archivematica-storage-service/Dockerfile
@@ -1,15 +1,30 @@
 FROM ubuntu:bionic
 
-RUN apt-get update && \
-    apt-get install -y dpkg-dev git build-essential wget debhelper \
-    devscripts equivs quilt apt-transport-https python-minimal
-
-RUN wget -O /tmp/pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && python /tmp/pip.py
-
-RUN wget -O - https://packages.archivematica.org/1.7.x/key.asc |\
-    apt-key add - && \
-    echo "deb [arch=amd64] http://packages.archivematica.org/1.7.x/ubuntu-externals xenial main" >> /etc/apt/sources.list
-
-# Dependencies are also obtained in the debian build script, this speeds up the process of building packages
-RUN apt-get update &&  apt-get install -y libffi-dev libssl-dev libxslt-dev dh-virtualenv python-dev dh-systemd libmysqlclient-dev postgresql-server-dev-10 libldap2-dev libsasl2-dev
+ENV DEBIAN_FRONTEND noninteractive
 
+RUN apt-get update \
+	&& apt-get install -y \
+		dpkg-dev \
+		git \
+		build-essential \
+		debhelper \
+		devscripts \
+		equivs \
+		apt-transport-https \
+		python3.6-dev \
+		python3.6-distutils \
+		libffi-dev \
+		libssl-dev \
+		libxslt-dev \
+		dh-systemd \
+		libmysqlclient-dev \
+		postgresql-server-dev-10 \
+		libldap2-dev \
+		libsasl2-dev \
+		software-properties-common \
+	&& add-apt-repository ppa:jyrki-pulliainen/dh-virtualenv \
+	&& apt-get install -y dh-virtualenv \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 \
+	&& update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 \
+	&& curl -s https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py && python /tmp/get-pip.py
diff --git a/debs/bionic/archivematica-storage-service/Makefile b/debs/bionic/archivematica-storage-service/Makefile
@@ -1,75 +1,72 @@
 NAME          = am-packbuild
-PACKAGE	      ?= archivematica
+PACKAGE       = archivematica-storage-service
 DEB_TOPDIR    = "/debbuild"
 DOCKER_VOLUME = "/src"
-DOCKER_IMAGE  = "debbuild-$(NAME)-$(VERSION)"
-GPG_ID ?= 0F4A4D31
-PACKBUILD_EXTRA_ARGS ?=
-#PACKBUILD_EXTRA_ARGS ?= "-b 1"
-PACKAGE = archivematica-storage-service
-BRANCH     ?= qa/0.x
-VERSION     ?= 0.13.0
-RELEASE       ?= -3
-BUILD_TYPE         ?= ss
+DOCKER_IMAGE  = "debbuild-$(NAME)-$(PACKAGE)-$(VERSION)"
+GPG_ID        ?= 0F4A4D31
+BRANCH        ?= qa/0.x
+VERSION       ?= 0.18.0
+RELEASE       ?= -1
+GIT_REPO      = "https://github.com/artefactual/archivematica-storage-service"
 
-.PHONY: build-docker-image build deb-build deb-clean deb-test update-changelog
+.DEFAULT_GOAL := build
 
-all: build-docker-image deb-clean build
-
-build-docker-image: update-changelog
+.PHONY: build-docker-image
+build-docker-image:
 	@echo "==> Building Docker image with build environment."
 	@docker build --rm --tag "$(DOCKER_IMAGE)" .
 
-build:
+.PHONY: build
+build: build-docker-image
 	@echo "==> Building deb."
 	@docker run \
-                -e BRANCH=$(BRANCH) \
+		-e BRANCH=$(BRANCH) \
 		-e VERSION=$(VERSION) \
 		-e RELEASE=$(RELEASE) \
 		-e PACKAGE=$(PACKAGE) \
-                -e BUILD_TYPE=$(BUILD_TYPE) \
-                -e GPG_ID=$(GPG_ID) \
+		-e GPG_ID=$(GPG_ID) \
 		-e GPG_KEY \
-		-e PACKBUILD_EXTRA_ARGS="$(PACKBUILD_EXTRA_ARGS)" \
+		-e GIT_REPO="$(GIT_REPO)" \
  		--rm \
-                --volume "$(shell cd ../../ && pwd):$(DEB_TOPDIR)/$(NAME)" \
+		--volume "$(shell cd ../../ && pwd):$(DEB_TOPDIR)/$(NAME)" \
 		--volume "$(shell pwd):$(DOCKER_VOLUME)" \
-		 $(DOCKER_IMAGE) \
-                 make -C $(DOCKER_VOLUME) deb-build
+		$(DOCKER_IMAGE) \
+			make -C $(DOCKER_VOLUME) deb-build
 
-deb-build:
-	@echo "==> Install dependencies."
-	# Add GPG if available
-	@if [ -f "$(DOCKER_VOLUME)/GPG-KEY" ]; then gpg --import $(DOCKER_VOLUME)/GPG-KEY; fi
-	@if [ x"$$GPG_KEY" != x ]; then echo "$$GPG_KEY" | gpg --import - ; fi
+.PHONY: deb-build
+deb-build: deb-clean git-clone import-gpg-key
 	@cd /debbuild/$(NAME)
-	@echo "==> Clone code."
 	./build.sh
 	mkdir -p repo/
 	cp -rf src/archivematica-*.* repo
-	cd  repo && dpkg-scanpackages . | gzip > Packages.gz
+	cd repo && dpkg-scanpackages . | gzip > Packages.gz
 
-deb-clean:
-	@echo "==> Cleaning up previous builds."
-	@rm -rf $(DOCKER_VOLUME)/repo/
-	@rm -rf $(DOCKER_VOLUME)/src/
-	mkdir -p src/$(PACKAGE)
-	rm -rf src/
+.PHONY: import-gpg-key
+import-gpg-key:
+	@if [ -f "$(DOCKER_VOLUME)/GPG-KEY" ]; then gpg --import $(DOCKER_VOLUME)/GPG-KEY; fi
+	@if [ x"$$GPG_KEY" != x ]; then echo "$$GPG_KEY" | gpg --import - ; fi
+
+.PHONY: git-clone
+git-clone:
 	@git clone \
 		--quiet \
 		--branch $(BRANCH) \
 		--depth 1 \
 		--single-branch \
-			https://github.com/artefactual/$(PACKAGE) src/$(PACKAGE)
+			$(GIT_REPO) src/$(PACKAGE)
 
-cleanup:
+.PHONY: deb-clean
+deb-clean:
+	@echo "==> Cleaning up previous builds."
+	@rm -rf $(DOCKER_VOLUME)/repo/
+	@rm -rf $(DOCKER_VOLUME)/src/
+
+.PHONY: cleanup
+cleanup: build-docker-image
 	@echo "==> Remove artifacts created as root."
 	@docker run -i \
 		--rm \
 		--volume "$(shell cd ../../ && pwd):$(DEB_TOPDIR)/$(NAME)" \
 		--volume "$(shell pwd):$(DOCKER_VOLUME)" \
 		$(DOCKER_IMAGE) \
 		rm -rf /src/repo/ /src/src/
-
-deb-test:
-	@docker run --rm --volume="$(shell pwd):$(DOCKER_VOLUME)" ubuntu:trusty echo "Create repo and install package"
diff --git a/debs/bionic/archivematica-storage-service/build.sh b/debs/bionic/archivematica-storage-service/build.sh
@@ -1,15 +1,16 @@
-#!/bin/bash -x
+#!/usr/bin/env bash
+
+set -euxo
 
 BASE="$(pwd)"
 SOURCE=${BASE}/src/archivematica-storage-service/
 export DEBFULLNAME="Artefactual Systems"
 export DEBEMAIL="[email protected]"
+export DEB_BUILD_OPTIONS="noddebs"
 
 cd $SOURCE
 COMMIT=$(git rev-parse HEAD)
 cp -rf ${BASE}/debian-storage-service debian
-QUILT_PATCHES="debian/patches" quilt push -a || true
-pip download -d lib --no-binary :all: -r requirements.txt
 yes | mk-build-deps -i debian/control
 dch -v 1:${VERSION}${RELEASE}~18.04 commit: $(echo $COMMIT)
 dch -v 1:${VERSION}${RELEASE}~18.04 checkout: $(echo $BRANCH) 

diff --git a/debs/bionic/archivematica-storage-service/changelog.sh b/debs/bionic/archivematica-storage-service/changelog.sh
@@ -1,9 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
+
 export DEBFULLNAME="Artefactual Systems"
 export DEBEMAIL="[email protected]"
 BRANCH="$(git branch | cut -d\  -f2-)"
 COMMIT=$(git rev-parse HEAD)
 
 dch -v 1:${VERSION}${RELEASE} commit: $(echo $COMMIT)
 dch -v 1:${VERSION}${RELEASE} checkout: $(echo $BRANCH) 
-dch -r --distribution xenial --urgency high ignored
+dch -r --distribution bionic --urgency high ignored
diff --git a/...rchivematica-storage-service/debian-storage-service/archivematica-storage-service.install b/...rchivematica-storage-service/debian-storage-service/archivematica-storage-service.install
@@ -5,4 +5,4 @@ install/storageService.logging.json /etc/archivematica/
 install/storage etc/nginx/sites-available/
 install/.storage-service var/archivematica/
 install/make_key.py var/archivematica/storage-service/
-storage_service/* /usr/share/archivematica/virtualenvs/archivematica-storage-service/lib/python2.7/site-packages/storage_service/
+storage_service/* /usr/share/archivematica/virtualenvs/archivematica-storage-service/lib/python3.6/site-packages/storage_service/
diff --git a/debs/bionic/archivematica-storage-service/debian-storage-service/control b/debs/bionic/archivematica-storage-service/debian-storage-service/control
@@ -1,28 +1,36 @@
 Source: archivematica-storage-service
 Section: utils
 Priority: extra
-Maintainer: Artefactual Systems Inc. <[email protected]>
-Build-Depends: 
-    python (>= 2.7.3),
-    debhelper (>= 9),
+Maintainer: Artefactual Systems Inc. <[email protected]>
+Build-Depends:
+    python3.6-dev,
+    debhelper,
+    dh-virtualenv (>= 1.2.2),
     gnupg,
     libmysqlclient-dev,
     libffi-dev,
     libssl-dev,
     libxml2-dev,
     libxslt-dev,
-    python-dev,
-    dh-virtualenv (>= 0.8),
     dh-systemd,
+    dh-virtualenv,
     libldap2-dev,
     libsasl2-dev,
     zlib1g-dev
-Standards-Version: 3.9.3
-X-Python-Version: 2.7
+Standards-Version: 3.9.8
+X-Python-Version: 3.6
 
 Package: archivematica-storage-service
-Architecture: i386 amd64
-Depends: ${python:Depends}, ${misc:Depends}, nginx, unar (>= 1.8.1-4~), python (>= 2.7.3), rng-tools, gnupg1, gettext
+Architecture: amd64
+Depends:
+    ${misc:Depends},
+    nginx,
+    unar (>= 1.8.1-4~),
+    python3,
+    python3-distutils,
+    rng-tools,
+    gnupg1,
+    gettext
 Description: Django webapp for managing storage in an Archivematica
     installation.
 Homepage: http://archivematica.org
diff --git a/debs/bionic/archivematica-storage-service/debian-storage-service/postinst b/debs/bionic/archivematica-storage-service/debian-storage-service/postinst
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 SS_ENV_DIR="/usr/share/archivematica/virtualenvs/archivematica-storage-service"
 
@@ -13,19 +13,8 @@ else
   adduser --uid 333 --group --system --home /var/lib/archivematica/ archivematica
 fi
 
-# Remove dh-virtualenv build path in editable pip requirements and other local/bin files
-# https://github.com/spotify/dh-virtualenv/issues/134
-# https://github.com/archivematica/Issues/issues/903
-for filename in /usr/share/archivematica/virtualenvs/archivematica-storage-service/local/bin/* \
-    /usr/share/archivematica/virtualenvs/archivematica-storage-service/lib/python2.7/site-packages/*.pth \
-    /usr/share/archivematica/virtualenvs/archivematica-storage-service/lib/python2.7/site-packages/*.egg-link ; do
-    if [ -f "$filename" ]; then
-        sed -i "s/\/src\/src\/archivematica-storage-service\/debian\/archivematica-storage-service//g" $filename
-    fi
-done
-
 echo "creating django secret key"
-KEYCMD=$(python /var/archivematica/storage-service/make_key.py 2>&1)
+KEYCMD=$(python3 /var/archivematica/storage-service/make_key.py 2>&1)
 echo $KEYCMD
 
 sed -i "s/CHANGE_ME_WITH_A_SECRET_KEY/\"$KEYCMD\"/g" /etc/default/archivematica-storage-service
@@ -40,31 +29,12 @@ source /etc/default/archivematica-storage-service
 set +a
 echo "creating symlink in /usr/lib/archivematica"
 rm -f /usr/lib/archivematica/storage-service
-ln -s -f ${SS_ENV_DIR}/lib/python2.7/site-packages/storage_service/ /usr/lib/archivematica/storage-service
+ln -s -f ${SS_ENV_DIR}/lib/python3.6/site-packages/storage_service/ /usr/lib/archivematica/storage-service
 
 cd /usr/lib/archivematica/storage-service
 
 echo "configuring django database and static files"
-find /usr/lib/archivematica/storage-service/ -name '*.pyc'  -delete
-# 0.3.0 will automatically fake the first migration
-if [[ $2 == '0.7.'* || $2 == '0.6.'* || $2 == '0.5.'* || $2 == '0.4.'* ]]; then
-    echo 'Faking the 0.4 migration'
-    ${SS_ENV_DIR}/bin/python manage.py migrate locations 0002_v0_4 --fake
-fi
-if [[ $2 == '0.7.'* || $2 == '0.6.'* || $2 == '0.5.'* ]]; then
-    echo 'Faking 0.5 migration'
-    ${SS_ENV_DIR}/bin/python manage.py migrate locations 0003_v0_5 --fake
-fi
-
-if [[ $2 == '0.7.'* || $2 == '0.6.'* ]]; then
-    echo 'No migrations in 0.6'
-fi
-
-if [[ $2 == '1:0.7.'* ]]; then
-    echo 'Faking 0.7 migration'
-    ${SS_ENV_DIR}/bin/python manage.py migrate locations 0004_v0_7 --fake
-    ${SS_ENV_DIR}/bin/python manage.py migrate --fake-initial
-fi
+find /usr/lib/archivematica/storage-service/ -name '*.pyc' -delete
 
 ${SS_ENV_DIR}/bin/python manage.py migrate
 mkdir -p /usr/lib/archivematica/storage-service/assets

diff --git a/debs/bionic/archivematica-storage-service/debian-storage-service/postrm b/debs/bionic/archivematica-storage-service/debian-storage-service/postrm
@@ -1,11 +1,8 @@
 #!/bin/bash
 
-#postrm 
-#run after the package is uninstalled
-
 echo "postrm" $1
+
 if [ $1 = "remove" ]; then
-    #clean up storage service directories
     rm -rf /usr/share/python/archivematica-storage-service
     rm -rf /usr/share/archivematica/virtualenvs/archivematica-storage-service
     rm -rf /var/archivematica/storage*

diff --git a/debs/bionic/archivematica-storage-service/debian-storage-service/preinst b/debs/bionic/archivematica-storage-service/debian-storage-service/preinst
@@ -1,12 +1,3 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
-echo "preinst called with:" $1 $2
-
-# version 0.3.0.release postrm had a bug that prevents upgrades from working
-if [ x$2 == x"0.3.0.release" ]; then
-  echo "saving original files"
-  rm -f /var/lib/dpkg/info/archivematica-storage-service.postrm
-  # backup important user files
-  mkdir -p /etc/archivematica/storage-service
-  cp /var/archivematica/.storage-service /etc/archivematica/storage-service/.storage-service
-fi
+echo "preinst" $1 $2
diff --git a/debs/bionic/archivematica-storage-service/debian-storage-service/rules b/debs/bionic/archivematica-storage-service/debian-storage-service/rules
@@ -7,11 +7,7 @@ export DH_VIRTUALENV_INSTALL_ROOT=/usr/share/archivematica/virtualenvs
 	dh $@ --with python-virtualenv --with systemd
 
 override_dh_virtualenv:
-	# Preinstalling pip 9.0.3 solves a problem with the shebangs inside the
-	# virtual environment. See:
-	# - https://github.com/artefactual/archivematica/issues/1042
-	# - https://github.com/artefactual-labs/am-packbuild/pull/159
-	dh_virtualenv --preinstall "pip==9.0.3" --extra-pip-arg --find-links=lib
+	dh_virtualenv --python=python3 --requirements=requirements/production-py3.txt --skip-install
 
 # See https://readthedocs.org/projects/dh-virtualenv/downloads/pdf/latest/ (2.3.3 section)
 override_dh_shlibdeps:

diff --git a/debs/bionic/archivematica/Dockerfile b/debs/bionic/archivematica/Dockerfile
@@ -1,18 +1,32 @@
 FROM ubuntu:bionic
 
-RUN apt-get update && \
-    apt-get install -y dpkg-dev git build-essential wget debhelper \
-    devscripts equivs quilt apt-transport-https python-minimal
-
-RUN wget -O /tmp/pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && python /tmp/pip.py
-
-RUN wget -O - https://packages.archivematica.org/1.7.x/key.asc |\
-    apt-key add - && \
-    echo "deb [arch=amd64] http://packages.archivematica.org/1.7.x/ubuntu-externals xenial main" >> /etc/apt/sources.list
-
-
-RUN apt-get update && apt-get install -y libffi-dev libssl-dev libxslt-dev dh-virtualenv python-dev dh-systemd libmysqlclient-dev libldap2-dev libsasl2-dev curl
-
-RUN curl -sL https://deb.nodesource.com/setup_8.x |  bash - && apt-get install -y nodejs
-
-
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update \
+        && apt-get install -y \
+                dpkg-dev \
+                git \
+                build-essential \
+                debhelper \
+                devscripts \
+                equivs \
+                apt-transport-https \
+                python3.6-dev \
+                python3.6-distutils \
+                libffi-dev \
+                libssl-dev \
+                libxslt-dev \
+                dh-systemd \
+                libmysqlclient-dev \
+                libldap2-dev \
+                libsasl2-dev \
+        && update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 \
+        && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 \
+        && curl -s https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py && python /tmp/get-pip.py \
+	&& curl -sL https://deb.nodesource.com/setup_8.x | bash - && apt-get install -y nodejs
+
+# Install recent version of dh-virtualenv.
+RUN apt-get install -y software-properties-common \
+	&& add-apt-repository ppa:jyrki-pulliainen/dh-virtualenv \
+	&& apt-get update \
+	&& apt-get install -y dh-virtualenv