From 5120cd01a750039e1753970b715092f6a71883ec Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 23 Jul 2024 13:54:15 +0200 Subject: [PATCH 1/9] Switch to building packages using a Docker image This replaces the old Ansible / Vagrant setup with Docker. All the building steps are run in the relevant `ubuntu:X.Y` image, and the resulting deb package is copied to the client host. As with the old scripts, you can pass build args to control the process. The full command is: ``` docker buildx build \ --output type=local,dest=. \ --build-arg CKAN_VERSION=2.11 \ --build-arg CKAN_BRANCH=dev-v2.11 \ --build-arg DATAPUSHER_VERSION=0.0.21 \ --build-arg DISTRIBUTION=noble \ --build-arg UBUNTU_VERSION=24.04 \ . ``` For now we are sticking with a single Dockerfile for all Ubuntu versions as there is only one minor difference in focal, but if there are more differences added in the future we will switch to different Dockerfiles. --- Dockerfile | 109 ++++++++++++++++++++++++++ common/{after_web.j2 => after_web.sh} | 0 2 files changed, 109 insertions(+) create mode 100644 Dockerfile rename common/{after_web.j2 => after_web.sh} (100%) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6a6e31d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,109 @@ +ARG UBUNTU_VERSION=22.04 +FROM ubuntu:${UBUNTU_VERSION} AS builder + +ARG UBUNTU_VERSION=${UBUNTU_VERSION} +ARG CKAN_VERSION=2.11 +ARG CKAN_BRANCH=dev-v2.11 +ARG DATAPUSHER_VERSION=0.0.21 +ARG DISTRIBUTION=jammy +ARG ITERATION + +# Install Ubuntu packages +RUN apt-get update -q && \ + apt-get upgrade -y -q && \ + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y -q install \ + git \ + python3-dev \ + python3-pip \ + python3-venv \ + libpq-dev \ + libxml2-dev \ + libxslt1-dev \ + build-essential \ + rubygems-integration \ + ruby-dev \ + nginx + +# Install FPM +RUN if [ "$UBUNTU_VERSION" = "20.04" ] ; then gem install dotenv -v 2.8.1 ; fi && \ + gem install fpm -- creates=/usr/local/bin/fpm + +# Create dirs +RUN mkdir -p /etc/ckan/default && \ + mkdir -p /etc/ckan/datapusher && \ + mkdir -p /etc/supervisor/conf.d && \ + mkdir /output + + +# Create venv +RUN python3 -m venv /usr/lib/ckan/default + +# Pull CKAN source +RUN git clone --depth=1 --branch=${CKAN_VERSION} https://github.com/ckan/ckan /usr/lib/ckan/default/src/ckan && \ + rm -rf /usr/lib/ckan/default/src/ckan/.git/ + +# Install CKAN and its requirements +RUN /usr/lib/ckan/default/bin/pip install -U pip && \ + /usr/lib/ckan/default/bin/pip install -r /usr/lib/ckan/default/src/ckan/requirements.txt uwsgi && \ + /usr/lib/ckan/default/bin/pip install /usr/lib/ckan/default/src/ckan + +# Configure CKAN + uWSGI +RUN cp /usr/lib/ckan/default/src/ckan/wsgi.py /etc/ckan/default/wsgi.py && \ + cp /usr/lib/ckan/default/src/ckan/ckan-uwsgi.ini /etc/ckan/default/ckan-uwsgi.ini + +# Install DataPusher +RUN python3 -m venv /usr/lib/ckan/datapusher && \ + # Clone source + git clone --depth=1 --branch=${DATAPUSHER_VERSION} https://github.com/ckan/datapusher /usr/lib/ckan/datapusher/src/datapusher && \ + rm -rf /usr/lib/ckan/datapusher/src/datapusher/.git/ && \ + # Install requirements and datapusher + /usr/lib/ckan/datapusher/bin/pip install -U pip && \ + /usr/lib/ckan/datapusher/bin/pip install -r /usr/lib/ckan/datapusher/src/datapusher/requirements.txt uwsgi && \ + /usr/lib/ckan/datapusher/bin/pip install /usr/lib/ckan/datapusher/src/datapusher + +# Configure DataPusher + uWSGI +RUN cp /usr/lib/ckan/datapusher/src/datapusher/deployment/datapusher.wsgi /etc/ckan/datapusher && \ + cp /usr/lib/ckan/datapusher/src/datapusher/deployment/datapusher_settings.py /etc/ckan/datapusher && \ + cp /usr/lib/ckan/datapusher/src/datapusher/deployment/datapusher-uwsgi.ini /etc/ckan/datapusher && \ + # Enable threads in DataPusher uwsgi conf + echo "workers = 2\nthreads = 2\nlazy-apps = true" >> /etc/ckan/datapusher/datapusher-uwsgi.ini && \ + # Replace datapusher wsgi file path + sed -i 's/\/usr\/lib\/ckan\/datapusher\/src\/datapusher\/deployment\/datapusher.wsgi/\/etc\/ckan\/datapusher\/datapusher.wsgi/g' /etc/ckan/datapusher/datapusher-uwsgi.ini + +# Copy conf files +COPY common/etc/nginx/sites-available/ckan /etc/nginx/sites-available/ +COPY common/etc/cron.daily/remove_old_sessions /etc/cron.daily/ +COPY common/etc/supervisor/conf.d/ckan-uwsgi.conf /etc/supervisor/conf.d/ +COPY common/etc/supervisor/conf.d/ckan-worker.conf /etc/supervisor/conf.d/ +COPY common/etc/supervisor/conf.d/ckan-datapusher.conf /etc/supervisor/conf.d/ +COPY --chmod=744 common/usr/bin/ckan /usr/bin/ckan +COPY --chmod=744 common/after_web.sh /tmp/after_web.sh + +# Create the deb package +RUN fpm \ + -t deb -s dir \ + --package /output \ + --name python-ckan \ + --description='CKAN is an open-source DMS (data management system) for powering data hubs and data portals.' \ + --license='AGPL v3.0' \ + --maintainer='CKAN Tech Team ' \ + --vendor='CKAN Association' \ + --url='https://ckan.org' \ + --after-install=/tmp/after_web.sh \ + --iteration "${DISTRIBUTION}""${ITERATION}" \ + --version "${CKAN_VERSION}" \ + --depends nginx \ + --depends libpq5 \ + --config-files /etc/nginx/sites-available/ckan \ + /usr/lib/ckan/ \ + /etc/ckan/ \ + /usr/bin/ckan \ + /etc/cron.daily/remove_old_sessions \ + /etc/nginx/sites-available/ckan \ + /etc/supervisor/conf.d + +RUN ls -la /output + + +FROM scratch AS export +COPY --from=builder /output . diff --git a/common/after_web.j2 b/common/after_web.sh similarity index 100% rename from common/after_web.j2 rename to common/after_web.sh From 4215ec08548f275d9d705b5cb928c6a740316ae5 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 23 Jul 2024 14:15:34 +0200 Subject: [PATCH 2/9] Remove vagrant/ansible related files --- Vagrantfile | 46 ----------------------- package.yml | 89 --------------------------------------------- py3/apt.yml | 18 --------- py3/datapusher.yml | 54 --------------------------- py3/installckan.yml | 29 --------------- py3/virtualenv.yml | 9 ----- 6 files changed, 245 deletions(-) delete mode 100644 Vagrantfile delete mode 100644 package.yml delete mode 100644 py3/apt.yml delete mode 100644 py3/datapusher.yml delete mode 100644 py3/installckan.yml delete mode 100644 py3/virtualenv.yml diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index de40ef5..0000000 --- a/Vagrantfile +++ /dev/null @@ -1,46 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - - config.vm.define "focal" do |focal| - focal.vm.box = "ubuntu/focal64" - end - - config.vm.define "jammy" do |focal| - focal.vm.box = "ubuntu/jammy64" - end - - config.ssh.forward_agent = true - config.vm.provider "virtualbox" do |v| - v.memory = 1024 - v.customize ["modifyvm", :id, "--uart1", "0x3F8", "4"] - v.customize ["modifyvm", :id, "--uartmode1", "file", File::NULL] - - end - - config.vm.provision "ansible" do |ansible| - ansible.playbook = "package.yml" - ansible.host_key_checking = false - ansible.become = true - ansible.compatibility_mode = "2.0" - - ansible.verbose = ENV.fetch("CKAN_PACKAGE_ANSIBLE_VERBOSE", "vv") - ansible.extra_vars = {} - - ansible.extra_vars["datapusher"] = ENV.fetch("CKAN_PACKAGE_DATAPUSHER", "y") - - if ENV.has_key?("CKAN_PACKAGE_VERSION") - ansible.extra_vars["version"] = ENV["CKAN_PACKAGE_VERSION"] - end - if ENV.has_key?("PYTHON_VERSION") - ansible.extra_vars["pythonversion"] = ENV["PYTHON_VERSION"] - end - if ENV.has_key?("CKAN_PACKAGE_ITERATION") - ansible.extra_vars["iteration"] = ENV["CKAN_PACKAGE_ITERATION"] - end - end -end diff --git a/package.yml b/package.yml deleted file mode 100644 index 03fc5dc..0000000 --- a/package.yml +++ /dev/null @@ -1,89 +0,0 @@ ---- -- hosts: all - vars_prompt: - - name: version - prompt: "CKAN Version (branch or tag)" - private: True - - name: iteration - prompt: "Iteration" - private: True - - name: datapusher - prompt: "Enable DataPusher? [Y/n]" - default: "y" - private: True - tasks: - - name: set regex for CKAN version - set_fact: - re: '\D+\D+(?P\d+\.\d+)' - - set_fact: - ckanver: "{{ version | regex_search(re, '\\g') }}" - register: ckanver - - debug: - var: ckanver[0] - - # Install Ubuntu apt packages - - name: Install Ubuntu apt Packages - include_tasks: "py3/apt.yml" - - - name: make sure nginx is installed - action: apt pkg=nginx state=present update_cache=yes - - - name: install fpm requirements - action: command gem install dotenv -v 2.8.1 - when: ansible_distribution_version == "20.04" - - - name: install fpm - action: command gem install -v 1.15.0 fpm -- creates=/usr/local/bin/fpm - - - name: delete old directories - action: file path={{ item }} state=absent - with_items: - - /etc/ckan - - /usr/lib/ckan - - /root/.cache/pip - - # Install pip and virtualenv - - name: Install apt Packages - include_tasks: "py3/virtualenv.yml" - - # Pull CKAN - - name: pull ckan software - action: git repo=https://github.com/ckan/ckan dest=/usr/lib/ckan/default/src/ckan version={{ version }} - - # Clean up CKAN distro - - name: Recursively remove .git directory - file: - path: /usr/lib/ckan/default/src/ckan/.git - state: absent - - # Install ckan and dependancies - - name: Install ckan and dependancies - include_tasks: "py3/installckan.yml" - - # Install datapusher - - name: Install datapusher - include_tasks: "py3/datapusher.yml" - when: datapusher == 'y' - - # Clean up datapusher distro - - name: Recursively remove .git directory - file: - path: /usr/lib/ckan/datapusher/src/datapusher/.git - state: absent - when: datapusher == 'y' - - # Install web files and dependancies - - name: Install web and dependancies - include_tasks: "common/web.yml" - - # Copy other application files CKAN - - name: Copy other application files for CKAN - include_tasks: "common/app.yml" - - # template the after web script - - name: template the after web script - action: template src=common/after_web.j2 dest=/tmp/after_web.sh mode=744 - - # Build deb package for CKAN - - name: Build deb package for CKAN - include_tasks: "common/build.yml" diff --git a/py3/apt.yml b/py3/apt.yml deleted file mode 100644 index d15a7b8..0000000 --- a/py3/apt.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- - - name: Make sure Python3 apt packages are installed - apt: - update_cache: yes - force_apt_get: yes - name: - - 'git-core' - - 'python3-dev' - - 'python3-pip' - - 'python3-venv' - - 'python3-distutils' - - 'postgresql' - - 'libpq-dev' - - 'libxml2-dev' - - 'libxslt1-dev' - - 'build-essential' - - 'rubygems-integration' - - 'ruby-dev' diff --git a/py3/datapusher.yml b/py3/datapusher.yml deleted file mode 100644 index 52b5a75..0000000 --- a/py3/datapusher.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- - - name: pull datapusher version - action: git repo=https://github.com/ckan/datapusher dest=/usr/lib/ckan/datapusher/src/datapusher version=0.0.21 - - - name: Install Python3 Virtual Environment for datapusher for Python 3 - action: command python3 -m venv /usr/lib/ckan/datapusher - - - name: install requirements for datapusher for Python 3 - pip: - requirements: /usr/lib/ckan/datapusher/src/datapusher/requirements.txt - virtualenv: /usr/lib/ckan/datapusher - virtualenv_command: pyvenv - - - name: install uWSGI for DataPusher - pip: - name: uwsgi - virtualenv: /usr/lib/ckan/datapusher - virtualenv_command: pyvenv - - - name: run setup.py develop for datapusher - action: command chdir=/usr/lib/ckan/datapusher/src/datapusher/ ../../bin/python setup.py develop - - - name: create directories for DataPusher - action: file path={{ item }} state=directory - with_items: - - /etc/ckan/datapusher - - /etc/supervisor/conf.d - - - name: copy all deployment files needed for the DataPusher - action: command cp /usr/lib/ckan/datapusher/src/datapusher/deployment/{{ item }} /etc/ckan/datapusher/{{ item }} - with_items: - - datapusher.wsgi - - datapusher_settings.py - - datapusher-uwsgi.ini - - - name: enable threads in DataPusher uwsgi conf - blockinfile: - path: /etc/ckan/datapusher/datapusher-uwsgi.ini - insertafter: "EOF" - block: | - workers = 2 - threads = 2 - lazy-apps = true - - - name: copy all common needed files for DataPusher - action: copy src=common/{{ item }} dest=/{{ item }} - with_items: - - etc/supervisor/conf.d/ckan-datapusher.conf - - - name: replace datapusher wsgi file path - replace: - path: /etc/ckan/datapusher/datapusher-uwsgi.ini - regexp: '/usr/lib/ckan/datapusher/src/datapusher/deployment/datapusher.wsgi' - replace: '/etc/ckan/datapusher/datapusher.wsgi' diff --git a/py3/installckan.yml b/py3/installckan.yml deleted file mode 100644 index 7632bce..0000000 --- a/py3/installckan.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- - - name: install requirements version for Python3 - pip: - requirements: /usr/lib/ckan/default/src/ckan/requirements.txt - virtualenv: /usr/lib/ckan/default - virtualenv_command: pyvenv - - - name: install setuptools version for Python3 - pip: - requirements: /usr/lib/ckan/default/src/ckan/requirement-setuptools.txt - virtualenv: /usr/lib/ckan/default - virtualenv_command: pyvenv - when: ckanver[0] == "2.9" - - - name: install setuptools version for Python3 - pip: - name: - - setuptools>44 - - wheel - virtualenv: /usr/lib/ckan/default - virtualenv_command: pyvenv - when: ckanver[0] != "2.9" - - - name: run setup.py develop for Python3 ckan - action: command chdir=/usr/lib/ckan/default/src/ckan/ ../../bin/python3 setup.py develop - - - name: get ckan version - command: /usr/lib/ckan/default/bin/python3 -c "import ckan; print(ckan.__version__)" - register: ckan_version diff --git a/py3/virtualenv.yml b/py3/virtualenv.yml deleted file mode 100644 index 7ff8719..0000000 --- a/py3/virtualenv.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- - - name: Get pip - action: get_url url=https://bootstrap.pypa.io/get-pip.py dest=/tmp - - - name: Install pip for Python3 - action: command python3 /tmp/get-pip.py - - - name: Install virtualenv for Python3 - action: command python3 -m venv /usr/lib/ckan/default From 68bf2589832d177a14dadea3f9b8cce5fd166842 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 23 Jul 2024 14:24:32 +0200 Subject: [PATCH 3/9] New readme --- README.md | 37 +++++++++++++++++ README.rst | 116 ----------------------------------------------------- 2 files changed, 37 insertions(+), 116 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/README.md b/README.md new file mode 100644 index 0000000..a58be25 --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +# CKAN Packaging Scripts + +These scripts are used to build Debian packages (deb files) for CKAN releases. + +> [!NOTE] As of July 2024, Docker is used as a building environment to create the deb packages. +> The previous Ansible / Vagrant setup is no longer used + +## Overview + +To create Debian packages of CKAN, install Docker and run: + +``` +docker buildx build \ + --output type=local,dest=. \ + --build-arg CKAN_VERSION=2.11 \ + --build-arg CKAN_BRANCH=dev-v2.11 \ + --build-arg DATAPUSHER_VERSION=0.0.21 \ + --build-arg DISTRIBUTION=noble \ + --build-arg UBUNTU_VERSION=24.04 \ + . +``` + +The currently supported packages are: + +| CKAN version | Ubuntu version | +| ------------ | -------------- | +| 2.10 | 20.04 (focal) | +| 2.10 | 22.04 (jammy) | +| 2.11 | 22.04 (jammy) | +| 2.11 | 24.04 (noble) | + +Any other combination is not officially supported, although you might be able to +build it tweaking the parameters above. + + + +TODO diff --git a/README.rst b/README.rst deleted file mode 100644 index ba56499..0000000 --- a/README.rst +++ /dev/null @@ -1,116 +0,0 @@ -CKAN Packaging Scripts -====================== - -These scripts are used to build Debian packages (deb files) for CKAN releases. - -**Note:** This version of the scripts can only package CKAN 2.9 and higher. -For previous CKAN versions use the `build-ckan-2.8` tag. - - -Overview --------- - -To create Debian packages of CKAN, install Vagrant and Ansible and run:: - - ./ckan-package -v dev-v2.10 -i 1 -t jammy - -If you omit the parameters you will be prompted for them. - - CKAN 2.10 Python3 jammy, focal - CKAN 2.9 Python3 jammy, focal - - -The following combinations of packages are no longer supported:: - - CKAN 2.9 Python3, Python2 focal, bionic, xenial - CKAN 2.8 Python2 focal, bionic, xenial - CKAN 2.7 Python2 focal, bionic, xenial - - -Keep reading for more options and to learn how it works. - - -How it works ------------- - -This repository contains a `Vagrant `_ environment -configured in a `multi-machine `_ setup. - -Each machine is running one of the supported distributions that we target, currently: - -* Ubuntu 20.04 64bit (focal) -* Ubuntu 22.04 64bit (jammy) - -We use `Ansible `_ to provision the Vagrant machines, which -results in the creation of the package. Ansible is configured via -`playbooks `_. These are a set of -configurations and tasks that are run in the remote server. The file we use -is `package.yml` and it is quite straight-forward to follow what is going on. - -The actual packaging is done in the building maching by -`FPM `_, a tool that aims to make building -packages quickly and easily. - -To pass parameters like the CKAN version or the iteration to the Vagrant file and -ultimately to Ansible we need to use env vars. To make this a little bit more -convenient, the ``ckan-package`` script is included, which essentially sets up the -necessary env vars and calls ``vagrant up`` or ``vagrant provision`` as appropiate. - - -Install and setup ------------------ - -1. Install Vagrant. On an Ubuntu machine:: - - sudo apt-get install vagrant - - Or for an OSX machine use the installer at https://www.vagrantup.com/downloads.html - -2. Install Ansible. If you have a virtualenv you can install it in that:: - - pip install ansible - - Otherwise simply install it globally:: - - sudo easy_install ansible - -3. Checkout this repository in your virtualenv or elsewhere:: - - git clone https://github.com/ckan/ckan-packaging.git - cd ckan-packaging - -You are now ready to build packages. - - -Building a package ------------------- - -Packages are built using the ``ckan-package`` script. Check the following for a -full list of options and parameters:: - - ./ckan-package --help - -Most of the times you will want to run something like the following:: - - ./ckan-package -v dev-v2.10 -i 1 -t jammy - -Where: - - * -v (version) relates to the CKAN branch or tag to build, eg master, dev-v2.10, ckan-v2.9.8 - * -i (iteration) e.g. `beta1` for a beta or for a proper release use a number e.g. `1` - * -t (target) will build just for that version of Ubuntu - - -The first time that you run the build commands Vagrant will -need to download the OS images from the central repository, this might take a while. -After the first run, the image will be already on your machine so it won't take as much. -Most of the steps in the Ansible playbook (like installing required packages) will be also -skipped, so subsequent builds should be much faster. - -You will see the output of the different tasks, both for Vagrant and Ansible. -Once the process is finished, you should see a summary of the tasks. -All tasks should be `ok` or `changed`. - -The packages will be created in the same directory. - -TODO: upload the packages to S3 (http://packaging.ckan.org/build/). From ea8f745b496af72266d99e1b251012bd4715e8fa Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 3 Dec 2024 13:05:07 +0100 Subject: [PATCH 4/9] Reduce number of arguments --- Dockerfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6a6e31d..d2e1f2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,16 +2,15 @@ ARG UBUNTU_VERSION=22.04 FROM ubuntu:${UBUNTU_VERSION} AS builder ARG UBUNTU_VERSION=${UBUNTU_VERSION} -ARG CKAN_VERSION=2.11 -ARG CKAN_BRANCH=dev-v2.11 +ARG CKAN_REF=dev-v2.11 ARG DATAPUSHER_VERSION=0.0.21 -ARG DISTRIBUTION=jammy ARG ITERATION # Install Ubuntu packages RUN apt-get update -q && \ apt-get upgrade -y -q && \ DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y -q install \ + lsb-release \ git \ python3-dev \ python3-pip \ @@ -34,12 +33,11 @@ RUN mkdir -p /etc/ckan/default && \ mkdir -p /etc/supervisor/conf.d && \ mkdir /output - # Create venv RUN python3 -m venv /usr/lib/ckan/default # Pull CKAN source -RUN git clone --depth=1 --branch=${CKAN_VERSION} https://github.com/ckan/ckan /usr/lib/ckan/default/src/ckan && \ +RUN git clone --depth=1 --branch=${CKAN_REF} https://github.com/ckan/ckan /usr/lib/ckan/default/src/ckan && \ rm -rf /usr/lib/ckan/default/src/ckan/.git/ # Install CKAN and its requirements @@ -79,8 +77,10 @@ COPY common/etc/supervisor/conf.d/ckan-datapusher.conf /etc/supervisor/conf.d/ COPY --chmod=744 common/usr/bin/ckan /usr/bin/ckan COPY --chmod=744 common/after_web.sh /tmp/after_web.sh -# Create the deb package -RUN fpm \ +# Get the actual CKAN version and create the deb package +RUN DISTRIBUTION=$(lsb_release -c -s) && \ + CKAN_VERSION=$(/usr/lib/ckan/default/bin/python3 -c "import ckan; print(ckan.__version__)") && \ + fpm \ -t deb -s dir \ --package /output \ --name python-ckan \ @@ -90,8 +90,8 @@ RUN fpm \ --vendor='CKAN Association' \ --url='https://ckan.org' \ --after-install=/tmp/after_web.sh \ - --iteration "${DISTRIBUTION}""${ITERATION}" \ - --version "${CKAN_VERSION}" \ + --iteration "$DISTRIBUTION""${ITERATION}" \ + --version "$CKAN_VERSION" \ --depends nginx \ --depends libpq5 \ --config-files /etc/nginx/sites-available/ckan \ From 298ddfaee8acce416da71ed032770864863f2715 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 3 Dec 2024 13:05:27 +0100 Subject: [PATCH 5/9] Update and simplify helper script --- UBUNTU_VERSIONS.txt | 3 + ckan-package | 161 +++++++++----------------------------------- 2 files changed, 36 insertions(+), 128 deletions(-) create mode 100644 UBUNTU_VERSIONS.txt diff --git a/UBUNTU_VERSIONS.txt b/UBUNTU_VERSIONS.txt new file mode 100644 index 0000000..d526444 --- /dev/null +++ b/UBUNTU_VERSIONS.txt @@ -0,0 +1,3 @@ +20.04 focal +22.04 jammy +24.04 noble diff --git a/ckan-package b/ckan-package index 2e54b56..95e8b65 100755 --- a/ckan-package +++ b/ckan-package @@ -1,152 +1,57 @@ #!/usr/bin/env python +import argparse import subprocess -import re -import os import sys -import argparse - - -def set_env_vars(version, iteration, datapusher="y", ansible_verbose="vv"): - - os.environ["CKAN_PACKAGE_VERSION"] = version - os.environ["CKAN_PACKAGE_ITERATION"] = iteration - os.environ["CKAN_PACKAGE_DATAPUSHER"] = datapusher - os.environ["CKAN_PACKAGE_ANSIBLE_VERBOSE"] = ansible_verbose - - -def clear_env_vars(): - - for key in [ - "CKAN_PACKAGE_VERSION", - "CKAN_PACKAGE_ITERATION", - "CKAN_PACKAGE_DATAPUSHER", - "CKAN_PACKAGE_ANSIBLE_VERBOSE", - ]: - if key in os.environ: - del os.environ[key] - - -def get_status(out, target): - return re.search(target + "\s*(\w+)", out).group(1) - - -def run(target=None): - - status_jammy = None - status_focal = None - - print("Checking Vagrant machines status...") - out = subprocess.check_output(["vagrant", "status"]).decode("utf8") - - if target is None or target == "focal": - status_focal = get_status(out, "focal") - print('Machine "focal" is ' + status_focal) - if target is None or target == "jammy": - status_jammy = get_status(out, "jammy") - print('Machine "jammy" is ' + status_jammy) - - if (status_jammy and status_jammy != "running") or ( - status_focal and status_focal != "running" - ): - print("Starting up machine(s)") - command = ["vagrant", "up", "--provision"] - else: - print("Machine(s) already started, provisioning") - command = ["vagrant", "provision"] - - if target: - command.append(target) - - subprocess.call(command) - - -def minor_version(version_string): - pattern = r"\D+\D+(?P\d+\.\d+)" - version_number = re.search(pattern, version_string) - if version_number: - parts = version_number.groups(0)[0].split(".") - try: - if len(parts) == 3: - # ckan-2.8.5 - return int(parts[1]) - else: - # dev-v2.8 - return int(parts[-1]) - except ValueError: - pass - raise ValueError("Could not parse version number: %s" % version_string) if __name__ == "__main__": description = """Builds CKAN deb packages. -This script essentially sets up the necessary env vars and calls `vagrant up` -or `vagrant provision` as appropiate.""" +This script essentially sets up the necessary vars and calls `docker buildx build`.""" parser = argparse.ArgumentParser(description=description) parser.add_argument( - "-v", - "--version", - help="""The CKAN branch or tag to build, eg master, dev-v2.6, release-v2.5.3. -If not provided you will be propmt for it""", - ) - parser.add_argument( - "-i", - "--iteration", - help="""The iteration number to add to the package name. -If not provided you will be prompt for it""", - ) - parser.add_argument( - "-d", - "--datapusher", - action="store_true", - default="y", - help="""Whether to add the DataPusher to the package, -defaults to true""", + "ref", + help="The CKAN branch or tag to build (e.g. master, dev-v2.11, ckan-2.10.6...)", ) parser.add_argument( - "-t", - "--target", - help="""The distribution to target (focal or jammy). -If omitted, all are built""", + "target", help="The Ubuntu distribution to target (e.g. focal, jammy, noble...)" ) parser.add_argument( - "-a", - "--ansible-verbose", - default="vv", - help='Ansible verbosity level, defaults to "vv"', + "-i", + "--iteration", + default="1", + help="The iteration number to add to the package name.", ) - def _check_arg(args, var, prompt): - if getattr(args, var): - return getattr(args, var) - try: - var = raw_input("{0}:".format(prompt)) - except NameError: - var = input("{0}:".format(prompt)) - if not var: - print("Please provide a value for the {0}".format(prompt)) - sys.exit(1) - return var - args = parser.parse_args() - version = _check_arg(args, "version", "CKAN Version (branch or tag)") - iteration = _check_arg(args, "iteration", "Iteration") - datapusher = args.datapusher + ref = args.ref target = args.target + iteration = args.iteration - if minor_version(version) < 9: - print("This version of the scripts can only package CKAN 2.9 onwards") - sys.exit(1) + supported_targets = {} + with open("UBUNTU_VERSIONS.txt") as f: + for line in f.readlines(): + [version, name] = line.split() + supported_targets[name] = version - if target and target not in ("focal", "jammy"): - print("Wrong target: " + target) + if target not in supported_targets.keys(): + print(f"Wrong target: {target}") sys.exit(1) - ansible_verbose = args.ansible_verbose + command = [ + "docker", + "buildx", + "build", + "--output", + "type=local,dest=.", + "--build-arg", + f"CKAN_REF={ref}", + "--build-arg", + f"ITERATION={iteration}", + "--build-arg", + f"UBUNTU_VERSION={supported_targets[target]}", + ".", + ] - set_env_vars(version, iteration, datapusher, ansible_verbose) - try: - run(target) - finally: - clear_env_vars() + subprocess.call(command) From 2ade49e489ddbd643daf695f30e774ec7719cdf4 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 3 Dec 2024 13:14:32 +0100 Subject: [PATCH 6/9] Update README --- README.md | 45 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index a58be25..856aef2 100644 --- a/README.md +++ b/README.md @@ -2,24 +2,38 @@ These scripts are used to build Debian packages (deb files) for CKAN releases. -> [!NOTE] As of July 2024, Docker is used as a building environment to create the deb packages. +> [!WARNING] These scripts are used by CKAN maintainers. If you want to install CKAN, including +> via Debian package, check the [installation documentation](https://docs.ckan.org/en/latest/maintaining/installing/index.html) + +> [!NOTE] As of December 2024, Docker is used as a building environment to create the deb packages. > The previous Ansible / Vagrant setup is no longer used ## Overview -To create Debian packages of CKAN, install Docker and run: +To create Debian packages of CKAN, use the `ckan-package` executable: ``` -docker buildx build \ - --output type=local,dest=. \ - --build-arg CKAN_VERSION=2.11 \ - --build-arg CKAN_BRANCH=dev-v2.11 \ - --build-arg DATAPUSHER_VERSION=0.0.21 \ - --build-arg DISTRIBUTION=noble \ - --build-arg UBUNTU_VERSION=24.04 \ - . +/ckan-package --help +usage: ckan-package [-h] [-i ITERATION] ref target + +Builds CKAN deb packages. This script essentially sets up the necessary vars and calls `docker buildx build`. + +positional arguments: + ref The CKAN branch or tag to build (e.g. master, dev-v2.11, ckan-2.10.6...) + target The Ubuntu distribution to target (e.g. focal, jammy, noble...) + +optional arguments: + -h, --help show this help message and exit + -i ITERATION, --iteration ITERATION + The iteration number to add to the package name. ``` +For instance: + + ./ckan-package ckan-2.11.1 noble + ./ckan-package dev-v2.11 jammy + ./ckan-package master noble + The currently supported packages are: | CKAN version | Ubuntu version | @@ -32,6 +46,15 @@ The currently supported packages are: Any other combination is not officially supported, although you might be able to build it tweaking the parameters above. +# How it works +Under the hood, the `ckan-package` command just calls `docker buildx build`. You can +call it directly using the appropiate build arguments: -TODO +``` +docker buildx build \ + --output type=local,dest=. \ + --build-arg CKAN_REF=ckan-2.11.0 \ + --build-arg UBUNTU_VERSION=24.04 \ + . +``` From 82ac1a0740b91a3a96131c383aad4e67d219a268 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 3 Dec 2024 15:45:45 +0100 Subject: [PATCH 7/9] Test github actions --- .github/workflows/build-ckan-2.10-focal.yml | 10 +++++ .../workflows/reusable-build-deb-package.yml | 44 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 .github/workflows/build-ckan-2.10-focal.yml create mode 100644 .github/workflows/reusable-build-deb-package.yml diff --git a/.github/workflows/build-ckan-2.10-focal.yml b/.github/workflows/build-ckan-2.10-focal.yml new file mode 100644 index 0000000..0e0f100 --- /dev/null +++ b/.github/workflows/build-ckan-2.10-focal.yml @@ -0,0 +1,10 @@ +name: Build CKAN 2.10 on Ubuntu 20.04 (focal) + +on: push + +jobs: + call-reusable-workflow: + uses: ./.github/workflows/reusable-build-deb-package.yml + with: + ckan-ref: "dev-v2.10" + ubuntu-version: "20.04" diff --git a/.github/workflows/reusable-build-deb-package.yml b/.github/workflows/reusable-build-deb-package.yml new file mode 100644 index 0000000..d74eb37 --- /dev/null +++ b/.github/workflows/reusable-build-deb-package.yml @@ -0,0 +1,44 @@ +name: Reusable CKAN deb packages build workflow + +on: + workflow_call: + inputs: + ckan-ref: + required: true + type: string + ubuntu-version: + required: true + type: string + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build package + uses: docker/build-push-action@v6 + with: + push: false + outputs: "type=local,dest=/tmp" + build-args: | + CKAN_REF=${{ inputs.ckan-ref }} + UBUNTU_VERSION=${{ inputs.ubuntu-version }} + + - name: Rename file + # To remove patch version .e.g python-ckan_2.11.1b0-jammy1_amd64.deb -> python-ckan_2.11-jammy1_amd64.deb + run: | + for f in /tmp/python-ckan_*; do mv "$f" "$(echo "$f" | sed 's/\([0-9]\+\.[0-9]\+\)[^-]*-/\1-/')"; done + OUTPUT_FILE=$(basename /tmp/python-ckan*) + echo "OUTPUT_FILE=$OUTPUT_FILE" >> $GITHUB_ENV + echo "Generated file: $OUTPUT_FILE" + + - name: Upload deb file + uses: actions/upload-artifact@v4 + with: + name: ${{ env.OUTPUT_FILE }} + path: /tmp/${{ env.OUTPUT_FILE }} From 5c25a90253dffaab9a82276b6ac9ba40723c373c Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 4 Dec 2024 12:34:53 +0100 Subject: [PATCH 8/9] Use dynamic matrix to build all versions --- .github/workflows/build-ckan-2.10-focal.yml | 10 --- .github/workflows/build-deb-package.yml | 62 +++++++++++++++++++ .../workflows/reusable-build-deb-package.yml | 44 ------------- README.md | 8 +-- UBUNTU_VERSIONS.txt | 3 - VERSIONS.json | 18 ++++++ ckan-package | 15 ++--- common/app.yml | 8 --- common/build.yml | 22 ------- common/web.yml | 26 -------- 10 files changed, 90 insertions(+), 126 deletions(-) delete mode 100644 .github/workflows/build-ckan-2.10-focal.yml create mode 100644 .github/workflows/build-deb-package.yml delete mode 100644 .github/workflows/reusable-build-deb-package.yml delete mode 100644 UBUNTU_VERSIONS.txt create mode 100644 VERSIONS.json delete mode 100644 common/app.yml delete mode 100644 common/build.yml delete mode 100644 common/web.yml diff --git a/.github/workflows/build-ckan-2.10-focal.yml b/.github/workflows/build-ckan-2.10-focal.yml deleted file mode 100644 index 0e0f100..0000000 --- a/.github/workflows/build-ckan-2.10-focal.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: Build CKAN 2.10 on Ubuntu 20.04 (focal) - -on: push - -jobs: - call-reusable-workflow: - uses: ./.github/workflows/reusable-build-deb-package.yml - with: - ckan-ref: "dev-v2.10" - ubuntu-version: "20.04" diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml new file mode 100644 index 0000000..45690d5 --- /dev/null +++ b/.github/workflows/build-deb-package.yml @@ -0,0 +1,62 @@ +name: CKAN deb packages build workflow + +on: + push: + paths-ignore: + - 'README.md' + +jobs: + get-build-versions: + runs-on: ubuntu-latest + outputs: + versions: ${{ steps.get-versions.outputs.versions }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Get versions to build from VERSIONS file + id: get-versions + run: | + # Remove whitespace and line breaks + VERSIONS=$(cat "VERSIONS.json" | tr -d '[:space:]\n') + echo "versions=$VERSIONS" >> $GITHUB_OUTPUT + + build: + needs: get-build-versions + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + build: ${{ fromJson(needs.get-build-versions.outputs.versions) }} + + name: CKAN ref ${{ matrix.build.ckan_ref }} on Ubuntu ${{ matrix.build.ubuntu_version }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build package + uses: docker/build-push-action@v6 + with: + push: false + outputs: "type=local,dest=." + build-args: | + CKAN_REF=${{ matrix.build.ckan_ref }} + UBUNTU_VERSION=${{ matrix.build.ubuntu_version }} + + - name: Rename file + # To remove patch version .e.g python-ckan_2.11.1b0-jammy_amd64.deb -> python-ckan_2.11-jammy_amd64.deb + run: | + for f in python-ckan_*; do mv "$f" "$(echo "$f" | sed 's/\([0-9]\+\.[0-9]\+\)[^-]*-/\1-/')"; done + OUTPUT_FILE=$(basename python-ckan*) + echo "OUTPUT_FILE=$OUTPUT_FILE" >> $GITHUB_ENV + echo "Generated file: $OUTPUT_FILE" + + - name: Upload deb file + uses: actions/upload-artifact@v4 + with: + name: ${{ env.OUTPUT_FILE }} + path: ${{ env.OUTPUT_FILE }} diff --git a/.github/workflows/reusable-build-deb-package.yml b/.github/workflows/reusable-build-deb-package.yml deleted file mode 100644 index d74eb37..0000000 --- a/.github/workflows/reusable-build-deb-package.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Reusable CKAN deb packages build workflow - -on: - workflow_call: - inputs: - ckan-ref: - required: true - type: string - ubuntu-version: - required: true - type: string - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build package - uses: docker/build-push-action@v6 - with: - push: false - outputs: "type=local,dest=/tmp" - build-args: | - CKAN_REF=${{ inputs.ckan-ref }} - UBUNTU_VERSION=${{ inputs.ubuntu-version }} - - - name: Rename file - # To remove patch version .e.g python-ckan_2.11.1b0-jammy1_amd64.deb -> python-ckan_2.11-jammy1_amd64.deb - run: | - for f in /tmp/python-ckan_*; do mv "$f" "$(echo "$f" | sed 's/\([0-9]\+\.[0-9]\+\)[^-]*-/\1-/')"; done - OUTPUT_FILE=$(basename /tmp/python-ckan*) - echo "OUTPUT_FILE=$OUTPUT_FILE" >> $GITHUB_ENV - echo "Generated file: $OUTPUT_FILE" - - - name: Upload deb file - uses: actions/upload-artifact@v4 - with: - name: ${{ env.OUTPUT_FILE }} - path: /tmp/${{ env.OUTPUT_FILE }} diff --git a/README.md b/README.md index 856aef2..38006c0 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Builds CKAN deb packages. This script essentially sets up the necessary vars and positional arguments: ref The CKAN branch or tag to build (e.g. master, dev-v2.11, ckan-2.10.6...) - target The Ubuntu distribution to target (e.g. focal, jammy, noble...) + target The Ubuntu version to target (e.g. 20.04, 22.04, 24.04...) optional arguments: -h, --help show this help message and exit @@ -30,9 +30,9 @@ optional arguments: For instance: - ./ckan-package ckan-2.11.1 noble - ./ckan-package dev-v2.11 jammy - ./ckan-package master noble + ./ckan-package ckan-2.11.1 24.04 + ./ckan-package dev-v2.11 22.04 + ./ckan-package master 24.04 The currently supported packages are: diff --git a/UBUNTU_VERSIONS.txt b/UBUNTU_VERSIONS.txt deleted file mode 100644 index d526444..0000000 --- a/UBUNTU_VERSIONS.txt +++ /dev/null @@ -1,3 +0,0 @@ -20.04 focal -22.04 jammy -24.04 noble diff --git a/VERSIONS.json b/VERSIONS.json new file mode 100644 index 0000000..9848540 --- /dev/null +++ b/VERSIONS.json @@ -0,0 +1,18 @@ +[ + { + "ubuntu_version": "20.04", + "ckan_ref": "ckan-2.10.5" + }, + { + "ubuntu_version": "22.04", + "ckan_ref": "ckan-2.10.5" + }, + { + "ubuntu_version": "22.04", + "ckan_ref": "ckan-2.11.0" + }, + { + "ubuntu_version": "24.04", + "ckan_ref": "ckan-2.11.0" + } +] diff --git a/ckan-package b/ckan-package index 95e8b65..de75b54 100755 --- a/ckan-package +++ b/ckan-package @@ -1,10 +1,10 @@ #!/usr/bin/env python import argparse +import json import subprocess import sys - if __name__ == "__main__": description = """Builds CKAN deb packages. This script essentially sets up the necessary vars and calls `docker buildx build`.""" @@ -14,7 +14,7 @@ This script essentially sets up the necessary vars and calls `docker buildx buil help="The CKAN branch or tag to build (e.g. master, dev-v2.11, ckan-2.10.6...)", ) parser.add_argument( - "target", help="The Ubuntu distribution to target (e.g. focal, jammy, noble...)" + "target", help="The Ubuntu distribution to target (e.g. 20.04, 22.04, 24.04...)" ) parser.add_argument( "-i", @@ -29,13 +29,10 @@ This script essentially sets up the necessary vars and calls `docker buildx buil target = args.target iteration = args.iteration - supported_targets = {} - with open("UBUNTU_VERSIONS.txt") as f: - for line in f.readlines(): - [version, name] = line.split() - supported_targets[name] = version + with open("VERSIONS.json") as f: + supported_versions = json.load(f) - if target not in supported_targets.keys(): + if target not in [v["ubuntu_version"] for v in supported_versions]: print(f"Wrong target: {target}") sys.exit(1) @@ -50,7 +47,7 @@ This script essentially sets up the necessary vars and calls `docker buildx buil "--build-arg", f"ITERATION={iteration}", "--build-arg", - f"UBUNTU_VERSION={supported_targets[target]}", + f"UBUNTU_VERSION={target}", ".", ] diff --git a/common/app.yml b/common/app.yml deleted file mode 100644 index 9a31f60..0000000 --- a/common/app.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- - - name: copy source who.ini to configuration folder - action: command cp /usr/lib/ckan/default/src/ckan/who.ini /etc/ckan/default/who.ini - - - name: copy executables - action: copy src=common/{{ item }} dest=/{{ item }} mode=744 - with_items: - - usr/bin/ckan diff --git a/common/build.yml b/common/build.yml deleted file mode 100644 index 761b650..0000000 --- a/common/build.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- - - name: create the /vagrant directory - file: - path: /vagrant - state: directory - - - name: build deb main - action: command chdir=/vagrant fpm - -t deb -s dir - --name python-ckan - --description='CKAN is an open-source DMS (data management system) for powering data hubs and data portals.' - --license='AGPL v3.0' - --maintainer='CKAN team ' - --vendor='CKAN Association' - --url='https://ckan.org' - --after-install=/tmp/after_web.sh - --iteration {{ ansible_distribution_release }}{{ iteration }} - --version {{ ckan_version.stdout }} - --depends nginx --depends libpq5 - --config-files /etc/nginx/sites-available/ckan --config-files /etc/ckan/default/who.ini - /usr/lib/ckan/ /etc/ckan/ /usr/bin/ckan /etc/cron.daily/remove_old_sessions /etc/nginx/sites-available/ckan - /etc/supervisor/conf.d diff --git a/common/web.yml b/common/web.yml deleted file mode 100644 index 92af25c..0000000 --- a/common/web.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- - - name: install uWSGI - pip: - name: uwsgi - virtualenv: /usr/lib/ckan/default - virtualenv_command: pyvenv - - - name: create directories - action: file path={{ item }} state=directory - with_items: - - /etc/ckan/default - - /etc/supervisor/conf.d - - - name: copy source wsgi.py to configuration folder - action: command cp /usr/lib/ckan/default/src/ckan/wsgi.py /etc/ckan/default/wsgi.py - - - name: copy source ckan-uwsgi.ini to configuration folder - action: command cp /usr/lib/ckan/default/src/ckan/ckan-uwsgi.ini /etc/ckan/default/ckan-uwsgi.ini - - - name: copy all common needed web (and other) files - action: copy src=common/{{ item }} dest=/{{ item }} - with_items: - - etc/nginx/sites-available/ckan - - etc/cron.daily/remove_old_sessions - - etc/supervisor/conf.d/ckan-uwsgi.conf - - etc/supervisor/conf.d/ckan-worker.conf From 8f85d5f96a7c987b1ef796cc8039756fb9116d04 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 9 Dec 2024 11:38:25 +0100 Subject: [PATCH 9/9] Create GitHub workflows for building and publishing There are two separate workflows: * `build.yml` builds the deb packages (based on the versions i supplied in `VERSIONS.json`) and stores them as artifacfts in the workflow run page. This is triggered on every push. * Additionally, when a tag is pushed, `publish.yml` also builds the packages and: 1. Uploads them to the S3 bucket powering https://packaging.ckan.org 2. Creates a new GitHub release with the packages attached as asset --- .github/workflows/build.yml | 12 ++++ .github/workflows/publish.yml | 71 +++++++++++++++++++ ...package.yml => reusable-build-package.yml} | 9 ++- Dockerfile | 1 - README.md | 17 +++++ 5 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/publish.yml rename .github/workflows/{build-deb-package.yml => reusable-build-package.yml} (93%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..1b75842 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,12 @@ +name: CKAN deb packages build workflow + +on: + push: + branches: + - '**' + tags-ignore: + - v* + +jobs: + call-build-workflow: + uses: ./.github/workflows/reusable-build-package.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..12b8100 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,71 @@ +name: CKAN deb packages publish workflow + +on: + push: + tags: + - v* + +jobs: + call-build-workflow: + uses: ./.github/workflows/reusable-build-package.yml + + upload-to-s3: + needs: call-build-workflow + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + steps: + - uses: actions/download-artifact@v4 + with: + pattern: python-ckan* + merge-multiple: true + - name: Generate hash and upload + run: | + # Download current md5sum file + aws s3 cp s3://${{ secrets.AWS_BUCKET }}/md5sum . + + for file in python-ckan*; do + # Remove current md5sum entry + sed -i "/$file/d" md5sum + + # Add updated entry to md5sum file + md5sum $file >> md5sum + + # Upload deb file + aws s3 cp $file s3://${{ secrets.AWS_BUCKET }}/staging/$file + done + + # Upload updated md5sum file + aws s3 cp md5sum s3://${{ secrets.AWS_BUCKET }}/staging/md5sum + + upload-to-release: + needs: call-build-workflow + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: python-ckan* + merge-multiple: true + - name: Create release and upload the deb files + env: + GH_TOKEN: ${{ github.token }} + run: | + VERSIONS=$(cat "VERSIONS.json") + + LIST=$(echo $VERSIONS | jq -r ' + (.[] | ["* \(.ckan_ref) on Ubuntu \(.ubuntu_version)"]) | + .[] + ') + + NOTES="This release includes deb packages for the following versions. + + $LIST + + Please check the relevant file in the Assets section below. + Packages are also available at https://packaging.ckan.org." + + gh release create ${{ github.ref_name }} ./python-ckan* --verify-tag --notes "$NOTES" diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/reusable-build-package.yml similarity index 93% rename from .github/workflows/build-deb-package.yml rename to .github/workflows/reusable-build-package.yml index 45690d5..9f2b62d 100644 --- a/.github/workflows/build-deb-package.yml +++ b/.github/workflows/reusable-build-package.yml @@ -1,9 +1,7 @@ -name: CKAN deb packages build workflow +name: Reusable CKAN deb packages build workflow on: - push: - paths-ignore: - - 'README.md' + workflow_call: jobs: get-build-versions: @@ -21,7 +19,7 @@ jobs: VERSIONS=$(cat "VERSIONS.json" | tr -d '[:space:]\n') echo "versions=$VERSIONS" >> $GITHUB_OUTPUT - build: + build-package: needs: get-build-versions runs-on: ubuntu-latest strategy: @@ -54,6 +52,7 @@ jobs: OUTPUT_FILE=$(basename python-ckan*) echo "OUTPUT_FILE=$OUTPUT_FILE" >> $GITHUB_ENV echo "Generated file: $OUTPUT_FILE" + dpkg --info $OUTPUT_FILE - name: Upload deb file uses: actions/upload-artifact@v4 diff --git a/Dockerfile b/Dockerfile index d2e1f2a..dab82a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -104,6 +104,5 @@ RUN DISTRIBUTION=$(lsb_release -c -s) && \ RUN ls -la /output - FROM scratch AS export COPY --from=builder /output . diff --git a/README.md b/README.md index 38006c0..5985ace 100644 --- a/README.md +++ b/README.md @@ -58,3 +58,20 @@ docker buildx build \ --build-arg UBUNTU_VERSION=24.04 \ . ``` + +# Release process + +There are two separate workflows: + +* `build.yml` builds the deb packages (based on the versions supplied in `VERSIONS.json`) and stores them as artifacfts in the workflow run page. This is triggered on every push. + + +* Additionally, when a tag is pushed, `publish.yml` also builds the packages and: + 1. Uploads them to the S3 bucket powering https://packaging.ckan.org + 2. Creates a new GitHub release with the packages attached as assets. + +With this, the suggested release process is the following: + +* Whenever there is a new CKAN release in the works, or fixes need to be applied to the packages, a new branch and pull request is created. This will trigger the workflows that will create the packages for that version of the code. The `ckan_ref` should be the relvant development branch (e.g. `dev-v2.11`). +* The packages can be downloded from the workflow page to test locally. Once everthing looks fine the PR is merged. +* A new tag in the form `vYYYYMMDD` is pushed to trigger the publication of the packages and the creation of the release.