Allow configuring multiple buckets for process_incoming.py (#68)

The incoming package may now declare which bucket it wants to go in using an identifier, and a mapping from bucket identifiers to actual bucket names can be specified in the new `[common.buckets]` config key.
edgedb · Dec 14, 2023 · 2f1e801 · 2f1e801
1 parent 41398f8
commit 2f1e801
Show file tree

Hide file tree

Showing 18 changed files with 102 additions and 16 deletions.
diff --git a/Makefile b/Makefile
@@ -72,6 +72,10 @@ ifneq ($(PKG_SUBDIST),)
 	EXTRAENV += -e PKG_SUBDIST="$(PKG_SUBDIST)"
 endif
 
+ifneq ($(PKG_TAGS),)
+	EXTRAENV += -e PKG_TAGS="$(PKG_TAGS)"
+endif
+
 ifneq ($(PKG_INSTALL_REF),)
 	EXTRAENV += -e PKG_INSTALL_REF="$(PKG_INSTALL_REF)"
 endif

diff --git a/integration/linux/build/centos-7/Dockerfile b/integration/linux/build/centos-7/Dockerfile
@@ -185,7 +185,7 @@ RUN cd /usr/local/bin \
 	&& ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
 	\
@@ -246,6 +246,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/centos-8/Dockerfile b/integration/linux/build/centos-8/Dockerfile
@@ -185,7 +185,7 @@ RUN cd /usr/local/bin \
 	&& ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
 	\
@@ -246,6 +246,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/debian-bookworm/Dockerfile b/integration/linux/build/debian-bookworm/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
 	&& ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
 	\
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/debian-bullseye/Dockerfile b/integration/linux/build/debian-bullseye/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
 	&& ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
 	\
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/debian-buster/Dockerfile b/integration/linux/build/debian-buster/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
 	&& ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
 	\
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/entrypoint.sh b/integration/linux/build/entrypoint.sh
@@ -39,6 +39,10 @@ if [ -n "${PKG_SUBDIST}" ]; then
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"
 fi
 
+if [ -n "${PKG_TAGS}" ]; then
+    extraopts+=" --pkg-tags=${PKG_TAGS}"
+fi
+
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then
     extraopts+=" --extra-optimizations"
 fi

diff --git a/integration/linux/build/fedora-29/Dockerfile b/integration/linux/build/fedora-29/Dockerfile
@@ -90,7 +90,7 @@ RUN cd /usr/local/bin \
 	&& ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
 	\
@@ -151,6 +151,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/linux-aarch64/Dockerfile b/integration/linux/build/linux-aarch64/Dockerfile
@@ -218,7 +218,7 @@ RUN cd /usr/local/bin \
 	&& ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
 	\
@@ -279,6 +279,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/linux-x86_64/Dockerfile b/integration/linux/build/linux-x86_64/Dockerfile
@@ -218,7 +218,7 @@ RUN cd /usr/local/bin \
 	&& ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
 	\
@@ -279,6 +279,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/linuxmusl-aarch64/Dockerfile b/integration/linux/build/linuxmusl-aarch64/Dockerfile
@@ -82,6 +82,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/linuxmusl-x86_64/Dockerfile b/integration/linux/build/linuxmusl-x86_64/Dockerfile
@@ -82,6 +82,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/rockylinux-9/Dockerfile b/integration/linux/build/rockylinux-9/Dockerfile
@@ -148,6 +148,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/ubuntu-bionic/Dockerfile b/integration/linux/build/ubuntu-bionic/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
 	&& ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
 	\
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/ubuntu-focal/Dockerfile b/integration/linux/build/ubuntu-focal/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
 	&& ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
 	\
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/ubuntu-hirsute/Dockerfile b/integration/linux/build/ubuntu-hirsute/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
 	&& ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
 	\
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/ubuntu-jammy/Dockerfile b/integration/linux/build/ubuntu-jammy/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
 	&& ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
 	\
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
     extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+    extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
     extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/server/process_incoming.py b/server/process_incoming.py
@@ -23,13 +23,15 @@
 import textwrap
 
 import boto3
+import boto3.session
 import click
 import filelock
 import semver
 import tomli
 
 from debian import debian_support
 
+import mypy_boto3_s3
 from mypy_boto3_s3 import type_defs as s3types
 from mypy_boto3_s3 import service_resource as s3
 
@@ -46,6 +48,7 @@
 
 class CommonConfig(TypedDict):
     signing_key: str
+    buckets: dict[str, str]
 
 
 class GenericConfig(TypedDict):
@@ -268,7 +271,7 @@ def remove_old(
     channel: str | None = None,
 ) -> None:
     logger.info("remove_old: %s %s %s %s", bucket, prefix, keep, channel)
-    index: dict[str, dict[str, list[str]]] = {}
+    index: dict[str, dict[semver.VersionInfo, list[str]]] = {}
     prefix_str = str(prefix) + "/"
     for obj in bucket.objects.filter(Prefix=prefix_str):
         if is_metadata_object(obj.key):
@@ -532,19 +535,26 @@ def sync_to_s3(
 
 @click.command()
 @click.option("-c", "--config", default="/etc/genrepo.toml")
-@click.option("--bucket", default="edgedb-packages")
+@click.option("--bucket")
 @click.option("--incoming-dir")
 @click.option("--local-dir")
 @click.argument("upload_listing")  # a single file with a listing of many files
 def main(
     config: str,
-    bucket: str,
+    bucket: Optional[str],
     incoming_dir: str,
     local_dir: str,
     upload_listing: str,
 ) -> None:
     with open(config, "rb") as cf:
         cfg = cast(Config, tomli.load(cf))
+        if "common" not in cfg:
+            raise ValueError("missing required [common] section in config")
+        if not cfg["common"].get("buckets"):
+            cfg["common"]["buckets"] = {}
+
+    if bucket:
+        cfg["common"]["buckets"]["default"] = bucket
 
     os.chdir(incoming_dir)
     with open(upload_listing) as upload_listing_file:
@@ -553,7 +563,7 @@ def main(
 
     region = os.environ.get("AWS_REGION", "us-east-2")
     session = boto3.session.Session(region_name=region)
-    s3 = session.resource("s3")
+    s3: mypy_boto3_s3.S3ServiceResource = session.resource("s3")  # pyright: ignore
 
     for path_str in uploads:
         path = pathlib.Path(path_str)
@@ -578,6 +588,14 @@ def main(
 
                 metadata = json.loads(metadata_file.read())
                 repository = metadata.get("repository")
+                tags = metadata.get("tags") or {}
+                target_bucket = tags.get("bucket", "default")
+                bucket = cfg["common"]["buckets"].get(target_bucket)
+                if bucket is None:
+                    raise RuntimeError(
+                        f"invalid target bucket in metadata: {target_bucket!r}"
+                        f", configure it in genrepo.toml"
+                    )
 
                 local_dir_path = pathlib.Path(local_dir)
                 temp_dir_path = pathlib.Path(temp_dir)