diff --git a/Makefile b/Makefile index 37e7149..cf2e958 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,10 @@ ifneq ($(PKG_SUBDIST),) EXTRAENV += -e PKG_SUBDIST="$(PKG_SUBDIST)" endif +ifneq ($(PKG_TAGS),) + EXTRAENV += -e PKG_TAGS="$(PKG_TAGS)" +endif + ifneq ($(PKG_INSTALL_REF),) EXTRAENV += -e PKG_INSTALL_REF="$(PKG_INSTALL_REF)" endif diff --git a/integration/linux/build/centos-7/Dockerfile b/integration/linux/build/centos-7/Dockerfile index c3a83c8..b401f57 100644 --- a/integration/linux/build/centos-7/Dockerfile +++ b/integration/linux/build/centos-7/Dockerfile @@ -185,7 +185,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -246,6 +246,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/centos-8/Dockerfile b/integration/linux/build/centos-8/Dockerfile index 36ecfd1..778f4a2 100644 --- a/integration/linux/build/centos-8/Dockerfile +++ b/integration/linux/build/centos-8/Dockerfile @@ -185,7 +185,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -246,6 +246,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/debian-bookworm/Dockerfile b/integration/linux/build/debian-bookworm/Dockerfile index 6f82542..d172925 100644 --- a/integration/linux/build/debian-bookworm/Dockerfile +++ b/integration/linux/build/debian-bookworm/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/debian-bullseye/Dockerfile b/integration/linux/build/debian-bullseye/Dockerfile index b1d47fb..e274d4f 100644 --- a/integration/linux/build/debian-bullseye/Dockerfile +++ b/integration/linux/build/debian-bullseye/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/debian-buster/Dockerfile b/integration/linux/build/debian-buster/Dockerfile index 74c332c..be9478a 100644 --- a/integration/linux/build/debian-buster/Dockerfile +++ b/integration/linux/build/debian-buster/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/entrypoint.sh b/integration/linux/build/entrypoint.sh index 59bb1de..5bde044 100755 --- a/integration/linux/build/entrypoint.sh +++ b/integration/linux/build/entrypoint.sh @@ -39,6 +39,10 @@ if [ -n "${PKG_SUBDIST}" ]; then extraopts+=" --pkg-subdist=${PKG_SUBDIST}" fi +if [ -n "${PKG_TAGS}" ]; then + extraopts+=" --pkg-tags=${PKG_TAGS}" +fi + if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then extraopts+=" --extra-optimizations" fi diff --git a/integration/linux/build/fedora-29/Dockerfile b/integration/linux/build/fedora-29/Dockerfile index db14ba9..770d438 100644 --- a/integration/linux/build/fedora-29/Dockerfile +++ b/integration/linux/build/fedora-29/Dockerfile @@ -90,7 +90,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -151,6 +151,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/linux-aarch64/Dockerfile b/integration/linux/build/linux-aarch64/Dockerfile index d71f4a6..86c3040 100644 --- a/integration/linux/build/linux-aarch64/Dockerfile +++ b/integration/linux/build/linux-aarch64/Dockerfile @@ -218,7 +218,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -279,6 +279,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/linux-x86_64/Dockerfile b/integration/linux/build/linux-x86_64/Dockerfile index b544682..77fb1c4 100644 --- a/integration/linux/build/linux-x86_64/Dockerfile +++ b/integration/linux/build/linux-x86_64/Dockerfile @@ -218,7 +218,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -279,6 +279,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/linuxmusl-aarch64/Dockerfile b/integration/linux/build/linuxmusl-aarch64/Dockerfile index fbc0a63..8ced143 100644 --- a/integration/linux/build/linuxmusl-aarch64/Dockerfile +++ b/integration/linux/build/linuxmusl-aarch64/Dockerfile @@ -82,6 +82,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/linuxmusl-x86_64/Dockerfile b/integration/linux/build/linuxmusl-x86_64/Dockerfile index fbc0a63..8ced143 100644 --- a/integration/linux/build/linuxmusl-x86_64/Dockerfile +++ b/integration/linux/build/linuxmusl-x86_64/Dockerfile @@ -82,6 +82,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/rockylinux-9/Dockerfile b/integration/linux/build/rockylinux-9/Dockerfile index 299f3ce..d088f56 100644 --- a/integration/linux/build/rockylinux-9/Dockerfile +++ b/integration/linux/build/rockylinux-9/Dockerfile @@ -148,6 +148,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/ubuntu-bionic/Dockerfile b/integration/linux/build/ubuntu-bionic/Dockerfile index a5561ee..a5a9548 100644 --- a/integration/linux/build/ubuntu-bionic/Dockerfile +++ b/integration/linux/build/ubuntu-bionic/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/ubuntu-focal/Dockerfile b/integration/linux/build/ubuntu-focal/Dockerfile index ef79d5b..8127a8a 100644 --- a/integration/linux/build/ubuntu-focal/Dockerfile +++ b/integration/linux/build/ubuntu-focal/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/ubuntu-hirsute/Dockerfile b/integration/linux/build/ubuntu-hirsute/Dockerfile index 56a2a0c..8272ffc 100644 --- a/integration/linux/build/ubuntu-hirsute/Dockerfile +++ b/integration/linux/build/ubuntu-hirsute/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/ubuntu-jammy/Dockerfile b/integration/linux/build/ubuntu-jammy/Dockerfile index 024d94a..07c0041 100644 --- a/integration/linux/build/ubuntu-jammy/Dockerfile +++ b/integration/linux/build/ubuntu-jammy/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/server/process_incoming.py b/server/process_incoming.py index 218a562..ea5533d 100755 --- a/server/process_incoming.py +++ b/server/process_incoming.py @@ -23,6 +23,7 @@ import textwrap import boto3 +import boto3.session import click import filelock import semver @@ -30,6 +31,7 @@ from debian import debian_support +import mypy_boto3_s3 from mypy_boto3_s3 import type_defs as s3types from mypy_boto3_s3 import service_resource as s3 @@ -46,6 +48,7 @@ class CommonConfig(TypedDict): signing_key: str + buckets: dict[str, str] class GenericConfig(TypedDict): @@ -268,7 +271,7 @@ def remove_old( channel: str | None = None, ) -> None: logger.info("remove_old: %s %s %s %s", bucket, prefix, keep, channel) - index: dict[str, dict[str, list[str]]] = {} + index: dict[str, dict[semver.VersionInfo, list[str]]] = {} prefix_str = str(prefix) + "/" for obj in bucket.objects.filter(Prefix=prefix_str): if is_metadata_object(obj.key): @@ -532,19 +535,26 @@ def sync_to_s3( @click.command() @click.option("-c", "--config", default="/etc/genrepo.toml") -@click.option("--bucket", default="edgedb-packages") +@click.option("--bucket") @click.option("--incoming-dir") @click.option("--local-dir") @click.argument("upload_listing") # a single file with a listing of many files def main( config: str, - bucket: str, + bucket: Optional[str], incoming_dir: str, local_dir: str, upload_listing: str, ) -> None: with open(config, "rb") as cf: cfg = cast(Config, tomli.load(cf)) + if "common" not in cfg: + raise ValueError("missing required [common] section in config") + if not cfg["common"].get("buckets"): + cfg["common"]["buckets"] = {} + + if bucket: + cfg["common"]["buckets"]["default"] = bucket os.chdir(incoming_dir) with open(upload_listing) as upload_listing_file: @@ -553,7 +563,7 @@ def main( region = os.environ.get("AWS_REGION", "us-east-2") session = boto3.session.Session(region_name=region) - s3 = session.resource("s3") + s3: mypy_boto3_s3.S3ServiceResource = session.resource("s3") # pyright: ignore for path_str in uploads: path = pathlib.Path(path_str) @@ -578,6 +588,14 @@ def main( metadata = json.loads(metadata_file.read()) repository = metadata.get("repository") + tags = metadata.get("tags") or {} + target_bucket = tags.get("bucket", "default") + bucket = cfg["common"]["buckets"].get(target_bucket) + if bucket is None: + raise RuntimeError( + f"invalid target bucket in metadata: {target_bucket!r}" + f", configure it in genrepo.toml" + ) local_dir_path = pathlib.Path(local_dir) temp_dir_path = pathlib.Path(temp_dir)