Cray-HPE · mtupitsyn · Jun 26, 2024 · Jun 26, 2024 · Jun 26, 2024
@@ -73,5 +73,23 @@ for arch in "${CN_ARCH[@]}"; do
     \)
 done
 
-HPE_SIGNING_KEY=https://artifactory.algol60.net/artifactory/gpg-keys/hpe-signing-key.asc
-HPE_SIGNING_KEY_FIPS=https://artifactory.algol60.net/artifactory/gpg-keys/hpe-signing-key-fips.asc
+# Public keys for RPM signature validation.
+#
+# hpe-signing-key.asc - for all packages signed by HPE Code Signing DST/CSM old key (expires 2025-12-07)
+# hpe-signing-key-fips.asc - for all packages signed by HPE Code Signing, DST new key (expires 2026-09-01), for example kernel-mft-mlnx-kmp-default
+# hpe-sdr-signing-key.asc - older HPE key used by SDR repos (Qlogic driver - qlgc-fastlinq-kmp-default)
+# google-package-key.asc - for kubelet/kubeadm/kubectl from https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
+# suse-package-key.asc - for most SUSE packages in embedded repo
+# opensuse-obs-filesystems.asc - for packages copied into /csm-rpms/stable from OpenSUSE filesystems (such as csm-rpms/hpe/stable/sle-15sp5/ceph-common-17.2.6.865+g60870edfe2e-lp155.1.1.x86_64.rpm): https://download.opensuse.org/repositories/filesystems:/ceph:/quincy:/upstream/openSUSE_Leap_15.5/repodata/repomd.xml.key
+# opensuse-obs-backports.asc - for packages in /sles-mirror/Backports/SLE-15-SP5_x86_64 (dkms, perl-File-BaseDir)
+# suse_ptf_key.asc - for SUSE PTF kernel packages, see https://www.suse.com/support/kb/doc/?id=000018545
+HPE_RPM_SIGNING_KEYS=(
+    https://artifactory.algol60.net/artifactory/gpg-keys/hpe-signing-key.asc
+    https://artifactory.algol60.net/artifactory/gpg-keys/hpe-signing-key-fips.asc
+    https://artifactory.algol60.net/artifactory/gpg-keys/hpe-sdr-signing-key.asc
+    https://artifactory.algol60.net/artifactory/gpg-keys/google-package-key.asc
+    https://artifactory.algol60.net/artifactory/gpg-keys/suse-package-key.asc
+    https://artifactory.algol60.net/artifactory/gpg-keys/opensuse-obs-filesystems-15-sp5.asc
+    https://artifactory.algol60.net/artifactory/gpg-keys/opensuse-obs-backports-15-sp5.asc
+    https://artifactory.algol60.net/artifactory/gpg-keys/suse_ptf_key.asc
+)
@@ -67,6 +67,7 @@ if [ -n "$KERNEL_DEFAULT_DEBUGINFO_VERSION" ]; then
 fi
 
 # Try repos for SLES 15 SP4 and SP5
+# Filter out openSUSE:Backports repos - we have these packages in SLES RMT and should prefer that
 (
     cat "${TMPDIR}/ncn.repo-list.releasever" \
         | sed -e "s/\${basearch}/${NCN_ARCH}/g" \
@@ -78,42 +79,18 @@ fi
         | sed -e "s/\${releasever_major}/15/g" \
         | sed -e "s/\${releasever_minor}/5/g" \
         | sed -e "s/\${releasever}/15.5/g"
-) | sort -u > "${TMPDIR}/ncn.repo-list.unverified"
+) \
+    | grep -v openSUSE:Backports \
+    | sort -u > "${TMPDIR}/ncn.repo-list.unverified"
 
 # Filter out non-existent repos and generate directory names for rpm-index input
 echo -ne > "${TMPDIR}/ncn.repo-list"
-SIGNING_KEYS=""
-if [ "${VALIDATE}" != "1" ]; then
-    echo "Downloading additional signing keys ..."
-    mkdir -p "${BUILDDIR}/security"
-    # google-package-key.asc - from https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
-    # hpe-signing-key.asc - for all packages signed by HPE Code Signing
-    # opensuse-ceph-quincy.asc - for Ceph 17.2 packages copied from OpenSUSE: https://download.opensuse.org/repositories/filesystems:/ceph:/quincy:/upstream/openSUSE_Leap_15.5/repodata/repomd.xml.key
-    # suse_ptf_key.asc - for SUSE PTF kernel packages, see https://www.suse.com/support/kb/doc/?id=000018545
-    for key in google-package-key.asc hpe-signing-key.asc opensuse-ceph-quincy.asc suse_ptf_key.asc; do
-        echo -ne "Downloading ${key} ... "
-        acurl -Ss -o "${BUILDDIR}/security/${key}" "https://artifactory.algol60.net/artifactory/gpg-keys/${key}"
-        echo "ok"
-        SIGNING_KEYS="${SIGNING_KEYS} -k /keys/${key}"
-    done
-fi
-
 while read -r url; do
     if acurl -I -Ss -f "$url/repodata/repomd.xml" >/dev/null 2>/dev/null; then
         dir="${url#https://}"
         dir="${dir#artifactory.algol60.net/artifactory/}"
         dir="${dir//-mirror/}"
         echo "$url" "$dir" >> "${TMPDIR}/ncn.repo-list"
-        if [ "${VALIDATE}" != "1" ]; then
-            echo -ne "Looking for GPG key in ${url} ... "
-            mkdir -p "${TARGET_DIR}/${dir}/repodata"
-            if acurl -Ss -f -o "${TARGET_DIR}/${dir}/repodata/repomd.xml.key" "$url/repodata/repomd.xml.key" >/dev/null 2>/dev/null; then
-                echo "ok"
-                SIGNING_KEYS="${SIGNING_KEYS} -k /data/${dir}/repodata/repomd.xml.key"
-            else
-                echo "no key"
-            fi
-        fi
     fi
 done < "${TMPDIR}/ncn.repo-list.unverified"
 
@@ -155,12 +132,26 @@ echo "Building RPM package index ..."
 if [ "${VALIDATE}" == "1" ]; then
     echo "All RPM packages were resolved successfully"
 else
+    # Download and store RPM signing keys (if not yet downloaded by rpm.sh)
+    SIGNING_KEYS=""
+    mkdir -p "${BUILDDIR}/security/keys/rpm"
+    for key_url in "${HPE_RPM_SIGNING_KEYS[@]}"; do
+        key=$(basename "${key_url}")
+        if [ -f "${BUILDDIR}/security/keys/rpm/${key}" ]; then
+            echo "Signing key ${key} is already downloaded"
+        else
+            echo "Downloading ${key} signing key"
+            acurl -Ss -o "${BUILDDIR}/security/keys/rpm/${key}" "${key_url}"
+        fi
+        SIGNING_KEYS="${SIGNING_KEYS} -k /keys/${key}"
+    done
+
     echo "Downloading RPM packages into ${TARGET_DIR} ..."
     mkdir -p "${TARGET_DIR}"
     docker run ${REPO_CREDS_DOCKER_OPTIONS} --rm -i -u "$(id -u):$(id -g)" \
             -v "$(realpath "${TMPDIR}/embedded.yaml"):/index.yaml:ro" \
             -v "$(realpath "${TARGET_DIR}"):/data" \
-            -v "$(realpath "${BUILDDIR}/security/"):/keys" \
+            -v "$(realpath "${BUILDDIR}/security/keys/rpm/"):/keys" \
             "${PACKAGING_TOOLS_IMAGE}" \
             rpm-sync ${REPO_CREDS_RPMSYNC_OPTIONS} -n 1 -s -v ${SIGNING_KEYS} -d /data /index.yaml
 

@@ -12,6 +12,7 @@ if [ $# -ne 1 ] || ([ "${1}" != "--validate" ] && [ "${1}" != "--download" ]); t
 fi
 
 [ "${1}" == "--validate" ] && VALIDATE=1 || VALIDATE=0
+SIGNING_KEYS=""
 
 function rpm-sync() {
     index="${1}"
@@ -25,9 +26,9 @@ function rpm-sync() {
         docker run ${REPO_CREDS_DOCKER_OPTIONS} --rm -i -u "$(id -u):$(id -g)" \
             -v "$(realpath "${index}"):/index.yaml:ro" \
             -v "$(realpath "${destdir}"):/data" \
-            -v "$(realpath "${BUILDDIR}/security/"):/keys" \
+            -v "$(realpath "${BUILDDIR}/security/keys/rpm/"):/keys" \
             "${PACKAGING_TOOLS_IMAGE}" \
-            rpm-sync ${REPO_CREDS_RPMSYNC_OPTIONS} -n 1 -s -v -k /keys/hpe-signing-key.asc -k /keys/hpe-signing-key-fips.asc -d /data /index.yaml
+            rpm-sync ${REPO_CREDS_RPMSYNC_OPTIONS} -n 1 -s -v ${SIGNING_KEYS} -d /data /index.yaml
     fi
 }
 
@@ -76,16 +77,27 @@ function createrepo() {
 }
 
 if [ "${VALIDATE}" != "1" ]; then
-    if ! [ -f "${BUILDDIR}/security/hpe-signing-key.asc" ]; then
-        echo "Downloading HPE signing key"
-        mkdir -p "${BUILDDIR}/security"
-        acurl -Ss -o "${BUILDDIR}/security/hpe-signing-key.asc" "${HPE_SIGNING_KEY}"
-    fi
-    if ! [ -f "${BUILDDIR}/security/hpe-signing-key-fips.asc" ]; then
-        echo "Downloading new HPE signing key"
-        mkdir -p "${BUILDDIR}/security"
-        acurl -Ss -o "${BUILDDIR}/security/hpe-signing-key-fips.asc" "${HPE_SIGNING_KEY_FIPS}"
-    fi
+    # Special processing for docs-csm, as we don't know exact version before build starts, so can't include it into rpm indexes.
+    # Can't include docs-csm-latest either, because it is not unique. Get version from right docs-csm-latest, then download actual rpm file.
+    DOCS_CSM_MAJOR_MINOR="${DOCS_CSM_MAJOR_MINOR:-${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}}"
+    DOCS_CSM_VERSION=$(acurl -sSL "https://artifactory.algol60.net/artifactory/api/storage/csm-rpms/hpe/stable/noos/docs-csm/${DOCS_CSM_MAJOR_MINOR}/noarch/docs-csm-latest.noarch.rpm?properties" | jq -r '.properties["rpm.metadata.version"][0] // ""')
+    echo "Downloading docs-csm-${DOCS_CSM_VERSION}-1.noarch.rpm ..."
+    mkdir -p "${BUILDDIR}/rpm/cray/csm/noos/noarch"
+    acurl -sSL -o "${BUILDDIR}/rpm/cray/csm/noos/noarch/docs-csm-${DOCS_CSM_VERSION}-1.noarch.rpm" \
+        "https://artifactory.algol60.net/artifactory/csm-rpms/hpe/stable/noos/docs-csm/${DOCS_CSM_MAJOR_MINOR}/noarch/docs-csm-${DOCS_CSM_VERSION}-1.noarch.rpm"
+
+    # Download and store RPM signing keys.
+    mkdir -p "${BUILDDIR}/security/keys/rpm"
+    for key_url in "${HPE_RPM_SIGNING_KEYS[@]}"; do
+        key=$(basename "${key_url}")
+        if [ -f "${BUILDDIR}/security/keys/rpm/${key}" ]; then
+            echo "Signing key ${key} is already downloaded"
+        else
+            echo "Downloading ${key} signing key"
+            acurl -Ss -o "${BUILDDIR}/security/keys/rpm/${key}" "${key_url}"
+        fi
+        SIGNING_KEYS="${SIGNING_KEYS} -k /keys/${key}"
+    done
 fi
 
 rpm-sync-with-csm-base "rpm/cray/csm/sle-15sp2"
@@ -98,15 +110,6 @@ if [ "${VALIDATE}" == "1" ]; then
     echo "RPM indexes validated successfully"
 else
     echo "RPM indexes synchronized successfully"
-    # Special processing for docs-csm, as we don't know exact version before build starts, so can't include it into rpm indexes.
-    # Can't include docs-csm-latest either, because it is not unique. Get version from right docs-csm-latest, then download actual rpm file.
-    DOCS_CSM_MAJOR_MINOR="${DOCS_CSM_MAJOR_MINOR:-${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}}"
-    DOCS_CSM_VERSION=$(acurl -sSL "https://artifactory.algol60.net/artifactory/api/storage/csm-rpms/hpe/stable/noos/docs-csm/${DOCS_CSM_MAJOR_MINOR}/noarch/docs-csm-latest.noarch.rpm?properties" | jq -r '.properties["rpm.metadata.version"][0]')
-    mkdir -p "${BUILDDIR}/rpm/cray/csm/noos/noarch"
-    acurl -sSL -o "${BUILDDIR}/rpm/cray/csm/noos/noarch/docs-csm-${DOCS_CSM_VERSION}-1.noarch.rpm" \
-        "https://artifactory.algol60.net/artifactory/csm-rpms/hpe/stable/noos/docs-csm/${DOCS_CSM_MAJOR_MINOR}/noarch/docs-csm-${DOCS_CSM_VERSION}-1.noarch.rpm"
-    rpm -qpi "${BUILDDIR}/rpm/cray/csm/noos/noarch/docs-csm-${DOCS_CSM_VERSION}-1.noarch.rpm" | grep -q -E "Signature\s*:\s*\(none\)" && (echo "ERROR: RPM package docs-csm-${DOCS_CSM_VERSION}-1.noarch.rpm is not signed"; exit 1)
-
     # Fix-up cray directories by removing misc subdirectories
     {
         find "${BUILDDIR}/rpm/cray" -name '*-team' -type d

@@ -79,10 +79,13 @@ deploy "${BUILDDIR}/manifests/storage.yaml"
 deploy "${BUILDDIR}/manifests/platform.yaml"
 deploy "${BUILDDIR}/manifests/keycloak-gatekeeper.yaml"
 
-# Create secret with HPE signing key
-if [[ -f "${ROOTDIR}/security/hpe-signing-key.asc" ]]; then
-    kubectl create secret generic hpe-signing-key -n services --from-file=gpg-pubkey="${ROOTDIR}/security/hpe-signing-key.asc" --dry-run=client --save-config -o yaml | kubectl apply -f -
-fi
+# Create secret with RPM signing keys
+# For backward compatibility, also import hpe-signing-key.asc under the name "gpg-pubkey"
+RPM_SIGNING_KEYS_OPT="--from-file gpg-pubkey=${ROOTDIR}/security/keys/rpm/hpe-signing-key.asc"
+for key in ${ROOTDIR}/security/keys/rpm/*.asc; do
+        RPM_SIGNING_KEYS_OPT="${RPM_SIGNING_KEYS_OPT} --from-file ${key}"
+done
+kubectl create secret generic hpe-signing-key -n services ${RPM_SIGNING_KEYS_OPT} --dry-run=client --save-config -o yaml | kubectl apply -f -
 
 # Upload SLS Input file to S3
 csi upload-sls-file --sls-file "$SLS_INPUT_FILE"

@@ -196,7 +196,7 @@ spec:
     namespace: services
   - name: csm-config
     source: csm-algol60
-    version: 1.16.31
+    version: 1.16.33
     namespace: services
     values:
       cray-import-config:

@@ -75,10 +75,13 @@ deploy "${BUILDDIR}/manifests/keycloak-gatekeeper.yaml"
 # Deploy metal-lb configuration
 # kubectl apply -f "$METALLB_YAML"
 
-# Create secret with HPE signing key
-if [[ -f "${ROOTDIR}/security/hpe-signing-key.asc" ]]; then
-    kubectl create secret generic hpe-signing-key -n services --from-file=gpg-pubkey="${ROOTDIR}/security/hpe-signing-key.asc" --dry-run=client --save-config -o yaml | kubectl apply -f -
-fi
+# Create secret with RPM signing keys
+# For backward compatibility, also import hpe-signing-key.asc under the name "gpg-pubkey"
+RPM_SIGNING_KEYS_OPT="--from-file gpg-pubkey=${ROOTDIR}/security/keys/rpm/hpe-signing-key.asc"
+for key in ${ROOTDIR}/security/keys/rpm/*.asc; do
+        RPM_SIGNING_KEYS_OPT="${RPM_SIGNING_KEYS_OPT} --from-file ${key}"
+done
+kubectl create secret generic hpe-signing-key -n services ${RPM_SIGNING_KEYS_OPT} --dry-run=client --save-config -o yaml | kubectl apply -f -
 
 # Save previous Unbound IP
 pre_upgrade_unbound_ip="$(kubectl get -n services service cray-dns-unbound-udp-nmn -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"