Update airgap patch

.github/workflows/manual-kubespray-release.yaml

@@ -95,6 +95,7 @@ jobs:
         build-args: |
           SPRAY_TAG=${{ needs.check-inputs.outputs.kubespray_sha1_short }}
           REPO=${{ needs.check-inputs.outputs.image_repo }}
+          CR_NAME_POSTFIX: ${{ needs.check-inputs.outputs.tag }}
         github-token: ${{ secrets.GITHUB_TOKEN }}
         push: true
         provenance: false
@@ -126,14 +127,15 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: sprayjob image build
+    - name: airgap-patch image build
       uses: docker/[email protected]
       with:
         context: ./
         file: build/images/airgap-patch/Dockerfile
         build-args: |
           SPRAY_TAG=${{ needs.check-inputs.outputs.kubespray_sha1_short }}
           REPO=${{ needs.check-inputs.outputs.image_repo }}
+          CR_NAME_POSTFIX: ${{ needs.check-inputs.outputs.tag }}
         github-token: ${{ secrets.GITHUB_TOKEN }}
         push: true
         provenance: false

artifacts/offline_patch.py → artifacts/airgap_patch.py

@@ -14,7 +14,7 @@
 
 OPTION = os.getenv("OPTION", default="all")  # create_files create_images create_offlineversion_cr
 
-KUBESPRAY_DIR = os.path.join(CUR_DIR, "kubespray")
+SPRAY_REPO_PATH = os.path.join(CUR_DIR, "kubespray")
 MANIFEST_YML_FILE = os.getenv("MANIFEST_CONF", default="manifest.yml")
 ZONE = os.getenv("ZONE", default="Other")  ## Other or CN
 
@@ -23,9 +23,11 @@
 OFFLINE_VER_CR_TEMP = os.getenv("OFFLINEVERSION_CR_TEMPLATE",
                                 default=os.path.join(CUR_DIR,
                                                      "artifacts/template/localartifactset.template.yml"))
+SPRAY_RELEASE = os.getenv("SPRAY_RELEASE")
+CR_NAME_POSTFIX = os.getenv("CR_NAME_POSTFIX")
 
-FILE_LIST_TEMP_PATH = os.path.join(KUBESPRAY_DIR, "contrib/offline/temp/files.list")
-IMAGE_LIST_TEMP_PATH = os.path.join(KUBESPRAY_DIR, "contrib/offline/temp/images.list")
+FILE_LIST_TEMP_PATH = os.path.join(SPRAY_REPO_PATH, "contrib/offline/temp/files.list")
+IMAGE_LIST_TEMP_PATH = os.path.join(SPRAY_REPO_PATH, "contrib/offline/temp/images.list")
 KUBEAN_TAG = "v_offline_patch"
 print(f"CUR_DIR:{CUR_DIR}")
 
@@ -42,7 +44,7 @@ def extra_line_str_with_pattern(filepath, *patterns):
 
 def fetch_info_list(env_dict, *patterns):
     print(f"generating info list for {env_dict}")
-    os.chdir(KUBESPRAY_DIR)
+    os.chdir(SPRAY_REPO_PATH)
     if os.path.exists("contrib/offline/temp"):
         shutil.rmtree("contrib/offline/temp")
     cmd = ["bash", "contrib/offline/generate_list.sh"]
@@ -67,11 +69,11 @@ def fetch_info_list(env_dict, *patterns):
 
 
 def check_dependencies():
-    if not os.path.exists(KUBESPRAY_DIR):
-        print(f"not found kubespray git repo")
+    if not os.path.exists(SPRAY_REPO_PATH):
+        print("kubespray repo path not found")
         sys.exit(1)
     if subprocess.run(["which", "skopeo"]).returncode != 0:
-        print("need skopeo")
+        print("skopeo command not found")
         sys.exit(1)
 
 
@@ -98,7 +100,6 @@ def get_manifest_version(key, manifest_dict):
 
 
 def execute_generate_offline_package(arg_option, arch):
-    script_name = "generate_offline_package.sh"
     if not os.path.exists("artifacts/generate_offline_package.sh"):
         print("generate_offline_package.sh not found in artifacts")
         sys.exit(1)
@@ -113,12 +114,9 @@ def execute_generate_offline_package(arg_option, arch):
 
 
 def create_files(file_urls, arch):
-    file_content = "\n".join(file_urls)
-    if ZONE == "CN":
-        file_content = file_content.replace("https://github.com", "https://files.m.daocloud.io/github.com")
     os.chdir(CUR_DIR)
     with open(FILE_LIST_TEMP_PATH, "w") as f:
-        f.write(file_content)
+        f.write("\n".join(file_urls))
         f.flush()
     execute_generate_offline_package("files", arch)
 
@@ -140,7 +138,13 @@ def create_offlineversion_cr():
     offlineversion_cr_dict = yaml.load(template_file, Loader=yaml.loader.FullLoader)  # dict
     template_file.close()
     offlineversion_cr_dict["spec"]["docker"] = []
-    offlineversion_cr_dict["metadata"]["name"] = f"offlineversion-patch-{int(datetime.now().timestamp())}"
+    offlineversion_cr_dict["metadata"]["labels"] = {}
+    if CR_NAME_POSTFIX != "":
+        offlineversion_cr_dict["metadata"]["name"] = f"localartifactset-{CR_NAME_POSTFIX}"
+        offlineversion_cr_dict["metadata"]["labels"]["sprayRelease"] = SPRAY_RELEASE
+    else:
+        offlineversion_cr_dict["metadata"]["name"] = f"localartifactset-patch-{int(datetime.now().timestamp())}"
+        offlineversion_cr_dict["metadata"]["labels"]["sprayRelease"] = "master"
     items_array = offlineversion_cr_dict["spec"]["items"]
 
     for index in range(len(items_array)):
@@ -161,7 +165,7 @@ def create_offlineversion_cr():
 
     offlineversion_cr_dict["spec"]["items"] = items_array
     kubeanofflineversion_file = open(
-        os.path.join(KUBEAN_TAG, "kubeanofflineversion.cr.patch.yaml"),
+        os.path.join(KUBEAN_TAG, "localartifactset.cr.yaml"),
         "w",
         encoding="utf-8")
     yaml.dump(offlineversion_cr_dict, kubeanofflineversion_file)

artifacts/generate_offline_package.sh

@@ -79,37 +79,73 @@ function generate_temp_list() {
   mv contrib/offline/temp/images.list contrib/offline/temp/images.list.old
   cat contrib/offline/temp/images.list.old | egrep -v ${remove_images} > contrib/offline/temp/images.list
   add_pause_image_addr contrib/offline/temp/images.list
-  cp contrib/offline/temp/*.list $OFFLINE_PACKAGE_DIR
+  cp contrib/offline/temp/*.list ${OFFLINE_PACKAGE_DIR}
+}
+
+function update_binaris_cn_mirror() {
+  local file_list="${CURRENT_DIR}/kubespray/contrib/offline/temp/files.list"
+  # 1. backup files list
+  mv "${file_list}" "${file_list}.bak"
+  # 2. update cn mirror
+  local binary_mirror_addr="files.m.daocloud.io"
+  while read -r binary_addr; do
+    echo "${binary_addr}" | sed "s/https:\/\//&${binary_mirror_addr}\//" >> "${file_list}"
+  done <<< "$(cat "${file_list}.bak" || true)"
+  # 3. clear files list backup file
+  rm -rf "${file_list}.bak"
+}
+
+function update_images_cn_mirror() {
+  local image_list="${CURRENT_DIR}/kubespray/contrib/offline/temp/images.list"
+  # 1. backup images list
+  mv "${image_list}" "${image_list}.bak"
+  # 2. update cn mirror
+  while read -r image_addr; do
+    image_addr=${image_addr/docker.io/docker.m.daocloud.io}
+    image_addr=${image_addr/gcr.io/gcr.m.daocloud.io}
+    image_addr=${image_addr/ghcr.io/ghcr.m.daocloud.io}
+    image_addr=${image_addr/k8s.gcr.io/k8s-gcr.m.daocloud.io}
+    image_addr=${image_addr/registry.k8s.io/k8s.m.daocloud.io}
+    image_addr=${image_addr/quay.io/quay.m.daocloud.io}
+    echo "${image_addr}" >> "${image_list}"
+  done <<< "$(cat "${image_list}.bak" || true)"
+  # 3. clear images list backup file
+  rm -rf "${image_list}.bak"
 }
 
 function create_files() {
   cd $CURRENT_DIR/kubespray/contrib/offline/
+  if [[ "${ZONE}" == "CN" ]]; then
+    update_binaris_cn_mirror
+  fi
+
   NO_HTTP_SERVER=true bash manage-offline-files.sh
   cp offline-files.tar.gz $OFFLINE_FILES_DIR
 }
 
 function create_images() {
   cd $CURRENT_DIR/artifacts
+  if [[ "${ZONE}" == "CN" ]]; then
+    update_images_cn_mirror
+  fi
 
   if which skopeo; then
-    echo "skopeo check successfully"
+    echo "skopeo check successfully."
   else
     echo "please install skopeo first"
     exit 1
   fi
 
-  IMG_LIST=$CURRENT_DIR/kubespray/contrib/offline/temp/images.list
-
-  echo "begin to download images"
-  images_list_content=$(cat "$IMG_LIST")
+  echo "begin to download images."
+  local images_list_content
+  images_list_content=$(cat "${CURRENT_DIR}/kubespray/contrib/offline/temp/images.list")
 
   if [ ! -d "offline-images" ]; then
-    echo "create dir offline-images"
+    echo "create offline-images directory."
     mkdir offline-images
   fi
 
   while read -r image_name; do
-    image_name=$(replace_image_name "$image_name")
     echo "download image $image_name to local"
     ret=0
     skopeo copy --insecure-policy --retry-times=3 --override-os linux --override-arch ${ARCH} "docker://$image_name" "oci:offline-images:$image_name" || ret=$?
@@ -118,30 +154,13 @@ function create_images() {
       exit 1
     fi
     echo "$image_name" >> offline-images/images.list
-  done <<< "$images_list_content"
+  done <<< "${images_list_content}"
 
   tar -czvf $OFFLINE_IMAGES_DIR/offline-images.tar.gz offline-images
 
   echo "zipping images completed!"
 }
 
-function replace_image_name() {
-  local origin_address=$1
-
-  if [ "$ZONE" != "CN" ]; then
-    echo "$origin_address" 
-    return
-  fi
-
-  origin_address=${origin_address/docker.io/docker.m.daocloud.io}
-  origin_address=${origin_address/gcr.io/gcr.m.daocloud.io} 
-  origin_address=${origin_address/ghcr.io/ghcr.m.daocloud.io}
-  origin_address=${origin_address/k8s.gcr.io/k8s-gcr.m.daocloud.io}
-  origin_address=${origin_address/registry.k8s.io/k8s.m.daocloud.io}
-  origin_address=${origin_address/quay.io/quay.m.daocloud.io}
-  echo "$origin_address"
-}
-
 function copy_import_sh() {
     cp $CURRENT_DIR/artifacts/import_files.sh $OFFLINE_FILES_DIR
     cp $CURRENT_DIR/artifacts/import_images.sh $OFFLINE_IMAGES_DIR

build/images/airgap-patch/Dockerfile

@@ -1,29 +1,33 @@
 ARG SPRAY_TAG=master
 ARG REPO=kubean-io
-
+ARG CR_NAME_POSTFIX
 FROM ghcr.io/${REPO}/kubespray:${SPRAY_TAG}
 
 WORKDIR /kubean
 
 COPY artifacts /kubean/artifacts
-
 COPY charts /kubean/charts
 
+ARG SKOPEO_VERSION="v1.11.2"
+ARG YQ_VERSION="v4.33.3"
 RUN ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" \
-    && apk add --update --no-cache wget curl \
-    && wget https://github.com/lework/skopeo-binary/releases/download/v1.9.3/skopeo-linux-$ARCH \
-    && chmod +x skopeo-linux-$ARCH \
-    && mv skopeo-linux-$ARCH /usr/bin/skopeo \
-    && wget https://github.com/mikefarah/yq/releases/download/v4.30.8/yq_linux_$ARCH \
-    && chmod +x yq_linux_$ARCH \
-    && mv yq_linux_$ARCH /usr/bin/yq \
+    && apk add --update --no-cache curl wget \
+    && echo "install skopeo" \
+    && wget -O /usr/bin/skopeo https://github.com/lework/skopeo-binary/releases/download/${SKOPEO_VERSION}/skopeo-linux-${ARCH} \
+    && chmod +x /usr/bin/skopeo \
+    && echo "install yq" \
+    && wget -O /usr/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH} \
+    && chmod +x /usr/bin/yq \
     && ln -s /kubespray /kubean/kubespray
 
+ENV CR_NAME_POSTFIX=$CR_NAME_POSTFIX
+ENV SPRAY_RELEASE=${CR_NAME_POSTFIX%-*}
+
 CMD echo "show supporting versions" \
     && bash artifacts/manage_version_cr.sh create_manifest \
-    && echo $(yq -o=json  '.spec.components' charts/kubean/templates/manifest.cr.yaml) \
+    && echo $(yq -o=json '.spec.components' charts/kubean/templates/manifest.cr.yaml) \
     && echo "starting generating airgap patch files" \
-    && MANIFEST_CONF=/manifest.yml python3 artifacts/offline_patch.py \
+    && MANIFEST_CONF=/manifest.yml python3 artifacts/airgap_patch.py \
     && echo "copying data" \
     && mv v_* /data \
     && chmod -R 777 /data/ \

docs/en/usage/airgap_patch_usage.md

@@ -1,6 +1,6 @@
 # Generation and use of incremental offline packages
 
-To meet users' needs for components of certain versions, Kubean provides the script `artifacts/offline_patch.py` to generate a corresponding version of offline packages based on the configuration file `manifest.yml`.
+To meet users' needs for components of certain versions, Kubean provides the script `artifacts/airgap_patch.py` to generate a corresponding version of offline packages based on the configuration file `manifest.yml`.
 
 ## Generate an incremental offline package
 

docs/zh/usage/airgap_patch_usage.md

@@ -1,6 +1,6 @@
 # 增量离线包的生成和使用
 
-为了满足用户对于某些软件特定版本的需要，Kubean 提供脚本 `artifacts/offline_patch.py` 根据配置文件 `manifest.yml`
+为了满足用户对于某些软件特定版本的需要，Kubean 提供脚本 `artifacts/airgap_patch.py` 根据配置文件 `manifest.yml`
 来生成对应版本的离线包。
 
 ## 生成增量离线包