From adc3b8c5d2dff35716983228213a5de1a4964828 Mon Sep 17 00:00:00 2001 From: ErikJiang Date: Fri, 8 Sep 2023 11:15:38 +0000 Subject: [PATCH 1/3] See https://github.com/ErikJiang/kubean/commit/4e44908821b63272c8d5faf8394ab09d0fde8aa7 from refs/tags/v0.0.16 --- docs/overrides/releases/v0.0.16.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/overrides/releases/v0.0.16.md diff --git a/docs/overrides/releases/v0.0.16.md b/docs/overrides/releases/v0.0.16.md new file mode 100644 index 000000000..cda28b1da --- /dev/null +++ b/docs/overrides/releases/v0.0.16.md @@ -0,0 +1,5 @@ + + + + +**Full Changelog**: https://github.com/ErikJiang/kubean/compare/v0.0.6...v0.0.9 From 0d7c35108b1b3d93f5d6b500a39c19144708cb24 Mon Sep 17 00:00:00 2001 From: ErikJiang Date: Mon, 11 Sep 2023 03:10:36 +0000 Subject: [PATCH 2/3] See https://github.com/ErikJiang/kubean/commit/f4d8a10c30b2aff8d73ffb3d8bd619e04ceec07c from refs/tags/v0.0.17 --- docs/overrides/releases/v0.0.17.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/overrides/releases/v0.0.17.md diff --git a/docs/overrides/releases/v0.0.17.md b/docs/overrides/releases/v0.0.17.md new file mode 100644 index 000000000..0f0ceb1ea --- /dev/null +++ b/docs/overrides/releases/v0.0.17.md @@ -0,0 +1,5 @@ + + + + +**Full Changelog**: https://github.com/ErikJiang/kubean/compare/v0.0.6...v0.0.17 From fceb218cfeeb58867ebcd47f9bf178f960f9d755 Mon Sep 17 00:00:00 2001 From: "bo.jiang" Date: Tue, 14 Nov 2023 17:36:09 +0800 Subject: [PATCH 3/3] update airgap patch --- .../workflows/manual-kubespray-release.yaml | 4 +- .../{offline_patch.py => airgap_patch.py} | 32 +++++---- artifacts/generate_offline_package.sh | 71 ++++++++++++------- build/images/airgap-patch/Dockerfile | 26 ++++--- docs/en/usage/airgap_patch_usage.md | 2 +- docs/overrides/releases/v0.0.16.md | 5 -- docs/overrides/releases/v0.0.17.md | 5 -- docs/zh/usage/airgap_patch_usage.md | 2 +- 8 files changed, 83 insertions(+), 64 deletions(-) rename artifacts/{offline_patch.py => airgap_patch.py} (90%) delete mode 100644 docs/overrides/releases/v0.0.16.md delete mode 100644 docs/overrides/releases/v0.0.17.md diff --git a/.github/workflows/manual-kubespray-release.yaml b/.github/workflows/manual-kubespray-release.yaml index 10d0de9a7..b6dd96510 100644 --- a/.github/workflows/manual-kubespray-release.yaml +++ b/.github/workflows/manual-kubespray-release.yaml @@ -95,6 +95,7 @@ jobs: build-args: | SPRAY_TAG=${{ needs.check-inputs.outputs.kubespray_sha1_short }} REPO=${{ needs.check-inputs.outputs.image_repo }} + CR_NAME_POSTFIX: ${{ needs.check-inputs.outputs.tag }} github-token: ${{ secrets.GITHUB_TOKEN }} push: true provenance: false @@ -126,7 +127,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: sprayjob image build + - name: airgap-patch image build uses: docker/build-push-action@v4.1.1 with: context: ./ @@ -134,6 +135,7 @@ jobs: build-args: | SPRAY_TAG=${{ needs.check-inputs.outputs.kubespray_sha1_short }} REPO=${{ needs.check-inputs.outputs.image_repo }} + CR_NAME_POSTFIX: ${{ needs.check-inputs.outputs.tag }} github-token: ${{ secrets.GITHUB_TOKEN }} push: true provenance: false diff --git a/artifacts/offline_patch.py b/artifacts/airgap_patch.py similarity index 90% rename from artifacts/offline_patch.py rename to artifacts/airgap_patch.py index 565af4260..19e26e15f 100644 --- a/artifacts/offline_patch.py +++ b/artifacts/airgap_patch.py @@ -14,7 +14,7 @@ OPTION = os.getenv("OPTION", default="all") # create_files create_images create_offlineversion_cr -KUBESPRAY_DIR = os.path.join(CUR_DIR, "kubespray") +SPRAY_REPO_PATH = os.path.join(CUR_DIR, "kubespray") MANIFEST_YML_FILE = os.getenv("MANIFEST_CONF", default="manifest.yml") ZONE = os.getenv("ZONE", default="Other") ## Other or CN @@ -23,9 +23,11 @@ OFFLINE_VER_CR_TEMP = os.getenv("OFFLINEVERSION_CR_TEMPLATE", default=os.path.join(CUR_DIR, "artifacts/template/localartifactset.template.yml")) +SPRAY_RELEASE = os.getenv("SPRAY_RELEASE") +CR_NAME_POSTFIX = os.getenv("CR_NAME_POSTFIX") -FILE_LIST_TEMP_PATH = os.path.join(KUBESPRAY_DIR, "contrib/offline/temp/files.list") -IMAGE_LIST_TEMP_PATH = os.path.join(KUBESPRAY_DIR, "contrib/offline/temp/images.list") +FILE_LIST_TEMP_PATH = os.path.join(SPRAY_REPO_PATH, "contrib/offline/temp/files.list") +IMAGE_LIST_TEMP_PATH = os.path.join(SPRAY_REPO_PATH, "contrib/offline/temp/images.list") KUBEAN_TAG = "v_offline_patch" print(f"CUR_DIR:{CUR_DIR}") @@ -42,7 +44,7 @@ def extra_line_str_with_pattern(filepath, *patterns): def fetch_info_list(env_dict, *patterns): print(f"generating info list for {env_dict}") - os.chdir(KUBESPRAY_DIR) + os.chdir(SPRAY_REPO_PATH) if os.path.exists("contrib/offline/temp"): shutil.rmtree("contrib/offline/temp") cmd = ["bash", "contrib/offline/generate_list.sh"] @@ -67,11 +69,11 @@ def fetch_info_list(env_dict, *patterns): def check_dependencies(): - if not os.path.exists(KUBESPRAY_DIR): - print(f"not found kubespray git repo") + if not os.path.exists(SPRAY_REPO_PATH): + print("kubespray repo path not found") sys.exit(1) if subprocess.run(["which", "skopeo"]).returncode != 0: - print("need skopeo") + print("skopeo command not found") sys.exit(1) @@ -98,7 +100,6 @@ def get_manifest_version(key, manifest_dict): def execute_generate_offline_package(arg_option, arch): - script_name = "generate_offline_package.sh" if not os.path.exists("artifacts/generate_offline_package.sh"): print("generate_offline_package.sh not found in artifacts") sys.exit(1) @@ -113,12 +114,9 @@ def execute_generate_offline_package(arg_option, arch): def create_files(file_urls, arch): - file_content = "\n".join(file_urls) - if ZONE == "CN": - file_content = file_content.replace("https://github.com", "https://files.m.daocloud.io/github.com") os.chdir(CUR_DIR) with open(FILE_LIST_TEMP_PATH, "w") as f: - f.write(file_content) + f.write("\n".join(file_urls)) f.flush() execute_generate_offline_package("files", arch) @@ -140,7 +138,13 @@ def create_offlineversion_cr(): offlineversion_cr_dict = yaml.load(template_file, Loader=yaml.loader.FullLoader) # dict template_file.close() offlineversion_cr_dict["spec"]["docker"] = [] - offlineversion_cr_dict["metadata"]["name"] = f"offlineversion-patch-{int(datetime.now().timestamp())}" + offlineversion_cr_dict["metadata"]["labels"] = {} + if CR_NAME_POSTFIX != "": + offlineversion_cr_dict["metadata"]["name"] = f"localartifactset-{CR_NAME_POSTFIX}" + offlineversion_cr_dict["metadata"]["labels"]["sprayRelease"] = SPRAY_RELEASE + else: + offlineversion_cr_dict["metadata"]["name"] = f"localartifactset-patch-{int(datetime.now().timestamp())}" + offlineversion_cr_dict["metadata"]["labels"]["sprayRelease"] = "master" items_array = offlineversion_cr_dict["spec"]["items"] for index in range(len(items_array)): @@ -161,7 +165,7 @@ def create_offlineversion_cr(): offlineversion_cr_dict["spec"]["items"] = items_array kubeanofflineversion_file = open( - os.path.join(KUBEAN_TAG, "kubeanofflineversion.cr.patch.yaml"), + os.path.join(KUBEAN_TAG, "localartifactset.cr.yaml"), "w", encoding="utf-8") yaml.dump(offlineversion_cr_dict, kubeanofflineversion_file) diff --git a/artifacts/generate_offline_package.sh b/artifacts/generate_offline_package.sh index a5dbbb964..7dac49201 100644 --- a/artifacts/generate_offline_package.sh +++ b/artifacts/generate_offline_package.sh @@ -79,37 +79,73 @@ function generate_temp_list() { mv contrib/offline/temp/images.list contrib/offline/temp/images.list.old cat contrib/offline/temp/images.list.old | egrep -v ${remove_images} > contrib/offline/temp/images.list add_pause_image_addr contrib/offline/temp/images.list - cp contrib/offline/temp/*.list $OFFLINE_PACKAGE_DIR + cp contrib/offline/temp/*.list ${OFFLINE_PACKAGE_DIR} +} + +function update_binaris_cn_mirror() { + local file_list="${CURRENT_DIR}/kubespray/contrib/offline/temp/files.list" + # 1. backup files list + mv "${file_list}" "${file_list}.bak" + # 2. update cn mirror + local binary_mirror_addr="files.m.daocloud.io" + while read -r binary_addr; do + echo "${binary_addr}" | sed "s/https:\/\//&${binary_mirror_addr}\//" >> "${file_list}" + done <<< "$(cat "${file_list}.bak" || true)" + # 3. clear files list backup file + rm -rf "${file_list}.bak" +} + +function update_images_cn_mirror() { + local image_list="${CURRENT_DIR}/kubespray/contrib/offline/temp/images.list" + # 1. backup images list + mv "${image_list}" "${image_list}.bak" + # 2. update cn mirror + while read -r image_addr; do + image_addr=${image_addr/docker.io/docker.m.daocloud.io} + image_addr=${image_addr/gcr.io/gcr.m.daocloud.io} + image_addr=${image_addr/ghcr.io/ghcr.m.daocloud.io} + image_addr=${image_addr/k8s.gcr.io/k8s-gcr.m.daocloud.io} + image_addr=${image_addr/registry.k8s.io/k8s.m.daocloud.io} + image_addr=${image_addr/quay.io/quay.m.daocloud.io} + echo "${image_addr}" >> "${image_list}" + done <<< "$(cat "${image_list}.bak" || true)" + # 3. clear images list backup file + rm -rf "${image_list}.bak" } function create_files() { cd $CURRENT_DIR/kubespray/contrib/offline/ + if [[ "${ZONE}" == "CN" ]]; then + update_binaris_cn_mirror + fi + NO_HTTP_SERVER=true bash manage-offline-files.sh cp offline-files.tar.gz $OFFLINE_FILES_DIR } function create_images() { cd $CURRENT_DIR/artifacts + if [[ "${ZONE}" == "CN" ]]; then + update_images_cn_mirror + fi if which skopeo; then - echo "skopeo check successfully" + echo "skopeo check successfully." else echo "please install skopeo first" exit 1 fi - IMG_LIST=$CURRENT_DIR/kubespray/contrib/offline/temp/images.list - - echo "begin to download images" - images_list_content=$(cat "$IMG_LIST") + echo "begin to download images." + local images_list_content + images_list_content=$(cat "${CURRENT_DIR}/kubespray/contrib/offline/temp/images.list") if [ ! -d "offline-images" ]; then - echo "create dir offline-images" + echo "create offline-images directory." mkdir offline-images fi while read -r image_name; do - image_name=$(replace_image_name "$image_name") echo "download image $image_name to local" ret=0 skopeo copy --insecure-policy --retry-times=3 --override-os linux --override-arch ${ARCH} "docker://$image_name" "oci:offline-images:$image_name" || ret=$? @@ -118,30 +154,13 @@ function create_images() { exit 1 fi echo "$image_name" >> offline-images/images.list - done <<< "$images_list_content" + done <<< "${images_list_content}" tar -czvf $OFFLINE_IMAGES_DIR/offline-images.tar.gz offline-images echo "zipping images completed!" } -function replace_image_name() { - local origin_address=$1 - - if [ "$ZONE" != "CN" ]; then - echo "$origin_address" - return - fi - - origin_address=${origin_address/docker.io/docker.m.daocloud.io} - origin_address=${origin_address/gcr.io/gcr.m.daocloud.io} - origin_address=${origin_address/ghcr.io/ghcr.m.daocloud.io} - origin_address=${origin_address/k8s.gcr.io/k8s-gcr.m.daocloud.io} - origin_address=${origin_address/registry.k8s.io/k8s.m.daocloud.io} - origin_address=${origin_address/quay.io/quay.m.daocloud.io} - echo "$origin_address" -} - function copy_import_sh() { cp $CURRENT_DIR/artifacts/import_files.sh $OFFLINE_FILES_DIR cp $CURRENT_DIR/artifacts/import_images.sh $OFFLINE_IMAGES_DIR diff --git a/build/images/airgap-patch/Dockerfile b/build/images/airgap-patch/Dockerfile index ee1f19c27..86e453cad 100644 --- a/build/images/airgap-patch/Dockerfile +++ b/build/images/airgap-patch/Dockerfile @@ -1,29 +1,33 @@ ARG SPRAY_TAG=master ARG REPO=kubean-io - +ARG CR_NAME_POSTFIX FROM ghcr.io/${REPO}/kubespray:${SPRAY_TAG} WORKDIR /kubean COPY artifacts /kubean/artifacts - COPY charts /kubean/charts +ARG SKOPEO_VERSION="v1.11.2" +ARG YQ_VERSION="v4.33.3" RUN ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" \ - && apk add --update --no-cache wget curl \ - && wget https://github.com/lework/skopeo-binary/releases/download/v1.9.3/skopeo-linux-$ARCH \ - && chmod +x skopeo-linux-$ARCH \ - && mv skopeo-linux-$ARCH /usr/bin/skopeo \ - && wget https://github.com/mikefarah/yq/releases/download/v4.30.8/yq_linux_$ARCH \ - && chmod +x yq_linux_$ARCH \ - && mv yq_linux_$ARCH /usr/bin/yq \ + && apk add --update --no-cache curl wget \ + && echo "install skopeo" \ + && wget -O /usr/bin/skopeo https://github.com/lework/skopeo-binary/releases/download/${SKOPEO_VERSION}/skopeo-linux-${ARCH} \ + && chmod +x /usr/bin/skopeo \ + && echo "install yq" \ + && wget -O /usr/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH} \ + && chmod +x /usr/bin/yq \ && ln -s /kubespray /kubean/kubespray +ENV CR_NAME_POSTFIX=$CR_NAME_POSTFIX +ENV SPRAY_RELEASE=${CR_NAME_POSTFIX%-*} + CMD echo "show supporting versions" \ && bash artifacts/manage_version_cr.sh create_manifest \ - && echo $(yq -o=json '.spec.components' charts/kubean/templates/manifest.cr.yaml) \ + && echo $(yq -o=json '.spec.components' charts/kubean/templates/manifest.cr.yaml) \ && echo "starting generating airgap patch files" \ - && MANIFEST_CONF=/manifest.yml python3 artifacts/offline_patch.py \ + && MANIFEST_CONF=/manifest.yml python3 artifacts/airgap_patch.py \ && echo "copying data" \ && mv v_* /data \ && chmod -R 777 /data/ \ diff --git a/docs/en/usage/airgap_patch_usage.md b/docs/en/usage/airgap_patch_usage.md index 5022a4a0b..02262b852 100644 --- a/docs/en/usage/airgap_patch_usage.md +++ b/docs/en/usage/airgap_patch_usage.md @@ -1,6 +1,6 @@ # Generation and use of incremental offline packages -To meet users' needs for components of certain versions, Kubean provides the script `artifacts/offline_patch.py` to generate a corresponding version of offline packages based on the configuration file `manifest.yml`. +To meet users' needs for components of certain versions, Kubean provides the script `artifacts/airgap_patch.py` to generate a corresponding version of offline packages based on the configuration file `manifest.yml`. ## Generate an incremental offline package diff --git a/docs/overrides/releases/v0.0.16.md b/docs/overrides/releases/v0.0.16.md deleted file mode 100644 index cda28b1da..000000000 --- a/docs/overrides/releases/v0.0.16.md +++ /dev/null @@ -1,5 +0,0 @@ - - - - -**Full Changelog**: https://github.com/ErikJiang/kubean/compare/v0.0.6...v0.0.9 diff --git a/docs/overrides/releases/v0.0.17.md b/docs/overrides/releases/v0.0.17.md deleted file mode 100644 index 0f0ceb1ea..000000000 --- a/docs/overrides/releases/v0.0.17.md +++ /dev/null @@ -1,5 +0,0 @@ - - - - -**Full Changelog**: https://github.com/ErikJiang/kubean/compare/v0.0.6...v0.0.17 diff --git a/docs/zh/usage/airgap_patch_usage.md b/docs/zh/usage/airgap_patch_usage.md index 805b998d7..dc66acca1 100644 --- a/docs/zh/usage/airgap_patch_usage.md +++ b/docs/zh/usage/airgap_patch_usage.md @@ -1,6 +1,6 @@ # 增量离线包的生成和使用 -为了满足用户对于某些软件特定版本的需要，Kubean 提供脚本 `artifacts/offline_patch.py` 根据配置文件 `manifest.yml` +为了满足用户对于某些软件特定版本的需要，Kubean 提供脚本 `artifacts/airgap_patch.py` 根据配置文件 `manifest.yml` 来生成对应版本的离线包。 ## 生成增量离线包