diff --git a/artifacts/offline_patch.py b/artifacts/airgap_patch.py similarity index 94% rename from artifacts/offline_patch.py rename to artifacts/airgap_patch.py index 565af4260..be93bbc2f 100644 --- a/artifacts/offline_patch.py +++ b/artifacts/airgap_patch.py @@ -14,7 +14,7 @@ OPTION = os.getenv("OPTION", default="all") # create_files create_images create_offlineversion_cr -KUBESPRAY_DIR = os.path.join(CUR_DIR, "kubespray") +SPRAY_REPO_PATH = os.path.join(CUR_DIR, "kubespray") MANIFEST_YML_FILE = os.getenv("MANIFEST_CONF", default="manifest.yml") ZONE = os.getenv("ZONE", default="Other") ## Other or CN @@ -24,8 +24,8 @@ default=os.path.join(CUR_DIR, "artifacts/template/localartifactset.template.yml")) -FILE_LIST_TEMP_PATH = os.path.join(KUBESPRAY_DIR, "contrib/offline/temp/files.list") -IMAGE_LIST_TEMP_PATH = os.path.join(KUBESPRAY_DIR, "contrib/offline/temp/images.list") +FILE_LIST_TEMP_PATH = os.path.join(SPRAY_REPO_PATH, "contrib/offline/temp/files.list") +IMAGE_LIST_TEMP_PATH = os.path.join(SPRAY_REPO_PATH, "contrib/offline/temp/images.list") KUBEAN_TAG = "v_offline_patch" print(f"CUR_DIR:{CUR_DIR}") @@ -42,7 +42,7 @@ def extra_line_str_with_pattern(filepath, *patterns): def fetch_info_list(env_dict, *patterns): print(f"generating info list for {env_dict}") - os.chdir(KUBESPRAY_DIR) + os.chdir(SPRAY_REPO_PATH) if os.path.exists("contrib/offline/temp"): shutil.rmtree("contrib/offline/temp") cmd = ["bash", "contrib/offline/generate_list.sh"] @@ -67,11 +67,11 @@ def fetch_info_list(env_dict, *patterns): def check_dependencies(): - if not os.path.exists(KUBESPRAY_DIR): - print(f"not found kubespray git repo") + if not os.path.exists(SPRAY_REPO_PATH): + print("kubespray repo path not found") sys.exit(1) if subprocess.run(["which", "skopeo"]).returncode != 0: - print("need skopeo") + print("skopeo command not found") sys.exit(1) @@ -98,7 +98,6 @@ def get_manifest_version(key, manifest_dict): def execute_generate_offline_package(arg_option, arch): - script_name = "generate_offline_package.sh" if not os.path.exists("artifacts/generate_offline_package.sh"): print("generate_offline_package.sh not found in artifacts") sys.exit(1) @@ -113,12 +112,9 @@ def execute_generate_offline_package(arg_option, arch): def create_files(file_urls, arch): - file_content = "\n".join(file_urls) - if ZONE == "CN": - file_content = file_content.replace("https://github.com", "https://files.m.daocloud.io/github.com") os.chdir(CUR_DIR) with open(FILE_LIST_TEMP_PATH, "w") as f: - f.write(file_content) + f.write(file_urls) f.flush() execute_generate_offline_package("files", arch) diff --git a/artifacts/generate_offline_package.sh b/artifacts/generate_offline_package.sh index a5dbbb964..7dac49201 100644 --- a/artifacts/generate_offline_package.sh +++ b/artifacts/generate_offline_package.sh @@ -79,37 +79,73 @@ function generate_temp_list() { mv contrib/offline/temp/images.list contrib/offline/temp/images.list.old cat contrib/offline/temp/images.list.old | egrep -v ${remove_images} > contrib/offline/temp/images.list add_pause_image_addr contrib/offline/temp/images.list - cp contrib/offline/temp/*.list $OFFLINE_PACKAGE_DIR + cp contrib/offline/temp/*.list ${OFFLINE_PACKAGE_DIR} +} + +function update_binaris_cn_mirror() { + local file_list="${CURRENT_DIR}/kubespray/contrib/offline/temp/files.list" + # 1. backup files list + mv "${file_list}" "${file_list}.bak" + # 2. update cn mirror + local binary_mirror_addr="files.m.daocloud.io" + while read -r binary_addr; do + echo "${binary_addr}" | sed "s/https:\/\//&${binary_mirror_addr}\//" >> "${file_list}" + done <<< "$(cat "${file_list}.bak" || true)" + # 3. clear files list backup file + rm -rf "${file_list}.bak" +} + +function update_images_cn_mirror() { + local image_list="${CURRENT_DIR}/kubespray/contrib/offline/temp/images.list" + # 1. backup images list + mv "${image_list}" "${image_list}.bak" + # 2. update cn mirror + while read -r image_addr; do + image_addr=${image_addr/docker.io/docker.m.daocloud.io} + image_addr=${image_addr/gcr.io/gcr.m.daocloud.io} + image_addr=${image_addr/ghcr.io/ghcr.m.daocloud.io} + image_addr=${image_addr/k8s.gcr.io/k8s-gcr.m.daocloud.io} + image_addr=${image_addr/registry.k8s.io/k8s.m.daocloud.io} + image_addr=${image_addr/quay.io/quay.m.daocloud.io} + echo "${image_addr}" >> "${image_list}" + done <<< "$(cat "${image_list}.bak" || true)" + # 3. clear images list backup file + rm -rf "${image_list}.bak" } function create_files() { cd $CURRENT_DIR/kubespray/contrib/offline/ + if [[ "${ZONE}" == "CN" ]]; then + update_binaris_cn_mirror + fi + NO_HTTP_SERVER=true bash manage-offline-files.sh cp offline-files.tar.gz $OFFLINE_FILES_DIR } function create_images() { cd $CURRENT_DIR/artifacts + if [[ "${ZONE}" == "CN" ]]; then + update_images_cn_mirror + fi if which skopeo; then - echo "skopeo check successfully" + echo "skopeo check successfully." else echo "please install skopeo first" exit 1 fi - IMG_LIST=$CURRENT_DIR/kubespray/contrib/offline/temp/images.list - - echo "begin to download images" - images_list_content=$(cat "$IMG_LIST") + echo "begin to download images." + local images_list_content + images_list_content=$(cat "${CURRENT_DIR}/kubespray/contrib/offline/temp/images.list") if [ ! -d "offline-images" ]; then - echo "create dir offline-images" + echo "create offline-images directory." mkdir offline-images fi while read -r image_name; do - image_name=$(replace_image_name "$image_name") echo "download image $image_name to local" ret=0 skopeo copy --insecure-policy --retry-times=3 --override-os linux --override-arch ${ARCH} "docker://$image_name" "oci:offline-images:$image_name" || ret=$? @@ -118,30 +154,13 @@ function create_images() { exit 1 fi echo "$image_name" >> offline-images/images.list - done <<< "$images_list_content" + done <<< "${images_list_content}" tar -czvf $OFFLINE_IMAGES_DIR/offline-images.tar.gz offline-images echo "zipping images completed!" } -function replace_image_name() { - local origin_address=$1 - - if [ "$ZONE" != "CN" ]; then - echo "$origin_address" - return - fi - - origin_address=${origin_address/docker.io/docker.m.daocloud.io} - origin_address=${origin_address/gcr.io/gcr.m.daocloud.io} - origin_address=${origin_address/ghcr.io/ghcr.m.daocloud.io} - origin_address=${origin_address/k8s.gcr.io/k8s-gcr.m.daocloud.io} - origin_address=${origin_address/registry.k8s.io/k8s.m.daocloud.io} - origin_address=${origin_address/quay.io/quay.m.daocloud.io} - echo "$origin_address" -} - function copy_import_sh() { cp $CURRENT_DIR/artifacts/import_files.sh $OFFLINE_FILES_DIR cp $CURRENT_DIR/artifacts/import_images.sh $OFFLINE_IMAGES_DIR diff --git a/build/images/airgap-patch/Dockerfile b/build/images/airgap-patch/Dockerfile index ee1f19c27..4dac6be54 100644 --- a/build/images/airgap-patch/Dockerfile +++ b/build/images/airgap-patch/Dockerfile @@ -1,5 +1,7 @@ ARG SPRAY_TAG=master ARG REPO=kubean-io +ARG SKOPEO_VERSION=v1.13.3 +ARG YQ_VERSION=v4.33.3 FROM ghcr.io/${REPO}/kubespray:${SPRAY_TAG} @@ -10,20 +12,20 @@ COPY artifacts /kubean/artifacts COPY charts /kubean/charts RUN ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" \ - && apk add --update --no-cache wget curl \ - && wget https://github.com/lework/skopeo-binary/releases/download/v1.9.3/skopeo-linux-$ARCH \ - && chmod +x skopeo-linux-$ARCH \ - && mv skopeo-linux-$ARCH /usr/bin/skopeo \ - && wget https://github.com/mikefarah/yq/releases/download/v4.30.8/yq_linux_$ARCH \ - && chmod +x yq_linux_$ARCH \ - && mv yq_linux_$ARCH /usr/bin/yq \ + && apk add --update --no-cache curl \ + && echo "install skopeo" \ + && curl -Lo /usr/bin/skopeo https://github.com/lework/skopeo-binary/releases/download/${SKOPEO_VERSION}/skopeo-linux-${ARCH} \ + && chmod +x /usr/bin/skopeo \ + && echo "install yq" \ + && curl -Lo /usr/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH} \ + && chmod +x /usr/bin/yq \ && ln -s /kubespray /kubean/kubespray CMD echo "show supporting versions" \ && bash artifacts/manage_version_cr.sh create_manifest \ && echo $(yq -o=json '.spec.components' charts/kubean/templates/manifest.cr.yaml) \ && echo "starting generating airgap patch files" \ - && MANIFEST_CONF=/manifest.yml python3 artifacts/offline_patch.py \ + && MANIFEST_CONF=/manifest.yml python3 artifacts/airgap_patch.py \ && echo "copying data" \ && mv v_* /data \ && chmod -R 777 /data/ \ diff --git a/docs/en/usage/airgap_patch_usage.md b/docs/en/usage/airgap_patch_usage.md index 5022a4a0b..02262b852 100644 --- a/docs/en/usage/airgap_patch_usage.md +++ b/docs/en/usage/airgap_patch_usage.md @@ -1,6 +1,6 @@ # Generation and use of incremental offline packages -To meet users' needs for components of certain versions, Kubean provides the script `artifacts/offline_patch.py` to generate a corresponding version of offline packages based on the configuration file `manifest.yml`. +To meet users' needs for components of certain versions, Kubean provides the script `artifacts/airgap_patch.py` to generate a corresponding version of offline packages based on the configuration file `manifest.yml`. ## Generate an incremental offline package diff --git a/docs/overrides/releases/v0.0.16.md b/docs/overrides/releases/v0.0.16.md deleted file mode 100644 index cda28b1da..000000000 --- a/docs/overrides/releases/v0.0.16.md +++ /dev/null @@ -1,5 +0,0 @@ - - - - -**Full Changelog**: https://github.com/ErikJiang/kubean/compare/v0.0.6...v0.0.9 diff --git a/docs/overrides/releases/v0.0.17.md b/docs/overrides/releases/v0.0.17.md deleted file mode 100644 index 0f0ceb1ea..000000000 --- a/docs/overrides/releases/v0.0.17.md +++ /dev/null @@ -1,5 +0,0 @@ - - - - -**Full Changelog**: https://github.com/ErikJiang/kubean/compare/v0.0.6...v0.0.17 diff --git a/docs/zh/usage/airgap_patch_usage.md b/docs/zh/usage/airgap_patch_usage.md index 805b998d7..dc66acca1 100644 --- a/docs/zh/usage/airgap_patch_usage.md +++ b/docs/zh/usage/airgap_patch_usage.md @@ -1,6 +1,6 @@ # 增量离线包的生成和使用 -为了满足用户对于某些软件特定版本的需要,Kubean 提供脚本 `artifacts/offline_patch.py` 根据配置文件 `manifest.yml` +为了满足用户对于某些软件特定版本的需要,Kubean 提供脚本 `artifacts/airgap_patch.py` 根据配置文件 `manifest.yml` 来生成对应版本的离线包。 ## 生成增量离线包