Skip to content

Commit

Permalink
update airgap patch
Browse files Browse the repository at this point in the history
  • Loading branch information
ErikJiang committed Nov 14, 2023
1 parent adaf40e commit 249329f
Show file tree
Hide file tree
Showing 7 changed files with 65 additions and 58 deletions.
20 changes: 8 additions & 12 deletions artifacts/offline_patch.py → artifacts/airgap_patch.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

OPTION = os.getenv("OPTION", default="all") # create_files create_images create_offlineversion_cr

KUBESPRAY_DIR = os.path.join(CUR_DIR, "kubespray")
SPRAY_REPO_PATH = os.path.join(CUR_DIR, "kubespray")
MANIFEST_YML_FILE = os.getenv("MANIFEST_CONF", default="manifest.yml")
ZONE = os.getenv("ZONE", default="Other") ## Other or CN

Expand All @@ -24,8 +24,8 @@
default=os.path.join(CUR_DIR,
"artifacts/template/localartifactset.template.yml"))

FILE_LIST_TEMP_PATH = os.path.join(KUBESPRAY_DIR, "contrib/offline/temp/files.list")
IMAGE_LIST_TEMP_PATH = os.path.join(KUBESPRAY_DIR, "contrib/offline/temp/images.list")
FILE_LIST_TEMP_PATH = os.path.join(SPRAY_REPO_PATH, "contrib/offline/temp/files.list")
IMAGE_LIST_TEMP_PATH = os.path.join(SPRAY_REPO_PATH, "contrib/offline/temp/images.list")
KUBEAN_TAG = "v_offline_patch"
print(f"CUR_DIR:{CUR_DIR}")

Expand All @@ -42,7 +42,7 @@ def extra_line_str_with_pattern(filepath, *patterns):

def fetch_info_list(env_dict, *patterns):
print(f"generating info list for {env_dict}")
os.chdir(KUBESPRAY_DIR)
os.chdir(SPRAY_REPO_PATH)
if os.path.exists("contrib/offline/temp"):
shutil.rmtree("contrib/offline/temp")
cmd = ["bash", "contrib/offline/generate_list.sh"]
Expand All @@ -67,11 +67,11 @@ def fetch_info_list(env_dict, *patterns):


def check_dependencies():
if not os.path.exists(KUBESPRAY_DIR):
print(f"not found kubespray git repo")
if not os.path.exists(SPRAY_REPO_PATH):
print("kubespray repo path not found")
sys.exit(1)
if subprocess.run(["which", "skopeo"]).returncode != 0:
print("need skopeo")
print("skopeo command not found")
sys.exit(1)


Expand All @@ -98,7 +98,6 @@ def get_manifest_version(key, manifest_dict):


def execute_generate_offline_package(arg_option, arch):
script_name = "generate_offline_package.sh"
if not os.path.exists("artifacts/generate_offline_package.sh"):
print("generate_offline_package.sh not found in artifacts")
sys.exit(1)
Expand All @@ -113,12 +112,9 @@ def execute_generate_offline_package(arg_option, arch):


def create_files(file_urls, arch):
file_content = "\n".join(file_urls)
if ZONE == "CN":
file_content = file_content.replace("https://github.com", "https://files.m.daocloud.io/github.com")
os.chdir(CUR_DIR)
with open(FILE_LIST_TEMP_PATH, "w") as f:
f.write(file_content)
f.write(file_urls)
f.flush()
execute_generate_offline_package("files", arch)

Expand Down
71 changes: 45 additions & 26 deletions artifacts/generate_offline_package.sh
Original file line number Diff line number Diff line change
Expand Up @@ -79,37 +79,73 @@ function generate_temp_list() {
mv contrib/offline/temp/images.list contrib/offline/temp/images.list.old
cat contrib/offline/temp/images.list.old | egrep -v ${remove_images} > contrib/offline/temp/images.list
add_pause_image_addr contrib/offline/temp/images.list
cp contrib/offline/temp/*.list $OFFLINE_PACKAGE_DIR
cp contrib/offline/temp/*.list ${OFFLINE_PACKAGE_DIR}
}

function update_binaris_cn_mirror() {
local file_list="${CURRENT_DIR}/kubespray/contrib/offline/temp/files.list"
# 1. backup files list
mv "${file_list}" "${file_list}.bak"
# 2. update cn mirror
local binary_mirror_addr="files.m.daocloud.io"
while read -r binary_addr; do
echo "${binary_addr}" | sed "s/https:\/\//&${binary_mirror_addr}\//" >> "${file_list}"
done <<< "$(cat "${file_list}.bak" || true)"
# 3. clear files list backup file
rm -rf "${file_list}.bak"
}

function update_images_cn_mirror() {
local image_list="${CURRENT_DIR}/kubespray/contrib/offline/temp/images.list"
# 1. backup images list
mv "${image_list}" "${image_list}.bak"
# 2. update cn mirror
while read -r image_addr; do
image_addr=${image_addr/docker.io/docker.m.daocloud.io}
image_addr=${image_addr/gcr.io/gcr.m.daocloud.io}
image_addr=${image_addr/ghcr.io/ghcr.m.daocloud.io}
image_addr=${image_addr/k8s.gcr.io/k8s-gcr.m.daocloud.io}
image_addr=${image_addr/registry.k8s.io/k8s.m.daocloud.io}
image_addr=${image_addr/quay.io/quay.m.daocloud.io}
echo "${image_addr}" >> "${image_list}"
done <<< "$(cat "${image_list}.bak" || true)"
# 3. clear images list backup file
rm -rf "${image_list}.bak"
}

function create_files() {
cd $CURRENT_DIR/kubespray/contrib/offline/
if [[ "${ZONE}" == "CN" ]]; then
update_binaris_cn_mirror
fi

NO_HTTP_SERVER=true bash manage-offline-files.sh
cp offline-files.tar.gz $OFFLINE_FILES_DIR
}

function create_images() {
cd $CURRENT_DIR/artifacts
if [[ "${ZONE}" == "CN" ]]; then
update_images_cn_mirror
fi

if which skopeo; then
echo "skopeo check successfully"
echo "skopeo check successfully."
else
echo "please install skopeo first"
exit 1
fi

IMG_LIST=$CURRENT_DIR/kubespray/contrib/offline/temp/images.list

echo "begin to download images"
images_list_content=$(cat "$IMG_LIST")
echo "begin to download images."
local images_list_content
images_list_content=$(cat "${CURRENT_DIR}/kubespray/contrib/offline/temp/images.list")

if [ ! -d "offline-images" ]; then
echo "create dir offline-images"
echo "create offline-images directory."
mkdir offline-images
fi

while read -r image_name; do
image_name=$(replace_image_name "$image_name")
echo "download image $image_name to local"
ret=0
skopeo copy --insecure-policy --retry-times=3 --override-os linux --override-arch ${ARCH} "docker://$image_name" "oci:offline-images:$image_name" || ret=$?
Expand All @@ -118,30 +154,13 @@ function create_images() {
exit 1
fi
echo "$image_name" >> offline-images/images.list
done <<< "$images_list_content"
done <<< "${images_list_content}"

tar -czvf $OFFLINE_IMAGES_DIR/offline-images.tar.gz offline-images

echo "zipping images completed!"
}

function replace_image_name() {
local origin_address=$1

if [ "$ZONE" != "CN" ]; then
echo "$origin_address"
return
fi

origin_address=${origin_address/docker.io/docker.m.daocloud.io}
origin_address=${origin_address/gcr.io/gcr.m.daocloud.io}
origin_address=${origin_address/ghcr.io/ghcr.m.daocloud.io}
origin_address=${origin_address/k8s.gcr.io/k8s-gcr.m.daocloud.io}
origin_address=${origin_address/registry.k8s.io/k8s.m.daocloud.io}
origin_address=${origin_address/quay.io/quay.m.daocloud.io}
echo "$origin_address"
}

function copy_import_sh() {
cp $CURRENT_DIR/artifacts/import_files.sh $OFFLINE_FILES_DIR
cp $CURRENT_DIR/artifacts/import_images.sh $OFFLINE_IMAGES_DIR
Expand Down
18 changes: 10 additions & 8 deletions build/images/airgap-patch/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
ARG SPRAY_TAG=master
ARG REPO=kubean-io
ARG SKOPEO_VERSION=v1.13.3
ARG YQ_VERSION=v4.33.3

FROM ghcr.io/${REPO}/kubespray:${SPRAY_TAG}

Expand All @@ -10,20 +12,20 @@ COPY artifacts /kubean/artifacts
COPY charts /kubean/charts

RUN ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" \
&& apk add --update --no-cache wget curl \
&& wget https://github.com/lework/skopeo-binary/releases/download/v1.9.3/skopeo-linux-$ARCH \
&& chmod +x skopeo-linux-$ARCH \
&& mv skopeo-linux-$ARCH /usr/bin/skopeo \
&& wget https://github.com/mikefarah/yq/releases/download/v4.30.8/yq_linux_$ARCH \
&& chmod +x yq_linux_$ARCH \
&& mv yq_linux_$ARCH /usr/bin/yq \
&& apk add --update --no-cache curl \
&& echo "install skopeo" \
&& curl -Lo /usr/bin/skopeo https://github.com/lework/skopeo-binary/releases/download/${SKOPEO_VERSION}/skopeo-linux-${ARCH} \
&& chmod +x /usr/bin/skopeo \
&& echo "install yq" \
&& curl -Lo /usr/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH} \
&& chmod +x /usr/bin/yq \
&& ln -s /kubespray /kubean/kubespray

CMD echo "show supporting versions" \
&& bash artifacts/manage_version_cr.sh create_manifest \
&& echo $(yq -o=json '.spec.components' charts/kubean/templates/manifest.cr.yaml) \
&& echo "starting generating airgap patch files" \
&& MANIFEST_CONF=/manifest.yml python3 artifacts/offline_patch.py \
&& MANIFEST_CONF=/manifest.yml python3 artifacts/airgap_patch.py \
&& echo "copying data" \
&& mv v_* /data \
&& chmod -R 777 /data/ \
Expand Down
2 changes: 1 addition & 1 deletion docs/en/usage/airgap_patch_usage.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Generation and use of incremental offline packages

To meet users' needs for components of certain versions, Kubean provides the script `artifacts/offline_patch.py` to generate a corresponding version of offline packages based on the configuration file `manifest.yml`.
To meet users' needs for components of certain versions, Kubean provides the script `artifacts/airgap_patch.py` to generate a corresponding version of offline packages based on the configuration file `manifest.yml`.

## Generate an incremental offline package

Expand Down
5 changes: 0 additions & 5 deletions docs/overrides/releases/v0.0.16.md

This file was deleted.

5 changes: 0 additions & 5 deletions docs/overrides/releases/v0.0.17.md

This file was deleted.

2 changes: 1 addition & 1 deletion docs/zh/usage/airgap_patch_usage.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# 增量离线包的生成和使用

为了满足用户对于某些软件特定版本的需要,Kubean 提供脚本 `artifacts/offline_patch.py` 根据配置文件 `manifest.yml`
为了满足用户对于某些软件特定版本的需要,Kubean 提供脚本 `artifacts/airgap_patch.py` 根据配置文件 `manifest.yml`
来生成对应版本的离线包。

## 生成增量离线包
Expand Down

0 comments on commit 249329f

Please sign in to comment.