diff --git a/.gitignore b/.gitignore
index 46ad4f505..dff224e54 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,7 +33,6 @@ controller/**/log
 !controller/pkg/registry/log
 controller/tests/testdata/crd/
 go.work.sum
-external/*
 e2e/log
 .idea
 # helm charts
diff --git a/.licenserc.yaml b/.licenserc.yaml
index b4d54f3a4..ad551a982 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -43,7 +43,6 @@ header:
     - 'site/layouts/**'
     - 'api/internal/cookie/cookie.go'
     - 'controller/registries/nacos/config.go'
-    - 'external/**'
   comment: on-failure
 
 dependency:
diff --git a/Makefile b/Makefile
index 4d7540c84..8e42311ad 100644
--- a/Makefile
+++ b/Makefile
@@ -119,7 +119,7 @@ fmt-proto: dev-tools
 
 .PHONY: fmt-proto-local
 fmt-proto-local:
-	find . -name '*.proto' | grep -v './external' | xargs clang-format -i
+	find . -name '*.proto' | xargs clang-format -i
 
 
 LICENSE_CHECKER_VERSION = 0.6.0
@@ -144,7 +144,7 @@ lint-spell: dev-tools
 		${DEV_TOOLS_IMAGE} \
 		make lint-spell-local
 
-CODESPELL = codespell --skip '.ignore_words,test-envoy,go.mod,go.sum,*.svg,*.patch,./site/public/**,external,.git,.idea,go.work.sum' --check-filenames --check-hidden --ignore-words ./.ignore_words .
+CODESPELL = codespell --skip '.ignore_words,test-envoy,go.mod,go.sum,*.svg,*.patch,./site/public/**,.git,.idea,go.work.sum' --check-filenames --check-hidden --ignore-words ./.ignore_words .
 .PHONY: lint-spell-local
 lint-spell-local:
 	$(CODESPELL)
@@ -187,8 +187,7 @@ lint-website: $(LOCALBIN)
 .PHONY: lint-markdown
 lint-markdown:
 	if ! command -v markdownlint >/dev/null 2>&1; then npm install -g markdownlint-cli; fi
-	@# ignore markdown under 'external/istio'
-	markdownlint '{*.md,site/**/*.md}' --disable MD012 MD013 MD029 MD033 MD034 MD036 MD041
+	markdownlint '**/*.md' --disable MD012 MD013 MD029 MD033 MD034 MD036 MD041
 
 # We don’t use if ! command -v yamllint because some environments might have a pre-installed Python version.
 # Checking the specific path ensures we're using the Node.js version to avoid conflicts.
@@ -215,7 +214,8 @@ fmt: fmt-go fmt-proto fix-spell fix-cjk
 verify-example:
 	cd ./examples/dev_your_plugin && ./verify.sh
 
-TARGET_ISTIO_DIR ?= $(shell pwd)/external/istio
+# use a path outside repo so the linters won't lint the istio files
+TARGET_ISTIO_DIR ?= $(shell pwd)/../istio
 
 .PHONY: prebuild
 prebuild:
diff --git a/manifests/images/cp/Dockerfile b/manifests/images/cp/Dockerfile
index 0fdc49eb1..1c2800dae 100644
--- a/manifests/images/cp/Dockerfile
+++ b/manifests/images/cp/Dockerfile
@@ -34,8 +34,22 @@ COPY api/ api/
 COPY types/ types/
 COPY plugins/ plugins/
 COPY controller/ controller/
-# Remember to run `make prebuild` before building the image
-COPY external/istio/ external/istio/
+# Copy files used by `make prebuild`
+COPY Makefile common.mk ./
+COPY patch/ patch/
+# Download istio to external/istio
+# Note that if we switch to non-debian base image these commands will break...
+RUN cat <<_EOF > /etc/apt/sources.list.d/debian.sources
+Types: deb
+URIs: https://mirrors.tuna.tsinghua.edu.cn/debian
+Suites: bookworm bookworm-updates bookworm-backports
+Components: main contrib non-free non-free-firmware
+Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
+_EOF
+
+# hadolint ignore=DL3008
+RUN apt-get update && apt-get install -y patch --no-install-recommends
+RUN TARGET_ISTIO_DIR=$(pwd)/external/istio make prebuild
 WORKDIR /workspace/external/istio
 RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -C pilot/cmd/pilot-discovery -a -o /workspace/pilot-discovery
 
diff --git a/manifests/images/dp/Dockerfile b/manifests/images/dp/Dockerfile
index 7717a9023..75c9f0cf5 100644
--- a/manifests/images/dp/Dockerfile
+++ b/manifests/images/dp/Dockerfile
@@ -33,8 +33,22 @@ COPY api/ api/
 COPY types/ types/
 COPY plugins/ plugins/
 COPY controller/ controller/
-# Remember to run `make prebuild` before building the image
-COPY external/istio/ external/istio/
+# Copy files used by `make prebuild`
+COPY Makefile common.mk ./
+COPY patch/ patch/
+# Download istio to external/istio
+# Note that if we switch to non-debian base image these commands will break...
+RUN cat <<_EOF > /etc/apt/sources.list.d/debian.sources
+Types: deb
+URIs: https://mirrors.tuna.tsinghua.edu.cn/debian
+Suites: bookworm bookworm-updates bookworm-backports
+Components: main contrib non-free non-free-firmware
+Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
+_EOF
+
+# hadolint ignore=DL3008
+RUN apt-get update && apt-get install -y patch --no-install-recommends
+RUN TARGET_ISTIO_DIR=$(pwd)/external/istio make prebuild
 
 COPY patch/switch-envoy-go-version.sh patch/switch-envoy-go-version.sh
 COPY common.mk common.mk
diff --git a/patch/apply-patch.sh b/patch/apply-patch.sh
index 5dcb8a1a9..753cd219e 100755
--- a/patch/apply-patch.sh
+++ b/patch/apply-patch.sh
@@ -25,17 +25,6 @@ done
 
 pushd "$TARGET_ISTIO_DIR"
 go mod tidy
-go install golang.org/x/tools/cmd/goimports@latest # required by codegen
-if [ "$(uname -s)" = "Darwin" ]; then
-    sed -i '' -e 's/.\/site/.\/external\/istio/' ../../go.work
-else
-    sed -i -e 's/.\/site/.\/external\/istio/' ../../go.work
-fi
-# go run will fail without adding `./external/istio` to go.work
+go install golang.org/x/tools/cmd/goimports@v0.24.0 # required by codegen. Version higher than 0.24.0 doesn't support Go 1.21.
 go run pkg/config/schema/codegen/tools/collections.main.go
-if [ "$(uname -s)" = "Darwin" ]; then
-    sed -i '' -e 's/.\/external\/istio/.\/site/' ../../go.work
-else
-    sed -i -e 's/.\/external\/istio/.\/site/' ../../go.work
-fi
 popd