diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 892d8aec..e6c98729 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,17 @@ jobs: api-module-test: timeout-minutes: 10 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + envoy_version: + - 1.29 + - 1.31 + env: + ENVOY_API_VERSION: ${{ matrix.envoy_version }} + # patch version should not contain API breaking changes, so we just pick the first one + FULL_ENVOY_VERSION: ${{ matrix.envoy_version }}.0 + PROXY_IMAGE: envoyproxy/envoy:contrib-v${{ matrix.envoy_version }}.0 defaults: run: working-directory: ./api @@ -34,6 +45,11 @@ jobs: with: go-version: '1.21' cache-dependency-path: "**/*.sum" + - name: Choose the Envoy API + run: | + pushd .. + ./patch/switch-envoy-go-version.sh ${FULL_ENVOY_VERSION} + popd - name: Unit test run: make unit-test @@ -47,7 +63,7 @@ jobs: if: failure() with: # upload artifact can be found in https://github.com/mosn/htnn/actions/runs/$id - name: api-module-test-logs + name: api-module-test-logs-${{ matrix.envoy_version }} path: ./test-envoy - name: Generate coverage @@ -58,7 +74,7 @@ jobs: if: always() # always upload coverage, so the coverage percents won't affect by the failed tests uses: actions/upload-artifact@v4 with: - name: api-module-test-cover + name: api-module-test-cover-${{ matrix.envoy_version }} path: | ./api/cover.out ./api/cover_integration.out @@ -213,8 +229,10 @@ jobs: with: fail_ci_if_error: true files: | - ./api-module-test-cover/cover.out, - ./api-module-test-cover/cover_integration.out, + ./api-module-test-cover-1.29/cover.out, + ./api-module-test-cover-1.29/cover_integration.out, + ./api-module-test-cover-1.31/cover.out, + ./api-module-test-cover-1.31/cover_integration.out, ./types-module-test-cover/cover.out, ./plugins-unit-test-cover/cover.out, ./plugins-integration-test-cover/cover.out, diff --git a/MAINTAIN.md b/MAINTAIN.md index 0c83d516..9db3363a 100644 --- a/MAINTAIN.md +++ b/MAINTAIN.md @@ -22,8 +22,7 @@ To release a new version, please follow the steps below: To upgrade Istio, please follow the steps below: * Discuss the impact of the upgrade. For example, is there any break change, do we need to upgrade K8S, etc. -* Update the base image used in the integration tests. -* Update the ISTIO_VERSION we define in the `common.mk`. +* Update the ISTIO_VERSION we define in the `common.mk` and the dataplane image's Dockerfile. * Update the link `/envoy/v1.xx.y/configuration/` in the doc to the new Envoy version. And `istio/istio/xxx` to the new Istio version. * Update the charts' dependency versions used in the `manifests/charts/*/Chart.yaml`. diff --git a/README.md b/README.md index fffdb1f6..6b86149c 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,10 @@ HTNN (Hyper Trust-Native Network) is Ant Group's internally developed cloud-nati * [Quick Start](https://github.com/mosn/htnn/blob/main/site/content/en/docs/getting-started/quick_start.md) * [Get Involved](https://github.com/mosn/htnn/blob/main/site/content/en/docs/developer-guide/get_involved.md) +If you want to extend Envoy via Go, you can consider using the dataplane of HTNN only. Please read the documentation: + +* [Dataplane Support](https://github.com/mosn/htnn/blob/main/site/content/en/docs/developer-guide/dataplane_support.md) + ## Community ### Chinese diff --git a/api/Makefile b/api/Makefile index b2863752..3552c4fe 100644 --- a/api/Makefile +++ b/api/Makefile @@ -16,7 +16,7 @@ include ../common.mk .PHONY: unit-test unit-test: - go test ${TEST_OPTION} $(shell go list ./... | \ + go test -tags envoy${ENVOY_API_VERSION} ${TEST_OPTION} $(shell go list ./... | \ grep -v tests/integration) # We can't specify -race to `go build` because it seems that @@ -24,7 +24,7 @@ unit-test: # the race detector will allocate memory out of 48bits address which is not allowed in x64. .PHONY: build-test-so-local build-test-so-local: - CGO_ENABLED=1 go build -tags so \ + CGO_ENABLED=1 go build -tags so,envoy${ENVOY_API_VERSION} \ -ldflags "-B 0x$(shell head -c20 /dev/urandom|od -An -tx1|tr -d ' \n') " \ --buildmode=c-shared \ -cover -covermode=atomic -coverpkg=${PROJECT_NAME}/... \ @@ -39,6 +39,7 @@ build-test-so: -v $(PWD)/..:/go/src/${PROJECT_NAME} \ -w /go/src/${PROJECT_NAME}/api \ -e GOPROXY \ + -e ENVOY_API_VERSION \ ${BUILD_IMAGE} \ bash -c "git config --global --add safe.directory '*' && make build-test-so-local" @@ -47,10 +48,10 @@ build-test-so: integration-test: test -d /tmp/htnn_coverage && rm -rf /tmp/htnn_coverage || true $(foreach PKG, $(shell go list ./tests/integration/...), \ - go test -v ${PKG} || exit 1; \ + go test -tags envoy${ENVOY_API_VERSION} -v ${PKG} || exit 1; \ ) # The benchmark running time can be controlled via env var HTNN_DATA_PLANE_BENCHMARK_DURATION .PHONY: benchmark benchmark: - go test -tags benchmark -v ./tests/integration/ -run TestBenchmark + go test -tags benchmark,envoy${ENVOY_API_VERSION} -v ./tests/integration/ -run TestBenchmark diff --git a/api/go.mod b/api/go.mod index b7d2089a..242f20f6 100644 --- a/api/go.mod +++ b/api/go.mod @@ -21,7 +21,7 @@ go 1.21 require ( github.com/agiledragon/gomonkey/v2 v2.11.0 github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa - github.com/envoyproxy/envoy v1.29.4 + github.com/envoyproxy/envoy v1.31.0 github.com/envoyproxy/go-control-plane v0.12.1-0.20240117015050-472addddff92 // version used by istio 1.21 github.com/envoyproxy/protoc-gen-validate v1.0.4 github.com/go-logr/logr v1.4.1 @@ -31,7 +31,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/net v0.24.0 google.golang.org/grpc v1.63.2 - google.golang.org/protobuf v1.33.0 + google.golang.org/protobuf v1.34.1 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/api/go.sum b/api/go.sum index caad57b4..778f3637 100644 --- a/api/go.sum +++ b/api/go.sum @@ -6,8 +6,8 @@ github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/P github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/envoy v1.29.4 h1:1c52LYxzA6arnSjpTfDyxCSrtztwVAnzekcGHirvq8Y= -github.com/envoyproxy/envoy v1.29.4/go.mod h1:c2OGLXJVY9BaTYPiWFRz6eUNaC+3TfqFKmkWS9brdKo= +github.com/envoyproxy/envoy v1.31.0 h1:NsTo+medzu0bMffXAjl+zKaViLOShKuIZWQnKKYq0/4= +github.com/envoyproxy/envoy v1.31.0/go.mod h1:ujBFxE543X8OePZG+FbeR9LnpBxTLu64IAU7A20EB9A= github.com/envoyproxy/go-control-plane v0.12.1-0.20240117015050-472addddff92 h1:/3bsjkhOTh0swUKDBxL1+3MrXCxrf/sEEMseiIEJg00= github.com/envoyproxy/go-control-plane v0.12.1-0.20240117015050-472addddff92/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= @@ -59,8 +59,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/api/pkg/consumer/consumer.go b/api/pkg/consumer/consumer.go index 980e8ea1..5bb5688b 100644 --- a/api/pkg/consumer/consumer.go +++ b/api/pkg/consumer/consumer.go @@ -36,16 +36,14 @@ type consumerManager struct { conf *consumerManagerConfig } -func ConsumerManagerFactory(c interface{}) capi.StreamFilterFactory { +func ConsumerManagerFactory(c interface{}, callbacks capi.FilterCallbackHandler) capi.StreamFilter { conf, ok := c.(*consumerManagerConfig) if !ok { panic(fmt.Sprintf("wrong config type: %s", reflect.TypeOf(c))) } - return func(callbacks capi.FilterCallbackHandler) capi.StreamFilter { - return &consumerManager{ - callbacks: callbacks, - conf: conf, - } + return &consumerManager{ + callbacks: callbacks, + conf: conf, } } diff --git a/api/pkg/consumer/consumer_test.go b/api/pkg/consumer/consumer_test.go index aef97156..d9141cf1 100644 --- a/api/pkg/consumer/consumer_test.go +++ b/api/pkg/consumer/consumer_test.go @@ -76,6 +76,6 @@ func TestParse(t *testing.T) { func TestConsumerManagerFactory(t *testing.T) { assert.PanicsWithValuef(t, "wrong config type: *struct {}", func() { - ConsumerManagerFactory(&struct{}{}) + ConsumerManagerFactory(&struct{}{}, nil) }, "check if the panic message contains the wrong type") } diff --git a/api/pkg/dynamicconfig/dynamicconfig.go b/api/pkg/dynamicconfig/dynamicconfig.go index 6417b66c..904eaa6e 100644 --- a/api/pkg/dynamicconfig/dynamicconfig.go +++ b/api/pkg/dynamicconfig/dynamicconfig.go @@ -41,11 +41,9 @@ type dynamicConfigFilter struct { callbacks capi.FilterCallbackHandler } -func DynamicConfigFactory(c interface{}) capi.StreamFilterFactory { - return func(callbacks capi.FilterCallbackHandler) capi.StreamFilter { - return &dynamicConfigFilter{ - callbacks: callbacks, - } +func DynamicConfigFactory(_ interface{}, callbacks capi.FilterCallbackHandler) capi.StreamFilter { + return &dynamicConfigFilter{ + callbacks: callbacks, } } diff --git a/api/pkg/filtermanager/api/api.go b/api/pkg/filtermanager/api/api.go index 2d5b6348..ccfad7fb 100644 --- a/api/pkg/filtermanager/api/api.go +++ b/api/pkg/filtermanager/api/api.go @@ -17,7 +17,6 @@ package api import ( "net/http" "net/url" - "sync/atomic" "github.com/envoyproxy/envoy/contrib/golang/common/go/api" "google.golang.org/protobuf/reflect/protoreflect" @@ -172,6 +171,8 @@ type StreamInfo interface { FilterState() FilterState // VirtualClusterName returns the name of the virtual cluster which got matched VirtualClusterName() (string, bool) + // WorkerID returns the ID of the Envoy worker thread + WorkerID() uint32 // Methods added by HTNN @@ -194,13 +195,10 @@ type Consumer interface { PluginConfig(name string) PluginConsumerConfig } -// FilterCallbackHandler provides API that is used during request processing -type FilterCallbackHandler interface { +// StreamFilterCallbacks provides API that is used during request processing +type StreamFilterCallbacks interface { // StreamInfo provides API to get/set current stream's context. StreamInfo() StreamInfo - // RecoverPanic covers panic to 500 response to avoid crashing Envoy. If you create goroutine - // in your Filter, please add `defer RecoverPanic()` to avoid crash by panic. - RecoverPanic() // GetProperty fetch Envoy attribute and return the value as a string. // The list of attributes can be found in https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/advanced/attributes. // If the fetch succeeded, a string will be returned. @@ -212,6 +210,9 @@ type FilterCallbackHandler interface { // * ErrSerializationFailure (Currently, fetching attributes in List/Map type are unsupported) // * ErrValueNotFound GetProperty(key string) (string, error) + // ClearRouteCache clears the route cache for the current request, and filtermanager will re-fetch the route in the next filter. + // Please be careful to invoke it, since filtermanager will raise an 404 route_not_found response when failed to re-fetch a route. + ClearRouteCache() // Methods added by HTNN @@ -229,7 +230,7 @@ type FilterCallbackHandler interface { // WithLogArg injectes `key: value` as the suffix of application log created by this // callback's Log* methods. The injected log arguments are only valid in the current request. // This method can be used to inject IDs or other context information into the logs. - WithLogArg(key string, value any) FilterCallbackHandler + WithLogArg(key string, value any) StreamFilterCallbacks LogTracef(format string, v ...any) LogTrace(message string) LogDebugf(format string, v ...any) @@ -242,6 +243,31 @@ type FilterCallbackHandler interface { LogError(message string) } +// FilterProcessCallbacks is the interface for filter to process request/response in decode/encode phase. +type FilterProcessCallbacks interface { + SendLocalReply(responseCode int, bodyText string, headers map[string][]string, grpcStatus int64, details string) + // RecoverPanic recover panic in defer and terminate the request by SendLocalReply with 500 status code. + RecoverPanic() + + // hide Continue() method from the user +} + +type DecoderFilterCallbacks interface { + FilterProcessCallbacks +} + +type EncoderFilterCallbacks interface { + FilterProcessCallbacks +} + +type FilterCallbackHandler interface { + StreamFilterCallbacks + // DecoderFilterCallbacks could only be used in DecodeXXX phases. + DecoderFilterCallbacks() DecoderFilterCallbacks + // EncoderFilterCallbacks could only be used in EncodeXXX phases. + EncoderFilterCallbacks() EncoderFilterCallbacks +} + // FilterFactory returns a per-request Filter which has configuration bound to it. // This function should be a pure builder and should not have any side effect. type FilterFactory func(config interface{}, callbacks FilterCallbackHandler) Filter @@ -282,102 +308,6 @@ var ( LogLevelCritical = api.Critical ) -// Drop our log optimization once https://github.com/envoyproxy/envoy/commit/591fb13817ddf1f54945186e3c6de4e0345508d2 -// is used. - -var ( - currLogLevel atomic.Int32 -) - -func GetLogLevel() LogType { - lv := currLogLevel.Load() - return LogType(lv) -} - -func LogTrace(message string) { - if GetLogLevel() > LogLevelTrace { - return - } - api.LogTrace(message) -} - -func LogDebug(message string) { - if GetLogLevel() > LogLevelDebug { - return - } - api.LogDebug(message) -} - -func LogInfo(message string) { - if GetLogLevel() > LogLevelInfo { - return - } - api.LogInfo(message) -} - -func LogWarn(message string) { - if GetLogLevel() > LogLevelWarn { - return - } - api.LogWarn(message) -} - -func LogError(message string) { - if GetLogLevel() > LogLevelError { - return - } - api.LogError(message) -} - -func LogCritical(message string) { - if GetLogLevel() > LogLevelCritical { - return - } - api.LogCritical(message) -} - -func LogTracef(format string, v ...any) { - if GetLogLevel() > LogLevelTrace { - return - } - api.LogTracef(format, v...) -} - -func LogDebugf(format string, v ...any) { - if GetLogLevel() > LogLevelDebug { - return - } - api.LogDebugf(format, v...) -} - -func LogInfof(format string, v ...any) { - if GetLogLevel() > LogLevelInfo { - return - } - api.LogInfof(format, v...) -} - -func LogWarnf(format string, v ...any) { - if GetLogLevel() > LogLevelWarn { - return - } - api.LogWarnf(format, v...) -} - -func LogErrorf(format string, v ...any) { - if GetLogLevel() > LogLevelError { - return - } - api.LogErrorf(format, v...) -} - -func LogCriticalf(format string, v ...any) { - if GetLogLevel() > LogLevelCritical { - return - } - api.LogCriticalf(format, v...) -} - var ( ErrInternalFailure = api.ErrInternalFailure ErrSerializationFailure = api.ErrSerializationFailure diff --git a/api/pkg/filtermanager/api/api_129.go b/api/pkg/filtermanager/api/api_129.go new file mode 100644 index 00000000..e622612f --- /dev/null +++ b/api/pkg/filtermanager/api/api_129.go @@ -0,0 +1,116 @@ +// Copyright The HTNN Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build envoy1.29 + +package api + +import ( + "sync/atomic" + + "github.com/envoyproxy/envoy/contrib/golang/common/go/api" +) + +var ( + currLogLevel atomic.Int32 +) + +func GetLogLevel() LogType { + lv := currLogLevel.Load() + return LogType(lv) +} + +func LogTrace(message string) { + if GetLogLevel() > LogLevelTrace { + return + } + api.LogTrace(message) +} + +func LogDebug(message string) { + if GetLogLevel() > LogLevelDebug { + return + } + api.LogDebug(message) +} + +func LogInfo(message string) { + if GetLogLevel() > LogLevelInfo { + return + } + api.LogInfo(message) +} + +func LogWarn(message string) { + if GetLogLevel() > LogLevelWarn { + return + } + api.LogWarn(message) +} + +func LogError(message string) { + if GetLogLevel() > LogLevelError { + return + } + api.LogError(message) +} + +func LogCritical(message string) { + if GetLogLevel() > LogLevelCritical { + return + } + api.LogCritical(message) +} + +func LogTracef(format string, v ...any) { + if GetLogLevel() > LogLevelTrace { + return + } + api.LogTracef(format, v...) +} + +func LogDebugf(format string, v ...any) { + if GetLogLevel() > LogLevelDebug { + return + } + api.LogDebugf(format, v...) +} + +func LogInfof(format string, v ...any) { + if GetLogLevel() > LogLevelInfo { + return + } + api.LogInfof(format, v...) +} + +func LogWarnf(format string, v ...any) { + if GetLogLevel() > LogLevelWarn { + return + } + api.LogWarnf(format, v...) +} + +func LogErrorf(format string, v ...any) { + if GetLogLevel() > LogLevelError { + return + } + api.LogErrorf(format, v...) +} + +func LogCriticalf(format string, v ...any) { + if GetLogLevel() > LogLevelCritical { + return + } + api.LogCriticalf(format, v...) +} diff --git a/api/pkg/filtermanager/api/no_so.go b/api/pkg/filtermanager/api/api_129_no_so.go similarity index 96% rename from api/pkg/filtermanager/api/no_so.go rename to api/pkg/filtermanager/api/api_129_no_so.go index f951975e..0faef059 100644 --- a/api/pkg/filtermanager/api/no_so.go +++ b/api/pkg/filtermanager/api/api_129_no_so.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !so +//go:build envoy1.29 && !so package api diff --git a/api/pkg/filtermanager/api/so.go b/api/pkg/filtermanager/api/api_129_so.go similarity index 98% rename from api/pkg/filtermanager/api/so.go rename to api/pkg/filtermanager/api/api_129_so.go index 2facd5ed..bcc0b61f 100644 --- a/api/pkg/filtermanager/api/so.go +++ b/api/pkg/filtermanager/api/api_129_so.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build so +//go:build envoy1.29 && so package api diff --git a/api/pkg/filtermanager/api/api_129_test.go b/api/pkg/filtermanager/api/api_129_test.go new file mode 100644 index 00000000..be9d0b7c --- /dev/null +++ b/api/pkg/filtermanager/api/api_129_test.go @@ -0,0 +1,44 @@ +// Copyright The HTNN Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build envoy1.29 + +package api + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLogAPI(t *testing.T) { + currLogLevel.Store(int32(LogLevelCritical)) + assert.Equal(t, LogLevelCritical, GetLogLevel()) + + for _, s := range []struct { + level string + logf func(string, ...any) + log func(string) + }{ + {"Trace", LogTracef, LogTrace}, + {"Debug", LogDebugf, LogDebug}, + {"Info", LogInfof, LogInfo}, + {"Warn", LogWarnf, LogWarn}, + {"Error", LogErrorf, LogError}, + } { + s.logf("test %s", s.level) + s.log(s.level) + // should not call api.LogXX directly - which will panic + } +} diff --git a/api/pkg/filtermanager/api/api_latest.go b/api/pkg/filtermanager/api/api_latest.go new file mode 100644 index 00000000..cd17eede --- /dev/null +++ b/api/pkg/filtermanager/api/api_latest.go @@ -0,0 +1,38 @@ +// Copyright The HTNN Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !envoy1.29 + +package api + +import ( + "github.com/envoyproxy/envoy/contrib/golang/common/go/api" +) + +var ( + LogTrace = api.LogTrace + LogDebug = api.LogDebug + LogInfo = api.LogInfo + LogWarn = api.LogWarn + LogError = api.LogError + LogCritical = api.LogCritical + LogTracef = api.LogTracef + LogDebugf = api.LogDebugf + LogInfof = api.LogInfof + LogWarnf = api.LogWarnf + LogErrorf = api.LogErrorf + LogCriticalf = api.LogCriticalf + + GetLogLevel = api.GetLogLevel +) diff --git a/api/pkg/filtermanager/api_impl.go b/api/pkg/filtermanager/api_impl.go index 574f9a6c..24078e62 100644 --- a/api/pkg/filtermanager/api_impl.go +++ b/api/pkg/filtermanager/api_impl.go @@ -207,7 +207,7 @@ func (cb *filterManagerCallbackHandler) PluginState() api.PluginState { return cb.pluginState } -func (cb *filterManagerCallbackHandler) WithLogArg(key string, value any) api.FilterCallbackHandler { +func (cb *filterManagerCallbackHandler) WithLogArg(key string, value any) api.StreamFilterCallbacks { // As the log is embedded into the Envoy's log, it's not so necessary to use structural logging // here. So far the value is just an ID string, introduce complex processions like quoting is // overkill. diff --git a/api/pkg/filtermanager/api_impl_129.go b/api/pkg/filtermanager/api_impl_129.go new file mode 100644 index 00000000..5e632485 --- /dev/null +++ b/api/pkg/filtermanager/api_impl_129.go @@ -0,0 +1,46 @@ +// Copyright The HTNN Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build envoy1.29 + +package filtermanager + +import ( + "runtime/debug" + + capi "github.com/envoyproxy/envoy/contrib/golang/common/go/api" + + "mosn.io/htnn/api/pkg/filtermanager/api" +) + +func (s *filterManagerStreamInfo) WorkerID() uint32 { + api.LogErrorf("WorkerID is not implemented: %s", debug.Stack()) + return 0 +} + +func (cb *filterManagerCallbackHandler) ClearRouteCache() { + api.LogErrorf("ClearRouteCache is not implemented: %s", debug.Stack()) +} + +func (cb *filterManagerCallbackHandler) DecoderFilterCallbacks() api.DecoderFilterCallbacks { + return cb.FilterCallbackHandler +} + +func (cb *filterManagerCallbackHandler) EncoderFilterCallbacks() api.EncoderFilterCallbacks { + return cb.FilterCallbackHandler +} + +func (cb *filterManagerCallbackHandler) Continue(st capi.StatusType, _ bool) { + cb.FilterCallbackHandler.Continue(st) +} diff --git a/api/pkg/filtermanager/api_impl_latest.go b/api/pkg/filtermanager/api_impl_latest.go new file mode 100644 index 00000000..c8e1c891 --- /dev/null +++ b/api/pkg/filtermanager/api_impl_latest.go @@ -0,0 +1,39 @@ +// Copyright The HTNN Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !envoy1.29 + +package filtermanager + +import ( + capi "github.com/envoyproxy/envoy/contrib/golang/common/go/api" + + "mosn.io/htnn/api/pkg/filtermanager/api" +) + +func (cb *filterManagerCallbackHandler) DecoderFilterCallbacks() api.DecoderFilterCallbacks { + return cb.FilterCallbackHandler.DecoderFilterCallbacks() +} + +func (cb *filterManagerCallbackHandler) EncoderFilterCallbacks() api.EncoderFilterCallbacks { + return cb.FilterCallbackHandler.EncoderFilterCallbacks() +} + +func (cb *filterManagerCallbackHandler) Continue(st capi.StatusType, decoding bool) { + if decoding { + cb.FilterCallbackHandler.DecoderFilterCallbacks().Continue(st) + } else { + cb.FilterCallbackHandler.EncoderFilterCallbacks().Continue(st) + } +} diff --git a/api/pkg/filtermanager/api_impl_test.go b/api/pkg/filtermanager/api_impl_test.go index c9398deb..3f1cf2d6 100644 --- a/api/pkg/filtermanager/api_impl_test.go +++ b/api/pkg/filtermanager/api_impl_test.go @@ -78,7 +78,7 @@ func TestPluginState(t *testing.T) { Factory: getPluginStateFilterFactory, }, } - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) h := http.Header{} hdr := envoy.NewRequestHeaderMap(h) m.DecodeHeaders(hdr, true) @@ -143,7 +143,7 @@ func TestAccessCacheFieldsConcurrently(t *testing.T) { for i := 0; i < n; i++ { go func(i int) { cb := envoy.NewCAPIFilterCallbackHandler() - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) h := http.Header{} hdr := envoy.NewRequestHeaderMap(h) m.DecodeHeaders(hdr, true) @@ -223,7 +223,7 @@ func TestLogWithArgs(t *testing.T) { Factory: testLogFilterFactory, }, } - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) h := http.Header{} hdr := envoy.NewRequestHeaderMap(h) m.DecodeHeaders(hdr, true) diff --git a/api/pkg/filtermanager/filtermanager.go b/api/pkg/filtermanager/filtermanager.go index 593aff3b..66107c4a 100644 --- a/api/pkg/filtermanager/filtermanager.go +++ b/api/pkg/filtermanager/filtermanager.go @@ -37,7 +37,8 @@ type filterManager struct { decodeRequestNeeded bool decodeIdx int - reqHdr api.RequestHeaderMap + reqHdr api.RequestHeaderMap // don't access it in Encode phases + contentType string encodeResponseNeeded bool encodeIdx int @@ -66,6 +67,7 @@ func (m *filterManager) Reset() { m.decodeRequestNeeded = false m.decodeIdx = -1 m.reqHdr = nil + m.contentType = "" m.encodeResponseNeeded = false m.encodeIdx = -1 @@ -127,7 +129,17 @@ func needLogExecution() bool { return api.GetLogLevel() <= api.LogLevelDebug } -func FilterManagerFactory(c interface{}) capi.StreamFilterFactory { +func FilterManagerFactory(c interface{}, cb capi.FilterCallbackHandler) (streamFilter capi.StreamFilter) { + // the RecoverPanic requires the underline Go req to be created. However, the Go req is created + // after the FilterManagerFactory is called. So we implement our own RecoverPanic here to avoid breaking + // the creation of Go req. + defer func() { + if p := recover(); p != nil { + api.LogErrorf("panic: %v\n%s", p, debug.Stack()) + streamFilter = InternalErrorFactoryForCAPI(c, cb) + } + }() + conf, ok := c.(*filterManagerConfig) if !ok { panic(fmt.Sprintf("wrong config type: %s", reflect.TypeOf(c))) @@ -135,96 +147,86 @@ func FilterManagerFactory(c interface{}) capi.StreamFilterFactory { parsedConfig := conf.parsed - return func(cb capi.FilterCallbackHandler) (streamFilter capi.StreamFilter) { - // TODO: remove this protection once we upgrade to the new Envoy version - defer func() { - if p := recover(); p != nil { - api.LogErrorf("panic: %v\n%s", p, debug.Stack()) - streamFilter = InternalErrorFactoryForCAPI(c, cb) - } - }() - - data := conf.pool.Get() - fm, ok := data.(*filterManager) - if !ok { - panic(fmt.Sprintf("unexpected type: %s", reflect.TypeOf(data))) - } + data := conf.pool.Get() + fm, ok := data.(*filterManager) + if !ok { + panic(fmt.Sprintf("unexpected type: %s", reflect.TypeOf(data))) + } - fm.callbacks.FilterCallbackHandler = cb + fm.callbacks.FilterCallbackHandler = cb - canSkipMethod := fm.canSkipMethod - if canSkipMethod == nil { - canSkipMethod = newSkipMethodsMap() - } + canSkipMethod := fm.canSkipMethod + if canSkipMethod == nil { + canSkipMethod = newSkipMethodsMap() + } - filters := make([]*model.FilterWrapper, len(parsedConfig)) - logExecution := needLogExecution() - for i, fc := range parsedConfig { - factory := fc.Factory - config := fc.ParsedConfig - f := factory(config, fm.callbacks) - // Technically, the factory might create different f for different calls. We don't support this edge case for now. - if fm.canSkipMethod == nil { - definedMethod := make(map[string]bool, len(canSkipMethod)) - for meth := range canSkipMethod { - definedMethod[meth] = false - } - for meth := range canSkipMethod { - overridden, err := reflectx.IsMethodOverridden(f, meth) - if err != nil { - api.LogErrorf("failed to check method %s in plugin %s: %v", meth, fc.Name, err) - // canSkipMethod[meth] will be false - } - canSkipMethod[meth] = canSkipMethod[meth] && !overridden - definedMethod[meth] = overridden + filters := make([]*model.FilterWrapper, len(parsedConfig)) + logExecution := needLogExecution() + for i, fc := range parsedConfig { + factory := fc.Factory + config := fc.ParsedConfig + f := factory(config, fm.callbacks) + // Technically, the factory might create different f for different calls. We don't support this edge case for now. + if fm.canSkipMethod == nil { + definedMethod := make(map[string]bool, len(canSkipMethod)) + for meth := range canSkipMethod { + definedMethod[meth] = false + } + for meth := range canSkipMethod { + overridden, err := reflectx.IsMethodOverridden(f, meth) + if err != nil { + api.LogErrorf("failed to check method %s in plugin %s: %v", meth, fc.Name, err) + // canSkipMethod[meth] will be false } + canSkipMethod[meth] = canSkipMethod[meth] && !overridden + definedMethod[meth] = overridden + } - if definedMethod["DecodeRequest"] { - if !definedMethod["DecodeHeaders"] { - api.LogErrorf("plugin %s has DecodeRequest but not DecodeHeaders. To run DecodeRequest, we need to return api.WaitAllData from DecodeHeaders", fc.Name) - } + if definedMethod["DecodeRequest"] { + if !definedMethod["DecodeHeaders"] { + api.LogErrorf("plugin %s has DecodeRequest but not DecodeHeaders. To run DecodeRequest, we need to return api.WaitAllData from DecodeHeaders", fc.Name) + } - p := pkgPlugins.LoadPluginType(fc.Name) - if p != nil { - order := p.Order() - if order.Position <= pkgPlugins.OrderPositionAuthn { - api.LogErrorf("plugin %s has DecodeRequest which is not supported because the order of plugin", fc.Name) - } + p := pkgPlugins.LoadPluginType(fc.Name) + if p != nil { + order := p.Order() + if order.Position <= pkgPlugins.OrderPositionAuthn { + api.LogErrorf("plugin %s has DecodeRequest which is not supported because the order of plugin", fc.Name) } } - if definedMethod["EncodeResponse"] && !definedMethod["EncodeHeaders"] { - api.LogErrorf("plugin %s has EncodeResponse but not EncodeHeaders. To run EncodeResponse, we need to return api.WaitAllData from EncodeHeaders", fc.Name) - } } - - if logExecution { - filters[i] = model.NewFilterWrapper(fc.Name, NewLogExecutionFilter(fc.Name, f, fm.callbacks)) - } else { - filters[i] = model.NewFilterWrapper(fc.Name, f) + if definedMethod["EncodeResponse"] && !definedMethod["EncodeHeaders"] { + api.LogErrorf("plugin %s has EncodeResponse but not EncodeHeaders. To run EncodeResponse, we need to return api.WaitAllData from EncodeHeaders", fc.Name) } + } - if fm.DebugModeEnabled() { - filters[i] = model.NewFilterWrapper(fc.Name, NewDebugFilter(fc.Name, filters[i].Filter, fm.callbacks)) - } + if logExecution { + filters[i] = model.NewFilterWrapper(fc.Name, NewLogExecutionFilter(fc.Name, f, fm.callbacks)) + } else { + filters[i] = model.NewFilterWrapper(fc.Name, f) } - if fm.canSkipMethod == nil { - fm.canSkipMethod = canSkipMethod + if fm.DebugModeEnabled() { + filters[i] = model.NewFilterWrapper(fc.Name, NewDebugFilter(fc.Name, filters[i].Filter, fm.callbacks)) } + } - // We can't cache the slice of filters as it may be changed by consumer - fm.filters = filters + if fm.canSkipMethod == nil { + fm.canSkipMethod = canSkipMethod + } - // The skip check is based on the compiled code. So if the DecodeRequest is defined, - // even it is not called, DecodeData will not be skipped. Same as EncodeResponse. - fm.canSkipDecodeHeaders = fm.canSkipMethod["DecodeHeaders"] && fm.canSkipMethod["DecodeRequest"] && fm.config.initOnce == nil - fm.canSkipDecodeData = fm.canSkipMethod["DecodeData"] && fm.canSkipMethod["DecodeRequest"] - fm.canSkipEncodeHeaders = fm.canSkipMethod["EncodeHeaders"] - fm.canSkipEncodeData = fm.canSkipMethod["EncodeData"] && fm.canSkipMethod["EncodeResponse"] - fm.canSkipOnLog = fm.canSkipMethod["OnLog"] + // We can't cache the slice of filters as it may be changed by consumer + fm.filters = filters - return fm - } + // The skip check is based on the compiled code. So if the DecodeRequest is defined, + // even it is not called, DecodeData will not be skipped. Same as EncodeResponse. + fm.canSkipDecodeHeaders = fm.canSkipMethod["DecodeHeaders"] && fm.canSkipMethod["DecodeRequest"] && fm.config.initOnce == nil + fm.canSkipDecodeData = fm.canSkipMethod["DecodeData"] && fm.canSkipMethod["DecodeRequest"] + fm.canSkipEncodeHeaders = fm.canSkipMethod["EncodeHeaders"] + fm.canSkipEncodeData = fm.canSkipMethod["EncodeData"] && fm.canSkipMethod["EncodeResponse"] + fm.canSkipOnLog = fm.canSkipMethod["OnLog"] + + return fm } func (m *filterManager) recordLocalReplyPluginName(name string) { @@ -254,7 +256,7 @@ func (m *filterManager) handleAction(res api.ResultAction, phase phase, filter * switch v := res.(type) { case *api.LocalResponse: m.recordLocalReplyPluginName(filter.Name) - m.localReply(v) + m.localReply(v, phase < phaseEncodeHeaders) return true default: api.LogErrorf("unknown result action: %+v", v) @@ -266,7 +268,7 @@ type jsonReply struct { Msg string `json:"msg"` } -func (m *filterManager) localReply(v *api.LocalResponse) { +func (m *filterManager) localReply(v *api.LocalResponse, decoding bool) { var hdr map[string][]string if v.Header != nil { hdr = map[string][]string(v.Header) @@ -294,8 +296,10 @@ func (m *filterManager) localReply(v *api.LocalResponse) { isJSON = true } } else { - ct, ok = m.reqHdr.Get("content-type") - if !ok || ct == "application/json" { + // use the Content-Type header passed by the client, not the header + // provided by the gateway if have. + ct = m.contentType + if ct == "" || ct == "application/json" { isJSON = true } } @@ -310,14 +314,22 @@ func (m *filterManager) localReply(v *api.LocalResponse) { hdr["Content-Type"] = []string{"application/json"} } } - m.callbacks.SendLocalReply(v.Code, msg, hdr, 0, "") + + var cb api.FilterProcessCallbacks + if decoding { + cb = m.callbacks.DecoderFilterCallbacks() + } else { + cb = m.callbacks.EncoderFilterCallbacks() + } + cb.SendLocalReply(v.Code, msg, hdr, 0, "") } func (m *filterManager) DecodeHeaders(headers capi.RequestHeaderMap, endStream bool) capi.StatusType { + m.contentType, _ = headers.Get("content-type") + // Ensure the headers are cached on the Go side. // FIXME: remove this once we support OnLog phase headers in Envoy Go. if m.DebugModeEnabled() { - headers.Get("test") headers := &filterManagerRequestHeaderMap{ RequestHeaderMap: headers, } @@ -332,7 +344,7 @@ func (m *filterManager) DecodeHeaders(headers capi.RequestHeaderMap, endStream b go func() { defer m.MarkRunningInGoThread(false) - defer m.callbacks.RecoverPanic() + defer m.callbacks.DecoderFilterCallbacks().RecoverPanic() var res api.ResultAction m.config.InitOnce() @@ -341,7 +353,7 @@ func (m *filterManager) DecodeHeaders(headers capi.RequestHeaderMap, endStream b m.recordLocalReplyPluginName(m.config.initFailedPluginName) m.localReply(&api.LocalResponse{ Code: 500, - }) + }, true) return } @@ -370,7 +382,7 @@ func (m *filterManager) DecodeHeaders(headers capi.RequestHeaderMap, endStream b m.localReply(&api.LocalResponse{ Code: 401, Msg: "consumer not found", - }) + }, true) return } @@ -484,7 +496,7 @@ func (m *filterManager) DecodeHeaders(headers capi.RequestHeaderMap, endStream b m.decodeIdx = i // some filters, like authorization with request body, need to // have a whole body before passing to the next filter - m.callbacks.Continue(capi.StopAndBuffer) + m.callbacks.Continue(capi.StopAndBuffer, true) return } @@ -496,7 +508,7 @@ func (m *filterManager) DecodeHeaders(headers capi.RequestHeaderMap, endStream b } } - m.callbacks.Continue(capi.Continue) + m.callbacks.Continue(capi.Continue, true) }() return capi.Running @@ -511,7 +523,7 @@ func (m *filterManager) DecodeData(buf capi.BufferInstance, endStream bool) capi go func() { defer m.MarkRunningInGoThread(false) - defer m.callbacks.RecoverPanic() + defer m.callbacks.DecoderFilterCallbacks().RecoverPanic() var res api.ResultAction // We have discussed a lot about how to support processing data both streamingly and @@ -537,7 +549,7 @@ func (m *filterManager) DecodeData(buf capi.BufferInstance, endStream bool) capi return } } - m.callbacks.Continue(capi.Continue) + m.callbacks.Continue(capi.Continue, true) } else { for i := 0; i < m.decodeIdx; i++ { @@ -592,7 +604,7 @@ func (m *filterManager) DecodeData(buf capi.BufferInstance, endStream bool) capi } } - m.callbacks.Continue(capi.Continue) + m.callbacks.Continue(capi.Continue, true) } }() @@ -615,7 +627,7 @@ func (m *filterManager) EncodeHeaders(headers capi.ResponseHeaderMap, endStream go func() { defer m.MarkRunningInGoThread(false) - defer m.callbacks.RecoverPanic() + defer m.callbacks.EncoderFilterCallbacks().RecoverPanic() var res api.ResultAction m.hdrLock.Lock() @@ -633,7 +645,7 @@ func (m *filterManager) EncodeHeaders(headers capi.ResponseHeaderMap, endStream m.encodeResponseNeeded = false if !endStream { m.encodeIdx = i - m.callbacks.Continue(capi.StopAndBuffer) + m.callbacks.Continue(capi.StopAndBuffer, false) return } @@ -645,7 +657,7 @@ func (m *filterManager) EncodeHeaders(headers capi.ResponseHeaderMap, endStream } } - m.callbacks.Continue(capi.Continue) + m.callbacks.Continue(capi.Continue, false) }() return capi.Running @@ -660,7 +672,7 @@ func (m *filterManager) EncodeData(buf capi.BufferInstance, endStream bool) capi go func() { defer m.MarkRunningInGoThread(false) - defer m.callbacks.RecoverPanic() + defer m.callbacks.EncoderFilterCallbacks().RecoverPanic() var res api.ResultAction n := len(m.filters) @@ -673,7 +685,7 @@ func (m *filterManager) EncodeData(buf capi.BufferInstance, endStream bool) capi return } } - m.callbacks.Continue(capi.Continue) + m.callbacks.Continue(capi.Continue, false) } else { for i := n - 1; i > m.encodeIdx; i-- { @@ -724,7 +736,7 @@ func (m *filterManager) EncodeData(buf capi.BufferInstance, endStream bool) capi } } - m.callbacks.Continue(capi.Continue) + m.callbacks.Continue(capi.Continue, false) } }() diff --git a/api/pkg/filtermanager/filtermanager_benchmark_test.go b/api/pkg/filtermanager/filtermanager_benchmark_test.go index e680cc0a..f5eca7c7 100644 --- a/api/pkg/filtermanager/filtermanager_benchmark_test.go +++ b/api/pkg/filtermanager/filtermanager_benchmark_test.go @@ -44,7 +44,7 @@ func BenchmarkFilterManagerAllPhase(b *testing.B) { respBuf := envoy.NewBufferInstance([]byte{}) for n := 0; n < b.N; n++ { - m := FilterManagerFactory(config)(cb) + m := FilterManagerFactory(config, cb) m.DecodeHeaders(reqHdr, false) cb.WaitContinued() m.DecodeData(reqBuf, true) @@ -88,7 +88,7 @@ func BenchmarkFilterManagerRegular(b *testing.B) { reqHdr := envoy.NewRequestHeaderMap(http.Header{}) for n := 0; n < b.N; n++ { - m := FilterManagerFactory(config)(cb) + m := FilterManagerFactory(config, cb) m.DecodeHeaders(reqHdr, false) cb.WaitContinued() m.OnLog() @@ -129,7 +129,7 @@ func BenchmarkFilterManagerConsumerWithFilter(b *testing.B) { } for n := 0; n < b.N; n++ { - m := FilterManagerFactory(config)(cb) + m := FilterManagerFactory(config, cb) m.DecodeHeaders(reqHdrs[n%num], false) cb.WaitContinued() m.OnLog() diff --git a/api/pkg/filtermanager/filtermanager_test.go b/api/pkg/filtermanager/filtermanager_test.go index b7445040..0aeee79e 100644 --- a/api/pkg/filtermanager/filtermanager_test.go +++ b/api/pkg/filtermanager/filtermanager_test.go @@ -41,7 +41,7 @@ func TestPassThrough(t *testing.T) { }, } for i := 0; i < 2; i++ { - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) hdr := envoy.NewRequestHeaderMap(http.Header{}) m.DecodeHeaders(hdr, false) cb.WaitContinued() @@ -109,7 +109,7 @@ func TestLocalReplyJSON_UseReqHeader(t *testing.T) { Factory: PassThroughFactory, }, } - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) patches := gomonkey.ApplyMethodReturn(m.filters[0].Filter, "DecodeHeaders", &api.LocalResponse{ Code: 200, Msg: "msg", @@ -182,7 +182,7 @@ func TestLocalReplyJSON_UseRespHeader(t *testing.T) { Factory: PassThroughFactory, }, } - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) patches := gomonkey.ApplyMethodReturn(m.filters[0].Filter, "EncodeHeaders", &api.LocalResponse{ Code: 200, Msg: "msg", @@ -218,7 +218,7 @@ func TestLocalReplyJSON_DoNotChangeMsgIfContentTypeIsGiven(t *testing.T) { Factory: PassThroughFactory, }, } - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) patches := gomonkey.ApplyMethodReturn(m.filters[0].Filter, "DecodeHeaders", &api.LocalResponse{ Msg: "msg", Header: http.Header(map[string][]string{"Content-Type": {"text/plain"}}), @@ -280,7 +280,7 @@ func TestInitFailed(t *testing.T) { for i := 0; i < n; i++ { go func(i int) { cb := envoy.NewCAPIFilterCallbackHandler() - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) h := http.Header{} hdr := envoy.NewRequestHeaderMap(h) m.DecodeHeaders(hdr, true) @@ -303,7 +303,7 @@ func TestInitFailed(t *testing.T) { for i := 0; i < n; i++ { go func(i int) { cb := envoy.NewCAPIFilterCallbackHandler() - m := FilterManagerFactory(config2)(cb).(*filterManager) + m := FilterManagerFactory(config2, cb).(*filterManager) h := http.Header{} hdr := envoy.NewRequestHeaderMap(h) m.DecodeHeaders(hdr, true) @@ -371,7 +371,7 @@ func TestSkipMethodWhenThereAreMultiFilters(t *testing.T) { } for i := 0; i < 2; i++ { - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) assert.Equal(t, false, m.canSkipOnLog) assert.Equal(t, false, m.canSkipDecodeHeaders) assert.Equal(t, true, m.canSkipDecodeData) @@ -452,7 +452,7 @@ func TestFiltersFromConsumer(t *testing.T) { for i := 0; i < 2*n; i++ { go func(i int) { cb := envoy.NewCAPIFilterCallbackHandler() - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) assert.Equal(t, true, m.canSkipOnLog) assert.Equal(t, 2, len(m.filters)) h := http.Header{} @@ -545,7 +545,7 @@ func TestDoNotRecycleInUsedFilterManager(t *testing.T) { for i := 0; i < n; i++ { go func(i int) { cb := envoy.NewCAPIFilterCallbackHandler() - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) h := http.Header{} hdr := envoy.NewRequestHeaderMap(h) m.DecodeHeaders(hdr, true) @@ -560,7 +560,7 @@ func TestDoNotRecycleInUsedFilterManager(t *testing.T) { for i := 0; i < n; i++ { go func(i int) { cb := envoy.NewCAPIFilterCallbackHandler() - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) h := http.Header{} hdr := envoy.NewRequestHeaderMap(h) m.DecodeHeaders(hdr, false) @@ -577,7 +577,7 @@ func TestDoNotRecycleInUsedFilterManager(t *testing.T) { for i := 0; i < n; i++ { go func(i int) { cb := envoy.NewCAPIFilterCallbackHandler() - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) h := http.Header{} hdr := envoy.NewRequestHeaderMap(h) m.DecodeHeaders(hdr, true) @@ -595,7 +595,7 @@ func TestDoNotRecycleInUsedFilterManager(t *testing.T) { for i := 0; i < n; i++ { go func(i int) { cb := envoy.NewCAPIFilterCallbackHandler() - m := FilterManagerFactory(config)(cb).(*filterManager) + m := FilterManagerFactory(config, cb).(*filterManager) h := http.Header{} hdr := envoy.NewRequestHeaderMap(h) m.DecodeHeaders(hdr, true) diff --git a/api/pkg/filtermanager/internal_error.go b/api/pkg/filtermanager/internal_error.go index fe78392e..72f52cc0 100644 --- a/api/pkg/filtermanager/internal_error.go +++ b/api/pkg/filtermanager/internal_error.go @@ -46,16 +46,18 @@ func NewInternalErrorFactory(plugin string, err error) api.FilterFactory { type internalErrorFilterForCAPI struct { capi.PassThroughStreamFilter - callbacks capi.FilterCallbacks + callbacks api.FilterCallbackHandler } func (f *internalErrorFilterForCAPI) DecodeHeaders(headers capi.RequestHeaderMap, endStream bool) capi.StatusType { - f.callbacks.SendLocalReply(500, "", nil, 0, "") + f.callbacks.DecoderFilterCallbacks().SendLocalReply(500, "", nil, 0, "") return capi.LocalReply } func InternalErrorFactoryForCAPI(cfg interface{}, callbacks capi.FilterCallbackHandler) capi.StreamFilter { return &internalErrorFilterForCAPI{ - callbacks: callbacks, + callbacks: &filterManagerCallbackHandler{ + FilterCallbackHandler: callbacks, + }, } } diff --git a/api/plugins/tests/integration/dataplane/data_plane.go b/api/plugins/tests/integration/dataplane/data_plane.go index e9c59e0d..21d3fdd7 100644 --- a/api/plugins/tests/integration/dataplane/data_plane.go +++ b/api/plugins/tests/integration/dataplane/data_plane.go @@ -145,7 +145,7 @@ func StartDataPlane(t *testing.T, opt *Option) (*DataPlane, error) { // Since we only care about the coverage in CI, it is fine so far. } - image := "m.daocloud.io/docker.io/envoyproxy/envoy:contrib-v1.29.5" + image := "m.daocloud.io/docker.io/envoyproxy/envoy:contrib-v1.31.2" specifiedImage := os.Getenv("PROXY_IMAGE") if specifiedImage != "" { diff --git a/api/plugins/tests/pkg/envoy/capi.go b/api/plugins/tests/pkg/envoy/capi.go index 8b0a2af7..cc99442d 100644 --- a/api/plugins/tests/pkg/envoy/capi.go +++ b/api/plugins/tests/pkg/envoy/capi.go @@ -36,8 +36,12 @@ func init() { capi.SetCommonCAPI(&fakeCapi{}) } +var ( + logLevel = capi.Info +) + func DisableLogInTest() { - api.SetLogLevel(capi.Critical) + logLevel = capi.Critical } func logInGo(level capi.LogType, message string) { @@ -51,7 +55,7 @@ func (a *fakeCapi) Log(level capi.LogType, message string) { } func (a *fakeCapi) LogLevel() capi.LogType { - return 0 + return logLevel } type HeaderMap struct { @@ -70,6 +74,10 @@ func (i *HeaderMap) Get(key string) (string, bool) { return v, true } +func (i *HeaderMap) GetAllHeaders() map[string][]string { + return i.Header.Clone() +} + func (i *HeaderMap) Values(key string) []string { return i.Header.Values(key) } @@ -144,6 +152,18 @@ func (i *RequestHeaderMap) Path() string { return path } +func (i *RequestHeaderMap) SetMethod(method string) { + i.Set(":method", method) +} + +func (i *RequestHeaderMap) SetPath(path string) { + i.Set(":path", path) +} + +func (i *RequestHeaderMap) SetHost(host string) { + i.Set(":authority", host) +} + func (i *RequestHeaderMap) URL() *url.URL { path := i.Path() u, _ := url.ParseRequestURI(path) @@ -499,6 +519,9 @@ func (i *filterCallbackHandler) GetProperty(key string) (string, error) { return "", nil } +func (i *filterCallbackHandler) ClearRouteCache() { +} + func (i *filterCallbackHandler) LookupConsumer(_, _ string) (api.Consumer, bool) { return nil, false } @@ -518,7 +541,7 @@ func (i *filterCallbackHandler) PluginState() api.PluginState { return i.pluginState } -func (i *filterCallbackHandler) WithLogArg(key string, value any) api.FilterCallbackHandler { +func (i *filterCallbackHandler) WithLogArg(key string, value any) api.StreamFilterCallbacks { return i } @@ -552,6 +575,16 @@ func (i *filterCallbackHandler) LogErrorf(format string, v ...any) { func (i *filterCallbackHandler) LogError(message string) { } +func (i *filterCallbackHandler) DecoderFilterCallbacks() api.DecoderFilterCallbacks { + // we don't distinguish between decoder and encoder filter callbacks in the test helper + return i +} + +func (i *filterCallbackHandler) EncoderFilterCallbacks() api.EncoderFilterCallbacks { + // we don't distinguish between decoder and encoder filter callbacks in the test helper + return i +} + var _ api.FilterCallbackHandler = (*filterCallbackHandler)(nil) type capiFilterCallbackHandler struct { diff --git a/api/plugins/tests/pkg/envoy/capi_129.go b/api/plugins/tests/pkg/envoy/capi_129.go new file mode 100644 index 00000000..53be497d --- /dev/null +++ b/api/plugins/tests/pkg/envoy/capi_129.go @@ -0,0 +1,17 @@ +// Copyright The HTNN Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build envoy1.29 && !so + +package envoy diff --git a/api/plugins/tests/pkg/envoy/capi_latest.go b/api/plugins/tests/pkg/envoy/capi_latest.go new file mode 100644 index 00000000..d4724b1b --- /dev/null +++ b/api/plugins/tests/pkg/envoy/capi_latest.go @@ -0,0 +1,29 @@ +// Copyright The HTNN Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !envoy1.29 && !so + +package envoy + +import ( + capi "github.com/envoyproxy/envoy/contrib/golang/common/go/api" +) + +func (cb *capiFilterCallbackHandler) DecoderFilterCallbacks() capi.DecoderFilterCallbacks { + return cb.filterCallbackHandler.DecoderFilterCallbacks().(*filterCallbackHandler) +} + +func (cb *capiFilterCallbackHandler) EncoderFilterCallbacks() capi.EncoderFilterCallbacks { + return cb.filterCallbackHandler.EncoderFilterCallbacks().(*filterCallbackHandler) +} diff --git a/api/tests/integration/libgolang/main.go b/api/tests/integration/libgolang/main.go index 4b278ca2..0af71b5b 100644 --- a/api/tests/integration/libgolang/main.go +++ b/api/tests/integration/libgolang/main.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build so +//go:build so && !envoy1.29 package main @@ -27,9 +27,9 @@ import ( ) func init() { - http.RegisterHttpFilterConfigFactoryAndParser("fm", filtermanager.FilterManagerFactory, &filtermanager.FilterManagerConfigParser{}) - http.RegisterHttpFilterConfigFactoryAndParser("cm", consumer.ConsumerManagerFactory, &consumer.ConsumerManagerConfigParser{}) - http.RegisterHttpFilterConfigFactoryAndParser("dc", dynamicconfig.DynamicConfigFactory, &dynamicconfig.DynamicConfigParser{}) + http.RegisterHttpFilterFactoryAndConfigParser("fm", filtermanager.FilterManagerFactory, &filtermanager.FilterManagerConfigParser{}) + http.RegisterHttpFilterFactoryAndConfigParser("cm", consumer.ConsumerManagerFactory, &consumer.ConsumerManagerConfigParser{}) + http.RegisterHttpFilterFactoryAndConfigParser("dc", dynamicconfig.DynamicConfigFactory, &dynamicconfig.DynamicConfigParser{}) } func main() {} diff --git a/api/tests/integration/libgolang/main_129.go b/api/tests/integration/libgolang/main_129.go new file mode 100644 index 00000000..88ccee54 --- /dev/null +++ b/api/tests/integration/libgolang/main_129.go @@ -0,0 +1,54 @@ +// Copyright The HTNN Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build so && envoy1.29 + +package main + +import ( + capi "github.com/envoyproxy/envoy/contrib/golang/common/go/api" + "github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http" + + "mosn.io/htnn/api/pkg/consumer" + "mosn.io/htnn/api/pkg/dynamicconfig" + "mosn.io/htnn/api/pkg/filtermanager" + _ "mosn.io/htnn/api/plugins/tests/integration/dataplane" // for utility plugins provided in the test framework + _ "mosn.io/htnn/api/tests/integration" // for plugins used in the test +) + +var ( + filterManagerFactoryWrapper = func(c interface{}) capi.StreamFilterFactory { + return func(cb capi.FilterCallbackHandler) (streamFilter capi.StreamFilter) { + return filtermanager.FilterManagerFactory(c, cb) + } + } + consumerManagerFactoryWrapper = func(c interface{}) capi.StreamFilterFactory { + return func(cb capi.FilterCallbackHandler) (streamFilter capi.StreamFilter) { + return consumer.ConsumerManagerFactory(c, cb) + } + } + dynamicConfigFactoryWrapper = func(c interface{}) capi.StreamFilterFactory { + return func(cb capi.FilterCallbackHandler) (streamFilter capi.StreamFilter) { + return dynamicconfig.DynamicConfigFactory(c, cb) + } + } +) + +func init() { + http.RegisterHttpFilterConfigFactoryAndParser("fm", filterManagerFactoryWrapper, &filtermanager.FilterManagerConfigParser{}) + http.RegisterHttpFilterConfigFactoryAndParser("cm", consumerManagerFactoryWrapper, &consumer.ConsumerManagerConfigParser{}) + http.RegisterHttpFilterConfigFactoryAndParser("dc", dynamicConfigFactoryWrapper, &dynamicconfig.DynamicConfigParser{}) +} + +func main() {} diff --git a/api/tests/integration/test_plugins.go b/api/tests/integration/test_plugins.go index 4b515cae..c7208a64 100644 --- a/api/tests/integration/test_plugins.go +++ b/api/tests/integration/test_plugins.go @@ -180,16 +180,20 @@ type localReplyFilter struct { callbacks api.FilterCallbackHandler config *Config - reqHdr api.RequestHeaderMap + + reqHdr api.RequestHeaderMap + runFilters []string } -func (f *localReplyFilter) NewLocalResponse(reply string) *api.LocalResponse { +func (f *localReplyFilter) NewLocalResponse(reply string, decoding bool) *api.LocalResponse { hdr := http.Header{} hdr.Set("local", reply) - runFilters := f.reqHdr.Values("run") - if len(runFilters) > 0 { - hdr.Set("order", strings.Join(runFilters, "|")) + if decoding { + f.runFilters = f.reqHdr.Values("run") + } + if len(f.runFilters) > 0 { + hdr.Set("order", strings.Join(f.runFilters, "|")) } msg := "ok" @@ -202,8 +206,9 @@ func (f *localReplyFilter) NewLocalResponse(reply string) *api.LocalResponse { func (f *localReplyFilter) DecodeRequest(headers api.RequestHeaderMap, buf api.BufferInstance, trailer api.RequestTrailerMap) api.ResultAction { api.LogInfof("traceback: %s", string(debug.Stack())) f.reqHdr = headers + f.runFilters = headers.Values("run") if f.config.Decode { - return f.NewLocalResponse("reply") + return f.NewLocalResponse("reply", true) } return api.Continue } @@ -212,7 +217,7 @@ func (f *localReplyFilter) EncodeResponse(headers api.ResponseHeaderMap, buf api api.LogInfof("traceback: %s", string(debug.Stack())) if f.config.Encode { r, _ := headers.Get("echo-from") - return f.NewLocalResponse(r) + return f.NewLocalResponse(r, false) } return api.Continue } @@ -223,8 +228,9 @@ func (f *localReplyFilter) DecodeHeaders(headers api.RequestHeaderMap, endStream return api.WaitAllData } f.reqHdr = headers + f.runFilters = headers.Values("run") if f.config.Decode && f.config.Headers { - return f.NewLocalResponse("reply") + return f.NewLocalResponse("reply", true) } return api.Continue } @@ -232,7 +238,7 @@ func (f *localReplyFilter) DecodeHeaders(headers api.RequestHeaderMap, endStream func (f *localReplyFilter) DecodeData(data api.BufferInstance, endStream bool) api.ResultAction { api.LogInfof("traceback: %s", string(debug.Stack())) if f.config.Decode && f.config.Data { - return f.NewLocalResponse("reply") + return f.NewLocalResponse("reply", true) } return api.Continue } @@ -244,7 +250,7 @@ func (f *localReplyFilter) EncodeHeaders(headers api.ResponseHeaderMap, endStrea } if f.config.Encode && f.config.Headers { r, _ := headers.Get("echo-from") - return f.NewLocalResponse(r) + return f.NewLocalResponse(r, false) } return api.Continue } @@ -252,7 +258,7 @@ func (f *localReplyFilter) EncodeHeaders(headers api.ResponseHeaderMap, endStrea func (f *localReplyFilter) EncodeData(data api.BufferInstance, endStream bool) api.ResultAction { api.LogInfof("traceback: %s", string(debug.Stack())) if f.config.Encode && f.config.Data { - return f.NewLocalResponse("reply") + return f.NewLocalResponse("reply", false) } return api.Continue } diff --git a/common.mk b/common.mk index 83b68a73..b3245237 100644 --- a/common.mk +++ b/common.mk @@ -26,10 +26,13 @@ $(LOCALBIN): TARGET_SO = libgolang.so PROJECT_NAME = mosn.io/htnn DOCKER_MIRROR = m.daocloud.io/ + # Both images use glibc 2.31. Ensure libc in the images match each other. BUILD_IMAGE ?= $(DOCKER_MIRROR)docker.io/library/golang:1.21-bullseye -# We don't use istio/proxyv2 because it is not designed to be run separately (need to work around permission issue). -PROXY_IMAGE ?= $(DOCKER_MIRROR)docker.io/envoyproxy/envoy:contrib-v1.29.5 +ENVOY_API_VERSION ?= 1.31 +PROXY_IMAGE ?= $(DOCKER_MIRROR)docker.io/envoyproxy/envoy:contrib-v1.31.2 +# We also support other Envoy versions. See https://github.com/mosn/htnn/tree/main/site/content/en/docs/developer-guide/dataplane_support.md + # We may need to use timestamp if we need to update the image in one PR REAL_DEV_TOOLS_IMAGE ?= ghcr.io/mosn/htnn-dev-tools:2024-07-12 DEV_TOOLS_IMAGE ?= $(DOCKER_MIRROR)$(REAL_DEV_TOOLS_IMAGE) diff --git a/manifests/Makefile b/manifests/Makefile index 5c6bd625..d7fcc197 100644 --- a/manifests/Makefile +++ b/manifests/Makefile @@ -25,12 +25,8 @@ PROXY_BASE_IMAGE ?= istio/proxyv2:$(ISTIO_VERSION) CONTROLLER_IMAGE ?= htnn/controller:latest CONTROLLER_BASE_IMAGE ?= docker.io/istio/pilot:$(ISTIO_VERSION) -.PHONY: build-so -build-so: - cd ../plugins/ && make build-so - .PHONY: build-proxy-image -build-proxy-image: build-so +build-proxy-image: cd .. && $(CONTAINER_TOOL) build -t ${PROXY_IMAGE} --build-arg GOPROXY=${GOPROXY} --build-arg PROXY_BASE_IMAGE=${PROXY_BASE_IMAGE} \ -f manifests/images/dp/Dockerfile . diff --git a/manifests/images/dp/Dockerfile b/manifests/images/dp/Dockerfile index 8c65ef09..eb2a9886 100644 --- a/manifests/images/dp/Dockerfile +++ b/manifests/images/dp/Dockerfile @@ -33,6 +33,14 @@ COPY plugins/ plugins/ COPY controller/ controller/ # Remember to run `make prebuild` before building the image COPY external/istio/ external/istio/ + +COPY patch/switch-envoy-go-version.sh patch/switch-envoy-go-version.sh +COPY common.mk common.mk +# hadolint ignore=DL3003 +RUN ./patch/switch-envoy-go-version.sh 1.29.5 && \ + cd plugins/ && \ + ENVOY_API_VERSION=1.29 make build-so-local + WORKDIR /workspace/external/istio RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -C pilot/cmd/pilot-agent -a -o /workspace/pilot-agent @@ -46,5 +54,5 @@ LABEL org.opencontainers.image.description="This is image used in the HTNN data LABEL org.opencontainers.image.licenses="Apache-2.0" COPY --from=builder /workspace/pilot-agent /usr/local/bin/ -COPY plugins/libgolang.so /etc/libgolang.so +COPY --from=builder /workspace/plugins/libgolang.so /etc/libgolang.so CMD ["envoy"] diff --git a/patch/switch-envoy-go-version.sh b/patch/switch-envoy-go-version.sh new file mode 100755 index 00000000..fa56891d --- /dev/null +++ b/patch/switch-envoy-go-version.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Copyright The HTNN Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is used to switch the envoy version in go.mod files +set -euo pipefail + +envoy_version=$1 + +# Check if arguments were provided +if [ -z "$envoy_version" ]; then + echo "Usage: $0 " + exit 1 +fi + +if [[ ! "$envoy_version" =~ ^[0-9]+\.[0-9]+\. ]]; then + echo "Envoy version $envoy_version should be in the format of x.y.z" + exit 1 +fi + +if [[ "$envoy_version" =~ ^1\.31\. ]]; then + echo "Envoy version $envoy_version is already in used" + exit 0 +fi + +if [[ ! "$envoy_version" =~ ^1\.(29)\. ]]; then + echo "Unsupported envoy version $envoy_version" + exit 1 +fi + +append_if_need() { + search_string=$1 + target_file=$2 + # Search for the string in the file + if ! grep -q "$search_string" "$target_file"; then + # Append the string to the file + echo "$search_string" >> "$target_file" + echo "String '$search_string' appended to $target_file" + else + echo "String '$search_string' already exists in $target_file" + fi +} + +# append to go.mod is easier than maintaining a go.mod file for -modfile flag +append_if_need "replace github.com/envoyproxy/envoy => github.com/envoyproxy/envoy v$envoy_version" api/go.mod +append_if_need "replace github.com/envoyproxy/envoy => github.com/envoyproxy/envoy v$envoy_version" plugins/go.mod diff --git a/plugins/Makefile b/plugins/Makefile index f11514d5..ed54b001 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -18,7 +18,7 @@ PROJECT_ROOT := $(PWD)/.. .PHONY: unit-test unit-test: - go test ${TEST_OPTION} $(shell go list ./... | \ + go test -tags envoy${ENVOY_API_VERSION} ${TEST_OPTION} $(shell go list ./... | \ grep -v tests/integration) # We can't specify -race to `go build` because it seems that @@ -26,7 +26,7 @@ unit-test: # the race detector will allocate memory out of 48bits address which is not allowed in x64. .PHONY: build-test-so-local build-test-so-local: - CGO_ENABLED=1 go build -tags so \ + CGO_ENABLED=1 go build -tags so,envoy${ENVOY_API_VERSION} \ -ldflags "-B 0x$(shell head -c20 /dev/urandom|od -An -tx1|tr -d ' \n')" \ --buildmode=c-shared \ -cover -covermode=atomic -coverpkg=${PROJECT_NAME}/... \ @@ -39,6 +39,7 @@ build-test-so-local: build-test-so: docker run --rm ${MOUNT_GOMOD_CACHE} -v ${PROJECT_ROOT}:/go/src/${PROJECT_NAME} -w /go/src/${PROJECT_NAME}/plugins \ -e GOPROXY \ + -e ENVOY_API_VERSION \ ${BUILD_IMAGE} \ bash -c "git config --global --add safe.directory '*' && make build-test-so-local" @@ -55,12 +56,12 @@ stop-service: integration-test: test -d /tmp/htnn_coverage && rm -rf /tmp/htnn_coverage || true $(foreach PKG, $(shell go list ./tests/integration/...), \ - go test -v ${PKG} || exit 1; \ + go test -tags envoy${ENVOY_API_VERSION} -v ${PKG} || exit 1; \ ) .PHONY: build-so-local build-so-local: - CGO_ENABLED=1 go build -tags so \ + CGO_ENABLED=1 go build -tags so,envoy${ENVOY_API_VERSION} \ -ldflags "-B 0x$(shell head -c20 /dev/urandom|od -An -tx1|tr -d ' \n')" \ --buildmode=c-shared \ -v -o ${TARGET_SO} \ @@ -70,6 +71,7 @@ build-so-local: build-so: docker run --rm ${MOUNT_GOMOD_CACHE} -v ${PROJECT_ROOT}:/go/src/${PROJECT_NAME} -w /go/src/${PROJECT_NAME}/plugins \ -e GOPROXY \ + -e ENVOY_API_VERSION \ ${BUILD_IMAGE} \ bash -c "git config --global --add safe.directory '*' && make build-so-local" diff --git a/plugins/cmd/libgolang/main.go b/plugins/cmd/libgolang/main.go index 32a7465b..4981b2a4 100644 --- a/plugins/cmd/libgolang/main.go +++ b/plugins/cmd/libgolang/main.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build so +//go:build so && !envoy1.29 package main @@ -26,9 +26,9 @@ import ( ) func init() { - http.RegisterHttpFilterConfigFactoryAndParser("fm", filtermanager.FilterManagerFactory, &filtermanager.FilterManagerConfigParser{}) - http.RegisterHttpFilterConfigFactoryAndParser("cm", consumer.ConsumerManagerFactory, &consumer.ConsumerManagerConfigParser{}) - http.RegisterHttpFilterConfigFactoryAndParser("dc", dynamicconfig.DynamicConfigFactory, &dynamicconfig.DynamicConfigParser{}) + http.RegisterHttpFilterFactoryAndConfigParser("fm", filtermanager.FilterManagerFactory, &filtermanager.FilterManagerConfigParser{}) + http.RegisterHttpFilterFactoryAndConfigParser("cm", consumer.ConsumerManagerFactory, &consumer.ConsumerManagerConfigParser{}) + http.RegisterHttpFilterFactoryAndConfigParser("dc", dynamicconfig.DynamicConfigFactory, &dynamicconfig.DynamicConfigParser{}) } func main() {} diff --git a/plugins/cmd/libgolang/main_129.go b/plugins/cmd/libgolang/main_129.go new file mode 100644 index 00000000..12a1777c --- /dev/null +++ b/plugins/cmd/libgolang/main_129.go @@ -0,0 +1,53 @@ +// Copyright The HTNN Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build so && envoy1.29 + +package main + +import ( + capi "github.com/envoyproxy/envoy/contrib/golang/common/go/api" + "github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http" + + "mosn.io/htnn/api/pkg/consumer" + "mosn.io/htnn/api/pkg/dynamicconfig" + "mosn.io/htnn/api/pkg/filtermanager" + _ "mosn.io/htnn/plugins" +) + +var ( + filterManagerFactoryWrapper = func(c interface{}) capi.StreamFilterFactory { + return func(cb capi.FilterCallbackHandler) (streamFilter capi.StreamFilter) { + return filtermanager.FilterManagerFactory(c, cb) + } + } + consumerManagerFactoryWrapper = func(c interface{}) capi.StreamFilterFactory { + return func(cb capi.FilterCallbackHandler) (streamFilter capi.StreamFilter) { + return consumer.ConsumerManagerFactory(c, cb) + } + } + dynamicConfigFactoryWrapper = func(c interface{}) capi.StreamFilterFactory { + return func(cb capi.FilterCallbackHandler) (streamFilter capi.StreamFilter) { + return dynamicconfig.DynamicConfigFactory(c, cb) + } + } +) + +func init() { + http.RegisterHttpFilterConfigFactoryAndParser("fm", filterManagerFactoryWrapper, &filtermanager.FilterManagerConfigParser{}) + http.RegisterHttpFilterConfigFactoryAndParser("cm", consumerManagerFactoryWrapper, &consumer.ConsumerManagerConfigParser{}) + http.RegisterHttpFilterConfigFactoryAndParser("dc", dynamicConfigFactoryWrapper, &dynamicconfig.DynamicConfigParser{}) +} + +func main() {} diff --git a/plugins/go.mod b/plugins/go.mod index 30e349b8..c11949cb 100644 --- a/plugins/go.mod +++ b/plugins/go.mod @@ -26,7 +26,7 @@ require ( github.com/avast/retry-go v3.0.0+incompatible github.com/casbin/casbin/v2 v2.88.0 github.com/coreos/go-oidc/v3 v3.10.0 - github.com/envoyproxy/envoy v1.29.4 + github.com/envoyproxy/envoy v1.31.0 github.com/fsnotify/fsnotify v1.7.0 github.com/google/cel-go v0.20.1 github.com/gorilla/securecookie v1.1.2 diff --git a/plugins/go.sum b/plugins/go.sum index a8966014..6ee2ee3e 100644 --- a/plugins/go.sum +++ b/plugins/go.sum @@ -50,8 +50,8 @@ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/envoyproxy/envoy v1.29.4 h1:1c52LYxzA6arnSjpTfDyxCSrtztwVAnzekcGHirvq8Y= -github.com/envoyproxy/envoy v1.29.4/go.mod h1:c2OGLXJVY9BaTYPiWFRz6eUNaC+3TfqFKmkWS9brdKo= +github.com/envoyproxy/envoy v1.31.0 h1:NsTo+medzu0bMffXAjl+zKaViLOShKuIZWQnKKYq0/4= +github.com/envoyproxy/envoy v1.31.0/go.mod h1:ujBFxE543X8OePZG+FbeR9LnpBxTLu64IAU7A20EB9A= github.com/envoyproxy/go-control-plane v0.12.1-0.20240621013728-1eb8caab5155 h1:IgJPqnrlY2Mr4pYB6oaMKvFvwJ9H+X6CCY5x1vCTcpc= github.com/envoyproxy/go-control-plane v0.12.1-0.20240621013728-1eb8caab5155/go.mod h1:5Wkq+JduFtdAXihLmeTJf+tRYIT4KBc2vPXDhwVo1pA= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= diff --git a/plugins/pkg/request/request.go b/plugins/pkg/request/request.go index 75fa2838..62121811 100644 --- a/plugins/pkg/request/request.go +++ b/plugins/pkg/request/request.go @@ -15,7 +15,7 @@ package request import ( - "github.com/envoyproxy/envoy/contrib/golang/common/go/api" + "mosn.io/htnn/api/pkg/filtermanager/api" ) // GetHeaders returns a plain map represents the headers. The returned headers won't diff --git a/plugins/tests/integration/libgolang/main.go b/plugins/tests/integration/libgolang/main.go index 064a275b..91528340 100644 --- a/plugins/tests/integration/libgolang/main.go +++ b/plugins/tests/integration/libgolang/main.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build so +//go:build so && !envoy1.29 package main @@ -27,9 +27,9 @@ import ( ) func init() { - http.RegisterHttpFilterConfigFactoryAndParser("fm", filtermanager.FilterManagerFactory, &filtermanager.FilterManagerConfigParser{}) - http.RegisterHttpFilterConfigFactoryAndParser("cm", consumer.ConsumerManagerFactory, &consumer.ConsumerManagerConfigParser{}) - http.RegisterHttpFilterConfigFactoryAndParser("dc", dynamicconfig.DynamicConfigFactory, &dynamicconfig.DynamicConfigParser{}) + http.RegisterHttpFilterFactoryAndConfigParser("fm", filtermanager.FilterManagerFactory, &filtermanager.FilterManagerConfigParser{}) + http.RegisterHttpFilterFactoryAndConfigParser("cm", consumer.ConsumerManagerFactory, &consumer.ConsumerManagerConfigParser{}) + http.RegisterHttpFilterFactoryAndConfigParser("dc", dynamicconfig.DynamicConfigFactory, &dynamicconfig.DynamicConfigParser{}) } func main() {} diff --git a/plugins/tests/integration/libgolang/main_129.go b/plugins/tests/integration/libgolang/main_129.go new file mode 100644 index 00000000..2b7640db --- /dev/null +++ b/plugins/tests/integration/libgolang/main_129.go @@ -0,0 +1,54 @@ +// Copyright The HTNN Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build so && envoy1.29 + +package main + +import ( + capi "github.com/envoyproxy/envoy/contrib/golang/common/go/api" + "github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http" + + "mosn.io/htnn/api/pkg/consumer" + "mosn.io/htnn/api/pkg/dynamicconfig" + "mosn.io/htnn/api/pkg/filtermanager" + _ "mosn.io/htnn/api/plugins/tests/integration/dataplane" // for utility plugins provided in the test framework + _ "mosn.io/htnn/plugins" +) + +var ( + filterManagerFactoryWrapper = func(c interface{}) capi.StreamFilterFactory { + return func(cb capi.FilterCallbackHandler) (streamFilter capi.StreamFilter) { + return filtermanager.FilterManagerFactory(c, cb) + } + } + consumerManagerFactoryWrapper = func(c interface{}) capi.StreamFilterFactory { + return func(cb capi.FilterCallbackHandler) (streamFilter capi.StreamFilter) { + return consumer.ConsumerManagerFactory(c, cb) + } + } + dynamicConfigFactoryWrapper = func(c interface{}) capi.StreamFilterFactory { + return func(cb capi.FilterCallbackHandler) (streamFilter capi.StreamFilter) { + return dynamicconfig.DynamicConfigFactory(c, cb) + } + } +) + +func init() { + http.RegisterHttpFilterConfigFactoryAndParser("fm", filterManagerFactoryWrapper, &filtermanager.FilterManagerConfigParser{}) + http.RegisterHttpFilterConfigFactoryAndParser("cm", consumerManagerFactoryWrapper, &consumer.ConsumerManagerConfigParser{}) + http.RegisterHttpFilterConfigFactoryAndParser("dc", dynamicConfigFactoryWrapper, &dynamicconfig.DynamicConfigParser{}) +} + +func main() {} diff --git a/site/content/en/docs/developer-guide/dataplane_support.md b/site/content/en/docs/developer-guide/dataplane_support.md new file mode 100644 index 00000000..c64485df --- /dev/null +++ b/site/content/en/docs/developer-guide/dataplane_support.md @@ -0,0 +1,56 @@ +--- +title: HTNN's Multi-Version Support for Envoy +--- + +For users who only need the data plane of HTNN, this document will introduce how to choose HTNN features according to their own situations. + +HTNN compiles its Go code into a shared library and loads it into Envoy. So its data plane can be divided into two parts, one is Envoy itself, and the other is the shared library compiled from the Go code developed by HTNN. If users want to use HTNN's data plane code separately, it usually means using HTNN's Go code in combination with their own Envoy. + +## Introduction to the Data Plane Code + +HTNN's data plane code is located in the `./api` and `./plugins` modules. The introduction and extension of these two modules can be found in [How to develop HTNN to fit your purpose](./get_involved.md). Both of these modules can be compiled independently into shared libraries. The difference between them is that `./api` only contains the minimalist implementation of HTNN, providing the necessary interfaces for running Go plugins. While `./plugins` provides a set of official Go plugins based on `./api`. If users want to use the official Go plugins, they can import the `./plugins` module in their own `package main`. This way, when compiling into a shared library, it will include the official Go plugins. For a specific implementation, please refer to https://github.com/mosn/htnn/blob/main/plugins/cmd/libgolang/main.go. + +Note that due to the dependencies of the `./plugins` module being managed by `go.work`, if a user wants to import a version of the `./plugins` module that has not been tagged, they need to manually keep the versions of the `./api` and `./types` modules it depends on consistent. As shown below: + +```go.mod +require ( + mosn.io/htnn/api v0.3.3-0.20240910021016-dd32dd2d331f // indirect + mosn.io/htnn/plugins v0.3.3-0.20240912020652-82b6aa8de677 + mosn.io/htnn/types v0.3.3-0.20240910021016-dd32dd2d331f +) +``` + +If it's a tagged version, such as `mosn.io/htnn/plugins v0.3.2`, you can directly require that module. + +## Choosing the Target Data Plane Version + +Since the Envoy Golang filter is still under development, almost every version introduces breaking changes. To address this, HTNN introduces a data plane API version selection mechanism, allowing developers to choose the corresponding HTNN data plane code according to their Envoy version. + +By default, the target API version of HTNN's data plane code is the latest officially released Envoy version. At the same time, it supports compiling a shared library that can run on previously released Envoy versions by using build tags. The currently supported versions are as follows: + +| Version | build tag | +|---------|-------------------------------------| +| 1.29 | envoy1.29 | +| 1.31 | Latest version, no build tag needed | + +For example, to compile a shared library that can run on Envoy 1.29, you need to execute the following commands: + +Modify `go.mod` and replace the Envoy SDK with the version consistent with Envoy: + +```go.mod +replace github.com/envoyproxy/envoy => github.com/envoyproxy/envoy v1.29.5 +``` + +Then compile: + +```shell +CGO_ENABLED=1 go build -tags so,envoy1.29 --buildmode=c-shared ... +``` + +If the target is the latest officially released Envoy version, no additional build tag is needed: + +```shell +CGO_ENABLED=1 go build -tags so --buildmode=c-shared ... +``` + +If an interface that only exists in the latest Envoy is executed on an older Envoy, the compatibility layer provided by this suite will execute an virtual interface, output an error log, and return a null value. diff --git a/site/content/zh-hans/docs/developer-guide/dataplane_support.md b/site/content/zh-hans/docs/developer-guide/dataplane_support.md new file mode 100644 index 00000000..42d95cc5 --- /dev/null +++ b/site/content/zh-hans/docs/developer-guide/dataplane_support.md @@ -0,0 +1,56 @@ +--- +title: HTNN 对 Envoy 的多版本支持 +--- + +对于只需要 HTNN 数据面的用户，本文将介绍如何根据自己的情况选用 HTNN 的功能。 + +HTNN 会将自己的 Go 代码编译成 shared library，加载到 Envoy 当中。所以它的数据面可以分成两部分，一个是 Envoy 本身，另一个是 HTNN 开发的 Go 代码所编译出的 shared library。用户如果要单独使用 HTNN 的数据面代码，通常是指结合自己的 Envoy 来使用 HTNN 的 Go 代码。 + +## 数据面代码介绍 + +HTNN 数据面代码位于 `./api` 和 `./plugins` 模块。这两个模块的介绍以及拓展的方式，可以参考 [如何二次开发 HTNN](./get_involved.md)。这两个模块都能独立编译成 shared library。他们的区别在于 `./api` 只包含了最小化的 HTNN 实现，提供了运行 Go 插件所需的接口。而 `./plugins` 在 `./api` 的基础上提供了一组官方的 Go 插件。如果用户想要使用官方的 Go 插件，可以在自己的 `package main` 里 import `./plugins` 模块。这样在编译成 shared library 时就会包含官方 Go 插件。具体实现可以参考 https://github.com/mosn/htnn/blob/main/plugins/cmd/libgolang/main.go 这个文件。 + +注意由于 `./plugins` 模块的依赖是通过 `go.work` 管理的，如果用户要 import 没有打过 tag 的 `./plugins` 模块版本，需要手动将它所依赖的 `./api` 和 `./types` 模块版本保持一致。如下所示： + +```go.mod +require ( + mosn.io/htnn/api v0.3.3-0.20240910021016-dd32dd2d331f // indirect + mosn.io/htnn/plugins v0.3.3-0.20240912020652-82b6aa8de677 + mosn.io/htnn/types v0.3.3-0.20240910021016-dd32dd2d331f +) +``` + +如果是打过 tag 的版本，如 `mosn.io/htnn/plugins v0.3.2`，直接 require 该模块即可。 + +## 选择目标数据面版本 + +由于 Envoy Golang filter 尚处于发展阶段，几乎每个版本都会引入 break change。为此 HTNN 引入了一套数据面 API 版本选择机制，开发者能够根据自己的 Envoy 版本选择对应的 HTNN 数据面代码。 + +默认情况下 HTNN 数据面代码的目标 API 版本是最新正式发布的 Envoy 版本。同时支持通过 build tag，编译出能够在之前发布的 Envoy 上运行的 shared library。目前支持的版本如下： + +| 版本 | build tag | +|------|----------------------------| +| 1.29 | envoy1.29 | +| 1.31 | 最新版本，不需要 build tag | + +举个例子，编译在 Envoy 1.29 上可运行的 shared library 需要执行下面命令： + +修改 `go.mod`，把 Envoy SDK 替换成和 Envoy 一致的版本： + +```go.mod +replace github.com/envoyproxy/envoy => github.com/envoyproxy/envoy v1.29.5 +``` + +然后编译： + +```shell +CGO_ENABLED=1 go build -tags so,envoy1.29 --buildmode=c-shared ... +``` + +如果目标是最新正式发布的 Envoy 版本，则不需要额外的 build tag： + +```shell +CGO_ENABLED=1 go build -tags so --buildmode=c-shared ... +``` + +如果在旧的 Envoy 上执行只有最新 Envoy 才存在的接口，会执行到这套兼容层提供的虚假接口，输出错误日志并返回空值。 diff --git a/types/registries/nacos/config.pb.go b/types/registries/nacos/config.pb.go index 67eac5fb..a6ea6092 100644 --- a/types/registries/nacos/config.pb.go +++ b/types/registries/nacos/config.pb.go @@ -42,6 +42,7 @@ type Config struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // the version is used to choose the Nacos version between v1 and v2 Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` ServerUrl string `protobuf:"bytes,2,opt,name=server_url,json=serverUrl,proto3" json:"server_url,omitempty"` Namespace string `protobuf:"bytes,3,opt,name=namespace,proto3" json:"namespace,omitempty"`