From 000b65c1eb364e952605e40540b55c354a1f7b66 Mon Sep 17 00:00:00 2001 From: barnettZQG Date: Mon, 17 Apr 2023 09:48:37 +0800 Subject: [PATCH] Feat: Support to compress and cache the static files (#777) * Feat: Support to compress and cache the static files Signed-off-by: barnettZQG * Fix: Code style Signed-off-by: barnettZQG * Fix: Code style Signed-off-by: barnettZQG --------- Signed-off-by: barnettZQG --- go.mod | 2 +- pkg/features/server.go | 3 + pkg/server/server.go | 19 +++- pkg/server/utils/filters/gzip.go | 69 ++++++++++++++ pkg/server/utils/filters/gzip_test.go | 50 +++++++++++ pkg/server/utils/filters/js-cache.go | 104 ++++++++++++++++++++++ pkg/server/utils/filters/js-cache_test.go | 71 +++++++++++++++ 7 files changed, 313 insertions(+), 5 deletions(-) create mode 100644 pkg/server/utils/filters/gzip.go create mode 100644 pkg/server/utils/filters/gzip_test.go create mode 100644 pkg/server/utils/filters/js-cache.go create mode 100644 pkg/server/utils/filters/js-cache_test.go diff --git a/go.mod b/go.mod index 870bf8f86..b71c87e25 100644 --- a/go.mod +++ b/go.mod @@ -109,6 +109,7 @@ require ( require github.com/oam-dev/kubevela v1.8.0-rc.1.0.20230414094557-fcd721ffed60 require ( + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da github.com/grafana/grafana v1.9.2-0.20230216173926-a0bea04a0274 github.com/julienschmidt/httprouter v1.3.0 ) @@ -181,7 +182,6 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect diff --git a/pkg/features/server.go b/pkg/features/server.go index 87886021a..b12c7c2a1 100644 --- a/pkg/features/server.go +++ b/pkg/features/server.go @@ -34,11 +34,14 @@ const ( APIServerEnableImpersonation featuregate.Feature = "EnableImpersonation" // APIServerEnableAdminImpersonation whether to disable User admin impersonation for APIServer APIServerEnableAdminImpersonation featuregate.Feature = "EnableAdminImpersonation" + // APIServerEnableCacheJSFile whether to cache the JS file to memory + APIServerEnableCacheJSFile featuregate.Feature = "EnableCacheJSFile" ) func init() { runtime.Must(APIServerMutableFeatureGate.Add(map[featuregate.Feature]featuregate.FeatureSpec{ APIServerEnableImpersonation: {Default: false, PreRelease: featuregate.Alpha}, APIServerEnableAdminImpersonation: {Default: true, PreRelease: featuregate.Alpha}, + APIServerEnableCacheJSFile: {Default: false, PreRelease: featuregate.Alpha}, })) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 52a512fcf..6badd913c 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -39,10 +39,11 @@ import ( "github.com/oam-dev/kubevela/apis/types" pkgaddon "github.com/oam-dev/kubevela/pkg/addon" pkgconfig "github.com/oam-dev/kubevela/pkg/config" - "github.com/oam-dev/kubevela/pkg/features" + pkgUtils "github.com/oam-dev/kubevela/pkg/utils" "github.com/oam-dev/kubevela/pkg/utils/apply" + "github.com/kubevela/velaux/pkg/features" "github.com/kubevela/velaux/pkg/plugin/proxy" "github.com/kubevela/velaux/pkg/plugin/router" plugintypes "github.com/kubevela/velaux/pkg/plugin/types" @@ -57,6 +58,7 @@ import ( "github.com/kubevela/velaux/pkg/server/utils" "github.com/kubevela/velaux/pkg/server/utils/bcode" "github.com/kubevela/velaux/pkg/server/utils/container" + "github.com/kubevela/velaux/pkg/server/utils/filters" ) const ( @@ -348,15 +350,22 @@ func enrichSwaggerObject(swo *spec.Swagger) { } func (s *restServer) ServeHTTP(res http.ResponseWriter, req *http.Request) { + staticFilters := []utils.FilterFunction{} + if features.APIServerFeatureGate.Enabled(features.APIServerEnableCacheJSFile) { + staticFilters = append(staticFilters, filters.JSCache) + } + staticFilters = append(staticFilters, filters.Gzip) switch { case strings.HasPrefix(req.URL.Path, SwaggerConfigRoutePath): s.webContainer.ServeHTTP(res, req) return case strings.HasPrefix(req.URL.Path, BuildPublicRoutePath): - s.staticFiles(res, req, "./") + utils.NewFilterChain(func(req *http.Request, res http.ResponseWriter) { + s.staticFiles(res, req, "./") + }, staticFilters...).ProcessFilter(req, res) return case strings.HasPrefix(req.URL.Path, PluginPublicRoutePath): - utils.NewFilterChain(s.getPluginAssets).ProcessFilter(req, res) + utils.NewFilterChain(s.getPluginAssets, staticFilters...).ProcessFilter(req, res) return case strings.HasPrefix(req.URL.Path, PluginProxyRoutePath): utils.NewFilterChain(s.proxyPluginBackend, api.AuthTokenCheck, api.AuthUserCheck(s.UserService)).ProcessFilter(req, res) @@ -373,7 +382,9 @@ func (s *restServer) ServeHTTP(res http.ResponseWriter, req *http.Request) { } // Rewrite to index.html, which means this route is handled by frontend. req.URL.Path = "/" - s.staticFiles(res, req, BuildPublicPath) + utils.NewFilterChain(func(req *http.Request, res http.ResponseWriter) { + s.staticFiles(res, req, BuildPublicPath) + }, staticFilters...).ProcessFilter(req, res) } } diff --git a/pkg/server/utils/filters/gzip.go b/pkg/server/utils/filters/gzip.go new file mode 100644 index 000000000..33e95fae1 --- /dev/null +++ b/pkg/server/utils/filters/gzip.go @@ -0,0 +1,69 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filters + +import ( + "net/http" + "strings" + + "github.com/emicklei/go-restful/v3" + "k8s.io/klog/v2" + + "github.com/kubevela/velaux/pkg/server/utils" +) + +// Gzip static file compression +func Gzip(req *http.Request, res http.ResponseWriter, chain *utils.FilterChain) { + doCompress, encoding := wantsCompressedResponse(req, res) + if doCompress { + w, err := restful.NewCompressingResponseWriter(res, encoding) + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + return + } + defer func() { + if err = w.Close(); err != nil { + klog.Errorf("failed to close the compressing writer, err: %s", err.Error()) + } + }() + chain.ProcessFilter(req, w) + return + } + chain.ProcessFilter(req, res) +} + +// WantsCompressedResponse reads the Accept-Encoding header to see if and which encoding is requested. +// It also inspects the httpWriter whether its content-encoding is already set (non-empty). +func wantsCompressedResponse(httpRequest *http.Request, httpWriter http.ResponseWriter) (bool, string) { + if contentEncoding := httpWriter.Header().Get(restful.HEADER_ContentEncoding); contentEncoding != "" { + return false, "" + } + header := httpRequest.Header.Get(restful.HEADER_AcceptEncoding) + gi := strings.Index(header, restful.ENCODING_GZIP) + zi := strings.Index(header, restful.ENCODING_DEFLATE) + // use in order of appearance + if gi == -1 { + return zi != -1, restful.ENCODING_DEFLATE + } + if zi == -1 { + return gi != -1, restful.ENCODING_GZIP + } + if gi < zi { + return true, restful.ENCODING_GZIP + } + return true, restful.ENCODING_DEFLATE +} diff --git a/pkg/server/utils/filters/gzip_test.go b/pkg/server/utils/filters/gzip_test.go new file mode 100644 index 000000000..5a9366749 --- /dev/null +++ b/pkg/server/utils/filters/gzip_test.go @@ -0,0 +1,50 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filters + +import ( + "compress/gzip" + "io" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/emicklei/go-restful/v3" + "github.com/stretchr/testify/assert" + + "github.com/kubevela/velaux/pkg/server/utils" +) + +func TestGZip(t *testing.T) { + chain := utils.NewFilterChain(loadJS, Gzip) + res1 := httptest.NewRecorder() + u, err := url.Parse("/test.js?v=1") + assert.Equal(t, err, nil) + reqHeader := http.Header{} + reqHeader.Set(restful.HEADER_AcceptEncoding, restful.ENCODING_GZIP) + chain.ProcessFilter(&http.Request{Method: "GET", URL: u, Header: reqHeader}, res1) + assert.Equal(t, res1.Code, 200) + assert.Equal(t, res1.HeaderMap.Get(restful.HEADER_ContentEncoding), restful.ENCODING_GZIP) + + // Gzip decode + reader, err := gzip.NewReader(res1.Body) + assert.Equal(t, err, nil) + body, err := io.ReadAll(reader) + assert.Equal(t, err, nil) + assert.Equal(t, string(body), jsContent) +} diff --git a/pkg/server/utils/filters/js-cache.go b/pkg/server/utils/filters/js-cache.go new file mode 100644 index 000000000..731d742ff --- /dev/null +++ b/pkg/server/utils/filters/js-cache.go @@ -0,0 +1,104 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filters + +import ( + "bytes" + "net/http" + "strings" + + "github.com/golang/groupcache/lru" + "k8s.io/klog/v2" + + "github.com/kubevela/velaux/pkg/server/utils" +) + +var jsFileCache = lru.New(100) + +// HeaderHitCache the header key +var HeaderHitCache = "Hit-Cache" + +func matchCacheCondition(req *http.Request) bool { + return strings.HasSuffix(req.URL.Path, ".js") && req.Method == "GET" +} + +// JSCache cache the JS static file. +func JSCache(req *http.Request, res http.ResponseWriter, chain *utils.FilterChain) { + if matchCacheCondition(req) { + if value, ok := jsFileCache.Get(req.URL.String()); ok { + if cacheData, ok := value.(*cacheData); ok { + cacheData.Write(res) + return + } + } + } + + if matchCacheCondition(req) { + res.Header().Set(HeaderHitCache, "false") + cacheWriter := &CacheWriter{writer: res, cacheData: &cacheData{}} + chain.ProcessFilter(req, cacheWriter) + jsFileCache.Add(req.URL.String(), cacheWriter.cacheData) + return + } + chain.ProcessFilter(req, res) +} + +type cacheData struct { + code int + data bytes.Buffer + header http.Header +} + +func (c *cacheData) Write(w http.ResponseWriter) { + for k, values := range c.header { + for _, value := range values { + w.Header().Add(k, value) + } + } + w.Header().Set(HeaderHitCache, "true") + w.WriteHeader(c.code) + if _, err := w.Write(c.data.Bytes()); err != nil { + klog.Errorf("failed to write the cache content, err: %s", err.Error()) + } +} + +// CacheWriter generate the cache item the response body and status +type CacheWriter struct { + writer http.ResponseWriter + cacheData *cacheData +} + +// Header cache the header +func (c *CacheWriter) Header() http.Header { + header := c.writer.Header() + c.cacheData.header = header + return header +} + +// Write cache the data +func (c *CacheWriter) Write(b []byte) (int, error) { + if _, err := c.cacheData.data.Write(b); err != nil { + return -1, err + } + return c.writer.Write(b) +} + +// WriteHeader cache the status code +func (c *CacheWriter) WriteHeader(statusCode int) { + c.writer.WriteHeader(statusCode) + c.cacheData.code = statusCode +} diff --git a/pkg/server/utils/filters/js-cache_test.go b/pkg/server/utils/filters/js-cache_test.go new file mode 100644 index 000000000..a32e2d67a --- /dev/null +++ b/pkg/server/utils/filters/js-cache_test.go @@ -0,0 +1,71 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filters + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/emicklei/go-restful/v3" + "github.com/stretchr/testify/assert" + + "github.com/kubevela/velaux/pkg/server/utils" +) + +func TestJSCache(t *testing.T) { + chain := utils.NewFilterChain(loadJS, JSCache) + res1 := httptest.NewRecorder() + u, err := url.Parse("/test.js?v=1") + assert.Equal(t, err, nil) + chain.ProcessFilter(&http.Request{Method: "GET", URL: u}, res1) + assert.Equal(t, res1.Code, 200) + assert.Equal(t, res1.Body.String(), jsContent) + assert.Equal(t, res1.HeaderMap.Get(HeaderHitCache), "false") + assert.Equal(t, jsFileCache.Len(), 1) + data, ok := jsFileCache.Get(u.String()) + assert.Equal(t, ok, true) + assert.Equal(t, data.(*cacheData).data.String(), jsContent) + + res2 := httptest.NewRecorder() + chain = utils.NewFilterChain(loadJS, JSCache) + chain.ProcessFilter(&http.Request{Method: "GET", URL: u}, res2) + assert.Equal(t, res2.Code, 200) + assert.Equal(t, res2.Body.String(), jsContent) + + assert.Equal(t, res2.HeaderMap.Get(HeaderHitCache), "true") + + u2, err := url.Parse("/test.js?v=2") + assert.Equal(t, err, nil) + res3 := httptest.NewRecorder() + chain = utils.NewFilterChain(loadJS, JSCache) + chain.ProcessFilter(&http.Request{Method: "GET", URL: u2}, res3) + assert.Equal(t, res3.Code, 200) + assert.Equal(t, res3.Body.String(), jsContent) + assert.Equal(t, res3.HeaderMap.Get(HeaderHitCache), "false") +} + +var jsContent = "console.log(\"hello\")" + +func loadJS(req *http.Request, res http.ResponseWriter) { + fmt.Printf("miss cache,path:%s \n", req.URL.String()) + res.WriteHeader(200) + res.Write([]byte(jsContent)) + res.Header().Add(restful.HEADER_ContentType, "application/javascript") +}