This repository has been archived by the owner on Nov 20, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Qatzstd to Release-1.21-intel (#262)
* zstd compressor refactor * add qatzstd to contrib * fix decompressor * support qatzstd * add qatzstd test * fix api * use logger id compression * add doc and format * trim whitespace * fix memcpy * remove dictionary due to not support for qatzstd * fix BUILD * fix proto format * fix docs * fix clang build * fix unused parameter * remove unused vars * use operator instead if * use protected * fix comments * add more api comments * fix proto format Signed-off-by: giantcroc <[email protected]>
- Loading branch information
1 parent
93f8e2e
commit 75a4160
Showing
37 changed files
with
965 additions
and
68 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
9 changes: 9 additions & 0 deletions
9
api/contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha/BUILD
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| # DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. | ||
|
|
||
| load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") | ||
|
|
||
| licenses(["notice"]) # Apache 2 | ||
|
|
||
| api_proto_package( | ||
| deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], | ||
| ) |
69 changes: 69 additions & 0 deletions
69
api/contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha/qatzstd.proto
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| syntax = "proto3"; | ||
|
|
||
| package envoy.extensions.compression.qatzstd.compressor.v3alpha; | ||
|
|
||
| import "google/protobuf/wrappers.proto"; | ||
|
|
||
| import "udpa/annotations/status.proto"; | ||
| import "validate/validate.proto"; | ||
|
|
||
| option java_package = "io.envoyproxy.envoy.extensions.compression.qatzstd.compressor.v3alpha"; | ||
| option java_outer_classname = "QatzstdProto"; | ||
| option java_multiple_files = true; | ||
| option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha"; | ||
| option (udpa.annotations.file_status).package_version_status = ACTIVE; | ||
|
|
||
| // [#protodoc-title: Qatzstd Compressor] | ||
| // Qatzstd :ref:`configuration overview <config_qatzstd>`. | ||
| // [#extension: envoy.compression.qatzstd.compressor] | ||
|
|
||
| // [#next-free-field: 8] | ||
| message Qatzstd { | ||
| // Reference to http://facebook.github.io/zstd/zstd_manual.html | ||
| enum Strategy { | ||
| DEFAULT = 0; | ||
| FAST = 1; | ||
| DFAST = 2; | ||
| GREEDY = 3; | ||
| LAZY = 4; | ||
| LAZY2 = 5; | ||
| BTLAZY2 = 6; | ||
| BTOPT = 7; | ||
| BTULTRA = 8; | ||
| BTULTRA2 = 9; | ||
| } | ||
|
|
||
| // Set compression parameters according to pre-defined compression level table. | ||
| // Note that exact compression parameters are dynamically determined, | ||
| // depending on both compression level and source content size (when known). | ||
| // Value 0 means default, and default level is 3. | ||
| // | ||
| // Setting a level does not automatically set all other compression parameters | ||
| // to default. Setting this will however eventually dynamically impact the compression | ||
| // parameters which have not been manually set. The manually set | ||
| // ones will 'stick'. | ||
| google.protobuf.UInt32Value compression_level = 1 [(validate.rules).uint32 = {lte: 22 gte: 1}]; | ||
|
|
||
| // A 32-bits checksum of content is written at end of frame. If not set, defaults to false. | ||
| bool enable_checksum = 2; | ||
|
|
||
| // The higher the value of selected strategy, the more complex it is, | ||
| // resulting in stronger and slower compression. | ||
| // | ||
| // Special: value 0 means "use default strategy". | ||
| Strategy strategy = 3 [(validate.rules).enum = {defined_only: true}]; | ||
|
|
||
| // Value for compressor's next output buffer. If not set, defaults to 4096. | ||
| google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; | ||
|
|
||
| // Enable QAT to accelerate Zstd compression or not. If not set, defaults to false. | ||
| // | ||
| // This is useful in the case that users want to enable QAT for a period of time and disable QAT for another period of time, | ||
| // they don't have to change the config too much or prepare for another config that has software zstd compressor and just changing the value of this filed. | ||
| bool enable_qat_zstd = 6; | ||
|
|
||
| // Fallback to software for Qatzstd when input size is less than this value. | ||
| // Valid only ``enable_qat_zstd`` is ``true``. 0 means no fallback at all. If not set, defaults to 4000. | ||
| google.protobuf.UInt32Value qat_zstd_fallback_threshold = 7 | ||
| [(validate.rules).uint32 = {lte: 65536 gte: 0}]; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| diff --git a/src/Makefile b/src/Makefile | ||
| index 1abf10d..a0c7e9a 100644 | ||
| --- a/src/Makefile | ||
| +++ b/src/Makefile | ||
| @@ -41,6 +41,7 @@ LIBDIR ?= $(INSTALLDIR)/lib | ||
| INCLUDEDIR ?= $(INSTALLDIR)/include | ||
|
|
||
| CP ?= cp | ||
| +MKDIR ?= mkdir | ||
|
|
||
| ENABLE_USDM_DRV ?= 0 | ||
| ifneq ($(ICP_ROOT), ) | ||
| @@ -55,10 +56,8 @@ ifneq ($(ICP_ROOT), ) | ||
| else | ||
| QATFLAGS = -DINTREE | ||
| LDFLAGS = -lqat | ||
| - ifneq ($(ENABLE_USDM_DRV), 0) | ||
| - QATFLAGS += -DENABLE_USDM_DRV | ||
| - LDFLAGS += -lusdm | ||
| - endif | ||
| + QATFLAGS += -DENABLE_USDM_DRV | ||
| + LDFLAGS += -lusdm | ||
| endif | ||
|
|
||
| ifdef ZSTDLIB | ||
| @@ -69,8 +68,8 @@ CFLAGS += -Wall -Werror -Wextra -Wcast-align -Wshadow -Wstrict-aliasing=1 \ | ||
| -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ | ||
| -Wundef -Wpointer-arith -Wvla -Wformat=2 -Winit-self \ | ||
| -Wfloat-equal -Wwrite-strings -Wredundant-decls -Wc++-compat \ | ||
| - -pedantic -fstack-protector-strong -fPIE -fPIC \ | ||
| - -fno-delete-null-pointer-checks -fwrapv -fno-strict-overflow | ||
| + -pedantic -fstack-protector-strong \ | ||
| + -fno-delete-null-pointer-checks -fwrapv | ||
|
|
||
| DEBUGLEVEL ?=0 | ||
|
|
||
| @@ -81,27 +80,30 @@ else | ||
| QATFLAGS += -O3 | ||
| endif | ||
|
|
||
| +$(info INSTALLDIR="$(INSTALLDIR)") | ||
| +$(info CPPFLAGS="$(CPPFLAGS)") | ||
| + | ||
| qatseqprod.o: qatseqprod.c | ||
| - $(CC) -c $(CFLAGS) $(QATFLAGS) $(DEBUGFLAGS) $^ -o $@ | ||
| + $(CC) -c $(CPPFLAGS) $(CFLAGS) $(QATFLAGS) $(DEBUGFLAGS) $^ -o $@ | ||
|
|
||
| lib: qatseqprod.o | ||
| $(AR) rc libqatseqprod.a $^ | ||
| - $(CC) -shared $^ $(LDFLAGS) -o libqatseqprod.so | ||
| + @echo qatseqprod library successfully build | ||
|
|
||
| .PHONY: install | ||
| install: lib | ||
| + $(MKDIR) -p $(LIBDIR) | ||
| + $(MKDIR) -p $(INCLUDEDIR) | ||
| $(CP) libqatseqprod.a $(LIBDIR) | ||
| - $(CP) libqatseqprod.so $(LIBDIR) | ||
| $(CP) qatseqprod.h $(INCLUDEDIR) | ||
| @echo qatseqprod library successfully installed | ||
|
|
||
| .PHONY: uninstall | ||
| uninstall: | ||
| $(RM) $(LIBDIR)/libqatseqprod.a | ||
| - $(RM) $(LIBDIR)/libqatseqprod.so | ||
| $(RM) $(INCLUDEDIR)/qatseqprod.h | ||
| @echo qatseqprod library successfully uninstalled | ||
|
|
||
| clean: | ||
| $(RM) *.o | ||
| - $(RM) libqatseqprod.a libqatseqprod.so | ||
| + $(RM) libqatseqprod.a | ||
| diff --git a/test/Makefile b/test/Makefile | ||
| index dff0c8e..4ba01b2 100644 | ||
| --- a/test/Makefile | ||
| +++ b/test/Makefile | ||
| @@ -34,7 +34,7 @@ | ||
| # ####################################################################### | ||
| LIB = ../src | ||
|
|
||
| -LDFLAGS = $(LIB)/libqatseqprod.a -I$(LIB) | ||
| +LDFLAGS = $(LIB)/libqatseqprod.a -I$(LIB) -L$(LIB) -l:libqatseqprod.a -l:libqat.a -l:libusdm.a -l:libzstd.a -lpthread -lcrypto | ||
|
|
||
| ifneq ($(ICP_ROOT), ) | ||
| LDFLAGS += -lqat_s -lusdm_drv_s -Wl,-rpath,$(ICP_ROOT)/build -L$(ICP_ROOT)/build |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| load("@rules_foreign_cc//foreign_cc:defs.bzl", "make") | ||
| load( | ||
| "//bazel:envoy_build_system.bzl", | ||
| "envoy_cc_contrib_extension", | ||
| "envoy_cc_library", | ||
| "envoy_contrib_package", | ||
| ) | ||
|
|
||
| licenses(["notice"]) # Apache 2 | ||
|
|
||
| envoy_contrib_package() | ||
|
|
||
| make( | ||
| name = "qat-zstd", | ||
| build_data = ["@com_github_qat_zstd//:all"], | ||
| env = select({ | ||
| "//bazel:clang_build": { | ||
| "CFLAGS": "-Wno-error=unused-parameter", | ||
| }, | ||
| "//conditions:default": {}, | ||
| }), | ||
| includes = [], | ||
| lib_source = "@com_github_qat_zstd//:all", | ||
| out_static_libs = ["libqatseqprod.a"], | ||
| tags = ["skip_on_windows"], | ||
| target_compatible_with = [ | ||
| "@platforms//os:linux", | ||
| "@platforms//cpu:x86_64", | ||
| ], | ||
| targets = [ | ||
| "", | ||
| "install", | ||
| ], | ||
| deps = [ | ||
| "//contrib/qat:qatlib", | ||
| "//external:zstd", | ||
| ], | ||
| ) | ||
|
|
||
| envoy_cc_library( | ||
| name = "compressor_lib", | ||
| srcs = ["qatzstd_compressor_impl.cc"], | ||
| hdrs = ["qatzstd_compressor_impl.h"], | ||
| deps = [ | ||
| ":qat-zstd", | ||
| "//envoy/compression/compressor:compressor_interface", | ||
| "//envoy/server:factory_context_interface", | ||
| "//source/common/buffer:buffer_lib", | ||
| "//source/common/compression/zstd/common:zstd_base_lib", | ||
| "//source/common/compression/zstd/compressor:compressor_base", | ||
| ], | ||
| ) | ||
|
|
||
| envoy_cc_contrib_extension( | ||
| name = "config", | ||
| srcs = ["config.cc"], | ||
| hdrs = ["config.h"], | ||
| deps = [ | ||
| ":compressor_lib", | ||
| ":qat-zstd", | ||
| "//envoy/event:dispatcher_interface", | ||
| "//envoy/thread_local:thread_local_interface", | ||
| "//source/common/http:headers_lib", | ||
| "//source/extensions/compression/common/compressor:compressor_factory_base_lib", | ||
| "@envoy_api//contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha:pkg_cc_proto", | ||
| ], | ||
| ) |
Oops, something went wrong.