Skip to content

Commit

Permalink
[EXPORTER] Gzip compression support for OTLP/HTTP and OTLP/gRPC expor…
Browse files Browse the repository at this point in the history
…ter (#2530)
  • Loading branch information
perhapsmaple authored Feb 29, 2024
1 parent eaaf6cd commit 07f6cb5
Show file tree
Hide file tree
Showing 39 changed files with 402 additions and 22 deletions.
13 changes: 13 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,19 @@ Increment the:

## [Unreleased]

* [EXPORTER] Gzip compression support for OTLP/HTTP and OTLP/gRPC exporter
[#2530](https://github.com/open-telemetry/opentelemetry-cpp/pull/2530)

Important changes:

* [EXPORTER] Gzip compression support for OTLP/HTTP and OTLP/gRPC exporter
[#2530](https://github.com/open-telemetry/opentelemetry-cpp/pull/2530)
* In the `OtlpHttpExporterOptions` and `OtlpGrpcExporterOptions`, a new
field called compression has been introduced. This field can be set
to "gzip” to enable gzip compression.
* The CMake option `WITH_OTLP_HTTP_COMPRESSION` is introduced to enable
gzip compression support for the OTLP HTTP Exporter and includes a
dependency on zlib.
* [SDK] Change OTLP HTTP content_type default to binary
[#2558](https://github.com/open-telemetry/opentelemetry-cpp/pull/2558)

Expand Down
17 changes: 17 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,11 @@ option(WITH_OTLP_GRPC "Whether to include the OTLP gRPC exporter in the SDK"
option(WITH_OTLP_HTTP "Whether to include the OTLP http exporter in the SDK"
OFF)

option(
WITH_OTLP_HTTP_COMPRESSION
"Whether to include gzip compression for the OTLP http exporter in the SDK"
OFF)

option(WITH_ZIPKIN "Whether to include the Zipkin exporter in the SDK" OFF)

option(WITH_PROMETHEUS "Whether to include the Prometheus Client in the SDK"
Expand Down Expand Up @@ -448,6 +453,18 @@ if((NOT WITH_API_ONLY) AND WITH_HTTP_CLIENT_CURL)
message(STATUS "Found CURL: ${CURL_LIBRARIES}, version ${CURL_VERSION}")
endif()

#
# Do we need ZLIB ?
#

if((NOT WITH_API_ONLY)
AND WITH_HTTP_CLIENT_CURL
AND WITH_OTLP_HTTP_COMPRESSION)
# No specific version required.
find_package(ZLIB REQUIRED)
message(STATUS "Found ZLIB: ${ZLIB_LIBRARIES}, version ${ZLIB_VERSION}")
endif()

#
# Do we need NLOHMANN_JSON ?
#
Expand Down
5 changes: 5 additions & 0 deletions api/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,11 @@ if(WITH_METRICS_EXEMPLAR_PREVIEW)
INTERFACE ENABLE_METRICS_EXEMPLAR_PREVIEW)
endif()

if(WITH_OTLP_HTTP_COMPRESSION)
target_compile_definitions(opentelemetry_api
INTERFACE ENABLE_OTLP_COMPRESSION_PREVIEW)
endif()

include(${PROJECT_SOURCE_DIR}/cmake/pkgconfig.cmake)

if(OPENTELEMETRY_INSTALL)
Expand Down
13 changes: 13 additions & 0 deletions bazel/repository.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -183,3 +183,16 @@ def opentelemetry_cpp_deps():
"https://github.com/opentracing/opentracing-cpp/archive/refs/tags/v1.6.0.tar.gz",
],
)

# Zlib (optional)
maybe(
http_archive,
name = "zlib",
build_file = "@io_opentelemetry_cpp//bazel:zlib.BUILD",
sha256 = "d14c38e313afc35a9a8760dadf26042f51ea0f5d154b0630a31da0540107fb98",
strip_prefix = "zlib-1.2.13",
urls = [
"https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.xz",
"https://zlib.net/zlib-1.2.13.tar.xz",
],
)
74 changes: 74 additions & 0 deletions bazel/zlib.BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# Copyright The OpenTelemetry Authors
# SPDX-License-Identifier: Apache-2.0

# Builds ZLIB from a distribution.
# Copied from https://github.com/protocolbuffers/protobuf/blob/master/third_party/zlib.BUILD

licenses(["notice"]) # BSD/MIT-like license (for zlib)

exports_files(["zlib.BUILD"])

_ZLIB_HEADERS = [
"crc32.h",
"deflate.h",
"gzguts.h",
"inffast.h",
"inffixed.h",
"inflate.h",
"inftrees.h",
"trees.h",
"zconf.h",
"zlib.h",
"zutil.h",
]

_ZLIB_PREFIXED_HEADERS = ["zlib/include/" + hdr for hdr in _ZLIB_HEADERS]

# In order to limit the damage from the `includes` propagation
# via `:zlib`, copy the public headers to a subdirectory and
# expose those.
genrule(
name = "copy_public_headers",
srcs = _ZLIB_HEADERS,
outs = _ZLIB_PREFIXED_HEADERS,
cmd_bash = "cp $(SRCS) $(@D)/zlib/include/",
cmd_bat = " && ".join(
["@copy /Y $(location %s) $(@D)\\zlib\\include\\ >NUL" %
s for s in _ZLIB_HEADERS],
),
)

cc_library(
name = "zlib",
srcs = [
"adler32.c",
"compress.c",
"crc32.c",
"deflate.c",
"gzclose.c",
"gzlib.c",
"gzread.c",
"gzwrite.c",
"infback.c",
"inffast.c",
"inflate.c",
"inftrees.c",
"trees.c",
"uncompr.c",
"zutil.c",
# Include the un-prefixed headers in srcs to work
# around the fact that zlib isn't consistent in its
# choice of <> or "" delimiter when including itself.
] + _ZLIB_HEADERS,
hdrs = _ZLIB_PREFIXED_HEADERS,
copts = select({
"@platforms//os:windows": [],
"//conditions:default": [
"-Wno-deprecated-non-prototype",
"-Wno-unused-variable",
"-Wno-implicit-function-declaration",
],
}),
includes = ["zlib/include/"],
visibility = ["//visibility:public"],
)
4 changes: 4 additions & 0 deletions ci/do_ci.sh
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ elif [[ "$1" == "cmake.maintainer.sync.test" ]]; then
-DWITH_ASYNC_EXPORT_PREVIEW=OFF \
-DOTELCPP_MAINTAINER_MODE=ON \
-DWITH_NO_DEPRECATED_CODE=ON \
-DWITH_OTLP_HTTP_COMPRESSION=ON \
${IWYU} \
"${SRC_DIR}"
eval "$MAKE_COMMAND"
Expand All @@ -140,6 +141,7 @@ elif [[ "$1" == "cmake.maintainer.async.test" ]]; then
-DWITH_ASYNC_EXPORT_PREVIEW=ON \
-DOTELCPP_MAINTAINER_MODE=ON \
-DWITH_NO_DEPRECATED_CODE=ON \
-DWITH_OTLP_HTTP_COMPRESSION=ON \
${IWYU} \
"${SRC_DIR}"
eval "$MAKE_COMMAND"
Expand All @@ -161,6 +163,7 @@ elif [[ "$1" == "cmake.maintainer.cpp11.async.test" ]]; then
-DWITH_ASYNC_EXPORT_PREVIEW=ON \
-DOTELCPP_MAINTAINER_MODE=ON \
-DWITH_NO_DEPRECATED_CODE=ON \
-DWITH_OTLP_HTTP_COMPRESSION=ON \
"${SRC_DIR}"
make -k -j $(nproc)
make test
Expand All @@ -182,6 +185,7 @@ elif [[ "$1" == "cmake.maintainer.abiv2.test" ]]; then
-DWITH_NO_DEPRECATED_CODE=ON \
-DWITH_ABI_VERSION_1=OFF \
-DWITH_ABI_VERSION_2=ON \
-DWITH_OTLP_HTTP_COMPRESSION=ON \
${IWYU} \
"${SRC_DIR}"
eval "$MAKE_COMMAND"
Expand Down
6 changes: 6 additions & 0 deletions docs/dependencies.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,12 @@ Both these dependencies are listed here:
- protobuf serialized otlp messages are encoded in JSON format using this
library.
- License: `MIT License`
- [zlib](https://www.zlib.net/): A Massively Spiffy Yet Delicately
Unobtrusive Compression Library.
- The `http_client` utilizes zlib to compress the message body and send
it in gzip format.
- License: The library is licensed
[here](https://www.zlib.net/zlib_license.html)

- [OTLP/gRPC](/exporters/otlp)
exporter:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,10 @@ inline OtlpHeaders GetOtlpDefaultHeaders()
return GetOtlpDefaultTracesHeaders();
}

std::string GetOtlpDefaultTracesCompression();
std::string GetOtlpDefaultMetricsCompression();
std::string GetOtlpDefaultLogsCompression();

} // namespace otlp
} // namespace exporter
OPENTELEMETRY_END_NAMESPACE
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ struct OtlpGrpcClientOptions
/** max number of threads that can be allocated from this */
std::size_t max_threads;

/** Compression type. */
std::string compression;

#ifdef ENABLE_ASYNC_EXPORT
// Concurrent requests
std::size_t max_concurrent_requests;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ struct OtlpHttpClientOptions
// This option is ignored if content_type is not kJson
JsonBytesMappingKind json_bytes_mapping = JsonBytesMappingKind::kHexId;

// By default, do not compress data
std::string compression = "none";

// If using the json name of protobuf field to set the key of json. By default, we will use the
// field name just like proto files.
bool use_json_name = false;
Expand Down Expand Up @@ -94,6 +97,7 @@ struct OtlpHttpClientOptions
nostd::string_view input_ssl_cipher_suite,
HttpRequestContentType input_content_type,
JsonBytesMappingKind input_json_bytes_mapping,
nostd::string_view input_compression,
bool input_use_json_name,
bool input_console_debug,
std::chrono::system_clock::duration input_timeout,
Expand All @@ -116,6 +120,7 @@ struct OtlpHttpClientOptions
input_ssl_cipher_suite),
content_type(input_content_type),
json_bytes_mapping(input_json_bytes_mapping),
compression(input_compression),
use_json_name(input_use_json_name),
console_debug(input_console_debug),
timeout(input_timeout),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,9 @@ struct OPENTELEMETRY_EXPORT OtlpHttpExporterOptions

/** TLS cipher suite. */
std::string ssl_cipher_suite;

/** Compression type. */
std::string compression;
};

} // namespace otlp
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,9 @@ struct OPENTELEMETRY_EXPORT OtlpHttpLogRecordExporterOptions

/** TLS cipher suite. */
std::string ssl_cipher_suite;

/** Compression type. */
std::string compression;
};

} // namespace otlp
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,9 @@ struct OPENTELEMETRY_EXPORT OtlpHttpMetricExporterOptions

/** TLS cipher suite. */
std::string ssl_cipher_suite;

/** Compression type. */
std::string compression;
};

} // namespace otlp
Expand Down
51 changes: 51 additions & 0 deletions exporters/otlp/src/otlp_environment.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1069,6 +1069,57 @@ OtlpHeaders GetOtlpDefaultLogsHeaders()
return GetHeaders(kSignalEnv, kGenericEnv);
}

std::string GetOtlpDefaultTracesCompression()
{
constexpr char kSignalEnv[] = "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION";
constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_COMPRESSION";

std::string value;
bool exists;

exists = GetStringDualEnvVar(kSignalEnv, kGenericEnv, value);
if (exists)
{
return value;
}

return std::string{"none"};
}

std::string GetOtlpDefaultMetricsCompression()
{
constexpr char kSignalEnv[] = "OTEL_EXPORTER_OTLP_METRICS_COMPRESSION";
constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_COMPRESSION";

std::string value;
bool exists;

exists = GetStringDualEnvVar(kSignalEnv, kGenericEnv, value);
if (exists)
{
return value;
}

return std::string{"none"};
}

std::string GetOtlpDefaultLogsCompression()
{
constexpr char kSignalEnv[] = "OTEL_EXPORTER_OTLP_LOGS_COMPRESSION";
constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_COMPRESSION";

std::string value;
bool exists;

exists = GetStringDualEnvVar(kSignalEnv, kGenericEnv, value);
if (exists)
{
return value;
}

return std::string{"none"};
}

} // namespace otlp
} // namespace exporter
OPENTELEMETRY_END_NAMESPACE
5 changes: 5 additions & 0 deletions exporters/otlp/src/otlp_grpc_client.cc
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,11 @@ std::shared_ptr<grpc::Channel> OtlpGrpcClient::MakeChannel(const OtlpGrpcClientO
grpc_arguments.SetResourceQuota(quota);
}

if (options.compression == "gzip")
{
grpc_arguments.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP);
}

if (options.use_ssl_credentials)
{
grpc::SslCredentialsOptions ssl_opts;
Expand Down
2 changes: 2 additions & 0 deletions exporters/otlp/src/otlp_grpc_exporter_options.cc
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ OtlpGrpcExporterOptions::OtlpGrpcExporterOptions()
user_agent = GetOtlpDefaultUserAgent();

max_threads = 0;

compression = GetOtlpDefaultTracesCompression();
#ifdef ENABLE_ASYNC_EXPORT
max_concurrent_requests = 64;
#endif
Expand Down
2 changes: 2 additions & 0 deletions exporters/otlp/src/otlp_grpc_log_record_exporter_options.cc
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ OtlpGrpcLogRecordExporterOptions::OtlpGrpcLogRecordExporterOptions()
user_agent = GetOtlpDefaultUserAgent();

max_threads = 0;

compression = GetOtlpDefaultLogsCompression();
#ifdef ENABLE_ASYNC_EXPORT
max_concurrent_requests = 64;
#endif
Expand Down
2 changes: 2 additions & 0 deletions exporters/otlp/src/otlp_grpc_metric_exporter_options.cc
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ OtlpGrpcMetricExporterOptions::OtlpGrpcMetricExporterOptions()
aggregation_temporality = PreferredAggregationTemporality::kCumulative;

max_threads = 0;

compression = GetOtlpDefaultMetricsCompression();
#ifdef ENABLE_ASYNC_EXPORT
max_concurrent_requests = 64;
#endif
Expand Down
5 changes: 5 additions & 0 deletions exporters/otlp/src/otlp_http_client.cc
Original file line number Diff line number Diff line change
Expand Up @@ -958,6 +958,11 @@ OtlpHttpClient::createSession(
request->ReplaceHeader("Content-Type", content_type);
request->ReplaceHeader("User-Agent", options_.user_agent);

if (options_.compression == "gzip")
{
request->SetCompression(opentelemetry::ext::http::client::Compression::kGzip);
}

// Returns the created session data
return HttpSessionData{
std::move(session),
Expand Down
1 change: 1 addition & 0 deletions exporters/otlp/src/otlp_http_exporter.cc
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ OtlpHttpExporter::OtlpHttpExporter(const OtlpHttpExporterOptions &options)
options.ssl_cipher_suite,
options.content_type,
options.json_bytes_mapping,
options.compression,
options.use_json_name,
options.console_debug,
options.timeout,
Expand Down
Loading

2 comments on commit 07f6cb5

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Performance Alert ⚠️

Possible performance regression was detected for benchmark 'OpenTelemetry-cpp exporters Benchmark'.
Benchmark result of this commit is worse than the previous benchmark result exceeding threshold 2.

Benchmark suite Current: 07f6cb5 Previous: eaaf6cd Ratio
BM_OtlpExporterSparseSpans 191.85832504635312 ns/iter 93.81658777414067 ns/iter 2.05
BM_OtlpExporterDenseSpans 1331.227440987864 ns/iter 639.1109396877898 ns/iter 2.08

This comment was automatically generated by workflow using github-action-benchmark.

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Performance Alert ⚠️

Possible performance regression was detected for benchmark 'OpenTelemetry-cpp sdk Benchmark'.
Benchmark result of this commit is worse than the previous benchmark result exceeding threshold 2.

Benchmark suite Current: 07f6cb5 Previous: eaaf6cd Ratio
BM_BaselineBuffer/1 5855917.930603027 ns/iter 1993003.6067962646 ns/iter 2.94

This comment was automatically generated by workflow using github-action-benchmark.

Please sign in to comment.