-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
SNOW-1023673 Implement common encode and serialize methods using OTEL…
… v0.12.0
- Loading branch information
1 parent
3d1dacb
commit 402120b
Showing
15 changed files
with
1,662 additions
and
25 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -43,4 +43,5 @@ jobs: | |
pip install . | ||
- name: Test with pytest | ||
run: | | ||
pip install ./tests/snowflake-telemetry-test-utils | ||
pytest |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -11,15 +11,27 @@ The package is a stub for the full functionality when running in Snowflake. | |
|
||
## Getting started | ||
|
||
To install this package, run | ||
To install the latest release of this package as an end user, run | ||
|
||
```bash | ||
$ git clone [email protected]:snowflakedb/snowflake-telemetry-python.git | ||
$ cd snowflake-telemetry-python | ||
|
||
$ python3 -m venv venv | ||
$ source venv/bin/activate | ||
$ pip install --upgrade pip | ||
$ pip install . | ||
VERSION="0.2.0" | ||
curl -L "https://github.com/snowflakedb/snowflake-telemetry-python/archive/refs/tags/v${VERSION}.tar.gz" > "snowflake-telemetry-python-${VERSION}.tar.gz" | ||
tar -xvf "snowflake-telemetry-python-${VERSION}.tar.gz" | ||
cd "snowflake-telemetry-python-${VERSION}" | ||
python3 -m venv .venv | ||
source .venv/bin/activate | ||
pip install --upgrade pip | ||
pip install . | ||
``` | ||
|
||
To develop this package, run | ||
|
||
```bash | ||
git clone [email protected]:snowflakedb/snowflake-telemetry-python.git | ||
cd snowflake-telemetry-python | ||
|
||
python3 -m venv .venv | ||
source .venv/bin/activate | ||
pip install --upgrade pip | ||
pip install . ./tests/snowflake-telemetry-test-utils | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,8 @@ | ||
import os | ||
from setuptools import setup | ||
|
||
from setuptools import ( | ||
find_namespace_packages, | ||
setup, | ||
) | ||
|
||
DESCRIPTION = 'Snowflake Telemetry Python' | ||
LONG_DESCRIPTION = 'This package provides a set of telemetry APIs for use in Snowflake' | ||
|
@@ -9,26 +11,24 @@ | |
VERSION = None | ||
with open(os.path.join(SNOWFLAKE_TELEMETRY_SRC_DIR, "version.py"), encoding="utf-8") as f: | ||
exec(f.read()) | ||
VERSION_STR = ".".join([str(v) for v in VERSION if v is not None]) | ||
|
||
|
||
setup( | ||
name="snowflake-telemetry-python", | ||
version=VERSION_STR, | ||
version=VERSION, | ||
author="Snowflake, Inc", | ||
author_email="[email protected]", | ||
description=DESCRIPTION, | ||
long_description=LONG_DESCRIPTION, | ||
install_requires=[ | ||
"setuptools >= 40.0.0", | ||
"opentelemetry-sdk == 1.12.0", | ||
"setuptools >= 40.0.0, < 66.0.0", | ||
"opentelemetry-api == 1.12.0", | ||
"opentelemetry-exporter-otlp == 1.12.0", | ||
"opentelemetry-sdk == 1.12.0", | ||
], | ||
namespace_packages=["snowflake"], | ||
packages=[ | ||
"snowflake.telemetry", | ||
], | ||
packages=find_namespace_packages( | ||
where='src' | ||
), | ||
package_dir={ | ||
"": "src", | ||
}, | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
24 changes: 24 additions & 0 deletions
24
src/snowflake/telemetry/_internal/encoder/otlp/proto/common/log_encoder.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
# | ||
# Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. | ||
# | ||
|
||
from logging import LogRecord | ||
from typing import Sequence | ||
|
||
from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest | ||
from opentelemetry.proto.logs.v1.logs_pb2 import LogsData | ||
|
||
|
||
def _encode_logs(batch: Sequence[LogRecord]) -> ExportLogsServiceRequest: | ||
# TODO fix this no-op implementation of encode_logs | ||
return ExportLogsServiceRequest(resource_logs=[]) | ||
|
||
def serialize_logs_data(batch: Sequence[LogRecord]) -> bytes: | ||
return LogsData( | ||
resource_logs=_encode_logs(batch).resource_logs | ||
).SerializeToString() | ||
|
||
|
||
__all__ = [ | ||
"serialize_logs_data", | ||
] |
25 changes: 25 additions & 0 deletions
25
src/snowflake/telemetry/_internal/encoder/otlp/proto/common/metrics_encoder.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
# | ||
# Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. | ||
# | ||
|
||
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter | ||
from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest | ||
from opentelemetry.proto.metrics.v1.metrics_pb2 import MetricsData as PB2MetricsData | ||
from opentelemetry.sdk.metrics.export import MetricsData | ||
|
||
|
||
_exporter = OTLPMetricExporter() | ||
|
||
def _encode_metrics(data: MetricsData) -> ExportMetricsServiceRequest: | ||
# Will no longer rely on _translate_data after we upgrade to v1.19.0 or later | ||
return _exporter._translate_data(data) | ||
|
||
def serialize_metrics_data(data: MetricsData) -> bytes: | ||
return PB2MetricsData( | ||
resource_metrics=_encode_metrics(data).resource_metrics | ||
).SerializeToString() | ||
|
||
|
||
__all__ = [ | ||
"serialize_metrics_data", | ||
] |
29 changes: 29 additions & 0 deletions
29
src/snowflake/telemetry/_internal/encoder/otlp/proto/common/trace_encoder.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
# | ||
# Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. | ||
# | ||
|
||
from typing import Sequence | ||
|
||
from opentelemetry.exporter.otlp.proto.http.trace_exporter.encoder import _ProtobufEncoder | ||
from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest | ||
from opentelemetry.proto.trace.v1.trace_pb2 import TracesData | ||
from opentelemetry.sdk.trace import ReadableSpan | ||
|
||
|
||
def _encode_spans( | ||
sdk_spans: Sequence[ReadableSpan], | ||
) -> ExportTraceServiceRequest: | ||
# Will no longer rely on _ProtobufEncoder after we upgrade to v1.19.0 or later | ||
return _ProtobufEncoder.encode(sdk_spans) | ||
|
||
def serialize_traces_data( | ||
sdk_spans: Sequence[ReadableSpan], | ||
) -> bytes: | ||
return TracesData( | ||
resource_spans=_encode_spans(sdk_spans).resource_spans | ||
).SerializeToString() | ||
|
||
|
||
__all__ = [ | ||
"serialize_traces_data", | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,7 @@ | ||
#!/usr/bin/env python3 | ||
# | ||
# Copyright (c) 2012-2022 Snowflake Computing Inc. All rights reserved. | ||
# Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. | ||
# | ||
|
||
"""Update this for the versions.""" | ||
VERSION = (0, 2, 0) | ||
VERSION = "0.2.1.dev" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
# Snowflake Telemetry Test Utils | ||
|
||
## About | ||
|
||
`snowflake-telemetry-test-utils` is a utility module that supports the test written for the Snowflake Telemetry Python project. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
import os | ||
from setuptools import ( | ||
find_namespace_packages, | ||
setup, | ||
) | ||
|
||
DESCRIPTION = 'Snowflake Telemetry Test Utils' | ||
LONG_DESCRIPTION = 'This package provides test utils for testing snowflake-telemetry-python' | ||
|
||
|
||
setup( | ||
name="snowflake-telemetry-test-utils", | ||
version="0.0.1.dev", | ||
author="Snowflake, Inc", | ||
author_email="[email protected]", | ||
description=DESCRIPTION, | ||
long_description=LONG_DESCRIPTION, | ||
install_requires=[ | ||
"pytest >= 7.0.0", | ||
"snowflake-telemetry-python == 0.2.1.dev", | ||
], | ||
packages=find_namespace_packages( | ||
where='src' | ||
), | ||
package_dir={ | ||
"": "src", | ||
}, | ||
keywords="Snowflake db database cloud analytics warehouse", | ||
classifiers=[ | ||
"Development Status :: 1 - Planning", | ||
"Environment :: Console", | ||
"Environment :: Other Environment", | ||
"Intended Audience :: Developers", | ||
"Intended Audience :: Education", | ||
"Intended Audience :: Information Technology", | ||
"Intended Audience :: System Administrators", | ||
"License :: OSI Approved :: Apache Software License", | ||
"Operating System :: OS Independent", | ||
"Programming Language :: SQL", | ||
"Programming Language :: Python :: 3.6", | ||
"Programming Language :: Python :: 3.7", | ||
"Programming Language :: Python :: 3.8", | ||
"Programming Language :: Python :: 3.9", | ||
"Programming Language :: Python :: 3.10", | ||
"Topic :: Database", | ||
"Topic :: Software Development", | ||
"Topic :: Software Development :: Libraries", | ||
"Topic :: Software Development :: Libraries :: Application Frameworks", | ||
"Topic :: Software Development :: Libraries :: Python Modules", | ||
"Topic :: Scientific/Engineering :: Information Analysis", | ||
], | ||
zip_safe=True, | ||
) |
Empty file.
100 changes: 100 additions & 0 deletions
100
tests/snowflake-telemetry-test-utils/src/snowflake/telemetry/test/metrictestutil.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,100 @@ | ||
# Copyright The OpenTelemetry Authors | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
|
||
from opentelemetry.attributes import BoundedAttributes | ||
from opentelemetry.sdk.metrics.export import ( | ||
AggregationTemporality, | ||
Gauge, | ||
Metric, | ||
NumberDataPoint, | ||
Sum, | ||
) | ||
|
||
|
||
def _generate_metric( | ||
name, data, attributes=None, description=None, unit=None | ||
) -> Metric: | ||
if description is None: | ||
description = "foo" | ||
if unit is None: | ||
unit = "s" | ||
return Metric( | ||
name=name, | ||
description=description, | ||
unit=unit, | ||
data=data, | ||
) | ||
|
||
|
||
def _generate_sum( | ||
name, | ||
value, | ||
attributes=None, | ||
description=None, | ||
unit=None, | ||
is_monotonic=True, | ||
) -> Metric: | ||
if attributes is None: | ||
attributes = BoundedAttributes(attributes={"a": 1, "b": True}) | ||
return _generate_metric( | ||
name, | ||
Sum( | ||
data_points=[ | ||
NumberDataPoint( | ||
attributes=attributes, | ||
start_time_unix_nano=1641946015139533244, | ||
time_unix_nano=1641946016139533244, | ||
value=value, | ||
) | ||
], | ||
aggregation_temporality=AggregationTemporality.CUMULATIVE, | ||
is_monotonic=is_monotonic, | ||
), | ||
description=description, | ||
unit=unit, | ||
) | ||
|
||
|
||
def _generate_gauge( | ||
name, value, attributes=None, description=None, unit=None | ||
) -> Metric: | ||
if attributes is None: | ||
attributes = BoundedAttributes(attributes={"a": 1, "b": True}) | ||
return _generate_metric( | ||
name, | ||
Gauge( | ||
data_points=[ | ||
NumberDataPoint( | ||
attributes=attributes, | ||
start_time_unix_nano=1641946015139533244, | ||
time_unix_nano=1641946016139533244, | ||
value=value, | ||
) | ||
], | ||
), | ||
description=description, | ||
unit=unit, | ||
) | ||
|
||
|
||
def _generate_unsupported_metric( | ||
name, attributes=None, description=None, unit=None | ||
) -> Metric: | ||
return _generate_metric( | ||
name, | ||
None, | ||
description=description, | ||
unit=unit, | ||
) |
Oops, something went wrong.