SNOW-1023673 Implement common encode and serialize methods using OTEL…

… v0.12.0

.github/workflows/build-test.yml

@@ -43,4 +43,5 @@ jobs:
         pip install .
     - name: Test with pytest
       run: |
+        pip install ./tests/snowflake-telemetry-test-utils
         pytest

README.md

@@ -11,15 +11,27 @@ The package is a stub for the full functionality when running in Snowflake.
 
 ## Getting started
 
-To install this package, run
+To install the latest release of this package as an end user, run
 
 ```bash
-$ git clone [email protected]:snowflakedb/snowflake-telemetry-python.git
-$ cd snowflake-telemetry-python
-
-$ python3 -m venv venv
-$ source venv/bin/activate
-$ pip install --upgrade pip
-$ pip install .
+VERSION="0.2.0"
+curl -L "https://github.com/snowflakedb/snowflake-telemetry-python/archive/refs/tags/v${VERSION}.tar.gz" > "snowflake-telemetry-python-${VERSION}.tar.gz"
+tar -xvf "snowflake-telemetry-python-${VERSION}.tar.gz"
+cd "snowflake-telemetry-python-${VERSION}"
+python3 -m venv venv
+source venv/bin/activate
+pip install --upgrade pip
+pip install .
 ```
 
+To develop this package, run
+
+```bash
+git clone [email protected]:snowflakedb/snowflake-telemetry-python.git
+cd snowflake-telemetry-python
+
+python3 -m venv venv
+source venv/bin/activate
+pip install --upgrade pip
+pip install . ./tests/snowflake-telemetry-test-utils
+```

setup.py

@@ -1,6 +1,8 @@
 import os
-from setuptools import setup
-
+from setuptools import (
+    find_namespace_packages,
+    setup,
+)
 
 DESCRIPTION = 'Snowflake Telemetry Python'
 LONG_DESCRIPTION = 'This package provides a set of telemetry APIs for use in Snowflake'
@@ -9,26 +11,24 @@
 VERSION = None
 with open(os.path.join(SNOWFLAKE_TELEMETRY_SRC_DIR, "version.py"), encoding="utf-8") as f:
     exec(f.read())
-VERSION_STR = ".".join([str(v) for v in VERSION if v is not None])
 
 
 setup(
     name="snowflake-telemetry-python",
-    version=VERSION_STR,
+    version=VERSION,
     author="Snowflake, Inc",
     author_email="[email protected]",
     description=DESCRIPTION,
     long_description=LONG_DESCRIPTION,
     install_requires=[
-        "setuptools >= 40.0.0",
-        "opentelemetry-sdk == 1.12.0",
+        "setuptools >= 40.0.0, < 66.0.0",
         "opentelemetry-api == 1.12.0",
         "opentelemetry-exporter-otlp == 1.12.0",
+        "opentelemetry-sdk == 1.12.0",
     ],
-    namespace_packages=["snowflake"],
-    packages=[
-        "snowflake.telemetry",
-    ],
+    packages=find_namespace_packages(
+        where='src'
+    ),
     package_dir={
         "": "src",
     },

src/snowflake/telemetry/__init__.py

@@ -11,24 +11,27 @@
 Stored Procedures.
 """
 
-from opentelemetry.util import types
+from opentelemetry.util.types import (
+    AttributeValue,
+    Attributes,
+)
 
 from snowflake.telemetry.version import VERSION
 
-__version__ = ".".join(str(x) for x in VERSION if x is not None)
+__version__ = VERSION
 
 
 def add_event(
             name: str,
-            attributes: types.Attributes = None,
+            attributes: Attributes = None,
         ) -> None:
     """Add an event to the Snowflake auto-instrumented span.
 
     This is a stub for the full functionality when running in Snowflake.
     """
 
 
-def set_span_attribute(key: str, value: types.AttributeValue) -> None:
+def set_span_attribute(key: str, value: AttributeValue) -> None:
     """Set an attribute to the Snowflake auto-instrumented span.
 
     This is a stub for the full functionality when running in Snowflake.

src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/log_encoder.py

@@ -0,0 +1,22 @@
+from logging import LogRecord
+
+from typing import Sequence
+
+from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest
+from opentelemetry.proto.logs.v1.logs_pb2 import LogsData
+
+
+def encode_logs(batch: Sequence[LogRecord]) -> ExportLogsServiceRequest:
+    # TODO fix this no-op implementation of encode_logs
+    return ExportLogsServiceRequest(resource_logs=[])
+
+def serialize_logs_data(batch: Sequence[LogRecord]) -> bytes:
+    return LogsData(
+        resource_logs=encode_logs(batch).resource_logs
+    ).SerializeToString()
+
+
+__all__ = [
+    "encode_logs",
+    "serialize_logs_data",
+]

src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/metrics_encoder.py

@@ -0,0 +1,22 @@
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest
+from opentelemetry.proto.metrics.v1.metrics_pb2 import MetricsData as PB2MetricsData
+from opentelemetry.sdk.metrics.export import MetricsData
+
+
+_exporter = OTLPMetricExporter()
+
+def encode_metrics(data: MetricsData) -> ExportMetricsServiceRequest:
+    # Will no longer rely on _translate_data after we upgrade to v1.19.0 or later
+    return _exporter._translate_data(data)
+
+def serialize_metrics_data(data: MetricsData) -> bytes:
+    return PB2MetricsData(
+        resource_metrics=encode_metrics(data).resource_metrics
+    ).SerializeToString()
+
+
+__all__ = [
+    "encode_metrics",
+    "serialize_metrics_data",
+]

src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/trace_encoder.py

@@ -0,0 +1,28 @@
+from typing import Sequence
+
+from google.protobuf.json_format import MessageToJson
+
+from opentelemetry.exporter.otlp.proto.http.trace_exporter.encoder import _ProtobufEncoder
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest as PB2ExportTraceServiceRequest
+from opentelemetry.proto.trace.v1.trace_pb2 import TracesData
+from opentelemetry.sdk.trace import ReadableSpan
+
+
+def encode_spans(
+    sdk_spans: Sequence[ReadableSpan],
+) -> PB2ExportTraceServiceRequest:
+    # Will no longer rely on _ProtobufEncoder after we upgrade to v1.19.0 or later
+    return _ProtobufEncoder.encode(sdk_spans)
+
+def serialize_traces_data(
+    sdk_spans: Sequence[ReadableSpan],
+) -> bytes:
+    return TracesData(
+        resource_spans=encode_spans(sdk_spans).resource_spans
+    ).SerializeToString()
+
+
+__all__ = [
+    "encode_spans",
+    "serialize_traces_data",
+]

src/snowflake/telemetry/version.py

@@ -4,4 +4,4 @@
 #
 
 """Update this for the versions."""
-VERSION = (0, 2, 0)
+VERSION = "0.2.1.dev"

tests/snowflake-telemetry-test-utils/README.md

@@ -0,0 +1,5 @@
+# Snowflake Telemetry Test Utils
+
+## About
+
+`snowflake-telemetry-test-utils` is a utility module that supports the test written for the Snowflake Telemetry Python project.

tests/snowflake-telemetry-test-utils/setup.py

@@ -0,0 +1,53 @@
+import os
+from setuptools import (
+    find_namespace_packages,
+    setup,
+)
+
+DESCRIPTION = 'Snowflake Telemetry Test Utils'
+LONG_DESCRIPTION = 'This package provides test utils for testing snowflake-telemetry-python'
+
+
+setup(
+    name="snowflake-telemetry-test-utils",
+    version="0.0.1.dev",
+    author="Snowflake, Inc",
+    author_email="[email protected]",
+    description=DESCRIPTION,
+    long_description=LONG_DESCRIPTION,
+    install_requires=[
+        "pytest >= 7.0.0",
+        "snowflake-telemetry-python == 0.2.1.dev",
+    ],
+    packages=find_namespace_packages(
+        where='src'
+    ),
+    package_dir={
+        "": "src",
+    },
+    keywords="Snowflake db database cloud analytics warehouse",
+    classifiers=[
+        "Development Status :: 1 - Planning",
+        "Environment :: Console",
+        "Environment :: Other Environment",
+        "Intended Audience :: Developers",
+        "Intended Audience :: Education",
+        "Intended Audience :: Information Technology",
+        "Intended Audience :: System Administrators",
+        "License :: OSI Approved :: Apache Software License",
+        "Operating System :: OS Independent",
+        "Programming Language :: SQL",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Topic :: Database",
+        "Topic :: Software Development",
+        "Topic :: Software Development :: Libraries",
+        "Topic :: Software Development :: Libraries :: Application Frameworks",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        "Topic :: Scientific/Engineering :: Information Analysis",
+     ],
+    zip_safe=True,
+)

tests/snowflake-telemetry-test-utils/src/snowflake/telemetry/test/metrictestutil.py

@@ -0,0 +1,100 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from opentelemetry.attributes import BoundedAttributes
+from opentelemetry.sdk.metrics.export import (
+    AggregationTemporality,
+    Gauge,
+    Metric,
+    NumberDataPoint,
+    Sum,
+)
+
+
+def _generate_metric(
+    name, data, attributes=None, description=None, unit=None
+) -> Metric:
+    if description is None:
+        description = "foo"
+    if unit is None:
+        unit = "s"
+    return Metric(
+        name=name,
+        description=description,
+        unit=unit,
+        data=data,
+    )
+
+
+def _generate_sum(
+    name,
+    value,
+    attributes=None,
+    description=None,
+    unit=None,
+    is_monotonic=True,
+) -> Metric:
+    if attributes is None:
+        attributes = BoundedAttributes(attributes={"a": 1, "b": True})
+    return _generate_metric(
+        name,
+        Sum(
+            data_points=[
+                NumberDataPoint(
+                    attributes=attributes,
+                    start_time_unix_nano=1641946015139533244,
+                    time_unix_nano=1641946016139533244,
+                    value=value,
+                )
+            ],
+            aggregation_temporality=AggregationTemporality.CUMULATIVE,
+            is_monotonic=is_monotonic,
+        ),
+        description=description,
+        unit=unit,
+    )
+
+
+def _generate_gauge(
+    name, value, attributes=None, description=None, unit=None
+) -> Metric:
+    if attributes is None:
+        attributes = BoundedAttributes(attributes={"a": 1, "b": True})
+    return _generate_metric(
+        name,
+        Gauge(
+            data_points=[
+                NumberDataPoint(
+                    attributes=attributes,
+                    start_time_unix_nano=1641946015139533244,
+                    time_unix_nano=1641946016139533244,
+                    value=value,
+                )
+            ],
+        ),
+        description=description,
+        unit=unit,
+    )
+
+
+def _generate_unsupported_metric(
+    name, attributes=None, description=None, unit=None
+) -> Metric:
+    return _generate_metric(
+        name,
+        None,
+        description=description,
+        unit=unit,
+    )