From e16f7dc6f7316cf0bc8b1af25b11641a950550ac Mon Sep 17 00:00:00 2001 From: Douglas Barker Date: Mon, 16 Dec 2024 05:52:20 +0000 Subject: [PATCH 1/3] encode instrumentation scope schema url and attributes to otlp proto messages. update tests to cover the schema_url and attributes. --- .../otlp/proto/common/_internal/__init__.py | 1 + .../common/_internal/_log_encoder/__init__.py | 3 + .../_internal/metrics_encoder/__init__.py | 8 +- .../_internal/trace_encoder/__init__.py | 3 + .../tests/test_log_encoder.py | 114 ++++++++++++++++- .../tests/test_metrics_encoder.py | 114 +++++++++++++++-- .../tests/test_trace_encoder.py | 121 +++++++++++++++++- 7 files changed, 347 insertions(+), 17 deletions(-) diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py index 4e75a6bcbe5..58a7975e483 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py @@ -59,6 +59,7 @@ def _encode_instrumentation_scope( return PB2InstrumentationScope( name=instrumentation_scope.name, version=instrumentation_scope.version, + attributes=_encode_attributes(instrumentation_scope.attributes), ) diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py index 7213f89d4a0..b7933b0d139 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py @@ -82,6 +82,9 @@ def _encode_resource_logs(batch: Sequence[LogData]) -> List[ResourceLogs]: ScopeLogs( scope=(_encode_instrumentation_scope(sdk_instrumentation)), log_records=pb2_logs, + schema_url=sdk_instrumentation.schema_url + if sdk_instrumentation + else None, ) ) pb2_resource_logs.append( diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py index 0c2b153b3bd..5ac1e232a9c 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py @@ -17,13 +17,13 @@ from opentelemetry.exporter.otlp.proto.common._internal import ( _encode_attributes, + _encode_instrumentation_scope, _encode_span_id, _encode_trace_id, ) from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ( ExportMetricsServiceRequest, ) -from opentelemetry.proto.common.v1.common_pb2 import InstrumentationScope from opentelemetry.proto.metrics.v1 import metrics_pb2 as pb2 from opentelemetry.proto.resource.v1.resource_pb2 import ( Resource as PB2Resource, @@ -219,10 +219,8 @@ def _encode_resource_metrics(resource_metrics, resource_metrics_dict): # there is no need to check for existing instrumentation scopes # here. pb2_scope_metrics = pb2.ScopeMetrics( - scope=InstrumentationScope( - name=instrumentation_scope.name, - version=instrumentation_scope.version, - ) + scope=_encode_instrumentation_scope(instrumentation_scope), + schema_url=instrumentation_scope.schema_url, ) scope_metrics_dict[instrumentation_scope] = pb2_scope_metrics diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py index d382159e418..388d229bab6 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py @@ -89,6 +89,9 @@ def _encode_resource_spans( PB2ScopeSpans( scope=(_encode_instrumentation_scope(sdk_instrumentation)), spans=pb2_spans, + schema_url=sdk_instrumentation.schema_url + if sdk_instrumentation + else None, ) ) pb2_resource_spans.append( diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_log_encoder.py b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_log_encoder.py index 70f4c821c9e..c7be8d3d8f9 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_log_encoder.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_log_encoder.py @@ -154,7 +154,54 @@ def _get_sdk_log_data() -> List[LogData]: ), ) - return [log1, log2, log3, log4] + log5 = LogData( + log_record=SDKLogRecord( + timestamp=1644650584292683022, + observed_timestamp=1644650584292683022, + trace_id=212592107417388365804938480559624925522, + span_id=6077757853989569222, + trace_flags=TraceFlags(0x01), + severity_text="ERROR", + severity_number=SeverityNumber.ERROR, + body="This instrumentation scope has a schema url", + resource=SDKResource( + {"first_resource": "value"}, + "resource_schema_url", + ), + attributes={"filename": "model.py", "func_name": "run_method"}, + ), + instrumentation_scope=InstrumentationScope( + "scope_with_url", + "scope_with_url_version", + "instrumentation_schema_url", + ), + ) + + log6 = LogData( + log_record=SDKLogRecord( + timestamp=1644650584292683033, + observed_timestamp=1644650584292683033, + trace_id=212592107417388365804938480559624925533, + span_id=6077757853989569233, + trace_flags=TraceFlags(0x01), + severity_text="FATAL", + severity_number=SeverityNumber.FATAL, + body="This instrumentation scope has a schema url and attributes", + resource=SDKResource( + {"first_resource": "value"}, + "resource_schema_url", + ), + attributes={"filename": "model.py", "func_name": "run_method"}, + ), + instrumentation_scope=InstrumentationScope( + "scope_with_attributes", + "scope_with_attributes_version", + "instrumentation_schema_url", + {"one": 1, "two": "2"}, + ), + ) + + return [log1, log2, log3, log4, log5, log6] def get_test_logs( self, @@ -229,6 +276,71 @@ def get_test_logs( ) ], ), + PB2ScopeLogs( + scope=PB2InstrumentationScope( + name="scope_with_url", + version="scope_with_url_version", + ), + schema_url="instrumentation_schema_url", + log_records=[ + PB2LogRecord( + time_unix_nano=1644650584292683022, + observed_time_unix_nano=1644650584292683022, + trace_id=_encode_trace_id( + 212592107417388365804938480559624925522 + ), + span_id=_encode_span_id( + 6077757853989569222 + ), + flags=int(TraceFlags(0x01)), + severity_text="ERROR", + severity_number=SeverityNumber.ERROR.value, + body=_encode_value( + "This instrumentation scope has a schema url" + ), + attributes=_encode_attributes( + { + "filename": "model.py", + "func_name": "run_method", + } + ), + ) + ], + ), + PB2ScopeLogs( + scope=PB2InstrumentationScope( + name="scope_with_attributes", + version="scope_with_attributes_version", + attributes=_encode_attributes( + {"one": 1, "two": "2"} + ), + ), + schema_url="instrumentation_schema_url", + log_records=[ + PB2LogRecord( + time_unix_nano=1644650584292683033, + observed_time_unix_nano=1644650584292683033, + trace_id=_encode_trace_id( + 212592107417388365804938480559624925533 + ), + span_id=_encode_span_id( + 6077757853989569233 + ), + flags=int(TraceFlags(0x01)), + severity_text="FATAL", + severity_number=SeverityNumber.FATAL.value, + body=_encode_value( + "This instrumentation scope has a schema url and attributes" + ), + attributes=_encode_attributes( + { + "filename": "model.py", + "func_name": "run_method", + } + ), + ) + ], + ), ], schema_url="resource_schema_url", ), diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_metrics_encoder.py b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_metrics_encoder.py index cdf12964856..d2ef292f93a 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_metrics_encoder.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_metrics_encoder.py @@ -110,7 +110,7 @@ def test_encode_sum_int(self): scope=SDKInstrumentationScope( name="first_name", version="first_version", - schema_url="insrumentation_scope_schema_url", + schema_url="instrumentation_scope_schema_url", ), metrics=[_generate_sum("sum_int", 33)], schema_url="instrumentation_scope_schema_url", @@ -137,6 +137,7 @@ def test_encode_sum_int(self): scope=InstrumentationScope( name="first_name", version="first_version" ), + schema_url="instrumentation_scope_schema_url", metrics=[ pb2.Metric( name="sum_int", @@ -190,7 +191,7 @@ def test_encode_sum_double(self): scope=SDKInstrumentationScope( name="first_name", version="first_version", - schema_url="insrumentation_scope_schema_url", + schema_url="instrumentation_scope_schema_url", ), metrics=[_generate_sum("sum_double", 2.98)], schema_url="instrumentation_scope_schema_url", @@ -217,6 +218,7 @@ def test_encode_sum_double(self): scope=InstrumentationScope( name="first_name", version="first_version" ), + schema_url="instrumentation_scope_schema_url", metrics=[ pb2.Metric( name="sum_double", @@ -270,7 +272,7 @@ def test_encode_gauge_int(self): scope=SDKInstrumentationScope( name="first_name", version="first_version", - schema_url="insrumentation_scope_schema_url", + schema_url="instrumentation_scope_schema_url", ), metrics=[_generate_gauge("gauge_int", 9000)], schema_url="instrumentation_scope_schema_url", @@ -297,6 +299,7 @@ def test_encode_gauge_int(self): scope=InstrumentationScope( name="first_name", version="first_version" ), + schema_url="instrumentation_scope_schema_url", metrics=[ pb2.Metric( name="gauge_int", @@ -348,7 +351,7 @@ def test_encode_gauge_double(self): scope=SDKInstrumentationScope( name="first_name", version="first_version", - schema_url="insrumentation_scope_schema_url", + schema_url="instrumentation_scope_schema_url", ), metrics=[_generate_gauge("gauge_double", 52.028)], schema_url="instrumentation_scope_schema_url", @@ -375,6 +378,7 @@ def test_encode_gauge_double(self): scope=InstrumentationScope( name="first_name", version="first_version" ), + schema_url="instrumentation_scope_schema_url", metrics=[ pb2.Metric( name="gauge_double", @@ -425,7 +429,7 @@ def test_encode_histogram(self): scope=SDKInstrumentationScope( name="first_name", version="first_version", - schema_url="insrumentation_scope_schema_url", + schema_url="instrumentation_scope_schema_url", ), metrics=[self.histogram], schema_url="instrumentation_scope_schema_url", @@ -452,6 +456,7 @@ def test_encode_histogram(self): scope=InstrumentationScope( name="first_name", version="first_version" ), + schema_url="instrumentation_scope_schema_url", metrics=[ pb2.Metric( name="histogram", @@ -537,7 +542,7 @@ def test_encode_multiple_scope_histogram(self): scope=SDKInstrumentationScope( name="first_name", version="first_version", - schema_url="insrumentation_scope_schema_url", + schema_url="instrumentation_scope_schema_url", ), metrics=[self.histogram, self.histogram], schema_url="instrumentation_scope_schema_url", @@ -546,7 +551,7 @@ def test_encode_multiple_scope_histogram(self): scope=SDKInstrumentationScope( name="second_name", version="second_version", - schema_url="insrumentation_scope_schema_url", + schema_url="instrumentation_scope_schema_url", ), metrics=[self.histogram], schema_url="instrumentation_scope_schema_url", @@ -555,7 +560,7 @@ def test_encode_multiple_scope_histogram(self): scope=SDKInstrumentationScope( name="third_name", version="third_version", - schema_url="insrumentation_scope_schema_url", + schema_url="instrumentation_scope_schema_url", ), metrics=[self.histogram], schema_url="instrumentation_scope_schema_url", @@ -582,6 +587,7 @@ def test_encode_multiple_scope_histogram(self): scope=InstrumentationScope( name="first_name", version="first_version" ), + schema_url="instrumentation_scope_schema_url", metrics=[ pb2.Metric( name="histogram", @@ -713,6 +719,7 @@ def test_encode_multiple_scope_histogram(self): scope=InstrumentationScope( name="second_name", version="second_version" ), + schema_url="instrumentation_scope_schema_url", metrics=[ pb2.Metric( name="histogram", @@ -782,6 +789,7 @@ def test_encode_multiple_scope_histogram(self): scope=InstrumentationScope( name="third_name", version="third_version" ), + schema_url="instrumentation_scope_schema_url", metrics=[ pb2.Metric( name="histogram", @@ -892,7 +900,7 @@ def test_encode_exponential_histogram(self): scope=SDKInstrumentationScope( name="first_name", version="first_version", - schema_url="insrumentation_scope_schema_url", + schema_url="instrumentation_scope_schema_url", ), metrics=[exponential_histogram], schema_url="instrumentation_scope_schema_url", @@ -919,6 +927,7 @@ def test_encode_exponential_histogram(self): scope=InstrumentationScope( name="first_name", version="first_version" ), + schema_url="instrumentation_scope_schema_url", metrics=[ pb2.Metric( name="exponential_histogram", @@ -989,7 +998,7 @@ def test_encoding_exception_reraise(self): scope=SDKInstrumentationScope( name="first_name", version="first_version", - schema_url="insrumentation_scope_schema_url", + schema_url="instrumentation_scope_schema_url", ), metrics=[_generate_sum("sum_double", big_number)], schema_url="instrumentation_scope_schema_url", @@ -1005,3 +1014,88 @@ def test_encoding_exception_reraise(self): # assert that the EncodingException wraps the metric and original exception assert isinstance(context.exception.metric, Metric) assert isinstance(context.exception.original_exception, ValueError) + + def test_encode_scope_with_attributes(self): + metrics_data = MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=Resource( + attributes=None, + schema_url="resource_schema_url", + ), + scope_metrics=[ + ScopeMetrics( + scope=SDKInstrumentationScope( + name="first_name", + version="first_version", + schema_url="instrumentation_scope_schema_url", + attributes={"one": 1, "two": "2"}, + ), + metrics=[_generate_sum("sum_int", 88)], + schema_url="instrumentation_scope_schema_url", + ) + ], + schema_url="resource_schema_url", + ) + ] + ) + expected = ExportMetricsServiceRequest( + resource_metrics=[ + pb2.ResourceMetrics( + schema_url="resource_schema_url", + resource=OTLPResource(), + scope_metrics=[ + pb2.ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + attributes=[ + KeyValue( + key="one", value=AnyValue(int_value=1) + ), + KeyValue( + key="two", + value=AnyValue(string_value="2"), + ), + ], + ), + schema_url="instrumentation_scope_schema_url", + metrics=[ + pb2.Metric( + name="sum_int", + unit="s", + description="foo", + sum=pb2.Sum( + data_points=[ + pb2.NumberDataPoint( + attributes=[ + KeyValue( + key="a", + value=AnyValue( + int_value=1 + ), + ), + KeyValue( + key="b", + value=AnyValue( + bool_value=True + ), + ), + ], + start_time_unix_nano=1641946015139533244, + time_unix_nano=1641946016139533244, + as_int=88, + ) + ], + aggregation_temporality=AggregationTemporality.CUMULATIVE, + is_monotonic=True, + ), + ) + ], + ) + ], + ) + ] + ) + actual = encode_metrics(metrics_data) + self.assertEqual(expected, actual) diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_trace_encoder.py b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_trace_encoder.py index 663b9ab309f..bf78526d7e4 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_trace_encoder.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_trace_encoder.py @@ -72,12 +72,16 @@ def get_exhaustive_otel_span_list() -> List[SDKSpan]: base_time + 150 * 10**6, base_time + 300 * 10**6, base_time + 400 * 10**6, + base_time + 500 * 10**6, + base_time + 600 * 10**6, ) end_times = ( start_times[0] + (50 * 10**6), start_times[1] + (100 * 10**6), start_times[2] + (200 * 10**6), start_times[3] + (300 * 10**6), + start_times[4] + (400 * 10**6), + start_times[5] + (500 * 10**6), ) parent_span_context = SDKSpanContext( @@ -151,7 +155,42 @@ def get_exhaustive_otel_span_list() -> List[SDKSpan]: span4.start(start_time=start_times[3]) span4.end(end_time=end_times[3]) - return [span1, span2, span3, span4] + span5 = SDKSpan( + name="test-span-5", + context=other_context, + parent=None, + resource=SDKResource( + attributes={"key_resource": "another_resource"}, + schema_url="resource_schema_url", + ), + instrumentation_scope=SDKInstrumentationScope( + name="scope_1_name", + version="scope_1_version", + schema_url="scope_1_schema_url", + ), + ) + span5.start(start_time=start_times[4]) + span5.end(end_time=end_times[4]) + + span6 = SDKSpan( + name="test-span-6", + context=other_context, + parent=None, + resource=SDKResource( + attributes={"key_resource": "another_resource"}, + schema_url="resource_schema_url", + ), + instrumentation_scope=SDKInstrumentationScope( + name="scope_2_name", + version="scope_2_version", + schema_url="scope_2_schema_url", + attributes={"one": "1", "two": 2}, + ), + ) + span6.start(start_time=start_times[5]) + span6.end(end_time=end_times[5]) + + return [span1, span2, span3, span4, span5, span6] def get_exhaustive_test_spans( self, @@ -356,6 +395,86 @@ def get_exhaustive_test_spans( ) ], ), + PB2ResourceSpans( + resource=PB2Resource( + attributes=[ + PB2KeyValue( + key="key_resource", + value=PB2AnyValue( + string_value="another_resource" + ), + ), + ], + ), + schema_url="resource_schema_url", + scope_spans=[ + PB2ScopeSpans( + scope=PB2InstrumentationScope( + name="scope_1_name", version="scope_1_version" + ), + schema_url="scope_1_schema_url", + spans=[ + PB2SPan( + trace_id=trace_id, + span_id=_encode_span_id( + otel_spans[4].context.span_id + ), + trace_state=None, + parent_span_id=None, + name=otel_spans[4].name, + kind=span_kind, + start_time_unix_nano=otel_spans[ + 4 + ].start_time, + end_time_unix_nano=otel_spans[4].end_time, + attributes=None, + events=None, + links=None, + status={}, + flags=0x100, + ), + ], + ), + PB2ScopeSpans( + scope=PB2InstrumentationScope( + name="scope_2_name", + version="scope_2_version", + attributes=[ + PB2KeyValue( + key="one", + value=PB2AnyValue(string_value="1"), + ), + PB2KeyValue( + key="two", + value=PB2AnyValue(int_value=2), + ), + ], + ), + schema_url="scope_2_schema_url", + spans=[ + PB2SPan( + trace_id=trace_id, + span_id=_encode_span_id( + otel_spans[5].context.span_id + ), + trace_state=None, + parent_span_id=None, + name=otel_spans[5].name, + kind=span_kind, + start_time_unix_nano=otel_spans[ + 5 + ].start_time, + end_time_unix_nano=otel_spans[5].end_time, + attributes=None, + events=None, + links=None, + status={}, + flags=0x100, + ), + ], + ), + ], + ), ] ) From 4edfefa9e574c3f55520760f45ed04892e2f031c Mon Sep 17 00:00:00 2001 From: Douglas Barker Date: Mon, 16 Dec 2024 06:33:07 +0000 Subject: [PATCH 2/3] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8b1a254db0..2d87de58f58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +- Fix OTLP encoders missing instrumentation scope schema url and attributes + ([#4359](https://github.com/open-telemetry/opentelemetry-python/pull/4359)) ## Version 1.29.0/0.50b0 (2024-12-11) From 3410495ba54fb5385915535855ed417e06f9ffe1 Mon Sep 17 00:00:00 2001 From: Doug Barker Date: Mon, 23 Dec 2024 09:40:41 -0700 Subject: [PATCH 3/3] Update CHANGELOG.md Co-authored-by: Riccardo Magliocchetti --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5017dfb5046..237c36fe79a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + - Fix OTLP encoders missing instrumentation scope schema url and attributes ([#4359](https://github.com/open-telemetry/opentelemetry-python/pull/4359)) - - Add `attributes` field in `metrics.get_meter` wrapper function ([#4364](https://github.com/open-telemetry/opentelemetry-python/pull/4364)) - Add Python 3.13 support