From 6c007613fc1abd746857d3eb4800a3ecd5fcfb9c Mon Sep 17 00:00:00 2001 From: YANGDB Date: Tue, 30 May 2023 16:00:34 -0700 Subject: [PATCH 01/18] Adding an Observability OTEL based demo for Observability playground - data-prepper - nginx - fluentBit - jaeger - otel-col Signed-off-by: YANGDB --- .../demo/observability/otel_demo/README.md | 97 + .../observability/otel_demo/data-prepper.yaml | 97 + .../observability/otel_demo/fluentbit.yaml | 99 + .../observability/otel_demo/img/DemoFlow.png | Bin 0 -> 77310 bytes .../img/docker-services-topology.png | Bin 0 -> 166354 bytes .../otel_demo/img/nginx_dashboard.png | Bin 0 -> 404439 bytes .../img/otelcol-data-flow-overview.png | Bin 0 -> 148284 bytes .../img/prometheus_federated_metrics.png | Bin 0 -> 550505 bytes .../otel_demo/img/service-graph.png | Bin 0 -> 192513 bytes .../observability/otel_demo/img/services.png | Bin 0 -> 488377 bytes .../otel_demo/img/trace_analytics.png | Bin 0 -> 487029 bytes .../observability/otel_demo/img/traces.png | Bin 0 -> 585338 bytes .../observability/otel_demo/integrations.yaml | 1497 ++++++++++ .../demo/observability/otel_demo/jaeger.yaml | 103 + .../demo/observability/otel_demo/nginx.yaml | 75 + .../otel_demo/opentelemetry-demo.yaml | 2402 +++++++++++++++++ 16 files changed, 4370 insertions(+) create mode 100644 config/playground/demo/observability/otel_demo/README.md create mode 100644 config/playground/demo/observability/otel_demo/data-prepper.yaml create mode 100644 config/playground/demo/observability/otel_demo/fluentbit.yaml create mode 100644 config/playground/demo/observability/otel_demo/img/DemoFlow.png create mode 100644 config/playground/demo/observability/otel_demo/img/docker-services-topology.png create mode 100644 config/playground/demo/observability/otel_demo/img/nginx_dashboard.png create mode 100644 config/playground/demo/observability/otel_demo/img/otelcol-data-flow-overview.png create mode 100644 config/playground/demo/observability/otel_demo/img/prometheus_federated_metrics.png create mode 100644 config/playground/demo/observability/otel_demo/img/service-graph.png create mode 100644 config/playground/demo/observability/otel_demo/img/services.png create mode 100644 config/playground/demo/observability/otel_demo/img/trace_analytics.png create mode 100644 config/playground/demo/observability/otel_demo/img/traces.png create mode 100644 config/playground/demo/observability/otel_demo/integrations.yaml create mode 100644 config/playground/demo/observability/otel_demo/jaeger.yaml create mode 100644 config/playground/demo/observability/otel_demo/nginx.yaml create mode 100644 config/playground/demo/observability/otel_demo/opentelemetry-demo.yaml diff --git a/config/playground/demo/observability/otel_demo/README.md b/config/playground/demo/observability/otel_demo/README.md new file mode 100644 index 0000000..2e58a24 --- /dev/null +++ b/config/playground/demo/observability/otel_demo/README.md @@ -0,0 +1,97 @@ +# Opensearch OTEL Demo Architecture +This document will review the OpenSearch architecture for the [OTEL demo](https://opentelemetry.io/docs/demo/) and will review how to use the new Observability capabilities +implemented into OpenSearch. +--- +This diagram provides an overview of the system components, showcasing the configuration derived from the OpenTelemetry Collector (otelcol) configuration file utilized by the OpenTelemetry demo application. + +Additionally, it highlights the observability data (traces and metrics) flow within the system. + +![](img/otelcol-data-flow-overview.png) + +--- +[OTEL DEMO](https://opentelemetry.io/docs/demo/architecture/) Describes the list of services that are composing the Astronomy Shop. + +They are combined of: + - [Accounting](https://opentelemetry.io/docs/demo/services/accounting/) + - [Ad](https://opentelemetry.io/docs/demo/services/ad/) + - [Cart](https://opentelemetry.io/docs/demo/services/cart/) + - [Checkout](https://opentelemetry.io/docs/demo/services/checkout/) + - [Currency](https://opentelemetry.io/docs/demo/services/currency/) + - [Email](https://opentelemetry.io/docs/demo/services/email/) + - [Feature Flag](https://opentelemetry.io/docs/demo/services/feature-flag/) + - [Fraud Detection](https://opentelemetry.io/docs/demo/services/fraud-detection/) + - [Frontend](https://opentelemetry.io/docs/demo/services/frontend/) + - [Kafka](https://opentelemetry.io/docs/demo/services/kafka/) + - [Payment](https://opentelemetry.io/docs/demo/services/payment/) + - [Product Catalog](https://opentelemetry.io/docs/demo/services/product-catalog/) + - [Quote](https://opentelemetry.io/docs/demo/services/quote/) + - [Recommendation](https://opentelemetry.io/docs/demo/services/recommendation/) + - [Shipping](https://opentelemetry.io/docs/demo/services/shipping/) + - [Fluent-Bit]() *(Nginx's otel log exported)* + - [Integrations]() *(Pre-canned OpenSearch assets)* + - [DataPrepper]() *(OpenSearch's ingestion pipeline)* + +Backend supportive services + - [Load Generator]() + - See [description]() + - [Frontend Nginx Proxy]() *(Replacement for _Frontend-Proxy_)* + - [OpenSearch]() + - [Dashboards]() + - [Prometheus]() + - [Feature-Flag]() + +### Services Topology +The next diagram shows the docker compose services dependencies + +![](img/docker-services-topology.png) +--- + +## Purpose +The purpose of this demo is to demonstrate the different capabilities of OpenSearch Observability to investigate and reflect your system. + +### Ingestion +The ingestion capabilities for OpenSearch is to be able to support multiple pipelines: + - [Data-Prepper](https://github.com/opensearch-project/data-prepper/) is an OpenSearch ingestion project that allows ingestion of OTEL standard signals using Otel-Collector + - [Jaeger](https://opensearch.org/docs/latest/observing-your-data/trace/trace-analytics-jaeger/) is an ingestion framework which has a build in capability for pushing OTEL signals into OpenSearch + - [Fluent-Bit](https://docs.fluentbit.io/manual/pipeline/outputs/opensearch) is an ingestion framework which has a build in capability for pushing OTEL signals into OpenSearch + +### Integrations - +The integration service is a list of pre-canned assets that are loaded in a combined manner to allow users the ability for simple and automatic way to discover and review their services topology. + +These (demo-sample) integrations contain the following assets: + - components & index template mapping + - datasources + - data-stream & indices + - queries + - dashboards + +Once they are loaded, the user can imminently review his OTEL demo services and dashboards that reflect the system state. + - [Nginx Dashboard]() - reflects the Nginx Proxy server that routes all the network communication to/from the frontend + - [Prometheus datasource]() - reflects the connectivity to the prometheus metric storage that allows us to federate metrics analytics queries + - [Logs Datastream]() - reflects the data-stream used by nginx logs ingestion and dashboards representing a well-structured [log schema](../src/integrations/mapping-templates/logs.mapping) + +Once these assets are loaded - the user can start reviewing its Observability dashboards and traces + +![Nginx Dashboard](img/nginx_dashboard.png) + +![Prometheus Metrics](img/prometheus_federated_metrics.png) + +![Trace Analytics](img/trace_analytics.png) + +![Service Maps](img/services.png) + +![Traces](img/traces.png) + +![ServiceGraph](img/service-graph.png) +--- + +### **Scenarios** + +How can you solve problems with OpenTelemetry? These scenarios walk you through some pre-configured problems and show you how to interpret OpenTelemetry data to solve them. + +- Generate a Product Catalog error for GetProduct requests with product id: OLJCESPC7Z using the Feature Flag service +- Discover a memory leak and diagnose it using metrics and traces. Read more + +### **Reference** +Project reference documentation, like requirements and feature matrices [here](https://opentelemetry.io/docs/demo/#reference) + diff --git a/config/playground/demo/observability/otel_demo/data-prepper.yaml b/config/playground/demo/observability/otel_demo/data-prepper.yaml new file mode 100644 index 0000000..336453e --- /dev/null +++ b/config/playground/demo/observability/otel_demo/data-prepper.yaml @@ -0,0 +1,97 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 +--- +# Data Prepper Deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: data-prepper + namespace: otel-demo +spec: + replicas: 1 + selector: + matchLabels: + app: data-prepper + template: + metadata: + labels: + app: data-prepper + spec: + containers: + - name: data-prepper + image: opensearchproject/data-prepper:latest + ports: + - containerPort: 21890 + volumeMounts: + - name: config-volume + mountPath: /usr/share/data-prepper/pipelines/ + subPath: pipelines.yaml + - name: config-volume + mountPath: /usr/share/data-prepper/config/ + subPath: data-prepper-config.yaml + volumes: + - name: config-volume + configMap: + name: data-prepper-config +--- +# Data Prepper config-map +apiVersion: v1 +kind: ConfigMap +metadata: + name: data-prepper-config + namespace: otel-demo +data: + trace_analytics_no_ssl_2x.yml: | + entry-pipeline: + delay: "100" + source: + otel_trace_source: + ssl: false + sink: + - pipeline: + name: "raw-pipeline" + - pipeline: + name: "service-map-pipeline" + raw-pipeline: + source: + pipeline: + name: "entry-pipeline" + processor: + - otel_trace_raw: + sink: + - opensearch: + hosts: [ "https://opensearch-cluster-leader:9200" ] + insecure: true + username: "{USR}" + password: "{PSWD}" + index_type: trace-analytics-raw + service-map-pipeline: + delay: "100" + source: + pipeline: + name: "entry-pipeline" + processor: + - service_map_stateful: + sink: + - opensearch: + hosts: ["https://opensearch-cluster-leader:9200"] + insecure: true + username: "{USR}" + password: "{PSWD}" + index_type: trace-analytics-service-map + data-prepper-config.yaml: | + ssl: false +--- +# Data Prepper Service +apiVersion: v1 +kind: Service +metadata: + name: data-prepper-service + namespace: otel-demo +spec: + selector: + app: data-prepper + ports: + - protocol: TCP + port: 21890 + targetPort: 21890 diff --git a/config/playground/demo/observability/otel_demo/fluentbit.yaml b/config/playground/demo/observability/otel_demo/fluentbit.yaml new file mode 100644 index 0000000..41175ba --- /dev/null +++ b/config/playground/demo/observability/otel_demo/fluentbit.yaml @@ -0,0 +1,99 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 +--- +# Fluentbit Deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fluentbit + namespace: otel-demo +spec: + replicas: 1 + selector: + matchLabels: + app: fluentbit + template: + metadata: + labels: + app: fluentbit + spec: + containers: + - name: fluentbit + image: fluent/fluent-bit:latest + ports: + - containerPort: 24224 + volumeMounts: + - name: config-volume + mountPath: /fluent-bit/etc/fluent-bit.conf + subPath: fluent-bit.conf + - name: config-volume + mountPath: /fluent-bit/etc/parsers.conf + subPath: parsers.conf + volumes: + - name: config-volume + configMap: + name: fluentbit-config +--- +# Fluentbit Service +apiVersion: v1 +kind: Service +metadata: + name: fluentbit-service + namespace: otel-demo +spec: + selector: + app: fluentbit + ports: + - protocol: TCP + port: 24224 + targetPort: 24224 +--- +# Fluentbit Config-Map +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluentbit-config + namespace: otel-demo +data: + fluent-bit.conf: | + [SERVICE] + Parsers_File parsers.conf + Log_Level info + Daemon off + [INPUT] + Name forward + Port 24224 + [FILTER] + Name parser + Match nginx.access + Key_Name log + Parser nginx + [FILTER] + Name lua + Match nginx.access + code function cb_filter(a,b,c)local d={}local e=os.date("!%Y-%m-%dT%H:%M:%S.000Z")d["observerTime"]=e;d["body"]=c.remote.." "..c.host.." "..c.user.." ["..os.date("%d/%b/%Y:%H:%M:%S %z").."] \""..c.method.." "..c.path.." HTTP/1.1\" "..c.code.." "..c.size.." \""..c.referer.."\" \""..c.agent.."\""d["trace_id"]="102981ABCD2901"d["span_id"]="abcdef1010"d["attributes"]={}d["attributes"]["data_stream"]={}d["attributes"]["data_stream"]["dataset"]="nginx.access"d["attributes"]["data_stream"]["namespace"]="production"d["attributes"]["data_stream"]["type"]="logs"d["event"]={}d["event"]["category"]={"web"}d["event"]["name"]="access"d["event"]["domain"]="nginx.access"d["event"]["kind"]="event"d["event"]["result"]="success"d["event"]["type"]={"access"}d["http"]={}d["http"]["request"]={}d["http"]["request"]["method"]=c.method;d["http"]["response"]={}d["http"]["response"]["bytes"]=tonumber(c.size)d["http"]["response"]["status_code"]=c.code;d["http"]["flavor"]="1.1"d["http"]["url"]=c.path;d["communication"]={}d["communication"]["source"]={}d["communication"]["source"]["address"]="127.0.0.1"d["communication"]["source"]["ip"]=c.remote;return 1,b,d end + call cb_filter + [OUTPUT] + Name opensearch + Match nginx.* + Host opensearch-cluster-leader + Port 9200 + tls On + tls.verify Off + HTTP_Passwd {USR} + HTTP_User {PSWD} + Index sso_logs-nginx-prod + Generate_ID On + Retry_Limit False + Suppress_Type_Name On + [OUTPUT] + Name stdout + Match nginx.access + parsers.conf: | + [PARSER] + Name nginx + Format regex + Regex ^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?