From 2e03596252af82a7eaecf2b7b04e8aac6fcac0b6 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 7 Oct 2024 14:31:51 +0200 Subject: [PATCH 01/27] Use fields instead of _source on APM queries --- packages/kbn-apm-types/src/es_fields/apm.ts | 21 +++ .../src/es_schemas/raw/fields/cloud.ts | 18 +- .../src/es_schemas/raw/fields/container.ts | 4 +- .../src/es_schemas/raw/fields/http.ts | 4 +- .../src/es_schemas/raw/fields/kubernetes.ts | 2 +- .../src/es_schemas/raw/fields/observer.ts | 4 +- .../src/es_schemas/raw/fields/page.ts | 2 +- .../src/es_schemas/raw/fields/service.ts | 8 +- .../src/es_schemas/raw/fields/url.ts | 2 +- .../src/es_schemas/raw/fields/user.ts | 2 +- .../object/unflatten_object.ts | 17 ++ .../__snapshots__/es_fields.test.ts.snap | 18 ++ .../apm/common/waterfall/typings.ts | 9 +- .../error_sample_contextual_insight.tsx | 24 ++- .../error_sampler/error_sample_detail.tsx | 14 +- .../error_sampler/error_tabs.tsx | 2 +- .../error_sampler/sample_summary.tsx | 4 +- ...redirect_to_transaction_detail_page_url.ts | 4 +- .../discover_links/discover_error_link.tsx | 14 +- .../metadata_table/error_metadata/index.tsx | 11 +- .../collect_data_telemetry/tasks.test.ts | 4 +- .../collect_data_telemetry/tasks.ts | 32 ++-- .../apm/server/lib/helpers/get_error_name.ts | 6 +- .../get_log_categories/index.ts | 18 +- .../get_container_id_from_signals.ts | 17 +- .../get_downstream_dependency_name.ts | 10 +- .../get_service_name_from_signals.ts | 14 +- .../get_metadata_for_dependency.ts | 15 +- .../dependencies/get_top_dependency_spans.ts | 43 +++-- .../get_error_group_main_statistics.ts | 59 +++++-- .../get_error_group_sample_ids.ts | 8 +- .../get_error_sample_details.ts | 91 +++++++++- .../apm/server/routes/errors/route.ts | 9 +- .../get_derived_service_annotations.ts | 20 ++- .../routes/services/get_service_agent.ts | 35 ++-- ...get_service_instance_container_metadata.ts | 21 ++- .../get_service_instance_metadata_details.ts | 15 +- .../services/get_service_metadata_details.ts | 19 +- .../services/get_service_metadata_icons.ts | 23 ++- .../routes/span_links/get_linked_children.ts | 50 ++++-- .../routes/span_links/get_linked_parents.ts | 2 +- .../span_links/get_span_links_details.ts | 154 ++++++++++------ .../server/routes/traces/get_trace_items.ts | 164 ++++++++++++----- .../traces/get_trace_samples_by_query.ts | 2 +- .../apm/server/routes/traces/route.ts | 13 +- .../routes/transactions/get_span/index.ts | 17 +- .../transactions/get_transaction/index.ts | 69 +++++++- .../get_transaction_by_name/index.ts | 28 ++- .../get_transaction_by_trace/index.ts | 53 +++++- .../transactions/trace_samples/index.ts | 23 ++- .../apm_data_access/server/utils.ts | 1 + .../utils/unflatten_known_fields.test.ts | 56 ++++++ .../server/utils/unflatten_known_fields.ts | 167 ++++++++++++++++++ 53 files changed, 1139 insertions(+), 303 deletions(-) create mode 100644 x-pack/packages/observability/observability_utils/object/unflatten_object.ts create mode 100644 x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts create mode 100644 x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index 6b0a68379f5d4..87bf76bb58ee6 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -8,6 +8,7 @@ */ export const TIMESTAMP = 'timestamp.us'; +export const AT_TIMESTAMP = '@timestamp'; export const AGENT = 'agent'; export const AGENT_NAME = 'agent.name'; export const AGENT_VERSION = 'agent.version'; @@ -21,9 +22,11 @@ export const CLOUD_PROVIDER = 'cloud.provider'; export const CLOUD_REGION = 'cloud.region'; export const CLOUD_MACHINE_TYPE = 'cloud.machine.type'; export const CLOUD_ACCOUNT_ID = 'cloud.account.id'; +export const CLOUD_ACCOUNT_NAME = 'cloud.account.name'; export const CLOUD_INSTANCE_ID = 'cloud.instance.id'; export const CLOUD_INSTANCE_NAME = 'cloud.instance.name'; export const CLOUD_SERVICE_NAME = 'cloud.service.name'; +export const CLOUD_PROJECT_NAME = 'cloud.project.name'; export const EVENT_SUCCESS_COUNT = 'event.success_count'; @@ -48,10 +51,14 @@ export const USER_ID = 'user.id'; export const USER_AGENT_ORIGINAL = 'user_agent.original'; export const USER_AGENT_NAME = 'user_agent.name'; +export const OBSERVER_VERSION = 'observer.version'; +export const OBSERVER_VERSION_MAJOR = 'observer.version_major'; export const OBSERVER_HOSTNAME = 'observer.hostname'; export const OBSERVER_LISTENING = 'observer.listening'; export const PROCESSOR_EVENT = 'processor.event'; +export const PROCESSOR_NAME = 'processor.name'; +export const TRANSACTION_AGENT_MARKS = 'transaction.agent.marks'; export const TRANSACTION_DURATION = 'transaction.duration.us'; export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram'; export const TRANSACTION_DURATION_SUMMARY = 'transaction.duration.summary'; @@ -110,6 +117,7 @@ export const ERROR_EXC_MESSAGE = 'error.exception.message'; // only to be used i export const ERROR_EXC_HANDLED = 'error.exception.handled'; // only to be used in es queries, since error.exception is now an array export const ERROR_EXC_TYPE = 'error.exception.type'; export const ERROR_PAGE_URL = 'error.page.url'; +export const ERROR_STACK_TRACE = 'error.stack_trace'; export const ERROR_TYPE = 'error.type'; // METRICS @@ -153,6 +161,15 @@ export const CONTAINER_IMAGE = 'container.image.name'; export const KUBERNETES = 'kubernetes'; export const KUBERNETES_POD_NAME = 'kubernetes.pod.name'; export const KUBERNETES_POD_UID = 'kubernetes.pod.uid'; +export const KUBERNETES_NAMESPACE = 'kubernetes.namespace'; +export const KUBERNETES_NAMESPACE_NAME = 'kubernetes.namespace.name'; +export const KUBERNETES_NODE_NAME = 'kubernetes.node.name'; +export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name'; +export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id'; +export const KUBERNETES_DEPLOYMENT = 'kubernetes.deployment'; +export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name'; +export const KUBERNETES_REPLICASET = 'kubernetes.replicaset'; +export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name'; export const FAAS_ID = 'faas.id'; export const FAAS_NAME = 'faas.name'; @@ -198,3 +215,7 @@ export const CLIENT_GEO_REGION_NAME = 'client.geo.region_name'; export const CHILD_ID = 'child.id'; export const LOG_LEVEL = 'log.level'; + +// Process +export const PROCESS_ARGS = 'process.args'; +export const PROCESS_PID = 'process.pid'; diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts index 7ee972faf7680..290be75091e18 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts @@ -10,26 +10,26 @@ export interface Cloud { availability_zone?: string; instance?: { - name: string; - id: string; + name?: string; + id?: string; }; machine?: { - type: string; + type?: string; }; project?: { - id: string; - name: string; + id?: string; + name?: string; }; provider?: string; region?: string; account?: { - id: string; - name: string; + id?: string; + name?: string; }; image?: { - id: string; + id?: string; }; service?: { - name: string; + name?: string; }; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts index 64dd497710b97..4c8d1ed4e52b4 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts @@ -9,5 +9,7 @@ export interface Container { id?: string | null; - image?: string | null; + image?: { + name?: string; + }; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts index 458731f690838..f3c62298ca8cb 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts @@ -8,7 +8,7 @@ */ export interface Http { - request?: { method: string; [key: string]: unknown }; - response?: { status_code: number; [key: string]: unknown }; + request?: { method?: string }; + response?: { status_code?: number }; version?: string; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts index 704d77f19f858..2a4f1465db9a5 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts @@ -8,7 +8,7 @@ */ export interface Kubernetes { - pod?: { uid?: string | null; [key: string]: unknown }; + pod?: { uid?: string | null; name?: string }; namespace?: string; replicaset?: { name?: string; diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts index 067ecb9436ff9..7d286d4c3581e 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts @@ -13,6 +13,6 @@ export interface Observer { id?: string; name?: string; type?: string; - version: string; - version_major: number; + version?: string; + version_major?: number; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts index 6cc058ef75642..a18f3c5578eb5 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts @@ -9,5 +9,5 @@ // only for RUM agent: shared by error and transaction export interface Page { - url: string; + url?: string; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts index bcd9af08706ec..bd52784576dce 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts @@ -11,18 +11,18 @@ export interface Service { name: string; environment?: string; framework?: { - name: string; + name?: string; version?: string; }; node?: { name?: string; }; runtime?: { - name: string; - version: string; + name?: string; + version?: string; }; language?: { - name: string; + name?: string; version?: string; }; version?: string; diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts index 3703763724f38..0f8cd3c814315 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts @@ -9,6 +9,6 @@ export interface Url { domain?: string; - full: string; + full?: string; original?: string; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts index 1c2235288a661..962ed1060b826 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts @@ -8,5 +8,5 @@ */ export interface User { - id: string; + id?: string; } diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.ts b/x-pack/packages/observability/observability_utils/object/unflatten_object.ts new file mode 100644 index 0000000000000..a04799cbaddcd --- /dev/null +++ b/x-pack/packages/observability/observability_utils/object/unflatten_object.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { set } from '@kbn/safer-lodash-set'; + +export function unflattenObject(source: Record, target: Record = {}) { + // eslint-disable-next-line guard-for-in + for (const key in source) { + const val = source[key as keyof typeof source]; + set(target, key, val); + } + return target; +} diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap b/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap index 6fa3e146a423d..22dbfb92aa0e0 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap @@ -230,6 +230,10 @@ exports[`Error OBSERVER_LISTENING 1`] = `undefined`; exports[`Error PARENT_ID 1`] = `"parentId"`; +exports[`Error PROCESS_ARGS 1`] = `undefined`; + +exports[`Error PROCESS_PID 1`] = `undefined`; + exports[`Error PROCESSOR_EVENT 1`] = `"error"`; exports[`Error SERVICE 1`] = ` @@ -308,6 +312,8 @@ exports[`Error TIMESTAMP 1`] = `1337`; exports[`Error TRACE_ID 1`] = `"trace id"`; +exports[`Error TRANSACTION_AGENT_MARKS 1`] = `undefined`; + exports[`Error TRANSACTION_DURATION 1`] = `undefined`; exports[`Error TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`; @@ -565,6 +571,10 @@ exports[`Span OBSERVER_LISTENING 1`] = `undefined`; exports[`Span PARENT_ID 1`] = `"parentId"`; +exports[`Span PROCESS_ARGS 1`] = `undefined`; + +exports[`Span PROCESS_PID 1`] = `undefined`; + exports[`Span PROCESSOR_EVENT 1`] = `"span"`; exports[`Span SERVICE 1`] = ` @@ -639,6 +649,8 @@ exports[`Span TIMESTAMP 1`] = `1337`; exports[`Span TRACE_ID 1`] = `"trace id"`; +exports[`Span TRANSACTION_AGENT_MARKS 1`] = `undefined`; + exports[`Span TRANSACTION_DURATION 1`] = `undefined`; exports[`Span TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`; @@ -910,6 +922,10 @@ exports[`Transaction OBSERVER_LISTENING 1`] = `undefined`; exports[`Transaction PARENT_ID 1`] = `"parentId"`; +exports[`Transaction PROCESS_ARGS 1`] = `undefined`; + +exports[`Transaction PROCESS_PID 1`] = `undefined`; + exports[`Transaction PROCESSOR_EVENT 1`] = `"transaction"`; exports[`Transaction SERVICE 1`] = ` @@ -988,6 +1004,8 @@ exports[`Transaction TIMESTAMP 1`] = `1337`; exports[`Transaction TRACE_ID 1`] = `"trace id"`; +exports[`Transaction TRANSACTION_AGENT_MARKS 1`] = `undefined`; + exports[`Transaction TRANSACTION_DURATION 1`] = `1337`; exports[`Transaction TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`; diff --git a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts index 49f282473e9dd..2fd0be94a5c5f 100644 --- a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts +++ b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts @@ -64,16 +64,17 @@ export interface WaterfallSpan { links?: SpanLink[]; }; transaction?: { - id: string; + id?: string; }; child?: { id: string[] }; } export interface WaterfallError { timestamp: TimestampUs; - trace?: { id: string }; - transaction?: { id: string }; - parent?: { id: string }; + trace?: { id?: string }; + transaction?: { id?: string }; + parent?: { id?: string }; + span?: { id?: string }; error: { id: string; log?: { diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx index 2e91865083b8c..20d5521b43ebf 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx @@ -8,9 +8,9 @@ import { EuiFlexItem, EuiSpacer } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import type { Message } from '@kbn/observability-ai-assistant-plugin/public'; import React, { useMemo, useState } from 'react'; +import { AT_TIMESTAMP } from '@kbn/apm-types'; import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context'; import { APMError } from '../../../../../typings/es_schemas/ui/apm_error'; -import { Transaction } from '../../../../../typings/es_schemas/ui/transaction'; import { ErrorSampleDetailTabContent } from './error_sample_detail'; import { exceptionStacktraceTab, logStacktraceTab } from './error_tabs'; @@ -18,8 +18,26 @@ export function ErrorSampleContextualInsight({ error, transaction, }: { - error: APMError; - transaction?: Transaction; + error: { + [AT_TIMESTAMP]: string; + error: Pick; + service: { + name: string; + environment?: string; + language?: { + name?: string; + }; + runtime?: { + name?: string; + version?: string; + }; + }; + }; + transaction?: { + transaction: { + name: string; + }; + }; }) { const { observabilityAIAssistant } = useApmPluginContext(); diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx index a38c4dfc96f63..90bffc1641d13 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx @@ -29,7 +29,7 @@ import { first } from 'lodash'; import React, { useEffect, useState } from 'react'; import { useHistory } from 'react-router-dom'; import useAsync from 'react-use/lib/useAsync'; -import { ERROR_GROUP_ID } from '../../../../../common/es_fields/apm'; +import { AT_TIMESTAMP, ERROR_GROUP_ID } from '../../../../../common/es_fields/apm'; import { TraceSearchType } from '../../../../../common/trace_explorer'; import { APMError } from '../../../../../typings/es_schemas/ui/apm_error'; import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context'; @@ -348,14 +348,22 @@ export function ErrorSampleDetailTabContent({ error, currentTab, }: { - error: APMError; + error: { + service: { + language?: { + name?: string; + }; + }; + [AT_TIMESTAMP]: string; + error: Pick; + }; currentTab: ErrorTab; }) { const codeLanguage = error?.service.language?.name; const exceptions = error?.error.exception || []; const logStackframes = error?.error.log?.stacktrace; const isPlaintextException = - !!error?.error.stack_trace && exceptions.length === 1 && !exceptions[0].stacktrace; + !!error.error.stack_trace && exceptions.length === 1 && !exceptions[0].stacktrace; switch (currentTab.key) { case ErrorTabKey.LogStackTrace: return ; diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx index 893e842513c8f..86b69eb480b3f 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx @@ -41,7 +41,7 @@ export const metadataTab: ErrorTab = { }), }; -export function getTabs(error: APMError) { +export function getTabs(error: { error: { log?: APMError['error']['log'] } }) { const hasLogStacktrace = !isEmpty(error?.error.log?.stacktrace); return [...(hasLogStacktrace ? [logStacktraceTab] : []), exceptionStacktraceTab, metadataTab]; } diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx index af05e8766994c..c7acbfee7e45e 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx @@ -18,7 +18,9 @@ const Label = euiStyled.div` `; interface Props { - error: APMError; + error: { + error: Pick; + }; } export function SampleSummary({ error }: Props) { const logMessage = error.error.log?.message; diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts b/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts index a3467d7272ff5..acd9e9445ad8f 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts +++ b/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts @@ -6,7 +6,7 @@ */ import { format } from 'url'; -import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; +import type { TransactionDetailRedirectInfo } from '../../../../server/routes/transactions/get_transaction_by_trace'; export const getRedirectToTransactionDetailPageUrl = ({ transaction, @@ -14,7 +14,7 @@ export const getRedirectToTransactionDetailPageUrl = ({ rangeTo, waterfallItemId, }: { - transaction: Transaction; + transaction: TransactionDetailRedirectInfo; rangeFrom?: string; rangeTo?: string; waterfallItemId?: string; diff --git a/x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx b/x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx index 2958d2af7d68f..a32c01f3b15e5 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx @@ -7,10 +7,18 @@ import React, { ReactNode } from 'react'; import { ERROR_GROUP_ID, SERVICE_NAME } from '../../../../../common/es_fields/apm'; -import { APMError } from '../../../../../typings/es_schemas/ui/apm_error'; import { DiscoverLink } from './discover_link'; -function getDiscoverQuery(error: APMError, kuery?: string) { +interface ErrorForDiscoverQuery { + service: { + name: string; + }; + error: { + grouping_key: string; + }; +} + +function getDiscoverQuery(error: ErrorForDiscoverQuery, kuery?: string) { const serviceName = error.service.name; const groupId = error.error.grouping_key; let query = `${SERVICE_NAME}:"${serviceName}" AND ${ERROR_GROUP_ID}:"${groupId}"`; @@ -36,7 +44,7 @@ function DiscoverErrorLink({ children, }: { children?: ReactNode; - readonly error: APMError; + readonly error: ErrorForDiscoverQuery; readonly kuery?: string; }) { return ; diff --git a/x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx index ae688f8917602..dab585180fce9 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx @@ -7,13 +7,16 @@ import React, { useMemo } from 'react'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { APMError } from '../../../../../typings/es_schemas/ui/apm_error'; +import { APMError, AT_TIMESTAMP } from '@kbn/apm-types'; import { getSectionsFromFields } from '../helper'; import { MetadataTable } from '..'; import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher'; interface Props { - error: APMError; + error: { + [AT_TIMESTAMP]: string; + error: Pick; + }; } export function ErrorMetadata({ error }: Props) { @@ -26,8 +29,8 @@ export function ErrorMetadata({ error }: Props) { id: error.error.id, }, query: { - start: error['@timestamp'], - end: error['@timestamp'], + start: error[AT_TIMESTAMP], + end: error[AT_TIMESTAMP], }, }, }); diff --git a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts index 319582f61b664..ed460257b0b10 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts @@ -101,7 +101,7 @@ describe('data telemetry collection tasks', () => { // a fixed date range. .mockReturnValueOnce({ hits: { - hits: [{ _source: { '@timestamp': new Date().toISOString() } }], + hits: [{ field: { '@timestamp': new Date().toISOString() } }], }, total: { value: 1, @@ -314,7 +314,7 @@ describe('data telemetry collection tasks', () => { ? { hits: { total: { value: 1 } } } : { hits: { - hits: [{ _source: { '@timestamp': 1 } }], + hits: [{ field: { '@timestamp': 1 } }], }, } ); diff --git a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index db6a6a918177a..40c4b9a8f6121 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -11,11 +11,13 @@ import { createHash } from 'crypto'; import { flatten, merge, pickBy, sortBy, sum, uniq } from 'lodash'; import { SavedObjectsClient } from '@kbn/core/server'; import type { APMIndices } from '@kbn/apm-data-access-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; import { AGENT_NAMES, RUM_AGENT_NAMES } from '../../../../common/agent_name'; import { AGENT_ACTIVATION_METHOD, AGENT_NAME, AGENT_VERSION, + AT_TIMESTAMP, CLIENT_GEO_COUNTRY_ISO_CODE, CLOUD_AVAILABILITY_ZONE, CLOUD_PROVIDER, @@ -29,6 +31,7 @@ import { METRICSET_INTERVAL, METRICSET_NAME, OBSERVER_HOSTNAME, + OBSERVER_VERSION, PARENT_ID, PROCESSOR_EVENT, SERVICE_ENVIRONMENT, @@ -54,10 +57,7 @@ import { SavedServiceGroup, } from '../../../../common/service_groups'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; -import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; -import { Span } from '../../../../typings/es_schemas/ui/span'; -import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; import { APMDataTelemetry, APMPerService, @@ -193,17 +193,19 @@ export const tasks: TelemetryTask[] = [ size: 1, track_total_hits: false, sort: { - '@timestamp': 'desc' as const, + [AT_TIMESTAMP]: 'desc' as const, }, + fields: [AT_TIMESTAMP], }, }) - ).hits.hits[0] as { _source: { '@timestamp': string } }; + ).hits.hits[0]; if (!lastTransaction) { return {}; } - const end = new Date(lastTransaction._source['@timestamp']).getTime() - 5 * 60 * 1000; + const end = + new Date(lastTransaction.fields[AT_TIMESTAMP]![0] as string).getTime() - 5 * 60 * 1000; const start = end - 60 * 1000; @@ -512,16 +514,16 @@ export const tasks: TelemetryTask[] = [ }, }, sort: { - '@timestamp': 'asc', + [AT_TIMESTAMP]: 'asc', }, - _source: ['@timestamp'], + fields: [AT_TIMESTAMP], }, }) : null; - const event = retainmentResponse?.hits.hits[0]?._source as + const event = retainmentResponse?.hits.hits[0]?.fields as | { - '@timestamp': number; + [AT_TIMESTAMP]: number[]; } | undefined; @@ -535,7 +537,7 @@ export const tasks: TelemetryTask[] = [ ? { retainment: { [processorEvent]: { - ms: new Date().getTime() - new Date(event['@timestamp']).getTime(), + ms: new Date().getTime() - new Date(event['@timestamp'][0]).getTime(), }, }, } @@ -690,16 +692,16 @@ export const tasks: TelemetryTask[] = [ sort: { '@timestamp': 'desc', }, + fields: asMutableArray([OBSERVER_VERSION] as const), }, }); - const hit = response.hits.hits[0]?._source as Pick; - - if (!hit || !hit.observer?.version) { + const event = unflattenKnownApmEventFields(response.hits.hits[0]?.fields); + if (!event || !event.observer?.version) { return {}; } - const [major, minor, patch] = hit.observer.version.split('.').map((part) => Number(part)); + const [major, minor, patch] = event.observer.version.split('.').map((part) => Number(part)); return { version: { diff --git a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts index 88d0040f70fc9..5d4977a73b42f 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts @@ -9,6 +9,10 @@ import { NOT_AVAILABLE_LABEL } from '../../../common/i18n'; import { Maybe } from '../../../typings/common'; import { APMError } from '../../../typings/es_schemas/ui/apm_error'; -export function getErrorName({ error }: { error: Maybe }): string { +export function getErrorName({ + error, +}: { + error: Maybe> & { log?: { message?: string } }; +}): string { return error?.log?.message || error?.exception?.[0]?.message || NOT_AVAILABLE_LABEL; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts index 5f36325031ccb..9d8917bbaad98 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts @@ -8,6 +8,9 @@ import datemath from '@elastic/datemath'; import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { flattenObject, KeyValuePair } from '../../../../common/utils/flatten_object'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { PROCESSOR_EVENT, TRACE_ID } from '../../../../common/es_fields/apm'; @@ -86,6 +89,7 @@ export async function getLogCategories({ const rawSamplingProbability = Math.min(100_000 / totalDocCount, 1); const samplingProbability = rawSamplingProbability < 0.5 ? rawSamplingProbability : 1; + const requiredFields = asMutableArray(['message', TRACE_ID] as const); const categorizedLogsRes = await search({ index, size: 1, @@ -108,7 +112,7 @@ export async function getLogCategories({ top_hits: { sort: { '@timestamp': 'desc' as const }, size: 1, - _source: ['message', TRACE_ID], + fields: requiredFields, }, }, }, @@ -120,9 +124,11 @@ export async function getLogCategories({ const promises = categorizedLogsRes.aggregations?.sampling.categories?.buckets.map( async ({ doc_count: docCount, key, sample }) => { - const hit = sample.hits.hits[0]._source as { message: string; trace?: { id: string } }; - const sampleMessage = hit?.message; - const sampleTraceId = hit?.trace?.id; + const hit = sample.hits.hits[0]; + const event = unflattenKnownApmEventFields(hit?.fields); + + const sampleMessage = event.message as string; + const sampleTraceId = event.trace?.id; const errorCategory = key as string; if (!sampleTraceId) { @@ -140,7 +146,9 @@ export async function getLogCategories({ } ); - const sampleDoc = categorizedLogsRes.hits.hits?.[0]?._source as Record; + const event = unflattenKnownApmEventFields(maybe(categorizedLogsRes.hits.hits[0])?.fields); + + const sampleDoc = event as Record; return { logCategories: await Promise.all(promises ?? []), diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts index e7f3ace07e2a1..93c55cf1a9a30 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts @@ -13,6 +13,10 @@ import moment from 'moment'; import { ESSearchRequest } from '@kbn/es-types'; import { alertDetailsContextRt } from '@kbn/observability-plugin/server/services'; import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; +import { CONTAINER_ID } from '@kbn/apm-types'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { ApmDocumentType } from '../../../../common/document_type'; import { APMEventClient, @@ -79,13 +83,17 @@ async function getContainerIdFromLogs({ esClient: ElasticsearchClient; logSourcesService: LogSourcesService; }) { + const requiredFields = asMutableArray([CONTAINER_ID] as const); const index = await logSourcesService.getFlattenedLogSources(); const res = await typedSearch<{ container: { id: string } }, any>(esClient, { index, ...params, + fields: requiredFields, }); - return res.hits.hits[0]?._source?.container?.id; + const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields); + + return event?.container.id; } async function getContainerIdFromTraces({ @@ -95,6 +103,7 @@ async function getContainerIdFromTraces({ params: APMEventESSearchRequest['body']; apmEventClient: APMEventClient; }) { + const requiredFields = asMutableArray([CONTAINER_ID] as const); const res = await apmEventClient.search('get_container_id_from_traces', { apm: { sources: [ @@ -104,8 +113,10 @@ async function getContainerIdFromTraces({ }, ], }, - body: params, + body: { ...params, fields: requiredFields }, }); - return res.hits.hits[0]?._source.container?.id; + const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields); + + return event?.container.id; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts index a5b75f76ff237..d957372285b02 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts @@ -6,6 +6,9 @@ */ import { rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; +import { maybe } from '../../../../common/utils/maybe'; import { ApmDocumentType } from '../../../../common/document_type'; import { termQuery } from '../../../../common/utils/term_query'; import { @@ -27,6 +30,7 @@ export async function getDownstreamServiceResource({ end: number; apmEventClient: APMEventClient; }) { + const requiredFields = asMutableArray([SPAN_DESTINATION_SERVICE_RESOURCE] as const); const response = await apmEventClient.search('get_error_group_main_statistics', { apm: { sources: [ @@ -50,9 +54,11 @@ export async function getDownstreamServiceResource({ ], }, }, + fields: requiredFields, }, }); - const hit = response.hits.hits[0]; - return hit?._source?.span.destination?.service.resource; + const event = unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields, requiredFields); + + return event?.span.destination.service.resource; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts index 0168431d0ac4e..bd966c500d1bc 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts @@ -12,6 +12,10 @@ import moment from 'moment'; import { ESSearchRequest } from '@kbn/es-types'; import { alertDetailsContextRt } from '@kbn/observability-plugin/server/services'; import type { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { SERVICE_NAME } from '@kbn/apm-types'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { ApmDocumentType } from '../../../../common/document_type'; import { APMEventClient, @@ -102,6 +106,7 @@ async function getServiceNameFromTraces({ params: APMEventESSearchRequest['body']; apmEventClient: APMEventClient; }) { + const requiredFields = asMutableArray([SERVICE_NAME] as const); const res = await apmEventClient.search('get_service_name_from_traces', { apm: { sources: [ @@ -111,8 +116,13 @@ async function getServiceNameFromTraces({ }, ], }, - body: params, + body: { + ...params, + fields: requiredFields, + }, }); - return res.hits.hits[0]?._source.service.name; + const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields); + + return event?.service.name; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts index ebb3d3f57d18a..d3d335620b155 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts @@ -7,8 +7,14 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { maybe } from '../../../common/utils/maybe'; -import { SPAN_DESTINATION_SERVICE_RESOURCE } from '../../../common/es_fields/apm'; +import { + SPAN_DESTINATION_SERVICE_RESOURCE, + SPAN_SUBTYPE, + SPAN_TYPE, +} from '../../../common/es_fields/apm'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; export interface MetadataForDependencyResponse { @@ -27,6 +33,7 @@ export async function getMetadataForDependency({ start: number; end: number; }): Promise { + const requiredFields = asMutableArray([SPAN_TYPE, SPAN_SUBTYPE] as const); const sampleResponse = await apmEventClient.search('get_metadata_for_dependency', { apm: { events: [ProcessorEvent.span], @@ -46,13 +53,17 @@ export async function getMetadataForDependency({ ], }, }, + fields: requiredFields, sort: { '@timestamp': 'desc', }, }, }); - const sample = maybe(sampleResponse.hits.hits[0])?._source; + const sample = unflattenKnownApmEventFields( + maybe(sampleResponse.hits.hits[0])?.fields, + requiredFields + ); return { spanType: sample?.span.type, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts index 1c8448579806f..2a5a804d57f04 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts @@ -8,8 +8,11 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { kqlQuery, rangeQuery, termQuery, termsQuery } from '@kbn/observability-plugin/server'; import { keyBy } from 'lodash'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { AGENT_NAME, + AT_TIMESTAMP, EVENT_OUTCOME, SERVICE_ENVIRONMENT, SERVICE_NAME, @@ -66,6 +69,19 @@ export async function getTopDependencySpans({ sampleRangeFrom?: number; sampleRangeTo?: number; }): Promise { + const topDedsRequiredFields = asMutableArray([ + SPAN_ID, + TRACE_ID, + TRANSACTION_ID, + SPAN_NAME, + SERVICE_NAME, + SERVICE_ENVIRONMENT, + AGENT_NAME, + SPAN_DURATION, + EVENT_OUTCOME, + AT_TIMESTAMP, + ] as const); + const spans = ( await apmEventClient.search('get_top_dependency_spans', { apm: { @@ -98,23 +114,18 @@ export async function getTopDependencySpans({ ], }, }, - _source: [ - SPAN_ID, - TRACE_ID, - TRANSACTION_ID, - SPAN_NAME, - SERVICE_NAME, - SERVICE_ENVIRONMENT, - AGENT_NAME, - SPAN_DURATION, - EVENT_OUTCOME, - '@timestamp', - ], + fields: topDedsRequiredFields, }, }) - ).hits.hits.map((hit) => hit._source); + ).hits.hits.map((hit) => unflattenKnownApmEventFields(hit.fields, topDedsRequiredFields)); + + const transactionIds = spans.map((span) => span.transaction.id); - const transactionIds = spans.map((span) => span.transaction!.id); + const txRequiredFields = asMutableArray([ + TRANSACTION_ID, + TRANSACTION_TYPE, + TRANSACTION_NAME, + ] as const); const transactions = ( await apmEventClient.search('get_transactions_for_dependency_spans', { @@ -129,13 +140,13 @@ export async function getTopDependencySpans({ filter: [...termsQuery(TRANSACTION_ID, ...transactionIds)], }, }, - _source: [TRANSACTION_ID, TRANSACTION_TYPE, TRANSACTION_NAME], + fields: txRequiredFields, sort: { '@timestamp': 'desc', }, }, }) - ).hits.hits.map((hit) => hit._source); + ).hits.hits.map((hit) => unflattenKnownApmEventFields(hit.fields, txRequiredFields)); const transactionsById = keyBy(transactions, (transaction) => transaction.transaction.id); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts index 1e9576ea2f7e4..6323ad5212ab3 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts @@ -7,13 +7,18 @@ import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types'; import { kqlQuery, rangeQuery, termQuery, wildcardQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { castArray } from 'lodash'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { + AT_TIMESTAMP, ERROR_CULPRIT, ERROR_EXC_HANDLED, ERROR_EXC_MESSAGE, ERROR_EXC_TYPE, ERROR_GROUP_ID, ERROR_GROUP_NAME, + ERROR_ID, ERROR_LOG_MESSAGE, SERVICE_NAME, TRACE_ID, @@ -93,6 +98,21 @@ export async function getErrorGroupMainStatistics({ ] : []; + const requiredFields = asMutableArray([ + TRACE_ID, + AT_TIMESTAMP, + ERROR_GROUP_ID, + ERROR_ID, + ] as const); + + const optionalFields = asMutableArray([ + ERROR_CULPRIT, + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const response = await apmEventClient.search('get_error_group_main_statistics', { apm: { sources: [ @@ -129,16 +149,8 @@ export async function getErrorGroupMainStatistics({ sample: { top_hits: { size: 1, - _source: [ - TRACE_ID, - ERROR_LOG_MESSAGE, - ERROR_EXC_MESSAGE, - ERROR_EXC_HANDLED, - ERROR_EXC_TYPE, - ERROR_CULPRIT, - ERROR_GROUP_ID, - '@timestamp', - ], + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE], sort: { '@timestamp': 'desc', }, @@ -157,15 +169,30 @@ export async function getErrorGroupMainStatistics({ const errorGroups = response.aggregations?.error_groups.buckets.map((bucket) => { + const errorSource = + 'error' in bucket.sample.hits.hits[0]._source + ? bucket.sample.hits.hits[0]._source + : undefined; + + const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields); + + const mergedEvent = { + ...event, + error: { + ...(event.error ?? {}), + exception: castArray(errorSource?.error.exception ?? event?.error.exception), + }, + }; + return { groupId: bucket.key as string, - name: getErrorName(bucket.sample.hits.hits[0]._source), - lastSeen: new Date(bucket.sample.hits.hits[0]._source['@timestamp']).getTime(), + name: getErrorName(mergedEvent), + lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(), occurrences: bucket.doc_count, - culprit: bucket.sample.hits.hits[0]._source.error.culprit, - handled: bucket.sample.hits.hits[0]._source.error.exception?.[0].handled, - type: bucket.sample.hits.hits[0]._source.error.exception?.[0].type, - traceId: bucket.sample.hits.hits[0]._source.trace?.id, + culprit: mergedEvent.error.culprit, + handled: mergedEvent.error.exception[0].handled, + type: mergedEvent.error.exception[0].type, + traceId: mergedEvent.trace?.id, }; }) ?? []; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts index 0a154d3ad13fa..fc80c3f411651 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts @@ -6,6 +6,7 @@ */ import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { ERROR_GROUP_ID, @@ -42,6 +43,7 @@ export async function getErrorGroupSampleIds({ start: number; end: number; }): Promise { + const requiredFields = asMutableArray([ERROR_ID] as const); const resp = await apmEventClient.search('get_error_group_sample_ids', { apm: { sources: [ @@ -66,7 +68,7 @@ export async function getErrorGroupSampleIds({ should: [{ term: { [TRANSACTION_SAMPLED]: true } }], // prefer error samples with related transactions }, }, - _source: [ERROR_ID, 'transaction'], + fields: requiredFields, sort: asMutableArray([ { _score: { order: 'desc' } }, // sort by _score first to ensure that errors with transaction.sampled:true ends up on top { '@timestamp': { order: 'desc' } }, // sort by timestamp to get the most recent error @@ -74,8 +76,8 @@ export async function getErrorGroupSampleIds({ }, }); const errorSampleIds = resp.hits.hits.map((item) => { - const source = item._source; - return source.error.id; + const event = unflattenKnownApmEventFields(item.fields, requiredFields); + return event.error?.id; }); return { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts index 348949d3ecca5..dfad2a71e1f0d 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts @@ -6,7 +6,28 @@ */ import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server'; -import { ERROR_ID, SERVICE_NAME } from '../../../../common/es_fields/apm'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { castArray } from 'lodash'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; +import { maybe } from '../../../../common/utils/maybe'; +import { + AGENT_NAME, + AGENT_VERSION, + AT_TIMESTAMP, + ERROR_EXCEPTION, + ERROR_GROUP_ID, + ERROR_ID, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + PROCESSOR_EVENT, + PROCESSOR_NAME, + SERVICE_NAME, + TIMESTAMP, + TRACE_ID, + TRANSACTION_ID, + ERROR_STACK_TRACE, +} from '../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../common/utils/environment_query'; import { ApmDocumentType } from '../../../../common/document_type'; import { RollupInterval } from '../../../../common/rollup'; @@ -17,7 +38,15 @@ import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; export interface ErrorSampleDetailsResponse { transaction: Transaction | undefined; - error: APMError; + error: Omit & { + transaction?: { id?: string; type?: string }; + error: { + id: string; + } & Omit & { + exception?: APMError['error']['exception']; + log?: APMError['error']['log']; + }; + }; } export async function getErrorSampleDetails({ @@ -36,7 +65,28 @@ export async function getErrorSampleDetails({ apmEventClient: APMEventClient; start: number; end: number; -}): Promise { +}): Promise> { + const requiredFields = asMutableArray([ + AGENT_NAME, + AGENT_VERSION, + PROCESSOR_EVENT, + TRACE_ID, + TIMESTAMP, + AT_TIMESTAMP, + SERVICE_NAME, + ERROR_ID, + ERROR_GROUP_ID, + ] as const); + + const optionalFields = asMutableArray([ + TRANSACTION_ID, + PROCESSOR_NAME, + ERROR_STACK_TRACE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const params = { apm: { sources: [ @@ -60,15 +110,29 @@ export async function getErrorSampleDetails({ ], }, }, + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_EXCEPTION, 'error.log'], }, }; const resp = await apmEventClient.search('get_error_sample_details', params); - const error = resp.hits.hits[0]?._source; - const transactionId = error?.transaction?.id; - const traceId = error?.trace?.id; + const hit = maybe(resp.hits.hits[0]); + + if (!hit) { + return { + transaction: undefined, + error: undefined, + }; + } + + const source = 'error' in hit._source ? hit._source : undefined; - let transaction; + const errorFromFields = unflattenKnownApmEventFields(hit.fields, requiredFields); + + const transactionId = errorFromFields.transaction?.id; + const traceId = errorFromFields.trace.id; + + let transaction: Transaction | undefined; if (transactionId && traceId) { transaction = await getTransaction({ transactionId, @@ -81,6 +145,17 @@ export async function getErrorSampleDetails({ return { transaction, - error, + error: { + ...errorFromFields, + processor: { + name: errorFromFields.processor.name as 'error', + event: errorFromFields.processor.event as 'error', + }, + error: { + ...errorFromFields.error, + exception: castArray(source?.error.exception ?? errorFromFields?.error.exception), + log: source?.error?.log, + }, + }, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts index dd262246a80b7..62d9d883ba896 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts @@ -7,6 +7,7 @@ import { jsonRt, toNumberRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; +import { notFound } from '@hapi/boom'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { ErrorDistributionResponse, getErrorDistribution } from './distribution/get_distribution'; import { environmentRt, kueryRt, rangeRt } from '../default_api_types'; @@ -205,7 +206,7 @@ const errorGroupSampleDetailsRoute = createApmServerRoute({ const { serviceName, errorId } = params.path; const { environment, kuery, start, end } = params.query; - return getErrorSampleDetails({ + const { transaction, error } = await getErrorSampleDetails({ environment, errorId, kuery, @@ -214,6 +215,12 @@ const errorGroupSampleDetailsRoute = createApmServerRoute({ start, end, }); + + if (!error) { + throw notFound(); + } + + return { error, transaction }; }, }); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts index e766d56c44ae4..3a7e4a64c80f5 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts @@ -7,9 +7,12 @@ import type { ESFilter } from '@kbn/es-types'; import { rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { isFiniteNumber } from '../../../../common/utils/is_finite_number'; import { Annotation, AnnotationType } from '../../../../common/annotations'; -import { SERVICE_NAME, SERVICE_VERSION } from '../../../../common/es_fields/apm'; +import { AT_TIMESTAMP, SERVICE_NAME, SERVICE_VERSION } from '../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../common/utils/environment_query'; import { getBackwardCompatibleDocumentTypeFilter, @@ -66,6 +69,8 @@ export async function getDerivedServiceAnnotations({ if (versions.length <= 1) { return []; } + + const requiredFields = asMutableArray([AT_TIMESTAMP] as const); const annotations = await Promise.all( versions.map(async (version) => { const response = await apmEventClient.search('get_first_seen_of_version', { @@ -86,8 +91,17 @@ export async function getDerivedServiceAnnotations({ }, }); - const firstSeen = new Date(response.hits.hits[0]._source['@timestamp']).getTime(); + const event = unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields, + requiredFields + ); + + const timestamp = event?.[AT_TIMESTAMP]; + if (!timestamp) { + throw new Error('First seen for version was unexpectedly undefined or null.'); + } + const firstSeen = new Date(timestamp).getTime(); if (!isFiniteNumber(firstSeen)) { throw new Error('First seen for version was unexpectedly undefined or null.'); } @@ -99,7 +113,7 @@ export async function getDerivedServiceAnnotations({ return { type: AnnotationType.VERSION, id: version, - '@timestamp': firstSeen, + [AT_TIMESTAMP]: firstSeen, text: version, }; }) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts index 94c5bcbac4e66..8aed821b10f2a 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts @@ -7,6 +7,8 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { AGENT_NAME, SERVICE_NAME, @@ -16,23 +18,7 @@ import { } from '../../../common/es_fields/apm'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { getServerlessTypeFromCloudData, ServerlessType } from '../../../common/serverless'; - -interface ServiceAgent { - agent?: { - name: string; - }; - service?: { - runtime?: { - name?: string; - }; - }; - cloud?: { - provider?: string; - service?: { - name?: string; - }; - }; -} +import { maybe } from '../../../common/utils/maybe'; export interface ServiceAgentResponse { agentName?: string; @@ -51,6 +37,13 @@ export async function getServiceAgent({ start: number; end: number; }): Promise { + const requiredFields = asMutableArray([ + AGENT_NAME, + SERVICE_RUNTIME_NAME, + CLOUD_PROVIDER, + CLOUD_SERVICE_NAME, + ] as const); + const params = { terminate_after: 1, apm: { @@ -90,6 +83,7 @@ export async function getServiceAgent({ ], }, }, + fields: requiredFields, sort: { _score: { order: 'desc' as const }, }, @@ -97,11 +91,14 @@ export async function getServiceAgent({ }; const response = await apmEventClient.search('get_service_agent_name', params); - if (response.hits.total.value === 0) { + const hit = maybe(response.hits.hits[0]); + if (!hit) { return {}; } - const { agent, service, cloud } = response.hits.hits[0]._source as ServiceAgent; + const event = unflattenKnownApmEventFields(hit.fields); + + const { agent, service, cloud } = event; const serverlessType = getServerlessTypeFromCloudData(cloud?.provider, cloud?.service?.name); return { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts index 400429617d803..1d82c26199586 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts @@ -6,19 +6,20 @@ */ import { rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { CONTAINER_ID, CONTAINER_IMAGE, KUBERNETES, KUBERNETES_POD_NAME, KUBERNETES_POD_UID, -} from '../../../common/es_fields/apm'; -import { KUBERNETES_CONTAINER_NAME, KUBERNETES_NAMESPACE, KUBERNETES_REPLICASET_NAME, KUBERNETES_DEPLOYMENT_NAME, -} from '../../../common/es_fields/infra_metrics'; + KUBERNETES_CONTAINER_ID, +} from '../../../common/es_fields/apm'; import { Kubernetes } from '../../../typings/es_schemas/raw/fields/kubernetes'; import { maybe } from '../../../common/utils/maybe'; import { InfraMetricsClient } from '../../lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client'; @@ -51,9 +52,21 @@ export const getServiceInstanceContainerMetadata = async ({ { exists: { field: KUBERNETES_DEPLOYMENT_NAME } }, ]; + const fields = asMutableArray([ + KUBERNETES_POD_NAME, + KUBERNETES_POD_UID, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_CONTAINER_ID, + KUBERNETES_CONTAINER_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_DEPLOYMENT_NAME, + ] as const); + const response = await infraMetricsClient.search({ size: 1, track_total_hits: false, + fields, query: { bool: { filter: [ @@ -69,7 +82,7 @@ export const getServiceInstanceContainerMetadata = async ({ }, }); - const sample = maybe(response.hits.hits[0])?._source as ServiceInstanceContainerMetadataDetails; + const sample = unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields); return { kubernetes: { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts index daa49d2ed59c8..62d714734d9b8 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts @@ -7,6 +7,8 @@ import { merge } from 'lodash'; import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; import { METRICSET_NAME, SERVICE_NAME, SERVICE_NODE_NAME } from '../../../common/es_fields/apm'; import { maybe } from '../../../common/utils/maybe'; import { @@ -70,7 +72,9 @@ export async function getServiceInstanceMetadataDetails({ } ); - return maybe(response.hits.hits[0]?._source); + return unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); } async function getTransactionEventSample() { @@ -89,7 +93,9 @@ export async function getServiceInstanceMetadataDetails({ } ); - return maybe(response.hits.hits[0]?._source); + return unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); } async function getTransactionMetricSample() { @@ -111,7 +117,10 @@ export async function getServiceInstanceMetadataDetails({ }, } ); - return maybe(response.hits.hits[0]?._source); + + return unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); } // we can expect the most detail of application metrics, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts index fb44638d8a6b0..1ba4c75a8fd3c 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts @@ -7,6 +7,8 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; import { environmentQuery } from '../../../common/utils/environment_query'; import { AGENT, @@ -27,17 +29,11 @@ import { FAAS_TRIGGER_TYPE, LABEL_TELEMETRY_AUTO_VERSION, } from '../../../common/es_fields/apm'; - import { ContainerType } from '../../../common/service_metadata'; -import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { should } from './get_service_metadata_icons'; import { isOpenTelemetryAgentName, hasOpenTelemetryPrefix } from '../../../common/agent_name'; - -type ServiceMetadataDetailsRaw = Pick< - TransactionRaw, - 'service' | 'agent' | 'host' | 'container' | 'kubernetes' | 'cloud' | 'labels' ->; +import { maybe } from '../../../common/utils/maybe'; export interface ServiceMetadataDetails { service?: { @@ -171,8 +167,11 @@ export async function getServiceMetadataDetails({ const response = await apmEventClient.search('get_service_metadata_details', params); - const hit = response.hits.hits[0]?._source as ServiceMetadataDetailsRaw | undefined; - if (!hit) { + const event = unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); + + if (!event) { return { service: undefined, container: undefined, @@ -180,7 +179,7 @@ export async function getServiceMetadataDetails({ }; } - const { service, agent, host, kubernetes, container, cloud, labels } = hit; + const { service, agent, host, kubernetes, container, cloud, labels } = event; const serviceMetadataDetails = { versions: response.aggregations?.serviceVersions.buckets.map((bucket) => bucket.key as string), diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts index 30580ddbf0ac8..9985d1259ffa5 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts @@ -7,27 +7,27 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../common/utils/maybe'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { AGENT_NAME, CLOUD_PROVIDER, CLOUD_SERVICE_NAME, CONTAINER_ID, - KUBERNETES, SERVICE_NAME, KUBERNETES_POD_NAME, HOST_OS_PLATFORM, LABEL_TELEMETRY_AUTO_VERSION, AGENT_VERSION, SERVICE_FRAMEWORK_NAME, + KUBERNETES, } from '../../../common/es_fields/apm'; import { ContainerType } from '../../../common/service_metadata'; -import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { getProcessorEventForTransactions } from '../../lib/helpers/transactions'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { ServerlessType, getServerlessTypeFromCloudData } from '../../../common/serverless'; -type ServiceMetadataIconsRaw = Pick; - export interface ServiceMetadataIcons { agentName?: string; containerType?: ContainerType; @@ -61,6 +61,14 @@ export async function getServiceMetadataIcons({ }): Promise { const filter = [{ term: { [SERVICE_NAME]: serviceName } }, ...rangeQuery(start, end)]; + const fields = asMutableArray([ + KUBERNETES, + CLOUD_PROVIDER, + CONTAINER_ID, + AGENT_NAME, + CLOUD_SERVICE_NAME, + ] as const); + const params = { apm: { events: [ @@ -72,8 +80,9 @@ export async function getServiceMetadataIcons({ body: { track_total_hits: 1, size: 1, - _source: [KUBERNETES, CLOUD_PROVIDER, CONTAINER_ID, AGENT_NAME, CLOUD_SERVICE_NAME], + _source: false, query: { bool: { filter, should } }, + fields, }, }; @@ -88,9 +97,9 @@ export async function getServiceMetadataIcons({ }; } - const { kubernetes, cloud, container, agent } = response.hits.hits[0] - ._source as ServiceMetadataIconsRaw; + const event = unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields); + const { kubernetes, cloud, container, agent } = event ?? {}; let containerType: ContainerType; if (!!kubernetes) { containerType = 'Kubernetes'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts index 3f9cf1275cace..8d3c5315f635d 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts @@ -7,6 +7,8 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { isEmpty } from 'lodash'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { PROCESSOR_EVENT, SPAN_ID, @@ -16,8 +18,6 @@ import { TRACE_ID, TRANSACTION_ID, } from '../../../common/es_fields/apm'; -import type { SpanRaw } from '../../../typings/es_schemas/raw/span_raw'; -import type { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { getBufferedTimerange } from './utils'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; @@ -39,12 +39,20 @@ async function fetchLinkedChildrenOfSpan({ end, }); + const requiredFields = asMutableArray([ + TRACE_ID, + SPAN_ID, + PROCESSOR_EVENT, + TRANSACTION_ID, + ] as const); + const response = await apmEventClient.search('fetch_linked_children_of_span', { apm: { events: [ProcessorEvent.span, ProcessorEvent.transaction], }, - _source: [SPAN_LINKS, TRACE_ID, SPAN_ID, PROCESSOR_EVENT, TRANSACTION_ID], + _source: [SPAN_LINKS], body: { + fields: requiredFields, track_total_hits: false, size: 1000, query: { @@ -58,19 +66,32 @@ async function fetchLinkedChildrenOfSpan({ }, }, }); + + const linkedChildren = response.hits.hits.map((hit) => { + const source = 'span' in hit._source ? hit._source : undefined; + const event = unflattenKnownApmEventFields(hit.fields, requiredFields); + + return { + ...event, + span: { + ...event.span, + links: source?.span?.links ?? [], + }, + }; + }); // Filter out documents that don't have any span.links that match the combination of traceId and spanId - return response.hits.hits.filter(({ _source: source }) => { - const spanLinks = source.span?.links?.filter((spanLink) => { + return linkedChildren.filter((linkedChild) => { + const spanLinks = linkedChild?.span?.links?.filter((spanLink) => { return spanLink.trace.id === traceId && (spanId ? spanLink.span.id === spanId : true); }); return !isEmpty(spanLinks); }); } -function getSpanId(source: TransactionRaw | SpanRaw) { - return source.processor.event === ProcessorEvent.span - ? (source as SpanRaw).span.id - : (source as TransactionRaw).transaction?.id; +function getSpanId(linkedChild: Awaited>[number]) { + return linkedChild.processor.event === ProcessorEvent.span + ? linkedChild.span?.id + : linkedChild.transaction?.id; } export async function getSpanLinksCountById({ @@ -90,8 +111,9 @@ export async function getSpanLinksCountById({ start, end, }); - return linkedChildren.reduce>((acc, { _source: source }) => { - source.span?.links?.forEach((link) => { + + return linkedChildren.reduce>((acc, item) => { + item.span?.links?.forEach((link) => { // Ignores span links that don't belong to this trace if (link.trace.id === traceId) { acc[link.span.id] = (acc[link.span.id] || 0) + 1; @@ -122,10 +144,10 @@ export async function getLinkedChildrenOfSpan({ end, }); - return linkedChildren.map(({ _source: source }) => { + return linkedChildren.map((item) => { return { - trace: { id: source.trace.id }, - span: { id: getSpanId(source) }, + trace: { id: item.trace.id }, + span: { id: getSpanId(item) }, }; }); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts index 2010cd5e86f2f..59e91e0b17e6b 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts @@ -56,7 +56,7 @@ export async function getLinkedParentsOfSpan({ }, }); - const source = response.hits.hits?.[0]?._source as TransactionRaw | SpanRaw; + const source = response.hits.hits?.[0]?._source as Pick; return source?.span?.links || []; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts index 13f47764af375..1ae02db257b88 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts @@ -7,6 +7,8 @@ import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { chunk, compact, isEmpty, keyBy } from 'lodash'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { SERVICE_NAME, SPAN_ID, @@ -25,8 +27,6 @@ import { import { Environment } from '../../../common/environment_rt'; import { SpanLinkDetails } from '../../../common/span_links'; import { SpanLink } from '../../../typings/es_schemas/raw/fields/span_links'; -import { SpanRaw } from '../../../typings/es_schemas/raw/span_raw'; -import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { getBufferedTimerange } from './utils'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; @@ -48,26 +48,36 @@ async function fetchSpanLinksDetails({ end, }); + const requiredFields = asMutableArray([ + TRACE_ID, + SPAN_ID, + SERVICE_NAME, + AGENT_NAME, + PROCESSOR_EVENT, + ] as const); + + const requiredTxFields = asMutableArray([ + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_DURATION, + ] as const); + + const requiredSpanFields = asMutableArray([ + SPAN_ID, + SPAN_NAME, + SPAN_DURATION, + SPAN_SUBTYPE, + SPAN_TYPE, + ] as const); + + const optionalFields = asMutableArray([SERVICE_ENVIRONMENT] as const); + const response = await apmEventClient.search('get_span_links_details', { apm: { events: [ProcessorEvent.span, ProcessorEvent.transaction], }, - _source: [ - TRACE_ID, - SPAN_ID, - TRANSACTION_ID, - SERVICE_NAME, - SPAN_NAME, - TRANSACTION_NAME, - TRANSACTION_DURATION, - SPAN_DURATION, - PROCESSOR_EVENT, - SPAN_SUBTYPE, - SPAN_TYPE, - AGENT_NAME, - SERVICE_ENVIRONMENT, - ], body: { + fields: [...requiredFields, ...requiredTxFields, ...requiredSpanFields, ...optionalFields], track_total_hits: false, size: 1000, query: { @@ -106,16 +116,67 @@ async function fetchSpanLinksDetails({ const spanIdsMap = keyBy(spanLinks, 'span.id'); - return response.hits.hits.filter(({ _source: source }) => { - // The above query might return other spans from the same transaction because siblings spans share the same transaction.id - // so, if it is a span we need to guarantee that the span.id is the same as the span links ids - if (source.processor.event === ProcessorEvent.span) { - const span = source as SpanRaw; - const hasSpanId = spanIdsMap[span.span.id] || false; - return hasSpanId; - } - return true; - }); + return response.hits.hits + .filter((hit) => { + // The above query might return other spans from the same transaction because siblings spans share the same transaction.id + // so, if it is a span we need to guarantee that the span.id is the same as the span links ids + if (hit.fields['processor.event']?.[0] === ProcessorEvent.span) { + const spanLink = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredSpanFields, + ]); + + const hasSpanId = Boolean(spanIdsMap[spanLink.span.id] || false); + return hasSpanId; + } + return true; + }) + .map((hit) => { + const commonEvent = unflattenKnownApmEventFields(hit.fields, requiredFields); + + const commonDetails = { + serviceName: commonEvent.service.name, + agentName: commonEvent.agent.name, + environment: commonEvent.service.environment as Environment, + }; + + if (commonEvent.processor.event === ProcessorEvent.transaction) { + const event = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredTxFields, + ]); + return { + traceId: event.trace.id, + spanId: event.transaction.id, + transactionId: event.transaction.id, + processorEvent: commonEvent.processor.event, + details: { + ...commonDetails, + spanName: event.transaction.name, + duration: event.transaction.duration.us, + }, + }; + } else { + const event = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredSpanFields, + ]); + + return { + traceId: event.trace.id, + spanId: event.span.id, + transactionId: event.transaction?.id, + processorEvent: commonEvent.processor.event, + details: { + ...commonDetails, + spanName: event.span.name, + duration: event.span.duration.us, + spanSubtype: event.span.subtype, + spanType: event.span.type, + }, + }; + } + }); } export async function getSpanLinksDetails({ @@ -153,39 +214,20 @@ export async function getSpanLinksDetails({ // Creates a map for all span links details found const spanLinksDetailsMap = linkedSpans.reduce>( - (acc, { _source: source }) => { - const commonDetails = { - serviceName: source.service.name, - agentName: source.agent.name, - environment: source.service.environment as Environment, - transactionId: source.transaction?.id, - }; - - if (source.processor.event === ProcessorEvent.transaction) { - const transaction = source as TransactionRaw; - const key = `${transaction.trace.id}:${transaction.transaction.id}`; + (acc, spanLink) => { + if (spanLink.processorEvent === ProcessorEvent.transaction) { + const key = `${spanLink.traceId}:${spanLink.transactionId}`; acc[key] = { - traceId: source.trace.id, - spanId: transaction.transaction.id, - details: { - ...commonDetails, - spanName: transaction.transaction.name, - duration: transaction.transaction.duration.us, - }, + traceId: spanLink.traceId, + spanId: spanLink.transactionId, + details: spanLink.details, }; } else { - const span = source as SpanRaw; - const key = `${span.trace.id}:${span.span.id}`; + const key = `${spanLink.traceId}:${spanLink.spanId}`; acc[key] = { - traceId: source.trace.id, - spanId: span.span.id, - details: { - ...commonDetails, - spanName: span.span.name, - duration: span.span.duration.us, - spanSubtype: span.span.subtype, - spanType: span.span.type, - }, + traceId: spanLink.traceId, + spanId: spanLink.spanId, + details: spanLink.details, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts index d38a49745653a..84fb558fc1480 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts @@ -10,7 +10,9 @@ import { SortResults } from '@elastic/elasticsearch/lib/api/types'; import { QueryDslQueryContainer, Sort } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { rangeQuery } from '@kbn/observability-plugin/server'; -import { last } from 'lodash'; +import { last, omit } from 'lodash'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { APMConfig } from '../..'; import { AGENT_NAME, @@ -84,6 +86,15 @@ export async function getTraceItems({ const maxTraceItems = maxTraceItemsFromUrlParam ?? config.ui.maxTraceItems; const excludedLogLevels = ['debug', 'info', 'warning']; + const requiredFields = asMutableArray([ + TIMESTAMP, + TRACE_ID, + SERVICE_NAME, + ERROR_ID, + ERROR_GROUP_ID, + PROCESSOR_EVENT, + ] as const); + const errorResponsePromise = apmEventClient.search('get_errors_docs', { apm: { sources: [ @@ -96,23 +107,14 @@ export async function getTraceItems({ body: { track_total_hits: false, size: 1000, - _source: [ - TIMESTAMP, - TRACE_ID, - TRANSACTION_ID, - PARENT_ID, - SERVICE_NAME, - ERROR_ID, - ERROR_LOG_MESSAGE, - ERROR_EXCEPTION, - ERROR_GROUP_ID, - ], query: { bool: { filter: [{ term: { [TRACE_ID]: traceId } }, ...rangeQuery(start, end)], must_not: { terms: { [ERROR_LOG_LEVEL]: excludedLogLevels } }, }, }, + fields: [...requiredFields, PARENT_ID, TRANSACTION_ID, SPAN_ID], + _source: [ERROR_LOG_MESSAGE, ERROR_EXCEPTION], }, }); @@ -133,8 +135,24 @@ export async function getTraceItems({ const traceDocsTotal = traceResponse.total; const exceedsMax = traceDocsTotal > maxTraceItems; - const traceDocs = traceResponse.hits.map((hit) => hit._source); - const errorDocs = errorResponse.hits.hits.map((hit) => hit._source); + + const traceDocs = traceResponse.hits.map(({ hit }) => hit); + + const errorDocs = errorResponse.hits.hits.map((hit) => { + const errorSource = 'error' in hit._source ? hit._source : undefined; + + const event = unflattenKnownApmEventFields(hit.fields, requiredFields); + + const waterfallErrorEvent: WaterfallError = { + ...event, + error: { + ...event.error, + log: errorSource?.error.log, + exception: errorSource?.error.exception, + }, + }; + return waterfallErrorEvent; + }); return { exceedsMax, @@ -220,41 +238,54 @@ async function getTraceDocsPerPage({ start: number; end: number; searchAfter?: SortResults; -}) { +}): Promise<{ + hits: Array<{ hit: WaterfallTransaction | WaterfallSpan; sort: SortResults | undefined }>; + total: number; +}> { const size = Math.min(maxTraceItems, MAX_ITEMS_PER_PAGE); + const requiredFields = asMutableArray([ + AGENT_NAME, + TIMESTAMP, + TRACE_ID, + SERVICE_NAME, + PROCESSOR_EVENT, + ] as const); + + const requiredTxFields = asMutableArray([ + TRANSACTION_ID, + TRANSACTION_DURATION, + TRANSACTION_NAME, + TRANSACTION_TYPE, + ] as const); + + const requiredSpanFields = asMutableArray([ + SPAN_ID, + SPAN_TYPE, + SPAN_NAME, + SPAN_DURATION, + ] as const); + + const optionalFields = asMutableArray([ + PARENT_ID, + SERVICE_ENVIRONMENT, + EVENT_OUTCOME, + TRANSACTION_RESULT, + FAAS_COLDSTART, + SPAN_SUBTYPE, + SPAN_ACTION, + SPAN_COMPOSITE_COUNT, + SPAN_COMPOSITE_COMPRESSION_STRATEGY, + SPAN_COMPOSITE_SUM, + SPAN_SYNC, + CHILD_ID, + ] as const); + const body = { track_total_hits: true, size, search_after: searchAfter, - _source: [ - TIMESTAMP, - TRACE_ID, - PARENT_ID, - SERVICE_NAME, - SERVICE_ENVIRONMENT, - AGENT_NAME, - EVENT_OUTCOME, - PROCESSOR_EVENT, - TRANSACTION_DURATION, - TRANSACTION_ID, - TRANSACTION_NAME, - TRANSACTION_TYPE, - TRANSACTION_RESULT, - FAAS_COLDSTART, - SPAN_ID, - SPAN_TYPE, - SPAN_SUBTYPE, - SPAN_ACTION, - SPAN_NAME, - SPAN_DURATION, - SPAN_LINKS, - SPAN_COMPOSITE_COUNT, - SPAN_COMPOSITE_COMPRESSION_STRATEGY, - SPAN_COMPOSITE_SUM, - SPAN_SYNC, - CHILD_ID, - ], + _source: [SPAN_LINKS], query: { bool: { filter: [ @@ -266,6 +297,7 @@ async function getTraceDocsPerPage({ }, }, }, + fields: [...requiredFields, ...requiredTxFields, ...requiredSpanFields, ...optionalFields], sort: [ { _score: 'asc' }, { @@ -291,7 +323,51 @@ async function getTraceDocsPerPage({ }); return { - hits: res.hits.hits, + hits: res.hits.hits.map((hit) => { + const sort = hit.sort; + const spanLinksSource = 'span' in hit._source ? hit._source.span?.links : undefined; + + if (hit.fields['processor.event']?.[0] === 'span') { + const spanEvent = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredSpanFields, + ]); + + const spanWaterfallEvent: WaterfallSpan = { + ...omit(spanEvent, 'child'), + processor: { + event: 'span', + }, + span: { + ...spanEvent.span, + composite: spanEvent.span.composite + ? (spanEvent.span.composite as Required['composite']) + : undefined, + links: spanLinksSource, + }, + ...(spanEvent.child ? { child: spanEvent.child as WaterfallSpan['child'] } : {}), + }; + + return { sort, hit: spanWaterfallEvent }; + } + + const txEvent = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredTxFields, + ]); + const txWaterfallEvent: WaterfallTransaction = { + ...txEvent, + processor: { + event: 'transaction', + }, + span: { + ...txEvent.span, + links: spanLinksSource, + }, + }; + + return { hit: txWaterfallEvent, sort }; + }), total: res.hits.total.value, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts index 033b666d0371c..d35ccbea81714 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts @@ -92,7 +92,7 @@ export async function getTraceSamplesByQuery({ filter_path: 'hits.sequences.events._source.trace.id', }) ).hits?.sequences?.flatMap((sequence) => - sequence.events.map((event) => (event._source as { trace: { id: string } }).trace.id) + sequence.events.map((event) => (event.fields as { 'trace.id': [string] })['trace.id'][0]) ) ?? []; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts index 328201ec9d143..dd9cd780f826c 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts @@ -12,7 +12,10 @@ import { getSearchTransactionsEvents } from '../../lib/helpers/transactions'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { environmentRt, kueryRt, probabilityRt, rangeRt } from '../default_api_types'; import { getTransaction } from '../transactions/get_transaction'; -import { getRootTransactionByTraceId } from '../transactions/get_transaction_by_trace'; +import { + type TransactionDetailRedirectInfo, + getRootTransactionByTraceId, +} from '../transactions/get_transaction_by_trace'; import { getTopTracesPrimaryStats, TopTracesPrimaryStatsResponse, @@ -128,7 +131,7 @@ const rootTransactionByTraceIdRoute = createApmServerRoute({ handler: async ( resources ): Promise<{ - transaction: Transaction; + transaction: TransactionDetailRedirectInfo | undefined; }> => { const { params: { @@ -155,7 +158,7 @@ const transactionByIdRoute = createApmServerRoute({ handler: async ( resources ): Promise<{ - transaction: Transaction; + transaction: Transaction | undefined; }> => { const { params: { @@ -191,7 +194,7 @@ const transactionByNameRoute = createApmServerRoute({ handler: async ( resources ): Promise<{ - transaction: Transaction; + transaction: TransactionDetailRedirectInfo | undefined; }> => { const { params: { @@ -295,7 +298,7 @@ const transactionFromTraceByIdRoute = createApmServerRoute({ query: rangeRt, }), options: { tags: ['access:apm'] }, - handler: async (resources): Promise => { + handler: async (resources): Promise => { const { params } = resources; const { path: { transactionId, traceId }, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts index 4e1b7020a98a6..0152d469d8d71 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts @@ -7,6 +7,10 @@ import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; +import { merge, omit } from 'lodash'; +import { maybe } from '../../../../common/utils/maybe'; import { SPAN_ID, TRACE_ID } from '../../../../common/es_fields/apm'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; @@ -60,5 +64,16 @@ export async function getSpan({ : undefined, ]); - return { span: spanResp.hits.hits[0]?._source, parentTransaction }; + const event = unflattenKnownApmEventFields( + maybe(spanResp.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); + + return { + span: event + ? merge({}, omit(event, 'span.links'), { + processor: { event: 'span' as const, name: 'transaction' as const }, + }) + : undefined, + parentTransaction, + }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts index 8854f3075e59b..ee379bbdd6f78 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts @@ -6,7 +6,26 @@ */ import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; -import { TRACE_ID, TRANSACTION_ID } from '../../../../common/es_fields/apm'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import type { Transaction } from '@kbn/apm-types'; +import { maybe } from '../../../../common/utils/maybe'; +import { + AGENT_NAME, + PROCESSOR_EVENT, + SERVICE_NAME, + TIMESTAMP, + TRACE_ID, + TRANSACTION_DURATION, + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_SAMPLED, + TRANSACTION_TYPE, + AT_TIMESTAMP, + PROCESSOR_NAME, + AGENT_VERSION, + SPAN_LINKS, + TRANSACTION_AGENT_MARKS, +} from '../../../../common/es_fields/apm'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { ApmDocumentType } from '../../../../common/document_type'; @@ -24,7 +43,22 @@ export async function getTransaction({ apmEventClient: APMEventClient; start: number; end: number; -}) { +}): Promise { + const requiredFields = asMutableArray([ + TRACE_ID, + AGENT_NAME, + AGENT_VERSION, + PROCESSOR_EVENT, + AT_TIMESTAMP, + TIMESTAMP, + SERVICE_NAME, + TRANSACTION_ID, + TRANSACTION_DURATION, + TRANSACTION_NAME, + TRANSACTION_SAMPLED, + TRANSACTION_TYPE, + ] as const); + const resp = await apmEventClient.search('get_transaction', { apm: { sources: [ @@ -47,8 +81,37 @@ export async function getTransaction({ ]), }, }, + fields: [...requiredFields, PROCESSOR_NAME], + _source: [SPAN_LINKS, TRANSACTION_AGENT_MARKS], }, }); - return resp.hits.hits[0]?._source; + const hit = maybe(resp.hits.hits[0]); + + if (!hit) { + return undefined; + } + + const event = unflattenKnownApmEventFields(hit.fields, requiredFields); + + const source = + 'span' in hit._source && 'transaction' in hit._source + ? (hit._source as { + transaction: Pick['transaction'], 'marks'>; + span?: Pick['span'], 'links'>; + }) + : undefined; + + return { + ...event, + transaction: { + ...event.transaction, + marks: source?.transaction.marks, + }, + processor: { + name: 'transaction', + event: 'transaction', + }, + span: source?.span, + }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts index 75b4655d70117..160e3f736580a 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts @@ -6,11 +6,22 @@ */ import { rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; import { ApmDocumentType } from '../../../../common/document_type'; -import { SERVICE_NAME, TRANSACTION_NAME } from '../../../../common/es_fields/apm'; +import { + AT_TIMESTAMP, + SERVICE_NAME, + TRACE_ID, + TRANSACTION_DURATION, + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_TYPE, +} from '../../../../common/es_fields/apm'; import { RollupInterval } from '../../../../common/rollup'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; +import { TransactionDetailRedirectInfo } from '../get_transaction_by_trace'; export async function getTransactionByName({ transactionName, @@ -24,7 +35,17 @@ export async function getTransactionByName({ apmEventClient: APMEventClient; start: number; end: number; -}) { +}): Promise { + const requiredFields = asMutableArray([ + AT_TIMESTAMP, + TRACE_ID, + TRANSACTION_ID, + TRANSACTION_TYPE, + TRANSACTION_NAME, + TRANSACTION_DURATION, + SERVICE_NAME, + ] as const); + const resp = await apmEventClient.search('get_transaction', { apm: { sources: [ @@ -47,8 +68,9 @@ export async function getTransactionByName({ ]), }, }, + fields: requiredFields, }, }); - return resp.hits.hits[0]?._source; + return unflattenKnownApmEventFields(maybe(resp.hits.hits[0])?.fields, requiredFields); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts index d27be0489f8da..803ae19a2228e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts @@ -7,9 +7,40 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { rangeQuery } from '@kbn/observability-plugin/server'; -import { TRACE_ID, PARENT_ID } from '../../../../common/es_fields/apm'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; +import { + TRACE_ID, + PARENT_ID, + AT_TIMESTAMP, + TRANSACTION_DURATION, + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_TYPE, + SERVICE_NAME, +} from '../../../../common/es_fields/apm'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; +export interface TransactionDetailRedirectInfo { + [AT_TIMESTAMP]: string; + trace: { + id: string; + }; + transaction: { + id: string; + type: string; + name: string; + + duration: { + us: number; + }; + }; + service: { + name: string; + }; +} + export async function getRootTransactionByTraceId({ traceId, apmEventClient, @@ -20,7 +51,19 @@ export async function getRootTransactionByTraceId({ apmEventClient: APMEventClient; start: number; end: number; -}) { +}): Promise<{ + transaction: TransactionDetailRedirectInfo | undefined; +}> { + const requiredFields = asMutableArray([ + TRACE_ID, + TRANSACTION_ID, + TRANSACTION_NAME, + AT_TIMESTAMP, + TRANSACTION_TYPE, + TRANSACTION_DURATION, + SERVICE_NAME, + ] as const); + const params = { apm: { events: [ProcessorEvent.transaction as const], @@ -45,11 +88,15 @@ export async function getRootTransactionByTraceId({ filter: [{ term: { [TRACE_ID]: traceId } }, ...rangeQuery(start, end)], }, }, + fields: requiredFields, }, }; const resp = await apmEventClient.search('get_root_transaction_by_trace_id', params); + + const event = unflattenKnownApmEventFields(maybe(resp.hits.hits[0])?.fields, requiredFields); + return { - transaction: resp.hits.hits[0]?._source, + transaction: event, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts index 191250d3781ee..18dad19635333 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts @@ -7,7 +7,10 @@ import { Sort, QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { + AT_TIMESTAMP, SERVICE_NAME, TRACE_ID, TRANSACTION_ID, @@ -77,6 +80,8 @@ export async function getTraceSamples({ }); } + const requiredFields = asMutableArray([TRANSACTION_ID, TRACE_ID, AT_TIMESTAMP] as const); + const response = await apmEventClient.search('get_trace_samples_hits', { apm: { events: [ProcessorEvent.transaction], @@ -94,6 +99,7 @@ export async function getTraceSamples({ }, }, size: TRACE_SAMPLES_SIZE, + fields: requiredFields, sort: [ { _score: { @@ -101,7 +107,7 @@ export async function getTraceSamples({ }, }, { - '@timestamp': { + [AT_TIMESTAMP]: { order: 'desc', }, }, @@ -109,12 +115,15 @@ export async function getTraceSamples({ }, }); - const traceSamples = response.hits.hits.map((hit) => ({ - score: hit._score, - timestamp: hit._source['@timestamp'], - transactionId: hit._source.transaction.id, - traceId: hit._source.trace.id, - })); + const traceSamples = response.hits.hits.map((hit) => { + const event = unflattenKnownApmEventFields(hit.fields, requiredFields); + return { + score: hit._score, + timestamp: event[AT_TIMESTAMP], + transactionId: event.transaction.id, + traceId: event.trace.id, + }; + }); return { traceSamples }; }); diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts index 2fac072a8cdb5..71ceac4002919 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts @@ -15,3 +15,4 @@ export { } from './lib/helpers'; export { withApmSpan } from './utils/with_apm_span'; +export { unflattenKnownApmEventFields } from './utils/unflatten_known_fields'; diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts new file mode 100644 index 0000000000000..6064fa028c144 --- /dev/null +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// unflatten_known_fields.test.ts +import { unflattenKnownApmEventFields } from './unflatten_known_fields'; + +describe('unflattenKnownApmEventFields', () => { + it('should return an empty object when input is empty', () => { + const input = {}; + const expectedOutput = {}; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should correctly unflatten a simple flat input', () => { + const input = { + '@timestamp': '2024-10-10T10:10:10.000Z', + }; + const expectedOutput = { + '@timestamp': '2024-10-10T10:10:10.000Z', + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should correctly unflatten nested fields', () => { + const input = { + 'service.name': 'node-svc', + }; + const expectedOutput = { + service: { + name: 'node-svc', + }, + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should correctly unflatten nested fields with mandatory field', () => { + const input = { + 'service.name': 'node-svc', + 'service.environment': 'production', + }; + + const requiredFields: ['service.name'] = ['service.name']; + + const expectedOutput = { + service: { + name: 'node-svc', + environment: 'production', + }, + }; + expect(unflattenKnownApmEventFields(input, requiredFields)).toEqual(expectedOutput); + }); +}); diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts new file mode 100644 index 0000000000000..4d2e3a394e342 --- /dev/null +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DedotObject } from '@kbn/utility-types'; +import * as APM_EVENT_FIELDS_MAP from '@kbn/apm-types/es_fields'; +import type { ValuesType } from 'utility-types'; +import { unflattenObject } from '@kbn/observability-utils/object/unflatten_object'; +import { mergePlainObjects } from '@kbn/observability-utils/object/merge_plain_objects'; +import { castArray, isArray } from 'lodash'; +import { AgentName } from '@kbn/elastic-agent-utils'; +import { EventOutcome } from '@kbn/apm-types/src/es_schemas/raw/fields'; +import { ProcessorEvent } from '@kbn/observability-plugin/common'; + +const { + CLOUD, + AGENT, + SERVICE, + ERROR_EXCEPTION, + SPAN_LINKS, + HOST, + KUBERNETES, + CONTAINER, + TIER, + INDEX, + DATA_STEAM_TYPE, + VALUE_OTEL_JVM_PROCESS_MEMORY_HEAP, + VALUE_OTEL_JVM_PROCESS_MEMORY_NON_HEAP, + SPAN_LINKS_SPAN_ID, + SPAN_LINKS_TRACE_ID, + ...CONCRETE_FIELDS +} = APM_EVENT_FIELDS_MAP; + +const ALL_FIELDS = Object.values(CONCRETE_FIELDS); + +const KNOWN_MULTI_VALUED_FIELDS = [ + APM_EVENT_FIELDS_MAP.CHILD_ID, + APM_EVENT_FIELDS_MAP.PROCESS_ARGS, +] as const; + +type KnownField = ValuesType; + +type KnownSingleValuedField = Exclude; +type KnownMultiValuedField = ValuesType; + +const KNOWN_SINGLE_VALUED_FIELDS = ALL_FIELDS.filter( + (field): field is KnownSingleValuedField => !KNOWN_MULTI_VALUED_FIELDS.includes(field as any) +); + +interface TypeOverrideMap { + [APM_EVENT_FIELDS_MAP.SPAN_DURATION]: number; + [APM_EVENT_FIELDS_MAP.AGENT_NAME]: AgentName; + [APM_EVENT_FIELDS_MAP.EVENT_OUTCOME]: EventOutcome; + [APM_EVENT_FIELDS_MAP.FAAS_COLDSTART]: true; + [APM_EVENT_FIELDS_MAP.TRANSACTION_DURATION]: number; + [APM_EVENT_FIELDS_MAP.TIMESTAMP]: number; + [APM_EVENT_FIELDS_MAP.PROCESSOR_EVENT]: ProcessorEvent; + [APM_EVENT_FIELDS_MAP.SPAN_COMPOSITE_COUNT]: number; + [APM_EVENT_FIELDS_MAP.SPAN_COMPOSITE_SUM]: number; + [APM_EVENT_FIELDS_MAP.SPAN_SYNC]: boolean; + [APM_EVENT_FIELDS_MAP.TRANSACTION_SAMPLED]: boolean; + [APM_EVENT_FIELDS_MAP.PROCESSOR_NAME]: 'transaction' | 'metric' | 'error'; + [APM_EVENT_FIELDS_MAP.HTTP_RESPONSE_STATUS_CODE]: number; + [APM_EVENT_FIELDS_MAP.PROCESS_PID]: number; + [APM_EVENT_FIELDS_MAP.OBSERVER_VERSION_MAJOR]: number; + [APM_EVENT_FIELDS_MAP.ERROR_EXC_HANDLED]: boolean; +} + +type MaybeMultiValue = T extends KnownMultiValuedField ? U[] : U; + +type TypeOfKnownField = MaybeMultiValue< + T, + T extends keyof TypeOverrideMap ? TypeOverrideMap[T] : string +>; + +type MapToSingleOrMultiValue> = { + [TKey in keyof T]: TKey extends KnownField + ? T[TKey] extends undefined + ? TypeOfKnownField | undefined + : TypeOfKnownField + : unknown; +}; + +type UnflattenedKnownFields> = DedotObject< + MapToSingleOrMultiValue +>; + +export type FlattenedApmEvent = Record; + +export type UnflattenedApmEvent = UnflattenedKnownFields; + +export function unflattenKnownApmEventFields | undefined = undefined>( + fields: T +): T extends Record ? UnflattenedKnownFields : undefined; + +export function unflattenKnownApmEventFields< + T extends Record | undefined, + U extends Array> +>( + fields: T, + required: U +): T extends Record + ? UnflattenedKnownFields & + (U extends any[] + ? UnflattenedKnownFields<{ + [TKey in ValuesType]: keyof T extends TKey ? T[TKey] : unknown[]; + }> + : {}) + : undefined; + +export function unflattenKnownApmEventFields( + hitFields?: Record, + requiredFields?: string[] +) { + if (!hitFields) { + return undefined; + } + const missingRequiredFields = + requiredFields?.filter((key) => { + const value = hitFields?.[key]; + return value === null || value === undefined || (isArray(value) && value.length === 0); + }) ?? []; + + if (missingRequiredFields.length > 0) { + throw new Error(`Missing required fields ${missingRequiredFields.join(', ')} in event`); + } + + const copy: Record = mapToSingleOrMultiValue({ + ...hitFields, + }); + + const [knownFields, unknownFields] = Object.entries(copy).reduce( + (prev, [key, value]) => { + if (ALL_FIELDS.includes(key as KnownField)) { + prev[0][key as KnownField] = value; + } else { + prev[1][key] = value; + } + return prev; + }, + [{} as Record, {} as Record] + ); + + const unflattened = mergePlainObjects( + {}, + unflattenObject(unknownFields), + unflattenObject(knownFields) + ); + + return unflattened; +} + +export function mapToSingleOrMultiValue>( + fields: T +): MapToSingleOrMultiValue { + KNOWN_SINGLE_VALUED_FIELDS.forEach((field) => { + const value = fields[field]; + if (value !== null && value !== undefined) { + fields[field as keyof T] = castArray(value)[0]; + } + }); + + return fields; +} From c02dd196c0685706e28fcbe9e8b64c94d30835ea Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:50:05 +0000 Subject: [PATCH 02/27] [CI] Auto-commit changed files from 'node scripts/notice' --- .../packages/observability/observability_utils/tsconfig.json | 1 + .../observability_solution/apm_data_access/tsconfig.json | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/packages/observability/observability_utils/tsconfig.json b/x-pack/packages/observability/observability_utils/tsconfig.json index 2ed47d10cfad9..b3f1a4a21c4e7 100644 --- a/x-pack/packages/observability/observability_utils/tsconfig.json +++ b/x-pack/packages/observability/observability_utils/tsconfig.json @@ -21,5 +21,6 @@ "@kbn/es-types", "@kbn/apm-utils", "@kbn/es-query", + "@kbn/safer-lodash-set", ] } diff --git a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json index ea3ebf77b25be..aeeb73bee2857 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json +++ b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json @@ -20,6 +20,8 @@ "@kbn/apm-utils", "@kbn/core-http-server", "@kbn/security-plugin-types-server", - "@kbn/observability-utils" + "@kbn/observability-utils", + "@kbn/utility-types", + "@kbn/elastic-agent-utils" ] } From 880e27069b34a217714113e9ee8b80182c14f50e Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 7 Oct 2024 15:28:45 +0200 Subject: [PATCH 03/27] Generic otel agent names; CR fixes --- packages/kbn-apm-types/src/es_fields/apm.ts | 2 +- .../src/agent_names.ts | 30 ++----------------- .../get_error_sample_details.ts | 4 +-- .../services/get_service_metadata_icons.ts | 1 - .../server/routes/traces/get_trace_items.ts | 6 ++-- .../transactions/get_transaction/index.ts | 4 +-- .../server/utils/unflatten_known_fields.ts | 2 +- 7 files changed, 12 insertions(+), 37 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index 87bf76bb58ee6..f83e5e7a4c123 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export const TIMESTAMP = 'timestamp.us'; +export const TIMESTAMP_US = 'timestamp.us'; export const AT_TIMESTAMP = '@timestamp'; export const AGENT = 'agent'; export const AGENT_NAME = 'agent.name'; diff --git a/packages/kbn-elastic-agent-utils/src/agent_names.ts b/packages/kbn-elastic-agent-utils/src/agent_names.ts index 0405da9cf2193..44b29ef5fd558 100644 --- a/packages/kbn-elastic-agent-utils/src/agent_names.ts +++ b/packages/kbn-elastic-agent-utils/src/agent_names.ts @@ -38,34 +38,10 @@ export const ELASTIC_AGENT_NAMES: ElasticAgentName[] = [ ]; export type OpenTelemetryAgentName = - | 'otlp' | 'opentelemetry' - | 'opentelemetry/cpp' - | 'opentelemetry/dotnet' - | 'opentelemetry/erlang' - | 'opentelemetry/go' - | 'opentelemetry/java' - | 'opentelemetry/nodejs' - | 'opentelemetry/php' - | 'opentelemetry/python' - | 'opentelemetry/ruby' - | 'opentelemetry/rust' - | 'opentelemetry/swift' - | 'opentelemetry/android' - | 'opentelemetry/webjs' - | 'otlp/cpp' - | 'otlp/dotnet' - | 'otlp/erlang' - | 'otlp/go' - | 'otlp/java' - | 'otlp/nodejs' - | 'otlp/php' - | 'otlp/python' - | 'otlp/ruby' - | 'otlp/rust' - | 'otlp/swift' - | 'otlp/android' - | 'otlp/webjs'; + | 'otlp' + | `opentelemetry/${string}` + | `otlp/${string}`; export const OPEN_TELEMETRY_AGENT_NAMES: OpenTelemetryAgentName[] = [ 'otlp', 'opentelemetry', diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts index dfad2a71e1f0d..409466aafd9ba 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts @@ -23,7 +23,7 @@ import { PROCESSOR_EVENT, PROCESSOR_NAME, SERVICE_NAME, - TIMESTAMP, + TIMESTAMP_US, TRACE_ID, TRANSACTION_ID, ERROR_STACK_TRACE, @@ -71,7 +71,7 @@ export async function getErrorSampleDetails({ AGENT_VERSION, PROCESSOR_EVENT, TRACE_ID, - TIMESTAMP, + TIMESTAMP_US, AT_TIMESTAMP, SERVICE_NAME, ERROR_ID, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts index 9985d1259ffa5..a5f7abaaf16da 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts @@ -80,7 +80,6 @@ export async function getServiceMetadataIcons({ body: { track_total_hits: 1, size: 1, - _source: false, query: { bool: { filter, should } }, fields, }, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts index 84fb558fc1480..aad1dba9380e6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts @@ -39,7 +39,7 @@ import { SPAN_SUBTYPE, SPAN_SYNC, SPAN_TYPE, - TIMESTAMP, + TIMESTAMP_US, TRACE_ID, TRANSACTION_DURATION, TRANSACTION_ID, @@ -87,7 +87,7 @@ export async function getTraceItems({ const excludedLogLevels = ['debug', 'info', 'warning']; const requiredFields = asMutableArray([ - TIMESTAMP, + TIMESTAMP_US, TRACE_ID, SERVICE_NAME, ERROR_ID, @@ -246,7 +246,7 @@ async function getTraceDocsPerPage({ const requiredFields = asMutableArray([ AGENT_NAME, - TIMESTAMP, + TIMESTAMP_US, TRACE_ID, SERVICE_NAME, PROCESSOR_EVENT, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts index ee379bbdd6f78..7ac8415ee4c7e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts @@ -13,7 +13,7 @@ import { AGENT_NAME, PROCESSOR_EVENT, SERVICE_NAME, - TIMESTAMP, + TIMESTAMP_US, TRACE_ID, TRANSACTION_DURATION, TRANSACTION_ID, @@ -50,7 +50,7 @@ export async function getTransaction({ AGENT_VERSION, PROCESSOR_EVENT, AT_TIMESTAMP, - TIMESTAMP, + TIMESTAMP_US, SERVICE_NAME, TRANSACTION_ID, TRANSACTION_DURATION, diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts index 4d2e3a394e342..eb56205b4a6be 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts @@ -56,7 +56,7 @@ interface TypeOverrideMap { [APM_EVENT_FIELDS_MAP.EVENT_OUTCOME]: EventOutcome; [APM_EVENT_FIELDS_MAP.FAAS_COLDSTART]: true; [APM_EVENT_FIELDS_MAP.TRANSACTION_DURATION]: number; - [APM_EVENT_FIELDS_MAP.TIMESTAMP]: number; + [APM_EVENT_FIELDS_MAP.TIMESTAMP_US]: number; [APM_EVENT_FIELDS_MAP.PROCESSOR_EVENT]: ProcessorEvent; [APM_EVENT_FIELDS_MAP.SPAN_COMPOSITE_COUNT]: number; [APM_EVENT_FIELDS_MAP.SPAN_COMPOSITE_SUM]: number; From 5756999ffe4f545f72df71daee37adc25b286907 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 7 Oct 2024 15:47:13 +0200 Subject: [PATCH 04/27] Change missing error queries for mobile --- .../get_crash_group_main_statistics.ts | 64 +++++++++++++------ .../get_mobile_error_group_main_statistics.ts | 64 +++++++++++++------ 2 files changed, 88 insertions(+), 40 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts index b0d3eabe0bab2..67660130d9728 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts @@ -8,6 +8,9 @@ import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types'; import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { castArray } from 'lodash'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../../common/utils/as_mutable_array'; import { ERROR_CULPRIT, ERROR_TYPE, @@ -19,6 +22,7 @@ import { SERVICE_NAME, TRANSACTION_NAME, TRANSACTION_TYPE, + AT_TIMESTAMP, } from '../../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../../common/utils/environment_query'; import { getErrorName } from '../../../../lib/helpers/get_error_name'; @@ -68,6 +72,16 @@ export async function getMobileCrashGroupMainStatistics({ ? { [maxTimestampAggKey]: sortDirection } : { _count: sortDirection }; + const requiredFields = asMutableArray([ERROR_GROUP_ID, AT_TIMESTAMP] as const); + + const optionalFields = asMutableArray([ + ERROR_CULPRIT, + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const response = await apmEventClient.search('get_crash_group_main_statistics', { apm: { events: [ProcessorEvent.error], @@ -99,22 +113,15 @@ export async function getMobileCrashGroupMainStatistics({ sample: { top_hits: { size: 1, - _source: [ - ERROR_LOG_MESSAGE, - ERROR_EXC_MESSAGE, - ERROR_EXC_HANDLED, - ERROR_EXC_TYPE, - ERROR_CULPRIT, - ERROR_GROUP_ID, - '@timestamp', - ], + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE], sort: { - '@timestamp': 'desc', + [AT_TIMESTAMP]: 'desc', }, }, }, ...(sortByLatestOccurrence - ? { [maxTimestampAggKey]: { max: { field: '@timestamp' } } } + ? { [maxTimestampAggKey]: { max: { field: AT_TIMESTAMP } } } : {}), }, }, @@ -123,14 +130,31 @@ export async function getMobileCrashGroupMainStatistics({ }); return ( - response.aggregations?.crash_groups.buckets.map((bucket) => ({ - groupId: bucket.key as string, - name: getErrorName(bucket.sample.hits.hits[0]._source), - lastSeen: new Date(bucket.sample.hits.hits[0]?._source['@timestamp']).getTime(), - occurrences: bucket.doc_count, - culprit: bucket.sample.hits.hits[0]?._source.error.culprit, - handled: bucket.sample.hits.hits[0]?._source.error.exception?.[0].handled, - type: bucket.sample.hits.hits[0]?._source.error.exception?.[0].type, - })) ?? [] + response.aggregations?.crash_groups.buckets.map((bucket) => { + const errorSource = + 'error' in bucket.sample.hits.hits[0]._source + ? bucket.sample.hits.hits[0]._source + : undefined; + + const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields); + + const mergedEvent = { + ...event, + error: { + ...(event.error ?? {}), + exception: castArray(errorSource?.error.exception ?? event?.error.exception), + }, + }; + + return { + groupId: event.error?.grouping_key, + name: getErrorName(mergedEvent), + lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(), + occurrences: bucket.doc_count, + culprit: mergedEvent.error.culprit, + handled: mergedEvent.error.exception[0].handled, + type: mergedEvent.error.exception[0].type, + }; + }) ?? [] ); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts index f259e17d6154c..9c7b46706db61 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts @@ -8,7 +8,11 @@ import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types'; import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { castArray } from 'lodash'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { + AT_TIMESTAMP, ERROR_CULPRIT, ERROR_EXC_HANDLED, ERROR_EXC_MESSAGE, @@ -67,6 +71,16 @@ export async function getMobileErrorGroupMainStatistics({ ? { [maxTimestampAggKey]: sortDirection } : { _count: sortDirection }; + const requiredFields = asMutableArray([ERROR_GROUP_ID, AT_TIMESTAMP] as const); + + const optionalFields = asMutableArray([ + ERROR_CULPRIT, + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const response = await apmEventClient.search('get_error_group_main_statistics', { apm: { events: [ProcessorEvent.error], @@ -100,22 +114,15 @@ export async function getMobileErrorGroupMainStatistics({ sample: { top_hits: { size: 1, - _source: [ - ERROR_LOG_MESSAGE, - ERROR_EXC_MESSAGE, - ERROR_EXC_HANDLED, - ERROR_EXC_TYPE, - ERROR_CULPRIT, - ERROR_GROUP_ID, - '@timestamp', - ], + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE], sort: { - '@timestamp': 'desc', + [AT_TIMESTAMP]: 'desc', }, }, }, ...(sortByLatestOccurrence - ? { [maxTimestampAggKey]: { max: { field: '@timestamp' } } } + ? { [maxTimestampAggKey]: { max: { field: AT_TIMESTAMP } } } : {}), }, }, @@ -124,14 +131,31 @@ export async function getMobileErrorGroupMainStatistics({ }); return ( - response.aggregations?.error_groups.buckets.map((bucket) => ({ - groupId: bucket.key as string, - name: getErrorName(bucket.sample.hits.hits[0]._source), - lastSeen: new Date(bucket.sample.hits.hits[0]?._source['@timestamp']).getTime(), - occurrences: bucket.doc_count, - culprit: bucket.sample.hits.hits[0]?._source.error.culprit, - handled: bucket.sample.hits.hits[0]?._source.error.exception?.[0].handled, - type: bucket.sample.hits.hits[0]?._source.error.exception?.[0].type, - })) ?? [] + response.aggregations?.error_groups.buckets.map((bucket) => { + const errorSource = + 'error' in bucket.sample.hits.hits[0]._source + ? bucket.sample.hits.hits[0]._source + : undefined; + + const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields); + + const mergedEvent = { + ...event, + error: { + ...(event.error ?? {}), + exception: castArray(errorSource?.error.exception ?? event?.error.exception), + }, + }; + + return { + groupId: event.error?.grouping_key, + name: getErrorName(mergedEvent), + lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(), + occurrences: bucket.doc_count, + culprit: mergedEvent.error.culprit, + handled: mergedEvent.error.exception[0].handled, + type: mergedEvent.error.exception[0].type, + }; + }) ?? [] ); } From 2adf0d1df38282d500daa315efd377c183c186e4 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 7 Oct 2024 18:48:51 +0200 Subject: [PATCH 05/27] Revert AgentName change; CR fixes --- packages/kbn-apm-types/src/es_fields/apm.ts | 3 -- .../src/es_schemas/raw/fields/kubernetes.ts | 4 ++- .../src/agent_names.ts | 32 ++++++++++++++--- .../object/unflatten_object.test.ts | 36 +++++++++++++++++++ .../get_error_group_main_statistics.ts | 6 +++- .../get_crash_group_main_statistics.ts | 6 +++- .../get_mobile_error_group_main_statistics.ts | 6 +++- ...get_service_instance_container_metadata.ts | 10 +++--- .../server/routes/traces/get_trace_items.ts | 31 ++++++++++++---- 9 files changed, 113 insertions(+), 21 deletions(-) create mode 100644 x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index f83e5e7a4c123..56e2377f2bb7d 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -161,14 +161,11 @@ export const CONTAINER_IMAGE = 'container.image.name'; export const KUBERNETES = 'kubernetes'; export const KUBERNETES_POD_NAME = 'kubernetes.pod.name'; export const KUBERNETES_POD_UID = 'kubernetes.pod.uid'; -export const KUBERNETES_NAMESPACE = 'kubernetes.namespace'; export const KUBERNETES_NAMESPACE_NAME = 'kubernetes.namespace.name'; export const KUBERNETES_NODE_NAME = 'kubernetes.node.name'; export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name'; export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id'; -export const KUBERNETES_DEPLOYMENT = 'kubernetes.deployment'; export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name'; -export const KUBERNETES_REPLICASET = 'kubernetes.replicaset'; export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name'; export const FAAS_ID = 'faas.id'; diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts index 2a4f1465db9a5..bd8ebfd8b7272 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts @@ -9,7 +9,9 @@ export interface Kubernetes { pod?: { uid?: string | null; name?: string }; - namespace?: string; + namespace?: { + name?: string; + }; replicaset?: { name?: string; }; diff --git a/packages/kbn-elastic-agent-utils/src/agent_names.ts b/packages/kbn-elastic-agent-utils/src/agent_names.ts index 44b29ef5fd558..946e72f4b94b0 100644 --- a/packages/kbn-elastic-agent-utils/src/agent_names.ts +++ b/packages/kbn-elastic-agent-utils/src/agent_names.ts @@ -36,12 +36,36 @@ export const ELASTIC_AGENT_NAMES: ElasticAgentName[] = [ 'rum-js', 'android/java', ]; - +export type OpenTelemetryGenericAgentName = `opentelemetry/${string}` | `otlp/${string}`; export type OpenTelemetryAgentName = - | 'opentelemetry' | 'otlp' - | `opentelemetry/${string}` - | `otlp/${string}`; + | 'opentelemetry' + | 'opentelemetry/cpp' + | 'opentelemetry/dotnet' + | 'opentelemetry/erlang' + | 'opentelemetry/go' + | 'opentelemetry/java' + | 'opentelemetry/nodejs' + | 'opentelemetry/php' + | 'opentelemetry/python' + | 'opentelemetry/ruby' + | 'opentelemetry/rust' + | 'opentelemetry/swift' + | 'opentelemetry/android' + | 'opentelemetry/webjs' + | 'otlp/cpp' + | 'otlp/dotnet' + | 'otlp/erlang' + | 'otlp/go' + | 'otlp/java' + | 'otlp/nodejs' + | 'otlp/php' + | 'otlp/python' + | 'otlp/ruby' + | 'otlp/rust' + | 'otlp/swift' + | 'otlp/android' + | 'otlp/webjs'; export const OPEN_TELEMETRY_AGENT_NAMES: OpenTelemetryAgentName[] = [ 'otlp', 'opentelemetry', diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts b/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts new file mode 100644 index 0000000000000..acf6009514bf2 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { unflattenObject } from './unflatten_object'; + +describe('unflattenObject', () => { + it('unflattens deeply nested objects', () => { + expect(unflattenObject({ 'first.second.third': 'third' })).toEqual({ + first: { + second: { + third: 'third', + }, + }, + }); + }); + + it('does not unflatten arrays', () => { + expect( + unflattenObject({ + simpleArray: ['0', '1', '2'], + complexArray: [{ one: 'one', two: 'two', three: 'three' }], + 'nested.array': [0, 1, 2], + }) + ).toEqual({ + simpleArray: ['0', '1', '2'], + complexArray: [{ one: 'one', two: 'two', three: 'three' }], + nested: { + array: [0, 1, 2], + }, + }); + }); +}); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts index 6323ad5212ab3..fbdb7fc2fa94d 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts @@ -180,7 +180,11 @@ export async function getErrorGroupMainStatistics({ ...event, error: { ...(event.error ?? {}), - exception: castArray(errorSource?.error.exception ?? event?.error.exception), + exception: castArray( + errorSource?.error.exception && errorSource?.error.exception?.length > 1 + ? errorSource?.error.exception + : event?.error.exception + ), }, }; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts index 67660130d9728..96322ed92ca72 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts @@ -142,7 +142,11 @@ export async function getMobileCrashGroupMainStatistics({ ...event, error: { ...(event.error ?? {}), - exception: castArray(errorSource?.error.exception ?? event?.error.exception), + exception: castArray( + errorSource?.error.exception && errorSource?.error.exception?.length > 1 + ? errorSource?.error.exception + : event?.error.exception + ), }, }; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts index 9c7b46706db61..2caf6589247a3 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts @@ -143,7 +143,11 @@ export async function getMobileErrorGroupMainStatistics({ ...event, error: { ...(event.error ?? {}), - exception: castArray(errorSource?.error.exception ?? event?.error.exception), + exception: castArray( + errorSource?.error.exception && errorSource?.error.exception?.length > 1 + ? errorSource?.error.exception + : event?.error.exception + ), }, }; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts index 1d82c26199586..a8efc356f241b 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts @@ -15,10 +15,10 @@ import { KUBERNETES_POD_NAME, KUBERNETES_POD_UID, KUBERNETES_CONTAINER_NAME, - KUBERNETES_NAMESPACE, KUBERNETES_REPLICASET_NAME, KUBERNETES_DEPLOYMENT_NAME, KUBERNETES_CONTAINER_ID, + KUBERNETES_NAMESPACE_NAME, } from '../../../common/es_fields/apm'; import { Kubernetes } from '../../../typings/es_schemas/raw/fields/kubernetes'; import { maybe } from '../../../common/utils/maybe'; @@ -45,7 +45,7 @@ export const getServiceInstanceContainerMetadata = async ({ { exists: { field: KUBERNETES } }, { exists: { field: CONTAINER_IMAGE } }, { exists: { field: KUBERNETES_CONTAINER_NAME } }, - { exists: { field: KUBERNETES_NAMESPACE } }, + { exists: { field: KUBERNETES_NAMESPACE_NAME } }, { exists: { field: KUBERNETES_POD_NAME } }, { exists: { field: KUBERNETES_POD_UID } }, { exists: { field: KUBERNETES_REPLICASET_NAME } }, @@ -58,7 +58,7 @@ export const getServiceInstanceContainerMetadata = async ({ KUBERNETES_DEPLOYMENT_NAME, KUBERNETES_CONTAINER_ID, KUBERNETES_CONTAINER_NAME, - KUBERNETES_NAMESPACE, + KUBERNETES_NAMESPACE_NAME, KUBERNETES_REPLICASET_NAME, KUBERNETES_DEPLOYMENT_NAME, ] as const); @@ -96,7 +96,9 @@ export const getServiceInstanceContainerMetadata = async ({ replicaset: { name: sample?.kubernetes?.replicaset?.name, }, - namespace: sample?.kubernetes?.namespace, + namespace: { + name: sample?.kubernetes?.namespace?.name, + }, container: { name: sample?.kubernetes?.container?.name, id: sample?.kubernetes?.container?.id, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts index aad1dba9380e6..86933873c81b1 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts @@ -10,14 +10,17 @@ import { SortResults } from '@elastic/elasticsearch/lib/api/types'; import { QueryDslQueryContainer, Sort } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { rangeQuery } from '@kbn/observability-plugin/server'; -import { last, omit } from 'lodash'; +import { castArray, last, omit } from 'lodash'; import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { APMConfig } from '../..'; import { AGENT_NAME, CHILD_ID, - ERROR_EXCEPTION, + ERROR_CULPRIT, + ERROR_EXC_HANDLED, + ERROR_EXC_MESSAGE, + ERROR_EXC_TYPE, ERROR_GROUP_ID, ERROR_ID, ERROR_LOG_LEVEL, @@ -95,6 +98,17 @@ export async function getTraceItems({ PROCESSOR_EVENT, ] as const); + const optionalFields = asMutableArray([ + PARENT_ID, + TRANSACTION_ID, + SPAN_ID, + ERROR_CULPRIT, + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const errorResponsePromise = apmEventClient.search('get_errors_docs', { apm: { sources: [ @@ -113,8 +127,8 @@ export async function getTraceItems({ must_not: { terms: { [ERROR_LOG_LEVEL]: excludedLogLevels } }, }, }, - fields: [...requiredFields, PARENT_ID, TRANSACTION_ID, SPAN_ID], - _source: [ERROR_LOG_MESSAGE, ERROR_EXCEPTION], + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE], }, }); @@ -146,11 +160,16 @@ export async function getTraceItems({ const waterfallErrorEvent: WaterfallError = { ...event, error: { - ...event.error, + ...(event.error ?? {}), + exception: castArray( + errorSource?.error.exception && errorSource?.error.exception?.length > 1 + ? errorSource?.error.exception + : event?.error.exception + ), log: errorSource?.error.log, - exception: errorSource?.error.exception, }, }; + return waterfallErrorEvent; }); From a466aa5e4fe275407a3e7c8b4a031ef9a0bc9d60 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 7 Oct 2024 19:28:31 +0200 Subject: [PATCH 06/27] Fix service metadata details --- .../get_connection_stats/get_destination_map.ts | 1 - .../register_transaction_duration_rule_type.ts | 1 + .../routes/services/get_service_metadata_details.ts | 9 +-------- .../server/utils/unflatten_known_fields.test.ts | 3 +-- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts b/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts index 4ab2e753832a4..cbcad6dea5baf 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts @@ -186,7 +186,6 @@ export const getDestinationMap = ({ }, size: destinationsBySpanId.size, fields: asMutableArray([SERVICE_NAME, SERVICE_ENVIRONMENT, AGENT_NAME, PARENT_ID] as const), - _source: false, }, }); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts index 96ddbe15c4287..dfc32ec9eb54e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts @@ -184,6 +184,7 @@ export function registerTransactionDurationRuleType({ body: { track_total_hits: false, size: 0, + _source: false as const, query: { bool: { filter: [ diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts index 1ba4c75a8fd3c..0319ae66039e5 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts @@ -11,23 +11,16 @@ import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; import { environmentQuery } from '../../../common/utils/environment_query'; import { - AGENT, - CONTAINER, - CLOUD, CLOUD_AVAILABILITY_ZONE, CLOUD_REGION, CLOUD_MACHINE_TYPE, CLOUD_SERVICE_NAME, CONTAINER_ID, - HOST, - KUBERNETES, - SERVICE, SERVICE_NAME, SERVICE_NODE_NAME, SERVICE_VERSION, FAAS_ID, FAAS_TRIGGER_TYPE, - LABEL_TELEMETRY_AUTO_VERSION, } from '../../../common/es_fields/apm'; import { ContainerType } from '../../../common/service_metadata'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; @@ -108,7 +101,6 @@ export async function getServiceMetadataDetails({ body: { track_total_hits: 1, size: 1, - _source: [SERVICE, AGENT, HOST, CONTAINER, KUBERNETES, CLOUD, LABEL_TELEMETRY_AUTO_VERSION], query: { bool: { filter, should } }, aggs: { serviceVersions: { @@ -162,6 +154,7 @@ export async function getServiceMetadataDetails({ }, totalNumberInstances: { cardinality: { field: SERVICE_NODE_NAME } }, }, + fields: ['*'], }, }; diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts index 6064fa028c144..0fca917f271ea 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts @@ -40,7 +40,7 @@ describe('unflattenKnownApmEventFields', () => { it('should correctly unflatten nested fields with mandatory field', () => { const input = { 'service.name': 'node-svc', - 'service.environment': 'production', + 'service.environment': undefined, }; const requiredFields: ['service.name'] = ['service.name']; @@ -48,7 +48,6 @@ describe('unflattenKnownApmEventFields', () => { const expectedOutput = { service: { name: 'node-svc', - environment: 'production', }, }; expect(unflattenKnownApmEventFields(input, requiredFields)).toEqual(expectedOutput); From e64e6285da61483c32e46784bc22075c2de4183b Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 7 Oct 2024 19:46:49 +0200 Subject: [PATCH 07/27] Improve test coverage --- .../utils/unflatten_known_fields.test.ts | 83 ++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts index 0fca917f271ea..3517092ee0aaf 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts @@ -25,15 +25,83 @@ describe('unflattenKnownApmEventFields', () => { expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); }); - it('should correctly unflatten nested fields', () => { + it('should override unknown fields', () => { const input = { 'service.name': 'node-svc', + 'service.name.text': 'node-svc', }; const expectedOutput = { service: { name: 'node-svc', }, }; + + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should correctly unflatten multiple nested fields', () => { + const input = { + 'service.name': 'node-svc', + 'service.version': '1.0.0', + 'service.environment': 'production', + 'agent.name': 'nodejs', + }; + const expectedOutput = { + service: { + name: 'node-svc', + version: '1.0.0', + environment: 'production', + }, + agent: { + name: 'nodejs', + }, + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should handle multiple values for multi-valued fields', () => { + const input = { + 'service.name': 'node-svc', + 'service.tags': ['foo', 'bar'], + }; + const expectedOutput = { + service: { + name: 'node-svc', + tags: ['foo', 'bar'], + }, + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should correctly unflatten with empty multi-valued fields', () => { + const input = { + 'service.name': 'node-svc', + 'service.tags': [], + }; + const expectedOutput = { + service: { + name: 'node-svc', + tags: [], + }, + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should retain unknown fields in the output', () => { + const input = { + 'service.name': 'node-svc', + 'unknown.field': 'foo', + unknonwField: 'bar', + }; + const expectedOutput = { + service: { + name: 'node-svc', + }, + unknown: { + field: 'foo', + }, + unknonwField: 'bar', + }; expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); }); @@ -52,4 +120,17 @@ describe('unflattenKnownApmEventFields', () => { }; expect(unflattenKnownApmEventFields(input, requiredFields)).toEqual(expectedOutput); }); + + it('should throw an exception when mandatory field is not in the input', () => { + const input = { + 'service.environment': 'PROD', + }; + + const requiredFields: ['service.name'] = ['service.name']; + + // @ts-expect-error + expect(() => unflattenKnownApmEventFields(input, requiredFields)).toThrowError( + 'Missing required fields service.name in event' + ); + }); }); From 7b5db6c9ac8af9d9adcd69cc235590b9fbbefcbd Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 7 Oct 2024 19:53:26 +0200 Subject: [PATCH 08/27] Clean up --- .../apm_data_access/server/utils/unflatten_known_fields.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts index 3517092ee0aaf..ef6f2f71365a1 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts @@ -5,7 +5,6 @@ * 2.0. */ -// unflatten_known_fields.test.ts import { unflattenKnownApmEventFields } from './unflatten_known_fields'; describe('unflattenKnownApmEventFields', () => { From 8e702803beca5dee7991f8282daf2b74cac95e67 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 7 Oct 2024 20:04:29 +0200 Subject: [PATCH 09/27] More tests --- .../apm_data_access/server/utils/unflatten_known_fields.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts index ef6f2f71365a1..9dc861a5292df 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts @@ -89,6 +89,7 @@ describe('unflattenKnownApmEventFields', () => { it('should retain unknown fields in the output', () => { const input = { 'service.name': 'node-svc', + 'unknown.texts': ['foo', 'bar'], 'unknown.field': 'foo', unknonwField: 'bar', }; @@ -98,6 +99,7 @@ describe('unflattenKnownApmEventFields', () => { }, unknown: { field: 'foo', + texts: ['foo', 'bar'], }, unknonwField: 'bar', }; From e71abf17d692755cedab9bbb625a8f4754ebd5f8 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 8 Oct 2024 10:22:38 +0200 Subject: [PATCH 10/27] Fix build --- .../src/es_schemas/raw/apm_base_doc.ts | 6 +- .../src/es_schemas/ui/fields/agent.ts | 2 +- .../object/flatten_object.test.ts | 12 ++++ .../__snapshots__/es_fields.test.ts.snap | 60 ++++++++++++++++++- .../apm/common/service_metadata.ts | 49 +++++++++++++++ .../collect_data_telemetry/tasks.test.ts | 6 +- .../collect_data_telemetry/tasks.ts | 2 +- .../get_log_categories/index.ts | 4 +- .../get_derived_service_annotations.ts | 1 + .../routes/services/get_service_agent.ts | 4 +- .../get_service_instance_metadata_details.ts | 35 +++++++++-- .../routes/transactions/get_span/index.ts | 1 + .../transactions/get_transaction/index.ts | 2 - .../__snapshots__/instance_details.spec.snap | 1 + 14 files changed, 164 insertions(+), 21 deletions(-) diff --git a/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts b/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts index b3a6066631346..14d26354e44ed 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts @@ -14,10 +14,10 @@ export interface APMBaseDoc { '@timestamp': string; agent: { name: string; - version: string; + version?: string; }; - parent?: { id: string }; // parent ID is not available on root transactions - trace?: { id: string }; + parent?: { id?: string }; // parent ID is not available on root transactions + trace?: { id?: string }; labels?: { [key: string]: string | number | boolean; }; diff --git a/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts b/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts index ea3ebf39555d2..e8734de141e83 100644 --- a/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts +++ b/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts @@ -14,5 +14,5 @@ export type { ElasticAgentName, OpenTelemetryAgentName, AgentName } from '@kbn/e export interface Agent { ephemeral_id?: string; name: AgentName; - version: string; + version?: string; } diff --git a/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts b/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts index deb7ed998c478..13a8174f4f1cf 100644 --- a/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts +++ b/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts @@ -21,6 +21,18 @@ describe('flattenObject', () => { }); }); + it('flattens arrays', () => { + expect( + flattenObject({ + child: { + id: [1, 2], + }, + }) + ).toEqual({ + 'child.id': [1, 2], + }); + }); + it('does not flatten arrays', () => { expect( flattenObject({ diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap b/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap index 22dbfb92aa0e0..c02bb4daa0a87 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap @@ -15,6 +15,8 @@ exports[`Error AGENT_VERSION 1`] = `"agent version"`; exports[`Error APP_LAUNCH_TIME 1`] = `undefined`; +exports[`Error AT_TIMESTAMP 1`] = `"Tue Oct 08 2024 08:59:49 GMT+0200 (Central European Summer Time)"`; + exports[`Error CHILD_ID 1`] = `undefined`; exports[`Error CLIENT_GEO_CITY_NAME 1`] = `undefined`; @@ -37,6 +39,8 @@ Object { exports[`Error CLOUD_ACCOUNT_ID 1`] = `undefined`; +exports[`Error CLOUD_ACCOUNT_NAME 1`] = `undefined`; + exports[`Error CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; exports[`Error CLOUD_INSTANCE_ID 1`] = `undefined`; @@ -45,6 +49,8 @@ exports[`Error CLOUD_INSTANCE_NAME 1`] = `undefined`; exports[`Error CLOUD_MACHINE_TYPE 1`] = `undefined`; +exports[`Error CLOUD_PROJECT_NAME 1`] = `undefined`; + exports[`Error CLOUD_PROVIDER 1`] = `"gcp"`; exports[`Error CLOUD_REGION 1`] = `"europe-west1"`; @@ -94,6 +100,8 @@ exports[`Error ERROR_LOG_MESSAGE 1`] = `undefined`; exports[`Error ERROR_PAGE_URL 1`] = `undefined`; +exports[`Error ERROR_STACK_TRACE 1`] = `undefined`; + exports[`Error ERROR_TYPE 1`] = `undefined`; exports[`Error EVENT_NAME 1`] = `undefined`; @@ -140,6 +148,8 @@ exports[`Error INDEX 1`] = `undefined`; exports[`Error KUBERNETES 1`] = `undefined`; +exports[`Error KUBERNETES_CONTAINER_ID 1`] = `undefined`; + exports[`Error KUBERNETES_CONTAINER_NAME 1`] = `undefined`; exports[`Error KUBERNETES_DEPLOYMENT 1`] = `undefined`; @@ -150,6 +160,8 @@ exports[`Error KUBERNETES_NAMESPACE 1`] = `undefined`; exports[`Error KUBERNETES_NAMESPACE_NAME 1`] = `undefined`; +exports[`Error KUBERNETES_NODE_NAME 1`] = `undefined`; + exports[`Error KUBERNETES_POD_NAME 1`] = `undefined`; exports[`Error KUBERNETES_POD_UID 1`] = `undefined`; @@ -228,6 +240,10 @@ exports[`Error OBSERVER_HOSTNAME 1`] = `undefined`; exports[`Error OBSERVER_LISTENING 1`] = `undefined`; +exports[`Error OBSERVER_VERSION 1`] = `"whatever"`; + +exports[`Error OBSERVER_VERSION_MAJOR 1`] = `8`; + exports[`Error PARENT_ID 1`] = `"parentId"`; exports[`Error PROCESS_ARGS 1`] = `undefined`; @@ -236,6 +252,8 @@ exports[`Error PROCESS_PID 1`] = `undefined`; exports[`Error PROCESSOR_EVENT 1`] = `"error"`; +exports[`Error PROCESSOR_NAME 1`] = `"error"`; + exports[`Error SERVICE 1`] = ` Object { "language": Object { @@ -308,7 +326,7 @@ exports[`Error SPAN_TYPE 1`] = `undefined`; exports[`Error TIER 1`] = `undefined`; -exports[`Error TIMESTAMP 1`] = `1337`; +exports[`Error TIMESTAMP_US 1`] = `1337`; exports[`Error TRACE_ID 1`] = `"trace id"`; @@ -369,6 +387,8 @@ exports[`Span AGENT_VERSION 1`] = `"agent version"`; exports[`Span APP_LAUNCH_TIME 1`] = `undefined`; +exports[`Span AT_TIMESTAMP 1`] = `"Tue Oct 08 2024 08:59:49 GMT+0200 (Central European Summer Time)"`; + exports[`Span CHILD_ID 1`] = `undefined`; exports[`Span CLIENT_GEO_CITY_NAME 1`] = `undefined`; @@ -391,6 +411,8 @@ Object { exports[`Span CLOUD_ACCOUNT_ID 1`] = `undefined`; +exports[`Span CLOUD_ACCOUNT_NAME 1`] = `undefined`; + exports[`Span CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; exports[`Span CLOUD_INSTANCE_ID 1`] = `undefined`; @@ -399,6 +421,8 @@ exports[`Span CLOUD_INSTANCE_NAME 1`] = `undefined`; exports[`Span CLOUD_MACHINE_TYPE 1`] = `undefined`; +exports[`Span CLOUD_PROJECT_NAME 1`] = `undefined`; + exports[`Span CLOUD_PROVIDER 1`] = `"gcp"`; exports[`Span CLOUD_REGION 1`] = `"europe-west1"`; @@ -439,6 +463,8 @@ exports[`Span ERROR_LOG_MESSAGE 1`] = `undefined`; exports[`Span ERROR_PAGE_URL 1`] = `undefined`; +exports[`Span ERROR_STACK_TRACE 1`] = `undefined`; + exports[`Span ERROR_TYPE 1`] = `undefined`; exports[`Span EVENT_NAME 1`] = `undefined`; @@ -481,6 +507,8 @@ exports[`Span INDEX 1`] = `undefined`; exports[`Span KUBERNETES 1`] = `undefined`; +exports[`Span KUBERNETES_CONTAINER_ID 1`] = `undefined`; + exports[`Span KUBERNETES_CONTAINER_NAME 1`] = `undefined`; exports[`Span KUBERNETES_DEPLOYMENT 1`] = `undefined`; @@ -491,6 +519,8 @@ exports[`Span KUBERNETES_NAMESPACE 1`] = `undefined`; exports[`Span KUBERNETES_NAMESPACE_NAME 1`] = `undefined`; +exports[`Span KUBERNETES_NODE_NAME 1`] = `undefined`; + exports[`Span KUBERNETES_POD_NAME 1`] = `undefined`; exports[`Span KUBERNETES_POD_UID 1`] = `undefined`; @@ -569,6 +599,10 @@ exports[`Span OBSERVER_HOSTNAME 1`] = `undefined`; exports[`Span OBSERVER_LISTENING 1`] = `undefined`; +exports[`Span OBSERVER_VERSION 1`] = `"whatever"`; + +exports[`Span OBSERVER_VERSION_MAJOR 1`] = `8`; + exports[`Span PARENT_ID 1`] = `"parentId"`; exports[`Span PROCESS_ARGS 1`] = `undefined`; @@ -577,6 +611,8 @@ exports[`Span PROCESS_PID 1`] = `undefined`; exports[`Span PROCESSOR_EVENT 1`] = `"span"`; +exports[`Span PROCESSOR_NAME 1`] = `"transaction"`; + exports[`Span SERVICE 1`] = ` Object { "name": "service name", @@ -645,7 +681,7 @@ exports[`Span SPAN_TYPE 1`] = `"span type"`; exports[`Span TIER 1`] = `undefined`; -exports[`Span TIMESTAMP 1`] = `1337`; +exports[`Span TIMESTAMP_US 1`] = `1337`; exports[`Span TRACE_ID 1`] = `"trace id"`; @@ -706,6 +742,8 @@ exports[`Transaction AGENT_VERSION 1`] = `"agent version"`; exports[`Transaction APP_LAUNCH_TIME 1`] = `undefined`; +exports[`Transaction AT_TIMESTAMP 1`] = `"Tue Oct 08 2024 08:59:49 GMT+0200 (Central European Summer Time)"`; + exports[`Transaction CHILD_ID 1`] = `undefined`; exports[`Transaction CLIENT_GEO_CITY_NAME 1`] = `undefined`; @@ -728,6 +766,8 @@ Object { exports[`Transaction CLOUD_ACCOUNT_ID 1`] = `undefined`; +exports[`Transaction CLOUD_ACCOUNT_NAME 1`] = `undefined`; + exports[`Transaction CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; exports[`Transaction CLOUD_INSTANCE_ID 1`] = `undefined`; @@ -736,6 +776,8 @@ exports[`Transaction CLOUD_INSTANCE_NAME 1`] = `undefined`; exports[`Transaction CLOUD_MACHINE_TYPE 1`] = `undefined`; +exports[`Transaction CLOUD_PROJECT_NAME 1`] = `undefined`; + exports[`Transaction CLOUD_PROVIDER 1`] = `"gcp"`; exports[`Transaction CLOUD_REGION 1`] = `"europe-west1"`; @@ -780,6 +822,8 @@ exports[`Transaction ERROR_LOG_MESSAGE 1`] = `undefined`; exports[`Transaction ERROR_PAGE_URL 1`] = `undefined`; +exports[`Transaction ERROR_STACK_TRACE 1`] = `undefined`; + exports[`Transaction ERROR_TYPE 1`] = `undefined`; exports[`Transaction EVENT_NAME 1`] = `undefined`; @@ -832,6 +876,8 @@ Object { } `; +exports[`Transaction KUBERNETES_CONTAINER_ID 1`] = `undefined`; + exports[`Transaction KUBERNETES_CONTAINER_NAME 1`] = `undefined`; exports[`Transaction KUBERNETES_DEPLOYMENT 1`] = `undefined`; @@ -842,6 +888,8 @@ exports[`Transaction KUBERNETES_NAMESPACE 1`] = `undefined`; exports[`Transaction KUBERNETES_NAMESPACE_NAME 1`] = `undefined`; +exports[`Transaction KUBERNETES_NODE_NAME 1`] = `undefined`; + exports[`Transaction KUBERNETES_POD_NAME 1`] = `undefined`; exports[`Transaction KUBERNETES_POD_UID 1`] = `"pod1234567890abcdef"`; @@ -920,6 +968,10 @@ exports[`Transaction OBSERVER_HOSTNAME 1`] = `undefined`; exports[`Transaction OBSERVER_LISTENING 1`] = `undefined`; +exports[`Transaction OBSERVER_VERSION 1`] = `"whatever"`; + +exports[`Transaction OBSERVER_VERSION_MAJOR 1`] = `8`; + exports[`Transaction PARENT_ID 1`] = `"parentId"`; exports[`Transaction PROCESS_ARGS 1`] = `undefined`; @@ -928,6 +980,8 @@ exports[`Transaction PROCESS_PID 1`] = `undefined`; exports[`Transaction PROCESSOR_EVENT 1`] = `"transaction"`; +exports[`Transaction PROCESSOR_NAME 1`] = `"transaction"`; + exports[`Transaction SERVICE 1`] = ` Object { "language": Object { @@ -1000,7 +1054,7 @@ exports[`Transaction SPAN_TYPE 1`] = `undefined`; exports[`Transaction TIER 1`] = `undefined`; -exports[`Transaction TIMESTAMP 1`] = `1337`; +exports[`Transaction TIMESTAMP_US 1`] = `1337`; exports[`Transaction TRACE_ID 1`] = `"trace id"`; diff --git a/x-pack/plugins/observability_solution/apm/common/service_metadata.ts b/x-pack/plugins/observability_solution/apm/common/service_metadata.ts index 0ccede67741b7..00560a2d6000b 100644 --- a/x-pack/plugins/observability_solution/apm/common/service_metadata.ts +++ b/x-pack/plugins/observability_solution/apm/common/service_metadata.ts @@ -4,5 +4,54 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_ID, + CLOUD_INSTANCE_NAME, + CLOUD_MACHINE_TYPE, + CLOUD_PROVIDER, + CONTAINER_ID, + HOST_NAME, + KUBERNETES_CONTAINER_NAME, + KUBERNETES_NAMESPACE_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_POD_NAME, + KUBERNETES_POD_UID, + KUBERNETES_REPLICASET_NAME, + SERVICE_NODE_NAME, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, + SERVICE_VERSION, +} from './es_fields/apm'; +import { asMutableArray } from './utils/as_mutable_array'; + +export const SERVICE_METADATA_SERVICE_KEYS = asMutableArray([ + SERVICE_NODE_NAME, + SERVICE_VERSION, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, +] as const); + +export const SERVICE_METADATA_CONTAINER_KEYS = asMutableArray([ + CONTAINER_ID, + HOST_NAME, + KUBERNETES_POD_UID, + KUBERNETES_POD_NAME, +] as const); + +export const SERVICE_METADATA_INFRA_METRICS_KEYS = asMutableArray([ + KUBERNETES_CONTAINER_NAME, + KUBERNETES_NAMESPACE_NAME, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_DEPLOYMENT_NAME, +] as const); + +export const SERVICE_METADATA_CLOUD_KEYS = asMutableArray([ + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_ID, + CLOUD_INSTANCE_NAME, + CLOUD_MACHINE_TYPE, + CLOUD_PROVIDER, +] as const); export type ContainerType = 'Kubernetes' | 'Docker' | undefined; diff --git a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts index ed460257b0b10..a2b6809f855e7 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts @@ -8,7 +8,7 @@ import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; import type { APMIndices } from '@kbn/apm-data-access-plugin/server'; import { tasks } from './tasks'; -import { SERVICE_NAME, SERVICE_ENVIRONMENT } from '../../../../common/es_fields/apm'; +import { SERVICE_NAME, SERVICE_ENVIRONMENT, AT_TIMESTAMP } from '../../../../common/es_fields/apm'; import { IndicesStatsResponse } from '../telemetry_client'; describe('data telemetry collection tasks', () => { @@ -101,7 +101,7 @@ describe('data telemetry collection tasks', () => { // a fixed date range. .mockReturnValueOnce({ hits: { - hits: [{ field: { '@timestamp': new Date().toISOString() } }], + hits: [{ fields: { [AT_TIMESTAMP]: [new Date().toISOString()] } }], }, total: { value: 1, @@ -314,7 +314,7 @@ describe('data telemetry collection tasks', () => { ? { hits: { total: { value: 1 } } } : { hits: { - hits: [{ field: { '@timestamp': 1 } }], + hits: [{ fields: { [AT_TIMESTAMP]: [1] } }], }, } ); diff --git a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index 40c4b9a8f6121..1347cbb4e3641 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -537,7 +537,7 @@ export const tasks: TelemetryTask[] = [ ? { retainment: { [processorEvent]: { - ms: new Date().getTime() - new Date(event['@timestamp'][0]).getTime(), + ms: new Date().getTime() - new Date(event[AT_TIMESTAMP][0]).getTime(), }, }, } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts index 9d8917bbaad98..7072639f8526e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts @@ -89,7 +89,7 @@ export async function getLogCategories({ const rawSamplingProbability = Math.min(100_000 / totalDocCount, 1); const samplingProbability = rawSamplingProbability < 0.5 ? rawSamplingProbability : 1; - const requiredFields = asMutableArray(['message', TRACE_ID] as const); + const fields = asMutableArray(['message', TRACE_ID] as const); const categorizedLogsRes = await search({ index, size: 1, @@ -112,7 +112,7 @@ export async function getLogCategories({ top_hits: { sort: { '@timestamp': 'desc' as const }, size: 1, - fields: requiredFields, + fields, }, }, }, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts index 3a7e4a64c80f5..c683e308b73b8 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts @@ -88,6 +88,7 @@ export async function getDerivedServiceAnnotations({ sort: { '@timestamp': 'asc', }, + fields: requiredFields, }, }); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts index 8aed821b10f2a..dd272eadf57d6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts @@ -37,7 +37,7 @@ export async function getServiceAgent({ start: number; end: number; }): Promise { - const requiredFields = asMutableArray([ + const fields = asMutableArray([ AGENT_NAME, SERVICE_RUNTIME_NAME, CLOUD_PROVIDER, @@ -83,7 +83,7 @@ export async function getServiceAgent({ ], }, }, - fields: requiredFields, + fields, sort: { _score: { order: 'desc' as const }, }, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts index 62d714734d9b8..3c139f2aee0de 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts @@ -9,7 +9,14 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; -import { METRICSET_NAME, SERVICE_NAME, SERVICE_NODE_NAME } from '../../../common/es_fields/apm'; +import { + AGENT_NAME, + AT_TIMESTAMP, + METRICSET_NAME, + SERVICE_ENVIRONMENT, + SERVICE_NAME, + SERVICE_NODE_NAME, +} from '../../../common/es_fields/apm'; import { maybe } from '../../../common/utils/maybe'; import { getBackwardCompatibleDocumentTypeFilter, @@ -22,6 +29,13 @@ import { Container } from '../../../typings/es_schemas/raw/fields/container'; import { Kubernetes } from '../../../typings/es_schemas/raw/fields/kubernetes'; import { Host } from '../../../typings/es_schemas/raw/fields/host'; import { Cloud } from '../../../typings/es_schemas/raw/fields/cloud'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; +import { + SERVICE_METADATA_CLOUD_KEYS, + SERVICE_METADATA_CONTAINER_KEYS, + SERVICE_METADATA_INFRA_METRICS_KEYS, + SERVICE_METADATA_SERVICE_KEYS, +} from '../../../common/service_metadata'; export interface ServiceInstanceMetadataDetailsResponse { '@timestamp': string; @@ -52,6 +66,18 @@ export async function getServiceInstanceMetadataDetails({ ...rangeQuery(start, end), ]; + const requiredKeys = asMutableArray([AT_TIMESTAMP, SERVICE_NAME, AGENT_NAME] as const); + + const optionalKeys = asMutableArray([ + SERVICE_ENVIRONMENT, + ...SERVICE_METADATA_SERVICE_KEYS, + ...SERVICE_METADATA_CLOUD_KEYS, + ...SERVICE_METADATA_CONTAINER_KEYS, + ...SERVICE_METADATA_INFRA_METRICS_KEYS, + ] as const); + + const fields = [...requiredKeys, ...optionalKeys]; + async function getApplicationMetricSample() { const response = await apmEventClient.search( 'get_service_instance_metadata_details_application_metric', @@ -68,13 +94,12 @@ export async function getServiceInstanceMetadataDetails({ filter: filter.concat({ term: { [METRICSET_NAME]: 'app' } }), }, }, + fields, }, } ); - return unflattenKnownApmEventFields( - maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent - ); + return unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields, requiredKeys); } async function getTransactionEventSample() { @@ -89,6 +114,7 @@ export async function getServiceInstanceMetadataDetails({ terminate_after: 1, size: 1, query: { bool: { filter } }, + fields, }, } ); @@ -114,6 +140,7 @@ export async function getServiceInstanceMetadataDetails({ filter: filter.concat(getBackwardCompatibleDocumentTypeFilter(true)), }, }, + fields, }, } ); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts index 0152d469d8d71..59a3262d21846 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts @@ -42,6 +42,7 @@ export async function getSpan({ track_total_hits: false, size: 1, terminate_after: 1, + fields: ['*'], query: { bool: { filter: asMutableArray([ diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts index 7ac8415ee4c7e..212fa0ea3d0c5 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts @@ -22,7 +22,6 @@ import { TRANSACTION_TYPE, AT_TIMESTAMP, PROCESSOR_NAME, - AGENT_VERSION, SPAN_LINKS, TRANSACTION_AGENT_MARKS, } from '../../../../common/es_fields/apm'; @@ -47,7 +46,6 @@ export async function getTransaction({ const requiredFields = asMutableArray([ TRACE_ID, AGENT_NAME, - AGENT_VERSION, PROCESSOR_EVENT, AT_TIMESTAMP, TIMESTAMP_US, diff --git a/x-pack/test/apm_api_integration/tests/service_overview/__snapshots__/instance_details.spec.snap b/x-pack/test/apm_api_integration/tests/service_overview/__snapshots__/instance_details.spec.snap index f3fb16ec38b15..660355102425e 100644 --- a/x-pack/test/apm_api_integration/tests/service_overview/__snapshots__/instance_details.spec.snap +++ b/x-pack/test/apm_api_integration/tests/service_overview/__snapshots__/instance_details.spec.snap @@ -14,6 +14,7 @@ Object { "kubernetes": Object { "container": Object {}, "deployment": Object {}, + "namespace": Object {}, "pod": Object { "uid": "234", }, From d0e7b1279cc9321e1dc1c8b42bf6cfa7194ac197 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 8 Oct 2024 10:35:00 +0200 Subject: [PATCH 11/27] Improve unflatten_object function and tests --- .../object/unflatten_object.test.ts | 4 ++++ .../observability_utils/object/unflatten_object.ts | 13 ++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts b/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts index acf6009514bf2..22cee17bb1a64 100644 --- a/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts +++ b/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts @@ -24,6 +24,7 @@ describe('unflattenObject', () => { simpleArray: ['0', '1', '2'], complexArray: [{ one: 'one', two: 'two', three: 'three' }], 'nested.array': [0, 1, 2], + 'complex.nested': [{ one: 'one', two: 'two', 'first.second': 'foo', 'first.third': 'bar' }], }) ).toEqual({ simpleArray: ['0', '1', '2'], @@ -31,6 +32,9 @@ describe('unflattenObject', () => { nested: { array: [0, 1, 2], }, + complex: { + nested: [{ one: 'one', two: 'two', first: { second: 'foo', third: 'bar' } }], + }, }); }); }); diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.ts b/x-pack/packages/observability/observability_utils/object/unflatten_object.ts index a04799cbaddcd..142ea2eea6461 100644 --- a/x-pack/packages/observability/observability_utils/object/unflatten_object.ts +++ b/x-pack/packages/observability/observability_utils/object/unflatten_object.ts @@ -11,7 +11,18 @@ export function unflattenObject(source: Record, target: Record { + if (item && typeof item === 'object' && !Array.isArray(item)) { + return unflattenObject(item); + } + return item; + }); + set(target, key, unflattenedArray); + } else { + set(target, key, val); + } } return target; } From 7da83115f1cd800246a0c9dccada9b0a51520b31 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 8 Oct 2024 15:10:31 +0200 Subject: [PATCH 12/27] Error fixes --- packages/kbn-apm-types/src/es_fields/apm.ts | 1 + .../kbn-elastic-agent-utils/src/agent_names.ts | 2 +- .../__snapshots__/es_fields.test.ts.snap | 6 +++--- .../apm/common/es_fields/es_fields.test.ts | 3 ++- .../routes/span_links/get_linked_children.ts | 18 +++++++----------- .../span_links/get_span_links_details.ts | 5 ++--- .../server/routes/traces/get_trace_items.ts | 2 +- .../routes/transactions/get_span/index.ts | 12 +++++++----- .../transactions/get_transaction/index.ts | 5 ++++- .../server/utils/unflatten_known_fields.ts | 1 + 10 files changed, 29 insertions(+), 26 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index 56e2377f2bb7d..25ebd6bc7c0bf 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -102,6 +102,7 @@ export const SPAN_COMPOSITE_SUM = 'span.composite.sum.us'; export const SPAN_COMPOSITE_COMPRESSION_STRATEGY = 'span.composite.compression_strategy'; export const SPAN_SYNC = 'span.sync'; +export const SPAN_STACKTRACE = 'span.stacktrace'; // Parent ID for a transaction or span export const PARENT_ID = 'parent.id'; diff --git a/packages/kbn-elastic-agent-utils/src/agent_names.ts b/packages/kbn-elastic-agent-utils/src/agent_names.ts index 946e72f4b94b0..0405da9cf2193 100644 --- a/packages/kbn-elastic-agent-utils/src/agent_names.ts +++ b/packages/kbn-elastic-agent-utils/src/agent_names.ts @@ -36,7 +36,7 @@ export const ELASTIC_AGENT_NAMES: ElasticAgentName[] = [ 'rum-js', 'android/java', ]; -export type OpenTelemetryGenericAgentName = `opentelemetry/${string}` | `otlp/${string}`; + export type OpenTelemetryAgentName = | 'otlp' | 'opentelemetry' diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap b/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap index c02bb4daa0a87..ccc55dffb5010 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap @@ -15,7 +15,7 @@ exports[`Error AGENT_VERSION 1`] = `"agent version"`; exports[`Error APP_LAUNCH_TIME 1`] = `undefined`; -exports[`Error AT_TIMESTAMP 1`] = `"Tue Oct 08 2024 08:59:49 GMT+0200 (Central European Summer Time)"`; +exports[`Error AT_TIMESTAMP 1`] = `"Tue Oct 08 2024 12:06:19 GMT+0200 (Central European Summer Time)"`; exports[`Error CHILD_ID 1`] = `undefined`; @@ -387,7 +387,7 @@ exports[`Span AGENT_VERSION 1`] = `"agent version"`; exports[`Span APP_LAUNCH_TIME 1`] = `undefined`; -exports[`Span AT_TIMESTAMP 1`] = `"Tue Oct 08 2024 08:59:49 GMT+0200 (Central European Summer Time)"`; +exports[`Span AT_TIMESTAMP 1`] = `"Tue Oct 08 2024 12:06:19 GMT+0200 (Central European Summer Time)"`; exports[`Span CHILD_ID 1`] = `undefined`; @@ -742,7 +742,7 @@ exports[`Transaction AGENT_VERSION 1`] = `"agent version"`; exports[`Transaction APP_LAUNCH_TIME 1`] = `undefined`; -exports[`Transaction AT_TIMESTAMP 1`] = `"Tue Oct 08 2024 08:59:49 GMT+0200 (Central European Summer Time)"`; +exports[`Transaction AT_TIMESTAMP 1`] = `"Tue Oct 08 2024 12:06:19 GMT+0200 (Central European Summer Time)"`; exports[`Transaction CHILD_ID 1`] = `undefined`; diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts b/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts index f33fddd430e8d..44f8be16d0e34 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts @@ -10,9 +10,10 @@ import { AllowUnknownProperties } from '../../typings/common'; import { APMError } from '../../typings/es_schemas/ui/apm_error'; import { Span } from '../../typings/es_schemas/ui/span'; import { Transaction } from '../../typings/es_schemas/ui/transaction'; -import * as apmFieldnames from './apm'; +import * as allApmFieldnames from './apm'; import * as infraMetricsFieldnames from './infra_metrics'; +const { AT_TIMESTAMP, ...apmFieldnames } = allApmFieldnames; const fieldnames = { ...apmFieldnames, ...infraMetricsFieldnames, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts index 8d3c5315f635d..2ff34698c20bc 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts @@ -39,12 +39,8 @@ async function fetchLinkedChildrenOfSpan({ end, }); - const requiredFields = asMutableArray([ - TRACE_ID, - SPAN_ID, - PROCESSOR_EVENT, - TRANSACTION_ID, - ] as const); + const requiredFields = asMutableArray([TRACE_ID, PROCESSOR_EVENT] as const); + const optionalFields = asMutableArray([SPAN_ID, TRANSACTION_ID] as const); const response = await apmEventClient.search('fetch_linked_children_of_span', { apm: { @@ -52,7 +48,7 @@ async function fetchLinkedChildrenOfSpan({ }, _source: [SPAN_LINKS], body: { - fields: requiredFields, + fields: [...requiredFields, ...optionalFields], track_total_hits: false, size: 1000, query: { @@ -88,10 +84,10 @@ async function fetchLinkedChildrenOfSpan({ }); } -function getSpanId(linkedChild: Awaited>[number]) { - return linkedChild.processor.event === ProcessorEvent.span - ? linkedChild.span?.id - : linkedChild.transaction?.id; +function getSpanId( + linkedChild: Awaited>[number] +): string { + return (linkedChild.span.id ?? linkedChild.transaction?.id) as string; } export async function getSpanLinksCountById({ diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts index 1ae02db257b88..d32fe93fff64e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts @@ -50,7 +50,6 @@ async function fetchSpanLinksDetails({ const requiredFields = asMutableArray([ TRACE_ID, - SPAN_ID, SERVICE_NAME, AGENT_NAME, PROCESSOR_EVENT, @@ -138,6 +137,7 @@ async function fetchSpanLinksDetails({ serviceName: commonEvent.service.name, agentName: commonEvent.agent.name, environment: commonEvent.service.environment as Environment, + transactionId: commonEvent.transaction?.id, }; if (commonEvent.processor.event === ProcessorEvent.transaction) { @@ -148,8 +148,8 @@ async function fetchSpanLinksDetails({ return { traceId: event.trace.id, spanId: event.transaction.id, - transactionId: event.transaction.id, processorEvent: commonEvent.processor.event, + transactionId: event.transaction.id, details: { ...commonDetails, spanName: event.transaction.name, @@ -165,7 +165,6 @@ async function fetchSpanLinksDetails({ return { traceId: event.trace.id, spanId: event.span.id, - transactionId: event.transaction?.id, processorEvent: commonEvent.processor.event, details: { ...commonDetails, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts index 86933873c81b1..7961f1f00e9c8 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts @@ -346,7 +346,7 @@ async function getTraceDocsPerPage({ const sort = hit.sort; const spanLinksSource = 'span' in hit._source ? hit._source.span?.links : undefined; - if (hit.fields['processor.event']?.[0] === 'span') { + if (hit.fields['processor.event']?.[0] === ProcessorEvent.span) { const spanEvent = unflattenKnownApmEventFields(hit.fields, [ ...requiredFields, ...requiredSpanFields, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts index 59a3262d21846..dc8ab0a6aab19 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts @@ -11,7 +11,7 @@ import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; import { merge, omit } from 'lodash'; import { maybe } from '../../../../common/utils/maybe'; -import { SPAN_ID, TRACE_ID } from '../../../../common/es_fields/apm'; +import { SPAN_ID, SPAN_STACKTRACE, TRACE_ID } from '../../../../common/es_fields/apm'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { getTransaction } from '../get_transaction'; @@ -43,6 +43,7 @@ export async function getSpan({ size: 1, terminate_after: 1, fields: ['*'], + _source: [SPAN_STACKTRACE], query: { bool: { filter: asMutableArray([ @@ -65,13 +66,14 @@ export async function getSpan({ : undefined, ]); - const event = unflattenKnownApmEventFields( - maybe(spanResp.hits.hits[0])?.fields as undefined | FlattenedApmEvent - ); + const hit = maybe(spanResp.hits.hits[0]); + const spanFromSource = hit && 'span' in hit._source ? hit._source : undefined; + + const event = unflattenKnownApmEventFields(hit?.fields as undefined | FlattenedApmEvent); return { span: event - ? merge({}, omit(event, 'span.links'), { + ? merge({}, omit(event, 'span.links'), spanFromSource, { processor: { event: 'span' as const, name: 'transaction' as const }, }) : undefined, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts index 212fa0ea3d0c5..8fc9d93ceff87 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts @@ -24,6 +24,7 @@ import { PROCESSOR_NAME, SPAN_LINKS, TRANSACTION_AGENT_MARKS, + SERVICE_LANGUAGE_NAME, } from '../../../../common/es_fields/apm'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; @@ -57,6 +58,8 @@ export async function getTransaction({ TRANSACTION_TYPE, ] as const); + const optionalFields = asMutableArray([PROCESSOR_NAME, SERVICE_LANGUAGE_NAME] as const); + const resp = await apmEventClient.search('get_transaction', { apm: { sources: [ @@ -79,7 +82,7 @@ export async function getTransaction({ ]), }, }, - fields: [...requiredFields, PROCESSOR_NAME], + fields: [...requiredFields, ...optionalFields], _source: [SPAN_LINKS, TRANSACTION_AGENT_MARKS], }, }); diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts index eb56205b4a6be..b9a4322269828 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts @@ -31,6 +31,7 @@ const { VALUE_OTEL_JVM_PROCESS_MEMORY_NON_HEAP, SPAN_LINKS_SPAN_ID, SPAN_LINKS_TRACE_ID, + SPAN_STACKTRACE, ...CONCRETE_FIELDS } = APM_EVENT_FIELDS_MAP; From f99b34df4b4b273bfa5abdbbd2d594ef53faa722 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 8 Oct 2024 16:23:03 +0200 Subject: [PATCH 13/27] Fix error sample query --- .../routes/errors/get_error_groups/get_error_sample_details.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts index 409466aafd9ba..3980c7ff1dfb6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts @@ -68,7 +68,6 @@ export async function getErrorSampleDetails({ }): Promise> { const requiredFields = asMutableArray([ AGENT_NAME, - AGENT_VERSION, PROCESSOR_EVENT, TRACE_ID, TIMESTAMP_US, @@ -80,6 +79,7 @@ export async function getErrorSampleDetails({ const optionalFields = asMutableArray([ TRANSACTION_ID, + AGENT_VERSION, PROCESSOR_NAME, ERROR_STACK_TRACE, ERROR_EXC_MESSAGE, From 486e30e7621b0b28f79d280ee42ad154ef8c95de Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 8 Oct 2024 17:15:53 +0200 Subject: [PATCH 14/27] CR fixes --- .../apm/common/es_fields/es_fields.test.ts | 10 +++++----- .../routes/services/get_service_metadata_icons.ts | 3 +-- .../server/routes/span_links/get_span_links_details.ts | 2 +- .../apm/server/routes/traces/get_trace_items.ts | 2 +- .../server/routes/traces/get_trace_samples_by_query.ts | 2 +- .../apm/server/routes/traces/route.ts | 6 +++--- 6 files changed, 12 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts b/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts index 44f8be16d0e34..12537d35afefe 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts @@ -10,13 +10,13 @@ import { AllowUnknownProperties } from '../../typings/common'; import { APMError } from '../../typings/es_schemas/ui/apm_error'; import { Span } from '../../typings/es_schemas/ui/span'; import { Transaction } from '../../typings/es_schemas/ui/transaction'; -import * as allApmFieldnames from './apm'; -import * as infraMetricsFieldnames from './infra_metrics'; +import * as allApmFieldNames from './apm'; +import * as infraMetricsFieldNames from './infra_metrics'; -const { AT_TIMESTAMP, ...apmFieldnames } = allApmFieldnames; +const { AT_TIMESTAMP, ...apmFieldNames } = allApmFieldNames; const fieldnames = { - ...apmFieldnames, - ...infraMetricsFieldnames, + ...apmFieldNames, + ...infraMetricsFieldNames, }; describe('Transaction', () => { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts index a5f7abaaf16da..74e13ee2783d6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts @@ -21,7 +21,6 @@ import { LABEL_TELEMETRY_AUTO_VERSION, AGENT_VERSION, SERVICE_FRAMEWORK_NAME, - KUBERNETES, } from '../../../common/es_fields/apm'; import { ContainerType } from '../../../common/service_metadata'; import { getProcessorEventForTransactions } from '../../lib/helpers/transactions'; @@ -62,7 +61,7 @@ export async function getServiceMetadataIcons({ const filter = [{ term: { [SERVICE_NAME]: serviceName } }, ...rangeQuery(start, end)]; const fields = asMutableArray([ - KUBERNETES, + KUBERNETES_POD_NAME, CLOUD_PROVIDER, CONTAINER_ID, AGENT_NAME, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts index d32fe93fff64e..669adb1008080 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts @@ -119,7 +119,7 @@ async function fetchSpanLinksDetails({ .filter((hit) => { // The above query might return other spans from the same transaction because siblings spans share the same transaction.id // so, if it is a span we need to guarantee that the span.id is the same as the span links ids - if (hit.fields['processor.event']?.[0] === ProcessorEvent.span) { + if (hit.fields[PROCESSOR_EVENT]?.[0] === ProcessorEvent.span) { const spanLink = unflattenKnownApmEventFields(hit.fields, [ ...requiredFields, ...requiredSpanFields, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts index 7961f1f00e9c8..4a1900ff3143a 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts @@ -346,7 +346,7 @@ async function getTraceDocsPerPage({ const sort = hit.sort; const spanLinksSource = 'span' in hit._source ? hit._source.span?.links : undefined; - if (hit.fields['processor.event']?.[0] === ProcessorEvent.span) { + if (hit.fields[PROCESSOR_EVENT]?.[0] === ProcessorEvent.span) { const spanEvent = unflattenKnownApmEventFields(hit.fields, [ ...requiredFields, ...requiredSpanFields, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts index d35ccbea81714..6e46739408040 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts @@ -92,7 +92,7 @@ export async function getTraceSamplesByQuery({ filter_path: 'hits.sequences.events._source.trace.id', }) ).hits?.sequences?.flatMap((sequence) => - sequence.events.map((event) => (event.fields as { 'trace.id': [string] })['trace.id'][0]) + sequence.events.map((event) => (event.fields as { [TRACE_ID]: [string] })[TRACE_ID][0]) ) ?? []; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts index dd9cd780f826c..0814bcdc5738f 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts @@ -131,7 +131,7 @@ const rootTransactionByTraceIdRoute = createApmServerRoute({ handler: async ( resources ): Promise<{ - transaction: TransactionDetailRedirectInfo | undefined; + transaction?: TransactionDetailRedirectInfo; }> => { const { params: { @@ -158,7 +158,7 @@ const transactionByIdRoute = createApmServerRoute({ handler: async ( resources ): Promise<{ - transaction: Transaction | undefined; + transaction?: Transaction; }> => { const { params: { @@ -194,7 +194,7 @@ const transactionByNameRoute = createApmServerRoute({ handler: async ( resources ): Promise<{ - transaction: TransactionDetailRedirectInfo | undefined; + transaction?: TransactionDetailRedirectInfo; }> => { const { params: { From 548efb859733579ad3009103c5c9d9926bab2b26 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 8 Oct 2024 19:08:25 +0200 Subject: [PATCH 15/27] More tests fixes --- .../es_fields/__snapshots__/es_fields.test.ts.snap | 12 ++++++------ .../routes/traces/get_trace_samples_by_query.ts | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap b/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap index ccc55dffb5010..88d00196e074b 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap @@ -15,8 +15,6 @@ exports[`Error AGENT_VERSION 1`] = `"agent version"`; exports[`Error APP_LAUNCH_TIME 1`] = `undefined`; -exports[`Error AT_TIMESTAMP 1`] = `"Tue Oct 08 2024 12:06:19 GMT+0200 (Central European Summer Time)"`; - exports[`Error CHILD_ID 1`] = `undefined`; exports[`Error CLIENT_GEO_CITY_NAME 1`] = `undefined`; @@ -318,6 +316,8 @@ exports[`Error SPAN_NAME 1`] = `undefined`; exports[`Error SPAN_SELF_TIME_SUM 1`] = `undefined`; +exports[`Error SPAN_STACKTRACE 1`] = `undefined`; + exports[`Error SPAN_SUBTYPE 1`] = `undefined`; exports[`Error SPAN_SYNC 1`] = `undefined`; @@ -387,8 +387,6 @@ exports[`Span AGENT_VERSION 1`] = `"agent version"`; exports[`Span APP_LAUNCH_TIME 1`] = `undefined`; -exports[`Span AT_TIMESTAMP 1`] = `"Tue Oct 08 2024 12:06:19 GMT+0200 (Central European Summer Time)"`; - exports[`Span CHILD_ID 1`] = `undefined`; exports[`Span CLIENT_GEO_CITY_NAME 1`] = `undefined`; @@ -673,6 +671,8 @@ exports[`Span SPAN_NAME 1`] = `"span name"`; exports[`Span SPAN_SELF_TIME_SUM 1`] = `undefined`; +exports[`Span SPAN_STACKTRACE 1`] = `undefined`; + exports[`Span SPAN_SUBTYPE 1`] = `"my subtype"`; exports[`Span SPAN_SYNC 1`] = `false`; @@ -742,8 +742,6 @@ exports[`Transaction AGENT_VERSION 1`] = `"agent version"`; exports[`Transaction APP_LAUNCH_TIME 1`] = `undefined`; -exports[`Transaction AT_TIMESTAMP 1`] = `"Tue Oct 08 2024 12:06:19 GMT+0200 (Central European Summer Time)"`; - exports[`Transaction CHILD_ID 1`] = `undefined`; exports[`Transaction CLIENT_GEO_CITY_NAME 1`] = `undefined`; @@ -1046,6 +1044,8 @@ exports[`Transaction SPAN_NAME 1`] = `undefined`; exports[`Transaction SPAN_SELF_TIME_SUM 1`] = `undefined`; +exports[`Transaction SPAN_STACKTRACE 1`] = `undefined`; + exports[`Transaction SPAN_SUBTYPE 1`] = `undefined`; exports[`Transaction SPAN_SYNC 1`] = `undefined`; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts index 6e46739408040..dd1330dea4e48 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts @@ -89,7 +89,7 @@ export async function getTraceSamplesByQuery({ }, event_category_field: PROCESSOR_EVENT, query, - filter_path: 'hits.sequences.events._source.trace.id', + fields: [TRACE_ID], }) ).hits?.sequences?.flatMap((sequence) => sequence.events.map((event) => (event.fields as { [TRACE_ID]: [string] })[TRACE_ID][0]) From 2b75e7ab1d49d8ebcf1f0140fe0d574eac16904f Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 9 Oct 2024 09:24:41 +0200 Subject: [PATCH 16/27] Revert kubernetes.namespace change --- packages/kbn-apm-types/src/es_fields/apm.ts | 2 +- .../src/es_schemas/raw/fields/kubernetes.ts | 4 +--- .../apm/common/service_metadata.ts | 4 ++-- ...get_service_instance_container_metadata.ts | 10 ++++---- .../__snapshots__/queries.test.ts.snap | 24 +++++++++++++++++++ 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index 25ebd6bc7c0bf..5d50833161979 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -162,7 +162,7 @@ export const CONTAINER_IMAGE = 'container.image.name'; export const KUBERNETES = 'kubernetes'; export const KUBERNETES_POD_NAME = 'kubernetes.pod.name'; export const KUBERNETES_POD_UID = 'kubernetes.pod.uid'; -export const KUBERNETES_NAMESPACE_NAME = 'kubernetes.namespace.name'; +export const KUBERNETES_NAMESPACE = 'kubernetes.namespace'; export const KUBERNETES_NODE_NAME = 'kubernetes.node.name'; export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name'; export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id'; diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts index bd8ebfd8b7272..2a4f1465db9a5 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts @@ -9,9 +9,7 @@ export interface Kubernetes { pod?: { uid?: string | null; name?: string }; - namespace?: { - name?: string; - }; + namespace?: string; replicaset?: { name?: string; }; diff --git a/x-pack/plugins/observability_solution/apm/common/service_metadata.ts b/x-pack/plugins/observability_solution/apm/common/service_metadata.ts index 00560a2d6000b..f4607fd699735 100644 --- a/x-pack/plugins/observability_solution/apm/common/service_metadata.ts +++ b/x-pack/plugins/observability_solution/apm/common/service_metadata.ts @@ -13,7 +13,7 @@ import { CONTAINER_ID, HOST_NAME, KUBERNETES_CONTAINER_NAME, - KUBERNETES_NAMESPACE_NAME, + KUBERNETES_NAMESPACE, KUBERNETES_DEPLOYMENT_NAME, KUBERNETES_POD_NAME, KUBERNETES_POD_UID, @@ -41,7 +41,7 @@ export const SERVICE_METADATA_CONTAINER_KEYS = asMutableArray([ export const SERVICE_METADATA_INFRA_METRICS_KEYS = asMutableArray([ KUBERNETES_CONTAINER_NAME, - KUBERNETES_NAMESPACE_NAME, + KUBERNETES_NAMESPACE, KUBERNETES_REPLICASET_NAME, KUBERNETES_DEPLOYMENT_NAME, ] as const); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts index a8efc356f241b..d16910f5984fc 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts @@ -18,7 +18,7 @@ import { KUBERNETES_REPLICASET_NAME, KUBERNETES_DEPLOYMENT_NAME, KUBERNETES_CONTAINER_ID, - KUBERNETES_NAMESPACE_NAME, + KUBERNETES_NAMESPACE, } from '../../../common/es_fields/apm'; import { Kubernetes } from '../../../typings/es_schemas/raw/fields/kubernetes'; import { maybe } from '../../../common/utils/maybe'; @@ -45,7 +45,7 @@ export const getServiceInstanceContainerMetadata = async ({ { exists: { field: KUBERNETES } }, { exists: { field: CONTAINER_IMAGE } }, { exists: { field: KUBERNETES_CONTAINER_NAME } }, - { exists: { field: KUBERNETES_NAMESPACE_NAME } }, + { exists: { field: KUBERNETES_NAMESPACE } }, { exists: { field: KUBERNETES_POD_NAME } }, { exists: { field: KUBERNETES_POD_UID } }, { exists: { field: KUBERNETES_REPLICASET_NAME } }, @@ -58,7 +58,7 @@ export const getServiceInstanceContainerMetadata = async ({ KUBERNETES_DEPLOYMENT_NAME, KUBERNETES_CONTAINER_ID, KUBERNETES_CONTAINER_NAME, - KUBERNETES_NAMESPACE_NAME, + KUBERNETES_NAMESPACE, KUBERNETES_REPLICASET_NAME, KUBERNETES_DEPLOYMENT_NAME, ] as const); @@ -96,9 +96,7 @@ export const getServiceInstanceContainerMetadata = async ({ replicaset: { name: sample?.kubernetes?.replicaset?.name, }, - namespace: { - name: sample?.kubernetes?.namespace?.name, - }, + namespace: sample?.kubernetes?.namespace, container: { name: sample?.kubernetes?.container?.name, id: sample?.kubernetes?.container?.id, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap b/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap index deb1dec096f08..c7f832fe5ca65 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap @@ -11,6 +11,25 @@ Object { ], }, "body": Object { + "_source": Array [ + "span.links", + "transaction.agent.marks", + ], + "fields": Array [ + "trace.id", + "agent.name", + "processor.event", + "@timestamp", + "timestamp.us", + "service.name", + "transaction.id", + "transaction.duration.us", + "transaction.name", + "transaction.sampled", + "transaction.type", + "processor.name", + "service.language.name", + ], "query": Object { "bool": Object { "filter": Array [ @@ -311,6 +330,11 @@ Object { ], }, "body": Object { + "fields": Array [ + "transaction.id", + "trace.id", + "@timestamp", + ], "query": Object { "bool": Object { "filter": Array [ From f0ab58361d1f10ebe8ad77b4af53b5f20dd27d0d Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 9 Oct 2024 10:00:16 +0200 Subject: [PATCH 17/27] More tests fixes --- .../apm/common/service_metadata.ts | 9 +++++++++ .../routes/services/get_service_metadata_icons.ts | 11 +++++++---- .../__snapshots__/instance_details.spec.snap | 1 - 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/common/service_metadata.ts b/x-pack/plugins/observability_solution/apm/common/service_metadata.ts index f4607fd699735..4136ea361392e 100644 --- a/x-pack/plugins/observability_solution/apm/common/service_metadata.ts +++ b/x-pack/plugins/observability_solution/apm/common/service_metadata.ts @@ -54,4 +54,13 @@ export const SERVICE_METADATA_CLOUD_KEYS = asMutableArray([ CLOUD_PROVIDER, ] as const); +export const SERVICE_METADATA_KUBERNETES_KEYS = asMutableArray([ + KUBERNETES_CONTAINER_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_POD_NAME, + KUBERNETES_POD_UID, + KUBERNETES_REPLICASET_NAME, +] as const); + export type ContainerType = 'Kubernetes' | 'Docker' | undefined; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts index 74e13ee2783d6..eb71eaa0362c5 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts @@ -8,6 +8,7 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import type { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; import { maybe } from '../../../common/utils/maybe'; import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { @@ -22,7 +23,7 @@ import { AGENT_VERSION, SERVICE_FRAMEWORK_NAME, } from '../../../common/es_fields/apm'; -import { ContainerType } from '../../../common/service_metadata'; +import { ContainerType, SERVICE_METADATA_KUBERNETES_KEYS } from '../../../common/service_metadata'; import { getProcessorEventForTransactions } from '../../lib/helpers/transactions'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { ServerlessType, getServerlessTypeFromCloudData } from '../../../common/serverless'; @@ -61,11 +62,11 @@ export async function getServiceMetadataIcons({ const filter = [{ term: { [SERVICE_NAME]: serviceName } }, ...rangeQuery(start, end)]; const fields = asMutableArray([ - KUBERNETES_POD_NAME, CLOUD_PROVIDER, CONTAINER_ID, AGENT_NAME, CLOUD_SERVICE_NAME, + ...SERVICE_METADATA_KUBERNETES_KEYS, ] as const); const params = { @@ -80,7 +81,7 @@ export async function getServiceMetadataIcons({ track_total_hits: 1, size: 1, query: { bool: { filter, should } }, - fields, + fields: [...fields, 'kubernetes*'], }, }; @@ -95,7 +96,9 @@ export async function getServiceMetadataIcons({ }; } - const event = unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields); + const event = unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); const { kubernetes, cloud, container, agent } = event ?? {}; let containerType: ContainerType; diff --git a/x-pack/test/apm_api_integration/tests/service_overview/__snapshots__/instance_details.spec.snap b/x-pack/test/apm_api_integration/tests/service_overview/__snapshots__/instance_details.spec.snap index 660355102425e..f3fb16ec38b15 100644 --- a/x-pack/test/apm_api_integration/tests/service_overview/__snapshots__/instance_details.spec.snap +++ b/x-pack/test/apm_api_integration/tests/service_overview/__snapshots__/instance_details.spec.snap @@ -14,7 +14,6 @@ Object { "kubernetes": Object { "container": Object {}, "deployment": Object {}, - "namespace": Object {}, "pod": Object { "uid": "234", }, From 8fc09c825485e2ddccbef48c6c7d2c897a90494b Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 9 Oct 2024 11:41:42 +0200 Subject: [PATCH 18/27] Update snapshot --- .../traces/__snapshots__/queries.test.ts.snap | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap b/x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap index d64c33a421e19..58ec97112e8f6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap @@ -12,15 +12,26 @@ Object { }, "body": Object { "_source": Array [ + "error.log.message", + "error.exception.message", + "error.exception.handled", + "error.exception.type", + ], + "fields": Array [ "timestamp.us", "trace.id", - "transaction.id", - "parent.id", "service.name", "error.id", - "error.log.message", - "error.exception", "error.grouping_key", + "processor.event", + "parent.id", + "transaction.id", + "span.id", + "error.culprit", + "error.log.message", + "error.exception.message", + "error.exception.handled", + "error.exception.type", ], "query": Object { "bool": Object { From cad4de0c76d5b347c9f9b8d04bd98e599b9450a6 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 9 Oct 2024 15:08:55 +0200 Subject: [PATCH 19/27] Clean up --- .../apm/server/routes/services/get_service_metadata_icons.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts index eb71eaa0362c5..ee0a857c9b719 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts @@ -81,7 +81,7 @@ export async function getServiceMetadataIcons({ track_total_hits: 1, size: 1, query: { bool: { filter, should } }, - fields: [...fields, 'kubernetes*'], + fields, }, }; From 4382f12ff645e1a7806d44e550988ba21ad0f7c3 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 9 Oct 2024 18:21:45 +0200 Subject: [PATCH 20/27] Fix get_metadata_for_dependency query --- .../dependencies/get_metadata_for_dependency.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts index d3d335620b155..5b84743064142 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts @@ -33,7 +33,7 @@ export async function getMetadataForDependency({ start: number; end: number; }): Promise { - const requiredFields = asMutableArray([SPAN_TYPE, SPAN_SUBTYPE] as const); + const fields = asMutableArray([SPAN_TYPE, SPAN_SUBTYPE] as const); const sampleResponse = await apmEventClient.search('get_metadata_for_dependency', { apm: { events: [ProcessorEvent.span], @@ -53,20 +53,17 @@ export async function getMetadataForDependency({ ], }, }, - fields: requiredFields, + fields, sort: { '@timestamp': 'desc', }, }, }); - const sample = unflattenKnownApmEventFields( - maybe(sampleResponse.hits.hits[0])?.fields, - requiredFields - ); + const sample = unflattenKnownApmEventFields(maybe(sampleResponse.hits.hits[0])?.fields); return { - spanType: sample?.span.type, - spanSubtype: sample?.span.subtype, + spanType: sample?.span?.type, + spanSubtype: sample?.span?.subtype, }; } From 06ec4ef54115524b059ad37038aab892361b0a87 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Fri, 11 Oct 2024 16:38:54 +0200 Subject: [PATCH 21/27] Fix error object handling --- .../get_error_group_main_statistics.ts | 12 +++++------- .../get_crash_group_main_statistics.ts | 12 +++++------- .../errors/get_mobile_error_group_main_statistics.ts | 12 +++++------- .../apm/server/routes/traces/get_trace_items.ts | 9 ++++----- 4 files changed, 19 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts index fbdb7fc2fa94d..3d6fa0f5a5ef6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts @@ -8,7 +8,6 @@ import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types'; import { kqlQuery, rangeQuery, termQuery, wildcardQuery } from '@kbn/observability-plugin/server'; import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; -import { castArray } from 'lodash'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { AT_TIMESTAMP, @@ -180,11 +179,10 @@ export async function getErrorGroupMainStatistics({ ...event, error: { ...(event.error ?? {}), - exception: castArray( - errorSource?.error.exception && errorSource?.error.exception?.length > 1 + exception: + (errorSource?.error.exception?.length ?? 0) > 1 ? errorSource?.error.exception - : event?.error.exception - ), + : event?.error.exception && [event.error.exception], }, }; @@ -194,8 +192,8 @@ export async function getErrorGroupMainStatistics({ lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(), occurrences: bucket.doc_count, culprit: mergedEvent.error.culprit, - handled: mergedEvent.error.exception[0].handled, - type: mergedEvent.error.exception[0].type, + handled: mergedEvent.error.exception?.[0].handled, + type: mergedEvent.error.exception?.[0].type, traceId: mergedEvent.trace?.id, }; }) ?? []; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts index 96322ed92ca72..c606a6b045a93 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts @@ -8,7 +8,6 @@ import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types'; import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { castArray } from 'lodash'; import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; import { asMutableArray } from '../../../../../common/utils/as_mutable_array'; import { @@ -142,11 +141,10 @@ export async function getMobileCrashGroupMainStatistics({ ...event, error: { ...(event.error ?? {}), - exception: castArray( - errorSource?.error.exception && errorSource?.error.exception?.length > 1 + exception: + (errorSource?.error.exception?.length ?? 0) > 1 ? errorSource?.error.exception - : event?.error.exception - ), + : event?.error.exception && [event.error.exception], }, }; @@ -156,8 +154,8 @@ export async function getMobileCrashGroupMainStatistics({ lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(), occurrences: bucket.doc_count, culprit: mergedEvent.error.culprit, - handled: mergedEvent.error.exception[0].handled, - type: mergedEvent.error.exception[0].type, + handled: mergedEvent.error.exception?.[0].handled, + type: mergedEvent.error.exception?.[0].type, }; }) ?? [] ); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts index 2caf6589247a3..1181aa5b02870 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts @@ -9,7 +9,6 @@ import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; -import { castArray } from 'lodash'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { AT_TIMESTAMP, @@ -143,11 +142,10 @@ export async function getMobileErrorGroupMainStatistics({ ...event, error: { ...(event.error ?? {}), - exception: castArray( - errorSource?.error.exception && errorSource?.error.exception?.length > 1 + exception: + (errorSource?.error.exception?.length ?? 0) > 1 ? errorSource?.error.exception - : event?.error.exception - ), + : event?.error.exception && [event.error.exception], }, }; @@ -157,8 +155,8 @@ export async function getMobileErrorGroupMainStatistics({ lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(), occurrences: bucket.doc_count, culprit: mergedEvent.error.culprit, - handled: mergedEvent.error.exception[0].handled, - type: mergedEvent.error.exception[0].type, + handled: mergedEvent.error.exception?.[0].handled, + type: mergedEvent.error.exception?.[0].type, }; }) ?? [] ); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts index 4a1900ff3143a..7d2c0443dfa32 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts @@ -10,7 +10,7 @@ import { SortResults } from '@elastic/elasticsearch/lib/api/types'; import { QueryDslQueryContainer, Sort } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { rangeQuery } from '@kbn/observability-plugin/server'; -import { castArray, last, omit } from 'lodash'; +import { last, omit } from 'lodash'; import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { APMConfig } from '../..'; @@ -161,11 +161,10 @@ export async function getTraceItems({ ...event, error: { ...(event.error ?? {}), - exception: castArray( - errorSource?.error.exception && errorSource?.error.exception?.length > 1 + exception: + (errorSource?.error.exception?.length ?? 0) > 1 ? errorSource?.error.exception - : event?.error.exception - ), + : event?.error.exception && [event.error.exception], log: errorSource?.error.log, }, }; From 7e4e43efbb04091648a21739bdc8693f92da7208 Mon Sep 17 00:00:00 2001 From: Jenny Date: Fri, 11 Oct 2024 17:45:39 +0200 Subject: [PATCH 22/27] Fix error correlation issue --- .../errors/get_error_groups/get_error_sample_details.ts | 4 +++- .../apm/server/routes/traces/get_trace_items.ts | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts index 3980c7ff1dfb6..f52767cd2cb46 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts @@ -27,6 +27,7 @@ import { TRACE_ID, TRANSACTION_ID, ERROR_STACK_TRACE, + SPAN_ID, } from '../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../common/utils/environment_query'; import { ApmDocumentType } from '../../../../common/document_type'; @@ -79,6 +80,7 @@ export async function getErrorSampleDetails({ const optionalFields = asMutableArray([ TRANSACTION_ID, + SPAN_ID, AGENT_VERSION, PROCESSOR_NAME, ERROR_STACK_TRACE, @@ -129,7 +131,7 @@ export async function getErrorSampleDetails({ const errorFromFields = unflattenKnownApmEventFields(hit.fields, requiredFields); - const transactionId = errorFromFields.transaction?.id; + const transactionId = errorFromFields.transaction?.id ?? errorFromFields.span?.id; const traceId = errorFromFields.trace.id; let transaction: Transaction | undefined; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts index 7d2c0443dfa32..55fb0aab47f38 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts @@ -159,6 +159,10 @@ export async function getTraceItems({ const waterfallErrorEvent: WaterfallError = { ...event, + parent: { + ...event?.parent, + id: event?.parent?.id ?? event?.span?.id, + }, error: { ...(event.error ?? {}), exception: From e8dce719aa7294795a11eab9bee3efa91e544219 Mon Sep 17 00:00:00 2001 From: Jenny Date: Mon, 14 Oct 2024 13:36:05 +0200 Subject: [PATCH 23/27] Fix missing exception error in error details --- .../error_sampler/error_sample_detail.tsx | 11 +++++------ .../get_error_groups/get_error_sample_details.ts | 6 ++++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx index 90bffc1641d13..2edb2c1a3fea6 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx @@ -36,7 +36,7 @@ import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plug import { useLegacyUrlParams } from '../../../../context/url_params_context/use_url_params'; import { useAnyOfApmParams } from '../../../../hooks/use_apm_params'; import { useApmRouter } from '../../../../hooks/use_apm_router'; -import { FETCH_STATUS, isPending } from '../../../../hooks/use_fetcher'; +import { FETCH_STATUS, isPending, isSuccess } from '../../../../hooks/use_fetcher'; import { useTraceExplorerEnabledSetting } from '../../../../hooks/use_trace_explorer_enabled_setting'; import { APIReturnType } from '../../../../services/rest/create_call_apm_api'; import { TransactionDetailLink } from '../../../shared/links/apm/transaction_detail_link'; @@ -111,8 +111,7 @@ export function ErrorSampleDetails({ const loadingErrorData = isPending(errorFetchStatus); const isLoading = loadingErrorSamplesData || loadingErrorData; - const isSucceded = - errorSamplesFetchStatus === FETCH_STATUS.SUCCESS && errorFetchStatus === FETCH_STATUS.SUCCESS; + const isSucceeded = isSuccess(errorSamplesFetchStatus) && isSuccess(errorFetchStatus); useEffect(() => { setSampleActivePage(0); @@ -137,7 +136,7 @@ export function ErrorSampleDetails({ }); }, [error, transaction, uiActions]); - if (!error && errorSampleIds?.length === 0 && isSucceded) { + if (!error && errorSampleIds?.length === 0 && isSucceeded) { return ( diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts index f52767cd2cb46..91da19224d83c 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts @@ -7,7 +7,6 @@ import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server'; import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; -import { castArray } from 'lodash'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { maybe } from '../../../../common/utils/maybe'; import { @@ -155,7 +154,10 @@ export async function getErrorSampleDetails({ }, error: { ...errorFromFields.error, - exception: castArray(source?.error.exception ?? errorFromFields?.error.exception), + exception: + (source?.error.exception?.length ?? 0) > 1 + ? source?.error.exception + : errorFromFields?.error.exception && [errorFromFields.error.exception], log: source?.error?.log, }, }, From 937e9f34ff2f380daae47ffd0e2b678c765c8378 Mon Sep 17 00:00:00 2001 From: Jenny Date: Mon, 14 Oct 2024 17:33:08 +0200 Subject: [PATCH 24/27] Make transaction fields optional because of the mobile services --- .../src/es_schemas/raw/transaction_raw.ts | 6 +++--- .../kbn-apm-types/src/es_schemas/ui/transaction.ts | 2 +- .../apm/common/waterfall/typings.ts | 8 ++++---- .../apm/server/routes/traces/get_trace_items.ts | 10 ++++------ .../routes/transactions/get_transaction/index.ts | 9 ++++++--- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/packages/kbn-apm-types/src/es_schemas/raw/transaction_raw.ts b/packages/kbn-apm-types/src/es_schemas/raw/transaction_raw.ts index 4e31712e1a002..676e8eee27fdb 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/transaction_raw.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/transaction_raw.ts @@ -35,7 +35,7 @@ export interface TransactionRaw extends APMBaseDoc { trace: { id: string }; // trace is required event?: { outcome?: EventOutcome }; transaction: { - duration: { us: number }; + duration?: { us?: number }; id: string; marks?: { // "agent": not defined by APM Server - only sent by RUM agent @@ -46,12 +46,12 @@ export interface TransactionRaw extends APMBaseDoc { name?: string; page?: Page; // special property for RUM: shared by error and transaction result?: string; - sampled: boolean; + sampled?: boolean; span_count?: { started?: number; dropped?: number; }; - type: string; + type?: string; custom?: Record; message?: { queue?: { name: string }; diff --git a/packages/kbn-apm-types/src/es_schemas/ui/transaction.ts b/packages/kbn-apm-types/src/es_schemas/ui/transaction.ts index 3e5986e86c36e..79783691b678a 100644 --- a/packages/kbn-apm-types/src/es_schemas/ui/transaction.ts +++ b/packages/kbn-apm-types/src/es_schemas/ui/transaction.ts @@ -15,7 +15,7 @@ import { Agent } from './fields/agent'; // and thus it doesn't make sense to treat it as optional type InnerTransaction = TransactionRaw['transaction']; interface InnerTransactionWithName extends InnerTransaction { - name: string; + name?: string; } export interface Transaction extends TransactionRaw { diff --git a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts index 2fd0be94a5c5f..46765721142dd 100644 --- a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts +++ b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts @@ -22,11 +22,11 @@ export interface WaterfallTransaction { event?: { outcome?: EventOutcome }; parent?: { id?: string }; processor: { event: 'transaction' }; - transaction: { - duration: { us: number }; + transaction?: { + duration?: { us?: number }; id: string; - name: string; - type: string; + name?: string; + type?: string; result?: string; }; faas?: { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts index 55fb0aab47f38..2b294b540b33a 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts @@ -274,12 +274,7 @@ async function getTraceDocsPerPage({ PROCESSOR_EVENT, ] as const); - const requiredTxFields = asMutableArray([ - TRANSACTION_ID, - TRANSACTION_DURATION, - TRANSACTION_NAME, - TRANSACTION_TYPE, - ] as const); + const requiredTxFields = asMutableArray([TRANSACTION_ID] as const); const requiredSpanFields = asMutableArray([ SPAN_ID, @@ -301,6 +296,9 @@ async function getTraceDocsPerPage({ SPAN_COMPOSITE_SUM, SPAN_SYNC, CHILD_ID, + TRANSACTION_NAME, + TRANSACTION_TYPE, + TRANSACTION_DURATION, ] as const); const body = { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts index 8fc9d93ceff87..b01a4a3ac6b26 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts @@ -52,14 +52,17 @@ export async function getTransaction({ TIMESTAMP_US, SERVICE_NAME, TRANSACTION_ID, + ] as const); + + const optionalFields = asMutableArray([ + PROCESSOR_NAME, + SERVICE_LANGUAGE_NAME, TRANSACTION_DURATION, TRANSACTION_NAME, TRANSACTION_SAMPLED, TRANSACTION_TYPE, ] as const); - const optionalFields = asMutableArray([PROCESSOR_NAME, SERVICE_LANGUAGE_NAME] as const); - const resp = await apmEventClient.search('get_transaction', { apm: { sources: [ @@ -83,7 +86,7 @@ export async function getTransaction({ }, }, fields: [...requiredFields, ...optionalFields], - _source: [SPAN_LINKS, TRANSACTION_AGENT_MARKS], + _source: [SPAN_LINKS, TRANSACTION_AGENT_MARKS, TRANSACTION_NAME], }, }); From 4134daa1da536f0d2e601935d47062159e2b4d2c Mon Sep 17 00:00:00 2001 From: Jenny Date: Mon, 14 Oct 2024 20:35:01 +0200 Subject: [PATCH 25/27] Make types optional to avoid error with undefined fields --- .../apm/common/waterfall/typings.ts | 2 +- .../error_sample_contextual_insight.tsx | 6 +++--- .../error_sampler/error_sample_detail.tsx | 2 +- ...redirect_to_transaction_detail_page_url.ts | 8 +++++--- .../distribution/index.tsx | 2 +- .../maybe_view_trace_link.tsx | 6 +++--- .../transaction_tabs.tsx | 6 +++--- .../waterfall/flyout_top_level_properties.tsx | 2 +- .../span_flyout/sticky_span_properties.tsx | 2 +- .../waterfall_helpers/waterfall_helpers.ts | 6 +++--- .../waterfall/waterfall_item.tsx | 20 ++++++++++--------- .../apm/transaction_detail_link/index.tsx | 4 ++-- .../shared/summary/duration_summary_item.tsx | 4 ++-- .../shared/summary/transaction_summary.tsx | 2 +- .../__snapshots__/queries.test.ts.snap | 5 +++-- .../get_transaction_by_trace/index.ts | 10 +++++----- 16 files changed, 46 insertions(+), 41 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts index 46765721142dd..083e8a21d0980 100644 --- a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts +++ b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts @@ -24,7 +24,7 @@ export interface WaterfallTransaction { processor: { event: 'transaction' }; transaction?: { duration?: { us?: number }; - id: string; + id?: string; name?: string; type?: string; result?: string; diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx index 20d5521b43ebf..25463912e62bd 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx @@ -34,8 +34,8 @@ export function ErrorSampleContextualInsight({ }; }; transaction?: { - transaction: { - name: string; + transaction?: { + name?: string; }; }; }) { @@ -49,7 +49,7 @@ export function ErrorSampleContextualInsight({ const languageName = error.service.language?.name ?? ''; const runtimeName = error.service.runtime?.name ?? ''; const runtimeVersion = error.service.runtime?.version ?? ''; - const transactionName = transaction?.transaction.name ?? ''; + const transactionName = transaction?.transaction?.name ?? ''; return observabilityAIAssistant?.getContextualInsightMessages({ message: `I'm looking at an exception and trying to understand what it means`, diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx index 2edb2c1a3fea6..91bf0edb49f23 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx @@ -260,7 +260,7 @@ export function ErrorSampleDetails({ { return format({ - pathname: `/services/${transaction.service.name}/transactions/view`, + pathname: `/services/${transaction?.service.name}/transactions/view`, query: { traceId: transaction.trace.id, transactionId: transaction.transaction.id, - transactionName: transaction.transaction.name, + transactionName: transaction?.transaction?.name, transactionType: transaction.transaction.type, rangeFrom: rangeFrom || @@ -36,7 +36,9 @@ export const getRedirectToTransactionDetailPageUrl = ({ rangeTo || roundToNearestMinute({ timestamp: transaction['@timestamp'], - diff: transaction.transaction.duration.us / 1000, + diff: transaction?.transaction?.duration?.us + ? transaction?.transaction?.duration?.us / 1000 + : 0, direction: 'up', }), waterfallItemId, diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/distribution/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/distribution/index.tsx index f7a976b3cc82d..7eec85582d59a 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/distribution/index.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/distribution/index.tsx @@ -63,7 +63,7 @@ export function TransactionDistribution({ const { serviceName } = useApmServiceContext(); const markerCurrentEvent = - waterfallFetchResult.waterfall.entryWaterfallTransaction?.doc.transaction.duration.us; + waterfallFetchResult.waterfall.entryWaterfallTransaction?.doc?.transaction?.duration?.us; const { chartData, hasData, percentileThresholdValue, status, totalDocCount } = useTransactionDistributionChartData(); diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/maybe_view_trace_link.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/maybe_view_trace_link.tsx index 0a0d69fa71169..04ca4a264c72f 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/maybe_view_trace_link.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/maybe_view_trace_link.tsx @@ -100,10 +100,10 @@ export function MaybeViewTraceLink({ return ( )} @@ -158,11 +158,11 @@ function LogsTabContent({ traceId, }: { timestamp: number; - duration: number; + duration?: number; traceId: string; }) { const startTimestamp = Math.floor(timestamp / 1000); - const endTimestamp = Math.ceil(startTimestamp + duration / 1000); + const endTimestamp = Math.ceil((startTimestamp + (duration ?? 0)) / 1000); const framePaddingMs = 1000 * 60 * 60 * 24; // 24 hours return ( { doc: TDocument; docType: TDoctype; - id: string; + id?: string; parent?: IWaterfallSpanOrTransaction; parentId?: string; color: string; @@ -129,9 +129,9 @@ function getTransactionItem( return { docType: 'transaction', doc: transaction, - id: transaction.transaction.id, + id: transaction?.transaction?.id, parentId: transaction.parent?.id, - duration: transaction.transaction.duration.us, + duration: transaction?.transaction?.duration?.us ?? 0, offset: 0, skew: 0, legendValues: getLegendValues(transaction), diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/waterfall_item.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/waterfall_item.tsx index 11d0f9bba9298..f4e39b03ad24c 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/waterfall_item.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/waterfall_item.tsx @@ -178,8 +178,8 @@ function Duration({ item }: { item: IWaterfallSpanOrTransaction }) { function HttpStatusCode({ item }: { item: IWaterfallSpanOrTransaction }) { // http status code for transactions of type 'request' const httpStatusCode = - item.docType === 'transaction' && item.doc.transaction.type === 'request' - ? item.doc.transaction.result + item.docType === 'transaction' && item.doc.transaction?.type === 'request' + ? item.doc.transaction?.result : undefined; if (!httpStatusCode) { @@ -206,7 +206,7 @@ function NameLabel({ item }: { item: IWaterfallSpanOrTransaction }) { case 'transaction': return ( -
{item.doc.transaction.name}
+
{item.doc.transaction?.name}
 ); } @@ -291,12 +291,14 @@ export function WaterfallItem({ {item.docType === 'span' && ( )} - + {item.id && ( + + )} {isServerlessColdstart && } diff --git a/x-pack/plugins/observability_solution/apm/public/components/shared/links/apm/transaction_detail_link/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/shared/links/apm/transaction_detail_link/index.tsx index 4cdd3fd5a6398..ef10c4f25fffa 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/shared/links/apm/transaction_detail_link/index.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/shared/links/apm/transaction_detail_link/index.tsx @@ -27,7 +27,7 @@ interface Props extends APMLinkExtendProps { serviceName: string; traceId?: string; transactionId?: string; - transactionName: string; + transactionName?: string; transactionType?: string; latencyAggregationType?: string; environment?: string; @@ -74,7 +74,7 @@ export function TransactionDetailLink({ search: location.search, }); - if (transactionName !== txGroupsDroppedBucketName) { + if (typeof transactionName === 'string' && transactionName !== txGroupsDroppedBucketName) { return ( {asDuration(duration)}   diff --git a/x-pack/plugins/observability_solution/apm/public/components/shared/summary/transaction_summary.tsx b/x-pack/plugins/observability_solution/apm/public/components/shared/summary/transaction_summary.tsx index db5ab9b4c7af5..ae1cff35128b6 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/shared/summary/transaction_summary.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/shared/summary/transaction_summary.tsx @@ -45,7 +45,7 @@ function TransactionSummary({ transaction, totalDuration, errorCount, coldStartB const items = [ , , diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap b/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap index c7f832fe5ca65..e25bf2220dc7a 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap @@ -14,6 +14,7 @@ Object { "_source": Array [ "span.links", "transaction.agent.marks", + "transaction.name", ], "fields": Array [ "trace.id", @@ -23,12 +24,12 @@ Object { "timestamp.us", "service.name", "transaction.id", + "processor.name", + "service.language.name", "transaction.duration.us", "transaction.name", "transaction.sampled", "transaction.type", - "processor.name", - "service.language.name", ], "query": Object { "bool": Object { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts index 803ae19a2228e..f5fb7f5858ba6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts @@ -28,12 +28,12 @@ export interface TransactionDetailRedirectInfo { id: string; }; transaction: { - id: string; - type: string; - name: string; + id?: string; + type?: string; + name?: string; - duration: { - us: number; + duration?: { + us?: number; }; }; service: { From 241a62f04d8a93a950bd487042bde65c2d5a91fa Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 15 Oct 2024 09:16:55 +0200 Subject: [PATCH 26/27] Revert "Make types optional to avoid error with undefined fields" This reverts commit 4134daa1da536f0d2e601935d47062159e2b4d2c. --- .../apm/common/waterfall/typings.ts | 2 +- .../error_sample_contextual_insight.tsx | 6 +++--- .../error_sampler/error_sample_detail.tsx | 2 +- ...redirect_to_transaction_detail_page_url.ts | 8 +++----- .../distribution/index.tsx | 2 +- .../maybe_view_trace_link.tsx | 6 +++--- .../transaction_tabs.tsx | 6 +++--- .../waterfall/flyout_top_level_properties.tsx | 2 +- .../span_flyout/sticky_span_properties.tsx | 2 +- .../waterfall_helpers/waterfall_helpers.ts | 6 +++--- .../waterfall/waterfall_item.tsx | 20 +++++++++---------- .../apm/transaction_detail_link/index.tsx | 4 ++-- .../shared/summary/duration_summary_item.tsx | 4 ++-- .../shared/summary/transaction_summary.tsx | 2 +- .../__snapshots__/queries.test.ts.snap | 5 ++--- .../get_transaction_by_trace/index.ts | 10 +++++----- 16 files changed, 41 insertions(+), 46 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts index 083e8a21d0980..46765721142dd 100644 --- a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts +++ b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts @@ -24,7 +24,7 @@ export interface WaterfallTransaction { processor: { event: 'transaction' }; transaction?: { duration?: { us?: number }; - id?: string; + id: string; name?: string; type?: string; result?: string; diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx index 25463912e62bd..20d5521b43ebf 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx @@ -34,8 +34,8 @@ export function ErrorSampleContextualInsight({ }; }; transaction?: { - transaction?: { - name?: string; + transaction: { + name: string; }; }; }) { @@ -49,7 +49,7 @@ export function ErrorSampleContextualInsight({ const languageName = error.service.language?.name ?? ''; const runtimeName = error.service.runtime?.name ?? ''; const runtimeVersion = error.service.runtime?.version ?? ''; - const transactionName = transaction?.transaction?.name ?? ''; + const transactionName = transaction?.transaction.name ?? ''; return observabilityAIAssistant?.getContextualInsightMessages({ message: `I'm looking at an exception and trying to understand what it means`, diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx index 91bf0edb49f23..2edb2c1a3fea6 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx @@ -260,7 +260,7 @@ export function ErrorSampleDetails({ { return format({ - pathname: `/services/${transaction?.service.name}/transactions/view`, + pathname: `/services/${transaction.service.name}/transactions/view`, query: { traceId: transaction.trace.id, transactionId: transaction.transaction.id, - transactionName: transaction?.transaction?.name, + transactionName: transaction.transaction.name, transactionType: transaction.transaction.type, rangeFrom: rangeFrom || @@ -36,9 +36,7 @@ export const getRedirectToTransactionDetailPageUrl = ({ rangeTo || roundToNearestMinute({ timestamp: transaction['@timestamp'], - diff: transaction?.transaction?.duration?.us - ? transaction?.transaction?.duration?.us / 1000 - : 0, + diff: transaction.transaction.duration.us / 1000, direction: 'up', }), waterfallItemId, diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/distribution/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/distribution/index.tsx index 7eec85582d59a..f7a976b3cc82d 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/distribution/index.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/distribution/index.tsx @@ -63,7 +63,7 @@ export function TransactionDistribution({ const { serviceName } = useApmServiceContext(); const markerCurrentEvent = - waterfallFetchResult.waterfall.entryWaterfallTransaction?.doc?.transaction?.duration?.us; + waterfallFetchResult.waterfall.entryWaterfallTransaction?.doc.transaction.duration.us; const { chartData, hasData, percentileThresholdValue, status, totalDocCount } = useTransactionDistributionChartData(); diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/maybe_view_trace_link.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/maybe_view_trace_link.tsx index 04ca4a264c72f..0a0d69fa71169 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/maybe_view_trace_link.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/maybe_view_trace_link.tsx @@ -100,10 +100,10 @@ export function MaybeViewTraceLink({ return ( )} @@ -158,11 +158,11 @@ function LogsTabContent({ traceId, }: { timestamp: number; - duration?: number; + duration: number; traceId: string; }) { const startTimestamp = Math.floor(timestamp / 1000); - const endTimestamp = Math.ceil((startTimestamp + (duration ?? 0)) / 1000); + const endTimestamp = Math.ceil(startTimestamp + duration / 1000); const framePaddingMs = 1000 * 60 * 60 * 24; // 24 hours return ( { doc: TDocument; docType: TDoctype; - id?: string; + id: string; parent?: IWaterfallSpanOrTransaction; parentId?: string; color: string; @@ -129,9 +129,9 @@ function getTransactionItem( return { docType: 'transaction', doc: transaction, - id: transaction?.transaction?.id, + id: transaction.transaction.id, parentId: transaction.parent?.id, - duration: transaction?.transaction?.duration?.us ?? 0, + duration: transaction.transaction.duration.us, offset: 0, skew: 0, legendValues: getLegendValues(transaction), diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/waterfall_item.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/waterfall_item.tsx index f4e39b03ad24c..11d0f9bba9298 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/waterfall_item.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/waterfall_item.tsx @@ -178,8 +178,8 @@ function Duration({ item }: { item: IWaterfallSpanOrTransaction }) { function HttpStatusCode({ item }: { item: IWaterfallSpanOrTransaction }) { // http status code for transactions of type 'request' const httpStatusCode = - item.docType === 'transaction' && item.doc.transaction?.type === 'request' - ? item.doc.transaction?.result + item.docType === 'transaction' && item.doc.transaction.type === 'request' + ? item.doc.transaction.result : undefined; if (!httpStatusCode) { @@ -206,7 +206,7 @@ function NameLabel({ item }: { item: IWaterfallSpanOrTransaction }) { case 'transaction': return ( -
{item.doc.transaction?.name}
+
{item.doc.transaction.name}
 ); } @@ -291,14 +291,12 @@ export function WaterfallItem({ {item.docType === 'span' && ( )} - {item.id && ( - - )} + {isServerlessColdstart && } diff --git a/x-pack/plugins/observability_solution/apm/public/components/shared/links/apm/transaction_detail_link/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/shared/links/apm/transaction_detail_link/index.tsx index ef10c4f25fffa..4cdd3fd5a6398 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/shared/links/apm/transaction_detail_link/index.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/shared/links/apm/transaction_detail_link/index.tsx @@ -27,7 +27,7 @@ interface Props extends APMLinkExtendProps { serviceName: string; traceId?: string; transactionId?: string; - transactionName?: string; + transactionName: string; transactionType?: string; latencyAggregationType?: string; environment?: string; @@ -74,7 +74,7 @@ export function TransactionDetailLink({ search: location.search, }); - if (typeof transactionName === 'string' && transactionName !== txGroupsDroppedBucketName) { + if (transactionName !== txGroupsDroppedBucketName) { return ( {asDuration(duration)}   diff --git a/x-pack/plugins/observability_solution/apm/public/components/shared/summary/transaction_summary.tsx b/x-pack/plugins/observability_solution/apm/public/components/shared/summary/transaction_summary.tsx index ae1cff35128b6..db5ab9b4c7af5 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/shared/summary/transaction_summary.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/shared/summary/transaction_summary.tsx @@ -45,7 +45,7 @@ function TransactionSummary({ transaction, totalDuration, errorCount, coldStartB const items = [ , , diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap b/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap index e25bf2220dc7a..c7f832fe5ca65 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap @@ -14,7 +14,6 @@ Object { "_source": Array [ "span.links", "transaction.agent.marks", - "transaction.name", ], "fields": Array [ "trace.id", @@ -24,12 +23,12 @@ Object { "timestamp.us", "service.name", "transaction.id", - "processor.name", - "service.language.name", "transaction.duration.us", "transaction.name", "transaction.sampled", "transaction.type", + "processor.name", + "service.language.name", ], "query": Object { "bool": Object { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts index f5fb7f5858ba6..803ae19a2228e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts @@ -28,12 +28,12 @@ export interface TransactionDetailRedirectInfo { id: string; }; transaction: { - id?: string; - type?: string; - name?: string; + id: string; + type: string; + name: string; - duration?: { - us?: number; + duration: { + us: number; }; }; service: { From db191655d6c99e8c71a1366944007a354e91303a Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 15 Oct 2024 09:17:57 +0200 Subject: [PATCH 27/27] Revert "Merge pull request #6 from jennypavlova/196161-apmotel-fix-an-error-with-mobile-services" This reverts commit 40d5fa9d3ba4b3fa260207d8b56b7a19a19da269, reversing changes made to cba684f45e10a8afa031fda17bb8b38a20ac7393. --- .../src/es_schemas/raw/transaction_raw.ts | 6 +++--- .../kbn-apm-types/src/es_schemas/ui/transaction.ts | 2 +- .../apm/common/waterfall/typings.ts | 8 ++++---- .../apm/server/routes/traces/get_trace_items.ts | 10 ++++++---- .../routes/transactions/get_transaction/index.ts | 9 +++------ 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/packages/kbn-apm-types/src/es_schemas/raw/transaction_raw.ts b/packages/kbn-apm-types/src/es_schemas/raw/transaction_raw.ts index 676e8eee27fdb..4e31712e1a002 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/transaction_raw.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/transaction_raw.ts @@ -35,7 +35,7 @@ export interface TransactionRaw extends APMBaseDoc { trace: { id: string }; // trace is required event?: { outcome?: EventOutcome }; transaction: { - duration?: { us?: number }; + duration: { us: number }; id: string; marks?: { // "agent": not defined by APM Server - only sent by RUM agent @@ -46,12 +46,12 @@ export interface TransactionRaw extends APMBaseDoc { name?: string; page?: Page; // special property for RUM: shared by error and transaction result?: string; - sampled?: boolean; + sampled: boolean; span_count?: { started?: number; dropped?: number; }; - type?: string; + type: string; custom?: Record; message?: { queue?: { name: string }; diff --git a/packages/kbn-apm-types/src/es_schemas/ui/transaction.ts b/packages/kbn-apm-types/src/es_schemas/ui/transaction.ts index 79783691b678a..3e5986e86c36e 100644 --- a/packages/kbn-apm-types/src/es_schemas/ui/transaction.ts +++ b/packages/kbn-apm-types/src/es_schemas/ui/transaction.ts @@ -15,7 +15,7 @@ import { Agent } from './fields/agent'; // and thus it doesn't make sense to treat it as optional type InnerTransaction = TransactionRaw['transaction']; interface InnerTransactionWithName extends InnerTransaction { - name?: string; + name: string; } export interface Transaction extends TransactionRaw { diff --git a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts index 46765721142dd..2fd0be94a5c5f 100644 --- a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts +++ b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts @@ -22,11 +22,11 @@ export interface WaterfallTransaction { event?: { outcome?: EventOutcome }; parent?: { id?: string }; processor: { event: 'transaction' }; - transaction?: { - duration?: { us?: number }; + transaction: { + duration: { us: number }; id: string; - name?: string; - type?: string; + name: string; + type: string; result?: string; }; faas?: { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts index 2b294b540b33a..55fb0aab47f38 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts @@ -274,7 +274,12 @@ async function getTraceDocsPerPage({ PROCESSOR_EVENT, ] as const); - const requiredTxFields = asMutableArray([TRANSACTION_ID] as const); + const requiredTxFields = asMutableArray([ + TRANSACTION_ID, + TRANSACTION_DURATION, + TRANSACTION_NAME, + TRANSACTION_TYPE, + ] as const); const requiredSpanFields = asMutableArray([ SPAN_ID, @@ -296,9 +301,6 @@ async function getTraceDocsPerPage({ SPAN_COMPOSITE_SUM, SPAN_SYNC, CHILD_ID, - TRANSACTION_NAME, - TRANSACTION_TYPE, - TRANSACTION_DURATION, ] as const); const body = { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts index b01a4a3ac6b26..8fc9d93ceff87 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts @@ -52,17 +52,14 @@ export async function getTransaction({ TIMESTAMP_US, SERVICE_NAME, TRANSACTION_ID, - ] as const); - - const optionalFields = asMutableArray([ - PROCESSOR_NAME, - SERVICE_LANGUAGE_NAME, TRANSACTION_DURATION, TRANSACTION_NAME, TRANSACTION_SAMPLED, TRANSACTION_TYPE, ] as const); + const optionalFields = asMutableArray([PROCESSOR_NAME, SERVICE_LANGUAGE_NAME] as const); + const resp = await apmEventClient.search('get_transaction', { apm: { sources: [ @@ -86,7 +83,7 @@ export async function getTransaction({ }, }, fields: [...requiredFields, ...optionalFields], - _source: [SPAN_LINKS, TRANSACTION_AGENT_MARKS, TRANSACTION_NAME], + _source: [SPAN_LINKS, TRANSACTION_AGENT_MARKS], }, });