From bb08db94e0207e4bb5785a287abb21c5a34762c8 Mon Sep 17 00:00:00 2001 From: Achyut Jhunjhunwala Date: Thu, 4 May 2023 17:06:43 +0200 Subject: [PATCH] Add logic to replace transaction histogram with summary (#155714) Closes https://github.com/elastic/kibana/issues/152694 ## Summary This PR replaces the transaction.duration.histogram with transaction.duration.summary field and maintains backward compatibility by switching to former when the later is no present for past data ### Checklist Delete any items that are not applicable to this PR. - [x] Switch for transactions table on the service overview page and transactions overview page - [x] Switch for Latency Charts on Transaction Details page - [x] Add logic to switch with backward compatibility - [x] Fix existing Unit and API tests - [x] Add new API Tests to make sure backward compatibility works ### Risk Matrix | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Backward compatibility to switch to histogram if summary field is not present on the data | Low | High | Integration tests will verify that all features are still supported in and duration switches to old field when new field is not available. | --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- x-pack/plugins/apm/common/data_source.ts | 7 + .../plugins/apm/common/time_range_metadata.ts | 4 +- ...ferred_bucket_size_and_data_source.test.ts | 22 +- ...t_preferred_bucket_size_and_data_source.ts | 14 +- .../shared/transactions_table/index.tsx | 13 +- ...k_time_range_metadata_context_provider.tsx | 4 + ...e_preferred_data_source_and_bucket_size.ts | 10 +- .../use_transaction_latency_chart_fetcher.ts | 20 +- .../lib/helpers/get_document_sources.ts | 193 +++++++++---- .../server/lib/helpers/transactions/index.ts | 7 +- ...e_transaction_group_detailed_statistics.ts | 65 ++--- .../get_service_transaction_groups.ts | 14 +- .../transactions/get_latency_charts/index.ts | 18 +- .../apm/server/routes/transactions/route.ts | 46 +-- .../tests/error_rate/service_apis.spec.ts | 1 + .../tests/latency/service_apis.spec.ts | 5 +- .../tests/throughput/service_apis.spec.ts | 1 + .../time_range_metadata/generate_data.ts | 76 +++++ .../time_range_metadata.spec.ts | 94 ++++++ .../tests/transactions/latency.spec.ts | 37 +++ ...actions_groups_detailed_statistics.spec.ts | 136 +++++++-- ...ransactions_groups_main_statistics.spec.ts | 267 +++++++++--------- 22 files changed, 750 insertions(+), 304 deletions(-) create mode 100644 x-pack/test/apm_api_integration/tests/time_range_metadata/generate_data.ts diff --git a/x-pack/plugins/apm/common/data_source.ts b/x-pack/plugins/apm/common/data_source.ts index 9282fb372ac72..f3450fe775429 100644 --- a/x-pack/plugins/apm/common/data_source.ts +++ b/x-pack/plugins/apm/common/data_source.ts @@ -22,3 +22,10 @@ export interface ApmDataSource< rollupInterval: RollupInterval; documentType: TDocumentType; } + +export type ApmDataSourceWithSummary< + T extends AnyApmDocumentType = AnyApmDocumentType +> = ApmDataSource & { + hasDurationSummaryField: boolean; + hasDocs: boolean; +}; diff --git a/x-pack/plugins/apm/common/time_range_metadata.ts b/x-pack/plugins/apm/common/time_range_metadata.ts index aaf8af4dc9a81..ff6d80ca1c745 100644 --- a/x-pack/plugins/apm/common/time_range_metadata.ts +++ b/x-pack/plugins/apm/common/time_range_metadata.ts @@ -9,5 +9,7 @@ import { ApmDataSource } from './data_source'; export interface TimeRangeMetadata { isUsingServiceDestinationMetrics: boolean; - sources: Array; + sources: Array< + ApmDataSource & { hasDocs: boolean; hasDurationSummaryField: boolean } + >; } diff --git a/x-pack/plugins/apm/common/utils/get_preferred_bucket_size_and_data_source.test.ts b/x-pack/plugins/apm/common/utils/get_preferred_bucket_size_and_data_source.test.ts index f4bd7a21af66d..e61b4da5e654e 100644 --- a/x-pack/plugins/apm/common/utils/get_preferred_bucket_size_and_data_source.test.ts +++ b/x-pack/plugins/apm/common/utils/get_preferred_bucket_size_and_data_source.test.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { ApmDataSource } from '../data_source'; +import { ApmDataSourceWithSummary } from '../data_source'; import { ApmDocumentType } from '../document_type'; import { RollupInterval } from '../rollup'; import { @@ -12,40 +12,54 @@ import { intervalToSeconds, } from './get_preferred_bucket_size_and_data_source'; -const serviceTransactionMetricSources: ApmDataSource[] = [ +const serviceTransactionMetricSources: ApmDataSourceWithSummary[] = [ { documentType: ApmDocumentType.ServiceTransactionMetric, rollupInterval: RollupInterval.OneMinute, + hasDurationSummaryField: true, + hasDocs: true, }, { documentType: ApmDocumentType.ServiceTransactionMetric, rollupInterval: RollupInterval.TenMinutes, + hasDurationSummaryField: true, + hasDocs: true, }, { documentType: ApmDocumentType.ServiceTransactionMetric, rollupInterval: RollupInterval.SixtyMinutes, + hasDurationSummaryField: true, + hasDocs: true, }, ]; -const txMetricSources: ApmDataSource[] = [ +const txMetricSources: ApmDataSourceWithSummary[] = [ { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.OneMinute, + hasDurationSummaryField: true, + hasDocs: true, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.TenMinutes, + hasDurationSummaryField: true, + hasDocs: true, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.SixtyMinutes, + hasDurationSummaryField: true, + hasDocs: true, }, ]; -const txEventSources: ApmDataSource[] = [ +const txEventSources: ApmDataSourceWithSummary[] = [ { documentType: ApmDocumentType.TransactionEvent, rollupInterval: RollupInterval.None, + hasDurationSummaryField: false, + hasDocs: false, }, ]; diff --git a/x-pack/plugins/apm/common/utils/get_preferred_bucket_size_and_data_source.ts b/x-pack/plugins/apm/common/utils/get_preferred_bucket_size_and_data_source.ts index 75dba269b93e9..3b2a2ce2e0a1f 100644 --- a/x-pack/plugins/apm/common/utils/get_preferred_bucket_size_and_data_source.ts +++ b/x-pack/plugins/apm/common/utils/get_preferred_bucket_size_and_data_source.ts @@ -6,7 +6,7 @@ */ import { parseInterval } from '@kbn/data-plugin/common'; import { orderBy, last } from 'lodash'; -import { ApmDataSource } from '../data_source'; +import { ApmDataSourceWithSummary } from '../data_source'; import { ApmDocumentType } from '../document_type'; import { RollupInterval } from '../rollup'; @@ -28,16 +28,18 @@ export function getPreferredBucketSizeAndDataSource({ sources, bucketSizeInSeconds, }: { - sources: ApmDataSource[]; + sources: ApmDataSourceWithSummary[]; bucketSizeInSeconds: number; }): { - source: ApmDataSource; + source: ApmDataSourceWithSummary; bucketSizeInSeconds: number; } { - let preferred: ApmDataSource | undefined; + let preferred: ApmDataSourceWithSummary | undefined; + + const sourcesWithDocs = sources.filter((source) => source.hasDocs); const sourcesInPreferredOrder = orderBy( - sources, + sourcesWithDocs, [ (source) => EVENT_PREFERENCE.indexOf(source.documentType), (source) => intervalToSeconds(source.rollupInterval), @@ -68,6 +70,8 @@ export function getPreferredBucketSizeAndDataSource({ preferred = { documentType: ApmDocumentType.TransactionEvent, rollupInterval: RollupInterval.None, + hasDurationSummaryField: false, + hasDocs: true, }; } diff --git a/x-pack/plugins/apm/public/components/shared/transactions_table/index.tsx b/x-pack/plugins/apm/public/components/shared/transactions_table/index.tsx index c4df8a4f7e9a9..000416c09e873 100644 --- a/x-pack/plugins/apm/public/components/shared/transactions_table/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/transactions_table/index.tsx @@ -141,6 +141,10 @@ export function TransactionsTable({ type: ApmDocumentType.TransactionMetric, }); + const shouldUseDurationSummary = + latencyAggregationType === 'avg' && + preferred?.source?.hasDurationSummaryField; + const { data = INITIAL_STATE, status } = useFetcher( (callApmApi) => { if (!latencyAggregationType || !transactionType || !preferred) { @@ -157,6 +161,7 @@ export function TransactionsTable({ start, end, transactionType, + useDurationSummary: shouldUseDurationSummary, latencyAggregationType: latencyAggregationType as LatencyAggregationType, documentType: preferred.source.documentType, @@ -227,7 +232,8 @@ export function TransactionsTable({ start && end && transactionType && - latencyAggregationType + latencyAggregationType && + preferred ) { return callApmApi( 'GET /internal/apm/services/{serviceName}/transactions/groups/detailed_statistics', @@ -239,8 +245,11 @@ export function TransactionsTable({ kuery, start, end, - numBuckets: 20, + bucketSizeInSeconds: preferred.bucketSizeInSeconds, transactionType, + documentType: preferred.source.documentType, + rollupInterval: preferred.source.rollupInterval, + useDurationSummary: shouldUseDurationSummary, latencyAggregationType: latencyAggregationType as LatencyAggregationType, transactionNames: JSON.stringify( diff --git a/x-pack/plugins/apm/public/context/time_range_metadata/mock_time_range_metadata_context_provider.tsx b/x-pack/plugins/apm/public/context/time_range_metadata/mock_time_range_metadata_context_provider.tsx index d0783529cb10d..4c08dd57db9eb 100644 --- a/x-pack/plugins/apm/public/context/time_range_metadata/mock_time_range_metadata_context_provider.tsx +++ b/x-pack/plugins/apm/public/context/time_range_metadata/mock_time_range_metadata_context_provider.tsx @@ -17,21 +17,25 @@ const DEFAULTS = { documentType: ApmDocumentType.ServiceTransactionMetric, hasDocs: true, rollupInterval: RollupInterval.OneMinute, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionMetric, hasDocs: true, rollupInterval: RollupInterval.OneMinute, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionEvent, hasDocs: true, rollupInterval: RollupInterval.None, + hasDurationSummaryField: false, }, { documentType: ApmDocumentType.ServiceDestinationMetric, hasDocs: true, rollupInterval: RollupInterval.None, + hasDurationSummaryField: false, }, ], }; diff --git a/x-pack/plugins/apm/public/hooks/use_preferred_data_source_and_bucket_size.ts b/x-pack/plugins/apm/public/hooks/use_preferred_data_source_and_bucket_size.ts index bd757ea870f2b..07ee3b88ad08c 100644 --- a/x-pack/plugins/apm/public/hooks/use_preferred_data_source_and_bucket_size.ts +++ b/x-pack/plugins/apm/public/hooks/use_preferred_data_source_and_bucket_size.ts @@ -6,7 +6,7 @@ */ import { useMemo } from 'react'; -import { ApmDataSource } from '../../common/data_source'; +import { ApmDataSourceWithSummary } from '../../common/data_source'; import { ApmDocumentType } from '../../common/document_type'; import { getBucketSize } from '../../common/utils/get_bucket_size'; import { getPreferredBucketSizeAndDataSource } from '../../common/utils/get_preferred_bucket_size_and_data_source'; @@ -30,7 +30,7 @@ export function usePreferredDataSourceAndBucketSize< type: TDocumentType; }): { bucketSizeInSeconds: number; - source: ApmDataSource< + source: ApmDataSourceWithSummary< TDocumentType extends ApmDocumentType.ServiceTransactionMetric ? | ApmDocumentType.ServiceTransactionMetric @@ -74,15 +74,13 @@ export function usePreferredDataSourceAndBucketSize< start: new Date(start).getTime(), end: new Date(end).getTime(), }).bucketSize, - sources: sources.filter( - (s) => s.hasDocs && suitableTypes.includes(s.documentType) - ), + sources: sources.filter((s) => suitableTypes.includes(s.documentType)), } ); return { bucketSizeInSeconds, - source: source as ApmDataSource, + source: source as ApmDataSourceWithSummary, }; }, [type, start, end, sources, numBuckets]); } diff --git a/x-pack/plugins/apm/public/hooks/use_transaction_latency_chart_fetcher.ts b/x-pack/plugins/apm/public/hooks/use_transaction_latency_chart_fetcher.ts index e6a4bcc34969c..0e0612102b466 100644 --- a/x-pack/plugins/apm/public/hooks/use_transaction_latency_chart_fetcher.ts +++ b/x-pack/plugins/apm/public/hooks/use_transaction_latency_chart_fetcher.ts @@ -48,6 +48,10 @@ export function useTransactionLatencyChartsFetcher({ type: ApmDocumentType.ServiceTransactionMetric, }); + const shouldUseDurationSummary = + latencyAggregationType === 'avg' && + preferred?.source?.hasDurationSummaryField; + const { data, error, status } = useFetcher( (callApmApi) => { if (!transactionType && transactionTypeStatus === FETCH_STATUS.SUCCESS) { @@ -73,6 +77,7 @@ export function useTransactionLatencyChartsFetcher({ start, end, transactionType, + useDurationSummary: shouldUseDurationSummary, transactionName: transactionName || undefined, latencyAggregationType, offset: @@ -89,18 +94,19 @@ export function useTransactionLatencyChartsFetcher({ } }, [ - environment, - kuery, + transactionType, + transactionTypeStatus, serviceName, start, end, - transactionName, - transactionType, - transactionTypeStatus, latencyAggregationType, - offset, - comparisonEnabled, preferred, + environment, + kuery, + shouldUseDurationSummary, + transactionName, + comparisonEnabled, + offset, ] ); diff --git a/x-pack/plugins/apm/server/lib/helpers/get_document_sources.ts b/x-pack/plugins/apm/server/lib/helpers/get_document_sources.ts index 2b66a0b581888..02258283ac554 100644 --- a/x-pack/plugins/apm/server/lib/helpers/get_document_sources.ts +++ b/x-pack/plugins/apm/server/lib/helpers/get_document_sources.ts @@ -5,11 +5,50 @@ * 2.0. */ import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; -import { ApmDataSource } from '../../../common/data_source'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ApmDocumentType } from '../../../common/document_type'; import { RollupInterval } from '../../../common/rollup'; import { APMEventClient } from './create_es_client/create_apm_event_client'; import { getConfigForDocumentType } from './create_es_client/document_type'; +import { TRANSACTION_DURATION_SUMMARY } from '../../../common/es_fields/apm'; +import { TimeRangeMetadata } from '../../../common/time_range_metadata'; + +const getRequest = ({ + documentType, + rollupInterval, + filters, +}: { + documentType: ApmDocumentType; + rollupInterval: RollupInterval; + filters: estypes.QueryDslQueryContainer[]; +}) => { + const searchParams = { + apm: { + sources: [ + { + documentType, + rollupInterval, + }, + ], + }, + body: { + track_total_hits: 1, + size: 0, + terminate_after: 1, + }, + }; + return { + ...searchParams, + body: { + ...searchParams.body, + query: { + bool: { + filter: filters, + }, + }, + }, + }; +}; export async function getDocumentSources({ apmEventClient, @@ -44,52 +83,71 @@ export async function getDocumentSources({ ? docTypeConfig.rollupIntervals : [RollupInterval.OneMinute] ).flatMap((rollupInterval) => { - const searchParams = { - apm: { - sources: [ + return { + documentType, + rollupInterval, + meta: { + checkSummaryFieldExists: false, + }, + before: getRequest({ + documentType, + rollupInterval, + filters: [...kql, ...beforeRange], + }), + current: getRequest({ + documentType, + rollupInterval, + filters: [...kql, ...currentRange], + }), + }; + }); + }); + + const sourcesToCheckWithSummary = [ + ApmDocumentType.TransactionMetric as const, + ].flatMap((documentType) => { + const docTypeConfig = getConfigForDocumentType(documentType); + + return ( + enableContinuousRollups + ? docTypeConfig.rollupIntervals + : [RollupInterval.OneMinute] + ).flatMap((rollupInterval) => { + const summaryExistsFilter = { + bool: { + filter: [ { - documentType, - rollupInterval, + exists: { + field: TRANSACTION_DURATION_SUMMARY, + }, }, ], }, - body: { - track_total_hits: 1, - size: 0, - terminate_after: 1, - }, }; return { documentType, rollupInterval, - before: { - ...searchParams, - body: { - ...searchParams.body, - query: { - bool: { - filter: [...kql, ...beforeRange], - }, - }, - }, - }, - current: { - ...searchParams, - body: { - ...searchParams.body, - query: { - bool: { - filter: [...kql, ...currentRange], - }, - }, - }, + meta: { + checkSummaryFieldExists: true, }, + before: getRequest({ + documentType, + rollupInterval, + filters: [...kql, ...beforeRange, summaryExistsFilter], + }), + current: getRequest({ + documentType, + rollupInterval, + filters: [...kql, ...currentRange, summaryExistsFilter], + }), }; }); }); - const allSearches = sourcesToCheck.flatMap(({ before, current }) => [ + const allSourcesToCheck = [...sourcesToCheck, ...sourcesToCheckWithSummary]; + + const allSearches = allSourcesToCheck.flatMap(({ before, current }) => [ before, current, ]); @@ -98,43 +156,71 @@ export async function getDocumentSources({ await apmEventClient.msearch('get_document_availability', ...allSearches) ).responses; - const checkedSources = sourcesToCheck.map((source, index) => { + const checkedSources = allSourcesToCheck.map((source, index) => { + const { documentType, rollupInterval } = source; const responseBefore = allResponses[index * 2]; const responseAfter = allResponses[index * 2 + 1]; - const { documentType, rollupInterval } = source; - const hasDataBefore = responseBefore.hits.total.value > 0; - const hasDataAfter = responseAfter.hits.total.value > 0; + const hasDocBefore = responseBefore.hits.total.value > 0; + const hasDocAfter = responseAfter.hits.total.value > 0; return { documentType, rollupInterval, - hasDataBefore, - hasDataAfter, + hasDocBefore, + hasDocAfter, + checkSummaryFieldExists: source.meta.checkSummaryFieldExists, }; }); - const hasAnyDataBefore = checkedSources.some( - (source) => source.hasDataBefore + const hasAnySourceDocBefore = checkedSources.some( + (source) => source.hasDocBefore ); - const sources: Array = - checkedSources.map((source) => { - const { documentType, hasDataAfter, hasDataBefore, rollupInterval } = - source; + const sourcesWithHasDocs = checkedSources.map((checkedSource) => { + const { + documentType, + hasDocAfter, + hasDocBefore, + rollupInterval, + checkSummaryFieldExists, + } = checkedSource; - const hasData = hasDataBefore || hasDataAfter; + const hasDocBeforeOrAfter = hasDocBefore || hasDocAfter; + // If there is any data before, we require that data is available before + // this time range to mark this source as available. If we don't do that, + // users that upgrade to a version that starts generating service tx metrics + // will see a mostly empty screen for a while after upgrading. + // If we only check before, users with a new deployment will use raw transaction + // events. + const hasDocs = hasAnySourceDocBefore ? hasDocBefore : hasDocBeforeOrAfter; + return { + documentType, + rollupInterval, + checkSummaryFieldExists, + hasDocs, + }; + }); + const sources: TimeRangeMetadata['sources'] = sourcesWithHasDocs + .filter((source) => !source.checkSummaryFieldExists) + .map((checkedSource) => { + const { documentType, hasDocs, rollupInterval } = checkedSource; return { documentType, rollupInterval, - // If there is any data before, we require that data is available before - // this time range to mark this source as available. If we don't do that, - // users that upgrade to a version that starts generating service tx metrics - // will see a mostly empty screen for a while after upgrading. - // If we only check before, users with a new deployment will use raw transaction - // events. - hasDocs: hasAnyDataBefore ? hasDataBefore : hasData, + hasDocs, + hasDurationSummaryField: + documentType === ApmDocumentType.ServiceTransactionMetric || + Boolean( + sourcesWithHasDocs.find((eSource) => { + return ( + eSource.documentType === documentType && + eSource.rollupInterval === rollupInterval && + eSource.checkSummaryFieldExists + ); + })?.hasDocs + ), }; }); @@ -142,5 +228,6 @@ export async function getDocumentSources({ documentType: ApmDocumentType.TransactionEvent, rollupInterval: RollupInterval.None, hasDocs: true, + hasDurationSummaryField: false, }); } diff --git a/x-pack/plugins/apm/server/lib/helpers/transactions/index.ts b/x-pack/plugins/apm/server/lib/helpers/transactions/index.ts index cf95500f25318..da987b35025f8 100644 --- a/x-pack/plugins/apm/server/lib/helpers/transactions/index.ts +++ b/x-pack/plugins/apm/server/lib/helpers/transactions/index.ts @@ -95,9 +95,11 @@ export function getDurationFieldForTransactions( | ApmDocumentType.ServiceTransactionMetric | ApmDocumentType.TransactionMetric | ApmDocumentType.TransactionEvent - | boolean + | boolean, + useDurationSummaryField?: boolean ) { let type: ApmDocumentType; + if (typeOrSearchAgggregatedTransactions === true) { type = ApmDocumentType.TransactionMetric; } else if (typeOrSearchAgggregatedTransactions === false) { @@ -111,6 +113,9 @@ export function getDurationFieldForTransactions( } if (type === ApmDocumentType.TransactionMetric) { + if (useDurationSummaryField) { + return TRANSACTION_DURATION_SUMMARY; + } return TRANSACTION_DURATION_HISTOGRAM; } diff --git a/x-pack/plugins/apm/server/routes/services/get_service_transaction_group_detailed_statistics.ts b/x-pack/plugins/apm/server/routes/services/get_service_transaction_group_detailed_statistics.ts index 5ff2ed1a5e02e..27d4dc23d8cbc 100644 --- a/x-pack/plugins/apm/server/routes/services/get_service_transaction_group_detailed_statistics.ts +++ b/x-pack/plugins/apm/server/routes/services/get_service_transaction_group_detailed_statistics.ts @@ -7,7 +7,7 @@ import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; import { keyBy } from 'lodash'; -import { ApmDocumentType } from '../../../common/document_type'; +import { ApmTransactionDocumentType } from '../../../common/document_type'; import { SERVICE_NAME, TRANSACTION_NAME, @@ -19,20 +19,16 @@ import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms'; import { offsetPreviousPeriodCoordinates } from '../../../common/utils/offset_previous_period_coordinate'; import { Coordinate } from '../../../typings/timeseries'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; -import { getBucketSizeForAggregatedTransactions } from '../../lib/helpers/get_bucket_size_for_aggregated_transactions'; import { getLatencyAggregation, getLatencyValue, } from '../../lib/helpers/latency_aggregation_type'; -import { - getDocumentTypeFilterForTransactions, - getDurationFieldForTransactions, - getProcessorEventForTransactions, -} from '../../lib/helpers/transactions'; +import { getDurationFieldForTransactions } from '../../lib/helpers/transactions'; import { calculateFailedTransactionRate, getOutcomeAggregation, } from '../../lib/helpers/transaction_error_rate'; +import { RollupInterval } from '../../../common/rollup'; interface ServiceTransactionGroupDetailedStat { transactionName: string; @@ -48,26 +44,30 @@ async function getServiceTransactionGroupDetailedStatistics({ serviceName, transactionNames, apmEventClient, - numBuckets, - searchAggregatedTransactions, transactionType, latencyAggregationType, start, end, offset, + documentType, + rollupInterval, + bucketSizeInSeconds, + useDurationSummary, }: { environment: string; kuery: string; serviceName: string; transactionNames: string[]; apmEventClient: APMEventClient; - numBuckets: number; - searchAggregatedTransactions: boolean; transactionType: string; latencyAggregationType: LatencyAggregationType; start: number; end: number; offset?: string; + documentType: ApmTransactionDocumentType; + rollupInterval: RollupInterval; + bucketSizeInSeconds: number; + useDurationSummary: boolean; }): Promise { const { startWithOffset, endWithOffset } = getOffsetInMs({ start, @@ -75,22 +75,18 @@ async function getServiceTransactionGroupDetailedStatistics({ offset, }); - const { intervalString } = getBucketSizeForAggregatedTransactions({ - start: startWithOffset, - end: endWithOffset, - numBuckets, - searchAggregatedTransactions, - }); + const intervalString = `${bucketSizeInSeconds}s`; - const field = getDurationFieldForTransactions(searchAggregatedTransactions); + const field = getDurationFieldForTransactions( + documentType, + useDurationSummary + ); const response = await apmEventClient.search( 'get_service_transaction_group_detailed_statistics', { apm: { - events: [ - getProcessorEventForTransactions(searchAggregatedTransactions), - ], + sources: [{ documentType, rollupInterval }], }, body: { track_total_hits: false, @@ -100,9 +96,6 @@ async function getServiceTransactionGroupDetailedStatistics({ filter: [ { term: { [SERVICE_NAME]: serviceName } }, { term: { [TRANSACTION_TYPE]: transactionType } }, - ...getDocumentTypeFilterForTransactions( - searchAggregatedTransactions - ), ...rangeQuery(startWithOffset, endWithOffset), ...environmentQuery(environment), ...kqlQuery(kuery), @@ -133,11 +126,7 @@ async function getServiceTransactionGroupDetailedStatistics({ }, aggs: { ...getLatencyAggregation(latencyAggregationType, field), - ...getOutcomeAggregation( - searchAggregatedTransactions - ? ApmDocumentType.TransactionMetric - : ApmDocumentType.TransactionEvent - ), + ...getOutcomeAggregation(documentType), }, }, }, @@ -190,9 +179,11 @@ export async function getServiceTransactionGroupDetailedStatisticsPeriods({ serviceName, transactionNames, apmEventClient, - numBuckets, - searchAggregatedTransactions, transactionType, + documentType, + rollupInterval, + bucketSizeInSeconds, + useDurationSummary, latencyAggregationType, environment, kuery, @@ -203,9 +194,11 @@ export async function getServiceTransactionGroupDetailedStatisticsPeriods({ serviceName: string; transactionNames: string[]; apmEventClient: APMEventClient; - numBuckets: number; - searchAggregatedTransactions: boolean; transactionType: string; + documentType: ApmTransactionDocumentType; + rollupInterval: RollupInterval; + bucketSizeInSeconds: number; + useDurationSummary: boolean; latencyAggregationType: LatencyAggregationType; environment: string; kuery: string; @@ -217,12 +210,14 @@ export async function getServiceTransactionGroupDetailedStatisticsPeriods({ apmEventClient, serviceName, transactionNames, - searchAggregatedTransactions, transactionType, - numBuckets, latencyAggregationType: latencyAggregationType as LatencyAggregationType, environment, kuery, + documentType, + rollupInterval, + bucketSizeInSeconds, + useDurationSummary, }; const currentPeriodPromise = getServiceTransactionGroupDetailedStatistics({ diff --git a/x-pack/plugins/apm/server/routes/services/get_service_transaction_groups.ts b/x-pack/plugins/apm/server/routes/services/get_service_transaction_groups.ts index 1a0bbc9cc7b5c..968e487eed439 100644 --- a/x-pack/plugins/apm/server/routes/services/get_service_transaction_groups.ts +++ b/x-pack/plugins/apm/server/routes/services/get_service_transaction_groups.ts @@ -30,13 +30,6 @@ import { const txGroupsDroppedBucketName = '_other'; -export type ServiceOverviewTransactionGroupSortField = - | 'name' - | 'latency' - | 'throughput' - | 'errorRate' - | 'impact'; - export interface ServiceTransactionGroupsResponse { transactionGroups: Array<{ transactionType: string; @@ -61,6 +54,7 @@ export async function getServiceTransactionGroups({ end, documentType, rollupInterval, + useDurationSummary, }: { environment: string; kuery: string; @@ -72,8 +66,12 @@ export async function getServiceTransactionGroups({ end: number; documentType: ApmTransactionDocumentType; rollupInterval: RollupInterval; + useDurationSummary: boolean; }): Promise { - const field = getDurationFieldForTransactions(documentType); + const field = getDurationFieldForTransactions( + documentType, + useDurationSummary + ); const response = await apmEventClient.search( 'get_service_transaction_groups', diff --git a/x-pack/plugins/apm/server/routes/transactions/get_latency_charts/index.ts b/x-pack/plugins/apm/server/routes/transactions/get_latency_charts/index.ts index 1f91238c9152c..23aa8960e6f44 100644 --- a/x-pack/plugins/apm/server/routes/transactions/get_latency_charts/index.ts +++ b/x-pack/plugins/apm/server/routes/transactions/get_latency_charts/index.ts @@ -30,10 +30,6 @@ import { } from '../../../lib/helpers/latency_aggregation_type'; import { getDurationFieldForTransactions } from '../../../lib/helpers/transactions'; -export type LatencyChartsSearchResponse = Awaited< - ReturnType ->; - function searchLatency({ environment, kuery, @@ -49,6 +45,7 @@ function searchLatency({ documentType, rollupInterval, bucketSizeInSeconds, + useDurationSummary, }: { environment: string; kuery: string; @@ -64,6 +61,7 @@ function searchLatency({ documentType: ApmServiceTransactionDocumentType; rollupInterval: RollupInterval; bucketSizeInSeconds: number; + useDurationSummary?: boolean; }) { const { startWithOffset, endWithOffset } = getOffsetInMs({ start, @@ -71,8 +69,10 @@ function searchLatency({ offset, }); - const transactionDurationField = - getDurationFieldForTransactions(documentType); + const transactionDurationField = getDurationFieldForTransactions( + documentType, + useDurationSummary + ); const params = { apm: { @@ -130,6 +130,7 @@ export async function getLatencyTimeseries({ documentType, rollupInterval, bucketSizeInSeconds, + useDurationSummary, }: { environment: string; kuery: string; @@ -145,6 +146,7 @@ export async function getLatencyTimeseries({ documentType: ApmServiceTransactionDocumentType; rollupInterval: RollupInterval; bucketSizeInSeconds: number; + useDurationSummary?: boolean; // TODO: Make this field mandatory before migrating this for other charts like serverless latency chart, latency history chart }) { const response = await searchLatency({ environment, @@ -161,6 +163,7 @@ export async function getLatencyTimeseries({ documentType, rollupInterval, bucketSizeInSeconds, + useDurationSummary, }); if (!response.aggregations) { @@ -209,6 +212,7 @@ export async function getLatencyPeriods({ documentType, rollupInterval, bucketSizeInSeconds, + useDurationSummary, }: { serviceName: string; transactionType: string | undefined; @@ -223,6 +227,7 @@ export async function getLatencyPeriods({ documentType: ApmServiceTransactionDocumentType; rollupInterval: RollupInterval; bucketSizeInSeconds: number; + useDurationSummary?: boolean; }): Promise { const options = { serviceName, @@ -234,6 +239,7 @@ export async function getLatencyPeriods({ documentType, rollupInterval, bucketSizeInSeconds, + useDurationSummary, }; const currentPeriodPromise = getLatencyTimeseries({ diff --git a/x-pack/plugins/apm/server/routes/transactions/route.ts b/x-pack/plugins/apm/server/routes/transactions/route.ts index 2783d3465c87f..1eb6fa3c7ecd3 100644 --- a/x-pack/plugins/apm/server/routes/transactions/route.ts +++ b/x-pack/plugins/apm/server/routes/transactions/route.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { jsonRt, toNumberRt } from '@kbn/io-ts-utils'; +import { jsonRt, toBooleanRt, toNumberRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; import { offsetRt } from '../../../common/comparison_rt'; import { @@ -61,6 +61,7 @@ const transactionGroupsMainStatisticsRoute = createApmServerRoute({ kueryRt, rangeRt, t.type({ + useDurationSummary: toBooleanRt, transactionType: t.string, latencyAggregationType: latencyAggregationTypeRt, }), @@ -84,6 +85,7 @@ const transactionGroupsMainStatisticsRoute = createApmServerRoute({ end, documentType, rollupInterval, + useDurationSummary, }, } = params; @@ -98,6 +100,7 @@ const transactionGroupsMainStatisticsRoute = createApmServerRoute({ end, documentType, rollupInterval, + useDurationSummary, }); }, }); @@ -111,10 +114,16 @@ const transactionGroupsDetailedStatisticsRoute = createApmServerRoute({ environmentRt, kueryRt, rangeRt, - offsetRt, + t.intersection([ + offsetRt, + transactionDataSourceRt, + t.type({ + bucketSizeInSeconds: toNumberRt, + useDurationSummary: toBooleanRt, + }), + ]), t.type({ transactionNames: jsonRt.pipe(t.array(t.string)), - numBuckets: toNumberRt, transactionType: t.string, latencyAggregationType: latencyAggregationTypeRt, }), @@ -127,40 +136,37 @@ const transactionGroupsDetailedStatisticsRoute = createApmServerRoute({ resources ): Promise => { const apmEventClient = await getApmEventClient(resources); - const { params, config } = resources; + const { params } = resources; const { path: { serviceName }, query: { environment, kuery, - transactionNames, - latencyAggregationType, - numBuckets, - transactionType, start, end, offset, + documentType, + rollupInterval, + bucketSizeInSeconds, + useDurationSummary, + transactionNames, + transactionType, + latencyAggregationType, }, } = params; - const searchAggregatedTransactions = await getSearchTransactionsEvents({ - config, - apmEventClient, - kuery, - start, - end, - }); - return getServiceTransactionGroupDetailedStatisticsPeriods({ environment, kuery, apmEventClient, serviceName, transactionNames, - searchAggregatedTransactions, transactionType, - numBuckets, + documentType, + rollupInterval, + bucketSizeInSeconds, + useDurationSummary, latencyAggregationType, start, end, @@ -182,7 +188,7 @@ const transactionLatencyChartsRoute = createApmServerRoute({ latencyAggregationType: latencyAggregationTypeRt, bucketSizeInSeconds: toNumberRt, }), - t.partial({ transactionName: t.string }), + t.partial({ transactionName: t.string, useDurationSummary: toBooleanRt }), t.intersection([environmentRt, kueryRt, rangeRt, offsetRt]), serviceTransactionDataSourceRt, ]), @@ -205,6 +211,7 @@ const transactionLatencyChartsRoute = createApmServerRoute({ documentType, rollupInterval, bucketSizeInSeconds, + useDurationSummary, } = params.query; const options = { @@ -220,6 +227,7 @@ const transactionLatencyChartsRoute = createApmServerRoute({ documentType, rollupInterval, bucketSizeInSeconds, + useDurationSummary, }; return getLatencyPeriods({ diff --git a/x-pack/test/apm_api_integration/tests/error_rate/service_apis.spec.ts b/x-pack/test/apm_api_integration/tests/error_rate/service_apis.spec.ts index 472b5e6097d54..57807ec28c91b 100644 --- a/x-pack/test/apm_api_integration/tests/error_rate/service_apis.spec.ts +++ b/x-pack/test/apm_api_integration/tests/error_rate/service_apis.spec.ts @@ -89,6 +89,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { kuery: `processor.event : "${processorEvent}"`, transactionType: 'request', latencyAggregationType: 'avg' as LatencyAggregationType, + useDurationSummary: false, ...(processorEvent === ProcessorEvent.metric ? { documentType: ApmDocumentType.TransactionMetric, diff --git a/x-pack/test/apm_api_integration/tests/latency/service_apis.spec.ts b/x-pack/test/apm_api_integration/tests/latency/service_apis.spec.ts index 0bed9dc6e01a5..0b99d9c17b8b4 100644 --- a/x-pack/test/apm_api_integration/tests/latency/service_apis.spec.ts +++ b/x-pack/test/apm_api_integration/tests/latency/service_apis.spec.ts @@ -26,9 +26,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { async function getLatencyValues({ processorEvent, latencyAggregationType = LatencyAggregationType.avg, + useDurationSummary = false, }: { processorEvent: 'transaction' | 'metric'; latencyAggregationType?: LatencyAggregationType; + useDurationSummary?: boolean; }) { const commonQuery = { start: new Date(start).toISOString(), @@ -91,6 +93,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { kuery: `processor.event : "${processorEvent}"`, transactionType: 'request', latencyAggregationType: 'avg' as LatencyAggregationType, + useDurationSummary, ...(processorEvent === ProcessorEvent.metric ? { documentType: ApmDocumentType.TransactionMetric, @@ -189,7 +192,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { before(async () => { [latencyTransactionValues, latencyMetricValues] = await Promise.all([ getLatencyValues({ processorEvent: 'transaction' }), - getLatencyValues({ processorEvent: 'metric' }), + getLatencyValues({ processorEvent: 'metric', useDurationSummary: true }), ]); }); diff --git a/x-pack/test/apm_api_integration/tests/throughput/service_apis.spec.ts b/x-pack/test/apm_api_integration/tests/throughput/service_apis.spec.ts index c1746975da533..c79a8e7eba04f 100644 --- a/x-pack/test/apm_api_integration/tests/throughput/service_apis.spec.ts +++ b/x-pack/test/apm_api_integration/tests/throughput/service_apis.spec.ts @@ -85,6 +85,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { kuery: `processor.event : "${processorEvent}"`, transactionType: 'request', latencyAggregationType: 'avg' as LatencyAggregationType, + useDurationSummary: false, ...(processorEvent === ProcessorEvent.metric ? { documentType: ApmDocumentType.TransactionMetric, diff --git a/x-pack/test/apm_api_integration/tests/time_range_metadata/generate_data.ts b/x-pack/test/apm_api_integration/tests/time_range_metadata/generate_data.ts new file mode 100644 index 0000000000000..e1f8720a103b0 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/time_range_metadata/generate_data.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { apm, timerange } from '@kbn/apm-synthtrace-client'; +import moment, { Moment } from 'moment'; +import { Transform, Readable } from 'stream'; +import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; + +export function getTransactionEvents(start: Moment, end: Moment) { + const serviceName = 'synth-go'; + const transactionName = 'GET /api/product/list'; + const GO_PROD_RATE = 75; + const GO_PROD_ERROR_RATE = 25; + + const serviceGoProdInstance = apm + .service({ name: serviceName, environment: 'production', agentName: 'go' }) + .instance('instance-a'); + + return [ + timerange(start, end) + .interval('1m') + .rate(GO_PROD_RATE) + .generator((timestamp) => + serviceGoProdInstance + .transaction({ transactionName }) + .timestamp(timestamp) + .duration(1000) + .success() + ), + + timerange(start, end) + .interval('1m') + .rate(GO_PROD_ERROR_RATE) + .generator((timestamp) => + serviceGoProdInstance + .transaction({ transactionName }) + .duration(1000) + .timestamp(timestamp) + .failure() + ), + ]; +} + +export function subtractDateDifference(start: Moment, end: Moment) { + const diff = moment(end).diff(moment(start)) + 1000; + const previousStart = moment(start).subtract(diff, 'milliseconds').format(); + const previousEnd = moment(end).subtract(diff, 'milliseconds').format(); + return { previousStart: moment(previousStart), previousEnd: moment(previousEnd) }; +} + +function deleteSummaryFieldTransform() { + return new Transform({ + objectMode: true, + transform(chunk: any, encoding, callback) { + delete chunk?.transaction?.duration?.summary; + callback(null, chunk); + }, + }); +} + +export function overwriteSynthPipelineWithSummaryFieldDeleteTransform({ + synthtraceEsClient, +}: { + synthtraceEsClient: ApmSynthtraceEsClient; +}) { + return (base: Readable) => { + const defaultPipeline = synthtraceEsClient.getDefaultPipeline()(base); + return (defaultPipeline as unknown as NodeJS.ReadableStream).pipe( + deleteSummaryFieldTransform() + ); + }; +} diff --git a/x-pack/test/apm_api_integration/tests/time_range_metadata/time_range_metadata.spec.ts b/x-pack/test/apm_api_integration/tests/time_range_metadata/time_range_metadata.spec.ts index fa1230bb8dcf0..4ee6221a0e7a9 100644 --- a/x-pack/test/apm_api_integration/tests/time_range_metadata/time_range_metadata.spec.ts +++ b/x-pack/test/apm_api_integration/tests/time_range_metadata/time_range_metadata.spec.ts @@ -12,6 +12,11 @@ import moment, { Moment } from 'moment'; import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type'; import { RollupInterval } from '@kbn/apm-plugin/common/rollup'; import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + getTransactionEvents, + subtractDateDifference, + overwriteSynthPipelineWithSummaryFieldDeleteTransform, +} from './generate_data'; export default function ApiTest({ getService }: FtrProviderContext) { const registry = getService('registry'); @@ -65,11 +70,69 @@ export default function ApiTest({ getService }: FtrProviderContext) { documentType: ApmDocumentType.TransactionEvent, rollupInterval: RollupInterval.None, hasDocs: true, + hasDurationSummaryField: false, }, ]); }); }); + registry.when( + 'Time range metadata when generating summary data', + { config: 'basic', archives: [] }, + () => { + describe('data loaded with and without summary field', () => { + const localStart = moment('2023-04-28T00:00:00.000Z'); + const localEnd = moment('2023-04-28T06:00:00.000Z'); + before(async () => { + const regularData = getTransactionEvents(localStart, localEnd); + await synthtraceEsClient.index([...regularData]); + const { previousStart, previousEnd } = subtractDateDifference(localStart, localEnd); + const previousDataWithoutSummaryField = getTransactionEvents(previousStart, previousEnd); + synthtraceEsClient.pipeline( + overwriteSynthPipelineWithSummaryFieldDeleteTransform({ + synthtraceEsClient, + }) + ); + await synthtraceEsClient.index([...previousDataWithoutSummaryField]); + }); + after(() => { + synthtraceEsClient.clean(); + synthtraceEsClient.pipeline(synthtraceEsClient.getDefaultPipeline()); + }); + describe('Values for hasDurationSummaryField for transaction metrics', () => { + it('returns true when summary field is available both inside and outside the range', async () => { + const response = await getTimeRangeMedata({ + start: moment(localStart).add(3, 'hours'), + end: moment(localEnd), + }); + + expect( + response.sources.filter( + (source) => + source.documentType === ApmDocumentType.TransactionMetric && + source.hasDurationSummaryField === true + ).length + ).to.eql(3); + }); + it('returns false when summary field is available inside but not outside the range', async () => { + const response = await getTimeRangeMedata({ + start: moment(localStart).subtract(30, 'minutes'), + end: moment(localEnd), + }); + + expect( + response.sources.filter( + (source) => + source.documentType === ApmDocumentType.TransactionMetric && + source.hasDurationSummaryField === false + ).length + ).to.eql(3); + }); + }); + }); + } + ); + registry.when( 'Time range metadata when generating data', { config: 'basic', archives: [] }, @@ -103,36 +166,43 @@ export default function ApiTest({ getService }: FtrProviderContext) { documentType: ApmDocumentType.ServiceTransactionMetric, rollupInterval: RollupInterval.TenMinutes, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.ServiceTransactionMetric, rollupInterval: RollupInterval.OneMinute, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.ServiceTransactionMetric, rollupInterval: RollupInterval.SixtyMinutes, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionEvent, rollupInterval: RollupInterval.None, hasDocs: true, + hasDurationSummaryField: false, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.TenMinutes, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.OneMinute, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.SixtyMinutes, hasDocs: true, + hasDurationSummaryField: true, }, ]); }); @@ -151,16 +221,19 @@ export default function ApiTest({ getService }: FtrProviderContext) { documentType: ApmDocumentType.ServiceTransactionMetric, rollupInterval: RollupInterval.OneMinute, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionEvent, rollupInterval: RollupInterval.None, hasDocs: true, + hasDurationSummaryField: false, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.OneMinute, hasDocs: true, + hasDurationSummaryField: true, }, ]); }); @@ -179,21 +252,25 @@ export default function ApiTest({ getService }: FtrProviderContext) { documentType: ApmDocumentType.TransactionEvent, rollupInterval: RollupInterval.None, hasDocs: true, + hasDurationSummaryField: false, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.TenMinutes, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.OneMinute, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.SixtyMinutes, hasDocs: true, + hasDurationSummaryField: true, }, ]); }); @@ -211,36 +288,43 @@ export default function ApiTest({ getService }: FtrProviderContext) { documentType: ApmDocumentType.ServiceTransactionMetric, rollupInterval: RollupInterval.TenMinutes, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.ServiceTransactionMetric, rollupInterval: RollupInterval.OneMinute, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.ServiceTransactionMetric, rollupInterval: RollupInterval.SixtyMinutes, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionEvent, rollupInterval: RollupInterval.None, hasDocs: true, + hasDurationSummaryField: false, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.TenMinutes, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.OneMinute, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.SixtyMinutes, hasDocs: true, + hasDurationSummaryField: true, }, ]); }); @@ -258,36 +342,43 @@ export default function ApiTest({ getService }: FtrProviderContext) { documentType: ApmDocumentType.ServiceTransactionMetric, rollupInterval: RollupInterval.TenMinutes, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.ServiceTransactionMetric, rollupInterval: RollupInterval.OneMinute, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.ServiceTransactionMetric, rollupInterval: RollupInterval.SixtyMinutes, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionEvent, rollupInterval: RollupInterval.None, hasDocs: true, + hasDurationSummaryField: false, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.TenMinutes, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.OneMinute, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.SixtyMinutes, hasDocs: true, + hasDurationSummaryField: true, }, ]); }); @@ -383,16 +474,19 @@ export default function ApiTest({ getService }: FtrProviderContext) { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.TenMinutes, hasDocs: true, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.OneMinute, hasDocs: false, + hasDurationSummaryField: false, }, { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.SixtyMinutes, hasDocs: true, + hasDurationSummaryField: true, }, ]); }); diff --git a/x-pack/test/apm_api_integration/tests/transactions/latency.spec.ts b/x-pack/test/apm_api_integration/tests/transactions/latency.spec.ts index 8fbd41fc86436..621fcb03c1648 100644 --- a/x-pack/test/apm_api_integration/tests/transactions/latency.spec.ts +++ b/x-pack/test/apm_api_integration/tests/transactions/latency.spec.ts @@ -49,6 +49,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.OneMinute, bucketSizeInSeconds: 60, + useDurationSummary: false, ...overrides?.query, }, }, @@ -238,6 +239,42 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(latencyChartReturn.currentPeriod.latencyTimeseries.length).to.be.eql(15); }); }); + + describe('should return same data with duration summary true and false', () => { + let responseWithSummaryDurationTrue: Awaited>; + let responseWithSummaryDurationFalse: Awaited>; + + before(async () => { + [responseWithSummaryDurationTrue, responseWithSummaryDurationFalse] = await Promise.all([ + fetchLatencyCharts({ + query: { + environment: 'production', + useDurationSummary: true, + }, + }), + fetchLatencyCharts({ + query: { + environment: 'production', + useDurationSummary: false, + }, + }), + ]); + }); + + it('returns average duration and timeseries', async () => { + const latencyChartWithSummaryDurationTrueReturn = + responseWithSummaryDurationTrue.body as LatencyChartReturnType; + const latencyChartWithSummaryDurationFalseReturn = + responseWithSummaryDurationFalse.body as LatencyChartReturnType; + [ + latencyChartWithSummaryDurationTrueReturn, + latencyChartWithSummaryDurationFalseReturn, + ].forEach((response) => { + expect(response.currentPeriod.overallAvgDuration).to.be(GO_PROD_DURATION * 1000); + expect(response.currentPeriod.latencyTimeseries.length).to.be.eql(15); + }); + }); + }); } ); } diff --git a/x-pack/test/apm_api_integration/tests/transactions/transactions_groups_detailed_statistics.spec.ts b/x-pack/test/apm_api_integration/tests/transactions/transactions_groups_detailed_statistics.spec.ts index 2d25cf5ff04c3..ddefb6b669cef 100644 --- a/x-pack/test/apm_api_integration/tests/transactions/transactions_groups_detailed_statistics.spec.ts +++ b/x-pack/test/apm_api_integration/tests/transactions/transactions_groups_detailed_statistics.spec.ts @@ -9,8 +9,9 @@ import expect from '@kbn/expect'; import { first, isEmpty, last, meanBy } from 'lodash'; import moment from 'moment'; import { LatencyAggregationType } from '@kbn/apm-plugin/common/latency_aggregation_types'; -import { asPercent } from '@kbn/apm-plugin/common/utils/formatters'; import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; +import { RollupInterval } from '@kbn/apm-plugin/common/rollup'; +import { ApmDocumentType, ApmTransactionDocumentType } from '@kbn/apm-plugin/common/document_type'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { roundNumber } from '../../utils'; @@ -24,7 +25,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const serviceName = 'synth-go'; const start = new Date('2021-01-01T00:00:00.000Z').getTime(); - const end = new Date('2021-01-01T00:15:59.999Z').getTime(); + const end = new Date('2021-01-01T01:00:00.000Z').getTime() - 1; const transactionNames = ['GET /api/product/list']; async function callApi(overrides?: { @@ -40,7 +41,10 @@ export default function ApiTest({ getService }: FtrProviderContext) { offset?: string; transactionNames?: string; latencyAggregationType?: LatencyAggregationType; - numBuckets?: number; + bucketSizeInSeconds?: number; + documentType?: ApmTransactionDocumentType; + rollupInterval?: RollupInterval; + useDurationSummary?: boolean; }; }) { const response = await apmApiClient.readUser({ @@ -50,8 +54,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { query: { start: new Date(start).toISOString(), end: new Date(end).toISOString(), - numBuckets: 20, + bucketSizeInSeconds: 60, + documentType: ApmDocumentType.TransactionMetric, + rollupInterval: RollupInterval.OneMinute, transactionType: 'request', + useDurationSummary: false, latencyAggregationType: 'avg' as LatencyAggregationType, transactionNames: JSON.stringify(transactionNames), environment: 'ENVIRONMENT_ALL', @@ -114,11 +121,42 @@ export default function ApiTest({ getService }: FtrProviderContext) { describe('without comparisons', () => { let transactionsStatistics: TransactionsGroupsDetailedStatistics; - let metricsStatistics: TransactionsGroupsDetailedStatistics; + let metricsStatisticsOneMinute: TransactionsGroupsDetailedStatistics; + let metricsStatisticsTenMinute: TransactionsGroupsDetailedStatistics; + let metricsStatisticsSixtyMinute: TransactionsGroupsDetailedStatistics; before(async () => { - [metricsStatistics, transactionsStatistics] = await Promise.all([ - callApi({ query: { kuery: 'processor.event : "metric"' } }), - callApi({ query: { kuery: 'processor.event : "transaction"' } }), + [ + metricsStatisticsOneMinute, + metricsStatisticsTenMinute, + metricsStatisticsSixtyMinute, + transactionsStatistics, + ] = await Promise.all([ + callApi({ + query: { + documentType: ApmDocumentType.TransactionMetric, + rollupInterval: RollupInterval.OneMinute, + }, + }), + callApi({ + query: { + documentType: ApmDocumentType.TransactionMetric, + rollupInterval: RollupInterval.TenMinutes, + bucketSizeInSeconds: 600, + }, + }), + callApi({ + query: { + documentType: ApmDocumentType.TransactionMetric, + rollupInterval: RollupInterval.SixtyMinutes, + bucketSizeInSeconds: 3600, + }, + }), + callApi({ + query: { + documentType: ApmDocumentType.TransactionEvent, + rollupInterval: RollupInterval.None, + }, + }), ]); }); @@ -127,41 +165,81 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('returns some metrics data', () => { - expect(isEmpty(metricsStatistics.currentPeriod)).to.be.equal(false); + expect(isEmpty(metricsStatisticsOneMinute.currentPeriod)).to.be.equal(false); + expect(isEmpty(metricsStatisticsTenMinute.currentPeriod)).to.be.equal(false); + expect(isEmpty(metricsStatisticsSixtyMinute.currentPeriod)).to.be.equal(false); }); it('has same latency mean value for metrics and transactions data', () => { const transactionsCurrentPeriod = transactionsStatistics.currentPeriod[transactionNames[0]]; - const metricsCurrentPeriod = metricsStatistics.currentPeriod[transactionNames[0]]; + const metricsOneMinPeriod = metricsStatisticsOneMinute.currentPeriod[transactionNames[0]]; + const metricsTenMinPeriod = metricsStatisticsTenMinute.currentPeriod[transactionNames[0]]; + const metricsSixtyMinPeriod = + metricsStatisticsSixtyMinute.currentPeriod[transactionNames[0]]; const transactionsLatencyMean = meanBy(transactionsCurrentPeriod.latency, 'y'); - const metricsLatencyMean = meanBy(metricsCurrentPeriod.latency, 'y'); - [transactionsLatencyMean, metricsLatencyMean].forEach((value) => - expect(value).to.be.equal(1000000) - ); + const metricsOneMinLatencyMean = meanBy(metricsOneMinPeriod.latency, 'y'); + const metricsTenMinLatencyMean = meanBy(metricsTenMinPeriod.latency, 'y'); + const metricsSixtyMinLatencyMean = meanBy(metricsSixtyMinPeriod.latency, 'y'); + [ + transactionsLatencyMean, + metricsOneMinLatencyMean, + metricsTenMinLatencyMean, + metricsSixtyMinLatencyMean, + ].forEach((value) => expect(value).to.be.equal(1000000)); }); it('has same error rate mean value for metrics and transactions data', () => { const transactionsCurrentPeriod = transactionsStatistics.currentPeriod[transactionNames[0]]; - const metricsCurrentPeriod = metricsStatistics.currentPeriod[transactionNames[0]]; + const metricsOneMinPeriod = metricsStatisticsOneMinute.currentPeriod[transactionNames[0]]; + const metricsTenMinPeriod = metricsStatisticsTenMinute.currentPeriod[transactionNames[0]]; + const metricsSixtyMinPeriod = + metricsStatisticsSixtyMinute.currentPeriod[transactionNames[0]]; const transactionsErrorRateMean = meanBy(transactionsCurrentPeriod.errorRate, 'y'); - const metricsErrorRateMean = meanBy(metricsCurrentPeriod.errorRate, 'y'); - [transactionsErrorRateMean, metricsErrorRateMean].forEach((value) => - expect(asPercent(value, 1)).to.be.equal(`${GO_PROD_ERROR_RATE}%`) - ); + + const metricsOneMinErrorRateMean = meanBy(metricsOneMinPeriod.errorRate, 'y'); + const metricsTenMinErrorRateMean = meanBy(metricsTenMinPeriod.errorRate, 'y'); + const metricsSixtyMinErrorRateMean = meanBy(metricsSixtyMinPeriod.errorRate, 'y'); + + [ + transactionsErrorRateMean, + metricsOneMinErrorRateMean, + metricsTenMinErrorRateMean, + metricsSixtyMinErrorRateMean, + ].forEach((value) => expect(value).to.be.equal(GO_PROD_ERROR_RATE / 100)); }); it('has same throughput mean value for metrics and transactions data', () => { const transactionsCurrentPeriod = transactionsStatistics.currentPeriod[transactionNames[0]]; - const metricsCurrentPeriod = metricsStatistics.currentPeriod[transactionNames[0]]; + const metricsOneMinPeriod = metricsStatisticsOneMinute.currentPeriod[transactionNames[0]]; + const metricsTenMinPeriod = metricsStatisticsTenMinute.currentPeriod[transactionNames[0]]; + const metricsSixtyMinPeriod = + metricsStatisticsSixtyMinute.currentPeriod[transactionNames[0]]; + const transactionsThroughputMean = roundNumber( meanBy(transactionsCurrentPeriod.throughput, 'y') ); - const metricsThroughputMean = roundNumber(meanBy(metricsCurrentPeriod.throughput, 'y')); - [transactionsThroughputMean, metricsThroughputMean].forEach((value) => + const metricsOneMinThroughputMean = roundNumber( + meanBy(metricsOneMinPeriod.throughput, 'y') + ); + const metricsTenMinThroughputMean = roundNumber( + meanBy(metricsTenMinPeriod.throughput, 'y') + ); + const metricsSixtyMinThroughputMean = roundNumber( + meanBy(metricsSixtyMinPeriod.throughput, 'y') + ); + + expect(metricsTenMinThroughputMean).to.be.equal( + roundNumber(10 * (GO_PROD_RATE + GO_PROD_ERROR_RATE)) + ); + expect(metricsSixtyMinThroughputMean).to.be.equal( + roundNumber(60 * (GO_PROD_RATE + GO_PROD_ERROR_RATE)) + ); + + [transactionsThroughputMean, metricsOneMinThroughputMean].forEach((value) => expect(value).to.be.equal(roundNumber(GO_PROD_RATE + GO_PROD_ERROR_RATE)) ); }); @@ -169,11 +247,17 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('has same impact value for metrics and transactions data', () => { const transactionsCurrentPeriod = transactionsStatistics.currentPeriod[transactionNames[0]]; - const metricsCurrentPeriod = metricsStatistics.currentPeriod[transactionNames[0]]; + const metricsOneMinPeriod = metricsStatisticsOneMinute.currentPeriod[transactionNames[0]]; + const metricsTenMinPeriod = metricsStatisticsTenMinute.currentPeriod[transactionNames[0]]; + const metricsSixtyMinPeriod = + metricsStatisticsSixtyMinute.currentPeriod[transactionNames[0]]; - const transactionsImpact = transactionsCurrentPeriod.impact; - const metricsImpact = metricsCurrentPeriod.impact; - [transactionsImpact, metricsImpact].forEach((value) => expect(value).to.be.equal(100)); + [ + transactionsCurrentPeriod.impact, + metricsOneMinPeriod.impact, + metricsTenMinPeriod.impact, + metricsSixtyMinPeriod.impact, + ].forEach((value) => expect(value).to.be.equal(100)); }); }); diff --git a/x-pack/test/apm_api_integration/tests/transactions/transactions_groups_main_statistics.spec.ts b/x-pack/test/apm_api_integration/tests/transactions/transactions_groups_main_statistics.spec.ts index f3c969f3eefbd..f2b554ea07688 100644 --- a/x-pack/test/apm_api_integration/tests/transactions/transactions_groups_main_statistics.spec.ts +++ b/x-pack/test/apm_api_integration/tests/transactions/transactions_groups_main_statistics.spec.ts @@ -6,163 +6,170 @@ */ import expect from '@kbn/expect'; -import { pick, sum } from 'lodash'; -import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; +import { sum } from 'lodash'; import { LatencyAggregationType } from '@kbn/apm-plugin/common/latency_aggregation_types'; -import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type'; +import { ApmDocumentType, ApmTransactionDocumentType } from '@kbn/apm-plugin/common/document_type'; import { RollupInterval } from '@kbn/apm-plugin/common/rollup'; +import { apm, timerange } from '@kbn/apm-synthtrace-client'; import { FtrProviderContext } from '../../common/ftr_provider_context'; -import archives from '../../common/fixtures/es_archiver/archives_metadata'; - -type TransactionsGroupsPrimaryStatistics = - APIReturnType<'GET /internal/apm/services/{serviceName}/transactions/groups/main_statistics'>; export default function ApiTest({ getService }: FtrProviderContext) { const registry = getService('registry'); const apmApiClient = getService('apmApiClient'); - - const archiveName = 'apm_8.0.0'; - const { start, end } = archives[archiveName]; + const synthtraceEsClient = getService('synthtraceEsClient'); + + const serviceName = 'synth-go'; + const start = new Date('2021-01-01T00:00:00.000Z').getTime(); + const end = new Date('2021-01-01T01:00:00.000Z').getTime() - 1; + + async function callApi(overrides?: { + path?: { + serviceName?: string; + }; + query?: { + start?: string; + end?: string; + transactionType?: string; + environment?: string; + kuery?: string; + latencyAggregationType?: LatencyAggregationType; + documentType?: ApmTransactionDocumentType; + rollupInterval?: RollupInterval; + useDurationSummary?: boolean; + }; + }) { + const response = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/services/{serviceName}/transactions/groups/main_statistics', + params: { + path: { serviceName, ...overrides?.path }, + query: { + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + latencyAggregationType: 'avg' as LatencyAggregationType, + transactionType: 'request', + environment: 'ENVIRONMENT_ALL', + useDurationSummary: false, + kuery: '', + documentType: ApmDocumentType.TransactionMetric, + rollupInterval: RollupInterval.OneMinute, + ...overrides?.query, + }, + }, + }); + expect(response.status).to.be(200); + return response.body; + } registry.when( 'Transaction groups main statistics when data is not loaded', { config: 'basic', archives: [] }, () => { it('handles the empty state', async () => { - const response = await apmApiClient.readUser({ - endpoint: 'GET /internal/apm/services/{serviceName}/transactions/groups/main_statistics', - params: { - path: { serviceName: 'opbeans-java' }, - query: { - start, - end, - latencyAggregationType: LatencyAggregationType.avg, - transactionType: 'request', - environment: 'ENVIRONMENT_ALL', - kuery: '', - documentType: ApmDocumentType.TransactionMetric, - rollupInterval: RollupInterval.OneMinute, - }, - }, - }); + const transactionsGroupsPrimaryStatistics = await callApi(); - expect(response.status).to.be(200); - const transctionsGroupsPrimaryStatistics = - response.body as TransactionsGroupsPrimaryStatistics; - expect(transctionsGroupsPrimaryStatistics.transactionGroups).to.empty(); - expect(transctionsGroupsPrimaryStatistics.maxTransactionGroupsExceeded).to.be(false); + expect(transactionsGroupsPrimaryStatistics.transactionGroups).to.empty(); + expect(transactionsGroupsPrimaryStatistics.maxTransactionGroupsExceeded).to.be(false); }); } ); - registry.when( - 'Transaction groups main statistics when data is loaded', - { config: 'basic', archives: [archiveName] }, - () => { + registry.when('when data is loaded', { config: 'basic', archives: [] }, () => { + describe('Transaction groups main statistics', () => { + const GO_PROD_RATE = 75; + const GO_PROD_ERROR_RATE = 25; + const transactions = [ + { + name: 'GET /api/product/list', + duration: 10, + }, + { + name: 'GET /api/product/list2', + duration: 100, + }, + { + name: 'GET /api/product/list3', + duration: 1000, + }, + ]; + before(async () => { + const serviceGoProdInstance = apm + .service({ name: serviceName, environment: 'production', agentName: 'go' }) + .instance('instance-a'); + + await synthtraceEsClient.index([ + timerange(start, end) + .interval('1m') + .rate(GO_PROD_RATE) + .generator((timestamp) => { + return transactions.map(({ name, duration }) => { + return serviceGoProdInstance + .transaction({ transactionName: name }) + .timestamp(timestamp) + .duration(duration) + .success(); + }); + }), + timerange(start, end) + .interval('1m') + .rate(GO_PROD_ERROR_RATE) + .generator((timestamp) => { + return transactions.map(({ name, duration }) => { + return serviceGoProdInstance + .transaction({ transactionName: name }) + .timestamp(timestamp) + .duration(duration) + .failure(); + }); + }), + ]); + }); + after(() => synthtraceEsClient.clean()); + it('returns the correct data', async () => { - const response = await apmApiClient.readUser({ - endpoint: 'GET /internal/apm/services/{serviceName}/transactions/groups/main_statistics', - params: { - path: { serviceName: 'opbeans-java' }, - query: { - start, - end, - latencyAggregationType: LatencyAggregationType.avg, - transactionType: 'request', - environment: 'ENVIRONMENT_ALL', - kuery: '', - documentType: ApmDocumentType.TransactionMetric, - rollupInterval: RollupInterval.OneMinute, - }, + const transactionsGroupsPrimaryStatistics = await callApi(); + const transactionsGroupsPrimaryStatisticsWithDurationSummaryTrue = await callApi({ + query: { + useDurationSummary: true, }, }); - expect(response.status).to.be(200); - - const transctionsGroupsPrimaryStatistics = - response.body as TransactionsGroupsPrimaryStatistics; - - expectSnapshot( - transctionsGroupsPrimaryStatistics.transactionGroups.map((group: any) => group.name) - ).toMatchInline(` - Array [ - "DispatcherServlet#doGet", - "ResourceHttpRequestHandler", - "APIRestController#topProducts", - "APIRestController#customer", - "APIRestController#order", - "APIRestController#stats", - "APIRestController#customerWhoBought", - "APIRestController#product", - "APIRestController#orders", - "APIRestController#products", - "APIRestController#customers", - "DispatcherServlet#doPost", - ] - `); - - const impacts = transctionsGroupsPrimaryStatistics.transactionGroups.map( - (group: any) => group.impact - ); - expectSnapshot(impacts).toMatchInline(` - Array [ - 98.4867713293593, - 0.0910992862692518, - 0.217172932411727, - 0.197499651612207, - 0.117088451625813, - 0.203168003440319, - 0.0956764857936742, - 0.353287132108131, - 0.043688393280619, - 0.0754467823979389, - 0.115710953190738, - 0.00339059851027124, - ] - `); - - expect(Math.round(sum(impacts))).to.eql(100); - - const firstItem = transctionsGroupsPrimaryStatistics.transactionGroups[0]; - - expectSnapshot(pick(firstItem, 'name', 'latency', 'throughput', 'errorRate', 'impact')) - .toMatchInline(` - Object { - "errorRate": 0.08, - "impact": 98.4867713293593, - "latency": 1816019.48, - "name": "DispatcherServlet#doGet", - "throughput": 1.66666666666667, - } - `); + [ + transactionsGroupsPrimaryStatistics, + transactionsGroupsPrimaryStatisticsWithDurationSummaryTrue, + ].forEach((statistics) => { + expect(statistics.transactionGroups.length).to.be(3); + expect(statistics.maxTransactionGroupsExceeded).to.be(false); + expect(statistics.transactionGroups.map(({ name }) => name)).to.eql( + transactions.map(({ name }) => name) + ); + + const impacts = statistics.transactionGroups.map((group: any) => group.impact); + + expect(Math.round(sum(impacts))).to.eql(100); + + const firstItem = statistics.transactionGroups[0]; + + expect(firstItem).to.eql({ + name: 'GET /api/product/list', + latency: 10000, + throughput: 100.00002777778549, + errorRate: 0.25, + impact: 0.9009009009009009, + transactionType: 'request', + }); + }); }); it('returns the correct data for latency aggregation 99th percentile', async () => { - const response = await apmApiClient.readUser({ - endpoint: 'GET /internal/apm/services/{serviceName}/transactions/groups/main_statistics', - params: { - path: { serviceName: 'opbeans-java' }, - query: { - start, - end, - latencyAggregationType: LatencyAggregationType.p99, - transactionType: 'request', - environment: 'ENVIRONMENT_ALL', - kuery: '', - documentType: ApmDocumentType.TransactionMetric, - rollupInterval: RollupInterval.OneMinute, - }, + const transactionsGroupsPrimaryStatistics = await callApi({ + query: { + latencyAggregationType: LatencyAggregationType.p99, }, }); - expect(response.status).to.be(200); - - const transctionsGroupsPrimaryStatistics = - response.body as TransactionsGroupsPrimaryStatistics; - - const firstItem = transctionsGroupsPrimaryStatistics.transactionGroups[0]; - expectSnapshot(firstItem.latency).toMatchInline(`66846719`); + const firstItem = transactionsGroupsPrimaryStatistics.transactionGroups[0]; + expect(firstItem.latency).to.be(10000); }); - } - ); + }); + }); }