From 965b00ac07fff9ff59ae4879d2d4a5682a209217 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Fri, 25 Oct 2024 23:18:57 +0000 Subject: [PATCH 1/3] Add px/differential_flamegraph script Signed-off-by: Dom Del Nano (cherry picked from commit 12881672c021e47a9090df3335c29f6048e99df3) --- .../differential_flamegraph/differential.pxl | 50 +++++++++++++++ .../px/differential_flamegraph/manifest.yaml | 5 ++ .../px/differential_flamegraph/vis.json | 64 +++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 src/pxl_scripts/px/differential_flamegraph/differential.pxl create mode 100644 src/pxl_scripts/px/differential_flamegraph/manifest.yaml create mode 100644 src/pxl_scripts/px/differential_flamegraph/vis.json diff --git a/src/pxl_scripts/px/differential_flamegraph/differential.pxl b/src/pxl_scripts/px/differential_flamegraph/differential.pxl new file mode 100644 index 00000000000..c215ced5593 --- /dev/null +++ b/src/pxl_scripts/px/differential_flamegraph/differential.pxl @@ -0,0 +1,50 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import px + +def export_flame_graph(start_time: str, namespace: str, pod: str, baseline_pod: str): + stack_traces = px.DataFrame(table='stack_traces.beta', start_time=start_time) + stack_traces.namespace = stack_traces.ctx['namespace'] + stack_traces = stack_traces[px.contains(stack_traces.namespace, namespace)] + stack_traces.node = px.Node(px._exec_hostname()) + stack_traces.pod = stack_traces.ctx['pod'] + stack_traces.keep_row = px.select(px.contains(stack_traces.pod, baseline_pod), True, False) + stack_traces.keep_row = px.select(stack_traces.keep_row or px.contains(stack_traces.pod, pod), True, False) + stack_traces = stack_traces[stack_traces.keep_row] + + stack_traces = stack_traces.groupby(['node', 'namespace', 'pod', 'stack_trace_id']).agg( + stack_trace=('stack_trace', px.any), + count=('count', px.sum) + ) + + pod1 = stack_traces[px.contains(stack_traces.pod, baseline_pod)] + pod1 = pod1.drop(['node', 'namespace', 'pod', 'stack_trace_id']) + + pod2 = stack_traces[px.contains(stack_traces.pod, pod)] + pod2 = pod2.drop(['node', 'namespace', 'pod', 'stack_trace_id']) + + merged = pod1.merge( + pod2, + how='right', + left_on='stack_trace', + right_on='stack_trace' + suffixes=['_1', '_2'], + ) + # Pixie's stack trace format permits spaces, but Brendan Gregg's scripts do not + merged.stack_trace_2 = px.replace(' ', merged.stack_trace_2, '') + merged.delta = merged.count_2 - merged.count_1 + return merged[['stack_trace_2', 'count_1', 'count_2', 'delta']] diff --git a/src/pxl_scripts/px/differential_flamegraph/manifest.yaml b/src/pxl_scripts/px/differential_flamegraph/manifest.yaml new file mode 100644 index 00000000000..8c8c41c7bbe --- /dev/null +++ b/src/pxl_scripts/px/differential_flamegraph/manifest.yaml @@ -0,0 +1,5 @@ +--- +short: Differential Flame Graph +long: > + This live view shows a differential CPU flame graph. This is helpful in identifying what code + paths have changed between deployments, different container instances, etc. diff --git a/src/pxl_scripts/px/differential_flamegraph/vis.json b/src/pxl_scripts/px/differential_flamegraph/vis.json new file mode 100644 index 00000000000..221a83899bd --- /dev/null +++ b/src/pxl_scripts/px/differential_flamegraph/vis.json @@ -0,0 +1,64 @@ +{ + "variables": [ + { + "name": "start_time", + "type": "PX_STRING", + "description": "The relative start time of the window. Current time is assumed to be now", + "defaultValue": "-5m" + }, + { + "name": "namespace", + "type": "PX_NAMESPACE", + "description": "The namespace to filter on." + }, + { + "name": "pod", + "type": "PX_POD", + "description": "The pod that will have its flamegraph analyzed compared to the baseline_pod" + }, + { + "name": "baseline_pod", + "type": "PX_POD", + "description": "The pod to serve as the baseline. The resulting flamegraph will show the difference from this pod's profile." + } + ], + "globalFuncs": [], + "widgets": [ + { + "name": "Flamegraph", + "position": { + "x": 0, + "y": 0, + "w": 12, + "h": 6 + }, + "func": { + "name": "export_flame_graph", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "namespace", + "variable": "namespace" + }, + { + "name": "pod", + "variable": "pod" + }, + { + "name": "baseline_pod", + "variable": "baseline_pod" + } + ] + }, + "displaySpec": { + "@type": "types.px.dev/px.vispb.StackTraceFlameGraph", + "stacktraceColumn": "stack_trace_2", + "countColumn": "count_2", + "differenceColumn": "delta" + } + } + ] +} From bf7e68f5f2000a179fcf439e9ad7430df4df5f90 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Mon, 4 Nov 2024 16:22:25 +0000 Subject: [PATCH 2/3] Update differential_flamegraph script to make supporting negation easy once it's possible Signed-off-by: Dom Del Nano --- .../differential_flamegraph/differential.pxl | 65 +++++++++++++------ .../px/differential_flamegraph/vis.json | 7 +- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/src/pxl_scripts/px/differential_flamegraph/differential.pxl b/src/pxl_scripts/px/differential_flamegraph/differential.pxl index c215ced5593..6457040ff4d 100644 --- a/src/pxl_scripts/px/differential_flamegraph/differential.pxl +++ b/src/pxl_scripts/px/differential_flamegraph/differential.pxl @@ -16,14 +16,50 @@ import px -def export_flame_graph(start_time: str, namespace: str, pod: str, baseline_pod: str): +negate = False +# TODO(ddelnano): negation requires switching the type of join from right to left +# or returning a different DataFrame. This might not be possible with pxl's current +# functionality, but this should be implemented once it's possible. + +def merge_and_compute_delta(pod1, pod2, negate: bool): + + diff = pod1.merge( + pod2, + how='right', + left_on='stack_trace', + right_on='stack_trace' + suffixes=['_1', '_2'], + ) + # TODO(ddelnano): This needs to be switched with pod1 if the flamegraph should + # be negated. + percentage_agg = pod2.groupby(['pod']).agg( + count=('count', px.sum), + ) + diff.pod = px.select(negate, diff.pod_1, diff.pod_2) + diff.stack_trace = px.select(negate, diff.stack_trace_1, diff.stack_trace_2) + diff.stack_trace = px.replace(' ', diff.stack_trace, '') + diff.count = px.select(negate, diff.count_1, diff.count_2) + diff.delta = diff.count_2 - diff.count_1 + diff.delta = px.select(negate, px.negate(diff.delta), diff.delta) + + merged = diff.merge( + percentage_agg, + how='inner', + left_on='pod', + right_on='pod', + suffixes=['', '_x'] + ) + merged.percent = 100 * merged.count / merged.count_x + return merged + +def differential_flamegraph(start_time: str, namespace: str, pod: str, baseline_pod: str): stack_traces = px.DataFrame(table='stack_traces.beta', start_time=start_time) stack_traces.namespace = stack_traces.ctx['namespace'] - stack_traces = stack_traces[px.contains(stack_traces.namespace, namespace)] + stack_traces = stack_traces[stack_traces.namespace == namespace] stack_traces.node = px.Node(px._exec_hostname()) stack_traces.pod = stack_traces.ctx['pod'] - stack_traces.keep_row = px.select(px.contains(stack_traces.pod, baseline_pod), True, False) - stack_traces.keep_row = px.select(stack_traces.keep_row or px.contains(stack_traces.pod, pod), True, False) + stack_traces.keep_row = stack_traces.pod == baseline_pod + stack_traces.keep_row = px.select(stack_traces.keep_row or stack_traces.pod == pod, True, False) stack_traces = stack_traces[stack_traces.keep_row] stack_traces = stack_traces.groupby(['node', 'namespace', 'pod', 'stack_trace_id']).agg( @@ -31,20 +67,11 @@ def export_flame_graph(start_time: str, namespace: str, pod: str, baseline_pod: count=('count', px.sum) ) - pod1 = stack_traces[px.contains(stack_traces.pod, baseline_pod)] - pod1 = pod1.drop(['node', 'namespace', 'pod', 'stack_trace_id']) + pod1 = stack_traces[stack_traces.pod == baseline_pod] + pod1 = pod1.drop(['node', 'namespace', 'stack_trace_id']) - pod2 = stack_traces[px.contains(stack_traces.pod, pod)] - pod2 = pod2.drop(['node', 'namespace', 'pod', 'stack_trace_id']) + pod2 = stack_traces[stack_traces.pod == pod] + pod2 = pod2.drop(['node', 'namespace', 'stack_trace_id']) - merged = pod1.merge( - pod2, - how='right', - left_on='stack_trace', - right_on='stack_trace' - suffixes=['_1', '_2'], - ) - # Pixie's stack trace format permits spaces, but Brendan Gregg's scripts do not - merged.stack_trace_2 = px.replace(' ', merged.stack_trace_2, '') - merged.delta = merged.count_2 - merged.count_1 - return merged[['stack_trace_2', 'count_1', 'count_2', 'delta']] + merged = merge_and_compute_delta(pod1, pod2, negate) + return merged[['stack_trace', 'count', 'delta', 'percent']] diff --git a/src/pxl_scripts/px/differential_flamegraph/vis.json b/src/pxl_scripts/px/differential_flamegraph/vis.json index 221a83899bd..0b4d0eac8c9 100644 --- a/src/pxl_scripts/px/differential_flamegraph/vis.json +++ b/src/pxl_scripts/px/differential_flamegraph/vis.json @@ -33,7 +33,7 @@ "h": 6 }, "func": { - "name": "export_flame_graph", + "name": "differential_flamegraph", "args": [ { "name": "start_time", @@ -55,8 +55,9 @@ }, "displaySpec": { "@type": "types.px.dev/px.vispb.StackTraceFlameGraph", - "stacktraceColumn": "stack_trace_2", - "countColumn": "count_2", + "stacktraceColumn": "stack_trace", + "countColumn": "count", + "percentageColumn": "percent", "differenceColumn": "delta" } } From a54f66098bc25560770fb1575f91621900a44083 Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Tue, 12 Nov 2024 22:22:25 +0000 Subject: [PATCH 3/3] Ensure stacktrace has a base line scope to avoid intermitted rendering issues Signed-off-by: Dom Del Nano --- src/pxl_scripts/px/differential_flamegraph/differential.pxl | 2 +- src/pxl_scripts/px/differential_flamegraph/vis.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pxl_scripts/px/differential_flamegraph/differential.pxl b/src/pxl_scripts/px/differential_flamegraph/differential.pxl index 6457040ff4d..61e48e0da8e 100644 --- a/src/pxl_scripts/px/differential_flamegraph/differential.pxl +++ b/src/pxl_scripts/px/differential_flamegraph/differential.pxl @@ -74,4 +74,4 @@ def differential_flamegraph(start_time: str, namespace: str, pod: str, baseline_ pod2 = pod2.drop(['node', 'namespace', 'stack_trace_id']) merged = merge_and_compute_delta(pod1, pod2, negate) - return merged[['stack_trace', 'count', 'delta', 'percent']] + return merged[['stack_trace', 'count', 'delta', 'percent', 'pod']] diff --git a/src/pxl_scripts/px/differential_flamegraph/vis.json b/src/pxl_scripts/px/differential_flamegraph/vis.json index 0b4d0eac8c9..3603d09cca1 100644 --- a/src/pxl_scripts/px/differential_flamegraph/vis.json +++ b/src/pxl_scripts/px/differential_flamegraph/vis.json @@ -58,6 +58,7 @@ "stacktraceColumn": "stack_trace", "countColumn": "count", "percentageColumn": "percent", + "podColumn": "pod", "differenceColumn": "delta" } }