From 5dc0ea4b0e2a77c09078b3cff563be30ea784bc9 Mon Sep 17 00:00:00 2001 From: Fionera Date: Fri, 2 Jun 2023 12:57:09 +0200 Subject: [PATCH] feat: allow definition of interface groups Change-Id: I7cea56ce9ee02c8b4f99dd5ea901c5f0dd5878f4 --- deploy/dashboards/NetMeta_Overview.cue | 83 ++++++++++++++++----- deploy/single-node/config.cue | 3 + deploy/single-node/k8s/clickhouse/files.cue | 13 +++- deploy/single-node/schema/functions.cue | 7 ++ 4 files changed, 86 insertions(+), 20 deletions(-) diff --git a/deploy/dashboards/NetMeta_Overview.cue b/deploy/dashboards/NetMeta_Overview.cue index e0ce5cd..31aedcb 100644 --- a/deploy/dashboards/NetMeta_Overview.cue +++ b/deploy/dashboards/NetMeta_Overview.cue @@ -103,6 +103,33 @@ _trafficStatisticQueries: { GROUP BY time, SamplerAddress, OutIf, FlowDirectionStr ORDER BY time """# + + "Traffic per Ingress Interface Group": + #""" + SELECT + $__timeInterval(TimeReceived) as time, + InterfaceGroup(SamplerAddress, InIf) AS InIfGroup, + (sum(Bytes * SamplingRate) * 8 / $__interval_s) AS BitsPerSecond, + if(isIncomingFlow(SamplerAddress, SrcAddr, DstAddr, SrcAS, DstAS, FlowDirection), 'in', 'out') AS FlowDirectionStr + FROM flows_raw + WHERE $__timeFilter(TimeReceived) + \#(_filtersWithHost) + GROUP BY time, SamplerAddress, InIfGroup, FlowDirectionStr + ORDER BY time + """# + "Traffic per Egress Interface Group": + #""" + SELECT + $__timeInterval(TimeReceived) as time, + InterfaceGroup(SamplerAddress, OutIf) AS OutIfGroup, + (sum(Bytes * SamplingRate) * 8 / $__interval_s) AS BitsPerSecond, + if(isIncomingFlow(SamplerAddress, SrcAddr, DstAddr, SrcAS, DstAS, FlowDirection), 'in', 'out') AS FlowDirectionStr + FROM flows_raw + WHERE $__timeFilter(TimeReceived) + \#(_filtersWithHost) + GROUP BY time, SamplerAddress, OutIfGroup, FlowDirectionStr + ORDER BY time + """# } _trafficStatistics: [{ @@ -191,6 +218,28 @@ _trafficStatistics: [{ targets: [{ rawSql: _trafficStatisticQueries[title] }] +}, { + title: "Traffic per Ingress Interface Group" + description: "The traffic based on the ingress interface group." + type: "timeseries" + gridPos: {h: 12, w: 12, x: 0, y: 55} + fieldConfig: overrides: [_negativeYOut] + fieldConfig: defaults: unit: "bps" + fieldConfig: defaults: displayName: "${__field.labels.InIfName} ${__field.labels.FlowDirectionStr}" + targets: [{ + rawSql: _trafficStatisticQueries[title] + }] +}, { + title: "Traffic per Egress Interface Group" + description: "The traffic based on the egress interface group." + type: "timeseries" + gridPos: {h: 12, w: 12, x: 12, y: 55} + fieldConfig: overrides: [_negativeYOut] + fieldConfig: defaults: unit: "bps" + fieldConfig: defaults: displayName: "${__field.labels.OutIfName} ${__field.labels.FlowDirectionStr}" + targets: [{ + rawSql: _trafficStatisticQueries[title] + }] }] _originASStatisticQueries: { @@ -279,7 +328,7 @@ _originASStatistics: [{ }, { title: "Top 100 Source ASN" type: "table" - gridPos: {h: 12, w: 12, x: 0, y: 56} + gridPos: {h: 12, w: 12, x: 0, y: 68} fieldConfig: defaults: color: mode: "fixed" fieldConfig: overrides: [{ matcher: { @@ -312,7 +361,7 @@ _originASStatistics: [{ }, { title: "Top 100 Destination ASN" type: "table" - gridPos: {h: 12, w: 12, x: 12, y: 56} + gridPos: {h: 12, w: 12, x: 12, y: 68} fieldConfig: defaults: color: mode: "fixed" fieldConfig: defaults: unit: "bytes" fieldConfig: overrides: [{ @@ -347,7 +396,7 @@ _originASStatistics: [{ }, { title: "Top 10 Source ASN" type: "timeseries" - gridPos: {h: 12, w: 12, x: 0, y: 80} + gridPos: {h: 12, w: 12, x: 0, y: 92} fieldConfig: defaults: unit: "bps" targets: [{ rawSql: _originASStatisticQueries[title] @@ -355,7 +404,7 @@ _originASStatistics: [{ }, { title: "Top 10 Destination ASN" type: "timeseries" - gridPos: {h: 12, w: 12, x: 12, y: 80} + gridPos: {h: 12, w: 12, x: 12, y: 92} fieldConfig: defaults: unit: "bps" targets: [{ rawSql: _originASStatisticQueries[title] @@ -455,7 +504,7 @@ _trafficDetails: [{ }, { title: "TCP Flags" type: "timeseries" - gridPos: {h: 12, w: 12, x: 0, y: 94} + gridPos: {h: 12, w: 12, x: 0, y: 106} fieldConfig: defaults: unit: "bps" fieldConfig: defaults: displayName: "${__field.labels.TCPFlagsName} ${__field.labels.FlowDirectionStr}" targets: [{ @@ -464,7 +513,7 @@ _trafficDetails: [{ }, { title: "VLAN ID" type: "timeseries" - gridPos: {h: 12, w: 12, x: 12, y: 94} + gridPos: {h: 12, w: 12, x: 12, y: 106} fieldConfig: defaults: unit: "bps" fieldConfig: defaults: displayName: "${__field.labels.Vlan} ${__field.labels.FlowDirectionStr}" targets: [{ @@ -473,7 +522,7 @@ _trafficDetails: [{ }, { title: "Top 10 Source Ports" type: "timeseries" - gridPos: {h: 12, w: 12, x: 0, y: 106} + gridPos: {h: 12, w: 12, x: 0, y: 118} fieldConfig: defaults: unit: "bps" fieldConfig: defaults: displayName: "${__field.labels.ProtoName}, ${__field.labels.SrcPortStr} ${__field.labels.FlowDirectionStr}" targets: [{ @@ -482,7 +531,7 @@ _trafficDetails: [{ }, { title: "Top 10 Destination Ports" type: "timeseries" - gridPos: {h: 12, w: 12, x: 12, y: 106} + gridPos: {h: 12, w: 12, x: 12, y: 118} fieldConfig: defaults: unit: "bps" fieldConfig: defaults: displayName: "${__field.labels.ProtoName}, ${__field.labels.DstPortStr} ${__field.labels.FlowDirectionStr}" targets: [{ @@ -576,7 +625,7 @@ _topHosts: [{ }, { title: "Top 10 Source IPs" type: "timeseries" - gridPos: {h: 12, w: 12, x: 0, y: 108} + gridPos: {h: 12, w: 12, x: 0, y: 120} fieldConfig: defaults: unit: "bps" fieldConfig: defaults: displayName: "${__field.labels.Src}, ${__field.labels.FlowDirectionStr}" targets: [{ @@ -585,7 +634,7 @@ _topHosts: [{ }, { title: "Top 10 Destination IPs" type: "timeseries" - gridPos: {h: 12, w: 12, x: 12, y: 108} + gridPos: {h: 12, w: 12, x: 12, y: 120} fieldConfig: defaults: unit: "bps" fieldConfig: defaults: displayName: "${__field.labels.Dst}, ${__field.labels.FlowDirectionStr}" targets: [{ @@ -594,7 +643,7 @@ _topHosts: [{ }, { title: "Top 100 Source IPs" type: "table" - gridPos: {h: 12, w: 12, x: 0, y: 120} + gridPos: {h: 12, w: 12, x: 0, y: 132} fieldConfig: defaults: unit: "bytes" fieldConfig: overrides: [{ matcher: { @@ -612,7 +661,7 @@ _topHosts: [{ }, { title: "Top 100 Destination IPs" type: "table" - gridPos: {h: 12, w: 12, x: 12, y: 120} + gridPos: {h: 12, w: 12, x: 12, y: 132} fieldConfig: defaults: unit: "bytes" fieldConfig: overrides: [{ matcher: { @@ -654,7 +703,7 @@ _topFlows: [{ }, { title: "Top 100 Flows" type: "table" - gridPos: {h: 12, w: 24, x: 0, y: 122} + gridPos: {h: 12, w: 24, x: 0, y: 134} fieldConfig: overrides: [{ matcher: { id: "byName" @@ -775,7 +824,7 @@ _heatmaps: [{ }, { title: "IP TTL" type: "heatmap" - gridPos: {h: 12, w: 12, x: 0, y: 135} + gridPos: {h: 12, w: 12, x: 0, y: 147} options: calculation: xBuckets: mode: "count" options: calculation: xBuckets: value: 150 options: calculation: yBuckets: mode: "size" @@ -788,7 +837,7 @@ _heatmaps: [{ }, { title: "Packet Size (Bytes)" type: "heatmap" - gridPos: {h: 12, w: 12, x: 12, y: 135} + gridPos: {h: 12, w: 12, x: 12, y: 147} options: calculation: xBuckets: mode: "count" options: calculation: xBuckets: value: 150 options: calculation: yBuckets: mode: "size" @@ -801,7 +850,7 @@ _heatmaps: [{ }, { title: "Source Port" type: "heatmap" - gridPos: {h: 12, w: 12, x: 0, y: 147} + gridPos: {h: 12, w: 12, x: 0, y: 159} options: calculation: xBuckets: mode: "count" options: calculation: xBuckets: value: 150 options: calculation: yBuckets: mode: "size" @@ -814,7 +863,7 @@ _heatmaps: [{ }, { title: "Destination Port" type: "heatmap" - gridPos: {h: 12, w: 12, x: 12, y: 147} + gridPos: {h: 12, w: 12, x: 12, y: 159} options: calculation: xBuckets: mode: "count" options: calculation: xBuckets: value: 150 options: calculation: yBuckets: mode: "size" diff --git a/deploy/single-node/config.cue b/deploy/single-node/config.cue index 22f6312..42550d7 100644 --- a/deploy/single-node/config.cue +++ b/deploy/single-node/config.cue @@ -123,6 +123,9 @@ _deviceAddressTest: { // Human-readable interface description to show in the frontend description: string + + // Human-readable interface group to show in the frontend + group: string | *description } vlan: [ID=string]: { diff --git a/deploy/single-node/k8s/clickhouse/files.cue b/deploy/single-node/k8s/clickhouse/files.cue index daaf8f9..f5b25d3 100644 --- a/deploy/single-node/k8s/clickhouse/files.cue +++ b/deploy/single-node/k8s/clickhouse/files.cue @@ -40,12 +40,12 @@ ClickHouseInstallation: netmeta: spec: configuration: files: { // Dictionary for user-defined interface name lookup _files: InterfaceNames: { data: strings.Join([ for s in #Config.sampler for i in s.interface { - strings.Join([s.device, "\(i.id)", i.description], "\t") + strings.Join([s.device, "\(i.id)", i.description, i.group], "\t") }], "\n") cfg: { layout: complex_key_hashed: null - structure: { + structure: [{ key: [{ attribute: { name: "Device" @@ -57,12 +57,19 @@ _files: InterfaceNames: { type: "UInt32" } }] + }, { attribute: { name: "Description" type: "String" null_value: null } - } + }, { + attribute: { + name: "Group" + type: "String" + null_value: null + } + }] } } diff --git a/deploy/single-node/schema/functions.cue b/deploy/single-node/schema/functions.cue index c3cbf86..3db502d 100644 --- a/deploy/single-node/schema/functions.cue +++ b/deploy/single-node/schema/functions.cue @@ -36,6 +36,13 @@ function: InterfaceToString: { """# } +function: InterfaceGroup: { + arguments: ["Sampler", "Interface"] + query: #""" + dictGetString('InterfaceNames', 'Group', (IPv6NumToString(Sampler), Interface)) + """# +} + function: isIncomingFlow: { arguments: ["SamplerAddress", "SrcAddr", "DstAddr", "SrcAS", "DstAS", "FlowDirection"]