From afbfefbe29aa0634485e32be9201d41e3ff7b73b Mon Sep 17 00:00:00 2001 From: Julien Pinsonneau Date: Thu, 17 Oct 2024 11:46:27 +0200 Subject: [PATCH 1/2] add update-config target --- Makefile | 4 + config/sample-config.yaml | 259 ++++++++++++++++++++++++-------------- 2 files changed, 167 insertions(+), 96 deletions(-) diff --git a/Makefile b/Makefile index 84706ab25..ee7969be7 100644 --- a/Makefile +++ b/Makefile @@ -144,6 +144,10 @@ endif generate-doc: ## Generate documentation of the flows JSON format cd web && npm run generate-doc +.PHONY: update-config +update-config: ## Update sample config from operator repo + ./scripts/update-config.sh + ##@ Develop frontend .PHONY: install-frontend diff --git a/config/sample-config.yaml b/config/sample-config.yaml index 80870395c..f6ad09876 100644 --- a/config/sample-config.yaml +++ b/config/sample-config.yaml @@ -23,20 +23,20 @@ loki: prometheus: timeout: 30s metrics: - - enabled: false - name: netobserv_node_egress_bytes_total - type: counter - valueField: Bytes - labels: - - SrcK8S_HostName - - DstK8S_HostName - - enabled: true - name: netobserv_node_ingress_bytes_total - type: counter - valueField: Bytes - labels: - - SrcK8S_HostName - - DstK8S_HostName + - enabled: false + name: netobserv_node_egress_bytes_total + type: counter + valueField: Bytes + labels: + - SrcK8S_HostName + - DstK8S_HostName + - enabled: true + name: netobserv_node_ingress_bytes_total + type: counter + valueField: Bytes + labels: + - SrcK8S_HostName + - DstK8S_HostName frontend: recordTypes: - flowLog @@ -82,76 +82,76 @@ frontend: merge: false # The following configuration is taken from Network Observability Operator # see https://github.com/netobserv/network-observability-operator/blob/main/controllers/consoleplugin/config/static-frontend-config.yaml - panels: - # Protocol - - Proto_Bytes - - sum_Proto_Bytes - - avg_Proto_Bytes - - Proto_Packets - - sum_Proto_Packets - - avg_Proto_Packets - - Proto_Flows - # DSCP - - Dscp_Bytes - - Dscp_Packets - - min_Dscp_TimeFlowRttNs - - max_Dscp_TimeFlowRttNs - - avg_Dscp_TimeFlowRttNs - - p90_Dscp_TimeFlowRttNs - - p99_Dscp_TimeFlowRttNs - - min_Dscp_DnsLatencyMs - - max_Dscp_DnsLatencyMs - - avg_Dscp_DnsLatencyMs - - p90_Dscp_DnsLatencyMs - - p99_Dscp_DnsLatencyMs - - Dscp_Flows - # Port numbers - - SrcPort_Bytes - - sum_SrcPort_Bytes - - SrcPort_Packets - - sum_SrcPort_Packets - - SrcPort_TimeFlowRttNs - - min_SrcPort_TimeFlowRttNs - - max_SrcPort_TimeFlowRttNs - - avg_SrcPort_TimeFlowRttNs - - p90_SrcPort_TimeFlowRttNs - - p99_SrcPort_TimeFlowRttNs - - DstPort_Bytes - - sum_DstPort_Bytes - - DstPort_Packets - - sum_DstPort_Packets - - min_DstPort_TimeFlowRttNs - - max_DstPort_TimeFlowRttNs - - avg_DstPort_TimeFlowRttNs - - p90_DstPort_TimeFlowRttNs - - p99_DstPort_TimeFlowRttNs - # Node Directions - - FlowDirection_Bytes - - FlowDirection_Packets - - FlowDirection_Flows - # TODO: implement a way to manage plurals for interfaces - # Interfaces Directions - #- IfDirections_Bytes - #- IfDirections_Packets - #- IfDirections_Flows - # Interfaces names - #- Interfaces_Bytes - #- sum_Interfaces_Bytes - #- avg_Interfaces_Bytes - #- Interfaces_Packets - #- sum_Interfaces_Packets - #- avg_Interfaces_Packets - #- Interfaces_Flows - # DNS capture errors - - DnsErrno_Flows - # Connection tracking flow count - - numFlowLogs_Flows - # Bytes / Packets rates on current scope - - Bytes - - Packets - # flow on current scope - - Flows - - DnsFlows + panels: + # Protocol + - Proto_Bytes + - sum_Proto_Bytes + - avg_Proto_Bytes + - Proto_Packets + - sum_Proto_Packets + - avg_Proto_Packets + - Proto_Flows + # DSCP + - Dscp_Bytes + - Dscp_Packets + - min_Dscp_TimeFlowRttNs + - max_Dscp_TimeFlowRttNs + - avg_Dscp_TimeFlowRttNs + - p90_Dscp_TimeFlowRttNs + - p99_Dscp_TimeFlowRttNs + - min_Dscp_DnsLatencyMs + - max_Dscp_DnsLatencyMs + - avg_Dscp_DnsLatencyMs + - p90_Dscp_DnsLatencyMs + - p99_Dscp_DnsLatencyMs + - Dscp_Flows + # Port numbers + - SrcPort_Bytes + - sum_SrcPort_Bytes + - SrcPort_Packets + - sum_SrcPort_Packets + - SrcPort_TimeFlowRttNs + - min_SrcPort_TimeFlowRttNs + - max_SrcPort_TimeFlowRttNs + - avg_SrcPort_TimeFlowRttNs + - p90_SrcPort_TimeFlowRttNs + - p99_SrcPort_TimeFlowRttNs + - DstPort_Bytes + - sum_DstPort_Bytes + - DstPort_Packets + - sum_DstPort_Packets + - min_DstPort_TimeFlowRttNs + - max_DstPort_TimeFlowRttNs + - avg_DstPort_TimeFlowRttNs + - p90_DstPort_TimeFlowRttNs + - p99_DstPort_TimeFlowRttNs + # Node Directions + - FlowDirection_Bytes + - FlowDirection_Packets + - FlowDirection_Flows + # TODO: implement a way to manage plurals for interfaces + # Interfaces Directions + #- IfDirections_Bytes + #- IfDirections_Packets + #- IfDirections_Flows + # Interfaces names + #- Interfaces_Bytes + #- sum_Interfaces_Bytes + #- avg_Interfaces_Bytes + #- Interfaces_Packets + #- sum_Interfaces_Packets + #- avg_Interfaces_Packets + #- Interfaces_Flows + # DNS capture errors + - DnsErrno_Flows + # Connection tracking flow count + - numFlowLogs_Flows + # Bytes / Packets rates on current scope + - Bytes + - Packets + # flow on current scope + - Flows + - DnsFlows columns: - id: StartTime name: Start Time @@ -305,6 +305,14 @@ frontend: default: false width: 15 feature: zones + - id: SrcSubnetLabel + group: Source + name: Subnet Label + field: SrcSubnetLabel + filter: src_subnet_label + default: false + width: 10 + feature: subnetLabels - id: DstK8S_Name group: Destination name: Name @@ -425,6 +433,14 @@ frontend: default: false width: 15 feature: zones + - id: DstSubnetLabel + group: Destination + name: Subnet Label + field: DstSubnetLabel + filter: dst_subnet_label + default: false + width: 10 + feature: subnetLabels - id: K8S_Name name: Names calculated: getSrcOrDstValue(SrcK8S_Name,DstK8S_Name) @@ -517,6 +533,9 @@ frontend: name: Type tooltip: The type of the ICMP message field: IcmpType + fields: + - Proto + - IcmpType filter: icmp_type default: false width: 10 @@ -525,9 +544,20 @@ frontend: name: Code tooltip: The code of the ICMP message field: IcmpCode + fields: + - Proto + - IcmpType + - IcmpCode filter: icmp_code default: false width: 10 + - id: TCPFlags + name: TCP Flags + tooltip: Logical OR combination of unique TCP flags comprised in the flow, as per RFC-9293, with additional custom values. + field: Flags + filter: tcp_flags + default: false + width: 10 - id: FlowDirection name: Node Direction tooltip: The interpreted direction of the flow observed at the Node observation point. @@ -635,6 +665,14 @@ frontend: default: true width: 5 feature: flowRTT + - id: NetworkEvents + name: Network Events + tooltip: Network events flow monitor + field: NetworkEvents + filter: network_events + default: true + width: 15 + feature: networkEvents filters: - id: cluster_name name: Cluster @@ -754,6 +792,16 @@ frontend: component: autocomplete category: destination hint: Specify a single zone. + - id: src_subnet_label + name: Subnet Label + component: autocomplete + category: source + hint: Specify a subnet label, or an empty string to get unmatched sources. + - id: dst_subnet_label + name: Subnet Label + component: autocomplete + category: destination + hint: Specify a subnet label, or an empty string to get unmatched destinations. - id: src_resource name: Resource component: autocomplete @@ -912,6 +960,24 @@ frontend: name: ICMP code component: number hint: Specify an ICMP code value as integer number. + - id: tcp_flags + name: TCP flags + component: autocomplete + hint: Specify a TCP flags value as integer number. + examples: |- + Logical OR combination of unique TCP flags comprised in the flow, as per RFC-9293, with additional custom flags + users can specify either numeric value or string representation of the flags as follows : + - FIN or 1, + - SYN or 2, + - RST or 4, + - PSH or 8, + - ACK or 16, + - URG or 32, + - ECE or 64, + - CWR or 128, + - SYN_ACK or 256, + - FIN_ACK or 512, + - RST_ACK or 1024, - id: node_direction name: Node Direction component: autocomplete @@ -982,6 +1048,10 @@ frontend: name: Flow RTT component: number hint: Specify a TCP smoothed Round Trip Time in nanoseconds. + - id: network_events + name: Network events flow monitoring + component: text + hint: Specify a single network event. fields: - name: TimeFlowStartMs type: number @@ -998,18 +1068,15 @@ frontend: - name: SrcK8S_Type type: string description: Kind of the source Kubernetes object, such as Pod, Service or Node. - lokiLabel: true - name: SrcK8S_OwnerName type: string description: Name of the source owner, such as Deployment name, StatefulSet name, etc. - lokiLabel: true - name: SrcK8S_OwnerType type: string description: Kind of the source owner, such as Deployment, StatefulSet, etc. - name: SrcK8S_Namespace type: string description: Source namespace - lokiLabel: true - name: SrcAddr type: string description: Source IP address (ipv4 or ipv6) @@ -1028,25 +1095,24 @@ frontend: - name: SrcK8S_Zone type: string description: Source availability zone - lokiLabel: true + - name: SrcSubnetLabel + type: string + description: Source subnet label - name: DstK8S_Name type: string description: Name of the destination Kubernetes object, such as Pod name, Service name or Node name. - name: DstK8S_Type type: string description: Kind of the destination Kubernetes object, such as Pod, Service or Node. - lokiLabel: true - name: DstK8S_OwnerName type: string description: Name of the destination owner, such as Deployment name, StatefulSet name, etc. - lokiLabel: true - name: DstK8S_OwnerType type: string description: Kind of the destination owner, such as Deployment, StatefulSet, etc. - name: DstK8S_Namespace type: string description: Destination namespace - lokiLabel: true - name: DstAddr type: string description: Destination IP address (ipv4 or ipv6) @@ -1065,7 +1131,9 @@ frontend: - name: DstK8S_Zone type: string description: Destination availability zone - lokiLabel: true + - name: DstSubnetLabel + type: string + description: Destination subnet label - name: K8S_FlowLayer type: string description: "Flow layer: 'app' or 'infra'" @@ -1084,7 +1152,6 @@ frontend: - name: Duplicate type: boolean description: Indicates if this flow was also captured from another interface on the same host - lokiLabel: true - name: FlowDirection type: number description: | @@ -1092,7 +1159,6 @@ frontend: - 0: Ingress (incoming traffic, from the node observation point) + - 1: Egress (outgoing traffic, from the node observation point) + - 2: Inner (with the same source and destination node) - lokiLabel: true - name: IfDirections type: number description: | @@ -1150,14 +1216,15 @@ frontend: - name: TimeFlowRttNs type: number description: TCP Smoothed Round Trip Time (SRTT), in nanoseconds + - name: NetworkEvents + type: string + description: Network events flow monitoring - name: K8S_ClusterName type: string description: Cluster name or identifier - lokiLabel: true - name: _RecordType type: string description: "Type of record: 'flowLog' for regular flow logs, or 'newConnection', 'heartbeat', 'endConnection' for conversation tracking" - lokiLabel: true - name: _HashId type: string description: In conversation tracking, the conversation identifier From c57e3ec5f612c7a86eba5cd8dab152fd09d2f3df Mon Sep 17 00:00:00 2001 From: Julien Pinsonneau Date: Fri, 18 Oct 2024 09:51:09 +0200 Subject: [PATCH 2/2] update config and manage drop columns --- config/sample-config.yaml | 47 +++++++++++++++++-- .../components/drawer/record/record-field.tsx | 13 ++++- .../components/drawer/record/record-panel.tsx | 10 +++- web/src/utils/columns.ts | 5 ++ 4 files changed, 68 insertions(+), 7 deletions(-) diff --git a/config/sample-config.yaml b/config/sample-config.yaml index f6ad09876..a5432e153 100644 --- a/config/sample-config.yaml +++ b/config/sample-config.yaml @@ -45,12 +45,12 @@ frontend: # - endConnection features: # eBPF agent features - # - pktDrop - # - dnsTracking - # - flowRTT + - pktDrop + - dnsTracking + - flowRTT # processor features - # - multiCluster - # - zones + - multiCluster + - zones portNaming: enable: true portNames: @@ -588,6 +588,7 @@ frontend: - id: Bytes name: Bytes tooltip: The total aggregated number of bytes. + field: Bytes fields: - Bytes - PktDropBytes @@ -596,6 +597,7 @@ frontend: - id: Packets name: Packets tooltip: The total aggregated number of packets. + field: Packets fields: - Packets - PktDropPackets @@ -621,6 +623,41 @@ frontend: calculated: substract(column.CollectionTime,TimeFlowEndMs) default: false width: 5 + - id: PktDropBytes + name: Dropped Bytes + tooltip: The total aggregated number of bytes dropped. + field: PktDropBytes + default: false + width: 5 + feature: pktDrop + - id: PktDropPackets + name: Dropped Packets + tooltip: The total aggregated number of packets dropped. + field: PktDropPackets + default: false + width: 5 + feature: pktDrop + - id: PktDropLatestState + name: Drop State + tooltip: TCP state on last dropped packet. + field: PktDropLatestState + default: false + width: 10 + feature: pktDrop + - id: PktDropLatestDropCause + name: Drop Cause + tooltip: TCP state on last dropped packet. + field: PktDropLatestDropCause + default: false + width: 10 + feature: pktDrop + - id: PktDropLatestFlags + name: Drop Flags + tooltip: TCP flags on last dropped packet. + field: PktDropLatestFlags + default: false + width: 10 + feature: pktDrop - id: DNSId group: DNS name: DNS Id diff --git a/web/src/components/drawer/record/record-field.tsx b/web/src/components/drawer/record/record-field.tsx index aadb3b473..ac5a00516 100644 --- a/web/src/components/drawer/record/record-field.tsx +++ b/web/src/components/drawer/record/record-field.tsx @@ -471,7 +471,8 @@ export const RecordField: React.FC = ({ } return singleContainer(child); } - case ColumnsId.tcpflags: { + case ColumnsId.tcpflags: + case ColumnsId.dropflags: { let child = emptyText(); if (typeof value === 'number' && !isNaN(value)) { const flags = decomposeTCPFlagsBitfield(value); @@ -619,6 +620,16 @@ export const RecordField: React.FC = ({ simpleTextWithTooltip(detailed ? `${String(value)} ${c.name.toLowerCase()} ${t('sent')}` : String(value)) ); } + case ColumnsId.dropbytes: + case ColumnsId.droppackets: + const droppedText = t('dropped'); + const droppedCount = String(value); + return singleContainer( + simpleTextWithTooltip( + detailed ? `${droppedCount} ${c.name.toLowerCase()} ${droppedText}` : droppedCount, + isDark ? '#C9190B' : '#A30000' + ) + ); case ColumnsId.dnsid: { return singleContainer( typeof value === 'number' && !isNaN(value) ? simpleTextWithTooltip(String(value)) : emptyDnsErrorText() diff --git a/web/src/components/drawer/record/record-panel.tsx b/web/src/components/drawer/record/record-panel.tsx index d10fdf303..622dc8f2f 100644 --- a/web/src/components/drawer/record/record-panel.tsx +++ b/web/src/components/drawer/record/record-panel.tsx @@ -79,7 +79,15 @@ export const RecordPanel: React.FC = ({ // hide empty columns const getVisibleColumns = React.useCallback(() => { - const forbiddenColumns = [ColumnsId.ifdirs, ColumnsId.interfaces]; + const forbiddenColumns = [ + ColumnsId.ifdirs, + ColumnsId.interfaces, + ColumnsId.dropbytes, + ColumnsId.droppackets, + ColumnsId.dropstate, + ColumnsId.dropcause, + ColumnsId.dropflags + ]; return columns.filter((c: Column) => { const value = c.value(record); return !forbiddenColumns.includes(c.id) && value !== null && value !== '' && !Number.isNaN(value); diff --git a/web/src/utils/columns.ts b/web/src/utils/columns.ts index 2e568a1ed..4600a632c 100644 --- a/web/src/utils/columns.ts +++ b/web/src/utils/columns.ts @@ -45,6 +45,11 @@ export enum ColumnsId { tcpflags = 'TCPFlags', bytes = 'Bytes', packets = 'Packets', + dropbytes = 'PktDropBytes', + droppackets = 'PktDropPackets', + dropstate = 'PktDropLatestState', + dropcause = 'PktDropLatestDropCause', + dropflags = 'PktDropLatestFlags', owner = 'K8S_OwnerName', srcowner = 'SrcK8S_OwnerName', dstowner = 'DstK8S_OwnerName',