-
Notifications
You must be signed in to change notification settings - Fork 92
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: generate metrics documentation (#1351)
## Which problem is this PR solving? This PR implements the logic to parse out all metrics defined in the Refinery code base and generates a markdown table for them. - part 2 of #1152 ## Short description of the changes - add two new commands to the `convert` tool to generate the metricsMeta.yaml file and a metrics.md file
- Loading branch information
1 parent
209cf99
commit eb061f0
Showing
8 changed files
with
608 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| # Metrics Documentation | ||
| # Automatically generated on 2024-09-26 at 18:58:58 UTC | ||
|
|
||
| This document contains the description of various metrics used in the system. | ||
|
|
||
| | Name | Type | Unit | Description | | ||
| |------|------|------|-------------| | ||
| | collect_cache_buffer_overrun | Counter | Dimensionless | The number of times the trace overwritten in the circular buffer has not yet been sent | | ||
| | collect_cache_capacity | Gauge | Dimensionless | The number of traces that can be stored in the cache | | ||
| | collect_cache_entries | Histogram | Dimensionless | The number of traces currently stored in the cache | | ||
| | cuckoo_current_capacity | Gauge | Dimensionless | current capacity of the cuckoo filter | | ||
| | cuckoo_future_load_factor | Gauge | Percent | the fraction of slots occupied in the future cuckoo filter | | ||
| | cuckoo_current_load_factor | Gauge | Percent | the fraction of slots occupied in the current cuckoo filter | | ||
| | cache_recent_dropped_traces | Gauge | Dimensionless | the current size of the most recent dropped trace cache | | ||
| | collect_sent_reasons_cache_entries | Histogram | Dimensionless | Number of entries in the sent reasons cache | | ||
| | is_ready | Gauge | Dimensionless | Whether the system is ready to receive traffic | | ||
| | is_alive | Gauge | Dimensionless | Whether the system is alive and reporting in | | ||
| | redis_pubsub_published | Counter | Dimensionless | Number of messages published to Redis PubSub | | ||
| | redis_pubsub_received | Counter | Dimensionless | Number of messages received from Redis PubSub | | ||
| | local_pubsub_published | Counter | Dimensionless | The total number of messages sent via the local pubsub implementation | | ||
| | local_pubsub_received | Counter | Dimensionless | The total number of messages received via the local pubsub implementation | | ||
| | num_file_peers | Gauge | Dimensionless | Number of peers in the file peer list | | ||
| | num_peers | Gauge | Dimensionless | the active number of peers in the cluster | | ||
| | peer_hash | Gauge | Dimensionless | the hash of the current list of peers | | ||
| | peer_messages | Counter | Dimensionless | the number of messages received by the peers service | | ||
| | _num_dropped_by_drop_rule | Counter | Dimensionless | Number of traces dropped by the drop rule | | ||
| | _num_dropped | Counter | Dimensionless | Number of traces dropped by configured sampler | | ||
| | _num_kept | Counter | Dimensionless | Number of traces kept by configured sampler | | ||
| | _sample_rate | Histogram | Dimensionless | Sample rate for traces | | ||
| | enqueue_errors | Counter | Dimensionless | The number of errors encountered when enqueueing events | | ||
| | response_20x | Counter | Dimensionless | The number of successful responses from Honeycomb | | ||
| | response_errors | Counter | Dimensionless | The number of errors encountered when sending events to Honeycomb | | ||
| | queued_items | UpDown | Dimensionless | The number of events queued for transmission to Honeycomb | | ||
| | queue_time | Histogram | Microseconds | The time spent in the queue before being sent to Honeycomb | | ||
| | trace_duration_ms | Histogram | Milliseconds | time taken to process a trace from arrival to send | | ||
| | trace_span_count | Histogram | Dimensionless | number of spans in a trace | | ||
| | collector_incoming_queue | Histogram | Dimensionless | number of spans currently in the incoming queue | | ||
| | collector_peer_queue_length | Gauge | Dimensionless | number of spans in the peer queue | | ||
| | collector_incoming_queue_length | Gauge | Dimensionless | number of spans in the incoming queue | | ||
| | collector_peer_queue | Histogram | Dimensionless | number of spans currently in the peer queue | | ||
| | collector_cache_size | Gauge | Dimensionless | number of traces currently stored in the trace cache | | ||
| | memory_heap_allocation | Gauge | Bytes | current heap allocation | | ||
| | span_received | Counter | Dimensionless | number of spans received by the collector | | ||
| | span_processed | Counter | Dimensionless | number of spans processed by the collector | | ||
| | spans_waiting | UpDown | Dimensionless | number of spans waiting to be processed by the collector | | ||
| | trace_sent_cache_hit | Counter | Dimensionless | number of late spans received for traces that have already been sent | | ||
| | trace_accepted | Counter | Dimensionless | number of new traces received by the collector | | ||
| | trace_send_kept | Counter | Dimensionless | number of traces that has been kept | | ||
| | trace_send_dropped | Counter | Dimensionless | number of traces that has been dropped | | ||
| | trace_send_has_root | Counter | Dimensionless | number of kept traces that have a root span | | ||
| | trace_send_no_root | Counter | Dimensionless | number of kept traces that do not have a root span | | ||
| | trace_forwarded_on_peer_change | Gauge | Dimensionless | number of traces forwarded due to peer membership change | | ||
| | trace_redistribution_count | Gauge | Dimensionless | number of traces redistributed due to peer membership change | | ||
| | trace_send_on_shutdown | Counter | Dimensionless | number of traces sent during shutdown | | ||
| | trace_forwarded_on_shutdown | Counter | Dimensionless | number of traces forwarded during shutdown | | ||
| | trace_send_got_root | Counter | Dimensionless | number of traces that are ready for decision due to root span arrival | | ||
| | trace_send_expired | Counter | Dimensionless | number of traces that are ready for decision due to TraceTimeout or SendDelay | | ||
| | trace_send_span_limit | Counter | Dimensionless | number of traces that are ready for decision due to span limit | | ||
| | trace_send_ejected_full | Counter | Dimensionless | number of traces that are ready for decision due to cache capacity overrun | | ||
| | trace_send_ejected_memsize | Counter | Dimensionless | number of traces that are ready for decision due to memory overrun | | ||
| | trace_send_late_span | Counter | Dimensionless | number of spans that are sent due to late span arrival | | ||
| | dropped_from_stress | Counter | Dimensionless | number of traces dropped due to stress relief | | ||
| | cluster_stress_level | Gauge | Dimensionless | The overall stress level of the cluster | | ||
| | individual_stress_level | Gauge | Dimensionless | The stress level of the individual node | | ||
| | stress_level | Gauge | Dimensionless | The stress level that's being used to determine whether to activate stress relief | | ||
| | stress_relief_activated | Gauge | Dimensionless | Whether stress relief is currently activated | | ||
| | _router_proxied | Counter | Dimensionless | the number of events proxied to another refinery | | ||
| | _router_event | Counter | Dimensionless | the number of events received | | ||
| | config_hash | Gauge | Dimensionless | The hash of the current configuration | | ||
| | rule_config_hash | Gauge | Dimensionless | The hash of the current rules configuration | | ||
| | queue_length | Gauge | Dimensionless | number of events waiting to be sent to destination | | ||
| | queue_overflow | Counter | Dimensionless | number of events dropped due to queue overflow | | ||
| | send_errors | Counter | Dimensionless | number of errors encountered while sending events to destination | | ||
| | send_retries | Counter | Dimensionless | number of times a batch of events was retried | | ||
| | batches_sent | Counter | Dimensionless | number of batches of events sent to destination | | ||
| | messages_sent | Counter | Dimensionless | number of messages sent to destination | | ||
| | response_decode_errors | Counter | Dimensionless | number of errors encountered while decoding responses from destination | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,157 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "go/ast" | ||
| "go/types" | ||
| "os" | ||
| "slices" | ||
| "strings" | ||
|
|
||
| "golang.org/x/exp/maps" | ||
| "golang.org/x/tools/go/packages" | ||
| "gopkg.in/yaml.v3" | ||
| ) | ||
|
|
||
| type MetricsUsage struct { | ||
| Name string | ||
| Type string | ||
| Unit string | ||
| Description string | ||
| } | ||
|
|
||
| const metricsImportPath = "github.com/honeycombio/refinery/metrics" | ||
|
|
||
| func GenerateMetricsMetadata() error { | ||
| output, err := os.Create("metricsMeta.yaml") | ||
| if err != nil { | ||
| return fmt.Errorf("error creating output file: %v", err) | ||
| } | ||
| defer output.Close() | ||
|
|
||
| // Configuration to load Go packages. | ||
| cfg := &packages.Config{ | ||
| Mode: packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedName | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo, | ||
| } | ||
|
|
||
| // Load the package from the current directory. | ||
| pkgs, err := packages.Load(cfg, "github.com/honeycombio/refinery/...") | ||
| if err != nil { | ||
| return fmt.Errorf("error loading packages: %v", err) | ||
| } | ||
|
|
||
| usages := make([]MetricsUsage, 0) | ||
| // Traverse each package and file. | ||
| for _, pkg := range pkgs { | ||
| if !slices.Contains(maps.Keys(pkg.Imports), metricsImportPath) { | ||
| continue | ||
| } | ||
|
|
||
| var found bool | ||
| // Iterate over the syntax trees (ASTs) of each file in the package. | ||
| for _, syntax := range pkg.Syntax { | ||
| // Inspect the AST for each file. | ||
| ast.Inspect(syntax, func(n ast.Node) bool { | ||
| // Look for all slice type declarations | ||
| if decl, ok := n.(*ast.CompositeLit); ok { | ||
|
|
||
| if arrayType, ok := decl.Type.(*ast.ArrayType); ok { | ||
| // Check if the element type of the array is a selector expression | ||
| if selector, ok := arrayType.Elt.(*ast.SelectorExpr); ok { | ||
| // Check if the package and type name match "metrics.Metadata" | ||
| if pkgIdent, ok := selector.X.(*ast.Ident); ok && pkgIdent.Name == "metrics" && selector.Sel.Name == "Metadata" { | ||
|
|
||
| // Now extract the fields from the composite literal | ||
| for _, elt := range decl.Elts { | ||
| if comp, ok := elt.(*ast.CompositeLit); ok { | ||
| var usage MetricsUsage | ||
| for _, elt := range comp.Elts { | ||
| if kvExpr, ok := elt.(*ast.KeyValueExpr); ok { | ||
| field := exprToString(kvExpr.Key, pkg) | ||
| value := exprToString(kvExpr.Value, pkg) | ||
|
|
||
| switch field { | ||
| case "Name": | ||
| usage.Name = value | ||
| case "Type": | ||
| usage.Type = value | ||
| case "Unit": | ||
| usage.Unit = value | ||
| case "Description": | ||
| usage.Description = value | ||
|
|
||
| } | ||
| } | ||
| } | ||
|
|
||
| if usage.Name == "" { | ||
| continue | ||
| } | ||
| usages = append(usages, usage) | ||
| found = true | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| return true | ||
| }) | ||
|
|
||
| } | ||
| if !found { | ||
| return fmt.Errorf("Missing metrics.Metadata declaration in the package %s", pkg.Name) | ||
| } | ||
| } | ||
|
|
||
| if len(usages) == 0 { | ||
| return fmt.Errorf("No metrics.Metadata declarations found in all packages") | ||
| } | ||
|
|
||
| err = writeMetricsToYAML(usages, output) | ||
| if err != nil { | ||
| return fmt.Errorf("error writing metrics to YAML: %v", err) | ||
| } | ||
|
|
||
| fmt.Printf("Metrics usages have been written to %s\n", output.Name()) | ||
| return nil | ||
|
|
||
| } | ||
|
|
||
| // exprToString is a helper function to convert ast.Expr to a string representation | ||
| func exprToString(expr ast.Expr, pkg *packages.Package) string { | ||
| var strVal string | ||
| switch v := expr.(type) { | ||
| case *ast.Ident: | ||
| if obj := pkg.TypesInfo.ObjectOf(v); obj != nil { | ||
| // Get the value of the variable (if constant) | ||
| if constVal, ok := obj.(*types.Const); ok { | ||
| strVal = constVal.Val().String() | ||
| break | ||
| } | ||
|
|
||
| } | ||
| strVal = v.Name | ||
| case *ast.BasicLit: | ||
| strVal = v.Value | ||
| case *ast.SelectorExpr: | ||
| strVal = v.Sel.Name | ||
| default: | ||
| strVal = fmt.Sprintf("%T", expr) | ||
| } | ||
|
|
||
| return strings.Trim(strVal, "\"") | ||
| } | ||
|
|
||
| func writeMetricsToYAML(metricsUsages []MetricsUsage, output *os.File) error { | ||
| // Create a new YAML encoder and write the metrics | ||
| encoder := yaml.NewEncoder(output) | ||
| defer encoder.Close() | ||
|
|
||
| err := encoder.Encode(metricsUsages) | ||
| if err != nil { | ||
| return fmt.Errorf("error encoding YAML: %v", err) | ||
| } | ||
|
|
||
| return nil | ||
| } |
Oops, something went wrong.