Splunkenterprisereceiver add health metric

receiver/splunkenterprisereceiver/documentation.md

@@ -41,6 +41,21 @@ Gauge tracking the number of buckets and their searchable status. *Note:** Searc
 | splunk.host | The name of the splunk host | Any Str |
 | splunk.indexer.searchable | The searchability status reported for a specific object | Any Str |
 
+### splunk.health
+
+The status (color) of the Splunk server.
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| {status} | Gauge | Int |
+
+#### Attributes
+
+| Name | Description | Values |
+| ---- | ----------- | ------ |
+| splunk.feature | The Feature name from the Splunk Health Introspection Endpoint | Any Str |
+| splunk.feature.health | The Health (in color form) of a Splunk Feature from the Splunk Health Introspection Endpoint | Any Str |
+
 ### splunk.indexer.avg.rate
 
 Gauge tracking the average rate of indexed data. **Note:** Search is best run against a Cluster Manager.

receiver/splunkenterprisereceiver/internal/metadata/testdata/config.yaml

@@ -19,6 +19,8 @@ all_set:
       enabled: true
     splunk.data.indexes.extended.total.size:
       enabled: true
+    splunk.health:
+      enabled: true
     splunk.indexer.avg.rate:
       enabled: true
     splunk.indexer.cpu.time:
@@ -101,6 +103,8 @@ none_set:
       enabled: false
     splunk.data.indexes.extended.total.size:
       enabled: false
+    splunk.health:
+      enabled: false
     splunk.indexer.avg.rate:
       enabled: false
     splunk.indexer.cpu.time:

receiver/splunkenterprisereceiver/metadata.yaml

@@ -39,6 +39,12 @@ attributes:
   splunk.searchartifacts.cache.type:
     description: The search artifacts cache type
     type: string
+  splunk.feature:
+    description: The Feature name from the Splunk Health Introspection Endpoint
+    type: string
+  splunk.feature.health:
+    description: The Health (in color form) of a Splunk Feature from the Splunk Health Introspection Endpoint
+    type: string
 
 metrics:
   splunk.license.index.usage:
@@ -345,6 +351,14 @@ metrics:
       aggregation_temporality: cumulative
       value_type: int
     attributes: [splunk.host]
+  #`services/server/health/splunkd/details`
+  splunk.health:
+    enabled: True
+    description: The status (color) of the Splunk server.
+    unit: "{status}"
+    gauge:
+      value_type: int
+    attributes: [splunk.feature, splunk.feature.health]
 
 tests:
   config:

receiver/splunkenterprisereceiver/scraper.go

@@ -101,6 +101,7 @@ func (s *splunkScraper) scrape(ctx context.Context) (pmetric.Metrics, error) {
 		s.scrapeIndexerAvgRate,
 		s.scrapeKVStoreStatus,
 		s.scrapeSearchArtifacts,
+		s.scrapeHealth,
 	}
 	errChan := make(chan error, len(metricScrapes))
 
@@ -1075,12 +1076,12 @@ func unmarshallSearchReq(res *http.Response, sr *searchResponse) error {
 
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
-		return fmt.Errorf("Failed to read response: %w", err)
+		return fmt.Errorf("failed to read response: %w", err)
 	}
 
 	err = xml.Unmarshal(body, &sr)
 	if err != nil {
-		return fmt.Errorf("Failed to unmarshall response: %w", err)
+		return fmt.Errorf("failed to unmarshall response: %w", err)
 	}
 
 	return nil
@@ -1733,3 +1734,50 @@ func (s *splunkScraper) scrapeSearchArtifacts(ctx context.Context, now pcommon.T
 		}
 	}
 }
+
+// Scrape Health Introspection Endpoint
+func (s *splunkScraper) scrapeHealth(ctx context.Context, now pcommon.Timestamp, errs chan error) {
+	if !s.conf.MetricsBuilderConfig.Metrics.SplunkHealth.Enabled {
+		return
+	}
+
+	ctx = context.WithValue(ctx, endpointType("type"), typeCm)
+
+	ept := apiDict[`SplunkHealth`]
+	var hd HealthDetails
+
+	req, err := s.splunkClient.createAPIRequest(ctx, ept)
+	if err != nil {
+		errs <- err
+		return
+	}
+
+	res, err := s.splunkClient.makeRequest(req)
+	if err != nil {
+		errs <- err
+		return
+	}
+	defer res.Body.Close()
+
+	if err := json.NewDecoder(res.Body).Decode(&hd); err != nil {
+		errs <- err
+		return
+	}
+
+	s.traverseHealthDetailFeatures(hd.Features, now)
+}
+
+func (s *splunkScraper) traverseHealthDetailFeatures(features map[string]HealthDetails, now pcommon.Timestamp) {
+	if features == nil {
+		return
+	}
+
+	for k, feature := range features {
+		if feature.Health != "red" {
+			s.mb.RecordSplunkHealthDataPoint(now, 1, k, feature.Health)
+		} else {
+			s.mb.RecordSplunkHealthDataPoint(now, 0, k, feature.Health)
+		}
+		s.traverseHealthDetailFeatures(feature.Features, now)
+	}
+}

receiver/splunkenterprisereceiver/search_result.go

@@ -25,6 +25,7 @@ var apiDict = map[string]string{
 	`SplunkIntrospectionQueues`: `/services/server/introspection/queues?output_mode=json&count=-1`,
 	`SplunkKVStoreStatus`:       `/services/kvstore/status?output_mode=json`,
 	`SplunkDispatchArtifacts`:   `/services/server/status/dispatch-artifacts?output_mode=json&count=-1`,
+	`SplunkHealth`:              `/services/server/health/splunkd/details?output_mode=json`,
 }
 
 type searchResponse struct {
@@ -156,3 +157,9 @@ type DispatchArtifactContent struct {
 	StatusCacheSize    string `json:"cached_job_status_status_csv_size_mb"`
 	CacheTotalEntries  string `json:"cached_job_status_total_entries"`
 }
+
+// '/services/server/health/splunkd/details
+type HealthDetails struct {
+	Health   string                   `json:"health"`
+	Features map[string]HealthDetails `json:"features,omitempty"`
+}