Merge cockroachdb#131232

131232: builtins: handle unavailable ranges in `crdb_internal.ranges` r=rytaft,mgartner a=miraradeva

Previously, when querying crdb_internal.{lease_holder, range_stats} (e.g. as part of querying crdb_internal.ranges), a single unavailable range would result in the entire query failing. This is problematic when investigating clusters with unavailable ranges as it fails to show information even for the available ranges.

This commit adds new variants of `crdb_internal.
{lease_holder, range_stats}`, `crdb_internal.
{lease_holder, range_stats}_with_errors`, that include any encountered errors in the JSON output instead of failing the entire query. These are then used in `crdb_internal.ranges`, which now includes the errors encountered while fetching the leaseholder and range stats.

Fixes: cockroachdb#128088

Release note (sql change): Two new builtins, crdb_internal. {lease_holder, range_stats}_with_errors, include errors encountered while fetching lease holder and range stats. These new builtins are used by `crdb_internal.ranges`, which includes a new column, `errors`, that combines the errors from the builtins.

----

Example output of `select range_id, lease_holder, range_size, errors from crdb_internal.ranges;`

<img width="1728" alt="Screenshot 2024-09-23 at 2 37 24 PM" src="https://github.com/user-attachments/assets/87a9b0ac-6de0-4677-8e5d-6031133beaa5">


Co-authored-by: Mira Radeva <[email protected]>

pkg/sql/catalog/colinfo/result_columns.go

@@ -351,16 +351,18 @@ var Ranges = append(
 	// The following columns are computed by RangesExtraRenders below.
 	ResultColumn{Name: "lease_holder", Typ: types.Int},
 	ResultColumn{Name: "range_size", Typ: types.Int},
+	ResultColumn{Name: "errors", Typ: types.String},
 )
 
 // RangesExtraRenders describes the extra projections in
 // crdb_internal.ranges not included in crdb_internal.ranges_no_leases.
 const RangesExtraRenders = `
-	crdb_internal.lease_holder(start_key) AS lease_holder,
-	(crdb_internal.range_stats(start_key)->>'key_bytes')::INT +
-	(crdb_internal.range_stats(start_key)->>'val_bytes')::INT +
-	coalesce((crdb_internal.range_stats(start_key)->>'range_key_bytes')::INT, 0) +
-	coalesce((crdb_internal.range_stats(start_key)->>'range_val_bytes')::INT, 0) AS range_size
+	(crdb_internal.lease_holder_with_errors(start_key)->>'Leaseholder')::INT AS lease_holder,
+	(crdb_internal.range_stats_with_errors(start_key)->'RangeStats'->>'key_bytes')::INT +
+	(crdb_internal.range_stats_with_errors(start_key)->'RangeStats'->>'val_bytes')::INT +
+	coalesce((crdb_internal.range_stats_with_errors(start_key)->'RangeStats'->>'range_key_bytes')::INT, 0) +
+	coalesce((crdb_internal.range_stats_with_errors(start_key)->'RangeStats'->>'range_val_bytes')::INT, 0) AS range_size,
+	concat(crdb_internal.lease_holder_with_errors(start_key)->>'Error', ' ', crdb_internal.range_stats_with_errors(start_key)->>'Error') AS errors
 `
 
 // IdentifySystemColumns is the schema for IDENTIFY_SYSTEM.

pkg/sql/colexec/BUILD.bazel

@@ -75,6 +75,7 @@ go_library(
         "//pkg/sql/sem/tree",
         "//pkg/sql/sqltelemetry",  # keep
         "//pkg/sql/types",
+        "//pkg/storage/enginepb",
         "//pkg/util/buildutil",
         "//pkg/util/duration",  # keep
         "//pkg/util/encoding",  # keep

pkg/sql/colexec/builtin_funcs.go

@@ -133,7 +133,17 @@ func NewBuiltinFunctionOperator(
 			)
 		}
 		return newRangeStatsOperator(
-			evalCtx.RangeStatsFetcher, allocator, argumentCols[0], outputIdx, input,
+			evalCtx.RangeStatsFetcher, allocator, argumentCols[0], outputIdx, input, false, /* withErrors */
+		)
+	case tree.CrdbInternalRangeStatsWithErrors:
+		if len(argumentCols) != 1 {
+			return nil, errors.AssertionFailedf(
+				"expected 1 input column to crdb_internal.range_stats, got %d",
+				len(argumentCols),
+			)
+		}
+		return newRangeStatsOperator(
+			evalCtx.RangeStatsFetcher, allocator, argumentCols[0], outputIdx, input, true, /* withErrors */
 		)
 	default:
 		return &defaultBuiltinFuncOperator{

pkg/sql/colexec/range_stats.go

@@ -14,6 +14,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/colexecop"
 	"github.com/cockroachdb/cockroach/pkg/sql/colmem"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/eval"
+	"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
 	"github.com/cockroachdb/cockroach/pkg/util/json"
 	"github.com/cockroachdb/errors"
 )
@@ -24,6 +25,9 @@ type rangeStatsOperator struct {
 	allocator   *colmem.Allocator
 	argumentCol int
 	outputIdx   int
+	// withErrors defines if the operator includes any encountered errors in the
+	// returned JSON struct. If true, these errors will not fail the query.
+	withErrors bool
 }
 
 var _ colexecop.Operator = (*rangeStatsOperator)(nil)
@@ -38,13 +42,15 @@ func newRangeStatsOperator(
 	argumentCol int,
 	outputIdx int,
 	input colexecop.Operator,
+	withErrors bool,
 ) (colexecop.Operator, error) {
 	return &rangeStatsOperator{
 		OneInputHelper: colexecop.MakeOneInputHelper(input),
 		allocator:      allocator,
 		argumentCol:    argumentCol,
 		outputIdx:      outputIdx,
 		fetcher:        fetcher,
+		withErrors:     withErrors,
 	}, nil
 }
 
@@ -124,17 +130,36 @@ func (r *rangeStatsOperator) Next() coldata.Batch {
 			// keys plus some constant multiple.
 			// TODO(yuzefovich): add unit tests that use the RunTests test
 			// harness.
-			res, err := r.fetcher.RangeStats(r.Ctx, keys...)
-			if err != nil {
-				colexecerror.ExpectedError(err)
+			res, rangeStatsErr := r.fetcher.RangeStats(r.Ctx, keys...)
+			if rangeStatsErr != nil && !r.withErrors {
+				colexecerror.ExpectedError(rangeStatsErr)
 			}
-			if len(res) != len(keys) {
-				colexecerror.InternalError(errors.AssertionFailedf(
-					"unexpected number of RangeStats responses %d: %d expected", len(res), len(keys),
-				))
+			if len(res) != len(keys) && !r.withErrors {
+				colexecerror.InternalError(
+					errors.AssertionFailedf(
+						"unexpected number of RangeStats responses %d: %d expected", len(res), len(keys),
+					),
+				)
 			}
 			for i, outputIdx := range keysOutputIdx {
-				jsonStr, err := gojson.Marshal(&res[i].MVCCStats)
+				rswe := &rangeStatsWithErrors{}
+				if rangeStatsErr != nil {
+					rswe.Error = rangeStatsErr.Error()
+				}
+				// Not all keys from the keysOutputIdx are guaranteed to be
+				// present in res (e.g. some may be missing if there were errors
+				// in fetcher.RangeStats and r.withErrors = true).
+				if i < len(res) {
+					rswe.RangeStats = &res[i].MVCCStats
+				}
+				var jsonStr []byte
+				var err error
+				if r.withErrors {
+					jsonStr, err = gojson.Marshal(rswe)
+				} else {
+					jsonStr, err = gojson.Marshal(&res[i].MVCCStats)
+				}
+
 				if err != nil {
 					colexecerror.ExpectedError(err)
 				}
@@ -144,6 +169,12 @@ func (r *rangeStatsOperator) Next() coldata.Batch {
 				}
 				jsonOutput.Set(outputIdx, jsonDatum)
 			}
-		})
+		},
+	)
 	return batch
 }
+
+type rangeStatsWithErrors struct {
+	RangeStats *enginepb.MVCCStats
+	Error      string
+}

pkg/sql/delegate/show_ranges.go

@@ -707,6 +707,10 @@ all_span_stats AS (
 			if colinfo.Ranges[i].Name == "lease_holder" {
 				continue
 			}
+			// Skip the errors column; it's used for internal purposes.
+			if colinfo.Ranges[i].Name == "errors" {
+				continue
+			}
 			fmt.Fprintf(&buf, ",\n  %s", tree.NameString(colinfo.Ranges[i].Name))
 		}
 		buf.WriteString(",\n  span_stats")

pkg/sql/logictest/testdata/logic_test/crdb_internal

@@ -531,10 +531,10 @@ SELECT * FROM crdb_internal.node_inflight_trace_spans WHERE span_id < 0
 ----
 trace_id  parent_span_id  span_id  goroutine_id  finished  start_time  duration  operation
 
-query ITTTTITTTTTTI colnames
+query ITTTTITTTTTTIT colnames
 SELECT * FROM crdb_internal.ranges WHERE range_id < 0
 ----
-range_id  start_key  start_pretty  end_key  end_pretty  replicas  replica_localities  voting_replicas  non_voting_replicas  learner_replicas  split_enforced_until  lease_holder  range_size
+range_id  start_key  start_pretty  end_key  end_pretty  replicas  replica_localities  voting_replicas  non_voting_replicas  learner_replicas  split_enforced_until  lease_holder  range_size  errors
 
 query ITTTTTTTTTT colnames
 SELECT * FROM crdb_internal.ranges_no_leases WHERE range_id < 0