Merge cockroachdb#132922

132922: sql, ui: introduce `sql.crud_query.count` QPS metric r=angles-n-daemons a=dhartunian

Previously, customers struggled to get an accurate picture of "QPS" on a cluster, due to a number of factors.

First, our `sql.query.count` metric includes counts of transaction control statements which greatly inflate the "QPS" in a confusing way to a customer.

Second, the Metrics page in DB Console included a "QPS" metric which was a client-side sum of the insert/update/delete/select counters. This metric wasn't available to scrape or view in line graphs which led to confusion.

In this PR we introduce the `sql.crud_query.count` and `sql.crud_query.started.count` metric which is incremented along with the insert/update/delete/select counters.

This metric is added to the SQL chart at the top of the overview dashboard. That chart is renamed to have the title `SQL Queries Per Second` for clarity and the box on the right hand side now uses this metric directly.

Resolves: CRDB-25876
Epic: CRDB-37557

Release note (ops change): Two new metrics are available `sql.crud_query.count` and `sql.crud_query.started.count` which measure the number of insert/update/delete/select queries executed and started respectively.

Co-authored-by: David Hartunian <[email protected]>

docs/generated/metrics/metrics.html

@@ -1561,6 +1561,10 @@
 <tr><td>APPLICATION</td><td>sql.copy.nonatomic.started.count.internal</td><td>Number of non-atomic COPY SQL statements started (internal queries)</td><td>SQL Internal Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>APPLICATION</td><td>sql.copy.started.count</td><td>Number of COPY SQL statements started</td><td>SQL Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>APPLICATION</td><td>sql.copy.started.count.internal</td><td>Number of COPY SQL statements started (internal queries)</td><td>SQL Internal Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
+<tr><td>APPLICATION</td><td>sql.crud_query.count</td><td>Number of SQL SELECT, INSERT, UPDATE, DELETE statements successfully executed</td><td>SQL Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
+<tr><td>APPLICATION</td><td>sql.crud_query.count.internal</td><td>Number of SQL SELECT, INSERT, UPDATE, DELETE statements successfully executed (internal queries)</td><td>SQL Internal Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
+<tr><td>APPLICATION</td><td>sql.crud_query.started.count</td><td>Number of SQL SELECT, INSERT, UPDATE, DELETE statements started</td><td>SQL Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
+<tr><td>APPLICATION</td><td>sql.crud_query.started.count.internal</td><td>Number of SQL SELECT, INSERT, UPDATE, DELETE statements started (internal queries)</td><td>SQL Internal Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>APPLICATION</td><td>sql.ddl.count</td><td>Number of SQL DDL statements successfully executed</td><td>SQL Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>APPLICATION</td><td>sql.ddl.count.internal</td><td>Number of SQL DDL statements successfully executed (internal queries)</td><td>SQL Internal Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>APPLICATION</td><td>sql.ddl.started.count</td><td>Number of SQL DDL statements started</td><td>SQL Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
@@ -1676,10 +1680,10 @@
 <tr><td>APPLICATION</td><td>sql.pre_serve.mem.cur</td><td>Current memory usage for SQL connections prior to routing the connection to the target SQL server</td><td>Memory</td><td>GAUGE</td><td>BYTES</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>APPLICATION</td><td>sql.pre_serve.mem.max</td><td>Memory usage for SQL connections prior to routing the connection to the target SQL server</td><td>Memory</td><td>HISTOGRAM</td><td>BYTES</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>APPLICATION</td><td>sql.pre_serve.new_conns</td><td>Number of SQL connections created prior to routing the connection to the target SQL server</td><td>Connections</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
-<tr><td>APPLICATION</td><td>sql.query.count</td><td>Number of SQL queries executed</td><td>SQL Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
-<tr><td>APPLICATION</td><td>sql.query.count.internal</td><td>Number of SQL queries executed (internal queries)</td><td>SQL Internal Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
-<tr><td>APPLICATION</td><td>sql.query.started.count</td><td>Number of SQL queries started</td><td>SQL Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
-<tr><td>APPLICATION</td><td>sql.query.started.count.internal</td><td>Number of SQL queries started (internal queries)</td><td>SQL Internal Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
+<tr><td>APPLICATION</td><td>sql.query.count</td><td>Number of SQL operations started including queries, and transaction control statements</td><td>SQL Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
+<tr><td>APPLICATION</td><td>sql.query.count.internal</td><td>Number of SQL operations started including queries, and transaction control statements (internal queries)</td><td>SQL Internal Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
+<tr><td>APPLICATION</td><td>sql.query.started.count</td><td>Number of SQL operations started including queries, and transaction control statements</td><td>SQL Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
+<tr><td>APPLICATION</td><td>sql.query.started.count.internal</td><td>Number of SQL operations started including queries, and transaction control statements (internal queries)</td><td>SQL Internal Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>APPLICATION</td><td>sql.restart_savepoint.count</td><td>Number of `SAVEPOINT cockroach_restart` statements successfully executed</td><td>SQL Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>APPLICATION</td><td>sql.restart_savepoint.count.internal</td><td>Number of `SAVEPOINT cockroach_restart` statements successfully executed (internal queries)</td><td>SQL Internal Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>APPLICATION</td><td>sql.restart_savepoint.release.count</td><td>Number of `RELEASE SAVEPOINT cockroach_restart` statements successfully executed</td><td>SQL Statements</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>

pkg/sql/conn_executor.go

@@ -4435,6 +4435,8 @@ type StatementCounters struct {
 	UpdateCount telemetry.CounterWithMetric
 	InsertCount telemetry.CounterWithMetric
 	DeleteCount telemetry.CounterWithMetric
+	// CRUDQueryCount includes all 4 CRUD statements above.
+	CRUDQueryCount telemetry.CounterWithMetric
 
 	// Transaction operations.
 	TxnBeginCount    telemetry.CounterWithMetric
@@ -4499,6 +4501,8 @@ func makeStartedStatementCounters(internal bool) StatementCounters {
 			getMetricMeta(MetaInsertStarted, internal)),
 		DeleteCount: telemetry.NewCounterWithMetric(
 			getMetricMeta(MetaDeleteStarted, internal)),
+		CRUDQueryCount: telemetry.NewCounterWithMetric(
+			getMetricMeta(MetaCRUDStarted, internal)),
 		DdlCount: telemetry.NewCounterWithMetric(
 			getMetricMeta(MetaDdlStarted, internal)),
 		CopyCount: telemetry.NewCounterWithMetric(
@@ -4542,6 +4546,8 @@ func makeExecutedStatementCounters(internal bool) StatementCounters {
 			getMetricMeta(MetaInsertExecuted, internal)),
 		DeleteCount: telemetry.NewCounterWithMetric(
 			getMetricMeta(MetaDeleteExecuted, internal)),
+		CRUDQueryCount: telemetry.NewCounterWithMetric(
+			getMetricMeta(MetaCRUDExecuted, internal)),
 		DdlCount: telemetry.NewCounterWithMetric(
 			getMetricMeta(MetaDdlExecuted, internal)),
 		CopyCount: telemetry.NewCounterWithMetric(
@@ -4562,12 +4568,16 @@ func (sc *StatementCounters) incrementCount(ex *connExecutor, stmt tree.Statemen
 		sc.TxnBeginCount.Inc()
 	case *tree.Select:
 		sc.SelectCount.Inc()
+		sc.CRUDQueryCount.Inc()
 	case *tree.Update:
 		sc.UpdateCount.Inc()
+		sc.CRUDQueryCount.Inc()
 	case *tree.Insert:
 		sc.InsertCount.Inc()
+		sc.CRUDQueryCount.Inc()
 	case *tree.Delete:
 		sc.DeleteCount.Inc()
+		sc.CRUDQueryCount.Inc()
 	case *tree.CommitTransaction:
 		sc.TxnCommitCount.Inc()
 	case *tree.RollbackTransaction:

pkg/sql/exec_util.go

@@ -833,7 +833,7 @@ var (
 	// Below are the metadata for the statement started counters.
 	MetaQueryStarted = metric.Metadata{
 		Name:        "sql.query.started.count",
-		Help:        "Number of SQL queries started",
+		Help:        "Number of SQL operations started including queries, and transaction control statements",
 		Measurement: "SQL Statements",
 		Unit:        metric.Unit_COUNT,
 	}
@@ -885,6 +885,12 @@ var (
 		Measurement: "SQL Statements",
 		Unit:        metric.Unit_COUNT,
 	}
+	MetaCRUDStarted = metric.Metadata{
+		Name:        "sql.crud_query.started.count",
+		Help:        "Number of SQL SELECT, INSERT, UPDATE, DELETE statements started",
+		Measurement: "SQL Statements",
+		Unit:        metric.Unit_COUNT,
+	}
 	MetaSavepointStarted = metric.Metadata{
 		Name:        "sql.savepoint.started.count",
 		Help:        "Number of SQL SAVEPOINT statements started",
@@ -949,7 +955,7 @@ var (
 	// Below are the metadata for the statement executed counters.
 	MetaQueryExecuted = metric.Metadata{
 		Name:        "sql.query.count",
-		Help:        "Number of SQL queries executed",
+		Help:        "Number of SQL operations started including queries, and transaction control statements",
 		Measurement: "SQL Statements",
 		Unit:        metric.Unit_COUNT,
 	}
@@ -1001,6 +1007,12 @@ var (
 		Measurement: "SQL Statements",
 		Unit:        metric.Unit_COUNT,
 	}
+	MetaCRUDExecuted = metric.Metadata{
+		Name:        "sql.crud_query.count",
+		Help:        "Number of SQL SELECT, INSERT, UPDATE, DELETE statements successfully executed",
+		Measurement: "SQL Statements",
+		Unit:        metric.Unit_COUNT,
+	}
 	MetaSavepointExecuted = metric.Metadata{
 		Name:        "sql.savepoint.count",
 		Help:        "Number of SQL SAVEPOINT statements successfully executed",

pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/overview.tsx

@@ -30,12 +30,12 @@ export default function (props: GraphDashboardProps) {
 
   return [
     <LineGraph
-      title="SQL Statements"
+      title="SQL Queries Per Second"
       isKvGraph={false}
       sources={nodeSources}
       tenantSource={tenantSource}
       tooltip={`A moving average of the number of SELECT, INSERT, UPDATE, and DELETE
-          statements successfully executed per second ${tooltipSelection}.`}
+          statements, and the sum of all four, successfully executed per second ${tooltipSelection}.`}
       showMetricsInTooltip={true}
       preCalcGraphSize={true}
     >
@@ -60,6 +60,11 @@ export default function (props: GraphDashboardProps) {
           title="Deletes"
           nonNegativeRate
         />
+        <Metric
+          name="cr.node.sql.crud_query.count"
+          title="Total Queries"
+          nonNegativeRate
+        />
       </Axis>
     </LineGraph>,
 

pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/sql.tsx

@@ -137,12 +137,12 @@ export default function (props: GraphDashboardProps) {
     </LineGraph>,
 
     <LineGraph
-      title="SQL Statements"
+      title="SQL Queries Per Second"
       isKvGraph={false}
       sources={nodeSources}
       tenantSource={tenantSource}
       tooltip={`A ten-second moving average of the # of SELECT, INSERT, UPDATE, and
-          DELETE statements successfully executed per second ${tooltipSelection}.`}
+          DELETE statements, and the sum of all four, successfully executed per second ${tooltipSelection}.`}
       showMetricsInTooltip={true}
     >
       <Axis label="queries">
@@ -166,6 +166,11 @@ export default function (props: GraphDashboardProps) {
           title="Deletes"
           nonNegativeRate
         />
+        <Metric
+          name="cr.node.sql.crud_query.count"
+          title="Total Queries"
+          nonNegativeRate
+        />
       </Axis>
     </LineGraph>,
 

pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/summaryBar.tsx

@@ -10,19 +10,19 @@ import { useSelector } from "react-redux";
 import { Link } from "react-router-dom";
 import { createSelector } from "reselect";
 
-import { Tooltip, Anchor } from "src/components";
+import { Anchor, Tooltip } from "src/components";
 import { nodeStatusesSelector, nodeSumsSelector } from "src/redux/nodes";
 import { howAreCapacityMetricsCalculated } from "src/util/docs";
 import { EventBox } from "src/views/cluster/containers/events";
 import { Metric } from "src/views/shared/components/metricQuery";
 import {
   SummaryBar,
   SummaryLabel,
+  SummaryMetricsAggregator,
   SummaryMetricStat,
   SummaryStat,
   SummaryStatBreakdown,
   SummaryStatMessage,
-  SummaryMetricsAggregator,
 } from "src/views/shared/components/summaryBar";
 
 /**
@@ -145,28 +145,7 @@ export default function (props: ClusterSummaryProps) {
         >
           <Metric
             sources={props.nodeSources}
-            name="cr.node.sql.select.count"
-            title="Queries/Sec"
-            tenantSource={props.tenantSource}
-            nonNegativeRate
-          />
-          <Metric
-            sources={props.nodeSources}
-            name="cr.node.sql.insert.count"
-            title="Queries/Sec"
-            tenantSource={props.tenantSource}
-            nonNegativeRate
-          />
-          <Metric
-            sources={props.nodeSources}
-            name="cr.node.sql.update.count"
-            title="Queries/Sec"
-            tenantSource={props.tenantSource}
-            nonNegativeRate
-          />
-          <Metric
-            sources={props.nodeSources}
-            name="cr.node.sql.delete.count"
+            name="cr.node.sql.crud_query.count"
             title="Queries/Sec"
             tenantSource={props.tenantSource}
             nonNegativeRate