Minor tweaks to query plan analysis summary output

go/cmd/root.go

@@ -24,32 +24,26 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/vitessio/vt/go/web"
+	"github.com/vitessio/vt/go/web/state"
 )
 
+//nolint:gochecknoglobals // the state is protected using mutexes
+var wstate *state.State
+
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
 	// rootCmd represents the base command when called without any subcommands
-	var port int64
-	webserverStarted := false
-	ch := make(chan int, 2)
 	root := &cobra.Command{
 		Use:   "vt",
 		Short: "Utils tools for testing, running and benchmarking Vitess.",
-		RunE: func(_ *cobra.Command, _ []string) error {
-			if port > 0 {
-				if webserverStarted {
-					return nil
-				}
-				webserverStarted = true
-				go startWebServer(port, ch)
-				<-ch
-			}
-			return nil
+		CompletionOptions: cobra.CompletionOptions{
+			DisableDefaultCmd: true,
 		},
 	}
+
+	var port int64
 	root.PersistentFlags().Int64VarP(&port, "port", "p", 8080, "Port to run the web server on")
-	root.CompletionOptions.HiddenDefaultCmd = true
 
 	root.AddCommand(summarizeCmd(&port))
 	root.AddCommand(testerCmd())
@@ -63,16 +57,20 @@ func Execute() {
 		panic(err)
 	}
 
-	if !webserverStarted && port > 0 {
-		webserverStarted = true
-		go startWebServer(port, ch)
+	// Start the web server for all commands no matter what
+	wstate = state.NewState(port)
+	ch := make(chan int, 1)
+	if port > 0 {
+		wstate.SetStarted(true)
+		go startWebServer(ch)
+		if !wstate.WaitUntilAvailable(10 * time.Second) {
+			fmt.Println("Timed out waiting for server to start")
+			os.Exit(1)
+		}
 	} else {
 		ch <- 1
 	}
 
-	// FIXME: add sync b/w webserver and root command, for now just add a wait to make sure webserver is running
-	time.Sleep(2 * time.Second)
-
 	err := root.Execute()
 	if err != nil {
 		fmt.Printf("Error: %v\n", err)
@@ -81,13 +79,15 @@ func Execute() {
 	<-ch
 }
 
-func startWebServer(port int64, ch chan int) {
-	if port > 0 && port != 8080 {
-		panic("(FIXME: make port configurable) Port is not 8080")
+func startWebServer(ch chan int) {
+	err := web.Run(wstate)
+	if err != nil {
+		panic(err)
 	}
-	web.Run(port)
-	if os.WriteFile("/dev/stderr", []byte("Web server is running, use Ctrl-C to exit\n"), 0o600) != nil {
-		panic("Failed to write to /dev/stderr")
+
+	_, err = fmt.Fprint(os.Stderr, "Web server is running, use Ctrl-C to exit\n")
+	if err != nil {
+		panic(err)
 	}
 	ch <- 1
 }

go/cmd/summarize.go

@@ -23,9 +23,9 @@ import (
 )
 
 func summarizeCmd(port *int64) *cobra.Command {
-	var hotMetric string
-	var showGraph bool
-	var outputFormat string
+	cfg := summarize.Config{
+		WState: wstate,
+	}
 
 	cmd := &cobra.Command{
 		Use:     "summarize old_file.json [new_file.json]",
@@ -34,13 +34,15 @@ func summarizeCmd(port *int64) *cobra.Command {
 		Example: "vt summarize old.json new.json",
 		Args:    cobra.RangeArgs(1, 2),
 		Run: func(_ *cobra.Command, args []string) {
-			summarize.Run(args, hotMetric, showGraph, outputFormat, port)
+			cfg.Files = args
+			cfg.Port = *port
+			summarize.Run(&cfg)
 		},
 	}
 
-	cmd.Flags().StringVar(&hotMetric, "hot-metric", "total-time", "Metric to determine hot queries (options: usage-count, total-rows-examined, avg-rows-examined, avg-time, total-time)")
-	cmd.Flags().BoolVar(&showGraph, "graph", false, "Show the query graph in the browser")
-	cmd.Flags().StringVar(&outputFormat, "format", "html", "Output format (options: html, markdown)")
+	cmd.Flags().StringVar(&cfg.HotMetric, "hot-metric", "total-time", "Metric to determine hot queries (options: usage-count, total-rows-examined, avg-rows-examined, avg-time, total-time)")
+	cmd.Flags().BoolVar(&cfg.ShowGraph, "graph", false, "Show the query graph in the browser")
+	cmd.Flags().StringVar(&cfg.OutputFormat, "format", "html", "Output format (options: html, markdown)")
 
 	return cmd
 }

go/summarize/markdown.go

@@ -27,29 +27,15 @@ import (
 
 	humanize "github.com/dustin/go-humanize"
 
-	"github.com/vitessio/vt/go/keys"
 	"github.com/vitessio/vt/go/markdown"
 	"github.com/vitessio/vt/go/planalyze"
 )
 
-func renderHotQueries(md *markdown.MarkDown, queries []keys.QueryAnalysisResult, metricReader getMetric) {
+func renderHotQueries(md *markdown.MarkDown, queries []HotQueryResult) {
 	if len(queries) == 0 {
 		return
 	}
 
-	hasTime := false
-	// Sort the queries in descending order of hotness
-	sort.Slice(queries, func(i, j int) bool {
-		if queries[i].QueryTime != 0 {
-			hasTime = true
-		}
-		return metricReader(queries[i]) > metricReader(queries[j])
-	})
-
-	if !hasTime {
-		return
-	}
-
 	md.PrintHeader("Top Queries", 2)
 
 	// Prepare table headers and rows
@@ -58,13 +44,12 @@ func renderHotQueries(md *markdown.MarkDown, queries []keys.QueryAnalysisResult,
 
 	for i, query := range queries {
 		queryID := fmt.Sprintf("Q%d", i+1)
-		avgQueryTime := query.QueryTime / float64(query.UsageCount)
 		rows = append(rows, []string{
 			queryID,
-			humanize.Comma(int64(query.UsageCount)),
-			fmt.Sprintf("%.2f", query.QueryTime),
-			fmt.Sprintf("%.2f", avgQueryTime),
-			humanize.Comma(int64(query.RowsExamined)),
+			humanize.Comma(int64(query.QueryAnalysisResult.UsageCount)),
+			fmt.Sprintf("%.2f", query.QueryAnalysisResult.QueryTime),
+			fmt.Sprintf("%.2f", query.AvgQueryTime),
+			humanize.Comma(int64(query.QueryAnalysisResult.RowsExamined)),
 		})
 	}
 
@@ -74,11 +59,24 @@ func renderHotQueries(md *markdown.MarkDown, queries []keys.QueryAnalysisResult,
 	// After the table, list the full queries with their IDs
 	md.PrintHeader("Query Details", 3)
 	for i, query := range queries {
+		hasPlanAnalysis := len(string(query.PlanAnalysis.PlanOutput)) > 0
+
 		queryID := fmt.Sprintf("Q%d", i+1)
+		if hasPlanAnalysis {
+			queryID += fmt.Sprintf(" (`%s`)", query.PlanAnalysis.Complexity.String())
+		}
+
 		md.PrintHeader(queryID, 4)
 		md.Println("```sql")
-		md.Println(query.QueryStructure)
+		md.Println(query.QueryAnalysisResult.QueryStructure)
 		md.Println("```")
+
+		if hasPlanAnalysis {
+			md.Println("```json")
+			md.Println(string(query.PlanAnalysis.PlanOutput))
+			md.Println("```")
+		}
+
 		md.NewLine()
 	}
 }
@@ -230,8 +228,7 @@ func renderTransactions(md *markdown.MarkDown, transactions []TransactionSummary
 }
 
 func renderPlansSection(md *markdown.MarkDown, analysis PlanAnalysis) error {
-	sum := analysis.PassThrough + analysis.SimpleRouted + analysis.Complex + analysis.Unplannable
-	if sum == 0 {
+	if analysis.Total == 0 {
 		return nil
 	}
 
@@ -243,25 +240,28 @@ func renderPlansSection(md *markdown.MarkDown, analysis PlanAnalysis) error {
 		{planalyze.SimpleRouted.String(), strconv.Itoa(analysis.SimpleRouted)},
 		{planalyze.Complex.String(), strconv.Itoa(analysis.Complex)},
 		{planalyze.Unplannable.String(), strconv.Itoa(analysis.Unplannable)},
-		{"Total", strconv.Itoa(sum)},
+		{"Total", strconv.Itoa(analysis.Total)},
 	}
 	md.PrintTable(headers, rows)
 	md.NewLine()
 
-	err := renderQueryPlans(md, analysis.simpleRouted, planalyze.SimpleRouted.String())
+	err := renderQueryPlans(md, analysis.SimpleRoutedQ, planalyze.SimpleRouted.String())
 	if err != nil {
 		return err
 	}
-	return renderQueryPlans(md, analysis.complex, planalyze.Complex.String())
+	return renderQueryPlans(md, analysis.ComplexQ, planalyze.Complex.String())
 }
 
 func renderQueryPlans(md *markdown.MarkDown, queries []planalyze.AnalyzedQuery, title string) error {
 	for i, query := range queries {
 		if i == 0 {
-			md.Printf("# %s Queries\n\n", title)
+			md.PrintHeader(fmt.Sprintf("%s Queries\n\n", title), 3)
 		}
-		md.Printf("## Query\n\n```sql\n%s\n```\n\n", query.QueryStructure)
-		md.Println("## Plan\n\n```json")
+		md.PrintHeader("Query", 4)
+		md.Printf("```sql\n%s\n```\n\n", query.QueryStructure)
+
+		md.PrintHeader("Plan", 4)
+		md.Println("```json")
 
 		// Indent the JSON output. If we don't do this, the json will be indented all wrong
 		var buf bytes.Buffer
@@ -274,6 +274,7 @@ func renderQueryPlans(md *markdown.MarkDown, queries []planalyze.AnalyzedQuery,
 		}
 		md.NewLine()
 		md.Println("```")
+		md.Println("---")
 		md.NewLine()
 	}
 	return nil

go/summarize/summarize-keys.go

@@ -247,8 +247,8 @@ func summarizeKeysQueries(summary *Summary, queries *keys.Output) error {
 
 	// First pass: collect all graphData and count occurrences
 	for _, query := range queries.Queries {
+		summary.addQueryResult(query)
 		gatherTableInfo(query, tableSummaries, tableUsageWriteCounts, tableUsageReadCounts)
-		checkQueryForHotness(&summary.HotQueries, query, summary.hotQueryFn)
 	}
 
 	// Second pass: calculate percentages
@@ -327,15 +327,25 @@ func summarizeKeysQueries(summary *Summary, queries *keys.Output) error {
 	return nil
 }
 
-func checkQueryForHotness(hotQueries *[]keys.QueryAnalysisResult, query keys.QueryAnalysisResult, metricReader getMetric) {
+func checkQueryForHotness(hotQueries *[]HotQueryResult, query QueryResult, metricReader getMetric) {
 	// todo: we should be able to choose different metrics for hotness - e.g. total time spent on query, number of rows examined, etc.
+	newHotQueryFn := func() HotQueryResult {
+		return HotQueryResult{
+			QueryResult: QueryResult{
+				QueryAnalysisResult: query.QueryAnalysisResult,
+				PlanAnalysis:        query.PlanAnalysis,
+			},
+			AvgQueryTime: query.QueryAnalysisResult.QueryTime / float64(query.QueryAnalysisResult.UsageCount),
+		}
+	}
+
 	switch {
 	case len(*hotQueries) < HotQueryCount:
 		// If we have not yet reached the limit, add the query
-		*hotQueries = append(*hotQueries, query)
-	case metricReader(query) > metricReader((*hotQueries)[0]):
+		*hotQueries = append(*hotQueries, newHotQueryFn())
+	case metricReader(query.QueryAnalysisResult) > metricReader((*hotQueries)[0].QueryAnalysisResult):
 		// If the current query has more usage than the least used hot query, replace it
-		(*hotQueries)[0] = query
+		(*hotQueries)[0] = newHotQueryFn()
 	default:
 		// If the current query is not hot enough, just return
 		return
@@ -344,7 +354,7 @@ func checkQueryForHotness(hotQueries *[]keys.QueryAnalysisResult, query keys.Que
 	// Sort the hot queries by query time so that the least used query is always at the front
 	sort.Slice(*hotQueries,
 		func(i, j int) bool {
-			return metricReader((*hotQueries)[i]) < metricReader((*hotQueries)[j])
+			return metricReader((*hotQueries)[i].QueryAnalysisResult) < metricReader((*hotQueries)[j].QueryAnalysisResult)
 		})
 }
 

go/summarize/summarize-keys_test.go

@@ -118,6 +118,9 @@ func TestSummarizeKeysWithHotnessFile(t *testing.T) {
 			err = fn(s)
 			require.NoError(t, err)
 
+			err = compileSummary(s)
+			require.NoError(t, err)
+
 			err = s.PrintMarkdown(sb, now)
 			require.NoError(t, err)
 

go/summarize/summarize-planalyze.go

@@ -21,14 +21,18 @@ import (
 )
 
 func summarizePlanAnalyze(s *Summary, data planalyze.Output) (err error) {
-	s.planAnalysis = PlanAnalysis{
+	s.PlanAnalysis = PlanAnalysis{
 		PassThrough:  len(data.PassThrough),
 		SimpleRouted: len(data.SimpleRouted),
 		Complex:      len(data.Complex),
 		Unplannable:  len(data.Unplannable),
 	}
+	s.PlanAnalysis.Total = s.PlanAnalysis.PassThrough + s.PlanAnalysis.SimpleRouted + s.PlanAnalysis.Complex + s.PlanAnalysis.Unplannable
 
-	s.planAnalysis.simpleRouted = append(s.planAnalysis.simpleRouted, data.SimpleRouted...)
-	s.planAnalysis.complex = append(s.planAnalysis.complex, data.Complex...)
+	s.addPlanResult(data.SimpleRouted)
+	s.addPlanResult(data.Complex)
+
+	s.PlanAnalysis.SimpleRoutedQ = append(s.PlanAnalysis.SimpleRoutedQ, data.SimpleRouted...)
+	s.PlanAnalysis.ComplexQ = append(s.PlanAnalysis.ComplexQ, data.Complex...)
 	return nil
 }

go/summarize/summarize-planalyze_test.go

@@ -27,15 +27,25 @@ import (
 )
 
 func TestSummarizePlans(t *testing.T) {
-	fn, err := readPlanalyzeFile("../testdata/planalyze-output/bigger_slow_query_plan_report.json")
+	fnPlan, err := readPlanalyzeFile("../testdata/planalyze-output/bigger_slow_query_plan_report.json")
 	require.NoError(t, err)
+
+	fnKeys, err := readKeysFile("../testdata/keys-output/bigger_slow_query_log.json")
+	require.NoError(t, err)
+
 	sb := &strings.Builder{}
 	now := time.Date(2024, time.January, 1, 1, 2, 3, 0, time.UTC)
 
-	s, err := NewSummary("")
+	s, err := NewSummary("usage-count")
+	require.NoError(t, err)
+
+	err = fnPlan(s)
+	require.NoError(t, err)
+
+	err = fnKeys(s)
 	require.NoError(t, err)
 
-	err = fn(s)
+	err = compileSummary(s)
 	require.NoError(t, err)
 
 	err = s.PrintMarkdown(sb, now)