From 43c0f1a61b288ce150968630de3aef0fd0aa980e Mon Sep 17 00:00:00 2001
From: Nikki Everett <neverett@users.noreply.github.com>
Date: Wed, 8 May 2024 13:38:51 -0500
Subject: [PATCH 1/6] add tracking pixel (#5336)

Signed-off-by: nikki everett <nikki@union.ai>
---
 docs/_templates/page.html | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/_templates/page.html b/docs/_templates/page.html
index d078e9dd5e..635cdd10f2 100644
--- a/docs/_templates/page.html
+++ b/docs/_templates/page.html
@@ -203,7 +203,10 @@
             {%- endif %}
           </div>
         </div>
-        {% endblock footer %}
+        {% endblock footer %}    
+        <div class="extra_footer">
+          <img referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=65df87cf-549c-4711-b01c-ee9e966c931c" />
+        </div>
       </footer>
     </div>
     <aside class="toc-drawer{% if furo_hide_toc %} no-toc{% endif %}">

From 41b1b84a43567d4d3cba94ec18ae0bcd164772fb Mon Sep 17 00:00:00 2001
From: Austin Liu <austin362667@gmail.com>
Date: Thu, 9 May 2024 14:52:13 +0800
Subject: [PATCH 2/6] docs: add `nested_type` example in
 `data_types_and_io/structured_dataset.py` (#5269)

---
 .../data_types_and_io/structureddataset.md      | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/docs/user_guide/data_types_and_io/structureddataset.md b/docs/user_guide/data_types_and_io/structureddataset.md
index 03ebfb5275..1ba165717f 100644
--- a/docs/user_guide/data_types_and_io/structureddataset.md
+++ b/docs/user_guide/data_types_and_io/structureddataset.md
@@ -237,7 +237,7 @@ You can now use `numpy.ndarray` to deserialize the parquet file to NumPy and ser
 
 ```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/master/examples/data_types_and_io/data_types_and_io/structured_dataset.py
 :caption: data_types_and_io/structured_dataset.py
-:lines: 134-147
+:lines: 134-149
 ```
 
 :::{note}
@@ -248,7 +248,20 @@ You can run the code locally as follows:
 
 ```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/master/examples/data_types_and_io/data_types_and_io/structured_dataset.py
 :caption: data_types_and_io/structured_dataset.py
-:lines: 151-155
+:lines: 153-157
+```
+
+### The nested typed columns
+
+Like most storage formats (e.g. Avro, Parquet, and BigQuery), StructuredDataset support nested field structures.
+
+:::{note}
+Nested field StructuredDataset should be run when flytekit version > 1.11.0.
+:::
+
+```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/master/examples/data_types_and_io/data_types_and_io/structured_dataset.py
+:caption: data_types_and_io/structured_dataset.py
+:lines: 159-270
 ```
 
 [flytesnacks]: https://github.com/flyteorg/flytesnacks/tree/master/examples/data_types_and_io/

From aa1f211d2944255ad042e26e3fe2946e8bb84f2d Mon Sep 17 00:00:00 2001
From: Brian Chen <50983601+zychen5186@users.noreply.github.com>
Date: Thu, 9 May 2024 01:03:24 -0700
Subject: [PATCH 3/6] Add prefetch functionality for paginator (#5310)

Signed-off-by: zychen5186 <brianchen5197@gmail.com>
---
 flytectl/cmd/get/execution.go                 |  16 +-
 flytectl/go.mod                               |   1 +
 flytectl/go.sum                               |   2 +
 .../pkg/bubbletea/bubbletea_pagination.go     | 123 ++++++++++++--
 .../bubbletea/bubbletea_pagination_util.go    | 155 ++++++++++--------
 5 files changed, 213 insertions(+), 84 deletions(-)

diff --git a/flytectl/cmd/get/execution.go b/flytectl/cmd/get/execution.go
index 910aa7640f..4c11830c86 100644
--- a/flytectl/cmd/get/execution.go
+++ b/flytectl/cmd/get/execution.go
@@ -115,12 +115,12 @@ func ExecutionToProtoMessages(l []*admin.Execution) []proto.Message {
 }
 
 func getCallBack(ctx context.Context, cmdCtx cmdCore.CommandContext) bubbletea.DataCallback {
-	return func(filter filters.Filters) []proto.Message {
+	return func(filter filters.Filters) ([]proto.Message, error) {
 		executionList, err := cmdCtx.AdminFetcherExt().ListExecution(ctx, config.GetConfig().Project, config.GetConfig().Domain, filter)
 		if err != nil {
-			return nil
+			return nil, err
 		}
-		return ExecutionToProtoMessages(executionList.Executions)
+		return ExecutionToProtoMessages(executionList.Executions), nil
 	}
 }
 
@@ -154,17 +154,15 @@ func getExecutionFunc(ctx context.Context, args []string, cmdCtx cmdCore.Command
 		return adminPrinter.Print(config.GetConfig().MustOutputFormat(), executionColumns,
 			ExecutionToProtoMessages(executions)...)
 	}
+	if config.GetConfig().Interactive {
+		err := bubbletea.Paginator(executionColumns, getCallBack(ctx, cmdCtx), execution.DefaultConfig.Filter)
+		return err
+	}
 	executionList, err := cmdCtx.AdminFetcherExt().ListExecution(ctx, config.GetConfig().Project, config.GetConfig().Domain, execution.DefaultConfig.Filter)
 	if err != nil {
 		return err
 	}
 	logger.Infof(ctx, "Retrieved %v executions", len(executionList.Executions))
-
-	if config.GetConfig().Interactive {
-		bubbletea.Paginator(executionColumns, getCallBack(ctx, cmdCtx))
-		return nil
-	}
-
 	return adminPrinter.Print(config.GetConfig().MustOutputFormat(), executionColumns,
 		ExecutionToProtoMessages(executionList.Executions)...)
 }
diff --git a/flytectl/go.mod b/flytectl/go.mod
index 0e1ce9d224..6ce6ece779 100644
--- a/flytectl/go.mod
+++ b/flytectl/go.mod
@@ -67,6 +67,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/charmbracelet/lipgloss v0.10.0 // indirect
 	github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
 	github.com/containerd/containerd v1.6.26 // indirect
 	github.com/containerd/log v0.1.0 // indirect
diff --git a/flytectl/go.sum b/flytectl/go.sum
index 0937d8c0d2..65a362dfec 100644
--- a/flytectl/go.sum
+++ b/flytectl/go.sum
@@ -153,6 +153,8 @@ github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/
 github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw=
 github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt5dywy4TcM=
 github.com/charmbracelet/bubbletea v0.25.0/go.mod h1:EN3QDR1T5ZdWmdfDzYcqOCAps45+QIJbLOBxmVNWNNg=
+github.com/charmbracelet/lipgloss v0.10.0 h1:KWeXFSexGcfahHX+54URiZGkBFazf70JNMtwg/AFW3s=
+github.com/charmbracelet/lipgloss v0.10.0/go.mod h1:Wig9DSfvANsxqkRsqj6x87irdy123SR4dOXlKa91ciE=
 github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
 github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
 github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764=
diff --git a/flytectl/pkg/bubbletea/bubbletea_pagination.go b/flytectl/pkg/bubbletea/bubbletea_pagination.go
index 3f9fc91bd6..3b8f5319b5 100644
--- a/flytectl/pkg/bubbletea/bubbletea_pagination.go
+++ b/flytectl/pkg/bubbletea/bubbletea_pagination.go
@@ -3,46 +3,128 @@ package bubbletea
 import (
 	"fmt"
 	"log"
+	"math"
 	"strings"
 
+	"github.com/charmbracelet/bubbles/key"
 	"github.com/charmbracelet/bubbles/paginator"
+	"github.com/charmbracelet/bubbles/spinner"
+	"github.com/charmbracelet/lipgloss"
+	"github.com/flyteorg/flyte/flytectl/pkg/filters"
 	"github.com/flyteorg/flyte/flytectl/pkg/printer"
 	"github.com/golang/protobuf/proto"
 
 	tea "github.com/charmbracelet/bubbletea"
 )
 
+var (
+	spin = false
+	// Avoid fetching multiple times while still fetching
+	fetchingBackward = false
+	fetchingForward  = false
+)
+
 type pageModel struct {
-	items     []proto.Message
+	items     *[]proto.Message
 	paginator paginator.Model
+	spinner   spinner.Model
 }
 
 func newModel(initMsg []proto.Message) pageModel {
 	p := paginator.New()
 	p.PerPage = msgPerPage
-	p.SetTotalPages(len(initMsg))
+	p.Page = int(filter.Page) - 1
+	// Set the upper bound of the page number
+	p.SetTotalPages(getLastMsgIdx())
+
+	s := spinner.New()
+	s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("56"))
+	s.Spinner = spinner.Points
 
 	return pageModel{
 		paginator: p,
-		items:     initMsg,
+		spinner:   s,
+		items:     &initMsg,
 	}
 }
 
 func (m pageModel) Init() tea.Cmd {
-	return nil
+	return m.spinner.Tick
 }
 
 func (m pageModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 	var cmd tea.Cmd
 	switch msg := msg.(type) {
+	case error:
+		return m, tea.Quit
 	case tea.KeyMsg:
 		switch msg.String() {
 		case "q", "esc", "ctrl+c":
 			return m, tea.Quit
 		}
+		switch {
+		case key.Matches(msg, m.paginator.KeyMap.PrevPage):
+			// If previous page will be out of the range of the first batch, don't update
+			if m.paginator.Page == firstBatchIndex*pagePerBatch {
+				return m, nil
+			}
+		}
+	case spinner.TickMsg:
+		m.spinner, cmd = m.spinner.Update(msg)
+		return m, cmd
+	case newDataMsg:
+		if msg.fetchDirection == forward {
+			// Update if current page is in the range of the last batch
+			// i.e. if user not in last batch when finished fetching, don't update
+			if m.paginator.Page/pagePerBatch >= lastBatchIndex {
+				*m.items = append(*m.items, msg.newItems...)
+				lastBatchIndex++
+				if lastBatchIndex-firstBatchIndex >= localBatchLimit {
+					*m.items = (*m.items)[batchLen[firstBatchIndex]:]
+					firstBatchIndex++
+				}
+			}
+			fetchingForward = false
+		} else {
+			// Update if current page is in the range of the first batch
+			// i.e. if user not in first batch when finished fetching, don't update
+			if m.paginator.Page/pagePerBatch <= firstBatchIndex {
+				*m.items = append(msg.newItems, *m.items...)
+				firstBatchIndex--
+				if lastBatchIndex-firstBatchIndex >= localBatchLimit {
+					*m.items = (*m.items)[:len(*m.items)-batchLen[lastBatchIndex]]
+					lastBatchIndex--
+				}
+			}
+			fetchingBackward = false
+		}
+		// Set the upper bound of the page number
+		m.paginator.SetTotalPages(getLastMsgIdx())
+		return m, nil
 	}
-	m.paginator, cmd = m.paginator.Update(msg)
-	preFetchBatch(&m)
+
+	m.paginator, _ = m.paginator.Update(msg)
+
+	switch msg := msg.(type) {
+	case tea.KeyMsg:
+		switch {
+		case key.Matches(msg, m.paginator.KeyMap.NextPage):
+			if (m.paginator.Page >= (lastBatchIndex+1)*pagePerBatch-prefetchThreshold) && !fetchingForward {
+				// If no more data, don't fetch again (won't show spinner)
+				value, ok := batchLen[lastBatchIndex+1]
+				if !ok || value != 0 {
+					fetchingForward = true
+					cmd = fetchDataCmd(lastBatchIndex+1, forward)
+				}
+			}
+		case key.Matches(msg, m.paginator.KeyMap.PrevPage):
+			if (m.paginator.Page <= firstBatchIndex*pagePerBatch+prefetchThreshold) && (firstBatchIndex > 0) && !fetchingBackward {
+				fetchingBackward = true
+				cmd = fetchDataCmd(firstBatchIndex-1, backward)
+			}
+		}
+	}
+
 	return m, cmd
 }
 
@@ -50,25 +132,46 @@ func (m pageModel) View() string {
 	var b strings.Builder
 	table, err := getTable(&m)
 	if err != nil {
-		return ""
+		return "Error rendering table"
 	}
 	b.WriteString(table)
-	b.WriteString(fmt.Sprintf("  PAGE - %d\n", m.paginator.Page+1))
+	b.WriteString(fmt.Sprintf("  PAGE - %d   ", m.paginator.Page+1))
+	if spin {
+		b.WriteString(fmt.Sprintf("%s%s", m.spinner.View(), " Loading new pages..."))
+	}
 	b.WriteString("\n\n  h/l ←/→ page • q: quit\n")
+
 	return b.String()
 }
 
-func Paginator(_listHeader []printer.Column, _callback DataCallback) {
+func Paginator(_listHeader []printer.Column, _callback DataCallback, _filter filters.Filters) error {
 	listHeader = _listHeader
 	callback = _callback
+	filter = _filter
+	filter.Page = int32(_max(int(filter.Page), 1))
+	firstBatchIndex = (int(filter.Page) - 1) / pagePerBatch
+	lastBatchIndex = firstBatchIndex
 
 	var msg []proto.Message
 	for i := firstBatchIndex; i < lastBatchIndex+1; i++ {
-		msg = append(msg, getMessageList(i)...)
+		newMessages, err := getMessageList(i)
+		if err != nil {
+			return err
+		}
+		if int(filter.Page)-(firstBatchIndex*pagePerBatch) > int(math.Ceil(float64(len(newMessages))/msgPerPage)) {
+			return fmt.Errorf("the specified page has no data, please enter a valid page number")
+		}
+		msg = append(msg, newMessages...)
 	}
 
 	p := tea.NewProgram(newModel(msg))
 	if _, err := p.Run(); err != nil {
 		log.Fatal(err)
 	}
+
+	if errMsg != nil {
+		return errMsg
+	}
+
+	return nil
 }
diff --git a/flytectl/pkg/bubbletea/bubbletea_pagination_util.go b/flytectl/pkg/bubbletea/bubbletea_pagination_util.go
index cab4255a15..21e624ffed 100644
--- a/flytectl/pkg/bubbletea/bubbletea_pagination_util.go
+++ b/flytectl/pkg/bubbletea/bubbletea_pagination_util.go
@@ -5,7 +5,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"sync"
 
+	tea "github.com/charmbracelet/bubbletea"
 	"github.com/flyteorg/flyte/flytectl/pkg/filters"
 	"github.com/flyteorg/flyte/flytectl/pkg/printer"
 
@@ -13,34 +15,49 @@ import (
 	"github.com/golang/protobuf/proto"
 )
 
-type DataCallback func(filter filters.Filters) []proto.Message
+type DataCallback func(filter filters.Filters) ([]proto.Message, error)
 
-type PrintableProto struct{ proto.Message }
+type printTableProto struct{ proto.Message }
+
+type direction int
+
+type newDataMsg struct {
+	newItems       []proto.Message
+	batchIndex     int
+	fetchDirection direction
+}
 
 const (
-	msgPerBatch  = 100 // Please set msgPerBatch as a multiple of msgPerPage
-	msgPerPage   = 10
-	pagePerBatch = msgPerBatch / msgPerPage
+	forward direction = iota
+	backward
+)
+
+const (
+	msgPerBatch       = 100 // Please set msgPerBatch as a multiple of msgPerPage
+	msgPerPage        = 10
+	pagePerBatch      = msgPerBatch / msgPerPage
+	prefetchThreshold = pagePerBatch - 1
+	localBatchLimit   = 10 // Please set localBatchLimit at least 2
 )
 
 var (
-	// Used for indexing local stored rows
-	localPageIndex int
-	// Recording batch index fetched from admin
-	firstBatchIndex int32 = 1
-	lastBatchIndex  int32 = 10
-	batchLen              = make(map[int32]int)
 	// Callback function used to fetch data from the module that called bubbletea pagination.
-	callback DataCallback
-	// The header of the table
+	callback   DataCallback
 	listHeader []printer.Column
-
-	marshaller = jsonpb.Marshaler{
-		Indent: "\t",
-	}
+	filter     filters.Filters
+	// Record the index of the first and last batch that is in cache
+	firstBatchIndex int
+	lastBatchIndex  int
+	// Record numbers of messages in each batch
+	batchLen = make(map[int]int)
+	// Avoid fetching back and forward at the same time
+	mutex sync.Mutex
+	// Used to catch error happened while running paginator
+	errMsg error = nil
 )
 
-func (p PrintableProto) MarshalJSON() ([]byte, error) {
+func (p printTableProto) MarshalJSON() ([]byte, error) {
+	marshaller := jsonpb.Marshaler{Indent: "\t"}
 	buf := new(bytes.Buffer)
 	err := marshaller.Marshal(buf, p.Message)
 	if err != nil {
@@ -49,28 +66,35 @@ func (p PrintableProto) MarshalJSON() ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
-func min(a, b int) int {
+func _max(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func _min(a, b int) int {
 	if a < b {
 		return a
 	}
 	return b
 }
 
-func getSliceBounds(idx int, length int) (start int, end int) {
-	start = idx * msgPerPage
-	end = min(idx*msgPerPage+msgPerPage, length)
+func getSliceBounds(m *pageModel) (start int, end int) {
+	start = (m.paginator.Page - firstBatchIndex*pagePerBatch) * msgPerPage
+	end = _min(start+msgPerPage, len(*m.items))
 	return start, end
 }
 
 func getTable(m *pageModel) (string, error) {
-	start, end := getSliceBounds(localPageIndex, len(m.items))
-	curShowMessage := m.items[start:end]
-	printableMessages := make([]*PrintableProto, 0, len(curShowMessage))
+	start, end := getSliceBounds(m)
+	curShowMessage := (*m.items)[start:end]
+	printTableMessages := make([]*printTableProto, 0, len(curShowMessage))
 	for _, m := range curShowMessage {
-		printableMessages = append(printableMessages, &PrintableProto{Message: m})
+		printTableMessages = append(printTableMessages, &printTableProto{Message: m})
 	}
 
-	jsonRows, err := json.Marshal(printableMessages)
+	jsonRows, err := json.Marshal(printTableMessages)
 	if err != nil {
 		return "", fmt.Errorf("failed to marshal proto messages")
 	}
@@ -84,53 +108,54 @@ func getTable(m *pageModel) (string, error) {
 	return buf.String(), nil
 }
 
-func getMessageList(batchIndex int32) []proto.Message {
-	msg := callback(filters.Filters{
+func getMessageList(batchIndex int) ([]proto.Message, error) {
+	mutex.Lock()
+	spin = true
+	defer func() {
+		spin = false
+		mutex.Unlock()
+	}()
+
+	msg, err := callback(filters.Filters{
 		Limit:  msgPerBatch,
-		Page:   batchIndex,
-		SortBy: "created_at",
-		Asc:    false,
+		Page:   int32(batchIndex + 1),
+		SortBy: filter.SortBy,
+		Asc:    filter.Asc,
 	})
+	if err != nil {
+		return nil, err
+	}
+
 	batchLen[batchIndex] = len(msg)
 
-	return msg
+	return msg, nil
 }
 
-func countTotalPages() int {
-	sum := 0
-	for _, l := range batchLen {
-		sum += l
+func fetchDataCmd(batchIndex int, fetchDirection direction) tea.Cmd {
+	return func() tea.Msg {
+		newItems, err := getMessageList(batchIndex)
+		if err != nil {
+			errMsg = err
+			return err
+		}
+		msg := newDataMsg{
+			newItems:       newItems,
+			batchIndex:     batchIndex,
+			fetchDirection: fetchDirection,
+		}
+		return msg
 	}
-	return sum
 }
 
-// Only (lastBatchIndex-firstBatchIndex)*msgPerBatch of rows are stored in local memory.
-// When user tries to get rows out of this range, this function will be triggered.
-func preFetchBatch(m *pageModel) {
-	localPageIndex = m.paginator.Page - int(firstBatchIndex-1)*pagePerBatch
-
-	// Triggers when user is at the last local page
-	if localPageIndex+1 == len(m.items)/msgPerPage {
-		newMessages := getMessageList(lastBatchIndex + 1)
-		m.paginator.SetTotalPages(countTotalPages())
-		if len(newMessages) != 0 {
-			lastBatchIndex++
-			m.items = append(m.items, newMessages...)
-			m.items = m.items[batchLen[firstBatchIndex]:] // delete the msgs in the "firstBatchIndex" batch
-			localPageIndex -= batchLen[firstBatchIndex] / msgPerPage
-			firstBatchIndex++
+func getLastMsgIdx() int {
+	sum := 0
+	for i := 0; i < lastBatchIndex+1; i++ {
+		length, ok := batchLen[i]
+		if ok {
+			sum += length
+		} else {
+			sum += msgPerBatch
 		}
-		return
-	}
-	// Triggers when user is at the first local page
-	if localPageIndex == 0 && firstBatchIndex > 1 {
-		newMessages := getMessageList(firstBatchIndex - 1)
-		m.paginator.SetTotalPages(countTotalPages())
-		firstBatchIndex--
-		m.items = append(newMessages, m.items...)
-		m.items = m.items[:len(m.items)-batchLen[lastBatchIndex]] // delete the msgs in the "lastBatchIndex" batch
-		localPageIndex += batchLen[firstBatchIndex] / msgPerPage
-		lastBatchIndex--
-		return
 	}
+	return sum
 }

From c0f5b10777fbdfcf2b08059a270b04f0c60d7678 Mon Sep 17 00:00:00 2001
From: Daniel Sola <40698988+dansola@users.noreply.github.com>
Date: Thu, 9 May 2024 09:29:15 -0700
Subject: [PATCH 4/6] add maptask cache clarification (#5340)

* add maptask cache clarification

Signed-off-by: Daniel Sola <daniel.sola@union.ai>

* grammar

Signed-off-by: Daniel Sola <daniel.sola@union.ai>

* Update docs/user_guide/advanced_composition/map_tasks.md

Co-authored-by: Nikki Everett <neverett@users.noreply.github.com>
Signed-off-by: Daniel Sola <40698988+dansola@users.noreply.github.com>

---------

Signed-off-by: Daniel Sola <daniel.sola@union.ai>
Signed-off-by: Daniel Sola <40698988+dansola@users.noreply.github.com>
Co-authored-by: Nikki Everett <neverett@users.noreply.github.com>
---
 docs/user_guide/advanced_composition/map_tasks.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/user_guide/advanced_composition/map_tasks.md b/docs/user_guide/advanced_composition/map_tasks.md
index 8c3127fc4e..bae49faa82 100644
--- a/docs/user_guide/advanced_composition/map_tasks.md
+++ b/docs/user_guide/advanced_composition/map_tasks.md
@@ -58,6 +58,7 @@ def map_workflow_with_metadata(data: list[int] = [10, 12, 11, 10, 13, 12, 100, 1
         data_point=data
     )
 ```
+When `cache` and `cache_version` are used in `TaskMetadata` for a map task, the cache hits occur on individual tasks being mapped over, rather than the parent map task operation. This means that if one input item in a list changes, each previously executed task is read from cache and only the task for the changed item is actually executed, rather than the task being re-executed for every item.  Note that this has the same effect as adding `cache` and `cache_version` in the `@task` decorator for a task being mapped over.
 
 You can also configure `concurrency` and `min_success_ratio` for a map task:
 - `concurrency` limits the number of mapped tasks that can run in parallel to the specified batch size.

From ee6037bfdb9498197baef48f4098a16d3a1bdb34 Mon Sep 17 00:00:00 2001
From: Haytham Abuelfutuh <haytham@afutuh.com>
Date: Thu, 9 May 2024 13:59:30 -0700
Subject: [PATCH 5/6] Add Google Tag Manager (#5341)

* Add Google Tag Manager

Signed-off-by: Haytham Abuelfutuh <haytham@afutuh.com>

* Remove scarf (to be added through GTM)

Signed-off-by: Haytham Abuelfutuh <haytham@afutuh.com>

* Update base.html

Signed-off-by: Haytham Abuelfutuh <haytham@afutuh.com>

* PR Feedback

Signed-off-by: Haytham Abuelfutuh <haytham@afutuh.com>

---------

Signed-off-by: Haytham Abuelfutuh <haytham@afutuh.com>
---
 docs/_templates/base.html | 11 +++++++++++
 docs/_templates/page.html |  5 +----
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/docs/_templates/base.html b/docs/_templates/base.html
index 0d8cf4ac36..63094834f5 100644
--- a/docs/_templates/base.html
+++ b/docs/_templates/base.html
@@ -1,6 +1,13 @@
 <!doctype html>
 <html class="no-js"{% if language is not none %} lang="{{ language }}"{% endif %}>
   <head>
+    <!-- Google Tag Manager -->
+    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
+    new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
+    j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
+    'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
+    })(window,document,'script','dataLayer','GTM-WC8V9XS');</script>
+    <!-- End Google Tag Manager -->
     {%- block site_meta -%}
     <meta charset="utf-8"/>
     <meta name="viewport" content="width=device-width,initial-scale=1"/>
@@ -107,6 +114,10 @@
       runllm-name="RunLLM" />
   </head>
   <body>
+    <!-- Google Tag Manager (noscript) -->
+    <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-WC8V9XS"
+    height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
+    <!-- End Google Tag Manager (noscript) -->
     {% block body %}
     <script>
       document.body.dataset.theme = localStorage.getItem("theme") || "auto";
diff --git a/docs/_templates/page.html b/docs/_templates/page.html
index 635cdd10f2..d078e9dd5e 100644
--- a/docs/_templates/page.html
+++ b/docs/_templates/page.html
@@ -203,10 +203,7 @@
             {%- endif %}
           </div>
         </div>
-        {% endblock footer %}    
-        <div class="extra_footer">
-          <img referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=65df87cf-549c-4711-b01c-ee9e966c931c" />
-        </div>
+        {% endblock footer %}
       </footer>
     </div>
     <aside class="toc-drawer{% if furo_hide_toc %} no-toc{% endif %}">

From 2f38d65e45b1f098393feea72d8906b5aa645ade Mon Sep 17 00:00:00 2001
From: Thomas Newton <thomas.w.newton@gmail.com>
Date: Sun, 12 May 2024 12:54:03 +0100
Subject: [PATCH 6/6] Grafana dashboard updates (#5255)

* Add enqueued workflows graph

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Update queue metrics

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Better aggregations

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Add enqueued workflows graph

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Fix trailing comma

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Add transition latency, etc

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Fix

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Check-in latest dashboard

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Fixes

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Add informer stats to dashboard

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Add more stats on workflow store

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* More round metrics

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Fix Y axis labels

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Check-in latest dashboard

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Add garbage collection stats

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Fix scale on cache hit rate

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Working flytepropeller grpc histograms

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Change to a single accumulated graph

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Add GRPC histogram on flyteadmin too

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Fix admin metrics name

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Update workqueue metric names for flyte 1.11.0

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Add other queues to queue graphs

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Add round success counter, remove workqueue latencies

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Split enqueues by type and add missing rate for skip rate metric

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Don't use rate for round latency graphs

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Fix metric name for plugin failures and make it a rate

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Fix duplicate refIds

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Fix typo

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Update lots of title and units

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Add round errors graph

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Add total round rate

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Group metrics onto the same graphs were relevant

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Small updates

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Fix streaks graph

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* rename legend

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Rename

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Add descriptions

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Add a missing title

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Remove unused imports

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* Adjust a description

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

* More minor updates

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>

---------

Signed-off-by: Thomas Newton <thomas.w.newton@gmail.com>
---
 .../prometheus/flyteadmin-dashboard.json      |  406 +-
 .../prometheus/flytepropeller-dashboard.json  | 3407 +++++++++++++++--
 stats/flyteadmin.dashboard.py                 |   35 +-
 stats/flytepropeller.dashboard.py             |  679 +++-
 4 files changed, 3785 insertions(+), 742 deletions(-)

diff --git a/deployment/stats/prometheus/flyteadmin-dashboard.json b/deployment/stats/prometheus/flyteadmin-dashboard.json
index daef170643..5e88094e4c 100644
--- a/deployment/stats/prometheus/flyteadmin-dashboard.json
+++ b/deployment/stats/prometheus/flyteadmin-dashboard.json
@@ -22,6 +22,124 @@
   "panels": [],
   "refresh": "10s",
   "rows": [
+    {
+      "collapse": true,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "index": 0,
+                    "line": true,
+                    "op": "gt",
+                    "value": "null",
+                    "yaxis": "left"
+                  },
+                  {
+                    "color": "red",
+                    "index": 1,
+                    "line": true,
+                    "op": "gt",
+                    "value": 80.0,
+                    "yaxis": "left"
+                  }
+                ]
+              }
+            }
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 1,
+          "interval": null,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "options": {
+            "displayMode": "gradient",
+            "fieldOptions": {
+              "calcs": [
+                "sum"
+              ],
+              "defaults": {
+                "decimals": null,
+                "links": [],
+                "max": 200,
+                "min": 0,
+                "title": null,
+                "unit": "none"
+              },
+              "limit": null,
+              "mappings": [],
+              "override": {},
+              "thresholds": [
+                {
+                  "color": "green",
+                  "index": 0,
+                  "line": true,
+                  "op": "gt",
+                  "value": "null",
+                  "yaxis": "left"
+                },
+                {
+                  "color": "red",
+                  "index": 1,
+                  "line": true,
+                  "op": "gt",
+                  "value": 80.0,
+                  "yaxis": "left"
+                }
+              ],
+              "values": false
+            },
+            "orientation": "vertical",
+            "showThresholdLabels": false,
+            "showThresholdMarkers": true
+          },
+          "repeat": null,
+          "repeatDirection": null,
+          "span": 12,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum by(le) (rate(grpc_server_handling_seconds_bucket[5m]))",
+              "format": "heatmap",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{le}}",
+              "metric": "",
+              "query": "sum by(le) (rate(grpc_server_handling_seconds_bucket[5m]))",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "All GRPC calls latency",
+          "transformations": [],
+          "transparent": false,
+          "type": "bargauge"
+        }
+      ],
+      "repeat": null,
+      "showTitle": true,
+      "title": "GRPC latency metrics"
+    },
     {
       "collapse": true,
       "editable": true,
@@ -53,7 +171,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 1,
+          "id": 2,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -180,7 +298,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 2,
+          "id": 3,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -307,7 +425,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 3,
+          "id": 4,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -434,7 +552,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 4,
+          "id": 5,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -561,7 +679,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 5,
+          "id": 6,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -688,7 +806,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 6,
+          "id": 7,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -815,7 +933,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 7,
+          "id": 8,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -952,7 +1070,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 8,
+          "id": 9,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -1079,7 +1197,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 9,
+          "id": 10,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -1206,7 +1324,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 10,
+          "id": 11,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -1333,7 +1451,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 11,
+          "id": 12,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -1460,7 +1578,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 12,
+          "id": 13,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -1587,7 +1705,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 13,
+          "id": 14,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -1714,7 +1832,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 14,
+          "id": 15,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -1851,7 +1969,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 15,
+          "id": 16,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -1978,7 +2096,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 16,
+          "id": 17,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -2105,7 +2223,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 17,
+          "id": 18,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -2232,7 +2350,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 18,
+          "id": 19,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -2359,7 +2477,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 19,
+          "id": 20,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -2486,7 +2604,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 20,
+          "id": 21,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -2613,7 +2731,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 21,
+          "id": 22,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -2750,7 +2868,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 22,
+          "id": 23,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -2877,7 +2995,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 23,
+          "id": 24,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3004,7 +3122,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 24,
+          "id": 25,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3131,7 +3249,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 25,
+          "id": 26,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3258,7 +3376,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 26,
+          "id": 27,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3385,7 +3503,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 27,
+          "id": 28,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3512,7 +3630,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 28,
+          "id": 29,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3649,7 +3767,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 29,
+          "id": 30,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3776,7 +3894,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 30,
+          "id": 31,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3903,7 +4021,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 31,
+          "id": 32,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4030,7 +4148,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 32,
+          "id": 33,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4157,7 +4275,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 33,
+          "id": 34,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4284,7 +4402,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 34,
+          "id": 35,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4411,7 +4529,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 35,
+          "id": 36,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4548,7 +4666,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 36,
+          "id": 37,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4675,7 +4793,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 37,
+          "id": 38,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4802,7 +4920,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 38,
+          "id": 39,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4929,7 +5047,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 39,
+          "id": 40,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -5056,7 +5174,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 40,
+          "id": 41,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -5183,7 +5301,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 41,
+          "id": 42,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -5310,7 +5428,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 42,
+          "id": 43,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -5447,7 +5565,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 43,
+          "id": 44,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -5574,7 +5692,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 44,
+          "id": 45,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -5701,7 +5819,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 45,
+          "id": 46,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -5828,7 +5946,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 46,
+          "id": 47,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -5955,7 +6073,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 47,
+          "id": 48,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -6082,7 +6200,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 48,
+          "id": 49,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -6209,7 +6327,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 49,
+          "id": 50,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -6346,7 +6464,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 50,
+          "id": 51,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -6473,7 +6591,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 51,
+          "id": 52,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -6600,7 +6718,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 52,
+          "id": 53,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -6727,7 +6845,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 53,
+          "id": 54,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -6854,7 +6972,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 54,
+          "id": 55,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -6981,7 +7099,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 55,
+          "id": 56,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -7108,7 +7226,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 56,
+          "id": 57,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -7245,7 +7363,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 57,
+          "id": 58,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -7372,7 +7490,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 58,
+          "id": 59,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -7499,7 +7617,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 59,
+          "id": 60,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -7626,7 +7744,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 60,
+          "id": 61,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -7753,7 +7871,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 61,
+          "id": 62,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -7880,7 +7998,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 62,
+          "id": 63,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -8007,7 +8125,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 63,
+          "id": 64,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -8144,7 +8262,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 64,
+          "id": 65,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -8271,7 +8389,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 65,
+          "id": 66,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -8398,7 +8516,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 66,
+          "id": 67,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -8525,7 +8643,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 67,
+          "id": 68,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -8652,7 +8770,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 68,
+          "id": 69,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -8779,7 +8897,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 69,
+          "id": 70,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -8906,7 +9024,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 70,
+          "id": 71,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -9043,7 +9161,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 71,
+          "id": 72,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -9170,7 +9288,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 72,
+          "id": 73,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -9297,7 +9415,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 73,
+          "id": 74,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -9424,7 +9542,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 74,
+          "id": 75,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -9551,7 +9669,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 75,
+          "id": 76,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -9678,7 +9796,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 76,
+          "id": 77,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -9805,7 +9923,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 77,
+          "id": 78,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -9942,7 +10060,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 78,
+          "id": 79,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -10069,7 +10187,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 79,
+          "id": 80,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -10196,7 +10314,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 80,
+          "id": 81,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -10323,7 +10441,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 81,
+          "id": 82,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -10450,7 +10568,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 82,
+          "id": 83,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -10577,7 +10695,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 83,
+          "id": 84,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -10704,7 +10822,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 84,
+          "id": 85,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -10841,7 +10959,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 85,
+          "id": 86,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11013,7 +11131,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 86,
+          "id": 87,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11155,7 +11273,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 87,
+          "id": 88,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11292,7 +11410,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 88,
+          "id": 89,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11464,7 +11582,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 89,
+          "id": 90,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11606,7 +11724,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 90,
+          "id": 91,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11743,7 +11861,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 91,
+          "id": 92,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11915,7 +12033,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 92,
+          "id": 93,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12057,7 +12175,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 93,
+          "id": 94,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12194,7 +12312,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 94,
+          "id": 95,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12366,7 +12484,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 95,
+          "id": 96,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12508,7 +12626,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 96,
+          "id": 97,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12645,7 +12763,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 97,
+          "id": 98,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12817,7 +12935,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 98,
+          "id": 99,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12959,7 +13077,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 99,
+          "id": 100,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13096,7 +13214,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 100,
+          "id": 101,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13268,7 +13386,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 101,
+          "id": 102,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13410,7 +13528,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 102,
+          "id": 103,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13547,7 +13665,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 103,
+          "id": 104,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13719,7 +13837,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 104,
+          "id": 105,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13861,7 +13979,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 105,
+          "id": 106,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13998,7 +14116,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 106,
+          "id": 107,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14170,7 +14288,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 107,
+          "id": 108,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14312,7 +14430,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 108,
+          "id": 109,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14449,7 +14567,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 109,
+          "id": 110,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14621,7 +14739,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 110,
+          "id": 111,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14763,7 +14881,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 111,
+          "id": 112,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14900,7 +15018,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 112,
+          "id": 113,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15072,7 +15190,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 113,
+          "id": 114,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15214,7 +15332,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 114,
+          "id": 115,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15351,7 +15469,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 115,
+          "id": 116,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15523,7 +15641,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 116,
+          "id": 117,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15665,7 +15783,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 117,
+          "id": 118,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15802,7 +15920,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 118,
+          "id": 119,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15974,7 +16092,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 119,
+          "id": 120,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16116,7 +16234,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 120,
+          "id": 121,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16253,7 +16371,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 121,
+          "id": 122,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16425,7 +16543,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 122,
+          "id": 123,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16567,7 +16685,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 123,
+          "id": 124,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16704,7 +16822,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 124,
+          "id": 125,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16876,7 +16994,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 125,
+          "id": 126,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17018,7 +17136,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 126,
+          "id": 127,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17155,7 +17273,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 127,
+          "id": 128,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17327,7 +17445,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 128,
+          "id": 129,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17469,7 +17587,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 129,
+          "id": 130,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17606,7 +17724,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 130,
+          "id": 131,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17778,7 +17896,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 131,
+          "id": 132,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17920,7 +18038,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 132,
+          "id": 133,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18057,7 +18175,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 133,
+          "id": 134,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18229,7 +18347,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 134,
+          "id": 135,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18371,7 +18489,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 135,
+          "id": 136,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18508,7 +18626,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 136,
+          "id": 137,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18680,7 +18798,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 137,
+          "id": 138,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18822,7 +18940,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 138,
+          "id": 139,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18959,7 +19077,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 139,
+          "id": 140,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19131,7 +19249,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 140,
+          "id": 141,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19273,7 +19391,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 141,
+          "id": 142,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19410,7 +19528,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 142,
+          "id": 143,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19582,7 +19700,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 143,
+          "id": 144,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19724,7 +19842,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 144,
+          "id": 145,
           "interval": null,
           "isNew": true,
           "legend": {
diff --git a/deployment/stats/prometheus/flytepropeller-dashboard.json b/deployment/stats/prometheus/flytepropeller-dashboard.json
index 8bb8a61a4e..12ffbd26c5 100644
--- a/deployment/stats/prometheus/flytepropeller-dashboard.json
+++ b/deployment/stats/prometheus/flytepropeller-dashboard.json
@@ -32,7 +32,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_PROM}",
-          "description": null,
+          "description": "The number of golang goroutines available to accept new work from the main workqueue. Each worker can process one item from the workqueue at a time.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -132,7 +132,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ops",
+              "format": "none",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -159,7 +159,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_PROM}",
-          "description": null,
+          "description": "Round success, error and total rates. Also includes total rate including streaks within a single round. The streak rate graph should match the difference between the totals with and without streaks.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -223,24 +223,69 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:round:abort_error[5m]))",
+              "expr": "sum(rate(flyte:propeller:all:round:success_count[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "",
+              "legendFormat": "success",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:round:abort_error[5m]))",
+              "query": "sum(rate(flyte:propeller:all:round:success_count[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:round:error_count[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "error",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:round:error_count[5m]))",
+              "refId": "B",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:round:round_total_ms_count[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "total",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:round:round_total_ms_count[5m]))",
+              "refId": "C",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:round:round_time_unlabeled_ms_count[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "total-including-streaks",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:round:round_time_unlabeled_ms_count[5m]))",
+              "refId": "D",
+              "step": 10,
+              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "System errors",
+          "title": "Round success/error rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -286,7 +331,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_PROM}",
-          "description": null,
+          "description": "Error rates for each type of failure that may occur as propeller is traversing the workflow DAG.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -350,24 +395,84 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(deriv(flyte:propeller:all:round:system_error_unlabeled[5m]))",
+              "expr": "sum(rate(flyte:propeller:all:round:system_error_unlabeled[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "",
+              "legendFormat": "system",
               "metric": "",
-              "query": "sum(deriv(flyte:propeller:all:round:system_error_unlabeled[5m]))",
+              "query": "sum(rate(flyte:propeller:all:round:system_error_unlabeled[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:round:abort_error_unlabeled[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "abort",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:round:abort_error_unlabeled[5m]))",
+              "refId": "B",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:round:panic_unlabeled[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "panic",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:round:panic_unlabeled[5m]))",
+              "refId": "C",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:round:not_found[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "not-found",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:round:not_found[5m]))",
+              "refId": "D",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:round:skipped[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "skipped",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:round:skipped[5m]))",
+              "refId": "E",
+              "step": 10,
+              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "System errors",
+          "title": "Error rate breakdown",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -413,7 +518,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_PROM}",
-          "description": null,
+          "description": "Streaks are when propeller iterates over the same workflow multiple times in a single round. This is an optimisation technique.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -477,7 +582,7 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "{__name__=~\"flyte:propeller:all:node:plugin:.*_failure_unlabeled\"}",
+              "expr": "sum(rate(flyte:propeller:all:round:streak_length_unlabeled[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -485,31 +590,16 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "{__name__=~\"flyte:propeller:all:node:plugin:.*_failure_unlabeled\"}",
+              "query": "sum(rate(flyte:propeller:all:round:streak_length_unlabeled[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
-            },
-            {
-              "datasource": null,
-              "expr": "{__name__=~\"flyte:propeller:all:node:plugin:.*_success_unlabeled\"}",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "",
-              "metric": "",
-              "query": "{__name__=~\"flyte:propeller:all:node:plugin:.*_success_unlabeled\"}",
-              "refId": "B",
-              "step": 10,
-              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Plugin Failures",
+          "title": "Streak rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -555,7 +645,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_PROM}",
-          "description": null,
+          "description": "Success vs failure rate for the various plugins used for each node e.g. k8s plugin or spark plugin.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -619,24 +709,39 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:round:raw_unlabeled_ms[5m])) by (quantile)",
+              "expr": "sum(rate({__name__=~\"flyte:propeller:all:plugin:.*_success_unlabeled\"}[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "",
+              "legendFormat": "success",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:round:raw_unlabeled_ms[5m])) by (quantile)",
+              "query": "sum(rate({__name__=~\"flyte:propeller:all:plugin:.*_success_unlabeled\"}[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate({__name__=~\"flyte:propeller:all:plugin:.*_failure_unlabeled\"}[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "failure",
+              "metric": "",
+              "query": "sum(rate({__name__=~\"flyte:propeller:all:plugin:.*_failure_unlabeled\"}[5m]))",
+              "refId": "B",
+              "step": 10,
+              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "round Latency by quantile",
+          "title": "Plugin Success/Failure rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -655,7 +760,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ms",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -682,7 +787,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_PROM}",
-          "description": null,
+          "description": "Round latency breakdown. When there are streaks within one round each iteration is measured separately.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -746,24 +851,69 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:round:raw_ms[5m])) by (wf)",
+              "expr": "sum(flyte:propeller:all:round:raw_unlabeled_ms) by (quantile)",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "",
+              "legendFormat": "traverse-P{{quantile}}",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:round:raw_ms[5m])) by (wf)",
+              "query": "sum(flyte:propeller:all:round:raw_unlabeled_ms) by (quantile)",
               "refId": "A",
               "step": 10,
               "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "avg(flyte:propeller:all:round:raw_unlabeled_ms_sum/flyte:propeller:all:round:raw_unlabeled_ms_count)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "traverse-mean",
+              "metric": "",
+              "query": "avg(flyte:propeller:all:round:raw_unlabeled_ms_sum/flyte:propeller:all:round:raw_unlabeled_ms_count)",
+              "refId": "B",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:round:round_time_ms) by (quantile)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "total-P{{quantile}}",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:round:round_time_ms) by (quantile)",
+              "refId": "C",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "avg(flyte:propeller:all:round:round_time_unlabeled_ms_sum/flyte:propeller:all:round:round_time_unlabeled_ms_count)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "total-mean",
+              "metric": "",
+              "query": "avg(flyte:propeller:all:round:round_time_unlabeled_ms_sum/flyte:propeller:all:round:round_time_unlabeled_ms_count)",
+              "refId": "D",
+              "step": 10,
+              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "round Latency per workflow",
+          "title": "Round Latency (includes streak rounds)",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -809,7 +959,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_PROM}",
-          "description": null,
+          "description": "Round latency by workflow name. When there are streaks within one round each iteration is measured separately.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -873,15 +1023,15 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:round:panic_unlabeled[5m]))",
+              "expr": "sum(flyte:propeller:all:round:raw_ms) by (wf)",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "",
+              "legendFormat": "{{wf}}",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:round:panic_unlabeled[5m]))",
+              "query": "sum(flyte:propeller:all:round:raw_ms) by (wf)",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -890,7 +1040,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Round panic",
+          "title": "Round traverse latency per workflow",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -909,7 +1059,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ops",
+              "format": "ms",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -936,7 +1086,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_PROM}",
-          "description": null,
+          "description": "Count of currently running workflows running per project",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -1036,7 +1186,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ops",
+              "format": "none",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -1057,23 +1207,13 @@
             "align": false,
             "alignLevel": 0
           }
-        }
-      ],
-      "repeat": null,
-      "showTitle": true,
-      "title": "Core metrics"
-    },
-    {
-      "collapse": true,
-      "editable": true,
-      "height": "250px",
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_PROM}",
-          "description": null,
+          "description": "Rate at which workflows are being enqueued by flytepropeller. These enqueues all pass through the sub-queue before going back into the main queue.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -1131,13 +1271,13 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 6,
+          "span": 2,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "(sum(rate(flyte:propeller:all:metastore:cache_hit[5m])) * 100) / (sum(rate(flyte:propeller:all:metastore:cache_miss[5m])) + sum(rate(flyte:propeller:all:metastore:cache_hit[5m])))",
+              "expr": "sum by(type) (rate(flyte:propeller:all:wf_enqueue[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -1145,7 +1285,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "(sum(rate(flyte:propeller:all:metastore:cache_hit[5m])) * 100) / (sum(rate(flyte:propeller:all:metastore:cache_miss[5m])) + sum(rate(flyte:propeller:all:metastore:cache_hit[5m])))",
+              "query": "sum by(type) (rate(flyte:propeller:all:wf_enqueue[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -1154,7 +1294,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "cache hit percentage",
+          "title": "Workflow enqueue rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -1173,7 +1313,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "percent",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -1194,13 +1334,23 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "showTitle": true,
+      "title": "Core metrics"
+    },
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_PROM}",
-          "description": null,
+          "description": "Rate at which items are actually added to the queue. If an item is already on the queue attempting to add it will be a no-op. Usually there is also rate limiting that will delay items from being added to the queue.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -1258,90 +1408,60 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 6,
+          "span": 3,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:metastore:head_failure_unlabeled[5m]))",
+              "expr": "sum(rate(flyte:propeller:all:main_adds[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "head-failure",
+              "legendFormat": "main",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:metastore:head_failure_unlabeled[5m]))",
+              "query": "sum(rate(flyte:propeller:all:main_adds[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
             },
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:metastore:bad_container_unlabeled[5m]))",
+              "expr": "sum(rate(flyte:propeller:all:sub_adds[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "bad-container",
+              "legendFormat": "sub",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:metastore:bad_container_unlabeled[5m]))",
+              "query": "sum(rate(flyte:propeller:all:sub_adds[5m]))",
               "refId": "B",
               "step": 10,
               "target": ""
             },
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:metastore:bad_key_unlabeled[5m]))",
+              "expr": "sum(rate(flyte:propeller:all:admin_launcher:_adds[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "bad-key",
+              "legendFormat": "admin_launcher",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:metastore:bad_key_unlabeled[5m]))",
+              "query": "sum(rate(flyte:propeller:all:admin_launcher:_adds[5m]))",
               "refId": "C",
               "step": 10,
               "target": ""
-            },
-            {
-              "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:metastore:read_failure_unlabeled[5m]))",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "read-failure",
-              "metric": "",
-              "query": "sum(rate(flyte:propeller:all:metastore:read_failure_unlabeled[5m]))",
-              "refId": "D",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:metastore:write_failure_unlabeled[5m]))",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "write-failure",
-              "metric": "",
-              "query": "sum(rate(flyte:propeller:all:metastore:write_failure_unlabeled[5m]))",
-              "refId": "E",
-              "step": 10,
-              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Failures from metastore",
+          "title": "Add rate to queue",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -1381,23 +1501,13 @@
             "align": false,
             "alignLevel": 0
           }
-        }
-      ],
-      "repeat": null,
-      "showTitle": true,
-      "title": "Metastore failures and cache"
-    },
-    {
-      "collapse": true,
-      "editable": true,
-      "height": "250px",
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_PROM}",
-          "description": null,
+          "description": "Tracks every rate limited add synchronously before rate limiting delays or deduplication",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -1461,24 +1571,54 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:metastore:copy:overall_unlabeled_ms) by (quantile)",
+              "expr": "sum(rate(flyte:propeller:all:main_retries[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "",
+              "legendFormat": "main",
               "metric": "",
-              "query": "sum(flyte:propeller:all:metastore:copy:overall_unlabeled_ms) by (quantile)",
+              "query": "sum(rate(flyte:propeller:all:main_retries[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:sub_retries[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "sub",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:sub_retries[5m]))",
+              "refId": "B",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:admin_launcher:_retries[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "admin_launcher",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:admin_launcher:_retries[5m]))",
+              "refId": "C",
+              "step": 10,
+              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Metastore copy latency",
+          "title": "Add rate before rate limiting and deduplication",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -1497,7 +1637,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ms",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -1524,7 +1664,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_PROM}",
-          "description": null,
+          "description": "The number of items that are currently in the queue but have not been processed yet.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -1588,24 +1728,54 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:metastore:write_ms) by (quantile, wf)",
+              "expr": "sum(flyte:propeller:all:main_depth)",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "",
+              "legendFormat": "main",
               "metric": "",
-              "query": "sum(flyte:propeller:all:metastore:write_ms) by (quantile, wf)",
+              "query": "sum(flyte:propeller:all:main_depth)",
               "refId": "A",
               "step": 10,
               "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:sub_depth)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "sub",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:sub_depth)",
+              "refId": "B",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:admin_launcher:_depth)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "admin_launcher",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:admin_launcher:_depth)",
+              "refId": "C",
+              "step": 10,
+              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Metastore write latency by workflow",
+          "title": "Unprocessed Queue depth",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -1624,7 +1794,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ms",
+              "format": "short",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -1651,7 +1821,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_PROM}",
-          "description": null,
+          "description": "Sum of the current in progress time of every in progress item in the queue.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -1715,24 +1885,54 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:metastore:read_open_ms) by (quantile, wf)",
+              "expr": "sum(flyte:propeller:all:main_unfinished_work_s)",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "",
+              "legendFormat": "main",
               "metric": "",
-              "query": "sum(flyte:propeller:all:metastore:read_open_ms) by (quantile, wf)",
+              "query": "sum(flyte:propeller:all:main_unfinished_work_s)",
               "refId": "A",
               "step": 10,
               "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:sub_unfinished_work_s)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "sub",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:sub_unfinished_work_s)",
+              "refId": "B",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:admin_launcher:_unfinished_work_s)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "admin_launcher",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:admin_launcher:_unfinished_work_s)",
+              "refId": "C",
+              "step": 10,
+              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Metastore read open latency by workflow",
+          "title": "Seconds of unfinished work in progress",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -1751,7 +1951,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ms",
+              "format": "s",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -1772,7 +1972,17 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "showTitle": true,
+      "title": "FlytePropeller Queue metrics"
+    },
+    {
+      "collapse": true,
+      "editable": true,
+      "height": "250px",
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
@@ -1836,13 +2046,13 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 3,
+          "span": 6,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:metastore:head_ms) by (quantile, wf)",
+              "expr": "(sum(rate(flyte:propeller:all:metastore:cache_hit[5m])) * 100) / (sum(rate(flyte:propeller:all:metastore:cache_miss[5m])) + sum(rate(flyte:propeller:all:metastore:cache_hit[5m])))",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -1850,7 +2060,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(flyte:propeller:all:metastore:head_ms) by (quantile, wf)",
+              "query": "(sum(rate(flyte:propeller:all:metastore:cache_hit[5m])) * 100) / (sum(rate(flyte:propeller:all:metastore:cache_miss[5m])) + sum(rate(flyte:propeller:all:metastore:cache_hit[5m])))",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -1859,7 +2069,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Metastore head latency by workflow",
+          "title": "cache hit percentage",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -1878,7 +2088,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ms",
+              "format": "percent",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -1963,45 +2173,90 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 3,
+          "span": 6,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:metastore:proto_fetch_ms) by (quantile, wf)",
+              "expr": "sum(rate(flyte:propeller:all:metastore:head_failure_unlabeled[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "proto-fetch",
+              "legendFormat": "head-failure",
               "metric": "",
-              "query": "sum(flyte:propeller:all:metastore:proto_fetch_ms) by (quantile, wf)",
+              "query": "sum(rate(flyte:propeller:all:metastore:head_failure_unlabeled[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
             },
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:metastore:remote_fetch_ms) by (quantile, wf)",
+              "expr": "sum(rate(flyte:propeller:all:metastore:bad_container_unlabeled[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "remote-fetch",
+              "legendFormat": "bad-container",
               "metric": "",
-              "query": "sum(flyte:propeller:all:metastore:remote_fetch_ms) by (quantile, wf)",
+              "query": "sum(rate(flyte:propeller:all:metastore:bad_container_unlabeled[5m]))",
               "refId": "B",
               "step": 10,
               "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:metastore:bad_key_unlabeled[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "bad-key",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:metastore:bad_key_unlabeled[5m]))",
+              "refId": "C",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:metastore:read_failure_unlabeled[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "read-failure",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:metastore:read_failure_unlabeled[5m]))",
+              "refId": "D",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:metastore:write_failure_unlabeled[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "write-failure",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:metastore:write_failure_unlabeled[5m]))",
+              "refId": "E",
+              "step": 10,
+              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Metastore fetch latency by workflow",
+          "title": "Metastore failure rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -2020,7 +2275,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ms",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -2045,7 +2300,7 @@
       ],
       "repeat": null,
       "showTitle": true,
-      "title": "Metastore latencies"
+      "title": "Metastore failures and cache"
     },
     {
       "collapse": true,
@@ -2121,7 +2376,7 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:node:node_exec_latency_us) by (quantile, wf) / 1000",
+              "expr": "sum(flyte:propeller:all:metastore:copy:overall_unlabeled_ms) by (quantile)",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -2129,7 +2384,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(flyte:propeller:all:node:node_exec_latency_us) by (quantile, wf) / 1000",
+              "query": "sum(flyte:propeller:all:metastore:copy:overall_unlabeled_ms) by (quantile)",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -2138,7 +2393,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Node Exec latency quantile and workflow",
+          "title": "Metastore copy latency",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -2248,7 +2503,7 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:node:node_input_latency_ms) by (quantile, wf)",
+              "expr": "sum(flyte:propeller:all:metastore:write_ms) by (quantile, wf)",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -2256,7 +2511,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(flyte:propeller:all:node:node_input_latency_ms) by (quantile, wf)",
+              "query": "sum(flyte:propeller:all:metastore:write_ms) by (quantile, wf)",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -2265,7 +2520,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Node Input latency quantile and workflow",
+          "title": "Metastore write latency by workflow",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -2375,7 +2630,7 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)",
+              "expr": "sum(flyte:propeller:all:metastore:read_open_ms) by (quantile, wf)",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -2383,31 +2638,16 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)",
+              "query": "sum(flyte:propeller:all:metastore:read_open_ms) by (quantile, wf)",
               "refId": "A",
               "step": 10,
               "target": ""
-            },
-            {
-              "datasource": null,
-              "expr": "sum(flyte:propeller:all:node:event_recording:failure_duration_ms) by (quantile, wf)",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "",
-              "metric": "",
-              "query": "sum(flyte:propeller:all:node:event_recording:failure_duration_ms) by (quantile, wf)",
-              "refId": "B",
-              "step": 10,
-              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Node Event event recording latency quantile and workflow",
+          "title": "Metastore read open latency by workflow",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -2517,54 +2757,24 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))",
+              "expr": "sum(flyte:propeller:all:metastore:head_ms) by (quantile, wf)",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "system error",
+              "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))",
+              "query": "sum(flyte:propeller:all:metastore:head_ms) by (quantile, wf)",
               "refId": "A",
               "step": 10,
               "target": ""
-            },
-            {
-              "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "user error",
-              "metric": "",
-              "query": "sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))",
-              "refId": "B",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "user error",
-              "metric": "",
-              "query": "sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))",
-              "refId": "C",
-              "step": 10,
-              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "node event recording count",
+          "title": "Metastore head latency by workflow",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -2583,7 +2793,149 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 20,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 3,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:metastore:proto_fetch_ms) by (quantile, wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "proto-fetch-P{{quantile}}",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:metastore:proto_fetch_ms) by (quantile, wf)",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:metastore:remote_fetch_ms) by (quantile, wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "remote-fetch-P{{quantile}}",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:metastore:remote_fetch_ms) by (quantile, wf)",
+              "refId": "B",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Metastore fetch latency by workflow",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ms",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -2608,7 +2960,7 @@
       ],
       "repeat": null,
       "showTitle": true,
-      "title": "Node Metrics"
+      "title": "Metastore latencies"
     },
     {
       "collapse": true,
@@ -2641,7 +2993,2282 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 20,
+          "id": 21,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 3,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:node:node_exec_latency_us) by (quantile, wf) / 1000",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:node:node_exec_latency_us) by (quantile, wf) / 1000",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Node Exec latency quantile and workflow",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 22,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 3,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:node:node_input_latency_ms) by (quantile, wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:node:node_input_latency_ms) by (quantile, wf)",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Node Input latency quantile and workflow",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 23,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 3,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:node:event_recording:failure_duration_ms) by (quantile, wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:node:event_recording:failure_duration_ms) by (quantile, wf)",
+              "refId": "B",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Node Event event recording latency quantile and workflow",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 24,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 3,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "system error",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "user error",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))",
+              "refId": "B",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "unknown error",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))",
+              "refId": "C",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "node event recording error rate breakdown",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ops",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        }
+      ],
+      "repeat": null,
+      "showTitle": true,
+      "title": "Node Metrics"
+    },
+    {
+      "collapse": true,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 25,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 2,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "wf event recording latency success",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 26,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 2,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "success",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "failure",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)",
+              "refId": "B",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "wf event recording rate",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ops",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 27,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 2,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "node event recording latency",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 28,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 2,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "success-{{wf}}",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "failure-{{wf}}",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)",
+              "refId": "B",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "node event recording rate",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ops",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 29,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 2,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "task event recording latency",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 30,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 2,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "success-{{wf}}",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "failure-{{wf}}",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)",
+              "refId": "B",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "task event recording rate",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ops",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 31,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 2,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Dynamic workflow build latency",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 32,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 2,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Dynamic workflow build rate",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ops",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": "Cache hit rate when admin launcher is queried for the status of a workflow. Admin launcher will update status of workflows in the cache on a polling interval.",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 33,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 2,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "hit",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": null,
+              "expr": "sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "miss",
+              "metric": "",
+              "query": "sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))",
+              "refId": "B",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Admin Launcher cache hit/miss rate",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ops",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        }
+      ],
+      "repeat": null,
+      "showTitle": true,
+      "title": "Perf metrics"
+    },
+    {
+      "collapse": true,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "index": 0,
+                    "line": true,
+                    "op": "gt",
+                    "value": "null",
+                    "yaxis": "left"
+                  },
+                  {
+                    "color": "red",
+                    "index": 1,
+                    "line": true,
+                    "op": "gt",
+                    "value": 80.0,
+                    "yaxis": "left"
+                  }
+                ]
+              }
+            }
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 34,
+          "interval": null,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "options": {
+            "displayMode": "gradient",
+            "fieldOptions": {
+              "calcs": [
+                "sum"
+              ],
+              "defaults": {
+                "decimals": null,
+                "links": [],
+                "max": 200,
+                "min": 0,
+                "title": null,
+                "unit": "none"
+              },
+              "limit": null,
+              "mappings": [],
+              "override": {},
+              "thresholds": [
+                {
+                  "color": "green",
+                  "index": 0,
+                  "line": true,
+                  "op": "gt",
+                  "value": "null",
+                  "yaxis": "left"
+                },
+                {
+                  "color": "red",
+                  "index": 1,
+                  "line": true,
+                  "op": "gt",
+                  "value": 80.0,
+                  "yaxis": "left"
+                }
+              ],
+              "values": false
+            },
+            "orientation": "vertical",
+            "showThresholdLabels": false,
+            "showThresholdMarkers": true
+          },
+          "repeat": null,
+          "repeatDirection": null,
+          "span": 12,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum by(le) (rate(grpc_client_handling_seconds_bucket[5m]))",
+              "format": "heatmap",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{le}}",
+              "metric": "",
+              "query": "sum by(le) (rate(grpc_client_handling_seconds_bucket[5m]))",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "All GRPC calls latency",
+          "transformations": [],
+          "transparent": false,
+          "type": "bargauge"
+        }
+      ],
+      "repeat": null,
+      "showTitle": true,
+      "title": "GRPC latency metrics"
+    },
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 35,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 2,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:workflow:acceptance_latency_ms) by (wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:workflow:acceptance_latency_ms) by (wf)",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Workflow acceptance latency per workflow",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 36,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 2,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:workflow:acceptance_latency_unlabeled_ms) by (quantile)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:workflow:acceptance_latency_unlabeled_ms) by (quantile)",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Workflow acceptance latency by quantile",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 37,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "sideWidth": null,
+            "sort": null,
+            "sortDesc": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "maxDataPoints": 100,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": 2,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": null,
+              "expr": "sum(flyte:propeller:all:node:transition_latency_ms) by (wf)",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "metric": "",
+              "query": "sum(flyte:propeller:all:node:transition_latency_ms) by (wf)",
+              "refId": "A",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Node transition latency per workflow",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_PROM}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": null,
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 38,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -2684,7 +5311,7 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)",
+              "expr": "sum(flyte:propeller:all:node:transition_latency_unlabeled_ms) by (quantile)",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -2692,7 +5319,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)",
+              "query": "sum(flyte:propeller:all:node:transition_latency_unlabeled_ms) by (quantile)",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -2701,7 +5328,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "wf event recording latency success",
+          "title": "Node transition latency by quantile",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -2768,7 +5395,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 21,
+          "id": 39,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -2811,39 +5438,24 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)",
+              "expr": "sum(flyte:propeller:all:node:queueing_latency_ms) by (wf)",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "success",
+              "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)",
+              "query": "sum(flyte:propeller:all:node:queueing_latency_ms) by (wf)",
               "refId": "A",
               "step": 10,
               "target": ""
-            },
-            {
-              "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "failure",
-              "metric": "",
-              "query": "sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)",
-              "refId": "B",
-              "step": 10,
-              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "wf event recording count",
+          "title": "Node queueing latency per workflow",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -2862,7 +5474,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "ms",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -2910,7 +5522,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 22,
+          "id": 40,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -2953,7 +5565,7 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)",
+              "expr": "sum(flyte:propeller:all:node:queueing_latency_unlabeled_ms) by (quantile)",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -2961,7 +5573,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)",
+              "query": "sum(flyte:propeller:all:node:queueing_latency_unlabeled_ms) by (quantile)",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -2970,7 +5582,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "node event recording latency success",
+          "title": "Node queueing latency by quantile",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -3037,7 +5649,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 23,
+          "id": 41,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3080,39 +5692,24 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)",
+              "expr": "sum(flyte:propeller:all:workflow:completion_latency_ms) by (wf)",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "success",
+              "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)",
+              "query": "sum(flyte:propeller:all:workflow:completion_latency_ms) by (wf)",
               "refId": "A",
               "step": 10,
               "target": ""
-            },
-            {
-              "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "failure",
-              "metric": "",
-              "query": "sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)",
-              "refId": "B",
-              "step": 10,
-              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "node event recording count",
+          "title": "Workflow completion latency per workflow",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -3131,7 +5728,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "ms",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -3179,7 +5776,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 24,
+          "id": 42,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3222,7 +5819,7 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)",
+              "expr": "sum(flyte:propeller:all:workflow:completion_latency_unlabeled_ms) by (quantile)",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -3230,7 +5827,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)",
+              "query": "sum(flyte:propeller:all:workflow:completion_latency_unlabeled_ms) by (quantile)",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -3239,7 +5836,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "task event recording latency",
+          "title": "Workflow completion latency by quantile",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -3279,7 +5876,17 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "showTitle": true,
+      "title": "Workflow latencies"
+    },
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
@@ -3306,7 +5913,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 25,
+          "id": 43,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3343,45 +5950,30 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 2,
+          "span": 3,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)",
+              "expr": "sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "success wf",
+              "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)",
+              "query": "sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)",
               "refId": "A",
               "step": 10,
               "target": ""
-            },
-            {
-              "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "failure",
-              "metric": "",
-              "query": "sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)",
-              "refId": "B",
-              "step": 10,
-              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "task event recording count",
+          "title": "wf update etcD latency",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -3400,7 +5992,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "ms",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -3448,7 +6040,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 26,
+          "id": 44,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3485,13 +6077,13 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 2,
+          "span": 3,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000",
+              "expr": "sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -3499,7 +6091,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000",
+              "query": "sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -3508,7 +6100,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Dynamic workflow build latency",
+          "title": "etcD write rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -3527,7 +6119,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ms",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -3575,7 +6167,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 27,
+          "id": 45,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3612,13 +6204,13 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 2,
+          "span": 3,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)",
+              "expr": "sum(rate(flyte:propeller:all:wf_update_conflict[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -3626,7 +6218,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)",
+              "query": "sum(rate(flyte:propeller:all:wf_update_conflict[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -3635,7 +6227,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Dynamic workflow build count",
+          "title": "etcD write conflict rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -3654,7 +6246,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -3702,7 +6294,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 28,
+          "id": 46,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3739,45 +6331,30 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 2,
+          "span": 3,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))",
+              "expr": "sum(rate(flyte:propeller:all:wf_update_failed[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "hit",
+              "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))",
+              "query": "sum(rate(flyte:propeller:all:wf_update_failed[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
-            },
-            {
-              "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "miss",
-              "metric": "",
-              "query": "sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))",
-              "refId": "B",
-              "step": 10,
-              "target": ""
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Admin Launcher cache",
+          "title": "etcD write failure rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -3796,7 +6373,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ms",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -3817,17 +6394,7 @@
             "align": false,
             "alignLevel": 0
           }
-        }
-      ],
-      "repeat": null,
-      "showTitle": true,
-      "title": "Perf metrics"
-    },
-    {
-      "collapse": false,
-      "editable": true,
-      "height": "250px",
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
@@ -3854,7 +6421,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 29,
+          "id": 47,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -3897,7 +6464,7 @@
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)",
+              "expr": "sum(rate(flyte:propeller:all:wf_too_large[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -3905,7 +6472,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)",
+              "query": "sum(rate(flyte:propeller:all:wf_too_large[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -3914,7 +6481,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "wf update etcD latency",
+          "title": "etcD write too large rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -3933,7 +6500,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ms",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -3954,7 +6521,17 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "showTitle": true,
+      "title": "etcD write metrics"
+    },
+    {
+      "collapse": true,
+      "editable": true,
+      "height": "250px",
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
@@ -3981,7 +6558,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 30,
+          "id": 48,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4018,13 +6595,13 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 3,
+          "span": 6,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))",
+              "expr": "sum(rate(flyte:propeller:all:node:container:container:informer_update[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -4032,7 +6609,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))",
+              "query": "sum(rate(flyte:propeller:all:node:container:container:informer_update[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -4041,7 +6618,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "etcD writes",
+          "title": "Update event rate from informer",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -4060,7 +6637,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -4108,7 +6685,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 31,
+          "id": 49,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4145,13 +6722,13 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 3,
+          "span": 6,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:wf_update_conflict[5m]))",
+              "expr": "sum(rate(flyte:propeller:all:node:container:container:informer_update_dropped[5m]))",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -4159,7 +6736,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:wf_update_conflict[5m]))",
+              "query": "sum(rate(flyte:propeller:all:node:container:container:informer_update_dropped[5m]))",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -4168,7 +6745,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "etcD write conflicts",
+          "title": "Update events drop rate becacuse they have the same resource version",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -4187,7 +6764,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -4208,7 +6785,17 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "showTitle": true,
+      "title": "K8s Pod Informer stats"
+    },
+    {
+      "collapse": true,
+      "editable": true,
+      "height": "250px",
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
@@ -4235,7 +6822,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 32,
+          "id": 50,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4272,13 +6859,13 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 3,
+          "span": 4,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:wf_update_failed[5m]))",
+              "expr": "sum(rate(flyte:propeller:all:wf_stale_unlabeled[5m])) by (quantile)",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -4286,7 +6873,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:wf_update_failed[5m]))",
+              "query": "sum(rate(flyte:propeller:all:wf_stale_unlabeled[5m])) by (quantile)",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -4295,7 +6882,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "etcD write fail",
+          "title": "Stale workflows rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -4314,7 +6901,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -4335,17 +6922,7 @@
             "align": false,
             "alignLevel": 0
           }
-        }
-      ],
-      "repeat": null,
-      "showTitle": true,
-      "title": "etcD write metrics"
-    },
-    {
-      "collapse": true,
-      "editable": true,
-      "height": "250px",
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
@@ -4372,7 +6949,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 33,
+          "id": 51,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4409,13 +6986,13 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 2,
+          "span": 4,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:main_adds[5m]))",
+              "expr": "sum(rate(flyte:propeller:all:wf_stale_unlabeled[5m])) by (quantile)",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -4423,7 +7000,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:main_adds[5m]))",
+              "query": "sum(rate(flyte:propeller:all:wf_stale_unlabeled[5m])) by (quantile)",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -4432,7 +7009,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "WF Adds to main queue",
+          "title": "Evict workflows rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -4451,7 +7028,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "short",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -4499,7 +7076,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 34,
+          "id": 52,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4536,13 +7113,13 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 2,
+          "span": 4,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:main_depth[5m]))",
+              "expr": "sum(rate(flyte:propeller:all:wf_redundant_unlabeled[5m])) by (quantile)",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -4550,7 +7127,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:main_depth[5m]))",
+              "query": "sum(rate(flyte:propeller:all:wf_redundant_unlabeled[5m])) by (quantile)",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -4559,7 +7136,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Unprocessed Queue depth",
+          "title": "Workflow redundant updates rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -4578,7 +7155,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "short",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -4599,7 +7176,17 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "showTitle": true,
+      "title": "Workflow store"
+    },
+    {
+      "collapse": true,
+      "editable": true,
+      "height": "250px",
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
@@ -4626,7 +7213,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 35,
+          "id": 53,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4663,13 +7250,13 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 2,
+          "span": 3,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:main_retries[5m]))",
+              "expr": "sum(flyte:propeller:all:gc_success)",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -4677,7 +7264,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:main_retries[5m]))",
+              "query": "sum(flyte:propeller:all:gc_success)",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -4686,7 +7273,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Item retries",
+          "title": "Namespace successful garbage collections",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -4753,7 +7340,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 36,
+          "id": 54,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4790,13 +7377,13 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 2,
+          "span": 3,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "flyte:propeller:all:main_unfinished_work_s",
+              "expr": "sum(flyte:propeller:all:gc_failure)",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -4804,7 +7391,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "flyte:propeller:all:main_unfinished_work_s",
+              "query": "sum(flyte:propeller:all:gc_failure)",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -4813,7 +7400,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Seconds of unfinished work in progress",
+          "title": "Namespace failed garbage collections",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -4832,7 +7419,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "s",
+              "format": "short",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -4880,7 +7467,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 37,
+          "id": 55,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -4917,13 +7504,13 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 2,
+          "span": 3,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:main_work_duration_us_sum[5m]) / rate(flyte:propeller:all:main_work_duration_us_count[5m]))",
+              "expr": "sum(flyte:propeller:all:gc_latency_ms_count)",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -4931,7 +7518,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:main_work_duration_us_sum[5m]) / rate(flyte:propeller:all:main_work_duration_us_count[5m]))",
+              "query": "sum(flyte:propeller:all:gc_latency_ms_count)",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -4940,7 +7527,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Workqueue work average duration",
+          "title": "Total completed garbage collection rounds",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -4959,7 +7546,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "s",
+              "format": "short",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -5007,7 +7594,7 @@
           "gridPos": null,
           "height": null,
           "hideTimeOverride": false,
-          "id": 38,
+          "id": 56,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -5044,13 +7631,13 @@
           "repeat": null,
           "repeatDirection": null,
           "seriesOverrides": [],
-          "span": 2,
+          "span": 3,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": null,
-              "expr": "sum(rate(flyte:propeller:all:main_queue_latency_us_sum[5m]) / rate(flyte:propeller:all:main_queue_latency_us_count[5m]))",
+              "expr": "sum(flyte:propeller:all:gc_latency_ms_sum) / sum(flyte:propeller:all:gc_latency_ms_count)",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -5058,7 +7645,7 @@
               "intervalFactor": 2,
               "legendFormat": "",
               "metric": "",
-              "query": "sum(rate(flyte:propeller:all:main_queue_latency_us_sum[5m]) / rate(flyte:propeller:all:main_queue_latency_us_count[5m]))",
+              "query": "sum(flyte:propeller:all:gc_latency_ms_sum) / sum(flyte:propeller:all:gc_latency_ms_count)",
               "refId": "A",
               "step": 10,
               "target": ""
@@ -5067,7 +7654,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Duration for which an item stays in queue - avg",
+          "title": "Average garbage collection round duration",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -5086,7 +7673,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "s",
+              "format": "ms",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -5111,7 +7698,7 @@
       ],
       "repeat": null,
       "showTitle": true,
-      "title": "FlytePropeller Queue metrics"
+      "title": "Workflow garbage collection"
     }
   ],
   "schemaVersion": 12,
diff --git a/stats/flyteadmin.dashboard.py b/stats/flyteadmin.dashboard.py
index ed709b1633..80df160daf 100644
--- a/stats/flyteadmin.dashboard.py
+++ b/stats/flyteadmin.dashboard.py
@@ -1,9 +1,8 @@
 import typing
-from grafanalib.core import (
-    Alert, AlertCondition, Dashboard, Graph,
-    GreaterThan, OP_AND, OPS_FORMAT, Row, RTYPE_SUM, SECONDS_FORMAT,
-    SHORT_FORMAT, single_y_axis, Target, TimeRange, YAxes, YAxis, DataSourceInput, MILLISECONDS_FORMAT
-)
+
+from grafanalib.core import (MILLISECONDS_FORMAT, OPS_FORMAT, SHORT_FORMAT,
+                             BarGauge, Dashboard, DataSourceInput, Graph, Row,
+                             Target, YAxes, YAxis, single_y_axis)
 
 # ------------------------------
 # For Gostats we recommend using
@@ -207,9 +206,35 @@ def create_all_apis(interval: int = 5) -> typing.List[Row]:
             rows.append(FlyteAdmin.create_api_row(api, collapse=True, interval=interval))
         return rows
 
+    @staticmethod
+    def grpc_latency_row() -> Graph:
+        return Row(
+            title="GRPC latency metrics",
+            collapse=True,
+            panels=[
+                BarGauge(
+                    title="All GRPC calls latency",
+                    calc="sum",
+                    dataSource=DATASOURCE,
+                    targets=[
+                        Target(
+                            expr="sum by(le) (rate(grpc_server_handling_seconds_bucket[5m]))",
+                            refId="A",
+                            format="heatmap",
+                            legendFormat=r"{{le}}",
+                        ),
+                    ],
+                    displayMode="gradient",
+                    orientation="vertical",
+                    max=200,
+                )
+            ]
+        )
+
     @staticmethod
     def create_all_rows(interval: int = 5) -> typing.List[Row]:
         rows = []
+        rows.extend([FlyteAdmin.grpc_latency_row()])
         rows.extend(FlyteAdmin.create_all_entity_db_rows(collapse=True, interval=interval))
         rows.extend(FlyteAdmin.create_all_apis(interval))
         return rows
diff --git a/stats/flytepropeller.dashboard.py b/stats/flytepropeller.dashboard.py
index d873daa1e9..513404cd5e 100644
--- a/stats/flytepropeller.dashboard.py
+++ b/stats/flytepropeller.dashboard.py
@@ -1,10 +1,9 @@
 import typing
 
-from grafanalib.core import (
-    Dashboard, Graph, Gauge, Stat,
-    OPS_FORMAT, Row, SHORT_FORMAT, single_y_axis, Target, YAxes, YAxis, MILLISECONDS_FORMAT, DataSourceInput,
-    PERCENT_FORMAT, NO_FORMAT, SECONDS_FORMAT
-)
+from grafanalib.core import (MILLISECONDS_FORMAT, NO_FORMAT, OPS_FORMAT,
+                             PERCENT_FORMAT, SECONDS_FORMAT, SHORT_FORMAT,
+                             BarGauge, Dashboard, DataSourceInput, Graph, Row,
+                             Target, YAxes, YAxis, single_y_axis)
 
 # ------------------------------
 # For Gostats we recommend using
@@ -16,61 +15,102 @@
 
 
 class FlytePropeller(object):
-
     @staticmethod
     def create_free_workers() -> Graph:
         return Graph(
             title="Free workers count",
+            description="The number of golang goroutines available to accept new work from the main workqueue. Each worker can process one item from the workqueue at a time.",
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr='sum(flyte:propeller:all:free_workers_count) by (kubernetes_pod_name)',
-                    refId='A',
+                    expr="sum(flyte:propeller:all:free_workers_count) by (kubernetes_pod_name)",
+                    refId="A",
                 ),
             ],
             yAxes=YAxes(
-                YAxis(format=OPS_FORMAT),
+                YAxis(format=NO_FORMAT),
                 YAxis(format=SHORT_FORMAT),
             ),
         )
 
     @staticmethod
-    def round_latency_per_wf(interval: int = 1) -> Graph:
+    def round_latency_per_wf() -> Graph:
         return Graph(
-            title=f"round Latency per workflow",
+            title=f"Round traverse latency per workflow",
+            description="Round latency by workflow name. When there are streaks within one round each iteration is measured separately.",
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr=f'sum(rate(flyte:propeller:all:round:raw_ms[{interval}m])) by (wf)',
-                    refId='A',
+                    expr=f"sum(flyte:propeller:all:round:raw_ms) by (wf)",
+                    refId="A",
+                    legendFormat="{{wf}}",
                 ),
             ],
             yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
         )
 
     @staticmethod
-    def round_latency(interval: int = 1) -> Graph:
+    def round_latency() -> Graph:
         return Graph(
-            title=f"round Latency by quantile",
+            title=f"Round Latency (includes streak rounds)",
+            description="Round latency breakdown. When there are streaks within one round each iteration is measured separately.",
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr=f'sum(rate(flyte:propeller:all:round:raw_unlabeled_ms[{interval}m])) by (quantile)',
-                    refId='A',
+                    expr="sum(flyte:propeller:all:round:raw_unlabeled_ms) by (quantile)",
+                    refId="A",
+                    legendFormat="traverse-P{{quantile}}",
+                ),
+                Target(
+                    expr="avg(flyte:propeller:all:round:raw_unlabeled_ms_sum/flyte:propeller:all:round:raw_unlabeled_ms_count)",
+                    refId="B",
+                    legendFormat="traverse-mean",
+                ),
+                Target(
+                    expr="sum(flyte:propeller:all:round:round_time_ms) by (quantile)",
+                    refId="C",
+                    legendFormat="total-P{{quantile}}",
+                ),
+                Target(
+                    expr="avg(flyte:propeller:all:round:round_time_unlabeled_ms_sum/flyte:propeller:all:round:round_time_unlabeled_ms_count)",
+                    refId="D",
+                    legendFormat="total-mean",
                 ),
             ],
             yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
         )
 
     @staticmethod
-    def round_panic() -> Graph:
+    def error_breakdown() -> Graph:
         return Graph(
-            title="Round panic",
+            title="Error rate breakdown",
+            description="Error rates for each type of failure that may occur as propeller is traversing the workflow DAG.",
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr='sum(rate(flyte:propeller:all:round:panic_unlabeled[5m]))',
-                    refId='A',
+                    expr="sum(rate(flyte:propeller:all:round:system_error_unlabeled[5m]))",
+                    refId="A",
+                    legendFormat="system",
+                ),
+                Target(
+                    expr="sum(rate(flyte:propeller:all:round:abort_error_unlabeled[5m]))",
+                    refId="B",
+                    legendFormat="abort",
+                ),
+                Target(
+                    expr="sum(rate(flyte:propeller:all:round:panic_unlabeled[5m]))",
+                    refId="C",
+                    legendFormat="panic",
+                ),
+                Target(
+                    expr="sum(rate(flyte:propeller:all:round:not_found[5m]))",
+                    refId="D",
+                    legendFormat="not-found",
+                ),
+                Target(
+                    expr="sum(rate(flyte:propeller:all:round:skipped[5m]))",
+                    refId="E",
+                    legendFormat="skipped",
                 ),
             ],
             yAxes=YAxes(
@@ -80,14 +120,15 @@ def round_panic() -> Graph:
         )
 
     @staticmethod
-    def streak_length() -> Graph:
+    def streak_rate() -> Graph:
         return Graph(
-            title="Avg streak length",
+            title="Streak rate",
+            description="Streaks are when propeller iterates over the same workflow multiple times in a single round. This is an optimisation technique.",
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr='avg(flyte:propeller:all:round:streak_length_unlabeled)',
-                    refId='A',
+                    expr="sum(rate(flyte:propeller:all:round:streak_length_unlabeled[5m]))",
+                    refId="A",
                 ),
             ],
             yAxes=YAxes(
@@ -97,14 +138,31 @@ def streak_length() -> Graph:
         )
 
     @staticmethod
-    def system_errors() -> Graph:
+    def round_rates() -> Graph:
         return Graph(
-            title="System errors",
+            title="Round success/error rate",
+            description="Round success, error and total rates. Also includes total rate including streaks within a single round. The streak rate graph should match the difference between the totals with and without streaks.",
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr='sum(deriv(flyte:propeller:all:round:system_error_unlabeled[5m]))',
-                    refId='A',
+                    expr="sum(rate(flyte:propeller:all:round:success_count[5m]))",
+                    refId="A",
+                    legendFormat="success",
+                ),
+                Target(
+                    expr="sum(rate(flyte:propeller:all:round:error_count[5m]))",
+                    refId="B",
+                    legendFormat="error",
+                ),
+                Target(
+                    expr="sum(rate(flyte:propeller:all:round:round_total_ms_count[5m]))",
+                    refId="C",
+                    legendFormat="total",
+                ),
+                Target(
+                    expr="sum(rate(flyte:propeller:all:round:round_time_unlabeled_ms_count[5m]))",
+                    refId="D",
+                    legendFormat="total-including-streaks",
                 ),
             ],
             yAxes=YAxes(
@@ -114,31 +172,33 @@ def system_errors() -> Graph:
         )
 
     @staticmethod
-    def abort_errors() -> Graph:
+    def workflows_per_project() -> Graph:
         return Graph(
-            title="System errors",
+            title=f"Running Workflows per project",
+            description="Count of currently running workflows running per project",
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr='sum(rate(flyte:propeller:all:round:abort_error[5m]))',
-                    refId='A',
+                    expr=f"sum(flyte:propeller:all:collector:flyteworkflow) by (project)",
+                    refId="A",
                 ),
             ],
             yAxes=YAxes(
-                YAxis(format=OPS_FORMAT),
+                YAxis(format=NO_FORMAT),
                 YAxis(format=SHORT_FORMAT),
             ),
         )
 
     @staticmethod
-    def workflows_per_project() -> Graph:
+    def enqueued_workflows() -> Graph:
         return Graph(
-            title=f"Running Workflows per project",
+            title="Workflow enqueue rate",
+            description="Rate at which workflows are being enqueued by flytepropeller. These enqueues all pass through the sub-queue before going back into the main queue.",
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr=f'sum(flyte:propeller:all:collector:flyteworkflow) by (project)',
-                    refId='A',
+                    expr="sum by(type) (rate(flyte:propeller:all:wf_enqueue[5m]))",
+                    refId="A",
                 ),
             ],
             yAxes=YAxes(
@@ -153,16 +213,19 @@ def plugin_success_vs_failures() -> Graph:
         TODO We need to convert the plugin names to be labels, so that prometheus can perform queries correctly
         """
         return Graph(
-            title=f"Plugin Failures",
+            title=f"Plugin Success/Failure rate",
+            description="Success vs failure rate for the various plugins used for each node e.g. k8s plugin or spark plugin.",
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr='{__name__=~"flyte:propeller:all:node:plugin:.*_failure_unlabeled"}',
-                    refId='A',
+                    expr='sum(rate({__name__=~"flyte:propeller:all:plugin:.*_success_unlabeled"}[5m]))',
+                    refId="A",
+                    legendFormat="success",
                 ),
                 Target(
-                    expr='{__name__=~"flyte:propeller:all:node:plugin:.*_success_unlabeled"}',
-                    refId='B',
+                    expr='sum(rate({__name__=~"flyte:propeller:all:plugin:.*_failure_unlabeled"}[5m]))',
+                    refId="B",
+                    legendFormat="failure",
                 ),
             ],
             yAxes=YAxes(
@@ -178,8 +241,8 @@ def node_exec_latency() -> Graph:
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr=f'sum(flyte:propeller:all:node:node_exec_latency_us) by (quantile, wf) / 1000',
-                    refId='A',
+                    expr=f"sum(flyte:propeller:all:node:node_exec_latency_us) by (quantile, wf) / 1000",
+                    refId="A",
                 ),
             ],
             yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
@@ -192,12 +255,12 @@ def node_event_recording_latency() -> Graph:
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr=f'sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)',
-                    refId='A',
+                    expr=f"sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)",
+                    refId="A",
                 ),
                 Target(
-                    expr=f'sum(flyte:propeller:all:node:event_recording:failure_duration_ms) by (quantile, wf)',
-                    refId='B',
+                    expr=f"sum(flyte:propeller:all:node:event_recording:failure_duration_ms) by (quantile, wf)",
+                    refId="B",
                 ),
             ],
             yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
@@ -210,8 +273,8 @@ def node_input_latency() -> Graph:
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr=f'sum(flyte:propeller:all:node:node_input_latency_ms) by (quantile, wf)',
-                    refId='A',
+                    expr=f"sum(flyte:propeller:all:node:node_input_latency_ms) by (quantile, wf)",
+                    refId="A",
                 ),
             ],
             yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
@@ -221,33 +284,33 @@ def node_input_latency() -> Graph:
     def metastore_failures():
         # Copy counts sum(rate(flyte:propeller:all:metastore:copy:overall_unlabeled_ms_count[5m]))
         return Graph(
-            title=f"Failures from metastore",
+            title=f"Metastore failure rate",
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr=f'sum(rate(flyte:propeller:all:metastore:head_failure_unlabeled[5m]))',
+                    expr=f"sum(rate(flyte:propeller:all:metastore:head_failure_unlabeled[5m]))",
                     legendFormat="head-failure",
-                    refId='A',
+                    refId="A",
                 ),
                 Target(
-                    expr=f'sum(rate(flyte:propeller:all:metastore:bad_container_unlabeled[5m]))',
+                    expr=f"sum(rate(flyte:propeller:all:metastore:bad_container_unlabeled[5m]))",
                     legendFormat="bad-container",
-                    refId='B',
+                    refId="B",
                 ),
                 Target(
-                    expr=f'sum(rate(flyte:propeller:all:metastore:bad_key_unlabeled[5m]))',
+                    expr=f"sum(rate(flyte:propeller:all:metastore:bad_key_unlabeled[5m]))",
                     legendFormat="bad-key",
-                    refId='C',
+                    refId="C",
                 ),
                 Target(
-                    expr=f'sum(rate(flyte:propeller:all:metastore:read_failure_unlabeled[5m]))',
+                    expr=f"sum(rate(flyte:propeller:all:metastore:read_failure_unlabeled[5m]))",
                     legendFormat="read-failure",
-                    refId='D',
+                    refId="D",
                 ),
                 Target(
-                    expr=f'sum(rate(flyte:propeller:all:metastore:write_failure_unlabeled[5m]))',
+                    expr=f"sum(rate(flyte:propeller:all:metastore:write_failure_unlabeled[5m]))",
                     legendFormat="write-failure",
-                    refId='E',
+                    refId="E",
                 ),
             ],
             yAxes=YAxes(
@@ -266,8 +329,8 @@ def metastore_cache_hit_percentage(interval: int) -> Graph:
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr=f'(sum(rate(flyte:propeller:all:metastore:cache_hit[{interval}m])) * 100) / (sum(rate(flyte:propeller:all:metastore:cache_miss[{interval}m])) + sum(rate(flyte:propeller:all:metastore:cache_hit[{interval}m])))',
-                    refId='A',
+                    expr=f"(sum(rate(flyte:propeller:all:metastore:cache_hit[{interval}m])) * 100) / (sum(rate(flyte:propeller:all:metastore:cache_miss[{interval}m])) + sum(rate(flyte:propeller:all:metastore:cache_hit[{interval}m])))",
+                    refId="A",
                 ),
             ],
             yAxes=single_y_axis(format=PERCENT_FORMAT),
@@ -284,8 +347,8 @@ def metastore_latencies(collapse: bool) -> Row:
                     dataSource=DATASOURCE,
                     targets=[
                         Target(
-                            expr=f'sum(flyte:propeller:all:metastore:copy:overall_unlabeled_ms) by (quantile)',
-                            refId='A',
+                            expr=f"sum(flyte:propeller:all:metastore:copy:overall_unlabeled_ms) by (quantile)",
+                            refId="A",
                         ),
                     ],
                     yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
@@ -295,8 +358,8 @@ def metastore_latencies(collapse: bool) -> Row:
                     dataSource=DATASOURCE,
                     targets=[
                         Target(
-                            expr='sum(flyte:propeller:all:metastore:write_ms) by (quantile, wf)',
-                            refId='A',
+                            expr="sum(flyte:propeller:all:metastore:write_ms) by (quantile, wf)",
+                            refId="A",
                         ),
                     ],
                     yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
@@ -306,8 +369,8 @@ def metastore_latencies(collapse: bool) -> Row:
                     dataSource=DATASOURCE,
                     targets=[
                         Target(
-                            expr='sum(flyte:propeller:all:metastore:read_open_ms) by (quantile, wf)',
-                            refId='A',
+                            expr="sum(flyte:propeller:all:metastore:read_open_ms) by (quantile, wf)",
+                            refId="A",
                         ),
                     ],
                     yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
@@ -317,8 +380,8 @@ def metastore_latencies(collapse: bool) -> Row:
                     dataSource=DATASOURCE,
                     targets=[
                         Target(
-                            expr='sum(flyte:propeller:all:metastore:head_ms) by (quantile, wf)',
-                            refId='A',
+                            expr="sum(flyte:propeller:all:metastore:head_ms) by (quantile, wf)",
+                            refId="A",
                         ),
                     ],
                     yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
@@ -328,41 +391,43 @@ def metastore_latencies(collapse: bool) -> Row:
                     dataSource=DATASOURCE,
                     targets=[
                         Target(
-                            expr='sum(flyte:propeller:all:metastore:proto_fetch_ms) by (quantile, wf)',
-                            legendFormat="proto-fetch",
-                            refId='A',
+                            expr="sum(flyte:propeller:all:metastore:proto_fetch_ms) by (quantile, wf)",
+                            legendFormat="proto-fetch-P{{quantile}}",
+                            refId="A",
                         ),
-
                         Target(
-                            expr='sum(flyte:propeller:all:metastore:remote_fetch_ms) by (quantile, wf)',
-                            legendFormat="remote-fetch",
-                            refId='B',
+                            expr="sum(flyte:propeller:all:metastore:remote_fetch_ms) by (quantile, wf)",
+                            legendFormat="remote-fetch-P{{quantile}}",
+                            refId="B",
                         ),
                     ],
                     yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
                 ),
-            ]
+            ],
         )
 
     @staticmethod
     def admin_launcher_cache() -> Graph:
         return Graph(
-            title="Admin Launcher cache",
+            title="Admin Launcher cache hit/miss rate",
+            description="Cache hit rate when admin launcher is queried for the status of a workflow. Admin launcher will update status of workflows in the cache on a polling interval.",
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr=f'sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))',
+                    expr=f"sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))",
                     legendFormat="hit",
-                    refId='A',
+                    refId="A",
                 ),
-
                 Target(
-                    expr=f'sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))',
+                    expr=f"sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))",
                     legendFormat="miss",
-                    refId='B',
+                    refId="B",
                 ),
             ],
-            yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
+            yAxes=YAxes(
+                YAxis(format=OPS_FORMAT),
+                YAxis(format=SHORT_FORMAT),
+            ),
         )
 
     @staticmethod
@@ -373,22 +438,22 @@ def dynamic_wf_build() -> typing.List[Graph]:
                 dataSource=DATASOURCE,
                 targets=[
                     Target(
-                        expr=f'sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000',
-                        refId='A',
+                        expr=f"sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000",
+                        refId="A",
                     ),
                 ],
                 yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
             ),
             Graph(
-                title="Dynamic workflow build count",
+                title="Dynamic workflow build rate",
                 dataSource=DATASOURCE,
                 targets=[
                     Target(
-                        expr=f'sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)',
-                        refId='A',
+                        expr=f"sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)",
+                        refId="A",
                     ),
                 ],
-                yAxes=single_y_axis(format=NO_FORMAT),
+                yAxes=single_y_axis(format=OPS_FORMAT),
             ),
         ]
 
@@ -400,28 +465,28 @@ def task_event_recording() -> typing.List[Graph]:
                 dataSource=DATASOURCE,
                 targets=[
                     Target(
-                        expr=f'sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)',
-                        refId='A',
+                        expr=f"sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)",
+                        refId="A",
                     ),
                 ],
                 yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
             ),
             Graph(
-                title="task event recording count",
+                title="task event recording rate",
                 dataSource=DATASOURCE,
                 targets=[
                     Target(
-                        expr=f'sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)',
-                        legendFormat="success wf",
-                        refId='A',
+                        expr=f"sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)",
+                        legendFormat="success-{{wf}}",
+                        refId="A",
                     ),
                     Target(
-                        expr=f'sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)',
-                        legendFormat="failure",
-                        refId='B',
+                        expr=f"sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)",
+                        legendFormat="failure-{{wf}}",
+                        refId="B",
                     ),
                 ],
-                yAxes=single_y_axis(format=NO_FORMAT),
+                yAxes=single_y_axis(format=OPS_FORMAT),
             ),
         ]
 
@@ -429,32 +494,32 @@ def task_event_recording() -> typing.List[Graph]:
     def node_event_recording() -> typing.List[Graph]:
         return [
             Graph(
-                title="node event recording latency success",
+                title="node event recording latency",
                 dataSource=DATASOURCE,
                 targets=[
                     Target(
-                        expr=f'sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)',
-                        refId='A',
+                        expr=f"sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)",
+                        refId="A",
                     ),
                 ],
                 yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
             ),
             Graph(
-                title="node event recording count",
+                title="node event recording rate",
                 dataSource=DATASOURCE,
                 targets=[
                     Target(
-                        expr=f'sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)',
-                        legendFormat="success",
-                        refId='A',
+                        expr=f"sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)",
+                        legendFormat="success-{{wf}}",
+                        refId="A",
                     ),
                     Target(
-                        expr=f'sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)',
-                        legendFormat="failure",
-                        refId='B',
+                        expr=f"sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)",
+                        legendFormat="failure-{{wf}}",
+                        refId="B",
                     ),
                 ],
-                yAxes=single_y_axis(format=NO_FORMAT),
+                yAxes=single_y_axis(format=OPS_FORMAT),
             ),
         ]
 
@@ -466,31 +531,50 @@ def wf_event_recording() -> typing.List[Graph]:
                 dataSource=DATASOURCE,
                 targets=[
                     Target(
-                        expr=f'sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)',
-                        refId='A',
+                        expr=f"sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)",
+                        refId="A",
                     ),
                 ],
                 yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
             ),
             Graph(
-                title="wf event recording count",
+                title="wf event recording rate",
                 dataSource=DATASOURCE,
                 targets=[
                     Target(
-                        expr=f'sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)',
+                        expr=f"sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)",
                         legendFormat="success",
-                        refId='A',
+                        refId="A",
                     ),
                     Target(
-                        expr=f'sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)',
+                        expr=f"sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)",
                         legendFormat="failure",
-                        refId='B',
+                        refId="B",
                     ),
                 ],
-                yAxes=single_y_axis(format=NO_FORMAT),
+                yAxes=single_y_axis(format=OPS_FORMAT),
             ),
         ]
 
+    @staticmethod
+    def grpc_latency_histogram() -> Graph:
+        return BarGauge(
+            title="All GRPC calls latency",
+            calc="sum",
+            dataSource=DATASOURCE,
+            targets=[
+                Target(
+                    expr="sum by(le) (rate(grpc_client_handling_seconds_bucket[5m]))",
+                    refId="A",
+                    format="heatmap",
+                    legendFormat=r"{{le}}",
+                ),
+            ],
+            displayMode="gradient",
+            orientation="vertical",
+            max=200,
+        )
+
     @staticmethod
     def wf_store_latency(collapse: bool) -> Row:
         return Row(
@@ -502,46 +586,58 @@ def wf_store_latency(collapse: bool) -> Row:
                     dataSource=DATASOURCE,
                     targets=[
                         Target(
-                            expr=f'sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)',
-                            refId='A',
+                            expr=f"sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)",
+                            refId="A",
                         ),
                     ],
                     yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
                 ),
                 Graph(
-                    title="etcD writes",
+                    title="etcD write rate",
+                    dataSource=DATASOURCE,
+                    targets=[
+                        Target(
+                            expr=f"sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))",
+                            refId="A",
+                        ),
+                    ],
+                    yAxes=single_y_axis(format=OPS_FORMAT),
+                ),
+                Graph(
+                    title="etcD write conflict rate",
                     dataSource=DATASOURCE,
                     targets=[
                         Target(
-                            expr=f'sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))',
-                            refId='A',
+                            expr=f"sum(rate(flyte:propeller:all:wf_update_conflict[5m]))",
+                            refId="A",
                         ),
                     ],
-                    yAxes=single_y_axis(format=NO_FORMAT),
+                    yAxes=single_y_axis(format=OPS_FORMAT),
                 ),
                 Graph(
-                    title="etcD write conflicts",
+                    title="etcD write failure rate",
                     dataSource=DATASOURCE,
                     targets=[
                         Target(
-                            expr=f'sum(rate(flyte:propeller:all:wf_update_conflict[5m]))',
-                            refId='A',
+                            expr=f"sum(rate(flyte:propeller:all:wf_update_failed[5m]))",
+                            refId="A",
                         ),
                     ],
-                    yAxes=single_y_axis(format=NO_FORMAT),
+                    yAxes=single_y_axis(format=OPS_FORMAT),
                 ),
                 Graph(
-                    title="etcD write fail",
+                    title="etcD write too large rate",
                     dataSource=DATASOURCE,
                     targets=[
                         Target(
-                            expr=f'sum(rate(flyte:propeller:all:wf_update_failed[5m]))',
-                            refId='A',
+                            expr=f"sum(rate(flyte:propeller:all:wf_too_large[5m]))",
+                            refId="A",
                         ),
                     ],
-                    yAxes=single_y_axis(format=NO_FORMAT),
+                    yAxes=single_y_axis(format=OPS_FORMAT),
                 ),
-            ])
+            ],
+        )
 
     @staticmethod
     def perf_metrics(collapse: bool) -> Row:
@@ -557,6 +653,17 @@ def perf_metrics(collapse: bool) -> Row:
         r.panels.append(FlytePropeller.admin_launcher_cache())
         return r
 
+    @staticmethod
+    def grpc_metrics(collapse: bool) -> Row:
+        r = Row(
+            title="GRPC latency metrics",
+            collapse=collapse,
+            panels=[
+                FlytePropeller.grpc_latency_histogram(),
+            ],
+        )
+        return r
+
     @staticmethod
     def metastore_metrics(interval: int, collapse: bool) -> Row:
         return Row(
@@ -571,26 +678,26 @@ def metastore_metrics(interval: int, collapse: bool) -> Row:
     @staticmethod
     def node_errors() -> Graph:
         return Graph(
-            title="node event recording count",
+            title="node event recording error rate breakdown",
             dataSource=DATASOURCE,
             targets=[
                 Target(
-                    expr=f'sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))',
+                    expr=f"sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))",
                     legendFormat="system error",
-                    refId='A',
+                    refId="A",
                 ),
                 Target(
-                    expr=f'sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))',
+                    expr=f"sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))",
                     legendFormat="user error",
-                    refId='B',
+                    refId="B",
                 ),
                 Target(
-                    expr=f'sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))',
-                    legendFormat="user error",
-                    refId='C',
+                    expr=f"sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))",
+                    legendFormat="unknown error",
+                    refId="C",
                 ),
             ],
-            yAxes=single_y_axis(format=NO_FORMAT),
+            yAxes=single_y_axis(format=OPS_FORMAT),
         )
 
     @staticmethod
@@ -600,67 +707,93 @@ def queue_metrics(collapse: bool) -> Row:
             collapse=collapse,
             panels=[
                 Graph(
-                    title="WF Adds to main queue",
+                    title="Add rate to queue",
+                    description="Rate at which items are actually added to the queue. If an item is already on the queue attempting to add it will be a no-op. Usually there is also rate limiting that will delay items from being added to the queue.",
                     dataSource=DATASOURCE,
                     targets=[
                         Target(
-                            expr=f'sum(rate(flyte:propeller:all:main_adds[5m]))',
-                            refId='A',
+                            expr="sum(rate(flyte:propeller:all:main_adds[5m]))",
+                            legendFormat="main",
+                            refId="A",
+                        ),
+                        Target(
+                            expr="sum(rate(flyte:propeller:all:sub_adds[5m]))",
+                            legendFormat="sub",
+                            refId="B",
+                        ),
+                        Target(
+                            expr="sum(rate(flyte:propeller:all:admin_launcher:_adds[5m]))",
+                            legendFormat="admin_launcher",
+                            refId="C",
                         ),
                     ],
-                    yAxes=single_y_axis(format=SHORT_FORMAT),
+                    yAxes=single_y_axis(format=OPS_FORMAT),
                 ),
                 Graph(
-                    title="Unprocessed Queue depth",
+                    title="Add rate before rate limiting and deduplication",
                     dataSource=DATASOURCE,
+                    description="Tracks every rate limited add synchronously before rate limiting delays or deduplication",
                     targets=[
                         Target(
-                            expr=f'sum(rate(flyte:propeller:all:main_depth[5m]))',
-                            refId='A',
+                            expr="sum(rate(flyte:propeller:all:main_retries[5m]))",
+                            legendFormat="main",
+                            refId="A",
+                        ),
+                        Target(
+                            expr="sum(rate(flyte:propeller:all:sub_retries[5m]))",
+                            legendFormat="sub",
+                            refId="B",
+                        ),
+                        Target(
+                            expr="sum(rate(flyte:propeller:all:admin_launcher:_retries[5m]))",
+                            legendFormat="admin_launcher",
+                            refId="C",
                         ),
                     ],
-                    yAxes=single_y_axis(format=SHORT_FORMAT),
+                    yAxes=single_y_axis(format=OPS_FORMAT),
                 ),
                 Graph(
-                    title="Item retries",
+                    title="Unprocessed Queue depth",
+                    description="The number of items that are currently in the queue but have not been processed yet.",
                     dataSource=DATASOURCE,
                     targets=[
                         Target(
-                            expr=f'sum(rate(flyte:propeller:all:main_retries[5m]))',
-                            refId='A',
+                            expr="sum(flyte:propeller:all:main_depth)",
+                            legendFormat="main",
+                            refId="A",
+                        ),
+                        Target(
+                            expr="sum(flyte:propeller:all:sub_depth)",
+                            legendFormat="sub",
+                            refId="B",
+                        ),
+                        Target(
+                            expr="sum(flyte:propeller:all:admin_launcher:_depth)",
+                            legendFormat="admin_launcher",
+                            refId="C",
                         ),
                     ],
                     yAxes=single_y_axis(format=SHORT_FORMAT),
                 ),
                 Graph(
                     title="Seconds of unfinished work in progress",
+                    description="Sum of the current in progress time of every in progress item in the queue.",
                     dataSource=DATASOURCE,
                     targets=[
                         Target(
-                            expr=f'flyte:propeller:all:main_unfinished_work_s',
-                            refId='A',
+                            expr="sum(flyte:propeller:all:main_unfinished_work_s)",
+                            legendFormat="main",
+                            refId="A",
                         ),
-                    ],
-                    yAxes=single_y_axis(format=SECONDS_FORMAT),
-                ),
-                Graph(
-                    title="Workqueue work average duration",
-                    dataSource=DATASOURCE,
-                    targets=[
                         Target(
-                            expr=f'sum(rate(flyte:propeller:all:main_work_duration_us_sum[5m]) / rate(flyte:propeller:all:main_work_duration_us_count[5m]))',
-                            refId='A',
+                            expr="sum(flyte:propeller:all:sub_unfinished_work_s)",
+                            legendFormat="sub",
+                            refId="B",
                         ),
-                    ],
-                    yAxes=single_y_axis(format=SECONDS_FORMAT),
-                ),
-                Graph(
-                    title="Duration for which an item stays in queue - avg",
-                    dataSource=DATASOURCE,
-                    targets=[
                         Target(
-                            expr=f'sum(rate(flyte:propeller:all:main_queue_latency_us_sum[5m]) / rate(flyte:propeller:all:main_queue_latency_us_count[5m]))',
-                            refId='A',
+                            expr="sum(flyte:propeller:all:admin_launcher:_unfinished_work_s)",
+                            legendFormat="admin_launcher",
+                            refId="C",
                         ),
                     ],
                     yAxes=single_y_axis(format=SECONDS_FORMAT),
@@ -688,13 +821,188 @@ def core_metrics(interval: int, collapse: bool) -> Row:
             collapse=collapse,
             panels=[
                 FlytePropeller.create_free_workers(),
-                FlytePropeller.abort_errors(),
-                FlytePropeller.system_errors(),
+                FlytePropeller.round_rates(),
+                FlytePropeller.error_breakdown(),
+                FlytePropeller.streak_rate(),
                 FlytePropeller.plugin_success_vs_failures(),
-                FlytePropeller.round_latency(interval),
-                FlytePropeller.round_latency_per_wf(interval),
-                FlytePropeller.round_panic(),
+                FlytePropeller.round_latency(),
+                FlytePropeller.round_latency_per_wf(),
                 FlytePropeller.workflows_per_project(),
+                FlytePropeller.enqueued_workflows(),
+            ],
+        )
+
+    @staticmethod
+    def workflow_latency_per_wf(latency_type: str) -> typing.List[Graph]:
+        latency_name = latency_type.replace(":", " ").capitalize()
+        return [
+            Graph(
+                title=f"{latency_name} latency per workflow",
+                dataSource=DATASOURCE,
+                targets=[
+                    Target(
+                        expr=f"sum(flyte:propeller:all:{latency_type}_latency_ms) by (wf)",
+                        refId="A",
+                    ),
+                ],
+                yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
+            ),
+            Graph(
+                title=f"{latency_name} latency by quantile",
+                dataSource=DATASOURCE,
+                targets=[
+                    Target(
+                        expr=f"sum(flyte:propeller:all:{latency_type}_latency_unlabeled_ms) by (quantile)",
+                        refId="A",
+                    ),
+                ],
+                yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
+            ),
+        ]
+
+    @staticmethod
+    def workflow_latencies(collapse: bool) -> Row:
+        return Row(
+            title="Workflow latencies",
+            collapse=collapse,
+            panels=[
+                panel
+                for panels in [
+                    FlytePropeller.workflow_latency_per_wf(latency_type)
+                    for latency_type in [
+                        "workflow:acceptance",
+                        "node:transition",
+                        "node:queueing",
+                        "workflow:completion",
+                    ]
+                ]
+                for panel in panels
+            ],
+        )
+
+    @staticmethod
+    def k8s_pod_informers(collapse: bool) -> Row:
+        return Row(
+            title="K8s Pod Informer stats",
+            collapse=collapse,
+            panels=[
+                Graph(
+                    title=f"Update event rate from informer",
+                    dataSource=DATASOURCE,
+                    targets=[
+                        Target(
+                            expr=f"sum(rate(flyte:propeller:all:node:container:container:informer_update[5m]))",
+                            refId="A",
+                        ),
+                    ],
+                    yAxes=single_y_axis(format=OPS_FORMAT),
+                ),
+                Graph(
+                    title=f"Update events drop rate becacuse they have the same resource version",
+                    dataSource=DATASOURCE,
+                    targets=[
+                        Target(
+                            expr=f"sum(rate(flyte:propeller:all:node:container:container:informer_update_dropped[5m]))",
+                            refId="A",
+                        ),
+                    ],
+                    yAxes=single_y_axis(format=OPS_FORMAT),
+                ),
+            ],
+        )
+
+    @staticmethod
+    def workflow_garbage_collection(collapse: bool) -> Row:
+        return Row(
+            title="Workflow garbage collection",
+            collapse=collapse,
+            panels=[
+                Graph(
+                    title="Namespace successful garbage collections",
+                    dataSource=DATASOURCE,
+                    targets=[
+                        Target(
+                            expr=f"sum(flyte:propeller:all:gc_success)",
+                            refId="A",
+                        ),
+                    ],
+                    yAxes=single_y_axis(format=SHORT_FORMAT),
+                ),
+                Graph(
+                    title="Namespace failed garbage collections",
+                    dataSource=DATASOURCE,
+                    targets=[
+                        Target(
+                            expr=f"sum(flyte:propeller:all:gc_failure)",
+                            refId="A",
+                        ),
+                    ],
+                    yAxes=single_y_axis(format=SHORT_FORMAT),
+                ),
+                Graph(
+                    title="Total completed garbage collection rounds",
+                    dataSource=DATASOURCE,
+                    targets=[
+                        Target(
+                            expr=f"sum(flyte:propeller:all:gc_latency_ms_count)",
+                            refId="A",
+                        ),
+                    ],
+                    yAxes=single_y_axis(format=SHORT_FORMAT),
+                ),
+                Graph(
+                    title="Average garbage collection round duration",
+                    dataSource=DATASOURCE,
+                    targets=[
+                        Target(
+                            expr=f"sum(flyte:propeller:all:gc_latency_ms_sum) / sum(flyte:propeller:all:gc_latency_ms_count)",
+                            refId="A",
+                        ),
+                    ],
+                    yAxes=single_y_axis(format=MILLISECONDS_FORMAT),
+                ),
+            ],
+        )
+
+    @staticmethod
+    def workflowstore(collapse: bool) -> Row:
+        return Row(
+            title="Workflow store",
+            collapse=collapse,
+            panels=[
+                Graph(
+                    title="Stale workflows rate",
+                    dataSource=DATASOURCE,
+                    targets=[
+                        Target(
+                            expr=f"sum(rate(flyte:propeller:all:wf_stale_unlabeled[5m])) by (quantile)",
+                            refId="A",
+                        ),
+                    ],
+                    yAxes=single_y_axis(format=OPS_FORMAT),
+                ),
+                Graph(
+                    title="Evict workflows rate",
+                    dataSource=DATASOURCE,
+                    targets=[
+                        Target(
+                            expr=f"sum(rate(flyte:propeller:all:wf_stale_unlabeled[5m])) by (quantile)",
+                            refId="A",
+                        ),
+                    ],
+                    yAxes=single_y_axis(format=OPS_FORMAT),
+                ),
+                Graph(
+                    title="Workflow redundant updates rate",
+                    dataSource=DATASOURCE,
+                    targets=[
+                        Target(
+                            expr=f"sum(rate(flyte:propeller:all:wf_redundant_unlabeled[5m])) by (quantile)",
+                            refId="A",
+                        ),
+                    ],
+                    yAxes=single_y_axis(format=OPS_FORMAT),
+                ),
             ],
         )
 
@@ -702,12 +1010,17 @@ def core_metrics(interval: int, collapse: bool) -> Row:
     def create_all_rows(interval: int = 5) -> typing.List[Row]:
         return [
             FlytePropeller.core_metrics(interval, False),
+            FlytePropeller.queue_metrics(False),
             FlytePropeller.metastore_metrics(interval, True),
             FlytePropeller.metastore_latencies(True),
             FlytePropeller.node_metrics(True),
             FlytePropeller.perf_metrics(True),
+            FlytePropeller.grpc_metrics(True),
+            FlytePropeller.workflow_latencies(False),
             FlytePropeller.wf_store_latency(False),
-            FlytePropeller.queue_metrics(True),
+            FlytePropeller.k8s_pod_informers(True),
+            FlytePropeller.workflowstore(True),
+            FlytePropeller.workflow_garbage_collection(True),
         ]