From 4d77c2a7b1a97cbcb20f3c159f72ef6bf05120f4 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 19 Mar 2024 17:15:43 -0500 Subject: [PATCH] [ui] Asset view support for column lineage metadata (#20525) ## Summary & Motivation This PR adds a new "Column" selector to the Asset > Lineage page for assets that emit `lineage` materialization event metadata. Choosing a column switches you to a column-oriented version of the graph, which supports the same upstream/downstream scoping + navigation as the normal lineage view, but shows column-level arrows and info. This PR makes a few changes to our asset layout system in order to support this new DAG styling: - The asset layout engine takes a wider range of it's hardcoded constants as configuration (previously just `direction`), and you can override Dagre layout settings. This allows us to run asset layout (with the worker, indexdb caching, etc), but with smaller `column` nodes. The options were already part of the cache keys, so this all works pretty smoothly! - I found an open PR on Dagre that fixes the issue with asset groups overlapping occasionally because the layout engine could not account for the presence of the "title bar" we render on them! I added the adjustments to our existing `dagre.patch` file. This is important in this PR because the nodes are the columns and the groups are the assets, so they overlap badly without this fix. ## How I Tested These Changes image image image --------- Co-authored-by: bengotow --- .../ui-components/src/components/Icon.tsx | 2 + .../src/icon-svgs/column_lineage.svg | 6 + .../dagster-ui/packages/ui-core/package.json | 2 +- .../src/asset-graph/AssetColumnsNode.tsx | 115 ++++++++++++ .../src/asset-graph/AssetGraphExplorer.tsx | 7 +- .../ui-core/src/asset-graph/AssetNode.tsx | 56 +++--- .../src/asset-graph/SidebarAssetInfo.tsx | 8 +- .../ui-core/src/asset-graph/Utils.tsx | 6 +- .../ui-core/src/asset-graph/layout.ts | 105 +++++++---- .../src/assets/AssetColumnLineageGraph.tsx | 166 ++++++++++++++++++ .../assets/AssetEventMetadataEntriesTable.tsx | 18 +- .../ui-core/src/assets/AssetNodeLineage.tsx | 75 +++++++- .../src/assets/AssetNodeLineageGraph.tsx | 33 +--- .../ui-core/src/assets/AssetNodeOverview.tsx | 55 +++--- .../packages/ui-core/src/assets/AssetView.tsx | 2 +- .../assets/LastMaterializationMetadata.tsx | 27 +-- .../ui-core/src/assets/SavedZoomLevel.tsx | 31 ++++ .../assets/buildConsolidatedColumnSchema.tsx | 51 ++++++ .../useColumnLineageDataForAssets.types.ts | 82 +++++++++ .../lineage/useColumnLineageDataForAssets.tsx | 158 +++++++++++++++++ .../packages/ui-core/src/assets/types.tsx | 1 + .../src/assets/useColumnLineageLayout.tsx | 142 +++++++++++++++ .../ui-core/src/graph/asyncGraphLayout.ts | 10 +- .../ui-core/src/metadata/MetadataEntry.tsx | 4 +- .../ui-core/src/metadata/TableSchema.tsx | 50 +++++- .../dagster-ui/patches/dagre+0.8.5.patch | 15 -- js_modules/dagster-ui/yarn.lock | 8 +- 27 files changed, 1056 insertions(+), 179 deletions(-) create mode 100644 js_modules/dagster-ui/packages/ui-components/src/icon-svgs/column_lineage.svg create mode 100644 js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx create mode 100644 js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx create mode 100644 js_modules/dagster-ui/packages/ui-core/src/assets/SavedZoomLevel.tsx create mode 100644 js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx create mode 100644 js_modules/dagster-ui/packages/ui-core/src/assets/lineage/types/useColumnLineageDataForAssets.types.ts create mode 100644 js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx create mode 100644 js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx delete mode 100644 js_modules/dagster-ui/patches/dagre+0.8.5.patch diff --git a/js_modules/dagster-ui/packages/ui-components/src/components/Icon.tsx b/js_modules/dagster-ui/packages/ui-components/src/components/Icon.tsx index 0ae0152f826a2..405c33c0ffc4f 100644 --- a/js_modules/dagster-ui/packages/ui-components/src/components/Icon.tsx +++ b/js_modules/dagster-ui/packages/ui-components/src/components/Icon.tsx @@ -42,6 +42,7 @@ import chevron_right from '../icon-svgs/chevron_right.svg'; import close from '../icon-svgs/close.svg'; import code_location from '../icon-svgs/code_location.svg'; import collapse_arrows from '../icon-svgs/collapse_arrows.svg'; +import column_lineage from '../icon-svgs/column_lineage.svg'; import concept_book from '../icon-svgs/concept-book.svg'; import console_icon from '../icon-svgs/console.svg'; import content_copy from '../icon-svgs/content_copy.svg'; @@ -284,6 +285,7 @@ export const Icons = { console: console_icon, content_copy, collapse_arrows, + column_lineage, corporate_fare, delete: deleteSVG, done, diff --git a/js_modules/dagster-ui/packages/ui-components/src/icon-svgs/column_lineage.svg b/js_modules/dagster-ui/packages/ui-components/src/icon-svgs/column_lineage.svg new file mode 100644 index 0000000000000..c1e6ff95e4075 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-components/src/icon-svgs/column_lineage.svg @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/js_modules/dagster-ui/packages/ui-core/package.json b/js_modules/dagster-ui/packages/ui-core/package.json index ac3b9e7f64a5d..0cf029f8e0df5 100644 --- a/js_modules/dagster-ui/packages/ui-core/package.json +++ b/js_modules/dagster-ui/packages/ui-core/package.json @@ -43,7 +43,7 @@ "codemirror": "^5.65.2", "color": "^3.0.0", "cronstrue": "^1.84.0", - "dagre": "^0.8.5", + "dagre": "dagster-io/dagre#0.8.5", "date-fns": "^2.28.0", "dayjs": "^1.11.7", "deepmerge": "^4.2.2", diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx new file mode 100644 index 0000000000000..af83205e3fb16 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetColumnsNode.tsx @@ -0,0 +1,115 @@ +import {Box, Caption, Colors, Icon, StyledTag, Tooltip} from '@dagster-io/ui-components'; +import {Link} from 'react-router-dom'; +import styled from 'styled-components'; + +import { + AssetInsetForHoverEffect, + AssetNameRow, + AssetNodeBox, + AssetNodeContainer, +} from './AssetNode'; +import {AssetNodeFragment} from './types/AssetNode.types'; +import {Timestamp} from '../app/time/Timestamp'; +import {assetDetailsPathForKey} from '../assets/assetDetailsPathForKey'; +import {AssetColumnLineageLocalColumn} from '../assets/lineage/useColumnLineageDataForAssets'; +import {AssetComputeKindTag} from '../graph/OpTags'; +import {AssetKeyInput} from '../graphql/types'; +import {iconForColumnType} from '../metadata/TableSchema'; +import {Description} from '../pipelines/Description'; + +export const AssetColumnsGroupNode = ({ + selected, + definition, + height, + asOf, +}: { + selected: boolean; + definition: AssetNodeFragment; + asOf: string | undefined; + height: number; +}) => { + return ( + + +
+ + + + + {asOf ? ( + + + + ) : undefined} + + + + + + ); +}; + +export const AssetColumnNode = ({ + assetKey, + column, + blueBackground, +}: { + assetKey: AssetKeyInput; + column: AssetColumnLineageLocalColumn; + blueBackground: boolean; +}) => { + const icon = iconForColumnType(column.type ?? ''); + + return ( + + + +
+ } + > + + {icon ? : } + + {column.name} + + + + + ); +}; + +const ColumnLink = styled(Link)<{$blueBackground: boolean}>` + height: 28px; + margin: 2px 12px; + padding-left: 2px; + padding-right: 4px; + display: flex; + gap: 4px; + align-items: center; + transition: background 100ms linear; + border-radius: 8px; + + ${StyledTag} { + background: none; + color: ${Colors.textLight()}; + } + ${(p) => + p.$blueBackground + ? ` + background: ${Colors.backgroundBlue()}` + : ` + &:hover { + text-decoration: none; + background: ${Colors.backgroundLightHover()}; + }`} +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx index e52795efc9103..9e1e6ff3d86de 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetGraphExplorer.tsx @@ -16,6 +16,7 @@ import pickBy from 'lodash/pickBy'; import uniq from 'lodash/uniq'; import without from 'lodash/without'; import * as React from 'react'; +import {useMemo} from 'react'; import styled from 'styled-components'; import {AssetEdges} from './AssetEdges'; @@ -269,7 +270,11 @@ const AssetGraphExplorerWithData = ({ }); const focusGroupIdAfterLayoutRef = React.useRef(''); - const {layout, loading, async} = useAssetLayout(assetGraphData, expandedGroups, direction); + const {layout, loading, async} = useAssetLayout( + assetGraphData, + expandedGroups, + useMemo(() => ({direction}), [direction]), + ); React.useEffect(() => { if (!loading) { diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx index 2148119e451cf..70641ee82ef90 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/AssetNode.tsx @@ -28,14 +28,16 @@ interface Props { } export const AssetNode = React.memo(({definition, selected}: Props) => { - const displayName = definition.assetKey.path[definition.assetKey.path.length - 1]!; const isSource = definition.isSource; const {liveData} = useAssetLiveData(definition.assetKey); return ( - + { - - - - -
- {withMiddleTruncation(displayName, { - maxLength: ASSET_NODE_NAME_MAX_LENGTH, - })} -
-
- + {definition.description ? ( @@ -83,6 +71,28 @@ export const AssetNode = React.memo(({definition, selected}: Props) => { ); }, isEqual); +export const AssetNameRow = ({definition}: {definition: AssetNodeFragment}) => { + const displayName = definition.assetKey.path[definition.assetKey.path.length - 1]!; + + return ( + + + + +
+ {withMiddleTruncation(displayName, { + maxLength: ASSET_NODE_NAME_MAX_LENGTH, + })} +
+
+ + ); +}; + const AssetNodeRowBox = styled(Box)` white-space: nowrap; line-height: 12px; @@ -244,7 +254,7 @@ export const ASSET_NODE_FRAGMENT = gql` } `; -const AssetInsetForHoverEffect = styled.div` +export const AssetInsetForHoverEffect = styled.div` padding: 10px 4px 2px 4px; height: 100%; @@ -253,7 +263,7 @@ const AssetInsetForHoverEffect = styled.div` } `; -const AssetNodeContainer = styled.div<{$selected: boolean}>` +export const AssetNodeContainer = styled.div<{$selected: boolean}>` user-select: none; cursor: pointer; padding: 6px; @@ -264,7 +274,11 @@ const AssetNodeShowOnHover = styled.span` display: none; `; -const AssetNodeBox = styled.div<{$isSource: boolean; $selected: boolean}>` +export const AssetNodeBox = styled.div<{ + $isSource: boolean; + $selected: boolean; + $noScale?: boolean; +}>` ${(p) => p.$isSource ? `border: 2px dashed ${p.$selected ? Colors.accentGrayHover() : Colors.accentGray()}` @@ -280,7 +294,7 @@ const AssetNodeBox = styled.div<{$isSource: boolean; $selected: boolean}>` &:hover { ${(p) => !p.$selected && `border: 2px solid ${Colors.lineageNodeBorderHover()};`}; box-shadow: ${Colors.shadowDefault()} 0px 1px 4px 0px; - scale: 1.03; + scale: ${(p) => (p.$noScale ? '1' : '1.03')}; ${AssetNodeShowOnHover} { display: initial; } diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx index 3ce4087a9cbb3..c6644d6256854 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx @@ -32,6 +32,7 @@ import { import {DagsterTypeSummary} from '../dagstertype/DagsterType'; import {DagsterTypeFragment} from '../dagstertype/types/DagsterType.types'; import {METADATA_ENTRY_FRAGMENT} from '../metadata/MetadataEntry'; +import {TableSchemaLineageContext} from '../metadata/TableSchema'; import {Description} from '../pipelines/Description'; import {SidebarSection, SidebarTitle} from '../pipelines/SidebarComponents'; import {ResourceContainer, ResourceHeader} from '../pipelines/SidebarOpHelpers'; @@ -159,7 +160,12 @@ export const SidebarAssetInfo = ({graphNode}: {graphNode: GraphNode}) => { {assetMetadata.length > 0 && ( - + + + )} diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx index 03b3191d33a90..f8489806cd64c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/Utils.tsx @@ -44,7 +44,7 @@ export function isHiddenAssetGroupJob(jobName: string) { // export type GraphId = string; export const toGraphId = (key: {path: string[]}): GraphId => JSON.stringify(key.path); -export const fromGraphID = (graphId: GraphId): AssetNodeKeyFragment => ({ +export const fromGraphId = (graphId: GraphId): AssetNodeKeyFragment => ({ path: JSON.parse(graphId), __typename: 'AssetKey', }); @@ -266,7 +266,7 @@ export const itemWithAssetKey = (key: {path: string[]}) => { return (asset: {assetKey: {path: string[]}}) => tokenForAssetKey(asset.assetKey) === token; }; -export const isGroupId = (str: string) => /^[^@:]+@[^@:]+:[^@:]+$/.test(str); +export const isGroupId = (str: string) => /^[^@:]+@[^@:]+:.+$/.test(str); export const groupIdForNode = (node: GraphNode) => [ @@ -281,7 +281,7 @@ export const groupIdForNode = (node: GraphNode) => export const getUpstreamNodes = memoize( (assetKey: AssetNodeKeyFragment, graphData: GraphData): AssetNodeKeyFragment[] => { const upstream = Object.keys(graphData.upstream[toGraphId(assetKey)] || {}); - const currentUpstream = upstream.map((graphId) => fromGraphID(graphId)); + const currentUpstream = upstream.map((graphId) => fromGraphId(graphId)); return [ assetKey, ...currentUpstream, diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts index c3b7ba28b882d..c4083745196fb 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/layout.ts @@ -35,8 +35,59 @@ export type AssetGraphLayout = { }; const MARGIN = 100; +export type LayoutAssetGraphConfig = dagre.GraphLabel & { + direction: AssetLayoutDirection; + /** Pass `auto` to use getAssetNodeDimensions, or a value to give nodes a fixed height */ + nodeHeight: number | 'auto'; + /** Our asset groups have "title bars" - use these numbers to adjust the bounding boxes. + * Note that these adjustments are applied post-dagre layout. For padding > nodesep, you + * may need to set "clusterpaddingtop", "clusterpaddingbottom" so Dagre lays out the boxes + * with more spacing. + */ + groupPaddingTop: number; + groupPaddingBottom: number; + groupRendering: 'if-varied' | 'always'; + + /** Supported in Dagre, just not documented. Additional spacing between group nodes */ + clusterpaddingtop?: number; + clusterpaddingbottom?: number; +}; + export type LayoutAssetGraphOptions = { direction: AssetLayoutDirection; + overrides?: Partial; +}; + +export const Config = { + horizontal: { + ranker: 'tight-tree', + direction: 'horizontal', + marginx: MARGIN, + marginy: MARGIN, + ranksep: 60, + rankdir: 'LR', + edgesep: 90, + nodesep: -10, + nodeHeight: 'auto', + groupPaddingTop: 65, + groupPaddingBottom: -15, + groupRendering: 'if-varied', + clusterpaddingtop: 100, + }, + vertical: { + ranker: 'tight-tree', + direction: 'horizontal', + marginx: MARGIN, + marginy: MARGIN, + ranksep: 20, + rankdir: 'TB', + nodesep: 40, + edgesep: 10, + nodeHeight: 'auto', + groupPaddingTop: 40, + groupPaddingBottom: -20, + groupRendering: 'if-varied', + }, }; export const layoutAssetGraph = ( @@ -44,30 +95,9 @@ export const layoutAssetGraph = ( opts: LayoutAssetGraphOptions, ): AssetGraphLayout => { const g = new dagre.graphlib.Graph({compound: true}); + const config = Object.assign({}, Config[opts.direction], opts.overrides || {}); - const ranker = 'tight-tree'; - - g.setGraph( - opts.direction === 'horizontal' - ? { - rankdir: 'LR', - marginx: MARGIN, - marginy: MARGIN, - nodesep: -10, - edgesep: 90, - ranksep: 60, - ranker, - } - : { - rankdir: 'TB', - marginx: MARGIN, - marginy: MARGIN, - nodesep: 40, - edgesep: 10, - ranksep: 20, - ranker, - }, - ); + g.setGraph(config); g.setDefaultEdgeLabel(() => ({})); // const shouldRender = (node?: GraphNode) => node && node.definition.opNames.length > 0; @@ -92,13 +122,17 @@ export const layoutAssetGraph = ( } // Add all the group boxes to the graph - const groupsPresent = Object.keys(groups).length > 1; + const groupsPresent = + config.groupRendering === 'if-varied' ? Object.keys(groups).length > 1 : true; + if (groupsPresent) { Object.keys(groups).forEach((groupId) => { if (expandedGroups.includes(groupId)) { - g.setNode(groupId, {}); // sized based on it's children + // sized based on it's children, but "border" tells Dagre we want cluster-level + // spacing between the node and others. Necessary because our groups have title bars. + g.setNode(groupId, {borderType: 'borderRight'}); } else { - g.setNode(groupId, {width: 320, height: 110}); + g.setNode(groupId, {width: ASSET_NODE_WIDTH, height: 110}); } }); } @@ -106,7 +140,12 @@ export const layoutAssetGraph = ( // Add all the nodes inside expanded groups to the graph renderedNodes.forEach((node) => { if (!groupsPresent || expandedGroups.includes(groupIdForNode(node))) { - g.setNode(node.id, getAssetNodeDimensions(node.definition)); + const label = + config.nodeHeight === 'auto' + ? getAssetNodeDimensions(node.definition) + : {width: ASSET_NODE_WIDTH, height: config.nodeHeight}; + + g.setNode(node.id, label); if (groupsPresent && node.definition.groupName) { g.setParent(node.id, groupIdForNode(node)); } @@ -205,10 +244,11 @@ export const layoutAssetGraph = ( } for (const group of Object.values(groups)) { if (group.expanded) { - group.bounds = - opts.direction === 'horizontal' - ? padBounds(group.bounds, {x: 15, top: 65, bottom: -15}) - : padBounds(group.bounds, {x: 15, top: 40, bottom: -20}); + group.bounds = padBounds(group.bounds, { + x: 15, + top: config.groupPaddingTop, + bottom: config.groupPaddingBottom, + }); } } } @@ -276,6 +316,7 @@ export const extendBounds = (a: IBounds, b: IBounds) => { return {x: xmin, y: ymin, width: xmax - xmin, height: ymax - ymin}; }; +export const ASSET_NODE_WIDTH = 320; export const ASSET_NODE_NAME_MAX_LENGTH = 38; export const getAssetNodeDimensions = (def: { @@ -289,7 +330,7 @@ export const getAssetNodeDimensions = (def: { computeKind: string | null; changedReasons?: ChangeReason[]; }) => { - const width = 320; + const width = ASSET_NODE_WIDTH; let height = 100; // top tags area + name + description diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx new file mode 100644 index 0000000000000..37a6c90fe9371 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetColumnLineageGraph.tsx @@ -0,0 +1,166 @@ +import {Box, Spinner} from '@dagster-io/ui-components'; +import {useMemo, useRef, useState} from 'react'; +import styled from 'styled-components'; + +import {SVGSaveZoomLevel, useLastSavedZoomLevel} from './SavedZoomLevel'; +import {AssetColumnLineages} from './lineage/useColumnLineageDataForAssets'; +import {fromColumnGraphId, useColumnLineageLayout} from './useColumnLineageLayout'; +import {AssetColumnNode, AssetColumnsGroupNode} from '../asset-graph/AssetColumnsNode'; +import {AssetEdges} from '../asset-graph/AssetEdges'; +import {AssetNodeContextMenuWrapper} from '../asset-graph/AssetNode'; +import {GraphData, fromGraphId, toGraphId} from '../asset-graph/Utils'; +import {DEFAULT_MAX_ZOOM, SVGViewport} from '../graph/SVGViewport'; +import {isNodeOffscreen} from '../graph/common'; +import {AssetKeyInput} from '../graphql/types'; + +export const AssetColumnLineageGraph = ({ + assetKey, + assetGraphData, + columnLineageData, + focusedColumn, +}: { + assetKey: AssetKeyInput; + assetGraphData: GraphData; + columnLineageData: AssetColumnLineages; + focusedColumn: string; +}) => { + const focusedAssetGraphId = toGraphId(assetKey); + + const [highlighted, setHighlighted] = useState(null); + + const {layout, loading} = useColumnLineageLayout( + assetGraphData, + focusedAssetGraphId, + focusedColumn, + columnLineageData, + ); + + const viewportEl = useRef(); + + useLastSavedZoomLevel(viewportEl, layout, focusedAssetGraphId); + + const blue = useMemo(() => { + const blue = new Set(); + if (!highlighted || !layout) { + return blue; + } + + for (const id of highlighted) { + blue.add(id); + layout.edges.filter((e) => e.fromId === id).forEach((e) => blue.add(e.toId)); + layout.edges.filter((e) => e.toId === id).forEach((e) => blue.add(e.fromId)); + } + return blue; + }, [layout, highlighted]); + + if (!layout || loading) { + return ( + + + + ); + } + + return ( + (viewportEl.current = r || undefined)} + interactor={SVGViewport.Interactors.PanAndZoom} + defaultZoom="zoom-to-fit" + graphWidth={layout.width} + graphHeight={layout.height} + onDoubleClick={(e) => { + viewportEl.current?.autocenter(true); + e.stopPropagation(); + }} + maxZoom={DEFAULT_MAX_ZOOM} + maxAutocenterZoom={DEFAULT_MAX_ZOOM} + > + {({scale}, viewportRect) => ( + + {viewportEl.current && } + + {Object.values(layout.groups) + .filter((node) => !isNodeOffscreen(node.bounds, viewportRect)) + .map(({id, bounds}) => { + const groupAssetGraphId = toGraphId({path: id.split(':').pop()!.split('>')}); + const graphNode = assetGraphData.nodes[groupAssetGraphId]; + const contextMenuProps = { + graphData: assetGraphData, + node: graphNode!, + }; + + const cols = columnLineageData[groupAssetGraphId] || {}; + const colsAsOf = Object.values(cols)[0]?.asOf; + + return ( + setHighlighted([id])} + onMouseLeave={() => setHighlighted(null)} + onDoubleClick={(e) => { + viewportEl.current?.zoomToSVGBox(bounds, true, 1.2); + e.stopPropagation(); + }} + > + + + + + ); + })} + + + + {Object.values(layout.nodes) + .filter((node) => !isNodeOffscreen(node.bounds, viewportRect)) + .map(({id, bounds}) => { + const {assetGraphId, column} = fromColumnGraphId(id); + const assetKey = fromGraphId(assetGraphId); + + const col = columnLineageData[assetGraphId]?.[column] || { + name: column, + description: 'Not found in column metadata', + type: null, + upstream: [], + asOf: undefined, + }; + + return ( + setHighlighted([id])} + onMouseLeave={() => setHighlighted(null)} + onDoubleClick={(e) => { + viewportEl.current?.zoomToSVGBox(bounds, true, 1.2); + e.stopPropagation(); + }} + > + + + ); + })} + + )} + + ); +}; + +const SVGContainer = styled.svg` + overflow: visible; + border-radius: 0; +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetEventMetadataEntriesTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetEventMetadataEntriesTable.tsx index 1aa46aafaf8df..32ffaa4168861 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetEventMetadataEntriesTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetEventMetadataEntriesTable.tsx @@ -11,7 +11,7 @@ import { } from './types/useRecentAssetEvents.types'; import {Timestamp} from '../app/time/Timestamp'; import {HIDDEN_METADATA_ENTRY_LABELS, MetadataEntry} from '../metadata/MetadataEntry'; -import {isCanonicalTableSchemaEntry} from '../metadata/TableSchema'; +import {isCanonicalColumnLineageEntry, isCanonicalColumnSchemaEntry} from '../metadata/TableSchema'; import {MetadataEntryFragment} from '../metadata/types/MetadataEntry.types'; import {titleForRun} from '../runs/RunUtils'; @@ -89,14 +89,14 @@ export const AssetEventMetadataEntriesTable = ({ const filteredRows = useMemo( () => - allRows.filter( - (row) => - !filter || - row.entry.label.toLowerCase().includes(filter.toLowerCase()) || - !HIDDEN_METADATA_ENTRY_LABELS.has(row.entry.label) || - !hideTableSchema || - !isCanonicalTableSchemaEntry(row.entry), - ), + allRows + .filter((row) => !filter || row.entry.label.toLowerCase().includes(filter.toLowerCase())) + .filter( + (row) => + !HIDDEN_METADATA_ENTRY_LABELS.has(row.entry.label) && + !(isCanonicalColumnSchemaEntry(row.entry) && hideTableSchema) && + !isCanonicalColumnLineageEntry(row.entry), + ), [allRows, filter, hideTableSchema], ); diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx index 69fff663b3b46..04306e03958b8 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineage.tsx @@ -5,17 +5,24 @@ import { Colors, Icon, JoinedButtons, + MenuItem, + Suggest, TextInput, } from '@dagster-io/ui-components'; -import {useEffect, useMemo, useState} from 'react'; +import React, {useEffect, useMemo, useState} from 'react'; import styled from 'styled-components'; +import {AssetColumnLineageGraph} from './AssetColumnLineageGraph'; import {AssetNodeLineageGraph} from './AssetNodeLineageGraph'; import {LaunchAssetExecutionButton} from './LaunchAssetExecutionButton'; +import {asAssetKeyInput} from './asInput'; +import {useColumnLineageDataForAssets} from './lineage/useColumnLineageDataForAssets'; import {AssetLineageScope, AssetViewParams} from './types'; -import {GraphData} from '../asset-graph/Utils'; +import {GraphData, toGraphId} from '../asset-graph/Utils'; import {AssetGraphQueryItem, calculateGraphDistances} from '../asset-graph/useAssetGraphData'; import {AssetKeyInput} from '../graphql/types'; +import {useQueryPersistedState} from '../hooks/useQueryPersistedState'; +import {ClearButton} from '../ui/ClearButton'; export const AssetNodeLineage = ({ params, @@ -45,6 +52,18 @@ export const AssetNodeLineage = ({ const currentDepth = Math.max(1, Math.min(maxDepth, requestedDepth)); + const assetGraphKeys = useMemo( + () => Object.values(assetGraphData.nodes).map(asAssetKeyInput), + [assetGraphData], + ); + const columnLineageData = useColumnLineageDataForAssets(assetGraphKeys); + const columnLineage = columnLineageData[toGraphId(assetKey)]; + const [column, setColumn] = useQueryPersistedState({ + queryKey: 'column', + decode: (qs) => qs.column || null, + encode: (column) => ({column: column || undefined}), + }); + return ( setParams({...params, lineageDepth: depth})} max={maxDepth} /> + {columnLineage || column ? ( + <> + Column + setColumn(null)} + style={{marginTop: 5, marginRight: 4}} + > + + + ) : undefined, + }} + selectedItem={column} + items={Object.keys(columnLineage || {})} + noResults="No matching columns" + onItemSelect={setColumn} + inputValueRenderer={(item) => item} + itemPredicate={(query, item) => + item.toLocaleLowerCase().includes(query.toLocaleLowerCase()) + } + itemRenderer={(item, itemProps) => ( + itemProps.handleClick(e)} + text={item} + key={item} + /> + )} + /> + + ) : undefined}
{Object.values(assetGraphData.nodes).length > 1 ? ( )} - + {column ? ( + + ) : ( + + )} ); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx index 13d99ed05388b..7d9df75ba23ab 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeLineageGraph.tsx @@ -1,8 +1,9 @@ import {Box, Spinner} from '@dagster-io/ui-components'; -import {useEffect, useMemo, useRef, useState} from 'react'; +import {useMemo, useRef, useState} from 'react'; import {useHistory} from 'react-router-dom'; import styled from 'styled-components'; +import {SVGSaveZoomLevel, useLastSavedZoomLevel} from './SavedZoomLevel'; import {assetDetailsPathForKey} from './assetDetailsPathForKey'; import {AssetKey, AssetViewParams} from './types'; import {AssetEdges} from '../asset-graph/AssetEdges'; @@ -11,13 +12,13 @@ import {AssetNode, AssetNodeContextMenuWrapper, AssetNodeMinimal} from '../asset import {ExpandedGroupNode, GroupOutline} from '../asset-graph/ExpandedGroupNode'; import {AssetNodeLink} from '../asset-graph/ForeignNode'; import {GraphData, GraphNode, groupIdForNode, toGraphId} from '../asset-graph/Utils'; +import {LayoutAssetGraphOptions} from '../asset-graph/layout'; import {DEFAULT_MAX_ZOOM, SVGViewport} from '../graph/SVGViewport'; import {useAssetLayout} from '../graph/asyncGraphLayout'; import {isNodeOffscreen} from '../graph/common'; import {AssetKeyInput} from '../graphql/types'; -import {getJSONForKey} from '../hooks/useStateWithStorage'; -const LINEAGE_GRAPH_ZOOM_LEVEL = 'lineageGraphZoomLevel'; +const LINEAGE_GRAPH_OPTIONS: LayoutAssetGraphOptions = {direction: 'horizontal'}; export const AssetNodeLineageGraph = ({ assetKey, @@ -42,7 +43,7 @@ export const AssetNodeLineageGraph = ({ const [highlighted, setHighlighted] = useState(null); - const {layout, loading} = useAssetLayout(assetGraphData, allGroups, 'horizontal'); + const {layout, loading} = useAssetLayout(assetGraphData, allGroups, LINEAGE_GRAPH_OPTIONS); const viewportEl = useRef(); const history = useHistory(); @@ -50,13 +51,7 @@ export const AssetNodeLineageGraph = ({ history.push(assetDetailsPathForKey(key, {...params, lineageScope: 'neighbors'})); }; - useEffect(() => { - if (viewportEl.current && layout) { - const lastZoomLevel = Number(getJSONForKey(LINEAGE_GRAPH_ZOOM_LEVEL)); - viewportEl.current.autocenter(false, lastZoomLevel); - viewportEl.current.focus(); - } - }, [viewportEl, layout, assetGraphId]); + useLastSavedZoomLevel(viewportEl, layout, assetGraphId); if (!layout || loading) { return ( @@ -113,10 +108,7 @@ export const AssetNodeLineageGraph = ({ .map((group) => ( @@ -174,17 +166,6 @@ export const AssetNodeLineageGraph = ({ ); }; -const SVGSaveZoomLevel = ({scale}: {scale: number}) => { - useEffect(() => { - try { - window.localStorage.setItem(LINEAGE_GRAPH_ZOOM_LEVEL, JSON.stringify(scale)); - } catch (err) { - // no-op - } - }, [scale]); - return <>; -}; - const SVGContainer = styled.svg` overflow: visible; border-radius: 0; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx index 3d99f1881739b..7fb5e537cf297 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeOverview.tsx @@ -36,6 +36,7 @@ import {SimpleStakeholderAssetStatus} from './SimpleStakeholderAssetStatus'; import {UnderlyingOpsOrGraph} from './UnderlyingOpsOrGraph'; import {AssetChecksStatusSummary} from './asset-checks/AssetChecksStatusSummary'; import {assetDetailsPathForKey} from './assetDetailsPathForKey'; +import {buildConsolidatedColumnSchema} from './buildConsolidatedColumnSchema'; import {globalAssetGraphPathForAssetsAndDescendants} from './globalAssetGraphPathToString'; import {AssetKey} from './types'; import {AssetNodeDefinitionFragment} from './types/AssetNodeDefinition.types'; @@ -55,7 +56,11 @@ import {DagsterTypeSummary} from '../dagstertype/DagsterType'; import {AssetComputeKindTag} from '../graph/OpTags'; import {useStateWithStorage} from '../hooks/useStateWithStorage'; import {useLaunchPadHooks} from '../launchpad/LaunchpadHooksContext'; -import {TableSchema, isCanonicalTableSchemaEntry} from '../metadata/TableSchema'; +import { + TableSchema, + TableSchemaLineageContext, + isCanonicalColumnLineageEntry, +} from '../metadata/TableSchema'; import {RepositoryLink} from '../nav/RepositoryLink'; import {ScheduleOrSensorTag} from '../nav/ScheduleOrSensorTag'; import {useRepositoryLocationForAddress} from '../nav/useRepositoryLocationForAddress'; @@ -102,35 +107,13 @@ export const AssetNodeOverview = ({ return ; } - const materializationTableSchema = materialization?.metadataEntries.find( - isCanonicalTableSchemaEntry, - ); - const materializationTableSchemaLoadTimestamp = materialization - ? Number(materialization.timestamp) - : undefined; - const definitionTableSchema = assetNode?.metadataEntries.find(isCanonicalTableSchemaEntry); - const definitionTableSchemaLoadTimestamp = assetNodeLoadTimestamp; - - let tableSchema = materializationTableSchema ?? definitionTableSchema; - const tableSchemaLoadTimestamp = - materializationTableSchemaLoadTimestamp ?? definitionTableSchemaLoadTimestamp; - - // Merge the descriptions from the definition table schema with the materialization table schema - if (materializationTableSchema && definitionTableSchema) { - const definitionTableSchemaColumnDescriptionsByName = Object.fromEntries( - definitionTableSchema.schema.columns.map((column) => [column.name, column.description]), - ); - const mergedColumns = materializationTableSchema.schema.columns.map((column) => { - const description = - definitionTableSchemaColumnDescriptionsByName[column.name] || column.description; - return {...column, description}; - }); - - tableSchema = { - ...materializationTableSchema, - schema: {...materializationTableSchema.schema, columns: mergedColumns}, - }; - } + const {tableSchema, tableSchemaLoadTimestamp} = buildConsolidatedColumnSchema({ + materialization, + definition: assetNode, + definitionLoadTimestamp: assetNodeLoadTimestamp, + }); + + const columnSchema = materialization?.metadataEntries.find(isCanonicalColumnLineageEntry); const renderStatusSection = () => ( @@ -411,10 +394,14 @@ export const AssetNodeOverview = ({ {tableSchema && ( - + + + )} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx index 219673952c2e2..01033017d8822 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx @@ -366,7 +366,7 @@ function getQueryForVisibleAssets( return {query: `+"${token}"+`, requestedDepth: 1}; } if (view === 'lineage') { - const defaultDepth = lineageScope === 'neighbors' ? 2 : 5; + const defaultDepth = 1; const requestedDepth = Number(lineageDepth) || defaultDepth; const depthStr = '+'.repeat(requestedDepth); diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/LastMaterializationMetadata.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/LastMaterializationMetadata.tsx index 6c3c9559b59c6..63263400d7b5a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/LastMaterializationMetadata.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/LastMaterializationMetadata.tsx @@ -15,6 +15,7 @@ import {Timestamp} from '../app/time/Timestamp'; import {LiveDataForNode, isHiddenAssetGroupJob} from '../asset-graph/Utils'; import {AssetKeyInput} from '../graphql/types'; import {MetadataEntry} from '../metadata/MetadataEntry'; +import {isCanonicalColumnLineageEntry} from '../metadata/TableSchema'; import {Description} from '../pipelines/Description'; import {PipelineReference} from '../pipelines/PipelineReference'; import {linkToRunEvent, titleForRun} from '../runs/RunUtils'; @@ -156,18 +157,20 @@ export const LatestMaterializationMetadata = ({ ) : null} - {latestEvent.metadataEntries.map((entry) => ( - - {entry.label} - - - - - ))} + {latestEvent.metadataEntries + .filter((entry) => !isCanonicalColumnLineageEntry(entry)) + .map((entry) => ( + + {entry.label} + + + + + ))} ) : ( diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/SavedZoomLevel.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/SavedZoomLevel.tsx new file mode 100644 index 0000000000000..3d75adb38f5c0 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/SavedZoomLevel.tsx @@ -0,0 +1,31 @@ +import {MutableRefObject, useEffect} from 'react'; + +import {SVGViewport} from '../graph/SVGViewport'; +import {getJSONForKey} from '../hooks/useStateWithStorage'; + +const LINEAGE_GRAPH_ZOOM_LEVEL = 'lineageGraphZoomLevel'; + +export const SVGSaveZoomLevel = ({scale}: {scale: number}) => { + useEffect(() => { + try { + window.localStorage.setItem(LINEAGE_GRAPH_ZOOM_LEVEL, JSON.stringify(scale)); + } catch (err) { + // no-op + } + }, [scale]); + return <>; +}; + +export function useLastSavedZoomLevel( + viewportEl: MutableRefObject, + layout: import('../asset-graph/layout').AssetGraphLayout | null, + graphFocusChangeKey: string, +) { + useEffect(() => { + if (viewportEl.current && layout) { + const lastZoomLevel = Number(getJSONForKey(LINEAGE_GRAPH_ZOOM_LEVEL)); + viewportEl.current.autocenter(false, lastZoomLevel); + viewportEl.current.focus(); + } + }, [viewportEl, layout, graphFocusChangeKey]); +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx new file mode 100644 index 0000000000000..c4081b0699bb0 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/buildConsolidatedColumnSchema.tsx @@ -0,0 +1,51 @@ +// eslint-disable-next-line no-restricted-imports + +import {AssetColumnLineageQuery} from './lineage/types/useColumnLineageDataForAssets.types'; +import {isCanonicalColumnSchemaEntry} from '../metadata/TableSchema'; + +type AssetDefinitionWithMetadata = AssetColumnLineageQuery['assetNodes'][0]; + +/** + * This helper pulls the `columns` metadata entry from the most recent materialization + * and the asset definition, blending the two together to produce the most current + * representation. (Sometimes descriptions are only in the definition-time version) + */ +export function buildConsolidatedColumnSchema({ + materialization, + definition, + definitionLoadTimestamp, +}: { + materialization: + | Pick + | undefined; + definition: Pick | undefined; + definitionLoadTimestamp: number | undefined; +}) { + const materializationTableSchema = materialization?.metadataEntries.find( + isCanonicalColumnSchemaEntry, + ); + const materializationTimestamp = materialization ? Number(materialization.timestamp) : undefined; + const definitionTableSchema = definition?.metadataEntries.find(isCanonicalColumnSchemaEntry); + + let tableSchema = materializationTableSchema ?? definitionTableSchema; + const tableSchemaLoadTimestamp = materializationTimestamp ?? definitionLoadTimestamp; + + // Merge the descriptions from the definition table schema with the materialization table schema + if (materializationTableSchema && definitionTableSchema) { + const definitionTableSchemaColumnDescriptionsByName = Object.fromEntries( + definitionTableSchema.schema.columns.map((column) => [column.name, column.description]), + ); + const mergedColumns = materializationTableSchema.schema.columns.map((column) => { + const description = + definitionTableSchemaColumnDescriptionsByName[column.name] || column.description; + return {...column, description}; + }); + + tableSchema = { + ...materializationTableSchema, + schema: {...materializationTableSchema.schema, columns: mergedColumns}, + }; + } + console.log(tableSchema); + return {tableSchema, tableSchemaLoadTimestamp}; +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/types/useColumnLineageDataForAssets.types.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/types/useColumnLineageDataForAssets.types.ts new file mode 100644 index 0000000000000..e9db8c746bf02 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/types/useColumnLineageDataForAssets.types.ts @@ -0,0 +1,82 @@ +// Generated GraphQL types, do not edit manually. + +import * as Types from '../../../graphql/types'; + +export type AssetColumnLineageQueryVariables = Types.Exact<{ + assetKeys: Array | Types.AssetKeyInput; +}>; + +export type AssetColumnLineageQuery = { + __typename: 'Query'; + assetNodes: Array<{ + __typename: 'AssetNode'; + id: string; + assetKey: {__typename: 'AssetKey'; path: Array}; + metadataEntries: Array< + | {__typename: 'AssetMetadataEntry'; label: string} + | {__typename: 'BoolMetadataEntry'; label: string} + | {__typename: 'FloatMetadataEntry'; label: string} + | {__typename: 'IntMetadataEntry'; label: string} + | {__typename: 'JobMetadataEntry'; label: string} + | {__typename: 'JsonMetadataEntry'; label: string} + | {__typename: 'MarkdownMetadataEntry'; label: string} + | {__typename: 'NotebookMetadataEntry'; label: string} + | {__typename: 'NullMetadataEntry'; label: string} + | {__typename: 'PathMetadataEntry'; label: string} + | {__typename: 'PipelineRunMetadataEntry'; label: string} + | {__typename: 'PythonArtifactMetadataEntry'; label: string} + | {__typename: 'TableMetadataEntry'; label: string} + | { + __typename: 'TableSchemaMetadataEntry'; + label: string; + schema: { + __typename: 'TableSchema'; + columns: Array<{ + __typename: 'TableColumn'; + name: string; + type: string; + description: string | null; + }>; + }; + } + | {__typename: 'TextMetadataEntry'; label: string} + | {__typename: 'TimestampMetadataEntry'; label: string} + | {__typename: 'UrlMetadataEntry'; label: string} + >; + assetMaterializations: Array<{ + __typename: 'MaterializationEvent'; + timestamp: string; + metadataEntries: Array< + | {__typename: 'AssetMetadataEntry'; label: string} + | {__typename: 'BoolMetadataEntry'; label: string} + | {__typename: 'FloatMetadataEntry'; label: string} + | {__typename: 'IntMetadataEntry'; label: string} + | {__typename: 'JobMetadataEntry'; label: string} + | {__typename: 'JsonMetadataEntry'; jsonString: string; label: string} + | {__typename: 'MarkdownMetadataEntry'; label: string} + | {__typename: 'NotebookMetadataEntry'; label: string} + | {__typename: 'NullMetadataEntry'; label: string} + | {__typename: 'PathMetadataEntry'; label: string} + | {__typename: 'PipelineRunMetadataEntry'; label: string} + | {__typename: 'PythonArtifactMetadataEntry'; label: string} + | {__typename: 'TableMetadataEntry'; label: string} + | { + __typename: 'TableSchemaMetadataEntry'; + label: string; + schema: { + __typename: 'TableSchema'; + columns: Array<{ + __typename: 'TableColumn'; + name: string; + type: string; + description: string | null; + }>; + }; + } + | {__typename: 'TextMetadataEntry'; label: string} + | {__typename: 'TimestampMetadataEntry'; label: string} + | {__typename: 'UrlMetadataEntry'; label: string} + >; + }>; + }>; +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx new file mode 100644 index 0000000000000..5140b7a95af8d --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/lineage/useColumnLineageDataForAssets.tsx @@ -0,0 +1,158 @@ +import {gql, useApolloClient} from '@apollo/client'; +import React, {useMemo, useRef, useState} from 'react'; + +import { + AssetColumnLineageQuery, + AssetColumnLineageQueryVariables, +} from './types/useColumnLineageDataForAssets.types'; +import {toGraphId} from '../../asset-graph/Utils'; +import {AssetKeyInput} from '../../graphql/types'; +import {isCanonicalColumnLineageEntry} from '../../metadata/TableSchema'; +import {buildConsolidatedColumnSchema} from '../buildConsolidatedColumnSchema'; + +export type AssetColumnLineageServer = { + [column: string]: { + // Note: This is [["key_part_1", "key_part_2"]] but the outer array + // only contains one item, it's a serialization odditiy. + upstream_asset_key: string[][]; + upstream_column_name: string; + }[]; +}; + +export type AssetColumnLineageLocalColumn = { + name: string; + type: string | null; + description: string | null; + asOf: string | undefined; // materialization timestamp + upstream: { + assetKey: AssetKeyInput; + columnName: string; + }[]; +}; + +export type AssetColumnLineageLocal = { + [column: string]: AssetColumnLineageLocalColumn; +}; + +export type AssetColumnLineages = {[graphId: string]: AssetColumnLineageLocal | undefined}; + +/** + * The column definitions and the column lineage are in two separate metadata entries, + * and the definitions may be specified in definition-time or materialization-time metadata. + * Parse them both and combine the results into a single representation of asset columns + * that is easier for the rest of the front-end to use. + */ +const getColumnLineage = ( + asset: AssetColumnLineageQuery['assetNodes'][0], +): AssetColumnLineageLocal | undefined => { + const materialization = asset.assetMaterializations[0]; + const lineageMetadata = materialization?.metadataEntries.find(isCanonicalColumnLineageEntry); + if (!lineageMetadata) { + return undefined; + } + + const {tableSchema} = buildConsolidatedColumnSchema({ + materialization, + definition: asset, + definitionLoadTimestamp: undefined, + }); + + const lineageParsed: AssetColumnLineageServer = JSON.parse(lineageMetadata.jsonString); + const schemaParsed = tableSchema?.schema + ? Object.fromEntries(tableSchema.schema.columns.map((col) => [col.name, col])) + : {}; + + return Object.fromEntries( + Object.entries(lineageParsed).map(([column, m]) => [ + column, + { + name: column, + asOf: materialization?.timestamp, + type: schemaParsed[column]?.type || null, + description: schemaParsed[column]?.description || null, + upstream: m.map((u) => ({ + assetKey: {path: u.upstream_asset_key[0]!}, + columnName: u.upstream_column_name, + })), + }, + ]), + ); +}; + +export function useColumnLineageDataForAssets(assetKeys: AssetKeyInput[]) { + const [loaded, setLoaded] = useState({}); + const client = useApolloClient(); + const fetching = useRef(false); + const missing = useMemo( + () => assetKeys.filter((a) => !loaded[toGraphId(a)]), + [assetKeys, loaded], + ); + + React.useEffect(() => { + const fetch = async () => { + fetching.current = true; + const {data} = await client.query({ + query: ASSET_COLUMN_LINEAGE_QUERY, + variables: {assetKeys: missing}, + }); + fetching.current = false; + + setLoaded((loaded) => ({ + ...loaded, + ...Object.fromEntries( + data.assetNodes.map((n) => [toGraphId(n.assetKey), getColumnLineage(n)]), + ), + })); + }; + if (!fetching.current && missing.length) { + void fetch(); + } + }, [client, missing]); + + return loaded; +} + +const ASSET_COLUMN_LINEAGE_QUERY = gql` + query AssetColumnLineage($assetKeys: [AssetKeyInput!]!) { + assetNodes(loadMaterializations: true, assetKeys: $assetKeys) { + id + assetKey { + path + } + metadataEntries { + __typename + label + ... on TableSchemaMetadataEntry { + label + schema { + columns { + name + type + description + } + } + } + } + assetMaterializations(limit: 1) { + timestamp + metadataEntries { + __typename + label + ... on TableSchemaMetadataEntry { + label + schema { + columns { + name + type + description + } + } + } + ... on JsonMetadataEntry { + jsonString + } + } + } + } + } +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/types.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/types.tsx index b61cf71a59ed6..b87c0bfaabdf1 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/types.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/types.tsx @@ -21,4 +21,5 @@ export interface AssetViewParams { evaluation?: string; checkDetail?: string; default_range?: string; + column?: string; } diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx new file mode 100644 index 0000000000000..376b60e6a069a --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/useColumnLineageLayout.tsx @@ -0,0 +1,142 @@ +import {useMemo} from 'react'; + +import {AssetColumnLineages} from './lineage/useColumnLineageDataForAssets'; +import {GraphData, groupIdForNode, toGraphId, tokenForAssetKey} from '../asset-graph/Utils'; +import {LayoutAssetGraphOptions} from '../asset-graph/layout'; +import {useAssetLayout} from '../graph/asyncGraphLayout'; + +const LINEAGE_GRAPH_COLUMN_LAYOUT_OPTIONS: LayoutAssetGraphOptions = { + direction: 'horizontal', + overrides: { + nodeHeight: 32, + nodesep: 0, + edgesep: 0, + clusterpaddingtop: 50, + groupPaddingBottom: -3, + groupPaddingTop: 68, + groupRendering: 'always', + }, +}; + +type Item = {assetGraphId: string; column: string; direction?: 'upstream' | 'downstream'}; + +export function toColumnGraphId(item: {assetGraphId: string; column: string}) { + return JSON.stringify({assetGraphId: item.assetGraphId, column: item.column}); +} +export function fromColumnGraphId(id: string) { + return JSON.parse(id) as {assetGraphId: string; column: string}; +} + +/** + * This function returns GraphData in which each `node` is a column of an asset and each `group` + * is an asset, essentially "zooming in" to the asset column level. This is a bit awkward but + * allows us to reuse the asset layout engine (and all it's caching, async dispatch, etc) for + * this view. + */ +export function useColumnLineageLayout( + assetGraphData: GraphData, + assetGraphId: string, + column: string, + columnLineageData: AssetColumnLineages, +) { + const {columnGraphData, groups} = useMemo(() => { + const columnGraphData: GraphData = { + nodes: {}, + downstream: {}, + upstream: {}, + }; + + const downstreams = buildReverseEdgeLookupTable(columnLineageData); + + const addEdge = (upstreamId: string, downstreamId: string) => { + columnGraphData.upstream[downstreamId] = columnGraphData.upstream[downstreamId] || {}; + columnGraphData.upstream[downstreamId]![upstreamId] = true; + columnGraphData.downstream[upstreamId] = columnGraphData.downstream[upstreamId] || {}; + columnGraphData.downstream[upstreamId]![downstreamId] = true; + }; + + const groups = new Set(); + const queue: Item[] = [{assetGraphId, column}]; + let item: Item | undefined; + + while ((item = queue.pop())) { + if (!item) { + continue; + } + const id = toColumnGraphId(item); + const {assetGraphId, column, direction} = item; + const assetNode = assetGraphData.nodes[assetGraphId]; + if (columnGraphData.nodes[id] || !assetNode) { + continue; // visited already + } + + const columnGraphNode = { + id, + assetKey: assetNode.assetKey, + definition: { + ...assetNode.definition, + groupName: `${tokenForAssetKey(assetNode.assetKey)}`, + }, + }; + columnGraphData.nodes[id] = columnGraphNode; + groups.add(groupIdForNode(columnGraphNode)); + + if (!direction || direction === 'upstream') { + const lineageForColumn = columnLineageData[assetGraphId]?.[column]; + for (const upstream of lineageForColumn?.upstream || []) { + const upstreamGraphId = toGraphId(upstream.assetKey); + const upstreamItem: Item = { + assetGraphId: upstreamGraphId, + column: upstream.columnName, + direction: 'upstream', + }; + if (assetGraphData.nodes[upstreamItem.assetGraphId]) { + queue.push(upstreamItem); + addEdge(toColumnGraphId(upstreamItem), id); + } + } + } + if (!direction || direction === 'downstream') { + for (const downstreamId of Object.keys(downstreams[id] || {})) { + const downstreamItem: Item = { + ...fromColumnGraphId(downstreamId), + direction: 'downstream', + }; + if (assetGraphData.nodes[downstreamItem.assetGraphId]) { + queue.push(downstreamItem); + addEdge(id, downstreamId); + } + } + } + } + + return {columnGraphData, groups: Array.from(groups)}; + }, [assetGraphData, column, columnLineageData, assetGraphId]); + + return useAssetLayout(columnGraphData, groups, LINEAGE_GRAPH_COLUMN_LAYOUT_OPTIONS); +} + +/** + * The column lineage data we get from asset metadata only gives us upstreams for each column. + * To efficiently build graph data we need both upstreams and downstreams for each column. + * This function visits every node and builds a downstreams lookup table. + */ +function buildReverseEdgeLookupTable(columnLineageData: AssetColumnLineages) { + const downstreams: {[id: string]: {[id: string]: true}} = {}; + + Object.entries(columnLineageData).forEach(([downstreamAssetGraphId, e]) => { + Object.entries(e || {}).forEach(([downstreamColumnName, {upstream}]) => { + const downstreamKey = toColumnGraphId({ + assetGraphId: downstreamAssetGraphId, + column: downstreamColumnName, + }); + for (const {assetKey, columnName} of upstream) { + const key = toColumnGraphId({assetGraphId: toGraphId(assetKey), column: columnName}); + downstreams[key] = downstreams[key] || {}; + downstreams[key]![downstreamKey] = true; + } + }); + }); + + return downstreams; +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts b/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts index d7ec4bf980477..fa47bbe03099b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/graph/asyncGraphLayout.ts @@ -5,12 +5,7 @@ import {ILayoutOp, LayoutOpGraphOptions, OpGraphLayout, layoutOpGraph} from './l import {useFeatureFlags} from '../app/Flags'; import {asyncMemoize, indexedDBAsyncMemoize} from '../app/Util'; import {GraphData} from '../asset-graph/Utils'; -import { - AssetGraphLayout, - AssetLayoutDirection, - LayoutAssetGraphOptions, - layoutAssetGraph, -} from '../asset-graph/layout'; +import {AssetGraphLayout, LayoutAssetGraphOptions, layoutAssetGraph} from '../asset-graph/layout'; const ASYNC_LAYOUT_SOLID_COUNT = 50; @@ -186,14 +181,13 @@ export function useOpLayout(ops: ILayoutOp[], parentOp?: ILayoutOp) { export function useAssetLayout( _graphData: GraphData, expandedGroups: string[], - direction: AssetLayoutDirection, + opts: LayoutAssetGraphOptions, ) { const [state, dispatch] = useReducer(reducer, initialState); const flags = useFeatureFlags(); const graphData = useMemo(() => ({..._graphData, expandedGroups}), [expandedGroups, _graphData]); - const opts = useMemo(() => ({direction}), [direction]); const cacheKey = _assetLayoutCacheKey(graphData, opts); const nodeCount = Object.keys(graphData.nodes).length; const runAsync = nodeCount >= ASYNC_LAYOUT_SOLID_COUNT; diff --git a/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx b/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx index 4f44ea1b94c79..a50b1a299139a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/metadata/MetadataEntry.tsx @@ -223,6 +223,7 @@ export const MetadataEntry = ({ ) : ( JSON.stringify(entry.schema, null, 2)} content={() => ( React.ReactNode; copyContent: () => string; }) => { @@ -377,7 +379,7 @@ const MetadataEntryModalAction = (props: { setOpen(true)}>{props.children} setOpen(false)} isOpen={open} diff --git a/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx b/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx index 7982db4f315e0..839e83cd86460 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/metadata/TableSchema.tsx @@ -11,12 +11,20 @@ import { Tooltip, } from '@dagster-io/ui-components'; import {Spacing} from '@dagster-io/ui-components/src/components/types'; -import {useState} from 'react'; +import {createContext, useContext, useState} from 'react'; import {TableSchemaFragment} from './types/TableSchema.types'; import {Timestamp} from '../app/time/Timestamp'; import {StyledTableWithHeader} from '../assets/AssetEventMetadataEntriesTable'; -import {MaterializationEvent, TableSchemaMetadataEntry} from '../graphql/types'; +import {assetDetailsPathForKey} from '../assets/assetDetailsPathForKey'; +import { + AssetKeyInput, + JsonMetadataEntry, + MaterializationEvent, + TableSchemaMetadataEntry, +} from '../graphql/types'; +import {Description} from '../pipelines/Description'; +import {AnchorButton} from '../ui/AnchorButton'; type ITableSchema = TableSchemaFragment; @@ -28,16 +36,26 @@ interface ITableSchemaProps { itemHorizontalPadding?: Spacing; } -export const isCanonicalTableSchemaEntry = ( +export const isCanonicalColumnSchemaEntry = ( m: Pick, ): m is TableSchemaMetadataEntry => m.__typename === 'TableSchemaMetadataEntry' && m.label === 'dagster/column_schema'; +export const isCanonicalColumnLineageEntry = ( + m: Pick, +): m is JsonMetadataEntry => m.__typename === 'JsonMetadataEntry' && m.label === 'lineage'; + +export const TableSchemaLineageContext = createContext<{assetKey: AssetKeyInput | null}>({ + assetKey: null, +}); + export const TableSchema = ({ schema, schemaLoadTimestamp, itemHorizontalPadding, }: ITableSchemaProps) => { + const {assetKey} = useContext(TableSchemaLineageContext); + const multiColumnConstraints = schema.constraints?.other || []; const [filter, setFilter] = useState(''); const rows = schema.columns.filter( @@ -76,6 +94,7 @@ export const TableSchema = ({ Column name Type Description + {assetKey ? : undefined} @@ -85,14 +104,29 @@ export const TableSchema = ({ {column.name} - + {!column.constraints.nullable && NonNullableTag} {column.constraints.unique && UniqueTag} {column.constraints.other.map((constraint, i) => ( ))} - {column.description} + + + + {assetKey ? ( + + + } + to={assetDetailsPathForKey(assetKey, { + view: 'lineage', + column: column.name, + })} + /> + + + ) : undefined} ))} {rows.length === 0 && ( @@ -108,7 +142,7 @@ export const TableSchema = ({ ); }; -const iconForType = (type: string): IconName | null => { +export const iconForColumnType = (type: string): IconName | null => { const lower = type.toLowerCase(); if (lower.includes('bool')) { return 'datatype_bool'; @@ -128,12 +162,14 @@ const iconForType = (type: string): IconName | null => { return null; }; -const TypeTag = ({type = '', icon}: {type: string; icon: IconName | null}) => { +export const TypeTag = ({type = ''}: {type: string}) => { if (type.trim().replace(/\?/g, '').length === 0) { // Do not render type '' or '?' or any other empty value. return ; } + const icon = iconForColumnType(type); + return ( diff --git a/js_modules/dagster-ui/patches/dagre+0.8.5.patch b/js_modules/dagster-ui/patches/dagre+0.8.5.patch deleted file mode 100644 index fab38165260b3..0000000000000 --- a/js_modules/dagster-ui/patches/dagre+0.8.5.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/node_modules/dagre/lib/order/index.js b/node_modules/dagre/lib/order/index.js -index 4ac2d9f..a2182fe 100644 ---- a/node_modules/dagre/lib/order/index.js -+++ b/node_modules/dagre/lib/order/index.js -@@ -73,7 +73,9 @@ function sweepLayerGraphs(layerGraphs, biasRight) { - function assignOrder(g, layering) { - _.forEach(layering, function(layer) { - _.forEach(layer, function(v, i) { -- g.node(v).order = i; -+ try { -+ g.node(v).order = i; -+ } catch (e) {} - }); - }); - } diff --git a/js_modules/dagster-ui/yarn.lock b/js_modules/dagster-ui/yarn.lock index f44fbb20646dc..fb58b04e57528 100644 --- a/js_modules/dagster-ui/yarn.lock +++ b/js_modules/dagster-ui/yarn.lock @@ -2598,7 +2598,7 @@ __metadata: codemirror: "npm:^5.65.2" color: "npm:^3.0.0" cronstrue: "npm:^1.84.0" - dagre: "npm:^0.8.5" + dagre: "dagster-io/dagre#0.8.5" date-fns: "npm:^2.28.0" dayjs: "npm:^1.11.7" deepmerge: "npm:^4.2.2" @@ -11528,13 +11528,13 @@ __metadata: languageName: node linkType: hard -"dagre@npm:^0.8.5": +"dagre@dagster-io/dagre#0.8.5": version: 0.8.5 - resolution: "dagre@npm:0.8.5" + resolution: "dagre@https://github.com/dagster-io/dagre.git#commit=c2a1821cc7f8a220e819461b82b6ddbf48189100" dependencies: graphlib: "npm:^2.1.8" lodash: "npm:^4.17.15" - checksum: f39899e29e9090581d67177ef6e2dd3ca5d7f764fbf3de81758d879bba66fee6fd8802d41d0c5d3d9a0563b334e99e1454a8d6ab4ce17e8e4f50836a3a403fdd + checksum: 6a94d8d9b1c3132b406b5921fd2bbd1a207c78fc1048216787fec68fad1d96f649cb084d1ae576f5456532a4275f22a113d140265f05937db60e5918a25adac5 languageName: node linkType: hard