diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AssetDaemonTicksQuery.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AssetDaemonTicksQuery.tsx new file mode 100644 index 0000000000000..f50b71b34cad8 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AssetDaemonTicksQuery.tsx @@ -0,0 +1,47 @@ +import {gql} from '@apollo/client'; + +import {PYTHON_ERROR_FRAGMENT} from '../../app/PythonErrorFragment'; + +export const ASSET_DAMEON_TICKS_QUERY = gql` + query AssetDaemonTicksQuery( + $dayRange: Int + $dayOffset: Int + $statuses: [InstigationTickStatus!] + $limit: Int + $cursor: String + ) { + autoMaterializeTicks( + dayRange: $dayRange + dayOffset: $dayOffset + statuses: $statuses + limit: $limit + cursor: $cursor + ) { + id + ...AssetDaemonTickFragment + } + } + + fragment AssetDaemonTickFragment on InstigationTick { + id + timestamp + endTimestamp + status + instigationType + error { + ...PythonErrorFragment + } + requestedAssetKeys { + path + } + requestedAssetMaterializationCount + autoMaterializeAssetEvaluationId + requestedMaterializationsForAssets { + assetKey { + path + } + partitionKeys + } + } + ${PYTHON_ERROR_FRAGMENT} +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx index b50c6a4b4795c..75a0299460f6c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx @@ -1,42 +1,275 @@ -import {Body2, Colors, Table} from '@dagster-io/ui-components'; +import { + BaseTag, + Body2, + Box, + Button, + ButtonGroup, + ButtonLink, + Checkbox, + Colors, + CursorHistoryControls, + Dialog, + DialogBody, + DialogFooter, + Spinner, + Table, + Tag, +} from '@dagster-io/ui-components'; import React from 'react'; +import {PythonErrorInfo} from '../../app/PythonErrorInfo'; +import {useQueryRefreshAtInterval} from '../../app/QueryRefresh'; import {Timestamp} from '../../app/time/Timestamp'; +import {InstigationTickStatus} from '../../graphql/types'; +import {useQueryPersistedState} from '../../hooks/useQueryPersistedState'; import {TimeElapsed} from '../../runs/TimeElapsed'; -import {AnchorButton} from '../../ui/AnchorButton'; +import {useCursorPaginatedQuery} from '../../runs/useCursorPaginatedQuery'; + +import {ASSET_DAMEON_TICKS_QUERY} from './AssetDaemonTicksQuery'; +import { + AssetDaemonTicksQuery, + AssetDaemonTicksQueryVariables, + AssetDaemonTickFragment, +} from './types/AssetDaemonTicksQuery.types'; + +const PAGE_SIZE = 15; + +export const AutomaterializationEvaluationHistoryTable = ({ + setSelectedTick, + setTableView, +}: { + setSelectedTick: (tick: AssetDaemonTickFragment | null) => void; + setTableView: (view: 'evaluations' | 'runs') => void; +}) => { + const [statuses, setStatuses] = useQueryPersistedState>({ + queryKey: 'statuses', + decode: React.useCallback(({statuses}: {statuses?: string}) => { + return new Set( + statuses + ? JSON.parse(statuses) + : [ + InstigationTickStatus.STARTED, + InstigationTickStatus.SUCCESS, + InstigationTickStatus.FAILURE, + InstigationTickStatus.SKIPPED, + ], + ); + }, []), + encode: React.useCallback((raw: Set) => { + return {statuses: JSON.stringify(Array.from(raw))}; + }, []), + }); + + const {queryResult, paginationProps} = useCursorPaginatedQuery< + AssetDaemonTicksQuery, + AssetDaemonTicksQueryVariables + >({ + query: ASSET_DAMEON_TICKS_QUERY, + variables: { + statuses: React.useMemo(() => Array.from(statuses), [statuses]), + }, + nextCursorForResult: (data) => { + const ticks = data.autoMaterializeTicks; + if (!ticks.length) { + return undefined; + } + return ticks[PAGE_SIZE - 1]?.id; + }, + getResultArray: (data) => { + if (!data?.autoMaterializeTicks) { + return []; + } + return data.autoMaterializeTicks; + }, + pageSize: PAGE_SIZE, + }); + // Only refresh if we're on the first page + useQueryRefreshAtInterval(queryResult, !paginationProps.hasPrevCursor ? 10000 : 60 * 60 * 1000); + + return ( + + + + { + setTableView(id); + }} + /> + {!queryResult.data ? : null} + + + + + + + + + + + + + + + + + + + {/* Use previous data to stop page from jumping while new data loads */} + {(queryResult.data || queryResult.previousData)?.autoMaterializeTicks.map((tick) => ( + + + + + + + ))} + +
TimestampStatusDurationResult
+ + + + + + + {[InstigationTickStatus.SKIPPED, InstigationTickStatus.SUCCESS].includes( + tick.status, + ) ? ( + { + setSelectedTick(tick); + }} + > + + {tick.requestedAssetMaterializationCount} materializations requested + + + ) : ( + ' - ' + )} +
+
+ +
+ + ); +}; + +const StatusTag = ({tick}: {tick: AssetDaemonTickFragment}) => { + const {status, error, requestedAssetMaterializationCount} = tick; + const count = requestedAssetMaterializationCount; + const [showErrors, setShowErrors] = React.useState(false); + const tag = React.useMemo(() => { + switch (status) { + case InstigationTickStatus.STARTED: + return ( + + Evaluating + + ); + case InstigationTickStatus.SKIPPED: + return ; + case InstigationTickStatus.FAILURE: + return ( + + Failure + {error ? ( + { + setShowErrors(true); + }} + > + View + + ) : null} + + ); + case InstigationTickStatus.SUCCESS: + return {count} requested; + } + }, [error, count, status]); -export const AutomaterializationEvaluationHistoryTable = () => { - // TODO return ( - - - - - - - - - - - - - - - - - - - -
TimestampStatusDurationResult
- - -
-
- - - No runs launched - - View details -
+ <> + {tag} + {error ? ( + + + + + + + + + ) : null} + ); }; + +const StatusLabels = { + [InstigationTickStatus.SKIPPED]: 'None requested', + [InstigationTickStatus.STARTED]: 'Started', + [InstigationTickStatus.FAILURE]: 'Failed', + [InstigationTickStatus.SUCCESS]: 'Requested', +}; + +function StatusCheckbox({ + status, + statuses, + setStatuses, +}: { + status: InstigationTickStatus; + statuses: Set; + setStatuses: (statuses: Set) => void; +}) { + return ( + { + const newStatuses = new Set(statuses); + if (statuses.has(status)) { + newStatuses.delete(status); + } else { + newStatuses.add(status); + } + setStatuses(newStatuses); + }} + /> + ); +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationRoot.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationRoot.tsx index f0d0921f40ef6..8ca16372973fc 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationRoot.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationRoot.tsx @@ -1,4 +1,4 @@ -import {gql, useQuery} from '@apollo/client'; +import {useQuery} from '@apollo/client'; import { Alert, Box, @@ -10,27 +10,27 @@ import { Heading, PageHeader, Table, - ButtonGroup, } from '@dagster-io/ui-components'; import React from 'react'; import {useConfirmation} from '../../app/CustomConfirmationProvider'; import {useUnscopedPermissions} from '../../app/Permissions'; -import {PYTHON_ERROR_FRAGMENT} from '../../app/PythonErrorFragment'; import {useQueryRefreshAtInterval} from '../../app/QueryRefresh'; import {useTrackPageView} from '../../app/analytics'; +import {useQueryPersistedState} from '../../hooks/useQueryPersistedState'; import {LiveTickTimeline} from '../../instigation/LiveTickTimeline2'; import {OverviewTabs} from '../../overview/OverviewTabs'; import {useAutomaterializeDaemonStatus} from '../AutomaterializeDaemonStatusTag'; +import {ASSET_DAMEON_TICKS_QUERY} from './AssetDaemonTicksQuery'; import {AutomaterializationEvaluationHistoryTable} from './AutomaterializationEvaluationHistoryTable'; import {AutomaterializationTickDetailDialog} from './AutomaterializationTickDetailDialog'; import {AutomaterializeRunHistoryTable} from './AutomaterializeRunHistoryTable'; import { - AssetDameonTicksQuery, - AssetDameonTicksQueryVariables, + AssetDaemonTicksQuery, + AssetDaemonTicksQueryVariables, AssetDaemonTickFragment, -} from './types/AutomaterializationRoot.types'; +} from './types/AssetDaemonTicksQuery.types'; const MINUTE = 60 * 1000; const THREE_MINUTES = 3 * MINUTE; @@ -44,7 +44,7 @@ export const AutomaterializationRoot = () => { const {permissions: {canToggleAutoMaterialize} = {}} = useUnscopedPermissions(); - const queryResult = useQuery( + const queryResult = useQuery( ASSET_DAMEON_TICKS_QUERY, ); const [isPaused, setIsPaused] = React.useState(false); @@ -52,7 +52,18 @@ export const AutomaterializationRoot = () => { const [selectedTick, setSelectedTick] = React.useState(null); - const [tableView, setTableView] = React.useState<'evaluations' | 'runs'>('evaluations'); + const [tableView, setTableView] = useQueryPersistedState<'evaluations' | 'runs'>( + React.useMemo( + () => ({ + queryKey: 'view', + decode: ({view}) => (view === 'runs' ? 'runs' : 'evaluations'), + encode: (raw) => { + return {view: raw, cursor: undefined, statuses: undefined}; + }, + }), + [], + ), + ); const ids = queryResult.data ? queryResult.data.autoMaterializeTicks.map((tick) => `${tick.id}:${tick.status}`) @@ -150,63 +161,16 @@ export const AutomaterializationRoot = () => { setSelectedTick(null); }} /> - - { - setTableView(id); - }} - /> - {tableView === 'evaluations' ? ( - + ) : ( - + )} )} ); }; - -const ASSET_DAMEON_TICKS_QUERY = gql` - query AssetDameonTicksQuery( - $dayRange: Int - $dayOffset: Int - $statuses: [InstigationTickStatus!] - $limit: Int - $cursor: String - ) { - autoMaterializeTicks( - dayRange: $dayRange - dayOffset: $dayOffset - statuses: $statuses - limit: $limit - cursor: $cursor - ) { - id - ...AssetDaemonTickFragment - } - } - - fragment AssetDaemonTickFragment on InstigationTick { - id - timestamp - endTimestamp - status - instigationType - error { - ...PythonErrorFragment - } - requestedAssetMaterializationCount - requestedAssetKeys { - path - } - autoMaterializeAssetEvaluationId - } - ${PYTHON_ERROR_FRAGMENT} -`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationTickDetailDialog.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationTickDetailDialog.tsx index 8e47d29eb31cd..78200382b5361 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationTickDetailDialog.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationTickDetailDialog.tsx @@ -1,3 +1,4 @@ +import {gql, useQuery} from '@apollo/client'; import { Box, Colors, @@ -9,25 +10,36 @@ import { DialogBody, DialogFooter, ButtonLink, + Icon, + Spinner, } from '@dagster-io/ui-components'; import {useVirtualizer} from '@tanstack/react-virtual'; import React from 'react'; +import {Link} from 'react-router-dom'; import styled from 'styled-components'; import {PythonErrorInfo} from '../../app/PythonErrorInfo'; import {formatElapsedTime} from '../../app/Util'; import {Timestamp} from '../../app/time/Timestamp'; import {PythonErrorFragment} from '../../app/types/PythonErrorFragment.types'; +import {tokenForAssetKey} from '../../asset-graph/Utils'; import {AssetKeyInput, InstigationTickStatus} from '../../graphql/types'; import {HeaderCell, Inner, Row, RowCell} from '../../ui/VirtualizedTable'; +import {buildRepoAddress} from '../../workspace/buildRepoAddress'; +import {workspacePathFromAddress} from '../../workspace/workspacePath'; import {AssetLink} from '../AssetLink'; import { AssetKeysDialog, AssetKeysDialogHeader, AssetKeysDialogEmptyState, } from '../AutoMaterializePolicyPage/AssetKeysDialog'; +import {assetDetailsPathForKey} from '../assetDetailsPathForKey'; -import {AssetDaemonTickFragment} from './types/AutomaterializationRoot.types'; +import {AssetDaemonTickFragment} from './types/AssetDaemonTicksQuery.types'; +import { + AssetGroupAndLocationQuery, + AssetGroupAndLocationQueryVariables, +} from './types/AutomaterializationTickDetailDialog.types'; const TEMPLATE_COLUMNS = '30% 17% 53%'; export const AutomaterializationTickDetailDialog = React.memo( @@ -64,6 +76,14 @@ export const AutomaterializationTickDetailDialog = React.memo( const totalHeight = rowVirtualizer.getTotalSize(); const items = rowVirtualizer.getVirtualItems(); + const assetKeyToPartitionsMap = React.useMemo(() => { + const map: Record = {}; + tick?.requestedMaterializationsForAssets.forEach(({assetKey, partitionKeys}) => { + map[tokenForAssetKey(assetKey)] = partitionKeys; + }); + return map; + }, [tick?.requestedMaterializationsForAssets]); + const content = React.useMemo(() => { if (queryString && !filteredAssetKeys.length) { return ( @@ -107,12 +127,21 @@ export const AutomaterializationTickDetailDialog = React.memo( {items.map(({index, key, size, start}) => { const assetKey = filteredAssetKeys[index]!; - return ; + return ( + + ); })} ); - }, [filteredAssetKeys, items, queryString, tick, totalHeight]); + }, [assetKeyToPartitionsMap, filteredAssetKeys, items, queryString, tick, totalHeight]); const intent = React.useMemo(() => { switch (tick?.status) { @@ -190,7 +219,11 @@ export const AutomaterializationTickDetailDialog = React.memo( Status - {tick?.requestedAssetMaterializationCount || 0} requested + {tick?.status === InstigationTickStatus.STARTED ? ( + 'Evaluating…' + ) : ( + <>{tick?.requestedAssetMaterializationCount ?? 0} requested + )} {tick?.error ? ( - 0 ? undefined : 'bottom'} - > - Materializations requested - - {content} + {tick?.status === InstigationTickStatus.STARTED ? null : ( + <> + 0 ? undefined : 'bottom'} + > + Materializations requested + + {content} + + )} } /> @@ -235,20 +272,64 @@ const AssetDetailRow = ({ $start, $height, assetKey, + partitionKeys, + evaluationId, }: { $start: number; $height: number; assetKey: AssetKeyInput; + partitionKeys?: string[]; + evaluationId: number; }) => { - // TODO (after daniel adds new fields) + const numMaterializations = partitionKeys?.length || 1; + const {data} = useQuery( + ASSET_GROUP_QUERY, + { + fetchPolicy: 'cache-and-network', + variables: { + assetKey: {path: assetKey.path}, + }, + }, + ); + const asset = data?.assetOrError.__typename === 'Asset' ? data.assetOrError : null; + const definition = asset?.definition; + const repoAddress = definition + ? buildRepoAddress(definition.repository.name, definition.repository.location.name) + : null; return ( - - + + {data ? ( + definition && definition.groupName && repoAddress ? ( + + + + {definition.groupName} + + + ) : ( + Asset not found + ) + ) : ( + + )} + + + + {numMaterializations} materialization{numMaterializations === 1 ? '' : 's'} requested + + ); @@ -262,3 +343,25 @@ const RowGrid = styled(Box)` padding-top: 26px 0px; } `; + +const ASSET_GROUP_QUERY = gql` + query AssetGroupAndLocationQuery($assetKey: AssetKeyInput!) { + assetOrError(assetKey: $assetKey) { + ... on Asset { + id + definition { + id + groupName + repository { + id + name + location { + id + name + } + } + } + } + } + } +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializeRunHistoryTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializeRunHistoryTable.tsx index d82b3eba414b6..60adb989dc57c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializeRunHistoryTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializeRunHistoryTable.tsx @@ -1,6 +1,79 @@ +import {ButtonGroup, Box, CursorHistoryControls} from '@dagster-io/ui-components'; import React from 'react'; +import styled from 'styled-components'; -export const AutomaterializeRunHistoryTable = () => { - // TODO - return
; +import {useQueryRefreshAtInterval} from '../../app/QueryRefresh'; +import {RunTable} from '../../runs/RunTable'; +import {RUNS_ROOT_QUERY} from '../../runs/RunsRoot'; +import {RunsRootQuery, RunsRootQueryVariables} from '../../runs/types/RunsRoot.types'; +import {useCursorPaginatedQuery} from '../../runs/useCursorPaginatedQuery'; + +const PAGE_SIZE = 15; + +export const AutomaterializeRunHistoryTable = ({ + setTableView, +}: { + setTableView: (view: 'evaluations' | 'runs') => void; +}) => { + const {queryResult, paginationProps} = useCursorPaginatedQuery< + RunsRootQuery, + RunsRootQueryVariables + >({ + nextCursorForResult: (runs) => { + if (runs.pipelineRunsOrError.__typename !== 'Runs') { + return undefined; + } + return runs.pipelineRunsOrError.results[PAGE_SIZE - 1]?.id; + }, + getResultArray: (data) => { + if (!data || data.pipelineRunsOrError.__typename !== 'Runs') { + return []; + } + return data.pipelineRunsOrError.results; + }, + variables: { + filter: { + tags: [{key: 'dagster/auto_materialize', value: 'true'}], + }, + }, + query: RUNS_ROOT_QUERY, + pageSize: PAGE_SIZE, + }); + + useQueryRefreshAtInterval(queryResult, 15 * 1000); + + const runData = (queryResult.data || queryResult.previousData)?.pipelineRunsOrError; + + return ( + + + + { + setTableView(id); + }} + /> + + + +
+ +
+ + ); }; + +// Super hacky but easiest solution to position the action button +const Wrapper = styled.div` + position: relative; + > *:nth-child(2) { + position: absolute; + right: 0; + top: 0; + } +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AutomaterializationRoot.types.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AssetDaemonTicksQuery.types.ts similarity index 78% rename from js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AutomaterializationRoot.types.ts rename to js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AssetDaemonTicksQuery.types.ts index c51dda1e34e16..d081cd44a39fc 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AutomaterializationRoot.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AssetDaemonTicksQuery.types.ts @@ -2,7 +2,7 @@ import * as Types from '../../../graphql/types'; -export type AssetDameonTicksQueryVariables = Types.Exact<{ +export type AssetDaemonTicksQueryVariables = Types.Exact<{ dayRange?: Types.InputMaybe; dayOffset?: Types.InputMaybe; statuses?: Types.InputMaybe | Types.InstigationTickStatus>; @@ -10,7 +10,7 @@ export type AssetDameonTicksQueryVariables = Types.Exact<{ cursor?: Types.InputMaybe; }>; -export type AssetDameonTicksQuery = { +export type AssetDaemonTicksQuery = { __typename: 'Query'; autoMaterializeTicks: Array<{ __typename: 'InstigationTick'; @@ -32,6 +32,11 @@ export type AssetDameonTicksQuery = { }>; } | null; requestedAssetKeys: Array<{__typename: 'AssetKey'; path: Array}>; + requestedMaterializationsForAssets: Array<{ + __typename: 'RequestedMaterializationsForAsset'; + partitionKeys: Array; + assetKey: {__typename: 'AssetKey'; path: Array}; + }>; }>; }; @@ -55,4 +60,9 @@ export type AssetDaemonTickFragment = { }>; } | null; requestedAssetKeys: Array<{__typename: 'AssetKey'; path: Array}>; + requestedMaterializationsForAssets: Array<{ + __typename: 'RequestedMaterializationsForAsset'; + partitionKeys: Array; + assetKey: {__typename: 'AssetKey'; path: Array}; + }>; }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AutomaterializationTickDetailDialog.types.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AutomaterializationTickDetailDialog.types.ts new file mode 100644 index 0000000000000..49b4098c6c41a --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AutomaterializationTickDetailDialog.types.ts @@ -0,0 +1,28 @@ +// Generated GraphQL types, do not edit manually. + +import * as Types from '../../../graphql/types'; + +export type AssetGroupAndLocationQueryVariables = Types.Exact<{ + assetKey: Types.AssetKeyInput; +}>; + +export type AssetGroupAndLocationQuery = { + __typename: 'Query'; + assetOrError: + | { + __typename: 'Asset'; + id: string; + definition: { + __typename: 'AssetNode'; + id: string; + groupName: string | null; + repository: { + __typename: 'Repository'; + id: string; + name: string; + location: {__typename: 'RepositoryLocation'; id: string; name: string}; + }; + } | null; + } + | {__typename: 'AssetNotFoundError'}; +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/instigation/LiveTickTimeline2.tsx b/js_modules/dagster-ui/packages/ui-core/src/instigation/LiveTickTimeline2.tsx index 27ea532498587..ee4273d31de5a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/instigation/LiveTickTimeline2.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/instigation/LiveTickTimeline2.tsx @@ -7,6 +7,7 @@ import styled from 'styled-components'; import {TimeContext} from '../app/time/TimeContext'; import {browserTimezone} from '../app/time/browserTimezone'; +import {AssetDaemonTickFragment} from '../assets/auto-materialization/types/AssetDaemonTicksQuery.types'; import {InstigationTickStatus, InstigationType} from '../graphql/types'; import {HistoryTickFragment} from './types/TickHistory.types'; @@ -51,7 +52,7 @@ type StrippedDownTickFragment = Pick< runs?: HistoryTickFragment['runs']; endTimestamp?: number | null; requestedAssetMaterializationCount?: number; -}; +} & Pick; export const LiveTickTimeline = ({ ticks,