From 19a3dac9765b051ce72299544d6b0a9c69c410b1 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Thu, 5 Oct 2023 15:33:34 -0400 Subject: [PATCH 01/10] amp timeline --- .../ui-core/src/app/CloudFeatureFlag.tsx | 11 ++++ .../ui-core/src/app/time/Timestamp.tsx | 9 +++- .../src/app/time/timestampToString.tsx | 5 +- ...omaterializationEvaluationHistoryTable.tsx | 51 ++++++++++++++++++- .../dagster_graphql/schema/instigation.py | 1 - 5 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 js_modules/dagster-ui/packages/ui-core/src/app/CloudFeatureFlag.tsx diff --git a/js_modules/dagster-ui/packages/ui-core/src/app/CloudFeatureFlag.tsx b/js_modules/dagster-ui/packages/ui-core/src/app/CloudFeatureFlag.tsx new file mode 100644 index 0000000000000..074cabbf7be17 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/app/CloudFeatureFlag.tsx @@ -0,0 +1,11 @@ +import React from 'react'; + +type CloudFeatureFlagContextType = { + enableAMPTimeline: boolean; +}; +const CloudFeatureFlagContext = React.createContext({ + enableAMPTimeline: false, +}); +export const useCloudFeatureFlag = () => { + return React.useContext(CloudFeatureFlagContext); +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/app/time/Timestamp.tsx b/js_modules/dagster-ui/packages/ui-core/src/app/time/Timestamp.tsx index 414cffe6e82c1..351909c1fc29e 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/app/time/Timestamp.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/app/time/Timestamp.tsx @@ -7,14 +7,19 @@ import {timestampToString} from './timestampToString'; interface Props { timestamp: {ms: number} | {unix: number}; timeFormat?: TimeFormat; + dateTimeSeparator?: string; } export const Timestamp: React.FC = (props) => { - const {timestamp, timeFormat} = props; + const {timestamp, timeFormat, dateTimeSeparator} = props; const { timezone: [timezone], hourCycle: [hourCycle], } = React.useContext(TimeContext); const locale = navigator.language; - return <>{timestampToString({timestamp, locale, timezone, timeFormat, hourCycle})}; + return ( + <> + {timestampToString({timestamp, locale, dateTimeSeparator, timezone, timeFormat, hourCycle})} + + ); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/app/time/timestampToString.tsx b/js_modules/dagster-ui/packages/ui-core/src/app/time/timestampToString.tsx index c1673ae35f9e3..c2cc13d378676 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/app/time/timestampToString.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/app/time/timestampToString.tsx @@ -6,6 +6,7 @@ type Config = { timestamp: {ms: number} | {unix: number}; locale: string; timezone: string; + dateTimeSeparator?: string; timeFormat?: TimeFormat; hourCycle?: HourCycle; }; @@ -17,6 +18,7 @@ export const timestampToString = (config: Config) => { timezone, timeFormat = DEFAULT_TIME_FORMAT, hourCycle = 'Automatic', + dateTimeSeparator = ', ', } = config; const msec = 'ms' in timestamp ? timestamp.ms : timestamp.unix * 1000; @@ -33,7 +35,7 @@ export const timestampToString = (config: Config) => { }); const sameYear = timestampYear === viewerYear; - return date.toLocaleDateString(locale, { + const stringDate = date.toLocaleDateString(locale, { month: 'short', day: 'numeric', year: sameYear ? undefined : 'numeric', @@ -44,4 +46,5 @@ export const timestampToString = (config: Config) => { timeZone: targetTimezone, timeZoneName: timeFormat.showTimezone ? 'short' : undefined, }); + return stringDate.replace(', ', dateTimeSeparator); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx index b50c6a4b4795c..67e3659aac63f 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx @@ -1,4 +1,4 @@ -import {Body2, Colors, Table} from '@dagster-io/ui-components'; +import {Body2, Box, ButtonLink, Colors, Dialog, Table, Tag} from '@dagster-io/ui-components'; import React from 'react'; import {Timestamp} from '../../app/time/Timestamp'; @@ -24,7 +24,7 @@ export const AutomaterializationEvaluationHistoryTable = () => { -
+ @@ -40,3 +40,50 @@ export const AutomaterializationEvaluationHistoryTable = () => { ); }; + +const StatusTag = ({ + status, + errors, +}: + | {status: 'Evaluating' | 'Skipped' | 'Complete'; errors?: null} + | { + status: 'Failure'; + errors: any; + }) => { + const [showErrors, setShowErrors] = React.useState(false); + const tag = React.useMemo(() => { + switch (status) { + case 'Evaluating': + return ( + + Evaluating + + ); + case 'Skipped': + return Skipped; + case 'Failure': + console.log({errors}); + return ( + + + { + setShowErrors(true); + }} + > + View errors + + + ); + case 'Complete': + return ; + } + }, []); + + return ( + <> + {tag} + errors + + ); +}; diff --git a/python_modules/dagster-graphql/dagster_graphql/schema/instigation.py b/python_modules/dagster-graphql/dagster_graphql/schema/instigation.py index 3dcde668801a4..bfa9ab91fc146 100644 --- a/python_modules/dagster-graphql/dagster_graphql/schema/instigation.py +++ b/python_modules/dagster-graphql/dagster_graphql/schema/instigation.py @@ -50,7 +50,6 @@ GrapheneInstigationType = graphene.Enum.from_enum(InstigatorType, "InstigationType") - class GrapheneInstigationStatus(graphene.Enum): RUNNING = "RUNNING" STOPPED = "STOPPED" From d5bedb2d3ed7dd56a0bf11c2fcdbffd998a0bed1 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Thu, 5 Oct 2023 15:46:22 -0400 Subject: [PATCH 02/10] export --- .../dagster-ui/packages/ui-core/src/app/CloudFeatureFlag.tsx | 4 ++-- .../AutomaterializationEvaluationHistoryTable.tsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/app/CloudFeatureFlag.tsx b/js_modules/dagster-ui/packages/ui-core/src/app/CloudFeatureFlag.tsx index 074cabbf7be17..323aba12915dd 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/app/CloudFeatureFlag.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/app/CloudFeatureFlag.tsx @@ -3,8 +3,8 @@ import React from 'react'; type CloudFeatureFlagContextType = { enableAMPTimeline: boolean; }; -const CloudFeatureFlagContext = React.createContext({ - enableAMPTimeline: false, +export const CloudFeatureFlagContext = React.createContext({ + enableAMPTimeline: true, }); export const useCloudFeatureFlag = () => { return React.useContext(CloudFeatureFlagContext); diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx index 67e3659aac63f..487e235334d1d 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx @@ -78,7 +78,7 @@ const StatusTag = ({ case 'Complete': return ; } - }, []); + }, [errors, status]); return ( <> From 62c701f25c1bfef27484a76a6742283483e59ecb Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Thu, 5 Oct 2023 16:05:38 -0400 Subject: [PATCH 03/10] ruff/pyright/black --- .../dagster-graphql/dagster_graphql/schema/instigation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_modules/dagster-graphql/dagster_graphql/schema/instigation.py b/python_modules/dagster-graphql/dagster_graphql/schema/instigation.py index bfa9ab91fc146..3dcde668801a4 100644 --- a/python_modules/dagster-graphql/dagster_graphql/schema/instigation.py +++ b/python_modules/dagster-graphql/dagster_graphql/schema/instigation.py @@ -50,6 +50,7 @@ GrapheneInstigationType = graphene.Enum.from_enum(InstigatorType, "InstigationType") + class GrapheneInstigationStatus(graphene.Enum): RUNNING = "RUNNING" STOPPED = "STOPPED" From 6ed185434103e96b478e4c76c8e5e78ca5e1790a Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Fri, 6 Oct 2023 10:53:21 -0400 Subject: [PATCH 04/10] feedback --- .../ui-core/src/app/time/Timestamp.tsx | 9 +--- .../src/app/time/timestampToString.tsx | 5 +- ...omaterializationEvaluationHistoryTable.tsx | 49 +------------------ 3 files changed, 4 insertions(+), 59 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/app/time/Timestamp.tsx b/js_modules/dagster-ui/packages/ui-core/src/app/time/Timestamp.tsx index 351909c1fc29e..414cffe6e82c1 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/app/time/Timestamp.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/app/time/Timestamp.tsx @@ -7,19 +7,14 @@ import {timestampToString} from './timestampToString'; interface Props { timestamp: {ms: number} | {unix: number}; timeFormat?: TimeFormat; - dateTimeSeparator?: string; } export const Timestamp: React.FC = (props) => { - const {timestamp, timeFormat, dateTimeSeparator} = props; + const {timestamp, timeFormat} = props; const { timezone: [timezone], hourCycle: [hourCycle], } = React.useContext(TimeContext); const locale = navigator.language; - return ( - <> - {timestampToString({timestamp, locale, dateTimeSeparator, timezone, timeFormat, hourCycle})} - - ); + return <>{timestampToString({timestamp, locale, timezone, timeFormat, hourCycle})}; }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/app/time/timestampToString.tsx b/js_modules/dagster-ui/packages/ui-core/src/app/time/timestampToString.tsx index c2cc13d378676..c1673ae35f9e3 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/app/time/timestampToString.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/app/time/timestampToString.tsx @@ -6,7 +6,6 @@ type Config = { timestamp: {ms: number} | {unix: number}; locale: string; timezone: string; - dateTimeSeparator?: string; timeFormat?: TimeFormat; hourCycle?: HourCycle; }; @@ -18,7 +17,6 @@ export const timestampToString = (config: Config) => { timezone, timeFormat = DEFAULT_TIME_FORMAT, hourCycle = 'Automatic', - dateTimeSeparator = ', ', } = config; const msec = 'ms' in timestamp ? timestamp.ms : timestamp.unix * 1000; @@ -35,7 +33,7 @@ export const timestampToString = (config: Config) => { }); const sameYear = timestampYear === viewerYear; - const stringDate = date.toLocaleDateString(locale, { + return date.toLocaleDateString(locale, { month: 'short', day: 'numeric', year: sameYear ? undefined : 'numeric', @@ -46,5 +44,4 @@ export const timestampToString = (config: Config) => { timeZone: targetTimezone, timeZoneName: timeFormat.showTimezone ? 'short' : undefined, }); - return stringDate.replace(', ', dateTimeSeparator); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx index 487e235334d1d..09850297807f0 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx @@ -24,7 +24,7 @@ export const AutomaterializationEvaluationHistoryTable = () => { - +
@@ -40,50 +40,3 @@ export const AutomaterializationEvaluationHistoryTable = () => { ); }; - -const StatusTag = ({ - status, - errors, -}: - | {status: 'Evaluating' | 'Skipped' | 'Complete'; errors?: null} - | { - status: 'Failure'; - errors: any; - }) => { - const [showErrors, setShowErrors] = React.useState(false); - const tag = React.useMemo(() => { - switch (status) { - case 'Evaluating': - return ( - - Evaluating - - ); - case 'Skipped': - return Skipped; - case 'Failure': - console.log({errors}); - return ( - - - { - setShowErrors(true); - }} - > - View errors - - - ); - case 'Complete': - return ; - } - }, [errors, status]); - - return ( - <> - {tag} - errors - - ); -}; From 8addd59749a6bb22a5891f262532f4d6e343ee4a Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Fri, 6 Oct 2023 12:13:57 -0400 Subject: [PATCH 05/10] unused imports --- .../AutomaterializationEvaluationHistoryTable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx index 09850297807f0..b50c6a4b4795c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx @@ -1,4 +1,4 @@ -import {Body2, Box, ButtonLink, Colors, Dialog, Table, Tag} from '@dagster-io/ui-components'; +import {Body2, Colors, Table} from '@dagster-io/ui-components'; import React from 'react'; import {Timestamp} from '../../app/time/Timestamp'; From 46f05f817b7e90df334038253b0bf2ffd822900b Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Mon, 9 Oct 2023 07:56:38 -0400 Subject: [PATCH 06/10] client side feature flag instead --- .../packages/ui-core/src/app/CloudFeatureFlag.tsx | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 js_modules/dagster-ui/packages/ui-core/src/app/CloudFeatureFlag.tsx diff --git a/js_modules/dagster-ui/packages/ui-core/src/app/CloudFeatureFlag.tsx b/js_modules/dagster-ui/packages/ui-core/src/app/CloudFeatureFlag.tsx deleted file mode 100644 index 323aba12915dd..0000000000000 --- a/js_modules/dagster-ui/packages/ui-core/src/app/CloudFeatureFlag.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import React from 'react'; - -type CloudFeatureFlagContextType = { - enableAMPTimeline: boolean; -}; -export const CloudFeatureFlagContext = React.createContext({ - enableAMPTimeline: true, -}); -export const useCloudFeatureFlag = () => { - return React.useContext(CloudFeatureFlagContext); -}; From 2ade4bde569fea8b934ce7f9c1e5e754de543841 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Thu, 5 Oct 2023 21:47:13 -0400 Subject: [PATCH 07/10] amp evaluations table and runs table --- .../AssetDaemonTicksQuery.tsx | 47 +++ ...omaterializationEvaluationHistoryTable.tsx | 309 ++++++++++++++++-- .../AutomaterializationRoot.tsx | 84 ++--- .../AutomaterializationTickDetailDialog.tsx | 109 +++++- .../AutomaterializeRunHistoryTable.tsx | 79 ++++- ...ypes.ts => AssetDaemonTicksQuery.types.ts} | 14 +- ...tomaterializationTickDetailDialog.types.ts | 28 ++ .../src/instigation/LiveTickTimeline2.tsx | 3 +- 8 files changed, 566 insertions(+), 107 deletions(-) create mode 100644 js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AssetDaemonTicksQuery.tsx rename js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/{AutomaterializationRoot.types.ts => AssetDaemonTicksQuery.types.ts} (78%) create mode 100644 js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AutomaterializationTickDetailDialog.types.ts diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AssetDaemonTicksQuery.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AssetDaemonTicksQuery.tsx new file mode 100644 index 0000000000000..f50b71b34cad8 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AssetDaemonTicksQuery.tsx @@ -0,0 +1,47 @@ +import {gql} from '@apollo/client'; + +import {PYTHON_ERROR_FRAGMENT} from '../../app/PythonErrorFragment'; + +export const ASSET_DAMEON_TICKS_QUERY = gql` + query AssetDaemonTicksQuery( + $dayRange: Int + $dayOffset: Int + $statuses: [InstigationTickStatus!] + $limit: Int + $cursor: String + ) { + autoMaterializeTicks( + dayRange: $dayRange + dayOffset: $dayOffset + statuses: $statuses + limit: $limit + cursor: $cursor + ) { + id + ...AssetDaemonTickFragment + } + } + + fragment AssetDaemonTickFragment on InstigationTick { + id + timestamp + endTimestamp + status + instigationType + error { + ...PythonErrorFragment + } + requestedAssetKeys { + path + } + requestedAssetMaterializationCount + autoMaterializeAssetEvaluationId + requestedMaterializationsForAssets { + assetKey { + path + } + partitionKeys + } + } + ${PYTHON_ERROR_FRAGMENT} +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx index b50c6a4b4795c..7d0fd8185e216 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx @@ -1,42 +1,283 @@ -import {Body2, Colors, Table} from '@dagster-io/ui-components'; +import { + BaseTag, + Body2, + Box, + Button, + ButtonGroup, + ButtonLink, + Checkbox, + Colors, + CursorHistoryControls, + Dialog, + DialogBody, + DialogFooter, + Spinner, + Table, + Tag, +} from '@dagster-io/ui-components'; import React from 'react'; +import {PythonErrorInfo} from '../../app/PythonErrorInfo'; +import {useQueryRefreshAtInterval} from '../../app/QueryRefresh'; import {Timestamp} from '../../app/time/Timestamp'; +import {InstigationTickStatus} from '../../graphql/types'; +import {useQueryPersistedState} from '../../hooks/useQueryPersistedState'; import {TimeElapsed} from '../../runs/TimeElapsed'; -import {AnchorButton} from '../../ui/AnchorButton'; +import {useCursorPaginatedQuery} from '../../runs/useCursorPaginatedQuery'; + +import {ASSET_DAMEON_TICKS_QUERY} from './AssetDaemonTicksQuery'; +import { + AssetDaemonTicksQuery, + AssetDaemonTicksQueryVariables, + AssetDaemonTickFragment, +} from './types/AssetDaemonTicksQuery.types'; + +const PAGE_SIZE = 15; + +export const AutomaterializationEvaluationHistoryTable = ({ + setSelectedTick, + setTableView, +}: { + setSelectedTick: (tick: AssetDaemonTickFragment | null) => void; + setTableView: (view: 'evaluations' | 'runs') => void; +}) => { + const [statuses, setStatuses] = useQueryPersistedState>({ + queryKey: 'statuses', + decode: React.useCallback(({statuses}: {statuses?: string}) => { + return new Set( + statuses + ? JSON.parse(statuses) + : [ + InstigationTickStatus.STARTED, + InstigationTickStatus.SUCCESS, + InstigationTickStatus.FAILURE, + InstigationTickStatus.SKIPPED, + ], + ); + }, []), + encode: React.useCallback((raw: Set) => { + return {statuses: JSON.stringify(Array.from(raw))}; + }, []), + }); + + const {queryResult, paginationProps} = useCursorPaginatedQuery< + AssetDaemonTicksQuery, + AssetDaemonTicksQueryVariables + >({ + query: ASSET_DAMEON_TICKS_QUERY, + variables: { + statuses: React.useMemo(() => Array.from(statuses), [statuses]), + }, + nextCursorForResult: (data) => { + const ticks = data.autoMaterializeTicks; + if (!ticks.length) { + return undefined; + } + return ticks[PAGE_SIZE - 1]?.id; + }, + getResultArray: (data) => { + if (!data?.autoMaterializeTicks) { + return []; + } + return data.autoMaterializeTicks; + }, + pageSize: PAGE_SIZE, + }); + // Only refresh if we're on the first page + useQueryRefreshAtInterval(queryResult, !paginationProps.hasPrevCursor ? 10000 : 60 * 60 * 1000); + + return ( + + + + { + setTableView(id); + }} + /> + {!queryResult.data ? : null} + + + + + + + + + + + + + + + + + + + + {/* Use previous data to stop page from jumping while new data loads */} + {(queryResult.data || queryResult.previousData)?.autoMaterializeTicks.map((tick) => ( + + + + + + + + ))} + +
TimestampStatusDurationResult
+ + + + + + + {tick.requestedAssetMaterializationCount ? ( + { + setSelectedTick(tick); + }} + > + + {tick.requestedAssetMaterializationCount} materializations requested + + + ) : ( + No runs launched + )} + + +
+
+ +
+ + ); +}; + +const StatusTag = ({tick}: {tick: AssetDaemonTickFragment}) => { + const {status, error, requestedMaterializationsForAssets} = tick; + const count = Object.keys(requestedMaterializationsForAssets).length; + const [showErrors, setShowErrors] = React.useState(false); + const tag = React.useMemo(() => { + switch (status) { + case InstigationTickStatus.STARTED: + return ( + + Evaluating + + ); + case InstigationTickStatus.SKIPPED: + return ; + case InstigationTickStatus.FAILURE: + return ( + + Failure + {error ? ( + { + setShowErrors(true); + }} + > + View + + ) : null} + + ); + case InstigationTickStatus.SUCCESS: + return {count} requested; + } + }, [error, count, status]); -export const AutomaterializationEvaluationHistoryTable = () => { - // TODO return ( - - - - - - - - - - - - - - - - - - - -
TimestampStatusDurationResult
- - -
-
- - - No runs launched - - View details -
+ <> + {tag} + {error ? ( + + + + + + + + + ) : null} + ); }; + +const StatusLabels = { + [InstigationTickStatus.SKIPPED]: 'None requested', + [InstigationTickStatus.STARTED]: 'Started', + [InstigationTickStatus.FAILURE]: 'Failed', + [InstigationTickStatus.SUCCESS]: 'Requested', +}; + +function StatusCheckbox({ + status, + statuses, + setStatuses, +}: { + status: InstigationTickStatus; + statuses: Set; + setStatuses: (statuses: Set) => void; +}) { + return ( + { + const newStatuses = new Set(statuses); + if (statuses.has(status)) { + newStatuses.delete(status); + } else { + newStatuses.add(status); + } + setStatuses(newStatuses); + }} + /> + ); +} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationRoot.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationRoot.tsx index f0d0921f40ef6..8ca16372973fc 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationRoot.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationRoot.tsx @@ -1,4 +1,4 @@ -import {gql, useQuery} from '@apollo/client'; +import {useQuery} from '@apollo/client'; import { Alert, Box, @@ -10,27 +10,27 @@ import { Heading, PageHeader, Table, - ButtonGroup, } from '@dagster-io/ui-components'; import React from 'react'; import {useConfirmation} from '../../app/CustomConfirmationProvider'; import {useUnscopedPermissions} from '../../app/Permissions'; -import {PYTHON_ERROR_FRAGMENT} from '../../app/PythonErrorFragment'; import {useQueryRefreshAtInterval} from '../../app/QueryRefresh'; import {useTrackPageView} from '../../app/analytics'; +import {useQueryPersistedState} from '../../hooks/useQueryPersistedState'; import {LiveTickTimeline} from '../../instigation/LiveTickTimeline2'; import {OverviewTabs} from '../../overview/OverviewTabs'; import {useAutomaterializeDaemonStatus} from '../AutomaterializeDaemonStatusTag'; +import {ASSET_DAMEON_TICKS_QUERY} from './AssetDaemonTicksQuery'; import {AutomaterializationEvaluationHistoryTable} from './AutomaterializationEvaluationHistoryTable'; import {AutomaterializationTickDetailDialog} from './AutomaterializationTickDetailDialog'; import {AutomaterializeRunHistoryTable} from './AutomaterializeRunHistoryTable'; import { - AssetDameonTicksQuery, - AssetDameonTicksQueryVariables, + AssetDaemonTicksQuery, + AssetDaemonTicksQueryVariables, AssetDaemonTickFragment, -} from './types/AutomaterializationRoot.types'; +} from './types/AssetDaemonTicksQuery.types'; const MINUTE = 60 * 1000; const THREE_MINUTES = 3 * MINUTE; @@ -44,7 +44,7 @@ export const AutomaterializationRoot = () => { const {permissions: {canToggleAutoMaterialize} = {}} = useUnscopedPermissions(); - const queryResult = useQuery( + const queryResult = useQuery( ASSET_DAMEON_TICKS_QUERY, ); const [isPaused, setIsPaused] = React.useState(false); @@ -52,7 +52,18 @@ export const AutomaterializationRoot = () => { const [selectedTick, setSelectedTick] = React.useState(null); - const [tableView, setTableView] = React.useState<'evaluations' | 'runs'>('evaluations'); + const [tableView, setTableView] = useQueryPersistedState<'evaluations' | 'runs'>( + React.useMemo( + () => ({ + queryKey: 'view', + decode: ({view}) => (view === 'runs' ? 'runs' : 'evaluations'), + encode: (raw) => { + return {view: raw, cursor: undefined, statuses: undefined}; + }, + }), + [], + ), + ); const ids = queryResult.data ? queryResult.data.autoMaterializeTicks.map((tick) => `${tick.id}:${tick.status}`) @@ -150,63 +161,16 @@ export const AutomaterializationRoot = () => { setSelectedTick(null); }} /> - - { - setTableView(id); - }} - /> - {tableView === 'evaluations' ? ( - + ) : ( - + )} )} ); }; - -const ASSET_DAMEON_TICKS_QUERY = gql` - query AssetDameonTicksQuery( - $dayRange: Int - $dayOffset: Int - $statuses: [InstigationTickStatus!] - $limit: Int - $cursor: String - ) { - autoMaterializeTicks( - dayRange: $dayRange - dayOffset: $dayOffset - statuses: $statuses - limit: $limit - cursor: $cursor - ) { - id - ...AssetDaemonTickFragment - } - } - - fragment AssetDaemonTickFragment on InstigationTick { - id - timestamp - endTimestamp - status - instigationType - error { - ...PythonErrorFragment - } - requestedAssetMaterializationCount - requestedAssetKeys { - path - } - autoMaterializeAssetEvaluationId - } - ${PYTHON_ERROR_FRAGMENT} -`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationTickDetailDialog.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationTickDetailDialog.tsx index 8e47d29eb31cd..054c4bdc3c658 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationTickDetailDialog.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationTickDetailDialog.tsx @@ -1,3 +1,4 @@ +import {gql, useQuery} from '@apollo/client'; import { Box, Colors, @@ -9,25 +10,36 @@ import { DialogBody, DialogFooter, ButtonLink, + Icon, + Spinner, } from '@dagster-io/ui-components'; import {useVirtualizer} from '@tanstack/react-virtual'; import React from 'react'; +import {Link} from 'react-router-dom'; import styled from 'styled-components'; import {PythonErrorInfo} from '../../app/PythonErrorInfo'; import {formatElapsedTime} from '../../app/Util'; import {Timestamp} from '../../app/time/Timestamp'; import {PythonErrorFragment} from '../../app/types/PythonErrorFragment.types'; +import {tokenForAssetKey} from '../../asset-graph/Utils'; import {AssetKeyInput, InstigationTickStatus} from '../../graphql/types'; import {HeaderCell, Inner, Row, RowCell} from '../../ui/VirtualizedTable'; +import {buildRepoAddress} from '../../workspace/buildRepoAddress'; +import {workspacePathFromAddress} from '../../workspace/workspacePath'; import {AssetLink} from '../AssetLink'; import { AssetKeysDialog, AssetKeysDialogHeader, AssetKeysDialogEmptyState, } from '../AutoMaterializePolicyPage/AssetKeysDialog'; +import {assetDetailsPathForKey} from '../assetDetailsPathForKey'; -import {AssetDaemonTickFragment} from './types/AutomaterializationRoot.types'; +import {AssetDaemonTickFragment} from './types/AssetDaemonTicksQuery.types'; +import { + AssetGroupAndLocationQuery, + AssetGroupAndLocationQueryVariables, +} from './types/AutomaterializationTickDetailDialog.types'; const TEMPLATE_COLUMNS = '30% 17% 53%'; export const AutomaterializationTickDetailDialog = React.memo( @@ -64,6 +76,14 @@ export const AutomaterializationTickDetailDialog = React.memo( const totalHeight = rowVirtualizer.getTotalSize(); const items = rowVirtualizer.getVirtualItems(); + const assetKeyToPartitionsMap = React.useMemo(() => { + const map: Record = {}; + tick?.requestedMaterializationsForAssets.forEach(({assetKey, partitionKeys}) => { + map[tokenForAssetKey(assetKey)] = partitionKeys; + }); + return map; + }, [tick?.requestedMaterializationsForAssets]); + const content = React.useMemo(() => { if (queryString && !filteredAssetKeys.length) { return ( @@ -107,12 +127,21 @@ export const AutomaterializationTickDetailDialog = React.memo( {items.map(({index, key, size, start}) => { const assetKey = filteredAssetKeys[index]!; - return ; + return ( + + ); })}
); - }, [filteredAssetKeys, items, queryString, tick, totalHeight]); + }, [assetKeyToPartitionsMap, filteredAssetKeys, items, queryString, tick, totalHeight]); const intent = React.useMemo(() => { switch (tick?.status) { @@ -190,7 +219,7 @@ export const AutomaterializationTickDetailDialog = React.memo( Status - {tick?.requestedAssetMaterializationCount || 0} requested + {tick?.requestedAssetMaterializationCount ?? 0} requested {tick?.error ? ( { - // TODO (after daniel adds new fields) + const numMaterializations = partitionKeys?.length || 1; + const {data} = useQuery( + ASSET_GROUP_QUERY, + { + fetchPolicy: 'cache-and-network', + variables: { + assetKey: {path: assetKey.path}, + }, + }, + ); + const asset = data?.assetOrError.__typename === 'Asset' ? data.assetOrError : null; + const definition = asset?.definition; + const repoAddress = definition + ? buildRepoAddress(definition.repository.name, definition.repository.location.name) + : null; return ( - - + + {data ? ( + definition && definition.groupName && repoAddress ? ( + + + + {definition.groupName} + + + ) : ( + Asset not found + ) + ) : ( + + )} + + + + {numMaterializations} materialization{numMaterializations === 1 ? '' : 's'} requested + + ); @@ -262,3 +335,25 @@ const RowGrid = styled(Box)` padding-top: 26px 0px; } `; + +const ASSET_GROUP_QUERY = gql` + query AssetGroupAndLocationQuery($assetKey: AssetKeyInput!) { + assetOrError(assetKey: $assetKey) { + ... on Asset { + id + definition { + id + groupName + repository { + id + name + location { + id + name + } + } + } + } + } + } +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializeRunHistoryTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializeRunHistoryTable.tsx index d82b3eba414b6..60adb989dc57c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializeRunHistoryTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializeRunHistoryTable.tsx @@ -1,6 +1,79 @@ +import {ButtonGroup, Box, CursorHistoryControls} from '@dagster-io/ui-components'; import React from 'react'; +import styled from 'styled-components'; -export const AutomaterializeRunHistoryTable = () => { - // TODO - return
; +import {useQueryRefreshAtInterval} from '../../app/QueryRefresh'; +import {RunTable} from '../../runs/RunTable'; +import {RUNS_ROOT_QUERY} from '../../runs/RunsRoot'; +import {RunsRootQuery, RunsRootQueryVariables} from '../../runs/types/RunsRoot.types'; +import {useCursorPaginatedQuery} from '../../runs/useCursorPaginatedQuery'; + +const PAGE_SIZE = 15; + +export const AutomaterializeRunHistoryTable = ({ + setTableView, +}: { + setTableView: (view: 'evaluations' | 'runs') => void; +}) => { + const {queryResult, paginationProps} = useCursorPaginatedQuery< + RunsRootQuery, + RunsRootQueryVariables + >({ + nextCursorForResult: (runs) => { + if (runs.pipelineRunsOrError.__typename !== 'Runs') { + return undefined; + } + return runs.pipelineRunsOrError.results[PAGE_SIZE - 1]?.id; + }, + getResultArray: (data) => { + if (!data || data.pipelineRunsOrError.__typename !== 'Runs') { + return []; + } + return data.pipelineRunsOrError.results; + }, + variables: { + filter: { + tags: [{key: 'dagster/auto_materialize', value: 'true'}], + }, + }, + query: RUNS_ROOT_QUERY, + pageSize: PAGE_SIZE, + }); + + useQueryRefreshAtInterval(queryResult, 15 * 1000); + + const runData = (queryResult.data || queryResult.previousData)?.pipelineRunsOrError; + + return ( + + + + { + setTableView(id); + }} + /> + + + +
+ +
+ + ); }; + +// Super hacky but easiest solution to position the action button +const Wrapper = styled.div` + position: relative; + > *:nth-child(2) { + position: absolute; + right: 0; + top: 0; + } +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AutomaterializationRoot.types.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AssetDaemonTicksQuery.types.ts similarity index 78% rename from js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AutomaterializationRoot.types.ts rename to js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AssetDaemonTicksQuery.types.ts index c51dda1e34e16..d081cd44a39fc 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AutomaterializationRoot.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AssetDaemonTicksQuery.types.ts @@ -2,7 +2,7 @@ import * as Types from '../../../graphql/types'; -export type AssetDameonTicksQueryVariables = Types.Exact<{ +export type AssetDaemonTicksQueryVariables = Types.Exact<{ dayRange?: Types.InputMaybe; dayOffset?: Types.InputMaybe; statuses?: Types.InputMaybe | Types.InstigationTickStatus>; @@ -10,7 +10,7 @@ export type AssetDameonTicksQueryVariables = Types.Exact<{ cursor?: Types.InputMaybe; }>; -export type AssetDameonTicksQuery = { +export type AssetDaemonTicksQuery = { __typename: 'Query'; autoMaterializeTicks: Array<{ __typename: 'InstigationTick'; @@ -32,6 +32,11 @@ export type AssetDameonTicksQuery = { }>; } | null; requestedAssetKeys: Array<{__typename: 'AssetKey'; path: Array}>; + requestedMaterializationsForAssets: Array<{ + __typename: 'RequestedMaterializationsForAsset'; + partitionKeys: Array; + assetKey: {__typename: 'AssetKey'; path: Array}; + }>; }>; }; @@ -55,4 +60,9 @@ export type AssetDaemonTickFragment = { }>; } | null; requestedAssetKeys: Array<{__typename: 'AssetKey'; path: Array}>; + requestedMaterializationsForAssets: Array<{ + __typename: 'RequestedMaterializationsForAsset'; + partitionKeys: Array; + assetKey: {__typename: 'AssetKey'; path: Array}; + }>; }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AutomaterializationTickDetailDialog.types.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AutomaterializationTickDetailDialog.types.ts new file mode 100644 index 0000000000000..49b4098c6c41a --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/types/AutomaterializationTickDetailDialog.types.ts @@ -0,0 +1,28 @@ +// Generated GraphQL types, do not edit manually. + +import * as Types from '../../../graphql/types'; + +export type AssetGroupAndLocationQueryVariables = Types.Exact<{ + assetKey: Types.AssetKeyInput; +}>; + +export type AssetGroupAndLocationQuery = { + __typename: 'Query'; + assetOrError: + | { + __typename: 'Asset'; + id: string; + definition: { + __typename: 'AssetNode'; + id: string; + groupName: string | null; + repository: { + __typename: 'Repository'; + id: string; + name: string; + location: {__typename: 'RepositoryLocation'; id: string; name: string}; + }; + } | null; + } + | {__typename: 'AssetNotFoundError'}; +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/instigation/LiveTickTimeline2.tsx b/js_modules/dagster-ui/packages/ui-core/src/instigation/LiveTickTimeline2.tsx index 27ea532498587..ee4273d31de5a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/instigation/LiveTickTimeline2.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/instigation/LiveTickTimeline2.tsx @@ -7,6 +7,7 @@ import styled from 'styled-components'; import {TimeContext} from '../app/time/TimeContext'; import {browserTimezone} from '../app/time/browserTimezone'; +import {AssetDaemonTickFragment} from '../assets/auto-materialization/types/AssetDaemonTicksQuery.types'; import {InstigationTickStatus, InstigationType} from '../graphql/types'; import {HistoryTickFragment} from './types/TickHistory.types'; @@ -51,7 +52,7 @@ type StrippedDownTickFragment = Pick< runs?: HistoryTickFragment['runs']; endTimestamp?: number | null; requestedAssetMaterializationCount?: number; -}; +} & Pick; export const LiveTickTimeline = ({ ticks, From e46513509188a33d571fb2b4f7a77c37f9f62fd4 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Thu, 5 Oct 2023 21:53:47 -0400 Subject: [PATCH 08/10] rebase --- .../AutomaterializationEvaluationHistoryTable.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx index 7d0fd8185e216..8df0339ab4cab 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx @@ -192,8 +192,8 @@ export const AutomaterializationEvaluationHistoryTable = ({ }; const StatusTag = ({tick}: {tick: AssetDaemonTickFragment}) => { - const {status, error, requestedMaterializationsForAssets} = tick; - const count = Object.keys(requestedMaterializationsForAssets).length; + const {status, error, requestedAssetMaterializationCount} = tick; + const count = requestedAssetMaterializationCount; const [showErrors, setShowErrors] = React.useState(false); const tag = React.useMemo(() => { switch (status) { From 2358818f2d94cb6d8e41ae2e1217e4745b5fae42 Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Thu, 5 Oct 2023 21:56:59 -0400 Subject: [PATCH 09/10] - --- .../AutomaterializationEvaluationHistoryTable.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx index 8df0339ab4cab..85653727fbd96 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx @@ -157,7 +157,9 @@ export const AutomaterializationEvaluationHistoryTable = ({ /> - {tick.requestedAssetMaterializationCount ? ( + {[InstigationTickStatus.SKIPPED, InstigationTickStatus.SUCCESS].includes( + tick.status, + ) ? ( { setSelectedTick(tick); @@ -168,7 +170,7 @@ export const AutomaterializationEvaluationHistoryTable = ({ ) : ( - No runs launched + ' - ' )} From a348d0e01e7f29353d40bc5c6a05e60539dacc7f Mon Sep 17 00:00:00 2001 From: Marco Salazar Date: Fri, 6 Oct 2023 15:49:05 -0400 Subject: [PATCH 10/10] feedback + rebase --- ...omaterializationEvaluationHistoryTable.tsx | 10 -------- .../AutomaterializationTickDetailDialog.tsx | 24 ++++++++++++------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx index 85653727fbd96..75a0299460f6c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationEvaluationHistoryTable.tsx @@ -137,7 +137,6 @@ export const AutomaterializationEvaluationHistoryTable = ({ Status Duration Result - @@ -173,15 +172,6 @@ export const AutomaterializationEvaluationHistoryTable = ({ ' - ' )} - - - ))} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationTickDetailDialog.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationTickDetailDialog.tsx index 054c4bdc3c658..78200382b5361 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationTickDetailDialog.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/auto-materialization/AutomaterializationTickDetailDialog.tsx @@ -219,7 +219,11 @@ export const AutomaterializationTickDetailDialog = React.memo( Status - {tick?.requestedAssetMaterializationCount ?? 0} requested + {tick?.status === InstigationTickStatus.STARTED ? ( + 'Evaluating…' + ) : ( + <>{tick?.requestedAssetMaterializationCount ?? 0} requested + )} {tick?.error ? (
- 0 ? undefined : 'bottom'} - > - Materializations requested - - {content} + {tick?.status === InstigationTickStatus.STARTED ? null : ( + <> + 0 ? undefined : 'bottom'} + > + Materializations requested + + {content} + + )}
} />