diff --git a/js_modules/dagster-ui/packages/ui-components/src/components/calculateMiddleTruncation.tsx b/js_modules/dagster-ui/packages/ui-components/src/components/calculateMiddleTruncation.tsx index e29d26a4b2100..cfecfa4377e2b 100644 --- a/js_modules/dagster-ui/packages/ui-components/src/components/calculateMiddleTruncation.tsx +++ b/js_modules/dagster-ui/packages/ui-components/src/components/calculateMiddleTruncation.tsx @@ -32,6 +32,5 @@ export const calculateMiddleTruncation = ( end = mid - 1; } } - return `${text.slice(0, end)}…${text.slice(-end)}`; }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx index 841854079d64e..b7e015fe32513 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/SidebarAssetInfo.tsx @@ -248,6 +248,9 @@ const SIDEBAR_ASSET_FRAGMENT = gql` description } } + backfillPolicy { + description + } partitionDefinition { description } diff --git a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/types/SidebarAssetInfo.types.ts b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/types/SidebarAssetInfo.types.ts index 11bde05c01ebc..130aea6245670 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/asset-graph/types/SidebarAssetInfo.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/asset-graph/types/SidebarAssetInfo.types.ts @@ -127,6 +127,7 @@ export type SidebarAssetFragment = { description: string; }>; } | null; + backfillPolicy: {__typename: 'BackfillPolicy'; description: string} | null; partitionDefinition: {__typename: 'PartitionDefinition'; description: string} | null; assetKey: {__typename: 'AssetKey'; path: Array}; op: { @@ -15697,6 +15698,7 @@ export type SidebarAssetQuery = { description: string; }>; } | null; + backfillPolicy: {__typename: 'BackfillPolicy'; description: string} | null; partitionDefinition: {__typename: 'PartitionDefinition'; description: string} | null; assetKey: {__typename: 'AssetKey'; path: Array}; op: { diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetLink.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetLink.tsx index fe3db1973d7de..93803fd45d422 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetLink.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetLink.tsx @@ -24,7 +24,7 @@ export const AssetLink: React.FC<{ style={{maxWidth: '100%'}} > {icon ? ( - + ) : null} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeDefinition.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeDefinition.tsx index 4a29e0d66aa02..ac3ffd5d3dec3 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeDefinition.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetNodeDefinition.tsx @@ -95,6 +95,7 @@ export const AssetNodeDefinition: React.FC<{ )} + {assetNode.freshnessPolicy && ( <> @@ -127,6 +128,21 @@ export const AssetNodeDefinition: React.FC<{ )} + + {assetNode.backfillPolicy && ( + <> + + Backfill policy + + + {assetNode.backfillPolicy.description} + + + )} + = ({ )} + {asset.backfillPolicy && ( + + + {asset.backfillPolicy.description} + + + )} + {loadedPartitionKeys.length > 1 ? null : ( <> void; +} +const TEMPLATE_COLUMNS = '1fr 1fr 1fr 1fr'; + +export const BackfillPreviewModal = ({ + isOpen, + setOpen, + assets, + keysFiltered, +}: BackfillPreviewModalProps) => { + const assetKeys = React.useMemo(() => assets.map(asAssetKeyInput), [assets]); + const parentRef = React.useRef(null); + + const rowVirtualizer = useVirtualizer({ + count: assets.length, + getScrollElement: () => parentRef.current, + estimateSize: () => 60, + overscan: 10, + }); + const totalHeight = rowVirtualizer.getTotalSize(); + const items = rowVirtualizer.getVirtualItems(); + + const {data} = useQuery( + BACKFILL_PREVIEW_QUERY, + { + variables: {partitionNames: keysFiltered, assetKeys}, + skip: !isOpen, + }, + ); + + const partitionsByAssetToken = React.useMemo(() => { + return Object.fromEntries( + (data?.assetBackfillPreview || []).map((d) => [tokenForAssetKey(d.assetKey), d.partitions]), + ); + }, [data]); + + // BG Note: The transform: scale(1) below fixes a bug with MiddleTruncate where the text size + // is measured while the dialog is animating open and the scale is < 1, causing it to think + // it needs to truncate. A more general fix for this seems like it'll require a lot of testing. + + return ( + setOpen(false)} + style={{width: '90vw', maxWidth: 1100, transform: 'scale(1)'}} + > + + + + {items.map(({index, size, start}) => { + const {assetKey, partitionDefinition, backfillPolicy} = assets[index]!; + const token = tokenForAssetKey(assetKey); + const partitions = partitionsByAssetToken[token]; + + return ( + + + + + + {backfillPolicy ? ( + {backfillPolicy?.description} + ) : ( + {'\u2013'} + )} + {partitionDefinition ? ( + + {partitionDefinition?.description} + + ) : ( + {'\u2013'} + )} + + {partitions ? ( + + ) : ( + + )} + + + + ); + })} + + + + + + + ); +}; + +const RowGrid = styled(Box)` + display: grid; + grid-template-columns: ${TEMPLATE_COLUMNS}; + height: 100%; +`; + +export const BackfillPreviewTableHeader = () => { + return ( + + Asset key + Backfill policy + Partition definition + Partitions to launch + + ); +}; + +export const BACKFILL_PREVIEW_QUERY = gql` + query BackfillPreviewQuery($partitionNames: [String!]!, $assetKeys: [AssetKeyInput!]!) { + assetBackfillPreview(params: {partitionNames: $partitionNames, assetSelection: $assetKeys}) { + assetKey { + path + } + partitions { + partitionKeys + ranges { + start + end + } + } + } + } +`; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/LaunchAssetChoosePartitionsDialog.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/LaunchAssetChoosePartitionsDialog.tsx index cf6cade712b62..3e325f2e68d43 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/LaunchAssetChoosePartitionsDialog.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/LaunchAssetChoosePartitionsDialog.tsx @@ -2,19 +2,19 @@ import {gql, useApolloClient, useQuery} from '@apollo/client'; // eslint-disable-next-line no-restricted-imports import {Radio} from '@blueprintjs/core'; import { + Alert, Box, Button, ButtonLink, + Checkbox, Colors, Dialog, DialogFooter, DialogHeader, - Tooltip, - Alert, - Checkbox, Icon, - Subheading, RadioContainer, + Subheading, + Tooltip, } from '@dagster-io/ui-components'; import reject from 'lodash/reject'; import React from 'react'; @@ -38,18 +38,18 @@ import { } from '../instance/backfill/types/BackfillUtils.types'; import {CONFIG_PARTITION_SELECTION_QUERY} from '../launchpad/ConfigEditorConfigPicker'; import {useLaunchPadHooks} from '../launchpad/LaunchpadHooksContext'; -import {TagEditor, TagContainer} from '../launchpad/TagEditor'; +import {TagContainer, TagEditor} from '../launchpad/TagEditor'; import { ConfigPartitionSelectionQuery, ConfigPartitionSelectionQueryVariables, } from '../launchpad/types/ConfigEditorConfigPicker.types'; import { - DaemonNotRunningAlert, DAEMON_NOT_RUNNING_ALERT_INSTANCE_FRAGMENT, + DaemonNotRunningAlert, + USING_DEFAULT_LAUNCHER_ALERT_INSTANCE_FRAGMENT, + UsingDefaultLauncherAlert, showBackfillErrorToast, showBackfillSuccessToast, - UsingDefaultLauncherAlert, - USING_DEFAULT_LAUNCHER_ALERT_INSTANCE_FRAGMENT, } from '../partitions/BackfillMessaging'; import {DimensionRangeWizard} from '../partitions/DimensionRangeWizard'; import {assembleIntoSpans, stringForSpan} from '../partitions/SpanRepresentation'; @@ -61,27 +61,30 @@ import {RepoAddress} from '../workspace/types'; import {partitionCountString} from './AssetNodePartitionCounts'; import {AssetPartitionStatus} from './AssetPartitionStatus'; +import {BackfillPreviewModal} from './BackfillPreviewModal'; import { - executionParamsForAssetJob, LaunchAssetsChoosePartitionsTarget, + executionParamsForAssetJob, } from './LaunchAssetExecutionButton'; import { explodePartitionKeysInSelectionMatching, mergedAssetHealth, partitionDefinitionsEqual, } from './MultipartitioningSupport'; -import {PartitionHealthSummary} from './PartitionHealthSummary'; import {RunningBackfillsNotice} from './RunningBackfillsNotice'; import {asAssetKeyInput} from './asInput'; import { LaunchAssetWarningsQuery, LaunchAssetWarningsQueryVariables, } from './types/LaunchAssetChoosePartitionsDialog.types'; -import {PartitionDefinitionForLaunchAssetFragment} from './types/LaunchAssetExecutionButton.types'; +import { + BackfillPolicyForLaunchAssetFragment, + PartitionDefinitionForLaunchAssetFragment, +} from './types/LaunchAssetExecutionButton.types'; import {usePartitionDimensionSelections} from './usePartitionDimensionSelections'; import { - keyCountInSelections, PartitionDimensionSelection, + keyCountInSelections, usePartitionHealthData, } from './usePartitionHealthData'; @@ -97,6 +100,7 @@ interface Props { assetChecks: Pick[]; opNames: string[]; partitionDefinition: PartitionDefinitionForLaunchAssetFragment | null; + backfillPolicy: BackfillPolicyForLaunchAssetFragment | null; }[]; upstreamAssetKeys: AssetKey[]; // single layer of upstream dependencies refetch?: () => Promise; @@ -146,12 +150,10 @@ const LaunchAssetChoosePartitionsDialogBody: React.FC = ({ disabledReasons, } = usePermissionsForLocation(repoAddress.location); const [launching, setLaunching] = React.useState(false); - const [tagEditorOpen, setTagEditorOpen] = React.useState(false); + const [tagEditorOpen, setTagEditorOpen] = React.useState(false); + const [previewOpen, setPreviewOpen] = React.useState(false); const [tags, setTags] = React.useState([]); - const [previewCount, setPreviewCount] = React.useState(0); - const morePreviewsCount = partitionedAssets.length - previewCount; - const showSingleRunBackfillToggle = useFeatureFlagForCodeLocation( repoAddress.location, 'SHOW_SINGLE_RUN_BACKFILL_TOGGLE', @@ -229,6 +231,8 @@ const LaunchAssetChoosePartitionsDialogBody: React.FC = ({ ['pureWithAnchorAsset', 'pureAll'].includes(target.type) || (!launchWithRangesAsTags && keysFiltered.length !== 1); + const backfillPolicyVaries = assets.some((a) => a.backfillPolicy !== assets[0]?.backfillPolicy); + React.useEffect(() => { !canLaunchWithRangesAsTags && setLaunchWithRangesAsTags(false); }, [canLaunchWithRangesAsTags]); @@ -398,11 +402,7 @@ const LaunchAssetChoosePartitionsDialogBody: React.FC = ({ if (launchAsBackfill && !canLaunchPartitionBackfill) { return ( - + ); } @@ -423,13 +423,7 @@ const LaunchAssetChoosePartitionsDialogBody: React.FC = ({ disabled={target.type === 'pureAll' ? false : keysFiltered.length === 0} loading={launching} > - {launching - ? 'Launching...' - : launchAsBackfill - ? target.type === 'job' - ? `Launch ${keysFiltered.length}-run backfill` - : 'Launch backfill' - : `Launch 1 run`} + {launching ? 'Launching...' : launchAsBackfill ? 'Launch backfill' : `Launch 1 run`} ); }; @@ -492,7 +486,7 @@ const LaunchAssetChoosePartitionsDialogBody: React.FC = ({ {selections.map((range, idx) => ( @@ -531,16 +525,37 @@ const LaunchAssetChoosePartitionsDialogBody: React.FC = ({ repoAddress={repoAddress} refetch={refetch} /> - - {target.type === 'pureWithAnchorAsset' && ( - - )} ))} + + + + {target.type === 'pureWithAnchorAsset' ? ( + setPreviewOpen(true)} + text={ + `Dagster will materialize all partitions downstream of the ` + + `selected partitions for the selected assets, using separate runs + ${backfillPolicyVaries ? `and obeying backfill policies.` : `as needed.`}` + } + /> + ) : backfillPolicyVaries ? ( + setPreviewOpen(true)} + text={ + `Dagster will materialize the selected partitions for the ` + + `selected assets using varying backfill policies.` + } + /> + ) : assets[0]?.backfillPolicy ? ( + + ) : undefined} )} = ({ - Backfill options} - isInitiallyOpen={true} - > - {target.type === 'job' && ( + {target.type === 'job' && ( + Options} + > = ({ ) : null} - )} - - - - {previewCount > 0 && ( - - {partitionedAssets.slice(0, previewCount).map((a) => ( - - ))} - {morePreviewsCount > 0 && ( - - setPreviewCount(partitionedAssets.length)}> - Show {morePreviewsCount} more {morePreviewsCount > 1 ? 'previews' : 'preview'} - - - )} - - )} - - {previewCount === 0 && partitionedAssets.length > 1 && ( - - setPreviewCount(5)}> - Show per-asset partition health - - - )} - + + )} ); }; + +const PartitionSelectionNotice = ({ + text, + onShowPreview, +}: { + text: string; + onShowPreview?: () => void; +}) => { + return ( + + + {text} + {onShowPreview && ( + + )} + + } + /> + + ); +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/LaunchAssetExecutionButton.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/LaunchAssetExecutionButton.tsx index d4da79d1b32da..d2d499d90a742 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/LaunchAssetExecutionButton.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/LaunchAssetExecutionButton.tsx @@ -39,13 +39,13 @@ import {partitionDefinitionsEqual} from './MultipartitioningSupport'; import {asAssetKeyInput, getAssetCheckHandleInputs} from './asInput'; import {AssetKey} from './types'; import { + LaunchAssetCheckUpstreamQuery, + LaunchAssetCheckUpstreamQueryVariables, LaunchAssetExecutionAssetNodeFragment, LaunchAssetLoaderQuery, LaunchAssetLoaderQueryVariables, LaunchAssetLoaderResourceQuery, LaunchAssetLoaderResourceQueryVariables, - LaunchAssetCheckUpstreamQuery, - LaunchAssetCheckUpstreamQueryVariables, } from './types/LaunchAssetExecutionButton.types'; export type LaunchAssetsChoosePartitionsTarget = @@ -678,6 +678,14 @@ const PARTITION_DEFINITION_FOR_LAUNCH_ASSET_FRAGMENT = gql` } `; +const BACKFILL_POLICY_FOR_LAUNCH_ASSET_FRAGMENT = gql` + fragment BackfillPolicyForLaunchAssetFragment on BackfillPolicy { + maxPartitionsPerRun + description + policyType + } +`; + const LAUNCH_ASSET_EXECUTION_ASSET_NODE_FRAGMENT = gql` fragment LaunchAssetExecutionAssetNodeFragment on AssetNode { id @@ -688,6 +696,9 @@ const LAUNCH_ASSET_EXECUTION_ASSET_NODE_FRAGMENT = gql` partitionDefinition { ...PartitionDefinitionForLaunchAssetFragment } + backfillPolicy { + ...BackfillPolicyForLaunchAssetFragment + } isObservable isExecutable isSource @@ -717,6 +728,7 @@ const LAUNCH_ASSET_EXECUTION_ASSET_NODE_FRAGMENT = gql` ${ASSET_NODE_CONFIG_FRAGMENT} ${PARTITION_DEFINITION_FOR_LAUNCH_ASSET_FRAGMENT} + ${BACKFILL_POLICY_FOR_LAUNCH_ASSET_FRAGMENT} `; export const LAUNCH_ASSET_LOADER_QUERY = gql` diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/__fixtures__/BackfillPreviewQuery.fixtures.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/__fixtures__/BackfillPreviewQuery.fixtures.tsx new file mode 100644 index 0000000000000..f65ebfbbf9de5 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/__fixtures__/BackfillPreviewQuery.fixtures.tsx @@ -0,0 +1,67 @@ +import {MockedResponse} from '@apollo/client/testing'; + +import { + buildQuery, + buildAssetPartitions, + buildAssetKey, + buildAssetBackfillTargetPartitions, + buildPartitionKeyRange, +} from '../../graphql/types'; +import {BACKFILL_PREVIEW_QUERY} from '../BackfillPreviewModal'; +import { + BackfillPreviewQuery, + BackfillPreviewQueryVariables, +} from '../types/BackfillPreviewModal.types'; + +export const BackfillPreviewQueryMockPartitionKeys = [ + '2023-07-02', + '2023-07-09', + '2023-07-16', + '2023-07-23', + '2023-07-30', + '2023-08-06', + '2023-08-13', + '2023-08-20', + '2023-08-27', + '2023-09-03', + '2023-09-10', + '2023-09-17', + '2023-09-24', + '2023-10-01', + '2023-10-08', + '2023-10-15', + '2023-10-22', +]; + +export const BackfillPreviewQueryMock: MockedResponse< + BackfillPreviewQuery, + BackfillPreviewQueryVariables +> = { + request: { + query: BACKFILL_PREVIEW_QUERY, + variables: { + partitionNames: BackfillPreviewQueryMockPartitionKeys, + assetKeys: [{path: ['asset_weekly']}, {path: ['asset_daily']}], + }, + }, + result: { + data: buildQuery({ + assetBackfillPreview: [ + buildAssetPartitions({ + assetKey: buildAssetKey({path: ['asset_daily']}), + partitions: buildAssetBackfillTargetPartitions({ + ranges: [buildPartitionKeyRange({start: '2023-07-02', end: '2023-10-22'})], + partitionKeys: null, + }), + }), + buildAssetPartitions({ + assetKey: buildAssetKey({path: ['asset_weekly']}), + partitions: buildAssetBackfillTargetPartitions({ + ranges: [buildPartitionKeyRange({start: '2023-07-07', end: '2023-10-21'})], + partitionKeys: null, + }), + }), + ], + }), + }, +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/__fixtures__/LaunchAssetExecutionButton.fixtures.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/__fixtures__/LaunchAssetExecutionButton.fixtures.ts index 013f19aa8dab1..3606b88c50f4c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/__fixtures__/LaunchAssetExecutionButton.fixtures.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/__fixtures__/LaunchAssetExecutionButton.fixtures.ts @@ -1,4 +1,5 @@ import {MockedResponse} from '@apollo/client/testing'; +import without from 'lodash/without'; import {tokenForAssetKey} from '../../asset-graph/Utils'; import {AssetNodeForGraphQueryFragment} from '../../asset-graph/types/useAssetGraphData.types'; @@ -239,6 +240,49 @@ export const buildLaunchAssetWarningsMock = ( }, }); +export const PartitionHealthAssetDailyMaterializedRanges = [ + { + status: PartitionRangeStatus.MATERIALIZED, + startTime: 1662940800.0, + endTime: 1663027200.0, + startKey: '2022-09-12', + endKey: '2022-09-12', + __typename: 'TimePartitionRangeStatus' as const, + }, + { + status: PartitionRangeStatus.MATERIALIZED, + startTime: 1663027200.0, + endTime: 1667088000.0, + startKey: '2022-09-13', + endKey: '2022-10-29', + __typename: 'TimePartitionRangeStatus' as const, + }, + { + status: PartitionRangeStatus.MATERIALIZED, + startTime: 1668816000.0, + endTime: 1670803200.0, + startKey: '2022-11-19', + endKey: '2022-12-11', + __typename: 'TimePartitionRangeStatus' as const, + }, + { + status: PartitionRangeStatus.MATERIALIZED, + startTime: 1671494400.0, + endTime: 1674086400.0, + startKey: '2022-12-20', + endKey: '2023-01-18', + __typename: 'TimePartitionRangeStatus' as const, + }, + { + status: PartitionRangeStatus.MATERIALIZED, + startTime: 1676851200.0, + endTime: 1676937600.0, + startKey: '2023-02-20', + endKey: '2023-02-20', + __typename: 'TimePartitionRangeStatus' as const, + }, +]; + export const PartitionHealthAssetDailyMock: MockedResponse = { request: { query: PARTITION_HEALTH_QUERY, @@ -262,48 +306,7 @@ export const PartitionHealthAssetDailyMock: MockedResponse }, ], assetPartitionStatuses: { - ranges: [ - { - status: PartitionRangeStatus.MATERIALIZED, - startTime: 1662940800.0, - endTime: 1663027200.0, - startKey: '2022-09-12', - endKey: '2022-09-12', - __typename: 'TimePartitionRangeStatus', - }, - { - status: PartitionRangeStatus.MATERIALIZED, - startTime: 1663027200.0, - endTime: 1667088000.0, - startKey: '2022-09-13', - endKey: '2022-10-29', - __typename: 'TimePartitionRangeStatus', - }, - { - status: PartitionRangeStatus.MATERIALIZED, - startTime: 1668816000.0, - endTime: 1670803200.0, - startKey: '2022-11-19', - endKey: '2022-12-11', - __typename: 'TimePartitionRangeStatus', - }, - { - status: PartitionRangeStatus.MATERIALIZED, - startTime: 1671494400.0, - endTime: 1674086400.0, - startKey: '2022-12-20', - endKey: '2023-01-18', - __typename: 'TimePartitionRangeStatus', - }, - { - status: PartitionRangeStatus.MATERIALIZED, - startTime: 1676851200.0, - endTime: 1676937600.0, - startKey: '2023-02-20', - endKey: '2023-02-20', - __typename: 'TimePartitionRangeStatus', - }, - ], + ranges: PartitionHealthAssetDailyMaterializedRanges, __typename: 'TimePartitionStatuses', }, __typename: 'AssetNode', @@ -312,6 +315,13 @@ export const PartitionHealthAssetDailyMock: MockedResponse }, }; +export const ASSET_DAILY_PARTITION_KEYS_MISSING = without( + ASSET_DAILY_PARTITION_KEYS, + ...PartitionHealthAssetDailyMaterializedRanges.flatMap((r) => + generateDailyTimePartitions(new Date(r.startTime * 1000 - 1), new Date(r.endTime * 1000 - 1)), + ), +); + export const PartitionHealthAssetWeeklyMock: MockedResponse = { request: { query: PARTITION_HEALTH_QUERY, @@ -647,6 +657,7 @@ export const LaunchAssetLoaderAssetDailyWeeklyMock: MockedResponse { + return ( + + {}} + assets={[ + buildAssetNode({ + assetKey: buildAssetKey({path: ['asset_weekly']}), + partitionDefinition: buildPartitionDefinition({ + description: 'Weekly at 4:00AM UTC', + }), + backfillPolicy: buildBackfillPolicy({ + description: 'Backfills using separate runs, with at most 5 partitions per run.', + }), + }), + buildAssetNode({ + assetKey: buildAssetKey({path: ['asset_daily']}), + partitionDefinition: buildPartitionDefinition({ + description: 'Daily at 4:00AM UTC', + }), + backfillPolicy: buildBackfillPolicy({ + description: 'Backfills in a single run.', + }), + }), + ]} + keysFiltered={BackfillPreviewQueryMockPartitionKeys} + /> + + ); +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/LaunchAssetExecutionButton.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/LaunchAssetExecutionButton.test.tsx index 50ea6cc9326d5..b9531957cf0c9 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/LaunchAssetExecutionButton.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/__tests__/LaunchAssetExecutionButton.test.tsx @@ -17,14 +17,10 @@ import { import { ASSET_DAILY, ASSET_DAILY_PARTITION_KEYS, + ASSET_DAILY_PARTITION_KEYS_MISSING, ASSET_WEEKLY, ASSET_WEEKLY_ROOT, - buildConfigPartitionSelectionLatestPartitionMock, - buildExpectedLaunchBackfillMutation, - buildExpectedLaunchSingleRunMutation, - buildLaunchAssetLoaderMock, LaunchAssetCheckUpstreamWeeklyRootMock, - buildLaunchAssetWarningsMock, LaunchAssetLoaderResourceJob7Mock, LaunchAssetLoaderResourceJob8Mock, LaunchAssetLoaderResourceMyAssetJobMock, @@ -32,8 +28,13 @@ import { UNPARTITIONED_ASSET, UNPARTITIONED_ASSET_OTHER_REPO, UNPARTITIONED_ASSET_WITH_REQUIRED_CONFIG, - UNPARTITIONED_SOURCE_ASSET, UNPARTITIONED_NON_EXECUTABLE_ASSET, + UNPARTITIONED_SOURCE_ASSET, + buildConfigPartitionSelectionLatestPartitionMock, + buildExpectedLaunchBackfillMutation, + buildExpectedLaunchSingleRunMutation, + buildLaunchAssetLoaderMock, + buildLaunchAssetWarningsMock, } from '../__fixtures__/LaunchAssetExecutionButton.fixtures'; // This file must be mocked because Jest can't handle `import.meta.url`. @@ -272,16 +273,34 @@ describe('LaunchAssetExecutionButton', () => { await clickMaterializeButton(); await screen.findByTestId('choose-partitions-dialog'); - const launchButton = await screen.findByTestId('launch-button'); + // verify that the executed mutation is correct + await expectLaunchExecutesMutationAndCloses('Launch backfill', launchMock); + }); - // verify that the missing only checkbox updates the number of runs - expect(launchButton.textContent).toEqual('Launch 1148-run backfill'); - await userEvent.click(screen.getByTestId('missing-only-checkbox')); - expect(launchButton.textContent).toEqual('Launch 1046-run backfill'); + it('should launch backfills with only missing partitions if requested', async () => { + const launchMock = buildExpectedLaunchBackfillMutation({ + selector: { + partitionSetName: 'my_asset_job_partition_set', + repositorySelector: {repositoryLocationName: 'test.py', repositoryName: 'repo'}, + }, + assetSelection: [{path: ['asset_daily']}], + partitionNames: ASSET_DAILY_PARTITION_KEYS_MISSING, + fromFailure: false, + tags: [], + }); + renderButton({ + scope: {all: [ASSET_DAILY]}, + preferredJobName: 'my_asset_job', + launchMock, + }); + await clickMaterializeButton(); + await screen.findByTestId('choose-partitions-dialog'); + + // verify that checking "missing only" triggers the mutation with fewer partitions await userEvent.click(screen.getByTestId('missing-only-checkbox')); // verify that the executed mutation is correct - await expectLaunchExecutesMutationAndCloses('Launch 1148-run backfill', launchMock); + await expectLaunchExecutesMutationAndCloses('Launch backfill', launchMock); }); it('should launch single runs via the hidden job if no job is in context', async () => { @@ -342,7 +361,7 @@ describe('LaunchAssetExecutionButton', () => { const rangesAsTags = screen.getByTestId('ranges-as-tags-true-radio'); await waitFor(async () => expect(rangesAsTags).toBeEnabled()); - await expectLaunchExecutesMutationAndCloses('Launch 1148-run backfill', launchMock); + await expectLaunchExecutesMutationAndCloses('Launch backfill', launchMock); }); it('should launch a single run if you choose to pass the partition range using tags', async () => { @@ -442,6 +461,30 @@ describe('LaunchAssetExecutionButton', () => { await expectLaunchExecutesMutationAndCloses('Launch backfill', LaunchMutationMock); }); + it('should offer a preview showing the exact ranges to be launched', async () => { + const LaunchMutationMock = buildExpectedLaunchBackfillMutation({ + selector: undefined, + assetSelection: [{path: ['asset_daily']}, {path: ['asset_weekly']}], + partitionNames: ASSET_DAILY_PARTITION_KEYS, + fromFailure: false, + tags: [], + }); + + renderButton({ + scope: {all: [ASSET_DAILY, ASSET_WEEKLY]}, + launchMock: LaunchMutationMock, + }); + + await clickMaterializeButton(); + + const preview = await screen.findByTestId('backfill-preview-button'); + await preview.click(); + + // Expect the modal to be displayed. We have separate test coverage for + // for the content of this modal + await screen.findByTestId('backfill-preview-modal-content'); + }); + it('should offer to materialize all partitions if roots have different partition defintions ("pureAll" case)', async () => { const LaunchPureAllMutationMock = buildExpectedLaunchBackfillMutation({ tags: [], diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/types/AssetNodeDefinition.types.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/types/AssetNodeDefinition.types.ts index e3450fefc1385..cfb658e96e6f1 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/types/AssetNodeDefinition.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/types/AssetNodeDefinition.types.ts @@ -32,6 +32,7 @@ export type AssetNodeDefinitionFragment = { cronSchedule: string | null; cronScheduleTimezone: string | null; } | null; + backfillPolicy: {__typename: 'BackfillPolicy'; description: string} | null; partitionDefinition: {__typename: 'PartitionDefinition'; description: string} | null; repository: { __typename: 'Repository'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/types/AssetView.types.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/types/AssetView.types.ts index e30e9b2690740..4c8cdfa3facdf 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/types/AssetView.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/types/AssetView.types.ts @@ -88,6 +88,7 @@ export type AssetViewDefinitionQuery = { cronSchedule: string | null; cronScheduleTimezone: string | null; } | null; + backfillPolicy: {__typename: 'BackfillPolicy'; description: string} | null; requiredResources: Array<{__typename: 'ResourceRequirement'; resourceKey: string}>; assetKey: {__typename: 'AssetKey'; path: Array}; configField: { @@ -15807,6 +15808,7 @@ export type AssetViewDefinitionNodeFragment = { cronSchedule: string | null; cronScheduleTimezone: string | null; } | null; + backfillPolicy: {__typename: 'BackfillPolicy'; description: string} | null; requiredResources: Array<{__typename: 'ResourceRequirement'; resourceKey: string}>; assetKey: {__typename: 'AssetKey'; path: Array}; configField: { diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/types/BackfillPreviewModal.types.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/types/BackfillPreviewModal.types.ts new file mode 100644 index 0000000000000..22541e8727a91 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/types/BackfillPreviewModal.types.ts @@ -0,0 +1,21 @@ +// Generated GraphQL types, do not edit manually. + +import * as Types from '../../graphql/types'; + +export type BackfillPreviewQueryVariables = Types.Exact<{ + partitionNames: Array | Types.Scalars['String']; + assetKeys: Array | Types.AssetKeyInput; +}>; + +export type BackfillPreviewQuery = { + __typename: 'Query'; + assetBackfillPreview: Array<{ + __typename: 'AssetPartitions'; + assetKey: {__typename: 'AssetKey'; path: Array}; + partitions: { + __typename: 'AssetBackfillTargetPartitions'; + partitionKeys: Array | null; + ranges: Array<{__typename: 'PartitionKeyRange'; start: string; end: string}> | null; + } | null; + }>; +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/types/LaunchAssetExecutionButton.types.ts b/js_modules/dagster-ui/packages/ui-core/src/assets/types/LaunchAssetExecutionButton.types.ts index c40cb08f554db..ccc60b6b48857 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/types/LaunchAssetExecutionButton.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/types/LaunchAssetExecutionButton.types.ts @@ -14,6 +14,13 @@ export type PartitionDefinitionForLaunchAssetFragment = { }>; }; +export type BackfillPolicyForLaunchAssetFragment = { + __typename: 'BackfillPolicy'; + maxPartitionsPerRun: number | null; + description: string; + policyType: Types.BackfillPolicyType; +}; + export type LaunchAssetExecutionAssetNodeFragment = { __typename: 'AssetNode'; id: string; @@ -35,6 +42,12 @@ export type LaunchAssetExecutionAssetNodeFragment = { dynamicPartitionsDefinitionName: string | null; }>; } | null; + backfillPolicy: { + __typename: 'BackfillPolicy'; + maxPartitionsPerRun: number | null; + description: string; + policyType: Types.BackfillPolicyType; + } | null; assetKey: {__typename: 'AssetKey'; path: Array}; assetChecks: Array<{ __typename: 'AssetCheck'; @@ -630,6 +643,12 @@ export type LaunchAssetLoaderQuery = { dynamicPartitionsDefinitionName: string | null; }>; } | null; + backfillPolicy: { + __typename: 'BackfillPolicy'; + maxPartitionsPerRun: number | null; + description: string; + policyType: Types.BackfillPolicyType; + } | null; assetKey: {__typename: 'AssetKey'; path: Array}; assetChecks: Array<{ __typename: 'AssetCheck'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/BackfillPage.tsx b/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/BackfillPage.tsx index 516f57ba014d1..5fd0d7f0401f2 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/BackfillPage.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/BackfillPage.tsx @@ -1,18 +1,15 @@ import {gql, useApolloClient, useQuery} from '@apollo/client'; import { - Page, - PageHeader, - Colors, Box, - Tag, - Table, - Spinner, - Dialog, - Button, - DialogFooter, ButtonLink, - NonIdealState, + Colors, Heading, + NonIdealState, + Page, + PageHeader, + Spinner, + Table, + Tag, } from '@dagster-io/ui-components'; import dayjs from 'dayjs'; import duration from 'dayjs/plugin/duration'; @@ -32,20 +29,17 @@ import {assetDetailsPathForKey} from '../../assets/assetDetailsPathForKey'; import {AssetViewParams} from '../../assets/types'; import {AssetKey, BulkActionStatus, RunStatus} from '../../graphql/types'; import {useDocumentTitle} from '../../hooks/useDocumentTitle'; -import {TruncatedTextWithFullTextOnHover} from '../../nav/getLeftNavItemsForOption'; import {RunFilterToken, runsPathWithFilters} from '../../runs/RunsFilterInput'; import {testId} from '../../testing/testId'; -import {VirtualizedItemListForDialog} from '../../ui/VirtualizedItemListForDialog'; -import {numberFormatter} from '../../ui/formatters'; import {BACKFILL_ACTIONS_BACKFILL_FRAGMENT, BackfillActionsMenu} from './BackfillActionsMenu'; import {BackfillStatusTagForPage} from './BackfillStatusTagForPage'; +import {TargetPartitionsDisplay} from './TargetPartitionsDisplay'; import { BackfillPartitionsForAssetKeyQuery, BackfillPartitionsForAssetKeyQueryVariables, BackfillStatusesByAssetQuery, BackfillStatusesByAssetQueryVariables, - PartitionBackfillFragment, } from './types/BackfillPage.types'; dayjs.extend(duration); @@ -200,9 +194,9 @@ export const BackfillPage = () => { } /> @@ -460,102 +454,6 @@ export const BACKFILL_PARTITIONS_FOR_ASSET_KEY_QUERY = gql` } `; -const COLLATOR = new Intl.Collator(navigator.language, {sensitivity: 'base', numeric: true}); - -type AssetBackfillData = Extract< - PartitionBackfillFragment['assetBackfillData'], - {__typename: 'AssetBackfillData'} ->; - -export const PartitionSelection = ({ - numPartitions, - rootTargetedPartitions, -}: { - numPartitions: number; - rootTargetedPartitions?: AssetBackfillData['rootTargetedPartitions']; -}) => { - const [isDialogOpen, setIsDialogOpen] = React.useState(false); - - const {partitionKeys, ranges} = rootTargetedPartitions || {}; - - if (partitionKeys) { - if (partitionKeys.length <= 3) { - return ( - - {partitionKeys.map((p) => ( - {p} - ))} - - ); - } - - return ( - <> - setIsDialogOpen(true)}> - {numberFormatter.format(numPartitions)} partitions - - setIsDialogOpen(false)} - > -
- COLLATOR.compare(a, b))} - renderItem={(assetKey) => ( -
- -
- )} - /> -
- - - - - - ); - } - - if (ranges) { - if (ranges.length === 1) { - const {start, end} = ranges[0]!; - return ( -
- {start}...{end} -
- ); - } - - return ( - <> - setIsDialogOpen(true)}> - {numberFormatter.format(numPartitions)} partitions - - setIsDialogOpen(false)} - > -
- { - return
{`${start}...${end}`}
; - }} - /> -
- - - - - - ); - } - - return
{numPartitions === 1 ? '1 partition' : `${numPartitions} partitions`}
; -}; - const formatDuration = (duration: number) => { const seconds = Math.floor((duration / 1000) % 60); const minutes = Math.floor((duration / (1000 * 60)) % 60); diff --git a/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/TargetPartitionsDisplay.tsx b/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/TargetPartitionsDisplay.tsx new file mode 100644 index 0000000000000..9dd3d783112b4 --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/TargetPartitionsDisplay.tsx @@ -0,0 +1,100 @@ +import {Box, Button, ButtonLink, Dialog, DialogFooter, Tag} from '@dagster-io/ui-components'; +import React from 'react'; + +import {AssetBackfillTargetPartitions} from '../../graphql/types'; +import {TruncatedTextWithFullTextOnHover} from '../../nav/getLeftNavItemsForOption'; +import {VirtualizedItemListForDialog} from '../../ui/VirtualizedItemListForDialog'; +import {numberFormatter} from '../../ui/formatters'; + +const COLLATOR = new Intl.Collator(navigator.language, {sensitivity: 'base', numeric: true}); + +export const TargetPartitionsDisplay = ({ + targetPartitionCount, + targetPartitions, +}: { + targetPartitionCount?: number; + targetPartitions?: Pick; +}) => { + const [isDialogOpen, setIsDialogOpen] = React.useState(false); + + const {partitionKeys, ranges} = targetPartitions || {}; + + if (partitionKeys) { + if (partitionKeys.length <= 3) { + return ( + + {partitionKeys.map((p) => ( + {p} + ))} + + ); + } + + return ( + <> + setIsDialogOpen(true)}> + {numberFormatter.format(partitionKeys.length)} partitions + + setIsDialogOpen(false)} + > +
+ COLLATOR.compare(a, b))} + renderItem={(assetKey) => ( +
+ +
+ )} + /> +
+ + + + + + ); + } + + if (ranges) { + if (ranges.length === 1) { + const {start, end} = ranges[0]!; + return ( +
+ {start}...{end} +
+ ); + } + + return ( + <> + setIsDialogOpen(true)}> + {numberFormatter.format(ranges.length)} ranges + + setIsDialogOpen(false)} + > +
+ { + return
{`${start}...${end}`}
; + }} + /> +
+ + + + + + ); + } + + return ( +
{targetPartitionCount === 1 ? '1 partition' : `${targetPartitionCount} partitions`}
+ ); +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/__tests__/BackfillPage.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/__tests__/BackfillPage.test.tsx index 00251e248c981..3fc411696ba8c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/__tests__/BackfillPage.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/__tests__/BackfillPage.test.tsx @@ -168,80 +168,3 @@ describe('BackfillPage', () => { expect(getAllByText(assetBRow, '-').length).toBe(3); }); }); - -describe('PartitionSelection', () => { - it('renders the targeted partitions when rootAssetTargetedPartitions is provided and length <= 3', async () => { - const {getByText} = render( - , - ); - - expect(getByText('1')).toBeInTheDocument(); - expect(getByText('2')).toBeInTheDocument(); - expect(getByText('3')).toBeInTheDocument(); - }); - - it('renders the targeted partitions in a dialog when rootAssetTargetedPartitions is provided and length > 3', async () => { - const {getByText} = render( - , - ); - - await userEvent.click(getByText('4 partitions')); - - expect(getByText('1')).toBeInTheDocument(); - expect(getByText('2')).toBeInTheDocument(); - expect(getByText('3')).toBeInTheDocument(); - expect(getByText('4')).toBeInTheDocument(); - }); - - it('renders the single targeted range when rootAssetTargetedRanges is provided and length === 1', async () => { - const {getByText} = render( - , - ); - - expect(getByText('1...2')).toBeInTheDocument(); - }); - - it('renders the targeted ranges in a dialog when rootAssetTargetedRanges is provided and length > 1', async () => { - const {getByText} = render( - , - ); - - await userEvent.click(getByText('2 partitions')); - - expect(getByText('1...2')).toBeInTheDocument(); - expect(getByText('3...4')).toBeInTheDocument(); - }); - - it('renders the numPartitions when neither rootAssetTargetedPartitions nor rootAssetTargetedRanges are provided', async () => { - const {getByText} = render(); - - expect(getByText('2 partitions')).toBeInTheDocument(); - }); -}); diff --git a/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/__tests__/TargetPartitionsDisplay.tsx b/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/__tests__/TargetPartitionsDisplay.tsx new file mode 100644 index 0000000000000..28891a293790e --- /dev/null +++ b/js_modules/dagster-ui/packages/ui-core/src/instance/backfill/__tests__/TargetPartitionsDisplay.tsx @@ -0,0 +1,92 @@ +import {render} from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import React from 'react'; + +import {buildAssetBackfillTargetPartitions, buildPartitionKeyRange} from '../../../graphql/types'; +import {TargetPartitionsDisplay} from '../TargetPartitionsDisplay'; + +// This file must be mocked because Jest can't handle `import.meta.url`. +jest.mock('../../../graph/asyncGraphLayout', () => ({})); +jest.mock('../../../app/QueryRefresh', () => { + return { + useQueryRefreshAtInterval: jest.fn(), + QueryRefreshCountdown: jest.fn(() =>
), + }; +}); + +describe('TargetPartitionsDisplay', () => { + it('renders the targeted partitions when rootAssetTargetedPartitions is provided and length <= 3', async () => { + const {getByText} = render( + , + ); + + expect(getByText('1')).toBeInTheDocument(); + expect(getByText('2')).toBeInTheDocument(); + expect(getByText('3')).toBeInTheDocument(); + }); + + it('renders the targeted partitions in a dialog when rootAssetTargetedPartitions is provided and length > 3', async () => { + const {getByText} = render( + , + ); + + await userEvent.click(getByText('4 partitions')); + + expect(getByText('1')).toBeInTheDocument(); + expect(getByText('2')).toBeInTheDocument(); + expect(getByText('3')).toBeInTheDocument(); + expect(getByText('4')).toBeInTheDocument(); + }); + + it('renders the single targeted range when rootAssetTargetedRanges is provided and length === 1', async () => { + const {getByText} = render( + , + ); + + expect(getByText('1...2')).toBeInTheDocument(); + }); + + it('renders the targeted ranges in a dialog when rootAssetTargetedRanges is provided and length > 1', async () => { + const {getByText} = render( + , + ); + + await userEvent.click(getByText('2 partitions')); + + expect(getByText('1...2')).toBeInTheDocument(); + expect(getByText('3...4')).toBeInTheDocument(); + }); + + it('renders the targetPartitionCount when neither rootAssetTargetedPartitions nor rootAssetTargetedRanges are provided', async () => { + const {getByText} = render(); + + expect(getByText('2 partitions')).toBeInTheDocument(); + }); +});