Skip to content

Commit

Permalink
Option to execute individual asset checks when launching runs via Lau…
Browse files Browse the repository at this point in the history
…nchpad
  • Loading branch information
bengotow committed Oct 9, 2023
1 parent 4b1ae14 commit d5aab3b
Show file tree
Hide file tree
Showing 12 changed files with 242 additions and 104 deletions.
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import memoize from 'lodash/memoize';
import * as React from 'react';

import {AssetKeyInput} from '../graphql/types';
import {AssetKeyInput, AssetCheck} from '../graphql/types';
import {useSetStateUpdateCallback} from '../hooks/useSetStateUpdateCallback';
import {getJSONForKey, useStateWithStorage} from '../hooks/useStateWithStorage';
import {
Expand Down Expand Up @@ -43,6 +43,8 @@ export interface IExecutionSession {
mode: string | null;
needsRefresh: boolean;
assetSelection: {assetKey: AssetKeyInput; opNames: string[]}[] | null;
assetChecksAvailable: Pick<AssetCheck, 'name' | 'canExecuteIndividually' | 'assetKey'>[];
includeSeparatelyExecutableChecks: boolean;
solidSelection: string[] | null;
solidSelectionQuery: string | null;
flattenGraphs: boolean;
Expand Down Expand Up @@ -95,6 +97,8 @@ export const createSingleSession = (initial: IExecutionSessionChanges = {}, key?
base: null,
needsRefresh: false,
assetSelection: null,
assetChecksAvailable: [],
includeSeparatelyExecutableChecks: true,
solidSelection: null,
solidSelectionQuery: '*',
flattenGraphs: false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,8 @@ describe('ExecutionSessionStorage', () => {
mode: 'default',
needsRefresh: false,
assetSelection: null,
assetChecksAvailable: [],
includeSeparatelyExecutableChecks: true,
solidSelection: null,
solidSelectionQuery: '*',
flattenGraphs: false,
Expand Down Expand Up @@ -271,6 +273,8 @@ describe('ExecutionSessionStorage', () => {
mode: 'default',
needsRefresh: false,
assetSelection: null,
assetChecksAvailable: [],
includeSeparatelyExecutableChecks: true,
solidSelection: null,
solidSelectionQuery: '*',
flattenGraphs: false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import * as React from 'react';
import {PYTHON_ERROR_FRAGMENT} from '../app/PythonErrorFragment';
import {displayNameForAssetKey} from '../asset-graph/Utils';

import {asAssetKeyInput} from './asInput';
import {AssetWipeMutation, AssetWipeMutationVariables} from './types/AssetWipeDialog.types';

interface AssetKey {
Expand All @@ -21,7 +22,7 @@ export const AssetWipeDialog: React.FC<{
const [requestWipe] = useMutation<AssetWipeMutation, AssetWipeMutationVariables>(
ASSET_WIPE_MUTATION,
{
variables: {assetKeys: assetKeys.map((key) => ({path: key.path || []}))},
variables: {assetKeys: assetKeys.map(asAssetKeyInput)},
refetchQueries: requery,
},
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import {
itemWithAssetKey,
} from '../asset-graph/Utils';
import {AssetKey} from '../assets/types';
import {LaunchBackfillParams, PartitionDefinitionType} from '../graphql/types';
import {AssetCheck, LaunchBackfillParams, PartitionDefinitionType} from '../graphql/types';
import {LAUNCH_PARTITION_BACKFILL_MUTATION} from '../instance/backfill/BackfillUtils';
import {
LaunchPartitionBackfillMutation,
Expand Down Expand Up @@ -72,6 +72,7 @@ import {
} from './MultipartitioningSupport';
import {PartitionHealthSummary} from './PartitionHealthSummary';
import {RunningBackfillsNotice} from './RunningBackfillsNotice';
import {asAssetKeyInput} from './asInput';
import {
LaunchAssetWarningsQuery,
LaunchAssetWarningsQueryVariables,
Expand All @@ -93,6 +94,7 @@ interface Props {
target: LaunchAssetsChoosePartitionsTarget;
assets: {
assetKey: AssetKey;
assetChecks: Pick<AssetCheck, 'name'>[];
opNames: string[];
partitionDefinition: PartitionDefinitionForLaunchAssetFragment | null;
}[];
Expand Down Expand Up @@ -348,7 +350,7 @@ const LaunchAssetChoosePartitionsDialogBody: React.FC<Props> = ({
target.type === 'job' && !isHiddenAssetGroupJob(target.jobName)
? {
tags,
assetSelection: assets.map((a) => ({path: a.assetKey.path})),
assetSelection: assets.map(asAssetKeyInput),
partitionNames: keysFiltered,
fromFailure: false,
selector: {
Expand All @@ -362,12 +364,12 @@ const LaunchAssetChoosePartitionsDialogBody: React.FC<Props> = ({
: target.type === 'pureAll'
? {
tags,
assetSelection: assets.map((a) => ({path: a.assetKey.path})),
assetSelection: assets.map(asAssetKeyInput),
allPartitions: true,
}
: {
tags,
assetSelection: assets.map((a) => ({path: a.assetKey.path})),
assetSelection: assets.map(asAssetKeyInput),
partitionNames: keysFiltered,
fromFailure: false,
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import {MULTIPLE_DEFINITIONS_WARNING} from './AssetDefinedInMultipleReposNotice'
import {LaunchAssetChoosePartitionsDialog} from './LaunchAssetChoosePartitionsDialog';
import {partitionDefinitionsEqual} from './MultipartitioningSupport';
import {isAssetMissing, isAssetStale} from './Stale';
import {asAssetKeyInput, getAssetCheckHandleInputs} from './asInput';
import {AssetKey} from './types';
import {
LaunchAssetExecutionAssetNodeFragment,
Expand Down Expand Up @@ -289,7 +290,7 @@ export const useMaterializationAction = (preferredJobName?: string) => {

const result = await client.query<LaunchAssetLoaderQuery, LaunchAssetLoaderQueryVariables>({
query: LAUNCH_ASSET_LOADER_QUERY,
variables: {assetKeys: assetKeys.map(({path}) => ({path}))},
variables: {assetKeys: assetKeys.map(asAssetKeyInput)},
});

if (result.data.assetNodeDefinitionCollisions.length) {
Expand Down Expand Up @@ -373,7 +374,7 @@ export const useMaterializationAction = (preferredJobName?: string) => {
LaunchAssetLoaderQueryVariables
>({
query: LAUNCH_ASSET_LOADER_QUERY,
variables: {assetKeys: state.assets.map(({assetKey}) => ({path: assetKey.path}))},
variables: {assetKeys: state.assets.map(asAssetKeyInput)},
});
const assets = result.data.assetNodes;
const next = await stateForLaunchingAssets(client, assets, false, preferredJobName);
Expand Down Expand Up @@ -502,6 +503,10 @@ async function stateForLaunchingAssets(
sessionPresets: {
flattenGraphs: true,
assetSelection: assets.map((a) => ({assetKey: a.assetKey, opNames: a.opNames})),
assetChecksAvailable: assets.flatMap((a) =>
a.assetChecks.map((check) => ({...check, assetKey: a.assetKey})),
),
includeSeparatelyExecutableChecks: true,
solidSelectionQuery: assetOpNames.map((name) => `"${name}"`).join(', '),
base: partitionSetName
? {partitionsSetName: partitionSetName, partitionName: null, tags: []}
Expand Down Expand Up @@ -600,7 +605,7 @@ async function upstreamAssetsWithNoMaterializations(
export function executionParamsForAssetJob(
repoAddress: RepoAddress,
jobName: string,
assets: {assetKey: AssetKey; opNames: string[]}[],
assets: {assetKey: AssetKey; opNames: string[]; assetChecks: {name: string}[]}[],
tags: {key: string; value: string}[],
): LaunchPipelineExecutionMutationVariables['executionParams'] {
return {
Expand All @@ -613,9 +618,8 @@ export function executionParamsForAssetJob(
repositoryLocationName: repoAddress.location,
repositoryName: repoAddress.name,
pipelineName: jobName,
assetSelection: assets.map((asset) => ({
path: asset.assetKey.path,
})),
assetSelection: assets.map(asAssetKeyInput),
assetCheckSelection: getAssetCheckHandleInputs(assets),
},
};
}
Expand Down Expand Up @@ -674,6 +678,10 @@ const LAUNCH_ASSET_EXECUTION_ASSET_NODE_FRAGMENT = gql`
assetKey {
path
}
assetChecks {
name
canExecuteIndividually
}
dependencyKeys {
path
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
getCommonJob,
LAUNCH_ASSET_LOADER_QUERY,
} from './LaunchAssetExecutionButton';
import {asAssetKeyInput} from './asInput';
import {
LaunchAssetExecutionAssetNodeFragment,
LaunchAssetLoaderQuery,
Expand Down Expand Up @@ -73,7 +74,7 @@ export const LaunchAssetObservationButton: React.FC<{

const result = await client.query<LaunchAssetLoaderQuery, LaunchAssetLoaderQueryVariables>({
query: LAUNCH_ASSET_LOADER_QUERY,
variables: {assetKeys: scopeAssets.map((a) => ({path: a.assetKey.path}))},
variables: {assetKeys: scopeAssets.map(asAssetKeyInput)},
});

if (result.data.assetNodeDefinitionCollisions.length) {
Expand Down
24 changes: 24 additions & 0 deletions js_modules/dagster-ui/packages/ui-core/src/assets/asInput.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import {AssetCheckHandleInput} from '../graphql/types';

import {AssetKey} from './types';

export function getAssetCheckHandleInputs(
assets: {assetKey: AssetKey; assetChecks: {name: string}[]}[],
): AssetCheckHandleInput[] {
return assets.flatMap((a) =>
a.assetChecks.map((check) => ({name: check.name, assetKey: {path: a.assetKey.path}})),
);
}

// The `.map` calls below sanitize the __typename and other possible fields out of the
// assetSelection / assetCheckSelection because GraphQL is strict about extra values.

export function asAssetKeyInput(assetOrAssetKey: {assetKey: {path: string[]}} | {path: string[]}) {
return 'path' in assetOrAssetKey
? {path: assetOrAssetKey.path}
: {path: assetOrAssetKey.assetKey.path};
}

export function asAssetCheckHandleInput(check: {name: string; assetKey: {path: string[]}}) {
return {name: check.name, assetKey: {path: check.assetKey.path}};
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {Button, Icon, Spinner, Tooltip} from '@dagster-io/ui-components';
import React, {useState} from 'react';

import {usePermissionsForLocation} from '../../app/Permissions';
import {AssetCheckCanExecuteIndividually} from '../../graphql/types';
import {AssetCheckCanExecuteIndividually, ExecutionParams} from '../../graphql/types';
import {useLaunchPadHooks} from '../../launchpad/LaunchpadHooksContext';

import {
Expand Down Expand Up @@ -68,24 +68,23 @@ export const ExecuteChecksButton = ({
}

const onClick = async () => {
const params = {
executionParams: {
mode: 'default',
executionMetadata: {},
runConfigData: '{}',
selector: {
repositoryLocationName: repository.location.name,
repositoryName: repository.name,
jobName,
assetCheckSelection: launchable.map((c) => ({
assetKey: {path: assetKey.path},
name: c.name,
})),
},
const executionParams: ExecutionParams = {
mode: 'default',
executionMetadata: {},
runConfigData: '{}',
selector: {
jobName,
repositoryLocationName: repository.location.name,
repositoryName: repository.name,
assetSelection: [],
assetCheckSelection: launchable.map((c) => ({
assetKey: {path: assetKey.path},
name: c.name,
})),
},
};
setLaunching(true);
await launchWithTelemetry(params, 'toast');
await launchWithTelemetry({executionParams}, 'toast');
setLaunching(false);
};

Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit d5aab3b

Please sign in to comment.