Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Policy evaluations frontend #19030

Merged
merged 58 commits into from
Jan 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
58 commits
Select commit Hold shift + click to select a range
5caf4b0
Update context objects
OwenKephart Dec 8, 2023
4059038
Add snapshot class
OwenKephart Dec 8, 2023
45a3173
RuleEvaluationData -> SubsetWithMetadata
OwenKephart Dec 8, 2023
75cf296
AssetAutomationCondition -> AssetCondition
OwenKephart Dec 8, 2023
92fede3
Serialize AssetConditionEvaluation
OwenKephart Dec 11, 2023
7008767
Simplify Context
OwenKephart Dec 12, 2023
80cbb07
AssetConditionCursor
OwenKephart Dec 13, 2023
e4595f0
move some changes from 8/n
OwenKephart Jan 11, 2024
9f7fe6e
Make AssetDaemonCursor whitelisted for serdes
OwenKephart Dec 15, 2023
4ce2404
Better cursor handling
OwenKephart Dec 15, 2023
f22fd81
Improve backcompat
OwenKephart Dec 20, 2023
ae20d64
Create AssetConditionEvaluationResult object
OwenKephart Dec 19, 2023
cde3a4a
Update AutoMaterializeRul return type
OwenKephart Dec 19, 2023
5a6cee2
Add timing information to evaluation objects
OwenKephart Dec 29, 2023
b1a03c9
Fix non-subsettable multi-asset handling
OwenKephart Jan 3, 2024
d27f79a
Rename AssetConditionCursor
OwenKephart Jan 4, 2024
67eaf9a
Reenable legacy queries
OwenKephart Jan 8, 2024
4098438
PoC: AssetConditionEvaluation GQL
OwenKephart Dec 13, 2023
78a294b
Return tree instead of recursive resolver
OwenKephart Jan 3, 2024
6263bd4
sofar
salazarm Jan 9, 2024
a1f62c4
??
salazarm Jan 9, 2024
46b5fc7
??
salazarm Jan 9, 2024
b70fc22
types and shiz
salazarm Jan 10, 2024
ca85d81
feature gate
salazarm Jan 10, 2024
8fd1acf
ts
salazarm Jan 10, 2024
53249eb
query persisted
salazarm Jan 10, 2024
b154d97
.
salazarm Jan 10, 2024
ae14b28
rm unused things
salazarm Jan 11, 2024
e417046
showMsec
salazarm Jan 16, 2024
6972ea7
...
salazarm Jan 16, 2024
aa96d78
collapsible
salazarm Jan 16, 2024
93254ae
partitions evaluated
salazarm Jan 17, 2024
fbf5dce
cursor
salazarm Jan 17, 2024
b6d3edd
update links
salazarm Jan 17, 2024
9a7c0a1
link
salazarm Jan 17, 2024
a4a084f
stories
salazarm Jan 17, 2024
85ec89d
remove false subset
salazarm Jan 17, 2024
fc54594
feedback
salazarm Jan 17, 2024
28a311a
rm unused SensorSwitchFragment
salazarm Jan 17, 2024
57f6764
add popover
salazarm Jan 17, 2024
8d51913
fix manage sensor link
salazarm Jan 17, 2024
fd48fc2
.
salazarm Jan 17, 2024
1853e2c
padding on alert
salazarm Jan 17, 2024
12149e0
.
salazarm Jan 17, 2024
9cde05d
.
salazarm Jan 17, 2024
7cbae05
reverse gating
salazarm Jan 17, 2024
2320f10
generate types
OwenKephart Jan 17, 2024
a247ddb
.
salazarm Jan 19, 2024
6d60bfd
rebase???
salazarm Jan 19, 2024
6238d1f
rerebase
salazarm Jan 19, 2024
6e30d5b
rerebase
salazarm Jan 19, 2024
8028e0e
rererebase
salazarm Jan 19, 2024
242123f
rm as any
salazarm Jan 19, 2024
1320da1
.
salazarm Jan 19, 2024
b20714b
reverse order of runs and policy evaluations
salazarm Jan 19, 2024
dd8f9d1
Merge branch 'master' of github.com:dagster-io/dagster into salazarm/…
salazarm Jan 19, 2024
d8b002a
storybook
salazarm Jan 19, 2024
8c6ca14
feedback
salazarm Jan 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -98,23 +98,25 @@ export const Suggest = <T,>(props: Props<T>) => {
}

return (
<List
style={{outline: 'none', marginRight: -5, paddingRight: 5}}
rowCount={props.filteredItems.length}
scrollToIndex={
props.activeItem && !isCreateNewItem(props.activeItem)
? props.filteredItems.indexOf(props.activeItem)
: undefined
}
rowHeight={itemHeight}
rowRenderer={(a: any) => (
<div key={a.index} style={a.style}>
{props.renderItem(props.filteredItems[a.index] as T, a.index)}
</div>
)}
width={menuWidth}
height={Math.min(props.filteredItems.length * itemHeight, itemHeight * VISIBLE_ITEMS)}
/>
<div style={{overscrollBehavior: 'contain'}}>
salazarm marked this conversation as resolved.
Show resolved Hide resolved
<List
style={{outline: 'none', marginRight: -5, paddingRight: 5}}
rowCount={props.filteredItems.length}
scrollToIndex={
props.activeItem && !isCreateNewItem(props.activeItem)
? props.filteredItems.indexOf(props.activeItem)
: undefined
}
rowHeight={itemHeight}
rowRenderer={(a: any) => (
<div key={a.index} style={a.style}>
{props.renderItem(props.filteredItems[a.index] as T, a.index)}
</div>
)}
width={menuWidth}
height={Math.min(props.filteredItems.length * itemHeight, itemHeight * VISIBLE_ITEMS)}
/>
</div>
);
}}
popoverProps={allPopoverProps}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ type Props = {
dropdownStyles?: React.CSSProperties;
rowWidth?: number;
rowHeight?: number;
closeOnSelect?: boolean;
usePortal?: boolean;
};

const defaultRenderTag = (tag: string, tagProps: TagSelectorTagProps) => {
Expand All @@ -58,7 +60,7 @@ const defaultRenderTag = (tag: string, tagProps: TagSelectorTagProps) => {
maxWidth: '120px',
}}
data-tooltip={tag}
data-tooltip-style={DefaultTagTooltipStyle}
data-tooltip-style={TagSelectorDefaultTagTooltipStyle}
>
<MiddleTruncate text={tag} />
<Box style={{cursor: 'pointer'}} onClick={tagProps.remove}>
Expand Down Expand Up @@ -101,6 +103,8 @@ export const TagSelector = ({
dropdownStyles,
renderTagList,
rowHeight = MENU_ITEM_HEIGHT,
closeOnSelect,
usePortal,
}: Props) => {
const [isDropdownOpen, setIsDropdownOpen] = React.useState(false);
const {viewport, containerProps} = useViewport();
Expand Down Expand Up @@ -135,6 +139,9 @@ export const TagSelector = ({
setSelectedTags(
selected ? selectedTags.filter((t) => t !== tag) : [...selectedTags, tag],
);
if (closeOnSelect) {
setIsDropdownOpen(false);
}
};
if (renderDropdownItem) {
return <div>{renderDropdownItem(tag, {toggle, selected})}</div>;
Expand All @@ -156,6 +163,7 @@ export const TagSelector = ({
return <Menu style={{width: viewport.width + 'px'}}>{dropdownContent}</Menu>;
}, [
allTags,
closeOnSelect,
dropdownStyles,
items,
renderDropdown,
Expand Down Expand Up @@ -199,27 +207,28 @@ export const TagSelector = ({
}
}
}}
content={<div ref={dropdownContainer}>{dropdown}</div>}
content={<div>{dropdown}</div>}
targetTagName="div"
onOpening={rowVirtualizer.measure}
onOpened={rowVirtualizer.measure}
usePortal={usePortal}
>
<Container
<TagSelectorContainer
onClick={() => {
setIsDropdownOpen((isOpen) => !isOpen);
}}
{...containerProps}
>
<TagsContainer flex={{grow: 1, gap: 6}}>{tagsContent}</TagsContainer>
<TagSelectorTagsContainer flex={{grow: 1, gap: 6}}>{tagsContent}</TagSelectorTagsContainer>
<div style={{cursor: 'pointer'}}>
<Icon name={isDropdownOpen ? 'expand_less' : 'expand_more'} />
</div>
</Container>
</TagSelectorContainer>
</Popover>
);
};

const Container = styled.div`
export const TagSelectorContainer = styled.div`
display: flex;
flex-direction: row;
align-items: center;
Expand All @@ -231,7 +240,7 @@ const Placeholder = styled.div`
color: ${Colors.textDisabled()};
`;

const TagsContainer = styled(Box)`
export const TagSelectorTagsContainer = styled(Box)`
overflow-x: auto;

&::-webkit-scrollbar {
Expand Down Expand Up @@ -297,7 +306,7 @@ export const TagSelectorWithSearch = (
);
};

const DefaultTagTooltipStyle = JSON.stringify({
export const TagSelectorDefaultTagTooltipStyle = JSON.stringify({
background: Colors.backgroundDefault(),
border: `1px solid ${Colors.borderDefault()}`,
color: Colors.textDefault(),
Expand Down
1 change: 1 addition & 0 deletions js_modules/dagster-ui/packages/ui-core/src/app/Flags.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ export const FeatureFlag = {
flagDisableWebsockets: 'flagDisableWebsockets' as const,
flagSidebarResources: 'flagSidebarResources' as const,
flagDisableAutoLoadDefaults: 'flagDisableAutoLoadDefaults' as const,
flagUseNewAutomationPage: 'flagUseNewAutomationPage' as const,
};
export type FeatureFlagType = keyof typeof FeatureFlag;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,8 @@ export const getVisibleFeatureFlagRows = () => [
key: 'Debug console logging',
flagType: FeatureFlag.flagDebugConsoleLogging,
},
{
key: 'Use new asset auto-materialize history page',
flagType: FeatureFlag.flagUseNewAutomationPage,
},
];
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,11 @@ export const AssetEventMetadataEntriesTable = ({
event,
observations,
}: {
event: AssetObservationFragment | AssetMaterializationFragment | null;
event:
| (Pick<AssetObservationFragment | AssetMaterializationFragment, 'metadataEntries'> & {
timestamp?: string | number;
})
| null;
observations?: (AssetObservationFragment | AssetMaterializationFragment)[];
}) => {
if (!event || (!event.metadataEntries.length && !observations?.length)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,8 @@ export const AssetSidebarActivitySummary = ({
padding={{horizontal: 24, vertical: 12}}
flex={{direction: 'row', gap: 4, alignItems: 'center'}}
>
<Link to={assetDetailsPathForKey(asset.assetKey, {view: 'auto-materialize-history'})}>
View auto-materialize history
<Link to={assetDetailsPathForKey(asset.assetKey, {view: 'automation'})}>
View automation history
</Link>
<Icon name="open_in_new" color={Colors.linkDefault()} />
</Box>
Expand Down
10 changes: 5 additions & 5 deletions js_modules/dagster-ui/packages/ui-core/src/assets/AssetTabs.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ export const DEFAULT_ASSET_TAB_ORDER = [
'plots',
'definition',
'lineage',
'auto-materialize-history',
'automation',
];

export type AssetTabConfigInput = {
Expand Down Expand Up @@ -89,10 +89,10 @@ export const buildAssetTabMap = (input: AssetTabConfigInput): Record<string, Ass
to: buildAssetViewParams({...params, view: 'lineage'}),
disabled: !definition,
},
'auto-materialize-history': {
id: 'auto-materialize-history',
title: 'Auto-materialize history',
to: buildAssetViewParams({...params, view: 'auto-materialize-history'}),
automation: {
id: 'automation',
title: 'Automation',
to: buildAssetViewParams({...params, view: 'automation'}),
disabled: !definition,
hidden: !definition?.autoMaterializePolicy,
},
Expand Down
11 changes: 9 additions & 2 deletions js_modules/dagster-ui/packages/ui-core/src/assets/AssetView.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {AssetPartitions} from './AssetPartitions';
import {AssetPlots} from './AssetPlots';
import {AssetTabs} from './AssetTabs';
import {AssetAutomaterializePolicyPage} from './AutoMaterializePolicyPage/AssetAutomaterializePolicyPage';
import {AssetAutomaterializePolicyPageOld} from './AutoMaterializePolicyPageOld/AssetAutomaterializePolicyPage';
import {AutomaterializeDaemonStatusTag} from './AutomaterializeDaemonStatusTag';
import {useAutomationPolicySensorFlag} from './AutomationPolicySensorFlag';
import {LaunchAssetExecutionButton} from './LaunchAssetExecutionButton';
Expand All @@ -28,6 +29,7 @@ import {
} from './types/AssetView.types';
import {healthRefreshHintFromLiveData} from './usePartitionHealthData';
import {useReportEventsModal} from './useReportEventsModal';
import {useFeatureFlags} from '../app/Flags';
import {Timestamp} from '../app/time/Timestamp';
import {AssetLiveDataRefresh, useAssetLiveData} from '../asset-data/AssetLiveDataProvider';
import {
Expand Down Expand Up @@ -178,12 +180,17 @@ export const AssetView = ({assetKey, trace}: Props) => {
);
};

const {flagUseNewAutomationPage} = useFeatureFlags();

const renderAutomaterializeHistoryTab = () => {
if (definitionQueryResult.loading && !definitionQueryResult.previousData) {
return <AssetLoadingDefinitionState />;
}
if (flagUseNewAutomationPage) {
return <AssetAutomaterializePolicyPage assetKey={assetKey} definition={definition} />;
}
return (
<AssetAutomaterializePolicyPage
<AssetAutomaterializePolicyPageOld
assetKey={assetKey}
assetHasDefinedPartitions={!!definition?.partitionDefinition}
/>
Expand Down Expand Up @@ -214,7 +221,7 @@ export const AssetView = ({assetKey, trace}: Props) => {
return renderEventsTab();
case 'plots':
return renderPlotsTab();
case 'auto-materialize-history':
case 'automation':
return renderAutomaterializeHistoryTab();
case 'checks':
return renderChecksTab();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,42 +1,39 @@
import {Box, Colors, Subheading} from '@dagster-io/ui-components';
import {Box, Colors, Spinner} from '@dagster-io/ui-components';
import * as React from 'react';
import {useMemo} from 'react';
import styled from 'styled-components';

import {AutoMaterializeExperimentalBanner} from './AutoMaterializeExperimentalBanner';
import {AutomaterializeLeftPanel} from './AutomaterializeLeftPanel';
import {AutomaterializeMiddlePanel} from './AutomaterializeMiddlePanel';
import {AutomaterializeRightPanel} from './AutomaterializeRightPanel';
import {useEvaluationsQueryResult} from './useEvaluationsQueryResult';
import {FIFTEEN_SECONDS, useQueryRefreshAtInterval} from '../../app/QueryRefresh';
import {useQueryPersistedState} from '../../hooks/useQueryPersistedState';
import {AssetKey} from '../types';
import {AssetViewDefinitionNodeFragment} from '../types/AssetView.types';

export const AssetAutomaterializePolicyPage = ({
assetKey,
assetHasDefinedPartitions,
definition,
}: {
assetKey: AssetKey;
assetHasDefinedPartitions: boolean;
definition?: AssetViewDefinitionNodeFragment | null;
}) => {
const {queryResult, paginationProps} = useEvaluationsQueryResult({assetKey});

useQueryRefreshAtInterval(queryResult, FIFTEEN_SECONDS);

const {evaluations} = useMemo(() => {
const evaluations = useMemo(() => {
if (
queryResult.data?.autoMaterializeAssetEvaluationsOrError?.__typename ===
'AutoMaterializeAssetEvaluationRecords' &&
queryResult.data?.assetConditionEvaluationRecordsOrError?.__typename ===
'AssetConditionEvaluationRecords' &&
queryResult.data?.assetNodeOrError?.__typename === 'AssetNode'
) {
return {
evaluations: queryResult.data?.autoMaterializeAssetEvaluationsOrError.records,
currentAutoMaterializeEvaluationId:
queryResult.data.assetNodeOrError.currentAutoMaterializeEvaluationId,
};
return queryResult.data?.assetConditionEvaluationRecordsOrError.records;
}
return {evaluations: [], currentAutoMaterializeEvaluationId: null};
return [];
}, [
queryResult.data?.autoMaterializeAssetEvaluationsOrError,
queryResult.data?.assetConditionEvaluationRecordsOrError,
queryResult.data?.assetNodeOrError,
]);

Expand All @@ -63,49 +60,48 @@ export const AssetAutomaterializePolicyPage = ({
return evaluations.find((evaluation) => evaluation.evaluationId === selectedEvaluationId);
}, [selectedEvaluationId, isFirstPage, evaluations]);

if (!queryResult.data && queryResult.loading) {
return (
<Box
style={{height: 390}}
flex={{direction: 'row', justifyContent: 'center', alignItems: 'center'}}
>
<Spinner purpose="page" />
</Box>
);
}

return (
<AutomaterializePage
style={{flex: 1, minHeight: 0, color: Colors.textLight(), overflow: 'hidden'}}
flex={{direction: 'column'}}
>
<Box padding={{horizontal: 24, vertical: 12}} border="bottom">
<AutoMaterializeExperimentalBanner />
</Box>
<AutoMaterializeExperimentalBanner />
<Box flex={{direction: 'row'}} style={{minHeight: 0, flex: 1}}>
<Box flex={{direction: 'column', grow: 1}}>
<Box
flex={{alignItems: 'center'}}
padding={{vertical: 16, horizontal: 24}}
border="bottom"
>
<Subheading>Evaluation history</Subheading>
<Box flex={{direction: 'row'}} style={{flex: 1, minHeight: 0}}>
<Box border="right" flex={{grow: 0, direction: 'column'}} style={{flex: '0 0 296px'}}>
<AutomaterializeLeftPanel
definition={definition}
evaluations={evaluations}
paginationProps={paginationProps}
onSelectEvaluation={(evaluation) => {
setSelectedEvaluationId(evaluation.evaluationId);
}}
selectedEvaluation={selectedEvaluation}
/>
</Box>
<Box flex={{direction: 'row'}} style={{flex: 1, minHeight: 0}}>
<Box border="right" flex={{grow: 0, direction: 'column'}} style={{flex: '0 0 296px'}}>
<AutomaterializeLeftPanel
assetHasDefinedPartitions={assetHasDefinedPartitions}
evaluations={evaluations}
paginationProps={paginationProps}
onSelectEvaluation={(evaluation) => {
setSelectedEvaluationId(evaluation.evaluationId);
}}
selectedEvaluation={selectedEvaluation}
/>
</Box>
<Box flex={{grow: 1}} style={{minHeight: 0, overflowY: 'auto'}}>
<AutomaterializeMiddlePanel
assetKey={assetKey}
assetHasDefinedPartitions={assetHasDefinedPartitions}
// Use the evaluation ID of the current evaluation object, if any. Otherwise
// fall back to the evaluation ID from the query parameter, if any.
selectedEvaluationId={selectedEvaluation?.evaluationId || selectedEvaluationId}
/>
</Box>
<Box flex={{grow: 1}} style={{minHeight: 0, overflowY: 'auto'}}>
<AutomaterializeMiddlePanel
key={selectedEvaluation?.evaluationId || selectedEvaluationId}
assetKey={assetKey}
// Use the evaluation ID of the current evaluation object, if any. Otherwise
// fall back to the evaluation ID from the query parameter, if any.
selectedEvaluationId={selectedEvaluation?.evaluationId || selectedEvaluationId}
selectedEvaluation={selectedEvaluation}
definition={definition}
/>
</Box>
</Box>
<Box border="left">
<AutomaterializeRightPanel assetKey={assetKey} />
</Box>
</Box>
</AutomaterializePage>
);
Expand Down
Loading