Skip to content

Commit

Permalink
[ui] Evaluations table: Don't load all partition keys up front (#23616)
Browse files Browse the repository at this point in the history 
## Summary & Motivation

Defer fetching partition keys affected by an automation policy evaluation.

For assets with tons of partitions, this page is slowed significantly by retrieving the parititon subsets up front. Instead, retrieve them when actually hovering over the evaluation data for the relevant row.

## How I Tested These Changes

Load a repo with an asset with 200k partitions, then perform some automation evaluations. View an evaluation result, and hover over the "200,000 True" tag. Verify loading state for the partition key query, and correct rendering of partition list.
  • Loading branch information
@hellendag
hellendag authored Aug 13, 2024
1 parent 1f67aed commit e082039
Show file tree
Hide file tree
Showing 14 changed files with 467 additions and 605 deletions.
26 changes: 15 additions & 11 deletions ...kages/ui-core/src/assets/AutoMaterializePolicyPage/AutomaterializeMiddlePanelWithData.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import styled from 'styled-components';

import {StatusDot} from './AutomaterializeLeftPanel';
import {AutomaterializeRunsTable} from './AutomaterializeRunsTable';
import {PartitionSubsetList} from './PartitionSegmentWithPopover';
import {PartitionSubsetList} from './PartitionSubsetList';
import {PolicyEvaluationTable} from './PolicyEvaluationTable';
import {
FullPartitionsQuery,
Expand Down Expand Up @@ -57,17 +57,20 @@ export const AutomaterializeMiddlePanelWithData = ({
() => evaluation?.evaluationNodes.find((node) => node.uniqueId === evaluation.rootUniqueId),
[evaluation],
);
const partitionDefinition = definition?.partitionDefinition;
const numRequested = selectedEvaluation?.numRequested;
const rootUniqueId = evaluation?.rootUniqueId;

const statusTag = useMemo(() => {
const trueSubset =
const partitionDefinition = definition?.partitionDefinition;
const assetKeyPath = definition?.assetKey.path || [];
const numRequested = selectedEvaluation?.numRequested;

const numTrue =
rootEvaluationNode?.__typename === 'PartitionedAssetConditionEvaluationNode'
? rootEvaluationNode.trueSubset
? rootEvaluationNode.numTrue
: null;

if (numRequested) {
if (partitionDefinition && trueSubset) {
if (partitionDefinition && rootUniqueId && numTrue) {
return (
<Box flex={{direction: 'row', gap: 4, alignItems: 'center'}}>
<Popover
Expand All @@ -78,7 +81,9 @@ export const AutomaterializeMiddlePanelWithData = ({
content={
<PartitionSubsetList
description="Requested assets"
subset={trueSubset}
assetKeyPath={assetKeyPath}
evaluationId={selectedEvaluation.evaluationId}
nodeUniqueId={rootUniqueId}
selectPartition={selectPartition}
/>
}
Expand All @@ -90,9 +95,6 @@ export const AutomaterializeMiddlePanelWithData = ({
</Box>
</Tag>
</Popover>
{numRequested === 1 ? (
<Tag icon="partition">{trueSubset.subsetValue.partitionKeys![0]}</Tag>
) : null}
</Box>
);
}
Expand All @@ -115,7 +117,7 @@ export const AutomaterializeMiddlePanelWithData = ({
</Box>
</Tag>
);
}, [partitionDefinition, selectPartition, numRequested, rootEvaluationNode]);
}, [definition, rootEvaluationNode, selectedEvaluation, rootUniqueId, selectPartition]);

const fullPartitionsQueryResult = useQuery<FullPartitionsQuery, FullPartitionsQueryVariables>(
FULL_PARTITIONS_QUERY,
Expand Down Expand Up @@ -256,6 +258,8 @@ export const AutomaterializeMiddlePanelWithData = ({
</Box>
) : null}
<PolicyEvaluationTable
assetKeyPath={definition?.assetKey.path ?? null}
evaluationId={selectedEvaluation.evaluationId}
evaluationNodes={
!selectedEvaluation.isLegacy
? selectedEvaluation.evaluationNodes
Expand Down
33 changes: 3 additions & 30 deletions .../dagster-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/GetEvaluationsQuery.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,6 @@ import {gql} from '@apollo/client';

import {METADATA_ENTRY_FRAGMENT} from '../../metadata/MetadataEntryFragment';

const AssetSubsetFragment = gql`
fragment AssetSubsetFragment on AssetSubset {
subsetValue {
isPartitioned
partitionKeys
partitionKeyRanges {
start
end
}
}
}
`;

const SpecificPartitionAssetConditionEvaluationNodeFragment = gql`
fragment SpecificPartitionAssetConditionEvaluationNodeFragment on SpecificPartitionAssetConditionEvaluationNode {
description
Expand Down Expand Up @@ -48,18 +35,11 @@ const PartitionedAssetConditionEvaluationNodeFragment = gql`
startTimestamp
endTimestamp
numTrue
numFalse
numSkipped
trueSubset {
...AssetSubsetFragment
}
candidateSubset {
...AssetSubsetFragment
}
uniqueId
childUniqueIds
numTrue
numCandidates
}
${AssetSubsetFragment}
`;

const NEW_EVALUATION_NODE_FRAGMENT = gql`
Expand All @@ -69,18 +49,11 @@ const NEW_EVALUATION_NODE_FRAGMENT = gql`
userLabel
startTimestamp
endTimestamp
numCandidates
numTrue
isPartitioned
childUniqueIds
trueSubset {
...AssetSubsetFragment
}
candidateSubset {
...AssetSubsetFragment
}
}
${AssetSubsetFragment}
`;

const AssetConditionEvaluationRecordFragment = gql`
Expand Down
185 changes: 21 additions & 164 deletions ...-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/PartitionSegmentWithPopover.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,61 +1,33 @@
import {
Box,
Colors,
Menu,
MenuItem,
MiddleTruncate,
Popover,
Tag,
TextInput,
TextInputContainer,
} from '@dagster-io/ui-components';
import {useVirtualizer} from '@tanstack/react-virtual';
import {useMemo, useRef, useState} from 'react';
import styled from 'styled-components';
import {Popover, Tag} from '@dagster-io/ui-components';

import {PolicyEvaluationStatusTag} from './PolicyEvaluationStatusTag';
import {assertUnreachable} from '../../app/Util';
import {AssetConditionEvaluationStatus, AssetSubsetValue} from '../../graphql/types';
import {Container, Inner, Row} from '../../ui/VirtualizedTable';
import {PartitionSubsetList} from './PartitionSubsetList';
import {AssetConditionEvaluationStatus} from '../../graphql/types';
import {numberFormatter} from '../../ui/formatters';

const statusToColors = (status: AssetConditionEvaluationStatus) => {
switch (status) {
case AssetConditionEvaluationStatus.TRUE:
return {color: Colors.accentGreen(), hoverColor: Colors.accentGreenHover()};
case AssetConditionEvaluationStatus.FALSE:
return {color: Colors.accentYellow(), hoverColor: Colors.accentYellowHover()};
case AssetConditionEvaluationStatus.SKIPPED:
return {color: Colors.accentGray(), hoverColor: Colors.accentGrayHover()};
default:
return assertUnreachable(status);
}
};

type AssetSusbsetWithoutTypenames = {
subsetValue: Omit<AssetSubsetValue, '__typename' | 'boolValue'>;
};

interface Props {
description: string;
subset: AssetSusbsetWithoutTypenames | null;
numTrue: number;
assetKeyPath: string[];
evaluationId: number;
nodeUniqueId: string;
selectPartition?: (partitionKey: string | null) => void;
}

export const PartitionSegmentWithPopover = ({description, selectPartition, subset}: Props) => {
if (!subset) {
return null;
}

const count = subset.subsetValue.partitionKeys?.length || 0;

export const PartitionSegmentWithPopover = ({
description,
selectPartition,
assetKeyPath,
evaluationId,
nodeUniqueId,
numTrue,
}: Props) => {
const tag = (
<Tag intent={count > 0 ? 'success' : 'none'} icon={count > 0 ? 'check_circle' : undefined}>
{numberFormatter.format(count)} True
<Tag intent={numTrue > 0 ? 'success' : 'none'} icon={numTrue > 0 ? 'check_circle' : undefined}>
{numberFormatter.format(numTrue)} True
</Tag>
);

if (count === 0) {
if (numTrue === 0) {
return tag;
}

Expand All @@ -69,7 +41,9 @@ export const PartitionSegmentWithPopover = ({description, selectPartition, subse
<PartitionSubsetList
description={description}
status={AssetConditionEvaluationStatus.TRUE}
subset={subset}
assetKeyPath={assetKeyPath}
evaluationId={evaluationId}
nodeUniqueId={nodeUniqueId}
selectPartition={selectPartition}
/>
}
Expand All @@ -78,120 +52,3 @@ export const PartitionSegmentWithPopover = ({description, selectPartition, subse
</Popover>
);
};

interface ListProps {
description: string;
status?: AssetConditionEvaluationStatus;
subset: AssetSusbsetWithoutTypenames;
selectPartition?: (partitionKey: string | null) => void;
}

const ITEM_HEIGHT = 32;
const MAX_ITEMS_BEFORE_TRUNCATION = 4;

export const PartitionSubsetList = ({description, status, subset, selectPartition}: ListProps) => {
const container = useRef<HTMLDivElement | null>(null);
const [searchValue, setSearchValue] = useState('');

const {color, hoverColor} = useMemo(
() => statusToColors(status ?? AssetConditionEvaluationStatus.TRUE),
[status],
);

const partitionKeys = useMemo(() => subset.subsetValue.partitionKeys || [], [subset]);

const filteredKeys = useMemo(() => {
const searchLower = searchValue.toLocaleLowerCase();
return partitionKeys.filter((key) => key.toLocaleLowerCase().includes(searchLower));
}, [partitionKeys, searchValue]);

const count = filteredKeys.length;

const rowVirtualizer = useVirtualizer({
count: filteredKeys.length,
getScrollElement: () => container.current,
estimateSize: () => ITEM_HEIGHT,
overscan: 10,
});

const totalHeight = rowVirtualizer.getTotalSize();
const virtualItems = rowVirtualizer.getVirtualItems();

return (
<div style={{width: '292px'}}>
<Box
padding={{vertical: 8, left: 12, right: 8}}
border="bottom"
flex={{direction: 'row', alignItems: 'center', justifyContent: 'space-between'}}
style={{display: 'grid', gridTemplateColumns: 'minmax(0, 1fr) auto', gap: 8}}
>
<strong>
<MiddleTruncate text={description} />
</strong>
{status ? <PolicyEvaluationStatusTag status={status} /> : null}
</Box>
{partitionKeys.length > MAX_ITEMS_BEFORE_TRUNCATION ? (
<SearchContainer padding={{vertical: 4, horizontal: 8}}>
<TextInput
icon="search"
placeholder="Filter partitions…"
value={searchValue}
onChange={(e) => setSearchValue(e.target.value)}
/>
</SearchContainer>
) : null}
<div
style={{
height: count > MAX_ITEMS_BEFORE_TRUNCATION ? '150px' : count * ITEM_HEIGHT + 16,
overflow: 'hidden',
}}
>
<Container ref={container}>
<Menu>
<Inner $totalHeight={totalHeight}>
{virtualItems.map(({index, key, size, start}) => {
const partitionKey = filteredKeys[index]!;
return (
<Row $height={size} $start={start} key={key}>
<MenuItem
onClick={() => {
selectPartition && selectPartition(partitionKey);
}}
text={
<Box flex={{direction: 'row', alignItems: 'center', gap: 8}}>
<PartitionStatusDot $color={color} $hoverColor={hoverColor} />
<div>
<MiddleTruncate text={partitionKey} />
</div>
</Box>
}
/>
</Row>
);
})}
</Inner>
</Menu>
</Container>
</div>
</div>
);
};

const SearchContainer = styled(Box)`
display: flex;
${TextInputContainer} {
flex: 1;
}
`;

const PartitionStatusDot = styled.div<{$color: string; $hoverColor: string}>`
background-color: ${({$color}) => $color};
height: 8px;
width: 8px;
border-radius: 50%;
transition: background-color 100ms linear;
:hover {
background-color: ${({$hoverColor}) => $hoverColor};
}
`;
Loading

1 comment on commit e082039

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deploy preview for dagit-core-storybook ready!

✅ Preview
https://dagit-core-storybook-3d2vnapal-elementl.vercel.app

Built with commit e082039.
This pull request is being automatically deployed with vercel-action

Please sign in to comment.