Skip to content

Commit

Permalink
[2.x] Add log pattern table (opensearch-project#1187) (opensearch-pro…
Browse files Browse the repository at this point in the history 
…ject#1212)

Signed-off-by: Joshua Li <[email protected]>
  • Loading branch information
@joshuali925
joshuali925 authored Nov 1, 2022
1 parent 0b412e0 commit 8a652dd
Show file tree
Hide file tree
Showing 16 changed files with 2,131 additions and 50 deletions.
3 changes: 3 additions & 0 deletions common/constants/explorer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ export const RAW_QUERY = 'rawQuery';
export const FINAL_QUERY = 'finalQuery';
export const SELECTED_DATE_RANGE = 'selectedDateRange';
export const INDEX = 'index';
export const SELECTED_PATTERN = 'selectedPattern';
export const SELECTED_TIMESTAMP = 'selectedTimestamp';
export const SELECTED_FIELDS = 'selectedFields';
export const UNSELECTED_FIELDS = 'unselectedFields';
Expand Down Expand Up @@ -74,6 +75,8 @@ export const REDUX_EXPL_SLICE_FIELDS = 'fields';
export const REDUX_EXPL_SLICE_QUERY_TABS = 'queryTabs';
export const REDUX_EXPL_SLICE_VISUALIZATION = 'explorerVisualization';
export const REDUX_EXPL_SLICE_COUNT_DISTRIBUTION = 'countDistributionVisualization';
export const REDUX_EXPL_SLICE_PATTERNS = 'patterns';
export const PLOTLY_GAUGE_COLUMN_NUMBER = 5;
export const APP_ANALYTICS_TAB_ID_REGEX = /application-analytics-tab.+/;
export const DEFAULT_AVAILABILITY_QUERY = 'stats count() by span( timestamp, 1h )';
export const PPL_PATTERNS_REGEX = /\|\s*patterns\s+\S+\s*\|\s*where\s+patterns_field\s*\=\s*'[^a-zA-Z0-9]+'/;
12 changes: 9 additions & 3 deletions common/types/explorer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,8 @@ export interface SavedQuery {
name: string;
query: string;
selected_date_range: { start: string; end: string; text: string };
selected_fields: { text: string; tokens: [{ name: string; type: string }] };
selected_timestamp: { name: string; type: string };
selected_fields: { text: string; tokens: IField[] };
selected_timestamp: IField;
}

export interface SavedVisualization {
Expand All @@ -129,7 +129,7 @@ export interface SavedVisualization {
query: string;
selected_date_range: { start: string; end: string; text: string };
selected_fields: { text: string; tokens: [] };
selected_timestamp: { name: string; type: string };
selected_timestamp: IField;
type: string;
application_id?: string;
}
Expand Down Expand Up @@ -227,3 +227,9 @@ export interface LiveTailProps {
isLiveTailPopoverOpen: boolean;
dataTestSubj: string;
}

export interface PatternTableData {
count: number;
pattern: string;
sampleLog: string;
}
9 changes: 8 additions & 1 deletion public/components/application_analytics/helpers/utils.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { FilterType } from 'public/components/trace_analytics/components/common/
import React, { Dispatch, ReactChild } from 'react';
import { batch } from 'react-redux';
import PPLService from 'public/services/requests/ppl';
import { IField } from '../../../../common/types/explorer';
import { preprocessQuery } from '../../../../common/utils/query_utils';
import { SPAN_REGEX } from '../../../../common/constants/shared';
import { fetchVisualizationById } from '../../../components/custom_panels/helpers/utils';
Expand All @@ -36,6 +37,10 @@ import {
remove as removeQueryResult,
} from '../../event_analytics/redux/slices/query_result_slice';
import { addTab, removeTab } from '../../event_analytics/redux/slices/query_tab_slice';
import {
init as initPatterns,
remove as removePatterns,
} from '../../event_analytics/redux/slices/patterns_slice';

// Name validation
export const isNameValid = (name: string, existingNames: string[]) => {
Expand Down Expand Up @@ -153,6 +158,7 @@ export const removeTabData = (
[NEW_SELECTED_QUERY_TAB]: newIdToFocus,
})
);
dispatch(removePatterns({ tabId: TabIdToBeClosed }));
});
};

Expand All @@ -172,6 +178,7 @@ export const initializeTabData = async (dispatch: Dispatch<any>, tabId: string,
},
})
);
dispatch(initPatterns({ tabId }));
});
};

Expand Down Expand Up @@ -234,7 +241,7 @@ export const calculateAvailability = async (
})
.then((res) => {
const stat = res.metadata.fields.filter(
(field: { name: string; type: string }) => !field.name.match(SPAN_REGEX)
(field: IField) => !field.name.match(SPAN_REGEX)
)[0].name;
const value = res.data[stat];
currValue = value[value.length - 1];
Expand Down
160 changes: 146 additions & 14 deletions public/components/event_analytics/explorer/explorer.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import React, { useState, useMemo, useEffect, useRef, useCallback, ReactElement
import { batch, useDispatch, useSelector } from 'react-redux';
import { isEmpty, cloneDeep, isEqual, has, reduce } from 'lodash';
import { FormattedMessage } from '@osd/i18n/react';
import { EuiLoadingSpinner, EuiSpacer } from '@elastic/eui';
import { EuiHorizontalRule, EuiLoadingSpinner, EuiSpacer, EuiTitle } from '@elastic/eui';
import {
EuiText,
EuiButtonIcon,
Expand Down Expand Up @@ -51,6 +51,8 @@ import {
TAB_CHART_ID,
DEFAULT_AVAILABILITY_QUERY,
DATE_PICKER_FORMAT,
PPL_PATTERNS_REGEX,
SELECTED_PATTERN,
} from '../../../../common/constants/explorer';
import {
PPL_STATS_REGEX,
Expand All @@ -77,6 +79,9 @@ import { getVizContainerProps } from '../../visualizations/charts/helpers';
import { parseGetSuggestions, onItemSelect } from '../../common/search/autocomplete_logic';
import { formatError } from '../utils';
import { sleep } from '../../common/live_tail/live_tail_button';
import { PatternsTable } from './log_patterns/patterns_table';
import { selectPatterns } from '../redux/slices/patterns_slice';
import { useFetchPatterns } from '../hooks/use_fetch_patterns';

const TYPE_TAB_MAPPING = {
[SAVED_QUERY]: TAB_EVENT_ID,
Expand Down Expand Up @@ -116,13 +121,18 @@ export const Explorer = ({
pplService,
requestParams,
});
const { getPatterns, setDefaultPatternsField } = useFetchPatterns({
pplService,
requestParams,
});
const appLogEvents = tabId.startsWith('application-analytics-tab');
const query = useSelector(selectQueries)[tabId];
const explorerData = useSelector(selectQueryResult)[tabId];
const explorerFields = useSelector(selectFields)[tabId];
const countDistribution = useSelector(selectCountDistribution)[tabId];
const explorerVisualizations = useSelector(selectExplorerVisualization)[tabId];
const userVizConfigs = useSelector(selectVisualizationConfig)[tabId] || {};
const patternsData = useSelector(selectPatterns)[tabId];
const [selectedContentTabId, setSelectedContentTab] = useState(TAB_EVENT_ID);
const [selectedCustomPanelOptions, setSelectedCustomPanelOptions] = useState([]);
const [selectedPanelName, setSelectedPanelName] = useState('');
Expand All @@ -132,6 +142,7 @@ export const Explorer = ({
const [isSidebarClosed, setIsSidebarClosed] = useState(false);
const [timeIntervalOptions, setTimeIntervalOptions] = useState(TIME_INTERVAL_OPTIONS);
const [isOverridingTimestamp, setIsOverridingTimestamp] = useState(false);
const [isOverridingPattern, setIsOverridingPattern] = useState(false);
const [tempQuery, setTempQuery] = useState(query[RAW_QUERY]);
const [isLiveTailPopoverOpen, setIsLiveTailPopoverOpen] = useState(false);
const [isLiveTailOn, setIsLiveTailOn] = useState(false);
Expand All @@ -141,7 +152,7 @@ export const Explorer = ({
const [browserTabFocus, setBrowserTabFocus] = useState(true);
const [liveTimestamp, setLiveTimestamp] = useState(DATE_PICKER_FORMAT);
const [triggerAvailability, setTriggerAvailability] = useState(false);

const [viewLogPatterns, setViewLogPatterns] = useState(false);
const queryRef = useRef();
const appBasedRef = useRef('');
appBasedRef.current = appBaseQuery;
Expand Down Expand Up @@ -196,6 +207,15 @@ export const Explorer = ({
});
});

const getErrorHandler = (title: string) => {
return (error: any) => {
const formattedError = formatError(error.name, error.message, error.body.message);
notifications.toasts.addError(formattedError, {
title,
});
};
};

const composeFinalQuery = (
curQuery: any,
startingTime: string,
Expand Down Expand Up @@ -327,6 +347,19 @@ export const Explorer = ({
}
}

let curPattern: string = curQuery![SELECTED_PATTERN];

if (isEmpty(curPattern)) {
const patternErrorHandler = getErrorHandler('Error fetching default pattern field');
await setDefaultPatternsField(curIndex, '', patternErrorHandler);
const newQuery = queryRef.current;
curPattern = newQuery![SELECTED_PATTERN];
if (isEmpty(curPattern)) {
setToast('Index does not contain a valid pattern field.', 'danger');
return;
}
}

if (isEqual(typeof startingTime, 'undefined') && isEqual(typeof endingTime, 'undefined')) {
startingTime = curQuery![SELECTED_DATE_RANGE][0];
endingTime = curQuery![SELECTED_DATE_RANGE][1];
Expand Down Expand Up @@ -358,21 +391,17 @@ export const Explorer = ({
} else {
findAutoInterval(startTime, endTime);
if (isLiveTailOnRef.current) {
getLiveTail(undefined, (error) => {
const formattedError = formatError(error.name, error.message, error.body.message);
notifications.toasts.addError(formattedError, {
title: 'Error fetching events',
});
});
getLiveTail(undefined, getErrorHandler('Error fetching events'));
} else {
getEvents(undefined, (error) => {
const formattedError = formatError(error.name, error.message, error.body.message);
notifications.toasts.addError(formattedError, {
title: 'Error fetching events',
});
});
getEvents(undefined, getErrorHandler('Error fetching events'));
}
getCountVisualizations(minInterval);

// to fetch patterns data on current query
if (!finalQuery.match(PPL_PATTERNS_REGEX)) {
const patternErrorHandler = getErrorHandler('Error fetching patterns');
getPatterns(patternErrorHandler);
}
}

// for comparing usage if for the same tab, user changed index from one to another
Expand Down Expand Up @@ -538,6 +567,17 @@ export const Explorer = ({
handleQuerySearch();
};

const handleOverridePattern = async (pattern: IField) => {
setIsOverridingPattern(true);
await setDefaultPatternsField(
'',
pattern.name,
getErrorHandler('Error overriding default pattern')
);
setIsOverridingPattern(false);
await getPatterns(getErrorHandler('Error fetching patterns'));
};

const totalHits: number = useMemo(() => {
if (isLiveTailOn && countDistribution?.data) {
const hits = reduce(
Expand All @@ -553,6 +593,21 @@ export const Explorer = ({
return 0;
}, [countDistribution?.data]);

const onPatternSelection = async (pattern: string) => {
let currQuery = queryRef.current![RAW_QUERY] as string;
const currPattern = queryRef.current![SELECTED_PATTERN] as string;
// Remove existing pattern selection if it exists
if (currQuery.match(PPL_PATTERNS_REGEX)) {
currQuery = currQuery.replace(PPL_PATTERNS_REGEX, '');
}
const patternSelectQuery = `${currQuery.trim()} | patterns ${currPattern} | where patterns_field = '${pattern}'`;
// Passing in empty string will remove pattern query
const newQuery = pattern ? patternSelectQuery : currQuery;
await setTempQuery(newQuery);
await updateQueryInStore(newQuery);
await handleTimeRangePickerRefresh(true);
};

const getMainContent = () => {
return (
<main className="container-fluid">
Expand All @@ -569,10 +624,13 @@ export const Explorer = ({
explorerFields={explorerFields}
explorerData={explorerData}
selectedTimestamp={query[SELECTED_TIMESTAMP]}
selectedPattern={query[SELECTED_PATTERN]}
handleOverrideTimestamp={handleOverrideTimestamp}
handleOverridePattern={handleOverridePattern}
handleAddField={(field: IField) => handleAddField(field)}
handleRemoveField={(field: IField) => handleRemoveField(field)}
isOverridingTimestamp={isOverridingTimestamp}
isOverridingPattern={isOverridingPattern}
isFieldToggleButtonDisabled={
isEmpty(explorerData.jsonData) ||
!isEmpty(queryRef.current![RAW_QUERY].match(PPL_STATS_REGEX))
Expand Down Expand Up @@ -628,6 +686,58 @@ export const Explorer = ({
</EuiFlexItem>
</EuiFlexGroup>
<CountDistribution countDistribution={countDistribution} />
<EuiHorizontalRule margin="xs" />
<EuiFlexGroup
justifyContent="spaceBetween"
alignItems="center"
style={{ margin: '8px' }}
gutterSize="xs"
>
<EuiFlexItem grow={false}>
{viewLogPatterns && (
<EuiTitle size="s">
<h3 style={{ margin: '0px' }}>
Patterns
<span className="pattern-header-count">
{' '}
(
{patternsData.patternTableData
? patternsData.patternTableData.length
: 0}
)
</span>
</h3>
</EuiTitle>
)}
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiFlexGroup>
<EuiFlexItem grow={false}>
{viewLogPatterns && (
<EuiLink onClick={() => onPatternSelection('')}>
Clear Selection
</EuiLink>
)}
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiLink onClick={() => setViewLogPatterns(!viewLogPatterns)}>
{`${viewLogPatterns ? 'Hide' : 'Show'} Patterns`}
</EuiLink>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>
</EuiFlexGroup>
<EuiHorizontalRule margin="xs" />
{viewLogPatterns && (
<>
<PatternsTable
tableData={patternsData.patternTableData || []}
onPatternSelection={onPatternSelection}
tabId={tabId}
/>
<EuiHorizontalRule margin="xs" />
</>
)}
</>
)}

Expand Down Expand Up @@ -662,6 +772,26 @@ export const Explorer = ({
<EuiSpacer size="m" />
</>
)}
{countDistribution?.data && (
<EuiTitle size="s">
<h3 style={{ margin: '0px', textAlign: 'left', marginLeft: '10px' }}>
Events
<span className="event-header-count">
{' '}
(
{reduce(
countDistribution.data['count()'],
(sum, n) => {
return sum + n;
},
0
)}
)
</span>
</h3>
</EuiTitle>
)}
<EuiHorizontalRule margin="xs" />
<DataGrid
http={http}
pplService={pplService}
Expand Down Expand Up @@ -775,6 +905,8 @@ export const Explorer = ({
visualizations,
query,
isLiveTailOnRef.current,
patternsData,
viewLogPatterns,
]);

const handleContentTabClick = (selectedTab: IQueryTab) => setSelectedContentTab(selectedTab.id);
Expand Down
Loading

0 comments on commit 8a652dd

Please sign in to comment.