Skip to content

Commit

Permalink
feat: DeliveryHog extra logging and filtering (#22870)
Browse files Browse the repository at this point in the history
  • Loading branch information
@benjackwhite
benjackwhite authored Jun 12, 2024
1 parent 0f50171 commit f324ec6
Show file tree
Hide file tree
Showing 16 changed files with 469 additions and 103 deletions.
Binary file modified frontend/__snapshots__/scenes-app-pipeline--pipeline-node-logs--dark.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified frontend/__snapshots__/scenes-app-pipeline--pipeline-node-logs--light.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified ...nd/__snapshots__/scenes-app-pipeline--pipeline-node-logs-batch-export--dark.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified ...d/__snapshots__/scenes-app-pipeline--pipeline-node-logs-batch-export--light.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
7 changes: 7 additions & 0 deletions frontend/src/lib/api.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { ActivityLogItem } from 'lib/components/ActivityLog/humanizeActivity'
import { apiStatusLogic } from 'lib/logic/apiStatusLogic'
import { objectClean, toParams } from 'lib/utils'
import posthog from 'posthog-js'
import { LogEntry } from 'scenes/pipeline/pipelineNodeLogsLogic'
import { SavedSessionRecordingPlaylistsResult } from 'scenes/session-recordings/saved-playlists/savedSessionRecordingPlaylistsLogic'

import { getCurrentExporterData } from '~/exporter/exporterViewLogic'
Expand Down Expand Up @@ -1659,6 +1660,12 @@ const api = {
async update(id: HogFunctionType['id'], data: Partial<HogFunctionType>): Promise<HogFunctionType> {
return await new ApiRequest().hogFunction(id).update({ data })
},
async searchLogs(
id: HogFunctionType['id'],
params: Record<string, any> = {}
): Promise<PaginatedResponse<LogEntry>> {
return await new ApiRequest().hogFunction(id).withAction('logs').withQueryString(params).get()
},
},

annotations: {
Expand Down
15 changes: 12 additions & 3 deletions frontend/src/scenes/pipeline/PipelineNodeLogs.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { LemonButton, LemonCheckbox, LemonInput, LemonTable } from '@posthog/lemon-ui'
import { IconSearch } from '@posthog/icons'
import { LemonButton, LemonCheckbox, LemonInput, LemonSnack, LemonTable } from '@posthog/lemon-ui'
import { useActions, useValues } from 'kea'
import { LOGS_PORTION_LIMIT } from 'lib/constants'
import { pluralize } from 'lib/utils'
Expand All @@ -9,8 +10,9 @@ import { PipelineLogLevel, pipelineNodeLogsLogic } from './pipelineNodeLogsLogic
export function PipelineNodeLogs({ id, stage }: PipelineNodeLogicProps): JSX.Element {
const logic = pipelineNodeLogsLogic({ id, stage })

const { logs, logsLoading, backgroundLogs, columns, isThereMoreToLoad, selectedLogLevels } = useValues(logic)
const { revealBackground, loadMoreLogs, setSelectedLogLevels, setSearchTerm } = useActions(logic)
const { logs, logsLoading, backgroundLogs, columns, isThereMoreToLoad, selectedLogLevels, instanceId } =
useValues(logic)
const { revealBackground, loadMoreLogs, setSelectedLogLevels, setSearchTerm, setInstanceId } = useActions(logic)

return (
<div className="ph-no-capture space-y-2 flex-1">
Expand All @@ -20,6 +22,13 @@ export function PipelineNodeLogs({ id, stage }: PipelineNodeLogicProps): JSX.Ele
fullWidth
onChange={setSearchTerm}
allowClear
prefix={
<>
<IconSearch />

{instanceId && <LemonSnack onClose={() => setInstanceId('')}>{instanceId}</LemonSnack>}
</>
}
/>
<div className="flex items-center gap-4">
<span className="mr-1">Show logs of level:</span>
Expand Down
2 changes: 1 addition & 1 deletion frontend/src/scenes/pipeline/hogfunctions/templates/hog-templates.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ export const HOG_FUNCTION_TEMPLATES: HogFunctionTemplateType[] = [
id: 'template-webhook',
name: 'HogHook',
description: 'Sends a webhook templated by the incoming event data',
hog: "fetch(inputs.url, {\n 'headers': inputs.headers,\n 'body': inputs.payload,\n 'method': inputs.method,\n 'payload': inputs.payload\n});",
hog: "fetch(inputs.url, {\n 'headers': inputs.headers,\n 'body': inputs.payload,\n 'method': inputs.method\n});",
inputs_schema: [
{
key: 'url',
Expand Down
126 changes: 108 additions & 18 deletions frontend/src/scenes/pipeline/pipelineNodeLogsLogic.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { LemonTableColumns } from '@posthog/lemon-ui'
import { TZLabel } from '@posthog/apps-common'
import { LemonTableColumns, Link } from '@posthog/lemon-ui'
import { actions, connect, events, kea, key, listeners, path, props, reducers, selectors } from 'kea'
import { loaders } from 'kea-loaders'
import { LOGS_PORTION_LIMIT } from 'lib/constants'
Expand Down Expand Up @@ -28,13 +29,14 @@ export const pipelineNodeLogsLogic = kea<pipelineNodeLogsLogicType>([
key(({ id }) => id),
path((key) => ['scenes', 'pipeline', 'pipelineNodeLogsLogic', key]),
connect((props: PipelineNodeLogicProps) => ({
values: [teamLogic(), ['currentTeamId'], pipelineNodeLogic(props), ['nodeBackend']],
values: [teamLogic(), ['currentTeamId'], pipelineNodeLogic(props), ['node']],
})),
actions({
setSelectedLogLevels: (levels: PipelineLogLevel[]) => ({
levels,
}),
setSearchTerm: (searchTerm: string) => ({ searchTerm }),
setInstanceId: (instanceId: string) => ({ instanceId }),
clearBackgroundLogs: true,
markLogsEnd: true,
}),
Expand All @@ -44,15 +46,24 @@ export const pipelineNodeLogsLogic = kea<pipelineNodeLogsLogicType>([
{
loadLogs: async () => {
let results: LogEntry[]
if (values.nodeBackend === PipelineBackend.BatchExport) {
if (values.node.backend === PipelineBackend.BatchExport) {
results = await api.batchExportLogs.search(
id as string,
values.node.id,
values.searchTerm,
values.selectedLogLevels
)
} else if (values.node.backend === PipelineBackend.HogFunction) {
const res = await api.hogFunctions.searchLogs(values.node.id, {
search: values.searchTerm,
levels: values.selectedLogLevels,
limit: LOGS_PORTION_LIMIT,
instance_id: values.instanceId,
})

results = res.results
} else {
results = await api.pluginLogs.search(
id as number,
values.node.id,
values.searchTerm,
logLevelsToTypeFilters(values.selectedLogLevels)
)
Expand All @@ -66,13 +77,23 @@ export const pipelineNodeLogsLogic = kea<pipelineNodeLogsLogicType>([
},
loadMoreLogs: async () => {
let results: LogEntry[]
if (values.nodeBackend === PipelineBackend.BatchExport) {
if (values.node.backend === PipelineBackend.BatchExport) {
results = await api.batchExportLogs.search(
id as string,
values.searchTerm,
values.selectedLogLevels,
values.trailingEntry as BatchExportLogEntry | null
)
} else if (values.node.backend === PipelineBackend.HogFunction) {
const res = await api.hogFunctions.searchLogs(values.node.id, {
search: values.searchTerm,
levels: values.selectedLogLevels,
limit: LOGS_PORTION_LIMIT,
before: values.trailingEntry?.timestamp,
instance_id: values.instanceId,
})

results = res.results
} else {
results = await api.pluginLogs.search(
id as number,
Expand Down Expand Up @@ -105,14 +126,24 @@ export const pipelineNodeLogsLogic = kea<pipelineNodeLogsLogicType>([
}

let results: LogEntry[]
if (values.nodeBackend === PipelineBackend.BatchExport) {
if (values.node.backend === PipelineBackend.BatchExport) {
results = await api.batchExportLogs.search(
id as string,
values.searchTerm,
values.selectedLogLevels,
null,
values.leadingEntry as BatchExportLogEntry | null
)
} else if (values.node.backend === PipelineBackend.HogFunction) {
const res = await api.hogFunctions.searchLogs(values.node.id, {
search: values.searchTerm,
levels: values.selectedLogLevels,
limit: LOGS_PORTION_LIMIT,
after: values.leadingEntry?.timestamp,
instance_id: values.instanceId,
})

results = res.results
} else {
results = await api.pluginLogs.search(
id as number,
Expand Down Expand Up @@ -147,6 +178,12 @@ export const pipelineNodeLogsLogic = kea<pipelineNodeLogsLogicType>([
setSearchTerm: (_, { searchTerm }) => searchTerm,
},
],
instanceId: [
'',
{
setInstanceId: (_, { instanceId }) => instanceId,
},
],
isThereMoreToLoad: [
true,
{
Expand All @@ -155,7 +192,7 @@ export const pipelineNodeLogsLogic = kea<pipelineNodeLogsLogicType>([
},
],
}),
selectors({
selectors(({ actions }) => ({
leadingEntry: [
(s) => [s.logs, s.backgroundLogs],
(logs: LogEntry[], backgroundLogs: LogEntry[]): LogEntry | null => {
Expand All @@ -181,26 +218,76 @@ export const pipelineNodeLogsLogic = kea<pipelineNodeLogsLogicType>([
},
],
columns: [
(s) => [s.nodeBackend],
(nodeBackend): LemonTableColumns<LogEntry> => {
(s) => [s.node],
(node): LemonTableColumns<LogEntry> => {
return [
{
title: 'Timestamp',
key: 'timestamp',
dataIndex: 'timestamp',
sorter: (a: LogEntry, b: LogEntry) => dayjs(a.timestamp).unix() - dayjs(b.timestamp).unix(),
render: (timestamp: string) => dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss.SSS UTC'),
render: (timestamp: string) => <TZLabel time={timestamp} />,
width: 0,
},
{
title: nodeBackend === PipelineBackend.BatchExport ? 'Run Id' : 'Source',
dataIndex: nodeBackend === PipelineBackend.BatchExport ? 'run_id' : 'source',
key: nodeBackend === PipelineBackend.BatchExport ? 'run_id' : 'source',
width: 0,
title:
node.backend == PipelineBackend.HogFunction
? 'Invocation'
: node.backend == PipelineBackend.BatchExport
? 'Run Id'
: 'Source',
dataIndex:
node.backend == PipelineBackend.HogFunction
? 'instance_id'
: node.backend == PipelineBackend.BatchExport
? 'run_id'
: 'source',
key:
node.backend == PipelineBackend.HogFunction
? 'instance_id'
: node.backend == PipelineBackend.BatchExport
? 'run_id'
: 'source',

render: (instanceId: string) => (
<code className="whitespace-nowrap">
{node.backend === PipelineBackend.HogFunction ? (
<Link
subtle
onClick={() => {
actions.setInstanceId(instanceId)
}}
>
{instanceId}
</Link>
) : (
instanceId
)}
</code>
),
},
{
width: 100,
title: 'Level',
key: nodeBackend === PipelineBackend.BatchExport ? 'level' : 'type',
dataIndex: nodeBackend === PipelineBackend.BatchExport ? 'level' : 'type',
render: nodeBackend === PipelineBackend.BatchExport ? LogLevelDisplay : LogTypeDisplay,
key:
node.backend == PipelineBackend.HogFunction
? 'level'
: node.backend == PipelineBackend.BatchExport
? 'level'
: 'type',
dataIndex:
node.backend == PipelineBackend.HogFunction
? 'level'
: node.backend == PipelineBackend.BatchExport
? 'level'
: 'type',
render:
node.backend == PipelineBackend.HogFunction
? LogLevelDisplay
: node.backend == PipelineBackend.BatchExport
? LogLevelDisplay
: LogTypeDisplay,
},
{
title: 'Message',
Expand All @@ -211,7 +298,7 @@ export const pipelineNodeLogsLogic = kea<pipelineNodeLogsLogicType>([
] as LemonTableColumns<LogEntry>
},
],
}),
})),
listeners(({ actions }) => ({
setSelectedLogLevels: () => {
actions.loadLogs()
Expand All @@ -222,6 +309,9 @@ export const pipelineNodeLogsLogic = kea<pipelineNodeLogsLogicType>([
}
actions.loadLogs()
},
setInstanceId: async () => {
actions.loadLogs()
},
})),
events(({ actions, cache }) => ({
afterMount: () => {
Expand Down
Loading

0 comments on commit f324ec6

Please sign in to comment.