From 595cd58edfc5c25b6b445a89cccbd63167f63e42 Mon Sep 17 00:00:00 2001 From: Janson Bunce Date: Mon, 9 Sep 2024 08:54:27 -0700 Subject: [PATCH 01/11] initial logging implementation --- backend/src/api/app.ts | 77 ++++++++++++----- backend/src/models/connection.ts | 4 +- backend/src/models/index.ts | 1 + backend/src/models/log.ts | 19 +++++ backend/src/tools/loggingMiddleware.ts | 109 +++++++++++++++++++++++++ frontend/src/components/Header.tsx | 5 +- 6 files changed, 188 insertions(+), 27 deletions(-) create mode 100644 backend/src/models/log.ts create mode 100644 backend/src/tools/loggingMiddleware.ts diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index 1c7f5ad3..928d3d52 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -28,6 +28,7 @@ import * as jwt from 'jsonwebtoken'; import { Request, Response, NextFunction } from 'express'; import fetch from 'node-fetch'; import * as searchOrganizations from './organizationSearch'; +import { Logger, RecordMessage } from '../tools/loggingMiddleware'; const sanitizer = require('sanitizer'); @@ -43,27 +44,35 @@ if ( setInterval(() => scheduler({}, {} as any, () => null), 30000); } -const handlerToExpress = (handler) => async (req, res) => { - const { statusCode, body } = await handler( - { - pathParameters: req.params, - query: req.query, - requestContext: req.requestContext, - body: JSON.stringify(req.body || '{}'), - headers: req.headers, - path: req.originalUrl - }, - {} - ); - try { - const parsedBody = JSON.parse(sanitizer.sanitize(body)); - res.status(statusCode).json(parsedBody); - } catch (e) { - // Not a JSON body - res.setHeader('content-type', 'text/plain'); - res.status(statusCode).send(sanitizer.sanitize(body)); - } -}; +const handlerToExpress = + (handler, message?: RecordMessage, action?: string) => async (req, res) => { + const logger = new Logger(req); + const { statusCode, body } = await handler( + { + pathParameters: req.params, + query: req.query, + requestContext: req.requestContext, + body: JSON.stringify(req.body || '{}'), + headers: req.headers, + path: req.originalUrl + }, + {} + ); + try { + const parsedBody = JSON.parse(sanitizer.sanitize(body)); + res.status(200).json(parsedBody); + if (message && action) { + logger.record(action, 'success', message); + } + } catch (e) { + // Not a JSON body + res.setHeader('content-type', 'text/plain'); + res.status(statusCode).send(sanitizer.sanitize(body)); + if (message && action) { + logger.record(action, 'fail', message); + } + } + }; const app = express(); @@ -181,8 +190,15 @@ app.post('/auth/okta-callback', async (req, res) => { const clientId = process.env.REACT_APP_COGNITO_CLIENT_ID; const callbackUrl = process.env.REACT_APP_COGNITO_CALLBACK_URL; const domain = process.env.REACT_APP_COGNITO_DOMAIN; + const logger = new Logger(req); if (!code) { + logger.record('USER LOGIN', 'fail', (req, user) => { + return { + timestamp: new Date(), + trace: console.trace() + }; + }); return res.status(400).json({ message: 'Missing authorization code' }); } @@ -268,6 +284,13 @@ app.post('/auth/okta-callback', async (req, res) => { res.cookie('id_token', signedToken, { httpOnly: true, secure: true }); + logger.record('USER LOGIN', 'success', (req, user) => { + return { + timestamp: new Date(), + stuff: 'hello', + userId: user?.data?.id + }; + }); return res.status(200).json({ token: signedToken, user: user @@ -671,7 +694,17 @@ authenticatedRoute.post( authenticatedRoute.put( '/users/:userId/register/approve', checkGlobalAdminOrRegionAdmin, - handlerToExpress(users.registrationApproval) + handlerToExpress( + users.registrationApproval, + (req, user) => { + return { + timestamp: new Date(), + trace: console.trace(), + userId: user?.data?.id + }; + }, + 'APPROVE USER' + ) ); authenticatedRoute.put( diff --git a/backend/src/models/connection.ts b/backend/src/models/connection.ts index 8f2cfdbf..a32ba88d 100644 --- a/backend/src/models/connection.ts +++ b/backend/src/models/connection.ts @@ -21,6 +21,7 @@ import { User, Vulnerability, Webpage, + Log, // Models for the Mini Data Lake database CertScan, @@ -194,7 +195,8 @@ const connectDb = async (logging?: boolean) => { Service, User, Vulnerability, - Webpage + Webpage, + Log ], synchronize: false, name: 'default', diff --git a/backend/src/models/index.ts b/backend/src/models/index.ts index 85627560..d0b3814f 100644 --- a/backend/src/models/index.ts +++ b/backend/src/models/index.ts @@ -19,6 +19,7 @@ export * from './service'; export * from './user'; export * from './vulnerability'; export * from './webpage'; +export * from './log'; // Mini data lake models export * from './mini_data_lake/cert_scans'; export * from './mini_data_lake/cidrs'; diff --git a/backend/src/models/log.ts b/backend/src/models/log.ts new file mode 100644 index 00000000..2d128b18 --- /dev/null +++ b/backend/src/models/log.ts @@ -0,0 +1,19 @@ +import { BaseEntity, Column, Entity, PrimaryGeneratedColumn } from 'typeorm'; + +@Entity() +export class Log extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column('json') + payload: Object; + + @Column({ nullable: false }) + createdAt: Date; + + @Column({ nullable: true }) + eventType: string; + + @Column({ nullable: false }) + result: string; +} diff --git a/backend/src/tools/loggingMiddleware.ts b/backend/src/tools/loggingMiddleware.ts new file mode 100644 index 00000000..0700f2f9 --- /dev/null +++ b/backend/src/tools/loggingMiddleware.ts @@ -0,0 +1,109 @@ +import { Request } from 'express'; +import { decode } from 'jsonwebtoken'; +import { User } from '../models'; +import { attempt, unescape } from 'lodash'; +import { Log } from '../models/log'; +import { getRepository, Repository } from 'typeorm'; + +type AccessTokenPayload = { + id: string; + email: string; + iat: string; + exp: string; +}; + +type LoggerUserState = { + data: User | undefined; + ready: boolean; + attempts: number; +}; + +type RecordPayload = object & { + timestamp: Date; +}; + +export type RecordMessage = + | ((request: Request, user: LoggerUserState) => RecordPayload) + | RecordPayload; + +export class Logger { + private request: Request; + private logId: string; + private token: AccessTokenPayload | undefined; + private user: LoggerUserState = { + data: undefined, + ready: false, + attempts: 0 + }; + private logRep: Repository; + + async record( + action: string, + result: 'success' | 'fail', + messageOrCB: RecordMessage | undefined + ) { + try { + if (!this.user.ready && this.user.attempts > 0) { + await this.fetchUser(); + } + + if (!this.logRep) { + const logRepository = getRepository(Log); + this.logRep = logRepository; + } + + const payload = + typeof messageOrCB === 'function' + ? messageOrCB(this.request, this.user) + : messageOrCB; + const logRecord = await this.logRep.create({ + payload: JSON.stringify(payload), + createdAt: payload?.timestamp, + result: result, + eventType: action + }); + + logRecord.save(); + } catch (error) { + console.warn('Error occured in loggingMiddleware', error); + } + } + + async fetchUser() { + if (this.token) { + const user = await User.findOne({ id: this.token.id }); + if (user) { + this.user = { + data: user, + ready: true, + attempts: 0 + }; + } + this.user = { + data: undefined, + ready: false, + attempts: this.user.attempts + 1 + }; + } + } + + // Constructor takes a request and sets it to a class variable + constructor(req: Request) { + this.request = req; + this.logId = '123123123123'; + const authToken = req.headers.authorization; + if (authToken) { + const tokenPayload = decode( + authToken as string + ) as unknown as AccessTokenPayload; + this.token = tokenPayload; + User.findOne({ id: this.token.id }).then((user) => { + if (user) { + this.user = { data: user, ready: true, attempts: 0 }; + } + }); + } + } +} + +// Database Tables diff --git a/frontend/src/components/Header.tsx b/frontend/src/components/Header.tsx index 994e61ef..fcfb91df 100644 --- a/frontend/src/components/Header.tsx +++ b/frontend/src/components/Header.tsx @@ -195,10 +195,7 @@ export const Header: React.FC = ({ width: 175, maxWidth: 175, padding: theme.spacing(), - paddingLeft: 0, - [theme.breakpoints.down('xl')]: { - display: 'flex' - } + paddingLeft: 0 }} alt="CyHy Dashboard Icon Navigate Home" /> From d0010fa0e1159fb600440627d8664a4e45d57749 Mon Sep 17 00:00:00 2001 From: Janson Bunce Date: Tue, 10 Sep 2024 09:15:48 -0700 Subject: [PATCH 02/11] initial logging implementation for user login and user approval --- backend/src/api/app.ts | 36 +++++++++++++++++++++++++++--------- backend/src/api/users.ts | 14 +++++++------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index 928d3d52..4d031f1c 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -58,19 +58,25 @@ const handlerToExpress = }, {} ); - try { - const parsedBody = JSON.parse(sanitizer.sanitize(body)); - res.status(200).json(parsedBody); + // Add additional status codes that we may return for succesfull requests + if (statusCode === 200) { if (message && action) { logger.record(action, 'success', message); } + } else { + if (message && action) { + logger.record(action, 'fail', message); + } + } + + try { + const parsedBody = JSON.parse(sanitizer.sanitize(body)); + res.status(200).json(parsedBody); } catch (e) { // Not a JSON body + console.log('Error?', e); res.setHeader('content-type', 'text/plain'); res.status(statusCode).send(sanitizer.sanitize(body)); - if (message && action) { - logger.record(action, 'fail', message); - } } }; @@ -644,8 +650,19 @@ authenticatedRoute.delete( ); authenticatedRoute.post( '/v2/organizations/:organizationId/users', - handlerToExpress(organizations.addUserV2) + handlerToExpress( + organizations.addUserV2, + (req, user) => { + return { + timestamp: new Date(), + userId: user?.data?.id, + updatePayload: req.body + }; + }, + 'UPDATE USER' + ) ); + authenticatedRoute.post( '/organizations/:organizationId/roles/:roleId/approve', handlerToExpress(organizations.approveRole) @@ -697,10 +714,11 @@ authenticatedRoute.put( handlerToExpress( users.registrationApproval, (req, user) => { + console.log('here', req.params); return { timestamp: new Date(), - trace: console.trace(), - userId: user?.data?.id + userId: user?.data?.id, + userToApprove: req.params.userId }; }, 'APPROVE USER' diff --git a/backend/src/api/users.ts b/backend/src/api/users.ts index da35e748..20b4c808 100644 --- a/backend/src/api/users.ts +++ b/backend/src/api/users.ts @@ -635,13 +635,13 @@ export const registrationApproval = wrapHandler(async (event) => { } // Send email notification - await sendRegistrationApprovedEmail( - user.email, - 'CyHy Dashboard Registration Approved', - user.firstName, - user.lastName, - 'crossfeed_approval_notification.html' - ); + // await sendRegistrationApprovedEmail( + // user.email, + // 'CyHy Dashboard Registration Approved', + // user.firstName, + // user.lastName, + // 'crossfeed_approval_notification.html' + // ); // TODO: Handle Response Output return { From e37e1581bc843960bb3d8567ab0f6885408b387b Mon Sep 17 00:00:00 2001 From: Janson Bunce Date: Wed, 11 Sep 2024 11:11:09 -0700 Subject: [PATCH 03/11] passes the response body into logger.record to capture response fields within the recorded log --- backend/src/api/app.ts | 20 +++++++++++++++++--- backend/src/tools/loggingMiddleware.ts | 18 ++++++++++++++---- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index 4d031f1c..4fc9b4f0 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -61,11 +61,11 @@ const handlerToExpress = // Add additional status codes that we may return for succesfull requests if (statusCode === 200) { if (message && action) { - logger.record(action, 'success', message); + logger.record(action, 'success', message, body); } } else { if (message && action) { - logger.record(action, 'fail', message); + logger.record(action, 'fail', message, body); } } @@ -684,7 +684,21 @@ authenticatedRoute.post( handlerToExpress(organizations.checkDomainVerification) ); authenticatedRoute.post('/stats', handlerToExpress(stats.get)); -authenticatedRoute.post('/users', handlerToExpress(users.invite)); +authenticatedRoute.post( + '/users', + handlerToExpress( + users.invite, + (req, user, responseBody) => { + return { + timestamp: new Date(), + userId: user.data?.id, + invitePayload: req.body, + createdUserRecord: responseBody + }; + }, + 'USER INVITED' + ) +); authenticatedRoute.get('/users', handlerToExpress(users.list)); authenticatedRoute.delete('/users/:userId', handlerToExpress(users.del)); authenticatedRoute.get( diff --git a/backend/src/tools/loggingMiddleware.ts b/backend/src/tools/loggingMiddleware.ts index 0700f2f9..5087f577 100644 --- a/backend/src/tools/loggingMiddleware.ts +++ b/backend/src/tools/loggingMiddleware.ts @@ -23,7 +23,11 @@ type RecordPayload = object & { }; export type RecordMessage = - | ((request: Request, user: LoggerUserState) => RecordPayload) + | (( + request: Request, + user: LoggerUserState, + responseBody?: object + ) => RecordPayload) | RecordPayload; export class Logger { @@ -40,7 +44,8 @@ export class Logger { async record( action: string, result: 'success' | 'fail', - messageOrCB: RecordMessage | undefined + messageOrCB: RecordMessage | undefined, + responseBody?: object | string ) { try { if (!this.user.ready && this.user.attempts > 0) { @@ -52,12 +57,17 @@ export class Logger { this.logRep = logRepository; } + const parsedResponseBody = + typeof responseBody === 'string' + ? JSON.parse(responseBody) + : responseBody; + const payload = typeof messageOrCB === 'function' - ? messageOrCB(this.request, this.user) + ? messageOrCB(this.request, this.user, parsedResponseBody) : messageOrCB; const logRecord = await this.logRep.create({ - payload: JSON.stringify(payload), + payload: payload as object, createdAt: payload?.timestamp, result: result, eventType: action From 2c5ac928d27e360a3c882e1f02f9b6e61c8a912c Mon Sep 17 00:00:00 2001 From: Janson Bunce Date: Thu, 12 Sep 2024 08:46:59 -0700 Subject: [PATCH 04/11] adds logging to user removal/denial --- backend/src/api/app.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index 4fc9b4f0..98a5215b 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -700,7 +700,19 @@ authenticatedRoute.post( ) ); authenticatedRoute.get('/users', handlerToExpress(users.list)); -authenticatedRoute.delete('/users/:userId', handlerToExpress(users.del)); +authenticatedRoute.delete( + '/users/:userId', + handlerToExpress( + users.del, + (req, user, res) => { + console.log(req.params); + return { + timestamp: new Date() + }; + }, + 'USER REMOVED' + ) +); authenticatedRoute.get( '/users/state/:state', handlerToExpress(users.getByState) From 51418da7e66839f7c2b1ed22062463623314d123 Mon Sep 17 00:00:00 2001 From: Janson Bunce Date: Tue, 17 Sep 2024 06:51:31 -0700 Subject: [PATCH 05/11] add UI for logs --- backend/src/api/app.ts | 10 ++- backend/src/api/logs.ts | 13 +++ .../tools/{loggingMiddleware.ts => logger.ts} | 0 frontend/src/components/Logs/Logs.tsx | 82 +++++++++++++++++++ frontend/src/pages/AdminTools/AdminTools.tsx | 5 ++ 5 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 backend/src/api/logs.ts rename backend/src/tools/{loggingMiddleware.ts => logger.ts} (100%) create mode 100644 frontend/src/components/Logs/Logs.tsx diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index cb7dda43..a974bd19 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -13,6 +13,7 @@ import * as search from './search'; import * as vulnerabilities from './vulnerabilities'; import * as organizations from './organizations'; import * as scans from './scans'; +import * as logs from './logs'; import * as users from './users'; import * as scanTasks from './scan-tasks'; import * as stats from './stats'; @@ -28,7 +29,7 @@ import * as jwt from 'jsonwebtoken'; import { Request, Response, NextFunction } from 'express'; import fetch from 'node-fetch'; import * as searchOrganizations from './organizationSearch'; -import { Logger, RecordMessage } from '../tools/loggingMiddleware'; +import { Logger, RecordMessage } from '../tools/logger'; const sanitizer = require('sanitizer'); @@ -287,11 +288,11 @@ app.post('/auth/okta-callback', async (req, res) => { res.cookie('id_token', signedToken, { httpOnly: true, secure: true }); - logger.record('USER LOGIN', 'success', (req, user) => { + logger.record('USER LOGIN', 'success', (req, us) => { + console.log('HERE', { req, us }); return { timestamp: new Date(), - stuff: 'hello', - userId: user?.data?.id + userId: user?.id }; }); return res.status(200).json({ @@ -590,6 +591,7 @@ authenticatedRoute.delete( handlerToExpress(savedSearches.del) ); authenticatedRoute.get('/scans', handlerToExpress(scans.list)); +authenticatedRoute.get('/logs', handlerToExpress(logs.list)); authenticatedRoute.get('/granularScans', handlerToExpress(scans.listGranular)); authenticatedRoute.post('/scans', handlerToExpress(scans.create)); authenticatedRoute.get('/scans/:scanId', handlerToExpress(scans.get)); diff --git a/backend/src/api/logs.ts b/backend/src/api/logs.ts new file mode 100644 index 00000000..d2ad80f0 --- /dev/null +++ b/backend/src/api/logs.ts @@ -0,0 +1,13 @@ +import { Log } from '../models'; +import { wrapHandler } from './helpers'; + +export const list = wrapHandler(async () => { + const logs = await Log.find(); + return { + statusCode: 200, + body: JSON.stringify({ + result: logs, + count: logs.length + }) + }; +}); diff --git a/backend/src/tools/loggingMiddleware.ts b/backend/src/tools/logger.ts similarity index 100% rename from backend/src/tools/loggingMiddleware.ts rename to backend/src/tools/logger.ts diff --git a/frontend/src/components/Logs/Logs.tsx b/frontend/src/components/Logs/Logs.tsx new file mode 100644 index 00000000..3c42db09 --- /dev/null +++ b/frontend/src/components/Logs/Logs.tsx @@ -0,0 +1,82 @@ +import { Box } from '@mui/system'; +import { DataGrid, GridColDef } from '@mui/x-data-grid'; +import { useAuthContext } from 'context'; +import { differenceInCalendarDays, parseISO } from 'date-fns'; +import React, { FC, useCallback, useEffect, useState } from 'react'; + +interface LogsProps {} + +export const Logs: FC = () => { + const { apiGet } = useAuthContext(); + const [logs, setLogs] = useState<{ count: Number; result: [] }>({ + count: 0, + result: [] + }); + + const fetchLogs = useCallback(async () => { + const results = await apiGet('/logs'); + setLogs(results); + }, []); + + useEffect(() => { + fetchLogs(); + }, []); + + const logCols: GridColDef[] = [ + { + field: 'eventType', + headerName: 'Event', + minWidth: 100, + flex: 1 + }, + { + field: 'result', + headerName: 'Result', + minWidth: 100, + flex: 1 + }, + { + field: 'createdAt', + headerName: 'Timestamp', + minWidth: 100, + flex: 1, + valueFormatter: (e) => { + return `${differenceInCalendarDays( + Date.now(), + parseISO(e.value) + )} days ago`; + } + }, + { + field: 'payload', + headerName: 'Payload', + minWidth: 300, + flex: 2, + renderCell: (cellValues) => { + return ( + + 
{JSON.stringify(cellValues.row.payload, null, 2)}
+ + ); + }, + valueFormatter: (e) => { + return JSON.stringify(e.value, null, 2); + } + } + ]; + + return ( + + + + ); +}; diff --git a/frontend/src/pages/AdminTools/AdminTools.tsx b/frontend/src/pages/AdminTools/AdminTools.tsx index e40cfc3a..d758fff7 100644 --- a/frontend/src/pages/AdminTools/AdminTools.tsx +++ b/frontend/src/pages/AdminTools/AdminTools.tsx @@ -5,6 +5,7 @@ import ScansView from 'pages/Scans/ScansView'; import ScanTasksView from 'pages/Scans/ScanTasksView'; import { Box, Container, Tab } from '@mui/material'; import { TabContext, TabList, TabPanel } from '@mui/lab'; +import { Logs } from 'components/Logs/Logs'; export const AdminTools: React.FC = () => { const [value, setValue] = React.useState('1'); @@ -21,6 +22,7 @@ export const AdminTools: React.FC = () => { + @@ -33,6 +35,9 @@ export const AdminTools: React.FC = () => { + + + From fb4de7e133b9478140dbcfa0ee57092098a727b3 Mon Sep 17 00:00:00 2001 From: Janson Bunce Date: Tue, 17 Sep 2024 07:48:24 -0700 Subject: [PATCH 06/11] removes a few routes --- backend/src/api/app.ts | 31 +++++++++++--------- frontend/src/pages/AdminTools/AdminTools.tsx | 2 +- frontend/src/pages/Risk/Risk.tsx | 2 +- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index a974bd19..66a10c3c 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -200,12 +200,12 @@ app.post('/auth/okta-callback', async (req, res) => { const logger = new Logger(req); if (!code) { - logger.record('USER LOGIN', 'fail', (req, user) => { - return { - timestamp: new Date(), - trace: console.trace() - }; - }); + // logger.record('USER LOGIN', 'fail', (req, user) => { + // return { + // timestamp: new Date(), + // trace: console.trace() + // }; + // }); return res.status(400).json({ message: 'Missing authorization code' }); } @@ -288,13 +288,13 @@ app.post('/auth/okta-callback', async (req, res) => { res.cookie('id_token', signedToken, { httpOnly: true, secure: true }); - logger.record('USER LOGIN', 'success', (req, us) => { - console.log('HERE', { req, us }); - return { - timestamp: new Date(), - userId: user?.id - }; - }); + // logger.record('USER LOGIN', 'success', (req, us) => { + // console.log('HERE', { req, us }); + // return { + // timestamp: new Date(), + // userId: user?.id + // }; + // }); return res.status(200).json({ token: signedToken, user: user @@ -526,7 +526,10 @@ const checkGlobalAdminOrRegionAdmin = async ( // needing to sign the terms of service yet const authenticatedNoTermsRoute = express.Router(); authenticatedNoTermsRoute.use(checkUserLoggedIn); -authenticatedNoTermsRoute.get('/users/me', handlerToExpress(users.me)); +authenticatedNoTermsRoute.get( + '/users/me', + handlerToExpress(users.me, (req, user, res) => {}, 'USER ME DATA') +); authenticatedNoTermsRoute.post( '/users/me/acceptTerms', handlerToExpress(users.acceptTerms) diff --git a/frontend/src/pages/AdminTools/AdminTools.tsx b/frontend/src/pages/AdminTools/AdminTools.tsx index d758fff7..4e068099 100644 --- a/frontend/src/pages/AdminTools/AdminTools.tsx +++ b/frontend/src/pages/AdminTools/AdminTools.tsx @@ -22,7 +22,7 @@ export const AdminTools: React.FC = () => { - + diff --git a/frontend/src/pages/Risk/Risk.tsx b/frontend/src/pages/Risk/Risk.tsx index cd071458..ca04692e 100644 --- a/frontend/src/pages/Risk/Risk.tsx +++ b/frontend/src/pages/Risk/Risk.tsx @@ -102,7 +102,7 @@ const Risk: React.FC = ({ filters, addFilter }) => { .range(['#c7e8ff', '#135787']); setStats(result); }, - [apiPost, riskFilters] + [riskFilters] ); useEffect(() => { From f4de74392e04f12189a1505b47e27a444d192a16 Mon Sep 17 00:00:00 2001 From: Janson Bunce Date: Thu, 26 Sep 2024 11:10:40 -0700 Subject: [PATCH 07/11] add dialog modal to display payload data --- backend/src/api/app.ts | 17 ++- backend/src/api/logs.ts | 174 +++++++++++++++++++++++++- backend/src/tools/logger.ts | 3 +- frontend/src/components/Logs/Logs.tsx | 134 ++++++++++++++++++-- frontend/src/pages/Risk/Risk.tsx | 1 + 5 files changed, 307 insertions(+), 22 deletions(-) diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index 66a10c3c..d3ca160e 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -526,10 +526,7 @@ const checkGlobalAdminOrRegionAdmin = async ( // needing to sign the terms of service yet const authenticatedNoTermsRoute = express.Router(); authenticatedNoTermsRoute.use(checkUserLoggedIn); -authenticatedNoTermsRoute.get( - '/users/me', - handlerToExpress(users.me, (req, user, res) => {}, 'USER ME DATA') -); +authenticatedNoTermsRoute.get('/users/me', handlerToExpress(users.me)); authenticatedNoTermsRoute.post( '/users/me/acceptTerms', handlerToExpress(users.acceptTerms) @@ -594,7 +591,7 @@ authenticatedRoute.delete( handlerToExpress(savedSearches.del) ); authenticatedRoute.get('/scans', handlerToExpress(scans.list)); -authenticatedRoute.get('/logs', handlerToExpress(logs.list)); +authenticatedRoute.post('/logs/search', handlerToExpress(logs.list)); authenticatedRoute.get('/granularScans', handlerToExpress(scans.listGranular)); authenticatedRoute.post('/scans', handlerToExpress(scans.create)); authenticatedRoute.get('/scans/:scanId', handlerToExpress(scans.get)); @@ -661,7 +658,7 @@ authenticatedRoute.post( updatePayload: req.body }; }, - 'UPDATE USER' + 'USER UPDATE' ) ); @@ -669,6 +666,8 @@ authenticatedRoute.post( '/organizations/:organizationId/roles/:roleId/approve', handlerToExpress(organizations.approveRole) ); + +// TO-DO Add logging => /users => user has an org and you change them to a new organization authenticatedRoute.post( '/organizations/:organizationId/roles/:roleId/remove', handlerToExpress(organizations.removeRole) @@ -698,7 +697,7 @@ authenticatedRoute.post( createdUserRecord: responseBody }; }, - 'USER INVITED' + 'USER INVITE' ) ); authenticatedRoute.get('/users', handlerToExpress(users.list)); @@ -712,7 +711,7 @@ authenticatedRoute.delete( timestamp: new Date() }; }, - 'USER REMOVED' + 'USER DENY/REMOVE' ) ); authenticatedRoute.get( @@ -749,7 +748,7 @@ authenticatedRoute.put( userToApprove: req.params.userId }; }, - 'APPROVE USER' + 'USER APPROVE' ) ); diff --git a/backend/src/api/logs.ts b/backend/src/api/logs.ts index d2ad80f0..c2136ed2 100644 --- a/backend/src/api/logs.ts +++ b/backend/src/api/logs.ts @@ -1,13 +1,177 @@ +import { SelectQueryBuilder } from 'typeorm'; import { Log } from '../models'; -import { wrapHandler } from './helpers'; +import { validateBody, wrapHandler } from './helpers'; +import { IsDate, IsOptional, IsString } from 'class-validator'; + +type ParsedQuery = { + [key: string]: string | ParsedQuery; +}; + +const parseQueryString = (query: string): ParsedQuery => { + // Parses a query string that is used to search the JSON payload of a record + // Example => createdUserPayload.userId: 123124121424 + const result: ParsedQuery = {}; + + const parts = query.match(/(\w+(\.\w+)*):\s*[^:]+/g); + + if (!parts) { + return result; + } + + parts.forEach((part) => { + const [key, value] = part.split(/:(.+)/); + + if (!key || value === undefined) return; + + const keyParts = key.trim().split('.'); + let current = result; + + keyParts.forEach((part, index) => { + if (index === keyParts.length - 1) { + current[part] = value.trim(); + } else { + if (!current[part]) { + current[part] = {}; + } + current = current[part] as ParsedQuery; + } + }); + }); + + return result; +}; + +const generateSqlConditions = ( + parsedQuery: ParsedQuery, + jsonPath: string[] = [] +): string[] => { + const conditions: string[] = []; + + for (const [key, value] of Object.entries(parsedQuery)) { + if (typeof value === 'object') { + const newPath = [...jsonPath, key]; + conditions.push(...generateSqlConditions(value, newPath)); + } else { + const jsonField = + jsonPath.length > 0 + ? `${jsonPath.map((path) => `'${path}'`).join('->')}->>'${key}'` + : `'${key}'`; + conditions.push( + `payload ${ + jsonPath.length > 0 ? '->' : '->>' + } ${jsonField} = '${value}'` + ); + } + } + + return conditions; +}; +class Filter { + @IsString() + value: string; + + @IsString() + operator?: string; +} + +class DateFilter { + @IsDate() + value: string; + + @IsString() + operator: + | 'is' + | 'not' + | 'after' + | 'onOrAfter' + | 'before' + | 'onOrBefore' + | 'empty' + | 'notEmpty'; +} +class LogSearch { + @IsOptional() + eventType?: Filter; + @IsOptional() + result?: Filter; + @IsOptional() + timestamp?: Filter; + @IsOptional() + payload?: Filter; +} + +const generateDateCondition = (filter: DateFilter): string => { + const { operator } = filter; + + switch (operator) { + case 'is': + return `log.createdAt = :timestamp`; + case 'not': + return `log.createdAt != :timestamp`; + case 'after': + return `log.createdAt > :timestamp`; + case 'onOrAfter': + return `log.createdAt >= :timestamp`; + case 'before': + return `log.createdAt < :timestamp`; + case 'onOrBefore': + return `log.createdAt <= :timestamp`; + case 'empty': + return `log.createdAt IS NULL`; + case 'notEmpty': + return `log.createdAt IS NOT NULL`; + default: + throw new Error('Invalid operator'); + } +}; + +const filterResultQueryset = async (qs: SelectQueryBuilder, filters) => { + if (filters?.eventType) { + qs.andWhere('log.eventType ILIKE :eventType', { + eventType: `%${filters?.eventType?.value}%` + }); + } + if (filters?.result) { + qs.andWhere('log.result ILIKE :result', { + result: `%${filters?.result?.value}%` + }); + } + if (filters?.payload) { + try { + const parsedQuery = parseQueryString(filters?.payload?.value); + const conditions = generateSqlConditions(parsedQuery); + qs.andWhere(conditions[0]); + } catch (error) {} + } + + if (filters?.timestamp) { + const timestampCondition = generateDateCondition(filters?.timestamp); + let date; + try { + date = new Date(filters?.timestamp?.value); + } catch (error) {} + qs.andWhere(timestampCondition, { + timestamp: new Date(filters?.timestamp?.value) + }); + } + + return qs; +}; + +export const list = wrapHandler(async (event) => { + const search = await validateBody(LogSearch, event.body); + + const qs = Log.createQueryBuilder('log'); + + const filterQs = await filterResultQueryset(qs, search); + + const [results, resultsCount] = await filterQs.getManyAndCount(); -export const list = wrapHandler(async () => { - const logs = await Log.find(); return { statusCode: 200, body: JSON.stringify({ - result: logs, - count: logs.length + result: results, + count: resultsCount }) }; }); diff --git a/backend/src/tools/logger.ts b/backend/src/tools/logger.ts index 5087f577..147ff918 100644 --- a/backend/src/tools/logger.ts +++ b/backend/src/tools/logger.ts @@ -58,7 +58,8 @@ export class Logger { } const parsedResponseBody = - typeof responseBody === 'string' + typeof responseBody === 'string' && + responseBody !== 'User registration approved.' ? JSON.parse(responseBody) : responseBody; diff --git a/frontend/src/components/Logs/Logs.tsx b/frontend/src/components/Logs/Logs.tsx index 3c42db09..25e1a240 100644 --- a/frontend/src/components/Logs/Logs.tsx +++ b/frontend/src/components/Logs/Logs.tsx @@ -1,26 +1,83 @@ +import { + Dialog, + DialogContent, + DialogTitle, + Icon, + IconButton +} from '@mui/material'; import { Box } from '@mui/system'; -import { DataGrid, GridColDef } from '@mui/x-data-grid'; +import { + DataGrid, + GridColDef, + GridFilterItem, + GridRenderEditCellParams, + GridToolbar, + GridToolbarColumnsButton, + GridToolbarDensitySelector, + GridToolbarFilterButton +} from '@mui/x-data-grid'; import { useAuthContext } from 'context'; import { differenceInCalendarDays, parseISO } from 'date-fns'; import React, { FC, useCallback, useEffect, useState } from 'react'; interface LogsProps {} +interface LogDetails { + createdAt: string; + eventType: string; + result: string; + payload: string; +} + +const CustomToolbar = () => { + return ( + + + + + + ); +}; + export const Logs: FC = () => { - const { apiGet } = useAuthContext(); - const [logs, setLogs] = useState<{ count: Number; result: [] }>({ + const { apiPost } = useAuthContext(); + const [filters, setFilters] = useState>([]); + const [openDialog, setOpenDialog] = useState(false); + const [dialogDetails, setDialogDetails] = useState< + (LogDetails & { id: number }) | null + >(null); + const [logs, setLogs] = useState<{ + count: Number; + result: Array; + }>({ count: 0, result: [] }); const fetchLogs = useCallback(async () => { - const results = await apiGet('/logs'); + const tableFilters = filters.reduce( + (acc: { [key: string]: { value: any; operator: any } }, cur) => { + return { + ...acc, + [cur.field]: { + value: cur.value, + operator: cur.operator + } + }; + }, + {} + ); + const results = await apiPost('/logs/search', { + body: { + ...tableFilters + } + }); setLogs(results); - }, []); + }, [apiPost, filters]); useEffect(() => { fetchLogs(); - }, []); + }, [fetchLogs]); const logCols: GridColDef[] = [ { @@ -38,6 +95,7 @@ export const Logs: FC = () => { { field: 'createdAt', headerName: 'Timestamp', + type: 'dateTime', minWidth: 100, flex: 1, valueFormatter: (e) => { @@ -50,6 +108,8 @@ export const Logs: FC = () => { { field: 'payload', headerName: 'Payload', + description: 'Click any payload cell to expand.', + sortable: false, minWidth: 300, flex: 2, renderCell: (cellValues) => { @@ -71,12 +131,72 @@ export const Logs: FC = () => { valueFormatter: (e) => { return JSON.stringify(e.value, null, 2); } + }, + { + field: 'details', + headerName: 'Details', + maxWidth: 70, + flex: 1, + renderCell: (cellValues: GridRenderEditCellParams) => { + return ( + { + setOpenDialog(true); + setDialogDetails(cellValues.row); + }} + > + info + + ); + } } ]; + useEffect(() => { + fetchLogs(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [filters]); + return ( - + { + setFilters(model.items); + }} + /> + setOpenDialog(false)} + scroll="paper" + fullWidth + maxWidth="lg" + > + Payload Details + + + 
{JSON.stringify(dialogDetails?.payload, null, 2)}
+ + + ); }; diff --git a/frontend/src/pages/Risk/Risk.tsx b/frontend/src/pages/Risk/Risk.tsx index ca04692e..02aac9f1 100644 --- a/frontend/src/pages/Risk/Risk.tsx +++ b/frontend/src/pages/Risk/Risk.tsx @@ -102,6 +102,7 @@ const Risk: React.FC = ({ filters, addFilter }) => { .range(['#c7e8ff', '#135787']); setStats(result); }, + // eslint-disable-next-line react-hooks/exhaustive-deps [riskFilters] ); From 05edd51a93cff505fda34747c5f6067f95307d26 Mon Sep 17 00:00:00 2001 From: Janson Bunce Date: Thu, 26 Sep 2024 11:22:53 -0700 Subject: [PATCH 08/11] remove dev silliness --- backend/src/api/app.ts | 14 -------------- backend/src/api/users.ts | 14 +++++++------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index d3ca160e..5326fd81 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -197,15 +197,8 @@ app.post('/auth/okta-callback', async (req, res) => { const clientId = process.env.REACT_APP_COGNITO_CLIENT_ID; const callbackUrl = process.env.REACT_APP_COGNITO_CALLBACK_URL; const domain = process.env.REACT_APP_COGNITO_DOMAIN; - const logger = new Logger(req); if (!code) { - // logger.record('USER LOGIN', 'fail', (req, user) => { - // return { - // timestamp: new Date(), - // trace: console.trace() - // }; - // }); return res.status(400).json({ message: 'Missing authorization code' }); } @@ -288,13 +281,6 @@ app.post('/auth/okta-callback', async (req, res) => { res.cookie('id_token', signedToken, { httpOnly: true, secure: true }); - // logger.record('USER LOGIN', 'success', (req, us) => { - // console.log('HERE', { req, us }); - // return { - // timestamp: new Date(), - // userId: user?.id - // }; - // }); return res.status(200).json({ token: signedToken, user: user diff --git a/backend/src/api/users.ts b/backend/src/api/users.ts index 20b4c808..da35e748 100644 --- a/backend/src/api/users.ts +++ b/backend/src/api/users.ts @@ -635,13 +635,13 @@ export const registrationApproval = wrapHandler(async (event) => { } // Send email notification - // await sendRegistrationApprovedEmail( - // user.email, - // 'CyHy Dashboard Registration Approved', - // user.firstName, - // user.lastName, - // 'crossfeed_approval_notification.html' - // ); + await sendRegistrationApprovedEmail( + user.email, + 'CyHy Dashboard Registration Approved', + user.firstName, + user.lastName, + 'crossfeed_approval_notification.html' + ); // TODO: Handle Response Output return { From e54876d3b960c9534731f11e43607facf7a16ef4 Mon Sep 17 00:00:00 2001 From: Janson Bunce Date: Fri, 27 Sep 2024 09:43:45 -0700 Subject: [PATCH 09/11] add paper to ui - remove console.log --- backend/src/api/app.ts | 8 +-- frontend/src/components/Logs/Logs.tsx | 75 ++++++++++++++------------- 2 files changed, 43 insertions(+), 40 deletions(-) diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index 5326fd81..5f1c6c19 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -74,7 +74,7 @@ const handlerToExpress = const parsedBody = JSON.parse(sanitizer.sanitize(body)); res.status(200).json(parsedBody); } catch (e) { - // Not a JSON body + // Not valid JSON - may be a string response. console.log('Error?', e); res.setHeader('content-type', 'text/plain'); res.status(statusCode).send(sanitizer.sanitize(body)); @@ -692,9 +692,10 @@ authenticatedRoute.delete( handlerToExpress( users.del, (req, user, res) => { - console.log(req.params); return { - timestamp: new Date() + timestamp: new Date(), + userPerformedRemoval: user.data?.id, + userRemoved: req.params.userId }; }, 'USER DENY/REMOVE' @@ -727,7 +728,6 @@ authenticatedRoute.put( handlerToExpress( users.registrationApproval, (req, user) => { - console.log('here', req.params); return { timestamp: new Date(), userId: user?.data?.id, diff --git a/frontend/src/components/Logs/Logs.tsx b/frontend/src/components/Logs/Logs.tsx index 25e1a240..57278e2e 100644 --- a/frontend/src/components/Logs/Logs.tsx +++ b/frontend/src/components/Logs/Logs.tsx @@ -3,7 +3,8 @@ import { DialogContent, DialogTitle, Icon, - IconButton + IconButton, + Paper } from '@mui/material'; import { Box } from '@mui/system'; import { @@ -162,41 +163,43 @@ export const Logs: FC = () => { return ( - { - setFilters(model.items); - }} - /> - setOpenDialog(false)} - scroll="paper" - fullWidth - maxWidth="lg" - > - Payload Details - - - 
{JSON.stringify(dialogDetails?.payload, null, 2)}
- - - + + { + setFilters(model.items); + }} + /> + setOpenDialog(false)} + scroll="paper" + fullWidth + maxWidth="lg" + > + Payload Details + + + 
{JSON.stringify(dialogDetails?.payload, null, 2)}
+ + + + ); }; From b6d7fcaf44d6804552b433d43e4493547d87fa99 Mon Sep 17 00:00:00 2001 From: Janson Bunce Date: Fri, 27 Sep 2024 09:49:35 -0700 Subject: [PATCH 10/11] removed unused date assignment --- backend/src/api/logs.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/src/api/logs.ts b/backend/src/api/logs.ts index c2136ed2..75087058 100644 --- a/backend/src/api/logs.ts +++ b/backend/src/api/logs.ts @@ -146,9 +146,7 @@ const filterResultQueryset = async (qs: SelectQueryBuilder, filters) => { if (filters?.timestamp) { const timestampCondition = generateDateCondition(filters?.timestamp); - let date; try { - date = new Date(filters?.timestamp?.value); } catch (error) {} qs.andWhere(timestampCondition, { timestamp: new Date(filters?.timestamp?.value) From 4dbd6bc7e45949e0b01a971dab0633fb8defe9bb Mon Sep 17 00:00:00 2001 From: Janson Bunce Date: Fri, 27 Sep 2024 11:19:02 -0700 Subject: [PATCH 11/11] expands payloads to allow users to search by name and org name --- backend/src/api/app.ts | 48 ++++++++++++++++++++++++++++++++----- backend/src/tools/logger.ts | 4 ++-- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index 5f1c6c19..a05b5ab7 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -23,7 +23,7 @@ import * as reports from './reports'; import * as savedSearches from './saved-searches'; import rateLimit from 'express-rate-limit'; import { createProxyMiddleware } from 'http-proxy-middleware'; -import { User, UserType, connectToDatabase } from '../models'; +import { Organization, User, UserType, connectToDatabase } from '../models'; import * as assessments from './assessments'; import * as jwt from 'jsonwebtoken'; import { Request, Response, NextFunction } from 'express'; @@ -637,14 +637,28 @@ authenticatedRoute.post( '/v2/organizations/:organizationId/users', handlerToExpress( organizations.addUserV2, - (req, user) => { + async (req, user) => { + const orgId = req?.params?.organizationId; + const userId = req?.body?.userId; + const role = req?.body?.role; + if (orgId && userId) { + const orgRecord = await Organization.findOne({ where: { id: orgId } }); + const userRecord = await User.findOne({ where: { id: userId } }); + return { + timestamp: new Date(), + userPerformedAssignment: user?.data?.id, + organization: orgRecord, + role: role, + user: userRecord + }; + } return { timestamp: new Date(), userId: user?.data?.id, updatePayload: req.body }; }, - 'USER UPDATE' + 'USER ASSIGNED' ) ); @@ -675,7 +689,17 @@ authenticatedRoute.post( '/users', handlerToExpress( users.invite, - (req, user, responseBody) => { + async (req, user, responseBody) => { + const userId = user?.data?.id; + if (userId) { + const userRecord = await User.findOne({ where: { id: userId } }); + return { + timestamp: new Date(), + userPerformedInvite: userRecord, + invitePayload: req.body, + createdUserRecord: responseBody + }; + } return { timestamp: new Date(), userId: user.data?.id, @@ -691,7 +715,19 @@ authenticatedRoute.delete( '/users/:userId', handlerToExpress( users.del, - (req, user, res) => { + async (req, user, res) => { + const userId = req?.params?.userId; + const userPerformedRemovalId = user?.data?.id; + if (userId && userPerformedRemovalId) { + const userPerformdRemovalRecord = await User.findOne({ + where: { id: userPerformedRemovalId } + }); + return { + timestamp: new Date(), + userPerformedRemoval: userPerformdRemovalRecord, + userRemoved: userId + }; + } return { timestamp: new Date(), userPerformedRemoval: user.data?.id, @@ -727,7 +763,7 @@ authenticatedRoute.put( checkGlobalAdminOrRegionAdmin, handlerToExpress( users.registrationApproval, - (req, user) => { + async (req, user) => { return { timestamp: new Date(), userId: user?.data?.id, diff --git a/backend/src/tools/logger.ts b/backend/src/tools/logger.ts index 147ff918..df414c8e 100644 --- a/backend/src/tools/logger.ts +++ b/backend/src/tools/logger.ts @@ -27,7 +27,7 @@ export type RecordMessage = request: Request, user: LoggerUserState, responseBody?: object - ) => RecordPayload) + ) => Promise) | RecordPayload; export class Logger { @@ -65,7 +65,7 @@ export class Logger { const payload = typeof messageOrCB === 'function' - ? messageOrCB(this.request, this.user, parsedResponseBody) + ? await messageOrCB(this.request, this.user, parsedResponseBody) : messageOrCB; const logRecord = await this.logRep.create({ payload: payload as object,