diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index 5c5dba9e..a05b5ab7 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -13,6 +13,7 @@ import * as search from './search'; import * as vulnerabilities from './vulnerabilities'; import * as organizations from './organizations'; import * as scans from './scans'; +import * as logs from './logs'; import * as users from './users'; import * as scanTasks from './scan-tasks'; import * as stats from './stats'; @@ -22,12 +23,13 @@ import * as reports from './reports'; import * as savedSearches from './saved-searches'; import rateLimit from 'express-rate-limit'; import { createProxyMiddleware } from 'http-proxy-middleware'; -import { User, UserType, connectToDatabase } from '../models'; +import { Organization, User, UserType, connectToDatabase } from '../models'; import * as assessments from './assessments'; import * as jwt from 'jsonwebtoken'; import { Request, Response, NextFunction } from 'express'; import fetch from 'node-fetch'; import * as searchOrganizations from './organizationSearch'; +import { Logger, RecordMessage } from '../tools/logger'; const sanitizer = require('sanitizer'); @@ -43,27 +45,41 @@ if ( setInterval(() => scheduler({}, {} as any, () => null), 30000); } -const handlerToExpress = (handler) => async (req, res) => { - const { statusCode, body } = await handler( - { - pathParameters: req.params, - query: req.query, - requestContext: req.requestContext, - body: JSON.stringify(req.body || '{}'), - headers: req.headers, - path: req.originalUrl - }, - {} - ); - try { - const parsedBody = JSON.parse(sanitizer.sanitize(body)); - res.status(statusCode).json(parsedBody); - } catch (e) { - // Not a JSON body - res.setHeader('content-type', 'text/plain'); - res.status(statusCode).send(sanitizer.sanitize(body)); - } -}; +const handlerToExpress = + (handler, message?: RecordMessage, action?: string) => async (req, res) => { + const logger = new Logger(req); + const { statusCode, body } = await handler( + { + pathParameters: req.params, + query: req.query, + requestContext: req.requestContext, + body: JSON.stringify(req.body || '{}'), + headers: req.headers, + path: req.originalUrl + }, + {} + ); + // Add additional status codes that we may return for succesfull requests + if (statusCode === 200) { + if (message && action) { + logger.record(action, 'success', message, body); + } + } else { + if (message && action) { + logger.record(action, 'fail', message, body); + } + } + + try { + const parsedBody = JSON.parse(sanitizer.sanitize(body)); + res.status(200).json(parsedBody); + } catch (e) { + // Not valid JSON - may be a string response. + console.log('Error?', e); + res.setHeader('content-type', 'text/plain'); + res.status(statusCode).send(sanitizer.sanitize(body)); + } + }; const app = express(); @@ -561,6 +577,7 @@ authenticatedRoute.delete( handlerToExpress(savedSearches.del) ); authenticatedRoute.get('/scans', handlerToExpress(scans.list)); +authenticatedRoute.post('/logs/search', handlerToExpress(logs.list)); authenticatedRoute.get('/granularScans', handlerToExpress(scans.listGranular)); authenticatedRoute.post('/scans', handlerToExpress(scans.create)); authenticatedRoute.get('/scans/:scanId', handlerToExpress(scans.get)); @@ -618,12 +635,39 @@ authenticatedRoute.delete( ); authenticatedRoute.post( '/v2/organizations/:organizationId/users', - handlerToExpress(organizations.addUserV2) + handlerToExpress( + organizations.addUserV2, + async (req, user) => { + const orgId = req?.params?.organizationId; + const userId = req?.body?.userId; + const role = req?.body?.role; + if (orgId && userId) { + const orgRecord = await Organization.findOne({ where: { id: orgId } }); + const userRecord = await User.findOne({ where: { id: userId } }); + return { + timestamp: new Date(), + userPerformedAssignment: user?.data?.id, + organization: orgRecord, + role: role, + user: userRecord + }; + } + return { + timestamp: new Date(), + userId: user?.data?.id, + updatePayload: req.body + }; + }, + 'USER ASSIGNED' + ) ); + authenticatedRoute.post( '/organizations/:organizationId/roles/:roleId/approve', handlerToExpress(organizations.approveRole) ); + +// TO-DO Add logging => /users => user has an org and you change them to a new organization authenticatedRoute.post( '/organizations/:organizationId/roles/:roleId/remove', handlerToExpress(organizations.removeRole) @@ -641,9 +685,58 @@ authenticatedRoute.post( handlerToExpress(organizations.checkDomainVerification) ); authenticatedRoute.post('/stats', handlerToExpress(stats.get)); -authenticatedRoute.post('/users', handlerToExpress(users.invite)); +authenticatedRoute.post( + '/users', + handlerToExpress( + users.invite, + async (req, user, responseBody) => { + const userId = user?.data?.id; + if (userId) { + const userRecord = await User.findOne({ where: { id: userId } }); + return { + timestamp: new Date(), + userPerformedInvite: userRecord, + invitePayload: req.body, + createdUserRecord: responseBody + }; + } + return { + timestamp: new Date(), + userId: user.data?.id, + invitePayload: req.body, + createdUserRecord: responseBody + }; + }, + 'USER INVITE' + ) +); authenticatedRoute.get('/users', handlerToExpress(users.list)); -authenticatedRoute.delete('/users/:userId', handlerToExpress(users.del)); +authenticatedRoute.delete( + '/users/:userId', + handlerToExpress( + users.del, + async (req, user, res) => { + const userId = req?.params?.userId; + const userPerformedRemovalId = user?.data?.id; + if (userId && userPerformedRemovalId) { + const userPerformdRemovalRecord = await User.findOne({ + where: { id: userPerformedRemovalId } + }); + return { + timestamp: new Date(), + userPerformedRemoval: userPerformdRemovalRecord, + userRemoved: userId + }; + } + return { + timestamp: new Date(), + userPerformedRemoval: user.data?.id, + userRemoved: req.params.userId + }; + }, + 'USER DENY/REMOVE' + ) +); authenticatedRoute.get( '/users/state/:state', handlerToExpress(users.getByState) @@ -668,7 +761,17 @@ authenticatedRoute.post( authenticatedRoute.put( '/users/:userId/register/approve', checkGlobalAdminOrRegionAdmin, - handlerToExpress(users.registrationApproval) + handlerToExpress( + users.registrationApproval, + async (req, user) => { + return { + timestamp: new Date(), + userId: user?.data?.id, + userToApprove: req.params.userId + }; + }, + 'USER APPROVE' + ) ); authenticatedRoute.put( diff --git a/backend/src/api/logs.ts b/backend/src/api/logs.ts new file mode 100644 index 00000000..75087058 --- /dev/null +++ b/backend/src/api/logs.ts @@ -0,0 +1,175 @@ +import { SelectQueryBuilder } from 'typeorm'; +import { Log } from '../models'; +import { validateBody, wrapHandler } from './helpers'; +import { IsDate, IsOptional, IsString } from 'class-validator'; + +type ParsedQuery = { + [key: string]: string | ParsedQuery; +}; + +const parseQueryString = (query: string): ParsedQuery => { + // Parses a query string that is used to search the JSON payload of a record + // Example => createdUserPayload.userId: 123124121424 + const result: ParsedQuery = {}; + + const parts = query.match(/(\w+(\.\w+)*):\s*[^:]+/g); + + if (!parts) { + return result; + } + + parts.forEach((part) => { + const [key, value] = part.split(/:(.+)/); + + if (!key || value === undefined) return; + + const keyParts = key.trim().split('.'); + let current = result; + + keyParts.forEach((part, index) => { + if (index === keyParts.length - 1) { + current[part] = value.trim(); + } else { + if (!current[part]) { + current[part] = {}; + } + current = current[part] as ParsedQuery; + } + }); + }); + + return result; +}; + +const generateSqlConditions = ( + parsedQuery: ParsedQuery, + jsonPath: string[] = [] +): string[] => { + const conditions: string[] = []; + + for (const [key, value] of Object.entries(parsedQuery)) { + if (typeof value === 'object') { + const newPath = [...jsonPath, key]; + conditions.push(...generateSqlConditions(value, newPath)); + } else { + const jsonField = + jsonPath.length > 0 + ? `${jsonPath.map((path) => `'${path}'`).join('->')}->>'${key}'` + : `'${key}'`; + conditions.push( + `payload ${ + jsonPath.length > 0 ? '->' : '->>' + } ${jsonField} = '${value}'` + ); + } + } + + return conditions; +}; +class Filter { + @IsString() + value: string; + + @IsString() + operator?: string; +} + +class DateFilter { + @IsDate() + value: string; + + @IsString() + operator: + | 'is' + | 'not' + | 'after' + | 'onOrAfter' + | 'before' + | 'onOrBefore' + | 'empty' + | 'notEmpty'; +} +class LogSearch { + @IsOptional() + eventType?: Filter; + @IsOptional() + result?: Filter; + @IsOptional() + timestamp?: Filter; + @IsOptional() + payload?: Filter; +} + +const generateDateCondition = (filter: DateFilter): string => { + const { operator } = filter; + + switch (operator) { + case 'is': + return `log.createdAt = :timestamp`; + case 'not': + return `log.createdAt != :timestamp`; + case 'after': + return `log.createdAt > :timestamp`; + case 'onOrAfter': + return `log.createdAt >= :timestamp`; + case 'before': + return `log.createdAt < :timestamp`; + case 'onOrBefore': + return `log.createdAt <= :timestamp`; + case 'empty': + return `log.createdAt IS NULL`; + case 'notEmpty': + return `log.createdAt IS NOT NULL`; + default: + throw new Error('Invalid operator'); + } +}; + +const filterResultQueryset = async (qs: SelectQueryBuilder, filters) => { + if (filters?.eventType) { + qs.andWhere('log.eventType ILIKE :eventType', { + eventType: `%${filters?.eventType?.value}%` + }); + } + if (filters?.result) { + qs.andWhere('log.result ILIKE :result', { + result: `%${filters?.result?.value}%` + }); + } + if (filters?.payload) { + try { + const parsedQuery = parseQueryString(filters?.payload?.value); + const conditions = generateSqlConditions(parsedQuery); + qs.andWhere(conditions[0]); + } catch (error) {} + } + + if (filters?.timestamp) { + const timestampCondition = generateDateCondition(filters?.timestamp); + try { + } catch (error) {} + qs.andWhere(timestampCondition, { + timestamp: new Date(filters?.timestamp?.value) + }); + } + + return qs; +}; + +export const list = wrapHandler(async (event) => { + const search = await validateBody(LogSearch, event.body); + + const qs = Log.createQueryBuilder('log'); + + const filterQs = await filterResultQueryset(qs, search); + + const [results, resultsCount] = await filterQs.getManyAndCount(); + + return { + statusCode: 200, + body: JSON.stringify({ + result: results, + count: resultsCount + }) + }; +}); diff --git a/backend/src/models/connection.ts b/backend/src/models/connection.ts index 8f2cfdbf..a32ba88d 100644 --- a/backend/src/models/connection.ts +++ b/backend/src/models/connection.ts @@ -21,6 +21,7 @@ import { User, Vulnerability, Webpage, + Log, // Models for the Mini Data Lake database CertScan, @@ -194,7 +195,8 @@ const connectDb = async (logging?: boolean) => { Service, User, Vulnerability, - Webpage + Webpage, + Log ], synchronize: false, name: 'default', diff --git a/backend/src/models/index.ts b/backend/src/models/index.ts index 85627560..d0b3814f 100644 --- a/backend/src/models/index.ts +++ b/backend/src/models/index.ts @@ -19,6 +19,7 @@ export * from './service'; export * from './user'; export * from './vulnerability'; export * from './webpage'; +export * from './log'; // Mini data lake models export * from './mini_data_lake/cert_scans'; export * from './mini_data_lake/cidrs'; diff --git a/backend/src/models/log.ts b/backend/src/models/log.ts new file mode 100644 index 00000000..2d128b18 --- /dev/null +++ b/backend/src/models/log.ts @@ -0,0 +1,19 @@ +import { BaseEntity, Column, Entity, PrimaryGeneratedColumn } from 'typeorm'; + +@Entity() +export class Log extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column('json') + payload: Object; + + @Column({ nullable: false }) + createdAt: Date; + + @Column({ nullable: true }) + eventType: string; + + @Column({ nullable: false }) + result: string; +} diff --git a/backend/src/tools/logger.ts b/backend/src/tools/logger.ts new file mode 100644 index 00000000..df414c8e --- /dev/null +++ b/backend/src/tools/logger.ts @@ -0,0 +1,120 @@ +import { Request } from 'express'; +import { decode } from 'jsonwebtoken'; +import { User } from '../models'; +import { attempt, unescape } from 'lodash'; +import { Log } from '../models/log'; +import { getRepository, Repository } from 'typeorm'; + +type AccessTokenPayload = { + id: string; + email: string; + iat: string; + exp: string; +}; + +type LoggerUserState = { + data: User | undefined; + ready: boolean; + attempts: number; +}; + +type RecordPayload = object & { + timestamp: Date; +}; + +export type RecordMessage = + | (( + request: Request, + user: LoggerUserState, + responseBody?: object + ) => Promise) + | RecordPayload; + +export class Logger { + private request: Request; + private logId: string; + private token: AccessTokenPayload | undefined; + private user: LoggerUserState = { + data: undefined, + ready: false, + attempts: 0 + }; + private logRep: Repository; + + async record( + action: string, + result: 'success' | 'fail', + messageOrCB: RecordMessage | undefined, + responseBody?: object | string + ) { + try { + if (!this.user.ready && this.user.attempts > 0) { + await this.fetchUser(); + } + + if (!this.logRep) { + const logRepository = getRepository(Log); + this.logRep = logRepository; + } + + const parsedResponseBody = + typeof responseBody === 'string' && + responseBody !== 'User registration approved.' + ? JSON.parse(responseBody) + : responseBody; + + const payload = + typeof messageOrCB === 'function' + ? await messageOrCB(this.request, this.user, parsedResponseBody) + : messageOrCB; + const logRecord = await this.logRep.create({ + payload: payload as object, + createdAt: payload?.timestamp, + result: result, + eventType: action + }); + + logRecord.save(); + } catch (error) { + console.warn('Error occured in loggingMiddleware', error); + } + } + + async fetchUser() { + if (this.token) { + const user = await User.findOne({ id: this.token.id }); + if (user) { + this.user = { + data: user, + ready: true, + attempts: 0 + }; + } + this.user = { + data: undefined, + ready: false, + attempts: this.user.attempts + 1 + }; + } + } + + // Constructor takes a request and sets it to a class variable + constructor(req: Request) { + this.request = req; + this.logId = '123123123123'; + const authToken = req.headers.authorization; + if (authToken) { + const tokenPayload = decode( + authToken as string + ) as unknown as AccessTokenPayload; + this.token = tokenPayload; + User.findOne({ id: this.token.id }).then((user) => { + if (user) { + this.user = { data: user, ready: true, attempts: 0 }; + } + }); + } + } +} + +// Database Tables diff --git a/frontend/src/components/Logs/Logs.tsx b/frontend/src/components/Logs/Logs.tsx new file mode 100644 index 00000000..57278e2e --- /dev/null +++ b/frontend/src/components/Logs/Logs.tsx @@ -0,0 +1,205 @@ +import { + Dialog, + DialogContent, + DialogTitle, + Icon, + IconButton, + Paper +} from '@mui/material'; +import { Box } from '@mui/system'; +import { + DataGrid, + GridColDef, + GridFilterItem, + GridRenderEditCellParams, + GridToolbar, + GridToolbarColumnsButton, + GridToolbarDensitySelector, + GridToolbarFilterButton +} from '@mui/x-data-grid'; +import { useAuthContext } from 'context'; +import { differenceInCalendarDays, parseISO } from 'date-fns'; +import React, { FC, useCallback, useEffect, useState } from 'react'; + +interface LogsProps {} + +interface LogDetails { + createdAt: string; + eventType: string; + result: string; + payload: string; +} + +const CustomToolbar = () => { + return ( + + + + + + ); +}; + +export const Logs: FC = () => { + const { apiPost } = useAuthContext(); + const [filters, setFilters] = useState>([]); + const [openDialog, setOpenDialog] = useState(false); + const [dialogDetails, setDialogDetails] = useState< + (LogDetails & { id: number }) | null + >(null); + const [logs, setLogs] = useState<{ + count: Number; + result: Array; + }>({ + count: 0, + result: [] + }); + + const fetchLogs = useCallback(async () => { + const tableFilters = filters.reduce( + (acc: { [key: string]: { value: any; operator: any } }, cur) => { + return { + ...acc, + [cur.field]: { + value: cur.value, + operator: cur.operator + } + }; + }, + {} + ); + const results = await apiPost('/logs/search', { + body: { + ...tableFilters + } + }); + setLogs(results); + }, [apiPost, filters]); + + useEffect(() => { + fetchLogs(); + }, [fetchLogs]); + + const logCols: GridColDef[] = [ + { + field: 'eventType', + headerName: 'Event', + minWidth: 100, + flex: 1 + }, + { + field: 'result', + headerName: 'Result', + minWidth: 100, + flex: 1 + }, + { + field: 'createdAt', + headerName: 'Timestamp', + type: 'dateTime', + minWidth: 100, + flex: 1, + valueFormatter: (e) => { + return `${differenceInCalendarDays( + Date.now(), + parseISO(e.value) + )} days ago`; + } + }, + { + field: 'payload', + headerName: 'Payload', + description: 'Click any payload cell to expand.', + sortable: false, + minWidth: 300, + flex: 2, + renderCell: (cellValues) => { + return ( + +
{JSON.stringify(cellValues.row.payload, null, 2)}
+
+ ); + }, + valueFormatter: (e) => { + return JSON.stringify(e.value, null, 2); + } + }, + { + field: 'details', + headerName: 'Details', + maxWidth: 70, + flex: 1, + renderCell: (cellValues: GridRenderEditCellParams) => { + return ( + { + setOpenDialog(true); + setDialogDetails(cellValues.row); + }} + > + info + + ); + } + } + ]; + + useEffect(() => { + fetchLogs(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [filters]); + + return ( + + + { + setFilters(model.items); + }} + /> + setOpenDialog(false)} + scroll="paper" + fullWidth + maxWidth="lg" + > + Payload Details + + +
{JSON.stringify(dialogDetails?.payload, null, 2)}
+
+
+
+
+
+ ); +}; diff --git a/frontend/src/pages/AdminTools/AdminTools.tsx b/frontend/src/pages/AdminTools/AdminTools.tsx index e40cfc3a..4e068099 100644 --- a/frontend/src/pages/AdminTools/AdminTools.tsx +++ b/frontend/src/pages/AdminTools/AdminTools.tsx @@ -5,6 +5,7 @@ import ScansView from 'pages/Scans/ScansView'; import ScanTasksView from 'pages/Scans/ScanTasksView'; import { Box, Container, Tab } from '@mui/material'; import { TabContext, TabList, TabPanel } from '@mui/lab'; +import { Logs } from 'components/Logs/Logs'; export const AdminTools: React.FC = () => { const [value, setValue] = React.useState('1'); @@ -21,6 +22,7 @@ export const AdminTools: React.FC = () => { + @@ -33,6 +35,9 @@ export const AdminTools: React.FC = () => { + + + diff --git a/frontend/src/pages/Risk/Risk.tsx b/frontend/src/pages/Risk/Risk.tsx index cd071458..02aac9f1 100644 --- a/frontend/src/pages/Risk/Risk.tsx +++ b/frontend/src/pages/Risk/Risk.tsx @@ -102,7 +102,8 @@ const Risk: React.FC = ({ filters, addFilter }) => { .range(['#c7e8ff', '#135787']); setStats(result); }, - [apiPost, riskFilters] + // eslint-disable-next-line react-hooks/exhaustive-deps + [riskFilters] ); useEffect(() => {