From cff0fa8e314af3e6ac11d676f38fe1cd12db7b97 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Wed, 5 Apr 2023 12:15:26 +0100 Subject: [PATCH 01/22] chore(deps): bump node images to 18.15.0 (#1500) --- packages/preview-service/Dockerfile | 4 ++-- packages/server/Dockerfile | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/preview-service/Dockerfile b/packages/preview-service/Dockerfile index 467f7b0f5c..3e00fd7eaa 100644 --- a/packages/preview-service/Dockerfile +++ b/packages/preview-service/Dockerfile @@ -1,7 +1,7 @@ # NOTE: Docker context should be set to git root directory, to include the viewer ARG NODE_ENV=production -FROM node:18.14.1-bullseye-slim as build-stage +FROM node:18.15.0-bullseye-slim as build-stage ARG NODE_ENV ENV NODE_ENV=${NODE_ENV} @@ -35,7 +35,7 @@ COPY packages/preview-service ./packages/preview-service/ # This way the foreach only builds the frontend and its deps RUN yarn workspaces foreach run build -FROM node:18.14.1-bullseye-slim as node +FROM node:18.15.0-bullseye-slim as node RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index 617fb7820d..b344ad6e37 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -1,7 +1,7 @@ ARG NODE_ENV=production ARG SPECKLE_SERVER_VERSION=custom -FROM node:18.14.1-bullseye-slim as build-stage +FROM node:18.15.0-bullseye-slim as build-stage ARG NODE_ENV ARG SPECKLE_SERVER_VERSION WORKDIR /speckle-server @@ -38,7 +38,7 @@ RUN yarn workspaces foreach run build # install only production dependencies # we need a clean environment, free of build dependencies -FROM node:18.14.1-bullseye-slim as dependency-stage +FROM node:18.15.0-bullseye-slim as dependency-stage ARG NODE_ENV ARG SPECKLE_SERVER_VERSION @@ -54,7 +54,7 @@ COPY packages/objectloader/package.json ./packages/objectloader/ WORKDIR /speckle-server/packages/server RUN yarn workspaces focus --production -FROM node:18.14.1-bullseye-slim as production-stage +FROM node:18.15.0-bullseye-slim as production-stage ARG NODE_ENV ARG SPECKLE_SERVER_VERSION ARG FILE_SIZE_LIMIT_MB=100 From 2a35fe6178efbe64a4f8bc5384267a5fa86fe43a Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Fri, 7 Apr 2023 12:49:45 +0100 Subject: [PATCH 02/22] fix(ratelimit): reduce /graphql limit based on incident (#1505) --- packages/server/modules/core/services/ratelimiter.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/modules/core/services/ratelimiter.ts b/packages/server/modules/core/services/ratelimiter.ts index 41769e45f1..1665ca9544 100644 --- a/packages/server/modules/core/services/ratelimiter.ts +++ b/packages/server/modules/core/services/ratelimiter.ts @@ -170,11 +170,11 @@ export const LIMITS: RateLimiterOptions = { }, 'POST /graphql': { regularOptions: { - limitCount: getIntFromEnv('RATELIMIT_POST_GRAPHQL', '50'), + limitCount: getIntFromEnv('RATELIMIT_POST_GRAPHQL', '10'), duration: 1 * TIME.second }, burstOptions: { - limitCount: getIntFromEnv('RATELIMIT_BURST_POST_GRAPHQL', '200'), + limitCount: getIntFromEnv('RATELIMIT_BURST_POST_GRAPHQL', '20'), duration: 1 * TIME.minute } }, From 59c0b7ada08ebb8d49a0eec6600f32a72811b94c Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Fri, 7 Apr 2023 15:17:03 +0100 Subject: [PATCH 03/22] feat(helm): rate limits are configurable via helm chart (#1507) * feat(helm): rate limits are configurable via helm chart * Document server.monitoring.mp * Update schema.json --- .../templates/server/deployment.yml | 90 +++++++++++++ utils/helm/speckle-server/values.schema.json | 120 ++++++++++++++++++ utils/helm/speckle-server/values.yaml | 46 +++++++ 3 files changed, 256 insertions(+) diff --git a/utils/helm/speckle-server/templates/server/deployment.yml b/utils/helm/speckle-server/templates/server/deployment.yml index 5639bd0f9a..98451e5eaa 100644 --- a/utils/helm/speckle-server/templates/server/deployment.yml +++ b/utils/helm/speckle-server/templates/server/deployment.yml @@ -287,6 +287,96 @@ spec: name: {{ default .Values.secretName .Values.server.monitoring.apollo.key.secretName }} key: {{ default "apollo_key" .Values.server.monitoring.apollo.key.secretKey }} {{- end }} + + # Rate Limiting + {{- if .Values.server.ratelimiting.all_requests }} + - name: RATELIMIT_ALL_REQUESTS + value: "{{ .Values.server.ratelimiting.all_requests }}" + {{- end }} + {{- if .Values.server.ratelimiting.burst_all_requests }} + - name: RATELIMIT_BURST_ALL_REQUESTS + value: "{{ .Values.server.ratelimiting.burst_all_requests }}" + {{- end }} + {{- if .Values.server.ratelimiting.user_create }} + - name: RATELIMIT_USER_CREATE + value: "{{ .Values.server.ratelimiting.user_create }}" + {{- end }} + {{- if .Values.server.ratelimiting.burst_user_create }} + - name: RATELIMIT_BURST_USER_CREATE + value: "{{ .Values.server.ratelimiting.burst_user_create }}" + {{- end }} + {{- if .Values.server.ratelimiting.stream_create }} + - name: RATELIMIT_STREAM_CREATE + value: "{{ .Values.server.ratelimiting.stream_create }}" + {{- end }} + {{- if .Values.server.ratelimiting.burst_stream_create }} + - name: RATELIMIT_BURST_STREAM_CREATE + value: "{{ .Values.server.ratelimiting.burst_stream_create }}" + {{- end }} + {{- if .Values.server.ratelimiting.commit_create }} + - name: RATELIMIT_COMMIT_CREATE + value: "{{ .Values.server.ratelimiting.commit_create }}" + {{- end }} + {{- if .Values.server.ratelimiting.burst_commit_create }} + - name: RATELIMIT_BURST_COMMIT_CREATE + value: "{{ .Values.server.ratelimiting.burst_commit_create }}" + {{- end }} + {{- if .Values.server.ratelimiting.post_getobjects_streamid }} + - name: RATELIMIT_POST_GETOBJECTS_STREAMID + value: "{{ .Values.server.ratelimiting.post_getobjects_streamid }}" + {{- end }} + {{- if .Values.server.ratelimiting.burst_post_getobjects_streamid }} + - name: RATELIMIT_BURST_POST_GETOBJECTS_STREAMID + value: "{{ .Values.server.ratelimiting.burst_post_getobjects_streamid }}" + {{- end }} + {{- if .Values.server.ratelimiting.post_diff_streamid }} + - name: RATELIMIT_POST_DIFF_STREAMID + value: "{{ .Values.server.ratelimiting.post_diff_streamid }}" + {{- end }} + {{- if .Values.server.ratelimiting.burst_post_diff_streamid }} + - name: RATELIMIT_BURST_POST_DIFF_STREAMID + value: "{{ .Values.server.ratelimiting.burst_post_diff_streamid }}" + {{- end }} + {{- if .Values.server.ratelimiting.post_objects_streamid }} + - name: RATELIMIT_POST_OBJECTS_STREAMID + value: "{{ .Values.server.ratelimiting.post_objects_streamid }}" + {{- end }} + {{- if .Values.server.ratelimiting.burst_post_objects_streamid }} + - name: RATELIMIT_BURST_POST_OBJECTS_STREAMID + value: "{{ .Values.server.ratelimiting.burst_post_objects_streamid }}" + {{- end }} + {{- if .Values.server.ratelimiting.get_objects_streamid_objectid }} + - name: RATELIMIT_GET_OBJECTS_STREAMID_OBJECTID + value: "{{ .Values.server.ratelimiting.get_objects_streamid_objectid }}" + {{- end }} + {{- if .Values.server.ratelimiting.burst_get_objects_streamid_objectid }} + - name: RATELIMIT_BURST_GET_OBJECTS_STREAMID_OBJECTID + value: "{{ .Values.server.ratelimiting.burst_get_objects_streamid_objectid }}" + {{- end }} + {{- if .Values.server.ratelimiting.get_objects_streamid_objectid_single }} + - name: RATELIMIT_GET_OBJECTS_STREAMID_OBJECTID_SINGLE + value: "{{ .Values.server.ratelimiting.get_objects_streamid_objectid_single }}" + {{- end }} + {{- if .Values.server.ratelimiting.burst_get_objects_streamid_objectid_single }} + - name: RATELIMIT_BURST_GET_OBJECTS_STREAMID_OBJECTID_SINGLE + value: "{{ .Values.server.ratelimiting.burst_get_objects_streamid_objectid_single }}" + {{- end }} + {{- if .Values.server.ratelimiting.post_graphql }} + - name: RATELIMIT_POST_GRAPHQL + value: "{{ .Values.server.ratelimiting.post_graphql }}" + {{- end }} + {{- if .Values.server.ratelimiting.burst_post_graphql }} + - name: RATELIMIT_BURST_POST_GRAPHQL + value: "{{ .Values.server.ratelimiting.burst_post_graphql }}" + {{- end }} + {{- if .Values.server.ratelimiting.get_auth }} + - name: RATELIMIT_GET_AUTH + value: "{{ .Values.server.ratelimiting.get_auth }}" + {{- end }} + {{- if .Values.server.ratelimiting.burst_get_auth }} + - name: RATELIMIT_BURST_GET_AUTH + value: "{{ .Values.server.ratelimiting.burst_get_auth }}" + {{- end }} {{- if .Values.server.affinity }} affinity: {{- include "speckle.renderTpl" (dict "value" .Values.server.affinity "context" $) | nindent 8 }} {{- end }} diff --git a/utils/helm/speckle-server/values.schema.json b/utils/helm/speckle-server/values.schema.json index b4e3390693..87e8a200c3 100644 --- a/utils/helm/speckle-server/values.schema.json +++ b/utils/helm/speckle-server/values.schema.json @@ -763,6 +763,121 @@ } } }, + "ratelimiting": { + "type": "object", + "properties": { + "all_requests": { + "type": "number", + "description": "The maximum number of requests that can be made to the Speckle server in a moving one second window.", + "default": 500 + }, + "burst_all_requests": { + "type": "number", + "description": "If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server in a moving one minute window.", + "default": 2000 + }, + "user_create": { + "type": "number", + "description": "The maximum number of requests that can be made to the Speckle server to create a new user in a moving one second window.", + "default": 6 + }, + "burst_user_create": { + "type": "number", + "description": "If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to create a new user in a moving one minute window.", + "default": 1000 + }, + "stream_create": { + "type": "number", + "description": "The maximum number of requests that can be made to the Speckle server to create a new stream in a moving one second window.", + "default": 1 + }, + "burst_stream_create": { + "type": "number", + "description": "If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to create a new stream in a moving one minute window.", + "default": 100 + }, + "commit_create": { + "type": "number", + "description": "The maximum number of requests that can be made to the Speckle server to create a new commit in a moving one second window.", + "default": 1 + }, + "burst_commit_create": { + "type": "number", + "description": "If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to create a new commit in a moving one minute window.", + "default": 100 + }, + "post_getobjects_streamid": { + "type": "number", + "description": "The maximum number of requests that can be made to the Speckle server to get a new object in a moving one second window.", + "default": 3 + }, + "burst_post_getobjects_streamid": { + "type": "number", + "description": "If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to get a new object in a moving one minute window.", + "default": 200 + }, + "post_diff_streamid": { + "type": "number", + "description": "The maximum number of requests that can be made to the Speckle server to undertake a diff in a moving one second window.", + "default": 10 + }, + "burst_post_diff_streamid": { + "type": "number", + "description": "If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to undertake a diff in a moving one minute window.", + "default": 1000 + }, + "post_objects_streamid": { + "type": "number", + "description": "The maximum number of requests that can be made to the Speckle server to post a new object in a moving one second window.", + "default": 6 + }, + "burst_post_objects_streamid": { + "type": "number", + "description": "If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to post a new object in a moving one minute window.", + "default": 400 + }, + "get_objects_streamid_objectid": { + "type": "number", + "description": "The maximum number of requests that can be made to the Speckle server to get an object in a moving one second window.", + "default": 3 + }, + "burst_get_objects_streamid_objectid": { + "type": "number", + "description": "If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to get an object in a moving one minute window.", + "default": 200 + }, + "get_objects_streamid_objectid_single": { + "type": "number", + "description": "The maximum number of requests that can be made to the Speckle server to get a single object in a moving one second window.", + "default": 3 + }, + "burst_get_objects_streamid_objectid_single": { + "type": "number", + "description": "If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to get a single object in a moving one minute window.", + "default": 200 + }, + "post_graphql": { + "type": "number", + "description": "The maximum number of requests that can be made to the GraphQL API in a moving one second window.", + "default": 10 + }, + "burst_post_grapqhl": { + "type": "number", + "description": "If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the GraphQL API in a moving one minute window.", + "default": 20 + }, + "get_auth": { + "type": "number", + "description": "The maximum number of requests that can be made to the Speckle server to authenticate in a moving one second window.", + "default": 2 + }, + "burst_get_auth": { + "type": "number", + "description": "If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to authenticate in a moving one minute window.", + "default": 20 + } + } + }, "serviceAccount": { "type": "object", "properties": { @@ -815,6 +930,11 @@ } } } + }, + "mp": { + "type": "object", + "description": "(Optional) If server.monitoring.mp.enabled is set to false, metrics will not be collected by the Speckle server.", + "default": {} } } }, diff --git a/utils/helm/speckle-server/values.yaml b/utils/helm/speckle-server/values.yaml index b814641371..fd4797b389 100644 --- a/utils/helm/speckle-server/values.yaml +++ b/utils/helm/speckle-server/values.yaml @@ -549,6 +549,51 @@ server: ## ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ ## memory: 3Gi + ratelimiting: + ## @param server.ratelimiting.all_requests The maximum number of requests that can be made to the Speckle server in a moving one second window. + all_requests: 500 + ## @param server.ratelimiting.burst_all_requests If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server in a moving one minute window. + burst_all_requests: 2000 + ## @param server.ratelimiting.user_create The maximum number of requests that can be made to the Speckle server to create a new user in a moving one second window. + user_create: 6 + ## @param server.ratelimiting.burst_user_create If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to create a new user in a moving one minute window. + burst_user_create: 1000 + ## @param server.ratelimiting.stream_create The maximum number of requests that can be made to the Speckle server to create a new stream in a moving one second window. + stream_create: 1 + ## @param server.ratelimiting.burst_stream_create If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to create a new stream in a moving one minute window. + burst_stream_create: 100 + ## @param server.ratelimiting.commit_create The maximum number of requests that can be made to the Speckle server to create a new commit in a moving one second window. + commit_create: 1 + ## @param server.ratelimiting.burst_commit_create If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to create a new commit in a moving one minute window. + burst_commit_create: 100 + ## @param server.ratelimiting.post_getobjects_streamid The maximum number of requests that can be made to the Speckle server to get a new object in a moving one second window. + post_getobjects_streamid: 3 + ## @param server.ratelimiting.burst_post_getobjects_streamid If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to get a new object in a moving one minute window. + burst_post_getobjects_streamid: 200 + ## @param server.ratelimiting.post_diff_streamid The maximum number of requests that can be made to the Speckle server to undertake a diff in a moving one second window. + post_diff_streamid: 10 + ## @param server.ratelimiting.burst_post_diff_streamid If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to undertake a diff in a moving one minute window. + burst_post_diff_streamid: 1000 + ## @param server.ratelimiting.post_objects_streamid The maximum number of requests that can be made to the Speckle server to post a new object in a moving one second window. + post_objects_streamid: 6 + ## @param server.ratelimiting.burst_post_objects_streamid If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to post a new object in a moving one minute window. + burst_post_objects_streamid: 400 + ## @param server.ratelimiting.get_objects_streamid_objectid The maximum number of requests that can be made to the Speckle server to get an object in a moving one second window. + get_objects_streamid_objectid: 3 + ## @param server.ratelimiting.burst_get_objects_streamid_objectid If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to get an object in a moving one minute window. + burst_get_objects_streamid_objectid: 200 + ## @param server.ratelimiting.get_objects_streamid_objectid_single The maximum number of requests that can be made to the Speckle server to get a single object in a moving one second window. + get_objects_streamid_objectid_single: 3 + ## @param server.ratelimiting.burst_get_objects_streamid_objectid_single If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to get a single object in a moving one minute window. + burst_get_objects_streamid_objectid_single: 200 + ## @param server.ratelimiting.post_graphql The maximum number of requests that can be made to the GraphQL API in a moving one second window. + post_graphql: 10 + ## @param server.ratelimiting.burst_post_grapqhl If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the GraphQL API in a moving one minute window. + burst_post_grapqhl: 20 + ## @param server.ratelimiting.get_auth The maximum number of requests that can be made to the Speckle server to authenticate in a moving one second window. + get_auth: 2 + ## @param server.ratelimiting.burst_get_auth If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to authenticate in a moving one minute window. + burst_get_auth: 20 serviceAccount: ## @param server.serviceAccount.create If enabled, a Kubernetes Service Account will be created for this pod. ## This provides additional security by limiting this pod's access to the Kubernetes API and to Secrets on the Kubernetes cluster. @@ -578,6 +623,7 @@ server: ## @param server.monitoring.apollo.key.secretKey The key within the Kubernetes Secret holding the Apollo monitoring key as its value. ## secretKey: '' + ## @param server.monitoring.mp (Optional) If server.monitoring.mp.enabled is set to false, metrics will not be collected by the Speckle server. mp: {} ## @param server.sentry_dns (Optional) The Data Source Name that was provided by Sentry.io ## Sentry.io allows events within Speckle to be monitored From ed005f8e896ee26c4435d3ecfa5f3c67a97e18e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Apr 2023 10:59:48 +0100 Subject: [PATCH 04/22] build(deps-dev): bump vite from 2.9.8 to 3.1.0 (#975) * build(deps-dev): bump vite from 2.9.8 to 2.9.13 Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 2.9.8 to 2.9.13. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/main/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v2.9.13/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: direct:development ... Signed-off-by: dependabot[bot] * Bumps viewer-sandbox vite to match frontend version * It was complaining about some html tags thing. Fixed it --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Iain Sproat <68657+iainsproat@users.noreply.github.com> Co-authored-by: AlexandruPopovici --- packages/viewer-sandbox/index.html | 2 +- packages/viewer-sandbox/package.json | 2 +- yarn.lock | 262 +-------------------------- 3 files changed, 6 insertions(+), 260 deletions(-) diff --git a/packages/viewer-sandbox/index.html b/packages/viewer-sandbox/index.html index 742881361b..fd0deee753 100644 --- a/packages/viewer-sandbox/index.html +++ b/packages/viewer-sandbox/index.html @@ -8,7 +8,7 @@ -
+
diff --git a/packages/viewer-sandbox/package.json b/packages/viewer-sandbox/package.json index 0c3f800a9f..ab78d52aaa 100644 --- a/packages/viewer-sandbox/package.json +++ b/packages/viewer-sandbox/package.json @@ -29,6 +29,6 @@ "eslint": "^8.11.0", "eslint-config-prettier": "^8.5.0", "typescript": "^4.5.4", - "vite": "^2.9.13" + "vite": "^3.1.0" } } diff --git a/yarn.lock b/yarn.lock index f0bee06213..3571f55755 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5375,7 +5375,7 @@ __metadata: eslint-config-prettier: ^8.5.0 tweakpane: ^3.0.8 typescript: ^4.5.4 - vite: ^2.9.13 + vite: ^3.1.0 languageName: unknown linkType: soft @@ -10511,13 +10511,6 @@ __metadata: languageName: node linkType: hard -"esbuild-android-64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-android-64@npm:0.14.38" - conditions: os=android & cpu=x64 - languageName: node - linkType: hard - "esbuild-android-64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-android-64@npm:0.15.7" @@ -10525,13 +10518,6 @@ __metadata: languageName: node linkType: hard -"esbuild-android-arm64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-android-arm64@npm:0.14.38" - conditions: os=android & cpu=arm64 - languageName: node - linkType: hard - "esbuild-android-arm64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-android-arm64@npm:0.15.7" @@ -10539,13 +10525,6 @@ __metadata: languageName: node linkType: hard -"esbuild-darwin-64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-darwin-64@npm:0.14.38" - conditions: os=darwin & cpu=x64 - languageName: node - linkType: hard - "esbuild-darwin-64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-darwin-64@npm:0.15.7" @@ -10553,13 +10532,6 @@ __metadata: languageName: node linkType: hard -"esbuild-darwin-arm64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-darwin-arm64@npm:0.14.38" - conditions: os=darwin & cpu=arm64 - languageName: node - linkType: hard - "esbuild-darwin-arm64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-darwin-arm64@npm:0.15.7" @@ -10567,13 +10539,6 @@ __metadata: languageName: node linkType: hard -"esbuild-freebsd-64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-freebsd-64@npm:0.14.38" - conditions: os=freebsd & cpu=x64 - languageName: node - linkType: hard - "esbuild-freebsd-64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-freebsd-64@npm:0.15.7" @@ -10581,13 +10546,6 @@ __metadata: languageName: node linkType: hard -"esbuild-freebsd-arm64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-freebsd-arm64@npm:0.14.38" - conditions: os=freebsd & cpu=arm64 - languageName: node - linkType: hard - "esbuild-freebsd-arm64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-freebsd-arm64@npm:0.15.7" @@ -10595,13 +10553,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-32@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-linux-32@npm:0.14.38" - conditions: os=linux & cpu=ia32 - languageName: node - linkType: hard - "esbuild-linux-32@npm:0.15.7": version: 0.15.7 resolution: "esbuild-linux-32@npm:0.15.7" @@ -10609,13 +10560,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-linux-64@npm:0.14.38" - conditions: os=linux & cpu=x64 - languageName: node - linkType: hard - "esbuild-linux-64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-linux-64@npm:0.15.7" @@ -10623,13 +10567,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-arm64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-linux-arm64@npm:0.14.38" - conditions: os=linux & cpu=arm64 - languageName: node - linkType: hard - "esbuild-linux-arm64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-linux-arm64@npm:0.15.7" @@ -10637,13 +10574,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-arm@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-linux-arm@npm:0.14.38" - conditions: os=linux & cpu=arm - languageName: node - linkType: hard - "esbuild-linux-arm@npm:0.15.7": version: 0.15.7 resolution: "esbuild-linux-arm@npm:0.15.7" @@ -10651,13 +10581,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-mips64le@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-linux-mips64le@npm:0.14.38" - conditions: os=linux & cpu=mips64el - languageName: node - linkType: hard - "esbuild-linux-mips64le@npm:0.15.7": version: 0.15.7 resolution: "esbuild-linux-mips64le@npm:0.15.7" @@ -10665,13 +10588,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-ppc64le@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-linux-ppc64le@npm:0.14.38" - conditions: os=linux & cpu=ppc64 - languageName: node - linkType: hard - "esbuild-linux-ppc64le@npm:0.15.7": version: 0.15.7 resolution: "esbuild-linux-ppc64le@npm:0.15.7" @@ -10679,13 +10595,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-riscv64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-linux-riscv64@npm:0.14.38" - conditions: os=linux & cpu=riscv64 - languageName: node - linkType: hard - "esbuild-linux-riscv64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-linux-riscv64@npm:0.15.7" @@ -10693,13 +10602,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-s390x@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-linux-s390x@npm:0.14.38" - conditions: os=linux & cpu=s390x - languageName: node - linkType: hard - "esbuild-linux-s390x@npm:0.15.7": version: 0.15.7 resolution: "esbuild-linux-s390x@npm:0.15.7" @@ -10707,13 +10609,6 @@ __metadata: languageName: node linkType: hard -"esbuild-netbsd-64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-netbsd-64@npm:0.14.38" - conditions: os=netbsd & cpu=x64 - languageName: node - linkType: hard - "esbuild-netbsd-64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-netbsd-64@npm:0.15.7" @@ -10721,13 +10616,6 @@ __metadata: languageName: node linkType: hard -"esbuild-openbsd-64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-openbsd-64@npm:0.14.38" - conditions: os=openbsd & cpu=x64 - languageName: node - linkType: hard - "esbuild-openbsd-64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-openbsd-64@npm:0.15.7" @@ -10735,13 +10623,6 @@ __metadata: languageName: node linkType: hard -"esbuild-sunos-64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-sunos-64@npm:0.14.38" - conditions: os=sunos & cpu=x64 - languageName: node - linkType: hard - "esbuild-sunos-64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-sunos-64@npm:0.15.7" @@ -10749,13 +10630,6 @@ __metadata: languageName: node linkType: hard -"esbuild-windows-32@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-windows-32@npm:0.14.38" - conditions: os=win32 & cpu=ia32 - languageName: node - linkType: hard - "esbuild-windows-32@npm:0.15.7": version: 0.15.7 resolution: "esbuild-windows-32@npm:0.15.7" @@ -10763,13 +10637,6 @@ __metadata: languageName: node linkType: hard -"esbuild-windows-64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-windows-64@npm:0.14.38" - conditions: os=win32 & cpu=x64 - languageName: node - linkType: hard - "esbuild-windows-64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-windows-64@npm:0.15.7" @@ -10777,13 +10644,6 @@ __metadata: languageName: node linkType: hard -"esbuild-windows-arm64@npm:0.14.38": - version: 0.14.38 - resolution: "esbuild-windows-arm64@npm:0.14.38" - conditions: os=win32 & cpu=arm64 - languageName: node - linkType: hard - "esbuild-windows-arm64@npm:0.15.7": version: 0.15.7 resolution: "esbuild-windows-arm64@npm:0.15.7" @@ -10791,77 +10651,6 @@ __metadata: languageName: node linkType: hard -"esbuild@npm:^0.14.27": - version: 0.14.38 - resolution: "esbuild@npm:0.14.38" - dependencies: - esbuild-android-64: 0.14.38 - esbuild-android-arm64: 0.14.38 - esbuild-darwin-64: 0.14.38 - esbuild-darwin-arm64: 0.14.38 - esbuild-freebsd-64: 0.14.38 - esbuild-freebsd-arm64: 0.14.38 - esbuild-linux-32: 0.14.38 - esbuild-linux-64: 0.14.38 - esbuild-linux-arm: 0.14.38 - esbuild-linux-arm64: 0.14.38 - esbuild-linux-mips64le: 0.14.38 - esbuild-linux-ppc64le: 0.14.38 - esbuild-linux-riscv64: 0.14.38 - esbuild-linux-s390x: 0.14.38 - esbuild-netbsd-64: 0.14.38 - esbuild-openbsd-64: 0.14.38 - esbuild-sunos-64: 0.14.38 - esbuild-windows-32: 0.14.38 - esbuild-windows-64: 0.14.38 - esbuild-windows-arm64: 0.14.38 - dependenciesMeta: - esbuild-android-64: - optional: true - esbuild-android-arm64: - optional: true - esbuild-darwin-64: - optional: true - esbuild-darwin-arm64: - optional: true - esbuild-freebsd-64: - optional: true - esbuild-freebsd-arm64: - optional: true - esbuild-linux-32: - optional: true - esbuild-linux-64: - optional: true - esbuild-linux-arm: - optional: true - esbuild-linux-arm64: - optional: true - esbuild-linux-mips64le: - optional: true - esbuild-linux-ppc64le: - optional: true - esbuild-linux-riscv64: - optional: true - esbuild-linux-s390x: - optional: true - esbuild-netbsd-64: - optional: true - esbuild-openbsd-64: - optional: true - esbuild-sunos-64: - optional: true - esbuild-windows-32: - optional: true - esbuild-windows-64: - optional: true - esbuild-windows-arm64: - optional: true - bin: - esbuild: bin/esbuild - checksum: d7523a36bd28016c010829c527386dbc0c6b9f514920abf5ac8003f346665161aa61026fd6822c5091fc1c1af52fe26c9281a81740fc06f2994cdbb7c2880297 - languageName: node - linkType: hard - "esbuild@npm:^0.15.6": version: 0.15.7 resolution: "esbuild@npm:0.15.7" @@ -16767,7 +16556,7 @@ __metadata: languageName: node linkType: hard -"postcss@npm:^8.3.11, postcss@npm:^8.4.13": +"postcss@npm:^8.3.11": version: 8.4.13 resolution: "postcss@npm:8.4.13" dependencies: @@ -17653,7 +17442,7 @@ __metadata: languageName: node linkType: hard -"resolve@npm:^1.1.7, resolve@npm:^1.14.2, resolve@npm:^1.17.0, resolve@npm:^1.19.0, resolve@npm:^1.20.0, resolve@npm:^1.22.0, resolve@npm:^1.9.0": +"resolve@npm:^1.1.7, resolve@npm:^1.14.2, resolve@npm:^1.17.0, resolve@npm:^1.19.0, resolve@npm:^1.20.0, resolve@npm:^1.9.0": version: 1.22.0 resolution: "resolve@npm:1.22.0" dependencies: @@ -17679,7 +17468,7 @@ __metadata: languageName: node linkType: hard -"resolve@patch:resolve@^1.1.7#~builtin, resolve@patch:resolve@^1.14.2#~builtin, resolve@patch:resolve@^1.17.0#~builtin, resolve@patch:resolve@^1.19.0#~builtin, resolve@patch:resolve@^1.20.0#~builtin, resolve@patch:resolve@^1.22.0#~builtin, resolve@patch:resolve@^1.9.0#~builtin": +"resolve@patch:resolve@^1.1.7#~builtin, resolve@patch:resolve@^1.14.2#~builtin, resolve@patch:resolve@^1.17.0#~builtin, resolve@patch:resolve@^1.19.0#~builtin, resolve@patch:resolve@^1.20.0#~builtin, resolve@patch:resolve@^1.9.0#~builtin": version: 1.22.0 resolution: "resolve@patch:resolve@npm%3A1.22.0#~builtin::version=1.22.0&hash=07638b" dependencies: @@ -17906,20 +17695,6 @@ __metadata: languageName: node linkType: hard -"rollup@npm:>=2.59.0 <2.78.0": - version: 2.77.3 - resolution: "rollup@npm:2.77.3" - dependencies: - fsevents: ~2.3.2 - dependenciesMeta: - fsevents: - optional: true - bin: - rollup: dist/bin/rollup - checksum: b179c68249584565ddb5664a241e8e48c293b2207718d885b08ee25797d98857a383f06b544bb89819407da5a71557f4713309a278f61c4778bb32b1d3321a1c - languageName: node - linkType: hard - "rollup@npm:^2.70.1": version: 2.72.0 resolution: "rollup@npm:2.72.0" @@ -20138,35 +19913,6 @@ __metadata: languageName: node linkType: hard -"vite@npm:^2.9.13": - version: 2.9.15 - resolution: "vite@npm:2.9.15" - dependencies: - esbuild: ^0.14.27 - fsevents: ~2.3.2 - postcss: ^8.4.13 - resolve: ^1.22.0 - rollup: ">=2.59.0 <2.78.0" - peerDependencies: - less: "*" - sass: "*" - stylus: "*" - dependenciesMeta: - fsevents: - optional: true - peerDependenciesMeta: - less: - optional: true - sass: - optional: true - stylus: - optional: true - bin: - vite: bin/vite.js - checksum: 5edf8afc132a598f82a1339aa503514a0e86f6264babe94531b9000d801f157c9a32ae022542404678acca43ee260ade04c4763c4b7b8d381a6b50511b4a1447 - languageName: node - linkType: hard - "vite@npm:^3.1.0": version: 3.1.0 resolution: "vite@npm:3.1.0" From f4ce7a2a5d2eaa987e2f9d3862dcfbf1c40559a2 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Mon, 10 Apr 2023 11:00:44 +0100 Subject: [PATCH 05/22] chore(deps): remove unused aws-sdk (#1510) * Remove aws-sdk as it does not appear to be used --- packages/fileimport-service/package.json | 1 - yarn.lock | 116 +---------------------- 2 files changed, 1 insertion(+), 116 deletions(-) diff --git a/packages/fileimport-service/package.json b/packages/fileimport-service/package.json index 2b33545c06..56189ecd8f 100644 --- a/packages/fileimport-service/package.json +++ b/packages/fileimport-service/package.json @@ -24,7 +24,6 @@ }, "dependencies": { "@speckle/shared": "workspace:^", - "aws-sdk": "^2.1075.0", "bcrypt": "^5.0.1", "crypto-random-string": "^3.3.1", "knex": "^2.4.1", diff --git a/yarn.lock b/yarn.lock index 3571f55755..97d33ce149 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5053,7 +5053,6 @@ __metadata: resolution: "@speckle/fileimport-service@workspace:packages/fileimport-service" dependencies: "@speckle/shared": "workspace:^" - aws-sdk: ^2.1075.0 bcrypt: ^5.0.1 cross-env: ^7.0.3 crypto-random-string: ^3.3.1 @@ -7999,23 +7998,6 @@ __metadata: languageName: node linkType: hard -"aws-sdk@npm:^2.1075.0": - version: 2.1128.0 - resolution: "aws-sdk@npm:2.1128.0" - dependencies: - buffer: 4.9.2 - events: 1.1.1 - ieee754: 1.1.13 - jmespath: 0.16.0 - querystring: 0.2.0 - sax: 1.2.1 - url: 0.10.3 - uuid: 3.3.2 - xml2js: 0.4.19 - checksum: e821059e10138e77e16ab471f79d8abc3681bd8f00182a87315b718b3f82048650892347100823337e57a0f3b7830710b644315b159ce8ab8dcd9b8594145fb6 - languageName: node - linkType: hard - "aws-sign2@npm:~0.7.0": version: 0.7.0 resolution: "aws-sign2@npm:0.7.0" @@ -8412,17 +8394,6 @@ __metadata: languageName: node linkType: hard -"buffer@npm:4.9.2": - version: 4.9.2 - resolution: "buffer@npm:4.9.2" - dependencies: - base64-js: ^1.0.2 - ieee754: ^1.1.4 - isarray: ^1.0.0 - checksum: 8801bc1ba08539f3be70eee307a8b9db3d40f6afbfd3cf623ab7ef41dffff1d0a31de0addbe1e66e0ca5f7193eeb667bfb1ecad3647f8f1b0750de07c13295c3 - languageName: node - linkType: hard - "buffer@npm:5.6.0": version: 5.6.0 resolution: "buffer@npm:5.6.0" @@ -11001,13 +10972,6 @@ __metadata: languageName: node linkType: hard -"events@npm:1.1.1": - version: 1.1.1 - resolution: "events@npm:1.1.1" - checksum: 40431eb005cc4c57861b93d44c2981a49e7feb99df84cf551baed299ceea4444edf7744733f6a6667e942af687359b1f4a87ec1ec4f21d5127dac48a782039b9 - languageName: node - linkType: hard - "events@npm:3.3.0, events@npm:^3.2.0, events@npm:^3.3.0": version: 3.3.0 resolution: "events@npm:3.3.0" @@ -12710,13 +12674,6 @@ __metadata: languageName: node linkType: hard -"ieee754@npm:1.1.13": - version: 1.1.13 - resolution: "ieee754@npm:1.1.13" - checksum: 102df1ba662e316e6160f7ce29c7c7fa3e04f2014c288336c5a9ff40bbcc2a27d209fa2a81ebfb33f28b1941021343d30e9ad8ee85a2d61f79f5936c35edc33d - languageName: node - linkType: hard - "ieee754@npm:^1.1.13, ieee754@npm:^1.1.4, ieee754@npm:^1.2.1": version: 1.2.1 resolution: "ieee754@npm:1.2.1" @@ -13262,7 +13219,7 @@ __metadata: languageName: node linkType: hard -"isarray@npm:^1.0.0, isarray@npm:~1.0.0": +"isarray@npm:~1.0.0": version: 1.0.0 resolution: "isarray@npm:1.0.0" checksum: f032df8e02dce8ec565cf2eb605ea939bdccea528dbcf565cdf92bfa2da9110461159d86a537388ef1acef8815a330642d7885b29010e8f7eac967c9993b65ab @@ -13436,13 +13393,6 @@ __metadata: languageName: node linkType: hard -"jmespath@npm:0.16.0": - version: 0.16.0 - resolution: "jmespath@npm:0.16.0" - checksum: 2d602493a1e4addfd1350ac8c9d54b1b03ed09e305fd863bab84a4ee1f52868cf939dd1a08c5cdea29ce9ba8f86875ebb458b6ed45dab3e1c3f2694503fb2fd9 - languageName: node - linkType: hard - "join-images@npm:^1.1.3": version: 1.1.3 resolution: "join-images@npm:1.1.3" @@ -16900,13 +16850,6 @@ __metadata: languageName: node linkType: hard -"punycode@npm:1.3.2": - version: 1.3.2 - resolution: "punycode@npm:1.3.2" - checksum: b8807fd594b1db33335692d1f03e8beeddde6fda7fbb4a2e32925d88d20a3aa4cd8dcc0c109ccaccbd2ba761c208dfaaada83007087ea8bfb0129c9ef1b99ed6 - languageName: node - linkType: hard - "punycode@npm:^1.3.2": version: 1.4.1 resolution: "punycode@npm:1.4.1" @@ -16980,13 +16923,6 @@ __metadata: languageName: node linkType: hard -"querystring@npm:0.2.0": - version: 0.2.0 - resolution: "querystring@npm:0.2.0" - checksum: 8258d6734f19be27e93f601758858c299bdebe71147909e367101ba459b95446fbe5b975bf9beb76390156a592b6f4ac3a68b6087cea165c259705b8b4e56a69 - languageName: node - linkType: hard - "queue-microtask@npm:^1.2.2": version: 1.2.3 resolution: "queue-microtask@npm:1.2.3" @@ -17857,20 +17793,6 @@ __metadata: languageName: node linkType: hard -"sax@npm:1.2.1": - version: 1.2.1 - resolution: "sax@npm:1.2.1" - checksum: 8dca7d5e1cd7d612f98ac50bdf0b9f63fbc964b85f0c4e2eb271f8b9b47fd3bf344c4d6a592e69ecf726d1485ca62cd8a52e603bbc332d18a66af25a9a1045ad - languageName: node - linkType: hard - -"sax@npm:>=0.6.0": - version: 1.2.4 - resolution: "sax@npm:1.2.4" - checksum: d3df7d32b897a2c2f28e941f732c71ba90e27c24f62ee918bd4d9a8cfb3553f2f81e5493c7f0be94a11c1911b643a9108f231dd6f60df3fa9586b5d2e3e9e1fe - languageName: node - linkType: hard - "schema-utils@npm:^2.6.5": version: 2.7.1 resolution: "schema-utils@npm:2.7.1" @@ -19740,16 +19662,6 @@ __metadata: languageName: node linkType: hard -"url@npm:0.10.3": - version: 0.10.3 - resolution: "url@npm:0.10.3" - dependencies: - punycode: 1.3.2 - querystring: 0.2.0 - checksum: 7b83ddb106c27bf9bde8629ccbe8d26e9db789c8cda5aa7db72ca2c6f9b8a88a5adf206f3e10db78e6e2d042b327c45db34c7010c1bf0d9908936a17a2b57d05 - languageName: node - linkType: hard - "util-deprecate@npm:^1.0.1, util-deprecate@npm:^1.0.2, util-deprecate@npm:~1.0.1": version: 1.0.2 resolution: "util-deprecate@npm:1.0.2" @@ -19771,15 +19683,6 @@ __metadata: languageName: node linkType: hard -"uuid@npm:3.3.2": - version: 3.3.2 - resolution: "uuid@npm:3.3.2" - bin: - uuid: ./bin/uuid - checksum: 8793629d2799f500aeea9fcd0aec6c4e9fbcc4d62ed42159ad96be345c3fffac1bbf61a23e18e2782600884fee05e6d4012ce4b70d0037c8e987533ae6a77870 - languageName: node - linkType: hard - "uuid@npm:8.3.2, uuid@npm:^8.3.0, uuid@npm:^8.3.2": version: 8.3.2 resolution: "uuid@npm:8.3.2" @@ -20613,16 +20516,6 @@ __metadata: languageName: node linkType: hard -"xml2js@npm:0.4.19": - version: 0.4.19 - resolution: "xml2js@npm:0.4.19" - dependencies: - sax: ">=0.6.0" - xmlbuilder: ~9.0.1 - checksum: ca8b2fee430d450a18947786bfd7cd1a353ee00fc6fd550acbc8a8e65f1b4df5e9786fcb2990c1a5514ecd554d445fb74e1d716b3a4fcfffc10554aeb5db482b - languageName: node - linkType: hard - "xml@npm:^1.0.0": version: 1.0.1 resolution: "xml@npm:1.0.1" @@ -20630,13 +20523,6 @@ __metadata: languageName: node linkType: hard -"xmlbuilder@npm:~9.0.1": - version: 9.0.7 - resolution: "xmlbuilder@npm:9.0.7" - checksum: 8193bb323806a002764f013bea0c6e9ff2dc26fd29109408761b16b59a8ad2214c2abe8e691755fd8b525586e3a0e1efeb92335947d7b0899032b779f1705a53 - languageName: node - linkType: hard - "xss@npm:^1.0.8": version: 1.0.11 resolution: "xss@npm:1.0.11" From ead926dea66a9360815a971bb8469e47f7d65d9b Mon Sep 17 00:00:00 2001 From: Snyk bot Date: Mon, 10 Apr 2023 22:25:26 +0800 Subject: [PATCH 06/22] [Snyk] Security upgrade numpy from 1.21.3 to 1.22.2 (#1512) * fix: packages/fileimport-service/requirements.txt to reduce vulnerabilities The following vulnerabilities are fixed by pinning transitive dependencies: - https://snyk.io/vuln/SNYK-PYTHON-NUMPY-2321964 - https://snyk.io/vuln/SNYK-PYTHON-NUMPY-2321966 - https://snyk.io/vuln/SNYK-PYTHON-NUMPY-2321970 * chore(deps): file-import python requirements should include required peers * Improve comment on requirement --------- Co-authored-by: Iain Sproat <68657+iainsproat@users.noreply.github.com> --- packages/fileimport-service/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/fileimport-service/requirements.txt b/packages/fileimport-service/requirements.txt index e636dde721..e3135282d1 100644 --- a/packages/fileimport-service/requirements.txt +++ b/packages/fileimport-service/requirements.txt @@ -1,3 +1,5 @@ numpy-stl==3.0.0 specklepy==2.9.1 structlog==22.3.0 +numpy==1.24.2 # not directly required, pinned to avoid a vulnerability in <1.22.2 +python-util==1.2.1 # not directly required, peer dependency of numpy-stl From 6c66049248a81e32c49c7cbac75fd824709f8be1 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Tue, 11 Apr 2023 18:42:40 +0100 Subject: [PATCH 07/22] feat(logging): log apollo (graphql) requests and responses (#1509) * feat(logging): log apollo (graphql) requests and responses --- packages/server/logging/apolloPlugin.js | 40 ++++++++++++++++--- packages/server/logging/logging.ts | 1 + .../server/modules/shared/middleware/index.ts | 11 ++++- 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/packages/server/logging/apolloPlugin.js b/packages/server/logging/apolloPlugin.js index 218853cb84..81d83e2f3a 100644 --- a/packages/server/logging/apolloPlugin.js +++ b/packages/server/logging/apolloPlugin.js @@ -1,8 +1,8 @@ /* istanbul ignore file */ -// const { logger } = require('@/logging/logging') const Sentry = require('@sentry/node') const { ApolloError } = require('apollo-server-express') const prometheusClient = require('prom-client') +const { graphqlLogger } = require('@/logging/logging') const metricCallCount = new prometheusClient.Counter({ name: 'speckle_server_apollo_calls', @@ -20,13 +20,20 @@ module.exports = { return } + let logger = ctx.log || graphqlLogger + + const op = `GQL ${ctx.operation.operation} ${ctx.operation.selectionSet.selections[0].name.value}` + const name = `GQL ${ctx.operation.selectionSet.selections[0].name.value}` + logger = logger.child({ op, name }) + const transaction = Sentry.startTransaction({ - op: `GQL ${ctx.operation.operation} ${ctx.operation.selectionSet.selections[0].name.value}`, - name: `GQL ${ctx.operation.selectionSet.selections[0].name.value}` + op, + name }) try { const actionName = `${ctx.operation.operation} ${ctx.operation.selectionSet.selections[0].name.value}` + logger = logger.child({ actionName }) metricCallCount.labels(actionName).inc() // logger.debug(actionName) } catch (e) { @@ -35,18 +42,36 @@ module.exports = { Sentry.configureScope((scope) => scope.setSpan(transaction)) ctx.request.transaction = transaction + ctx.log = logger }, didEncounterErrors(ctx) { if (!ctx.operation) return + let logger = ctx.log || graphqlLogger + for (const err of ctx.errors) { if (err instanceof ApolloError) { continue } + + const kind = ctx.operation.operation + const query = ctx.request.query + const variables = ctx.request.variables + + logger = logger.child({ + kind, + query, + variables + }) + if (err.path) { + logger = logger.child({ 'query-path': err.path.join(' > ') }) + } + logger.error(err, 'graphql error') + Sentry.withScope((scope) => { - scope.setTag('kind', ctx.operation.operation) - scope.setExtra('query', ctx.request.query) - scope.setExtra('variables', ctx.request.variables) + scope.setTag('kind', kind) + scope.setExtra('query', query) + scope.setExtra('variables', variables) if (err.path) { // We can also add the path as breadcrumb scope.addBreadcrumb({ @@ -60,6 +85,9 @@ module.exports = { } }, willSendResponse(ctx) { + const logger = ctx.log || graphqlLogger + logger.info('graphql response') + if (ctx.request.transaction) { ctx.request.transaction.finish() } diff --git a/packages/server/logging/logging.ts b/packages/server/logging/logging.ts index 890ee93889..9b8759d315 100644 --- a/packages/server/logging/logging.ts +++ b/packages/server/logging/logging.ts @@ -25,3 +25,4 @@ export const servicesLogger = extendLoggerComponent(logger, 'services') export const rateLimiterLogger = extendLoggerComponent(logger, 'rate-limiter') export const redisLogger = extendLoggerComponent(logger, 'redis') export const mixpanelLogger = extendLoggerComponent(logger, 'mixpanel') +export const graphqlLogger = extendLoggerComponent(logger, 'graphql') diff --git a/packages/server/modules/shared/middleware/index.ts b/packages/server/modules/shared/middleware/index.ts index a94748fa13..61e6686604 100644 --- a/packages/server/modules/shared/middleware/index.ts +++ b/packages/server/modules/shared/middleware/index.ts @@ -19,6 +19,8 @@ import { import { getUser } from '@/modules/core/repositories/users' import { resolveMixpanelUserId } from '@speckle/shared' import { mixpanel } from '@/modules/shared/utils/mixpanel' +import { Observability } from '@speckle/shared' +import { pino } from 'pino' export const authMiddlewareCreator = (steps: AuthPipelineFunction[]) => { const pipeline = authPipelineCreator(steps) @@ -104,6 +106,8 @@ export function addLoadersToCtx(ctx: AuthContext): GraphQLContext { return { ...ctx, loaders } } +type ApolloContext = AuthContext & { log?: pino.Logger } + /** * Build context for GQL operations */ @@ -114,10 +118,15 @@ export async function buildContext({ req: MaybeNullOrUndefined token: Nullable }): Promise { - const ctx = + const ctx: ApolloContext = req?.context || (await createAuthContextFromToken(token ?? getTokenFromRequest(req))) + ctx.log = Observability.extendLoggerComponent( + req?.log || Observability.getLogger(), + 'graphql' + ) + // Adding request data loaders return addLoadersToCtx(ctx) } From 4c723781b5e36b7e1f2002a7147c7b47ac9152ee Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Tue, 11 Apr 2023 18:43:46 +0100 Subject: [PATCH 08/22] feat(server): authentication middleware should log auth context creation status (#1508) * feat(server): authentication middleware should log auth context creation status - this uses the pino http logger provided via prior express middleware, ensuring a request ID is associated with the log messages - userID, scopes and roles will be logged * Appends the authContext to the req.log, which makes it available on all subsequent calls --- packages/server/modules/shared/middleware/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/modules/shared/middleware/index.ts b/packages/server/modules/shared/middleware/index.ts index 61e6686604..6d2e53ad22 100644 --- a/packages/server/modules/shared/middleware/index.ts +++ b/packages/server/modules/shared/middleware/index.ts @@ -89,12 +89,14 @@ export async function authContextMiddleware( ) { const token = getTokenFromRequest(req) const authContext = await createAuthContextFromToken(token) + req.log = req.log.child({ authContext }) if (!authContext.auth && authContext.err) { let message = 'Unknown Auth context error' let status = 500 message = authContext.err?.message || message if (authContext.err instanceof UnauthorizedError) status = 401 if (authContext.err instanceof ForbiddenError) status = 403 + req.log.warn('Auth context creation failed.') return res.status(status).json({ error: message }) } req.context = authContext From d61138e1576bacb35e9e8792cb84034e100814af Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Tue, 11 Apr 2023 20:11:56 +0100 Subject: [PATCH 09/22] feat(log): logs all headers, except authorization and cookie (#1517) * feat(log): logs all headers, except authorization and cookie * fix(logging): print headers for raw response --- packages/server/logging/expressLogging.ts | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/packages/server/logging/expressLogging.ts b/packages/server/logging/expressLogging.ts index 219e02b5e9..753f988bf3 100644 --- a/packages/server/logging/expressLogging.ts +++ b/packages/server/logging/expressLogging.ts @@ -44,12 +44,11 @@ export const LoggingExpressMiddleware = HttpLogger({ method: req.raw.method, path: req.raw.url?.split('?')[0], // Remove query params which might be sensitive // Allowlist useful headers - headers: { - host: req.raw.headers.host, - 'user-agent': req.raw.headers['user-agent'], - 'x-request-id': req.raw.headers[REQUEST_ID_HEADER], - referer: req.raw.headers.referer - } + headers: Object.fromEntries( + Object.entries(req.raw.headers).filter( + ([key]) => !['cookie', 'authorization'].includes(key.toLocaleLowerCase()) + ) + ) } }), res: pino.stdSerializers.wrapResponseSerializer((res) => { @@ -61,15 +60,7 @@ export const LoggingExpressMiddleware = HttpLogger({ return { statusCode: res.raw.statusCode, // Allowlist useful headers - headers: { - 'content-length': resRaw.raw.headers['content-length'], - 'content-type': resRaw.raw.headers['content-type'], - 'retry-after': resRaw.raw.headers['retry-after'], - 'x-ratelimit-remaining': resRaw.raw.headers['x-ratelimit-remaining'], - 'x-ratelimit-reset': resRaw.raw.headers['x-ratelimit-reset'], - 'x-request-id': resRaw.raw.headers['x-request-id'], - 'x-speckle-meditation': resRaw.raw.headers['x-speckle-meditation'] - } + headers: resRaw.raw.headers } }) } From 84ea2b1043a917de0b26be8a7d76b15cb4fb7edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Jedlicska?= <57442769+gjedlicska@users.noreply.github.com> Date: Wed, 12 Apr 2023 14:39:03 +0200 Subject: [PATCH 10/22] fix(server): make sure apollo logging works and it doesn't leak sensitive stuff (#1520) --- packages/server/logging/apolloPlugin.js | 8 ++++---- packages/server/modules/shared/middleware/index.ts | 8 ++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/server/logging/apolloPlugin.js b/packages/server/logging/apolloPlugin.js index 81d83e2f3a..7212150720 100644 --- a/packages/server/logging/apolloPlugin.js +++ b/packages/server/logging/apolloPlugin.js @@ -20,7 +20,7 @@ module.exports = { return } - let logger = ctx.log || graphqlLogger + let logger = ctx.context.log || graphqlLogger const op = `GQL ${ctx.operation.operation} ${ctx.operation.selectionSet.selections[0].name.value}` const name = `GQL ${ctx.operation.selectionSet.selections[0].name.value}` @@ -42,12 +42,12 @@ module.exports = { Sentry.configureScope((scope) => scope.setSpan(transaction)) ctx.request.transaction = transaction - ctx.log = logger + ctx.context.log = logger }, didEncounterErrors(ctx) { if (!ctx.operation) return - let logger = ctx.log || graphqlLogger + let logger = ctx.context.log || graphqlLogger for (const err of ctx.errors) { if (err instanceof ApolloError) { @@ -85,7 +85,7 @@ module.exports = { } }, willSendResponse(ctx) { - const logger = ctx.log || graphqlLogger + const logger = ctx.context.log || graphqlLogger logger.info('graphql response') if (ctx.request.transaction) { diff --git a/packages/server/modules/shared/middleware/index.ts b/packages/server/modules/shared/middleware/index.ts index 6d2e53ad22..f3d5954610 100644 --- a/packages/server/modules/shared/middleware/index.ts +++ b/packages/server/modules/shared/middleware/index.ts @@ -89,14 +89,18 @@ export async function authContextMiddleware( ) { const token = getTokenFromRequest(req) const authContext = await createAuthContextFromToken(token) - req.log = req.log.child({ authContext }) + const loggedContext = Object.fromEntries( + Object.entries(authContext).filter( + ([key]) => !['token'].includes(key.toLocaleLowerCase()) + ) + ) + req.log = req.log.child({ authContext: loggedContext }) if (!authContext.auth && authContext.err) { let message = 'Unknown Auth context error' let status = 500 message = authContext.err?.message || message if (authContext.err instanceof UnauthorizedError) status = 401 if (authContext.err instanceof ForbiddenError) status = 403 - req.log.warn('Auth context creation failed.') return res.status(status).json({ error: message }) } req.context = authContext From 271888ccd6148e7479551b6858c51ebbcd83a8e7 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Wed, 12 Apr 2023 18:30:29 +0100 Subject: [PATCH 11/22] fix(log): graphql logger should log everything (#1521) - improve log field naming - remove duplicated configuring of log fields --- packages/server/logging/apolloPlugin.js | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/server/logging/apolloPlugin.js b/packages/server/logging/apolloPlugin.js index 7212150720..99c4f289f3 100644 --- a/packages/server/logging/apolloPlugin.js +++ b/packages/server/logging/apolloPlugin.js @@ -1,3 +1,4 @@ +/* eslint-disable camelcase */ /* istanbul ignore file */ const Sentry = require('@sentry/node') const { ApolloError } = require('apollo-server-express') @@ -24,7 +25,17 @@ module.exports = { const op = `GQL ${ctx.operation.operation} ${ctx.operation.selectionSet.selections[0].name.value}` const name = `GQL ${ctx.operation.selectionSet.selections[0].name.value}` - logger = logger.child({ op, name }) + const kind = ctx.operation.operation + const query = ctx.request.query + const variables = ctx.request.variables + + logger = logger.child({ + graphql_operation_kind: kind, + graphql_query: query, + graphql_variables: variables, + graphql_operation_value: op, + grqphql_operation_name: name + }) const transaction = Sentry.startTransaction({ op, @@ -58,11 +69,6 @@ module.exports = { const query = ctx.request.query const variables = ctx.request.variables - logger = logger.child({ - kind, - query, - variables - }) if (err.path) { logger = logger.child({ 'query-path': err.path.join(' > ') }) } From 477fc109d0e04c0eb901aa095346a79ff3519e74 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Thu, 13 Apr 2023 11:24:01 +0100 Subject: [PATCH 12/22] fix(logging): redact email values from graphql log messages (#1523) --- packages/server/.mocharc.js | 2 +- packages/server/logging/apolloPlugin.js | 3 +- packages/server/logging/loggingHelper.js | 36 +++++++++ packages/server/logging/loggingHelper.spec.js | 78 +++++++++++++++++++ 4 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 packages/server/logging/loggingHelper.js create mode 100644 packages/server/logging/loggingHelper.spec.js diff --git a/packages/server/.mocharc.js b/packages/server/.mocharc.js index cf18e32173..d5276047b7 100644 --- a/packages/server/.mocharc.js +++ b/packages/server/.mocharc.js @@ -2,7 +2,7 @@ /** @type {import("mocha").MochaOptions} */ const config = { - spec: ['modules/**/*.spec.js', 'modules/**/*.spec.ts'], + spec: ['modules/**/*.spec.js', 'modules/**/*.spec.ts', 'logging/**/*.spec.js'], require: ['ts-node/register', 'test/hooks.js'], slow: 0, timeout: '150000', diff --git a/packages/server/logging/apolloPlugin.js b/packages/server/logging/apolloPlugin.js index 99c4f289f3..0ed1e43011 100644 --- a/packages/server/logging/apolloPlugin.js +++ b/packages/server/logging/apolloPlugin.js @@ -4,6 +4,7 @@ const Sentry = require('@sentry/node') const { ApolloError } = require('apollo-server-express') const prometheusClient = require('prom-client') const { graphqlLogger } = require('@/logging/logging') +const { redactSensitiveVariables } = require('@/logging/loggingHelper') const metricCallCount = new prometheusClient.Counter({ name: 'speckle_server_apollo_calls', @@ -32,7 +33,7 @@ module.exports = { logger = logger.child({ graphql_operation_kind: kind, graphql_query: query, - graphql_variables: variables, + graphql_variables: redactSensitiveVariables(variables), graphql_operation_value: op, grqphql_operation_name: name }) diff --git a/packages/server/logging/loggingHelper.js b/packages/server/logging/loggingHelper.js new file mode 100644 index 0000000000..ba6bd65f49 --- /dev/null +++ b/packages/server/logging/loggingHelper.js @@ -0,0 +1,36 @@ +const redactSensitiveVariables = (variables) => { + if (!variables) { + return variables + } + + if (Array.isArray(variables)) { + return variables.map((v) => redactSensitiveVariables(v)) + } + + if (typeof variables !== 'object') { + return variables + } + + return Object.entries(variables).reduce((acc, [key, val]) => { + if (typeof val === 'object') { + acc[key] = redactSensitiveVariables(val) + return acc + } + + if ( + ['email', 'emailaddress', 'email_address', 'emails'].includes( + key.toLocaleLowerCase() + ) + ) { + acc[key] = '[REDACTED]' + return acc + } + + acc[key] = val + return acc + }, {}) +} + +module.exports = { + redactSensitiveVariables +} diff --git a/packages/server/logging/loggingHelper.spec.js b/packages/server/logging/loggingHelper.spec.js new file mode 100644 index 0000000000..bd21db14b5 --- /dev/null +++ b/packages/server/logging/loggingHelper.spec.js @@ -0,0 +1,78 @@ +const expect = require('chai').expect +const { redactSensitiveVariables } = require('@/logging/loggingHelper') + +describe('loggingHelper', () => { + describe('filterSensitiveVariables', () => { + it('should filter sensitive variables at root', () => { + const variables = { + email: 'exampleValue', + emailaddress: 'exampleValue', + // eslint-disable-next-line camelcase + email_address: 'exampleValue', + emails: 'exampleValue', + notsensitive: 'exampleValue' + } + const result = redactSensitiveVariables(variables) + expect(result).to.deep.equal({ + email: '[REDACTED]', + emailaddress: '[REDACTED]', + // eslint-disable-next-line camelcase + email_address: '[REDACTED]', + emails: '[REDACTED]', + notsensitive: 'exampleValue' + }) + }) + it('should filter nested sensitive variables', () => { + const variables = { + nest1: { + email: 'exampleValue', + emailaddress: 'exampleValue', + // eslint-disable-next-line camelcase + email_address: 'exampleValue', + emails: 'exampleValue' + } + } + const result = redactSensitiveVariables(variables) + expect(result).to.deep.equal({ + nest1: { + email: '[REDACTED]', + emailaddress: '[REDACTED]', + // eslint-disable-next-line camelcase + email_address: '[REDACTED]', + emails: '[REDACTED]' + } + }) + }) + it('should filter sensitive variables in tree structure', () => { + const variables = { + nest1: { + nest2: { + // eslint-disable-next-line camelcase + email_address: 'exampleValue', + emails: 'exampleValue', + notsensitive: 'exampleValue' + }, + nest3: { + email: 'exampleValue', + emailaddress: 'exampleValue' + } + } + } + const result = redactSensitiveVariables(variables) + expect(result).to.deep.equal({ + nest1: { + nest2: { + // eslint-disable-next-line camelcase + email_address: '[REDACTED]', + emails: '[REDACTED]', + notsensitive: 'exampleValue' + }, + nest3: { + email: '[REDACTED]', + emailaddress: '[REDACTED]' + } + } + }) + }) + }) +}) From 122f4c731fd188ff125d3eecd97c9ecf32e29908 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Thu, 13 Apr 2023 14:57:07 +0100 Subject: [PATCH 13/22] feat(log): log the ip address if a user is not logged in (#1527) - we do not log both the ip if the user is signed in, as this may be a privacy issue - the ip is only logged if there is no associated user information --- packages/server/app.ts | 2 ++ packages/server/logging/expressLogging.ts | 28 +++++++++++++++++-- .../server/modules/shared/middleware/index.ts | 19 +++++++++++++ packages/server/modules/shared/utils/ip.js | 9 +++++- 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/packages/server/app.ts b/packages/server/app.ts index 22ffaa143d..465613fd76 100644 --- a/packages/server/app.ts +++ b/packages/server/app.ts @@ -44,6 +44,7 @@ import { get, has, isString, toNumber } from 'lodash' import { authContextMiddleware, buildContext, + determineClientIpAddressMiddleware, mixpanelTrackerHelperMiddleware } from '@/modules/shared/middleware' @@ -205,6 +206,7 @@ export async function init() { // Log errors app.use(errorLoggingMiddleware) + app.use(determineClientIpAddressMiddleware) app.use(authContextMiddleware) app.use(createRateLimiterMiddleware()) app.use(mixpanelTrackerHelperMiddleware) diff --git a/packages/server/logging/expressLogging.ts b/packages/server/logging/expressLogging.ts index 753f988bf3..73e04e1abc 100644 --- a/packages/server/logging/expressLogging.ts +++ b/packages/server/logging/expressLogging.ts @@ -5,9 +5,15 @@ import { IncomingMessage } from 'http' import { NextFunction, Response } from 'express' import pino, { SerializedResponse } from 'pino' import { GenReqId } from 'pino-http' +import { X_SPECKLE_CLIENT_IP_HEADER } from '@/modules/shared/middleware' +import { AuthContext } from '@/modules/shared/authz' const REQUEST_ID_HEADER = 'x-request-id' +interface RawRequestWithAuthContext extends IncomingMessage { + context: AuthContext +} + const GenerateRequestId: GenReqId = (req: IncomingMessage) => DetermineRequestId(req) const DetermineRequestId = ( @@ -45,9 +51,25 @@ export const LoggingExpressMiddleware = HttpLogger({ path: req.raw.url?.split('?')[0], // Remove query params which might be sensitive // Allowlist useful headers headers: Object.fromEntries( - Object.entries(req.raw.headers).filter( - ([key]) => !['cookie', 'authorization'].includes(key.toLocaleLowerCase()) - ) + Object.entries(req.raw.headers).filter(([key]) => { + if ( + (req.raw as RawRequestWithAuthContext).context?.userId && + key.toLocaleLowerCase() === X_SPECKLE_CLIENT_IP_HEADER + ) { + // we cannot log the IP (even if hashed) if we also have a User ID + // Having both together could be used to link an IP to a specific User + // and that's a privacy issue. + return false + } + ![ + 'cookie', + 'authorization', + 'cf-connecting-ip', + 'true-client-ip', + 'x-real-ip', + 'x-forwarded-for' + ].includes(key.toLocaleLowerCase()) + }) ) } }), diff --git a/packages/server/modules/shared/middleware/index.ts b/packages/server/modules/shared/middleware/index.ts index f3d5954610..e80edfcbd0 100644 --- a/packages/server/modules/shared/middleware/index.ts +++ b/packages/server/modules/shared/middleware/index.ts @@ -21,6 +21,7 @@ import { resolveMixpanelUserId } from '@speckle/shared' import { mixpanel } from '@/modules/shared/utils/mixpanel' import { Observability } from '@speckle/shared' import { pino } from 'pino' +import { getIpFromRequest } from '@/modules/shared/utils/ip' export const authMiddlewareCreator = (steps: AuthPipelineFunction[]) => { const pipeline = authPipelineCreator(steps) @@ -153,3 +154,21 @@ export async function mixpanelTrackerHelperMiddleware( req.mixpanel = mp next() } + +export const X_SPECKLE_CLIENT_IP_HEADER = 'x-speckle-client-ip' + +/** + * Determine the IP address of the request source and add it as a header to the request object. + * This is used to correlate anonymous/unauthenticated requests with external data sources. + * @param req HTTP request object + * @param _res HTTP response object + * @param next Express middleware-compatible next function + */ +export async function determineClientIpAddressMiddleware( + req: Request, + _res: Response, + next: NextFunction +) { + req.headers[X_SPECKLE_CLIENT_IP_HEADER] = getIpFromRequest(req) + next() +} diff --git a/packages/server/modules/shared/utils/ip.js b/packages/server/modules/shared/utils/ip.js index f8db74a66f..11fe03e26f 100644 --- a/packages/server/modules/shared/utils/ip.js +++ b/packages/server/modules/shared/utils/ip.js @@ -1,7 +1,14 @@ const getIpFromRequest = (req) => { let ip try { - ip = req.headers['cf-connecting-ip'] || req.ip || req.connection.remoteAddress || '' + ip = + req.headers['cf-connecting-ip'] || + req.headers['true-client-ip'] || + req.headers['x-real-ip'] || + req.headers['x-forwarded-for'] || + req.ip || + req.connection.remoteAddress || + '' } catch { ip = '' } From 1515e2fee60dbaf09ad82ba328b672a7fbd15f4b Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Thu, 13 Apr 2023 15:24:51 +0100 Subject: [PATCH 14/22] revert(ratelimit): defaults should remain as was prior to `2a35fe6` (#1528) * Revert "fix(ratelimit): reduce /graphql limit based on incident (#1505)" This reverts commit 2a35fe6178efbe64a4f8bc5384267a5fa86fe43a. * Revert helm chart defaults to value in code - fix typo --- packages/server/modules/core/services/ratelimiter.ts | 4 ++-- utils/helm/speckle-server/values.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/server/modules/core/services/ratelimiter.ts b/packages/server/modules/core/services/ratelimiter.ts index 1665ca9544..41769e45f1 100644 --- a/packages/server/modules/core/services/ratelimiter.ts +++ b/packages/server/modules/core/services/ratelimiter.ts @@ -170,11 +170,11 @@ export const LIMITS: RateLimiterOptions = { }, 'POST /graphql': { regularOptions: { - limitCount: getIntFromEnv('RATELIMIT_POST_GRAPHQL', '10'), + limitCount: getIntFromEnv('RATELIMIT_POST_GRAPHQL', '50'), duration: 1 * TIME.second }, burstOptions: { - limitCount: getIntFromEnv('RATELIMIT_BURST_POST_GRAPHQL', '20'), + limitCount: getIntFromEnv('RATELIMIT_BURST_POST_GRAPHQL', '200'), duration: 1 * TIME.minute } }, diff --git a/utils/helm/speckle-server/values.yaml b/utils/helm/speckle-server/values.yaml index fd4797b389..8bb8e4c916 100644 --- a/utils/helm/speckle-server/values.yaml +++ b/utils/helm/speckle-server/values.yaml @@ -587,9 +587,9 @@ server: ## @param server.ratelimiting.burst_get_objects_streamid_objectid_single If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to get a single object in a moving one minute window. burst_get_objects_streamid_objectid_single: 200 ## @param server.ratelimiting.post_graphql The maximum number of requests that can be made to the GraphQL API in a moving one second window. - post_graphql: 10 + post_graphql: 50 ## @param server.ratelimiting.burst_post_grapqhl If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the GraphQL API in a moving one minute window. - burst_post_grapqhl: 20 + burst_post_graphql: 200 ## @param server.ratelimiting.get_auth The maximum number of requests that can be made to the Speckle server to authenticate in a moving one second window. get_auth: 2 ## @param server.ratelimiting.burst_get_auth If the regular limit is exceeded, the limit is increased to the burst limit. This is the maximum number of requests that can be made to the Speckle server to authenticate in a moving one minute window. From 6f718d8d91be9d85cc0ad266b46d0a2d19dd89bb Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Thu, 13 Apr 2023 15:30:34 +0100 Subject: [PATCH 15/22] fix(logging): should log headers in request; add missing return statement (#1529) --- packages/server/logging/expressLogging.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/logging/expressLogging.ts b/packages/server/logging/expressLogging.ts index 73e04e1abc..5e9cf71d98 100644 --- a/packages/server/logging/expressLogging.ts +++ b/packages/server/logging/expressLogging.ts @@ -61,7 +61,7 @@ export const LoggingExpressMiddleware = HttpLogger({ // and that's a privacy issue. return false } - ![ + return ![ 'cookie', 'authorization', 'cf-connecting-ip', From c6e36cdfce10482f2d0899db544aaa91da228ae5 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Thu, 13 Apr 2023 16:47:31 +0100 Subject: [PATCH 16/22] fix(logging): log level for graphql errors (#1530) --- packages/server/logging/apolloPlugin.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/logging/apolloPlugin.js b/packages/server/logging/apolloPlugin.js index 0ed1e43011..0846aecd6d 100644 --- a/packages/server/logging/apolloPlugin.js +++ b/packages/server/logging/apolloPlugin.js @@ -5,6 +5,7 @@ const { ApolloError } = require('apollo-server-express') const prometheusClient = require('prom-client') const { graphqlLogger } = require('@/logging/logging') const { redactSensitiveVariables } = require('@/logging/loggingHelper') +const { GraphQLError } = require('graphql') const metricCallCount = new prometheusClient.Counter({ name: 'speckle_server_apollo_calls', @@ -73,7 +74,11 @@ module.exports = { if (err.path) { logger = logger.child({ 'query-path': err.path.join(' > ') }) } - logger.error(err, 'graphql error') + if (err instanceof GraphQLError && err.extensions?.code === 'FORBIDDEN') { + logger.info(err, 'graphql error') + } else { + logger.error(err, 'graphql error') + } Sentry.withScope((scope) => { scope.setTag('kind', kind) From 8bc04f97d980516a13410dd3ba737f96d6307bf9 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Thu, 13 Apr 2023 17:38:17 +0100 Subject: [PATCH 17/22] fix(logging): hash ip for all requests with an ip (#1531) --- packages/server/app.ts | 2 +- packages/server/logging/expressLogging.ts | 37 ++++++------------- .../server/modules/shared/middleware/index.ts | 7 ++-- 3 files changed, 17 insertions(+), 29 deletions(-) diff --git a/packages/server/app.ts b/packages/server/app.ts index 465613fd76..8ac33af578 100644 --- a/packages/server/app.ts +++ b/packages/server/app.ts @@ -192,6 +192,7 @@ export async function init() { await knex.migrate.latest() app.use(DetermineRequestIdMiddleware) + app.use(determineClientIpAddressMiddleware) app.use(LoggingExpressMiddleware) if (process.env.COMPRESSION) { @@ -206,7 +207,6 @@ export async function init() { // Log errors app.use(errorLoggingMiddleware) - app.use(determineClientIpAddressMiddleware) app.use(authContextMiddleware) app.use(createRateLimiterMiddleware()) app.use(mixpanelTrackerHelperMiddleware) diff --git a/packages/server/logging/expressLogging.ts b/packages/server/logging/expressLogging.ts index 5e9cf71d98..515cfdce80 100644 --- a/packages/server/logging/expressLogging.ts +++ b/packages/server/logging/expressLogging.ts @@ -5,15 +5,9 @@ import { IncomingMessage } from 'http' import { NextFunction, Response } from 'express' import pino, { SerializedResponse } from 'pino' import { GenReqId } from 'pino-http' -import { X_SPECKLE_CLIENT_IP_HEADER } from '@/modules/shared/middleware' -import { AuthContext } from '@/modules/shared/authz' const REQUEST_ID_HEADER = 'x-request-id' -interface RawRequestWithAuthContext extends IncomingMessage { - context: AuthContext -} - const GenerateRequestId: GenReqId = (req: IncomingMessage) => DetermineRequestId(req) const DetermineRequestId = ( @@ -51,25 +45,17 @@ export const LoggingExpressMiddleware = HttpLogger({ path: req.raw.url?.split('?')[0], // Remove query params which might be sensitive // Allowlist useful headers headers: Object.fromEntries( - Object.entries(req.raw.headers).filter(([key]) => { - if ( - (req.raw as RawRequestWithAuthContext).context?.userId && - key.toLocaleLowerCase() === X_SPECKLE_CLIENT_IP_HEADER - ) { - // we cannot log the IP (even if hashed) if we also have a User ID - // Having both together could be used to link an IP to a specific User - // and that's a privacy issue. - return false - } - return ![ - 'cookie', - 'authorization', - 'cf-connecting-ip', - 'true-client-ip', - 'x-real-ip', - 'x-forwarded-for' - ].includes(key.toLocaleLowerCase()) - }) + Object.entries(req.raw.headers).filter( + ([key]) => + ![ + 'cookie', + 'authorization', + 'cf-connecting-ip', + 'true-client-ip', + 'x-real-ip', + 'x-forwarded-for' + ].includes(key.toLocaleLowerCase()) + ) ) } }), @@ -94,6 +80,7 @@ export const DetermineRequestIdMiddleware = ( next: NextFunction ) => { const id = DetermineRequestId(req) + req.headers[REQUEST_ID_HEADER] = id res.setHeader(REQUEST_ID_HEADER, id) next() } diff --git a/packages/server/modules/shared/middleware/index.ts b/packages/server/modules/shared/middleware/index.ts index e80edfcbd0..d63be4b5e8 100644 --- a/packages/server/modules/shared/middleware/index.ts +++ b/packages/server/modules/shared/middleware/index.ts @@ -22,6 +22,7 @@ import { mixpanel } from '@/modules/shared/utils/mixpanel' import { Observability } from '@speckle/shared' import { pino } from 'pino' import { getIpFromRequest } from '@/modules/shared/utils/ip' +import { md5 } from '@/modules/shared/helpers/cryptoHelper' export const authMiddlewareCreator = (steps: AuthPipelineFunction[]) => { const pipeline = authPipelineCreator(steps) @@ -155,8 +156,7 @@ export async function mixpanelTrackerHelperMiddleware( next() } -export const X_SPECKLE_CLIENT_IP_HEADER = 'x-speckle-client-ip' - +const X_SPECKLE_CLIENT_IP_HEADER = 'x-speckle-client-ip' /** * Determine the IP address of the request source and add it as a header to the request object. * This is used to correlate anonymous/unauthenticated requests with external data sources. @@ -169,6 +169,7 @@ export async function determineClientIpAddressMiddleware( _res: Response, next: NextFunction ) { - req.headers[X_SPECKLE_CLIENT_IP_HEADER] = getIpFromRequest(req) + const ip = getIpFromRequest(req) + if (ip) req.headers[X_SPECKLE_CLIENT_IP_HEADER] = md5(ip) next() } From 43c339bccfa8da98f1fd447ac50fbc6cc165aed1 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Thu, 13 Apr 2023 19:43:54 +0100 Subject: [PATCH 18/22] fix(logging): use broadcast address to mask logged ip (#1532) --- packages/server/modules/shared/middleware/index.ts | 7 +++++-- packages/server/package.json | 2 ++ yarn.lock | 9 +++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/server/modules/shared/middleware/index.ts b/packages/server/modules/shared/middleware/index.ts index d63be4b5e8..c210d18bb4 100644 --- a/packages/server/modules/shared/middleware/index.ts +++ b/packages/server/modules/shared/middleware/index.ts @@ -22,7 +22,7 @@ import { mixpanel } from '@/modules/shared/utils/mixpanel' import { Observability } from '@speckle/shared' import { pino } from 'pino' import { getIpFromRequest } from '@/modules/shared/utils/ip' -import { md5 } from '@/modules/shared/helpers/cryptoHelper' +import { Netmask } from 'netmask' export const authMiddlewareCreator = (steps: AuthPipelineFunction[]) => { const pipeline = authPipelineCreator(steps) @@ -170,6 +170,9 @@ export async function determineClientIpAddressMiddleware( next: NextFunction ) { const ip = getIpFromRequest(req) - if (ip) req.headers[X_SPECKLE_CLIENT_IP_HEADER] = md5(ip) + if (ip) { + const mask = new Netmask(`${ip}/24`) + req.headers[X_SPECKLE_CLIENT_IP_HEADER] = mask.broadcast + } next() } diff --git a/packages/server/package.json b/packages/server/package.json index b223d61953..bb89d5b470 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -66,6 +66,7 @@ "mixpanel": "^0.17.0", "mjml": "^4.13.0", "module-alias": "^2.2.2", + "netmask": "^2.0.2", "node-cron": "^3.0.2", "node-machine-id": "^1.1.12", "nodemailer": "^6.5.0", @@ -115,6 +116,7 @@ "@types/mocha": "^10.0.0", "@types/mock-require": "^2.0.1", "@types/module-alias": "^2.0.1", + "@types/netmask": "^2.0.0", "@types/node-cron": "^3.0.2", "@types/nodemailer": "^6.4.5", "@types/sanitize-html": "^2.6.2", diff --git a/yarn.lock b/yarn.lock index 97d33ce149..37beefa5fb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5251,6 +5251,7 @@ __metadata: "@types/mocha": ^10.0.0 "@types/mock-require": ^2.0.1 "@types/module-alias": ^2.0.1 + "@types/netmask": ^2.0.0 "@types/node-cron": ^3.0.2 "@types/nodemailer": ^6.4.5 "@types/pino-http": ^5.8.1 @@ -5301,6 +5302,7 @@ __metadata: mocha-junit-reporter: ^2.0.2 mock-require: ^3.0.3 module-alias: ^2.2.2 + netmask: ^2.0.2 node-cron: ^3.0.2 node-machine-id: ^1.1.12 node-mocks-http: ^1.12.1 @@ -6231,6 +6233,13 @@ __metadata: languageName: node linkType: hard +"@types/netmask@npm:^2.0.0": + version: 2.0.0 + resolution: "@types/netmask@npm:2.0.0" + checksum: 63301fb59f014cd03e1aa9d3b2f41334e2cd7a2620e4741222c7abdf0553fb0941f00cd6596506dee69f29e6c3b33cf9b9c84f2e10e3ec722691229051d83478 + languageName: node + linkType: hard + "@types/node-cron@npm:^3.0.2": version: 3.0.2 resolution: "@types/node-cron@npm:3.0.2" From 6cc1046a860b2d1cd3d6220c5046d05bf1df7daf Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Sat, 15 Apr 2023 12:39:24 +0100 Subject: [PATCH 19/22] fix(server): logging ipv6 catches --- packages/server/modules/shared/middleware/index.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/server/modules/shared/middleware/index.ts b/packages/server/modules/shared/middleware/index.ts index c210d18bb4..c8b1744b3d 100644 --- a/packages/server/modules/shared/middleware/index.ts +++ b/packages/server/modules/shared/middleware/index.ts @@ -171,8 +171,18 @@ export async function determineClientIpAddressMiddleware( ) { const ip = getIpFromRequest(req) if (ip) { - const mask = new Netmask(`${ip}/24`) - req.headers[X_SPECKLE_CLIENT_IP_HEADER] = mask.broadcast + try{ + const isV6 = ip.includes(':') + if(isV6) { + req.headers[X_SPECKLE_CLIENT_IP_HEADER] = ip + } else { + const mask = new Netmask(`${ip}/24`) + req.headers[X_SPECKLE_CLIENT_IP_HEADER] = mask.broadcast + } + } + catch(e) { + req.headers[X_SPECKLE_CLIENT_IP_HEADER] = ip || 'ip-parse-error' + } } next() } From d402837f9aefcd1555d1032060698cfac2c8bd4e Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Sat, 15 Apr 2023 12:46:02 +0100 Subject: [PATCH 20/22] fix(server): prettier saves the day as always (snark) --- packages/server/modules/shared/middleware/index.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/server/modules/shared/middleware/index.ts b/packages/server/modules/shared/middleware/index.ts index c8b1744b3d..7531556435 100644 --- a/packages/server/modules/shared/middleware/index.ts +++ b/packages/server/modules/shared/middleware/index.ts @@ -171,16 +171,15 @@ export async function determineClientIpAddressMiddleware( ) { const ip = getIpFromRequest(req) if (ip) { - try{ + try { const isV6 = ip.includes(':') - if(isV6) { + if (isV6) { req.headers[X_SPECKLE_CLIENT_IP_HEADER] = ip } else { const mask = new Netmask(`${ip}/24`) req.headers[X_SPECKLE_CLIENT_IP_HEADER] = mask.broadcast } - } - catch(e) { + } catch (e) { req.headers[X_SPECKLE_CLIENT_IP_HEADER] = ip || 'ip-parse-error' } } From e8521c5c981d404c850ab0acd199f722b9e343e2 Mon Sep 17 00:00:00 2001 From: Jenessa Man Date: Mon, 17 Apr 2023 17:19:58 -0400 Subject: [PATCH 21/22] Add resolver for globals field on stream --- .../modules/core/graph/resolvers/objects.js | 26 +++++++++++++++++++ packages/server/package.json | 1 + yarn.lock | 12 ++++++++- 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/server/modules/core/graph/resolvers/objects.js b/packages/server/modules/core/graph/resolvers/objects.js index 6528a55434..0501451ba9 100644 --- a/packages/server/modules/core/graph/resolvers/objects.js +++ b/packages/server/modules/core/graph/resolvers/objects.js @@ -12,6 +12,12 @@ const { getObjectChildrenQuery } = require('../../services/objects') +const { getBranchesByStreamId } = require('../../services/branches') + +const { getCommitsByBranchName } = require('../../services/commits') + +const omitDeep = require('omit-deep-lodash') + module.exports = { Stream: { async object(parent, args) { @@ -20,6 +26,26 @@ module.exports = { obj.streamId = parent.id return obj + }, + async globals(parent) { + const branches = await getBranchesByStreamId({ streamId: parent.id }) + if (!branches || !branches.items.some((b) => b.name === 'globals')) return null + + const { commits } = await getCommitsByBranchName({ + streamId: parent.id, + branchName: 'globals', + limit: 1 + }) + if (!commits || commits.length === 0) return null + + const refObj = commits[0].referencedObject + const obj = await getObject({ streamId: parent.id, objectId: refObj }) + if (!obj) return null + + const items = omitDeep(obj.data, ['totalChildrenCount', 'speckle_type', 'id']) + const totalCount = Object.keys(items).length + + return { totalCount, items } } }, Object: { diff --git a/packages/server/package.json b/packages/server/package.json index 12aa4726d1..122e9f44ed 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -72,6 +72,7 @@ "node-machine-id": "^1.1.12", "nodemailer": "^6.5.0", "odata": "^1.3.1", + "omit-deep-lodash": "^1.1.7", "openid-client": "^5.1.7", "passport": "^0.6.0", "passport-azure-ad": "^4.3.4", diff --git a/yarn.lock b/yarn.lock index b84f9a7167..75c2d50548 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5329,6 +5329,7 @@ __metadata: nodemon: ^2.0.20 nyc: ^15.0.1 odata: ^1.3.1 + omit-deep-lodash: ^1.1.7 openid-client: ^5.1.7 passport: ^0.6.0 passport-azure-ad: ^4.3.4 @@ -14276,7 +14277,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:4.17.21, lodash@npm:^4.11.2, lodash@npm:^4.17.0, lodash@npm:^4.17.10, lodash@npm:^4.17.11, lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.19, lodash@npm:^4.17.20, lodash@npm:^4.17.21, lodash@npm:~4.17.0": +"lodash@npm:4.17.21, lodash@npm:^4.11.2, lodash@npm:^4.17.0, lodash@npm:^4.17.10, lodash@npm:^4.17.11, lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.19, lodash@npm:^4.17.20, lodash@npm:^4.17.21, lodash@npm:~4.17.0, lodash@npm:~4.17.21": version: 4.17.21 resolution: "lodash@npm:4.17.21" checksum: eb835a2e51d381e561e508ce932ea50a8e5a68f4ebdd771ea240d3048244a8d13658acbd502cd4829768c56f2e16bdd4340b9ea141297d472517b83868e677f7 @@ -15944,6 +15945,15 @@ __metadata: languageName: node linkType: hard +"omit-deep-lodash@npm:^1.1.7": + version: 1.1.7 + resolution: "omit-deep-lodash@npm:1.1.7" + dependencies: + lodash: ~4.17.21 + checksum: a93ba4d6aef93d5f8e8e9284e03613b84230db017b44cbe481b29f7e3c6809a02a537aec54c8827b9f090aad678b34e08ca55c92f945fb106ea2ef28bb7ff25f + languageName: node + linkType: hard + "on-exit-leak-free@npm:^2.1.0": version: 2.1.0 resolution: "on-exit-leak-free@npm:2.1.0" From 02569688d73cd96611bed2f3450e59fe8a08b33e Mon Sep 17 00:00:00 2001 From: Jenessa Man Date: Mon, 17 Apr 2023 18:35:14 -0400 Subject: [PATCH 22/22] Add secret for posthog key in frontend Dockerfile --- packages/frontend/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/frontend/Dockerfile b/packages/frontend/Dockerfile index 273f303e56..4af3bfb1e4 100644 --- a/packages/frontend/Dockerfile +++ b/packages/frontend/Dockerfile @@ -28,7 +28,9 @@ COPY packages/frontend ./packages/frontend/ COPY packages/shared ./packages/shared/ # This way the foreach only builds the frontend and its deps -RUN yarn workspaces foreach run build +RUN --mount=type=secret,id=posthog_api_key \ + export VITE_POSTHOG_API_KEY=$(cat /run/secrets/posthog_api_key) && \ + yarn workspaces foreach run build RUN DEBIAN_FRONTEND=noninteractive \ apt-get -q update && \