From 8c1f7fae15eb46c6d2e39e3061f32103f77c40bb Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Mon, 30 Sep 2024 17:20:37 +0400 Subject: [PATCH 1/2] feat: allow creation of multiple project envs --- backend/src/keystore/keystore.ts | 7 + backend/src/server/routes/index.ts | 1 + .../project-env/project-env-service.ts | 174 +++++++++++++----- 3 files changed, 133 insertions(+), 49 deletions(-) diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 3b19012e76..c78c428b0a 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -16,6 +16,13 @@ export const KeyStorePrefixes = { WaitUntilReadyKmsOrgKeyCreation: "wait-until-ready-kms-org-key-creation-", WaitUntilReadyKmsOrgDataKeyCreation: "wait-until-ready-kms-org-data-key-creation-", + ProjectEnvironmentCreation: "project-environment-creation-lock", + ProjectEnvironmentUpdate: "project-environment-update-lock", + ProjectEnvironmentDelete: "project-environment-delete-lock", + WaitUntilReadyCreateProjectEnvironment: "wait-until-ready-create-project-environments-", + WaitUntilReadyUpdateProjectEnvironment: "wait-until-ready-update-project-environments-", + WaitUntilReadyDeleteProjectEnvironment: "wait-until-ready-delete-project-environments-", + SyncSecretIntegrationLock: (projectId: string, environmentSlug: string, secretPath: string) => `sync-integration-mutex-${projectId}-${environmentSlug}-${secretPath}` as const, SyncSecretIntegrationLastRunTimestamp: (projectId: string, environmentSlug: string, secretPath: string) => diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 5d9632215a..8a9578454c 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -742,6 +742,7 @@ export const registerRoutes = async ( const projectEnvService = projectEnvServiceFactory({ permissionService, projectEnvDAL, + keyStore, licenseService, projectDAL, folderDAL diff --git a/backend/src/services/project-env/project-env-service.ts b/backend/src/services/project-env/project-env-service.ts index c7af817bd2..2fc2207a3b 100644 --- a/backend/src/services/project-env/project-env-service.ts +++ b/backend/src/services/project-env/project-env-service.ts @@ -3,7 +3,9 @@ import { ForbiddenError } from "@casl/ability"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { logger } from "@app/lib/logger"; import { TProjectDALFactory } from "../project/project-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; @@ -16,6 +18,7 @@ type TProjectEnvServiceFactoryDep = { projectDAL: Pick; permissionService: Pick; licenseService: Pick; + keyStore: Pick; }; export type TProjectEnvServiceFactory = ReturnType; @@ -24,6 +27,7 @@ export const projectEnvServiceFactory = ({ projectEnvDAL, permissionService, licenseService, + keyStore, projectDAL, folderDAL }: TProjectEnvServiceFactoryDep) => { @@ -45,32 +49,56 @@ export const projectEnvServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Environments); - const envs = await projectEnvDAL.find({ projectId }); - const existingEnv = envs.find(({ slug: envSlug }) => envSlug === slug); - if (existingEnv) - throw new BadRequestError({ - message: "Environment with slug already exist", - name: "Create envv" - }); + const lock = await keyStore + .acquireLock([KeyStorePrefixes.ProjectEnvironmentCreation, projectId], 5000) + .catch(() => null); + + try { + if (!lock) { + await keyStore.waitTillReady({ + key: `${KeyStorePrefixes.WaitUntilReadyCreateProjectEnvironment}${projectId}`, + keyCheckCb: (val) => val === "true", + waitingCb: () => logger.debug("Create project environment. Waiting for "), + delay: 500 + }); + } + + const envs = await projectEnvDAL.find({ projectId }); + const existingEnv = envs.find(({ slug: envSlug }) => envSlug === slug); + if (existingEnv) + throw new BadRequestError({ + message: "Environment with slug already exist", + name: "CreateEnvironment" + }); + + const project = await projectDAL.findById(projectId); + const plan = await licenseService.getPlan(project.orgId); + if (plan.environmentLimit !== null && envs.length >= plan.environmentLimit) { + // case: limit imposed on number of environments allowed + // case: number of environments used exceeds the number of environments allowed + throw new BadRequestError({ + message: + "Failed to create environment due to environment limit reached. Upgrade plan to create more environments." + }); + } - const project = await projectDAL.findById(projectId); - const plan = await licenseService.getPlan(project.orgId); - if (plan.environmentLimit !== null && envs.length >= plan.environmentLimit) { - // case: limit imposed on number of environments allowed - // case: number of environments used exceeds the number of environments allowed - throw new BadRequestError({ - message: - "Failed to create environment due to environment limit reached. Upgrade plan to create more environments." + const env = await projectEnvDAL.transaction(async (tx) => { + const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); + const doc = await projectEnvDAL.create({ slug, name, projectId, position: lastPos + 1 }, tx); + await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); + return doc; }); - } - const env = await projectEnvDAL.transaction(async (tx) => { - const lastPos = await projectEnvDAL.findLastEnvPosition(projectId, tx); - const doc = await projectEnvDAL.create({ slug, name, projectId, position: lastPos + 1 }, tx); - await folderDAL.create({ name: "root", parentId: null, envId: doc.id, version: 1 }, tx); - return doc; - }); - return env; + await keyStore.setItemWithExpiry( + `${KeyStorePrefixes.WaitUntilReadyCreateProjectEnvironment}${projectId}`, + 10, + "true" + ); + + return env; + } finally { + await lock?.release(); + } }; const updateEnvironment = async ({ @@ -93,26 +121,50 @@ export const projectEnvServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.Environments); - const oldEnv = await projectEnvDAL.findOne({ id, projectId }); - if (!oldEnv) throw new BadRequestError({ message: "Environment not found" }); - - if (slug) { - const existingEnv = await projectEnvDAL.findOne({ slug, projectId }); - if (existingEnv && existingEnv.id !== id) { - throw new BadRequestError({ - message: "Environment with slug already exist", - name: "Create envv" + const lock = await keyStore + .acquireLock([KeyStorePrefixes.ProjectEnvironmentUpdate, projectId], 5000) + .catch(() => null); + + try { + if (!lock) { + await keyStore.waitTillReady({ + key: `${KeyStorePrefixes.WaitUntilReadyUpdateProjectEnvironment}${projectId}`, + keyCheckCb: (val) => val === "true", + waitingCb: () => logger.debug("Update project environment. Waiting for project environment update"), + delay: 500 }); } - } - const env = await projectEnvDAL.transaction(async (tx) => { - if (position) { - await projectEnvDAL.updateAllPosition(projectId, oldEnv.position, position, tx); + const oldEnv = await projectEnvDAL.findOne({ id, projectId }); + if (!oldEnv) throw new NotFoundError({ message: "Environment not found", name: "UpdateEnvironment" }); + + if (slug) { + const existingEnv = await projectEnvDAL.findOne({ slug, projectId }); + if (existingEnv && existingEnv.id !== id) { + throw new BadRequestError({ + message: "Environment with slug already exist", + name: "UpdateEnvironment" + }); + } } - return projectEnvDAL.updateById(oldEnv.id, { name, slug, position }, tx); - }); - return { environment: env, old: oldEnv }; + + const env = await projectEnvDAL.transaction(async (tx) => { + if (position) { + await projectEnvDAL.updateAllPosition(projectId, oldEnv.position, position, tx); + } + return projectEnvDAL.updateById(oldEnv.id, { name, slug, position }, tx); + }); + + await keyStore.setItemWithExpiry( + `${KeyStorePrefixes.WaitUntilReadyUpdateProjectEnvironment}${projectId}`, + 10, + "true" + ); + + return { environment: env, old: oldEnv }; + } finally { + await lock?.release(); + } }; const deleteEnvironment = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod, id }: TDeleteEnvDTO) => { @@ -125,18 +177,42 @@ export const projectEnvServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.Environments); - const env = await projectEnvDAL.transaction(async (tx) => { - const [doc] = await projectEnvDAL.delete({ id, projectId }, tx); - if (!doc) - throw new BadRequestError({ - message: "Env doesn't exist", - name: "Re-order env" + const lock = await keyStore + .acquireLock([KeyStorePrefixes.ProjectEnvironmentDelete, projectId], 5000) + .catch(() => null); + + try { + if (!lock) { + await keyStore.waitTillReady({ + key: `${KeyStorePrefixes.WaitUntilReadyDeleteProjectEnvironment}${projectId}`, + keyCheckCb: (val) => val === "true", + waitingCb: () => logger.debug("Delete project environment. Waiting for "), + delay: 500 }); + } - await projectEnvDAL.updateAllPosition(projectId, doc.position, -1, tx); - return doc; - }); - return env; + const env = await projectEnvDAL.transaction(async (tx) => { + const [doc] = await projectEnvDAL.delete({ id, projectId }, tx); + if (!doc) + throw new NotFoundError({ + message: "Environment doesn't exist", + name: "DeleteEnvironment" + }); + + await projectEnvDAL.updateAllPosition(projectId, doc.position, -1, tx); + return doc; + }); + + await keyStore.setItemWithExpiry( + `${KeyStorePrefixes.WaitUntilReadyDeleteProjectEnvironment}${projectId}`, + 10, + "true" + ); + + return env; + } finally { + await lock?.release(); + } }; const getEnvironmentById = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod, id }: TGetEnvDTO) => { From 27a5515fa7e1c1fe8ec09ae86b56cc9b33ff74dd Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Mon, 30 Sep 2024 17:25:00 +0400 Subject: [PATCH 2/2] fix: variable naming --- backend/src/keystore/keystore.ts | 2 +- backend/src/services/project-env/project-env-service.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index c78c428b0a..37555f3754 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -16,7 +16,7 @@ export const KeyStorePrefixes = { WaitUntilReadyKmsOrgKeyCreation: "wait-until-ready-kms-org-key-creation-", WaitUntilReadyKmsOrgDataKeyCreation: "wait-until-ready-kms-org-data-key-creation-", - ProjectEnvironmentCreation: "project-environment-creation-lock", + ProjectEnvironmentCreate: "project-environment-creation-lock", ProjectEnvironmentUpdate: "project-environment-update-lock", ProjectEnvironmentDelete: "project-environment-delete-lock", WaitUntilReadyCreateProjectEnvironment: "wait-until-ready-create-project-environments-", diff --git a/backend/src/services/project-env/project-env-service.ts b/backend/src/services/project-env/project-env-service.ts index 2fc2207a3b..90d8c9fc32 100644 --- a/backend/src/services/project-env/project-env-service.ts +++ b/backend/src/services/project-env/project-env-service.ts @@ -50,7 +50,7 @@ export const projectEnvServiceFactory = ({ ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Environments); const lock = await keyStore - .acquireLock([KeyStorePrefixes.ProjectEnvironmentCreation, projectId], 5000) + .acquireLock([KeyStorePrefixes.ProjectEnvironmentCreate, projectId], 5000) .catch(() => null); try {