Merge branch 'crowd-linux' into improvement/identity-optimizations

backend/package.json

@@ -43,7 +43,8 @@
     "script:merge-similar-organizations": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/merge-similar-organizations.ts",
     "script:cache-dashboard": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/cache-dashboard.ts",
     "script:purge-tenants-and-data": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/purge-tenants-and-data.ts",
-    "script:import-lfx-memberships": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/import-lfx-memberships.ts"
+    "script:import-lfx-memberships": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/import-lfx-memberships.ts",
+    "script:fix-missing-org-displayName": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/fix-missing-org-displayName.ts"
   },
   "dependencies": {
     "@aws-sdk/client-comprehend": "^3.159.0",

backend/src/api/index.ts

@@ -242,6 +242,7 @@ setImmediate(async () => {
   require('./eventTracking').default(routes)
   require('./customViews').default(routes)
   require('./dashboard').default(routes)
+  require('./mergeAction').default(routes)
   // Loads the Tenant if the :tenantId param is passed
   routes.param('tenantId', tenantMiddleware)
   routes.param('tenantId', segmentMiddleware)

backend/src/api/mergeAction/index.ts

@@ -0,0 +1,5 @@
+import { safeWrap } from '../../middlewares/errorMiddleware'
+
+export default (app) => {
+  app.get(`/tenant/:tenantId/mergeActions`, safeWrap(require('./mergeActionQuery').default))
+}

backend/src/api/mergeAction/mergeActionQuery.ts

@@ -0,0 +1,29 @@
+import MergeActionsService from '@/services/MergeActionsService'
+import Permissions from '../../security/permissions'
+import PermissionChecker from '../../services/user/permissionChecker'
+
+/**
+ * GET /tenant/{tenantId}/mergeAction
+ * @summary Query mergeActions
+ * @tag MergeActions
+ * @security Bearer
+ * @description Query mergeActions. It accepts filters and pagination.
+ * @pathParam {string} tenantId - Your workspace/tenant ID
+ * @queryParam {string} entityId - ID of the entity
+ * @queryParam {string} type - type of the entity (e.g., org or member)
+ * @queryParam {number} [limit] - number of records to return (optional, default to 20)
+ * @queryParam {number} [offset] - number of records to skip (optional, default to 0)
+ * @response 200 - Ok
+ * @responseContent {MergeActionList} 200.application/json
+ * @responseExample {MergeActionList} 200.application/json.MergeAction
+ * @response 401 - Unauthorized
+ * @response 404 - Not found
+ * @response 429 - Too many requests
+ */
+export default async (req, res) => {
+  new PermissionChecker(req).validateHas(Permissions.values.mergeActionRead)
+
+  const payload = await new MergeActionsService(req).query(req.query)
+
+  await req.responseHandler.success(req, res, payload)
+}

backend/src/bin/scripts/fix-missing-org-displayName.ts

@@ -0,0 +1,164 @@
+/* eslint-disable @typescript-eslint/dot-notation */
+/* eslint-disable no-console */
+/* eslint-disable import/no-extraneous-dependencies */
+
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+
+import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor'
+import { databaseInit } from '@/database/databaseConnection'
+import SequelizeRepository from '@/database/repositories/sequelizeRepository'
+import { IRepositoryOptions } from '@/database/repositories/IRepositoryOptions'
+import OrganizationRepository from '@/database/repositories/organizationRepository'
+
+const options = [
+  {
+    name: 'help',
+    alias: 'h',
+    type: Boolean,
+    description: 'Print this usage guide.',
+  },
+  {
+    name: 'tenantId',
+    alias: 't',
+    type: String,
+    description: 'Tenant Id',
+  },
+]
+const sections = [
+  {
+    header: `Fix empty displayName in organizations`,
+    content: 'Script will fix organizations with empty displayName',
+  },
+  {
+    header: 'Options',
+    optionList: options,
+  },
+]
+
+const usage = commandLineUsage(sections)
+const parameters = commandLineArgs(options)
+
+function getOrgsWithoutDisplayName(
+  qx: QueryExecutor,
+  tenantId: string,
+  { limit = 50, countOnly = false },
+) {
+  return qx.select(
+    `
+        SELECT
+        ${countOnly ? 'COUNT(*)' : 'o.id'}
+        FROM organizations o
+        WHERE o."tenantId" = $(tenantId)
+        AND o."displayName" IS NULL
+        ${countOnly ? '' : 'LIMIT $(limit)'}
+    `,
+    { tenantId, limit },
+  )
+}
+
+async function getOrgIdentities(qx: QueryExecutor, orgId: string, tenantId: string) {
+  return qx.select(
+    `
+      SELECT value
+      FROM "organizationIdentities"
+      WHERE "organizationId" = $(orgId)
+      AND "tenantId" = $(tenantId)
+      LIMIT 1
+    `,
+    { orgId, tenantId },
+  )
+}
+
+async function getOrgAttributes(qx: QueryExecutor, orgId: string) {
+  return qx.select(
+    `
+      SELECT value
+      FROM "orgAttributes"
+      WHERE "organizationId" = $(orgId)
+      AND name = 'name'
+      LIMIT 1
+    `,
+    { orgId },
+  )
+}
+
+async function updateOrgDisplayName(qx: QueryExecutor, orgId: string, displayName: string) {
+  await qx.result(
+    `
+      UPDATE organizations
+      SET "displayName" = $(displayName)
+      WHERE id = $(id)
+    `,
+    { id: orgId, displayName },
+  )
+}
+
+if (parameters.help || !parameters.tenantId) {
+  console.log(usage)
+} else {
+  setImmediate(async () => {
+    const prodDb = await databaseInit()
+    const tenantId = parameters.tenantId
+    const qx = SequelizeRepository.getQueryExecutor({
+      database: prodDb,
+    } as IRepositoryOptions)
+
+    const options = await SequelizeRepository.getDefaultIRepositoryOptions()
+
+    const BATCH_SIZE = 50
+    let processed = 0
+
+    const totalOrgs = await getOrgsWithoutDisplayName(qx, tenantId, { countOnly: true })
+
+    console.log(`Total organizations without displayName: ${totalOrgs[0].count}`)
+
+    let orgs = await getOrgsWithoutDisplayName(qx, tenantId, { limit: BATCH_SIZE })
+
+    while (totalOrgs[0].count > processed) {
+      for (const org of orgs) {
+        let displayName
+        let updateAttributes = false
+
+        const attributes = await getOrgAttributes(qx, org.id)
+
+        if (attributes.length > 0) {
+          displayName = attributes[0]?.value
+        } else {
+          const identities = await getOrgIdentities(qx, org.id, tenantId)
+          displayName = identities && identities[0]?.value
+          updateAttributes = true
+        }
+
+        if (displayName) {
+          await updateOrgDisplayName(qx, org.id, displayName)
+
+          if (updateAttributes) {
+            await OrganizationRepository.updateOrgAttributes(
+              org.id,
+              {
+                attributes: {
+                  name: {
+                    custom: [displayName],
+                    default: displayName,
+                  },
+                },
+              },
+              options,
+            )
+          }
+        } else {
+          console.log(`Organization ${org.id} does not have displayName`)
+        }
+
+        processed++
+      }
+
+      console.log(`Processed ${processed}/${totalOrgs[0].count} organizations`)
+
+      orgs = await getOrgsWithoutDisplayName(qx, tenantId, { limit: BATCH_SIZE })
+    }
+
+    process.exit(0)
+  })
+}

backend/src/bin/scripts/merge-similar-organizations.ts

@@ -145,16 +145,19 @@ if (parameters.help || (!parameters.tenant && !parameters.allTenants)) {
                 row.organizationId,
                 row.toMergeId,
                 userContext,
+                undefined,
               )
               await orgService.mergeSync(row.organizationId, row.toMergeId, null)
             } catch (err) {
               console.log('Error merging organizations - continuing with the rest', err)
-              await MergeActionsRepository.setState(
+              await MergeActionsRepository.setMergeAction(
                 MergeActionType.ORG,
                 row.organizationId,
                 row.toMergeId,
-                MergeActionState.ERROR,
                 userContext,
+                {
+                  state: MergeActionState.ERROR,
+                },
               )
             }
 

backend/src/database/migrations/U1721311363__addStepToMergeActions.sql

@@ -0,0 +1 @@
+alter table "mergeActions" drop column "step";

backend/src/database/migrations/V1721311363__addStepToMergeActions.sql

@@ -0,0 +1,2 @@
+alter table "mergeActions" add column "step" text;
+

backend/src/database/repositories/mergeActionsRepository.ts

@@ -8,6 +8,7 @@ import {
   IUnmergeBackup,
   MemberIdentityType,
   MergeActionState,
+  MergeActionStep,
   MergeActionType,
   OrganizationIdentityType,
 } from '@crowd/types'
@@ -20,6 +21,7 @@ class MergeActionsRepository {
     primaryId: string,
     secondaryId: string,
     options: IRepositoryOptions,
+    step: MergeActionStep,
     state: MergeActionState = MergeActionState.PENDING,
     backup: IUnmergeBackup<IMemberUnmergeBackup | IOrganizationUnmergeBackup> = undefined,
   ) {
@@ -29,8 +31,8 @@ class MergeActionsRepository {
 
     await options.database.sequelize.query(
       `
-        INSERT INTO "mergeActions" ("tenantId", "type", "primaryId", "secondaryId", state, "unmergeBackup", "actionBy")
-        VALUES (:tenantId, :type, :primaryId, :secondaryId, :state, :backup, :userId)
+        INSERT INTO "mergeActions" ("tenantId", "type", "primaryId", "secondaryId", state, step, "unmergeBackup", "actionBy")
+        VALUES (:tenantId, :type, :primaryId, :secondaryId, :state, :step, :backup, :userId)
         ON CONFLICT ("tenantId", "type", "primaryId", "secondaryId")
         DO UPDATE SET state = :state, "unmergeBackup" = :backup
       `,
@@ -40,6 +42,7 @@ class MergeActionsRepository {
           type,
           primaryId,
           secondaryId,
+          step,
           state,
           backup: backup ? JSON.stringify(backup) : null,
           userId,
@@ -50,35 +53,49 @@ class MergeActionsRepository {
     )
   }
 
-  static async setState(
+  static async setMergeAction(
     type: MergeActionType,
     primaryId: string,
     secondaryId: string,
-    state: MergeActionState,
     options: IRepositoryOptions,
+    data: {
+      step?: MergeActionStep
+      state?: MergeActionState
+    },
   ) {
     const transaction = SequelizeRepository.getTransaction(options)
     const tenantId = options.currentTenant.id
 
+    const setClauses = []
+    const replacements: any = {
+      tenantId,
+      type,
+      primaryId,
+      secondaryId,
+    }
+
+    if (data.step) {
+      setClauses.push(`step = :step`)
+      replacements.step = data.step
+    }
+
+    if (data.state) {
+      setClauses.push(`state = :state`)
+      replacements.state = data.state
+    }
+
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     const [_, rowCount] = await options.database.sequelize.query(
       `
         UPDATE "mergeActions"
-        SET state = :state
+        SET ${setClauses.join(', ')}
         WHERE "tenantId" = :tenantId
           AND type = :type
           AND "primaryId" = :primaryId
           AND "secondaryId" = :secondaryId
-          AND state != :state
       `,
       {
-        replacements: {
-          tenantId,
-          type,
-          primaryId,
-          secondaryId,
-          state,
-        },
+        replacements,
         type: QueryTypes.UPDATE,
         transaction,
       },

backend/src/security/permissions.ts

@@ -1195,6 +1195,17 @@ class Permissions {
           TenantPlans.Scale,
         ],
       },
+      mergeActionRead: {
+        id: 'mergeActionRead',
+        allowedRoles: [roles.admin, roles.projectAdmin, roles.readonly],
+        allowedPlans: [
+          TenantPlans.Essential,
+          TenantPlans.Growth,
+          TenantPlans.EagleEye,
+          TenantPlans.Enterprise,
+          TenantPlans.Scale,
+        ],
+      },
     }
   }
 

backend/src/services/MergeActionsService.ts

@@ -0,0 +1,37 @@
+import { LoggerBase } from '@crowd/logging'
+import { queryMergeActions } from '@crowd/data-access-layer/src/mergeActions/repo'
+import { IServiceOptions } from './IServiceOptions'
+import SequelizeRepository from '@/database/repositories/sequelizeRepository'
+
+export default class MergeActionsService extends LoggerBase {
+  options: IServiceOptions
+
+  constructor(options: IServiceOptions) {
+    super(options.log)
+    this.options = options
+  }
+
+  async query(args) {
+    const qx = SequelizeRepository.getQueryExecutor(this.options)
+    const results = await queryMergeActions(qx, args)
+
+    return results.map((result) => ({
+      primaryId: result.primaryId,
+      secondaryId: result.secondaryId,
+      state: result.state,
+      // derive operation type from step and if step is null, default to merge
+      operationType: result.step ? MergeActionsService.getOperationType(result.step) : 'merge',
+    }))
+  }
+
+  static getOperationType(step) {
+    if (step.startsWith('merge')) {
+      return 'merge'
+    }
+    if (step.startsWith('unmerge')) {
+      return 'unmerge'
+    }
+
+    throw new Error(`Unrecognized merge action step: ${step}`)
+  }
+}