diff --git a/backend/src/bin/scripts/import-lfx-memberships.ts b/backend/src/bin/scripts/import-lfx-memberships.ts index d9a68a0463..6edeaf7652 100644 --- a/backend/src/bin/scripts/import-lfx-memberships.ts +++ b/backend/src/bin/scripts/import-lfx-memberships.ts @@ -13,7 +13,10 @@ import moment from 'moment' import { findProjectGroupByName } from '@crowd/data-access-layer/src/segments' import { insertLfxMembership, LfxMembership } from '@crowd/data-access-layer/src/lfx_memberships' -import { findOrgByDisplayName, findOrgByWebsite } from '@crowd/data-access-layer/src/organizations' +import { + findOrgIdByDisplayName, + findOrgIdByWebsite, +} from '@crowd/data-access-layer/src/organizations' import { databaseInit } from '@/database/databaseConnection' import SequelizeRepository from '@/database/repositories/sequelizeRepository' import { IRepositoryOptions } from '@/database/repositories/IRepositoryOptions' @@ -66,23 +69,27 @@ function parseDomains(domains: string) { ) } -async function findOrg(qx, tenantId, record) { - let org = await findOrgByWebsite(qx, tenantId, [record['Account Domain']]) +async function findOrgId(qx, tenantId, record) { + let org = await findOrgIdByWebsite(qx, tenantId, [record['Account Domain']]) if (org) { return org } - org = await findOrgByWebsite(qx, tenantId, record['Domain Alias']) + org = await findOrgIdByWebsite(qx, tenantId, record['Domain Alias']) if (org) { return org } - org = await findOrgByDisplayName(qx, { tenantId, orgName: record['Account Name'], exact: true }) + org = await findOrgIdByDisplayName(qx, { tenantId, orgName: record['Account Name'], exact: true }) if (org) { return org } - org = await findOrgByDisplayName(qx, { tenantId, orgName: record['Account Name'], exact: false }) + org = await findOrgIdByDisplayName(qx, { + tenantId, + orgName: record['Account Name'], + exact: false, + }) return org } @@ -115,10 +122,10 @@ if (parameters.help || !parameters.file || !parameters.tenantId) { tenantId, name: record['Project'], }) - const org = await findOrg(qx, tenantId, record) + const orgId = await findOrgId(qx, tenantId, record) const row = { tenantId: parameters.tenantId, - organizationId: org?.id, + organizationId: orgId, segmentId: segment?.id, accountName: orgName, parentAccount: record['Parent Account'], diff --git a/backend/src/database/migrations/U1718289198__org-attributes.sql b/backend/src/database/migrations/U1718289198__org-attributes.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/database/migrations/V1718289198__org-attributes.sql b/backend/src/database/migrations/V1718289198__org-attributes.sql new file mode 100644 index 0000000000..3ab9907f59 --- /dev/null +++ b/backend/src/database/migrations/V1718289198__org-attributes.sql @@ -0,0 +1,141 @@ +CREATE TABLE "orgAttributes" ( + "id" uuid PRIMARY KEY NOT NULL DEFAULT uuid_generate_v4(), + "createdAt" TIMESTAMP NOT NULL DEFAULT NOW(), + "updatedAt" TIMESTAMP NOT NULL DEFAULT NOW(), + "organizationId" uuid REFERENCES "organizations" ("id") NOT NULL, + "type" VARCHAR(255) NOT NULL, + "name" VARCHAR(255) NOT NULL, + "source" VARCHAR(255) NOT NULL, + "default" BOOLEAN NOT NULL DEFAULT FALSE, + "value" TEXT NOT NULL +); + +-- make sure there is only one default attribute +CREATE UNIQUE INDEX "orgAttributes_organizationId_name_default" ON "orgAttributes" ("organizationId", "name", "default") WHERE "default"; + +CREATE OR REPLACE FUNCTION add_org_attribute( + _org_id UUID, + _type TEXT, + _name TEXT, + _source TEXT, + _default BOOLEAN, + _value anyelement +) +RETURNS VOID AS $$ +BEGIN + IF _value IS NOT NULL THEN + INSERT INTO "orgAttributes" ("organizationId", "type", "name", "source", "default", "value") + VALUES (_org_id, _type, _name, _source, _default, _value::TEXT) + ON CONFLICT DO NOTHING; + END IF; +END; +$$ LANGUAGE plpgsql; + + +DO $$ +DECLARE + _org RECORD; + _value TEXT; + _source TEXT; +BEGIN + -- types: string | decimal | integer | boolean | object | array + FOR _org IN SELECT * FROM organizations LOOP + _source := CASE + WHEN _org."lastEnrichedAt" is null and NOT _org."manuallyCreated" THEN 'integration' + WHEN _org."lastEnrichedAt" is null and _org."manuallyCreated" THEN 'custom' + WHEN _org."lastEnrichedAt" is not null THEN 'peopledatalabs' + ELSE 'unknown' + END; + + PERFORM add_org_attribute(_org.id, 'string', 'description', _source, TRUE, _org.description); + + PERFORM add_org_attribute(_org.id, 'string', 'logo', _source, TRUE, _org.logo); + PERFORM add_org_attribute(_org.id, 'string', 'location', _source, TRUE, _org.location); + PERFORM add_org_attribute(_org.id, 'string', 'type', _source, TRUE, _org.type); + PERFORM add_org_attribute(_org.id, 'string', 'geoLocation', _source, TRUE, _org."geoLocation"); + PERFORM add_org_attribute(_org.id, 'string', 'size', _source, TRUE, _org.size); + PERFORM add_org_attribute(_org.id, 'string', 'ticker', _source, TRUE, _org.ticker); + PERFORM add_org_attribute(_org.id, 'string', 'headline', _source, TRUE, _org.headline); + PERFORM add_org_attribute(_org.id, 'string', 'industry', _source, TRUE, _org.industry); + PERFORM add_org_attribute(_org.id, 'string', 'gicsSector', _source, TRUE, _org."gicsSector"); + PERFORM add_org_attribute(_org.id, 'string', 'ultimateParent', _source, TRUE, _org."ultimateParent"); + PERFORM add_org_attribute(_org.id, 'string', 'immediateParent', _source, TRUE, _org."immediateParent"); + + PERFORM add_org_attribute(_org.id, 'integer', 'employees', _source, TRUE, _org.employees); + PERFORM add_org_attribute(_org.id, 'integer', 'founded', _source, TRUE, _org.founded); + + PERFORM add_org_attribute(_org.id, 'decimal', 'averageEmployeeTenure', _source, TRUE, _org."averageEmployeeTenure"); + + PERFORM add_org_attribute(_org.id, 'object', 'revenueRange', _source, TRUE, _org."revenueRange"); + PERFORM add_org_attribute(_org.id, 'object', 'employeeCountByCountry', _source, TRUE, _org."employeeCountByCountry"); + PERFORM add_org_attribute(_org.id, 'object', 'address', _source, TRUE, _org.address); + PERFORM add_org_attribute(_org.id, 'object', 'averageTenureByLevel', _source, TRUE, _org."averageTenureByLevel"); + PERFORM add_org_attribute(_org.id, 'object', 'averageTenureByRole', _source, TRUE, _org."averageTenureByRole"); + PERFORM add_org_attribute(_org.id, 'object', 'employeeChurnRate', _source, TRUE, _org."employeeChurnRate"); + PERFORM add_org_attribute(_org.id, 'object', 'employeeCountByMonth', _source, TRUE, _org."employeeCountByMonth"); + PERFORM add_org_attribute(_org.id, 'object', 'employeeGrowthRate', _source, TRUE, _org."employeeGrowthRate"); + PERFORM add_org_attribute(_org.id, 'object', 'employeeCountByMonthByLevel', _source, TRUE, _org."employeeCountByMonthByLevel"); + PERFORM add_org_attribute(_org.id, 'object', 'employeeCountByMonthByRole', _source, TRUE, _org."employeeCountByMonthByRole"); + PERFORM add_org_attribute(_org.id, 'object', 'grossAdditionsByMonth', _source, TRUE, _org."grossAdditionsByMonth"); + PERFORM add_org_attribute(_org.id, 'object', 'grossDeparturesByMonth', _source, TRUE, _org."grossDeparturesByMonth"); + PERFORM add_org_attribute(_org.id, 'object', 'naics', _source, TRUE, _org.naics); + + FOR _value IN SELECT UNNEST(_org."emails") AS value LOOP + PERFORM add_org_attribute(_org.id, 'string', 'email', _source, FALSE, _value); + END LOOP; + FOR _value IN SELECT UNNEST(_org."names") AS value LOOP + PERFORM add_org_attribute(_org.id, 'string', 'name', _source, FALSE, _value); + END LOOP; + PERFORM add_org_attribute(_org.id, 'string', 'name', _source, TRUE, _org."displayName"); + FOR _value IN SELECT UNNEST(_org."phoneNumbers") AS value LOOP + PERFORM add_org_attribute(_org.id, 'string', 'phoneNumber', _source, FALSE, _value); + END LOOP; + FOR _value IN SELECT UNNEST(_org."tags") AS value LOOP + PERFORM add_org_attribute(_org.id, 'string', 'tag', _source, FALSE, _value); + END LOOP; + FOR _value IN SELECT UNNEST(_org."profiles") AS value LOOP + PERFORM add_org_attribute(_org.id, 'string', 'profile', _source, FALSE, _value); + END LOOP; + FOR _value IN SELECT UNNEST(_org."allSubsidiaries") AS value LOOP + PERFORM add_org_attribute(_org.id, 'string', 'subsidiary', _source, FALSE, _value); + END LOOP; + FOR _value IN SELECT UNNEST(_org."alternativeNames") AS value LOOP + PERFORM add_org_attribute(_org.id, 'string', 'alternativeName', _source, FALSE, _value); + END LOOP; + FOR _value IN SELECT UNNEST(_org."directSubsidiaries") AS value LOOP + PERFORM add_org_attribute(_org.id, 'string', 'directSubsidiary', _source, FALSE, _value); + END LOOP; +-- EXCEPTION WHEN OTHERS THEN +-- RAISE EXCEPTION '_org: %', _org.id; +-- END; + END LOOP; +END; +$$; + +DROP FUNCTION add_org_attribute(UUID, TEXT, TEXT, TEXT, BOOLEAN, anyelement); + +ALTER TABLE organizations RENAME COLUMN "emails" TO "old_emails"; +ALTER TABLE organizations RENAME COLUMN "phoneNumbers" TO "old_phoneNumbers"; +ALTER TABLE organizations RENAME COLUMN "employeeCountByCountry" TO "old_employeeCountByCountry"; +ALTER TABLE organizations RENAME COLUMN "geoLocation" TO "old_geoLocation"; +ALTER TABLE organizations RENAME COLUMN "ticker" TO "old_ticker"; +ALTER TABLE organizations RENAME COLUMN "profiles" TO "old_profiles"; +ALTER TABLE organizations RENAME COLUMN "address" TO "old_address"; +ALTER TABLE organizations RENAME COLUMN "attributes" TO "old_attributes"; +ALTER TABLE organizations RENAME COLUMN "allSubsidiaries" TO "old_allSubsidiaries"; +ALTER TABLE organizations RENAME COLUMN "alternativeNames" TO "old_alternativeNames"; +ALTER TABLE organizations RENAME COLUMN "averageEmployeeTenure" TO "old_averageEmployeeTenure"; +ALTER TABLE organizations RENAME COLUMN "averageTenureByLevel" TO "old_averageTenureByLevel"; +ALTER TABLE organizations RENAME COLUMN "averageTenureByRole" TO "old_averageTenureByRole"; +ALTER TABLE organizations RENAME COLUMN "directSubsidiaries" TO "old_directSubsidiaries"; +ALTER TABLE organizations RENAME COLUMN "employeeCountByMonth" TO "old_employeeCountByMonth"; +ALTER TABLE organizations RENAME COLUMN "employeeCountByMonthByLevel" TO "old_employeeCountByMonthByLevel"; +ALTER TABLE organizations RENAME COLUMN "employeeCountByMonthByRole" TO "old_employeeCountByMonthByRole"; +ALTER TABLE organizations RENAME COLUMN "gicsSector" TO "old_gicsSector"; +ALTER TABLE organizations RENAME COLUMN "grossAdditionsByMonth" TO "old_grossAdditionsByMonth"; +ALTER TABLE organizations RENAME COLUMN "grossDeparturesByMonth" TO "old_grossDeparturesByMonth"; +ALTER TABLE organizations RENAME COLUMN "ultimateParent" TO "old_ultimateParent"; +ALTER TABLE organizations RENAME COLUMN "immediateParent" TO "old_immediateParent"; +ALTER TABLE organizations RENAME COLUMN "manuallyChangedFields" TO "old_manuallyChangedFields"; +ALTER TABLE organizations RENAME COLUMN "naics" TO "old_naics"; +ALTER TABLE organizations RENAME COLUMN "names" TO "old_names"; diff --git a/backend/src/database/models/organization.ts b/backend/src/database/models/organization.ts index 1b6004bbce..01aa98a81e 100644 --- a/backend/src/database/models/organization.ts +++ b/backend/src/database/models/organization.ts @@ -9,64 +9,6 @@ export default (sequelize) => { defaultValue: DataTypes.UUIDV4, primaryKey: true, }, - displayName: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - location: { - type: DataTypes.TEXT, - allowNull: true, - }, - description: { - type: DataTypes.TEXT, - allowNull: true, - comment: 'A detailed description of the company', - }, - immediateParent: { - type: DataTypes.TEXT, - allowNull: true, - }, - ultimateParent: { - type: DataTypes.TEXT, - allowNull: true, - }, - emails: { - type: DataTypes.ARRAY(DataTypes.TEXT), - allowNull: true, - default: [], - }, - names: { - type: DataTypes.ARRAY(DataTypes.TEXT), - allowNull: true, - default: [], - }, - phoneNumbers: { - type: DataTypes.ARRAY(DataTypes.TEXT), - allowNull: true, - default: [], - }, - logo: { - type: DataTypes.TEXT, - allowNull: true, - }, - tags: { - type: DataTypes.ARRAY(DataTypes.TEXT), - allowNull: true, - default: [], - }, - employees: { - type: DataTypes.INTEGER, - allowNull: true, - comment: 'total employee count of the company', - }, - revenueRange: { - type: DataTypes.JSONB, - allowNull: true, - comment: 'inferred revenue range of the company', - }, importHash: { type: DataTypes.STRING(255), allowNull: true, @@ -79,56 +21,7 @@ export default (sequelize) => { defaultValue: false, allowNull: false, }, - founded: { - type: DataTypes.INTEGER, - allowNull: true, - }, - industry: { - type: DataTypes.TEXT, - allowNull: true, - }, - size: { - type: DataTypes.TEXT, - allowNull: true, - comment: 'A range representing the size of the company.', - }, - naics: { - type: DataTypes.JSONB, - allowNull: true, - comment: 'industry classifications for a company according to NAICS', - }, - headline: { - type: DataTypes.TEXT, - allowNull: true, - comment: 'A brief description of the company', - }, - ticker: { - type: DataTypes.TEXT, - allowNull: true, - comment: "the company's stock symbol", - }, - geoLocation: { - type: DataTypes.STRING, - allowNull: true, - }, - type: { - type: DataTypes.TEXT, - allowNull: true, - comment: "The company's type. For example NGO", - }, - employeeCountByCountry: { - type: DataTypes.JSONB, - allowNull: true, - }, - address: { - type: DataTypes.JSONB, - allowNull: true, - comment: "granular information about the location of the company's current headquarters.", - }, - profiles: { - type: DataTypes.ARRAY(DataTypes.TEXT), - allowNull: true, - }, + lastEnrichedAt: { type: DataTypes.DATE, allowNull: true, @@ -138,71 +31,6 @@ export default (sequelize) => { allowNull: false, defaultValue: false, }, - attributes: { - type: DataTypes.JSONB, - defaultValue: {}, - }, - allSubsidiaries: { - type: DataTypes.ARRAY(DataTypes.TEXT), - allowNull: true, - }, - alternativeNames: { - type: DataTypes.ARRAY(DataTypes.TEXT), - allowNull: true, - }, - averageEmployeeTenure: { - type: DataTypes.FLOAT, - allowNull: true, - }, - averageTenureByLevel: { - type: DataTypes.JSONB, - allowNull: true, - }, - averageTenureByRole: { - type: DataTypes.JSONB, - allowNull: true, - }, - directSubsidiaries: { - type: DataTypes.ARRAY(DataTypes.TEXT), - allowNull: true, - }, - employeeChurnRate: { - type: DataTypes.JSONB, - allowNull: true, - }, - employeeCountByMonth: { - type: DataTypes.JSONB, - allowNull: true, - }, - employeeGrowthRate: { - type: DataTypes.JSONB, - allowNull: true, - }, - employeeCountByMonthByLevel: { - type: DataTypes.JSONB, - allowNull: true, - }, - employeeCountByMonthByRole: { - type: DataTypes.JSONB, - allowNull: true, - }, - gicsSector: { - type: DataTypes.TEXT, - allowNull: true, - }, - grossAdditionsByMonth: { - type: DataTypes.JSONB, - allowNull: true, - }, - grossDeparturesByMonth: { - type: DataTypes.JSONB, - allowNull: true, - }, - manuallyChangedFields: { - type: DataTypes.ARRAY(DataTypes.TEXT), - allowNull: true, - default: [], - }, }, { indexes: [ diff --git a/backend/src/database/repositories/organizationRepository.ts b/backend/src/database/repositories/organizationRepository.ts index cf616c9d82..a9302ae1ad 100644 --- a/backend/src/database/repositories/organizationRepository.ts +++ b/backend/src/database/repositories/organizationRepository.ts @@ -10,17 +10,22 @@ import { cleanUpOrgIdentities, fetchOrgIdentities, fetchManyOrgIdentities, - updateOrgIdentity, -} from '@crowd/data-access-layer/src/org_identities' + updateOrgIdentityVerifiedFlag, + findOrgAttributes, + queryOrgIdentities, + OrgIdentityField, + findManyOrgAttributes, + upsertOrgAttributes, + IDbOrgAttribute, + deleteOrgAttributes, +} from '@crowd/data-access-layer/src/organizations' import { FieldTranslatorFactory, OpensearchQueryParser } from '@crowd/opensearch' import { FeatureFlag, - IEnrichableOrganization, IMemberRenderFriendlyRole, IMemberRoleWithOrganization, IOrganization, IOrganizationIdentity, - IOrganizationMergeSuggestion, MergeActionState, MergeActionType, OpenSearchIndex, @@ -29,11 +34,13 @@ import { SegmentProjectGroupNestedData, SegmentProjectNestedData, } from '@crowd/types' -import lodash, { chunk, uniq } from 'lodash' +import lodash, { uniq } from 'lodash' import Sequelize, { QueryTypes } from 'sequelize' import validator from 'validator' import { findManyLfxMemberships } from '@crowd/data-access-layer/src/lfx_memberships' -import { fetchManyOrgSegments } from '@crowd/data-access-layer/src/org_segments' +import { fetchManyOrgSegments } from '@crowd/data-access-layer/src/organizations/segments' +import { OrganizationField, findOrgById } from '@crowd/data-access-layer/src/orgs' +import { findAttribute } from '@crowd/data-access-layer/src/organizations/attributesConfig' import { IFetchOrganizationMergeSuggestionArgs, SimilarityScoreRange, @@ -51,11 +58,6 @@ interface IOrganizationId { id: string } -interface IOrganizationNoMerge { - organizationId: string - noMergeId: string -} - class OrganizationRepository { public static QUERY_FILTER_COLUMN_MAP: Map = new Map([ // id fields @@ -95,183 +97,9 @@ class OrganizationRepository { // org fields for display ['logo', 'o."logo"'], - ['naics', 'o."naics"'], - ['profiles', 'o."profiles"'], - ['ticker', 'o."ticker"'], - ['address', 'o."address"'], - ['geoLocation', 'o."geoLocation"'], - ['employeeCountByCountry', 'o."employeeCountByCountry"'], ['description', 'o."description"'], - ['allSubsidiaries', 'o."allSubsidiaries"'], - ['alternativeNames', 'o."alternativeNames"'], - ['averageEmployeeTenure', 'o."averageEmployeeTenure"'], - ['averageTenureByLevel', 'o."averageTenureByLevel"'], - ['averageTenureByRole', 'o."averageTenureByRole"'], - ['directSubsidiaries', 'o."directSubsidiaries"'], - ['employeeChurnRate', 'o."employeeChurnRate"'], - ['employeeCountByMonth', 'o."employeeCountByMonth"'], - ['employeeCountByMonthByLevel', 'o."employeeCountByMonthByLevel"'], - ['employeeCountByMonthByRole', 'o."employeeCountByMonthByRole"'], - ['gicsSector', 'o."gicsSector"'], - ['grossAdditionsByMonth', 'o."grossAdditionsByMonth"'], - ['grossDeparturesByMonth', 'o."grossDeparturesByMonth"'], - ['ultimateParent', 'o."ultimateParent"'], - ['immediateParent', 'o."immediateParent"'], - ['names', 'o.names'], ]) - static async filterByPayingTenant( - tenantId: string, - limit: number, - options: IRepositoryOptions, - ): Promise { - const database = SequelizeRepository.getSequelize(options) - const transaction = SequelizeRepository.getTransaction(options) - const query = ` - with org_activities as (select a."organizationId", count(a.id) as "orgActivityCount" - from activities a - where a."tenantId" = :tenantId - and a."deletedAt" is null - group by a."organizationId" - having count(id) > 0), - identities as (select oi."organizationId", jsonb_agg(oi) as "identities" - from "organizationIdentities" oi - where oi."tenantId" = :tenantId - group by oi."organizationId") - select org.id, - i.identities, - org.emails, - org.names, - org."displayName", - org."location", - org."lastEnrichedAt", - org."employees", - org."size", - org."founded", - org."industry", - org."naics", - org."profiles", - org."headline", - org."ticker", - org."type", - org."address", - org."geoLocation", - org."employeeCountByCountry", - org."description", - org."revenueRange", - org."tags", - org."allSubsidiaries", - org."alternativeNames", - org."averageEmployeeTenure", - org."averageTenureByLevel", - org."averageTenureByRole", - org."directSubsidiaries", - org."employeeChurnRate", - org."employeeCountByMonth", - org."employeeGrowthRate", - org."employeeCountByMonthByLevel", - org."employeeCountByMonthByRole", - org."gicsSector", - org."grossAdditionsByMonth", - org."grossDeparturesByMonth", - org."ultimateParent", - org."immediateParent", - activity."orgActivityCount" - from "organizations" as org - join org_activities activity on activity."organizationId" = org."id" - join identities i on i."organizationId" = org.id - where :tenantId = org."tenantId" - and (org."lastEnrichedAt" is null or date_part('month', age(now(), org."lastEnrichedAt")) >= 6) - order by org."lastEnrichedAt" asc, activity."orgActivityCount" desc, org."createdAt" desc - limit :limit - ` - const orgs: IEnrichableOrganization[] = await database.query(query, { - type: QueryTypes.SELECT, - transaction, - replacements: { - tenantId, - limit, - }, - }) - return orgs - } - - static async filterByActiveLastYear( - tenantId: string, - limit: number, - options: IRepositoryOptions, - ): Promise { - const database = SequelizeRepository.getSequelize(options) - const transaction = SequelizeRepository.getTransaction(options) - const query = ` - with org_activities as (select a."organizationId", count(a.id) as "orgActivityCount" - from activities a - where a."tenantId" = :tenantId - and a."deletedAt" is null - and a."createdAt" > (CURRENT_DATE - INTERVAL '1 year') - group by a."organizationId" - having count(id) > 0), - identities as (select oi."organizationId", jsonb_agg(oi) as "identities" - from "organizationIdentities" oi - where oi."tenantId" = :tenantId - group by oi."organizationId") - select org.id, - i.identities, - org.names, - org.emails, - org."displayName", - org."location", - org."lastEnrichedAt", - org."employees", - org."size", - org."founded", - org."industry", - org."naics", - org."profiles", - org."headline", - org."ticker", - org."type", - org."address", - org."geoLocation", - org."employeeCountByCountry", - org."description", - org."revenueRange", - org."tags", - org."allSubsidiaries", - org."alternativeNames", - org."averageEmployeeTenure", - org."averageTenureByLevel", - org."averageTenureByRole", - org."directSubsidiaries", - org."employeeChurnRate", - org."employeeCountByMonth", - org."employeeGrowthRate", - org."employeeCountByMonthByLevel", - org."employeeCountByMonthByRole", - org."gicsSector", - org."grossAdditionsByMonth", - org."grossDeparturesByMonth", - org."ultimateParent", - org."immediateParent", - activity."orgActivityCount" - from "organizations" as org - join org_activities activity on activity."organizationId" = org."id" - join identities i on i."organizationId" = org.id - where :tenantId = org."tenantId" - order by org."lastEnrichedAt" asc, activity."orgActivityCount" desc, org."createdAt" desc - limit :limit - ` - const orgs: IEnrichableOrganization[] = await database.query(query, { - type: QueryTypes.SELECT, - transaction, - replacements: { - tenantId, - limit, - }, - }) - return orgs - } - static async create(data, options: IRepositoryOptions) { const currentUser = SequelizeRepository.getCurrentUser(options) @@ -284,45 +112,10 @@ class OrganizationRepository { } const toInsert = { ...lodash.pick(data, [ - 'displayName', - 'description', - 'names', - 'emails', - 'phoneNumbers', - 'logo', - 'tags', - 'location', - 'employees', - 'revenueRange', 'importHash', 'isTeamOrganization', - 'employeeCountByCountry', - 'type', - 'ticker', - 'headline', - 'profiles', - 'naics', - 'industry', - 'founded', - 'size', 'lastEnrichedAt', 'manuallyCreated', - 'allSubsidiaries', - 'alternativeNames', - 'averageEmployeeTenure', - 'averageTenureByLevel', - 'averageTenureByRole', - 'directSubsidiaries', - 'employeeChurnRate', - 'employeeCountByMonth', - 'employeeGrowthRate', - 'employeeCountByMonthByLevel', - 'employeeCountByMonthByRole', - 'gicsSector', - 'grossAdditionsByMonth', - 'grossDeparturesByMonth', - 'ultimateParent', - 'immediateParent', ]), tenantId: tenant.id, @@ -356,58 +149,6 @@ class OrganizationRepository { return this.findById(record.id, options) } - static async bulkUpdate( - data: T, - fields: string[], - options: IRepositoryOptions, - isEnrichment: boolean = false, - ): Promise { - const transaction = SequelizeRepository.getTransaction(options) - - // Ensure every organization has a non-undefine primary ID - const isValid = new Set(data.filter((org) => org.id).map((org) => org.id)).size !== data.length - if (isValid) return [] as T - - if (isEnrichment) { - // Fetch existing organizations - const existingOrgs = await options.database.organization.findAll({ - where: { - id: { - [options.database.Sequelize.Op.in]: data.map((org) => org.id), - }, - }, - }) - - // Append new tags to existing tags instead of overwriting - if (fields.includes('tags')) { - // @ts-ignore - data = data.map((org) => { - const existingOrg = existingOrgs.find((o) => o.id === org.id) - if (existingOrg && existingOrg.tags) { - // Merge existing and new tags without duplicates - const incomingTags = org.tags || [] - org.tags = lodash.uniq([...existingOrg.tags, ...incomingTags]) - } - return org - }) - } - } - - // Using bulk insert to update on duplicate primary ID - try { - const orgs = await options.database.organization.bulkCreate(data, { - fields: ['id', 'tenantId', ...fields], - updateOnDuplicate: fields, - returning: fields, - transaction, - }) - return orgs - } catch (error) { - options.log.error('Error while bulk updating organizations!', error) - throw error - } - } - static async includeOrganizationToSegments(organizationId: string, options: IRepositoryOptions) { const seq = SequelizeRepository.getSequelize(options) @@ -478,45 +219,26 @@ class OrganizationRepository { } static ORGANIZATION_UPDATE_COLUMNS = [ - 'displayName', - 'description', - 'emails', - 'phoneNumbers', - 'logo', - 'tags', - 'location', - 'employees', - 'revenueRange', 'importHash', 'isTeamOrganization', - 'employeeCountByCountry', - 'type', - 'ticker', 'headline', - 'profiles', - 'naics', + 'lastEnrichedAt', + + // default attributes + 'type', 'industry', 'founded', 'size', 'employees', - 'lastEnrichedAt', - 'allSubsidiaries', - 'alternativeNames', - 'averageEmployeeTenure', - 'averageTenureByLevel', - 'averageTenureByRole', - 'directSubsidiaries', + 'displayName', + 'description', + 'logo', + 'tags', + 'location', + 'employees', + 'revenueRange', 'employeeChurnRate', - 'employeeCountByMonth', 'employeeGrowthRate', - 'employeeCountByMonthByLevel', - 'employeeCountByMonthByRole', - 'gicsSector', - 'grossAdditionsByMonth', - 'grossDeparturesByMonth', - 'ultimateParent', - 'immediateParent', - 'attributes', ] static isEqual = { @@ -530,6 +252,53 @@ class OrganizationRepository { attributes: (a, b) => lodash.isEqual(a, b), } + static convertOrgAttributesForInsert(data: any) { + const orgAttributes = [] + + for (const [name, attribute] of Object.entries(data.attributes)) { + const attributeDefinition = findAttribute(name) + + if (!(attribute as any).custom) { + continue // eslint-disable-line no-continue + } + + for (const value of (attribute as any).custom) { + const isDefault = value === (attribute as any).default + + orgAttributes.push({ + type: attributeDefinition.type, + name, + source: 'custom', + default: isDefault, + value, + }) + + if (isDefault && attributeDefinition.defaultColumn) { + data[attributeDefinition.defaultColumn] = value + } + } + } + + return orgAttributes + } + + static convertOrgAttributesForDisplay(attributes: IDbOrgAttribute[]) { + return attributes.reduce((acc, a) => { + if (!acc[a.name]) { + acc[a.name] = {} + } + if (!acc[a.name][a.source]) { + acc[a.name][a.source] = [] + } + + acc[a.name][a.source].push(a.value) + if (a.default) { + acc[a.name].default = a.value + } + return acc + }, {}) + } + static async update( id, data, @@ -603,65 +372,27 @@ class OrganizationRepository { } } - // exclude syncRemote attributes, since these are populated from organizationSyncRemote table - if (data.attributes?.syncRemote) { - delete data.attributes.syncRemote - } - - if (manualChange) { - const manuallyChangedFields: string[] = record.manuallyChangedFields || [] - - for (const column of this.ORGANIZATION_UPDATE_COLUMNS) { - let changed = false - - // only check fields that are in the data object that will be updated - if (column in data) { - if ( - record[column] !== null && - column in data && - (data[column] === null || data[column] === undefined) - ) { - // column was removed in the update -> will be set to null by sequelize - changed = true - } else if ( - record[column] === null && - data[column] !== null && - data[column] !== undefined && - // also ignore empty arrays - (!Array.isArray(data[column]) || data[column].length > 0) - ) { - // column was null before now it's not anymore - changed = true - } else if ( - this.isEqual[column] && - this.isEqual[column](record[column], data[column]) === false - ) { - // column value has changed - changed = true - } - } + if (data.attributes) { + const qx = SequelizeRepository.getQueryExecutor(options, transaction) - if (changed && !manuallyChangedFields.includes(column)) { - manuallyChangedFields.push(column) - } - } - - data.manuallyChangedFields = manuallyChangedFields - } else { - // ignore columns that were manually changed - // by rewriting them with db data - const manuallyChangedFields: string[] = record.manuallyChangedFields || [] - for (const manuallyChangedColumn of manuallyChangedFields) { - data[manuallyChangedColumn] = record[manuallyChangedColumn] - } + const orgAttributes = OrganizationRepository.convertOrgAttributesForInsert(data) + const existingAttributes = await findOrgAttributes(qx, record.id) - data.manuallyChangedFields = manuallyChangedFields + await deleteOrgAttributes( + qx, + existingAttributes + .filter((attr) => + // remove those we want to upsert + orgAttributes.find((a) => a.name === attr.name && a.source === attr.source), + ) + .map((a) => a.id), + ) + await upsertOrgAttributes(qx, record.id, orgAttributes) } const updatedData = { ...lodash.pick(data, this.ORGANIZATION_UPDATE_COLUMNS), updatedById: currentUser.id, - manuallyChangedFields: data.manuallyChangedFields, } captureNewState(updatedData) await options.database.organization.update(updatedData, { @@ -844,7 +575,7 @@ class OrganizationRepository { const qx = SequelizeRepository.getQueryExecutor(options, transaction) - await cleanUpOrgIdentities(qx, { organizationId, tenantId: currentTenant.id }) + await cleanUpOrgIdentities(qx, organizationId, currentTenant.id) await OrganizationRepository.addIdentities(organizationId, identities, options) } @@ -869,7 +600,7 @@ class OrganizationRepository { const qx = SequelizeRepository.getQueryExecutor(options, transaction) - await updateOrgIdentity(qx, { + await updateOrgIdentityVerifiedFlag(qx, { organizationId, tenantId: currentTenant.id, platform: identity.platform, @@ -1072,126 +803,6 @@ class OrganizationRepository { } } - static async findNoMergeIds(id: string, options: IRepositoryOptions): Promise { - const transaction = SequelizeRepository.getTransaction(options) - const seq = SequelizeRepository.getSequelize(options) - - const query = `select onm."organizationId", onm."noMergeId" from "organizationNoMerge" onm - where onm."organizationId" = :id or onm."noMergeId" = :id;` - - try { - const results: IOrganizationNoMerge[] = await seq.query(query, { - type: QueryTypes.SELECT, - replacements: { - id, - }, - transaction, - }) - - return Array.from( - results.reduce((acc, r) => { - if (id === r.organizationId) { - acc.add(r.noMergeId) - } else if (id === r.noMergeId) { - acc.add(r.organizationId) - } - return acc - }, new Set()), - ) - } catch (error) { - options.log.error('error while getting non existing organizations from db', error) - throw error - } - } - - static async addToMerge( - suggestions: IOrganizationMergeSuggestion[], - options: IRepositoryOptions, - ): Promise { - const transaction = SequelizeRepository.getTransaction(options) - const seq = SequelizeRepository.getSequelize(options) - - // Remove possible duplicates - suggestions = lodash.uniqWith(suggestions, (a, b) => - lodash.isEqual(lodash.sortBy(a.organizations), lodash.sortBy(b.organizations)), - ) - - // check all suggestion ids exists in the db - const uniqueOrganizationIds = Array.from( - suggestions.reduce((acc, suggestion) => { - acc.add(suggestion.organizations[0]) - acc.add(suggestion.organizations[1]) - return acc - }, new Set()), - ) - - // filter non existing org ids from suggestions - const nonExistingIds = await OrganizationRepository.findNonExistingIds( - uniqueOrganizationIds, - options, - ) - - suggestions = suggestions.filter( - (s) => - !nonExistingIds.includes(s.organizations[0]) && - !nonExistingIds.includes(s.organizations[1]), - ) - - // Process suggestions in chunks of 100 or less - const suggestionChunks: IOrganizationMergeSuggestion[][] = chunk(suggestions, 100) - - const insertValues = ( - organizationId: string, - toMergeId: string, - similarity: number | null, - index: number, - ) => { - const idPlaceholder = (key: string) => `${key}${index}` - return { - query: `(:${idPlaceholder('organizationId')}, :${idPlaceholder( - 'toMergeId', - )}, :${idPlaceholder('similarity')}, NOW(), NOW())`, - replacements: { - [idPlaceholder('organizationId')]: organizationId, - [idPlaceholder('toMergeId')]: toMergeId, - [idPlaceholder('similarity')]: similarity === null ? null : similarity, - }, - } - } - - for (const suggestionChunk of suggestionChunks) { - const placeholders: string[] = [] - let replacements: Record = {} - - suggestionChunk.forEach((suggestion, index) => { - const { query, replacements: chunkReplacements } = insertValues( - suggestion.organizations[0], - suggestion.organizations[1], - suggestion.similarity, - index, - ) - placeholders.push(query) - replacements = { ...replacements, ...chunkReplacements } - }) - - const query = ` - INSERT INTO "organizationToMerge" ("organizationId", "toMergeId", "similarity", "createdAt", "updatedAt") - VALUES ${placeholders.join(', ')} - on conflict do nothing; - ` - try { - await seq.query(query, { - replacements, - type: QueryTypes.INSERT, - transaction, - }) - } catch (error) { - options.log.error('error adding organizations to merge', error) - throw error - } - } - } - static async countOrganizationMergeSuggestions( organizationFilter: string, similarityFilter: string, @@ -1516,76 +1127,53 @@ class OrganizationRepository { static async findByVerifiedIdentities( identities: IOrganizationIdentity[], options: IRepositoryOptions, - ): Promise { + ): Promise { const transaction = SequelizeRepository.getTransaction(options) - const sequelize = SequelizeRepository.getSequelize(options) - const currentTenant = SequelizeRepository.getCurrentTenant(options) - - const identityConditions = identities - .map( - (_, index) => ` - (oi.platform = :platform${index} and oi.value = :value${index} and oi.type = :type${index} and oi.verified = true) - `, - ) - .join(' or ') + const qx = SequelizeRepository.getQueryExecutor(options, transaction) - const results = await sequelize.query( - ` - with - "organizationsWithIdentity" as ( - select oi."organizationId" - from "organizationIdentities" oi - where ${identityConditions} - ), - "organizationsWithCounts" as ( - select o.id, count(oi."organizationId") as total_counts - from organizations o - join "organizationIdentities" oi on o.id = oi."organizationId" - where o.id in (select "organizationId" from "organizationsWithIdentity") - group by o.id - ) - select o.id, - o.description, - o.emails, - o.logo, - o.tags, - o.employees, - o.location, - o.type, - o.size, - o.headline, - o.industry, - o.founded, - o.attributes - from organizations o - inner join "organizationsWithCounts" oc on o.id = oc.id - where o."tenantId" = :tenantId - order by oc.total_counts desc - limit 1; - `, - { - replacements: { - tenantId: currentTenant.id, - ...identities.reduce( - (acc, identity, index) => ({ - ...acc, - [`platform${index}`]: identity.platform, - [`value${index}`]: identity.value, - [`type${index}`]: identity.type, - }), - {}, - ), - }, - type: QueryTypes.SELECT, - transaction, + const foundOrgs = await queryOrgIdentities(qx, { + fields: [OrgIdentityField.ORGANIZATION_ID], + filter: { + or: identities.map((identity) => ({ + and: [ + { platform: { eq: identity.platform } }, + { value: { eq: identity.value } }, + { type: { eq: identity.type } }, + { verified: { eq: true } }, + ], + })), }, - ) + }) - if (results.length === 0) { + if (foundOrgs.length === 0) { return null } - const result = results[0] as IOrganization + const foundOrgsIdentities = await fetchManyOrgIdentities( + qx, + foundOrgs.map((o) => o.organizationId), + options.currentTenant.id, + ) + + const orgIdWithMostIdentities = foundOrgsIdentities.sort( + (a, b) => b.identities.length - a.identities.length, + )[0].organizationId + + const result = await findOrgById(qx, orgIdWithMostIdentities, { + fields: [ + OrganizationField.ID, + OrganizationField.DESCRIPTION, + OrganizationField.LOGO, + OrganizationField.TAGS, + OrganizationField.EMPLOYEES, + OrganizationField.LOCATION, + OrganizationField.TYPE, + OrganizationField.SIZE, + OrganizationField.HEADLINE, + OrganizationField.INDUSTRY, + OrganizationField.FOUNDED, + ], + }) return result } @@ -1597,6 +1185,11 @@ class OrganizationRepository { limit: 1, offset: 0, segmentId, + include: { + attributes: true, + lfxMemberships: true, + identities: true, + }, }, options, ) @@ -1605,106 +1198,13 @@ class OrganizationRepository { throw new Error404() } - return rows[0] - } - - static async findByName(name, options: IRepositoryOptions) { - const transaction = SequelizeRepository.getTransaction(options) - - const include = [] - - const currentTenant = SequelizeRepository.getCurrentTenant(options) - - const record = await options.database.organization.findOne({ - where: { - name, - tenantId: currentTenant.id, - }, - include, - transaction, - }) - - if (!record) { - return null - } - - return record.get({ plain: true }) - } - - static async findByUrl(url, options: IRepositoryOptions) { - const transaction = SequelizeRepository.getTransaction(options) - - const include = [] - - const currentTenant = SequelizeRepository.getCurrentTenant(options) - - const record = await options.database.organization.findOne({ - where: { - url, - tenantId: currentTenant.id, - }, - include, - transaction, - }) - - if (!record) { - return null - } - - return record.get({ plain: true }) - } + const organization = rows[0] - static async findOrCreateByDomain(domain: string, options: IRepositoryOptions) { - const transaction = SequelizeRepository.getTransaction(options) - const currentTenant = SequelizeRepository.getCurrentTenant(options) - const seq = SequelizeRepository.getSequelize(options) - - // check if domain already exists in another organization in the same tenant - const existingOrg = (await seq.query( - ` - select "organizationId" - from "organizationIdentities" - where - "tenantId" = :tenantId and - type = :type and - value = :value and - verified = true - `, - { - replacements: { - tenantId: currentTenant.id, - type: OrganizationIdentityType.PRIMARY_DOMAIN, - value: domain, - }, - type: QueryTypes.SELECT, - transaction, - }, - )) as any[] - - let organization = existingOrg && existingOrg.length > 0 ? existingOrg[0] : null - - if (!organization) { - const data = { - displayName: domain, - names: [domain], - identities: [ - { - value: domain, - type: OrganizationIdentityType.PRIMARY_DOMAIN, - verified: true, - platform: 'email', - }, - ], - tenantId: currentTenant.id, - } - organization = await this.create(data, options) - } - - return organization.id - } + const qx = SequelizeRepository.getQueryExecutor(options) + const attributes = await findOrgAttributes(qx, id) + organization.attributes = OrganizationRepository.convertOrgAttributesForDisplay(attributes) - static async filterIdInTenant(id, options: IRepositoryOptions) { - return lodash.get(await this.filterIdsInTenant([id], options), '[0]', null) + return organization } static async filterIdsInTenant(ids, options: IRepositoryOptions) { @@ -1763,73 +1263,6 @@ class OrganizationRepository { }) } - static async findOrganizationActivities( - organizationId: string, - limit: number, - offset: number, - options: IRepositoryOptions, - ): Promise { - const seq = SequelizeRepository.getSequelize(options) - - const results = await seq.query( - `select "id", "organizationId" - from "activities" - where "organizationId" = :organizationId - order by "createdAt" - limit :limit offset :offset`, - { - replacements: { - organizationId, - limit, - offset, - }, - type: QueryTypes.SELECT, - }, - ) - - return results - } - - static async findByIdOpensearch( - id: string, - options: IRepositoryOptions, - segmentId?: string, - ): Promise { - const segments = segmentId ? [segmentId] : SequelizeRepository.getSegmentIds(options) - - const response = await this.findAndCountAllOpensearch( - { - filter: { - and: [ - { - id: { - eq: id, - }, - }, - ], - }, - isProfileQuery: true, - limit: 1, - offset: 0, - segments, - }, - options, - ) - - if (response.count === 0) { - throw new Error404() - } - - const result = response.rows[0] - - // Parse attributes that are indexed as strings - if (result.attributes) { - result.attributes = JSON.parse(result.attributes) - } - - return result - } - static async findAndCountAllOpensearch( { filter = {} as any, @@ -2191,7 +1624,13 @@ class OrganizationRepository { identities: true, lfxMemberships: true, segments: false, - } as { identities?: boolean; lfxMemberships?: boolean; segments?: boolean }, + attributes: false, + } as { + identities?: boolean + lfxMemberships?: boolean + segments?: boolean + attributes?: boolean + }, }, options: IRepositoryOptions, ) { @@ -2309,10 +1748,7 @@ class OrganizationRepository { }) } if (include.identities) { - const identities = await fetchManyOrgIdentities(qx, { - organizationIds: orgIds, - tenantId: options.currentTenant.id, - }) + const identities = await fetchManyOrgIdentities(qx, orgIds, options.currentTenant.id) rows.forEach((org) => { org.identities = identities.find((i) => i.organizationId === org.id)?.identities || [] @@ -2325,6 +1761,13 @@ class OrganizationRepository { org.segments = orgSegments.find((i) => i.organizationId === org.id)?.segments || [] }) } + if (include.attributes) { + const attributes = await findManyOrgAttributes(qx, orgIds) + + rows.forEach((org) => { + org.attributes = attributes.find((a) => a.organizationId === org.id)?.attributes || [] + }) + } return { rows, count, limit, offset } } diff --git a/backend/src/services/memberService.ts b/backend/src/services/memberService.ts index bde0fc366f..0d67b5adba 100644 --- a/backend/src/services/memberService.ts +++ b/backend/src/services/memberService.ts @@ -416,7 +416,12 @@ export default class MemberService extends LoggerBase { const org = await organizationService.createOrUpdate( { displayName: domain, - names: [domain], + attributes: { + name: { + default: domain, + custom: [domain], + }, + }, identities: [ { value: domain, diff --git a/backend/src/services/organizationService.ts b/backend/src/services/organizationService.ts index 7f2167a547..d73566c99f 100644 --- a/backend/src/services/organizationService.ts +++ b/backend/src/services/organizationService.ts @@ -1,6 +1,6 @@ import { captureApiChange, organizationMergeAction } from '@crowd/audit-logs' import { Error400, websiteNormalizer } from '@crowd/common' -import { findLfxMembership, hasLfxMembership } from '@crowd/data-access-layer/src/lfx_memberships' +import { hasLfxMembership } from '@crowd/data-access-layer/src/lfx_memberships' import { LoggerBase } from '@crowd/logging' import { IOrganization, @@ -17,6 +17,11 @@ import { } from '@crowd/types' import { randomUUID } from 'crypto' import lodash from 'lodash' +import { + findOrgAttributes, + upsertOrgAttributes, + upsertOrgIdentities, +} from '@crowd/data-access-layer/src/organizations' import getObjectWithoutKey from '@/utils/getObjectWithoutKey' import { IActiveOrganizationFilter } from '@/database/repositories/types/organizationTypes' import MemberOrganizationRepository from '@/database/repositories/memberOrganizationRepository' @@ -45,48 +50,15 @@ export default class OrganizationService extends LoggerBase { } static ORGANIZATION_MERGE_FIELDS = [ + 'displayName', 'description', - 'names', - 'emails', - 'phoneNumbers', 'logo', - 'tags', - 'type', + 'headline', + 'joinedAt', - 'employees', - 'revenueRange', - 'location', 'isTeamOrganization', - 'employeeCountByCountry', - 'geoLocation', - 'size', - 'ticker', - 'headline', - 'profiles', - 'naics', - 'address', - 'industry', - 'founded', - 'displayName', - 'attributes', - 'allSubsidiaries', - 'alternativeNames', - 'averageEmployeeTenure', - 'averageTenureByLevel', - 'averageTenureByRole', - 'directSubsidiaries', - 'employeeChurnRate', - 'employeeCountByMonth', - 'employeeGrowthRate', - 'employeeCountByMonthByLevel', - 'employeeCountByMonthByRole', - 'gicsSector', - 'grossAdditionsByMonth', - 'grossDeparturesByMonth', - 'ultimateParent', - 'immediateParent', 'manuallyCreated', - 'manuallyChangedFields', + 'activityCount', 'memberCount', ] @@ -100,6 +72,11 @@ export default class OrganizationService extends LoggerBase { const identities = await OrganizationRepository.getIdentities([organizationId], this.options) + const qx = SequelizeRepository.getQueryExecutor(this.options) + const attributes = OrganizationRepository.convertOrgAttributesForDisplay( + await findOrgAttributes(qx, organization.id), + ) + if ( !identities.some( (i) => @@ -125,34 +102,6 @@ export default class OrganizationService extends LoggerBase { const primaryBackup = mergeAction.unmergeBackup.primary as IOrganizationUnmergeBackup const secondaryBackup = mergeAction.unmergeBackup.secondary as IOrganizationUnmergeBackup - for (const key of OrganizationService.ORGANIZATION_MERGE_FIELDS) { - if (!(organization.manuallyChangedFields || []).includes(key)) { - // handle string arrays - if ( - key in - [ - 'names', - 'emails', - 'phoneNumbers', - 'tags', - 'profiles', - 'allSubsidiaries', - 'alternativeNames', - 'directSubsidiaries', - ] - ) { - organization[key] = organization[key].filter( - (k) => !secondaryBackup[key].some((f) => f === k), - ) - } else if ( - primaryBackup[key] !== organization[key] && - secondaryBackup[key] === organization[key] - ) { - organization[key] = null - } - } - } - // identities organization.identities = organization.identities.filter( (i) => @@ -170,6 +119,7 @@ export default class OrganizationService extends LoggerBase { primary: { ...lodash.pick(organization, OrganizationService.ORGANIZATION_MERGE_FIELDS), identities: organization.identities, + attributes, activityCount: primaryBackup.activityCount, memberCount: primaryBackup.memberCount, }, @@ -219,50 +169,25 @@ export default class OrganizationService extends LoggerBase { primary: { ...lodash.pick(organization, OrganizationService.ORGANIZATION_MERGE_FIELDS), identities: primaryIdentities, + attributes, memberCount: organization.memberCount - secondaryMemberCount, activityCount: organization.activityCount - secondaryActivityCount, }, secondary: { id: randomUUID(), identities: secondaryIdentities, - names: [identity.value], displayName: identity.platform === 'linkedin' ? identity.value.split(':').pop() : identity.value, - description: null, + attributes: { + name: { + default: identity.value, + custom: [identity.value], + }, + }, activityCount: secondaryActivityCount, memberCount: secondaryMemberCount, - emails: [], - phoneNumbers: [], - logo: null, - tags: [], - employees: null, - location: null, isTeamOrganization: false, - employeeCountByCountry: null, - geoLocation: null, - size: null, - ticker: null, - headline: null, - profiles: [], - naics: null, - address: null, - industry: null, - founded: null, - attributes: {}, searchSyncedAt: null, - allSubsidiaries: [], - alternativeNames: [], - averageEmployeeTenure: null, - averageTenureByLevel: null, - averageTenureByRole: null, - directSubsidiaries: [], - employeeChurnRate: null, - employeeCountByMonth: {}, - employeeGrowthRate: null, - employeeCountByMonthByLevel: {}, - employeeCountByMonthByRole: {}, - gicsSector: null, - grossAdditionsByMonth: {}, }, } } catch (err) { @@ -712,15 +637,6 @@ export default class OrganizationService extends LoggerBase { static organizationsMerge(originalObject, toMergeObject) { return merge(originalObject, toMergeObject, { - description: keepPrimaryIfExists, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - names: mergeUniqueStringArrayItems, - emails: mergeUniqueStringArrayItems, - phoneNumbers: mergeUniqueStringArrayItems, - logo: keepPrimaryIfExists, - tags: mergeUniqueStringArrayItems, - employees: keepPrimaryIfExists, - revenueRange: keepPrimaryIfExists, importHash: keepPrimary, createdAt: keepPrimary, updatedAt: keepPrimary, @@ -728,40 +644,26 @@ export default class OrganizationService extends LoggerBase { tenantId: keepPrimary, createdById: keepPrimary, updatedById: keepPrimary, - location: keepPrimaryIfExists, isTeamOrganization: keepPrimaryIfExists, lastEnrichedAt: keepPrimary, - employeeCountByCountry: keepPrimaryIfExists, + searchSyncedAt: keepPrimary, + manuallyCreated: keepPrimary, + + // default attributes + description: keepPrimaryIfExists, + logo: keepPrimaryIfExists, + tags: mergeUniqueStringArrayItems, + employees: keepPrimaryIfExists, + revenueRange: keepPrimaryIfExists, + location: keepPrimaryIfExists, type: keepPrimaryIfExists, - geoLocation: keepPrimaryIfExists, size: keepPrimaryIfExists, - ticker: keepPrimaryIfExists, headline: keepPrimaryIfExists, - profiles: mergeUniqueStringArrayItems, - naics: keepPrimaryIfExists, - address: keepPrimaryIfExists, industry: keepPrimaryIfExists, founded: keepPrimaryIfExists, displayName: keepPrimary, - attributes: keepPrimary, - searchSyncedAt: keepPrimary, - allSubsidiaries: mergeUniqueStringArrayItems, - alternativeNames: mergeUniqueStringArrayItems, - averageEmployeeTenure: keepPrimaryIfExists, - averageTenureByLevel: keepPrimaryIfExists, - averageTenureByRole: keepPrimaryIfExists, - directSubsidiaries: mergeUniqueStringArrayItems, employeeChurnRate: keepPrimaryIfExists, - employeeCountByMonth: keepPrimaryIfExists, employeeGrowthRate: keepPrimaryIfExists, - employeeCountByMonthByLevel: keepPrimaryIfExists, - employeeCountByMonthByRole: keepPrimaryIfExists, - gicsSector: keepPrimaryIfExists, - grossAdditionsByMonth: keepPrimaryIfExists, - grossDeparturesByMonth: keepPrimaryIfExists, - ultimateParent: keepPrimaryIfExists, - immediateParent: keepPrimaryIfExists, - manuallyCreated: keepPrimary, }) } @@ -821,13 +723,6 @@ export default class OrganizationService extends LoggerBase { } try { - const primaryIdentity = verifiedIdentities[0] - const name = primaryIdentity.value - - if (!data.names) { - data.names = [name] - } - // Normalize the website identities for (const i of data.identities.filter((i) => [ @@ -851,90 +746,20 @@ export default class OrganizationService extends LoggerBase { this.options, ) - if (existing) { - // Set displayName if it doesn't exist - if (!existing.displayName) { - data.displayName = name - } - - // if it does exists update it - const updateData: Partial = {} - const fields = [ - 'displayName', - 'description', - 'names', - 'emails', - 'logo', - 'tags', - 'employees', - 'location', - 'type', - 'size', - 'headline', - 'industry', - 'founded', - 'attributes', - ] - fields.forEach((field) => { - if (!existing[field] && data[field]) { - updateData[field] = data[field] - } - }) - - if (Object.keys(updateData).length > 0) { - record = await OrganizationRepository.update(existing.id, updateData, { - ...this.options, - transaction, - }) - } else { - record = existing - } - - const existingIdentities = await OrganizationRepository.getIdentities(record.id, { - ...this.options, - transaction, - }) + const qx = SequelizeRepository.getQueryExecutor(this.options, transaction) - const toUpdate: IOrganizationIdentity[] = [] - const toCreate: IOrganizationIdentity[] = [] + if (existing) { + record = existing - for (const i of data.identities) { - const existing = existingIdentities.find( - (ei) => ei.value === i.value && ei.platform === i.platform && ei.type === i.type, - ) - if (!existing) { - toCreate.push(i) - } else if (existing && existing.verified !== i.verified) { - toUpdate.push(i) - } - } + if (record.attributes) { + const attributes = OrganizationRepository.convertOrgAttributesForInsert(record) - if (toCreate.length > 0) { - for (const i of toCreate) { - // add the identity - await OrganizationRepository.addIdentity(record.id, i, { - ...this.options, - transaction, - }) - } + await upsertOrgAttributes(qx, record.id, attributes) } - if (toUpdate.length > 0) { - for (const i of toUpdate) { - // update the identity - await OrganizationRepository.updateIdentity(record.id, i, { - ...this.options, - transaction, - }) - } - } + await upsertOrgIdentities(qx, record.id, record.tenantId, data.identities) } else { - const organization = { - ...data, // to keep uncacheable data (like identities, weakIdentities) - displayName: name, - } - - record = await OrganizationRepository.create(organization, { + record = await OrganizationRepository.create(data, { ...this.options, transaction, }) @@ -953,6 +778,11 @@ export default class OrganizationService extends LoggerBase { transaction, }) } + + if (record.attributes) { + const attributes = OrganizationRepository.convertOrgAttributesForInsert(record) + await upsertOrgAttributes(qx, record.id, attributes) + } } await SequelizeRepository.commitTransaction(transaction) @@ -1162,27 +992,6 @@ export default class OrganizationService extends LoggerBase { ) } - async findByUrl(url) { - return OrganizationRepository.findByUrl(url, this.options) - } - - async findOrCreateByDomain(domain) { - return OrganizationRepository.findOrCreateByDomain(domain, this.options) - } - - async findByIdOpensearch(id: string, segmentId?: string) { - const org = await OrganizationRepository.findByIdOpensearch(id, this.options, segmentId) - - const qx = SequelizeRepository.getQueryExecutor(this.options) - - org.lfxMembership = await findLfxMembership(qx, { - organizationId: id, - tenantId: this.options.currentTenant.id, - }) - - return org - } - async query(data) { const { filter, orderBy, limit, offset, segments } = data return OrganizationRepository.findAndCountAll( diff --git a/backend/src/types/enrichmentTypes.ts b/backend/src/types/enrichmentTypes.ts deleted file mode 100644 index 08a8e0869d..0000000000 --- a/backend/src/types/enrichmentTypes.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { IOrganizationIdentity } from '@crowd/types' -import { CompanyEnrichmentParams, CompanyResponse } from 'peopledatalabs' - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export type IEnrichmentResponse = CompanyResponse & { address?: any; geoLocation?: string } - -export type EnrichmentParams = CompanyEnrichmentParams -export type IOrganizations = IOrganization[] - -export interface IOrganization { - id: string - name: string - tenantId?: string - website?: string - location?: string - orgActivityCount?: number - revenueRange?: { max: number; min: number } - tags?: IEnrichmentResponse['tags'] - description?: IEnrichmentResponse['summary'] - employeeCountByCountry?: IEnrichmentResponse['employee_count_by_country'] - type?: IEnrichmentResponse['type'] - ticker?: IEnrichmentResponse['ticker'] - headline?: IEnrichmentResponse['headline'] - profiles?: IEnrichmentResponse['profiles'] - naics?: IEnrichmentResponse['naics'] - industry?: IEnrichmentResponse['industry'] - founded?: IEnrichmentResponse['founded'] - employees?: IEnrichmentResponse['employee_count'] - twitter?: ISocialNetwork - github?: ISocialNetwork - linkedin?: ISocialNetwork - crunchbase?: ISocialNetwork - lastEnrichedAt?: Date - geoLocation?: string - address?: IEnrichmentResponse['location'] - ultimateParent: IEnrichmentResponse['ultimate_parent'] - immediateParent: IEnrichmentResponse['immediate_parent'] - affiliatedProfiles?: IEnrichmentResponse['affiliated_profiles'] - allSubsidiaries?: IEnrichmentResponse['all_subsidiaries'] - alternativeDomains?: IEnrichmentResponse['alternative_domains'] - alternativeNames?: IEnrichmentResponse['alternative_names'] - averageEmployeeTenure?: IEnrichmentResponse['average_employee_tenure'] - averageTenureByLevel?: IEnrichmentResponse['average_tenure_by_level'] - averageTenureByRole?: IEnrichmentResponse['average_tenure_by_role'] - directSubsidiaries?: IEnrichmentResponse['direct_subsidiaries'] - employeeChurnRate?: IEnrichmentResponse['employee_churn_rate'] - employeeCountByMonth?: IEnrichmentResponse['employee_count_by_month'] - employeeGrowthRate?: IEnrichmentResponse['employee_growth_rate'] - employeeCountByMonthByLevel?: IEnrichmentResponse['employee_count_by_month_by_level'] - employeeCountByMonthByRole?: IEnrichmentResponse['employee_count_by_month_by_role'] - gicsSector?: IEnrichmentResponse['gics_sector'] - grossAdditionsByMonth?: IEnrichmentResponse['gross_additions_by_month'] - grossDeparturesByMonth?: IEnrichmentResponse['gross_departures_by_month'] - inferredRevenue?: IEnrichmentResponse['inferred_revenue'] - identities?: IOrganizationIdentity[] -} - -interface ISocialNetwork { - url: string - handle: string -} diff --git a/frontend/src/modules/organization/components/details/header/organization-details-header-logo.vue b/frontend/src/modules/organization/components/details/header/organization-details-header-logo.vue index b2161079b2..95e4a044cf 100644 --- a/frontend/src/modules/organization/components/details/header/organization-details-header-logo.vue +++ b/frontend/src/modules/organization/components/details/header/organization-details-header-logo.vue @@ -2,8 +2,8 @@
(); -const { isNew } = useOrganizationHelpers(); +const { isNew, logo, displayName } = useOrganizationHelpers(); const isEditModalOpen = ref(false); diff --git a/frontend/src/modules/organization/components/details/overview/attributes/organization-attribute-source.vue b/frontend/src/modules/organization/components/details/overview/attributes/organization-attribute-source.vue new file mode 100644 index 0000000000..bf2a8c7730 --- /dev/null +++ b/frontend/src/modules/organization/components/details/overview/attributes/organization-attribute-source.vue @@ -0,0 +1,46 @@ + + + + + diff --git a/frontend/src/modules/organization/components/details/overview/organization-details-attributes.vue b/frontend/src/modules/organization/components/details/overview/organization-details-attributes.vue index e63606feb4..9e4e7e006b 100644 --- a/frontend/src/modules/organization/components/details/overview/organization-details-attributes.vue +++ b/frontend/src/modules/organization/components/details/overview/organization-details-attributes.vue @@ -6,34 +6,6 @@
- -
-
-

- Description -

-
-
- -
-
- - -
-
-

- Location -

-
-
- -
-
-
{{ attribute.label }}

-

- Source: Enrichment -

+
-
+

No organization details yet @@ -94,21 +64,24 @@ import LfOrganizationAttributeArray import LfOrganizationAttributeJson from '@/modules/organization/components/details/overview/attributes/organization-attribute-json.vue'; import LfIcon from '@/ui-kit/icon/Icon.vue'; +import LfOrganizationAttributeSource + from '@/modules/organization/components/details/overview/attributes/organization-attribute-source.vue'; const props = defineProps<{ organization: Organization, }>(); -const description = computed(() => props.organization.description); -const location = computed(() => props.organization.location); - const visibleAttributes = computed(() => enrichmentAttributes - .filter((a) => ((props.organization[a.name] && a.type !== AttributeType.ARRAY && a.type !== AttributeType.JSON) - || (a.type === AttributeType.ARRAY && props.organization[a.name]?.length) - || (a.type === AttributeType.JSON && props.organization[a.name] && Object.keys(props.organization[a.name]).length)) && a.showInAttributes)); + .filter((a) => ((props.organization.attributes[a.name]?.default && a.type !== AttributeType.ARRAY && a.type !== AttributeType.JSON) + || (a.type === AttributeType.ARRAY && props.organization.attributes[a.name]?.default?.length) + || (a.type === AttributeType.JSON && props.organization.attributes[a.name]?.default + && Object.keys(props.organization.attributes[a.name]?.default).length)) && a.showInAttributes)); const getValue = (attribute: OrganizationEnrichmentConfig) => { - const value = props.organization[attribute.name]; + let value = props.organization.attributes[attribute.name]?.default; + if (attribute.type === AttributeType.JSON) { + value = JSON.parse(value); + } if (attribute.formatValue) { return attribute.formatValue(value); } diff --git a/frontend/src/modules/organization/components/edit/organization-edit-logo.vue b/frontend/src/modules/organization/components/edit/organization-edit-logo.vue index 611d49c62b..1f254cefd9 100644 --- a/frontend/src/modules/organization/components/edit/organization-edit-logo.vue +++ b/frontend/src/modules/organization/components/edit/organization-edit-logo.vue @@ -5,7 +5,7 @@

@@ -52,7 +52,7 @@ (); const { updateOrganization } = useOrganizationStore(); +const { displayName, logo } = useOrganizationHelpers(); + const form = reactive({ - logo: props.organization.logo, + logo: logo(props.organization), }); const rules = { @@ -105,7 +108,12 @@ const sending = ref(false); const update = () => { sending.value = true; updateOrganization(props.organization.id, { - logo: form.logo, + attributes: { + logo: { + default: form.logo, + custom: [form.logo], + }, + }, }) .then(() => { Message.success('Organization logo updated successfully!'); diff --git a/frontend/src/modules/organization/components/edit/organization-edit-name.vue b/frontend/src/modules/organization/components/edit/organization-edit-name.vue index dc510f7e69..d95361811f 100644 --- a/frontend/src/modules/organization/components/edit/organization-edit-name.vue +++ b/frontend/src/modules/organization/components/edit/organization-edit-name.vue @@ -19,15 +19,17 @@ import { required } from '@vuelidate/validators'; import useVuelidate from '@vuelidate/core'; import { Organization } from '@/modules/organization/types/Organization'; import { useOrganizationStore } from '@/modules/organization/store/pinia'; +import useOrganizationHelpers from '@/modules/organization/helpers/organization.helpers'; const props = defineProps<{ organization: Organization, }>(); const { updateOrganization } = useOrganizationStore(); +const { displayName } = useOrganizationHelpers(); const form = reactive({ - name: props.organization.displayName, + name: displayName(props.organization), }); const rules = { @@ -39,15 +41,20 @@ const rules = { const $v = useVuelidate(rules, form); const update = () => { - if (form.name === props.organization.displayName) { + if (form.name === displayName(props.organization)) { return; } if ($v.value.$invalid) { - form.name = props.organization.displayName; + form.name = displayName(props.organization); return; } updateOrganization(props.organization.id, { - displayName: form.name, + attributes: { + name: { + default: form.name, + custom: [form.name], + }, + }, }) .then(() => { Message.success('Organization name updated successfully!'); diff --git a/frontend/src/modules/organization/components/form/organization-form-details.vue b/frontend/src/modules/organization/components/form/organization-form-details.vue index 789b1b1790..26920cb507 100644 --- a/frontend/src/modules/organization/components/form/organization-form-details.vue +++ b/frontend/src/modules/organization/components/form/organization-form-details.vue @@ -14,77 +14,12 @@
- - - - - - - - - - - - - - - - - - - -