diff --git a/src/meta/assessment/table.ts b/src/meta/assessment/table.ts index 93fa10697f..0642ebe08b 100644 --- a/src/meta/assessment/table.ts +++ b/src/meta/assessment/table.ts @@ -21,6 +21,8 @@ export enum TableNames { growingStockTotal = 'growingStockTotal', primaryDesignatedManagementObjective = 'primaryDesignatedManagementObjective', specificForestCategories = 'specificForestCategories', + sustainableDevelopment15_2_1_1 = 'sustainableDevelopment15_2_1_1', + sustainableDevelopment15_2_1_2 = 'sustainableDevelopment15_2_1_2', sustainableDevelopment15_2_1_5 = 'sustainableDevelopment15_2_1_5', totalAreaWithDesignatedManagementObjective = 'totalAreaWithDesignatedManagementObjective', // Used to append ODP data to tableData diff --git a/src/test/migrations/steps/20241118084404-step-fix-fra2020-sdg-tables.ts b/src/test/migrations/steps/20241118084404-step-fix-fra2020-sdg-tables.ts new file mode 100644 index 0000000000..8ca6d16a97 --- /dev/null +++ b/src/test/migrations/steps/20241118084404-step-fix-fra2020-sdg-tables.ts @@ -0,0 +1,74 @@ +import { AssessmentNames, TableNames } from 'meta/assessment' + +import { AssessmentController } from 'server/controller/assessment' +import { BaseProtocol, Schemas } from 'server/db' + +import { updateDependencies } from 'test/migrations/steps/utils/updateDependencies' + +const assessmentName = AssessmentNames.fra +const cycleName = '2020' + +export default async (client: BaseProtocol) => { + const { assessment, cycle } = await AssessmentController.getOneWithCycle({ assessmentName, cycleName }, client) + + const { uuid: cycleUuid } = cycle + const schemaAssessment = Schemas.getName(assessment) + + await client.query(` + with src as + (select c.id + from ${schemaAssessment}.col c + left join ${schemaAssessment}.row r on r.id = c.row_id + left join ${schemaAssessment}."table" t on t.id = r.table_id + where t.props ->> 'name' = '${TableNames.sustainableDevelopment15_2_1_1}' + and c.props ->> 'colType' = 'header' + and r.props ->> 'index' = 'header_1' + and c.props ->> 'index' = '10') + update ${schemaAssessment}.col c + set props = c.props #- '{style,${cycleUuid}}' + #- '{labels,${cycleUuid}}' + || jsonb_build_object('cycles', (c.props -> 'cycles') - '${cycleUuid}') + from src + where c.id = src.id; + + with src as + (select c.id + from ${schemaAssessment}.col c + left join ${schemaAssessment}.row r on r.id = c.row_id + left join ${schemaAssessment}."table" t on t.id = r.table_id + where t.props ->> 'name' = '${TableNames.sustainableDevelopment15_2_1_2}' + and c.props ->> 'colName' = '2020' + order by c.props -> 'index') + update ${schemaAssessment}.col c + set props = jsonb_set( + jsonb_set( + c.props, '{style,${cycleUuid}}', '{}'::jsonb + ), + '{calculateFn,${cycleUuid}}', + '"biomassStock.forest_above_ground"' + ) + || + jsonb_build_object('cycles', (c.props -> 'cycles') || '["${cycleUuid}"]') + from src + where c.id = src.id; + `) + + await AssessmentController.generateMetadataCache({ assessment }, client) + + const update = await AssessmentController.getOneWithCycle({ assessmentName, cycleName, metaCache: true }, client) + await updateDependencies( + { + ...update, + includeSourceNodes: true, + nodes: [ + { + tableName: TableNames.sustainableDevelopment15_2_1_2, + variableName: 'aboveGroundBiomassStockForests', + colName: '2020', + value: { raw: undefined }, + }, + ], + }, + client + ) +} diff --git a/src/test/migrations/steps/utils/calculateRow.ts b/src/test/migrations/steps/utils/calculateRow.ts deleted file mode 100644 index bfbf315877..0000000000 --- a/src/test/migrations/steps/utils/calculateRow.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { Objects } from 'utils/objects' - -import { CountryIso } from 'meta/area' -import { Assessment, AssessmentMetaCaches, Cycle, Row } from 'meta/assessment' -import { ExpressionEvaluator } from 'meta/expressionEvaluator' - -import { CycleDataController } from 'server/controller/cycleData' -import { MetadataController } from 'server/controller/metadata' -import { BaseProtocol } from 'server/db' - -import { NodeRow } from 'test/dataMigration/types' - -/** - * @deprecated - */ -export const calculateRow = async ( - props: { - assessment: Assessment - cycle: Cycle - countryISOs: Array - row: Row - tableName: string - calculatedVariables: Record> - }, - client: BaseProtocol -): Promise> => { - const { assessment, cycle, countryISOs, row, tableName, calculatedVariables } = props - const values: Array = [] - const visited: Record>> = {} - - // console.log('====== calculating ', tableName, row.props.variableName) - const dependencies = AssessmentMetaCaches.getCalculationsDependencies({ - assessment, - cycle, - tableName, - variableName: row.props.variableName, - }) - - const data = await CycleDataController.getTableData( - { - assessment, - cycle, - countryISOs, - dependencies, - aggregate: false, - columns: [], - mergeOdp: true, - tableNames: [], - variables: [], - }, - client - ) - - const table = await MetadataController.getTable({ assessment, cycle, tableName }) - - for (let i = 0; i < countryISOs.length; i += 1) { - const countryIso = countryISOs[i] - for (let j = 0; j < table.props.columnNames[cycle.uuid].length; j += 1) { - const colName = table.props.columnNames[cycle.uuid][j] - - const col = row.cols.find((c) => c.props.colName === colName) - // eslint-disable-next-line no-continue - if (!col) continue - - const expression = row.props.calculateFn?.[cycle.uuid] ?? col.props.calculateFn?.[cycle.uuid] - - const raw = ExpressionEvaluator.evalFormula({ - assessment, - countryIso, - cycle, - data, - colName, - row, - formula: expression, - }) - - const value: NodeRow = { - country_iso: countryIso, - row_uuid: row.uuid, - col_uuid: col.uuid, - value: { raw: !Objects.isEmpty(raw) ? String(raw) : null, calculated: true }, - } - - if ( - values.find( - (v) => v.country_iso === value.country_iso && v.row_uuid === value.row_uuid && v.col_uuid === value.col_uuid - ) - ) { - throw new Error(`Duplicate node ${JSON.stringify(value)}`) - } - - if (visited[countryIso]?.[row.uuid]?.[col.uuid]) { - throw new Error(`Duplicate node ${JSON.stringify(value)}`) - } - - values.push(value) - if (!visited[countryIso]) visited[countryIso] = {} - if (!visited[countryIso][row.uuid]) visited[countryIso][row.uuid] = {} - visited[countryIso][row.uuid][col.uuid] = true - } - } - - if (!calculatedVariables[tableName]) calculatedVariables[tableName] = {} - calculatedVariables[tableName][row.props.variableName] = true - - // eslint-disable-next-line consistent-return - return values -} diff --git a/src/test/migrations/steps/utils/getRow.ts b/src/test/migrations/steps/utils/getRow.ts deleted file mode 100644 index 17b05d5e37..0000000000 --- a/src/test/migrations/steps/utils/getRow.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { Objects } from 'utils/objects' - -import { Assessment, Cycle, Row } from 'meta/assessment' - -import { BaseProtocol, Schemas } from 'server/db' -import { ColAdapter } from 'server/repository/adapter' - -export const getRow = async ( - props: { - tableName: string - variableName: string - cycle: Cycle - assessment: Assessment - }, - client: BaseProtocol -) => { - const { tableName, variableName, cycle, assessment } = props - const schema = Schemas.getName(assessment) - - return client.one( - ` - select r.*, - t.props ->> 'name' as table_name, - jsonb_agg(c.*) filter (where c.props -> 'cycles' ? '${cycle.uuid}') as cols - from ${schema}.row r - left join ${schema}."table" t on r.table_id = t.id - left join ${schema}.col c on r.id = c.row_id - where t.props -> 'cycles' ? '${cycle.uuid}' - and t.props ->> 'name' = '${tableName}' - and r.props -> 'cycles' ? '${cycle.uuid}' - and r.props ->> 'variableName' = '${variableName}' - and c.props -> 'cycles' ? '${cycle.uuid}' - and ( - (r.props ->> 'calculateFn' is not null and r.props -> 'calculateFn' ->> '${cycle.uuid}' is not null) - or - (c.props ->> 'calculateFn' is not null and c.props -> 'calculateFn' ->> '${cycle.uuid}' is not null) - ) - group by r.id, r.uuid, r.props, t.props ->> 'name' - order by r.id`, - [], - (row) => { - return { - ...Objects.camelize(row), - cols: row.cols.map(ColAdapter), - props: { - ...Objects.camelize(row.props), - calculateFn: row.props.calculateFn, - linkToSection: row.props.linkToSection, - validateFns: row.props.validateFns, - chart: row.props.chart, - }, - } - } - ) -} diff --git a/src/test/migrations/steps/utils/runCalculations.ts b/src/test/migrations/steps/utils/runCalculations.ts deleted file mode 100644 index 08878c0fb9..0000000000 --- a/src/test/migrations/steps/utils/runCalculations.ts +++ /dev/null @@ -1,64 +0,0 @@ -import * as pgPromise from 'pg-promise' -import { Objects } from 'utils/objects' - -import { Assessment, Cycle } from 'meta/assessment' - -import { AreaController } from 'server/controller/area' -import { BaseProtocol, Schemas } from 'server/db' - -import { calculateRow } from './calculateRow' -import { getRow } from './getRow' - -/** - * @deprecated - */ -export const runCalculations = async ( - props: { - assessment: Assessment - cycle: Cycle - tableName: string - variableName: string - }, - client: BaseProtocol -): Promise => { - const { assessment, cycle, tableName, variableName } = props - - if (Objects.isEmpty(assessment.metaCache)) throw new Error('Meta cache is missing!') - - const row = await getRow({ tableName, variableName, cycle, assessment }, client) - - const calculatedVariables: Record> = {} - const countries = await AreaController.getCountries({ assessment, cycle }, client) - const countryISOs = countries.map((c) => c.countryIso) - - const schemaCycle = Schemas.getNameCycle(assessment, cycle) - const pgp = pgPromise() - const cs = new pgp.helpers.ColumnSet( - [ - 'country_iso', - { - name: 'row_uuid', - cast: 'uuid', - }, - { - name: 'col_uuid', - cast: 'uuid', - }, - { - name: 'value', - cast: 'jsonb', - }, - ], - { - table: { table: 'node', schema: schemaCycle }, - } - ) - - // ===== calculation row - const values = await calculateRow({ assessment, cycle, countryISOs, row, tableName, calculatedVariables }, client) - const query = `${pgp.helpers.insert( - values, - cs - )} on conflict ("country_iso", "row_uuid", "col_uuid") do update set "value" = excluded."value"` - await client.query(query) -} diff --git a/src/test/migrations/steps/utils/updateCalculatedVariable.ts b/src/test/migrations/steps/utils/updateCalculatedVariable.ts deleted file mode 100644 index d7afef0207..0000000000 --- a/src/test/migrations/steps/utils/updateCalculatedVariable.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { CountryIso } from 'meta/area' -import { Assessment, Cycle, RowCaches } from 'meta/assessment' -import { NodeUpdate, NodeUpdates } from 'meta/data' - -import { AreaController } from 'server/controller/area' -import { ContextFactory } from 'server/controller/cycleData/updateDependencies/context' -import { updateCalculationDependencies } from 'server/controller/cycleData/updateDependencies/updateCalculationDependencies' -import { MetadataController } from 'server/controller/metadata' -import { BaseProtocol } from 'server/db' -import { NodeDb, NodeRepository } from 'server/repository/assessmentCycle/node' -import { DataRedisRepository } from 'server/repository/redis/data' -import { RowRedisRepository } from 'server/repository/redis/row' -import { SectionRedisRepository } from 'server/repository/redis/section' -import { Logger } from 'server/utils/logger' - -type Props = { - assessment: Assessment - cycle: Cycle - sectionName: string - tableName: string - variableName: string -} - -const _updateCache = async (props: Props) => { - const { assessment, cycle, sectionName, tableName, variableName } = props - - const force = true - const sectionNames = [sectionName] - const rowKeys = [RowCaches.getKey({ tableName, variableName })] - - await Promise.all([ - SectionRedisRepository.getManyMetadata({ assessment, cycle, sectionNames, force }), - RowRedisRepository.getRows({ assessment, rowKeys, force }), - ]) -} - -/** - * @deprecated - */ -export const updateCalculatedVariable = async (props: Props, client: BaseProtocol) => { - const { assessment, cycle, sectionName, variableName, tableName } = props - - await _updateCache({ assessment, cycle, sectionName, variableName, tableName }) - - const table = await MetadataController.getTable({ assessment, cycle, tableName }, client) - - const _nodes = table.props.columnNames?.[cycle.uuid].map((colName) => { - return { tableName, variableName, colName, value: undefined } - }) - - const countryISOs = (await AreaController.getCountries({ assessment, cycle }, client)).map((c) => c.countryIso) - - const allNodesDb: Array = [] - const allNodes: Array<{ - nodes: Record - countryIso: CountryIso - }> = [] - - await Promise.all( - countryISOs.map(async (countryIso) => { - const assessmentName = assessment.props.name - const cycleName = cycle.name - - const nodeUpdates: NodeUpdates = { assessmentName, cycleName, countryIso, nodes: _nodes } - const contextProps = { assessment, cycle, isODP: false, nodeUpdates, includeSourceNodes: true } - const context = await ContextFactory.newInstance(contextProps, client) - const { nodesDb, nodes } = updateCalculationDependencies({ context, jobId: `migration_step-${Date.now()}` }) - - if (nodesDb.length > 0) { - allNodesDb.push(...nodesDb) - allNodes.push({ nodes, countryIso }) - } - }) - ) - - try { - if (allNodesDb.length > 0) { - await NodeRepository.massiveInsert({ assessment, cycle, nodes: allNodesDb }, client) - await Promise.all( - allNodes.map(async ({ countryIso, nodes }) => { - await DataRedisRepository.updateNodes({ assessment, cycle, countryIso, nodes }) - }) - ) - } - } catch (e) { - Logger.error('Persisting nodes failed') - Logger.error(e) - } -} diff --git a/src/test/migrations/steps/utils/updateRowCalculateFn.ts b/src/test/migrations/steps/utils/updateRowCalculateFn.ts deleted file mode 100644 index 6e13f941fa..0000000000 --- a/src/test/migrations/steps/utils/updateRowCalculateFn.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { Assessment, Cycle } from 'meta/assessment' - -import { AssessmentController } from 'server/controller/assessment' -import { BaseProtocol, Schemas } from 'server/db' - -import { runCalculations } from 'test/migrations/steps/utils/runCalculations' - -export const updateRowCalculateFn = async ( - props: { assessment: Assessment; cycle: Cycle; formula: string; tableName: string; variableName: string }, - client: BaseProtocol -) => { - const { assessment, cycle, formula, tableName, variableName } = props - const schemaName = Schemas.getName(assessment) - - await client.query( - ` - update ${schemaName}.row r - set props = jsonb_set(r.props, '{calculateFn,${cycle.uuid}}', '"${formula}"') - where id = ( - select id from ${schemaName}.row r - where r.props->>'variableName' = $2 - and table_id = ( - select id from ${schemaName}.table where props->>'name' = $1 - ) - ); - `, - [tableName, variableName] - ) - - await AssessmentController.generateMetaCache( - { - assessment, - cycle, - }, - client - ) - - await runCalculations( - { - assessment, - cycle, - variableName, - tableName, - }, - client - ) -}