diff --git a/src/features/import-projects/helpers.ts b/src/features/import-projects/helpers.ts index 8394712..534cff6 100644 --- a/src/features/import-projects/helpers.ts +++ b/src/features/import-projects/helpers.ts @@ -7,7 +7,7 @@ import { SourceConfig } from "./types"; export const updateOrCreateProject = async ( project: any, - sourConfig: SourceConfig + sourceConfig: SourceConfig ) => { const { source, @@ -18,7 +18,8 @@ export const updateOrCreateProject = async ( urlField, imageField, rfRoundField, - } = sourConfig; + prelimResult, + } = sourceConfig; const projectId = project[idField].toLowerCase(); const id = `${source}-${projectId}`; @@ -44,24 +45,31 @@ export const updateOrCreateProject = async ( const descriptionHtml = descriptionHtmlField && project[descriptionHtmlField]; const rfRound = rfRoundField && project[rfRoundField]; + // Skip project if prelimResult is "Remove" + if (prelimResult && project[prelimResult] === "Remove") { + return; + } + + const descriptionSummary = getHtmlTextSummary(descriptionHtml || description); + if (existingProject) { + // Update existing project const isUpdated = existingProject.title !== title || existingProject.description !== description || existingProject.url !== url || existingProject.image !== image || (rfRound && !existingProject.rfRounds?.some((rfr) => rfr === rfRound)) || - existingProject.descriptionHtml !== descriptionHtml || + existingProject.descriptionHtml != descriptionHtml || (!existingProject.descriptionSummary && description); - const descriptionSummary = getHtmlTextSummary( - descriptionHtml || description - ); - if (isUpdated) { - const rfRounds = new Set(existingProject.rfRounds || []); - rfRound && rfRounds.add(rfRound); - const updatedProject = new Project({ + // Add the current round to rfRounds if not already present + const rfRoundsSet = new Set(existingProject.rfRounds || []); + if (rfRound) { + rfRoundsSet.add(rfRound); + } + const updatedProject = { ...existingProject, title, description, @@ -70,25 +78,29 @@ export const updateOrCreateProject = async ( descriptionHtml, descriptionSummary, lastUpdatedTimestamp: new Date(), - rfRounds: Array.from(rfRounds), + rfRounds: Array.from(rfRoundsSet), imported: true, - }); + }; - await dataSource - .createQueryBuilder() - .update(Project) - .set(updatedProject) - .where("id = :id", { id }) - .execute(); + try { + await dataSource + .createQueryBuilder() + .update(Project) + .set(updatedProject) + .where("id = :id", { id }) + .execute(); - console.log( - `[${new Date().toISOString()}] - INFO: Project Updated. Project ID: ${id}` - ); + console.log( + `[${new Date().toISOString()}] - INFO: Project Updated. Project ID: ${id}` + ); + } catch (error: any) { + console.log( + `[${new Date().toISOString()}] - ERROR: Failed to update project. Project ID: ${id}, Error: ${error.message}` + ); + } } } else { - const descriptionSummary = getHtmlTextSummary( - descriptionHtml || description - ); + // Create new project const newProject = new Project({ id, title, @@ -107,16 +119,22 @@ export const updateOrCreateProject = async ( imported: true, }); - await dataSource - .createQueryBuilder() - .insert() - .into(Project) - .values([newProject]) - .execute(); + try { + await dataSource + .createQueryBuilder() + .insert() + .into(Project) + .values([newProject]) + .execute(); - console.log( - `[${new Date().toISOString()}] - INFO: Project Created. Project ID: ${id}` - ); + console.log( + `[${new Date().toISOString()}] - INFO: Project Created. Project ID: ${id}` + ); + } catch (error: any) { + console.log( + `[${new Date().toISOString()}] - ERROR: Failed to create project. Project ID: ${id}, Error: ${error.message}` + ); + } } }; diff --git a/src/features/import-projects/retroList/constants.ts b/src/features/import-projects/retroList/constants.ts index b0fc722..af740d1 100644 --- a/src/features/import-projects/retroList/constants.ts +++ b/src/features/import-projects/retroList/constants.ts @@ -17,4 +17,5 @@ export const rlSourceConfig: SourceConfig = { imageField: "bannerImageUrl", urlField: "url", rfRoundField: "rfRound", + prelimResult: "prelimResult", }; diff --git a/src/features/import-projects/retroList/helper.ts b/src/features/import-projects/retroList/helper.ts index ec93872..c247cb1 100644 --- a/src/features/import-projects/retroList/helper.ts +++ b/src/features/import-projects/retroList/helper.ts @@ -1,5 +1,172 @@ +import { getDataSource } from "../../../helpers/db"; +import { Project } from "../../../model"; +import { SourceConfig } from "../types"; import { type RlProjectInfo } from "./type"; export const generateRlUrl = (project: RlProjectInfo) => { return `/project/${project.id}`; }; + +export const manageProjectRemovals = async ( + newList: RlProjectInfo[] | null, + sourceConfig: SourceConfig, + round: number // Pass the current round +) => { + if (newList === null) { + console.log( + `[${new Date().toISOString()}] - ERROR: Failed to manage project removals. New list is null.` + ); + return; + } + try { + const dataSource = await getDataSource(); + if (!dataSource) { + console.log( + `[${new Date().toISOString()}] - ERROR: Failed to remove projects. Data source not found.` + ); + return; + } + + const shouldKeepProjects = newList + .filter((project) => project.prelimResult === "Keep") + .map((project) => `${sourceConfig.source}-${project.id}`); + console.log("shouldKeepProjects", shouldKeepProjects); + + const existingProjectsIds = await dataSource + .getRepository(Project) + .createQueryBuilder("project") + .select("project.id") + .where("project.source = :source", { source: sourceConfig.source }) + .andWhere(":round = ANY(project.rfRounds)", { round }) // Only projects in the current round + .getMany() + .then((projects) => projects.map((proj) => proj.id)); + + console.log("existingProjectsIds", existingProjectsIds); + + const projectIdsToManipulate = existingProjectsIds.filter( + (id) => !shouldKeepProjects.includes(id) + ); + + console.log("projectIdsToManipulate", projectIdsToManipulate); + + if (projectIdsToManipulate.length === 0) { + // No projects to Manipulate + return; + } + + // Fetch projects to Remove, including their attests + const projectsToManipulate = await dataSource + .getRepository(Project) + .createQueryBuilder("project") + .leftJoinAndSelect("project.attests", "attests") + .where("project.id IN (:...ids)", { ids: projectIdsToManipulate }) + .getMany(); + + const projectsToUpdateRfRounds: Array<{ + id: string; + updatedRfRounds: number[]; + }> = []; + const projectsToRemove: Project[] = []; + const projectsToMakeUnImported: Project[] = []; + + for (const project of projectsToManipulate) { + const updatedRfRounds = (project.rfRounds || []).filter( + (rfr) => rfr !== round + ); + if (updatedRfRounds.length > 0) { + // Collect projects to update + projectsToUpdateRfRounds.push({ id: project.id, updatedRfRounds }); + } else { + if (project.attests.length > 0) { + projectsToMakeUnImported.push(project); + } else { + projectsToRemove.push(project); + } + } + } + + const updateRfRoundsPromises = projectsToUpdateRfRounds.map( + async ({ id, updatedRfRounds }) => { + try { + await dataSource + .createQueryBuilder() + .update(Project) + .set({ rfRounds: updatedRfRounds }) + .where("id = :id", { id }) + .execute(); + console.log( + `[INFO]: Project updated with removed round. Project ID: ${id}` + ); + } catch (updateError: any) { + console.log( + `[ERROR]: Failed to update project rfRounds. Project ID: ${id}. Error: ${updateError.message}` + ); + throw updateError; // Re-throw to let Promise.all handle it + } + } + ); + try { + await Promise.all(updateRfRoundsPromises); + } catch (error: any) { + console.log( + `[ERROR]: One or more updates failed. Error: ${error.message}` + ); + } + + // Bulk delete projects without attests + if (projectsToRemove.length > 0) { + const projectIdsToDelete = projectsToRemove.map((project) => project.id); + try { + await dataSource + .createQueryBuilder() + .delete() + .from(Project) + .where("id IN (:...ids)", { ids: projectIdsToDelete }) + .execute(); + + console.log( + `[${new Date().toISOString()}] - INFO: Projects Deleted. Project IDs: ${projectIdsToDelete.join( + ", " + )}` + ); + } catch (error: any) { + console.log( + `[${new Date().toISOString()}] - ERROR: Failed to delete projects. Project IDs: ${projectIdsToDelete.join( + ", " + )}. Error: ${error.message}` + ); + } + } + + // Bulk update projects with attests to mark as not imported + if (projectsToMakeUnImported.length > 0) { + const projectIdsToMakeUnImported = projectsToMakeUnImported.map( + (project) => project.id + ); + try { + await dataSource + .createQueryBuilder() + .update(Project) + .set({ imported: false, rfRounds: [round] }) + .where("id IN (:...ids)", { ids: projectIdsToMakeUnImported }) + .execute(); + + console.log( + `[${new Date().toISOString()}] - INFO: Projects marked as not imported. Project IDs: ${projectIdsToMakeUnImported.join( + ", " + )}` + ); + } catch (error: any) { + console.log( + `[${new Date().toISOString()}] - ERROR: Failed to mark projects as not imported. Project IDs: ${projectIdsToMakeUnImported.join( + ", " + )}. Error: ${error.message}` + ); + } + } + } catch (error: any) { + console.log( + `[${new Date().toISOString()}] - ERROR: Failed to manage project removals. Error: ${error.message}` + ); + } +}; diff --git a/src/features/import-projects/retroList/index.ts b/src/features/import-projects/retroList/index.ts index 3c15c54..0292903 100644 --- a/src/features/import-projects/retroList/index.ts +++ b/src/features/import-projects/retroList/index.ts @@ -1,12 +1,15 @@ import { updateOrCreateProject } from "../helpers"; import { rlSourceConfig } from "./constants"; -import { generateRlUrl } from "./helper"; +import { generateRlUrl, manageProjectRemovals } from "./helper"; import { fetchRlProjects } from "./service"; export const fetchAndProcessRlProjects = async (round: number) => { try { const data = await fetchRlProjects(round); if (!data) return; + + const processedProjectIds: string[] = []; + for (const project of data) { const processedProject = { ...project, @@ -16,6 +19,9 @@ export const fetchAndProcessRlProjects = async (round: number) => { await updateOrCreateProject(processedProject, rlSourceConfig); } + + // After processing all new projects, handle projects not in the new dataset for the current round + await manageProjectRemovals(data, rlSourceConfig, round); } catch (error: any) { console.log("error on fetchAndProcessRlProjects", error.message); } diff --git a/src/features/import-projects/types.ts b/src/features/import-projects/types.ts index 18958bb..5ba2b95 100644 --- a/src/features/import-projects/types.ts +++ b/src/features/import-projects/types.ts @@ -7,4 +7,5 @@ export interface SourceConfig { urlField: string; imageField: string; rfRoundField?: string; + prelimResult?: string; }