From 589227a6f83fb347bb99c7432f84b84d325d8432 Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 02:13:02 +0330 Subject: [PATCH 01/14] add rfRounds column --- schema.graphql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schema.graphql b/schema.graphql index b2a6f13..8690c44 100644 --- a/schema.graphql +++ b/schema.graphql @@ -67,13 +67,13 @@ type Project @entity { url: String "Image of the project" image: String - "Project status at the source. e.g. Keeped, Removed, Missing in rf4" - sourceStatus: String "project data is imported from a source or not" imported: Boolean! @index lastUpdatedTimestamp: DateTime! attests: [ProjectAttestation!]! @derivedFrom(field: "project") attestedOrganisations: [OrganisationProject!]! @derivedFrom(field: "project") + "Rounds in which the project is included" + rfRounds: [Int!] } type OrganisationProject @entity { From bf7f645c55943fe661c7f8072ee2103334cf0185 Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 02:14:13 +0330 Subject: [PATCH 02/14] migration for adding rf_rounds and deleting source_status --- db/migrations/1724279465327-Data.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 db/migrations/1724279465327-Data.js diff --git a/db/migrations/1724279465327-Data.js b/db/migrations/1724279465327-Data.js new file mode 100644 index 0000000..38c0e82 --- /dev/null +++ b/db/migrations/1724279465327-Data.js @@ -0,0 +1,13 @@ +module.exports = class Data1724279465327 { + name = "Data1724279465327"; + + async up(db) { + await db.query(`ALTER TABLE "project" DROP COLUMN "source_status"`); + await db.query(`ALTER TABLE "project" ADD "rf_rounds" integer array`); + } + + async down(db) { + await db.query(`ALTER TABLE "project" DROP COLUMN "rf_rounds"`); + await db.query(`ALTER TABLE "project" ADD "source_status" text`); + } +}; From 6c3f9e9bbd6b919d4fe5cf7c66d94c562d7e9804 Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 02:15:16 +0330 Subject: [PATCH 03/14] add rfRoundField to SourceConfig --- src/features/import-projects/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/features/import-projects/types.ts b/src/features/import-projects/types.ts index 7d0363d..18958bb 100644 --- a/src/features/import-projects/types.ts +++ b/src/features/import-projects/types.ts @@ -6,5 +6,5 @@ export interface SourceConfig { descriptionHtmlField?: string; urlField: string; imageField: string; - sourceStatusField?: string; + rfRoundField?: string; } From 1fb2d8335ff0817a84b7433a5875ee64d16397f2 Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 02:15:25 +0330 Subject: [PATCH 04/14] add rf constants --- src/features/import-projects/rf/constants.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/features/import-projects/rf/constants.ts diff --git a/src/features/import-projects/rf/constants.ts b/src/features/import-projects/rf/constants.ts new file mode 100644 index 0000000..dbc58e4 --- /dev/null +++ b/src/features/import-projects/rf/constants.ts @@ -0,0 +1,14 @@ +import { SourceConfig } from "../types"; + +export const RF_API_URL = + process.env.RF4_API_URL || "https://vote.optimism.io/api/v1"; + +export const rfSourceConfig: SourceConfig = { + source: "rf", + idField: "id", + titleField: "name", + descriptionField: "description", + imageField: "projectCoverImageUrl", + urlField: "url", + rfRoundField: "rfRound", +}; From bb7faf82566a1cd3d40a8cf698421097765c1a92 Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 02:16:02 +0330 Subject: [PATCH 05/14] add rfRounds to model --- src/model/generated/project.model.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/model/generated/project.model.ts b/src/model/generated/project.model.ts index 0972f31..23f4ce6 100644 --- a/src/model/generated/project.model.ts +++ b/src/model/generated/project.model.ts @@ -82,12 +82,6 @@ export class Project { @Column_("text", {nullable: true}) image!: string | undefined | null - /** - * Project status at the source. e.g. Keeped, Removed, Missing in rf4 - */ - @Column_("text", {nullable: true}) - sourceStatus!: string | undefined | null - /** * project data is imported from a source or not */ @@ -103,4 +97,10 @@ export class Project { @OneToMany_(() => OrganisationProject, e => e.project) attestedOrganisations!: OrganisationProject[] + + /** + * Rounds in which the project is included + */ + @Column_("int4", {array: true, nullable: true}) + rfRounds!: (number)[] | undefined | null } From 755d63e09a5847ae2d9cf9f791aa5628b7c1be2c Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 02:16:12 +0330 Subject: [PATCH 06/14] add rf type --- src/features/import-projects/rf/type.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/features/import-projects/rf/type.ts diff --git a/src/features/import-projects/rf/type.ts b/src/features/import-projects/rf/type.ts new file mode 100644 index 0000000..851eb66 --- /dev/null +++ b/src/features/import-projects/rf/type.ts @@ -0,0 +1,22 @@ +export interface RfProjectInfo { + id: string; + category: string; + organization: string | null; + name: string; + description: string; + profileAvatarUrl: string; + projectCoverImageUrl: string; + team: string[]; + github: string[]; + packages: string[]; + links: string[]; +} + +export interface RfApiResponse { + meta: { + has_next: boolean; + total_returned: number; + next_offset: number; + }; + data: RfProjectInfo[]; +} From 389eb12718da50ae17ac79696599a5bbf4bbacb2 Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 02:16:32 +0330 Subject: [PATCH 07/14] add rf helper --- src/features/import-projects/rf/helpers.ts | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/features/import-projects/rf/helpers.ts diff --git a/src/features/import-projects/rf/helpers.ts b/src/features/import-projects/rf/helpers.ts new file mode 100644 index 0000000..f77c20e --- /dev/null +++ b/src/features/import-projects/rf/helpers.ts @@ -0,0 +1,26 @@ +import { updateOrCreateProject } from "../helpers"; +import { rfSourceConfig } from "./constants"; +import { RfProjectInfo } from "./type"; + +export const generateRf4Url = (project: RfProjectInfo) => { + return `/project/${project.id}`; +}; + +const processProject = (project: RfProjectInfo, round: number) => { + const projectData = { + ...project, + url: generateRf4Url(project), + rfRound: round, + }; + return projectData; +}; + +export const saveBatchProjects = async ( + projects: RfProjectInfo[], + round: number +) => { + for (const _project of projects) { + const project = processProject(_project, round); + await updateOrCreateProject(project, rfSourceConfig); + } +}; From 6aa94cb7dc67d22496131a224facfcec33a76f6f Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 02:16:52 +0330 Subject: [PATCH 08/14] add fetchRFProjectsByRound --- src/features/import-projects/rf/index.ts | 44 ++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/features/import-projects/rf/index.ts diff --git a/src/features/import-projects/rf/index.ts b/src/features/import-projects/rf/index.ts new file mode 100644 index 0000000..5397d9e --- /dev/null +++ b/src/features/import-projects/rf/index.ts @@ -0,0 +1,44 @@ +import { RF_API_URL } from "./constants"; +import { saveBatchProjects } from "./helpers"; +import { RfApiResponse, RfProjectInfo } from "./type"; + +export const fetchRFProjectsByRound = async (round: number) => { + let allProjects: RfProjectInfo[] = []; + let offset = 0; + const limit = 10; + let hasNext = true; + + try { + while (hasNext) { + const response = await fetch( + `${RF_API_URL}/retrofunding/rounds/${round}/projects?limit=${limit}&offset=${offset}`, + { + method: "GET", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${process.env.AGORA_API_KEY}`, + }, + } + ); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const res: RfApiResponse = await response.json(); + + // allProjects = allProjects.concat(res.data); + + await saveBatchProjects(res.data, round); + + hasNext = res.meta.has_next; + offset = res.meta.next_offset; + } + + console.log(allProjects); + } catch (error) { + console.error("Error fetching projects:", error); + } + + return allProjects; +}; From f5f8aa1b6e776502729a0b1d375ad2dff663b1ea Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 02:17:31 +0330 Subject: [PATCH 09/14] update updateOrCreateProject to handle rounds --- src/features/import-projects/helpers.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/features/import-projects/helpers.ts b/src/features/import-projects/helpers.ts index 99e9fbe..12c0ebf 100644 --- a/src/features/import-projects/helpers.ts +++ b/src/features/import-projects/helpers.ts @@ -17,7 +17,7 @@ export const updateOrCreateProject = async ( descriptionHtmlField, urlField, imageField, - sourceStatusField, + rfRoundField, } = sourConfig; const projectId = project[idField].toLowerCase(); @@ -42,7 +42,7 @@ export const updateOrCreateProject = async ( const url = project[urlField]; const image = project[imageField]; const descriptionHtml = descriptionHtmlField && project[descriptionHtmlField]; - const sourceStatus = sourceStatusField && project[sourceStatusField]; + const rfRound = rfRoundField && project[rfRoundField]; if (existingProject) { const isUpdated = @@ -50,7 +50,7 @@ export const updateOrCreateProject = async ( existingProject.description !== description || existingProject.url !== url || existingProject.image !== image || - existingProject.sourceStatus !== sourceStatus || + (rfRound && !existingProject.rfRounds?.some((rfr) => (rfr = rfRound))) || existingProject.descriptionHtml !== descriptionHtml || (!existingProject.descriptionSummary && description); @@ -59,6 +59,8 @@ export const updateOrCreateProject = async ( ); if (isUpdated) { + const rfRounds = new Set(existingProject.rfRounds || []); + rfRound && rfRounds.add(rfRound); const updatedProject = new Project({ ...existingProject, title, @@ -67,8 +69,8 @@ export const updateOrCreateProject = async ( url, descriptionHtml, descriptionSummary, - sourceStatus, lastUpdatedTimestamp: new Date(), + rfRounds: Array.from(rfRounds), imported: true, }); @@ -97,7 +99,7 @@ export const updateOrCreateProject = async ( descriptionSummary, projectId, source, - sourceStatus, + rfRounds: rfRound ? [rfRound] : [], totalVouches: 0, totalFlags: 0, totalAttests: 0, From 2e2899f129e9c0c236f5a818b8107176c764a8c0 Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 02:59:46 +0330 Subject: [PATCH 10/14] handle old rf4 attests --- src/controllers/projectVerificationAttestation.ts | 14 +++++++++----- src/features/import-projects/index.ts | 6 ++++-- src/features/import-projects/rf4/constants.ts | 1 - 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/controllers/projectVerificationAttestation.ts b/src/controllers/projectVerificationAttestation.ts index 41f8444..631a56a 100644 --- a/src/controllers/projectVerificationAttestation.ts +++ b/src/controllers/projectVerificationAttestation.ts @@ -63,11 +63,15 @@ export const handleProjectAttestation = async ( ctx.log.debug(`Processing project attestation with uid: ${uid}`); - const project = await getProject( - ctx, - projectVerificationAttestation.projectSource, - projectVerificationAttestation.projectId - ); + let projectId = projectVerificationAttestation.projectId; + let projectSource = projectVerificationAttestation.projectSource; + + if (projectVerificationAttestation.projectSource === "rf4") { + projectId = projectVerificationAttestation.projectId.replace("rf4-", "rf-"); + projectSource = "rf"; + } + + const project = await getProject(ctx, projectSource, projectId); // Delete the previous attestation await removeDuplicateProjectAttestations( diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index 516b56f..25efbca 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -4,13 +4,15 @@ import { fetchAndProcessGivethProjects } from "./giveth/index"; import { fetchAndProcessRpgf3Projects } from "./rpgf"; import { fetchAndProcessGitcoinProjects } from "./gitcoin"; import { fetchAndProcessRf4Projects } from "./rf4"; +import { fetchRFProjectsByRound } from "./rf"; export const task = async () => { console.log("Importing Projects", new Date()); fetchAndProcessGivethProjects(); - // fetchAndProcessRpgf3Projects(); fetchAndProcessGitcoinProjects(); - fetchAndProcessRf4Projects(); + // fetchAndProcessRpgf3Projects(); + await fetchRFProjectsByRound(4); + await fetchRFProjectsByRound(5); }; export const importProjects = async () => { diff --git a/src/features/import-projects/rf4/constants.ts b/src/features/import-projects/rf4/constants.ts index 536da83..6e54bc4 100644 --- a/src/features/import-projects/rf4/constants.ts +++ b/src/features/import-projects/rf4/constants.ts @@ -10,5 +10,4 @@ export const rf4SourceConfig: SourceConfig = { descriptionField: "description", imageField: "bannerImageUrl", urlField: "url", - sourceStatusField: "prelimResult", }; From e80b288a3caff095c6a9449f3030f87230a20ad8 Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 12:22:57 +0330 Subject: [PATCH 11/14] fix error handling --- src/features/import-projects/rf/index.ts | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/features/import-projects/rf/index.ts b/src/features/import-projects/rf/index.ts index 5397d9e..b4c4016 100644 --- a/src/features/import-projects/rf/index.ts +++ b/src/features/import-projects/rf/index.ts @@ -3,7 +3,6 @@ import { saveBatchProjects } from "./helpers"; import { RfApiResponse, RfProjectInfo } from "./type"; export const fetchRFProjectsByRound = async (round: number) => { - let allProjects: RfProjectInfo[] = []; let offset = 0; const limit = 10; let hasNext = true; @@ -22,23 +21,22 @@ export const fetchRFProjectsByRound = async (round: number) => { ); if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); + throw new Error( + `HTTP error! status: ${response.status} for round: ${round} at offset: ${offset}` + ); } const res: RfApiResponse = await response.json(); - // allProjects = allProjects.concat(res.data); - await saveBatchProjects(res.data, round); hasNext = res.meta.has_next; offset = res.meta.next_offset; } - - console.log(allProjects); } catch (error) { - console.error("Error fetching projects:", error); + console.error( + `Error fetching projects for round: ${round} at offset: ${offset}`, + error + ); } - - return allProjects; }; From 35af541e68b55485d233e9879b31654ba5773597 Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 12:23:03 +0330 Subject: [PATCH 12/14] fix condition --- src/features/import-projects/helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/features/import-projects/helpers.ts b/src/features/import-projects/helpers.ts index 12c0ebf..8394712 100644 --- a/src/features/import-projects/helpers.ts +++ b/src/features/import-projects/helpers.ts @@ -50,7 +50,7 @@ export const updateOrCreateProject = async ( existingProject.description !== description || existingProject.url !== url || existingProject.image !== image || - (rfRound && !existingProject.rfRounds?.some((rfr) => (rfr = rfRound))) || + (rfRound && !existingProject.rfRounds?.some((rfr) => rfr === rfRound)) || existingProject.descriptionHtml !== descriptionHtml || (!existingProject.descriptionSummary && description); From a25e657d5e6d7ec1d92f5d2e996ed2052008a04c Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 19:04:39 +0330 Subject: [PATCH 13/14] fix name --- src/features/import-projects/rf/helpers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/features/import-projects/rf/helpers.ts b/src/features/import-projects/rf/helpers.ts index f77c20e..b980295 100644 --- a/src/features/import-projects/rf/helpers.ts +++ b/src/features/import-projects/rf/helpers.ts @@ -2,14 +2,14 @@ import { updateOrCreateProject } from "../helpers"; import { rfSourceConfig } from "./constants"; import { RfProjectInfo } from "./type"; -export const generateRf4Url = (project: RfProjectInfo) => { +export const generateRfUrl = (project: RfProjectInfo) => { return `/project/${project.id}`; }; const processProject = (project: RfProjectInfo, round: number) => { const projectData = { ...project, - url: generateRf4Url(project), + url: generateRfUrl(project), rfRound: round, }; return projectData; From a77811240882b903f7c9bbfe69a0028a1f70bc5a Mon Sep 17 00:00:00 2001 From: Cherik Date: Thu, 22 Aug 2024 19:05:03 +0330 Subject: [PATCH 14/14] remove old RF4 feature --- src/features/import-projects/rf4/constants.ts | 13 ----------- src/features/import-projects/rf4/helpers.ts | 6 ----- src/features/import-projects/rf4/index.ts | 22 ------------------- src/features/import-projects/rf4/service.ts | 19 ---------------- src/features/import-projects/rf4/type.ts | 16 -------------- 5 files changed, 76 deletions(-) delete mode 100644 src/features/import-projects/rf4/constants.ts delete mode 100644 src/features/import-projects/rf4/helpers.ts delete mode 100644 src/features/import-projects/rf4/index.ts delete mode 100644 src/features/import-projects/rf4/service.ts delete mode 100644 src/features/import-projects/rf4/type.ts diff --git a/src/features/import-projects/rf4/constants.ts b/src/features/import-projects/rf4/constants.ts deleted file mode 100644 index 6e54bc4..0000000 --- a/src/features/import-projects/rf4/constants.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { SourceConfig } from "../types"; - -export const RF4_API_URL = - process.env.RF4_API_URL || "https://round4-api-eas.retrolist.app/projects"; - -export const rf4SourceConfig: SourceConfig = { - source: "rf4", - idField: "id", - titleField: "name", - descriptionField: "description", - imageField: "bannerImageUrl", - urlField: "url", -}; diff --git a/src/features/import-projects/rf4/helpers.ts b/src/features/import-projects/rf4/helpers.ts deleted file mode 100644 index 4089325..0000000 --- a/src/features/import-projects/rf4/helpers.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { type Rf4ProjectInfo } from "./type"; - -export const generateRf4Url = (project: Rf4ProjectInfo) => { - return `/project/${project.id}`; -}; - diff --git a/src/features/import-projects/rf4/index.ts b/src/features/import-projects/rf4/index.ts deleted file mode 100644 index 139adf1..0000000 --- a/src/features/import-projects/rf4/index.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { updateOrCreateProject } from "../helpers"; -import { rf4SourceConfig } from "./constants"; -import { generateRf4Url } from "./helpers"; -import { fetchRf4Projects } from "./service"; - -export const fetchAndProcessRf4Projects = async () => { - try { - const data = await fetchRf4Projects(); - if (!data) return; - for (const project of data) { - if (project.prelimResult !== "Keep") continue; - - const processedProject = { - ...project, - url: generateRf4Url(project), - }; - await updateOrCreateProject(processedProject, rf4SourceConfig); - } - } catch (error: any) { - console.log("error on fetchAndProcessRf4Projects", error.message); - } -}; diff --git a/src/features/import-projects/rf4/service.ts b/src/features/import-projects/rf4/service.ts deleted file mode 100644 index 978bf85..0000000 --- a/src/features/import-projects/rf4/service.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { RF4_API_URL } from "./constants"; -import { Rf4ProjectInfo } from "./type"; - -export const fetchRf4Projects = async () => { - try { - const res = await fetch(RF4_API_URL, { - method: "GET", - headers: { - "Content-Type": "application/json", - }, - }); - const data: Rf4ProjectInfo[] = await res.json(); - - return data; - } catch (error: any) { - console.log("error on fetching fetchRf4Projects", error.message); - return null; - } -}; diff --git a/src/features/import-projects/rf4/type.ts b/src/features/import-projects/rf4/type.ts deleted file mode 100644 index d835007..0000000 --- a/src/features/import-projects/rf4/type.ts +++ /dev/null @@ -1,16 +0,0 @@ -export type Rf4ProjectInfo = { - id: string; - name: string; - displayName: string; - description: string; - bio: string; - address: string; - bannerImageUrl: string; - profileImageUrl: string; - impactCategory: string[]; - primaryCategory: string; - recategorization: string; - prelimResult: string; - reportReason: string; - includedInBallots: number; -};