Merge pull request #105 from Giveth/feat/handle-rf-projects-rounds

Feat/handle rf projects rounds

db/migrations/1724279465327-Data.js

@@ -0,0 +1,13 @@
+module.exports = class Data1724279465327 {
+  name = "Data1724279465327";
+
+  async up(db) {
+    await db.query(`ALTER TABLE "project" DROP COLUMN "source_status"`);
+    await db.query(`ALTER TABLE "project" ADD "rf_rounds" integer array`);
+  }
+
+  async down(db) {
+    await db.query(`ALTER TABLE "project" DROP COLUMN "rf_rounds"`);
+    await db.query(`ALTER TABLE "project" ADD "source_status" text`);
+  }
+};

schema.graphql

@@ -67,13 +67,13 @@ type Project @entity {
   url: String
   "Image of the project"
   image: String
-  "Project status at the source. e.g. Keeped, Removed, Missing in rf4"
-  sourceStatus: String
   "project data is imported from a source or not"
   imported: Boolean! @index
   lastUpdatedTimestamp: DateTime!
   attests: [ProjectAttestation!]! @derivedFrom(field: "project")
   attestedOrganisations: [OrganisationProject!]! @derivedFrom(field: "project")
+  "Rounds in which the project is included"
+  rfRounds: [Int!]
 }
 
 type OrganisationProject @entity {

src/controllers/projectVerificationAttestation.ts

@@ -63,11 +63,15 @@ export const handleProjectAttestation = async (
 
   ctx.log.debug(`Processing project attestation with uid: ${uid}`);
 
-  const project = await getProject(
-    ctx,
-    projectVerificationAttestation.projectSource,
-    projectVerificationAttestation.projectId
-  );
+  let projectId = projectVerificationAttestation.projectId;
+  let projectSource = projectVerificationAttestation.projectSource;
+
+  if (projectVerificationAttestation.projectSource === "rf4") {
+    projectId = projectVerificationAttestation.projectId.replace("rf4-", "rf-");
+    projectSource = "rf";
+  }
+
+  const project = await getProject(ctx, projectSource, projectId);
 
   // Delete the previous attestation
   await removeDuplicateProjectAttestations(

src/features/import-projects/helpers.ts

@@ -17,7 +17,7 @@ export const updateOrCreateProject = async (
     descriptionHtmlField,
     urlField,
     imageField,
-    sourceStatusField,
+    rfRoundField,
   } = sourConfig;
 
   const projectId = project[idField].toLowerCase();
@@ -42,15 +42,15 @@ export const updateOrCreateProject = async (
   const url = project[urlField];
   const image = project[imageField];
   const descriptionHtml = descriptionHtmlField && project[descriptionHtmlField];
-  const sourceStatus = sourceStatusField && project[sourceStatusField];
+  const rfRound = rfRoundField && project[rfRoundField];
 
   if (existingProject) {
     const isUpdated =
       existingProject.title !== title ||
       existingProject.description !== description ||
       existingProject.url !== url ||
       existingProject.image !== image ||
-      existingProject.sourceStatus !== sourceStatus ||
+      (rfRound && !existingProject.rfRounds?.some((rfr) => rfr === rfRound)) ||
       existingProject.descriptionHtml !== descriptionHtml ||
       (!existingProject.descriptionSummary && description);
 
@@ -59,6 +59,8 @@ export const updateOrCreateProject = async (
     );
 
     if (isUpdated) {
+      const rfRounds = new Set(existingProject.rfRounds || []);
+      rfRound && rfRounds.add(rfRound);
       const updatedProject = new Project({
         ...existingProject,
         title,
@@ -67,8 +69,8 @@ export const updateOrCreateProject = async (
         url,
         descriptionHtml,
         descriptionSummary,
-        sourceStatus,
         lastUpdatedTimestamp: new Date(),
+        rfRounds: Array.from(rfRounds),
         imported: true,
       });
 
@@ -97,7 +99,7 @@ export const updateOrCreateProject = async (
       descriptionSummary,
       projectId,
       source,
-      sourceStatus,
+      rfRounds: rfRound ? [rfRound] : [],
       totalVouches: 0,
       totalFlags: 0,
       totalAttests: 0,

src/features/import-projects/index.ts

@@ -4,13 +4,15 @@ import { fetchAndProcessGivethProjects } from "./giveth/index";
 import { fetchAndProcessRpgf3Projects } from "./rpgf";
 import { fetchAndProcessGitcoinProjects } from "./gitcoin";
 import { fetchAndProcessRf4Projects } from "./rf4";
+import { fetchRFProjectsByRound } from "./rf";
 
 export const task = async () => {
   console.log("Importing Projects", new Date());
   fetchAndProcessGivethProjects();
-  // fetchAndProcessRpgf3Projects();
   fetchAndProcessGitcoinProjects();
-  fetchAndProcessRf4Projects();
+  // fetchAndProcessRpgf3Projects();
+  await fetchRFProjectsByRound(4);
+  await fetchRFProjectsByRound(5);
 };
 
 export const importProjects = async () => {

src/features/import-projects/rf/constants.ts

@@ -0,0 +1,14 @@
+import { SourceConfig } from "../types";
+
+export const RF_API_URL =
+  process.env.RF4_API_URL || "https://vote.optimism.io/api/v1";
+
+export const rfSourceConfig: SourceConfig = {
+  source: "rf",
+  idField: "id",
+  titleField: "name",
+  descriptionField: "description",
+  imageField: "projectCoverImageUrl",
+  urlField: "url",
+  rfRoundField: "rfRound",
+};

src/features/import-projects/rf/helpers.ts

@@ -0,0 +1,26 @@
+import { updateOrCreateProject } from "../helpers";
+import { rfSourceConfig } from "./constants";
+import { RfProjectInfo } from "./type";
+
+export const generateRfUrl = (project: RfProjectInfo) => {
+  return `/project/${project.id}`;
+};
+
+const processProject = (project: RfProjectInfo, round: number) => {
+  const projectData = {
+    ...project,
+    url: generateRfUrl(project),
+    rfRound: round,
+  };
+  return projectData;
+};
+
+export const saveBatchProjects = async (
+  projects: RfProjectInfo[],
+  round: number
+) => {
+  for (const _project of projects) {
+    const project = processProject(_project, round);
+    await updateOrCreateProject(project, rfSourceConfig);
+  }
+};

src/features/import-projects/rf/index.ts

@@ -0,0 +1,42 @@
+import { RF_API_URL } from "./constants";
+import { saveBatchProjects } from "./helpers";
+import { RfApiResponse, RfProjectInfo } from "./type";
+
+export const fetchRFProjectsByRound = async (round: number) => {
+  let offset = 0;
+  const limit = 10;
+  let hasNext = true;
+
+  try {
+    while (hasNext) {
+      const response = await fetch(
+        `${RF_API_URL}/retrofunding/rounds/${round}/projects?limit=${limit}&offset=${offset}`,
+        {
+          method: "GET",
+          headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${process.env.AGORA_API_KEY}`,
+          },
+        }
+      );
+
+      if (!response.ok) {
+        throw new Error(
+          `HTTP error! status: ${response.status} for round: ${round} at offset: ${offset}`
+        );
+      }
+
+      const res: RfApiResponse = await response.json();
+
+      await saveBatchProjects(res.data, round);
+
+      hasNext = res.meta.has_next;
+      offset = res.meta.next_offset;
+    }
+  } catch (error) {
+    console.error(
+      `Error fetching projects for round: ${round} at offset: ${offset}`,
+      error
+    );
+  }
+};

src/features/import-projects/rf/type.ts

@@ -0,0 +1,22 @@
+export interface RfProjectInfo {
+  id: string;
+  category: string;
+  organization: string | null;
+  name: string;
+  description: string;
+  profileAvatarUrl: string;
+  projectCoverImageUrl: string;
+  team: string[];
+  github: string[];
+  packages: string[];
+  links: string[];
+}
+
+export interface RfApiResponse {
+  meta: {
+    has_next: boolean;
+    total_returned: number;
+    next_offset: number;
+  };
+  data: RfProjectInfo[];
+}

src/features/import-projects/types.ts

@@ -6,5 +6,5 @@ export interface SourceConfig {
   descriptionHtmlField?: string;
   urlField: string;
   imageField: string;
-  sourceStatusField?: string;
+  rfRoundField?: string;
 }

src/model/generated/project.model.ts

@@ -82,12 +82,6 @@ export class Project {
     @Column_("text", {nullable: true})
     image!: string | undefined | null
 
-    /**
-     * Project status at the source. e.g. Keeped, Removed, Missing in rf4
-     */
-    @Column_("text", {nullable: true})
-    sourceStatus!: string | undefined | null
-
     /**
      * project data is imported from a source or not
      */
@@ -103,4 +97,10 @@ export class Project {
 
     @OneToMany_(() => OrganisationProject, e => e.project)
     attestedOrganisations!: OrganisationProject[]
+
+    /**
+     * Rounds in which the project is included
+     */
+    @Column_("int4", {array: true, nullable: true})
+    rfRounds!: (number)[] | undefined | null
 }