Merge pull request #59 from Giveth/add-rpgf3-projects

Add rpgf3 projects

.env.template

@@ -21,4 +21,5 @@ PROJECT_VERIFY_ATTESTATION_SCHEMA=0x3D5854AF182F27966DEA837C446A051B3509224DDC03
 PROJECT_GIVBACK_ELIGIBLE_ATTESTATION_SCHEMA=0x0000000000000000000000000000000000000000000000000000000000000000
 
 #APIs
-GIVETH_API_URL=Https://somewhere.giveth.io/XXXXX
+GIVETH_API_URL=https://somewhere.giveth.io/XXXXX
+RPGF3_API_URL=https://somewhere.RPGF3.io/XXXXX

src/features/import-projects/giveth/constants.ts

@@ -1,2 +1,11 @@
 export const GIVETH_API_URL =
   process.env.GIVETH_API_URL || "https://mainnet.serve.giveth.io/graphql";
+
+export const givethSourceConfig: SourceConfig = {
+  source: "giveth",
+  idField: "id",
+  titleField: "title",
+  descriptionField: "descriptionSummary",
+  slugField: "slug",
+  imageField: "image",
+};

src/features/import-projects/giveth/helpers.ts

@@ -1,92 +1,12 @@
-import { DataSource } from "typeorm";
 import { getDataSource } from "../../../helpers/db";
-import { Project } from "../../../model";
-import { fetchGivethProjectsBatch } from "./service";
-import { GivethProjectInfo } from "./type";
+import { type GivethProjectInfo } from "./type";
+import { updateOrCreateProject } from "../helpers";
+import { givethSourceConfig } from "./constants";
 
-const updateOrCreateProject = async (
-  dataSource: DataSource,
-  project: GivethProjectInfo
+export const processProjectsBatch = async (
+  projectsBatch: GivethProjectInfo[]
 ) => {
-  const existingProject = await dataSource
-    .getRepository(Project)
-    .createQueryBuilder("project")
-    .where("project.id = :id", { id: `giveth-${project.id}` })
-    .getOne();
-
-  if (existingProject) {
-    const isUpdated =
-      existingProject.title !== project.title ||
-      existingProject.description !== project.descriptionSummary ||
-      existingProject.slug !== project.slug ||
-      existingProject.image !== project.image;
-
-    if (isUpdated) {
-      const updatedProject = new Project({
-        ...existingProject,
-        title: project.title,
-        description: project.descriptionSummary,
-        image: project.image,
-        slug: project.slug,
-        lastUpdatedTimestamp: new Date(),
-      });
-
-      await dataSource
-        .createQueryBuilder()
-        .update(Project)
-        .set(updatedProject)
-        .where("id = :id", { id: `giveth-${project.id}` })
-        .execute();
-    }
-  } else {
-    const newProject = new Project({
-      id: `giveth-${project.id}`,
-      title: project.title,
-      description: project.descriptionSummary,
-      image: project.image,
-      slug: project.slug,
-      projectId: project.id,
-      source: "giveth",
-      totalVouches: 0,
-      totalFlags: 0,
-      totalAttests: 0,
-      lastUpdatedTimestamp: new Date(),
-    });
-
-    await dataSource
-      .createQueryBuilder()
-      .insert()
-      .into(Project)
-      .values([newProject])
-      .execute();
-  }
-};
-
-const processProjectsBatch = async (projectsBatch: GivethProjectInfo[]) => {
-  const dataSource = await getDataSource();
-  if (!dataSource) return;
   for (const project of projectsBatch) {
-    console.log("Processing Project: Giveth", project.id);
-    await updateOrCreateProject(dataSource, project);
-  }
-};
-
-export const fetchAndProcessGivethProjects = async () => {
-  try {
-    let hasMoreProjects = true;
-    let skip = 0;
-    const limit = 10;
-
-    while (hasMoreProjects) {
-      const projectsBatch = await fetchGivethProjectsBatch(limit, skip);
-      if (projectsBatch.length > 0) {
-        await processProjectsBatch(projectsBatch);
-        skip += limit;
-      } else {
-        hasMoreProjects = false;
-      }
-    }
-  } catch (error: any) {
-    console.log("error on fetching giveth projects", error.message);
+    await updateOrCreateProject(project, givethSourceConfig);
   }
 };

src/features/import-projects/giveth/index.ts

@@ -0,0 +1,22 @@
+import { processProjectsBatch } from "./helpers";
+import { fetchGivethProjectsBatch } from "./service";
+
+export const fetchAndProcessGivethProjects = async () => {
+  try {
+    let hasMoreProjects = true;
+    let skip = 0;
+    const limit = 10;
+
+    while (hasMoreProjects) {
+      const projectsBatch = await fetchGivethProjectsBatch(limit, skip);
+      if (projectsBatch.length > 0) {
+        await processProjectsBatch(projectsBatch);
+        skip += limit;
+      } else {
+        hasMoreProjects = false;
+      }
+    }
+  } catch (error: any) {
+    console.log("error on fetchAndProcessGivethProjects", error.message);
+  }
+};

src/features/import-projects/giveth/service.ts

@@ -3,31 +3,36 @@ import { GIVETH_API_URL } from "./constants";
 import { GivethProjectInfo } from "./type";
 
 export const fetchGivethProjectsBatch = async (limit: number, skip: number) => {
-  const res = await graphQLRequest(
-    GIVETH_API_URL,
-    `query ($limit: Int, $skip: Int, $sortingBy: SortingField) {
-      allProjects(
-        limit: $limit
-        skip: $skip
-        sortingBy: $sortingBy
-      ) {
-        projects {
-          id
-          title
-          image
-          slug
-          descriptionSummary
+  try {
+    const res = await graphQLRequest(
+      GIVETH_API_URL,
+      `query ($limit: Int, $skip: Int, $sortingBy: SortingField) {
+        allProjects(
+          limit: $limit
+          skip: $skip
+          sortingBy: $sortingBy
+        ) {
+          projects {
+            id
+            title
+            image
+            slug
+            descriptionSummary
+          }
         }
+      }`,
+      {
+        limit,
+        skip,
+        sortingBy: "Newest",
       }
-    }`,
-    {
-      limit,
-      skip,
-      sortingBy: "Newest",
-    }
-  );
+    );
 
-  return res.data.allProjects.projects;
+    return res.data.allProjects.projects;
+  } catch (error: any) {
+    console.log("error on fetchGivethProjectsBatch", error.message);
+    return [];
+  }
 };
 
 export const fetchGivethProjects = async () => {

src/features/import-projects/helpers.ts

@@ -0,0 +1,88 @@
+import { type DataSource } from "typeorm";
+import { Project } from "../../model";
+import { getDataSource } from "../../helpers/db";
+
+export const updateOrCreateProject = async (
+  project: any,
+  sourConfig: SourceConfig
+) => {
+  const {
+    source,
+    idField,
+    titleField,
+    descriptionField,
+    slugField,
+    imageField,
+  } = sourConfig;
+
+  const id = `${source}-${project[idField]}`;
+
+  const dataSource = await getDataSource();
+  if (!dataSource) {
+    console.log(
+      `[${new Date().toISOString()}] - ERROR: Failed to UPSERT project. Data source not found. Project ID: ${id}`
+    );
+    return;
+  }
+
+  const existingProject = await dataSource
+    .getRepository(Project)
+    .createQueryBuilder("project")
+    .where("project.id = :id", { id })
+    .getOne();
+
+  if (existingProject) {
+    const isUpdated =
+      existingProject.title !== project[titleField] ||
+      existingProject.description !== project[descriptionField] ||
+      existingProject.slug !== project[slugField] ||
+      existingProject.image !== project[imageField];
+
+    if (isUpdated) {
+      const updatedProject = new Project({
+        ...existingProject,
+        title: project[titleField],
+        description: project[descriptionField],
+        image: project[imageField],
+        slug: project[slugField],
+        lastUpdatedTimestamp: new Date(),
+      });
+
+      await dataSource
+        .createQueryBuilder()
+        .update(Project)
+        .set(updatedProject)
+        .where("id = :id", { id })
+        .execute();
+
+      console.log(
+        `[${new Date().toISOString()}] - INFO: Project Updated. Project ID: ${id}`
+      );
+    }
+  } else {
+    const newProject = new Project({
+      id,
+      title: project[titleField],
+      description: project[descriptionField],
+      image: project[imageField],
+      slug: project[slugField],
+      projectId: project[idField],
+      source: source,
+      totalVouches: 0,
+      totalFlags: 0,
+      totalAttests: 0,
+      lastUpdatedTimestamp: new Date(),
+    });
+
+    await dataSource
+      .createQueryBuilder()
+      .insert()
+      .into(Project)
+      .values([newProject])
+      .execute();
+
+    console.log(
+      `[${new Date().toISOString()}] - INFO: Project Created. Project ID: ${id}`
+    );
+  }
+};

src/features/import-projects/index.ts

@@ -1,10 +1,12 @@
 import cron from "node-cron";
-import { fetchAndProcessGivethProjects } from "./giveth/helpers";
 import { IMPORT_PROJECT_CRON_SCHEDULE } from "../../constants";
+import { fetchAndProcessGivethProjects } from "./giveth/index";
+import { fetchAndProcessRpgf3Projects } from "./rpgf";
 
 export const task = async () => {
   console.log("Importing Projects", new Date());
-  await fetchAndProcessGivethProjects();
+  fetchAndProcessGivethProjects();
+  fetchAndProcessRpgf3Projects();
 };
 
 export const importProjects = async () => {
@@ -17,6 +19,6 @@ export const importProjects = async () => {
       timezone: "UTC",
     });
   } catch (error) {
-    console.log("Error", error);
+    console.log("Error on scheduling importing project:", error);
   }
 };

src/features/import-projects/rpgf/constants.ts

@@ -0,0 +1,11 @@
+export const RPGF3_API_URL =
+  process.env.RPGF3_API_URL || "https://backend.pairwise.vote/mock/projects";
+
+export const rpgf3SourceConfig: SourceConfig = {
+  source: "rpgf3",
+  idField: "RPGF3Id",
+  titleField: "name",
+  descriptionField: "impactDescription",
+  slugField: "url",
+  imageField: "image",
+};

src/features/import-projects/rpgf/index.ts

@@ -0,0 +1,17 @@
+import { getDataSource } from "../../../helpers/db";
+import { updateOrCreateProject } from "../helpers";
+import { rpgf3SourceConfig } from "./constants";
+import { fetchRpgf3Projects } from "./service";
+
+export const fetchAndProcessRpgf3Projects = async () => {
+  try {
+    const data = await fetchRpgf3Projects();
+    if (!data) return;
+    for (const project of data) {
+      if (project.type.toLowerCase() !== "project") continue;
+      await updateOrCreateProject(project, rpgf3SourceConfig);
+    }
+  } catch (error: any) {
+    console.log("error on fetchAndProcessRpgf3Projects", error.message);
+  }
+};

src/features/import-projects/rpgf/service.ts

@@ -0,0 +1,19 @@
+import { RPGF3_API_URL } from "./constants";
+import { Rpgf3ProjectInfo } from "./type";
+
+export const fetchRpgf3Projects = async () => {
+  try {
+    const res = await fetch(RPGF3_API_URL, {
+      method: "GET",
+      headers: {
+        "Content-Type": "application/json",
+      },
+    });
+    const data: Rpgf3ProjectInfo[] = await res.json();
+
+    return data;
+  } catch (error: any) {
+    console.log("error on fetching fetchRpgf3Projects", error.message);
+    return null;
+  }
+};

src/features/import-projects/rpgf/type.ts

@@ -0,0 +1,14 @@
+export type Rpgf3ProjectInfo = {
+  id: string;
+  name: string;
+  poll_id: string;
+  url: string;
+  impactDescription: string;
+  contributionDescription: string;
+  RPGF3Id: string;
+  parentId: string;
+  image: string;
+  metadataUrl: string;
+  created_at: string;
+  type: string;
+};

src/features/import-projects/types.ts

@@ -0,0 +1,8 @@
+interface SourceConfig {
+  source: string;
+  idField: string;
+  titleField: string;
+  descriptionField: string;
+  slugField: string;
+  imageField: string;
+}