Merge pull request #50 from Giveth/add-fetching-project

Add fetching project

.env.template

@@ -19,3 +19,6 @@ SCHEMA_CONTRACT_ADDRESS=0x0a7E2Ff54e76B8E6659aedc9103FB21c038050D0
 
 PROJECT_VERIFY_ATTESTATION_SCHEMA=0x3D5854AF182F27966DEA837C446A051B3509224DDC03150E55097B362D111B1B
 PROJECT_GIVBACK_ELIGIBLE_ATTESTATION_SCHEMA=0x0000000000000000000000000000000000000000000000000000000000000000
+
+#APIs
+GIVETH_API_URL=Https://somewhere.giveth.io/XXXXX

.github/workflows/pipeline-develop.yml

@@ -22,7 +22,7 @@ jobs:
           git checkout -- .
           git pull
           docker compose stop devouch squid-db
-          docker compose rm -v -f devouch squid-db
-          docker rmi -f $(docker images | grep 'devouch' | awk '{print $3}')
-          docker volume rm $(docker volume ls | grep 'db-data' | awk '{print $2}')
+          # docker compose rm -v -f devouch squid-db
+          # docker rmi -f $(docker images | grep 'devouch' | awk '{print $3}')
+          # docker volume rm $(docker volume ls | grep 'db-data' | awk '{print $2}')
           docker compose up -d --build

db/migrations/1716207083370-Data.js

@@ -0,0 +1,13 @@
+module.exports = class Data1716207083370 {
+    name = 'Data1716207083370'
+
+    async up(db) {
+        await db.query(`ALTER TABLE "project" ADD "slug" text`)
+        await db.query(`ALTER TABLE "project" ADD "image" text`)
+    }
+
+    async down(db) {
+        await db.query(`ALTER TABLE "project" DROP COLUMN "slug"`)
+        await db.query(`ALTER TABLE "project" DROP COLUMN "image"`)
+    }
+}

package.json

@@ -19,6 +19,7 @@
     "@subsquid/typeorm-store": "^1.4.0",
     "dotenv": "^16.4.4",
     "ethers": "^6.12.1",
+    "node-cron": "^3.0.3",
     "pg": "^8.11.5",
     "type-graphql": "^1.2.0-rc.1",
     "typeorm": "^0.3.20",
@@ -30,6 +31,7 @@
     "@subsquid/typeorm-codegen": "^1.3.3",
     "@types/jest": "^29.5.12",
     "@types/node": "^20.11.17",
+    "@types/node-cron": "^3.0.11",
     "jest": "^29.7.0",
     "ts-jest": "^29.1.2",
     "ts-node": "^10.9.2",

schema.graphql

@@ -59,6 +59,10 @@ type Project @entity {
   totalFlags: Int!
   "Total attests"
   totalAttests: Int!
+  "Slug of the project"
+  slug: String
+  "Image of the project"
+  image: String
   lastUpdatedTimestamp: DateTime!
   attests: [ProjectAttestation!]! @derivedFrom(field: "project")
   attestedOrganisations: [OrganisationProject!]! @derivedFrom(field: "project")

src/features/import-projects/config.ts

@@ -0,0 +1 @@
+export const CRON_SCHEDULE = "0 0 * * *"; // UTC

src/features/import-projects/giveth/constants.ts

@@ -0,0 +1,2 @@
+export const GIVETH_API_URL =
+  process.env.GIVETH_API_URL || "https://mainnet.serve.giveth.io/graphql";

src/features/import-projects/giveth/helpers.ts

@@ -0,0 +1,92 @@
+import { DataSource } from "typeorm";
+import { getDataSource } from "../../../helpers/db";
+import { Project } from "../../../model";
+import { fetchGivethProjectsBatch } from "./service";
+import { GivethProjectInfo } from "./type";
+
+const updateOrCreateProject = async (
+  dataSource: DataSource,
+  project: GivethProjectInfo
+) => {
+  const existingProject = await dataSource
+    .getRepository(Project)
+    .createQueryBuilder("project")
+    .where("project.id = :id", { id: `giveth-${project.id}` })
+    .getOne();
+
+  if (existingProject) {
+    const isUpdated =
+      existingProject.title !== project.title ||
+      existingProject.description !== project.descriptionSummary ||
+      existingProject.slug !== project.slug ||
+      existingProject.image !== project.image;
+
+    if (isUpdated) {
+      const updatedProject = new Project({
+        ...existingProject,
+        title: project.title,
+        description: project.descriptionSummary,
+        image: project.image,
+        slug: project.slug,
+        lastUpdatedTimestamp: new Date(),
+      });
+
+      await dataSource
+        .createQueryBuilder()
+        .update(Project)
+        .set(updatedProject)
+        .where("id = :id", { id: `giveth-${project.id}` })
+        .execute();
+    }
+  } else {
+    const newProject = new Project({
+      id: `giveth-${project.id}`,
+      title: project.title,
+      description: project.descriptionSummary,
+      image: project.image,
+      slug: project.slug,
+      projectId: project.id,
+      source: "giveth",
+      totalVouches: 0,
+      totalFlags: 0,
+      totalAttests: 0,
+      lastUpdatedTimestamp: new Date(),
+    });
+
+    await dataSource
+      .createQueryBuilder()
+      .insert()
+      .into(Project)
+      .values([newProject])
+      .execute();
+  }
+};
+
+const processProjectsBatch = async (projectsBatch: GivethProjectInfo[]) => {
+  const dataSource = await getDataSource();
+  if (!dataSource) return;
+  for (const project of projectsBatch) {
+    console.log("Processing Project: Giveth", project.id);
+    await updateOrCreateProject(dataSource, project);
+  }
+};
+
+export const fetchAndProcessGivethProjects = async () => {
+  try {
+    let hasMoreProjects = true;
+    let skip = 0;
+    const limit = 10;
+
+    while (hasMoreProjects) {
+      const projectsBatch = await fetchGivethProjectsBatch(limit, skip);
+      if (projectsBatch.length > 0) {
+        await processProjectsBatch(projectsBatch);
+        skip += limit;
+      } else {
+        hasMoreProjects = false;
+      }
+    }
+  } catch (error: any) {
+    console.log("error on fetching giveth projects", error.message);
+  }
+};

src/features/import-projects/giveth/service.ts

@@ -0,0 +1,50 @@
+import { graphQLRequest } from "../../../helpers/request";
+import { GIVETH_API_URL } from "./constants";
+import { GivethProjectInfo } from "./type";
+
+export const fetchGivethProjectsBatch = async (limit: number, skip: number) => {
+  const res = await graphQLRequest(
+    GIVETH_API_URL,
+    `query ($limit: Int, $skip: Int, $sortingBy: SortingField) {
+      allProjects(
+        limit: $limit
+        skip: $skip
+        sortingBy: $sortingBy
+      ) {
+        projects {
+          id
+          title
+          image
+          slug
+          descriptionSummary
+        }
+      }
+    }`,
+    {
+      limit,
+      skip,
+      sortingBy: "Newest",
+    }
+  );
+
+  return res.data.allProjects.projects;
+};
+
+export const fetchGivethProjects = async () => {
+  let allProjects: GivethProjectInfo[] = [];
+  let hasMoreProjects = true;
+  let skip = 0;
+  const limit = 10;
+
+  while (hasMoreProjects) {
+    const projectsBatch = await fetchGivethProjectsBatch(limit, skip);
+    if (projectsBatch.length > 0) {
+      allProjects = allProjects.concat(projectsBatch);
+      skip += limit;
+    } else {
+      hasMoreProjects = false;
+    }
+  }
+
+  return allProjects;
+};

src/features/import-projects/giveth/type.ts

@@ -0,0 +1,7 @@
+export type GivethProjectInfo = {
+  id: string;
+  title: string;
+  descriptionSummary: string;
+  slug: string;
+  image: string;
+};

src/features/import-projects/index.ts

@@ -0,0 +1,20 @@
+import cron from "node-cron";
+import { fetchAndProcessGivethProjects } from "./giveth/helpers";
+import { CRON_SCHEDULE } from "./config";
+
+export const task = async () => {
+  console.log("Importing Projects", new Date());
+  await fetchAndProcessGivethProjects();
+};
+
+export const importProjects = async () => {
+  try {
+    console.log("Importing Projects has been scheduled.");
+    cron.schedule(CRON_SCHEDULE, task, {
+      scheduled: true,
+      timezone: "UTC",
+    });
+  } catch (error) {
+    console.log("Error", error);
+  }
+};

src/features/import-projects/standalone.ts

@@ -0,0 +1,3 @@
+import { task } from ".";
+
+task();

src/helpers/db.ts

@@ -0,0 +1,25 @@
+import { createOrmConfig } from "@subsquid/typeorm-config";
+import { DataSource } from "typeorm";
+
+let dataSource: DataSource | undefined = undefined;
+
+export const getDataSource = async () => {
+  try {
+    if (!dataSource || !dataSource.isInitialized) {
+      let cfg = createOrmConfig({ projectDir: __dirname + "/../.." });
+      (cfg.entities as string[]).push(__dirname + "/../model/generated/*.ts");
+
+      dataSource = await new DataSource(cfg).initialize();
+    }
+    return dataSource;
+  } catch (error) {
+    console.error("Failed to initialize DataSource", error);
+    return null;
+  }
+};
+
+export const createEntityManager = async () => {
+  const dataSource = await getDataSource();
+  if (!dataSource) return null;
+  return dataSource.createEntityManager();
+};

src/helpers/request.ts

@@ -0,0 +1,14 @@
+export const graphQLRequest = async (
+  url: string,
+  query: string,
+  variables: any
+) => {
+  const res = await fetch(url, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({ query, variables }),
+  });
+  return await res.json();
+};

src/main.ts

@@ -3,6 +3,7 @@ import { processor } from "./processor";
 import * as EASContract from "./abi/EAS";
 import { processAttest } from "./mappings/attest";
 import { processRevokeLog } from "./mappings/revoke";
+import { importProjects } from "./features/import-projects/index";
 
 processor.run(new TypeormDatabase({ supportHotBlocks: false }), async (ctx) => {
   for (let _block of ctx.blocks) {
@@ -18,3 +19,5 @@ processor.run(new TypeormDatabase({ supportHotBlocks: false }), async (ctx) => {
     }
   }
 });
+
+importProjects();

src/model/generated/project.model.ts

@@ -58,6 +58,18 @@ export class Project {
     @Column_("int4", {nullable: false})
     totalAttests!: number
 
+    /**
+     * Slug of the project
+     */
+    @Column_("text", {nullable: true})
+    slug!: string | undefined | null
+
+    /**
+     * Image of the project
+     */
+    @Column_("text", {nullable: true})
+    image!: string | undefined | null
+
     @Column_("timestamp with time zone", {nullable: false})
     lastUpdatedTimestamp!: Date