From 7d8a833014d10fbbaf69353dd8902bec20aca9aa Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 12:29:11 +0330 Subject: [PATCH 01/25] add getDataSource --- src/helpers/db.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/helpers/db.ts diff --git a/src/helpers/db.ts b/src/helpers/db.ts new file mode 100644 index 0000000..d136ceb --- /dev/null +++ b/src/helpers/db.ts @@ -0,0 +1,14 @@ +import { createOrmConfig } from "@subsquid/typeorm-config"; +import { DataSource } from "typeorm"; + +let ds: DataSource | undefined = undefined; + +export const getDataSource = async () => { + if (!ds || !ds.isInitialized) { + let cfg = createOrmConfig({ projectDir: __dirname + "/../.." }); + (cfg.entities as string[]).push(__dirname + "/../model/generated/*.ts"); + + ds = await new DataSource(cfg).initialize(); + } + return ds; +}; From 80bd1a111955994c86f3fdf90c1efb43b444f0d5 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 12:58:08 +0330 Subject: [PATCH 02/25] add createEntityManager --- src/helpers/db.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/helpers/db.ts b/src/helpers/db.ts index d136ceb..1831412 100644 --- a/src/helpers/db.ts +++ b/src/helpers/db.ts @@ -1,14 +1,19 @@ import { createOrmConfig } from "@subsquid/typeorm-config"; import { DataSource } from "typeorm"; -let ds: DataSource | undefined = undefined; +let dataSource: DataSource | undefined = undefined; export const getDataSource = async () => { - if (!ds || !ds.isInitialized) { + if (!dataSource || !dataSource.isInitialized) { let cfg = createOrmConfig({ projectDir: __dirname + "/../.." }); (cfg.entities as string[]).push(__dirname + "/../model/generated/*.ts"); - ds = await new DataSource(cfg).initialize(); + dataSource = await new DataSource(cfg).initialize(); } - return ds; + return dataSource; +}; + +export const createEntityManager = async () => { + const dataSource = await getDataSource(); + return dataSource.createEntityManager(); }; From dea7480d5df2c2fcb8cdb987c633f805bb6e5375 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 14:26:50 +0330 Subject: [PATCH 03/25] add graphQLRequest --- src/helpers/request.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 src/helpers/request.ts diff --git a/src/helpers/request.ts b/src/helpers/request.ts new file mode 100644 index 0000000..670f1b5 --- /dev/null +++ b/src/helpers/request.ts @@ -0,0 +1,10 @@ +export const graphQLRequest = async (query: string, variables: any) => { + const res = await fetch("https://mainnet.serve.giveth.io/graphql", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ query, variables }), + }); + return await res.json(); +}; From 1f1b3ba25134267c49eda0aa934f9a3cde3756e8 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 14:27:36 +0330 Subject: [PATCH 04/25] add fetchGivethProjects --- src/features/import-projects/service.ts | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/features/import-projects/service.ts diff --git a/src/features/import-projects/service.ts b/src/features/import-projects/service.ts new file mode 100644 index 0000000..4b6ae1e --- /dev/null +++ b/src/features/import-projects/service.ts @@ -0,0 +1,27 @@ +import { graphQLRequest } from "../../helpers/request"; + +export const fetchGivethProjects = async () => { + const res = await graphQLRequest( + `query ($limit: Int, $skip: Int, $sortingBy: SortingField) { + allProjects( + limit: $limit + skip: $skip + sortingBy: $sortingBy + ) { + projects { + id + title + image + slug + descriptionSummary + } + } + }`, + { + limit: 10, + skip: 0, + sortingBy: "Newest", + } + ); + return res.data.allProjects.projects; +}; From f56613728754c11672f4b623db9da866a7f97dde Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 14:28:28 +0330 Subject: [PATCH 05/25] run importProjects on start --- src/features/import-projects/index.ts | 17 +++++++++++++++++ src/main.ts | 3 +++ 2 files changed, 20 insertions(+) create mode 100644 src/features/import-projects/index.ts diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts new file mode 100644 index 0000000..ca031b4 --- /dev/null +++ b/src/features/import-projects/index.ts @@ -0,0 +1,17 @@ +import { Project } from "../../model"; +import { createEntityManager, getDataSource } from "../../helpers/db"; +import { fetchGivethProjects } from "./service"; + +export const importProjects = async () => { + try { + console.log("importing functions,"); + const em = await createEntityManager(); + const exitingProjects = await em.find(Project); + console.log("exitingProjects", exitingProjects); + + const projects = await fetchGivethProjects(); + console.log("projects", projects); + } catch (error: any) { + console.log("error", error.message); + } +}; diff --git a/src/main.ts b/src/main.ts index dcf6af9..95e6ade 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,6 +3,7 @@ import { processor } from "./processor"; import * as EASContract from "./abi/EAS"; import { processAttest } from "./mappings/attest"; import { processRevokeLog } from "./mappings/revoke"; +import { importProjects } from "./features/import-projects/index"; processor.run(new TypeormDatabase({ supportHotBlocks: false }), async (ctx) => { for (let _block of ctx.blocks) { @@ -18,3 +19,5 @@ processor.run(new TypeormDatabase({ supportHotBlocks: false }), async (ctx) => { } } }); + +importProjects(); From 6dfc889a79e3164841771b79d87e72639ea4a377 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 14:32:10 +0330 Subject: [PATCH 06/25] add slug and image to project schema --- schema.graphql | 4 ++++ src/model/generated/project.model.ts | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/schema.graphql b/schema.graphql index e8861ea..661e7fd 100644 --- a/schema.graphql +++ b/schema.graphql @@ -59,6 +59,10 @@ type Project @entity { totalFlags: Int! "Total attests" totalAttests: Int! + "Slug of the project" + slug: String! + "Image of the project" + image: String lastUpdatedTimestamp: DateTime! attests: [ProjectAttestation!]! @derivedFrom(field: "project") attestedOrganisations: [OrganisationProject!]! @derivedFrom(field: "project") diff --git a/src/model/generated/project.model.ts b/src/model/generated/project.model.ts index 37ffca4..642e642 100644 --- a/src/model/generated/project.model.ts +++ b/src/model/generated/project.model.ts @@ -58,6 +58,18 @@ export class Project { @Column_("int4", {nullable: false}) totalAttests!: number + /** + * Slug of the project + */ + @Column_("text", {nullable: false}) + slug!: string + + /** + * Image of the project + */ + @Column_("text", {nullable: true}) + image!: string | undefined | null + @Column_("timestamp with time zone", {nullable: false}) lastUpdatedTimestamp!: Date From d89b343dc38fc54d1096bb789fef3178adf3cd95 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 15:30:51 +0330 Subject: [PATCH 07/25] add giveth projects --- src/features/import-projects/index.ts | 49 +++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index ca031b4..21c1854 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -1,17 +1,52 @@ import { Project } from "../../model"; import { createEntityManager, getDataSource } from "../../helpers/db"; import { fetchGivethProjects } from "./service"; +import { type EntityManager } from "typeorm"; + +const updateGivethProject = async () => { + const projects = await fetchGivethProjects(); + console.log("Giveth Project Length", projects.Length); + for (const project of projects) { + console.log("Project", project); + const dataSource = await getDataSource(); + const exitingProject = await dataSource + .getRepository(Project) + .createQueryBuilder("project") + .where("project.id = :id", { id: `giveth-${project.id}` }) + .getOne(); + if (exitingProject) { + console.log("Project exists", exitingProject); + continue; + } + const newProject = new Project({ + id: `giveth-${project.id}`, + title: project.title, + description: project.descriptionSummary, + // image: project.image, + // slug: project.slug, + projectId: project.id, + source: "giveth", + totalVouches: 0, + totalFlags: 0, + totalAttests: 0, + lastUpdatedTimestamp: new Date(), + }); + console.log("newProject", newProject); + await dataSource + .createQueryBuilder() + .insert() + .into(Project) + .values([newProject]) + .execute(); + } +}; export const importProjects = async () => { try { - console.log("importing functions,"); + console.log("Importing Projects"); const em = await createEntityManager(); - const exitingProjects = await em.find(Project); - console.log("exitingProjects", exitingProjects); - - const projects = await fetchGivethProjects(); - console.log("projects", projects); + await updateGivethProject(); } catch (error: any) { - console.log("error", error.message); + console.log("error", error); } }; From b332543bcd03b06a7e01a38f9f3ddfaa8ae6f22d Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 16:25:21 +0330 Subject: [PATCH 08/25] add slug and image --- db/migrations/1716207083370-Data.js | 13 +++++++++++++ schema.graphql | 2 +- src/model/generated/project.model.ts | 4 ++-- 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 db/migrations/1716207083370-Data.js diff --git a/db/migrations/1716207083370-Data.js b/db/migrations/1716207083370-Data.js new file mode 100644 index 0000000..0fc1c3a --- /dev/null +++ b/db/migrations/1716207083370-Data.js @@ -0,0 +1,13 @@ +module.exports = class Data1716207083370 { + name = 'Data1716207083370' + + async up(db) { + await db.query(`ALTER TABLE "project" ADD "slug" text`) + await db.query(`ALTER TABLE "project" ADD "image" text`) + } + + async down(db) { + await db.query(`ALTER TABLE "project" DROP COLUMN "slug"`) + await db.query(`ALTER TABLE "project" DROP COLUMN "image"`) + } +} diff --git a/schema.graphql b/schema.graphql index 661e7fd..d36c109 100644 --- a/schema.graphql +++ b/schema.graphql @@ -60,7 +60,7 @@ type Project @entity { "Total attests" totalAttests: Int! "Slug of the project" - slug: String! + slug: String "Image of the project" image: String lastUpdatedTimestamp: DateTime! diff --git a/src/model/generated/project.model.ts b/src/model/generated/project.model.ts index 642e642..bce0129 100644 --- a/src/model/generated/project.model.ts +++ b/src/model/generated/project.model.ts @@ -61,8 +61,8 @@ export class Project { /** * Slug of the project */ - @Column_("text", {nullable: false}) - slug!: string + @Column_("text", {nullable: true}) + slug!: string | undefined | null /** * Image of the project From 6387d80b8ff79f3f68572e744bc81ebb1ebe8130 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 16:38:18 +0330 Subject: [PATCH 09/25] update exiting project --- src/features/import-projects/index.ts | 63 ++++++++++++++++++--------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index 21c1854..6acac29 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -16,28 +16,49 @@ const updateGivethProject = async () => { .getOne(); if (exitingProject) { console.log("Project exists", exitingProject); - continue; + if ( + exitingProject?.title === project.title && + exitingProject?.description === project.descriptionSummary && + exitingProject?.slug === project.slug && + exitingProject?.image === project.image + ) { + continue; + } + const updatedProject = new Project({ + ...exitingProject, + title: project.title, + description: project.descriptionSummary, + image: project.image, + slug: project.slug, + lastUpdatedTimestamp: new Date(), + }); + await dataSource + .createQueryBuilder() + .update(Project) + .set(updatedProject) + .where("id = :id", { id: `giveth-${project.id}` }) + .execute(); + } else { + const newProject = new Project({ + id: `giveth-${project.id}`, + title: project.title, + description: project.descriptionSummary, + image: project.image, + slug: project.slug, + projectId: project.id, + source: "giveth", + totalVouches: 0, + totalFlags: 0, + totalAttests: 0, + lastUpdatedTimestamp: new Date(), + }); + await dataSource + .createQueryBuilder() + .insert() + .into(Project) + .values([newProject]) + .execute(); } - const newProject = new Project({ - id: `giveth-${project.id}`, - title: project.title, - description: project.descriptionSummary, - // image: project.image, - // slug: project.slug, - projectId: project.id, - source: "giveth", - totalVouches: 0, - totalFlags: 0, - totalAttests: 0, - lastUpdatedTimestamp: new Date(), - }); - console.log("newProject", newProject); - await dataSource - .createQueryBuilder() - .insert() - .into(Project) - .values([newProject]) - .execute(); } }; From 6de355d331c620cc6f8393999a00b3059f8dc45a Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 16:56:38 +0330 Subject: [PATCH 10/25] add GivethProjectInfo type --- src/features/import-projects/type.ts | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/features/import-projects/type.ts diff --git a/src/features/import-projects/type.ts b/src/features/import-projects/type.ts new file mode 100644 index 0000000..a7e9588 --- /dev/null +++ b/src/features/import-projects/type.ts @@ -0,0 +1,7 @@ +export type GivethProjectInfo = { + id: string; + title: string; + descriptionSummary: string; + slug: string; + image: string; +}; From 4ff3c13c0ae293b8cdd98bf7f056617c009e9058 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 16:56:49 +0330 Subject: [PATCH 11/25] refactor --- src/features/import-projects/index.ts | 108 +++++++++++++++----------- 1 file changed, 61 insertions(+), 47 deletions(-) diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index 6acac29..6cc04d2 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -1,73 +1,87 @@ import { Project } from "../../model"; import { createEntityManager, getDataSource } from "../../helpers/db"; import { fetchGivethProjects } from "./service"; -import { type EntityManager } from "typeorm"; +import { DataSource } from "typeorm"; +import { GivethProjectInfo } from "./type"; -const updateGivethProject = async () => { - const projects = await fetchGivethProjects(); - console.log("Giveth Project Length", projects.Length); - for (const project of projects) { - console.log("Project", project); - const dataSource = await getDataSource(); - const exitingProject = await dataSource - .getRepository(Project) - .createQueryBuilder("project") - .where("project.id = :id", { id: `giveth-${project.id}` }) - .getOne(); - if (exitingProject) { - console.log("Project exists", exitingProject); - if ( - exitingProject?.title === project.title && - exitingProject?.description === project.descriptionSummary && - exitingProject?.slug === project.slug && - exitingProject?.image === project.image - ) { - continue; - } +const updateOrCreateProject = async ( + dataSource: DataSource, + project: GivethProjectInfo +) => { + const existingProject = await dataSource + .getRepository(Project) + .createQueryBuilder("project") + .where("project.id = :id", { id: `giveth-${project.id}` }) + .getOne(); + + if (existingProject) { + console.log("Project exists", existingProject); + + const isUpdated = + existingProject.title !== project.title || + existingProject.description !== project.descriptionSummary || + existingProject.slug !== project.slug || + existingProject.image !== project.image; + + if (isUpdated) { const updatedProject = new Project({ - ...exitingProject, + ...existingProject, title: project.title, description: project.descriptionSummary, image: project.image, slug: project.slug, lastUpdatedTimestamp: new Date(), }); + await dataSource .createQueryBuilder() .update(Project) .set(updatedProject) .where("id = :id", { id: `giveth-${project.id}` }) .execute(); - } else { - const newProject = new Project({ - id: `giveth-${project.id}`, - title: project.title, - description: project.descriptionSummary, - image: project.image, - slug: project.slug, - projectId: project.id, - source: "giveth", - totalVouches: 0, - totalFlags: 0, - totalAttests: 0, - lastUpdatedTimestamp: new Date(), - }); - await dataSource - .createQueryBuilder() - .insert() - .into(Project) - .values([newProject]) - .execute(); } + } else { + const newProject = new Project({ + id: `giveth-${project.id}`, + title: project.title, + description: project.descriptionSummary, + image: project.image, + slug: project.slug, + projectId: project.id, + source: "giveth", + totalVouches: 0, + totalFlags: 0, + totalAttests: 0, + lastUpdatedTimestamp: new Date(), + }); + + await dataSource + .createQueryBuilder() + .insert() + .into(Project) + .values([newProject]) + .execute(); + } +}; + +const updateGivethProjects = async () => { + const projects = await fetchGivethProjects(); + console.log("Giveth Project Length", projects.length); + + const dataSource = await getDataSource(); + + for (const project of projects) { + console.log("Project", project); + await updateOrCreateProject(dataSource, project); } }; export const importProjects = async () => { try { console.log("Importing Projects"); - const em = await createEntityManager(); - await updateGivethProject(); - } catch (error: any) { - console.log("error", error); + await createEntityManager(); + await updateGivethProjects(); + } catch (error) { + console.log("Error", error); } }; From 3e625dc9d6bd171bae674e3eb10e495ae2bd0c81 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 17:08:02 +0330 Subject: [PATCH 12/25] fetch all projects --- src/features/import-projects/index.ts | 1 - src/features/import-projects/service.ts | 55 +++++++++++++++++-------- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index 6cc04d2..bd2706e 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -79,7 +79,6 @@ const updateGivethProjects = async () => { export const importProjects = async () => { try { console.log("Importing Projects"); - await createEntityManager(); await updateGivethProjects(); } catch (error) { console.log("Error", error); diff --git a/src/features/import-projects/service.ts b/src/features/import-projects/service.ts index 4b6ae1e..a9f3140 100644 --- a/src/features/import-projects/service.ts +++ b/src/features/import-projects/service.ts @@ -1,27 +1,48 @@ import { graphQLRequest } from "../../helpers/request"; +import { GivethProjectInfo } from "./type"; -export const fetchGivethProjects = async () => { +const fetchGivethProjectsBatch = async (limit: number, skip: number) => { const res = await graphQLRequest( `query ($limit: Int, $skip: Int, $sortingBy: SortingField) { - allProjects( - limit: $limit - skip: $skip - sortingBy: $sortingBy - ) { - projects { - id - title - image - slug - descriptionSummary - } - } - }`, + allProjects( + limit: $limit + skip: $skip + sortingBy: $sortingBy + ) { + projects { + id + title + image + slug + descriptionSummary + } + } + }`, { - limit: 10, - skip: 0, + limit, + skip, sortingBy: "Newest", } ); + return res.data.allProjects.projects; }; + +export const fetchGivethProjects = async () => { + let allProjects: GivethProjectInfo[] = []; + let hasMoreProjects = true; + let skip = 0; + const limit = 10; + + while (hasMoreProjects) { + const projectsBatch = await fetchGivethProjectsBatch(limit, skip); + if (projectsBatch.length > 0) { + allProjects = allProjects.concat(projectsBatch); + skip += limit; + } else { + hasMoreProjects = false; + } + } + + return allProjects; +}; From 8644377acfb0272f38e52c5e3dd0252f50585ad7 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 17:17:19 +0330 Subject: [PATCH 13/25] process each giveth projects batch separately --- src/features/import-projects/index.ts | 32 +++++++++++++++++-------- src/features/import-projects/service.ts | 2 +- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index bd2706e..281bbc7 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -1,6 +1,6 @@ import { Project } from "../../model"; -import { createEntityManager, getDataSource } from "../../helpers/db"; -import { fetchGivethProjects } from "./service"; +import { getDataSource } from "../../helpers/db"; +import { fetchGivethProjectsBatch } from "./service"; import { DataSource } from "typeorm"; import { GivethProjectInfo } from "./type"; @@ -64,22 +64,34 @@ const updateOrCreateProject = async ( } }; -const updateGivethProjects = async () => { - const projects = await fetchGivethProjects(); - console.log("Giveth Project Length", projects.length); - +const processProjectsBatch = async (projectsBatch: GivethProjectInfo[]) => { const dataSource = await getDataSource(); - - for (const project of projects) { - console.log("Project", project); + for (const project of projectsBatch) { + console.log("Processing Project", project); await updateOrCreateProject(dataSource, project); } }; +export const fetchAndProcessGivethProjects = async () => { + let hasMoreProjects = true; + let skip = 0; + const limit = 10; + + while (hasMoreProjects) { + const projectsBatch = await fetchGivethProjectsBatch(limit, skip); + if (projectsBatch.length > 0) { + await processProjectsBatch(projectsBatch); + skip += limit; + } else { + hasMoreProjects = false; + } + } +}; + export const importProjects = async () => { try { console.log("Importing Projects"); - await updateGivethProjects(); + await fetchAndProcessGivethProjects(); } catch (error) { console.log("Error", error); } diff --git a/src/features/import-projects/service.ts b/src/features/import-projects/service.ts index a9f3140..fe36c06 100644 --- a/src/features/import-projects/service.ts +++ b/src/features/import-projects/service.ts @@ -1,7 +1,7 @@ import { graphQLRequest } from "../../helpers/request"; import { GivethProjectInfo } from "./type"; -const fetchGivethProjectsBatch = async (limit: number, skip: number) => { +export const fetchGivethProjectsBatch = async (limit: number, skip: number) => { const res = await graphQLRequest( `query ($limit: Int, $skip: Int, $sortingBy: SortingField) { allProjects( From 360538320369459e9a1517283ee4573c05372f52 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 17:19:02 +0330 Subject: [PATCH 14/25] wrap in try catch --- src/features/import-projects/index.ts | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index 281bbc7..84de4f5 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -73,18 +73,22 @@ const processProjectsBatch = async (projectsBatch: GivethProjectInfo[]) => { }; export const fetchAndProcessGivethProjects = async () => { - let hasMoreProjects = true; - let skip = 0; - const limit = 10; + try { + let hasMoreProjects = true; + let skip = 0; + const limit = 10; - while (hasMoreProjects) { - const projectsBatch = await fetchGivethProjectsBatch(limit, skip); - if (projectsBatch.length > 0) { - await processProjectsBatch(projectsBatch); - skip += limit; - } else { - hasMoreProjects = false; + while (hasMoreProjects) { + const projectsBatch = await fetchGivethProjectsBatch(limit, skip); + if (projectsBatch.length > 0) { + await processProjectsBatch(projectsBatch); + skip += limit; + } else { + hasMoreProjects = false; + } } + } catch (error: any) { + console.log("error on fetching giveth projects", error.message); } }; From 6ef1eebce00fc730ee714fede90e8997fb880fc7 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 17:22:22 +0330 Subject: [PATCH 15/25] only log project id --- src/features/import-projects/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index 84de4f5..a8f259f 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -15,8 +15,6 @@ const updateOrCreateProject = async ( .getOne(); if (existingProject) { - console.log("Project exists", existingProject); - const isUpdated = existingProject.title !== project.title || existingProject.description !== project.descriptionSummary || @@ -67,7 +65,7 @@ const updateOrCreateProject = async ( const processProjectsBatch = async (projectsBatch: GivethProjectInfo[]) => { const dataSource = await getDataSource(); for (const project of projectsBatch) { - console.log("Processing Project", project); + console.log("Processing Project: Giveth", project.id); await updateOrCreateProject(dataSource, project); } }; From 8e17d0946d8f89715b35e4a45977bfa93fe27463 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 17:39:43 +0330 Subject: [PATCH 16/25] remove export --- src/features/import-projects/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index a8f259f..27b5f6f 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -70,7 +70,7 @@ const processProjectsBatch = async (projectsBatch: GivethProjectInfo[]) => { } }; -export const fetchAndProcessGivethProjects = async () => { +const fetchAndProcessGivethProjects = async () => { try { let hasMoreProjects = true; let skip = 0; From 81b458b579b431db6c32f37c9e57470e5b29a19a Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 17:41:10 +0330 Subject: [PATCH 17/25] add GIVETH_API_URL --- .env.template | 3 +++ src/features/import-projects/constants.ts | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 src/features/import-projects/constants.ts diff --git a/.env.template b/.env.template index b40d2ad..eb10b9e 100644 --- a/.env.template +++ b/.env.template @@ -19,3 +19,6 @@ SCHEMA_CONTRACT_ADDRESS=0x0a7E2Ff54e76B8E6659aedc9103FB21c038050D0 PROJECT_VERIFY_ATTESTATION_SCHEMA=0x3D5854AF182F27966DEA837C446A051B3509224DDC03150E55097B362D111B1B PROJECT_GIVBACK_ELIGIBLE_ATTESTATION_SCHEMA=0x0000000000000000000000000000000000000000000000000000000000000000 + +#APIs +GIVETH_API_URL=Https://somewhere.giveth.io/XXXXX diff --git a/src/features/import-projects/constants.ts b/src/features/import-projects/constants.ts new file mode 100644 index 0000000..dca2815 --- /dev/null +++ b/src/features/import-projects/constants.ts @@ -0,0 +1,2 @@ +export const GIVETH_API_URL = + process.env.GIVETH_API_URL || "https://mainnet.serve.giveth.io/graphql"; From 28fb55a9a46c522622f08ae1c4c8515fc3845172 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 17:41:22 +0330 Subject: [PATCH 18/25] pass url to graphQLRequest --- src/features/import-projects/service.ts | 2 ++ src/helpers/request.ts | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/features/import-projects/service.ts b/src/features/import-projects/service.ts index fe36c06..0ddde4c 100644 --- a/src/features/import-projects/service.ts +++ b/src/features/import-projects/service.ts @@ -1,8 +1,10 @@ import { graphQLRequest } from "../../helpers/request"; +import { GIVETH_API_URL } from "./constants"; import { GivethProjectInfo } from "./type"; export const fetchGivethProjectsBatch = async (limit: number, skip: number) => { const res = await graphQLRequest( + GIVETH_API_URL, `query ($limit: Int, $skip: Int, $sortingBy: SortingField) { allProjects( limit: $limit diff --git a/src/helpers/request.ts b/src/helpers/request.ts index 670f1b5..f7ee8a8 100644 --- a/src/helpers/request.ts +++ b/src/helpers/request.ts @@ -1,5 +1,9 @@ -export const graphQLRequest = async (query: string, variables: any) => { - const res = await fetch("https://mainnet.serve.giveth.io/graphql", { +export const graphQLRequest = async ( + url: string, + query: string, + variables: any +) => { + const res = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json", From e2c0844d0f4499b0eed8fee6528ee270d2f48798 Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 17:43:30 +0330 Subject: [PATCH 19/25] add try catch to getDataSource --- src/features/import-projects/index.ts | 1 + src/helpers/db.ts | 16 +++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index 27b5f6f..3e0bf01 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -64,6 +64,7 @@ const updateOrCreateProject = async ( const processProjectsBatch = async (projectsBatch: GivethProjectInfo[]) => { const dataSource = await getDataSource(); + if (!dataSource) return; for (const project of projectsBatch) { console.log("Processing Project: Giveth", project.id); await updateOrCreateProject(dataSource, project); diff --git a/src/helpers/db.ts b/src/helpers/db.ts index 1831412..3e5d0f6 100644 --- a/src/helpers/db.ts +++ b/src/helpers/db.ts @@ -4,16 +4,22 @@ import { DataSource } from "typeorm"; let dataSource: DataSource | undefined = undefined; export const getDataSource = async () => { - if (!dataSource || !dataSource.isInitialized) { - let cfg = createOrmConfig({ projectDir: __dirname + "/../.." }); - (cfg.entities as string[]).push(__dirname + "/../model/generated/*.ts"); + try { + if (!dataSource || !dataSource.isInitialized) { + let cfg = createOrmConfig({ projectDir: __dirname + "/../.." }); + (cfg.entities as string[]).push(__dirname + "/../model/generated/*.ts"); - dataSource = await new DataSource(cfg).initialize(); + dataSource = await new DataSource(cfg).initialize(); + } + return dataSource; + } catch (error) { + console.error("Failed to initialize DataSource", error); + return null; } - return dataSource; }; export const createEntityManager = async () => { const dataSource = await getDataSource(); + if (!dataSource) return null; return dataSource.createEntityManager(); }; From 18c7c16273ecefdc09afacf194412d20c660cefd Mon Sep 17 00:00:00 2001 From: Cherik Date: Mon, 20 May 2024 17:50:58 +0330 Subject: [PATCH 20/25] move giveth related files to giveth directory --- .../import-projects/{ => giveth}/constants.ts | 0 .../import-projects/giveth/helpers.ts | 92 ++++++++++++++++++ .../import-projects/{ => giveth}/service.ts | 2 +- .../import-projects/{ => giveth}/type.ts | 0 src/features/import-projects/index.ts | 93 +------------------ 5 files changed, 94 insertions(+), 93 deletions(-) rename src/features/import-projects/{ => giveth}/constants.ts (100%) create mode 100644 src/features/import-projects/giveth/helpers.ts rename src/features/import-projects/{ => giveth}/service.ts (94%) rename src/features/import-projects/{ => giveth}/type.ts (100%) diff --git a/src/features/import-projects/constants.ts b/src/features/import-projects/giveth/constants.ts similarity index 100% rename from src/features/import-projects/constants.ts rename to src/features/import-projects/giveth/constants.ts diff --git a/src/features/import-projects/giveth/helpers.ts b/src/features/import-projects/giveth/helpers.ts new file mode 100644 index 0000000..c4f1def --- /dev/null +++ b/src/features/import-projects/giveth/helpers.ts @@ -0,0 +1,92 @@ +import { DataSource } from "typeorm"; +import { getDataSource } from "../../../helpers/db"; +import { Project } from "../../../model"; +import { fetchGivethProjectsBatch } from "./service"; +import { GivethProjectInfo } from "./type"; + +const updateOrCreateProject = async ( + dataSource: DataSource, + project: GivethProjectInfo +) => { + const existingProject = await dataSource + .getRepository(Project) + .createQueryBuilder("project") + .where("project.id = :id", { id: `giveth-${project.id}` }) + .getOne(); + + if (existingProject) { + const isUpdated = + existingProject.title !== project.title || + existingProject.description !== project.descriptionSummary || + existingProject.slug !== project.slug || + existingProject.image !== project.image; + + if (isUpdated) { + const updatedProject = new Project({ + ...existingProject, + title: project.title, + description: project.descriptionSummary, + image: project.image, + slug: project.slug, + lastUpdatedTimestamp: new Date(), + }); + + await dataSource + .createQueryBuilder() + .update(Project) + .set(updatedProject) + .where("id = :id", { id: `giveth-${project.id}` }) + .execute(); + } + } else { + const newProject = new Project({ + id: `giveth-${project.id}`, + title: project.title, + description: project.descriptionSummary, + image: project.image, + slug: project.slug, + projectId: project.id, + source: "giveth", + totalVouches: 0, + totalFlags: 0, + totalAttests: 0, + lastUpdatedTimestamp: new Date(), + }); + + await dataSource + .createQueryBuilder() + .insert() + .into(Project) + .values([newProject]) + .execute(); + } +}; + +const processProjectsBatch = async (projectsBatch: GivethProjectInfo[]) => { + const dataSource = await getDataSource(); + if (!dataSource) return; + for (const project of projectsBatch) { + console.log("Processing Project: Giveth", project.id); + await updateOrCreateProject(dataSource, project); + } +}; + +export const fetchAndProcessGivethProjects = async () => { + try { + let hasMoreProjects = true; + let skip = 0; + const limit = 10; + + while (hasMoreProjects) { + const projectsBatch = await fetchGivethProjectsBatch(limit, skip); + if (projectsBatch.length > 0) { + await processProjectsBatch(projectsBatch); + skip += limit; + } else { + hasMoreProjects = false; + } + } + } catch (error: any) { + console.log("error on fetching giveth projects", error.message); + } +}; diff --git a/src/features/import-projects/service.ts b/src/features/import-projects/giveth/service.ts similarity index 94% rename from src/features/import-projects/service.ts rename to src/features/import-projects/giveth/service.ts index 0ddde4c..4854729 100644 --- a/src/features/import-projects/service.ts +++ b/src/features/import-projects/giveth/service.ts @@ -1,4 +1,4 @@ -import { graphQLRequest } from "../../helpers/request"; +import { graphQLRequest } from "../../../helpers/request"; import { GIVETH_API_URL } from "./constants"; import { GivethProjectInfo } from "./type"; diff --git a/src/features/import-projects/type.ts b/src/features/import-projects/giveth/type.ts similarity index 100% rename from src/features/import-projects/type.ts rename to src/features/import-projects/giveth/type.ts diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index 3e0bf01..6d5a531 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -1,95 +1,4 @@ -import { Project } from "../../model"; -import { getDataSource } from "../../helpers/db"; -import { fetchGivethProjectsBatch } from "./service"; -import { DataSource } from "typeorm"; -import { GivethProjectInfo } from "./type"; - -const updateOrCreateProject = async ( - dataSource: DataSource, - project: GivethProjectInfo -) => { - const existingProject = await dataSource - .getRepository(Project) - .createQueryBuilder("project") - .where("project.id = :id", { id: `giveth-${project.id}` }) - .getOne(); - - if (existingProject) { - const isUpdated = - existingProject.title !== project.title || - existingProject.description !== project.descriptionSummary || - existingProject.slug !== project.slug || - existingProject.image !== project.image; - - if (isUpdated) { - const updatedProject = new Project({ - ...existingProject, - title: project.title, - description: project.descriptionSummary, - image: project.image, - slug: project.slug, - lastUpdatedTimestamp: new Date(), - }); - - await dataSource - .createQueryBuilder() - .update(Project) - .set(updatedProject) - .where("id = :id", { id: `giveth-${project.id}` }) - .execute(); - } - } else { - const newProject = new Project({ - id: `giveth-${project.id}`, - title: project.title, - description: project.descriptionSummary, - image: project.image, - slug: project.slug, - projectId: project.id, - source: "giveth", - totalVouches: 0, - totalFlags: 0, - totalAttests: 0, - lastUpdatedTimestamp: new Date(), - }); - - await dataSource - .createQueryBuilder() - .insert() - .into(Project) - .values([newProject]) - .execute(); - } -}; - -const processProjectsBatch = async (projectsBatch: GivethProjectInfo[]) => { - const dataSource = await getDataSource(); - if (!dataSource) return; - for (const project of projectsBatch) { - console.log("Processing Project: Giveth", project.id); - await updateOrCreateProject(dataSource, project); - } -}; - -const fetchAndProcessGivethProjects = async () => { - try { - let hasMoreProjects = true; - let skip = 0; - const limit = 10; - - while (hasMoreProjects) { - const projectsBatch = await fetchGivethProjectsBatch(limit, skip); - if (projectsBatch.length > 0) { - await processProjectsBatch(projectsBatch); - skip += limit; - } else { - hasMoreProjects = false; - } - } - } catch (error: any) { - console.log("error on fetching giveth projects", error.message); - } -}; +import { fetchAndProcessGivethProjects } from "./giveth/helpers"; export const importProjects = async () => { try { From ce6fca67b8aedd8399427e09af8e9b6d256414c4 Mon Sep 17 00:00:00 2001 From: Cherik Date: Tue, 21 May 2024 11:12:03 +0330 Subject: [PATCH 21/25] add node-cron --- package-lock.json | 27 +++++++++++++++++++++++++++ package.json | 2 ++ 2 files changed, 29 insertions(+) diff --git a/package-lock.json b/package-lock.json index ed3f44b..310a5b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@subsquid/typeorm-store": "^1.4.0", "dotenv": "^16.4.4", "ethers": "^6.12.1", + "node-cron": "^3.0.3", "pg": "^8.11.5", "type-graphql": "^1.2.0-rc.1", "typeorm": "^0.3.20", @@ -25,6 +26,7 @@ "@subsquid/typeorm-codegen": "^1.3.3", "@types/jest": "^29.5.12", "@types/node": "^20.11.17", + "@types/node-cron": "^3.0.11", "jest": "^29.7.0", "ts-jest": "^29.1.2", "ts-node": "^10.9.2", @@ -4305,6 +4307,12 @@ "undici-types": "~5.26.4" } }, + "node_modules/@types/node-cron": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.11.tgz", + "integrity": "sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==", + "dev": true + }, "node_modules/@types/pbkdf2": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@types/pbkdf2/-/pbkdf2-3.1.2.tgz", @@ -9988,6 +9996,25 @@ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" }, + "node_modules/node-cron": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz", + "integrity": "sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==", + "dependencies": { + "uuid": "8.3.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/node-cron/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", diff --git a/package.json b/package.json index c6ae277..6a21468 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@subsquid/typeorm-store": "^1.4.0", "dotenv": "^16.4.4", "ethers": "^6.12.1", + "node-cron": "^3.0.3", "pg": "^8.11.5", "type-graphql": "^1.2.0-rc.1", "typeorm": "^0.3.20", @@ -30,6 +31,7 @@ "@subsquid/typeorm-codegen": "^1.3.3", "@types/jest": "^29.5.12", "@types/node": "^20.11.17", + "@types/node-cron": "^3.0.11", "jest": "^29.7.0", "ts-jest": "^29.1.2", "ts-node": "^10.9.2", From 2c82074878e67657cad7c626a03da7b481a7b18a Mon Sep 17 00:00:00 2001 From: Cherik Date: Tue, 21 May 2024 11:28:19 +0330 Subject: [PATCH 22/25] set cron job --- src/features/import-projects/index.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index 6d5a531..c207ddc 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -1,9 +1,18 @@ +import cron from "node-cron"; import { fetchAndProcessGivethProjects } from "./giveth/helpers"; +const task = async () => { + console.log("Importing Projects", new Date()); + await fetchAndProcessGivethProjects(); +}; + export const importProjects = async () => { try { - console.log("Importing Projects"); - await fetchAndProcessGivethProjects(); + console.log("Importing Projects has been scheduled."); + cron.schedule("0 0 * * *", task, { + scheduled: true, + timezone: "UTC", + }); } catch (error) { console.log("Error", error); } From eabbbbb49cbeba31a36ab2aa82587e3bfb41b97a Mon Sep 17 00:00:00 2001 From: Cherik Date: Tue, 21 May 2024 12:28:37 +0330 Subject: [PATCH 23/25] add standalone file --- src/features/import-projects/index.ts | 2 +- src/features/import-projects/standalone.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 src/features/import-projects/standalone.ts diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index c207ddc..87f3b35 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -1,7 +1,7 @@ import cron from "node-cron"; import { fetchAndProcessGivethProjects } from "./giveth/helpers"; -const task = async () => { +export const task = async () => { console.log("Importing Projects", new Date()); await fetchAndProcessGivethProjects(); }; diff --git a/src/features/import-projects/standalone.ts b/src/features/import-projects/standalone.ts new file mode 100644 index 0000000..1eb1bfe --- /dev/null +++ b/src/features/import-projects/standalone.ts @@ -0,0 +1,3 @@ +import { task } from "."; + +task(); From 891c3ac9694293dd16e5e51edd8c0b968daf32a4 Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Tue, 21 May 2024 12:55:09 +0330 Subject: [PATCH 24/25] Disabled clean up on develop deploy action --- .github/workflows/pipeline-develop.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pipeline-develop.yml b/.github/workflows/pipeline-develop.yml index c706ce0..78ea5f7 100644 --- a/.github/workflows/pipeline-develop.yml +++ b/.github/workflows/pipeline-develop.yml @@ -22,7 +22,7 @@ jobs: git checkout -- . git pull docker compose stop devouch squid-db - docker compose rm -v -f devouch squid-db - docker rmi -f $(docker images | grep 'devouch' | awk '{print $3}') - docker volume rm $(docker volume ls | grep 'db-data' | awk '{print $2}') + # docker compose rm -v -f devouch squid-db + # docker rmi -f $(docker images | grep 'devouch' | awk '{print $3}') + # docker volume rm $(docker volume ls | grep 'db-data' | awk '{print $2}') docker compose up -d --build From 4f57d84df9ab9c0ea2141acd6e1f3c1f59e7e335 Mon Sep 17 00:00:00 2001 From: Cherik Date: Tue, 21 May 2024 13:14:52 +0330 Subject: [PATCH 25/25] move to config file --- src/features/import-projects/config.ts | 1 + src/features/import-projects/index.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 src/features/import-projects/config.ts diff --git a/src/features/import-projects/config.ts b/src/features/import-projects/config.ts new file mode 100644 index 0000000..be5f17d --- /dev/null +++ b/src/features/import-projects/config.ts @@ -0,0 +1 @@ +export const CRON_SCHEDULE = "0 0 * * *"; // UTC diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index 87f3b35..4ef3178 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -1,5 +1,6 @@ import cron from "node-cron"; import { fetchAndProcessGivethProjects } from "./giveth/helpers"; +import { CRON_SCHEDULE } from "./config"; export const task = async () => { console.log("Importing Projects", new Date()); @@ -9,7 +10,7 @@ export const task = async () => { export const importProjects = async () => { try { console.log("Importing Projects has been scheduled."); - cron.schedule("0 0 * * *", task, { + cron.schedule(CRON_SCHEDULE, task, { scheduled: true, timezone: "UTC", });