From 304efe8b428e1fc6c4b3d9b47a27a0face4114c9 Mon Sep 17 00:00:00 2001 From: Carlos Date: Fri, 29 Sep 2023 15:32:16 -0500 Subject: [PATCH 1/3] add qfFilter with estimatedMatchingView --- package-lock.json | 2 +- src/entities/project.ts | 1 + src/repositories/projectRepository.ts | 20 ++++++++++++++++++++ src/resolvers/projectResolver.ts | 8 ++++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index dd3a6ad55..aded15275 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23513,7 +23513,7 @@ "integrity": "sha512-wgpWy002tA+wgmO27buH/9KzyEOQnKsG/R0yrcjPT9BOFm0zRBVQbZ95nRGXWMywS8YR5knRbpohio0bcJABxQ==", "dev": true, "requires": { - "semver-regex": "3.1.4" + "semver-regex": "^3.1.2" } }, "find-yarn-workspace-root": { diff --git a/src/entities/project.ts b/src/entities/project.ts index 81aa09209..3f30400b6 100644 --- a/src/entities/project.ts +++ b/src/entities/project.ts @@ -76,6 +76,7 @@ export enum SortingField { QualityScore = 'QualityScore', GIVPower = 'GIVPower', InstantBoosting = 'InstantBoosting', + QfRoundRaisedFunds = 'QfRoundRaisedFunds', } export enum FilterField { diff --git a/src/repositories/projectRepository.ts b/src/repositories/projectRepository.ts index 72f7003b4..3c7a4a294 100644 --- a/src/repositories/projectRepository.ts +++ b/src/repositories/projectRepository.ts @@ -17,6 +17,8 @@ import { import { User, publicSelectionFields } from '../entities/user'; import { ResourcesTotalPerMonthAndYear } from '../resolvers/donationResolver'; import { OrderDirection, ProjectResolver } from '../resolvers/projectResolver'; +import { ProjectEstimatedMatchingView } from '../entities/ProjectEstimatedMatchingView'; +import { findActiveQfRound } from './qfRoundRepository'; export const findProjectById = (projectId: number): Promise => { // return Project.findOne({ id: projectId }); @@ -59,6 +61,7 @@ export type FilterProjectQueryInputParams = { slugArray?: string[]; sortingBy?: SortingField; qfRoundId?: number; + activeQfRoundId?: number; }; export const filterProjectsQuery = (params: FilterProjectQueryInputParams) => { const { @@ -71,6 +74,7 @@ export const filterProjectsQuery = (params: FilterProjectQueryInputParams) => { sortingBy, slugArray, qfRoundId, + activeQfRoundId, } = params; let query = Project.createQueryBuilder('project') @@ -168,6 +172,22 @@ export const filterProjectsQuery = (params: FilterProjectQueryInputParams) => { 'NULLS LAST', ); break; + case SortingField.QfRoundRaisedFunds: + if (activeQfRoundId) { + query + .leftJoin( + ProjectEstimatedMatchingView, + 'projectEstimatedMatchingView', + 'project.projectId = projectEstimatedMatchingView.projectId AND projectEstimatedMatchingView.qfRoundId = :qfRoundId', + { qfRoundId: activeQfRoundId }, + ) + .addOrderBy( + `projectEstimatedMatchingView.sumValueUsd`, + OrderDirection.DESC, + 'NULLS LAST', + ); + } + break; default: query .orderBy('projectInstantPower.totalPower', OrderDirection.DESC) diff --git a/src/resolvers/projectResolver.ts b/src/resolvers/projectResolver.ts index 2ec7ab8b2..50b0a1541 100644 --- a/src/resolvers/projectResolver.ts +++ b/src/resolvers/projectResolver.ts @@ -108,6 +108,7 @@ import { FeaturedUpdate } from '../entities/featuredUpdate'; import { PROJECT_UPDATE_CONTENT_MAX_LENGTH } from '../constants/validators'; import { calculateGivbackFactor } from '../services/givbackService'; import { ProjectBySlugResponse } from './types/projectResolver'; +import { findActiveQfRound } from '../repositories/qfRoundRepository'; @ObjectType() class AllProjects { @@ -759,6 +760,12 @@ export class ProjectResolver { ): Promise { let projects: Project[]; let totalCount: number; + let activeQfRoundId: number | undefined; + + if (sortingBy === SortingField.QfRoundRaisedFunds) { + activeQfRoundId = (await findActiveQfRound())?.id; + } + const filterQueryParams: FilterProjectQueryInputParams = { limit, skip, @@ -768,6 +775,7 @@ export class ProjectResolver { filters, sortingBy, qfRoundId, + activeQfRoundId, }; let campaign; if (campaignSlug) { From 484f6ff7a0785dc3e9aa70487cd4b7579dd9a179 Mon Sep 17 00:00:00 2001 From: Carlos Date: Mon, 2 Oct 2023 02:26:22 -0500 Subject: [PATCH 2/3] add sorting by qfround raised funds --- src/entities/ProjectEstimatedMatchingView.ts | 19 ++++-- src/entities/entities.ts | 2 + src/entities/project.ts | 2 +- src/repositories/donationRepository.ts | 22 +++++++ src/repositories/projectRepository.ts | 21 ++---- src/resolvers/projectResolver.test.ts | 68 ++++++++++++++++++++ src/resolvers/projectResolver.ts | 32 ++++++++- 7 files changed, 146 insertions(+), 20 deletions(-) diff --git a/src/entities/ProjectEstimatedMatchingView.ts b/src/entities/ProjectEstimatedMatchingView.ts index b63ff7903..73b209c0c 100644 --- a/src/entities/ProjectEstimatedMatchingView.ts +++ b/src/entities/ProjectEstimatedMatchingView.ts @@ -1,6 +1,17 @@ -import { Entity, Column, Index, PrimaryColumn } from 'typeorm'; +import { ObjectType } from 'type-graphql'; +import { + Entity, + Column, + Index, + PrimaryColumn, + BaseEntity, + ViewEntity, + ManyToOne, + RelationId, +} from 'typeorm'; +import { Project } from './project'; -@Entity({ name: 'project_estimated_matching_view' }) +@ViewEntity('project_estimated_matching_view', { synchronize: false }) @Index('project_estimated_matching_view_project_id_qfround_id', [ 'projectId', 'qfRoundId', @@ -13,8 +24,8 @@ import { Entity, Column, Index, PrimaryColumn } from 'typeorm'; @Index('project_estimated_matching_view_unique_donation_count', [ 'uniqueDonationCount', ]) -export class ProjectEstimatedMatchingView { - // Project ID associated with the donations +@ObjectType() +export class ProjectEstimatedMatchingView extends BaseEntity { @PrimaryColumn() projectId: number; diff --git a/src/entities/entities.ts b/src/entities/entities.ts index ea5c39429..158f4df3f 100644 --- a/src/entities/entities.ts +++ b/src/entities/entities.ts @@ -41,6 +41,7 @@ import { ProjectInstantPowerView } from '../views/projectInstantPowerView'; import { QfRound } from './qfRound'; import { ReferredEvent } from './referredEvent'; import { QfRoundHistory } from './qfRoundHistory'; +import { ProjectEstimatedMatchingView } from './ProjectEstimatedMatchingView'; export const getEntities = (): DataSourceOptions['entities'] => { return [ @@ -81,6 +82,7 @@ export const getEntities = (): DataSourceOptions['entities'] => { LastSnapshotProjectPowerView, ProjectInstantPowerView, ProjectUserInstantPowerView, + ProjectEstimatedMatchingView, // historic snapshots PowerSnapshotHistory, diff --git a/src/entities/project.ts b/src/entities/project.ts index 3f30400b6..018486f36 100644 --- a/src/entities/project.ts +++ b/src/entities/project.ts @@ -76,7 +76,7 @@ export enum SortingField { QualityScore = 'QualityScore', GIVPower = 'GIVPower', InstantBoosting = 'InstantBoosting', - QfRoundRaisedFunds = 'QfRoundRaisedFunds', + ActiveQfRoundRaisedFunds = 'ActiveQfRoundRaisedFunds', } export enum FilterField { diff --git a/src/repositories/donationRepository.ts b/src/repositories/donationRepository.ts index 9bf2825fc..b9ae10c63 100644 --- a/src/repositories/donationRepository.ts +++ b/src/repositories/donationRepository.ts @@ -354,6 +354,28 @@ export async function sumDonationValueUsdForQfRound(params: { return result[0] ? result[0].sumValueUsd : 0; } +export async function projectsInQfRoundOrderedBySumValueUsd( + qfRoundId: number, + limit: number, + skip: number, +): Promise { + const result = await AppDataSource.getDataSource().query( + ` + SELECT "projectId", "sumValueUsd" + FROM project_estimated_matching_view + WHERE "qfRoundId" = $1 + ORDER BY "sumValueUsd" DESC + LIMIT $2 + OFFSET $3; + `, + [qfRoundId, limit, skip], + ); + + return result?.length > 0 + ? result.map(project => project.projectId) + : undefined; +} + export async function countUniqueDonors(projectId: number): Promise { const result = await AppDataSource.getDataSource().query( ` diff --git a/src/repositories/projectRepository.ts b/src/repositories/projectRepository.ts index 3c7a4a294..3da2e8836 100644 --- a/src/repositories/projectRepository.ts +++ b/src/repositories/projectRepository.ts @@ -62,6 +62,7 @@ export type FilterProjectQueryInputParams = { sortingBy?: SortingField; qfRoundId?: number; activeQfRoundId?: number; + qfRoundProjectsIds?: number[]; }; export const filterProjectsQuery = (params: FilterProjectQueryInputParams) => { const { @@ -75,6 +76,7 @@ export const filterProjectsQuery = (params: FilterProjectQueryInputParams) => { slugArray, qfRoundId, activeQfRoundId, + qfRoundProjectsIds, } = params; let query = Project.createQueryBuilder('project') @@ -172,20 +174,11 @@ export const filterProjectsQuery = (params: FilterProjectQueryInputParams) => { 'NULLS LAST', ); break; - case SortingField.QfRoundRaisedFunds: - if (activeQfRoundId) { - query - .leftJoin( - ProjectEstimatedMatchingView, - 'projectEstimatedMatchingView', - 'project.projectId = projectEstimatedMatchingView.projectId AND projectEstimatedMatchingView.qfRoundId = :qfRoundId', - { qfRoundId: activeQfRoundId }, - ) - .addOrderBy( - `projectEstimatedMatchingView.sumValueUsd`, - OrderDirection.DESC, - 'NULLS LAST', - ); + case SortingField.ActiveQfRoundRaisedFunds: + if (activeQfRoundId && qfRoundProjectsIds) { + query.andWhere('project.id IN (:...qfRoundProjectsIds)', { + qfRoundProjectsIds, + }); } break; default: diff --git a/src/resolvers/projectResolver.test.ts b/src/resolvers/projectResolver.test.ts index 0aaae662f..ca6308ddb 100644 --- a/src/resolvers/projectResolver.test.ts +++ b/src/resolvers/projectResolver.test.ts @@ -597,6 +597,74 @@ function allProjectsTestCases() { ); }); + it('should return projects, sort by project raised funds in the active QF round DESC', async () => { + const donor = await saveUserDirectlyToDb(generateRandomEtheriumAddress()); + const project1 = await saveProjectDirectlyToDb({ + ...createProjectData(), + title: String(new Date().getTime()), + slug: String(new Date().getTime()), + }); + const project2 = await saveProjectDirectlyToDb({ + ...createProjectData(), + title: String(new Date().getTime()), + slug: String(new Date().getTime()), + }); + + const qfRound = await QfRound.create({ + isActive: true, + name: 'test filter by qfRoundId', + minimumPassportScore: 10, + allocatedFund: 100, + beginDate: new Date(), + endDate: moment().add(1, 'day').toDate(), + }).save(); + project1.qfRounds = [qfRound]; + await project1.save(); + project2.qfRounds = [qfRound]; + await project2.save(); + + const donation1 = await saveDonationDirectlyToDb( + { + ...createDonationData(), + status: 'verified', + qfRoundId: qfRound.id, + valueUsd: 2, + }, + donor.id, + project1.id, + ); + + const donation2 = await saveDonationDirectlyToDb( + { + ...createDonationData(), + status: 'verified', + qfRoundId: qfRound.id, + valueUsd: 20, + }, + donor.id, + project2.id, + ); + + await refreshProjectEstimatedMatchingView(); + await refreshProjectDonationSummaryView(); + + const result = await axios.post(graphqlUrl, { + query: fetchMultiFilterAllProjectsQuery, + variables: { + sortingBy: SortingField.ActiveQfRoundRaisedFunds, + limit: 10, + }, + }); + + assert.equal(result.data.data.allProjects.projects.length, 2); + assert.equal(result.data.data.allProjects.projects[0].id, project2.id); + result.data.data.allProjects.projects.forEach(project => { + assert.equal(project.qfRounds[0].id, qfRound.id); + }); + qfRound.isActive = false; + await qfRound.save(); + }); + it('should return projects, sort by project instant power DESC', async () => { await PowerBoosting.clear(); await InstantPowerBalance.clear(); diff --git a/src/resolvers/projectResolver.ts b/src/resolvers/projectResolver.ts index 50b0a1541..12d8fe296 100644 --- a/src/resolvers/projectResolver.ts +++ b/src/resolvers/projectResolver.ts @@ -109,6 +109,7 @@ import { PROJECT_UPDATE_CONTENT_MAX_LENGTH } from '../constants/validators'; import { calculateGivbackFactor } from '../services/givbackService'; import { ProjectBySlugResponse } from './types/projectResolver'; import { findActiveQfRound } from '../repositories/qfRoundRepository'; +import { projectsInQfRoundOrderedBySumValueUsd } from '../repositories/donationRepository'; @ObjectType() class AllProjects { @@ -761,9 +762,18 @@ export class ProjectResolver { let projects: Project[]; let totalCount: number; let activeQfRoundId: number | undefined; + let qfRoundProjectsIds: number[] | undefined; - if (sortingBy === SortingField.QfRoundRaisedFunds) { + if (sortingBy === SortingField.ActiveQfRoundRaisedFunds) { activeQfRoundId = (await findActiveQfRound())?.id; + + if (activeQfRoundId) { + qfRoundProjectsIds = await projectsInQfRoundOrderedBySumValueUsd( + activeQfRoundId, + limit, + skip, + ); + } } const filterQueryParams: FilterProjectQueryInputParams = { @@ -776,6 +786,7 @@ export class ProjectResolver { sortingBy, qfRoundId, activeQfRoundId, + qfRoundProjectsIds, }; let campaign; if (campaignSlug) { @@ -804,6 +815,25 @@ export class ProjectResolver { .cache(projectsQueryCacheKey, projectFiltersCacheDuration) .getManyAndCount(); + // No join or custom subquery worked, solution is to write a full raw sql which is too much work + // Sorting will be required to be done in-memory + if ( + sortingBy === SortingField.ActiveQfRoundRaisedFunds && + activeQfRoundId && + qfRoundProjectsIds + ) { + const orderMap: { [key: string]: number } = {}; + qfRoundProjectsIds.forEach((id, index) => { + orderMap[id] = index; + }); + + const orderedProjects = projects.sort((a, b) => { + return orderMap[a.id] - orderMap[b.id]; + }); + + projects = orderedProjects; + } + const userId = connectedWalletUserId || user?.userId; if (projects.length > 0 && userId) { const userReactions = await findUserReactionsByProjectIds( From 3620351fcf947ae7729a4a6bb23267361e50459f Mon Sep 17 00:00:00 2001 From: Carlos Date: Tue, 3 Oct 2023 01:09:05 -0500 Subject: [PATCH 3/3] fix relationships for estimated matching filters --- src/entities/ProjectEstimatedMatchingView.ts | 17 +++++++++- src/entities/project.ts | 8 +++++ src/repositories/donationRepository.ts | 22 ------------- src/repositories/projectRepository.ts | 33 +++++++++++--------- src/resolvers/projectResolver.ts | 30 ------------------ 5 files changed, 43 insertions(+), 67 deletions(-) diff --git a/src/entities/ProjectEstimatedMatchingView.ts b/src/entities/ProjectEstimatedMatchingView.ts index 73b209c0c..518e70ce9 100644 --- a/src/entities/ProjectEstimatedMatchingView.ts +++ b/src/entities/ProjectEstimatedMatchingView.ts @@ -1,4 +1,4 @@ -import { ObjectType } from 'type-graphql'; +import { Field, ObjectType } from 'type-graphql'; import { Entity, Column, @@ -8,6 +8,8 @@ import { ViewEntity, ManyToOne, RelationId, + ViewColumn, + JoinColumn, } from 'typeorm'; import { Project } from './project'; @@ -26,26 +28,39 @@ import { Project } from './project'; ]) @ObjectType() export class ProjectEstimatedMatchingView extends BaseEntity { + @Field(type => Project) + @ManyToOne(type => Project, project => project.projectEstimatedMatchingView) + @JoinColumn({ referencedColumnName: 'id' }) + project: Project; + + @Field() + @ViewColumn() @PrimaryColumn() projectId: number; // QF Round ID associated with the donations + @ViewColumn() + @Field() @PrimaryColumn() qfRoundId: number; // Sum of the square root of the value in USD of the donations + @ViewColumn() @Column('double precision') sqrtRootSum: number; // Count of unique donations per user per project per QF round + @ViewColumn() @Column('int') uniqueDonationCount: number; // Sum of the value in USD of the donations for active QF rounds where the donation status is verified + @ViewColumn() @Column('double precision') sumValueUsd: number; // Count of unique donors who have verified donations for each project + @ViewColumn() @Column('int') uniqueDonorsCount: number; } diff --git a/src/entities/project.ts b/src/entities/project.ts index 018486f36..a39f4714a 100644 --- a/src/entities/project.ts +++ b/src/entities/project.ts @@ -52,6 +52,7 @@ import { } from '../repositories/qfRoundRepository'; import { EstimatedMatching } from '../types/qfTypes'; import { Campaign } from './campaign'; +import { ProjectEstimatedMatchingView } from './ProjectEstimatedMatchingView'; // tslint:disable-next-line:no-var-requires const moment = require('moment'); @@ -342,6 +343,13 @@ export class Project extends BaseEntity { @OneToMany(type => SocialProfile, socialProfile => socialProfile.project) socialProfiles?: SocialProfile[]; + @Field(type => [ProjectEstimatedMatchingView], { nullable: true }) + @OneToMany( + type => ProjectEstimatedMatchingView, + projectEstimatedMatchingView => projectEstimatedMatchingView.project, + ) + projectEstimatedMatchingView?: ProjectEstimatedMatchingView[]; + @Field(type => Float) @Column({ type: 'real' }) totalDonations: number; diff --git a/src/repositories/donationRepository.ts b/src/repositories/donationRepository.ts index b9ae10c63..9bf2825fc 100644 --- a/src/repositories/donationRepository.ts +++ b/src/repositories/donationRepository.ts @@ -354,28 +354,6 @@ export async function sumDonationValueUsdForQfRound(params: { return result[0] ? result[0].sumValueUsd : 0; } -export async function projectsInQfRoundOrderedBySumValueUsd( - qfRoundId: number, - limit: number, - skip: number, -): Promise { - const result = await AppDataSource.getDataSource().query( - ` - SELECT "projectId", "sumValueUsd" - FROM project_estimated_matching_view - WHERE "qfRoundId" = $1 - ORDER BY "sumValueUsd" DESC - LIMIT $2 - OFFSET $3; - `, - [qfRoundId, limit, skip], - ); - - return result?.length > 0 - ? result.map(project => project.projectId) - : undefined; -} - export async function countUniqueDonors(projectId: number): Promise { const result = await AppDataSource.getDataSource().query( ` diff --git a/src/repositories/projectRepository.ts b/src/repositories/projectRepository.ts index 3da2e8836..4f58c04bd 100644 --- a/src/repositories/projectRepository.ts +++ b/src/repositories/projectRepository.ts @@ -9,17 +9,10 @@ import { } from '../entities/project'; import { ProjectVerificationForm } from '../entities/projectVerificationForm'; import { ProjectAddress } from '../entities/projectAddress'; -import { - errorMessages, - i18n, - translationErrorMessagesKeys, -} from '../utils/errorMessages'; +import { i18n, translationErrorMessagesKeys } from '../utils/errorMessages'; import { User, publicSelectionFields } from '../entities/user'; import { ResourcesTotalPerMonthAndYear } from '../resolvers/donationResolver'; import { OrderDirection, ProjectResolver } from '../resolvers/projectResolver'; -import { ProjectEstimatedMatchingView } from '../entities/ProjectEstimatedMatchingView'; -import { findActiveQfRound } from './qfRoundRepository'; - export const findProjectById = (projectId: number): Promise => { // return Project.findOne({ id: projectId }); @@ -62,7 +55,6 @@ export type FilterProjectQueryInputParams = { sortingBy?: SortingField; qfRoundId?: number; activeQfRoundId?: number; - qfRoundProjectsIds?: number[]; }; export const filterProjectsQuery = (params: FilterProjectQueryInputParams) => { const { @@ -76,7 +68,6 @@ export const filterProjectsQuery = (params: FilterProjectQueryInputParams) => { slugArray, qfRoundId, activeQfRoundId, - qfRoundProjectsIds, } = params; let query = Project.createQueryBuilder('project') @@ -175,10 +166,24 @@ export const filterProjectsQuery = (params: FilterProjectQueryInputParams) => { ); break; case SortingField.ActiveQfRoundRaisedFunds: - if (activeQfRoundId && qfRoundProjectsIds) { - query.andWhere('project.id IN (:...qfRoundProjectsIds)', { - qfRoundProjectsIds, - }); + if (activeQfRoundId) { + query + .innerJoin( + 'project.projectEstimatedMatchingView', + 'projectEstimatedMatchingView', + ) + .addSelect([ + 'projectEstimatedMatchingView.sumValueUsd', + 'projectEstimatedMatchingView.qfRoundId', + ]) + .andWhere('projectEstimatedMatchingView.qfRoundId = :qfRoundId', { + qfRoundId: activeQfRoundId, + }) + .orderBy( + 'projectEstimatedMatchingView.sumValueUsd', + OrderDirection.DESC, + ) + .addOrderBy(`project.verified`, OrderDirection.DESC); } break; default: diff --git a/src/resolvers/projectResolver.ts b/src/resolvers/projectResolver.ts index 12d8fe296..fe847665f 100644 --- a/src/resolvers/projectResolver.ts +++ b/src/resolvers/projectResolver.ts @@ -109,7 +109,6 @@ import { PROJECT_UPDATE_CONTENT_MAX_LENGTH } from '../constants/validators'; import { calculateGivbackFactor } from '../services/givbackService'; import { ProjectBySlugResponse } from './types/projectResolver'; import { findActiveQfRound } from '../repositories/qfRoundRepository'; -import { projectsInQfRoundOrderedBySumValueUsd } from '../repositories/donationRepository'; @ObjectType() class AllProjects { @@ -762,18 +761,9 @@ export class ProjectResolver { let projects: Project[]; let totalCount: number; let activeQfRoundId: number | undefined; - let qfRoundProjectsIds: number[] | undefined; if (sortingBy === SortingField.ActiveQfRoundRaisedFunds) { activeQfRoundId = (await findActiveQfRound())?.id; - - if (activeQfRoundId) { - qfRoundProjectsIds = await projectsInQfRoundOrderedBySumValueUsd( - activeQfRoundId, - limit, - skip, - ); - } } const filterQueryParams: FilterProjectQueryInputParams = { @@ -786,7 +776,6 @@ export class ProjectResolver { sortingBy, qfRoundId, activeQfRoundId, - qfRoundProjectsIds, }; let campaign; if (campaignSlug) { @@ -815,25 +804,6 @@ export class ProjectResolver { .cache(projectsQueryCacheKey, projectFiltersCacheDuration) .getManyAndCount(); - // No join or custom subquery worked, solution is to write a full raw sql which is too much work - // Sorting will be required to be done in-memory - if ( - sortingBy === SortingField.ActiveQfRoundRaisedFunds && - activeQfRoundId && - qfRoundProjectsIds - ) { - const orderMap: { [key: string]: number } = {}; - qfRoundProjectsIds.forEach((id, index) => { - orderMap[id] = index; - }); - - const orderedProjects = projects.sort((a, b) => { - return orderMap[a.id] - orderMap[b.id]; - }); - - projects = orderedProjects; - } - const userId = connectedWalletUserId || user?.userId; if (projects.length > 0 && userId) { const userReactions = await findUserReactionsByProjectIds(