Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Changed project sync delay time #202

Merged
merged 2 commits into from
Mar 22, 2024
Merged

Changed project sync delay time #202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3925338
Changed project sync delay time
DavidGershony Mar 22, 2024
be5cbd4
Added block for syncing project and investment until after IBD
DavidGershony Mar 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 101 additions & 65 deletions src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,74 +25,24 @@ public ProjectInvestmentsSyncRunner(IOptions<IndexerSettings> configuration, IL
Delay = TimeSpan.FromMinutes(1);
}

private bool CanRunInvestmentsSync()
{
return !( //sync with other runners
!Runner.GlobalState.IndexModeCompleted ||
Runner.GlobalState.Blocked ||
Runner.GlobalState.ReorgMode ||
Runner.GlobalState.StoreTip == null ||
Runner.GlobalState.IndexMode);
}

public override async Task<bool> OnExecute()
{
var investmentsInProjectOutputs = await angorMongoDb.ProjectTable.Aggregate(PipelineDefinition<Project, BsonDocument>.Create(
new[]
{
new BsonDocument("$lookup",
new BsonDocument
{
{ "from", "Investment" },
{ "let", new BsonDocument("angorProjectId", "_id") },
{
"pipeline", new BsonArray
{
new BsonDocument("$match",
new BsonDocument("$expr",
new BsonDocument("$eq",
new BsonArray { "$AngorKey", "$$angorProjectId" }))),
new BsonDocument("$group",
new BsonDocument
{
{ "_id", "$AngorKey" },
{ "projectMaxBlockScanned", new BsonDocument("$max", "$BlockIndex") }
}),
new BsonDocument("$project",
new BsonDocument("projectMaxBlockScanned", 1))
}
},
{ "as", "joinedData" }
}),
new BsonDocument("$unwind",
new BsonDocument { { "path", "$joinedData" }, { "preserveNullAndEmptyArrays", true } }),
new BsonDocument("$project",
new BsonDocument
{
{ "AddressOnFeeOutput", 1 }, { "TransactionId", 1 }, { "projectMaxBlockScanned", new BsonDocument("$ifNull", new BsonArray { "$joinedData.projectMaxBlockScanned", 0 }) }
}),
new BsonDocument("$lookup",
new BsonDocument
{
{ "from", "Output" },
{ "let", new BsonDocument{{"address" , "$AddressOnFeeOutput" },{"trx", "$TransactionId"},{"projectMaxBlockScanned", "$projectMaxBlockScanned"}}},
{ "pipeline", new BsonArray
{
new BsonDocument("$match",
new BsonDocument("$expr",
new BsonDocument("$eq",
new BsonArray { "$Address", "$$address" }))),
new BsonDocument("$match",
new BsonDocument("$expr",
new BsonDocument("$and", new BsonArray
{
new BsonDocument("$gt",
new BsonArray { "$BlockIndex","$$projectMaxBlockScanned"}),
new BsonDocument("$ne",
new BsonArray { "$$trx","$Outpoint.TransactionId"})
})))
} },
{ "as", "o" }
}),
new BsonDocument("$unwind", "$o"),
new BsonDocument("$project",
new BsonDocument
{
{ "OutputTransactionId", "$o.Outpoint.TransactionId" },
{ "OutputBlockIndex", "$o.BlockIndex" }
})
}))
if (!CanRunInvestmentsSync())
return false;

var investmentsInProjectOutputs = await angorMongoDb.ProjectTable
.Aggregate(PipelineDefinition<Project, BsonDocument>
.Create(MongoDbLookupForInvestments()))
.ToListAsync();

var investmentTasks = investmentsInProjectOutputs.Select(ValidateAndCreateInvestmentAsync).ToList();
Expand Down Expand Up @@ -166,4 +116,90 @@ public override async Task<bool> OnExecute()
StageOutpoint = stages.Select(x => x.Outpoint).ToList()
};
}

private BsonDocument[] MongoDbLookupForInvestments()
{
return new[]
{
//Left join to investment table on Angor key get max block index to only look an new blocks
new BsonDocument("$lookup",
new BsonDocument
{
{ "from", "Investment" },
{ "let", new BsonDocument("angorProjectId", "_id") },
{
"pipeline", new BsonArray
{
new BsonDocument("$match",
new BsonDocument("$expr",
new BsonDocument("$eq",
new BsonArray { "$AngorKey", "$$angorProjectId" }))),
new BsonDocument("$group",
new BsonDocument
{
{ "_id", "$AngorKey" },
{ "projectMaxBlockScanned", new BsonDocument("$max", "$BlockIndex") }
}),
new BsonDocument("$project",
new BsonDocument("projectMaxBlockScanned", 1))
}
},
{ "as", "joinedData" }
}),
new BsonDocument("$unwind",
new BsonDocument { { "path", "$joinedData" }, { "preserveNullAndEmptyArrays", true } }),
//Only take address transaction id and max block
new BsonDocument("$project",
new BsonDocument
{
{ "AddressOnFeeOutput", 1 },
{ "TransactionId", 1 },
{
"projectMaxBlockScanned",
new BsonDocument("$ifNull", new BsonArray { "$joinedData.projectMaxBlockScanned", 0 })
}
}),
//Inner join with output on the indexed address and greater than block index and filter by trensaction id
new BsonDocument("$lookup",
new BsonDocument
{
{ "from", "Output" },
{
"let",
new BsonDocument
{
{ "address", "$AddressOnFeeOutput" },
{ "trx", "$TransactionId" },
{ "projectMaxBlockScanned", "$projectMaxBlockScanned" }
}
},
{
"pipeline", new BsonArray
{
new BsonDocument("$match",
new BsonDocument("$expr",
new BsonDocument("$eq",
new BsonArray { "$Address", "$$address" }))),
new BsonDocument("$match",
new BsonDocument("$expr",
new BsonDocument("$and", new BsonArray
{
new BsonDocument("$gt",
new BsonArray { "$BlockIndex", "$$projectMaxBlockScanned" }),
new BsonDocument("$ne",
new BsonArray { "$$trx", "$Outpoint.TransactionId" })
})))
}
},
{ "as", "o" }
}),
new BsonDocument("$unwind", "$o"),
// Only take fields needed for performance
new BsonDocument("$project",
new BsonDocument
{
{ "OutputTransactionId", "$o.Outpoint.TransactionId" }, { "OutputBlockIndex", "$o.BlockIndex" }
})
};
}
}
15 changes: 15 additions & 0 deletions src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,26 @@ public ProjectsSyncRunner(IOptions<IndexerSettings> configuration, ILogger<Proje
AngorMongoDb = angorMongoDb;
this.syncConnection = syncConnection;

Delay = TimeSpan.FromMinutes(1);

extendedPublicKey = new BitcoinExtPubKey(AngorTestKey, new BitcoinSignet()).ExtPubKey;
}

private bool CanRunProjectSync()
{
return !( //sync with other runners
!Runner.GlobalState.IndexModeCompleted ||
Runner.GlobalState.Blocked ||
Runner.GlobalState.ReorgMode ||
Runner.GlobalState.StoreTip == null ||
Runner.GlobalState.IndexMode);
}

public override async Task<bool> OnExecute()
{
if (!CanRunProjectSync())
return false;

var blockIndexed = await AngorMongoDb.ProjectTable.EstimatedDocumentCountAsync() > 0
? AngorMongoDb.ProjectTable.AsQueryable().Max(p => p.BlockIndex)
: 0;
Expand Down
Loading