Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Export distribution versions #2014

Open
wants to merge 50 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
02f5d61
Export Distribution Only
Kbhat1 Dec 26, 2024
9e47cc5
Migrate distribution tool
Kbhat1 Dec 27, 2024
894b9cb
Check read
Kbhat1 Dec 27, 2024
f13a6d4
More logs
Kbhat1 Dec 27, 2024
d68dd2b
Timestamps
Kbhat1 Dec 27, 2024
b64e9c2
prefix db
Kbhat1 Dec 27, 2024
5dba22a
Fix call
Kbhat1 Dec 27, 2024
705b461
concurrent goroutines
Kbhat1 Dec 27, 2024
cd3bef5
Concurrent export
Kbhat1 Dec 30, 2024
e305148
Update to seidb migration
Kbhat1 Dec 31, 2024
8e51c44
Test
Kbhat1 Dec 31, 2024
a5e47ba
More dense logging
Kbhat1 Dec 31, 2024
36004b5
More dense
Kbhat1 Dec 31, 2024
0dd34fa
Update seidb
Kbhat1 Dec 31, 2024
929e575
More logging
Kbhat1 Dec 31, 2024
9faf21a
Remove log
Kbhat1 Dec 31, 2024
6000bd2
Every million log
Kbhat1 Dec 31, 2024
f47a2af
Update
Kbhat1 Dec 31, 2024
ad12d17
Correct home dir
Kbhat1 Dec 31, 2024
2ecf3e9
Update to write
Kbhat1 Dec 31, 2024
5db3504
Update whole distribution
Kbhat1 Jan 5, 2025
373d424
Verify distribution
Kbhat1 Jan 8, 2025
9c90cb3
Update to set on each mismatch
Kbhat1 Jan 8, 2025
94b196e
Only update specific versions
Kbhat1 Jan 9, 2025
ebe673b
new catchup
Kbhat1 Jan 9, 2025
4b99f79
Raw iterate versions
Kbhat1 Jan 9, 2025
60145e8
update version
Kbhat1 Jan 9, 2025
2d36b02
Update
Kbhat1 Jan 9, 2025
2cbe605
Update seidb
Kbhat1 Jan 9, 2025
e5e4508
more logging
Kbhat1 Jan 9, 2025
73bfe13
Bump seidb
Kbhat1 Jan 9, 2025
b0df739
Update
Kbhat1 Jan 9, 2025
6d0b6c5
Update
Kbhat1 Jan 9, 2025
c9c89da
Update log
Kbhat1 Jan 9, 2025
45c9bbb
Less logs
Kbhat1 Jan 9, 2025
8fbf0de
totalExported
Kbhat1 Jan 9, 2025
49f5e9e
remove unncessary increment
Kbhat1 Jan 9, 2025
80be213
tools
Kbhat1 Jan 10, 2025
ee265c0
Update sei-db
Kbhat1 Jan 10, 2025
19ca4a1
Update verification
Kbhat1 Jan 10, 2025
10df0f2
Remove logs
Kbhat1 Jan 10, 2025
05c6504
Panic
Kbhat1 Jan 10, 2025
fb04412
Skip more
Kbhat1 Jan 10, 2025
e7a8c7e
Sync mode
Kbhat1 Jan 10, 2025
98f2ba7
Update
Kbhat1 Jan 10, 2025
e4433d1
Verify once more
Kbhat1 Jan 12, 2025
d0a884c
Verification
Kbhat1 Jan 12, 2025
35b7421
Set
Kbhat1 Jan 13, 2025
bc56bc7
Update new counter
Kbhat1 Jan 13, 2025
87c2926
Verify
Kbhat1 Jan 13, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
prefix db
Kbhat1 committed Dec 27, 2024
commit b64e9c286692e5dbc594cb2f2f6e89562525c244
104 changes: 74 additions & 30 deletions tools/migration/ss/migrator.go
Original file line number Diff line number Diff line change
@@ -62,51 +62,95 @@ func exportDistributionLeafNodes(
db dbm.DB,
ch chan<- types.RawSnapshotNode,
startVersion, endVersion int64,
numGoroutines int,
) error {
fmt.Printf("Starting export at time: %s\n", time.Now().Format(time.RFC3339))

// Total counters for logging.
totalExported := 0
startTime := time.Now()
Fixed Show fixed Hide fixed

Check warning

Code scanning / CodeQL

Calling the system time Warning

Calling the system time may be a possible source of non-determinism

for ver := startVersion; ver <= endVersion; ver++ {
// Load only the distribution module prefix at this version.
tree, err := ReadTree(db, ver, []byte(utils.BuildTreePrefix("distribution")))
// for ver := startVersion; ver <= endVersion; ver++ {
// // Load only the distribution module prefix at this version.
// tree, err := ReadTree(db, ver, []byte(utils.BuildTreePrefix("distribution")))
// if err != nil {
// fmt.Printf("[%s] Error loading distribution tree at version %d: %s\n", time.Now().Format(time.RFC3339), ver, err.Error())
// return err
// }

// var count int
// _, err = tree.Iterate(func(key, value []byte) bool {
// // Each leaf node is a single K/V for this version
// // ch <- types.RawSnapshotNode{
// // StoreKey: "distribution",
// // Key: key,
// // Value: value,
// // Version: ver,
// // }
// count++
// totalExported++
// if count%1000000 == 0 {
// fmt.Printf("[%s] Exported %d distribution keys at version %d so far\n", time.Now().Format(time.RFC3339), count, ver)
// // Optionally add metrics here if desired.
// metrics.IncrCounterWithLabels([]string{"sei", "migration", "leaf_nodes_exported"}, float32(count), []metrics.Label{
// {Name: "module", Value: "distribution"},
// })
// }
// return false // continue iteration
// })
// if err != nil {
// fmt.Printf("[%s] Error iterating distribution tree for version %d: %s\n", time.Now().Format(time.RFC3339), ver, err.Error())
// return err
// }
// fmt.Printf("[%s] Finished version %d: exported %d distribution keys.\n", time.Now().Format(time.RFC3339), ver, count)
// }

prefixDB := dbm.NewPrefixDB(db, []byte(utils.BuildRawPrefix("distribution")))
var itr dbm.Iterator
var err error
leafNodeCount := 0
count := 0

// If there is a starting key, seek to it, otherwise start from the beginning
itr, err = prefixDB.Iterator(nil, nil)

if err != nil {
fmt.Printf("SeiDB Archive Migration: Error creating iterator: %+v\n", err)
return fmt.Errorf("failed to create iterator: %w", err)
}
defer itr.Close()

for ; itr.Valid(); itr.Next() {
value := bytes.Clone(itr.Value())

node, err := iavl.MakeNode(value)
if err != nil {
fmt.Printf("[%s] Error loading distribution tree at version %d: %s\n", time.Now().Format(time.RFC3339), ver, err.Error())
return err
fmt.Printf("SeiDB Archive Migration: Failed to make node: %+v\n", err)
return fmt.Errorf("failed to make node: %w", err)
}

var count int
_, err = tree.Iterate(func(key, value []byte) bool {
// Each leaf node is a single K/V for this version
// ch <- types.RawSnapshotNode{
// StoreKey: "distribution",
// Key: key,
// Value: value,
// Version: ver,
// }
count++
totalExported++
if count%1000000 == 0 {
fmt.Printf("[%s] Exported %d distribution keys at version %d so far\n", time.Now().Format(time.RFC3339), count, ver)
// Optionally add metrics here if desired.
metrics.IncrCounterWithLabels([]string{"sei", "migration", "leaf_nodes_exported"}, float32(count), []metrics.Label{
{Name: "module", Value: "distribution"},
})
// Only export leaf nodes
if node.GetHeight() == 0 {
version := node.GetVersion()
if version < startVersion || version > endVersion {
continue
}
return false // continue iteration
})
if err != nil {
fmt.Printf("[%s] Error iterating distribution tree for version %d: %s\n", time.Now().Format(time.RFC3339), ver, err.Error())
return err
leafNodeCount++
}

count++
if count%1000000 == 0 {
fmt.Printf("[%s] SeiDB Archive Migration: Last 1,000,000 iterations. Total scanned: %d, leaf nodes exported: %d\n", time.Now().Format(time.RFC3339), count, leafNodeCount)
}
fmt.Printf("[%s] Finished version %d: exported %d distribution keys.\n", time.Now().Format(time.RFC3339), ver, count)
}

if err := itr.Error(); err != nil {
fmt.Printf("Iterator error: %+v\n", err)
return fmt.Errorf("iterator error: %w", err)
}

fmt.Printf(
"[%s] Completed exporting distribution module from %d to %d. Total keys: %d. Duration: %s\n",
time.Now().Format(time.RFC3339), startVersion, endVersion, totalExported, time.Since(startTime),
time.Now().Format(time.RFC3339), startVersion, endVersion, leafNodeCount, time.Since(startTime),
)
fmt.Printf("Finished export at time: %s\n", time.Now().Format(time.RFC3339))
Fixed Show fixed Hide fixed
return nil