Warpsync: epoch syncing

Makefile

@@ -20,11 +20,10 @@ proto: $(PROTO_GO_FILES)
 # If $GOPATH/bin/protoc-gen-go does not exist, we'll run this command to install it.
 $(PROTOC_GEN_GO):
 	go install google.golang.org/protobuf/cmd/protoc-gen-go
-	go install google.golang.org/grpc/cmd/protoc-gen-go-grpc
 
 # Implicit compile rule for GRPC/proto files
 %.pb.go: %.proto | $(PROTOC_GEN_GO)
-	protoc $< --go_out=paths=source_relative:. --go-grpc_out=paths=source_relative:.
+	protoc $< --go_out=paths=source_relative:.
 
 .PHONY: clean_proto
 clean_proto:

packages/core/consensus/acceptance/gadget.go

@@ -26,7 +26,7 @@ const (
 
 var (
 	// DefaultConflictTranslation is the default function to translate the approval weight to confirmation.State of a conflict.
-	DefaultConflictTranslation ConflictThresholdTranslation = func(conflictID utxo.TransactionID, aw float64) confirmation.State {
+	DefaultConflictTranslation ConflictThresholdTranslation = func(_ utxo.TransactionID, aw float64) confirmation.State {
 		if aw >= acceptanceThreshold {
 			return confirmation.Accepted
 		}

packages/core/epoch/types.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/iotaledger/hive.go/core/byteutils"
 	"github.com/iotaledger/hive.go/core/generics/model"
 	"github.com/iotaledger/hive.go/core/serix"
 	"github.com/mr-tron/base58"
@@ -78,7 +79,7 @@ type (
 	EC  = MerkleRoot
 )
 
-func NewMerkleRoot(bytes []byte) (mr MerkleRoot) {
+func NewMerkleRoot(bytes []byte) MerkleRoot {
 	b := [blake2b.Size256]byte{}
 	copy(b[:], bytes[:])
 	return b
@@ -92,15 +93,24 @@ func (m MerkleRoot) Bytes() []byte {
 	return m[:]
 }
 
+// CommitmentRoots contains roots of trees of an epoch.
+type CommitmentRoots struct {
+	TangleRoot        MerkleRoot `serix:"0"`
+	StateMutationRoot MerkleRoot `serix:"1"`
+	StateRoot         MerkleRoot `serix:"2"`
+	ManaRoot          MerkleRoot `serix:"3"`
+}
+
 // ECRecord is a storable object represents the ecRecord of an epoch.
 type ECRecord struct {
 	model.Storable[Index, ECRecord, *ECRecord, ecRecord] `serix:"0"`
 }
 
 type ecRecord struct {
-	EI     Index `serix:"0"`
-	ECR    ECR   `serix:"1"`
-	PrevEC EC    `serix:"2"`
+	EI     Index            `serix:"0"`
+	ECR    ECR              `serix:"1"`
+	PrevEC EC               `serix:"2"`
+	Roots  *CommitmentRoots `serix:"3"`
 }
 
 // NewECRecord creates and returns a ECRecord of the given EI.
@@ -109,6 +119,7 @@ func NewECRecord(ei Index) (new *ECRecord) {
 		EI:     ei,
 		ECR:    MerkleRoot{},
 		PrevEC: MerkleRoot{},
+		Roots:  &CommitmentRoots{},
 	})
 	new.SetID(ei)
 	return
@@ -177,6 +188,43 @@ func (e *ECRecord) FromBytes(bytes []byte) (err error) {
 	return
 }
 
+// Roots returns the CommitmentRoots of an ECRecord.
+func (e *ECRecord) Roots() *CommitmentRoots {
+	e.RLock()
+	defer e.RUnlock()
+
+	return e.M.Roots
+}
+
+// SetRoots sets the CommitmentRoots of an ECRecord.
+func (e *ECRecord) SetRoots(roots *CommitmentRoots) {
+	e.Lock()
+	defer e.Unlock()
+
+	e.M.Roots = roots
+	e.SetModified()
+}
+
+// ComputeEC calculates the epoch commitment hash from the given ECRecord.
+func (e *ECRecord) ComputeEC() (ec EC) {
+	ecHash := blake2b.Sum256(byteutils.ConcatBytes(e.EI().Bytes(), e.ECR().Bytes(), e.PrevEC().Bytes()))
+
+	return NewMerkleRoot(ecHash[:])
+}
+
+// region hashing functions ////////////////////////////////////////////////////////////////////////////////////////////
+
+// ComputeECR calculates an ECR from the tree roots.
+func ComputeECR(tangleRoot, stateMutationRoot, stateRoot, manaRoot MerkleRoot) ECR {
+	branch1Hashed := blake2b.Sum256(byteutils.ConcatBytes(tangleRoot.Bytes(), stateMutationRoot.Bytes()))
+	branch2Hashed := blake2b.Sum256(byteutils.ConcatBytes(stateRoot.Bytes(), manaRoot.Bytes()))
+	rootHashed := blake2b.Sum256(byteutils.ConcatBytes(branch1Hashed[:], branch2Hashed[:]))
+
+	return NewMerkleRoot(rootHashed[:])
+}
+
+// endregion ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+
 // region NodesActivityLog //////////////////////////////////////////////////////////////////////////////////////////////////
 
 type NodesActivityLog map[Index]*ActivityLog

packages/core/ledger/models.go

@@ -612,6 +612,7 @@ func NewOutputWithMetadata(outputID utxo.OutputID, output utxo.Output, creationT
 	new = model.NewStorable[utxo.OutputID, OutputWithMetadata](&outputWithMetadataModel{
 		OutputID:              outputID,
 		Output:                output,
+		CreationTime:          creationTime,
 		ConsensusManaPledgeID: consensusManaPledgeID,
 		AccessManaPledgeID:    accessManaPledgeID,
 	})

packages/core/notarization/commitments.go

@@ -2,9 +2,9 @@ package notarization
 
 import (
 	"context"
+
 	"github.com/celestiaorg/smt"
 	"github.com/cockroachdb/errors"
-	"github.com/iotaledger/hive.go/core/byteutils"
 	"github.com/iotaledger/hive.go/core/identity"
 	"github.com/iotaledger/hive.go/core/serix"
 
@@ -93,20 +93,14 @@ func (f *EpochCommitmentFactory) ManaRoot() []byte {
 	return f.manaRootTree.Root()
 }
 
-// ECR retrieves the epoch commitment root.
-func (f *EpochCommitmentFactory) ECR(ei epoch.Index) (ecr epoch.ECR, err error) {
-	epochRoots, err := f.newEpochRoots(ei)
+// ECRandRoots retrieves the epoch commitment root.
+func (f *EpochCommitmentFactory) ECRandRoots(ei epoch.Index) (ecr epoch.ECR, roots *epoch.CommitmentRoots, err error) {
+	roots, err = f.newEpochRoots(ei)
 	if err != nil {
-		return epoch.MerkleRoot{}, errors.Wrap(err, "ECR could not be created")
+		return epoch.MerkleRoot{}, nil, errors.Wrap(err, "ECR could not be created")
 	}
 
-	branch1aHashed := blake2b.Sum256(byteutils.ConcatBytes(epochRoots.tangleRoot[:], epochRoots.stateMutationRoot[:]))
-	branch1bHashed := blake2b.Sum256(byteutils.ConcatBytes(epochRoots.stateRoot[:], epochRoots.manaRoot[:]))
-	branch1Hashed := blake2b.Sum256(byteutils.ConcatBytes(branch1aHashed[:], branch1bHashed[:]))
-	branch2Hashed := blake2b.Sum256(epochRoots.activityRoot[:])
-	rootHashed := blake2b.Sum256(byteutils.ConcatBytes(branch1Hashed[:], branch2Hashed[:]))
-
-	return epoch.NewMerkleRoot(rootHashed[:]), nil
+	return epoch.ComputeECR(roots.TangleRoot, roots.StateMutationRoot, roots.StateRoot, roots.ManaRoot), roots, nil
 }
 
 // removeStateLeaf removes the output ID from the ledger sparse merkle tree.
@@ -218,7 +212,7 @@ func (f *EpochCommitmentFactory) ecRecord(ei epoch.Index) (ecRecord *epoch.ECRec
 		return ecRecord, nil
 	}
 	// We never committed this epoch before, create and roll to a new epoch.
-	ecr, ecrErr := f.ECR(ei)
+	ecr, roots, ecrErr := f.ECRandRoots(ei)
 	if ecrErr != nil {
 		return nil, ecrErr
 	}
@@ -230,7 +224,8 @@ func (f *EpochCommitmentFactory) ecRecord(ei epoch.Index) (ecRecord *epoch.ECRec
 	// Store and return.
 	f.storage.CachedECRecord(ei, epoch.NewECRecord).Consume(func(e *epoch.ECRecord) {
 		e.SetECR(ecr)
-		e.SetPrevEC(EC(prevECRecord))
+		e.SetRoots(roots)
+		e.SetPrevEC(prevECRecord.ComputeEC())
 		ecRecord = e
 	})
 
@@ -241,6 +236,7 @@ func (f *EpochCommitmentFactory) loadECRecord(ei epoch.Index) (ecRecord *epoch.E
 	f.storage.CachedECRecord(ei).Consume(func(record *epoch.ECRecord) {
 		ecRecord = epoch.NewECRecord(ei)
 		ecRecord.SetECR(record.ECR())
+		ecRecord.SetRoots(record.Roots())
 		ecRecord.SetPrevEC(record.PrevEC())
 	})
 	return
@@ -334,7 +330,7 @@ func (f *EpochCommitmentFactory) newCommitmentTrees(ei epoch.Index) *CommitmentT
 }
 
 // newEpochRoots creates a new commitment with the given ei, by advancing the corresponding data structures.
-func (f *EpochCommitmentFactory) newEpochRoots(ei epoch.Index) (commitmentRoots *CommitmentRoots, commitmentTreesErr error) {
+func (f *EpochCommitmentFactory) newEpochRoots(ei epoch.Index) (commitmentRoots *epoch.CommitmentRoots, commitmentTreesErr error) {
 	// TODO: what if a node restarts and we have incomplete trees?
 	commitmentTrees, commitmentTreesErr := f.getCommitmentTrees(ei)
 	if commitmentTreesErr != nil {
@@ -350,12 +346,11 @@ func (f *EpochCommitmentFactory) newEpochRoots(ei epoch.Index) (commitmentRoots
 	// We advance the LedgerState to the next epoch.
 	f.commitLedgerState(ei - epoch.Index(f.snapshotDepth))
 
-	commitmentRoots = &CommitmentRoots{
-		EI:                ei,
-		stateRoot:         epoch.NewMerkleRoot(stateRoot),
-		manaRoot:          epoch.NewMerkleRoot(manaRoot),
-		tangleRoot:        epoch.NewMerkleRoot(commitmentTrees.tangleTree.Root()),
-		stateMutationRoot: epoch.NewMerkleRoot(commitmentTrees.stateMutationTree.Root()),
+	commitmentRoots = &epoch.CommitmentRoots{
+		StateRoot:         epoch.NewMerkleRoot(stateRoot),
+		ManaRoot:          epoch.NewMerkleRoot(manaRoot),
+		TangleRoot:        epoch.NewMerkleRoot(commitmentTrees.tangleTree.Root()),
+		StateMutationRoot: epoch.NewMerkleRoot(commitmentTrees.stateMutationTree.Root()),
 	}
 
 	// We are never going to use this epoch's commitment trees again.
@@ -431,18 +426,6 @@ func (f *EpochCommitmentFactory) newStateRoots(ei epoch.Index) (stateRoot []byte
 
 // region extra functions //////////////////////////////////////////////////////////////////////////////////////////////
 
-// EC calculates the epoch commitment hash from the given ECRecord.
-func EC(ecRecord *epoch.ECRecord) (ec epoch.EC) {
-	concatenated := make([]byte, 0)
-	concatenated = append(concatenated, ecRecord.EI().Bytes()...)
-	concatenated = append(concatenated, ecRecord.ECR().Bytes()...)
-	concatenated = append(concatenated, ecRecord.PrevEC().Bytes()...)
-
-	ecHash := blake2b.Sum256(concatenated)
-
-	return epoch.NewMerkleRoot(ecHash[:])
-}
-
 // insertLeaf inserts the outputID to the provided sparse merkle tree.
 func insertLeaf(tree *smt.SparseMerkleTree, keyBytes, valueBytes []byte) error {
 	_, err := tree.Update(keyBytes, valueBytes)

packages/core/notarization/events.go

@@ -0,0 +1,125 @@
+package notarization
+
+import (
+	"github.com/iotaledger/goshimmer/packages/core/epoch"
+	"github.com/iotaledger/goshimmer/packages/core/ledger"
+	"github.com/iotaledger/goshimmer/packages/core/ledger/utxo"
+	"github.com/iotaledger/goshimmer/packages/core/tangleold"
+	"github.com/iotaledger/hive.go/core/generics/event"
+	"github.com/iotaledger/hive.go/core/identity"
+)
+
+// region Events ///////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// Events is a container that acts as a dictionary for the existing events of a notarization manager.
+type Events struct {
+	// EpochCommittable is an event that gets triggered whenever an epoch commitment is committable.
+	EpochCommittable *event.Event[*EpochCommittableEvent]
+	// EpochConfirmed is an event that gets triggered whenever an epoch is confirmed.
+	EpochConfirmed *event.Event[*EpochConfirmedEvent]
+	// CompetingCommitmentDetected is an event that gets triggered whenever a competing epoch commitment is detected.
+	CompetingCommitmentDetected *event.Event[*CompetingCommitmentDetectedEvent]
+	// ManaVectorUpdate is an event that gets triggered whenever the consensus mana vector needs to be updated.
+	ManaVectorUpdate *event.Event[*ManaVectorUpdateEvent]
+	// TangleTreeInserted is an event that gets triggered when a Block is inserted into the Tangle smt.
+	TangleTreeInserted *event.Event[*TangleTreeUpdatedEvent]
+	// TangleTreeRemoved is an event that gets triggered when a Block is removed from Tangle smt.
+	TangleTreeRemoved *event.Event[*TangleTreeUpdatedEvent]
+	// StateMutationTreeInserted is an event that gets triggered when a transaction is inserted into the state mutation smt.
+	StateMutationTreeInserted *event.Event[*StateMutationTreeUpdatedEvent]
+	// StateMutationTreeRemoved is an event that gets triggered when a transaction is removed from state mutation smt.
+	StateMutationTreeRemoved *event.Event[*StateMutationTreeUpdatedEvent]
+	// UTXOTreeInserted is an event that gets triggered when UTXOs are stored into the UTXO smt.
+	UTXOTreeInserted *event.Event[*UTXOUpdatedEvent]
+	// UTXOTreeRemoved is an event that gets triggered when UTXOs are removed from the UTXO smt.
+	UTXOTreeRemoved *event.Event[*UTXOUpdatedEvent]
+	// Bootstrapped is an event that gets triggered when a notarization manager has the last committable epoch relatively close to current epoch.
+	Bootstrapped *event.Event[*BootstrappedEvent]
+	// SyncRange is an event that gets triggered when an entire range of epochs needs to be requested, validated and solidified
+	SyncRange *event.Event[*SyncRangeEvent]
+	// ActivityTreeInserted is an event that gets triggered when nodeID is added to the activity tree.
+        ActivityTreeInserted *event.Event[*ActivityTreeUpdatedEvent]
+        // ActivityTreeRemoved is an event that gets triggered when nodeID is removed from activity tree.
+        ActivityTreeRemoved *event.Event[*ActivityTreeUpdatedEvent]
+
+}
+
+// TangleTreeUpdatedEvent is a container that acts as a dictionary for the TangleTree inserted/removed event related parameters.
+type TangleTreeUpdatedEvent struct {
+	// EI is the index of the block.
+	EI epoch.Index
+	// BlockID is the blockID that inserted/removed to/from the tangle smt.
+	BlockID tangleold.BlockID
+}
+
+// BootstrappedEvent is an event that gets triggered when a notarization manager has the last committable epoch relatively close to current epoch.
+type BootstrappedEvent struct {
+	// EI is the index of the last commitable epoch
+	EI epoch.Index
+}
+
+// StateMutationTreeUpdatedEvent is a container that acts as a dictionary for the State mutation tree inserted/removed event related parameters.
+type StateMutationTreeUpdatedEvent struct {
+	// EI is the index of the transaction.
+	EI epoch.Index
+	// TransactionID is the transaction ID that inserted/removed to/from the state mutation smt.
+	TransactionID utxo.TransactionID
+}
+
+// UTXOUpdatedEvent is a container that acts as a dictionary for the UTXO update event related parameters.
+type UTXOUpdatedEvent struct {
+	// EI is the index of updated UTXO.
+	EI epoch.Index
+	// Created are the outputs created in a transaction.
+	Created []*ledger.OutputWithMetadata
+	// Spent are outputs that is spent in a transaction.
+	Spent []*ledger.OutputWithMetadata
+}
+
+// EpochCommittableEvent is a container that acts as a dictionary for the EpochCommittable event related parameters.
+type EpochCommittableEvent struct {
+	// EI is the index of committable epoch.
+	EI epoch.Index
+	// ECRecord is the ec root of committable epoch.
+	ECRecord *epoch.ECRecord
+}
+
+// EpochConfirmedEvent is a container that acts as a dictionary for the EpochConfirmed event related parameters.
+type EpochConfirmedEvent struct {
+	// EI is the index of committable epoch.
+	EI epoch.Index
+}
+
+// CompetingCommitmentDetectedEvent is a container that acts as a dictionary for the CompetingCommitmentDetectedEvent event related parameters.
+type CompetingCommitmentDetectedEvent struct {
+	// Block is the block that contains the competing commitment.
+	Block *tangleold.Block
+}
+
+// ManaVectorUpdateEvent is a container that acts as a dictionary for the EpochCommittable event related parameters.
+type ManaVectorUpdateEvent struct {
+	// EI is the index of committable epoch.
+	EI epoch.Index
+	// EpochDiffCreated is the list of outputs created in the epoch.
+	EpochDiffCreated []*ledger.OutputWithMetadata
+	// EpochDiffSpent is the list of outputs spent in the epoch.
+	EpochDiffSpent []*ledger.OutputWithMetadata
+}
+
+// SyncRangeEvent is a container that acts as a dictionary for the SyncRange event related parameters.
+type SyncRangeEvent struct {
+	StartEI   epoch.Index
+	EndEI     epoch.Index
+	StartEC   epoch.EC
+	EndPrevEC epoch.EC
+}
+
+// ActivityTreeUpdatedEvent is a container that acts as a dictionary for the ActivityTree inserted/removed event related parameters.
+type ActivityTreeUpdatedEvent struct {
+        // EI is the index of the epoch.
+        EI epoch.Index
+        // NodeID is the issuer nodeID.
+        NodeID identity.ID
+}
+
+// endregion ///////////////////////////////////////////////////////////////////////////////////////////////////////////