Merge pull request #2414 from iotaledger/develop

CHANGELOG.md

@@ -1,3 +1,11 @@
+# v0.9.6 - 2022-09-01
+
+> This release introduces major bugfixes to epoch notarization and networking.
+
+- Fix: several bugs in Notarization and Networking (#2412)
+- Wrap faucet errors and add faucet logging (#2411)
+- Fix concurrent map writes in integration network shutdown (#2410)
+
 # v0.9.5 - 2022-08-31
 
 > This release introduces a warpsync plugin for fast epochs retrieval, a simplified faucet, local snapshot improvements, and network and general bug fixes.

deploy/ansible/roles/goshimmer-node/templates/docker-compose-analysis-server.yml.j2

@@ -29,6 +29,6 @@ services:
       --metrics.local=false
       --metrics.global=true
       --node.enablePlugins=analysisServer,analysisDashboard,prometheus
-      --node.disablePlugins=activity,analysisClient,chat,consensus,dashboard,faucet,gossip,firewall,issuer,mana,manualpeering,blockLayer,metrics,networkdelay,portcheck,pow,syncBeaconFollower,webAPIBroadcastDataEndpoint,WebAPIDataEndpoint,WebAPIHealthzEndpoint,WebAPIFaucetRequestEndpoint,webAPIFindTransactionHashesEndpoint,webAPIGetNeighborsEndpoint,webAPIGetTransactionObjectsByHashEndpoint,webAPIGetTransactionTrytesByHashEndpoint,WebAPIInfoEndpoint,WebAPILedgerstateEndpoint,WebAPIBlockEndpoint,WebAPIToolsBlockEndpoint,WebAPIWeightProviderEndpoint,remotelog,remotelogmetrics,DAGsVisualizer,WebAPIRateSetterEndpoint,WebAPISchedulerEndpoint,ManaInitializer,Notarization,EpochStorage,WebAPIEpochEndpoint,BootstrapManager
+      --node.disablePlugins=activity,analysisClient,chat,consensus,dashboard,faucet,gossip,firewall,issuer,mana,manualpeering,blockLayer,metrics,networkdelay,portcheck,pow,syncBeaconFollower,webAPIBroadcastDataEndpoint,WebAPIDataEndpoint,WebAPIHealthzEndpoint,WebAPIFaucetRequestEndpoint,webAPIFindTransactionHashesEndpoint,webAPIGetNeighborsEndpoint,webAPIGetTransactionObjectsByHashEndpoint,webAPIGetTransactionTrytesByHashEndpoint,WebAPIInfoEndpoint,WebAPILedgerstateEndpoint,WebAPIBlockEndpoint,WebAPIToolsBlockEndpoint,WebAPIWeightProviderEndpoint,remotelog,remotelogmetrics,DAGsVisualizer,WebAPIRateSetterEndpoint,WebAPISchedulerEndpoint,ManaInitializer,Notarization,EpochStorage,WebAPIEpochEndpoint,BootstrapManager,Warpsync,Snapshot
       --logger.level={{ logLevel }}
       --logger.outputPaths=stdout

go.mod

@@ -36,6 +36,7 @@ require (
 	go.uber.org/atomic v1.9.0
 	go.uber.org/dig v1.15.0
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/src-d/go-git.v4 v4.13.1
 )
@@ -184,7 +185,6 @@ require (
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.22.0 // indirect
 	golang.org/x/net v0.0.0-20220809012201-f428fae20770 // indirect
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
 	golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect

packages/core/epoch/types.go

@@ -19,7 +19,7 @@ import (
 
 var (
 	// GenesisTime is the time (Unix in seconds) of the genesis.
-	GenesisTime int64 = 1661859573
+	GenesisTime int64 = 1662035280
 	// Duration is the default epoch duration in seconds.
 	Duration int64 = 10
 )

packages/core/mana/events.go

@@ -9,6 +9,8 @@ import (
 	"github.com/iotaledger/hive.go/core/stringify"
 	"github.com/mr-tron/base58"
 
+	"github.com/iotaledger/goshimmer/packages/core/epoch"
+	"github.com/iotaledger/goshimmer/packages/core/ledger"
 	"github.com/iotaledger/goshimmer/packages/core/ledger/utxo"
 )
 
@@ -58,6 +60,16 @@ type Event interface {
 	String() string
 }
 
+// ManaVectorUpdateEvent is a container that acts as a dictionary for the EpochCommittable event related parameters.
+type ManaVectorUpdateEvent struct {
+	// EI is the index of committable epoch.
+	EI epoch.Index
+	// Spent are outputs that is spent in a transaction.
+	Spent []*ledger.OutputWithMetadata
+	// Created are the outputs created in a transaction.
+	Created []*ledger.OutputWithMetadata
+}
+
 // PledgedEvent is the struct that is passed along with triggering a Pledged event.
 type PledgedEvent struct {
 	NodeID        identity.ID

packages/core/notarization/commitments.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/iotaledger/hive.go/core/generics/lo"
 	"github.com/iotaledger/hive.go/core/generics/objectstorage"
+	"github.com/iotaledger/hive.go/core/generics/shrinkingmap"
 	"github.com/iotaledger/hive.go/core/kvstore"
 	"golang.org/x/crypto/blake2b"
 
@@ -49,7 +50,7 @@ type CommitmentTrees struct {
 
 // EpochCommitmentFactory manages epoch commitmentTrees.
 type EpochCommitmentFactory struct {
-	commitmentTrees map[epoch.Index]*CommitmentTrees
+	commitmentTrees *shrinkingmap.ShrinkingMap[epoch.Index, *CommitmentTrees]
 
 	storage *EpochCommitmentStorage
 	tangle  *tangleold.Tangle
@@ -74,7 +75,7 @@ func NewEpochCommitmentFactory(store kvstore.KVStore, tangle *tangleold.Tangle,
 	manaRootTreeValueStore := objectstorage.NewStoreWithRealm(epochCommitmentStorage.baseStore, database.PrefixNotarization, prefixManaTreeValues)
 
 	return &EpochCommitmentFactory{
-		commitmentTrees: make(map[epoch.Index]*CommitmentTrees),
+		commitmentTrees: shrinkingmap.New[epoch.Index, *CommitmentTrees](),
 		storage:         epochCommitmentStorage,
 		tangle:          tangle,
 		snapshotDepth:   snapshotDepth,
@@ -344,7 +345,10 @@ func (f *EpochCommitmentFactory) newEpochRoots(ei epoch.Index) (commitmentRoots
 	}
 
 	// We advance the LedgerState to the next epoch.
-	f.commitLedgerState(ei - epoch.Index(f.snapshotDepth))
+	epochToCommit := ei - epoch.Index(f.snapshotDepth)
+	if epochToCommit > 0 {
+		f.commitLedgerState(epochToCommit)
+	}
 
 	commitmentRoots = &epoch.CommitmentRoots{
 		StateRoot:         epoch.NewMerkleRoot(stateRoot),
@@ -354,7 +358,7 @@ func (f *EpochCommitmentFactory) newEpochRoots(ei epoch.Index) (commitmentRoots
 	}
 
 	// We are never going to use this epoch's commitment trees again.
-	delete(f.commitmentTrees, ei)
+	f.commitmentTrees.Delete(ei)
 
 	return commitmentRoots, nil
 }
@@ -383,10 +387,10 @@ func (f *EpochCommitmentFactory) getCommitmentTrees(ei epoch.Index) (commitmentT
 	if ei <= lastCommittedEpoch {
 		return nil, errors.Errorf("cannot get commitment trees for epoch %d, because it is already committed", ei)
 	}
-	commitmentTrees, ok := f.commitmentTrees[ei]
+	commitmentTrees, ok := f.commitmentTrees.Get(ei)
 	if !ok {
 		commitmentTrees = f.newCommitmentTrees(ei)
-		f.commitmentTrees[ei] = commitmentTrees
+		f.commitmentTrees.Set(ei, commitmentTrees)
 	}
 	return
 }

packages/core/notarization/events.go

@@ -4,6 +4,7 @@ import (
 	"github.com/iotaledger/goshimmer/packages/core/epoch"
 	"github.com/iotaledger/goshimmer/packages/core/ledger"
 	"github.com/iotaledger/goshimmer/packages/core/ledger/utxo"
+	"github.com/iotaledger/goshimmer/packages/core/mana"
 	"github.com/iotaledger/goshimmer/packages/core/tangleold"
 	"github.com/iotaledger/hive.go/core/generics/event"
 	"github.com/iotaledger/hive.go/core/identity"
@@ -20,7 +21,7 @@ type Events struct {
 	// CompetingCommitmentDetected is an event that gets triggered whenever a competing epoch commitment is detected.
 	CompetingCommitmentDetected *event.Event[*CompetingCommitmentDetectedEvent]
 	// ManaVectorUpdate is an event that gets triggered whenever the consensus mana vector needs to be updated.
-	ManaVectorUpdate *event.Event[*ManaVectorUpdateEvent]
+	ManaVectorUpdate *event.Event[*mana.ManaVectorUpdateEvent]
 	// TangleTreeInserted is an event that gets triggered when a Block is inserted into the Tangle smt.
 	TangleTreeInserted *event.Event[*TangleTreeUpdatedEvent]
 	// TangleTreeRemoved is an event that gets triggered when a Block is removed from Tangle smt.
@@ -69,10 +70,10 @@ type StateMutationTreeUpdatedEvent struct {
 type UTXOUpdatedEvent struct {
 	// EI is the index of updated UTXO.
 	EI epoch.Index
-	// Created are the outputs created in a transaction.
-	Created []*ledger.OutputWithMetadata
 	// Spent are outputs that is spent in a transaction.
 	Spent []*ledger.OutputWithMetadata
+	// Created are the outputs created in a transaction.
+	Created []*ledger.OutputWithMetadata
 }
 
 // EpochCommittableEvent is a container that acts as a dictionary for the EpochCommittable event related parameters.
@@ -95,12 +96,6 @@ type CompetingCommitmentDetectedEvent struct {
 	Block *tangleold.Block
 }
 
-// ManaVectorUpdateEvent is a container that acts as a dictionary for the EpochCommittable event related parameters.
-type ManaVectorUpdateEvent struct {
-	// EI is the index of committable epoch.
-	EI epoch.Index
-}
-
 // SyncRangeEvent is a container that acts as a dictionary for the SyncRange event related parameters.
 type SyncRangeEvent struct {
 	StartEI   epoch.Index

packages/core/notarization/manager.go

@@ -10,9 +10,11 @@ import (
 
 	"github.com/iotaledger/hive.go/core/generics/event"
 	"github.com/iotaledger/hive.go/core/generics/lo"
+	"github.com/iotaledger/hive.go/core/generics/shrinkingmap"
 	"github.com/iotaledger/hive.go/core/logger"
 
 	"github.com/iotaledger/goshimmer/packages/core/conflictdag"
+	"github.com/iotaledger/goshimmer/packages/core/mana"
 	"github.com/iotaledger/goshimmer/packages/node/clock"
 
 	"github.com/iotaledger/goshimmer/packages/core/epoch"
@@ -35,7 +37,7 @@ type Manager struct {
 	epochCommitmentFactoryMutex sync.RWMutex
 	bootstrapMutex              sync.RWMutex
 	options                     *ManagerOptions
-	pendingConflictsCounters    map[epoch.Index]uint64
+	pendingConflictsCounters    *shrinkingmap.ShrinkingMap[epoch.Index, uint64]
 	log                         *logger.Logger
 	Events                      *Events
 	bootstrapped                bool
@@ -55,7 +57,7 @@ func NewManager(epochCommitmentFactory *EpochCommitmentFactory, t *tangleold.Tan
 	new = &Manager{
 		tangle:                   t,
 		epochCommitmentFactory:   epochCommitmentFactory,
-		pendingConflictsCounters: make(map[epoch.Index]uint64),
+		pendingConflictsCounters: shrinkingmap.New[epoch.Index, uint64](),
 		log:                      options.Log,
 		options:                  options,
 		Events: &Events{
@@ -66,7 +68,7 @@ func NewManager(epochCommitmentFactory *EpochCommitmentFactory, t *tangleold.Tan
 			UTXOTreeInserted:          event.New[*UTXOUpdatedEvent](),
 			UTXOTreeRemoved:           event.New[*UTXOUpdatedEvent](),
 			EpochCommittable:          event.New[*EpochCommittableEvent](),
-			ManaVectorUpdate:          event.New[*ManaVectorUpdateEvent](),
+			ManaVectorUpdate:          event.New[*mana.ManaVectorUpdateEvent](),
 			Bootstrapped:              event.New[*BootstrappedEvent](),
 			SyncRange:                 event.New[*SyncRangeEvent](),
 			ActivityTreeInserted:      event.New[*ActivityTreeUpdatedEvent](),
@@ -172,7 +174,7 @@ func (m *Manager) LoadOutputsWithMetadata(outputsWithMetadatas []*ledger.OutputW
 	}
 }
 
-// LoadEpochDiffs updates the state tree from a given snapshot.
+// LoadEpochDiff loads an epoch diff.
 func (m *Manager) LoadEpochDiff(epochDiff *ledger.EpochDiff) {
 	m.epochCommitmentFactoryMutex.Lock()
 	defer m.epochCommitmentFactoryMutex.Unlock()
@@ -328,8 +330,6 @@ func (m *Manager) OnBlockStored(block *tangleold.Block) {
 	latestCommittableEI := lo.PanicOnErr(m.epochCommitmentFactory.storage.latestCommittableEpochIndex())
 	epochDeltaSeconds := time.Duration(int64(blockEI-latestCommittableEI)*epoch.Duration) * time.Second
 
-	m.log.Debugf("block committing to epoch %d stored, latest committable epoch is %d", blockEI, latestCommittableEI)
-
 	// If we are too far behind, we will warpsync
 	if epochDeltaSeconds > m.options.BootstrapWindow {
 		m.Events.SyncRange.Trigger(&SyncRangeEvent{
@@ -451,7 +451,7 @@ func (m *Manager) OnConflictAccepted(conflictID utxo.TransactionID) {
 }
 
 // OnConflictConfirmed is the handler for conflict confirmed event.
-func (m *Manager) onConflictAccepted(conflictID utxo.TransactionID) ([]*EpochCommittableEvent, []*ManaVectorUpdateEvent) {
+func (m *Manager) onConflictAccepted(conflictID utxo.TransactionID) ([]*EpochCommittableEvent, []*mana.ManaVectorUpdateEvent) {
 	m.epochCommitmentFactoryMutex.Lock()
 	defer m.epochCommitmentFactoryMutex.Unlock()
 
@@ -485,7 +485,7 @@ func (m *Manager) OnConflictRejected(conflictID utxo.TransactionID) {
 }
 
 // OnConflictRejected is the handler for conflict created event.
-func (m *Manager) onConflictRejected(conflictID utxo.TransactionID) ([]*EpochCommittableEvent, []*ManaVectorUpdateEvent) {
+func (m *Manager) onConflictRejected(conflictID utxo.TransactionID) ([]*EpochCommittableEvent, []*mana.ManaVectorUpdateEvent) {
 	m.epochCommitmentFactoryMutex.Lock()
 	defer m.epochCommitmentFactoryMutex.Unlock()
 
@@ -505,7 +505,7 @@ func (m *Manager) OnAcceptanceTimeUpdated(newTime time.Time) {
 }
 
 // OnAcceptanceTimeUpdated is the handler for time updated event and returns events to be triggered.
-func (m *Manager) onAcceptanceTimeUpdated(newTime time.Time) ([]*EpochCommittableEvent, []*ManaVectorUpdateEvent) {
+func (m *Manager) onAcceptanceTimeUpdated(newTime time.Time) ([]*EpochCommittableEvent, []*mana.ManaVectorUpdateEvent) {
 	m.epochCommitmentFactoryMutex.Lock()
 	defer m.epochCommitmentFactoryMutex.Unlock()
 
@@ -532,21 +532,14 @@ func (m *Manager) PendingConflictsCountAll() (pendingConflicts map[epoch.Index]u
 	m.epochCommitmentFactoryMutex.RLock()
 	defer m.epochCommitmentFactoryMutex.RUnlock()
 
-	pendingConflicts = make(map[epoch.Index]uint64, len(m.pendingConflictsCounters))
-	for k, v := range m.pendingConflictsCounters {
+	pendingConflicts = make(map[epoch.Index]uint64, m.pendingConflictsCounters.Size())
+	m.pendingConflictsCounters.ForEach(func(k epoch.Index, v uint64) bool {
 		pendingConflicts[k] = v
-	}
+		return true
+	})
 	return pendingConflicts
 }
 
-// GetEpochDiff returns the epoch diff of an epoch.
-func (m *Manager) GetEpochDiff(ei epoch.Index) (spent []*ledger.OutputWithMetadata, created []*ledger.OutputWithMetadata) {
-	m.epochCommitmentFactoryMutex.Lock()
-	defer m.epochCommitmentFactoryMutex.Unlock()
-	spent, created = m.epochCommitmentFactory.loadDiffUTXOs(ei)
-	return
-}
-
 // Bootstrapped returns the current value of pendingConflictsCount per epoch.
 func (m *Manager) Bootstrapped() bool {
 	m.bootstrapMutex.RLock()
@@ -563,16 +556,20 @@ func (m *Manager) Shutdown() {
 	m.epochCommitmentFactory.storage.shutdown()
 }
 
-func (m *Manager) decreasePendingConflictCounter(ei epoch.Index) ([]*EpochCommittableEvent, []*ManaVectorUpdateEvent) {
-	m.pendingConflictsCounters[ei]--
-	if m.pendingConflictsCounters[ei] == 0 {
+func (m *Manager) decreasePendingConflictCounter(ei epoch.Index) ([]*EpochCommittableEvent, []*mana.ManaVectorUpdateEvent) {
+	count, _ := m.pendingConflictsCounters.Get(ei)
+	count--
+	m.pendingConflictsCounters.Set(ei, count)
+	if count == 0 {
 		return m.moveLatestCommittableEpoch(ei)
 	}
 	return nil, nil
 }
 
 func (m *Manager) increasePendingConflictCounter(ei epoch.Index) {
-	m.pendingConflictsCounters[ei]++
+	count, _ := m.pendingConflictsCounters.Get(ei)
+	count++
+	m.pendingConflictsCounters.Set(ei, count)
 }
 
 func (m *Manager) includeTransactionInEpoch(txID utxo.TransactionID, ei epoch.Index, spent, created []*ledger.OutputWithMetadata) (err error) {
@@ -611,7 +608,7 @@ func (m *Manager) allPastConflictsAreResolved(ei epoch.Index) (conflictsResolved
 	}
 	// epoch is not committable if there are any not resolved conflicts in this and past epochs
 	for index := lastEI; index <= ei; index++ {
-		if m.pendingConflictsCounters[index] != 0 {
+		if count, _ := m.pendingConflictsCounters.Get(index); count != 0 {
 			return false
 		}
 	}
@@ -675,21 +672,31 @@ func (m *Manager) resolveOutputs(tx utxo.Transaction) (spentOutputsWithMetadata,
 	return
 }
 
-func (m *Manager) manaVectorUpdate(ei epoch.Index) (event *ManaVectorUpdateEvent) {
-	return &ManaVectorUpdateEvent{
-		EI: ei,
+func (m *Manager) manaVectorUpdate(ei epoch.Index) (event *mana.ManaVectorUpdateEvent) {
+	manaEpoch := ei - epoch.Index(m.options.ManaEpochDelay)
+	spent := []*ledger.OutputWithMetadata{}
+	created := []*ledger.OutputWithMetadata{}
+
+	if manaEpoch > 0 {
+		spent, created = m.epochCommitmentFactory.loadDiffUTXOs(manaEpoch)
+	}
+
+	return &mana.ManaVectorUpdateEvent{
+		EI:      ei,
+		Spent:   spent,
+		Created: created,
 	}
 }
 
-func (m *Manager) moveLatestCommittableEpoch(currentEpoch epoch.Index) ([]*EpochCommittableEvent, []*ManaVectorUpdateEvent) {
+func (m *Manager) moveLatestCommittableEpoch(currentEpoch epoch.Index) ([]*EpochCommittableEvent, []*mana.ManaVectorUpdateEvent) {
 	latestCommittable, err := m.epochCommitmentFactory.storage.latestCommittableEpochIndex()
 	if err != nil {
 		m.log.Errorf("could not obtain last committed epoch index: %v", err)
 		return nil, nil
 	}
 
 	epochCommittableEvents := make([]*EpochCommittableEvent, 0)
-	manaVectorUpdateEvents := make([]*ManaVectorUpdateEvent, 0)
+	manaVectorUpdateEvents := make([]*mana.ManaVectorUpdateEvent, 0)
 	for ei := latestCommittable + 1; ei <= currentEpoch; ei++ {
 		if !m.isCommittable(ei) {
 			break
@@ -715,11 +722,14 @@ func (m *Manager) moveLatestCommittableEpoch(currentEpoch epoch.Index) ([]*Epoch
 		if manaVectorUpdateEvent := m.manaVectorUpdate(ei); manaVectorUpdateEvent != nil {
 			manaVectorUpdateEvents = append(manaVectorUpdateEvents, manaVectorUpdateEvent)
 		}
+
+		// We do not need to track pending conflicts for a committed epoch anymore.
+		m.pendingConflictsCounters.Delete(ei)
 	}
 	return epochCommittableEvents, manaVectorUpdateEvents
 }
 
-func (m *Manager) triggerEpochEvents(epochCommittableEvents []*EpochCommittableEvent, manaVectorUpdateEvents []*ManaVectorUpdateEvent) {
+func (m *Manager) triggerEpochEvents(epochCommittableEvents []*EpochCommittableEvent, manaVectorUpdateEvents []*mana.ManaVectorUpdateEvent) {
 	for _, epochCommittableEvent := range epochCommittableEvents {
 		m.Events.EpochCommittable.Trigger(epochCommittableEvent)
 	}
@@ -755,6 +765,15 @@ type ManagerOptions struct {
 	MinCommittableEpochAge time.Duration
 	BootstrapWindow        time.Duration
 	Log                    *logger.Logger
+	ManaEpochDelay         uint
+}
+
+// ManaEpochDelay specifies how many epochs the consensus mana booking is delayed with respect to the latest committable
+// epoch.
+func ManaEpochDelay(manaEpochDelay uint) ManagerOption {
+	return func(options *ManagerOptions) {
+		options.ManaEpochDelay = manaEpochDelay
+	}
 }
 
 // MinCommittableEpochAge specifies how old an epoch has to be for it to be committable.