Merge pull request #2179 from Algo-devops-service/relstable2.6.0

go-algorand 2.6.0-stable

.travis.yml

@@ -165,6 +165,7 @@ jobs:
 cache:
   directories:
     - crypto/lib
+    - $HOME/docker_cache
 
 before_install:
 - |-
@@ -189,6 +190,7 @@ before_install:
      export MAKE=mingw32-make  # so that Autotools can find it
      ;;
    esac
+   docker load -i $HOME/docker_cache/images.tar || true
 
 before_cache:
 - |-
@@ -198,6 +200,7 @@ before_cache:
      $msys2 pacman --sync --clean --noconfirm
      ;;
    esac
+   docker save -o $HOME/docker_cache/images.tar $(docker images -a -q)
 
 addons:
   apt:

Makefile

@@ -107,7 +107,18 @@ generate: deps
 msgp: $(patsubst %,%/msgp_gen.go,$(MSGP_GENERATE))
 
 %/msgp_gen.go: deps ALWAYS
-	$(GOPATH1)/bin/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand
+		@set +e; \
+		printf "msgp: $(@D)..."; \
+		$(GOPATH1)/bin/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand > ./$@.out 2>&1; \
+		if [ "$$?" != "0" ]; then \
+			printf "failed:\n$(GOPATH1)/bin/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand\n"; \
+			cat ./$@.out; \
+			rm ./$@.out; \
+			exit 1; \
+		else \
+			echo " done."; \
+		fi; \
+		rm -f ./$@.out
 ALWAYS:
 
 # build our fork of libsodium, placing artifacts into crypto/lib/ and crypto/include/
@@ -210,17 +221,17 @@ $(GOPATH1)/bin/%:
 	cp -f $< $@
 
 test: build
-	go test $(GOTAGS) -race $(UNIT_TEST_SOURCES) -timeout 3600s
+	go test $(GOTAGS) -race $(UNIT_TEST_SOURCES) -timeout 3600s | logfilter
 
 fulltest: build-race
 	for PACKAGE_DIRECTORY in $(UNIT_TEST_SOURCES) ; do \
-		go test $(GOTAGS) -timeout 2500s -race $$PACKAGE_DIRECTORY; \
+		go test $(GOTAGS) -timeout 2500s -race $$PACKAGE_DIRECTORY | logfilter; \
 	done
 
 shorttest: build-race $(addprefix short_test_target_, $(UNIT_TEST_SOURCES))
 
 $(addprefix short_test_target_, $(UNIT_TEST_SOURCES)): build
-	@go test $(GOTAGS) -short -timeout 2500s -race $(subst short_test_target_,,$@)
+	@go test $(GOTAGS) -short -timeout 2500s -race $(subst short_test_target_,,$@) | logfilter
 
 integration: build-race
 	./test/scripts/run_integration_tests.sh

README.md

@@ -22,8 +22,10 @@ the [official Go documentation website](https://golang.org/doc/).
 ### Linux / OSX ###
 
 We currently strive to support Debian based distributions with Ubuntu 18.04
-being our official release target. Our core engineering team uses Linux and OSX,
-so both environments are well supported for development.
+being our official release target.
+Building on Arch Linux works as well.
+Our core engineering team uses Linux and OSX, so both environments are well
+supported for development.
 
 OSX only: [Homebrew (brew)](https://brew.sh) must be installed before
 continuing. [Here](https://docs.brew.sh/Installation) are the installation

agreement/abstractions.go

@@ -225,13 +225,10 @@ type LedgerWriter interface {
 
 // A KeyManager stores and deletes participation keys.
 type KeyManager interface {
-	// Keys returns an immutable array of participation intervals to
-	// participating accounts.
-	Keys() []account.Participation
-
-	// HasLiveKeys returns true if we have any Participation
-	// keys valid for the specified round range (inclusive)
-	HasLiveKeys(from, to basics.Round) bool
+	// VotingKeys returns an immutable array of voting keys that are
+	// valid for the provided votingRound, and were available at
+	// keysRound.
+	VotingKeys(votingRound, keysRound basics.Round) []account.Participation
 }
 
 // MessageHandle is an ID referring to a specific message.

agreement/agreementtest/keyManager.go

@@ -24,24 +24,15 @@ import (
 // SimpleKeyManager provides a simple implementation of a KeyManager.
 type SimpleKeyManager []account.Participation
 
-// Keys implements KeyManager.Keys.
-func (m SimpleKeyManager) Keys() []account.Participation {
+// VotingKeys implements KeyManager.VotingKeys.
+func (m SimpleKeyManager) VotingKeys(votingRound, _ basics.Round) []account.Participation {
 	var km []account.Participation
 	for _, acc := range m {
-		km = append(km, acc)
-	}
-	return km
-}
-
-// HasLiveKeys returns true if we have any Participation
-// keys valid for the specified round range (inclusive)
-func (m SimpleKeyManager) HasLiveKeys(from, to basics.Round) bool {
-	for _, acc := range m {
-		if acc.OverlapsInterval(from, to) {
-			return true
+		if acc.OverlapsInterval(votingRound, votingRound) {
+			km = append(km, acc)
 		}
 	}
-	return false
+	return km
 }
 
 // DeleteOldKeys implements KeyManager.DeleteOldKeys.

agreement/agreementtest/simulate_test.go

@@ -356,7 +356,8 @@ func generateNAccounts(t *testing.T, N int, firstRound, lastRound basics.Round,
 		if err != nil {
 			panic(err)
 		}
-		accounts = append(accounts, part)
+		accounts = append(accounts, part.Participation)
+		part.Close()
 	}
 	return
 }

agreement/cryptoVerifier_test.go

@@ -317,7 +317,8 @@ func BenchmarkCryptoVerifierProposalVertification(b *testing.B) {
 	}
 
 	Period := period(0)
-	participation := pn.getParticipations("BenchmarkCryptoVerifierProposalVertification", ledger.NextRound())
+	pn.loadRoundParticipationKeys(ledger.NextRound())
+	participation := pn.participationKeys
 
 	proposals, _ := pn.makeProposals(ledger.NextRound(), Period, participation)
 

agreement/demux.go

@@ -238,12 +238,19 @@ func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Durat
 	deadlineCh := s.Clock.TimeoutAt(deadline)
 	var fastDeadlineCh <-chan time.Time
 
-	proto, err := d.ledger.ConsensusVersion(ParamsRound(currentRound))
-	if err == nil && config.Consensus[proto].FastPartitionRecovery {
-		fastDeadlineCh = s.Clock.TimeoutAt(fastDeadline)
+	fastPartitionRecoveryEnabled := false
+	if proto, err := d.ledger.ConsensusVersion(ParamsRound(currentRound)); err != nil {
+		logging.Base().Warnf("demux: could not get consensus parameters for round %d: %v", ParamsRound(currentRound), err)
+		// this might happen during catchup, since the Ledger.Wait fires as soon as a new block is received by the ledger, which could be
+		// far before it's being committed. In these cases, it should be safe to default to the current consensus version. On subsequent
+		// iterations, it will get "corrected" since the ledger would finish flushing the blocks to disk.
+		fastPartitionRecoveryEnabled = config.Consensus[protocol.ConsensusCurrentVersion].FastPartitionRecovery
+	} else {
+		fastPartitionRecoveryEnabled = config.Consensus[proto].FastPartitionRecovery
 	}
-	if err != nil {
-		logging.Base().Errorf("could not get consensus parameters for round %d: %v", ParamsRound(currentRound), err)
+
+	if fastPartitionRecoveryEnabled {
+		fastDeadlineCh = s.Clock.TimeoutAt(fastDeadline)
 	}
 
 	d.UpdateEventsQueue(eventQueueDemux, 0)

agreement/fuzzer/fuzzer_test.go

@@ -84,7 +84,7 @@ func MakeFuzzer(config FuzzerConfig) *Fuzzer {
 		crashAccessors:   make([]db.Accessor, config.NodesCount),
 		accounts:         make([]account.Participation, config.NodesCount),
 		balances:         make(map[basics.Address]basics.AccountData),
-		accountAccessors: make([]db.Accessor, config.NodesCount*2),
+		accountAccessors: make([]db.Accessor, config.NodesCount),
 		ledgers:          make([]*testLedger, config.NodesCount),
 		agreementParams:  make([]agreement.Parameters, config.NodesCount),
 		tickGranularity:  time.Millisecond * 300,
@@ -196,7 +196,7 @@ func (n *Fuzzer) initAccountsAndBalances(rootSeed []byte, onlineNodes []bool) er
 		if err != nil {
 			return err
 		}
-		n.accountAccessors[i*2+0] = rootAccess
+		n.accountAccessors[i] = rootAccess
 
 		seed = sha256.Sum256(seed[:])
 		root, err := account.ImportRoot(rootAccess, seed)
@@ -205,27 +205,12 @@ func (n *Fuzzer) initAccountsAndBalances(rootSeed []byte, onlineNodes []bool) er
 		}
 		rootAddress := root.Address()
 
-		partAccess, err := db.MakeAccessor(n.networkName+"part"+strconv.Itoa(i+off), false, true)
-
-		if err != nil {
-			return err
-		}
-
-		n.accountAccessors[i*2+1] = partAccess
-
 		n.accounts[i] = account.Participation{
 			Parent:     rootAddress,
 			VRF:        generatePseudoRandomVRF(i),
 			Voting:     readOnlyParticipationVotes[i],
 			FirstValid: firstValid,
 			LastValid:  lastValid,
-			Store:      partAccess,
-		}
-
-		err = n.accounts[i].Persist()
-
-		if err != nil {
-			panic(err)
 		}
 
 		acctData := basics.AccountData{

agreement/fuzzer/keyManager_test.go

@@ -23,15 +23,12 @@ import (
 
 type simpleKeyManager []account.Participation
 
-func (m simpleKeyManager) Keys() []account.Participation {
-	return m
-}
-
-func (m simpleKeyManager) HasLiveKeys(from, to basics.Round) bool {
+func (m simpleKeyManager) VotingKeys(votingRound, _ basics.Round) []account.Participation {
+	var km []account.Participation
 	for _, acc := range m {
-		if acc.OverlapsInterval(from, to) {
-			return true
+		if acc.OverlapsInterval(votingRound, votingRound) {
+			km = append(km, acc)
 		}
 	}
-	return false
+	return km
 }

agreement/gossip/networkFull_test.go

@@ -202,12 +202,12 @@ func testNetworkImplMixed(t *testing.T, nodesCount int) {
 	nets, counters := spinNetwork(t, nodesCount)
 	defer shutdownNetwork(nets, counters)
 
-	nets[0].broadcastTimeout(protocol.AgreementVoteTag, []byte{1}, testNetTimeout)
-	nets[0].broadcastTimeout(protocol.ProposalPayloadTag, []byte{1}, testNetTimeout)
-	nets[0].broadcastTimeout(protocol.ProposalPayloadTag, []byte{1}, testNetTimeout)
-	nets[0].broadcastTimeout(protocol.VoteBundleTag, []byte{1}, testNetTimeout)
-	nets[0].broadcastTimeout(protocol.VoteBundleTag, []byte{1}, testNetTimeout)
-	nets[0].broadcastTimeout(protocol.VoteBundleTag, []byte{1}, testNetTimeout)
+	nets[0].Broadcast(protocol.AgreementVoteTag, []byte{1})
+	nets[0].Broadcast(protocol.ProposalPayloadTag, []byte{1})
+	nets[0].Broadcast(protocol.ProposalPayloadTag, []byte{1})
+	nets[0].Broadcast(protocol.VoteBundleTag, []byte{1})
+	nets[0].Broadcast(protocol.VoteBundleTag, []byte{1})
+	nets[0].Broadcast(protocol.VoteBundleTag, []byte{1})
 	for i, counter := range counters {
 		if i != 0 {
 			if !counter.verify(t, 1, 2, 3) {
@@ -228,14 +228,14 @@ func testNetworkImplMixed2(t *testing.T, nodesCount int) {
 
 	const loadSize = 12
 	for i := byte(0); i < loadSize; i++ {
-		ok := nets[0].broadcastTimeout(protocol.AgreementVoteTag, []byte{i}, testNetTimeout)
+		ok := nets[0].Broadcast(protocol.AgreementVoteTag, []byte{i})
 		assert.NoError(t, ok)
 		if i%2 == 0 {
-			ok = nets[0].broadcastTimeout(protocol.ProposalPayloadTag, []byte{i}, testNetTimeout)
+			ok = nets[0].Broadcast(protocol.ProposalPayloadTag, []byte{i})
 			assert.NoError(t, ok)
 		}
 		if i%4 == 0 {
-			ok = nets[0].broadcastTimeout(protocol.VoteBundleTag, []byte{i}, testNetTimeout)
+			ok = nets[0].Broadcast(protocol.VoteBundleTag, []byte{i})
 			assert.NoError(t, ok)
 		}
 	}
@@ -266,14 +266,14 @@ func testNetworkImplReordered(t *testing.T, nodesCount int) {
 	wg.Add(loadSize)
 	for i := byte(0); i < loadSize; i++ {
 		go func(i byte) {
-			ok := nets[0].broadcastTimeout(protocol.AgreementVoteTag, []byte{i}, testNetTimeout)
+			ok := nets[0].Broadcast(protocol.AgreementVoteTag, []byte{i})
 			assert.NoError(t, ok)
 			if i%2 == 0 {
-				ok = nets[0].broadcastTimeout(protocol.ProposalPayloadTag, []byte{i}, testNetTimeout)
+				ok = nets[0].Broadcast(protocol.ProposalPayloadTag, []byte{i})
 				assert.NoError(t, ok)
 			}
 			if i%4 == 0 {
-				ok = nets[0].broadcastTimeout(protocol.VoteBundleTag, []byte{i}, testNetTimeout)
+				ok = nets[0].Broadcast(protocol.VoteBundleTag, []byte{i})
 				assert.NoError(t, ok)
 			}
 			wg.Done()
@@ -323,7 +323,7 @@ func testNetworkImplRebroadcast(t *testing.T, nodesCount int) {
 		rebroadcastNodes = 3
 	}
 	for i := byte(0); i < byte(rebroadcastNodes); i++ {
-		ok := nets[i].broadcastTimeout(protocol.AgreementVoteTag, []byte{i, i + 1}, testNetTimeout)
+		ok := nets[i].Broadcast(protocol.AgreementVoteTag, []byte{i, i + 1})
 		assert.NoError(t, ok)
 	}