go-algorand 3.18.0-stable Release PR

.circleci/config.yml

@@ -457,7 +457,7 @@ commands:
           shell: bash.exe
           command: |
             choco install -y msys2 pacman make wget --force
-            choco install -y golang --version=1.20.5 --force
+            choco install -y golang --version=$(./scripts/get_golang_version.sh) --force
             choco install -y python3 --version=3.7.3 --force
             export msys2='cmd //C RefreshEnv.cmd '
             export msys2+='& set MSYS=winsymlinks:nativestrict '
@@ -602,7 +602,6 @@ commands:
             export PACKAGE_NAMES=$(echo $PACKAGES | tr -d '\n')
             export PARTITION_TOTAL=${CIRCLE_NODE_TOTAL}
             export PARTITION_ID=${CIRCLE_NODE_INDEX}
-            export GOEXPERIMENT="none"
             gotestsum --format standard-verbose --junitfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml --jsonfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json -- --tags "sqlite_unlock_notify sqlite_omit_load_extension" << parameters.short_test_flag >> -race -timeout 1h -coverprofile=coverage.txt -covermode=atomic -p 1 $PACKAGE_NAMES
       - store_artifacts:
           path: << parameters.result_path >>

.github/workflows/reviewdog.yml

@@ -56,7 +56,7 @@ jobs:
         uses: actions/[email protected]
         with:
           path: cicdtmp/golangci-lint/golangci-lint-cgo
-          key: cicd-golangci-lint-cgo-v0.0.2
+          key: cicd-golangci-lint-cgo-v0.0.2-${{ env.GO_VERSION }}
 
       - name: Build custom golangci-lint with CGO_ENABLED
         if: steps.cache-golangci-lint.outputs.cache-hit != 'true'

.github/workflows/tools.yml

@@ -9,9 +9,6 @@ on:
       - 'tools/block-generator/**'
       - 'tools/x-repo-types/**'
   pull_request:
-    paths:
-      - 'tools/block-generator/**'
-      - 'tools/x-repo-types/**'
 
 jobs:
   tools_test:

Dockerfile

@@ -1,6 +1,6 @@
 FROM ubuntu:20.04 as builder
 
-ARG GO_VERSION="1.20.5"
+ARG GO_VERSION="1.20.7"
 
 ARG CHANNEL
 ARG URL

Makefile

@@ -69,11 +69,6 @@ ifeq ($(SHORT_PART_PERIOD), 1)
 export SHORT_PART_PERIOD_FLAG := -s
 endif
 
-# Disable go experiments during build as of go 1.20.5 due to
-# https://github.com/golang/go/issues/60825
-# Likely fix: https://go-review.googlesource.com/c/go/+/503937/6/src/runtime/race_arm64.s
-export GOEXPERIMENT=none
-
 GOTAGS      := --tags "$(GOTAGSLIST)"
 GOTRIMPATH	:= $(shell GOPATH=$(GOPATH) && go help build | grep -q .-trimpath && echo -trimpath)
 
@@ -91,7 +86,7 @@ ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd d
 
 GOMOD_DIRS := ./tools/block-generator ./tools/x-repo-types
 
-MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2
+MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/store/trackerdb/generickv ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2
 
 default: build
 
@@ -212,9 +207,11 @@ build: buildsrc buildsrc-special
 # get around a bug in go build where it will fail
 # to cache binaries from time to time on empty NFS
 # dirs
-buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN
-	mkdir -p "${GOCACHE}" && \
-	touch "${GOCACHE}"/file.txt && \
+${GOCACHE}/file.txt:
+	mkdir -p "${GOCACHE}"
+	touch "${GOCACHE}"/file.txt
+
+buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN ${GOCACHE}/file.txt
 	go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
 
 buildsrc-special:

agreement/agreementtest/simulate.go

@@ -31,7 +31,6 @@ import (
 	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/logging"
-	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/util/db"
 	"github.com/algorand/go-algorand/util/timers"
 )
@@ -52,15 +51,15 @@ func makeInstant() *instant {
 	return i
 }
 
-func (i *instant) Decode([]byte) (timers.Clock, error) {
+func (i *instant) Decode([]byte) (timers.Clock[agreement.TimeoutType], error) {
 	return i, nil
 }
 
 func (i *instant) Encode() []byte {
 	return nil
 }
 
-func (i *instant) TimeoutAt(d time.Duration) <-chan time.Time {
+func (i *instant) TimeoutAt(d time.Duration, timeoutType agreement.TimeoutType) <-chan time.Time {
 	ta := make(chan time.Time)
 	select {
 	case <-i.timeoutAtCalled:
@@ -69,13 +68,13 @@ func (i *instant) TimeoutAt(d time.Duration) <-chan time.Time {
 		return ta
 	}
 
-	if d == agreement.FilterTimeout(0, protocol.ConsensusCurrentVersion) && !i.HasPending("pseudonode") {
+	if timeoutType == agreement.TimeoutFilter && !i.HasPending("pseudonode") {
 		close(ta)
 	}
 	return ta
 }
 
-func (i *instant) Zero() timers.Clock {
+func (i *instant) Zero() timers.Clock[agreement.TimeoutType] {
 	i.Z0 <- struct{}{}
 	// pause here until runRound is called
 	i.Z1 <- struct{}{}

agreement/demux.go

@@ -19,7 +19,6 @@ package agreement
 import (
 	"context"
 	"fmt"
-	"time"
 
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/logging"
@@ -190,7 +189,7 @@ func (d *demux) verifyBundle(ctx context.Context, m message, r round, p period,
 // next blocks until it observes an external input event of interest for the state machine.
 //
 // If ok is false, there are no more events so the agreement service should quit.
-func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Duration, currentRound round) (e externalEvent, ok bool) {
+func (d *demux) next(s *Service, deadline Deadline, fastDeadline Deadline, currentRound round) (e externalEvent, ok bool) {
 	defer func() {
 		if !ok {
 			return
@@ -250,8 +249,8 @@ func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Durat
 	}
 
 	ledgerNextRoundCh := s.Ledger.Wait(nextRound)
-	deadlineCh := s.Clock.TimeoutAt(deadline)
-	fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline)
+	deadlineCh := s.Clock.TimeoutAt(deadline.Duration, deadline.Type)
+	fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline.Duration, fastDeadline.Type)
 
 	d.UpdateEventsQueue(eventQueueDemux, 0)
 	d.monitor.dec(demuxCoserviceType)

agreement/demux_test.go

@@ -422,14 +422,14 @@ func TestDemuxNext(t *testing.T) {
 }
 
 // implement timers.Clock
-func (t *demuxTester) Zero() timers.Clock {
+func (t *demuxTester) Zero() timers.Clock[TimeoutType] {
 	// we don't care about this function in this test.
 	return t
 }
 
 // implement timers.Clock
-func (t *demuxTester) TimeoutAt(delta time.Duration) <-chan time.Time {
-	if delta == fastTimeoutChTime {
+func (t *demuxTester) TimeoutAt(delta time.Duration, timeoutType TimeoutType) <-chan time.Time {
+	if timeoutType == TimeoutFastRecovery {
 		return nil
 	}
 
@@ -450,7 +450,7 @@ func (t *demuxTester) Encode() []byte {
 }
 
 // implement timers.Clock
-func (t *demuxTester) Decode([]byte) (timers.Clock, error) {
+func (t *demuxTester) Decode([]byte) (timers.Clock[TimeoutType], error) {
 	// we don't care about this function in this test.
 	return t, nil
 }
@@ -675,7 +675,7 @@ func (t *demuxTester) TestUsecase(testcase demuxTestUsecase) bool {
 		close(s.quit)
 	}
 
-	e, ok := dmx.next(s, time.Second, fastTimeoutChTime, 300)
+	e, ok := dmx.next(s, Deadline{Duration: time.Second, Type: TimeoutDeadline}, Deadline{Duration: fastTimeoutChTime, Type: TimeoutFastRecovery}, 300)
 
 	if !assert.Equal(t, testcase.ok, ok) {
 		return false

agreement/fuzzer/fuzzer_test.go

@@ -47,7 +47,7 @@ type Fuzzer struct {
 	wallClock        int32
 	agreements       []*agreement.Service
 	facades          []*NetworkFacade
-	clocks           []timers.Clock
+	clocks           []timers.Clock[agreement.TimeoutType]
 	disconnected     [][]bool
 	crashAccessors   []db.Accessor
 	router           *Router
@@ -80,7 +80,7 @@ func MakeFuzzer(config FuzzerConfig) *Fuzzer {
 		networkName:      config.FuzzerName,
 		agreements:       make([]*agreement.Service, config.NodesCount),
 		facades:          make([]*NetworkFacade, config.NodesCount),
-		clocks:           make([]timers.Clock, config.NodesCount),
+		clocks:           make([]timers.Clock[agreement.TimeoutType], config.NodesCount),
 		disconnected:     make([][]bool, config.NodesCount),
 		crashAccessors:   make([]db.Accessor, config.NodesCount),
 		accounts:         make([]account.Participation, config.NodesCount),

agreement/fuzzer/networkFacade_test.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/algorand/go-deadlock"
 
+	"github.com/algorand/go-algorand/agreement"
 	"github.com/algorand/go-algorand/network"
 	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/util/timers"
@@ -48,7 +49,7 @@ type NetworkFacadeMessage struct {
 type NetworkFacade struct {
 	network.GossipNode
 	NetworkFilter
-	timers.Clock
+	timers.Clock[agreement.TimeoutType]
 	nodeID                         int
 	mux                            *network.Multiplexer
 	fuzzer                         *Fuzzer
@@ -77,7 +78,7 @@ func MakeNetworkFacade(fuzzer *Fuzzer, nodeID int) *NetworkFacade {
 	n := &NetworkFacade{
 		fuzzer:         fuzzer,
 		nodeID:         nodeID,
-		mux:            network.MakeMultiplexer(fuzzer.log),
+		mux:            network.MakeMultiplexer(),
 		clocks:         make(map[int]chan time.Time),
 		eventsQueues:   make(map[string]int),
 		eventsQueuesCh: make(chan int, 1000),
@@ -345,7 +346,7 @@ func (n *NetworkFacade) Disconnect(sender network.Peer) {
 	n.fuzzer.Disconnect(n.nodeID, sourceNode)
 }
 
-func (n *NetworkFacade) Zero() timers.Clock {
+func (n *NetworkFacade) Zero() timers.Clock[agreement.TimeoutType] {
 	n.clockSync.Lock()
 	defer n.clockSync.Unlock()
 
@@ -375,7 +376,7 @@ func (n *NetworkFacade) Rezero() {
 // Since implements the Clock interface.
 func (n *NetworkFacade) Since() time.Duration { return 0 }
 
-func (n *NetworkFacade) TimeoutAt(d time.Duration) <-chan time.Time {
+func (n *NetworkFacade) TimeoutAt(d time.Duration, timeoutType agreement.TimeoutType) <-chan time.Time {
 	defer n.timeoutAtInitOnce.Do(func() {
 		n.timeoutAtInitWait.Done()
 	})
@@ -414,7 +415,7 @@ func (n *NetworkFacade) Encode() []byte {
 	return buf.Bytes()
 }
 
-func (n *NetworkFacade) Decode(in []byte) (timers.Clock, error) {
+func (n *NetworkFacade) Decode(in []byte) (timers.Clock[agreement.TimeoutType], error) {
 	n.clockSync.Lock()
 	defer n.clockSync.Unlock()
 

agreement/gossip/network_test.go

@@ -323,7 +323,7 @@ func makewhiteholeNetwork(domain *whiteholeDomain) *whiteholeNetwork {
 	w := &whiteholeNetwork{
 		peer:         atomic.AddUint32(&domain.peerIdx, 1),
 		lastMsgRead:  uint32(len(domain.messages)),
-		mux:          network.MakeMultiplexer(domain.log),
+		mux:          network.MakeMultiplexer(),
 		domain:       domain,
 		disconnected: make(map[uint32]bool),
 	}

agreement/message.go

@@ -28,7 +28,7 @@ type message struct {
 
 	// this field is for backwards compatibility with crash state serialized using go-codec prior to explicit unexport.
 	// should be removed after the next consensus update.
-	MessageHandle msgp.Raw
+	MessageHandle msgp.Raw `codec:"MessageHandle,omitempty"`
 	// explicitly unexport this field since we can't define serializers for interface{} type
 	// the only implementation of this is gossip.messageMetadata which doesn't have exported fields to serialize.
 	messageHandle MessageHandle

agreement/message_test.go

@@ -111,6 +111,7 @@ func TestMessageBackwardCompatibility(t *testing.T) {
 		Tag: protocol.ProposalPayloadTag,
 	}
 
+	require.Containsf(t, string(encoded), "MessageHandle", "encoded message does not contain MessageHandle field")
 	var m1, m2, m3, m4 message
 	// Both msgp and reflection should decode the message containing old MessageHandle successfully
 	err = protocol.Decode(encoded, &m1)
@@ -123,6 +124,7 @@ func TestMessageBackwardCompatibility(t *testing.T) {
 	e1 := protocol.Encode(&m1)
 	e2 := protocol.EncodeReflect(&m2)
 	require.Equal(t, e1, e2)
+	require.NotContainsf(t, string(e1), "MessageHandle", "encoded message still contains MessageHandle field")
 	err = protocol.DecodeReflect(e1, &m3)
 	require.NoError(t, err)
 	err = protocol.Decode(e2, &m4)