Merge pull request #5849 from Algo-devops-service/relbeta3.20.1

buildnumber.dat

@@ -1 +1 @@
-0
+1

catchup/catchpointService.go

@@ -373,8 +373,9 @@ func (cs *CatchpointCatchupService) processStageLatestBlockDownload() (err error
 	var blk *bookkeeping.Block
 	var cert *agreement.Certificate
 	// check to see if the current ledger might have this block. If so, we should try this first instead of downloading anything.
-	if ledgerBlock, err := cs.ledger.Block(blockRound); err == nil {
+	if ledgerBlock, ledgerCert, err0 := cs.ledger.BlockCert(blockRound); err0 == nil {
 		blk = &ledgerBlock
+		cert = &ledgerCert
 	}
 	var protoParams config.ConsensusParams
 	var ok bool
@@ -551,15 +552,17 @@ func (cs *CatchpointCatchupService) processStageBlocksDownload() (err error) {
 		}
 
 		blk = nil
+		cert = nil
 		// check to see if the current ledger might have this block. If so, we should try this first instead of downloading anything.
-		if ledgerBlock, err := cs.ledger.Block(topBlock.Round() - basics.Round(blocksFetched)); err == nil {
+		if ledgerBlock, ledgerCert, err0 := cs.ledger.BlockCert(topBlock.Round() - basics.Round(blocksFetched)); err0 == nil {
 			blk = &ledgerBlock
+			cert = &ledgerCert
 		} else {
-			switch err.(type) {
+			switch err0.(type) {
 			case ledgercore.ErrNoEntry:
 				// this is expected, ignore this one.
 			default:
-				cs.log.Warnf("processStageBlocksDownload encountered the following error when attempting to retrieve the block for round %d : %v", topBlock.Round()-basics.Round(blocksFetched), err)
+				cs.log.Warnf("processStageBlocksDownload encountered the following error when attempting to retrieve the block for round %d : %v", topBlock.Round()-basics.Round(blocksFetched), err0)
 			}
 		}
 

catchup/catchpointService_test.go

@@ -22,34 +22,37 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/algorand/go-algorand/agreement"
 	"github.com/algorand/go-algorand/components/mocks"
 	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/ledger"
 	"github.com/algorand/go-algorand/ledger/ledgercore"
+	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/test/partitiontest"
 )
 
 type catchpointCatchupLedger struct {
 }
 
-func (l *catchpointCatchupLedger) Block(rnd basics.Round) (blk bookkeeping.Block, err error) {
+func (l *catchpointCatchupLedger) BlockCert(rnd basics.Round) (blk bookkeeping.Block, cert agreement.Certificate, err error) {
 	blk = bookkeeping.Block{
 		BlockHeader: bookkeeping.BlockHeader{
 			UpgradeState: bookkeeping.UpgradeState{
 				CurrentProtocol: protocol.ConsensusCurrentVersion,
 			},
 		},
 	}
+	cert = agreement.Certificate{}
 	commitments, err := blk.PaysetCommit()
 	if err != nil {
-		return blk, err
+		return blk, cert, err
 	}
 	blk.TxnCommitments = commitments
 
-	return blk, nil
+	return blk, cert, nil
 }
 
 func (l *catchpointCatchupLedger) GenesisHash() (d crypto.Digest) {
@@ -95,3 +98,64 @@ func TestCatchpointServicePeerRank(t *testing.T) {
 	err := cs.processStageLatestBlockDownload()
 	require.NoError(t, err)
 }
+
+type catchpointAccessorMock struct {
+	mocks.MockCatchpointCatchupAccessor
+	t      *testing.T
+	topBlk bookkeeping.Block
+}
+
+func (m *catchpointAccessorMock) EnsureFirstBlock(ctx context.Context) (blk bookkeeping.Block, err error) {
+	return m.topBlk, nil
+}
+
+func (m *catchpointAccessorMock) StoreBlock(ctx context.Context, blk *bookkeeping.Block, cert *agreement.Certificate) (err error) {
+	require.NotNil(m.t, blk)
+	require.NotNil(m.t, cert)
+	return nil
+}
+
+type catchpointCatchupLedger2 struct {
+	catchpointCatchupLedger
+	blk bookkeeping.Block
+}
+
+func (l *catchpointCatchupLedger2) BlockCert(rnd basics.Round) (blk bookkeeping.Block, cert agreement.Certificate, err error) {
+	return l.blk, agreement.Certificate{}, nil
+}
+
+// TestProcessStageBlocksDownloadNilCert ensures StoreBlock does not receive a nil certificate when ledger has already had a block.
+// It uses two mocks catchpointAccessorMock and catchpointCatchupLedger2 and pre-crafted blocks to make a single iteration of processStageBlocksDownload.
+func TestProcessStageBlocksDownloadNilCert(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	var err error
+	blk1 := bookkeeping.Block{
+		BlockHeader: bookkeeping.BlockHeader{
+			Round: 1,
+			UpgradeState: bookkeeping.UpgradeState{
+				CurrentProtocol: protocol.ConsensusCurrentVersion,
+			},
+		},
+	}
+	blk1.TxnCommitments, err = blk1.PaysetCommit()
+	require.NoError(t, err)
+
+	blk2 := blk1
+	blk2.BlockHeader.Round = 2
+	blk2.BlockHeader.Branch = blk1.Hash()
+	blk2.TxnCommitments, err = blk2.PaysetCommit()
+	require.NoError(t, err)
+
+	ctx, cf := context.WithCancel(context.Background())
+	cs := CatchpointCatchupService{
+		ctx:            ctx,
+		cancelCtxFunc:  cf,
+		ledgerAccessor: &catchpointAccessorMock{topBlk: blk2, t: t},
+		ledger:         &catchpointCatchupLedger2{blk: blk1},
+		log:            logging.TestingLog(t),
+	}
+
+	err = cs.processStageBlocksDownload()
+	require.NoError(t, err)
+}

cmd/goal/application.go

@@ -488,7 +488,7 @@ var createAppCmd = &cobra.Command{
 				reportErrorf(errorBroadcastingTX, err2)
 			}
 
-			reportInfof("Attempting to create app (approval size %d, hash %v; clear size %d, hash %v)", len(approvalProg), crypto.HashObj(logic.Program(approvalProg)), len(clearProg), crypto.HashObj(logic.Program(clearProg)))
+			reportInfof("Attempting to create app (approval size %d, hash %v; clear size %d, hash %v)", len(approvalProg), crypto.HashObj(logic.Program(approvalProg)), len(clearProg), logic.HashProgram(clearProg))
 			reportInfof("Issued transaction from account %s, txid %s (fee %d)", tx.Sender, txid, tx.Fee.Raw)
 
 			if !noWaitAfterSend {
@@ -563,7 +563,7 @@ var updateAppCmd = &cobra.Command{
 				reportErrorf(errorBroadcastingTX, err2)
 			}
 
-			reportInfof("Attempting to update app (approval size %d, hash %v; clear size %d, hash %v)", len(approvalProg), crypto.HashObj(logic.Program(approvalProg)), len(clearProg), crypto.HashObj(logic.Program(clearProg)))
+			reportInfof("Attempting to update app (approval size %d, hash %v; clear size %d, hash %v)", len(approvalProg), crypto.HashObj(logic.Program(approvalProg)), len(clearProg), logic.HashProgram(clearProg))
 			reportInfof("Issued transaction from account %s, txid %s (fee %d)", tx.Sender, txid, tx.Fee.Raw)
 
 			if !noWaitAfterSend {
@@ -1455,9 +1455,9 @@ var methodAppCmd = &cobra.Command{
 
 		// Report tx details to user
 		if methodCreatesApp {
-			reportInfof("Attempting to create app (approval size %d, hash %v; clear size %d, hash %v)", len(approvalProg), crypto.HashObj(logic.Program(approvalProg)), len(clearProg), crypto.HashObj(logic.Program(clearProg)))
+			reportInfof("Attempting to create app (approval size %d, hash %v; clear size %d, hash %v)", len(approvalProg), crypto.HashObj(logic.Program(approvalProg)), len(clearProg), logic.HashProgram(clearProg))
 		} else if onCompletionEnum == transactions.UpdateApplicationOC {
-			reportInfof("Attempting to update app (approval size %d, hash %v; clear size %d, hash %v)", len(approvalProg), crypto.HashObj(logic.Program(approvalProg)), len(clearProg), crypto.HashObj(logic.Program(clearProg)))
+			reportInfof("Attempting to update app (approval size %d, hash %v; clear size %d, hash %v)", len(approvalProg), crypto.HashObj(logic.Program(approvalProg)), len(clearProg), logic.HashProgram(clearProg))
 		}
 
 		reportInfof("Issued %d transaction(s):", len(signedTxnGroup))

cmd/goal/interact.go

@@ -625,9 +625,9 @@ var appExecuteCmd = &cobra.Command{
 			}
 
 			if appIdx == 0 {
-				reportInfof("Attempting to create app (global ints %d, global blobs %d, local ints %d, local blobs %d, approval size %d, hash %v; clear size %d, hash %v)", globalSchema.NumUint, globalSchema.NumByteSlice, localSchema.NumUint, localSchema.NumByteSlice, len(approvalProg), crypto.HashObj(logic.Program(approvalProg)), len(clearProg), crypto.HashObj(logic.Program(clearProg)))
+				reportInfof("Attempting to create app (global ints %d, global blobs %d, local ints %d, local blobs %d, approval size %d, hash %v; clear size %d, hash %v)", globalSchema.NumUint, globalSchema.NumByteSlice, localSchema.NumUint, localSchema.NumByteSlice, len(approvalProg), logic.HashProgram(approvalProg), len(clearProg), crypto.HashObj(logic.Program(clearProg)))
 			} else if onCompletion == transactions.UpdateApplicationOC {
-				reportInfof("Attempting to update app (approval size %d, hash %v; clear size %d, hash %v)", len(approvalProg), crypto.HashObj(logic.Program(approvalProg)), len(clearProg), crypto.HashObj(logic.Program(clearProg)))
+				reportInfof("Attempting to update app (approval size %d, hash %v; clear size %d, hash %v)", len(approvalProg), crypto.HashObj(logic.Program(approvalProg)), len(clearProg), logic.HashProgram(clearProg))
 			}
 			reportInfof("Issued transaction from account %s, txid %s (fee %d)", tx.Sender, txid, tx.Fee.Raw)
 

cmd/goal/tealsign.go

@@ -139,7 +139,7 @@ The base64 encoding of the signature will always be printed to stdout. Optionall
 				reportErrorf(tealsignEmptyLogic)
 			}
 
-			progHash = crypto.HashObj(logic.Program(stxn.Lsig.Logic))
+			progHash = logic.HashProgram(stxn.Lsig.Logic)
 		} else {
 			// Otherwise, the contract address is the logic hash
 			parsedAddr, err := basics.UnmarshalChecksumAddress(contractAddr)

config/consensus.go

@@ -419,6 +419,11 @@ type ConsensusParams struct {
 	// their account balances.
 	StateProofExcludeTotalWeightWithRewards bool
 
+	// StateProofBlockHashInLightHeader specifies that the LightBlockHeader
+	// committed to by state proofs should contain the BlockHash of each
+	// block, instead of the seed.
+	StateProofBlockHashInLightHeader bool
+
 	// EnableAssetCloseAmount adds an extra field to the ApplyData. The field contains the amount of the remaining
 	// asset that were sent to the close-to address.
 	EnableAssetCloseAmount bool
@@ -759,15 +764,22 @@ func LoadConfigurableConsensusProtocols(dataDirectory string) error {
 		return err
 	}
 	if newConsensus != nil {
-		Consensus = newConsensus
-		// Set allocation limits
-		for _, p := range Consensus {
-			checkSetAllocBounds(p)
-		}
+		SetConfigurableConsensusProtocols(newConsensus)
 	}
 	return nil
 }
 
+// SetConfigurableConsensusProtocols sets the configurable protocols.
+func SetConfigurableConsensusProtocols(newConsensus ConsensusProtocols) ConsensusProtocols {
+	oldConsensus := Consensus
+	Consensus = newConsensus
+	// Set allocation limits
+	for _, p := range Consensus {
+		checkSetAllocBounds(p)
+	}
+	return oldConsensus
+}
+
 // PreloadConfigurableConsensusProtocols loads the configurable protocols from the data directory
 // and merge it with a copy of the Consensus map. Then, it returns it to the caller.
 func PreloadConfigurableConsensusProtocols(dataDirectory string) (ConsensusProtocols, error) {
@@ -1377,6 +1389,8 @@ func initConsensusProtocols() {
 	vFuture.LogicSigVersion = 10 // When moving this to a release, put a new higher LogicSigVersion here
 	vFuture.EnableLogicSigCostPooling = true
 
+	vFuture.StateProofBlockHashInLightHeader = true
+
 	// Setting DynamicFilterTimeout in vFuture will impact e2e test performance
 	// by reducing round time. Hence, it is commented out for now.
 	// vFuture.DynamicFilterTimeout = true

config/localTemplate.go

@@ -42,7 +42,7 @@ type Local struct {
 	// Version tracks the current version of the defaults so we can migrate old -> new
 	// This is specifically important whenever we decide to change the default value
 	// for an existing parameter. This field tag must be updated any time we add a new version.
-	Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28" version[29]:"29" version[30]:"30" version[31]:"31" version[32]:"32"`
+	Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28" version[29]:"29" version[30]:"30" version[31]:"31" version[32]:"32" version[33]:"33"`
 
 	// Archival nodes retain a full copy of the block history. Non-Archival nodes will delete old blocks and only retain what's need to properly validate blockchain messages (the precise number of recent blocks depends on the consensus parameters. Currently the last 1321 blocks are required). This means that non-Archival nodes require significantly less storage than Archival nodes. Relays (nodes with a valid NetAddress) are always Archival, regardless of this setting. This may change in the future. If setting this to true for the first time, the existing ledger may need to be deleted to get the historical values stored as the setting only effects current blocks forward. To do this, shutdown the node and delete all .sqlite files within the data/testnet-version directory, except the crash.sqlite file. Restart the node and wait for the node to sync.
 	Archival bool `version[0]:"false"`
@@ -375,6 +375,10 @@ type Local struct {
 	// 0 means don't store any, -1 mean unlimited and positive number suggest the maximum number of most recent catchpoint files to store.
 	CatchpointFileHistoryLength int `version[7]:"365"`
 
+	// EnableGossipService enables the gossip network HTTP websockets endpoint. The functionality of this depends on NetAddress, which must also be provided.
+	// This functionality is required for serving gossip traffic.
+	EnableGossipService bool `version[33]:"true"`
+
 	// EnableLedgerService enables the ledger serving service. The functionality of this depends on NetAddress, which must also be provided.
 	// This functionality is required for the catchpoint catchup.
 	EnableLedgerService bool `version[7]:"false"`

config/local_defaults.go

@@ -20,7 +20,7 @@
 package config
 
 var defaultLocal = Local{
-	Version:                                    32,
+	Version:                                    33,
 	AccountUpdatesStatsInterval:                5000000000,
 	AccountsRebuildSynchronousMode:             1,
 	AgreementIncomingBundlesQueueLength:        15,
@@ -70,6 +70,7 @@ var defaultLocal = Local{
 	EnableExperimentalAPI:                      false,
 	EnableFollowMode:                           false,
 	EnableGossipBlockService:                   true,
+	EnableGossipService:                        true,
 	EnableIncomingMessageFilter:                false,
 	EnableLedgerService:                        false,
 	EnableMetricReporting:                      false,

crypto/batchverifier.c

@@ -0,0 +1,20 @@
+#include "sodium.h"
+int ed25519_batch_wrapper(const unsigned char **messages2D,
+                          const unsigned char **publicKeys2D,
+                          const unsigned char **signatures2D,
+                          const unsigned char *messages1D,
+                          const unsigned long long *mlen,
+                          const unsigned char *publicKeys1D,
+                          const unsigned char *signatures1D,
+                          size_t num,
+                          int *valid) {
+    // fill 2-D arrays for messages, pks, sigs from provided 1-D arrays
+    unsigned long long mpos = 0;
+    for (size_t i = 0; i < num; i++) {
+        messages2D[i] = &messages1D[mpos];
+        mpos += mlen[i];
+        publicKeys2D[i] = &publicKeys1D[i*crypto_sign_ed25519_PUBLICKEYBYTES];
+        signatures2D[i] = &signatures1D[i*crypto_sign_ed25519_BYTES];
+    }
+    return crypto_sign_ed25519_open_batch(messages2D, mlen, publicKeys2D, signatures2D, num, valid);
+}