From af50da3ecfd5358b0ea12ca57126a1f6afe9d14a Mon Sep 17 00:00:00 2001 From: Philip Offtermatt <57488781+p-offtermatt@users.noreply.github.com> Date: Mon, 27 Nov 2023 09:12:39 +0100 Subject: [PATCH] Add fine-grained control of time (#88) * Start implementing auto-include-tx flag * Split testnet initialization and run * Add script to restart testnet from beginning * Start implementing auto-include-tx flag * Delete backup directory before taking backup * Uncomment killing processes with app binary * Rewrite TxQueue * Fix fire events method * Implement tx filtering and rechecks * Start implementing basic test for Tx effect * Ensure the script is not blocking * Rename Dockerfile-test to Dockerfile.test * Add Contains method * Remove initial setup from Dockerfile * Remove sleep and add -p 1 * Use big.Int for community pool size * Use testnet restart in tests instead of a new setup each time * Add cometmock args to be taken by startup script * Pull time handling into TimeHandler class * Fix CheckTx and time * Add test cases for AutoTx and block production * Add time control to README * Fix auto-tx flag * Add test for starting timestamp * Add test for starting time=system time (cherry picked from commit 7edb4c16410d9e6f5c3d26c1b62fb4a1031a4442) # Conflicts: # Dockerfile-test # cometmock/abci_client/client.go # cometmock/main.go # cometmock/rpc_server/routes.go # e2e-tests/local-testnet-singlechain.sh # e2e-tests/main_test.go --- .gitignore | 1 + Dockerfile.test | 48 ++ Makefile | 6 +- README.md | 12 +- cometmock/abci_client/client.go | 655 +++++++++++++++++- cometmock/abci_client/time_handler.go | 119 ++++ cometmock/main.go | 105 ++- cometmock/rpc_server/routes.go | 36 +- cometmock/utils/txs.go | 17 + e2e-tests/.gitignore | 4 + .../local-testnet-singlechain-restart.sh | 27 + e2e-tests/local-testnet-singlechain-setup.sh | 228 ++++++ e2e-tests/local-testnet-singlechain-start.sh | 9 + e2e-tests/local-testnet-singlechain.sh | 21 +- e2e-tests/main_test.go | 280 +++++++- e2e-tests/test_utils.go | 166 +++++ 16 files changed, 1674 insertions(+), 60 deletions(-) create mode 100644 Dockerfile.test create mode 100644 cometmock/abci_client/time_handler.go create mode 100644 cometmock/utils/txs.go create mode 100644 e2e-tests/.gitignore create mode 100755 e2e-tests/local-testnet-singlechain-restart.sh create mode 100755 e2e-tests/local-testnet-singlechain-setup.sh create mode 100755 e2e-tests/local-testnet-singlechain-start.sh create mode 100644 e2e-tests/test_utils.go diff --git a/.gitignore b/.gitignore index 4061428..e71a340 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ # Test binary, built with `go test -c` *.test +!Dockerfile.test # Output of the go coverage tool, specifically when used with LiteIDE *.out diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 0000000..c889496 --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,48 @@ +# import simd from ibc-go +FROM ghcr.io/cosmos/simapp:0.50.0-rc.1 AS simapp-builder + +FROM golang:1.21-alpine as cometmock-builder + +ENV PACKAGES curl make git libc-dev bash gcc linux-headers +RUN apk add --no-cache $PACKAGES + +ENV CGO_ENABLED=0 +ENV GOOS=linux +ENV GOFLAGS="-buildvcs=false" + +# cache gomodules for cometmock +ADD ./go.mod /go.mod +ADD ./go.sum /go.sum +RUN go mod download + +# Add CometMock and install it +ADD . /CometMock +WORKDIR /CometMock +RUN go build -o /usr/local/bin/cometmock ./cometmock + +RUN apk update +RUN apk add --no-cache which iputils procps-ng tmux net-tools htop jq gcompat + +FROM golang:1.21-alpine as test-env + +ENV PACKAGES curl make git libc-dev bash gcc linux-headers +RUN apk add --no-cache $PACKAGES +RUN apk update +RUN apk add --no-cache which iputils procps-ng tmux net-tools htop jq gcompat + +ENV CGO_ENABLED=0 +ENV GOOS=linux +ENV GOFLAGS="-buildvcs=false" + +ADD ./go.mod /go.mod +ADD ./go.sum /go.sum +RUN go mod download + +ADD ./e2e-tests /CometMock/e2e-tests + +COPY --from=simapp-builder /usr/bin/simd /usr/local/bin/simd + +WORKDIR /CometMock/e2e-tests +RUN /CometMock/e2e-tests/local-testnet-singlechain-setup.sh simd "" + +COPY --from=cometmock-builder /usr/local/bin/cometmock /usr/local/bin/cometmock \ No newline at end of file diff --git a/Makefile b/Makefile index 343ad1d..45ecc32 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,12 @@ install: go install ./cometmock test-locally: - go test -timeout 600s ./e2e-tests -test.v + go test -timeout 600s -p 1 ./e2e-tests -test.v test-docker: # Build the Docker image - docker build -f Dockerfile-test -t cometmock-test . + docker build -f Dockerfile.test -t cometmock-test . # Start a container and execute the test command inside docker rm cometmock-test-instance || true - docker run --name cometmock-test-instance --workdir /CometMock cometmock-test go test -timeout 600s ./e2e-tests -test.v \ No newline at end of file + docker run --name cometmock-test-instance --workdir /CometMock cometmock-test go test -p 1 -timeout 600s ./e2e-tests -test.v \ No newline at end of file diff --git a/README.md b/README.md index 2d088c5..f426856 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,19 @@ CometMock was tested with `go version go1.20.3 darwin/arm64`. To run CometMock, start your (cosmos-sdk) application instances with the flags ```--with-tendermint=false, --transport=grpc```. After the applications started, start CometMock like this ``` -cometmock [--block-time=XXX] {app_address1,app_address2,...} {genesis_file} {cometmock_listen_address} {home_folder1,home_folder2,...} {connection_mode} +cometmock [--block-time=value] [--auto-tx=] [--block-production-interval=] [--starting-timestamp=] [--starting-timestamp-from-genesis=] {app_address1,app_address2,...} {genesis_file} {cometmock_listen_address} {home_folder1,home_folder2,...} {connection_mode} ``` where: -* The `--block-time` flag is optional and specifies the time in milliseconds between blocks. The default is 1000ms(=1s). Values <= 0 mean that automatic block production is disabled. In this case, blocks can be produced by calling the `advance_blocks` endpoint or by broadcasting transactions (each transaction will be included in a fresh block). Note that all flags have to come before positional arguments. +* The `--block-time` flag is optional and specifies the time in milliseconds between the timestamps of consecutive blocks. +Values <= 0 mean that the timestamps are taken from the system time. The default value is -1. +* The `--auto-tx` flag is optional. If it is set to true, when a transaction is broadcasted, it will be automatically included in the next block. The default value is false. +* The `--block-production-interval` flag is optional and specifies the time (in milliseconds) to sleep between the production of consecutive blocks. +This does not mean that blocks are produced this fast, just that CometMock will sleep by this amount between producing two blocks. +The default value is 1000ms=1s. +* The `--starting-timestamp` flag is optional and specifies the starting timestamp of the blockchain. If not specified, the starting timestamp is taken from the system time. +* The `--starting-timestamp-from-genesis` flag is optional and can be used to override the starting timestamp of the blockchain with the timestamp of the genesis file. +In that case, the first block will have a timestamp of Genesis timestamp + block time or, if block time is <= 0, Genesis timestamp + some small, unspecified amount depending on system time. * The `app_addresses` are the `--address` flags of the applications. This is by default `"tcp://0.0.0.0:26658"` * The `genesis_file` is the genesis json that is also used by apps. * The `cometmock_listen_address` can be freely chosen and will be the address that requests that would normally go to CometBFT rpc endpoints need to be directed to. diff --git a/cometmock/abci_client/client.go b/cometmock/abci_client/client.go index 4f40e90..8b25e33 100644 --- a/cometmock/abci_client/client.go +++ b/cometmock/abci_client/client.go @@ -13,13 +13,18 @@ import ( "github.com/cometbft/cometbft/crypto/merkle" cometlog "github.com/cometbft/cometbft/libs/log" cmtmath "github.com/cometbft/cometbft/libs/math" +<<<<<<< HEAD cmtstate "github.com/cometbft/cometbft/proto/tendermint/state" cmttypes "github.com/cometbft/cometbft/proto/tendermint/types" +======= + cmtproto "github.com/cometbft/cometbft/proto/tendermint/types" +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) "github.com/cometbft/cometbft/state" blockindexkv "github.com/cometbft/cometbft/state/indexer/block/kv" "github.com/cometbft/cometbft/state/txindex" indexerkv "github.com/cometbft/cometbft/state/txindex/kv" "github.com/cometbft/cometbft/types" + cmttypes "github.com/cometbft/cometbft/types" "github.com/informalsystems/CometMock/cometmock/storage" "github.com/informalsystems/CometMock/cometmock/utils" ) @@ -43,6 +48,13 @@ const ( Equivocation ) +<<<<<<< HEAD +======= +// hardcode max data bytes to -1 (unlimited) since we do not utilize a mempool +// to pick evidence/txs out of +const maxDataBytes = cmttypes.MaxBlockSizeBytes + +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) // AbciClient facilitates calls to the ABCI interface of multiple nodes. // It also tracks the current state and a common logger. type AbciClient struct { @@ -68,24 +80,34 @@ type AbciClient struct { signingStatus map[string]bool signingStatusMutex sync.RWMutex - // time offset. whenever we qury the time, we add this offset to it - // this means after modifying this, blocks will have the timestamp offset by this value. - // this will look to the app like one block took a long time to be produced. - timeOffset time.Duration + // The TimeHandler that will be queried + // to obtain the block timestamp for each block. + TimeHandler TimeHandler + + // If this is true, then when broadcastTx is called, + // a block will automatically be produced immediately. + // If not, the transaction will be added to the TxQueue + // and consumed when the next block is created. + AutoIncludeTx bool + + // A list of transactions that will be included in the next block that is created. + // When transaction FreshTxQueue[i] is included, it will be removed from the FreshTxQueue, + // and the result will be sent to ResponseChannelQueue[i]. + // + FreshTxQueue []types.Tx + StaleTxQueue []types.Tx } -func (a *AbciClient) GetTimeOffset() time.Duration { - return a.timeOffset +func (a *AbciClient) QueueTx(tx types.Tx) { + // lock the block mutex so txs are not queued while a block is being run + blockMutex.Lock() + a.FreshTxQueue = append(a.FreshTxQueue, tx) + blockMutex.Unlock() } -func (a *AbciClient) IncrementTimeOffset(additionalOffset time.Duration) error { - if additionalOffset < 0 { - a.Logger.Error("time offset cannot be decremented, please provide a positive offset") - return fmt.Errorf("time offset cannot be decremented, please provide a positive offset") - } - a.Logger.Debug("Incrementing time offset", "additionalOffset", additionalOffset.String()) - a.timeOffset = a.timeOffset + additionalOffset - return nil +func (a *AbciClient) ClearTxs() { + a.FreshTxQueue = make([]types.Tx, 0) + a.StaleTxQueue = make([]types.Tx, 0) } func (a *AbciClient) CauseLightClientAttack(address string, misbehaviourType string) error { @@ -109,7 +131,11 @@ func (a *AbciClient) CauseLightClientAttack(address string, misbehaviourType str return fmt.Errorf("unknown misbehaviour type %s, possible types are: Equivocation, Lunatic, Amnesia", misbehaviourType) } +<<<<<<< HEAD _, _, _, _, _, err = a.RunBlockWithEvidence(nil, map[*types.Validator]MisbehaviourType{validator: misbehaviour}) +======= + err = a.RunBlockWithEvidence(map[*types.Validator]MisbehaviourType{validator: misbehaviour}) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) return err } @@ -121,8 +147,12 @@ func (a *AbciClient) CauseDoubleSign(address string) error { return err } +<<<<<<< HEAD _, _, _, _, _, err = a.RunBlockWithEvidence(nil, map[*types.Validator]MisbehaviourType{validator: DuplicateVote}) return err +======= + return a.RunBlockWithEvidence(map[*types.Validator]MisbehaviourType{validator: DuplicateVote}) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } func (a *AbciClient) GetValidatorFromAddress(address string) (*types.Validator, error) { @@ -192,7 +222,20 @@ func CreateAndStartIndexerService(eventBus *types.EventBus, logger cometlog.Logg return indexerService, txIndexer, blockIndexer, indexerService.Start() } +<<<<<<< HEAD func NewAbciClient(clients []AbciCounterpartyClient, logger cometlog.Logger, curState state.State, lastBlock *types.Block, lastCommit *types.Commit, storage storage.Storage, privValidators map[string]types.PrivValidator, errorOnUnequalResponses bool) *AbciClient { +======= +func NewAbciClient( + clients map[string]AbciCounterpartyClient, + logger cometlog.Logger, + curState state.State, + lastBlock *types.Block, + lastCommit *types.ExtendedCommit, + storage storage.Storage, + timeHandler TimeHandler, + errorOnUnequalResponses bool, +) *AbciClient { +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) signingStatus := make(map[string]bool) for addr := range privValidators { signingStatus[addr] = true @@ -222,8 +265,10 @@ func NewAbciClient(clients []AbciCounterpartyClient, logger cometlog.Logger, cur IndexerService: indexerService, TxIndex: txIndex, BlockIndex: blockIndex, + TimeHandler: timeHandler, ErrorOnUnequalResponses: errorOnUnequalResponses, signingStatus: signingStatus, + FreshTxQueue: make([]types.Tx, 0), } } @@ -545,10 +590,11 @@ func (a *AbciClient) SendCommit() (*abcitypes.ResponseCommit, error) { return responses[0].(*abcitypes.ResponseCommit), nil } -func (a *AbciClient) SendCheckTx(tx *[]byte) (*abcitypes.ResponseCheckTx, error) { +func (a *AbciClient) SendCheckTx(checkType abcitypes.CheckTxType, tx *[]byte) (*abcitypes.ResponseCheckTx, error) { // build the CheckTx request checkTxRequest := abcitypes.RequestCheckTx{ - Tx: *tx, + Tx: *tx, + Type: checkType, } // send CheckTx to all clients and collect the responses @@ -635,7 +681,11 @@ func (a *AbciClient) SendAbciQuery(data []byte, path string, height int64, prove // RunEmptyBlocks runs a specified number of empty blocks through ABCI. func (a *AbciClient) RunEmptyBlocks(numBlocks int) error { for i := 0; i < numBlocks; i++ { +<<<<<<< HEAD _, _, _, _, _, err := a.RunBlock(nil) +======= + err := a.RunBlock() +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) if err != nil { return err } @@ -643,16 +693,137 @@ func (a *AbciClient) RunEmptyBlocks(numBlocks int) error { return nil } +<<<<<<< HEAD // RunBlock runs a block with a specified transaction through the ABCI application. // It calls RunBlockWithTimeAndProposer with the current time and the LastValidators.Proposer. func (a *AbciClient) RunBlock(tx *[]byte) (*abcitypes.ResponseBeginBlock, *abcitypes.ResponseCheckTx, *abcitypes.ResponseDeliverTx, *abcitypes.ResponseEndBlock, *abcitypes.ResponseCommit, error) { return a.RunBlockWithTimeAndProposer(tx, time.Now().Add(a.timeOffset), a.CurState.LastValidators.Proposer, make(map[*types.Validator]MisbehaviourType, 0)) +======= +func (a *AbciClient) decideProposal( + proposerApp *AbciCounterpartyClient, + proposerVal *types.Validator, + height int64, + round int32, + txs *types.Txs, + misbehaviour []types.Evidence, +) (*types.Proposal, *types.Block, error) { + var block *types.Block + var blockParts *types.PartSet + + // Create a new proposal block from state/txs from the mempool. + var err error + numTxs := len(*txs) + _ = numTxs + block, err = a.CreateProposalBlock( + proposerApp, + proposerVal, + height, + a.CurState, + a.LastCommit, + txs, + &misbehaviour, + ) + if err != nil { + return nil, nil, err + } else if block == nil { + panic("Method createProposalBlock should not provide a nil block without errors") + } + blockParts, err = block.MakePartSet(types.BlockPartSizeBytes) + if err != nil { + return nil, nil, fmt.Errorf("unable to create proposal block part set: %v", err) + } + + // Make proposal + propBlockID := types.BlockID{Hash: block.Hash(), PartSetHeader: blockParts.Header()} + proposal := types.NewProposal(height, round, 0, propBlockID) + p := proposal.ToProto() + if err := proposerApp.PrivValidator.SignProposal(a.CurState.ChainID, p); err == nil { + proposal.Signature = p.Signature + + // TODO: evaluate if we need to emulate message sending + // send proposal and block parts on internal msg queue + // cs.sendInternalMessage(msgInfo{&ProposalMessage{proposal}, ""}) + + // for i := 0; i < int(blockParts.Total()); i++ { + // part := blockParts.GetPart(i) + // cs.sendInternalMessage(msgInfo{&BlockPartMessage{cs.Height, cs.Round, part}, ""}) + // } + + a.Logger.Debug("signed proposal", "height", height, "round", round, "proposal", proposal) + } else { + a.Logger.Error("propose step; failed signing proposal", "height", height, "round", round, "err", err) + } + + return proposal, block, nil +} + +// Create a proposal block with the given height and proposer, +// and including the given tx and misbehaviour. +// Essentially a hollowed-out version of CreateProposalBlock in CometBFT, see +// https://github.com/cometbft/cometbft/blob/33d276831843854881e6365b9696ac39dda12922/state/execution.go#L101 +func (a *AbciClient) CreateProposalBlock( + proposerApp *AbciCounterpartyClient, + proposerVal *types.Validator, + height int64, + curState state.State, + lastExtCommit *types.ExtendedCommit, + txs *types.Txs, + misbehaviour *[]types.Evidence, +) (*types.Block, error) { + commit := lastExtCommit.ToCommit() + + block := curState.MakeBlock(height, *txs, commit, *misbehaviour, proposerVal.Address) + + request := &abcitypes.RequestPrepareProposal{ + MaxTxBytes: maxDataBytes, + Txs: block.Txs.ToSliceOfBytes(), + LocalLastCommit: utils.BuildExtendedCommitInfo(lastExtCommit, curState.LastValidators, curState.InitialHeight, curState.ConsensusParams.ABCI), + Misbehavior: block.Evidence.Evidence.ToABCI(), + Height: block.Height, + Time: block.Time, + NextValidatorsHash: block.NextValidatorsHash, + ProposerAddress: block.ProposerAddress, + } + + ctx, cancel := context.WithTimeout(context.Background(), ABCI_TIMEOUT) + response, err := proposerApp.Client.PrepareProposal(ctx, request) + cancel() + if err != nil { + // We panic, since there is no meaninful recovery we can perform here. + panic(err) + } + + modifiedTxs := response.GetTxs() + txl := types.ToTxs(modifiedTxs) + if err := txl.Validate(maxDataBytes); err != nil { + return nil, err + } + + return curState.MakeBlock(height, txl, commit, *misbehaviour, block.ProposerAddress), nil +} + +// RunBlock runs a block with a specified transaction through the ABCI application. +// It calls RunBlockWithTimeAndProposer with the current time and the LastValidators.Proposer. +func (a *AbciClient) RunBlock() error { + blockTime := a.TimeHandler.GetBlockTime(a.LastBlock.Time) + return a.RunBlockWithTimeAndProposer(blockTime, a.CurState.LastValidators.Proposer, make(map[*types.Validator]MisbehaviourType, 0)) +} + +func (a *AbciClient) RunBlockWithTime(t time.Time) error { + return a.RunBlockWithTimeAndProposer(t, a.CurState.LastValidators.Proposer, make(map[*types.Validator]MisbehaviourType, 0)) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } // RunBlockWithEvidence runs a block with a specified transaction through the ABCI application. // It also produces the specified evidence for the specified misbehaving validators. +<<<<<<< HEAD func (a *AbciClient) RunBlockWithEvidence(tx *[]byte, misbehavingValidators map[*types.Validator]MisbehaviourType) (*abcitypes.ResponseBeginBlock, *abcitypes.ResponseCheckTx, *abcitypes.ResponseDeliverTx, *abcitypes.ResponseEndBlock, *abcitypes.ResponseCommit, error) { return a.RunBlockWithTimeAndProposer(tx, time.Now().Add(a.timeOffset), a.CurState.LastValidators.Proposer, misbehavingValidators) +======= +func (a *AbciClient) RunBlockWithEvidence(misbehavingValidators map[*types.Validator]MisbehaviourType) error { + blockTime := a.TimeHandler.GetBlockTime(a.LastBlock.Time) + return a.RunBlockWithTimeAndProposer(blockTime, a.CurState.LastValidators.Proposer, misbehavingValidators) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } func (a *AbciClient) ConstructDuplicateVoteEvidence(v *types.Validator) (*types.DuplicateVoteEvidence, error) { @@ -672,24 +843,24 @@ func (a *AbciClient) ConstructDuplicateVoteEvidence(v *types.Validator) (*types. index, valInLastState := lastState.Validators.GetByAddress(v.Address) // produce vote A. - voteA := &cmttypes.Vote{ + voteA := &cmtproto.Vote{ ValidatorAddress: v.Address, ValidatorIndex: int32(index), Height: lastBlock.Height, Round: 1, - Timestamp: time.Now().Add(a.timeOffset), - Type: cmttypes.PrecommitType, + Timestamp: lastBlock.Time, + Type: cmtproto.PrecommitType, BlockID: blockId.ToProto(), } // produce vote B, which just has a different round. - voteB := &cmttypes.Vote{ + voteB := &cmtproto.Vote{ ValidatorAddress: v.Address, ValidatorIndex: int32(index), Height: lastBlock.Height, Round: 2, // this is what differentiates the votes - Timestamp: time.Now().Add(a.timeOffset), - Type: cmttypes.PrecommitType, + Timestamp: lastBlock.Time, + Type: cmtproto.PrecommitType, BlockID: blockId.ToProto(), } @@ -778,6 +949,7 @@ func (a *AbciClient) ConstructLightClientAttackEvidence( }, nil } +<<<<<<< HEAD // RunBlock runs a block with a specified transaction through the ABCI application. // It calls BeginBlock, DeliverTx, EndBlock, Commit and then // updates the state. @@ -795,6 +967,144 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( defer blockMutex.Unlock() defer a.Logger.Debug("Unlocking mutex") +======= +// Calls ProcessProposal on a provided app, with the given block as +// proposed block. +func (a *AbciClient) ProcessProposal( + app *AbciCounterpartyClient, + block *types.Block, +) (bool, error) { + // call the temporary function on the client + timeoutContext, cancel := context.WithTimeout(context.Background(), ABCI_TIMEOUT) + defer cancel() + + response, err := app.Client.ProcessProposal(timeoutContext, &abcitypes.RequestProcessProposal{ + Hash: block.Header.Hash(), + Height: block.Header.Height, + Time: block.Header.Time, + Txs: block.Data.Txs.ToSliceOfBytes(), + ProposedLastCommit: utils.BuildLastCommitInfo(block, a.CurState.Validators, a.CurState.InitialHeight), + Misbehavior: block.Evidence.Evidence.ToABCI(), + ProposerAddress: block.ProposerAddress, + NextValidatorsHash: block.NextValidatorsHash, + }) + if err != nil { + return false, err + } + if response.IsStatusUnknown() { + panic(fmt.Sprintf("ProcessProposal responded with status %s", response.Status.String())) + } + + return response.IsAccepted(), nil +} + +func (a *AbciClient) ExtendAndSignVote( + app *AbciCounterpartyClient, + validator *types.Validator, + valIndex int32, + block *types.Block, +) (*types.Vote, error) { + // get the index of this validator in the current validator set + blockParts, err := block.MakePartSet(types.BlockPartSizeBytes) + if err != nil { + panic(fmt.Sprintf("error making block part set: %v", err)) + } + + vote := &types.Vote{ + ValidatorAddress: validator.Address, + ValidatorIndex: int32(valIndex), + Height: block.Height, + Round: block.LastCommit.Round, + Timestamp: block.Time, + Type: cmtproto.PrecommitType, + BlockID: types.BlockID{ + Hash: block.Hash(), + PartSetHeader: blockParts.Header(), + }, + } + + if a.CurState.ConsensusParams.ABCI.VoteExtensionsEnabled(vote.Height) { + ext, err := app.Client.ExtendVote(context.TODO(), &abcitypes.RequestExtendVote{ + Hash: vote.BlockID.Hash, + Height: vote.Height, + Time: block.Time, + Txs: block.Txs.ToSliceOfBytes(), + ProposedLastCommit: utils.BuildLastCommitInfo(block, a.CurState.Validators, a.CurState.InitialHeight), + Misbehavior: block.Evidence.Evidence.ToABCI(), + NextValidatorsHash: block.NextValidatorsHash, + ProposerAddress: block.ProposerAddress, + }) + if err != nil { + return nil, fmt.Errorf("error extending vote %v:\n %v", vote.String(), err) + } + vote.Extension = ext.VoteExtension + } + // going through ToProto looks weird but this is + // how signing is done in CometBFT https://github.com/cometbft/cometbft/blob/f63499c82c7defcdd82696f262f5a2eb495a3ac7/types/vote.go#L405 + protoVote := vote.ToProto() + err = app.PrivValidator.SignVote(a.CurState.ChainID, protoVote) + vote.Signature = protoVote.Signature + + vote.ExtensionSignature = nil + if a.CurState.ConsensusParams.ABCI.VoteExtensionsEnabled(vote.Height) { + vote.ExtensionSignature = protoVote.ExtensionSignature + } + if err != nil { + return nil, fmt.Errorf("error signing vote %v:\n %v", vote.String(), err) + } + return vote, nil +} + +// SendFinalizeBlock sends a FinalizeBlock request to all clients and collects the responses. +// The last commit of the AbciClient needs to be set when calling this. +func (a *AbciClient) SendFinalizeBlock( + block *types.Block, + lastCommitInfo *abcitypes.CommitInfo, +) (*abcitypes.ResponseFinalizeBlock, error) { + // build the FinalizeBlock request + request := abcitypes.RequestFinalizeBlock{ + Txs: block.Txs.ToSliceOfBytes(), + DecidedLastCommit: *lastCommitInfo, + Misbehavior: block.Evidence.Evidence.ToABCI(), + Height: block.Height, + Hash: block.Hash(), + Time: block.Time, + ProposerAddress: block.ProposerAddress, + NextValidatorsHash: block.NextValidatorsHash, + } + + // send FinalizeBlock to all clients and collect the responses + responses := make([]*abcitypes.ResponseFinalizeBlock, 0) + for _, client := range a.Clients { + ctx, cancel := context.WithTimeout(context.Background(), ABCI_TIMEOUT) + response, err := client.Client.FinalizeBlock(ctx, &request) + cancel() + if err != nil { + return nil, err + } + responses = append(responses, response) + } + + if a.ErrorOnUnequalResponses { + // return an error if the responses are not all equal + for i := 1; i < len(responses); i++ { + if !reflect.DeepEqual(responses[i], responses[0]) { + return nil, fmt.Errorf("responses are not all equal: %v is not equal to %v", responses[i], responses[0]) + } + } + } + + return responses[0], nil +} + +// internal method that runs a block. +// Should only be used after locking the blockMutex. +func (a *AbciClient) runBlock_helper( + blockTime time.Time, + proposer *types.Validator, + misbehavingValidators map[*types.Validator]MisbehaviourType, +) error { +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) a.Logger.Info("Running block") if verbose { a.Logger.Info("State at start of block", "state", a.CurState) @@ -802,17 +1112,51 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( newHeight := a.CurState.LastBlockHeight + 1 +<<<<<<< HEAD txs := make([]types.Tx, 0) if tx != nil { txs = append(txs, *tx) +======= + var err error + + for index, tx := range a.FreshTxQueue { + txBytes := []byte(tx) + resCheckTx, err := a.SendCheckTx(abcitypes.CheckTxType_New, &txBytes) + if err != nil { + return fmt.Errorf("error from CheckTx: %v", err) + } + // if the CheckTx code is != 0 + if resCheckTx.Code != abcitypes.CodeTypeOK { + // drop the tx by setting the index to empty + a.FreshTxQueue[index] = cmttypes.Tx{} + } +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } - var resCheckTx *abcitypes.ResponseCheckTx - var err error - if tx != nil { - resCheckTx, err = a.SendCheckTx(tx) + // recheck txs from the stale queue + for index, tx := range a.StaleTxQueue { + txBytes := []byte(tx) + resCheckTx, err := a.SendCheckTx(abcitypes.CheckTxType_Recheck, &txBytes) if err != nil { +<<<<<<< HEAD return nil, nil, nil, nil, nil, err +======= + return fmt.Errorf("error from CheckTx: %v", err) + } + // if the CheckTx code is != 0 + if resCheckTx.Code != abcitypes.CodeTypeOK { + // drop the tx by setting the index to empty + a.StaleTxQueue[index] = cmttypes.Tx{} + } + } + + // filter all empty txs from the queues + newTxQueue := make([]cmttypes.Tx, 0) + for _, tx := range append(a.FreshTxQueue, a.StaleTxQueue...) { + txBytes := []byte(tx) + if len(txBytes) > 0 { + newTxQueue = append(newTxQueue, tx) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } } @@ -836,12 +1180,17 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( } if err != nil { +<<<<<<< HEAD return nil, nil, nil, nil, nil, err +======= + return fmt.Errorf("error constructing evidence: %v", err) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } evidences = append(evidences, evidence) } +<<<<<<< HEAD block, _ := a.CurState.MakeBlock(a.CurState.LastBlockHeight+1, txs, a.LastCommit, evidences, proposerAddress) // override the block time, since we do not actually get votes from peers to median the time out of block.Time = blockTime @@ -852,11 +1201,82 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( commitSigs := []types.CommitSig{} +======= + var proposerApp *AbciCounterpartyClient + for _, c := range a.Clients { + if c.ValidatorAddress == proposerAddress.String() { + proposerApp = &c + break + } + } + + if proposerApp == nil { + return fmt.Errorf("could not find proposer app for address %v", proposerAddress) + } + + // The proposer runs PrepareProposal + txs := cmttypes.Txs(newTxQueue) + _, block, err := a.decideProposal( + proposerApp, + proposer, + a.CurState.LastBlockHeight+1, + 0, + &txs, + evidences, + ) + + // set the block time to the time passed as argument + block.Time = blockTime + + // clear the tx queues + a.ClearTxs() + + // for each tx not included in the block, + // put it in the stale queue + for _, tx := range newTxQueue { + if !utils.Contains(block.Txs, tx) { + a.StaleTxQueue = append(a.StaleTxQueue, tx) + } + } + + if err != nil { + return fmt.Errorf("error in decideProposal: %v", err) + } + + var nonProposers []*AbciCounterpartyClient + for _, val := range a.CurState.Validators.Validators { + client, err := a.GetCounterpartyFromAddress(val.Address.String()) + if err != nil { + return fmt.Errorf("error when getting counterparty client from address: address %v, error %v", val.Address.String(), err) + } + + if client.ValidatorAddress != proposerAddress.String() { + nonProposers = append(nonProposers, client) + } + } + + // non-proposers run ProcessProposal + for _, client := range nonProposers { + accepted, err := a.ProcessProposal(client, block) + if err != nil { + return fmt.Errorf("error in ProcessProposal for block %v, error %v", block.String(), err) + } + + if !accepted { + return fmt.Errorf("non-proposer %v did not accept the proposal for block %v", client.ValidatorAddress, block.String()) + } + } + + votes := []*types.Vote{} + + // sign the block with all current validators, and call ExtendVote (if necessary) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) for index, val := range a.CurState.Validators.Validators { privVal := a.PrivValidators[val.Address.String()] shouldSign, err := a.GetSigningStatus(val.Address.String()) if err != nil { +<<<<<<< HEAD return nil, nil, nil, nil, nil, err } @@ -870,6 +1290,19 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( Timestamp: time.Now().Add(a.timeOffset), Type: cmttypes.PrecommitType, BlockID: blockId.ToProto(), +======= + return fmt.Errorf("error getting signing status for validator %v, error %v", val.Address.String(), err) + } + + if shouldSign { + client, ok := a.Clients[val.Address.String()] + if !ok { + return fmt.Errorf("did not find privval for address: address %v", val.Address.String()) + } + vote, err := a.ExtendAndSignVote(&client, val, int32(index), block) + if err != nil { + return fmt.Errorf("error when signing vote for validator %v, error %v", val.Address.String(), err) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } err = privVal.SignVote(a.CurState.ChainID, vote) @@ -890,17 +1323,99 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( } } +<<<<<<< HEAD a.LastCommit = types.NewCommit( block.Height, 1, *blockId, commitSigs, ) +======= + // verify vote extensions if necessary + if a.CurState.ConsensusParams.ABCI.VoteExtensionsEnabled(block.Height) { + for _, val := range a.CurState.Validators.Validators { + a.Logger.Info("Verifying vote extension for validator", val.Address.String()) + client, err := a.GetCounterpartyFromAddress(val.Address.String()) + if err != nil { + return fmt.Errorf("error when getting counterparty client from address: address %v, error %v", val.Address.String(), err) + } + + for _, vote := range votes { + if vote != nil && vote.ValidatorAddress.String() != client.ValidatorAddress { + // make a context to time out the request + ctx, cancel := context.WithTimeout(context.Background(), ABCI_TIMEOUT) + + resp, err := client.Client.VerifyVoteExtension(ctx, &abcitypes.RequestVerifyVoteExtension{ + Hash: block.Hash(), + ValidatorAddress: vote.ValidatorAddress, + Height: block.Height, + VoteExtension: vote.Extension, + }) + cancel() + // recovering from errors of VerifyVoteExtension seems hard because applications + // are typically not supposed to reject valid extensions created by ExtendVote. + if err != nil { + panic(fmt.Errorf("verify vote extension failed with error %v", err)) + } + + if resp.IsStatusUnknown() { + panic(fmt.Sprintf("verify vote extension responded with status %s", resp.Status.String())) + } + + if !resp.IsAccepted() { + panic(fmt.Sprintf("Verify vote extension rejected an extension for vote %v", vote.String())) + } + } + } + } + } + + // if vote extensions are enabled, we need an extended vote set + // otherwise, we need a regular vote set + var voteSet *types.VoteSet + if a.CurState.ConsensusParams.ABCI.VoteExtensionsEnabled(block.Height) { + voteSet = types.NewExtendedVoteSet( + a.CurState.ChainID, + block.Height, + 0, // round is hardcoded to 0 + cmtproto.PrecommitType, + a.CurState.Validators, + ) + } else { + voteSet = types.NewVoteSet( + a.CurState.ChainID, + block.Height, + 0, // round is hardcoded to 0 + cmtproto.PrecommitType, + a.CurState.Validators, + ) + } + + // add the votes to the vote set + for _, vote := range votes { + if vote != nil { + added, err := voteSet.AddVote(vote) + if err != nil { + return fmt.Errorf("error adding vote %v to vote set: %v", vote.String(), err) + } + if !added { + return fmt.Errorf("could not add vote %v to vote set", vote.String()) + } + } + } + + // set the last commit to the vote set + a.LastCommit = voteSet.MakeExtendedCommit(a.CurState.ConsensusParams.ABCI) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) // sanity check that the commit is signed correctly err = a.CurState.Validators.VerifyCommitLightTrusting(a.CurState.ChainID, a.LastCommit, cmtmath.Fraction{Numerator: 1, Denominator: 3}) if err != nil { +<<<<<<< HEAD return nil, nil, nil, nil, nil, err +======= + return fmt.Errorf("error verifying commit %v: %v", a.LastCommit.ToCommit().StringIndented("\t"), err) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } // sanity check that the commit makes a proper light block @@ -917,11 +1432,16 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( err = lightBlock.ValidateBasic(a.CurState.ChainID) if err != nil { a.Logger.Error("Light block validation failed", "err", err) +<<<<<<< HEAD return nil, nil, nil, nil, nil, err +======= + return err +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } resBeginBlock, err := a.SendBeginBlock(block) if err != nil { +<<<<<<< HEAD return nil, nil, nil, nil, nil, err } @@ -943,6 +1463,9 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( deliverTxResponses := []*abcitypes.ResponseDeliverTx{} if tx != nil { deliverTxResponses = append(deliverTxResponses, resDeliverTx) +======= + return fmt.Errorf("error from FinalizeBlock for block %v: %v", block.String(), err) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } // lock the state update mutex while the stores are updated to avoid @@ -953,34 +1476,75 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( // copy state so that the historical state is not mutated state := a.CurState.Copy() +<<<<<<< HEAD // build components of the state update, then call the update function abciResponses := cmtstate.ABCIResponses{ DeliverTxs: deliverTxResponses, EndBlock: resEndBlock, BeginBlock: resBeginBlock, +======= + // insert entries into the storage + err = a.Storage.UpdateStores(newHeight, block, a.LastCommit.ToCommit(), &state, resFinalizeBlock) + if err != nil { + return fmt.Errorf("error updating stores: %v", err) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } // insert entries into the storage err = a.Storage.UpdateStores(newHeight, block, a.LastCommit, &state, &abciResponses) if err != nil { +<<<<<<< HEAD return nil, nil, nil, nil, nil, err +======= + return fmt.Errorf("error getting block id from block %v: %v", block.String(), err) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } // updates state as a side effect. returns an error if the state update fails err = a.UpdateStateFromBlock(blockId, block, abciResponses) if err != nil { +<<<<<<< HEAD return nil, nil, nil, nil, nil, err +======= + return fmt.Errorf("error updating state for result %v, block %v: %v", resFinalizeBlock.String(), block.String(), err) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } // unlock the state mutex, since we are done updating state a.Storage.UnlockAfterStateUpdate() - resCommit, err := a.SendCommit() + _, err = a.SendCommit() if err != nil { +<<<<<<< HEAD return nil, nil, nil, nil, nil, err +======= + return fmt.Errorf("error from Commit for block %v: %v", block.String(), err) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } a.CurState.AppHash = resCommit.Data +<<<<<<< HEAD return resBeginBlock, resCheckTx, resDeliverTx, resEndBlock, resCommit, nil +======= + return nil +} + +// RunBlock RunBlockWithTimeAndProposer runs a block through the ABCI application. +// RunBlock is safe for use by multiple goroutines simultaneously. +func (a *AbciClient) RunBlockWithTimeAndProposer( + blockTime time.Time, + proposer *types.Validator, + misbehavingValidators map[*types.Validator]MisbehaviourType, +) error { + // lock mutex to avoid running two blocks at the same time + a.Logger.Debug("Locking mutex") + blockMutex.Lock() + + err := a.runBlock_helper(blockTime, proposer, misbehavingValidators) + + blockMutex.Unlock() + a.Logger.Debug("Unlocking mutex") + return err +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } // UpdateStateFromBlock updates the AbciClients state @@ -1020,7 +1584,11 @@ func (a *AbciClient) UpdateStateFromBlock( // Events are fired after everything else. // NOTE: if we crash between Commit and Save, events wont be fired during replay +<<<<<<< HEAD fireEvents(a.Logger, &a.EventBus, block, &abciResponses, validatorUpdates) +======= + fireEvents(a.Logger, &a.EventBus, block, *blockId, finalizeBlockRes, validatorUpdates) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) return nil } @@ -1117,10 +1685,14 @@ func validateValidatorUpdates( return nil } +// Fire NewBlock, NewBlockHeader. +// Fire TxEvent for every tx. +// NOTE: if CometBFT crashes before commit, some or all of these events may be published again. func fireEvents( logger cometlog.Logger, eventBus types.BlockEventPublisher, block *types.Block, +<<<<<<< HEAD abciResponses *cmtstate.ABCIResponses, validatorUpdates []*types.Validator, ) { @@ -1128,19 +1700,40 @@ func fireEvents( Block: block, ResultBeginBlock: *abciResponses.BeginBlock, ResultEndBlock: *abciResponses.EndBlock, +======= + blockID types.BlockID, + abciResponse *abcitypes.ResponseFinalizeBlock, + validatorUpdates []*types.Validator, +) { + if err := eventBus.PublishEventNewBlock(types.EventDataNewBlock{ + Block: block, + BlockID: blockID, + ResultFinalizeBlock: *abciResponse, +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) }); err != nil { logger.Error("failed publishing new block", "err", err) } +<<<<<<< HEAD eventDataNewBlockHeader := types.EventDataNewBlockHeader{ Header: block.Header, NumTxs: int64(len(block.Txs)), ResultBeginBlock: *abciResponses.BeginBlock, ResultEndBlock: *abciResponses.EndBlock, +======= + if err := eventBus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{ + Header: block.Header, + }); err != nil { + logger.Error("failed publishing new block header", "err", err) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } - if err := eventBus.PublishEventNewBlockHeader(eventDataNewBlockHeader); err != nil { - logger.Error("failed publishing new block header", "err", err) + if err := eventBus.PublishEventNewBlockEvents(types.EventDataNewBlockEvents{ + Height: block.Height, + Events: abciResponse.Events, + NumTxs: int64(len(block.Txs)), + }); err != nil { + logger.Error("failed publishing new block events", "err", err) } if len(block.Evidence.Evidence) != 0 { @@ -1159,7 +1752,11 @@ func fireEvents( Height: block.Height, Index: uint32(i), Tx: tx, +<<<<<<< HEAD Result: *(abciResponses.DeliverTxs[i]), +======= + Result: *(abciResponse.TxResults[i]), +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) }}); err != nil { logger.Error("failed publishing event TX", "err", err) } diff --git a/cometmock/abci_client/time_handler.go b/cometmock/abci_client/time_handler.go new file mode 100644 index 0000000..e11a1a9 --- /dev/null +++ b/cometmock/abci_client/time_handler.go @@ -0,0 +1,119 @@ +package abci_client + +import ( + "sync" + "time" +) + +// A TimeHandler is responsible for +// deciding the timestamps of blocks. +// It will be called by AbciClient.RunBlock +// to decide on a block time. +// It may decide the time based on any number of factors, +// and the parameters of its methods might expand over time as needed. +// The TimeHandler does not have a way to decide the time of the first block, +// which is expected to be done externally, e.g. from the Genesis. +type TimeHandler interface { + // CONTRACT: TimeHandler.GetBlockTime will be called + // precisely once for each block after the first. + // It returns the timestamp of the next block. + GetBlockTime(lastBlockTimestamp time.Time) time.Time + + // AdvanceTime advances the timestamp of all following blocks by + // the given duration. + // The duration needs to be non-negative. + // It returns the timestamp that the next block would have if it + // was produced now. + AdvanceTime(duration time.Duration) time.Time +} + +// The SystemClockTimeHandler uses the system clock +// to decide the timestamps of blocks. +// It will return the system time + offset for each block. +// The offset is calculated by the initial timestamp +// + the sum of all durations passed to AdvanceTime. +type SystemClockTimeHandler struct { + // The offset to add to the system time. + curOffset time.Duration + + // A mutex that ensures that there are no concurrent calls + // to AdvanceTime + mutex sync.Mutex +} + +func NewSystemClockTimeHandler(initialTimestamp time.Time) *SystemClockTimeHandler { + return &SystemClockTimeHandler{ + curOffset: time.Since(initialTimestamp), + } +} + +func (s *SystemClockTimeHandler) GetBlockTime(lastBlockTimestamp time.Time) time.Time { + return time.Now().Add(s.curOffset) +} + +func (s *SystemClockTimeHandler) AdvanceTime(duration time.Duration) time.Time { + s.mutex.Lock() + defer s.mutex.Unlock() + + s.curOffset += duration + return time.Now().Add(s.curOffset) +} + +var _ TimeHandler = (*SystemClockTimeHandler)(nil) + +// The FixedBlockTimeHandler uses a fixed duration +// to advance the timestamp of a block compared to the previous block. +// The block timestamps therefore do not at all depend on the system time, +// but on the time of the previous block. +type FixedBlockTimeHandler struct { + // The fixed duration to add to the last block time + // when deciding the next block timestamp. + blockTime time.Duration + + // The offset to add to the last block time. + // This will be cleared after each block, + // but since the block time of the next block depends + // on the last block, + // this will shift the timestamps of all future blocks. + curBlockOffset time.Duration + + // A mutex that ensures that GetBlockTime and AdvanceTime + // are not called concurrently. + // Otherwise, the block offset might be put into a broken state. + mutex sync.Mutex + + // The timestamp of the last block we produced. + // If this is used before the first block is produced, + // it will be the zero time. + lastBlockTimestamp time.Time +} + +func NewFixedBlockTimeHandler(blockTime time.Duration) *FixedBlockTimeHandler { + return &FixedBlockTimeHandler{ + blockTime: blockTime, + curBlockOffset: 0, + } +} + +func (f *FixedBlockTimeHandler) GetBlockTime(lastBlockTimestamp time.Time) time.Time { + f.mutex.Lock() + defer f.mutex.Unlock() + + res := lastBlockTimestamp.Add(f.blockTime + f.curBlockOffset) + f.curBlockOffset = 0 + f.lastBlockTimestamp = res + return res +} + +// FixedBlockTimeHandler.AdvanceTime will only return the correct next block time +// after GetBlockTime has been called once, but it will +// still advance the time correctly before that - only the output will be wrong. +func (f *FixedBlockTimeHandler) AdvanceTime(duration time.Duration) time.Time { + f.mutex.Lock() + defer f.mutex.Unlock() + + f.curBlockOffset += duration + return f.lastBlockTimestamp.Add(f.blockTime + f.curBlockOffset) +} + +var _ TimeHandler = (*FixedBlockTimeHandler)(nil) diff --git a/cometmock/main.go b/cometmock/main.go index 46d7a09..52aebc4 100644 --- a/cometmock/main.go +++ b/cometmock/main.go @@ -40,7 +40,7 @@ func GetMockPVsFromNodeHomes(nodeHomes []string) []types.PrivValidator { func main() { logger := cometlog.NewTMLogger(cometlog.NewSyncWriter(os.Stdout)) - argumentString := "[--block-time=value] " + argumentString := "[--block-time=value] [--auto-tx=] [--block-production-interval=] [--starting-timestamp=] [--starting-timestamp-from-genesis=] " app := &cli.App{ Name: "cometmock", @@ -59,15 +59,51 @@ func main() { &cli.Int64Flag{ Name: "block-time", Usage: ` -Time between blocks in milliseconds. +The number of milliseconds by which the block timestamp should advance from one block to the next. +If this is <0, block timestamps will advance with the system time between the block productions. +Even then, it is still possible to shift the block time from the system time, e.g. by setting an initial timestamp +or by using the 'advance_time' endpoint.`, + Value: -1, + }, + &cli.BoolFlag{ + Name: "auto-tx", + Usage: ` +If this is true, transactions are included immediately +after they are received via broadcast_tx, i.e. a new block +is created when a BroadcastTx endpoint is hit. +If this is false, transactions are still included +upon creation of new blocks, but CometMock will not specifically produce +a new block when a transaction is broadcast.`, + Value: true, + }, + &cli.Int64Flag{ + Name: "block-production-interval", + Usage: ` +Time to sleep between blocks in milliseconds. To disable block production, set to 0. This will not necessarily mean block production is this fast - it is just the sleep time between blocks. -Setting this to a value <= 0 disables automatic block production. +Setting this to a value < 0 disables automatic block production. In this case, blocks are only produced when instructed explicitly either by advancing blocks or broadcasting transactions.`, Value: 1000, }, + &cli.Int64Flag{ + Name: "starting-timestamp", + Usage: ` +The timestamp to use for the first block, given in milliseconds since the unix epoch. +If this is < 0, the current system time is used. +If this is >= 0, the system time is ignored and this timestamp is used for the first block instead.`, + Value: -1, + }, + &cli.BoolFlag{ + Name: "starting-timestamp-from-genesis", + Usage: ` +If this is true, it overrides the starting-timestamp, and instead +bases the time for the first block on the genesis time, incremented by the block time +or the system time between creating the genesis request and producing the first block.`, + Value: false, + }, }, ArgsUsage: argumentString, Action: func(c *cli.Context) error { @@ -85,8 +121,8 @@ advancing blocks or broadcasting transactions.`, return cli.Exit(fmt.Sprintf("Invalid connection mode: %s. Connection mode must be either 'socket' or 'grpc'.\nUsage: %s", connectionMode, argumentString), 1) } - blockTime := c.Int("block-time") - fmt.Printf("Block time: %d\n", blockTime) + blockProductionInterval := c.Int("block-production-interval") + fmt.Printf("Block production interval: %d\n", blockProductionInterval) // read node homes from args nodeHomes := strings.Split(nodeHomesString, ",") @@ -104,8 +140,32 @@ advancing blocks or broadcasting transactions.`, logger.Error(err.Error()) } +<<<<<<< HEAD clients := []abci_client.AbciCounterpartyClient{} privValsMap := make(map[string]types.PrivValidator) +======= + // read starting timestamp from args + // if starting timestamp should be taken from genesis, + // read it from there + var startingTime time.Time + if c.Bool("starting-timestamp-from-genesis") { + startingTime = genesisDoc.GenesisTime + } else { + if c.Int64("starting-timestamp") < 0 { + startingTime = time.Now() + } else { + dur := time.Duration(c.Int64("starting-timestamp")) * time.Millisecond + startingTime = time.Unix(0, 0).Add(dur) + } + } + fmt.Printf("Starting time: %s\n", startingTime.Format(time.RFC3339)) + + // read block time from args + blockTime := time.Duration(c.Int64("block-time")) * time.Millisecond + fmt.Printf("Block time: %d\n", blockTime.Milliseconds()) + + clientMap := make(map[string]abci_client.AbciCounterpartyClient) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) for i, appAddress := range appAddresses { logger.Info("Connecting to client at %v", appAddress) @@ -147,6 +207,13 @@ advancing blocks or broadcasting transactions.`, privValsMap[addr.String()] = privVal } + var timeHandler abci_client.TimeHandler + if blockTime < 0 { + timeHandler = abci_client.NewSystemClockTimeHandler(startingTime) + } else { + timeHandler = abci_client.NewFixedBlockTimeHandler(blockTime) + } + abci_client.GlobalClient = abci_client.NewAbciClient( clients, logger, @@ -154,12 +221,21 @@ advancing blocks or broadcasting transactions.`, &types.Block{}, &types.Commit{}, &storage.MapStorage{}, +<<<<<<< HEAD privValsMap, true, ) // connect to clients abci_client.GlobalClient.RetryDisconnectedClients() +======= + timeHandler, + true, + ) + + abci_client.GlobalClient.AutoIncludeTx = c.Bool("auto-tx") + fmt.Printf("Auto include tx: %t\n", abci_client.GlobalClient.AutoIncludeTx) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) // initialize chain err = abci_client.GlobalClient.SendInitChain(curState, genesisDoc) @@ -168,8 +244,19 @@ advancing blocks or broadcasting transactions.`, panic(err) } + var firstBlockTime time.Time + if blockTime < 0 { + firstBlockTime = startingTime + } else { + firstBlockTime = startingTime.Add(blockTime) + } + // run an empty block +<<<<<<< HEAD _, _, _, _, _, err = abci_client.GlobalClient.RunBlock(nil) +======= + err = abci_client.GlobalClient.RunBlockWithTime(firstBlockTime) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) if err != nil { logger.Error(err.Error()) panic(err) @@ -177,15 +264,19 @@ advancing blocks or broadcasting transactions.`, go rpc_server.StartRPCServerWithDefaultConfig(cometMockListenAddress, logger) - if blockTime > 0 { + if blockProductionInterval > 0 { // produce blocks according to blockTime for { +<<<<<<< HEAD _, _, _, _, _, err := abci_client.GlobalClient.RunBlock(nil) +======= + err := abci_client.GlobalClient.RunBlock() +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) if err != nil { logger.Error(err.Error()) panic(err) } - time.Sleep(time.Millisecond * time.Duration(blockTime)) + time.Sleep(time.Millisecond * time.Duration(blockProductionInterval)) } } else { // wait forever diff --git a/cometmock/rpc_server/routes.go b/cometmock/rpc_server/routes.go index 86c272b..6bc10b3 100644 --- a/cometmock/rpc_server/routes.go +++ b/cometmock/rpc_server/routes.go @@ -10,7 +10,12 @@ import ( cmtmath "github.com/cometbft/cometbft/libs/math" cmtquery "github.com/cometbft/cometbft/libs/pubsub/query" "github.com/cometbft/cometbft/p2p" +<<<<<<< HEAD cometp2p "github.com/cometbft/cometbft/p2p" +======= + + abcitypes "github.com/cometbft/cometbft/abci/types" +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) ctypes "github.com/cometbft/cometbft/rpc/core/types" rpc "github.com/cometbft/cometbft/rpc/jsonrpc/server" rpctypes "github.com/cometbft/cometbft/rpc/jsonrpc/types" @@ -86,8 +91,8 @@ func AdvanceTime(ctx *rpctypes.Context, duration_in_seconds time.Duration) (*Res return nil, errors.New("duration to advance time by must be greater than 0") } - abci_client.GlobalClient.IncrementTimeOffset(duration_in_seconds * time.Second) - return &ResultAdvanceTime{time.Now().Add(abci_client.GlobalClient.GetTimeOffset())}, nil + res := abci_client.GlobalClient.TimeHandler.AdvanceTime(duration_in_seconds * time.Second) + return &ResultAdvanceTime{res}, nil } type ResultSetSigningStatus struct { @@ -433,16 +438,14 @@ func Health(ctx *rpctypes.Context) (*ctypes.ResultHealth, error) { return &ctypes.ResultHealth{}, nil } +// CURRENTLY UNSUPPORTED - THIS IS BECAUSE IT IS DISCOURAGED TO USE THIS BY COMETBFT +// needs some major changes to work with ABCI++ // BroadcastTxCommit broadcasts a transaction, // and wait until it is included in a block and and comitted. // In our case, this means running a block with just the the transition, // then return. func BroadcastTxCommit(ctx *rpctypes.Context, tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { - abci_client.GlobalClient.Logger.Info( - "BroadcastTxCommit called", "tx", tx) - - res, err := BroadcastTx(&tx) - return res, err + return nil, errors.New("BroadcastTxCommit is currently not supported. Try BroadcastTxSync or BroadcastTxAsync instead") } // BroadcastTxSync would normally broadcast a transaction and wait until it gets the result from CheckTx. @@ -481,25 +484,40 @@ func BroadcastTxAsync(ctx *rpctypes.Context, tx types.Tx) (*ctypes.ResultBroadca return &ctypes.ResultBroadcastTx{}, nil } -// BroadcastTx delivers a transaction to the ABCI client, includes it in the next block, then returns. func BroadcastTx(tx *types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { abci_client.GlobalClient.Logger.Info( "BroadcastTxs called", "tx", tx) +<<<<<<< HEAD byteTx := []byte(*tx) _, responseCheckTx, responseDeliverTx, _, _, err := abci_client.GlobalClient.RunBlock(&byteTx) +======= + txBytes := []byte(*tx) + checkTxResponse, err := abci_client.GlobalClient.SendCheckTx(abcitypes.CheckTxType_New, &txBytes) +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) if err != nil { return nil, err } + abci_client.GlobalClient.QueueTx(*tx) + + if abci_client.GlobalClient.AutoIncludeTx { + go abci_client.GlobalClient.RunBlock() + } - // TODO: fill the return value if necessary return &ctypes.ResultBroadcastTxCommit{ +<<<<<<< HEAD CheckTx: *responseCheckTx, DeliverTx: *responseDeliverTx, Height: abci_client.GlobalClient.LastBlock.Height, Hash: tx.Hash(), }, nil +======= + CheckTx: *checkTxResponse, + Hash: tx.Hash(), + Height: abci_client.GlobalClient.CurState.LastBlockHeight, + }, err +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) } func ABCIInfo(ctx *rpctypes.Context) (*ctypes.ResultABCIInfo, error) { diff --git a/cometmock/utils/txs.go b/cometmock/utils/txs.go new file mode 100644 index 0000000..92601e3 --- /dev/null +++ b/cometmock/utils/txs.go @@ -0,0 +1,17 @@ +package utils + +import ( + "bytes" + + cmttypes "github.com/cometbft/cometbft/types" +) + +// Contains returns true if txs contains tx, false otherwise. +func Contains(txs cmttypes.Txs, tx cmttypes.Tx) bool { + for _, ttx := range txs { + if bytes.Equal([]byte(ttx), []byte(tx)) { + return true + } + } + return false +} diff --git a/e2e-tests/.gitignore b/e2e-tests/.gitignore new file mode 100644 index 0000000..219563d --- /dev/null +++ b/e2e-tests/.gitignore @@ -0,0 +1,4 @@ +# Scripts that are generated by the testnet setup +start_apps.sh +start_cometmock.sh +cometmock_log \ No newline at end of file diff --git a/e2e-tests/local-testnet-singlechain-restart.sh b/e2e-tests/local-testnet-singlechain-restart.sh new file mode 100755 index 0000000..7edba3a --- /dev/null +++ b/e2e-tests/local-testnet-singlechain-restart.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# After the testnet was started, this script can restart it. +# It does so by killing the existing testnet, +# overwriting the node home directories with backups made +# right after initializatio, and then starting the testnet again. + + +BINARY_NAME=$1 + +set -eux + +ROOT_DIR=${HOME}/nodes/provider +BACKUP_DIR=${ROOT_DIR}_bkup + +if [ -z "$BINARY_NAME" ]; then + echo "Usage: $0 [cometmock_args]" + exit 1 +fi + +# Kill the testnet +pkill -f ^$BINARY_NAME &> /dev/null || true +pkill -f ^cometmock &> /dev/null || true + +# Restore the backup +rm -rf ${ROOT_DIR} +cp -r ${BACKUP_DIR} ${ROOT_DIR} \ No newline at end of file diff --git a/e2e-tests/local-testnet-singlechain-setup.sh b/e2e-tests/local-testnet-singlechain-setup.sh new file mode 100755 index 0000000..eb89c09 --- /dev/null +++ b/e2e-tests/local-testnet-singlechain-setup.sh @@ -0,0 +1,228 @@ +#!/bin/bash +## This script sets up the environment to run the single chain local testnet. +## Importantly, it does not actually start nodes (or cometmock) - instead, +## it will produce two scripts, start_apps.sh and start_cometmock.sh. +## After this script is done setting up, simply run these two scripts to run the +## testnet. +## The reason for this is that we want to be able to make the testnet setup +## differentiated from the actual run to allow for better caching in Docker. + +set -eux + +BINARY_NAME=$1 + +# User balance of stake tokens +USER_COINS="100000000000stake" +# Amount of stake tokens staked +STAKE="100000000stake" +# Node IP address +NODE_IP="127.0.0.1" + +# Home directory +HOME_DIR=$HOME + +rm -rf ./start_apps.sh +rm -rf ./start_cometmock.sh + +# Validator moniker +MONIKERS=("coordinator" "alice" "bob") +LEAD_VALIDATOR_MONIKER="coordinator" + +PROV_NODES_ROOT_DIR=${HOME_DIR}/nodes/provider +CONS_NODES_ROOT_DIR=${HOME_DIR}/nodes/consumer + +# Base port. Ports assigned after these ports sequentially by nodes. +RPC_LADDR_BASEPORT=29170 +P2P_LADDR_BASEPORT=29180 +GRPC_LADDR_BASEPORT=29190 +NODE_ADDRESS_BASEPORT=29200 +PPROF_LADDR_BASEPORT=29210 +CLIENT_BASEPORT=29220 + +# keeps a comma separated list of node addresses for provider and consumer +PROVIDER_NODE_LISTEN_ADDR_STR="" +CONSUMER_NODE_LISTEN_ADDR_STR="" + +# Strings that keep the homes of provider nodes and homes of consumer nodes +PROV_NODES_HOME_STR="" +CONS_NODES_HOME_STR="" + +PROVIDER_COMETMOCK_ADDR=tcp://$NODE_IP:22331 +CONSUMER_COMETMOCK_ADDR=tcp://$NODE_IP:22332 + +# Clean start +pkill -f ^$BINARY_NAME &> /dev/null || true +pkill -f ^cometmock &> /dev/null || true +sleep 1 +rm -rf ${PROV_NODES_ROOT_DIR} +rm -rf ${CONS_NODES_ROOT_DIR} + +# Let lead validator create genesis file +LEAD_VALIDATOR_PROV_DIR=${PROV_NODES_ROOT_DIR}/provider-${LEAD_VALIDATOR_MONIKER} +LEAD_VALIDATOR_CONS_DIR=${CONS_NODES_ROOT_DIR}/consumer-${LEAD_VALIDATOR_MONIKER} +LEAD_PROV_KEY=${LEAD_VALIDATOR_MONIKER}-key +LEAD_PROV_LISTEN_ADDR=tcp://${NODE_IP}:${RPC_LADDR_BASEPORT} + +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${CONS_NODES_ROOT_DIR}/consumer-${MONIKER} + + # Build genesis file and node directory structure + $BINARY_NAME init $MONIKER --chain-id provider --home ${PROV_NODE_DIR} + jq ".app_state.gov.params.voting_period = \"100000s\" | .app_state.staking.params.unbonding_time = \"86400s\" | .app_state.slashing.params.signed_blocks_window=\"1000\" " \ + ${PROV_NODE_DIR}/config/genesis.json > \ + ${PROV_NODE_DIR}/edited_genesis.json && mv ${PROV_NODE_DIR}/edited_genesis.json ${PROV_NODE_DIR}/config/genesis.json + + + sleep 1 + + # Create account keypair + $BINARY_NAME keys add $PROV_KEY --home ${PROV_NODE_DIR} --keyring-backend test --output json > ${PROV_NODE_DIR}/${PROV_KEY}.json 2>&1 + sleep 1 + + # copy genesis in, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json ${PROV_NODE_DIR}/config/genesis.json + fi + + # Add stake to user + PROV_ACCOUNT_ADDR=$(jq -r '.address' ${PROV_NODE_DIR}/${PROV_KEY}.json) + $BINARY_NAME genesis add-genesis-account $PROV_ACCOUNT_ADDR $USER_COINS --home ${PROV_NODE_DIR} --keyring-backend test + sleep 1 + + # copy genesis out, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${PROV_NODE_DIR}/config/genesis.json ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json + fi + + PPROF_LADDR=${NODE_IP}:$(($PPROF_LADDR_BASEPORT + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $index)) + + # adjust configs of this node + sed -i -r 's/timeout_commit = "5s"/timeout_commit = "3s"/g' ${PROV_NODE_DIR}/config/config.toml + sed -i -r 's/timeout_propose = "3s"/timeout_propose = "1s"/g' ${PROV_NODE_DIR}/config/config.toml + + # make address book non-strict. necessary for this setup + sed -i -r 's/addr_book_strict = true/addr_book_strict = false/g' ${PROV_NODE_DIR}/config/config.toml + + # avoid port double binding + sed -i -r "s/pprof_laddr = \"localhost:6060\"/pprof_laddr = \"${PPROF_LADDR}\"/g" ${PROV_NODE_DIR}/config/config.toml + + # allow duplicate IP addresses (all nodes are on the same machine) + sed -i -r 's/allow_duplicate_ip = false/allow_duplicate_ip = true/g' ${PROV_NODE_DIR}/config/config.toml +done + +for MONIKER in "${MONIKERS[@]}" +do + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # copy genesis in, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json* ${PROV_NODE_DIR}/config/genesis.json + fi + + # Stake 1/1000 user's coins + $BINARY_NAME genesis gentx $PROV_KEY $STAKE --chain-id provider --home ${PROV_NODE_DIR} --keyring-backend test --moniker $MONIKER + sleep 1 + + # Copy gentxs to the lead validator for possible future collection. + # Obviously we don't need to copy the first validator's gentx to itself + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${PROV_NODE_DIR}/config/gentx/* ${LEAD_VALIDATOR_PROV_DIR}/config/gentx/ + fi +done + +# Collect genesis transactions with lead validator +$BINARY_NAME genesis collect-gentxs --home ${LEAD_VALIDATOR_PROV_DIR} --gentx-dir ${LEAD_VALIDATOR_PROV_DIR}/config/gentx/ + +sleep 1 + +START_COMMANDS="" +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + + PERSISTENT_PEERS="" + + for peer_index in "${!MONIKERS[@]}" + do + if [ $index == $peer_index ]; then + continue + fi + PEER_MONIKER=${MONIKERS[$peer_index]} + + PEER_PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${PEER_MONIKER} + + PEER_NODE_ID=$($BINARY_NAME tendermint show-node-id --home ${PEER_PROV_NODE_DIR}) + + PEER_P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $peer_index)) + PERSISTENT_PEERS="$PERSISTENT_PEERS,$PEER_NODE_ID@${NODE_IP}:${PEER_P2P_LADDR_PORT}" + done + + # remove trailing comma from persistent peers + PERSISTENT_PEERS=${PERSISTENT_PEERS:1} + + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${PROV_NODES_ROOT_DIR}/consumer-${MONIKER} + + # copy genesis in, unless this validator is already the lead validator and thus it already has its genesis + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json ${PROV_NODE_DIR}/config/genesis.json + fi + + # enable vote extensions by setting .consesnsus.params.abci.vote_extensions_enable_height to 1, but 1 does not work currently - set it to 2 instead. see https://github.com/cosmos/cosmos-sdk/issues/18029#issuecomment-1754598598 + jq ".consensus.params.abci.vote_extensions_enable_height = \"2\"" ${PROV_NODE_DIR}/config/genesis.json > ${PROV_NODE_DIR}/edited_genesis.json && mv ${PROV_NODE_DIR}/edited_genesis.json ${PROV_NODE_DIR}/config/genesis.json + + RPC_LADDR_PORT=$(($RPC_LADDR_BASEPORT + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $index)) + GRPC_LADDR_PORT=$(($GRPC_LADDR_BASEPORT + $index)) + NODE_ADDRESS_PORT=$(($NODE_ADDRESS_BASEPORT + $index)) + + PROVIDER_NODE_LISTEN_ADDR_STR="${NODE_IP}:${NODE_ADDRESS_PORT},$PROVIDER_NODE_LISTEN_ADDR_STR" + PROV_NODES_HOME_STR="${PROV_NODE_DIR},$PROV_NODES_HOME_STR" + + rm -rf ${PROV_NODES_ROOT_DIR}_bkup + cp -r ${PROV_NODES_ROOT_DIR} ${PROV_NODES_ROOT_DIR}_bkup + + # Start gaia + echo $BINARY_NAME start \ + --home ${PROV_NODE_DIR} \ + --transport=grpc --with-tendermint=false \ + --p2p.persistent_peers ${PERSISTENT_PEERS} \ + --rpc.laddr tcp://${NODE_IP}:${RPC_LADDR_PORT} \ + --grpc.address ${NODE_IP}:${GRPC_LADDR_PORT} \ + --address tcp://${NODE_IP}:${NODE_ADDRESS_PORT} \ + --p2p.laddr tcp://${NODE_IP}:${P2P_LADDR_PORT} \ + --grpc-web.enable=false "&> ${PROV_NODE_DIR}/logs &" | tee -a start_apps.sh + + sleep 5 +done + +PROVIDER_NODE_LISTEN_ADDR_STR=${PROVIDER_NODE_LISTEN_ADDR_STR::${#PROVIDER_NODE_LISTEN_ADDR_STR}-1} +PROV_NODES_HOME_STR=${PROV_NODES_HOME_STR::${#PROV_NODES_HOME_STR}-1} + +echo "Testnet applications are set up! Starting CometMock..." +echo cometmock \$1 $PROVIDER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json $PROVIDER_COMETMOCK_ADDR $PROV_NODES_HOME_STR grpc "&> ${LEAD_VALIDATOR_PROV_DIR}/cometmock_log &" | tee -a start_cometmock.sh + +chmod +x start_apps.sh +chmod +x start_cometmock.sh + +# cometmock $PROVIDER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json $PROVIDER_COMETMOCK_ADDR $PROV_NODES_HOME_STR grpc $COMETMOCK_ARGS &> ${LEAD_VALIDATOR_PROV_DIR}/cometmock_log & \ No newline at end of file diff --git a/e2e-tests/local-testnet-singlechain-start.sh b/e2e-tests/local-testnet-singlechain-start.sh new file mode 100755 index 0000000..bad0414 --- /dev/null +++ b/e2e-tests/local-testnet-singlechain-start.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +## Starts the testnet, assuming that the scripts generated by local-testnet-singlechain-setup.sh +## are already present in the current directory. + +COMETMOCK_ARGS=$1 + +./start_apps.sh +./start_cometmock.sh "$1" \ No newline at end of file diff --git a/e2e-tests/local-testnet-singlechain.sh b/e2e-tests/local-testnet-singlechain.sh index fc6a100..26d4fbb 100755 --- a/e2e-tests/local-testnet-singlechain.sh +++ b/e2e-tests/local-testnet-singlechain.sh @@ -1,15 +1,17 @@ #!/bin/bash + +## This script sets up the local testnet and starts it. +## To run this, both the application binary and cometmock must be installed. set -eux +parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) +pushd "$parent_path" + BINARY_NAME=$1 -# User balance of stake tokens -USER_COINS="100000000000stake" -# Amount of stake tokens staked -STAKE="100000000stake" -# Node IP address -NODE_IP="127.0.0.1" +COMETMOCK_ARGS=$2 +<<<<<<< HEAD # Home directory HOME_DIR=$HOME @@ -204,4 +206,9 @@ PROV_NODES_HOME_STR=${PROV_NODES_HOME_STR::${#PROV_NODES_HOME_STR}-1} cometmock $PROVIDER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json $PROVIDER_COMETMOCK_ADDR $PROV_NODES_HOME_STR grpc &> ${LEAD_VALIDATOR_PROV_DIR}/cometmock_log & -sleep 5 \ No newline at end of file +sleep 5 +======= +# set up the net +./local-testnet-singlechain-setup.sh $BINARY_NAME "$COMETMOCK_ARGS" +./local-testnet-singlechain-start.sh +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) diff --git a/e2e-tests/main_test.go b/e2e-tests/main_test.go index d0f5ad2..a345554 100644 --- a/e2e-tests/main_test.go +++ b/e2e-tests/main_test.go @@ -1,15 +1,17 @@ package main import ( - "bytes" "encoding/json" "fmt" + "math/big" "os/exec" - "strconv" "testing" "time" + + "github.com/stretchr/testify/require" ) +<<<<<<< HEAD func runCommandWithOutput(cmd *exec.Cmd) (string, error) { var stdout, stderr bytes.Buffer cmd.Stdout = &stdout @@ -55,14 +57,26 @@ func extractHeightFromInfo(jsonBytes []byte) (int, error) { // Tests happy path functionality for Abci Info. func TestAbciInfo(t *testing.T) { +======= +func StartChain( + t *testing.T, + cometmockArgs string, +) error { +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) // execute the local-testnet-singlechain.sh script t.Log("Running local-testnet-singlechain.sh") - cmd := exec.Command("./local-testnet-singlechain.sh", "simd") + cmd := exec.Command("./local-testnet-singlechain-restart.sh", "simd") _, err := runCommandWithOutput(cmd) if err != nil { t.Fatalf("Error running local-testnet-singlechain.sh: %v", err) } + cmd = exec.Command("./local-testnet-singlechain-start.sh", cometmockArgs) + _, err = runCommandWithOutput(cmd) + if err != nil { + return fmt.Errorf("Error running local-testnet-singlechain.sh: %v", err) + } + t.Log("Done starting testnet") // wait until we are producing blocks @@ -75,6 +89,20 @@ func TestAbciInfo(t *testing.T) { t.Log("Waiting for blocks to be produced, latest output: ", string(out)) time.Sleep(1 * time.Second) } +<<<<<<< HEAD +======= + time.Sleep(5 * time.Second) + return nil +} + +// Tests happy path functionality for Abci Info. +func TestAbciInfo(t *testing.T) { + // start the chain + err := StartChain(t, "") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) // call the abci_info command by calling curl on the REST endpoint // curl -H 'Content-Type: application/json' -H 'Accept:application/json' --data '{"jsonrpc":"2.0","method":"abci_info","id":1}' 127.0.0.1:22331 @@ -112,3 +140,249 @@ func TestAbciInfo(t *testing.T) { t.Fatalf("Expected block height to increase, but it did not. First height was %v, second height was %v", height, height2) } } +<<<<<<< HEAD +======= + +func TestAbciQuery(t *testing.T) { + // start the chain + err := StartChain(t, "") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + + // call the abci_query command by submitting a query that hits the AbciQuery endpoint + // for simplicity, we query for the staking params here - any query would work, + // but ones without arguments are easier to construct + args := []string{"bash", "-c", "simd q staking params --node tcp://127.0.0.1:22331 --output json"} + cmd := exec.Command(args[0], args[1:]...) + out, err := runCommandWithOutput(cmd) + if err != nil { + t.Fatalf("Error running command: %v\noutput: %v\nerror: %v", cmd, string(out), err) + } + + // check that the output is valid JSON + var data map[string]interface{} + if err := json.Unmarshal([]byte(out), &data); err != nil { + t.Fatalf("Failed to unmarshal JSON %s \n error was %v", string(out), err) + } + + // check that the output contains the expected params field. its contents are not important + _, ok := data["params"] + if !ok { + t.Fatalf("Expected output to contain params field, but it did not. Output was %s", string(out)) + } +} + +func TestTx(t *testing.T) { + err := StartChain(t, "") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + + // check the current amount in the community pool + communityPoolSize, err := getCommunityPoolSize() + require.NoError(t, err) + + // send some tokens to the community pool + err = sendToCommunityPool(50000000000, "coordinator") + require.NoError(t, err) + + // check that the amount in the community pool has increased + communityPoolSize2, err := getCommunityPoolSize() + require.NoError(t, err) + + // cannot check for equality because the community pool gets dust over time + require.True(t, communityPoolSize2.Cmp(communityPoolSize.Add(communityPoolSize, big.NewInt(50000000000))) == +1) +} + +// TestBlockTime checks that the basic behaviour with a specified block-time is as expected, +// i.e. the time increases by the specified block time for each block. +func TestBlockTime(t *testing.T) { + err := StartChain(t, "--block-time=5000") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + + // get a block with height+time + blockString, err := QueryBlock() + require.NoError(t, err) + + // get the height and time from the block + height, err := GetHeightFromBlock(blockString) + require.NoError(t, err) + + blockTime, err := GetTimeFromBlock(blockString) + require.NoError(t, err) + + // wait for a couple of blocks to be produced + time.Sleep(10 * time.Second) + + // get the new height and time + blockString2, err := QueryBlock() + require.NoError(t, err) + + height2, err := GetHeightFromBlock(blockString2) + require.NoError(t, err) + + blockTime2, err := GetTimeFromBlock(blockString2) + require.NoError(t, err) + + blockDifference := height2 - height + // we expect that at least one block was produced, otherwise there is a problem + require.True(t, blockDifference >= 1) + + // get the expected time diff between blocks, as block time was set to 5000 millis = 5 seconds + expectedTimeDifference := time.Duration(blockDifference) * 5 * time.Second + + timeDifference := blockTime2.Sub(blockTime) + + require.Equal(t, expectedTimeDifference, timeDifference) +} + +// TestAutoBlockProductionOff checks that the basic behaviour with +// block-production-interval is as expected, i.e. blocks only +// appear when it is manually instructed. +func TestNoAutoBlockProduction(t *testing.T) { + err := StartChain(t, "--block-production-interval=-1 --block-time=0") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + + height, blockTime, err := GetHeightAndTime() + require.NoError(t, err) + + // wait a few seconds to detect it blocks are produced automatically + time.Sleep(10 * time.Second) + + // get the new height and time + height2, blockTime2, err := GetHeightAndTime() + require.NoError(t, err) + + // no blocks should have been produced + require.Equal(t, height, height2) + require.Equal(t, blockTime, blockTime2) + + // advance time by 5 seconds + err = AdvanceTime(5 * time.Second) + require.NoError(t, err) + + // get the height and time again, they should not have changed yet + height3, blockTime3, err := GetHeightAndTime() + require.NoError(t, err) + + require.Equal(t, height, height3) + require.Equal(t, blockTime, blockTime3) + + // produce a block + err = AdvanceBlocks(1) + require.NoError(t, err) + + // get the height and time again, they should have changed + height4, blockTime4, err := GetHeightAndTime() + require.NoError(t, err) + + require.Equal(t, height+1, height4) + require.Equal(t, blockTime.Add(5*time.Second), blockTime4) +} + +// TestNoAutoTx checks that without auto-tx, transactions are not included +// in blocks automatically. +func TestNoAutoTx(t *testing.T) { + err := StartChain(t, "--block-production-interval=-1 --auto-tx=false") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + + // produce a couple of blocks to initialize the community pool + err = AdvanceBlocks(10) + require.NoError(t, err) + + height, blockTime, err := GetHeightAndTime() + require.NoError(t, err) + + communityPoolBefore, err := getCommunityPoolSize() + require.NoError(t, err) + + // broadcast txs + err = sendToCommunityPool(50000000000, "coordinator") + require.NoError(t, err) + err = sendToCommunityPool(50000000000, "bob") + require.NoError(t, err) + + // get the new height and time + height2, blockTime2, err := GetHeightAndTime() + require.NoError(t, err) + + // no blocks should have been produced + require.Equal(t, height, height2) + require.Equal(t, blockTime, blockTime2) + + // produce a block + err = AdvanceBlocks(1) + require.NoError(t, err) + + // get the height and time again, they should have changed + height3, blockTime3, err := GetHeightAndTime() + require.NoError(t, err) + + require.Equal(t, height+1, height3) + // exact time does not matter, just that it is after the previous block + require.True(t, blockTime.Before(blockTime3)) + + // check that the community pool was increased + communityPoolAfter, err := getCommunityPoolSize() + require.NoError(t, err) + + // cannot check for equality because the community pool gets dust over time + require.True(t, communityPoolAfter.Cmp(communityPoolBefore.Add(communityPoolBefore, big.NewInt(100000000000))) == +1) +} + +func TestStartingTimestamp(t *testing.T) { + err := StartChain(t, "--block-production-interval=-1 --auto-tx=false --starting-timestamp=0 --block-time=1") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + + // produce a couple of blocks + err = AdvanceBlocks(10) + require.NoError(t, err) + + // get the time + _, blockTime, err := GetHeightAndTime() + require.NoError(t, err) + + // the time should be starting-timestamp + 10 * blockTime + 1 (for the first block needed after Genesis) + startingTimestamp := time.Unix(0, 0) + expectedTime := startingTimestamp.Add(11 * time.Millisecond) + + require.True(t, expectedTime.Compare(blockTime) == 0, "expectedTime: %v, blockTime: %v", expectedTime, blockTime) +} + +func TestSystemStartingTime(t *testing.T) { + err := StartChain(t, "--block-production-interval=-1 --auto-tx=false --starting-timestamp=-1 --block-time=1") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + startingTime := time.Now() + + // produce a couple of blocks + err = AdvanceBlocks(10) + require.NoError(t, err) + + // get the time + _, blockTime, err := GetHeightAndTime() + require.NoError(t, err) + + // the time should be starting-timestamp + 10 * blockTime + 1 (for the first block needed after Genesis) + expectedTime := startingTime.Add(11 * time.Millisecond) + + // since the starting timestamp is taken from the system time, + // we can only check that the time is close to the expected time + // since the chain startup is hard to time exactly + delta := 30 * time.Second + + diff := expectedTime.Sub(blockTime).Abs() + + require.True(t, diff <= delta, "expectedTime: %v, blockTime: %v", expectedTime, blockTime) +} +>>>>>>> 7edb4c1 (Add fine-grained control of time (#88)) diff --git a/e2e-tests/test_utils.go b/e2e-tests/test_utils.go new file mode 100644 index 0000000..0f92b56 --- /dev/null +++ b/e2e-tests/test_utils.go @@ -0,0 +1,166 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "math/big" + "os/exec" + "strconv" + "strings" + "time" +) + +// From the output of the AbciInfo command, extract the latest block height. +// The json bytes should look e.g. like this: +// {"jsonrpc":"2.0","id":1,"result":{"response":{"data":"interchain-security-p","last_block_height":"2566","last_block_app_hash":"R4Q3Si7+t7TIidl2oTHcQRDNEz+lP0IDWhU5OI89psg="}}} +func extractHeightFromInfo(jsonBytes []byte) (int, error) { + // Use a generic map to represent the JSON structure + var data map[string]interface{} + + if err := json.Unmarshal(jsonBytes, &data); err != nil { + return -1, fmt.Errorf("failed to unmarshal JSON %s \n error was %v", string(jsonBytes), err) + } + + // Navigate the map and use type assertions to get the last_block_height + result, ok := data["result"].(map[string]interface{}) + if !ok { + return -1, fmt.Errorf("failed to navigate abci_info output structure trying to access result: json was %s", string(jsonBytes)) + } + + response, ok := result["response"].(map[string]interface{}) + if !ok { + return -1, fmt.Errorf("failed to navigate abci_info output structure trying to access response: json was %s", string(jsonBytes)) + } + + lastBlockHeight, ok := response["last_block_height"].(string) + if !ok { + return -1, fmt.Errorf("failed to navigate abci_info output structure trying to access last_block_height: json was %s", string(jsonBytes)) + } + + return strconv.Atoi(lastBlockHeight) +} + +// Queries simd for the latest block. +func QueryBlock() (string, error) { + // execute the query command + cmd := exec.Command("bash", "-c", "simd q block --type height 0 --output json --node tcp://127.0.0.1:22331 --output json") + out, err := runCommandWithOutput(cmd) + if err != nil { + return "", fmt.Errorf("error running query command: %v", err) + } + + return out, nil +} + +type BlockInfo struct { + Header struct { + Height string `json:"height"` + Time string `json:"time"` + } `json:"header"` +} + +func GetHeightFromBlock(blockString string) (int, error) { + var block BlockInfo + err := json.Unmarshal([]byte(blockString), &block) + if err != nil { + return 0, err + } + + res, err := strconv.Atoi(block.Header.Height) + if err != nil { + return 0, err + } + + return res, nil +} + +func GetTimeFromBlock(blockBytes string) (time.Time, error) { + var block BlockInfo + err := json.Unmarshal([]byte(blockBytes), &block) + if err != nil { + return time.Time{}, err + } + + res, err := time.Parse(time.RFC3339, block.Header.Time) + if err != nil { + return time.Time{}, err + } + + return res, nil +} + +func GetHeightAndTime() (int, time.Time, error) { + blockBytes, err := QueryBlock() + if err != nil { + return 0, time.Time{}, err + } + + height, err := GetHeightFromBlock(blockBytes) + if err != nil { + return 0, time.Time{}, err + } + + timestamp, err := GetTimeFromBlock(blockBytes) + if err != nil { + return 0, time.Time{}, err + } + + return height, timestamp, nil +} + +// Queries the size of the community pool. +// For this, it will just check the number of tokens of the first denom in the community pool. +func getCommunityPoolSize() (*big.Int, error) { + // execute the query command + cmd := exec.Command("bash", "-c", "simd q distribution community-pool --output json --node tcp://127.0.0.1:22331 | jq -r '.pool[0].amount'") + out, err := runCommandWithOutput(cmd) + if err != nil { + return big.NewInt(-1), fmt.Errorf("error running query command: %v", err) + } + + res := new(big.Int) + + res, ok := res.SetString(strings.TrimSpace(out), 10) + if !ok { + return big.NewInt(-1), fmt.Errorf("error parsing community pool size: %v", err) + } + return res, err +} + +func sendToCommunityPool(amount int, sender string) error { + // execute the tx command + stringCmd := fmt.Sprintf("simd tx distribution fund-community-pool %vstake --chain-id provider --from %v-key --keyring-backend test --node tcp://127.0.0.1:22331 --home ~/nodes/provider/provider-%v -y", amount, sender, sender) + cmd := exec.Command("bash", "-c", stringCmd) + _, err := runCommandWithOutput(cmd) + return err +} + +func runCommandWithOutput(cmd *exec.Cmd) (string, error) { + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + return "", fmt.Errorf("error running command: %v\nstdout: %s\nstderr: %s", err, stdout.String(), stderr.String()) + } + + return stdout.String(), nil +} + +func AdvanceTime(duration time.Duration) error { + stringCmd := fmt.Sprintf("curl -H 'Content-Type: application/json' -H 'Accept:application/json' --data '{\"jsonrpc\":\"2.0\",\"method\":\"advance_time\",\"params\":{\"duration_in_seconds\": \"%v\"},\"id\":1}' 127.0.0.1:22331", duration.Seconds()) + + cmd := exec.Command("bash", "-c", stringCmd) + _, err := runCommandWithOutput(cmd) + return err +} + +func AdvanceBlocks(numBlocks int) error { + stringCmd := fmt.Sprintf("curl -H 'Content-Type: application/json' -H 'Accept:application/json' --data '{\"jsonrpc\":\"2.0\",\"method\":\"advance_blocks\",\"params\":{\"num_blocks\": \"%v\"},\"id\":1}' 127.0.0.1:22331", numBlocks) + + cmd := exec.Command("bash", "-c", stringCmd) + _, err := runCommandWithOutput(cmd) + return err +}