diff --git a/pkg/neotest/basic.go b/pkg/neotest/basic.go index 6acc8dbc2d..1540af8624 100644 --- a/pkg/neotest/basic.go +++ b/pkg/neotest/basic.go @@ -32,20 +32,23 @@ type Executor struct { Validator Signer Committee Signer CommitteeHash util.Uint160 - Contracts map[string]*Contract + // collectCoverage is true if coverage is being collected when running this executor. + collectCoverage bool } // NewExecutor creates a new executor instance from the provided blockchain and committee. +// By default coverage collection is enabled, but only when `go test` is running with coverage enabled. +// Use DisableCoverage and EnableCoverage to stop coverage collection for this executor when not desired. func NewExecutor(t testing.TB, bc *core.Blockchain, validator, committee Signer) *Executor { checkMultiSigner(t, validator) checkMultiSigner(t, committee) return &Executor{ - Chain: bc, - Validator: validator, - Committee: committee, - CommitteeHash: committee.ScriptHash(), - Contracts: make(map[string]*Contract), + Chain: bc, + Validator: validator, + Committee: committee, + CommitteeHash: committee.ScriptHash(), + collectCoverage: isCoverageEnabled(), } } @@ -145,6 +148,7 @@ func (e *Executor) DeployContract(t testing.TB, c *Contract, data any) util.Uint // data is an optional argument to `_deploy`. // It returns the hash of the deploy transaction. func (e *Executor) DeployContractBy(t testing.TB, signer Signer, c *Contract, data any) util.Uint256 { + e.trackCoverage(t, c) tx := e.NewDeployTxBy(t, signer, c, data) e.AddNewBlock(t, tx) e.CheckHalt(t, tx.Hash()) @@ -164,11 +168,22 @@ func (e *Executor) DeployContractBy(t testing.TB, signer Signer, c *Contract, da // DeployContractCheckFAULT compiles and deploys a contract to the bc using the validator // account. It checks that the deploy transaction FAULTed with the specified error. func (e *Executor) DeployContractCheckFAULT(t testing.TB, c *Contract, data any, errMessage string) { + e.trackCoverage(t, c) tx := e.NewDeployTx(t, c, data) e.AddNewBlock(t, tx) e.CheckFault(t, tx.Hash(), errMessage) } +// trackCoverage switches on coverage tracking for provided script if `go test` is running with coverage enabled. +func (e *Executor) trackCoverage(t testing.TB, c *Contract) { + if e.collectCoverage { + addScriptToCoverage(c) + t.Cleanup(func() { + reportCoverage(t) + }) + } +} + // InvokeScript adds a transaction with the specified script to the chain and // returns its hash. It does no faults check. func (e *Executor) InvokeScript(t testing.TB, script []byte, signers []Signer) util.Uint256 { @@ -401,6 +416,10 @@ func (e *Executor) TestInvoke(tx *transaction.Transaction) (*vm.VM, error) { ttx := *tx ic, _ := e.Chain.GetTestVM(trigger.Application, &ttx, b) + if e.collectCoverage { + ic.VM.SetOnExecHook(coverageHook) + } + defer ic.Finalize() ic.VM.LoadWithFlags(tx.Script, callflag.All) @@ -431,3 +450,13 @@ func (e *Executor) GetTxExecResult(t testing.TB, h util.Uint256) *state.AppExecR require.Equal(t, 1, len(aer)) return &aer[0] } + +// EnableCoverage enables coverage collection for this executor, but only when `go test` is running with coverage enabled. +func (e *Executor) EnableCoverage() { + e.collectCoverage = isCoverageEnabled() +} + +// DisableCoverage disables coverage collection for this executor until enabled explicitly through EnableCoverage. +func (e *Executor) DisableCoverage() { + e.collectCoverage = false +} diff --git a/pkg/neotest/client.go b/pkg/neotest/client.go index bd96bab476..3985d8b8c1 100644 --- a/pkg/neotest/client.go +++ b/pkg/neotest/client.go @@ -63,6 +63,10 @@ func (c *ContractInvoker) TestInvokeScript(t testing.TB, script []byte, signers } t.Cleanup(ic.Finalize) + if c.collectCoverage { + ic.VM.SetOnExecHook(coverageHook) + } + ic.VM.LoadWithFlags(tx.Script, callflag.All) err = ic.VM.Run() return ic.VM.Estack(), err @@ -78,6 +82,10 @@ func (c *ContractInvoker) TestInvoke(t testing.TB, method string, args ...any) ( } t.Cleanup(ic.Finalize) + if c.collectCoverage { + ic.VM.SetOnExecHook(coverageHook) + } + ic.VM.LoadWithFlags(tx.Script, callflag.All) err = ic.VM.Run() return ic.VM.Estack(), err diff --git a/pkg/neotest/compile.go b/pkg/neotest/compile.go index 70645c11cd..2967b2b1e6 100644 --- a/pkg/neotest/compile.go +++ b/pkg/neotest/compile.go @@ -16,9 +16,10 @@ import ( // Contract contains contract info for deployment. type Contract struct { - Hash util.Uint160 - NEF *nef.File - Manifest *manifest.Manifest + Hash util.Uint160 + NEF *nef.File + Manifest *manifest.Manifest + DebugInfo *compiler.DebugInfo } // contracts caches the compiled contracts from FS across multiple tests. @@ -36,9 +37,10 @@ func CompileSource(t testing.TB, sender util.Uint160, src io.Reader, opts *compi require.NoError(t, err) return &Contract{ - Hash: state.CreateContractHash(sender, ne.Checksum, m.Name), - NEF: ne, - Manifest: m, + Hash: state.CreateContractHash(sender, ne.Checksum, m.Name), + NEF: ne, + Manifest: m, + DebugInfo: di, } } @@ -73,9 +75,10 @@ func CompileFile(t testing.TB, sender util.Uint160, srcPath string, configPath s require.NoError(t, err) c := &Contract{ - Hash: state.CreateContractHash(sender, ne.Checksum, m.Name), - NEF: ne, - Manifest: m, + Hash: state.CreateContractHash(sender, ne.Checksum, m.Name), + NEF: ne, + Manifest: m, + DebugInfo: di, } contracts[srcPath] = c return c diff --git a/pkg/neotest/coverage.go b/pkg/neotest/coverage.go new file mode 100644 index 0000000000..a006679087 --- /dev/null +++ b/pkg/neotest/coverage.go @@ -0,0 +1,190 @@ +package neotest + +import ( + "flag" + "fmt" + "io" + "os" + "sync" + "testing" + + "github.com/nspcc-dev/neo-go/pkg/compiler" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm" + "github.com/nspcc-dev/neo-go/pkg/vm/opcode" +) + +var ( + // coverageLock protects all vars below from concurrent modification when tests are run in parallel. + coverageLock sync.Mutex + // rawCoverage maps script hash to coverage data, collected during testing. + rawCoverage = make(map[util.Uint160]*scriptRawCoverage) + // flagChecked is true if `go test` coverage flag was checked at any point. + flagChecked bool + // coverageEnabled is true if coverage is being collected on test execution. + coverageEnabled bool + // coverProfile specifies the file all coverage data is written to, unless empty. + coverProfile = "" +) + +type scriptRawCoverage struct { + debugInfo *compiler.DebugInfo + offsetsVisited []int +} + +type coverBlock struct { + // Line number for block start. + startLine uint + // Column number for block start. + startCol uint + // Line number for block end. + endLine uint + // Column number for block end. + endCol uint + // Number of statements included in this block. + stmts uint + // Number of times this block was executed. + counts uint +} + +// documentName makes it clear when a `string` maps path to the script file. +type documentName = string + +func isCoverageEnabled() bool { + coverageLock.Lock() + defer coverageLock.Unlock() + + if flagChecked { + return coverageEnabled + } + defer func() { flagChecked = true }() + + const coverProfileFlag = "test.coverprofile" + flag.VisitAll(func(f *flag.Flag) { + if f.Name == coverProfileFlag && f.Value != nil && f.Value.String() != "" { + coverageEnabled = true + coverProfile = f.Value.String() + } + }) + + if coverageEnabled { + // This is needed so go cover tool doesn't overwrite + // the file with our coverage when all tests are done. + err := flag.Set(coverProfileFlag, "") + if err != nil { + panic(err) + } + } + return coverageEnabled +} + +var coverageHook vm.OnExecHook = func(scriptHash util.Uint160, offset int, opcode opcode.Opcode) { + coverageLock.Lock() + defer coverageLock.Unlock() + if cov, ok := rawCoverage[scriptHash]; ok { + cov.offsetsVisited = append(cov.offsetsVisited, offset) + } +} + +func reportCoverage(t testing.TB) { + coverageLock.Lock() + defer coverageLock.Unlock() + f, err := os.Create(coverProfile) + if err != nil { + t.Fatalf("coverage: can't create file '%s' to write coverage report", coverProfile) + } + defer f.Close() + writeCoverageReport(f) +} + +func writeCoverageReport(w io.Writer) { + fmt.Fprintf(w, "mode: set\n") + cover := processCover() + for name, blocks := range cover { + for _, b := range blocks { + c := 0 + if b.counts > 0 { + c = 1 + } + fmt.Fprintf(w, "%s:%d.%d,%d.%d %d %d\n", name, + b.startLine, b.startCol, + b.endLine, b.endCol, + b.stmts, + c, + ) + } + } +} + +func processCover() map[documentName][]coverBlock { + documents := make(map[documentName]struct{}) + for _, scriptRawCoverage := range rawCoverage { + for _, documentName := range scriptRawCoverage.debugInfo.Documents { + documents[documentName] = struct{}{} + } + } + + cover := make(map[documentName][]coverBlock) + + for documentName := range documents { + mappedBlocks := make(map[int]*coverBlock) + + for _, scriptRawCoverage := range rawCoverage { + di := scriptRawCoverage.debugInfo + documentSeqPoints := documentSeqPoints(di, documentName) + + for _, point := range documentSeqPoints { + b := coverBlock{ + startLine: uint(point.StartLine), + startCol: uint(point.StartCol), + endLine: uint(point.EndLine), + endCol: uint(point.EndCol), + stmts: 1 + uint(point.EndLine) - uint(point.StartLine), + counts: 0, + } + mappedBlocks[point.Opcode] = &b + } + } + + for _, scriptRawCoverage := range rawCoverage { + di := scriptRawCoverage.debugInfo + documentSeqPoints := documentSeqPoints(di, documentName) + + for _, offset := range scriptRawCoverage.offsetsVisited { + for _, point := range documentSeqPoints { + if point.Opcode == offset { + mappedBlocks[point.Opcode].counts++ + } + } + } + } + + var blocks []coverBlock + for _, b := range mappedBlocks { + blocks = append(blocks, *b) + } + cover[documentName] = blocks + } + + return cover +} + +func documentSeqPoints(di *compiler.DebugInfo, doc documentName) []compiler.DebugSeqPoint { + var res []compiler.DebugSeqPoint + for _, methodDebugInfo := range di.Methods { + for _, p := range methodDebugInfo.SeqPoints { + if di.Documents[p.Document] == doc { + res = append(res, p) + } + } + } + return res +} + +func addScriptToCoverage(c *Contract) { + coverageLock.Lock() + defer coverageLock.Unlock() + if _, ok := rawCoverage[c.Hash]; !ok { + rawCoverage[c.Hash] = &scriptRawCoverage{debugInfo: c.DebugInfo} + } +} diff --git a/pkg/neotest/doc.go b/pkg/neotest/doc.go index bd223ae561..82eaff6a02 100644 --- a/pkg/neotest/doc.go +++ b/pkg/neotest/doc.go @@ -21,5 +21,12 @@ them in the same package with the smart contract iself can lead to unxpected results if smart contract has any init() functions. If that's the case they will be compiled into the testing binary even when using package_test and their execution can affect tests. See https://github.com/nspcc-dev/neo-go/issues/3120 for details. + +Test coverage for contracts is automatically enabled when `go test` is running with +coverage enabled. When not desired, it can be disabled for any Executor by using +EnableCoverage and DisableCoverage. Be aware that coverage data collected by `go test` +itself will not be saved because it will be replaced with contracts coverage instead. +Coverage is gathered by capturing VM instructions during test contract execution and +mapping them to the contract source code using the DebugInfo information. */ package neotest