feat(e2e): add parameter to set a custom output directory for testnet…

… files (cometbft#2433) Add the optional parameter `--testnet-dir` to `test/e2e/runner`. --- #### PR checklist - [ ] Tests written/updated - [x] Changelog entry added in `.changelog` (we use [unclog](https://github.com/informalsystems/unclog) to manage our changelog) - [x] Updated relevant documentation (`docs/` or `spec/`) and code comments - [X] Title follows the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) spec
berachain · Feb 26, 2024 · 4ce0277 · 4ce0277
1 parent e623c44
commit 4ce0277
Show file tree

Hide file tree

Showing 10 changed files with 46 additions and 15 deletions.
diff --git a/.changelog/unreleased/features/2433-e2e-testnet-dir.md b/.changelog/unreleased/features/2433-e2e-testnet-dir.md
@@ -0,0 +1,2 @@
+- `[e2e]` Add new `--testnet-dir` parameter to set a custom directory for the generated testnet files.
+  ([\#2433](https://github.com/cometbft/cometbft/pull/2433))
diff --git a/test/e2e/README.md b/test/e2e/README.md
@@ -9,6 +9,11 @@ make
 
 This creates and runs a testnet named `ci` under `networks/ci/`.
 
+To generate the testnet files in a different directory, run:
+```sh
+./build/runner -f networks/ci.toml -d networks/foo/bar/
+```
+
 ### Fast compiling
 
 If you need to run experiments on a testnet, you will probably want to compile the code multiple
@@ -141,6 +146,8 @@ To run tests manually, set the `E2E_MANIFEST` environment variable to the path o
 E2E_MANIFEST=networks/ci.toml go test -v ./tests/...
 ```
 
+If the testnet files are located in a custom directory, you need to set it in the `E2E_TESTNET_DIR` environment variable.
+
 Optionally, `E2E_NODE` specifies the name of a single testnet node to test.
 
 These environment variables can also be specified in `tests/e2e_test.go` to run tests from an editor or IDE:
@@ -150,6 +157,7 @@ func init() {
 	// This can be used to manually specify a testnet manifest and/or node to
 	// run tests against. The testnet must have been started by the runner first.
 	os.Setenv("E2E_MANIFEST", "networks/ci.toml")
+	os.Setenv("E2E_TESTNET_DIR", "networks/foo")
 	os.Setenv("E2E_NODE", "validator01")
 }
 ```

diff --git a/test/e2e/generator/generate_test.go b/test/e2e/generator/generate_test.go
@@ -24,7 +24,7 @@ func TestGenerator(t *testing.T) {
 		t.Run(fmt.Sprintf("Case%04d", idx), func(t *testing.T) {
 			infra, err := e2e.NewDockerInfrastructureData(m)
 			require.NoError(t, err)
-			_, err = e2e.NewTestnetFromManifest(m, filepath.Join(t.TempDir(), fmt.Sprintf("Case%04d", idx)), infra)
+			_, err = e2e.NewTestnetFromManifest(m, filepath.Join(t.TempDir(), fmt.Sprintf("Case%04d", idx)), infra, "")
 			require.NoError(t, err)
 		})
 	}

diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go
@@ -139,22 +139,26 @@ type Node struct {
 	Zone                    ZoneID
 }
 
-// LoadTestnet loads a testnet from a manifest file, using the filename to
-// determine the testnet name and directory (from the basename of the file).
+// LoadTestnet loads a testnet from a manifest file. The testnet files are
+// generated in the given directory, which is also use to determine the testnet
+// name (the directory's basename).
 // The testnet generation must be deterministic, since it is generated
 // separately by the runner and the test cases. For this reason, testnets use a
 // random seed to generate e.g. keys.
-func LoadTestnet(file string, ifd InfrastructureData) (*Testnet, error) {
+func LoadTestnet(file string, ifd InfrastructureData, dir string) (*Testnet, error) {
 	manifest, err := LoadManifest(file)
 	if err != nil {
 		return nil, err
 	}
-	return NewTestnetFromManifest(manifest, file, ifd)
+	return NewTestnetFromManifest(manifest, file, ifd, dir)
 }
 
 // NewTestnetFromManifest creates and validates a testnet from a manifest.
-func NewTestnetFromManifest(manifest Manifest, file string, ifd InfrastructureData) (*Testnet, error) {
-	dir := strings.TrimSuffix(file, filepath.Ext(file))
+func NewTestnetFromManifest(manifest Manifest, file string, ifd InfrastructureData, dir string) (*Testnet, error) {
+	if dir == "" {
+		// Set default testnet directory.
+		dir = strings.TrimSuffix(file, filepath.Ext(file))
+	}
 
 	keyGen := newKeyGenerator(randomSeed)
 	prometheusProxyPortGen := newPortGenerator(prometheusProxyPortFirst)

diff --git a/test/e2e/runner/cleanup.go b/test/e2e/runner/cleanup.go
@@ -63,7 +63,7 @@ func cleanupDir(dir string) error {
 		return err
 	}
 
-	logger.Info("cleanup dir", "msg", log.NewLazySprintf("Removing testnet directory %q", dir))
+	logger.Info("cleanup dir", "msg", log.NewLazySprintf("Removing testnet directory %#q", dir))
 
 	// On Linux, some local files in the volume will be owned by root since CometBFT
 	// runs as root inside the container, so we need to clean them up from within a

diff --git a/test/e2e/runner/load.go b/test/e2e/runner/load.go
@@ -70,7 +70,8 @@ func Load(ctx context.Context, testnet *e2e.Testnet) error {
 		// Log every ~1 second the number of sent transactions.
 		total := success + failed
 		if total%testnet.LoadTxBatchSize == 0 {
-			logger.Debug("load", "success", success, "failed", failed, "success/total", log.NewLazySprintf("%.1f", success/total), "tx/s", rate)
+			successRate := float64(success) / float64(total)
+			logger.Debug("load", "success", success, "failed", failed, "success/total", log.NewLazySprintf("%.2f", successRate), "tx/s", rate)
 		}
 
 		// Check if reached max number of allowed transactions to send.

diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go
@@ -80,7 +80,12 @@ func NewCLI() *CLI {
 				return fmt.Errorf("unknown infrastructure type '%s'", inft)
 			}
 
-			testnet, err := e2e.LoadTestnet(file, ifd)
+			testnetDir, err := cmd.Flags().GetString("testnet-dir")
+			if err != nil {
+				return err
+			}
+
+			testnet, err := e2e.LoadTestnet(file, ifd, testnetDir)
 			if err != nil {
 				return fmt.Errorf("loading testnet: %s", err)
 			}
@@ -175,6 +180,8 @@ func NewCLI() *CLI {
 	cli.root.PersistentFlags().StringP("file", "f", "", "Testnet TOML manifest")
 	_ = cli.root.MarkPersistentFlagRequired("file")
 
+	cli.root.PersistentFlags().StringP("testnet-dir", "d", "", "Set the directory for the testnet files generated during setup")
+
 	cli.root.PersistentFlags().StringP("infrastructure-type", "", "docker", "Backing infrastructure used to run the testnet. Either 'digital-ocean' or 'docker'")
 
 	cli.root.PersistentFlags().StringP("infrastructure-data", "", "", "path to the json file containing the infrastructure data. Only used if the 'infrastructure-type' is set to a value other than 'docker'")

diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go
@@ -40,7 +40,7 @@ const (
 
 // Setup sets up the testnet configuration.
 func Setup(testnet *e2e.Testnet, infp infra.Provider) error {
-	logger.Info("setup", "msg", log.NewLazySprintf("Generating testnet files in %q", testnet.Dir))
+	logger.Info("setup", "msg", log.NewLazySprintf("Generating testnet files in %#q", testnet.Dir))
 
 	if err := os.MkdirAll(testnet.Dir, os.ModePerm); err != nil {
 		return err

diff --git a/test/e2e/runner/test.go b/test/e2e/runner/test.go
@@ -16,6 +16,10 @@ func Test(testnet *e2e.Testnet, ifd *e2e.InfrastructureData) error {
 	if err != nil {
 		return err
 	}
+	err = os.Setenv("E2E_TESTNET_DIR", testnet.Dir)
+	if err != nil {
+		return err
+	}
 	if p := ifd.Path; p != "" {
 		err = os.Setenv("INFRASTRUCTURE_FILE", p)
 		if err != nil {

diff --git a/test/e2e/tests/e2e_test.go b/test/e2e/tests/e2e_test.go
@@ -3,7 +3,6 @@ package e2e_test
 import (
 	"context"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -16,13 +15,15 @@ import (
 	rpctypes "github.com/cometbft/cometbft/rpc/core/types"
 	"github.com/cometbft/cometbft/test/e2e/app"
 	e2e "github.com/cometbft/cometbft/test/e2e/pkg"
+	"github.com/cometbft/cometbft/test/e2e/pkg/infra/docker"
 	"github.com/cometbft/cometbft/types"
 )
 
 func init() {
 	// This can be used to manually specify a testnet manifest and/or node to
 	// run tests against. The testnet must have been started by the runner first.
 	// os.Setenv("E2E_MANIFEST", "networks/ci.toml")
+	// os.Setenv("E2E_TESTNET_DIR", "networks/ci")
 	// os.Setenv("E2E_NODE", "validator01")
 }
 
@@ -137,7 +138,12 @@ func loadTestnet(t *testing.T) e2e.Testnet {
 	}
 	require.NoError(t, err)
 
-	testnet, err := e2e.LoadTestnet(manifestFile, ifd)
+	testnetDir := os.Getenv("E2E_TESTNET_DIR")
+	if !filepath.IsAbs(testnetDir) {
+		testnetDir = filepath.Join("..", testnetDir)
+	}
+
+	testnet, err := e2e.LoadTestnet(manifestFile, ifd, testnetDir)
 	require.NoError(t, err)
 	testnetCache[manifestFile] = *testnet
 	return *testnet
@@ -227,6 +233,5 @@ func fetchABCIRequests(t *testing.T, nodeName string) ([][]*abci.Request, error)
 }
 
 func fetchNodeLogs(testnet e2e.Testnet) ([]byte, error) {
-	dir := filepath.Join(testnet.Dir, "docker-compose.yml")
-	return exec.Command("docker-compose", "-f", dir, "logs").Output()
+	return docker.ExecComposeOutput(context.Background(), testnet.Dir, "logs")
 }