[WIP] Proof of concept changes (test suites, HDR histograms)

cmd/agent.go

@@ -86,12 +86,72 @@ func getMetricsHook(
 	}
 }
 
+func loadAndConfigureDistTest(gs *state.GlobalState, distTest *distributed.Test) (*loadedAndConfiguredTest, error) {
+	var options lib.Options
+	if err := json.Unmarshal(distTest.Options, &options); err != nil {
+		return nil, err
+	}
+
+	arc, err := lib.ReadArchive(bytes.NewReader(distTest.Archive))
+	if err != nil {
+		return nil, err
+	}
+
+	registry := metrics.NewRegistry()
+	piState := &lib.TestPreInitState{
+		Logger: gs.Logger,
+		RuntimeOptions: lib.RuntimeOptions{
+			NoThresholds:      null.BoolFrom(true),
+			NoSummary:         null.BoolFrom(true),
+			Env:               arc.Env,
+			CompatibilityMode: null.StringFrom(arc.CompatibilityMode),
+		},
+		Registry:       registry,
+		BuiltinMetrics: metrics.RegisterBuiltinMetrics(registry),
+	}
+
+	initRunner, err := js.NewFromArchive(piState, arc)
+	if err != nil {
+		return nil, err
+	}
+
+	test := &loadedTest{
+		pwd:            arc.Pwd,
+		sourceRootPath: arc.Filename,
+		source: &loader.SourceData{
+			Data: distTest.Archive,
+			URL:  arc.FilenameURL,
+		},
+		fs:           afero.NewMemMapFs(), // TODO: figure out what should be here
+		fileSystems:  arc.Filesystems,
+		preInitState: piState,
+		initRunner:   initRunner,
+	}
+
+	pseudoConsoldatedConfig := applyDefault(Config{Options: options})
+	for _, thresholds := range pseudoConsoldatedConfig.Thresholds {
+		if err = thresholds.Parse(); err != nil {
+			return nil, err
+		}
+	}
+	derivedConfig, err := deriveAndValidateConfig(pseudoConsoldatedConfig, initRunner.IsExecutable, gs.Logger)
+	if err != nil {
+		return nil, err
+	}
+
+	return &loadedAndConfiguredTest{
+		loadedTest:         test,
+		consolidatedConfig: pseudoConsoldatedConfig,
+		derivedConfig:      derivedConfig,
+	}, nil
+}
+
 // TODO: a whole lot of cleanup, refactoring, error handling and hardening
-func getCmdAgent(gs *state.GlobalState) *cobra.Command { //nolint: funlen
+func getCmdAgent(gs *state.GlobalState) *cobra.Command {
 	c := &cmdsRunAndAgent{gs: gs}
 
-	c.loadConfiguredTest = func(cmd *cobra.Command, args []string) (
-		*loadedAndConfiguredTest, execution.Controller, error,
+	c.loadConfiguredTests = func(cmd *cobra.Command, args []string) (
+		[]*loadedAndConfiguredTest, execution.Controller, error,
 	) {
 		// TODO: add some gRPC authentication
 		conn, err := grpc.Dial(args[0], grpc.WithTransportCredentials(insecure.NewCredentials()))
@@ -110,74 +170,25 @@ func getCmdAgent(gs *state.GlobalState) *cobra.Command { //nolint: funlen
 			return nil, nil, err
 		}
 
-		c.metricsEngineHook = getMetricsHook(gs.Ctx, resp.InstanceID, client, gs.Logger)
-
-		controller, err := distributed.NewAgentController(gs.Ctx, resp.InstanceID, client, gs.Logger)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		var options lib.Options
-		if err = json.Unmarshal(resp.Options, &options); err != nil {
-			return nil, nil, err
-		}
-
-		arc, err := lib.ReadArchive(bytes.NewReader(resp.Archive))
-		if err != nil {
-			return nil, nil, err
-		}
-
-		registry := metrics.NewRegistry()
-		piState := &lib.TestPreInitState{
-			Logger: gs.Logger,
-			RuntimeOptions: lib.RuntimeOptions{
-				NoThresholds:      null.BoolFrom(true),
-				NoSummary:         null.BoolFrom(true),
-				Env:               arc.Env,
-				CompatibilityMode: null.StringFrom(arc.CompatibilityMode),
-			},
-			Registry:       registry,
-			BuiltinMetrics: metrics.RegisterBuiltinMetrics(registry),
-		}
-
-		initRunner, err := js.NewFromArchive(piState, arc)
-		if err != nil {
-			return nil, nil, err
+		var configuredTests []*loadedAndConfiguredTest
+		for _, test := range resp.Tests {
+			cTest, lcErr := loadAndConfigureDistTest(gs, test)
+			if lcErr != nil {
+				return nil, nil, lcErr
+			}
+			configuredTests = append(configuredTests, cTest)
 		}
 
-		test := &loadedTest{
-			pwd:            arc.Pwd,
-			sourceRootPath: arc.Filename,
-			source: &loader.SourceData{
-				Data: resp.Archive,
-				URL:  arc.FilenameURL,
-			},
-			fs:           afero.NewMemMapFs(), // TODO: figure out what should be here
-			fileSystems:  arc.Filesystems,
-			preInitState: piState,
-			initRunner:   initRunner,
-		}
+		c.metricsEngineHook = getMetricsHook(gs.Ctx, resp.InstanceID, client, gs.Logger)
 
-		pseudoConsoldatedConfig := applyDefault(Config{Options: options})
-		for _, thresholds := range pseudoConsoldatedConfig.Thresholds {
-			if err = thresholds.Parse(); err != nil {
-				return nil, nil, err
-			}
-		}
-		derivedConfig, err := deriveAndValidateConfig(pseudoConsoldatedConfig, initRunner.IsExecutable, gs.Logger)
+		controller, err := distributed.NewAgentController(gs.Ctx, resp.InstanceID, client, gs.Logger)
 		if err != nil {
 			return nil, nil, err
 		}
 
-		configuredTest := &loadedAndConfiguredTest{
-			loadedTest:         test,
-			consolidatedConfig: pseudoConsoldatedConfig,
-			derivedConfig:      derivedConfig,
-		}
-
 		gs.Flags.Address = "" // TODO: fix, this is a hack so agents don't start an API server
 
-		return configuredTest, controller, nil // TODO
+		return configuredTests, controller, nil // TODO
 	}
 
 	agentCmd := &cobra.Command{

cmd/archive.go

@@ -16,7 +16,11 @@ type cmdArchive struct {
 }
 
 func (c *cmdArchive) run(cmd *cobra.Command, args []string) error {
-	test, err := loadAndConfigureLocalTest(c.gs, cmd, args, getPartialConfig)
+	preInitState, err := getPreInitState(c.gs, cmd)
+	if err != nil {
+		return err
+	}
+	test, err := loadAndConfigureLocalTest(c.gs, preInitState, cmd, args[0], getPartialConfig)
 	if err != nil {
 		return err
 	}
@@ -88,7 +92,7 @@ func getCmdArchive(gs *state.GlobalState) *cobra.Command {
 
 An archive is a fully self-contained test run, and can be executed identically elsewhere.`,
 		Example: exampleText,
-		Args:    cobra.ExactArgs(1),
+		Args:    cobra.ExactArgs(1), // TODO: remove and support test suites?
 		RunE:    c.run,
 	}
 

cmd/cloud.go

@@ -84,7 +84,11 @@ func (c *cmdCloud) run(cmd *cobra.Command, args []string) error {
 	)
 	printBar(c.gs, progressBar)
 
-	test, err := loadAndConfigureLocalTest(c.gs, cmd, args, getPartialConfig)
+	preInitState, err := getPreInitState(c.gs, cmd)
+	if err != nil {
+		return err
+	}
+	test, err := loadAndConfigureLocalTest(c.gs, preInitState, cmd, args[0], getPartialConfig)
 	if err != nil {
 		return err
 	}

cmd/coordinator.go

@@ -28,35 +28,52 @@ type cmdCoordinator struct {
 func (c *cmdCoordinator) run(cmd *cobra.Command, args []string) (err error) { //nolint: funlen
 	ctx, runAbort := execution.NewTestRunContext(c.gs.Ctx, c.gs.Logger)
 
-	test, err := loadAndConfigureLocalTest(c.gs, cmd, args, getPartialConfig)
+	tests, err := loadAndConfigureLocalTests(c.gs, cmd, args, getPartialConfig)
 	if err != nil {
 		return err
 	}
 
-	// Only consolidated options, not derived
-	testRunState, err := test.buildTestRunState(test.consolidatedConfig.Options)
+	// TODO: refactor at some point, this limits us to handleSummary() from the first test
+	firstTest := tests[0]
+	// TODO: refactor - this is safe, preInitState is the same for all tests,
+	// but it's still icky to get it that way
+	preInitState := firstTest.preInitState
+	metricsEngine, err := engine.NewMetricsEngine(preInitState.Registry, c.gs.Logger)
 	if err != nil {
 		return err
 	}
 
-	metricsEngine, err := engine.NewMetricsEngine(testRunState.Registry, c.gs.Logger)
-	if err != nil {
-		return err
+	testArchives := make([]*lib.Archive, len(tests))
+	for i, test := range tests {
+		runState, rsErr := test.buildTestRunState(test.consolidatedConfig.Options)
+		if rsErr != nil {
+			return rsErr
+		}
+
+		// We get the thresholds from all tests
+		testArchives[i] = runState.Runner.MakeArchive()
+		err = metricsEngine.InitSubMetricsAndThresholds(
+			test.derivedConfig.Options,
+			preInitState.RuntimeOptions.NoThresholds.Bool,
+		)
+		if err != nil {
+			return err
+		}
 	}
 
 	coordinator, err := distributed.NewCoordinatorServer(
-		c.instanceCount, test.initRunner.MakeArchive(), metricsEngine, c.gs.Logger,
+		c.instanceCount, testArchives, metricsEngine, c.gs.Logger,
 	)
 	if err != nil {
 		return err
 	}
 
-	if !testRunState.RuntimeOptions.NoSummary.Bool {
+	if !preInitState.RuntimeOptions.NoSummary.Bool {
 		defer func() {
 			c.gs.Logger.Debug("Generating the end-of-test summary...")
-			summaryResult, serr := test.initRunner.HandleSummary(ctx, &lib.Summary{
+			summaryResult, serr := firstTest.initRunner.HandleSummary(ctx, &lib.Summary{
 				Metrics:         metricsEngine.ObservedMetrics,
-				RootGroup:       test.initRunner.GetDefaultGroup(),
+				RootGroup:       firstTest.initRunner.GetDefaultGroup(),
 				TestRunDuration: coordinator.GetCurrentTestRunDuration(),
 				NoColor:         c.gs.Flags.NoColor,
 				UIState: lib.UIState{
@@ -73,7 +90,7 @@ func (c *cmdCoordinator) run(cmd *cobra.Command, args []string) (err error) { //
 		}()
 	}
 
-	if !testRunState.RuntimeOptions.NoThresholds.Bool {
+	if !preInitState.RuntimeOptions.NoThresholds.Bool {
 		getCurrentTestDuration := coordinator.GetCurrentTestRunDuration
 		finalizeThresholds := metricsEngine.StartThresholdCalculations(nil, runAbort, getCurrentTestDuration)
 

cmd/inspect.go

@@ -19,9 +19,14 @@ func getCmdInspect(gs *state.GlobalState) *cobra.Command {
 		Use:   "inspect [file]",
 		Short: "Inspect a script or archive",
 		Long:  `Inspect a script or archive.`,
-		Args:  cobra.ExactArgs(1),
+		Args:  cobra.ExactArgs(1), // TODO: remove and support test suites?
 		RunE: func(cmd *cobra.Command, args []string) error {
-			test, err := loadLocalTest(gs, cmd, args)
+			preInitState, err := getPreInitState(gs, cmd)
+			if err != nil {
+				return err
+			}
+
+			test, err := loadLocalTest(gs, preInitState, args[0])
 			if err != nil {
 				return err
 			}