hydra-eval-jobs: hacky support for globs as string constituents

src/hydra-eval-jobs/hydra-eval-jobs.cc

@@ -27,6 +27,8 @@
 #include <sys/wait.h>
 #include <sys/resource.h>
 
+#include <fnmatch.h>
+
 #include <nlohmann/json.hpp>
 
 void check_pid_status_nonblocking(pid_t check_pid)
@@ -234,6 +236,10 @@ static void worker(
                         if (v->type() == nString)
                             job["namedConstituents"].push_back(v->string_view());
                     }
+
+                    auto glob = v->attrs()->get(state.symbols.create("_hydraGlobConstituents"));
+                    bool globConstituents = glob && state.forceBool(*glob->value, glob->pos, "while evaluating the `_hydraGlobConstituents` attribute");
+                    job["globConstituents"] = globConstituents;
                 }
 
                 /* Register the derivation as a GC root.  !!! This
@@ -497,46 +503,74 @@ int main(int argc, char * * argv)
             auto named = job.find("namedConstituents");
             if (named == job.end()) continue;
 
+            bool globConstituents = job.value<bool>("globConstituents", false);
+
             std::unordered_map<std::string, std::string> brokenJobs;
-            auto getNonBrokenJobOrRecordError = [&brokenJobs, &jobName, &state](
-                    const std::string & childJobName) -> std::optional<nlohmann::json> {
-                auto childJob = state->jobs.find(childJobName);
-                if (childJob == state->jobs.end()) {
-                    printError("aggregate job '%s' references non-existent job '%s'", jobName, childJobName);
-                    brokenJobs[childJobName] = "does not exist";
-                    return std::nullopt;
-                }
-                if (childJob->find("error") != childJob->end()) {
-                    std::string error = (*childJob)["error"];
+            auto isBroken = [&brokenJobs, &jobName](
+                    const std::string & childJobName, nlohmann::json & job) -> bool {
+                if (job.find("error") != job.end()) {
+                    std::string error = job["error"];
                     printError("aggregate job '%s' references broken job '%s': %s", jobName, childJobName, error);
                     brokenJobs[childJobName] = error;
-                    return std::nullopt;
+                    return true;
+                } else {
+                    return false;
                 }
-                return *childJob;
+            };
+            auto getNonBrokenJobsOrRecordError = [&state, &isBroken, &jobName, &brokenJobs, &globConstituents](
+                    const std::string & childJobName) -> std::vector<nlohmann::json> {
+                auto childJob = state->jobs.find(childJobName);
+                std::vector<nlohmann::json> results;
+                if (childJob == state->jobs.end()) {
+                    if (!globConstituents) {
+                        printError("aggregate job '%s' references non-existent job '%s'", jobName, childJobName);
+                        brokenJobs[childJobName] = "does not exist";
+                    } else {
+                        for (auto job = state->jobs.begin(); job != state->jobs.end(); job++) {
+                            auto jobName = job.key();
+                            if (fnmatch(childJobName.c_str(), jobName.c_str(), 0) == 0
+                                && !isBroken(jobName, *job)
+                            ) {
+                                results.push_back(*job);
+                            }
+                        }
+                        if (results.empty()) {
+                            warn("aggregate job '%s' references constituent glob pattern '%s' with no matches", jobName, childJobName);
+                            brokenJobs[childJobName] = "constituent glob pattern had no matches";
+                        }
+                    }
+                } else if (!isBroken(childJobName, *childJob)) {
+                    results.push_back(*childJob);
+                }
+                return results;
             };
 
             if (myArgs.dryRun) {
                 for (std::string jobName2 : *named) {
-                    auto job2 = getNonBrokenJobOrRecordError(jobName2);
-                    if (!job2) {
+                    auto foundJobs = getNonBrokenJobsOrRecordError(jobName2);
+                    if (foundJobs.empty()) {
                         continue;
                     }
-                    std::string drvPath2 = (*job2)["drvPath"];
-                    job["constituents"].push_back(drvPath2);
+                    for (auto & childJob : foundJobs) {
+                        std::string constituentDrvPath = childJob["drvPath"];
+                        job["constituents"].push_back(constituentDrvPath);
+                    }
                 }
             } else {
                 auto drvPath = store->parseStorePath((std::string) job["drvPath"]);
                 auto drv = store->readDerivation(drvPath);
 
                 for (std::string jobName2 : *named) {
-                    auto job2 = getNonBrokenJobOrRecordError(jobName2);
-                    if (!job2) {
+                    auto foundJobs = getNonBrokenJobsOrRecordError(jobName2);
+                    if (foundJobs.empty()) {
                         continue;
                     }
-                    auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]);
-                    auto drv2 = store->readDerivation(drvPath2);
-                    job["constituents"].push_back(store->printStorePath(drvPath2));
-                    drv.inputDrvs.map[drvPath2].value = {drv2.outputs.begin()->first};
+                    for (auto & childJob : foundJobs) {
+                        auto drvPath2 = store->parseStorePath((std::string) childJob["drvPath"]);
+                        auto drv2 = store->readDerivation(drvPath2);
+                        job["constituents"].push_back(store->printStorePath(drvPath2));
+                        drv.inputDrvs.map[drvPath2].value = {drv2.outputs.begin()->first};
+                    }
                 }
 
                 if (brokenJobs.empty()) {

t/evaluator/evaluate-constituents-globbing.t

@@ -0,0 +1,58 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+use Hydra::Helper::Exec;
+use Data::Dumper;
+
+my $ctx = test_context();
+
+my $jobsetCtx = $ctx->makeJobset(
+    expression => 'constituents-glob.nix',
+);
+my $jobset = $jobsetCtx->{"jobset"};
+
+my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+    ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+);
+
+subtest "non_match_aggregate failed" => sub {
+    ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
+    like(
+        $stderr,
+        qr/warning: aggregate job 'non_match_aggregate' references constituent glob pattern 'tests\.\*' with no matches/,
+        "The stderr record includes a relevant error message"
+    );
+
+    $jobset->discard_changes;  # refresh from DB
+    like(
+        $jobset->errormsg,
+        qr/tests\.\*: constituent glob pattern had no matches/,
+        "The jobset records a relevant error message"
+    );
+};
+
+my $builds = {};
+for my $build ($jobset->builds) {
+    $builds->{$build->job} = $build;
+}
+
+subtest "basic globbing works" => sub {
+    ok(defined $builds->{"ok_aggregate"}, "'ok_aggregate' is part of the jobset evaluation");
+    my @constituents = $builds->{"ok_aggregate"}->constituents->all;
+    is(2, scalar @constituents, "'ok_aggregate' has two constituents");
+
+    my @sortedConstituentNames = sort (map { $_->nixname } @constituents);
+
+    is($sortedConstituentNames[0], "empty-dir-A", "first constituent of 'ok_aggregate' is 'empty-dir-A'");
+    is($sortedConstituentNames[1], "empty-dir-B", "second constituent of 'ok_aggregate' is 'empty-dir-B'");
+};
+
+#subtest "transitivity is OK" => sub {
+    #ok(defined $builds->{"indirect_aggregate"}, "'indirect_aggregate' is part of the jobset evaluation");
+    #my @constituents = $builds->{"indirect_aggregate"}->constituents->all;
+    #is(1, scalar @constituents, "'indirect_aggregate' has one constituent");
+    #is($constituents[0]->nixname, "direct_aggregate", "'indirect_aggregate' has 'direct_aggregate' as single constituent");
+#};
+
+done_testing;

t/jobs/constituents-glob.nix

@@ -0,0 +1,41 @@
+with import ./config.nix;
+{
+  packages.constituentA = mkDerivation {
+    name = "empty-dir-A";
+    builder = ./empty-dir-builder.sh;
+  };
+
+  packages.constituentB = mkDerivation {
+    name = "empty-dir-B";
+    builder = ./empty-dir-builder.sh;
+  };
+
+  ok_aggregate = mkDerivation {
+    name = "direct_aggregate";
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [
+      "packages.*"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+
+  indirect_aggregate = mkDerivation {
+    name = "indirect_aggregate";
+    _hydraAggregate = true;
+    constituents = [
+      "ok_aggregate"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+
+  non_match_aggregate = mkDerivation {
+    name = "mixed_aggregate";
+    _hydraAggregate = true;
+    _hydraGlobConstituents = true;
+    constituents = [
+      "tests.*"
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+}