Automatically update git for jobs without CASEDIR/NEEDLES_DIR

dist/rpm/openQA.spec

@@ -18,7 +18,7 @@
 
 # can't use linebreaks here!
 %define openqa_main_service openqa-webui.service
-%define openqa_extra_services openqa-gru.service openqa-websockets.service openqa-scheduler.service openqa-enqueue-audit-event-cleanup.service openqa-enqueue-audit-event-cleanup.timer openqa-enqueue-asset-cleanup.service openqa-enqueue-asset-cleanup.timer openqa-enqueue-result-cleanup.service openqa-enqueue-result-cleanup.timer openqa-enqueue-bug-cleanup.service openqa-enqueue-bug-cleanup.timer
+%define openqa_extra_services openqa-gru.service openqa-websockets.service openqa-scheduler.service openqa-enqueue-audit-event-cleanup.service openqa-enqueue-audit-event-cleanup.timer openqa-enqueue-asset-cleanup.service openqa-enqueue-git-auto-update.service openqa-enqueue-asset-cleanup.timer openqa-enqueue-result-cleanup.service openqa-enqueue-result-cleanup.timer openqa-enqueue-bug-cleanup.service openqa-enqueue-bug-cleanup.timer openqa-enqueue-git-auto-update.timer
 %define openqa_services %{openqa_main_service} %{openqa_extra_services}
 %define openqa_worker_services openqa-worker.target openqa-slirpvde.service openqa-vde_switch.service openqa-worker-cacheservice.service openqa-worker-cacheservice-minion.service
 %if %{undefined tmpfiles_create}
@@ -603,6 +603,8 @@ fi
 %{_unitdir}/openqa-enqueue-audit-event-cleanup.timer
 %{_unitdir}/openqa-enqueue-asset-cleanup.service
 %{_unitdir}/openqa-enqueue-asset-cleanup.timer
+%{_unitdir}/openqa-enqueue-git-auto-update.service
+%{_unitdir}/openqa-enqueue-git-auto-update.timer
 %{_unitdir}/openqa-enqueue-result-cleanup.service
 %{_unitdir}/openqa-enqueue-result-cleanup.timer
 %{_unitdir}/openqa-enqueue-bug-cleanup.service
@@ -636,6 +638,7 @@ fi
 %{_datadir}/openqa/script/openqa-enqueue-asset-cleanup
 %{_datadir}/openqa/script/openqa-enqueue-audit-event-cleanup
 %{_datadir}/openqa/script/openqa-enqueue-bug-cleanup
+%{_datadir}/openqa/script/openqa-enqueue-git-auto-update
 %{_datadir}/openqa/script/openqa-enqueue-result-cleanup
 %{_datadir}/openqa/script/openqa-gru
 %{_datadir}/openqa/script/openqa-rollback

etc/openqa/openqa.ini

@@ -115,6 +115,8 @@
 #do_push = no
 ## whether to clone CASEDIR or NEEDLES_DIR on the web UI if that var points to a Git repo
 #git_auto_clone = yes
+## Experimental - Ensure a git update of all test code and needles
+#git_auto_update = no
 
 ## Authentication method to use for user management
 [auth]

lib/OpenQA/Setup.pm

@@ -68,6 +68,7 @@ sub read_config ($app) {
             do_push => 'no',
             do_cleanup => 'no',
             git_auto_clone => 'yes',
+            git_auto_update => 'no',
         },
         scheduler => {
             max_job_scheduled_time => 7,

lib/OpenQA/Shared/Plugin/Gru.pm

@@ -10,8 +10,10 @@ use DBIx::Class::Timestamps 'now';
 use OpenQA::Schema;
 use OpenQA::Shared::GruJob;
 use OpenQA::Log 'log_info';
+use OpenQA::Utils qw(sharedir);
 use Mojo::Pg;
 use Mojo::Promise;
+use Mojo::File qw(path);
 
 has app => undef, weak => 1;
 has 'dsn';
@@ -23,21 +25,20 @@ sub new ($class, $app = undef) {
 
 sub register_tasks ($self) {
     my $app = $self->app;
-    $app->plugin($_)
-      for (
-        qw(OpenQA::Task::AuditEvents::Limit),
-        qw(OpenQA::Task::Asset::Download),
-        qw(OpenQA::Task::Asset::Limit),
-        qw(OpenQA::Task::Git::Clone),
-        qw(OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete),
-        qw(OpenQA::Task::Job::Limit),
-        qw(OpenQA::Task::Job::ArchiveResults),
-        qw(OpenQA::Task::Job::FinalizeResults),
-        qw(OpenQA::Task::Job::HookScript),
-        qw(OpenQA::Task::Job::Restart),
-        qw(OpenQA::Task::Iso::Schedule),
-        qw(OpenQA::Task::Bug::Limit),
-      );
+    $app->plugin($_) for qw(
+      OpenQA::Task::AuditEvents::Limit
+      OpenQA::Task::Asset::Download
+      OpenQA::Task::Asset::Limit
+      OpenQA::Task::Git::Clone
+      OpenQA::Task::Needle::Scan OpenQA::Task::Needle::Save OpenQA::Task::Needle::Delete
+      OpenQA::Task::Job::Limit
+      OpenQA::Task::Job::ArchiveResults
+      OpenQA::Task::Job::FinalizeResults
+      OpenQA::Task::Job::HookScript
+      OpenQA::Task::Job::Restart
+      OpenQA::Task::Iso::Schedule
+      OpenQA::Task::Bug::Limit
+    );
 }
 
 # allow the continuously polled stats to be available on an
@@ -166,6 +167,32 @@ sub enqueue_download_jobs ($self, $downloads) {
     }
 }
 
+sub enqueue_git_update_all ($self) {
+    my $conf = OpenQA::App->singleton->config->{'scm git'};
+    return if $conf->{git_auto_clone} ne 'yes' || $conf->{git_auto_update} ne 'yes';
+    my %clones;
+    my $testdir = path(sharedir() . '/tests');
+    for my $distri ($testdir->list({dir => 1})->each) {
+        next unless -d $distri;    # no symlinks
+        next unless -e $distri->child('.git');
+        $clones{$distri} = undef;
+        if (-e $distri->child('products')) {
+            for my $product ($distri->child('products')->list({dir => 1})->each) {
+                next unless -d $product;    # no symlinks
+                my $needle = $product->child('needles');
+                next unless -e $needle->child('.git');
+                $clones{$needle} = undef;
+            }
+        }
+        else {
+            my $needle = $distri->child('needles');
+            next unless -e $needle->child('.git');
+            $clones{$needle} = undef;
+        }
+    }
+    $self->enqueue('git_clone', \%clones, {priority => 10});
+}
+
 sub enqueue_git_clones ($self, $clones, $job_ids) {
     return unless %$clones;
     return unless OpenQA::App->singleton->config->{'scm git'}->{git_auto_clone} eq 'yes';

lib/OpenQA/Task/Git/Clone.pm

@@ -5,7 +5,6 @@ package OpenQA::Task::Git::Clone;
 use Mojo::Base 'Mojolicious::Plugin', -signatures;
 use Mojo::Util 'trim';
 
-use OpenQA::Utils qw(run_cmd_with_log_return_error);
 use Mojo::File;
 use Time::Seconds 'ONE_HOUR';
 
@@ -54,18 +53,24 @@ sub _git_clone_all ($job, $clones) {
 
 sub _git_clone ($app, $job, $ctx, $path, $url) {
     my $git = OpenQA::Git->new(app => $app, dir => $path);
-    $ctx->debug(qq{Updating $path to $url});
-    $url = Mojo::URL->new($url);
-    my $requested_branch = $url->fragment;
-    $url->fragment(undef);
+    $ctx->debug(sprintf q{Updating '%s' to '%s'}, $path, ($url // 'n/a'));
+    my $requested_branch;
+    if ($url) {
+        $url = Mojo::URL->new($url);
+        $requested_branch = $url->fragment;
+        $url->fragment(undef);
 
-    # An initial clone fetches all refs, we are done
-    return $git->clone_url($url) unless -d $path;
+        # An initial clone fetches all refs, we are done
+        return $git->clone_url($url) unless -d $path;
 
-    my $origin_url = $git->get_origin_url;
-    if ($url ne $origin_url) {
-        $ctx->warn("Local checkout at $path has origin $origin_url but requesting to clone from $url");
-        return;
+        my $origin_url = $git->get_origin_url;
+        if ($url ne $origin_url) {
+            $ctx->warn("Local checkout at $path has origin $origin_url but requesting to clone from $url");
+            return;
+        }
+    }
+    else {
+        $url = $git->get_origin_url;
     }
 
     die "NOT updating dirty git checkout at $path" if !$git->is_workdir_clean();

lib/OpenQA/Utils.pm

@@ -603,6 +603,11 @@ sub create_downloads_list ($job_settings) {
 
 sub create_git_clone_list ($job_settings, $clones = {}) {
     my $distri = $job_settings->{DISTRI};
+    if (OpenQA::App->singleton->config->{'scm git'}->{git_auto_update} eq 'yes') {
+        # Potential existing git clones to update without having CASEDIR or NEEDLES_DIR
+        not $job_settings->{CASEDIR} and $clones->{testcasedir($distri)} = undef;
+        not $job_settings->{NEEDLES_DIR} and $clones->{needledir($distri)} = undef;
+    }
     my $case_url = Mojo::URL->new($job_settings->{CASEDIR} // '');
     my $needles_url = Mojo::URL->new($job_settings->{NEEDLES_DIR} // '');
     if ($case_url->scheme) {

script/openqa-enqueue-git-auto-update

@@ -0,0 +1,2 @@
+#!/bin/sh -e
+exec "$(dirname "$0")"/openqa eval -m production -V 'app->gru->enqueue_git_update_all()' "$@"

systemd/openqa-enqueue-git-auto-update.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Ensure a git update of all test code and needles regardless of test execution.
+After=postgresql.service openqa-setup-db.service
+Wants=openqa-setup-db.service
+
+[Service]
+Type=oneshot
+User=geekotest
+ExecStart=/usr/share/openqa/script/openqa-enqueue-git-update-all

systemd/openqa-enqueue-git-auto-update.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Ensure a git update of all test code and needles regardless of test execution.
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+
+[Install]
+WantedBy=timers.target