[IMP] runbot: commit exports

Source cleanup will check in multiple place for potentially used
commits. This maked the cleanup logic complex, plus limit the usable
commits in python steps. The current use case is to export the mergebase
commits.

The poposed solution is to save the exported commit and clean this list
when the build is done. All commit in this list cannot be cleaned.

runbot/models/build.py

@@ -729,6 +729,9 @@ def _schedule(self):
                 build._github_status()
             build._run_job()
 
+            if build.local_state == 'done':
+                self.env['runbot.commit.export'].search([('build_id', '=', build.id)]).unlink()
+
     def _run_job(self):
         # run job
         for build in self:
@@ -794,7 +797,7 @@ def _checkout(self):
             if build_export_path in exports:
                 self._log('_checkout', 'Multiple repo have same export path in build, some source may be missing for %s' % build_export_path, level='ERROR')
                 self._kill(result='ko')
-            exports[build_export_path] = commit.export()
+            exports[build_export_path] = commit.export(self)
 
         checkout_time = time.time() - start
         if checkout_time > 60:

runbot/models/commit.py

@@ -52,11 +52,11 @@ def _get_available_modules(self):
                     module = os.path.basename(os.path.dirname(manifest_path))
                     yield (addons_path, module, manifest_file_name)
 
-    def export(self):
+    def export(self, build):
         """Export a git repo into a sources"""
         #  TODO add automated tests
         self.ensure_one()
-
+        self.env['runbot.commit.export'].create({'commit_id': self.id, 'build_id': build.id})
         export_path = self._source_path()
 
         if os.path.isdir(export_path):
@@ -228,3 +228,13 @@ def send_github_status_async():
                 self._cr.after('commit', send_github_status_async)
             else:
                 send_github_status(self.env)
+
+
+class CommitExport(models.Model):
+    _name = 'runbot.commit.export'
+    _description = 'Commit export'
+
+    build_id = fields.Many2one('runbot.build', index=True)
+    commit_id = fields.Many2one('runbot.commit')
+
+    host = fields.Char(related='build_id.host', store=True)

runbot/models/runbot.py

@@ -315,10 +315,15 @@ def _source_cleanup(self):
             if self.pool._init:
                 return
             _logger.info('Source cleaning')
-            # we can remove a source only if no build are using them as name or rependency_ids aka as commit
+
+            cannot_be_deleted_path = set()
+            for commit in self.env['runbot.commit.export'].search([('host', '=', fqdn())]).mapped('commit_id'):
+                cannot_be_deleted_path.add(commit._source_path())
+
+
+            # the following part won't be usefull anymore once runbot.commit.export is populated
             cannot_be_deleted_builds = self.env['runbot.build'].search([('host', '=', fqdn()), ('local_state', '!=', 'done')])
             cannot_be_deleted_builds |= cannot_be_deleted_builds.mapped('params_id.builds_reference_ids')
-            cannot_be_deleted_path = set()
             for build in cannot_be_deleted_builds:
                 for build_commit in build.params_id.commit_link_ids:
                     cannot_be_deleted_path.add(build_commit.commit_id._source_path())

runbot/security/ir.model.access.csv

@@ -114,3 +114,5 @@ access_runbot_dockerfile_admin,access_runbot_dockerfile_admin,runbot.model_runbo
 
 access_runbot_codeowner_admin,runbot_codeowner_admin,runbot.model_runbot_codeowner,runbot.group_runbot_admin,1,1,1,1
 access_runbot_codeowner_user,runbot_codeowner_user,runbot.model_runbot_codeowner,group_user,1,0,0,0
+
+access_runbot_commit_export_admin,runbot_commit_export_admin,runbot.model_runbot_commit_export,runbot.group_runbot_admin,1,1,1,1