CI system in polemarch.

Add support for run template on sync when project is changed.
At now, allow run only one template without option specified.

See merge request polemarch/ce!155

doc/api_schema.yaml

@@ -19,9 +19,9 @@ info:
       url: https://gitlab.com/vstconsulting/polemarch.git
     Request:
       - name: Question
-        url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Ask&issue%5Btitle%5D=Ask%20about%20version%201.2.2
+        url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Ask&issue%5Btitle%5D=Ask%20about%20version%201.3.1
       - name: Bug report
-        url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Bug&issue%5Btitle%5D=Bug%20in%20version%201.2.2
+        url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Bug&issue%5Btitle%5D=Bug%20in%20version%201.3.1
       - name: Feature request
         url: https://gitlab.com/vstconsulting/polemarch/issues/new?issuable_template%5D=Feature%20request&issue%5Btitle%5D=
   x-menu:
@@ -54,9 +54,9 @@ info:
           span_class: fa fa-plug
           url: /hook
   x-versions:
-    application: 1.2.2
-    library: 1.2.2
-    vstutils: 2.2.0
+    application: 1.3.1
+    library: 1.3.1
+    vstutils: 2.3.1
     django: 2.2.1
     ansible: 2.8.1
   version: v2
@@ -11272,10 +11272,10 @@ definitions:
         type: string
         enum:
           - NEW
-          - WAIT_SYNC
-          - SYNC
           - ERROR
           - OK
+          - WAIT_SYNC
+          - SYNC
         readOnly: true
   ProjectCreateMaster:
     required:
@@ -11371,10 +11371,10 @@ definitions:
         type: string
         enum:
           - NEW
-          - WAIT_SYNC
-          - SYNC
           - ERROR
           - OK
+          - WAIT_SYNC
+          - SYNC
         readOnly: true
       revision:
         title: Revision
@@ -12204,9 +12204,11 @@ definitions:
         enum:
           - repo_type
           - repo_sync_on_run
+          - repo_sync_on_run_timeout
           - repo_branch
           - repo_password
           - repo_key
+          - ci_template
       value:
         title: Value
         type: string
@@ -12222,8 +12224,10 @@ definitions:
               - TAR
           field: key
           types:
+            ci_template: fk
             repo_key: secretfile
             repo_password: password
+            repo_sync_on_run_timeout: uptime
   Team:
     required:
       - name

polemarch/api/v2/serializers.py

@@ -1,4 +1,4 @@
-# pylint: disable=no-member,unused-argument,too-many-lines
+# pylint: disable=no-member,unused-argument,too-many-lines,c-extension-no-member
 from __future__ import unicode_literals
 from typing import Dict, List
 import json
@@ -19,6 +19,7 @@
 from ...main.utils import AnsibleArgumentsReference, AnsibleInventoryParser
 
 from ...main.models import Inventory
+from ...main.repo._base import _Base as base_repo_class
 from ...main import models
 from ..signals import api_post_save, api_pre_save
 
@@ -419,14 +420,16 @@ class PeriodicTaskVariableSerializer(VariableSerializer):
 class ProjectVariableSerializer(VariableSerializer):
     key = vst_fields.AutoCompletionField(
         required=True,
-        autocomplete=models.Project.VARS_KEY
+        autocomplete=models.Project.VARS_KEY+['ci_'+i for i in base_repo_class.handler_class.ci_types.keys()]
     )
     value = vst_fields.DependEnumField(allow_blank=True, field='key', choices={
         'repo_type': list(models.Project.repo_handlers.keys()),
         'repo_sync_on_run': [True, False]
     }, types={
         'repo_password': 'password',
-        'repo_key': 'secretfile'
+        'repo_key': 'secretfile',
+        'repo_sync_on_run_timeout': 'uptime',
+        'ci_template': 'fk'
     })
 
 

polemarch/main/ci.py

@@ -0,0 +1,40 @@
+from typing import Text, NoReturn
+import logging
+import traceback
+from .models.tasks import BModel, Template
+
+
+logger = logging.getLogger('polemarch')
+
+
+class DefaultHandler:
+    ci_types = {
+        'template': Template
+    }
+
+    def __init__(self, repo, results):
+        self.repo = repo
+        self.repo_obj, self.result = results
+        self.ci_vars = self.repo.proj.get_vars_prefixed('ci')
+
+    def event_log(self, message: Text, *args, **kwargs) -> NoReturn:
+        logger.info(message.format(*args, *kwargs))
+
+    def get_ci_call_object(self) -> [BModel, None]:
+        ci_keys = self.ci_vars.keys()
+        if not ci_keys:
+            return None
+        ci_type_name = list(ci_keys)[0]
+        ci_model = self.ci_types[ci_type_name]
+        return ci_model.objects.get(pk=self.ci_vars[ci_type_name])
+
+    def trigger_execution(self) -> NoReturn:
+        if self.result:
+            ci_object = self.get_ci_call_object()
+            if ci_object:
+                self.event_log('Start executing CI context.')
+                try:
+                    ci_object.ci_run()
+                except Exception:  # nocv
+                    self.event_log(traceback.format_exc())
+                    raise

polemarch/main/exceptions.py

@@ -27,3 +27,7 @@ class DataNotReady(PMException):
 
 class NotApplicable(exceptions.NotApplicable):
     pass
+
+
+class Conflict(PMException):
+    status = exceptions.status.HTTP_409_CONFLICT

polemarch/main/models/__init__.py

@@ -21,7 +21,7 @@
 from .tasks import PeriodicTask, History, HistoryLines, Template
 from .hooks import Hook
 from ..validators import RegexValidator, validate_hostname
-from ..exceptions import UnknownTypeException
+from ..exceptions import UnknownTypeException, Conflict
 from ..utils import AnsibleArgumentsReference, CmdExecutor
 
 
@@ -84,10 +84,32 @@ def check_variables_values(instance: Variable, *args, **kwargs) -> NoReturn:
     elif isinstance(content_object, Host):
         if instance.key == 'ansible_host':
             validate_hostname(instance.value)
-    elif isinstance(content_object, Project):
-        if instance.key[0:4] != 'env_' and instance.key not in Project.VARS_KEY:
-            msg = 'Unknown variable key \'{}\'. Key must be in {} or starts from \'env_\''
-            raise ValidationError(msg.format(instance.key, Project.VARS_KEY))
+
+
+@receiver(signals.pre_save, sender=Variable)
+def check_project_variables_values(instance: Variable, *args, **kwargs) -> NoReturn:
+    if 'loaddata' in sys.argv or kwargs.get('raw', False):  # nocv
+        return
+    if not isinstance(instance.content_object, Project):
+        return
+
+    project_object = instance.content_object
+
+    is_ci_var = instance.key.startswith('ci_')
+    key_startswith = instance.key.startswith('env_') or is_ci_var
+    if not key_startswith and instance.key not in Project.VARS_KEY:
+        msg = 'Unknown variable key \'{}\'. Key must be in {} or starts from \'env_\' or \'ci_\'.'
+        raise ValidationError(msg.format(instance.key, Project.VARS_KEY))
+
+    is_ci_template = instance.key == 'ci_template'
+    qs_variables = project_object.variables.all()
+
+    if is_ci_var and qs_variables.filter(key__startswith='repo_sync_on_run').exists():
+        raise Conflict('Couldnt install CI/CD to project with "repo_sync_on_run" settings.')
+    if instance.key.startswith('repo_sync_on_run') and project_object.get_vars_prefixed('ci'):
+        raise Conflict('Couldnt install "repo_sync_on_run" settings for CI/CD project.')
+    if is_ci_template and not project_object.template.filter(pk=instance.value).exists():
+        raise ValidationError('Template does not exists in this project.')
 
 
 @receiver(signals.pre_save, sender=Group)

polemarch/main/models/projects.py

@@ -5,6 +5,7 @@
 import os
 import logging
 import traceback
+import time
 import uuid
 import six
 import requests
@@ -119,6 +120,7 @@ class ReadMe(ReadMe):
     VARS_KEY = [
         'repo_type',
         'repo_sync_on_run',
+        'repo_sync_on_run_timeout',
         'repo_branch',
         'repo_password',
         'repo_key'
@@ -140,13 +142,16 @@ class ReadMe(ReadMe):
         'boolean': bool,
     }
 
-    STATUSES = [
-        'NEW',
+    BUSY_STATUSES = [
         'WAIT_SYNC',
         'SYNC',
+    ]
+
+    STATUSES = [
+        'NEW',
         'ERROR',
         'OK',
-    ]
+    ] + BUSY_STATUSES
 
     @classproperty
     def PROJECTS_DIR(cls) -> Text:
@@ -178,10 +183,7 @@ def repo_class(self):
 
     @property
     def env_vars(self) -> Dict[Text, Any]:
-        env_var_list = dict()
-        for var_obj in self.variables.filter(key__startswith='env_'):
-            env_var_list[var_obj.key[4:]] = var_obj.value
-        return env_var_list
+        return self.get_vars_prefixed('env')
 
     @property
     def type(self) -> Text:
@@ -190,6 +192,13 @@ def type(self) -> Text:
         except self.variables.model.DoesNotExist:  # nocv
             return 'MANUAL'
 
+    @property
+    def repo_sync_timeout(self):
+        try:
+            return self.variables.get(key="repo_sync_on_run_timeout").value
+        except self.variables.model.DoesNotExist:
+            return settings.PROJECT_REPOSYNC_WAIT_SECONDS
+
     @property
     def config(self) -> Dict[Text, Any]:
         return self.get_ansible_config_parser().get_data()
@@ -292,11 +301,18 @@ def _prepare_kw(self, kind: str, mod_name: str, inventory=None, **extra) -> Dict
     def hook(self, when, msg) -> NoReturn:
         Hook.objects.all().execute(when, msg)
 
-    def sync_on_execution_handler(self, history: BModel) -> NoReturn:
+    def sync_on_execution_handler(self) -> NoReturn:
         if not self.vars.get('repo_sync_on_run', False):
             return
+        timeout = self.repo_sync_timeout
         try:
-            self.sync()
+            for i in range(timeout):  # pylint: disable=unused-variable
+                self.refresh_from_db(fields=['status'])
+                if self.status not in self.BUSY_STATUSES:
+                    self.sync()
+                    return
+                time.sleep(1)  # nocv
+            raise Exception('Project busy (timeout={}).'.format(timeout))  # nocv
         except Exception as exc:  # nocv
             raise self.SyncError("ERROR on Sync operation: " + str(exc))
 

polemarch/main/models/tasks.py

@@ -112,6 +112,9 @@ def execute(self, user: User, option: str = None, **extra):
         )
         return response
 
+    def ci_run(self):
+        self.execute(self.project.owner)
+
     def _convert_to_data(self, value):
         if isinstance(value, (six.string_types, six.text_type)):
             return json.loads(value)  # nocv