Increase the request_timeout from default (#153)

* Increase the request_timeout from default - As AWX gets overloaded with jobs, the response time of HTTP requests to the api/v2/jobs endpoint - The result is that AnsibleJob objects launcht he job, then keep checking for the status of the job that was launch. The GET request times out, and it launches a new job, which complicates the problem further. - This behavior is seen when createing 100+ AnsibleJob objects at once - We should keep this HTTP request_timeout abnormally high until the resource operator grows a better way to handle this (smart checks for timeout errors, etc.) * Bump minimum awx.awx collection version to 22.7.0 for request_timeout support * Add ability to configure request_timeout on jobs and workflows Signed-off-by: Christian M. Adams <[email protected]>
ansible · Dec 6, 2023 · e2b61c4 · e2b61c4
1 parent bd06497
commit e2b61c4
Show file tree

Hide file tree

Showing 10 changed files with 32 additions and 5 deletions.
diff --git a/config/crd/bases/tower.ansible.com_ansiblejobs.yaml b/config/crd/bases/tower.ansible.com_ansiblejobs.yaml
@@ -40,6 +40,9 @@ spec:
                 type: string
               workflow_template_name:
                 type: string
+              request_timeout:
+                type: integer
+                description: Timeout in seconds to wait for a response when querying the API for the job status
               runner_image:
                 type: string
                 description: Runner image used when running jobs

diff --git a/config/crd/bases/tower.ansible.com_ansibleworkflows.yaml b/config/crd/bases/tower.ansible.com_ansibleworkflows.yaml
@@ -38,6 +38,9 @@ spec:
                 type: string
               workflow_template_name:
                 type: string
+              request_timeout:
+                type: integer
+                description: Timeout in seconds to wait for a response when querying the API for the job status
               runner_image:
                 type: string
                 description: Runner image used when running jobs

diff --git a/config/manifests/bases/awx-resource-operator.clusterserviceversion.yaml b/config/manifests/bases/awx-resource-operator.clusterserviceversion.yaml
@@ -47,6 +47,7 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
       - displayName: Job TTL
+        description: Time to live for k8s job object after the playbook run has finished
         path: job_ttl
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
@@ -61,6 +62,12 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:advanced
+      - displayName: Request Timeout
+        description: Time in seconds to wait for a response from the AWX API
+        path: request_timeout
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:advanced
       - displayName: Ansible Runner Image
         path: runner_image
         x-descriptors:
@@ -198,10 +205,17 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
       - displayName: Workflow TTL
+        description: Time to live for k8s job object after the playbook run has finished
         path: workflow_ttl
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:advanced
+      - displayName: Request Timeout
+        description: Time in seconds to wait for a response from the AWX API
+        path: request_timeout
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:advanced
       - displayName: Ansible Runner Image
         path: runner_image
         x-descriptors:

diff --git a/molecule/requirements.yml b/molecule/requirements.yml
@@ -5,6 +5,6 @@ collections:
   - name: kubernetes.core
     version: "2.3.2"
   - name: operator_sdk.util
-    version: "0.2.0"
+    version: "0.4.0"
   - name: awx.awx
-    version: ">=13.0.0"
+    version: ">=22.7.0"
diff --git a/requirements.yml b/requirements.yml
@@ -5,4 +5,4 @@ collections:
   - name: operator_sdk.util
     version: "0.4.0"
   - name: awx.awx
-    version: ">=21.8.0"
+    version: ">=22.7.0"
diff --git a/roles/job_runner/defaults/main.yml b/roles/job_runner/defaults/main.yml
@@ -0,0 +1,4 @@
+---
+
+# Time before request to AWX api/v2/jobs/{id}/stdout/ times out (in seconds)
+request_timeout: 600
diff --git a/roles/job_runner/tasks/launch_deprecated_job.yml b/roles/job_runner/tasks/launch_deprecated_job.yml
@@ -58,6 +58,7 @@
       awx.awx.job_wait:
         job_id: "{{ job.id }}"
         job_type: workflow_jobs
+        request_timeout: "{{ request_timeout }}"
       register: job_result
   rescue:
     - name: Update status if job results in an error

diff --git a/roles/job_runner/tasks/launch_job.yml b/roles/job_runner/tasks/launch_job.yml
@@ -53,6 +53,7 @@
     - name: Register Job result when complete
       awx.awx.job_wait:
         job_id: "{{ job.id }}"
+        request_timeout: "{{ request_timeout }}"
       register: job_result
   rescue:
     - name: Update status if job results in an error

diff --git a/roles/job_runner/tasks/launch_workflow.yml b/roles/job_runner/tasks/launch_workflow.yml
@@ -53,6 +53,7 @@
       awx.awx.job_wait:
         job_id: "{{ job.id }}"
         job_type: workflow_jobs
+        request_timeout: "{{ request_timeout }}"
       register: job_result
   rescue:
     - name: Update status if job results in an error

diff --git a/test-e2e/token-container/requirements.yml b/test-e2e/token-container/requirements.yml
@@ -3,6 +3,6 @@ collections:
   - name: kubernetes.core
     version: "2.3.2"
   - name: operator_sdk.util
-    version: "0.2.0"
+    version: "0.4.0"
   - name: awx.awx
-    version: ">=13.0.0"
+    version: ">=22.7.0"