From 4c75f1f079182938baaa6f2035d4cbe2c8afaecb Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Tue, 26 Dec 2023 08:41:14 +0200
Subject: [PATCH 01/15] add ec module

---
 .github/workflows/test-environment.yml |  2 +-
 deploy/cloud/modules/ec/main.tf        | 54 ++++++++++++++++++
 deploy/cloud/modules/ec/output.tf      | 26 +++++++++
 deploy/cloud/modules/ec/terraform.tf   | 10 ++++
 deploy/cloud/modules/ec/variables.tf   | 76 ++++++++++++++++++++++++++
 deploy/test-environments/main.tf       | 16 +++---
 deploy/test-environments/terraform.tf  |  2 +-
 7 files changed, 177 insertions(+), 9 deletions(-)
 create mode 100644 deploy/cloud/modules/ec/main.tf
 create mode 100644 deploy/cloud/modules/ec/output.tf
 create mode 100644 deploy/cloud/modules/ec/terraform.tf
 create mode 100644 deploy/cloud/modules/ec/variables.tf

diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml
index 740a0e421b..75173dda95 100644
--- a/.github/workflows/test-environment.yml
+++ b/.github/workflows/test-environment.yml
@@ -402,6 +402,6 @@ jobs:
           poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4
 
       - name: Cleanup Environment
-        if: ${{ inputs.cleanup-env == true }}
+        if: ${{ inputs.cleanup-env == true || inputs.cleanup-env == 'true' }}
         run: |
           just delete-cloud-env ${{ env.DEPLOYMENT_NAME }} '' "false"
diff --git a/deploy/cloud/modules/ec/main.tf b/deploy/cloud/modules/ec/main.tf
new file mode 100644
index 0000000000..bdb700ffc6
--- /dev/null
+++ b/deploy/cloud/modules/ec/main.tf
@@ -0,0 +1,54 @@
+locals {
+  version = var.stack_version
+  region                           = var.region
+  name_prefix                      = var.deployment_name_prefix
+  deployment_template              = var.deployment_template
+  es_docker_image                  = lookup(var.docker_image, "elasticsearch", "")
+  es_docker_image_tag_override     = lookup(var.docker_image_tag_override, "elasticsearch", "")
+  kibana_docker_image              = lookup(var.docker_image, "kibana", "")
+  kibana_docker_image_tag_override = lookup(var.docker_image_tag_override, "kibana", "")
+  apm_docker_image                 = lookup(var.docker_image, "apm", "")
+  apm_docker_image_tag_override    = lookup(var.docker_image_tag_override, "apm", "")
+}
+
+data "ec_stack" "deployment_version" {
+  version_regex = local.version
+  region        = local.region
+}
+
+resource "ec_deployment" "deployment" {
+  name                   = "${local.name_prefix}-${data.ec_stack.deployment_version.version}"
+  version                = data.ec_stack.deployment_version.version
+  region                 = local.region
+  deployment_template_id = local.deployment_template
+  tags                   = var.tags
+
+  elasticsearch = {
+    autoscale = var.elasticsearch_autoscale
+    strategy  = "rolling_all"
+    config = {
+      docker_image = local.es_docker_image_tag_override != "" ? "${local.es_docker_image}:${local.es_docker_image_tag_override}" : null
+    }
+
+    hot = {
+      autoscaling = {
+        max_size = "128g"
+      }
+      size       = var.elasticsearch_size
+      zone_count = var.elasticsearch_zone_count
+    }
+  }
+
+  kibana = {
+    config = {
+      docker_image = local.kibana_docker_image_tag_override != "" ? "${local.kibana_docker_image}:${local.kibana_docker_image_tag_override}" : null
+    }
+  }
+
+  integrations_server = {
+    config = {
+      docker_image = local.apm_docker_image_tag_override != "" ? "${local.apm_docker_image}:${local.apm_docker_image_tag_override}" : null
+    }
+  }
+
+}
\ No newline at end of file
diff --git a/deploy/cloud/modules/ec/output.tf b/deploy/cloud/modules/ec/output.tf
new file mode 100644
index 0000000000..bc31e69542
--- /dev/null
+++ b/deploy/cloud/modules/ec/output.tf
@@ -0,0 +1,26 @@
+output "kibana_url" {
+  value       = ec_deployment.deployment.kibana.https_endpoint
+  description = "The secure Kibana URL"
+}
+
+output "elasticsearch_url" {
+  value       = ec_deployment.deployment.elasticsearch.https_endpoint
+  description = "The secure Elasticsearch URL"
+}
+
+output "elasticsearch_username" {
+  value       = ec_deployment.deployment.elasticsearch_username
+  sensitive   = true
+  description = "The Elasticsearch username"
+}
+
+output "elasticsearch_password" {
+  value       = ec_deployment.deployment.elasticsearch_password
+  sensitive   = true
+  description = "The Elasticsearch password"
+}
+
+output "stack_version" {
+  value       = data.ec_stack.deployment_version.version
+  description = "The matching stack pack version from the provided stack_version"
+}
diff --git a/deploy/cloud/modules/ec/terraform.tf b/deploy/cloud/modules/ec/terraform.tf
new file mode 100644
index 0000000000..4389ad7a85
--- /dev/null
+++ b/deploy/cloud/modules/ec/terraform.tf
@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">= 1.3, <2.0.0"
+
+  required_providers {
+    ec = {
+      source  = "elastic/ec"
+      version = ">=0.9.0"
+    }
+  }
+}
diff --git a/deploy/cloud/modules/ec/variables.tf b/deploy/cloud/modules/ec/variables.tf
new file mode 100644
index 0000000000..c7791b7dc3
--- /dev/null
+++ b/deploy/cloud/modules/ec/variables.tf
@@ -0,0 +1,76 @@
+variable "ec_api_key" {
+  type = string
+}
+
+variable "stack_version" {
+  description = "Optional version of the Elastic Cloud deployment"
+  type        = string
+  default     = "latest"
+}
+
+variable "region" {
+  description = "Optional region of the Elastic Cloud deployment"
+  type        = string
+  default     = "gcp-us-west2"
+}
+
+variable "deployment_template" {
+  description = "Optional defaults to the CPU optimized template for GCP"
+  type        = string
+  default     = "gcp-compute-optimized-v3"
+}
+
+variable "deployment_name_prefix" {
+  description = "Prefix for the Elastic Cloud deployment name"
+  type        = string
+  default     = "cloud-security"
+}
+
+variable "tags" {
+  type = map(string)
+  default = {
+    "deployment"  = "cloud-security",
+    "environment" = "test-enviroment",
+  }
+  description = "Optional set of tags to use for all deployments"
+}
+
+variable "elasticsearch_size" {
+  default     = "8g"
+  type        = string
+  description = "Optional Elasticsearch instance size"
+}
+
+variable "elasticsearch_zone_count" {
+  default     = 2
+  type        = number
+  description = "Optional Elasticsearch zone count"
+}
+
+variable "elasticsearch_autoscale" {
+  default     = false
+  type        = bool
+  description = "Optional autoscale the Elasticsearch cluster"
+}
+
+# Docker image overrides
+
+variable "docker_image_tag_override" {
+  default = {
+    "elasticsearch" = "",
+    "kibana"        = "",
+    "apm"           = "",
+  }
+  description = "Optional docker image tag overrides, The full map needs to be specified"
+  type        = map(string)
+}
+
+variable "docker_image" {
+  default = {
+    "elasticsearch" = "docker.elastic.co/cloud-release/elasticsearch-cloud-ess",
+    "kibana"        = "docker.elastic.co/cloud-release/kibana-cloud",
+    "apm"           = "docker.elastic.co/cloud-release/elastic-agent-cloud",
+  }
+  type        = map(string)
+  description = "Optional docker image overrides. The full map needs to be specified"
+}
\ No newline at end of file
diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf
index 3422e20577..c0c6916bd5 100644
--- a/deploy/test-environments/main.tf
+++ b/deploy/test-environments/main.tf
@@ -14,6 +14,7 @@ locals {
     Content-type  = "application/json"
     Authorization = "ApiKey ${var.ec_api_key}"
   }
+  cleaned_version = length(regexall("(-[0-9a-z]{4})", var.stack_version)) > 0 ? split("-", var.stack_version)[0] : var.stack_version
 }
 
 # EC2 + kind deployment
@@ -55,24 +56,25 @@ provider "restapi" {
 # Elastic Cloud (EC) deployment
 module "ec_deployment" {
   count  = var.serverless_mode ? 0 : 1
-  source = "github.com/elastic/apm-server/testing/infra/terraform/modules/ec_deployment"
-
+  # source = "github.com/elastic/apm-server/testing/infra/terraform/modules/ec_deployment"
+  source = "../cloud/modules/ec"
+  ec_api_key = var.ec_api_key
   region        = var.ess_region
-  stack_version = var.stack_version
+  stack_version = local.cleaned_version
 
   deployment_template    = var.deployment_template
   deployment_name_prefix = "${var.deployment_name}-${random_string.suffix.result}"
 
-  integrations_server = true
+  # integrations_server = true
 
   elasticsearch_size       = var.elasticsearch_size
   elasticsearch_zone_count = var.elasticsearch_zone_count
 
   docker_image = var.docker_image_override
   docker_image_tag_override = {
-    "elasticsearch" : "",
-    "kibana" : "",
-    "apm" : ""
+    "elasticsearch" : "${var.stack_version}",
+    "kibana" : "${var.stack_version}",
+    "apm" : "${var.stack_version}"
   }
 }
 
diff --git a/deploy/test-environments/terraform.tf b/deploy/test-environments/terraform.tf
index 4494c3d8ba..4a1de8dd55 100644
--- a/deploy/test-environments/terraform.tf
+++ b/deploy/test-environments/terraform.tf
@@ -7,7 +7,7 @@ terraform {
 
     ec = {
       source  = "elastic/ec"
-      version = ">=0.5.0"
+      version = ">=0.9.0"
     }
 
     restapi = {

From c541fae359231ab863bed428c3662af10365300d Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Tue, 26 Dec 2023 08:54:59 +0200
Subject: [PATCH 02/15] update terraform format

---
 deploy/test-environments/main.tf | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf
index c0c6916bd5..aa44394288 100644
--- a/deploy/test-environments/main.tf
+++ b/deploy/test-environments/main.tf
@@ -55,10 +55,10 @@ provider "restapi" {
 
 # Elastic Cloud (EC) deployment
 module "ec_deployment" {
-  count  = var.serverless_mode ? 0 : 1
+  count = var.serverless_mode ? 0 : 1
   # source = "github.com/elastic/apm-server/testing/infra/terraform/modules/ec_deployment"
-  source = "../cloud/modules/ec"
-  ec_api_key = var.ec_api_key
+  source        = "../cloud/modules/ec"
+  ec_api_key    = var.ec_api_key
   region        = var.ess_region
   stack_version = local.cleaned_version
 

From f54a9c25a8e7af6b7500ee5cfa80ad72f7436b7d Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Tue, 26 Dec 2023 09:05:27 +0200
Subject: [PATCH 03/15] update ec module format

---
 deploy/cloud/modules/ec/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/cloud/modules/ec/main.tf b/deploy/cloud/modules/ec/main.tf
index bdb700ffc6..cc0b7de52b 100644
--- a/deploy/cloud/modules/ec/main.tf
+++ b/deploy/cloud/modules/ec/main.tf
@@ -1,5 +1,5 @@
 locals {
-  version = var.stack_version
+  version                          = var.stack_version
   region                           = var.region
   name_prefix                      = var.deployment_name_prefix
   deployment_template              = var.deployment_template

From eb9cdd557e7d1dd130e7917e8542ab4957e76fa7 Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Tue, 26 Dec 2023 10:46:22 +0200
Subject: [PATCH 04/15] update workflow

---
 .github/workflows/upgrade-environment.yml |  2 +-
 deploy/test-environments/variables.tf     | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/upgrade-environment.yml b/.github/workflows/upgrade-environment.yml
index ffc442a734..74871d060d 100644
--- a/.github/workflows/upgrade-environment.yml
+++ b/.github/workflows/upgrade-environment.yml
@@ -52,7 +52,7 @@ jobs:
             PREVIOUS_VERSION=$(./.ci/scripts/get-previous-version.sh "$VERSION")
             echo "PREVIOUS_VERSION=$PREVIOUS_VERSION" >> $GITHUB_OUTPUT
           else
-            echo "PREVIOUS_VERSION=${{ inputs.base-elk-stack-version }}" >> $GITHUB_ENV
+            echo "PREVIOUS_VERSION=${{ inputs.base-elk-stack-version }}" >> $GITHUB_OUTPUT
           fi
   deploy:
     uses: ./.github/workflows/test-environment.yml
diff --git a/deploy/test-environments/variables.tf b/deploy/test-environments/variables.tf
index dd397d3409..8769b200ee 100644
--- a/deploy/test-environments/variables.tf
+++ b/deploy/test-environments/variables.tf
@@ -66,9 +66,9 @@ variable "elasticsearch_zone_count" {
 
 variable "docker_image_tag_override" {
   default = {
-    "elasticsearch" : "",
-    "kibana" : "",
-    "apm" : "",
+    "elasticsearch" = "",
+    "kibana" = "",
+    "apm" = "",
   }
   description = "Optional docker image tag override"
   type        = map(string)
@@ -76,9 +76,9 @@ variable "docker_image_tag_override" {
 
 variable "docker_image_override" {
   default = {
-    "elasticsearch" : "docker.elastic.co/cloud-release/elasticsearch-cloud-ess",
-    "kibana" : "docker.elastic.co/cloud-release/kibana-cloud",
-    "apm" : "docker.elastic.co/cloud-release/elastic-agent-cloud",
+    "elasticsearch" = "docker.elastic.co/cloud-release/elasticsearch-cloud-ess",
+    "kibana" = "docker.elastic.co/cloud-release/kibana-cloud",
+    "apm" = "docker.elastic.co/cloud-release/elastic-agent-cloud",
   }
   type = map(string)
 }

From f444990c22467af0609696430feb7a4daa29a721 Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Tue, 26 Dec 2023 10:57:50 +0200
Subject: [PATCH 05/15] updat tf fmt

---
 deploy/test-environments/variables.tf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/deploy/test-environments/variables.tf b/deploy/test-environments/variables.tf
index 8769b200ee..561817784f 100644
--- a/deploy/test-environments/variables.tf
+++ b/deploy/test-environments/variables.tf
@@ -67,8 +67,8 @@ variable "elasticsearch_zone_count" {
 variable "docker_image_tag_override" {
   default = {
     "elasticsearch" = "",
-    "kibana" = "",
-    "apm" = "",
+    "kibana"        = "",
+    "apm"           = "",
   }
   description = "Optional docker image tag override"
   type        = map(string)
@@ -77,8 +77,8 @@ variable "docker_image_tag_override" {
 variable "docker_image_override" {
   default = {
     "elasticsearch" = "docker.elastic.co/cloud-release/elasticsearch-cloud-ess",
-    "kibana" = "docker.elastic.co/cloud-release/kibana-cloud",
-    "apm" = "docker.elastic.co/cloud-release/elastic-agent-cloud",
+    "kibana"        = "docker.elastic.co/cloud-release/kibana-cloud",
+    "apm"           = "docker.elastic.co/cloud-release/elastic-agent-cloud",
   }
   type = map(string)
 }

From 0e7139745eef927989f94d56036d86f8cef39989 Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Tue, 26 Dec 2023 18:42:17 +0200
Subject: [PATCH 06/15] update workflow and tf

---
 .github/workflows/upgrade-environment.yml | 24 ++++++++++++++++++-----
 deploy/cloud/modules/ec/main.tf           | 15 +++++++++++++-
 deploy/test-environments/main.tf          |  9 +++++----
 deploy/test-environments/variables.tf     |  2 +-
 4 files changed, 39 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/upgrade-environment.yml b/.github/workflows/upgrade-environment.yml
index 74871d060d..2870f77a45 100644
--- a/.github/workflows/upgrade-environment.yml
+++ b/.github/workflows/upgrade-environment.yml
@@ -38,8 +38,9 @@ jobs:
   init:
     runs-on: ubuntu-20.04
     outputs:
-      stack-version: ${{ steps.set-previous-version.outputs.PREVIOUS_VERSION }}
+      base-stack-version: ${{ steps.set-previous-version.outputs.PREVIOUS_VERSION }}
       ess-region: ${{ env.TF_VAR_ess_region }}
+      target-agent-version: ${{ steps.clean-version.outputs.AGENT_VERSION }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
@@ -54,6 +55,19 @@ jobs:
           else
             echo "PREVIOUS_VERSION=${{ inputs.base-elk-stack-version }}" >> $GITHUB_OUTPUT
           fi
+      - name: Process BC version
+        id: clean-version
+        run: |
+          # Extract the stack version
+          stack_version="${{ inputs.target-elk-stack-version }}"
+
+          # Check if the version contains a commit hash, remove it
+          if [[ $stack_version =~ -[a-f0-9]+ ]]; then
+            cleaned_version=$(echo $stack_version | awk -F"-" '{print $1}')
+            echo "AGENT_VERSION=$cleaned_version" >> $GITHUB_OUTPUT
+          else
+            echo "AGENT_VERSION=$stack_version" >> $GITHUB_OUTPUT
+          fi
   deploy:
     uses: ./.github/workflows/test-environment.yml
     needs: init
@@ -63,7 +77,7 @@ jobs:
       id-token: 'write'
     with:
       deployment_name: ${{ inputs.deployment_name }}
-      elk-stack-version: ${{ needs.init.outputs.stack-version }}
+      elk-stack-version: ${{ needs.init.outputs.base-stack-version }}
       ess-region: ${{ needs.init.outputs.ess-region }}
       run-sanity-tests: false # Set to true once the issue at https://github.com/elastic/kibana/pull/171200 is resolved.
       serverless_mode: false
@@ -167,7 +181,7 @@ jobs:
         working-directory: ./tests
         env:
           USE_K8S: false
-          AGENT_VERSION: ${{ needs.init.outputs.stack-version }}
+          AGENT_VERSION: ${{ needs.init.outputs.base-stack-version }}
         run: |
           poetry install
           poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4
@@ -203,7 +217,7 @@ jobs:
         working-directory: ${{ env.WORKING_DIR }}/${{ env.FLEET_API_DIR }}
         env:
           CNVM_STACK_NAME: ${{ needs.deploy.outputs.cnvm-stack-name }}
-          STACK_VERSION: ${{ inputs.target-elk-stack-version }}
+          STACK_VERSION: ${{ needs.init.outputs.target-agent-version }}
         run: |
           poetry run python upgrade_agents.py
 
@@ -211,7 +225,7 @@ jobs:
         if: success()
         working-directory: ./tests
         env:
-          AGENT_VERSION: ${{ inputs.target-elk-stack-version }}
+          AGENT_VERSION: ${{ needs.init.outputs.target-agent-version }}
           USE_K8S: false
         run: |
           poetry install
diff --git a/deploy/cloud/modules/ec/main.tf b/deploy/cloud/modules/ec/main.tf
index cc0b7de52b..79ff0a3cc4 100644
--- a/deploy/cloud/modules/ec/main.tf
+++ b/deploy/cloud/modules/ec/main.tf
@@ -30,13 +30,26 @@ resource "ec_deployment" "deployment" {
       docker_image = local.es_docker_image_tag_override != "" ? "${local.es_docker_image}:${local.es_docker_image_tag_override}" : null
     }
 
+    cold = {
+      autoscaling = {}
+    }
+
+    frozen = {
+      autoscaling = {}
+    }
+
     hot = {
       autoscaling = {
-        max_size = "128g"
+        max_size          = "128g"
+        max_size_resource = "memory"
       }
       size       = var.elasticsearch_size
       zone_count = var.elasticsearch_zone_count
     }
+
+    warm = {
+      autoscaling = {}
+    }
   }
 
   kibana = {
diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf
index aa44394288..4b935e6fac 100644
--- a/deploy/test-environments/main.tf
+++ b/deploy/test-environments/main.tf
@@ -61,20 +61,21 @@ module "ec_deployment" {
   ec_api_key    = var.ec_api_key
   region        = var.ess_region
   stack_version = local.cleaned_version
+  # stack_version = var.stack_version
 
   deployment_template    = var.deployment_template
   deployment_name_prefix = "${var.deployment_name}-${random_string.suffix.result}"
 
   # integrations_server = true
-
+  elasticsearch_autoscale  = true
   elasticsearch_size       = var.elasticsearch_size
   elasticsearch_zone_count = var.elasticsearch_zone_count
 
   docker_image = var.docker_image_override
   docker_image_tag_override = {
-    "elasticsearch" : "${var.stack_version}",
-    "kibana" : "${var.stack_version}",
-    "apm" : "${var.stack_version}"
+    "elasticsearch" = "${var.stack_version}",
+    "kibana"        = "${var.stack_version}",
+    "apm"           = "${var.stack_version}"
   }
 }
 
diff --git a/deploy/test-environments/variables.tf b/deploy/test-environments/variables.tf
index 561817784f..c27a645ef9 100644
--- a/deploy/test-environments/variables.tf
+++ b/deploy/test-environments/variables.tf
@@ -59,7 +59,7 @@ variable "elasticsearch_size" {
 }
 
 variable "elasticsearch_zone_count" {
-  default     = 1
+  default     = 2
   type        = number
   description = "Optional Elasticsearch zone count"
 }

From 056d798f14aafed81ba4ddd5a41d3fb518813fc6 Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Wed, 27 Dec 2023 08:18:26 +0200
Subject: [PATCH 07/15] add tags

---
 .github/workflows/destroy-environment.yml | 1 +
 .github/workflows/test-environment.yml    | 1 +
 deploy/test-environments/main.tf          | 3 ++-
 deploy/test-environments/variables.tf     | 5 +++++
 4 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/destroy-environment.yml b/.github/workflows/destroy-environment.yml
index 8e02f458fe..4d2e99b7a4 100644
--- a/.github/workflows/destroy-environment.yml
+++ b/.github/workflows/destroy-environment.yml
@@ -74,6 +74,7 @@ jobs:
 
       - uses: ./.github/actions/slack-notification
         if: always()
+        continue-on-error: true
         env:
           RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           JOB_STATUS_COLOR: "${{ job.status == 'success' && '#36a64f' || '#D40E0D' }}"
diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml
index 5a52ef3fbb..e86b02c0fc 100644
--- a/.github/workflows/test-environment.yml
+++ b/.github/workflows/test-environment.yml
@@ -423,6 +423,7 @@ jobs:
 
       - uses: ./.github/actions/slack-notification
         if: always()
+        continue-on-error: true
         with:
           vault-url: ${{ secrets.VAULT_ADDR }}
           vault-role-id: ${{ secrets.CSP_VAULT_ROLE_ID }}
diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf
index 4b935e6fac..a740a2aad6 100644
--- a/deploy/test-environments/main.tf
+++ b/deploy/test-environments/main.tf
@@ -8,6 +8,7 @@ locals {
     org      = "${var.org}"
     team     = "${var.team}"
     project  = "${var.project}"
+    owner    = "${var.owner}"
   }
   ec_url = "https://cloud.elastic.co"
   ec_headers = {
@@ -61,7 +62,7 @@ module "ec_deployment" {
   ec_api_key    = var.ec_api_key
   region        = var.ess_region
   stack_version = local.cleaned_version
-  # stack_version = var.stack_version
+  tags          = local.common_tags
 
   deployment_template    = var.deployment_template
   deployment_name_prefix = "${var.deployment_name}-${random_string.suffix.result}"
diff --git a/deploy/test-environments/variables.tf b/deploy/test-environments/variables.tf
index c27a645ef9..3c16153d45 100644
--- a/deploy/test-environments/variables.tf
+++ b/deploy/test-environments/variables.tf
@@ -107,4 +107,9 @@ variable "project" {
   description = "Optional project resource tag"
 }
 
+variable "owner" {
+  default     = "cloudbeat"
+  type        = string
+  description = "Optional owner tag"
+}
 # ============================================

From 558e2c3287c93859b8d6e9a3fa42347fd8c95f54 Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Wed, 27 Dec 2023 11:36:11 +0200
Subject: [PATCH 08/15] add ec api key input support

---
 .github/workflows/upgrade-environment.yml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/.github/workflows/upgrade-environment.yml b/.github/workflows/upgrade-environment.yml
index 2870f77a45..051876c86a 100644
--- a/.github/workflows/upgrade-environment.yml
+++ b/.github/workflows/upgrade-environment.yml
@@ -22,6 +22,10 @@ on:
       docker-image-override:
         required: false
         description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
+      ec-api-key:
+        type: string
+        description: "**Optional** By default, the environment will be created in our Cloud Security Organization. If you want to use your own cloud account, enter your Elastic Cloud API key."
+        required: false
 
 env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -33,6 +37,7 @@ env:
   TF_VAR_ess_region: gcp-us-west2
   TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
   DOCKER_IMAGE: ${{ inputs.docker-image-override }}
+  TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
 
 jobs:
   init:
@@ -41,10 +46,19 @@ jobs:
       base-stack-version: ${{ steps.set-previous-version.outputs.PREVIOUS_VERSION }}
       ess-region: ${{ env.TF_VAR_ess_region }}
       target-agent-version: ${{ steps.clean-version.outputs.AGENT_VERSION }}
+      ec-api-key: ${{ steps.set-ec-api-key.outputs.EC_KEY }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
 
+      - name: Mask Sensitive Data
+        if: inputs.ec-api-key != ''
+        id: set-ec-api-key
+        run: |
+          ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH)
+          echo "::add-mask::$ec_api_key"
+          echo "EC_KEY=$ec_api_key" >> $GITHUB_OUTPUT
+
       - name: Set Previous Version
         id: set-previous-version
         run: |
@@ -81,6 +95,7 @@ jobs:
       ess-region: ${{ needs.init.outputs.ess-region }}
       run-sanity-tests: false # Set to true once the issue at https://github.com/elastic/kibana/pull/171200 is resolved.
       serverless_mode: false
+      ec-api-key: ${{ needs.init.outputs.ec-api-key }}
     secrets: inherit
   upgrade:
     runs-on: ubuntu-20.04

From 62a121efc15c0da93331eb8908aaf9e24ac7ddef Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Wed, 27 Dec 2023 11:39:36 +0200
Subject: [PATCH 09/15] remove duplicate key

---
 .github/workflows/upgrade-environment.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/upgrade-environment.yml b/.github/workflows/upgrade-environment.yml
index 051876c86a..08c1364622 100644
--- a/.github/workflows/upgrade-environment.yml
+++ b/.github/workflows/upgrade-environment.yml
@@ -37,7 +37,6 @@ env:
   TF_VAR_ess_region: gcp-us-west2
   TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
   DOCKER_IMAGE: ${{ inputs.docker-image-override }}
-  TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
 
 jobs:
   init:

From 0eea065bc4901d2d2c53ead3763e0fcfc9f23beb Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Wed, 27 Dec 2023 16:13:04 +0200
Subject: [PATCH 10/15] update tf and workflow

---
 .github/workflows/test-environment.yml    |  3 ++-
 .github/workflows/upgrade-environment.yml | 21 ++++++---------------
 deploy/test-environments/main.tf          |  3 +--
 3 files changed, 9 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml
index e86b02c0fc..d67943751a 100644
--- a/.github/workflows/test-environment.yml
+++ b/.github/workflows/test-environment.yml
@@ -198,7 +198,8 @@ jobs:
           terraform apply --auto-approve \
             -var="deployment_name=${{ env.DEPLOYMENT_NAME }}" \
             -var="region=${{ env.AWS_REGION }}" \
-            -var="project=${{ github.actor }}"
+            -var="project=${{ github.actor }}" \
+            -var="owner=${{ github.actor }}"
 
       - name: Set Environment Output
         id: env-output
diff --git a/.github/workflows/upgrade-environment.yml b/.github/workflows/upgrade-environment.yml
index 08c1364622..58c576b894 100644
--- a/.github/workflows/upgrade-environment.yml
+++ b/.github/workflows/upgrade-environment.yml
@@ -22,10 +22,6 @@ on:
       docker-image-override:
         required: false
         description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
-      ec-api-key:
-        type: string
-        description: "**Optional** By default, the environment will be created in our Cloud Security Organization. If you want to use your own cloud account, enter your Elastic Cloud API key."
-        required: false
 
 env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -45,19 +41,10 @@ jobs:
       base-stack-version: ${{ steps.set-previous-version.outputs.PREVIOUS_VERSION }}
       ess-region: ${{ env.TF_VAR_ess_region }}
       target-agent-version: ${{ steps.clean-version.outputs.AGENT_VERSION }}
-      ec-api-key: ${{ steps.set-ec-api-key.outputs.EC_KEY }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
 
-      - name: Mask Sensitive Data
-        if: inputs.ec-api-key != ''
-        id: set-ec-api-key
-        run: |
-          ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH)
-          echo "::add-mask::$ec_api_key"
-          echo "EC_KEY=$ec_api_key" >> $GITHUB_OUTPUT
-
       - name: Set Previous Version
         id: set-previous-version
         run: |
@@ -94,7 +81,6 @@ jobs:
       ess-region: ${{ needs.init.outputs.ess-region }}
       run-sanity-tests: false # Set to true once the issue at https://github.com/elastic/kibana/pull/171200 is resolved.
       serverless_mode: false
-      ec-api-key: ${{ needs.init.outputs.ec-api-key }}
     secrets: inherit
   upgrade:
     runs-on: ubuntu-20.04
@@ -153,7 +139,11 @@ jobs:
         id: apply
         if: success()
         run: |
-          terraform apply --auto-approve -var="deployment_name=${{ inputs.deployment_name }}" -var="region=${{ env.AWS_REGION }}"
+          terraform apply --auto-approve \
+            -var="deployment_name=${{ inputs.deployment_name }}" \
+            -var="region=${{ env.AWS_REGION }}" \
+            -var="project=${{ github.actor }}" \
+            -var="owner=${{ github.actor }}"
 
       - name: Set Environment Output
         id: env-output
@@ -226,6 +216,7 @@ jobs:
               --name $(terraform output -raw deployment_name) --alias eks-config
           kubectl config use-context eks-config
           kubectl set image daemonset elastic-agent -n kube-system elastic-agent=${{ env.DOCKER_IMAGE }}
+          kubectl rollout restart daemonset/elastic-agent -n kube-system
 
       - name: Upgrade Linux agents
         working-directory: ${{ env.WORKING_DIR }}/${{ env.FLEET_API_DIR }}
diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf
index a740a2aad6..e3693741d4 100644
--- a/deploy/test-environments/main.tf
+++ b/deploy/test-environments/main.tf
@@ -57,7 +57,7 @@ provider "restapi" {
 # Elastic Cloud (EC) deployment
 module "ec_deployment" {
   count = var.serverless_mode ? 0 : 1
-  # source = "github.com/elastic/apm-server/testing/infra/terraform/modules/ec_deployment"
+
   source        = "../cloud/modules/ec"
   ec_api_key    = var.ec_api_key
   region        = var.ess_region
@@ -67,7 +67,6 @@ module "ec_deployment" {
   deployment_template    = var.deployment_template
   deployment_name_prefix = "${var.deployment_name}-${random_string.suffix.result}"
 
-  # integrations_server = true
   elasticsearch_autoscale  = true
   elasticsearch_size       = var.elasticsearch_size
   elasticsearch_zone_count = var.elasticsearch_zone_count

From 738d28da91101c4ceae60e1652d8fff561e66f6d Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Wed, 27 Dec 2023 17:37:49 +0200
Subject: [PATCH 11/15] update workflow and docs

---
 .github/workflows/test-environment.yml | 15 ++++++++++++++-
 dev-docs/Cloud-Env-Testing.md          |  6 +++---
 dev-docs/Cloud-Env-Upgrade.md          |  4 ++--
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml
index d67943751a..eae99839c8 100644
--- a/.github/workflows/test-environment.yml
+++ b/.github/workflows/test-environment.yml
@@ -95,6 +95,7 @@ env:
   AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments"
   GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments"
   TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
+  CLEANUP_ENV: ${{ inputs.cleanup-env }}
 
 jobs:
   Deploy:
@@ -152,6 +153,18 @@ jobs:
           echo "::add-mask::$ec_api_key"
           echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV
 
+      - name: Process BC version
+        id: remove-commit-hash
+        run: |
+          # Extract the stack version
+          stack_version="${{ inputs.elk-stack-version }}"
+
+          # Check if the version contains a commit hash, remove it
+          if [[ $stack_version =~ -[a-f0-9]+ ]]; then
+            cleaned_version=$(echo $stack_version | awk -F"-" '{print $1}')
+            echo "STACK_VERSION=$cleaned_version" >> $GITHUB_ENV
+          fi
+
       - name: Init Enrollment Token
         run: |
           enrollment_token="init"
@@ -404,7 +417,7 @@ jobs:
           poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4
 
       - name: Cleanup Environment
-        if: ${{ inputs.cleanup-env == true || inputs.cleanup-env == 'true' }}
+        if: ${{ '!cancelled()' && env.CLEANUP_ENV == 'true' }}
         run: |
           just delete-cloud-env ${{ env.DEPLOYMENT_NAME }} '' "false"
 
diff --git a/dev-docs/Cloud-Env-Testing.md b/dev-docs/Cloud-Env-Testing.md
index 727a0f7456..12572ddd83 100644
--- a/dev-docs/Cloud-Env-Testing.md
+++ b/dev-docs/Cloud-Env-Testing.md
@@ -23,8 +23,8 @@ Follow these steps to run the workflow:
 
     - **`elk-stack-version`**: Specify the version of Elastic Cloud stack, either a SNAPSHOT or a build candidate (BC)
       version. Check the available versions [here](https://artifacts-staging.elastic.co/dra-info/index.html).
-      For BC, enter only the version without additions/commit sha, e.g. `8.8.1`.
-      For SNAPSHOT, enter the full version, e.g. `8.8.1-SNAPSHOT`.
+      For BC, enter version with additions/commit sha, e.g. `8.12.0-61156bc6`.
+      For SNAPSHOT, enter the full version, e.g. `8.13.0-SNAPSHOT`.
 
     - **`ess-region`**: Indicate the Elastic Cloud deployment region. The default value is `gcp-us-west2`, which
       supports
@@ -34,7 +34,7 @@ Follow these steps to run the workflow:
 
 4. Optionally, modify other parameters if required:
 
-    - **`docker-image-override`** (**optional**): Use this to replace the default Docker image for build candidate (BC) or
+    - **`docker-image-override`** (**optional**): Use this to replace the default agent Docker image for build candidate (BC) or
       SNAPSHOT versions.
       Provide the full image path. Leave this field blank for snapshot versions. Follow this format for the image
       path: `docker.elastic.co/cloud-release/elastic-agent-cloud:8.8.1-9ac7eb02`. If you're not sure where to get this
diff --git a/dev-docs/Cloud-Env-Upgrade.md b/dev-docs/Cloud-Env-Upgrade.md
index 3d59ce11f8..95822220a4 100644
--- a/dev-docs/Cloud-Env-Upgrade.md
+++ b/dev-docs/Cloud-Env-Upgrade.md
@@ -22,8 +22,8 @@ Follow these steps to run the workflow:
       instance: `john-8-11-0-nov1`.
 
     - **`target-elk-stack-version`**: Specify the target version for the Elastic Cloud stack upgrade. This version represents the goal to which the workflow will upgrade the stack. Check the available versions [here](https://artifacts-staging.elastic.co/dra-info/index.html).
-      For BC, enter only the version without additions/commit sha, e.g. `8.11.0`.
-      For SNAPSHOT, enter the full version, e.g. `8.12.0-SNAPSHOT`.
+      For BC, enter version with additions/commit sha, e.g. `8.12.0-61156bc6`.
+      For SNAPSHOT, enter the full version, e.g. `8.13.0-SNAPSHOT`.
 
    ![Required Parameters](https://github.com/elastic/cloudbeat/assets/99176494/9475f553-70c9-4dd7-8330-260bbd704df8)
 

From 1a1e991cc32b884dd5898922e4d9e98f88899cfd Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Thu, 4 Jan 2024 09:56:34 +0200
Subject: [PATCH 12/15] update workflows

---
 .github/workflows/destroy-environment.yml | 1 -
 .github/workflows/test-environment.yml    | 4 +---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/destroy-environment.yml b/.github/workflows/destroy-environment.yml
index 4d2e99b7a4..8e02f458fe 100644
--- a/.github/workflows/destroy-environment.yml
+++ b/.github/workflows/destroy-environment.yml
@@ -74,7 +74,6 @@ jobs:
 
       - uses: ./.github/actions/slack-notification
         if: always()
-        continue-on-error: true
         env:
           RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           JOB_STATUS_COLOR: "${{ job.status == 'success' && '#36a64f' || '#D40E0D' }}"
diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml
index eae99839c8..6db3129d80 100644
--- a/.github/workflows/test-environment.yml
+++ b/.github/workflows/test-environment.yml
@@ -95,7 +95,6 @@ env:
   AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments"
   GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments"
   TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
-  CLEANUP_ENV: ${{ inputs.cleanup-env }}
 
 jobs:
   Deploy:
@@ -417,7 +416,7 @@ jobs:
           poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4
 
       - name: Cleanup Environment
-        if: ${{ '!cancelled()' && env.CLEANUP_ENV == 'true' }}
+        if: ${{ inputs.cleanup-env == true }}
         run: |
           just delete-cloud-env ${{ env.DEPLOYMENT_NAME }} '' "false"
 
@@ -437,7 +436,6 @@ jobs:
 
       - uses: ./.github/actions/slack-notification
         if: always()
-        continue-on-error: true
         with:
           vault-url: ${{ secrets.VAULT_ADDR }}
           vault-role-id: ${{ secrets.CSP_VAULT_ROLE_ID }}

From 63bdac7c7da1a3342bc69a3b8e00dddc61ba8749 Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Mon, 8 Jan 2024 17:28:30 +0200
Subject: [PATCH 13/15] updated documentation

---
 .github/workflows/test-environment.yml    | 4 ++--
 .github/workflows/upgrade-environment.yml | 2 +-
 deploy/cloud/modules/ec/variables.tf      | 4 ++++
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml
index 8c0b7cbcd5..c7493dc8cb 100644
--- a/.github/workflows/test-environment.yml
+++ b/.github/workflows/test-environment.yml
@@ -17,7 +17,7 @@ on:
         default: false
       elk-stack-version:
         required: true
-        description: "Stack version: For released/BC version use 8.x.y, for SNAPSHOT use 8.x.y-SNAPSHOT"
+        description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT"
         default: "8.11.0"
         type: string
       ess-region:
@@ -54,7 +54,7 @@ on:
         default: false
       elk-stack-version:
         required: true
-        description: "Stack version: For released/BC version use 8.x.y, for SNAPSHOT use 8.x.y-SNAPSHOT"
+        description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT"
         default: "8.10.0"
         type: string
       ess-region:
diff --git a/.github/workflows/upgrade-environment.yml b/.github/workflows/upgrade-environment.yml
index 58c576b894..d308987b0a 100644
--- a/.github/workflows/upgrade-environment.yml
+++ b/.github/workflows/upgrade-environment.yml
@@ -12,7 +12,7 @@ on:
         required: true
       target-elk-stack-version:
         required: true
-        description: "Target version of the ELK stack: For BC version use 8.x.y, for SNAPSHOT use 8.x.y-SNAPSHOT"
+        description: "Target version of the ELK stack: For BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT"
         default: "8.11.0"
         type: string
       base-elk-stack-version:
diff --git a/deploy/cloud/modules/ec/variables.tf b/deploy/cloud/modules/ec/variables.tf
index c7791b7dc3..ca702462c2 100644
--- a/deploy/cloud/modules/ec/variables.tf
+++ b/deploy/cloud/modules/ec/variables.tf
@@ -55,6 +55,10 @@ variable "elasticsearch_autoscale" {
 
 # Docker image overrides
 
+# Docker image tag override is used to override the default docker image tag
+# for BC reasons. This is used to test new versions of the cloud deployment
+# This option allow to pin the docker image tag to a specific version to prevent
+# unexpected changes in the deployment.
 variable "docker_image_tag_override" {
   default = {
     "elasticsearch" = "",

From a71888ebbf4981d713bae141474b21890b5a7146 Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Tue, 9 Jan 2024 16:24:28 +0200
Subject: [PATCH 14/15] fix review comments

---
 .github/workflows/test-environment.yml | 27 ++++++++++++++++++++++----
 deploy/cloud/modules/ec/main.tf        | 18 ++++++++---------
 deploy/test-environments/README.md     | 10 +++++++++-
 deploy/test-environments/main.tf       | 16 ++++++++-------
 deploy/test-environments/variables.tf  | 15 ++++++--------
 5 files changed, 56 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml
index 7e2a04784a..f79ad50de1 100644
--- a/.github/workflows/test-environment.yml
+++ b/.github/workflows/test-environment.yml
@@ -104,7 +104,6 @@ jobs:
       run:
         working-directory: ${{ env.WORKING_DIR }}
     env:
-      TF_VAR_stack_version: ${{ inputs.elk-stack-version }}
       TF_VAR_ess_region: ${{ inputs.ess-region }}
       DEPLOYMENT_NAME: ${{ inputs.deployment_name }}
       TF_VAR_serverless_mode: ${{ inputs.serverless_mode }}
@@ -112,7 +111,6 @@ jobs:
       S3_BASE_BUCKET: "s3://tf-state-bucket-test-infra"
       S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra"
       DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }}
-      STACK_VERSION: ${{ inputs.elk-stack-version }}
       CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack"
     # Add "id-token" with the intended permissions.
     permissions:
@@ -152,16 +150,37 @@ jobs:
           echo "::add-mask::$ec_api_key"
           echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV
 
-      - name: Process BC version
+      - name: Process Stack Version
         id: remove-commit-hash
         run: |
           # Extract the stack version
           stack_version="${{ inputs.elk-stack-version }}"
 
-          # Check if the version contains a commit hash, remove it
+          echo "TF_VAR_stack_version=$stack_version" >> $GITHUB_ENV
+          echo "STACK_VERSION=$stack_version" >> $GITHUB_ENV
+
+          # Handle BC versions with commit hash (e.g. 8.11.0-1234567890)
           if [[ $stack_version =~ -[a-f0-9]+ ]]; then
             cleaned_version=$(echo $stack_version | awk -F"-" '{print $1}')
+
+            # Versions with commit hash are not allowed for EC regular deployments and should be modified
+            # EC module resource:
+            # ec_deployment.deployment.version is required attribute and should be in format 8.x.y | 8.x.y-SNAPSHOT
+            # Therefore, we need to modify the version in the env variable
+            echo "TF_VAR_stack_version=$cleaned_version" >> $GITHUB_ENV
+
+            # env variable STACK_VERSION is used in sanity tests for findings validation
+            # findings are saved with version without commit hash
+            # therefore, we need to modify the version in the env variable
             echo "STACK_VERSION=$cleaned_version" >> $GITHUB_ENV
+
+            # TF_VAR_pin_version is used to override stack docker images
+            # for BC versions with commit hash
+            # This version will be used to override the docker images
+            # elasticsearch.config.docker_image
+            # kibana.config.docker_image
+            # integrations_server.config.docker_image
+            echo "TF_VAR_pin_version=$stack_version" >> $GITHUB_ENV
           fi
 
       - name: Init Enrollment Token
diff --git a/deploy/cloud/modules/ec/main.tf b/deploy/cloud/modules/ec/main.tf
index 79ff0a3cc4..47359622cc 100644
--- a/deploy/cloud/modules/ec/main.tf
+++ b/deploy/cloud/modules/ec/main.tf
@@ -26,9 +26,9 @@ resource "ec_deployment" "deployment" {
   elasticsearch = {
     autoscale = var.elasticsearch_autoscale
     strategy  = "rolling_all"
-    config = {
-      docker_image = local.es_docker_image_tag_override != "" ? "${local.es_docker_image}:${local.es_docker_image_tag_override}" : null
-    }
+    config = local.es_docker_image_tag_override != "" ? {
+      docker_image = "${local.es_docker_image}:${local.es_docker_image_tag_override}"
+    } : null
 
     cold = {
       autoscaling = {}
@@ -53,15 +53,15 @@ resource "ec_deployment" "deployment" {
   }
 
   kibana = {
-    config = {
-      docker_image = local.kibana_docker_image_tag_override != "" ? "${local.kibana_docker_image}:${local.kibana_docker_image_tag_override}" : null
-    }
+    config = local.kibana_docker_image_tag_override != "" ? {
+      docker_image = "${local.kibana_docker_image}:${local.kibana_docker_image_tag_override}"
+    } : null
   }
 
   integrations_server = {
-    config = {
-      docker_image = local.apm_docker_image_tag_override != "" ? "${local.apm_docker_image}:${local.apm_docker_image_tag_override}" : null
-    }
+    config = local.apm_docker_image_tag_override != "" ? {
+      docker_image = "${local.apm_docker_image}:${local.apm_docker_image_tag_override}"
+    } : null
   }
 
 }
\ No newline at end of file
diff --git a/deploy/test-environments/README.md b/deploy/test-environments/README.md
index 24bde89f23..43b1ddda1f 100644
--- a/deploy/test-environments/README.md
+++ b/deploy/test-environments/README.md
@@ -63,7 +63,8 @@ Please note that the customized image is currently available in the following re
 |:-------------:|:-------------:|:------------|
 | ec_api_key    |   None   | The API key for Elastic Cloud can also be defined using the `TF_VAR_ec_api_key` environment variable |
 | ess_region    | gcp-us-west2 | The ESS deployment region can also be defined using the `TF_VAR_stack_version` environment variable|
-| stack_version | latest | The ELK stack version can alsob be defined using the `TF_VAR_stack_version` environment variable |
+| stack_version | latest | The ELK stack version can also be defined using the `TF_VAR_stack_version` environment variable |
+| pin_version   | None | Optional: The ELK pin version (docker tag override) can also be defined using the `TF_VAR_pin_version` environment variable |
 
 ## Execution
 
@@ -104,6 +105,13 @@ terraform apply --auto-approve -target "module.aws_ec2_for_kspm"
 terraform apply --auto-approve -target "module.ec_deployment"
 ```
 
+BC version
+
+```bash
+terraform apply --auto-approve -var="stack_version=8.12.0" -var="pin_version=8.12.0-9f05a310" -target "module.ec_deployment"
+```
+
+
 - EKS Deployment
 
 ```bash
diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf
index e3693741d4..1d72c857b9 100644
--- a/deploy/test-environments/main.tf
+++ b/deploy/test-environments/main.tf
@@ -15,7 +15,6 @@ locals {
     Content-type  = "application/json"
     Authorization = "ApiKey ${var.ec_api_key}"
   }
-  cleaned_version = length(regexall("(-[0-9a-z]{4})", var.stack_version)) > 0 ? split("-", var.stack_version)[0] : var.stack_version
 }
 
 # EC2 + kind deployment
@@ -61,7 +60,7 @@ module "ec_deployment" {
   source        = "../cloud/modules/ec"
   ec_api_key    = var.ec_api_key
   region        = var.ess_region
-  stack_version = local.cleaned_version
+  stack_version = var.stack_version
   tags          = local.common_tags
 
   deployment_template    = var.deployment_template
@@ -71,11 +70,14 @@ module "ec_deployment" {
   elasticsearch_size       = var.elasticsearch_size
   elasticsearch_zone_count = var.elasticsearch_zone_count
 
-  docker_image = var.docker_image_override
-  docker_image_tag_override = {
-    "elasticsearch" = "${var.stack_version}",
-    "kibana"        = "${var.stack_version}",
-    "apm"           = "${var.stack_version}"
+  docker_image_tag_override = var.pin_version != "" ? {
+    "elasticsearch" = "${var.pin_version}",
+    "kibana"        = "${var.pin_version}",
+    "apm"           = "${var.pin_version}"
+  } : {
+    "elasticsearch" = "",
+    "kibana"        = "",
+    "apm"           = ""
   }
 }
 
diff --git a/deploy/test-environments/variables.tf b/deploy/test-environments/variables.tf
index 3c16153d45..43e8a199e8 100644
--- a/deploy/test-environments/variables.tf
+++ b/deploy/test-environments/variables.tf
@@ -35,6 +35,12 @@ variable "stack_version" {
   type        = string
 }
 
+variable "pin_version" {
+  default     = ""
+  description = "Optional pinned stack version for BC reasons"
+  type        = string
+}
+
 variable "serverless_mode" {
   default     = false
   description = "Set to true to create a serverless security project instead of an ESS deployment"
@@ -74,15 +80,6 @@ variable "docker_image_tag_override" {
   type        = map(string)
 }
 
-variable "docker_image_override" {
-  default = {
-    "elasticsearch" = "docker.elastic.co/cloud-release/elasticsearch-cloud-ess",
-    "kibana"        = "docker.elastic.co/cloud-release/kibana-cloud",
-    "apm"           = "docker.elastic.co/cloud-release/elastic-agent-cloud",
-  }
-  type = map(string)
-}
-
 variable "division" {
   default     = "engineering"
   type        = string

From 09f08a0acf0bd0d82f5bb98ffbf73e7d75d58c19 Mon Sep 17 00:00:00 2001
From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com>
Date: Tue, 9 Jan 2024 16:29:55 +0200
Subject: [PATCH 15/15] update format

---
 deploy/test-environments/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf
index 1d72c857b9..96c005dd94 100644
--- a/deploy/test-environments/main.tf
+++ b/deploy/test-environments/main.tf
@@ -74,7 +74,7 @@ module "ec_deployment" {
     "elasticsearch" = "${var.pin_version}",
     "kibana"        = "${var.pin_version}",
     "apm"           = "${var.pin_version}"
-  } : {
+    } : {
     "elasticsearch" = "",
     "kibana"        = "",
     "apm"           = ""