diff --git a/.gitignore b/.gitignore
index e37b30a13..d72e57b25 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,7 +15,7 @@ hmac
 # 3rd party packages
 kubernetes-prometheus
 
-# docker area 
+# docker area
 docker/sqoop/awg-ETL-crons/
 
 # skip configs/secrets
@@ -81,3 +81,5 @@ kubernetes/rucio/futureprod-db.yaml
 
 
 kubernetes/rucio/dev_grafana_password.txt
+
+!helm/crabserver/config
diff --git a/helm/crabserver/Chart.yaml b/helm/crabserver/Chart.yaml
index 507ceccdc..c608764b3 100644
--- a/helm/crabserver/Chart.yaml
+++ b/helm/crabserver/Chart.yaml
@@ -15,9 +15,9 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.11
+version: 0.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: "v3.240731"
+appVersion: "v3.240809"
diff --git a/helm/crabserver/README.md b/helm/crabserver/README.md
index 805314299..73a31e11d 100644
--- a/helm/crabserver/README.md
+++ b/helm/crabserver/README.md
@@ -1,19 +1,63 @@
-## Deploy CRAB
+# Deploy CRAB
 
-At the moment CRAB team is transitioning to using helm. However, we do noy like
-to use helm for deployment. See this 
-[comment](https://github.com/dmwm/CRABServer/issues/7843#issuecomment-2025085120)
-for some context.
+In this helm chart, we separate k8s resources into 3 resource groups:
 
-We like to use helm for templating, using the helm charts to generate the 
-manifest file, then applying it with `kubectl apply`.
+1. `crabserver` manifest for CRAB REST, including `crabserver-canary`.
+2. `logPipline`, the log pipeline which have logstash, filebeat and it config.
+3. `ciServiceAccount` with deployment permission, for deploying new image from CI. Note that only cluster admin could not `rolebinding` in testbed and production cluster.
 
-We therefore ask cmsweb operators to deploy crab following the procedure at
-[1].
+CRAB team will only handle the first resources group, `crabserver`, to deploy a new service version or modify how we start our service. The rest we kindly ask cluster admin to take care of.
 
-Changes to `../../kubernetes/cmsweb/services/crabserver.yaml` should not be committed.
+## Configuration (helm's value)
 
----
+There are helm value, `enabled`, that is used to control which resource group will be generated.
 
-[1] https://cmscrab.docs.cern.ch/technical/crab-rest/deploy.html#deploy-on-kubernetes-use-helm-template-to-generate-manifest-preferred
+#### enabled
 
+Default: `true`
+
+Generate `crabserver` resources group.
+
+#### canary.enabled
+
+Default: `false`
+
+Enable canary deployment, a.k.a. the `crabserver-canary` Deployment manifest.
+
+#### logPipeline.enabled
+
+Default: `false`
+
+Generate `logPipeline` resources group.
+
+#### ciServiceAccount.enabled
+
+Generate `ciServiceAccount` resources group. Need cluster admin for applying `role` and `rolebinding` of k8s service account.
+
+## Deploy
+
+### For cluster admins
+
+To deploy everything on new testbed and production cluster:
+
+1. Deploy `crabserver-secrets` credential.
+
+2. Deploy all services:
+    ```
+    helm install crabserver . -f values.yaml -f values-clusteradmin.yaml --set environment=preprod
+    ```
+    Please change `environment=preprod` to the cluster you are deploying (`preprod` for testbed and preprod cluster, `prod` for production cluster).
+
+Please leave test cluster to CRAB operators.
+
+### For CRAB operators
+
+Please consult [Deploying CRAB REST](https://cmscrab.docs.cern.ch/crab-components/crab-rest/deploy.html).
+
+## Regarding deploy with `helm install` command
+
+CRAB team does not use helm for deployment. We like to use helm for templating, using the helm charts to generate the manifest file, then applying it with `kubectl apply`. See this [comment](https://github.com/dmwm/CRABServer/issues/7843#issuecomment-2025085120) for some context.
+
+However, it should not have any conflict if cluster admins run `helm install` **before** CRAB operators apply new manifests with `kubectl apply`.
+
+In case of conflict, feel free to purge all resources and reinstall with helm again.
diff --git a/helm/crabserver/apply_logstash.sh b/helm/crabserver/apply_logstash.sh
new file mode 100644
index 000000000..aa6ec8b22
--- /dev/null
+++ b/helm/crabserver/apply_logstash.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -x
+set -euo pipefail
+ENV=test
+kubectl create configmap logstash-crab --from-file=config/${ENV}/logstash/logstash.conf --from-file config/${ENV}/logstash/logstash.yml --dry-run=client -oyaml | kubectl apply -f -
+kubectl delete pod $(kubectl get pod --no-headers -o custom-columns=":metadata.name" | grep logstash)
diff --git a/helm/crabserver/config/preprod/config.py b/helm/crabserver/config/preprod/config.py
new file mode 100644
index 000000000..6d717a0ae
--- /dev/null
+++ b/helm/crabserver/config/preprod/config.py
@@ -0,0 +1,49 @@
+from WMCore.Configuration import Configuration
+import re
+import socket
+import time
+import os
+
+myhost = socket.getfqdn().lower()
+
+conf = Configuration()
+main = conf.section_('main')
+srv = main.section_('server')
+srv.thread_pool = int(os.getenv('CRABSERVER_THREAD_POOL', 15))
+main.application = 'crabserver'
+main.port = 8270
+main.index = 'data'
+
+main.authz_defaults = { 'role': None, 'group': None, 'site': None }
+main.section_('tools').section_('cms_auth').key_file = "%s/auth/crabserver/header-auth-key" % __file__.rsplit('/', 3)[0]
+
+app = conf.section_('crabserver')
+app.admin = 'cms-service-webtools@cern.ch'
+app.description = 'CRABServer RESTFull API'
+app.title = 'CRABRESTFull'
+
+views = conf.section_('views')
+ui = views.section_('ui')
+ui.object = 'CRABInterface.Pages.FrontPage.FrontPage'
+
+data = views.section_('data')
+data.object = 'CRABInterface.RESTBaseAPI.RESTBaseAPI'
+data.phedexurl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod/'
+data.dbsurl = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
+data.defaultBlacklist = ['T0_CH_CERN']
+data.serverhostcert = "%s/auth/crabserver/dmwm-service-cert.pem" % __file__.rsplit('/', 3)[0]
+data.serverhostkey = "%s/auth/crabserver/dmwm-service-key.pem" % __file__.rsplit('/', 3)[0]
+data.credpath = '%s/state/crabserver/proxy/' % __file__.rsplit('/', 4)[0]
+data.backend = 'oracle'
+data.db = 'CRABServerAuth.dbconfig'
+data.s3 = 'CRABServerAuth.s3'
+data.workflowManager = 'HTCondorDataWorkflow'
+
+data.extconfigurl = 'http://gitlab.cern.ch/crab3/CRAB3ServerConfig/raw/master/cmsweb-rest-config.json'
+
+data.loggingLevel = 10
+data.loggingFile = '%s/logs/crabserver/CRAB-%s.log' % (__file__.rsplit('/', 4)[0], myhost)
+data.keptLogDays = 7
+data.mode = "cmsweb-preprod"
+
+data.delegateDN = "/DC=ch/DC=cern/OU=computers/CN=crab-(preprod|prod)-tw(01|02).cern.ch|/DC=ch/DC=cern/OU=computers/CN=crab-dev-tw(01|02|03|04).cern.ch|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cmscrab/CN=(817881|373708)/CN=Robot: cms crab|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=crabint1/CN=373708/CN=Robot: CMS CRAB Integration 1"
diff --git a/kubernetes/cmsweb/monitoring/crab/logstash.conf b/helm/crabserver/config/preprod/logstash/logstash.conf
similarity index 99%
rename from kubernetes/cmsweb/monitoring/crab/logstash.conf
rename to helm/crabserver/config/preprod/logstash/logstash.conf
index b191f9ccd..e47fb7852 100644
--- a/kubernetes/cmsweb/monitoring/crab/logstash.conf
+++ b/helm/crabserver/config/preprod/logstash/logstash.conf
@@ -176,8 +176,8 @@ filter {
       }
 
       grok {
-          # [07/Dec/2022:20:06:47] crabserver-56c69685b6-477jr 188.184.91.103:41924 "GET /crabserver/devthree/info HTTP/1.1" 200 OK [data: 9354 in 103 out 4870 us ] [auth: ok "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=tseethon/CN=856006/CN=Thanayut Seethongchuen" "" ] [ref: "https://cmsweb-test12.cern.ch" "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" ] Type=cherrypylog
-+         match => { "message" => '\[%{NOTSPACE:timestamp_temp}\] %{DATA:backend} %{IPORHOST:clientip}(|:%{NUMBER}) "%{WORD:method} %{NOTSPACE:request} %{DATA:httpversion}" %{NUMBER:code:int} %{DATA} \[data: (-|%{NUMBER:bytes_sent:int}) in (-|%{NUMBER:bytes_received:int}) out (-|%{NUMBER:time_spent_ms:int}) us \] \[auth: %{DATA} "%{DATA:dn}".*\] \[ref: "%{DATA}.*" "%{DATA:client}" \]' }
+          # [07/Dec/2022:20:06:47] crabserver-56c69685b6-477jr 188.184.91.103:41924 "GET /crabserver/devthree/info HTTP/1.1" 200 OK [data: 9354 in 103 out 4870 us ] [auth: ok "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=tseethon/CN=856006/CN=Thanayut Seethongchuen" "" ] [ref: "https://cmsweb-test12.cern.ch" "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" ] - Podname=crabserver-6cf49bcfff-bxvb6 Type=cherrypylog
+          match => { "message" => '\[%{NOTSPACE:timestamp_temp}\] %{DATA:backend} %{IPORHOST:clientip}(|:%{NUMBER}) "%{WORD:method} %{NOTSPACE:request} %{DATA:httpversion}" %{NUMBER:code:int} %{DATA} \[data: (-|%{NUMBER:bytes_sent:int}) in (-|%{NUMBER:bytes_received:int}) out (-|%{NUMBER:time_spent_ms:int}) us \] \[auth: %{DATA} "%{DATA:dn}".*\] \[ref: "%{DATA}.*" "%{DATA:client}" \]' }
       }
       grok {
         match => { "request" => '/%{WORD:system}%{UNIXPATH:uri_path}%{URIPARAM:uri_params}?' }
diff --git a/kubernetes/cmsweb/monitoring/crab/logstash.yml b/helm/crabserver/config/preprod/logstash/logstash.yml
similarity index 100%
rename from kubernetes/cmsweb/monitoring/crab/logstash.yml
rename to helm/crabserver/config/preprod/logstash/logstash.yml
diff --git a/helm/crabserver/config/prod/config.py b/helm/crabserver/config/prod/config.py
new file mode 100644
index 000000000..215f66ce0
--- /dev/null
+++ b/helm/crabserver/config/prod/config.py
@@ -0,0 +1,47 @@
+from WMCore.Configuration import Configuration
+import re
+import socket
+import time
+import os
+
+myhost = socket.getfqdn().lower()
+
+conf = Configuration()
+main = conf.section_('main')
+srv = main.section_('server')
+srv.thread_pool = int(os.getenv('CRABSERVER_THREAD_POOL', 15))
+main.application = 'crabserver'
+main.port = 8270
+main.index = 'data'
+
+main.authz_defaults = { 'role': None, 'group': None, 'site': None }
+main.section_('tools').section_('cms_auth').key_file = "%s/auth/crabserver/header-auth-key" % __file__.rsplit('/', 3)[0]
+
+app = conf.section_('crabserver')
+app.admin = 'cms-service-webtools@cern.ch'
+app.description = 'CRABServer RESTFull API'
+app.title = 'CRABRESTFull'
+
+views = conf.section_('views')
+ui = views.section_('ui')
+ui.object = 'CRABInterface.Pages.FrontPage.FrontPage'
+
+data = views.section_('data')
+data.object = 'CRABInterface.RESTBaseAPI.RESTBaseAPI'
+data.phedexurl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod/'
+data.dbsurl = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
+data.defaultBlacklist = ['T0_CH_CERN']
+data.serverhostcert = "%s/auth/crabserver/dmwm-service-cert.pem" % __file__.rsplit('/', 3)[0]
+data.serverhostkey = "%s/auth/crabserver/dmwm-service-key.pem" % __file__.rsplit('/', 3)[0]
+data.credpath = '%s/state/crabserver/proxy/' % __file__.rsplit('/', 4)[0]
+data.backend = 'oracle'
+data.db = 'CRABServerAuth.dbconfig'
+data.s3 = 'CRABServerAuth.s3'
+data.workflowManager = 'HTCondorDataWorkflow'
+
+data.extconfigurl = 'http://gitlab.cern.ch/crab3/CRAB3ServerConfig/raw/master/cmsweb-rest-config.json'
+
+data.loggingLevel = 10
+data.loggingFile = '%s/logs/crabserver/CRAB-%s.log' % (__file__.rsplit('/', 4)[0], myhost)
+data.keptLogDays = 7
+data.mode = "cmsweb-prod"
diff --git a/helm/crabserver/config/prod/logstash/logstash.conf b/helm/crabserver/config/prod/logstash/logstash.conf
new file mode 100644
index 000000000..e47fb7852
--- /dev/null
+++ b/helm/crabserver/config/prod/logstash/logstash.conf
@@ -0,0 +1,272 @@
+# Migrated from: https://github.com/dmwm/CRABServer/blob/master/src/script/Monitor/logstash/crabtaskworker.conf
+
+input { beats { port => 5044 } }
+
+filter {
+  # whitelisted tags
+  if "crabtaskworker" not in [tags] and "crabpublisher" not in [tags] and "crabhttpcall" not in [tags] and "crabrest" not in [tags] { drop { } }
+
+  mutate {
+    # ---- Common fields ---- [metadata]
+    add_field => {
+      "cmsweb_cluster" => "${CMSWEB_CLUSTER:NA}"
+      "cmsweb_env" => "${CMSWEB_ENV:NA}"
+      "filebeat_id" => "%{[agent][id]}"
+      "filebeat_name" => "%{[agent][name]}"
+      "filebeat_version" => "%{[agent][version]}"
+      "hostname" => "%{[host][name]}"
+      "log_file" => "%{[log][file][path]}"
+    }
+  }
+
+  if "crabtaskworker" in [tags] {
+      # ------------------------------------------------------------------------------------------------------------------------------------------------------------------
+      # TASKWORKER
+      mutate {
+        # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers!
+        replace => {
+          "producer" => "crab"
+          "type" => "crabtaskworker"
+        }
+      }
+      if "completed in" in [message] {
+        grok {
+          match => {
+            "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:DEBUG:Worker,%{INT}:Process-%{NOTSPACE:slaveID}: %{NOTSPACE:workType} work on %{NOTSPACE:taskName} completed in %{NUMBER:completionTime:int} seconds"
+          }
+          add_field => {"log_type" => "work_on_task_completed"}
+          overwrite => ["message"]
+        }
+
+      } else if ( [message] =~ /.*Starting.*at.*on.*/ ) {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:DEBUG:Worker,%{INT}:Process-%{NOTSPACE:slaveID}: Starting <function %{NOTSPACE:functionName} at %{DATA:taskID}> on %{NOTSPACE:taskName}"
+            }
+            add_field => {"log_type" => "start_new_task"}
+            overwrite => ["message"]
+          }
+      } else if "TWSTART" in [message] {
+          grok {
+              match => {
+                "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:MasterWorker,%{INT}:TWSTART: %{GREEDYDATA:tw_json_data}"
+              }
+              add_field => {"log_type" => "tw_config_data"}
+              overwrite => ["message"]
+          }
+          if [tw_json_data] {
+            json {
+              source => "tw_json_data"
+              target => "tw_json_data"
+            }
+          }
+
+      } else if ( [message] =~ /.*Finished.*object at .*/ ) {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:INFO:Handler,%{INT}:Finished <TaskWorker.Actions.%{NOTSPACE:action} object at %{DATA:taskID}> on %{NOTSPACE:taskName} in %{NUMBER:completionTime:int} seconds"
+            }
+            add_field => {"log_type" => "action_on_task_finished"}
+            overwrite => ["message"]
+          }
+      } else { drop {} }
+
+      date {
+        # Since date string does not provide timezone, we need to specify it explicitly
+        match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"]
+        timezone => "Europe/Zurich"
+        target => "rec_timestamp_str"
+      }
+      # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+  } else if "crabpublisher" in [tags] {
+      # ------------------------------------------------------------------------------------------------------------------------------------------------------------------
+      # CRABPUBLISHER
+      mutate {
+        # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers!
+        replace => {
+          "producer" => "crab"
+          "type" => "crabpublisher"
+        }
+      }
+
+      if "PUBSTART" in [message] {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:PUBSTART:%{GREEDYDATA:pb_json_data}"
+            }
+            add_field => {"log_type" => "publisher_config_data"}
+            overwrite => ["message"]
+          }
+          if [pb_json_data] {
+            json{
+              source => "pb_json_data"
+              target => "pb_json_data"
+            }
+          }
+
+      } else if ( [message] =~ /.*blocks failed.*files.*/ ) {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} : %{INT:blocks} blocks failed for a total of %{INT:files} files"
+            }
+            add_field => {"log_type" => "failed_publication"}
+            overwrite => ["message"]
+          }
+
+      } else if ( [message] =~ /.*Published.*files in.*/ ) {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} is %{NOTSPACE:publicationResult}. Published %{INT:filesPublished} files in %{INT:blocks} blocks."
+            }
+            add_field => {"log_type" => "successful_publication"}
+            overwrite => ["message"]
+          }
+
+      } else if ( [message] =~ /.*Exception.*TaskPublish.*/ ) {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:%{GREEDYDATA:exceptionHandled}"
+            }
+            add_field => {"log_type" => "publication_error"}
+            overwrite => ["message"]
+          }
+
+      # This filter is currently used for production publisher. After we deploy preprod to prod, this filter can be dropped
+      } else if "DEBUG:master" in [message] {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:master:%{SPACE}%{INT:acquiredFiles}%{SPACE}:%{SPACE}%{NOTSPACE:taskName}"
+            }
+            add_field => {"log_type" => "acquired_files"}
+            overwrite => ["message"]
+          }
+
+      # this filter mathces changes in #6861, which is already in preprod and dev
+      } else if "acquired_files" in [message] {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:acquired_files:%{SPACE}%{NOTSPACE:acquiredFilesStatus}%{SPACE}%{INT:acquiredFiles}%{SPACE}:%{SPACE}%{NOTSPACE:taskName}"
+            }
+            add_field => {"log_type" => "acquired_files_status"}
+            overwrite => ["message"]
+          }
+
+      } else { drop { } }
+      date {
+        # Since date string does not provide timezone, we need to specify it explicitly
+        match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"]
+        timezone => "Europe/Zurich"
+        target => "rec_timestamp_str"
+      }
+      # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+  } else if "crabhttpcall" in [tags] {
+      # ------------------------------------------------------------------------------------------------------------------------------------------------------------------
+      # CRABHTTPCALL (previously crabserver)
+      mutate {
+        # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers!
+        replace => {
+          "producer" => "crab"
+          "type" => "crabhttpcall"
+        }
+      }
+      # parse log path to get pod name
+      # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log
+      grok {
+          match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' }
+      }
+
+      grok {
+          # [07/Dec/2022:20:06:47] crabserver-56c69685b6-477jr 188.184.91.103:41924 "GET /crabserver/devthree/info HTTP/1.1" 200 OK [data: 9354 in 103 out 4870 us ] [auth: ok "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=tseethon/CN=856006/CN=Thanayut Seethongchuen" "" ] [ref: "https://cmsweb-test12.cern.ch" "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" ] - Podname=crabserver-6cf49bcfff-bxvb6 Type=cherrypylog
+          match => { "message" => '\[%{NOTSPACE:timestamp_temp}\] %{DATA:backend} %{IPORHOST:clientip}(|:%{NUMBER}) "%{WORD:method} %{NOTSPACE:request} %{DATA:httpversion}" %{NUMBER:code:int} %{DATA} \[data: (-|%{NUMBER:bytes_sent:int}) in (-|%{NUMBER:bytes_received:int}) out (-|%{NUMBER:time_spent_ms:int}) us \] \[auth: %{DATA} "%{DATA:dn}".*\] \[ref: "%{DATA}.*" "%{DATA:client}" \]' }
+      }
+      grok {
+        match => { "request" => '/%{WORD:system}%{UNIXPATH:uri_path}%{URIPARAM:uri_params}?' }
+      }
+      if ![uri_params] {
+          ruby { code => "event.set('uri_params','')" }
+      }
+      grok { match => { "uri_path" => '/.*/%{DATA:api}$' } }
+      if [client] {
+          grok { match => { "client" => '%{DATA:client_name}/%{DATA:client_version}$' } }
+      }
+      # !ATTENTION! Different date-time format than publisher and TW, means that special for crabserver.
+      date {
+         # Since date string does not provide timezone, we need to specify it explicitly
+         match => [ "timestamp_temp", "dd/MMM/yyyy:HH:mm:ss" ]
+         timezone => "Europe/Zurich"
+         target => "rec_timestamp_str"
+      }
+      mutate { gsub =>  [ "dn","/CN=\d+","" ] }
+      # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+  } else if "crabrest" in [tags] {
+      # ------------------------------------------------------------------------------------------------------------------------------------------------------------------
+      # CRABLOG
+      mutate {
+        # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers!
+        replace => {
+          "producer" => "crab"
+          "type" => "crabrest"
+        }
+      }
+      # parse log path to get pod name
+      # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log
+      grok {
+          match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' }
+      }
+
+      if ( [message] =~ /.*MeasureTime:seconds.*/ ) {
+          grok {
+            match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureTime:seconds - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' tot=%{NUMBER:tot:float} proc=%{NUMBER:proc:float} thread=%{NUMBER:thread:float}" }
+            add_field => {"log_type" => "measure_time"}
+          }
+      } else if ( [message] =~ /.*MeasureSize:bytes.*/ ) {
+          grok {
+            match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureSize:bytes - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' obj_size=%{INT:obj_size:int} get_size_time=%{NUMBER:get_size_time:float}" }
+            add_field => {"log_type" => "measure_size"}
+          }
+      }
+      date {
+        # Since date string does not provide timezone, we need to specify it explicitly
+        match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"]
+        timezone => "Europe/Zurich"
+        target => "rec_timestamp_str"
+      }
+      # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+  }
+
+  # ---- Common modifications ----
+  if "_grokparsefailure" in [tags] { drop { } }
+
+  # Date operations
+  ruby {
+    # "metadata.timestamp"(date in ES) equals to record_time(int in ES) which we need for latency calculation
+    # "producer_time" is filebeat process time stamp.
+    code => "event.set('timestamp', (event.get('rec_timestamp_str').to_f * 1000).to_i)
+           event.set('record_time', (event.get('rec_timestamp_str').to_f * 1000).to_i)
+           event.set('producer_time', (event.get('@timestamp').to_f * 1000).to_i)
+           "
+  }
+
+  # remove quotes from message entry since it will break the JSON
+  mutate { gsub => [ "message", "\n", "", "message", "\"", ""] }
+
+  # remove undesired fields and objects
+  mutate { remove_field => ["@timestamp", "@version", "timestamp_temp", "agent", "log", "input", "tags", "ecs", "host", "cloud", "event" ] }
+
+}
+
+output {
+  if [type] in ["crabtaskworker", "crabpublisher", "crabhttpcall", "crabrest"] {
+    http {
+      http_method => "post"
+      url => "http://monit-logs.cern.ch:10012/"
+      content_type => "application/json; charset=UTF-8"
+      format => "json_batch"
+      socket_timeout => 60
+      connect_timeout => 60
+    }
+  } else {
+    # For debugging, please keep it to catch problems
+    stdout { codec => json }
+  }
+}
diff --git a/helm/crabserver/config/prod/logstash/logstash.yml b/helm/crabserver/config/prod/logstash/logstash.yml
new file mode 100644
index 000000000..c826af0ea
--- /dev/null
+++ b/helm/crabserver/config/prod/logstash/logstash.yml
@@ -0,0 +1,7 @@
+http.host: "0.0.0.0"
+path.config: /usr/share/logstash/pipeline
+# disable connection to ES
+xpack.monitoring.enabled: false
+pipeline.workers: 16
+pipeline.batch.size: 256
+pipeline.batch.delay: 200
diff --git a/helm/crabserver/config/test/config.py b/helm/crabserver/config/test/config.py
new file mode 100644
index 000000000..3c3efb04f
--- /dev/null
+++ b/helm/crabserver/config/test/config.py
@@ -0,0 +1,51 @@
+from WMCore.Configuration import Configuration
+import re
+import socket
+import time
+import os
+
+myhost = socket.getfqdn().lower()
+
+conf = Configuration()
+main = conf.section_('main')
+srv = main.section_('server')
+srv.thread_pool = int(os.getenv('CRABSERVER_THREAD_POOL', 25))
+main.application = 'crabserver'
+main.port = 8270
+main.index = 'data'
+
+main.authz_defaults = { 'role': None, 'group': None, 'site': None }
+main.section_('tools').section_('cms_auth').key_file = "%s/auth/crabserver/header-auth-key" % __file__.rsplit('/', 3)[0]
+
+app = conf.section_('crabserver')
+app.admin = 'cms-service-webtools@cern.ch'
+app.description = 'CRABServer RESTFull API'
+app.title = 'CRABRESTFull'
+
+views = conf.section_('views')
+ui = views.section_('ui')
+ui.object = 'CRABInterface.Pages.FrontPage.FrontPage'
+
+data = views.section_('data')
+data.object = 'CRABInterface.RESTBaseAPI.RESTBaseAPI'
+data.phedexurl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod/'
+data.dbsurl = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
+data.defaultBlacklist = ['T0_CH_CERN']
+data.serverhostcert = "%s/auth/crabserver/dmwm-service-cert.pem" % __file__.rsplit('/', 3)[0]
+data.serverhostkey = "%s/auth/crabserver/dmwm-service-key.pem" % __file__.rsplit('/', 3)[0]
+data.credpath = '%s/state/crabserver/proxy/' % __file__.rsplit('/', 4)[0]
+data.backend = 'oracle'
+data.db = 'CRABServerAuth.dbconfig'
+data.s3 = 'CRABServerAuth.s3'
+data.workflowManager = 'HTCondorDataWorkflow'
+
+data.extconfigurl = 'http://gitlab.cern.ch/crab3/CRAB3ServerConfig/raw/master/cmsweb-rest-config.json'
+
+data.loggingLevel = 10
+data.loggingFile = '%s/logs/crabserver/CRAB-%s.log' % (__file__.rsplit('/', 4)[0], myhost)
+data.keptLogDays = 7
+data.mode = "cmsweb-test"
+
+data.enableQueryLoadAllRows = os.getenv('CRABSERVER_ENABLE_QUERY_LOAD_ALL_ROWS', 'True').lower() in ('true', '1', 't')
+
+data.delegateDN = "/DC=ch/DC=cern/OU=computers/CN=crab-(preprod|prod)-tw(01|02).cern.ch|/DC=ch/DC=cern/OU=computers/CN=crab-dev-tw(01|02|03|04).cern.ch|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cmscrab/CN=(817881|373708)/CN=Robot: cms crab|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=crabint1/CN=373708/CN=Robot: CMS CRAB Integration 1"
diff --git a/helm/crabserver/config/test/logstash/logstash.conf b/helm/crabserver/config/test/logstash/logstash.conf
new file mode 100644
index 000000000..e47fb7852
--- /dev/null
+++ b/helm/crabserver/config/test/logstash/logstash.conf
@@ -0,0 +1,272 @@
+# Migrated from: https://github.com/dmwm/CRABServer/blob/master/src/script/Monitor/logstash/crabtaskworker.conf
+
+input { beats { port => 5044 } }
+
+filter {
+  # whitelisted tags
+  if "crabtaskworker" not in [tags] and "crabpublisher" not in [tags] and "crabhttpcall" not in [tags] and "crabrest" not in [tags] { drop { } }
+
+  mutate {
+    # ---- Common fields ---- [metadata]
+    add_field => {
+      "cmsweb_cluster" => "${CMSWEB_CLUSTER:NA}"
+      "cmsweb_env" => "${CMSWEB_ENV:NA}"
+      "filebeat_id" => "%{[agent][id]}"
+      "filebeat_name" => "%{[agent][name]}"
+      "filebeat_version" => "%{[agent][version]}"
+      "hostname" => "%{[host][name]}"
+      "log_file" => "%{[log][file][path]}"
+    }
+  }
+
+  if "crabtaskworker" in [tags] {
+      # ------------------------------------------------------------------------------------------------------------------------------------------------------------------
+      # TASKWORKER
+      mutate {
+        # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers!
+        replace => {
+          "producer" => "crab"
+          "type" => "crabtaskworker"
+        }
+      }
+      if "completed in" in [message] {
+        grok {
+          match => {
+            "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:DEBUG:Worker,%{INT}:Process-%{NOTSPACE:slaveID}: %{NOTSPACE:workType} work on %{NOTSPACE:taskName} completed in %{NUMBER:completionTime:int} seconds"
+          }
+          add_field => {"log_type" => "work_on_task_completed"}
+          overwrite => ["message"]
+        }
+
+      } else if ( [message] =~ /.*Starting.*at.*on.*/ ) {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:DEBUG:Worker,%{INT}:Process-%{NOTSPACE:slaveID}: Starting <function %{NOTSPACE:functionName} at %{DATA:taskID}> on %{NOTSPACE:taskName}"
+            }
+            add_field => {"log_type" => "start_new_task"}
+            overwrite => ["message"]
+          }
+      } else if "TWSTART" in [message] {
+          grok {
+              match => {
+                "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:MasterWorker,%{INT}:TWSTART: %{GREEDYDATA:tw_json_data}"
+              }
+              add_field => {"log_type" => "tw_config_data"}
+              overwrite => ["message"]
+          }
+          if [tw_json_data] {
+            json {
+              source => "tw_json_data"
+              target => "tw_json_data"
+            }
+          }
+
+      } else if ( [message] =~ /.*Finished.*object at .*/ ) {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:INFO:Handler,%{INT}:Finished <TaskWorker.Actions.%{NOTSPACE:action} object at %{DATA:taskID}> on %{NOTSPACE:taskName} in %{NUMBER:completionTime:int} seconds"
+            }
+            add_field => {"log_type" => "action_on_task_finished"}
+            overwrite => ["message"]
+          }
+      } else { drop {} }
+
+      date {
+        # Since date string does not provide timezone, we need to specify it explicitly
+        match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"]
+        timezone => "Europe/Zurich"
+        target => "rec_timestamp_str"
+      }
+      # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+  } else if "crabpublisher" in [tags] {
+      # ------------------------------------------------------------------------------------------------------------------------------------------------------------------
+      # CRABPUBLISHER
+      mutate {
+        # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers!
+        replace => {
+          "producer" => "crab"
+          "type" => "crabpublisher"
+        }
+      }
+
+      if "PUBSTART" in [message] {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:PUBSTART:%{GREEDYDATA:pb_json_data}"
+            }
+            add_field => {"log_type" => "publisher_config_data"}
+            overwrite => ["message"]
+          }
+          if [pb_json_data] {
+            json{
+              source => "pb_json_data"
+              target => "pb_json_data"
+            }
+          }
+
+      } else if ( [message] =~ /.*blocks failed.*files.*/ ) {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} : %{INT:blocks} blocks failed for a total of %{INT:files} files"
+            }
+            add_field => {"log_type" => "failed_publication"}
+            overwrite => ["message"]
+          }
+
+      } else if ( [message] =~ /.*Published.*files in.*/ ) {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} is %{NOTSPACE:publicationResult}. Published %{INT:filesPublished} files in %{INT:blocks} blocks."
+            }
+            add_field => {"log_type" => "successful_publication"}
+            overwrite => ["message"]
+          }
+
+      } else if ( [message] =~ /.*Exception.*TaskPublish.*/ ) {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:%{GREEDYDATA:exceptionHandled}"
+            }
+            add_field => {"log_type" => "publication_error"}
+            overwrite => ["message"]
+          }
+
+      # This filter is currently used for production publisher. After we deploy preprod to prod, this filter can be dropped
+      } else if "DEBUG:master" in [message] {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:master:%{SPACE}%{INT:acquiredFiles}%{SPACE}:%{SPACE}%{NOTSPACE:taskName}"
+            }
+            add_field => {"log_type" => "acquired_files"}
+            overwrite => ["message"]
+          }
+
+      # this filter mathces changes in #6861, which is already in preprod and dev
+      } else if "acquired_files" in [message] {
+          grok {
+            match => {
+              "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:acquired_files:%{SPACE}%{NOTSPACE:acquiredFilesStatus}%{SPACE}%{INT:acquiredFiles}%{SPACE}:%{SPACE}%{NOTSPACE:taskName}"
+            }
+            add_field => {"log_type" => "acquired_files_status"}
+            overwrite => ["message"]
+          }
+
+      } else { drop { } }
+      date {
+        # Since date string does not provide timezone, we need to specify it explicitly
+        match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"]
+        timezone => "Europe/Zurich"
+        target => "rec_timestamp_str"
+      }
+      # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+  } else if "crabhttpcall" in [tags] {
+      # ------------------------------------------------------------------------------------------------------------------------------------------------------------------
+      # CRABHTTPCALL (previously crabserver)
+      mutate {
+        # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers!
+        replace => {
+          "producer" => "crab"
+          "type" => "crabhttpcall"
+        }
+      }
+      # parse log path to get pod name
+      # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log
+      grok {
+          match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' }
+      }
+
+      grok {
+          # [07/Dec/2022:20:06:47] crabserver-56c69685b6-477jr 188.184.91.103:41924 "GET /crabserver/devthree/info HTTP/1.1" 200 OK [data: 9354 in 103 out 4870 us ] [auth: ok "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=tseethon/CN=856006/CN=Thanayut Seethongchuen" "" ] [ref: "https://cmsweb-test12.cern.ch" "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" ] - Podname=crabserver-6cf49bcfff-bxvb6 Type=cherrypylog
+          match => { "message" => '\[%{NOTSPACE:timestamp_temp}\] %{DATA:backend} %{IPORHOST:clientip}(|:%{NUMBER}) "%{WORD:method} %{NOTSPACE:request} %{DATA:httpversion}" %{NUMBER:code:int} %{DATA} \[data: (-|%{NUMBER:bytes_sent:int}) in (-|%{NUMBER:bytes_received:int}) out (-|%{NUMBER:time_spent_ms:int}) us \] \[auth: %{DATA} "%{DATA:dn}".*\] \[ref: "%{DATA}.*" "%{DATA:client}" \]' }
+      }
+      grok {
+        match => { "request" => '/%{WORD:system}%{UNIXPATH:uri_path}%{URIPARAM:uri_params}?' }
+      }
+      if ![uri_params] {
+          ruby { code => "event.set('uri_params','')" }
+      }
+      grok { match => { "uri_path" => '/.*/%{DATA:api}$' } }
+      if [client] {
+          grok { match => { "client" => '%{DATA:client_name}/%{DATA:client_version}$' } }
+      }
+      # !ATTENTION! Different date-time format than publisher and TW, means that special for crabserver.
+      date {
+         # Since date string does not provide timezone, we need to specify it explicitly
+         match => [ "timestamp_temp", "dd/MMM/yyyy:HH:mm:ss" ]
+         timezone => "Europe/Zurich"
+         target => "rec_timestamp_str"
+      }
+      mutate { gsub =>  [ "dn","/CN=\d+","" ] }
+      # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+  } else if "crabrest" in [tags] {
+      # ------------------------------------------------------------------------------------------------------------------------------------------------------------------
+      # CRABLOG
+      mutate {
+        # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers!
+        replace => {
+          "producer" => "crab"
+          "type" => "crabrest"
+        }
+      }
+      # parse log path to get pod name
+      # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log
+      grok {
+          match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' }
+      }
+
+      if ( [message] =~ /.*MeasureTime:seconds.*/ ) {
+          grok {
+            match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureTime:seconds - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' tot=%{NUMBER:tot:float} proc=%{NUMBER:proc:float} thread=%{NUMBER:thread:float}" }
+            add_field => {"log_type" => "measure_time"}
+          }
+      } else if ( [message] =~ /.*MeasureSize:bytes.*/ ) {
+          grok {
+            match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureSize:bytes - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' obj_size=%{INT:obj_size:int} get_size_time=%{NUMBER:get_size_time:float}" }
+            add_field => {"log_type" => "measure_size"}
+          }
+      }
+      date {
+        # Since date string does not provide timezone, we need to specify it explicitly
+        match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"]
+        timezone => "Europe/Zurich"
+        target => "rec_timestamp_str"
+      }
+      # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+  }
+
+  # ---- Common modifications ----
+  if "_grokparsefailure" in [tags] { drop { } }
+
+  # Date operations
+  ruby {
+    # "metadata.timestamp"(date in ES) equals to record_time(int in ES) which we need for latency calculation
+    # "producer_time" is filebeat process time stamp.
+    code => "event.set('timestamp', (event.get('rec_timestamp_str').to_f * 1000).to_i)
+           event.set('record_time', (event.get('rec_timestamp_str').to_f * 1000).to_i)
+           event.set('producer_time', (event.get('@timestamp').to_f * 1000).to_i)
+           "
+  }
+
+  # remove quotes from message entry since it will break the JSON
+  mutate { gsub => [ "message", "\n", "", "message", "\"", ""] }
+
+  # remove undesired fields and objects
+  mutate { remove_field => ["@timestamp", "@version", "timestamp_temp", "agent", "log", "input", "tags", "ecs", "host", "cloud", "event" ] }
+
+}
+
+output {
+  if [type] in ["crabtaskworker", "crabpublisher", "crabhttpcall", "crabrest"] {
+    http {
+      http_method => "post"
+      url => "http://monit-logs.cern.ch:10012/"
+      content_type => "application/json; charset=UTF-8"
+      format => "json_batch"
+      socket_timeout => 60
+      connect_timeout => 60
+    }
+  } else {
+    # For debugging, please keep it to catch problems
+    stdout { codec => json }
+  }
+}
diff --git a/helm/crabserver/config/test/logstash/logstash.yml b/helm/crabserver/config/test/logstash/logstash.yml
new file mode 100644
index 000000000..c826af0ea
--- /dev/null
+++ b/helm/crabserver/config/test/logstash/logstash.yml
@@ -0,0 +1,7 @@
+http.host: "0.0.0.0"
+path.config: /usr/share/logstash/pipeline
+# disable connection to ES
+xpack.monitoring.enabled: false
+pipeline.workers: 16
+pipeline.batch.size: 256
+pipeline.batch.delay: 200
diff --git a/helm/crabserver/deploy.sh b/helm/crabserver/deploy.sh
index 747d15c44..6d0f02945 100755
--- a/helm/crabserver/deploy.sh
+++ b/helm/crabserver/deploy.sh
@@ -1,7 +1,7 @@
 #! /bin/bash
 set -euo pipefail
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
+ENVNAME=$1
 
 # hash table of clusters nickname-name:
 declare -A cluster_map=([prod]=prod)
@@ -11,20 +11,31 @@ cluster_map[testbed]=testbed
 cluster_map[test2]=test2
 cluster_map[test11]=test11
 cluster_map[test12]=test12
+cluster_map[test14]=test14
+
+declare -A valuefile_map=([prod]=prod)
+valuefile_map[prod]=values-prod.yaml
+valuefile_map[preprod]=values-preprod.yaml
+valuefile_map[testbed]=values-testbed.yaml
+valuefile_map[test2]=values-testx.yaml
+valuefile_map[test11]=values-testx.yaml
+valuefile_map[test12]=values-testx.yaml
+valuefile_map[test14]=values-testx.yaml
+
 if [[ $# -ne 1 ]]; then
     echo "Usage: deploy.sh ENVNAME"
-    echo " ENVNAME=(prod|preprod|testbed|test2|test11|test12)"
+    echo " ENVNAME=(prod|preprod|testbed|test2|test11|test12|test14)"
     exit 1
 fi
-desired_cluster="${cluster_map[$1]}"
+desired_cluster="${cluster_map[$ENVNAME]}"
+valuefile="${valuefile_map[$ENVNAME]}"
 
 # make sure that your current context points to the desired cluster
 current_cluster=$(kubectl config view -o json | jq '.["current-context"] as $context | .["contexts"][] | select(.name | contains($context))| .context.cluster')
-
-if [[ $current_cluster =~ $desired_cluster ]]; then 
-  echo "deploying to $desired_cluster"; 
-  helm template crabserver . -f values.yaml -f values-${1}-pypi.yaml | kubectl -n crab apply -f -
-else 
-  echo "wrong cluster: your are connected to $current_cluster"; 
+set -x
+if [[ $current_cluster =~ $desired_cluster ]]; then
+  echo "deploying to $desired_cluster";
+  helm template crabserver . -f values.yaml -f "${valuefile}" | kubectl -n crab apply -f -
+else
+  echo "Error: wrong cluster. your are connected to $current_cluster";
 fi
-
diff --git a/helm/crabserver/scripts/entrypoint.sh b/helm/crabserver/scripts/entrypoint.sh
new file mode 100644
index 000000000..2c0600dc4
--- /dev/null
+++ b/helm/crabserver/scripts/entrypoint.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -x
+set -euo pipefail
+
+# copy robotcert
+sudo cp /etc/robots/robotkey.pem /data/srv/current/auth/crabserver/dmwm-service-key.pem
+sudo cp /etc/robots/robotcert.pem /data/srv/current/auth/crabserver/dmwm-service-cert.pem
+sudo chown $USER:$USER /data/srv/current/auth/crabserver/dmwm-service-key.pem
+sudo chown $USER:$USER /data/srv/current/auth/crabserver/dmwm-service-cert.pem
+
+# hmac key
+sudo cp /etc/hmac/hmac /data/srv/current/auth/crabserver/header-auth-key
+sudo chown $USER:$USER /data/srv/current/auth/crabserver/header-auth-key
+
+# config.py
+sudo cp /opt/config/config.py /data/srv/current/config/crabserver/config.py
+sudo chown $USER:$USER /data/srv/current/config/crabserver/config.py
+
+# CRABServerAuth.py
+sudo cp /etc/secrets/CRABServerAuth.py /data/srv/current/auth/crabserver/CRABServerAuth.py
+sudo chown $USER:$USER /data/srv/current/auth/crabserver/CRABServerAuth.py
+
+# export proxy cert generated by cron-proxy cronjob
+# test if file exist
+ls /etc/proxy/proxy
+export X509_USER_PROXY=/etc/proxy/proxy
+
+# grid security
+sudo cp /host/etc/grid-security/* /etc/grid-security
+echo 'INFO Files in /etc/grid-security'
+ls -lahZ /etc/grid-security
+
+exec /usr/bin/tini -- "$@"
diff --git a/helm/crabserver/templates/ci-serviceaccount-rbac.yaml b/helm/crabserver/templates/ci-serviceaccount-rbac.yaml
new file mode 100644
index 000000000..013f8eeaa
--- /dev/null
+++ b/helm/crabserver/templates/ci-serviceaccount-rbac.yaml
@@ -0,0 +1,25 @@
+{{- if .Values.ciServiceAccount.enabled -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: crab-ns-full-access
+  namespace: crab
+subjects:
+- kind: ServiceAccount
+  name: crab-gitlab-ci
+  namespace: crab
+roleRef:
+  kind: Role
+  name: crab-ns-full-access
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: crab
+  name: crab-ns-full-access
+rules:
+- apiGroups: ["*"]
+  resources: ["*"]
+  verbs: ["*"]
+{{- end }}
diff --git a/helm/crabserver/templates/ci-serviceaccount-secret.yaml b/helm/crabserver/templates/ci-serviceaccount-secret.yaml
new file mode 100644
index 000000000..fa5adccf9
--- /dev/null
+++ b/helm/crabserver/templates/ci-serviceaccount-secret.yaml
@@ -0,0 +1,11 @@
+{{- if .Values.ciServiceAccount.enabled -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  annotations:
+    kubernetes.io/service-account.name: crab-gitlab-ci
+  name: crab-gitlab-ci-secret
+  labels:
+    {{- include "crabserver.labels" . | nindent 4 }}
+type: kubernetes.io/service-account-token
+{{- end }}
\ No newline at end of file
diff --git a/helm/crabserver/templates/ci-serviceaccount.yaml b/helm/crabserver/templates/ci-serviceaccount.yaml
new file mode 100644
index 000000000..0319bc93b
--- /dev/null
+++ b/helm/crabserver/templates/ci-serviceaccount.yaml
@@ -0,0 +1,8 @@
+{{- if .Values.ciServiceAccount.enabled -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: crab-gitlab-ci
+  labels:
+    {{- include "crabserver.labels" . | nindent 4 }}
+{{- end }}
diff --git a/helm/crabserver/templates/configMap.yaml b/helm/crabserver/templates/configMap.yaml
index 102f53ea3..3ddc639d1 100644
--- a/helm/crabserver/templates/configMap.yaml
+++ b/helm/crabserver/templates/configMap.yaml
@@ -1,3 +1,5 @@
+{{- $environment := .Values.environment | default dict }}
+{{- if .Values.enabled -}}
 kind: ConfigMap
 apiVersion: v1
 metadata:
@@ -6,48 +8,15 @@ metadata:
     app: crabserver
   namespace: crab
 data:
-  setup-certs-and-run.sh: |
-    #!/bin/bash
-    sudo cp /host/etc/grid-security/* /etc/grid-security
-    echo 'INFO Files in /etc/grid-security'
-    ls -lahZ /etc/grid-security
-    #su  -c "cd /data && /data/run.sh" --preserve-environment  _crabserver
-    if command -v tini &> /dev/null; then
-      /data/entrypoint.sh /data/run.sh
-    else
-      cd /data && /data/run.sh
-    fi
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  labels:
-    k8s-app: filebeat
-  name: filebeat-crab-config
-  namespace: crab
-data:
-  filebeat.yml: |-
-    filebeat.inputs:
-    - type: container
-      paths:
-        - /var/log/pods/crab_crabserver*/crabserver/*.log
-      include_lines:
-        - '\] crabserver-\w+-\w+ \d'
-      tags:
-        - "crabhttpcall"
-    - type: container
-      paths:
-        - /var/log/pods/crab_crabserver*/crabserver/*.log
-      include_lines:
-        - 'MeasureTime:seconds - '
-        - 'MeasureSize:bytes - '
-      tags:
-        - "crabrest"
-    output.logstash:
-      hosts: ["logstash:5044"]
-      compression_level: 3
-      bulk_max_size: 4096
-    queue.mem:
-      events: 65536
-    logging.metrics.enabled: false
-    logging.level: info
+  entrypoint.sh: |
+{{ (.Files.Get "scripts/entrypoint.sh" | indent 4) }}
+  config.py: |
+{{- if (eq (toString $environment) "prod") }}
+{{ (.Files.Get "config/prod/config.py" | indent 4) }}
+{{- else if (eq (toString $environment) "preprod")}}
+{{ (.Files.Get "config/preprod/config.py" | indent 4) }}
+{{- else }}
+{{ (.Files.Get "config/test/config.py" | indent 4) }}
+{{- end }}
+
+{{- end }}
diff --git a/helm/crabserver/templates/deployment-canary.yaml b/helm/crabserver/templates/deployment-canary.yaml
new file mode 100644
index 000000000..df833be62
--- /dev/null
+++ b/helm/crabserver/templates/deployment-canary.yaml
@@ -0,0 +1,119 @@
+{{- $environment := .Values.environment | default dict }}
+{{- if and (.Values.enabled) (.Values.canary.enabled) }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crabserver-canary
+  labels:
+    app: crabserver-canary
+  namespace: crab
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.canary.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      app: crabserver
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        app: crabserver
+        env: k8s-{{.Values.environment}}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: crabserver
+          image: "{{ .Values.canary.image.path }}:{{ .Values.canary.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.canary.image.pullPolicy }}
+          securityContext:
+           {{- toYaml .Values.securityContext | nindent 12 }}
+          ports:
+          - containerPort: 8270
+            protocol: TCP
+            name: crabserver
+          - containerPort: 18270
+            protocol: TCP
+            name: crabserver-mon
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          {{- if .Values.canary.image.command }}
+          command:
+          {{- toYaml .Values.canary.image.command | nindent 10 }}
+          {{- end }}
+          {{- if .Values.canary.image.args }}
+          args:
+          {{- toYaml .Values.canary.image.args | nindent 10 }}
+          {{- end }}
+          env:
+          {{- toYaml .Values.canary.image.env | nindent 10 }}
+          livenessProbe:
+          {{- if eq (toString $environment) "prod" }}
+          {{- toYaml .Values.livenessProbe | nindent 12 }}
+          {{- else if eq (toString $environment) "preprod" }}
+          {{- toYaml .Values.livenessProbePreProd | nindent 12 }}
+          {{- else }}
+          {{- toYaml .Values.livenessProbeTest | nindent 12 }}
+          {{- end }}
+          readinessProbe:
+          {{- if eq (toString $environment) "prod" }}
+          {{- toYaml .Values.readinessProbe | nindent 12 }}
+          {{- else if eq (toString $environment) "preprod" }}
+          {{- toYaml .Values.readinessProbePreProd | nindent 12 }}
+          {{- else }}
+          {{- toYaml .Values.readinessProbeTest | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+          - name: proxy-secrets
+            mountPath: /etc/proxy
+            readOnly: true
+          - name: secrets
+            mountPath: /etc/secrets
+            readOnly: true
+          - name: robot-secrets
+            mountPath: /etc/robots
+            readOnly: true
+          - name: hmac-secrets
+            mountPath: /etc/hmac
+            readOnly: true
+          - mountPath: /host/etc/grid-security
+            name: etc-grid-security
+            readOnly: true
+          - name: config
+            mountPath: /opt/config
+          - name: token-secrets
+            mountPath: /etc/token
+            readOnly: true
+          securityContext:
+            privileged: true
+      volumes:
+      - name: proxy-secrets
+        secret:
+          secretName: proxy-secrets
+      - name: secrets
+        secret:
+          secretName: crabserver-secrets
+      - name: robot-secrets
+        secret:
+          secretName: robot-secrets
+      - name: hmac-secrets
+        secret:
+          secretName: hmac-secrets
+      - name: etc-grid-security
+        hostPath:
+            path: /etc/grid-security
+      - name: config
+        configMap:
+          name: crabserver
+      - name: token-secrets
+        secret:
+          secretName: token-secrets
+{{- end }}
diff --git a/helm/crabserver/templates/deployment.yaml b/helm/crabserver/templates/deployment.yaml
index af8a10644..478ca1ed3 100644
--- a/helm/crabserver/templates/deployment.yaml
+++ b/helm/crabserver/templates/deployment.yaml
@@ -1,4 +1,5 @@
 {{- $environment := .Values.environment | default dict }}
+{{- if .Values.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -92,8 +93,8 @@ spec:
           - mountPath: /host/etc/grid-security
             name: etc-grid-security
             readOnly: true
-          - name: setup-certs-and-run
-            mountPath: /opt/setup-certs-and-run
+          - name: config
+            mountPath: /opt/config
           - name: token-secrets
             mountPath: /etc/token
             readOnly: true
@@ -115,9 +116,10 @@ spec:
       - name: etc-grid-security
         hostPath:
             path: /etc/grid-security
-      - name: setup-certs-and-run
+      - name: config
         configMap:
           name: crabserver
       - name: token-secrets
         secret:
           secretName: token-secrets
+{{- end }}
diff --git a/helm/crabserver/templates/filebeat-configmap.yaml b/helm/crabserver/templates/filebeat-configmap.yaml
new file mode 100644
index 000000000..c84c6d60f
--- /dev/null
+++ b/helm/crabserver/templates/filebeat-configmap.yaml
@@ -0,0 +1,37 @@
+{{- if .Values.logPipeline.enabled -}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    k8s-app: filebeat
+  name: filebeat-crab-config
+  namespace: crab
+data:
+  filebeat.yml: |-
+    filebeat.inputs:
+    - type: container
+      paths:
+        - /var/log/pods/crab_crabserver*/crabserver/*.log
+      include_lines:
+        - '\] (crabserver|crabserver-canary)-\w+-\w+ \d'
+      tags:
+        - "crabhttpcall"
+    - type: container
+      paths:
+        - /var/log/pods/crab_crabserver*/crabserver/*.log
+      include_lines:
+        - 'MeasureTime:seconds - '
+        - 'MeasureSize:bytes - '
+      tags:
+        - "crabrest"
+    output.logstash:
+      hosts: ["logstash-crab:5044"]
+      compression_level: 3
+      bulk_max_size: 4096
+    queue.mem:
+      events: 65536
+    logging.metrics.enabled: false
+    logging.level: info
+
+{{- end }}
diff --git a/helm/crabserver/templates/deployment-deamonset.yaml b/helm/crabserver/templates/filebeat-daemonset.yaml
similarity index 74%
rename from helm/crabserver/templates/deployment-deamonset.yaml
rename to helm/crabserver/templates/filebeat-daemonset.yaml
index 02a053c43..fa628947f 100644
--- a/helm/crabserver/templates/deployment-deamonset.yaml
+++ b/helm/crabserver/templates/filebeat-daemonset.yaml
@@ -1,3 +1,4 @@
+{{- if .Values.logPipeline.enabled -}}
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -20,14 +21,14 @@ spec:
     spec:
       containers:
       - args:
-        {{- toYaml .Values.deamonset.args | nindent 8 }}
+        {{- toYaml .Values.logPipeline.filebeat.args | nindent 8 }}
         env:
-        {{- toYaml .Values.deamonset.env | nindent 8 }}
-        image: {{ .Values.deamonset.path }}
-        imagePullPolicy: {{ .Values.deamonset.policy }}
-        name: {{ .Values.deamonset.name }}
+        {{- toYaml .Values.logPipeline.filebeat.env | nindent 8 }}
+        image: {{ .Values.logPipeline.filebeat.path }}
+        imagePullPolicy: {{ .Values.logPipeline.filebeat.policy }}
+        name: {{ .Values.logPipeline.filebeat.name }}
         resources:
-        {{- toYaml .Values.deamonset.resources | nindent 10 }}
+        {{- toYaml .Values.logPipeline.filebeat.resources | nindent 10 }}
         volumeMounts:
         - mountPath: /var/log/pods
           name: varlogpods
@@ -59,3 +60,5 @@ spec:
           defaultMode: 416
           name: filebeat-crab-config
         name: config
+
+{{- end }}
diff --git a/helm/crabserver/templates/hpa.yaml b/helm/crabserver/templates/hpa.yaml
index 81063b464..728361a56 100644
--- a/helm/crabserver/templates/hpa.yaml
+++ b/helm/crabserver/templates/hpa.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.autoscaling.enabled }}
+{{- if and (.Values.enabled) (.Values.autoscaling.enabled) }}
 apiVersion: autoscaling/v2beta1
 kind: HorizontalPodAutoscaler
 metadata:
diff --git a/helm/crabserver/templates/ingress.yaml b/helm/crabserver/templates/ingress.yaml
index e34d0ad0a..5cb99ced2 100644
--- a/helm/crabserver/templates/ingress.yaml
+++ b/helm/crabserver/templates/ingress.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.ingress.enabled -}}
+{{- if and (.Values.enabled) (.Values.ingress.enabled) -}}
 {{- $fullName := include "crabserver.fullname" . -}}
 {{- $svcPort := .Values.service.port -}}
 {{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
diff --git a/helm/crabserver/templates/logstash-configmap.yaml b/helm/crabserver/templates/logstash-configmap.yaml
new file mode 100644
index 000000000..a61c8e049
--- /dev/null
+++ b/helm/crabserver/templates/logstash-configmap.yaml
@@ -0,0 +1,29 @@
+{{- $environment := .Values.environment | default dict }}
+{{- if .Values.logPipeline.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    k8s-app: logstash
+  name: logstash-crab
+data:
+  # logstash config
+  logstash.yml: |
+{{- if (eq (toString $environment) "prod") }}
+{{ (.Files.Get "config/prod/logstash/logstash.yml" | indent 4) }}
+{{- else if (eq (toString $environment) "preprod")}}
+{{ (.Files.Get "config/preprod/logstash/logstash.yml" | indent 4) }}
+{{- else }}
+{{ (.Files.Get "config/test/logstash/logstash.yml" | indent 4) }}
+{{- end }}
+  # pipeline config
+  logstash.conf: |
+{{- if (eq (toString $environment) "prod") }}
+{{ (.Files.Get "config/prod/logstash/logstash.conf" | indent 4) }}
+{{- else if (eq (toString $environment) "preprod")}}
+{{ (.Files.Get "config/preprod/logstash/logstash.conf" | indent 4) }}
+{{- else }}
+{{ (.Files.Get "config/test/logstash/logstash.conf" | indent 4) }}
+{{- end }}
+
+{{- end }}
diff --git a/kubernetes/cmsweb/monitoring/crab/logstash.yaml b/helm/crabserver/templates/logstash-deployment.yaml
similarity index 56%
rename from kubernetes/cmsweb/monitoring/crab/logstash.yaml
rename to helm/crabserver/templates/logstash-deployment.yaml
index 9ec3d1104..ad227f801 100644
--- a/kubernetes/cmsweb/monitoring/crab/logstash.yaml
+++ b/helm/crabserver/templates/logstash-deployment.yaml
@@ -1,29 +1,46 @@
+{{- $environment := .Values.environment | default dict }}
+{{- if .Values.logPipeline.enabled -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: logstash
-  namespace: crab
+  name: logstash-crab
   labels:
-    app: logstash
+    app: logstash-crab
 spec:
   selector:
     matchLabels:
-      app: logstash
+      app: logstash-crab
   template:
     metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       labels:
-        app: logstash
+        app: logstash-crab
     spec:
       containers:
       - image: docker.elastic.co/logstash/logstash:8.1.3
         name: logstash
         env:
         - name: CMSWEB_CLUSTER
-          value: dev # cluster
+          {{- if (eq (toString $environment) "prod") }}
+          value: prod
+          {{- else if (eq (toString $environment) "preprod")}}
+          value: testbed
+          {{- else }}
+          value: dev
+          {{- end }}
         - name: CMSWEB_ENV
-          value: dev # cmsweb_env
+          {{- if (eq (toString $environment) "prod") }}
+          value: prod
+          {{- else if (eq (toString $environment) "preprod")}}
+          value: testbed
+          {{- else }}
+          value: dev
+          {{- end }}
         - name: LS_JAVA_OPTS
-          value: "-Xms4g -Xmx4g"
+          value: "-Xms256m -Xmx4g"
         ports:
         - name: logstash
           containerPort: 5044
@@ -39,41 +56,27 @@ spec:
         args:
         - -f
         - /usr/share/logstash/pipeline/logstash.conf
+        - -e
         #- --log.level=debug # in k8s, output stdout prints only in debug
         resources:
           requests:
-            memory: 4Gi
+            memory: 512Mi
             cpu: "400m"
           limits:
-            memory: 8Gi
+            memory: 5Gi
             cpu: "4000m"
       volumes:
       - name: pipeline
         configMap:
-          name: logstash
+          name: logstash-crab
           items:
           - key: logstash.conf
             path: logstash.conf
       - name: config
         configMap:
-          name: logstash
+          name: logstash-crab
           items:
           - key: logstash.yml
             path: logstash.yml
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: logstash
-  namespace: crab
-  labels:
-    app: logstash
-spec:
-  selector:
-    app: logstash
-  type: NodePort
-  ports:
-  - nodePort: 30044
-    targetPort: 5044
-    protocol: TCP
-    port: 5044
+
+{{- end }}
diff --git a/helm/crabserver/templates/logstash-service.yaml b/helm/crabserver/templates/logstash-service.yaml
new file mode 100644
index 000000000..b754f2b09
--- /dev/null
+++ b/helm/crabserver/templates/logstash-service.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.logPipeline.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: logstash-crab
+  labels:
+    app: logstash-crab
+spec:
+  selector:
+    app: logstash-crab
+  type: NodePort
+  ports:
+  - nodePort: 30044
+    targetPort: 5044
+    protocol: TCP
+    port: 5044
+{{- end }}
diff --git a/helm/crabserver/templates/service.yaml b/helm/crabserver/templates/service.yaml
index eb7acbbb3..8022584d8 100644
--- a/helm/crabserver/templates/service.yaml
+++ b/helm/crabserver/templates/service.yaml
@@ -1,3 +1,4 @@
+{{- if and (.Values.enabled) -}}
 kind: Service
 apiVersion: v1
 metadata:
@@ -13,4 +14,4 @@ spec:
     - port: 18270
       targetPort: 18270
       name: crabserver-mon
----
+{{- end }}
diff --git a/helm/crabserver/templates/serviceaccount.yaml b/helm/crabserver/templates/serviceaccount.yaml
index b3867549b..b0cd44582 100644
--- a/helm/crabserver/templates/serviceaccount.yaml
+++ b/helm/crabserver/templates/serviceaccount.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.serviceAccount.create -}}
+{{- if and (.Values.enabled) (.Values.serviceAccount.create)  -}}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
diff --git a/helm/crabserver/values-canary.yaml b/helm/crabserver/values-canary.yaml
new file mode 100644
index 000000000..7716ca584
--- /dev/null
+++ b/helm/crabserver/values-canary.yaml
@@ -0,0 +1,12 @@
+---
+# Deploying canary together with normal deployment
+environment: "preprod"
+
+image:
+  tag: "v3.240904-stable"
+
+canary:
+  enabled: true
+  replicaCount: 1
+  image:
+    tag: "v3.240904-stable"
diff --git a/helm/crabserver/values-clusteradmin.yaml b/helm/crabserver/values-clusteradmin.yaml
new file mode 100644
index 000000000..2890bfedf
--- /dev/null
+++ b/helm/crabserver/values-clusteradmin.yaml
@@ -0,0 +1,14 @@
+---
+enabled: true
+environment: "preprod"
+
+crabserver:
+  image:
+    tag: "v3.240904-stable"
+  canary:
+    enabled: false
+
+ciServiceAccount:
+  enabled: true
+logPipeline:
+  enabled: true
diff --git a/helm/crabserver/values-genall.yaml b/helm/crabserver/values-genall.yaml
new file mode 100644
index 000000000..e8aaa1350
--- /dev/null
+++ b/helm/crabserver/values-genall.yaml
@@ -0,0 +1,18 @@
+---
+# For debugging only. Generate all resources in this chart.
+enabled: true
+environment: "test"
+
+crabserver:
+  image:
+    tag: "pypi-test12-manual1"
+  canary:
+    enabled: true
+    replicaCount: 1
+    image:
+      tag: "pypi-test12-manual1"
+
+ciServiceAccount:
+  enabled: true
+logPipeline:
+  enabled: true
diff --git a/helm/crabserver/values-logpipeline.yaml b/helm/crabserver/values-logpipeline.yaml
new file mode 100644
index 000000000..4924616eb
--- /dev/null
+++ b/helm/crabserver/values-logpipeline.yaml
@@ -0,0 +1,6 @@
+environment: "test"
+
+enabled: false
+
+logPipeline:
+  enabled: true
diff --git a/helm/crabserver/values-preprod-pypi.yaml b/helm/crabserver/values-preprod-pypi.yaml
deleted file mode 100644
index 2855a4ce8..000000000
--- a/helm/crabserver/values-preprod-pypi.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-environment: "preprod"
-
-image:
-  path: registry.cern.ch/cmscrab/crabserver
-  pullPolicy: IfNotPresent
-  tag: "v3.240731-stable"
-  command:
-  - /data/entrypoint.sh
-  args:
-  - /bin/bash
-  - -c
-  - |
-    sudo cp -r /host/etc/grid-security/* /etc/grid-security \
-    && echo 'INFO Files in /etc/grid-security' \
-    && ls -lahZ /etc/grid-security \
-    && /data/run.sh
-
diff --git a/helm/crabserver/values-prod-pypi.yaml b/helm/crabserver/values-prod-pypi.yaml
deleted file mode 100644
index 30a862560..000000000
--- a/helm/crabserver/values-prod-pypi.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-environment: "prod"
-
-image:
-  path: registry.cern.ch/cmscrab/crabserver
-  pullPolicy: IfNotPresent
-  tag: "v3.240731-stable"
-  command:
-  - /data/entrypoint.sh
-  args:
-  - /bin/bash
-  - -c
-  - |
-    sudo cp -r /host/etc/grid-security/* /etc/grid-security \
-    && echo 'INFO Files in /etc/grid-security' \
-    && ls -lahZ /etc/grid-security \
-    && /data/run.sh
-
diff --git a/helm/crabserver/values-prod.yaml b/helm/crabserver/values-prod.yaml
index 8a358df5c..f63764119 100644
--- a/helm/crabserver/values-prod.yaml
+++ b/helm/crabserver/values-prod.yaml
@@ -1,5 +1,4 @@
 environment: "prod"
 
 image:
-  tag: "v3.240731"
-  path: "registry.cern.ch/cmsweb/crabserver"
+  tag: "v3.240904-stable"
diff --git a/helm/crabserver/values-test1.yaml b/helm/crabserver/values-test1.yaml
deleted file mode 100644
index 2df6ae869..000000000
--- a/helm/crabserver/values-test1.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-environment: "test"
-
-image:
-  tag: "v3.240709"
diff --git a/helm/crabserver/values-test11-pypi.yaml b/helm/crabserver/values-test11-pypi.yaml
deleted file mode 100644
index d5c69bdf7..000000000
--- a/helm/crabserver/values-test11-pypi.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
----
-environment: "test"
-
-image:
-  path: registry.cern.ch/cmscrab/crabserver
-  pullPolicy: IfNotPresent
-  tag: "pypi-test2-1716890482"
-  command:
-  - /data/entrypoint.sh
-  args:
-  - /bin/bash
-  - -c
-  - |
-    sudo cp -r /host/etc/grid-security/* /etc/grid-security \
-    && echo 'INFO Files in /etc/grid-security' \
-    && ls -lahZ /etc/grid-security \
-    && /data/run.sh
-
-#https://helm.sh/docs/chart_template_guide/values_files/#deleting-a-default-key
-livenessProbePreProd: null
-readinessProbePreProd: null
-readinessProbe: null
-livenessProbe: null
diff --git a/helm/crabserver/values-test11.yaml b/helm/crabserver/values-test11.yaml
deleted file mode 100644
index 1c3ab3927..000000000
--- a/helm/crabserver/values-test11.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-environment: "test"
-
-image:
-  tag: "v3.240515"
-
-livenessProbeTest:
-  failureThreshold: 3
-  initialDelaySeconds: 120
-  periodSeconds: 600
-  timeoutSeconds: 60
-
-readinessProbeTest:
-  periodSeconds: 600
-  timeoutSeconds: 60
diff --git a/helm/crabserver/values-test2.yaml b/helm/crabserver/values-test2.yaml
deleted file mode 100644
index 6e11536f4..000000000
--- a/helm/crabserver/values-test2.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-environment: "test"
-
-image:
-  tag: "v3.240515"
diff --git a/helm/crabserver/values-testbed-pypi.yaml b/helm/crabserver/values-testbed-pypi.yaml
deleted file mode 100644
index 8b8e2e8b5..000000000
--- a/helm/crabserver/values-testbed-pypi.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-environment: "testbed"
-
-image:
-  path: registry.cern.ch/cmscrab/crabserver
-  pullPolicy: IfNotPresent
-  tag: "v3.240731-stable"
-  command:
-  - /data/entrypoint.sh
-  args:
-  - /bin/bash
-  - -c
-  - |
-    sudo cp -r /host/etc/grid-security/* /etc/grid-security \
-    && echo 'INFO Files in /etc/grid-security' \
-    && ls -lahZ /etc/grid-security \
-    && /data/run.sh
-
diff --git a/helm/crabserver/values-testbed.yaml b/helm/crabserver/values-testbed.yaml
index 4abdbc172..64a304e6f 100644
--- a/helm/crabserver/values-testbed.yaml
+++ b/helm/crabserver/values-testbed.yaml
@@ -1,4 +1,4 @@
 environment: "preprod"
 
 image:
-  tag: "v3.240515"
+  tag: "v3.240904-stable"
diff --git a/helm/crabserver/values-test12.yaml b/helm/crabserver/values-testx-noprobe.yaml
similarity index 91%
rename from helm/crabserver/values-test12.yaml
rename to helm/crabserver/values-testx-noprobe.yaml
index 60ace5453..6f16d2302 100644
--- a/helm/crabserver/values-test12.yaml
+++ b/helm/crabserver/values-testx-noprobe.yaml
@@ -2,7 +2,7 @@
 environment: "test"
 
 image:
-  tag: "v3.240530"
+  tag: "v3.240809-stable"
 
 # disable liveness/readiness
 # https://helm.sh/docs/chart_template_guide/values_files/#deleting-a-default-key
diff --git a/helm/crabserver/values-testx.yaml b/helm/crabserver/values-testx.yaml
new file mode 100644
index 000000000..163272844
--- /dev/null
+++ b/helm/crabserver/values-testx.yaml
@@ -0,0 +1,5 @@
+---
+environment: "test"
+
+image:
+  tag: "v3.240809-stable"
diff --git a/helm/crabserver/values.yaml b/helm/crabserver/values.yaml
index a210351c7..dc3378b4a 100644
--- a/helm/crabserver/values.yaml
+++ b/helm/crabserver/values.yaml
@@ -2,25 +2,47 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-replicaCount: 
+enabled: true
+
+replicaCount:
   prod: 15
   preprod: 1
   test: 1
 
 image:
   path: registry.cern.ch/cmscrab/crabserver
-  pullPolicy: Always
+  pullPolicy: IfNotPresent
   # Overrides the image tag whose default is the chart appVersion.
   tag: "v3.231006"
   command:
   - /bin/bash
-  - /opt/setup-certs-and-run/setup-certs-and-run.sh
+  - /opt/config/entrypoint.sh
+  args:
+  - /data/run.sh
   env:
   - name: CRABSERVER_LOGSTDOUT
     value: "t"
   - name: CRABSERVER_THREAD_POOL
     value: "2"
 
+canary:
+  enabled: false
+  image:
+    path: registry.cern.ch/cmscrab/crabserver
+    pullPolicy: IfNotPresent
+    # Overrides the image tag whose default is the chart appVersion.
+    tag: "v3.231006"
+    command:
+    - /bin/bash
+    - /opt/config/entrypoint.sh
+    args:
+    - /data/run.sh
+    env:
+    - name: CRABSERVER_LOGSTDOUT
+      value: "t"
+    - name: CRABSERVER_THREAD_POOL
+      value: "2"
+
 environment:
 
 imagePullSecrets: []
@@ -36,7 +58,7 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name: ""
 
-podAnnotations: 
+podAnnotations:
   prometheus.io/scrape: 'true'
   prometheus.io/port: "18270"
 
@@ -48,7 +70,7 @@ podSecurityContext:
 imageFilebeatSecurityContext:
   allowPrivilegeEscalation: false
 
-securityContext: 
+securityContext:
   privileged: true
 
 serviceMon:
@@ -62,7 +84,7 @@ ingress:
     # kubernetes.io/ingress.class: nginx
     # kubernetes.io/tls-acme: "true"
 
-resources: 
+resources:
   limits:
     cpu: 1500m
     memory: 3Gi
@@ -70,38 +92,6 @@ resources:
     cpu: 200m
     memory: 256Mi
 
-
-deamonset:
-  name: filebeat
-  path: docker.elastic.co/beats/filebeat:8.5.1
-  policy: IfNotPresent
-  args:
-  - bash
-  - -c
-  - filebeat -c /etc/filebeat.yml --path.data /data/filebeat/${MY_NODE_NAME}/data -e
-  env: 
-  - name: MY_NODE_NAME
-    valueFrom:
-      fieldRef:
-        fieldPath: spec.nodeName
-  - name: NODE_NAME
-    valueFrom:
-      fieldRef:
-        fieldPath: spec.nodeName
-  - name: MY_NAMESPACE
-    valueFrom:
-      fieldRef:
-        apiVersion: v1
-        fieldPath: metadata.namespace
-  resources:
-    requests:
-      memory: "128Mi"
-      cpu: "200m"
-    limits: 
-      cpu: "1"
-      memory: "1Gi"
-
-
 livenessProbe:
   exec:
     command:
@@ -121,7 +111,7 @@ readinessProbe:
     - -c
     - |
       cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
-  periodSeconds: 60  
+  periodSeconds: 60
   timeoutSeconds: 60
 
 livenessProbePreProd:
@@ -143,7 +133,7 @@ readinessProbePreProd:
     - -c
     - |
       cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
-  periodSeconds: 60  
+  periodSeconds: 60
   timeoutSeconds: 60
 
 livenessProbeTest:
@@ -165,7 +155,7 @@ readinessProbeTest:
     - -c
     - |
       cmsweb-ping --url=http://localhost:8270/crabserver/dev/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]"
-  periodSeconds: 60  
+  periodSeconds: 60
   timeoutSeconds: 60
 
 autoscaling:
@@ -180,3 +170,39 @@ nodeSelector: {}
 tolerations: []
 
 affinity: {}
+
+
+ciServiceAccount:
+  enabled: false
+
+logPipeline:
+  enabled: false
+  filebeat:
+    name: filebeat
+    path: docker.elastic.co/beats/filebeat:8.5.1
+    policy: IfNotPresent
+    args:
+    - bash
+    - -c
+    - filebeat -c /etc/filebeat.yml --path.data /data/filebeat/${MY_NODE_NAME}/data -e
+    env:
+    - name: MY_NODE_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    - name: NODE_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    - name: MY_NAMESPACE
+      valueFrom:
+        fieldRef:
+          apiVersion: v1
+          fieldPath: metadata.namespace
+    resources:
+      requests:
+        memory: "128Mi"
+        cpu: "200m"
+      limits:
+        cpu: "1"
+        memory: "1Gi"
diff --git a/kubernetes/cmsweb/monitoring/crab/README.md b/kubernetes/cmsweb/monitoring/crab/README.md
deleted file mode 100644
index 86594b6da..000000000
--- a/kubernetes/cmsweb/monitoring/crab/README.md
+++ /dev/null
@@ -1,53 +0,0 @@
-## CRAB LOGSTASH
-
-Used for parsing CRAB Taskworker VM instance logs. Filebeat should send data with `crabtaskworker` tag.
-
-- Refs: https://monit-docs.web.cern.ch/metrics/http/
-- https://its.cern.ch/jira/browse/CMSMONIT-460
-
-#### Resorved keywords
-
-They are used in grok parsers (alpha order) !!please update the list when you update the grok!!
-
-```
-- acquiredFiles, acquiredFilesStatus, action,
-- blocks,
-- completionTime,
-- exceptionHandled,
-- files, filesPublished, functionName,
-- logMsg, log_type,
-- publicationResult, publisher_json_data,
-- slaveID,
-- taskID, taskName, timestamp_temp, tw_json_data,
-- Worker, workType,
-```
-
-#### Example log lines for grok
-
-There are currently 10 different grok definitions. Example log lines for each **log_type**:
-
-- `work_on_task_completed`
-    - `2020-10-21 00:11:41,973:DEBUG:Worker,111:Process-6: KILL work on 201020_214355:mimacken_crab_2017_LFVAnalysis_SingleEle_F completed in 2 seconds: Status: OK`
-- `publisher_config_data`
-    - `2021-05-19 09:15:34,006:INFO:PublisherMaster,149:PUBSTART: {"max_slaves": 5, "dryRun": false, "asoworker": "schedd", "DBShost": "cmsweb-prod.cern.ch", "instance": "test2", "version": "v3.210514"}`
-- `tw_config_data`
-    - `2021-05-25 18:50:55,460:INFO:MasterWorker,174:TWSTART: {"restHost": "X", "name": "Y", "recurringActions": ["RemovetmpDir", "BanDestinationSites", "TapeRecallStatus"], "DBSHostName": "Z", "instance": "other", "version": "development", "dbInstance": "dev", "nslaves": 1}`
-- `start_new_task`
-    - `2020-09-10 04:56:49,748:DEBUG:Worker,104:Process-6: Starting <function handleResubmit at 0x7f65ea4b5b90> on 200901_130305:wjang_crab_NanoAODv7_v0_QCD_HT2000toInf_TuneCUETP8M1_13TeV-madgraphMLM-pythia8`
-- `failed_publication`
-    - `2020-11-21 01:45:32,789:ERROR:PublisherMaster,550:Taskname 201118_182833:vcepaiti_crab_QCD_Pt-80to120_EMEnriched_TuneCUETP8M1_13TeV_pythia8-2016_NANOX_201117 : 1 blocks failed for a total of 11 files`
-- `successful_publication`
-    - `2020-11-27 14:25:16,191:INFO:PublisherMaster,545:Taskname 201127_011713:anstahll_crab_AODSkim_HIMinimumBias14_HIRun2018_04Apr2019_DiMuMassMin2_20201117 is OK. Published 37 files in 1 blocks.`
-- `publication_error`
-    - `2020-11-26 19:23:27,737:ERROR:PublisherMaster,554:Exception when calling TaskPublish!`
-- `acquired_files`
-    - `2021-04-15 22:23:39,566:DEBUG:master:          8 : 210415_093249:algomez_crab_QCDHT100to200TuneCP5PSWeights13TeV-madgraphMLM`
-- `acquired_files_status`
-    - `2021-12-03 20:13:59,965:DEBUG:PublisherMaster,413:acquired_files:   OK    89 : 211203_174945:cmsbot_crab_20211203_184942`
-- `action_on_task_finished`
-    - `2020-09-10 04:48:50,091:INFO:Handler,104:Finished <TaskWorker.Actions.MyProxyLogon.MyProxyLogon object at 0x7f65e821a610> on 200901_125412:wjang_crab_NanoAODv7_v0_QCD_HT700to1000_TuneCUETP8M1_13TeV-madgraphMLM-pythia8 in 1 seconds`
-
-
-#### Notes
-
-Tested with current filebeat version in VM: 7.10.0