Update script to automatically push templates

nicolas-f · Sep 7, 2023 · b143cfb · b143cfb
1 parent 3b09a1e
commit b143cfb
Show file tree

Hide file tree

Showing 4 changed files with 389 additions and 184 deletions.
diff --git a/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/feed_rpi_data.py b/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/feed_rpi_data.py
@@ -148,6 +148,19 @@ def main():
         api_key=(args.api_key_id, args.api_key),
         verify_certs=False, request_timeout=60
     )
+    # Check if templates are already pushed
+    # Templates are optional, but it will help
+    # to integrate well with kibana and reduce disk usage/increase performance
+    result = client.cat.templates(format="json")
+    templates = set([template["name"] for template in result])
+    templates_to_push = [filepath for filepath in
+                         os.listdir(os.path.dirname(__file__)) if
+                         filepath.endswith("template.json")]
+    for template in templates_to_push:
+        filename = os.path.basename(template)
+        if not filename[:filename.rfind(".json")] in templates:
+            print("Push index template " + filename)
+            client.indices.put_template(**json.load(open(template)))
     successes = 0
     try:
         for ok, action in streaming_bulk(client=client,

diff --git a/...ces/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_indicators_template.json b/...ces/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_indicators_template.json
@@ -0,0 +1,178 @@
+{
+  "name": "sensor_indicators_template",
+  "index_patterns": [
+    "sensor_indicators_*"
+  ],
+  "mappings": {
+    "properties": {
+      "12": {
+        "properties": {
+          "5HZ": {
+            "type": "scaled_float",
+            "scaling_factor": 100
+          }
+        }
+      },
+      "31": {
+        "properties": {
+          "5HZ": {
+            "type": "scaled_float",
+            "scaling_factor": 100
+          }
+        }
+      },
+      "10000HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "1000HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "100HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "10HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "12500HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "1250HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "125HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "1600HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "160HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "16HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "2000HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "200HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "20HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "2500HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "250HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "25HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "3150HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "315HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "4000HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "400HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "40HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "5000HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "500HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "50HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "6300HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "630HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "63HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "8000HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "800HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "80HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "8HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "LAeq": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "LCeq": {
+          "type": "scaled_float",
+          "scaling_factor": 100
+        },
+        "LZeq": {
+          "type": "scaled_float",
+          "scaling_factor": 100
+        },
+        "date_end": {
+          "type": "date"
+        },
+        "date_start": {
+          "type": "date"
+        },
+        "hwa": {
+          "type": "text",
+          "fields": {
+            "keyword": {
+              "type": "keyword",
+              "ignore_above": 256
+            }
+          }
+        }
+      }
+    }
+}
+