From b143cfb8e1a76242b9491779874e9e1ee400271e Mon Sep 17 00:00:00 2001
From: nicolas-f <1382241+nicolas-f@users.noreply.github.com>
Date: Thu, 7 Sep 2023 10:59:52 +0200
Subject: [PATCH] Update script to automatically push templates

---
 .../scripts_elasticsearch/feed_rpi_data.py    |  13 +
 .../sensor_indicators_template.json           | 178 +++++++++
 .../sensor_location_template.json             | 366 +++++++++---------
 .../sensor_yamnet_template.json               |  16 +
 4 files changed, 389 insertions(+), 184 deletions(-)
 create mode 100644 services/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_indicators_template.json
 create mode 100644 services/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_yamnet_template.json

diff --git a/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/feed_rpi_data.py b/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/feed_rpi_data.py
index 43d89c0..20c614e 100644
--- a/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/feed_rpi_data.py
+++ b/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/feed_rpi_data.py
@@ -148,6 +148,19 @@ def main():
         api_key=(args.api_key_id, args.api_key),
         verify_certs=False, request_timeout=60
     )
+    # Check if templates are already pushed
+    # Templates are optional, but it will help
+    # to integrate well with kibana and reduce disk usage/increase performance
+    result = client.cat.templates(format="json")
+    templates = set([template["name"] for template in result])
+    templates_to_push = [filepath for filepath in
+                         os.listdir(os.path.dirname(__file__)) if
+                         filepath.endswith("template.json")]
+    for template in templates_to_push:
+        filename = os.path.basename(template)
+        if not filename[:filename.rfind(".json")] in templates:
+            print("Push index template " + filename)
+            client.indices.put_template(**json.load(open(template)))
     successes = 0
     try:
         for ok, action in streaming_bulk(client=client,
diff --git a/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_indicators_template.json b/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_indicators_template.json
new file mode 100644
index 0000000..418a5d6
--- /dev/null
+++ b/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_indicators_template.json
@@ -0,0 +1,178 @@
+{
+  "name": "sensor_indicators_template",
+  "index_patterns": [
+    "sensor_indicators_*"
+  ],
+  "mappings": {
+    "properties": {
+      "12": {
+        "properties": {
+          "5HZ": {
+            "type": "scaled_float",
+            "scaling_factor": 100
+          }
+        }
+      },
+      "31": {
+        "properties": {
+          "5HZ": {
+            "type": "scaled_float",
+            "scaling_factor": 100
+          }
+        }
+      },
+      "10000HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "1000HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "100HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "10HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "12500HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "1250HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "125HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "1600HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "160HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "16HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "2000HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "200HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "20HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "2500HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "250HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "25HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "3150HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "315HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "4000HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "400HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "40HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "5000HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "500HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "50HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "6300HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "630HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "63HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "8000HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "800HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "80HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "8HZ": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "LAeq": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "LCeq": {
+          "type": "scaled_float",
+          "scaling_factor": 100
+        },
+        "LZeq": {
+          "type": "scaled_float",
+          "scaling_factor": 100
+        },
+        "date_end": {
+          "type": "date"
+        },
+        "date_start": {
+          "type": "date"
+        },
+        "hwa": {
+          "type": "text",
+          "fields": {
+            "keyword": {
+              "type": "keyword",
+              "ignore_above": 256
+            }
+          }
+        }
+      }
+    }
+}
+
diff --git a/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_location_template.json b/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_location_template.json
index 28f08fd..2604fe6 100644
--- a/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_location_template.json
+++ b/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_location_template.json
@@ -1,201 +1,199 @@
-PUT _index_template/sensor_location_template
 {
+  "name": "sensor_location_template",
   "index_patterns": ["sensor_location_*"],
-  "template": {
-    "mappings": {
-       "runtime": {
-        "temperature_4g": {
-          "type": "double",
-          "script": {
-            "source": "emit(Double.parseDouble(doc['temperature_module.keyword'].value.substring(doc['temperature_module.keyword'].value.indexOf(\",\") + 1)))"
-          }
+  "mappings": {
+     "runtime": {
+      "temperature_4g": {
+        "type": "double",
+        "script": {
+          "source": "emit(Double.parseDouble(doc['temperature_module.keyword'].value.substring(doc['temperature_module.keyword'].value.indexOf(\",\") + 1)))"
         }
-      },
-      "properties": {
-        "SKY": {
-          "properties": {
-            "class": {
-              "type": "text",
-              "fields": {
-                "keyword": {
-                  "type": "keyword",
-                  "ignore_above": 256
-                }
+      }
+    },
+    "properties": {
+      "SKY": {
+        "properties": {
+          "class": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
               }
-            },
-            "device": {
-              "type": "text",
-              "fields": {
-                "keyword": {
-                  "type": "keyword",
-                  "ignore_above": 256
-                }
+            }
+          },
+          "device": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
               }
-            },
-            "gdop": {
-              "type": "float"
-            },
-            "hdop": {
-              "type": "float"
-            },
-            "nSat": {
-              "type": "long"
-            },
-            "pdop": {
-              "type": "float"
-            },
-            "satellites": {
-              "properties": {
-                "PRN": {
-                  "type": "long"
-                },
-                "az": {
-                  "type": "float"
-                },
-                "el": {
-                  "type": "float"
-                },
-                "gnssid": {
-                  "type": "long"
-                },
-                "ss": {
-                  "type": "float"
-                },
-                "svid": {
-                  "type": "long"
-                },
-                "used": {
-                  "type": "boolean"
-                }
+            }
+          },
+          "gdop": {
+            "type": "float"
+          },
+          "hdop": {
+            "type": "float"
+          },
+          "nSat": {
+            "type": "long"
+          },
+          "pdop": {
+            "type": "float"
+          },
+          "satellites": {
+            "properties": {
+              "PRN": {
+                "type": "long"
+              },
+              "az": {
+                "type": "float"
+              },
+              "el": {
+                "type": "float"
+              },
+              "gnssid": {
+                "type": "long"
+              },
+              "ss": {
+                "type": "float"
+              },
+              "svid": {
+                "type": "long"
+              },
+              "used": {
+                "type": "boolean"
               }
-            },
-            "tdop": {
-              "type": "float"
-            },
-            "uSat": {
-              "type": "long"
-            },
-            "vdop": {
-              "type": "float"
-            },
-            "xdop": {
-              "type": "float"
-            },
-            "ydop": {
-              "type": "float"
             }
+          },
+          "tdop": {
+            "type": "float"
+          },
+          "uSat": {
+            "type": "long"
+          },
+          "vdop": {
+            "type": "float"
+          },
+          "xdop": {
+            "type": "float"
+          },
+          "ydop": {
+            "type": "float"
           }
-        },
-        "TPV": {
-          "properties": {
-            "alt": {
-              "type": "float"
-            },
-            "altHAE": {
-              "type": "float"
-            },
-            "altMSL": {
-              "type": "float"
-            },
-            "class": {
-              "type": "text",
-              "fields": {
-                "keyword": {
-                  "type": "keyword",
-                  "ignore_above": 256
-                }
+        }
+      },
+      "TPV": {
+        "properties": {
+          "alt": {
+            "type": "float"
+          },
+          "altHAE": {
+            "type": "float"
+          },
+          "altMSL": {
+            "type": "float"
+          },
+          "class": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
               }
-            },
-            "climb": {
-              "type": "float"
-            },
-            "device": {
-              "type": "text",
-              "fields": {
-                "keyword": {
-                  "type": "keyword",
-                  "ignore_above": 256
-                }
+            }
+          },
+          "climb": {
+            "type": "float"
+          },
+          "device": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
               }
-            },
-            "epc": {
-              "type": "float"
-            },
-            "eph": {
-              "type": "float"
-            },
-            "eps": {
-              "type": "float"
-            },
-            "ept": {
-              "type": "float"
-            },
-            "epv": {
-              "type": "float"
-            },
-            "epx": {
-              "type": "float"
-            },
-            "epy": {
-              "type": "float"
-            },
-            "geoidSep": {
-              "type": "float"
-            },
-            "lat": {
-              "type": "float"
-            },
-            "lon": {
-              "type": "float"
-            },
-            "magvar": {
-              "type": "float"
-            },
-            "mode": {
-              "type": "long"
-            },
-            "sep": {
-              "type": "float"
-            },
-            "speed": {
-              "type": "float"
-            },
-            "time": {
-              "type": "date"
             }
+          },
+          "epc": {
+            "type": "float"
+          },
+          "eph": {
+            "type": "float"
+          },
+          "eps": {
+            "type": "float"
+          },
+          "ept": {
+            "type": "float"
+          },
+          "epv": {
+            "type": "float"
+          },
+          "epx": {
+            "type": "float"
+          },
+          "epy": {
+            "type": "float"
+          },
+          "geoidSep": {
+            "type": "float"
+          },
+          "lat": {
+            "type": "float"
+          },
+          "lon": {
+            "type": "float"
+          },
+          "magvar": {
+            "type": "float"
+          },
+          "mode": {
+            "type": "long"
+          },
+          "sep": {
+            "type": "float"
+          },
+          "speed": {
+            "type": "float"
+          },
+          "time": {
+            "type": "date"
           }
-        },
-        "date": {
-          "type": "date"
-        },
-        "hwa": {
-          "type": "text",
-          "fields": {
-            "keyword": {
-              "type": "keyword",
-              "ignore_above": 256
-            }
+        }
+      },
+      "date": {
+        "type": "date"
+      },
+      "hwa": {
+        "type": "text",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "ignore_above": 256
           }
-        },
-        "location": {
-          "type": "geo_point"
-        },
-        "lte_strength": {
-          "type": "text",
-          "fields": {
-            "keyword": {
-              "type": "keyword",
-              "ignore_above": 256
-            }
+        }
+      },
+      "location": {
+        "type": "geo_point"
+      },
+      "lte_strength": {
+        "type": "text",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "ignore_above": 256
           }
-        },
-        "temperature_module": {
-          "type": "text",
-          "fields": {
-            "keyword": {
-              "type": "keyword",
-              "ignore_above": 256
-            }
+        }
+      },
+      "temperature_module": {
+        "type": "text",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "ignore_above": 256
           }
         }
       }
diff --git a/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_yamnet_template.json b/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_yamnet_template.json
new file mode 100644
index 0000000..bd0eac8
--- /dev/null
+++ b/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/sensor_yamnet_template.json
@@ -0,0 +1,16 @@
+{
+  "name": "sensor_yamnet_template",
+  "index_patterns": [
+    "sensor_yamnet_*"
+  ],
+  "mappings": {
+    "properties": {
+      "encrypted_audio": {
+        "type": "binary"
+      },
+      "spectrogram": {
+        "type": "binary"
+      }
+    }
+  }
+}