From 8b5c9f63625aaada583f36a9fc3ebb7593aff6d9 Mon Sep 17 00:00:00 2001
From: nicolas-f <1382241+nicolas-f@users.noreply.github.com>
Date: Wed, 23 Aug 2023 15:15:07 +0200
Subject: [PATCH] fix parsing json file

---
 .../scripts_elasticsearch/feed_rpi_data.py    | 45 ++++++++++---------
 services/telit/kibana_map.json                | 17 +++++++
 services/telit/zero_location.py               |  1 -
 3 files changed, 40 insertions(+), 23 deletions(-)
 create mode 100644 services/telit/kibana_map.json

diff --git a/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/feed_rpi_data.py b/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/feed_rpi_data.py
index 31e01e7..4474ade 100644
--- a/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/feed_rpi_data.py
+++ b/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/feed_rpi_data.py
@@ -57,28 +57,29 @@ def fetch_data(args):
                 if args.verbose:
                     print("Processing " + file_path)
                 with gzip.open(file_path, 'rb') as f:
-                    json_dict = json.loads(f.readline())
-                    if "_index" not in json_dict:
-                        # must create index as it is not specified in the
-                        # document
-                        epoch = os.path.getmtime(file_path)
-                        if "timestamp" in json_dict:
-                            epoch = json_dict["timestamp"]
-                        elif "_source" in json_dict and "timestamp" in \
-                                json_dict["_source"]:
-                            epoch = json_dict["_source"]["timestamp"]
-                        elif "date" in json_dict:
-                            epoch = calendar.timegm(datetime.datetime.strptime(
-                                json_dict["date"], "%Y-%m-%dT%H:%M:%S.%fZ")
-                                                    .timetuple())
-                        dt = datetime.datetime.utcfromtimestamp(epoch)
-                        stop_position = name.find("_")
-                        if stop_position == -1:
-                            stop_position = name.find(".")
-                        json_dict["_index"] = name[:stop_position] + "_" + dt.strftime(args.time_format)
-                    json_dict["_index"] = args.index_prepend + json_dict[
-                        "_index"]
-                    yield json_dict
+                    for line in f:
+                        json_dict = json.loads(line)
+                        if "_index" not in json_dict:
+                            # must create index as it is not specified in the
+                            # document
+                            epoch = os.path.getmtime(file_path)
+                            if "timestamp" in json_dict:
+                                epoch = json_dict["timestamp"]
+                            elif "_source" in json_dict and "timestamp" in \
+                                    json_dict["_source"]:
+                                epoch = json_dict["_source"]["timestamp"]
+                            elif "date" in json_dict:
+                                epoch = calendar.timegm(datetime.datetime.strptime(
+                                    json_dict["date"], "%Y-%m-%dT%H:%M:%S.%fZ")
+                                                        .timetuple())
+                            dt = datetime.datetime.utcfromtimestamp(epoch)
+                            stop_position = name.find("_")
+                            if stop_position == -1:
+                                stop_position = name.find(".")
+                            json_dict["_index"] = name[:stop_position] + "_" + dt.strftime(args.time_format)
+                        json_dict["_index"] = args.index_prepend + json_dict[
+                            "_index"]
+                        yield json_dict
                 if not args.keep_file:
                     destination = os.path.join(
                         args.json_archive_folder,
diff --git a/services/telit/kibana_map.json b/services/telit/kibana_map.json
new file mode 100644
index 0000000..689a1e9
--- /dev/null
+++ b/services/telit/kibana_map.json
@@ -0,0 +1,17 @@
+POST sensor_location_*/_update_by_query
+{
+  "script": {
+    "source": """
+    if(ctx._source.containsKey('TPV') && ctx._source.TPV.containsKey('lat')) {
+      ctx._source.location = [:];
+      ctx._source.location['lat']=ctx._source.TPV.lat;
+      ctx._source.location['lon']=ctx._source.TPV.lon;
+    }
+    """,
+    "lang": "painless"
+  },
+  "query": {
+    "match_all": {}
+  }
+}
+
diff --git a/services/telit/zero_location.py b/services/telit/zero_location.py
index cc98ebc..a869e51 100644
--- a/services/telit/zero_location.py
+++ b/services/telit/zero_location.py
@@ -8,7 +8,6 @@
 import subprocess  # For executing a shell command
 import datetime
 
-
 def ping(host):
     """
     Returns True if host (str) responds to a ping request.