fix parsing json file

nicolas-f · Aug 23, 2023 · 8b5c9f6 · 8b5c9f6
1 parent c373903
commit 8b5c9f6
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 23 deletions.
diff --git a/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/feed_rpi_data.py b/services/ansible_openvpn/playbooks/files/scripts_elasticsearch/feed_rpi_data.py
@@ -57,28 +57,29 @@ def fetch_data(args):
                 if args.verbose:
                     print("Processing " + file_path)
                 with gzip.open(file_path, 'rb') as f:
-                    json_dict = json.loads(f.readline())
-                    if "_index" not in json_dict:
-                        # must create index as it is not specified in the
-                        # document
-                        epoch = os.path.getmtime(file_path)
-                        if "timestamp" in json_dict:
-                            epoch = json_dict["timestamp"]
-                        elif "_source" in json_dict and "timestamp" in \
-                                json_dict["_source"]:
-                            epoch = json_dict["_source"]["timestamp"]
-                        elif "date" in json_dict:
-                            epoch = calendar.timegm(datetime.datetime.strptime(
-                                json_dict["date"], "%Y-%m-%dT%H:%M:%S.%fZ")
-                                                    .timetuple())
-                        dt = datetime.datetime.utcfromtimestamp(epoch)
-                        stop_position = name.find("_")
-                        if stop_position == -1:
-                            stop_position = name.find(".")
-                        json_dict["_index"] = name[:stop_position] + "_" + dt.strftime(args.time_format)
-                    json_dict["_index"] = args.index_prepend + json_dict[
-                        "_index"]
-                    yield json_dict
+                    for line in f:
+                        json_dict = json.loads(line)
+                        if "_index" not in json_dict:
+                            # must create index as it is not specified in the
+                            # document
+                            epoch = os.path.getmtime(file_path)
+                            if "timestamp" in json_dict:
+                                epoch = json_dict["timestamp"]
+                            elif "_source" in json_dict and "timestamp" in \
+                                    json_dict["_source"]:
+                                epoch = json_dict["_source"]["timestamp"]
+                            elif "date" in json_dict:
+                                epoch = calendar.timegm(datetime.datetime.strptime(
+                                    json_dict["date"], "%Y-%m-%dT%H:%M:%S.%fZ")
+                                                        .timetuple())
+                            dt = datetime.datetime.utcfromtimestamp(epoch)
+                            stop_position = name.find("_")
+                            if stop_position == -1:
+                                stop_position = name.find(".")
+                            json_dict["_index"] = name[:stop_position] + "_" + dt.strftime(args.time_format)
+                        json_dict["_index"] = args.index_prepend + json_dict[
+                            "_index"]
+                        yield json_dict
                 if not args.keep_file:
                     destination = os.path.join(
                         args.json_archive_folder,

diff --git a/services/telit/kibana_map.json b/services/telit/kibana_map.json
@@ -0,0 +1,17 @@
+POST sensor_location_*/_update_by_query
+{
+  "script": {
+    "source": """
+    if(ctx._source.containsKey('TPV') && ctx._source.TPV.containsKey('lat')) {
+      ctx._source.location = [:];
+      ctx._source.location['lat']=ctx._source.TPV.lat;
+      ctx._source.location['lon']=ctx._source.TPV.lon;
+    }
+    """,
+    "lang": "painless"
+  },
+  "query": {
+    "match_all": {}
+  }
+}
+
diff --git a/services/telit/zero_location.py b/services/telit/zero_location.py
@@ -8,7 +8,6 @@
 import subprocess  # For executing a shell command
 import datetime
 
-
 def ping(host):
     """
     Returns True if host (str) responds to a ping request.