From 6fdd5c9c0b8092a9fd564b313cd428b50fa75966 Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Mon, 8 Jul 2024 16:34:22 -0700
Subject: [PATCH 01/20] update config sample

---
 config.json.sample | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config.json.sample b/config.json.sample
index f6f9f36..a2ce89e 100644
--- a/config.json.sample
+++ b/config.json.sample
@@ -1,5 +1,6 @@
 {
   "broker": {
+    "enabled": true,
     "host": "mqtt.meshtastic.org",
     "port": 1883,
     "client_id_prefix": "meshinfo-dev",

From f851d903940da8609aef49e134f26667049baccd Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Mon, 8 Jul 2024 16:35:00 -0700
Subject: [PATCH 02/20] core refactoring

---
 data_renderer.py        |  40 +++
 main.py                 | 590 ++++------------------------------------
 memory_data_store.py    |  64 +++++
 mqtt.py                 | 244 +++++++++++++++++
 static_html_renderer.py | 294 ++++++++++++++++++++
 utils.py                |  23 ++
 6 files changed, 713 insertions(+), 542 deletions(-)
 create mode 100644 data_renderer.py
 create mode 100644 memory_data_store.py
 create mode 100644 mqtt.py
 create mode 100644 static_html_renderer.py
 create mode 100644 utils.py

diff --git a/data_renderer.py b/data_renderer.py
new file mode 100644
index 0000000..0d160bb
--- /dev/null
+++ b/data_renderer.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+import datetime
+import json
+from zoneinfo import ZoneInfo
+from jinja2 import Environment, FileSystemLoader
+
+from encoders import _JSONEncoder
+
+class DataRenderer:
+  def __init__(self, config, nodes, chat, telemetry, traceroutes):
+    self.config = config
+    self.nodes = nodes
+    self.chat = chat
+    self.telemetry = telemetry
+    self.traceroutes = traceroutes
+
+  def render(self):
+    self.save_file(self.chat, "chat.json")
+    print(f"Saved {len(self.chat['channels']['0']['messages'])} chat messages to file ({self.config['paths']['data']}/chat.json)")
+
+    old_nodes = nodes
+    nodes = {}
+    for id, node in old_nodes.items():
+        if id.startswith('!'):
+            id = id.replace('!', '')
+        nodes[id] = node
+
+    self.save_file(nodes, "nodes.json")
+    print(f"Saved {len(nodes)} nodes to file ({self.config['paths']['data']}/nodes.json)")
+
+    self.save_file(self.telemetry, "telemetry.json")
+    print(f"Saved {len(self.telemetry)} telemetry to file ({self.config['paths']['data']}/telemetry.json)")
+
+    self.save_file(self.traceroutes, "traceroutes.json")
+    print(f"Saved {len(self.traceroutes)} traceroutes to file ({self.config['paths']['data']}/traceroutes.json)")
+
+  def save_file(self, filename, data):
+    with open(f"{self.config['paths']['data']}/{filename}", "w", encoding='utf-8') as f:
+      json.dump(data, f, indent=2, sort_keys=True, cls=_JSONEncoder)
diff --git a/main.py b/main.py
index a367044..222f9d7 100644
--- a/main.py
+++ b/main.py
@@ -3,7 +3,6 @@
 import datetime
 import json
 from zoneinfo import ZoneInfo
-import paho.mqtt.client as mqtt_client
 import os
 from jinja2 import Environment, FileSystemLoader
 from dotenv import load_dotenv
@@ -11,89 +10,21 @@
 
 from config import Config
 from encoders import _JSONDecoder, _JSONEncoder
-from geo import distance_between_two_points
+from memory_data_store import MemoryDataStore
 from meshtastic import HardwareModel
 from models.node import Node
+import geo
+from mqtt import MQTT
+import utils
 
 load_dotenv()
 
 config = Config.load()
-chat: dict = {}
-chat['channels'] = {
-    '0': {
-        'name': 'General',
-        'messages': []
-    }
-}
-messages: list = []
-mqtt_messages: list = []
-mqtt_connect_time = datetime.datetime.now(ZoneInfo(config['server']['timezone']))
-nodes: dict = {}
-telemetry: list = []
-telemetry_by_node: dict = {}
-traceroutes: list = []
-traceroutes_by_node: dict = {}
-
-def connect_mqtt(broker, port, client_id, username, password):
-    def on_connect(client, userdata, flags, rc, properties=None):
-        global mqtt_connect_time
-        if rc == 0:
-            mqtt_connect_time = datetime.datetime.now(ZoneInfo(config['server']['timezone']))
-            print("Connected to MQTT broker at %s:%d (as client_id %s)" % (broker, port, client_id))
-        else:
-            print("Failed to connect, error: %s\n" % rc)
-
-    client = mqtt_client.Client(client_id=client_id, callback_api_version=mqtt_client.CallbackAPIVersion.VERSION2)
-    client.on_connect = on_connect
-    client.username_pw_set(username, password)
-    client.connect(broker, port)
-    return client
-
-def publish(client, topic, msg):
-    result = client.publish(topic, msg)
-
-    status = result[0]
-
-    if status == 0:
-        print(f"Send `{msg}` to topic `{topic}`")
-        print("Done!")
-        return True
-    else:
-        print(f"Failed to send message to topic {topic}")
-        return False
-
-def subscribe(client, topic):
-    def on_message(client, userdata, msg, properties=None):
-        try:
-            decoded = msg.payload.decode("utf-8")
-            j = json.loads(decoded, cls=_JSONDecoder)
-            handle_log(msg)
-            if j['type'] == "neighborinfo":
-                handle_neighborinfo(client, userdata, j)
-            if j['type'] == "nodeinfo":
-                handle_nodeinfo(client, userdata, j)
-            if j['type'] == "position":
-                handle_position(client, userdata, j)
-            if j['type'] == "telemetry":
-                handle_telemetry(client, userdata, j)
-            if j['type'] == "text":
-                handle_text(client, userdata, j)
-            if j['type'] == "traceroute":
-                handle_traceroute(client, userdata, j)
-            prune_expired_nodes()
-        except Exception as e:
-            print(e)
-
-    client.subscribe(topic)
-    client.on_message = on_message
-
-def update_node(id: str, node):
-    node['active'] = True
-    node['last_seen'] = datetime.datetime.now(ZoneInfo(config['server']['timezone']))
-    nodes[id] = node
+data = MemoryDataStore(config)
+data.update('mqtt_connect_time', datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
 
 def find_node_by_int_id(id: int):
-    return nodes.get(convert_node_id_from_int_to_hex(id), None)
+    return nodes.get(utils.convert_node_id_from_int_to_hex(id), None)
 
 def find_node_by_hex_id(id: str):
     return nodes.get(id, None)
@@ -104,26 +35,6 @@ def find_node_by_short_name(sn: str):
             return node
     return None
 
-def convert_node_id_from_int_to_hex(id: int):
-    return f'{id:x}'
-
-def convert_node_id_from_hex_to_int(id: str):
-    if id.startswith('!'):
-        id = id.replace('!', '')
-    return int(id, 16)
-
-def calculate_distance_between_nodes(node1, node2):
-    if node1 is None or node2 is None:
-        return None
-    if node1["position"] is None or node2["position"] is None:
-        return None
-    return round(distance_between_two_points(
-        node1["position"]["latitude_i"] / 10000000,
-        node1["position"]["longitude_i"] / 10000000,
-        node2["position"]["latitude_i"] / 10000000,
-        node2["position"]["longitude_i"] / 10000000
-    ), 2)
-
 def prune_expired_nodes():
     global config
     global nodes
@@ -139,150 +50,6 @@ def prune_expired_nodes():
     for id in ids_to_delete:
         nodes[id]['active'] = False
 
-def handle_log(msg):
-    global messages
-    global mqtt_messages
-    messages.append(msg.payload.decode("utf-8"))
-    mqtt_messages.append(msg)
-    print(f"MQTT >> {msg.topic} -- {msg.payload.decode("utf-8")}")
-    with open(f'{config["paths"]["data"]}/message-log.jsonl', 'a', encoding='utf-8') as f:
-        f.write(f"{msg.payload.decode("utf-8")}\n")
-
-def handle_neighborinfo(client, userdata, msg):
-    global nodes
-
-    msg['from'] = f'{msg["from"]:x}'
-    msg['to'] = f'{msg["to"]:x}'
-    if msg['sender'] and isinstance(msg['sender'], str):
-      msg['sender'] = msg['sender'].replace('!', '')
-
-    id = msg['from']
-    if id in nodes:
-      node = nodes[id]
-      node['neighborinfo'] = msg['payload']
-      update_node(id, node)
-      print(f"Node {id} updated with neighborinfo")
-    else:
-      node = Node.default_node(id)
-      node['neighborinfo'] = msg['payload']
-      update_node(id, node)
-      print(f"Node {id} skeleton added with neighborinfo")
-    save()
-
-def handle_nodeinfo(client, userdata, msg):
-    global nodes
-
-    msg['from'] = f'{msg["from"]:x}'
-    msg['to'] = f'{msg["to"]:x}'
-    if msg['sender'] and isinstance(msg['sender'], str):
-      msg['sender'] = msg['sender'].replace('!', '')
-
-    id = msg['payload']['id']
-    if id in nodes:
-        node = nodes[id]
-        node['hardware'] = msg['payload']['hardware']
-        node['longname'] = msg['payload']['longname']
-        node['shortname'] = msg['payload']['shortname']
-        update_node(id, node)
-        print(f"Node {id} updated")
-    else:
-        node = Node.default_node(id)
-        node['hardware'] = msg['payload']['hardware']
-        node['longname'] = msg['payload']['longname']
-        node['shortname'] = msg['payload']['shortname']
-        update_node(id, node)
-        print(f"Node {id} added")
-    sort_nodes_by_shortname()
-    save()
-
-def handle_position(client, userdata, msg):
-    global nodes
-
-    msg['from'] = f'{msg["from"]:x}'
-    msg['to'] = f'{msg["to"]:x}'
-    if msg['sender'] and isinstance(msg['sender'], str):
-      msg['sender'] = msg['sender'].replace('!', '')
-
-    id = msg['from']
-    if id in nodes:
-        node = nodes[id]
-        node['position'] = msg['payload'] if 'payload' in msg else None
-        update_node(id, node)
-        print(f"Node {id} updated with position")
-    else:
-        node = Node.default_node(id)
-        node['position'] = msg['payload'] if 'payload' in msg else None
-        update_node(id, node)
-        print(f"Node {id} skeleton added with position")
-    save()
-
-def handle_telemetry(client, userdata, msg):
-    global nodes
-    global telemetry
-
-    msg['from'] = f'{msg["from"]:x}'
-    msg['to'] = f'{msg["to"]:x}'
-    if msg['sender'] and isinstance(msg['sender'], str):
-      msg['sender'] = msg['sender'].replace('!', '')
-
-    id = msg['from']
-    if id in nodes:
-        node = nodes[id]
-        node['telemetry'] = msg['payload'] if 'payload' in msg else None
-        update_node(id, node)
-        print(f"Node {id} updated with telemetry")
-    else:
-        node = Node.default_node(id)
-        node['telemetry'] = msg['payload'] if 'payload' in msg else None
-        update_node(id, node)
-        print(f"Node {id} skeleton added with telemetry")
-
-    if id not in telemetry_by_node:
-      telemetry_by_node[id] = []
-    if 'payload' in msg:
-      telemetry.insert(0, msg)
-      telemetry_by_node[id].insert(0, msg)
-
-    save()
-
-def handle_text(client, userdata, msg):
-    global chat
-
-    msg['from'] = f'{msg["from"]:x}'
-    msg['to'] = f'{msg["to"]:x}'
-    if msg['sender'] and isinstance(msg['sender'], str):
-      msg['sender'] = msg['sender'].replace('!', '')
-
-    chat['channels'][str(msg['channel'])]['messages'].insert(0, {
-        'id': msg['id'],
-        'sender': msg['sender'],
-        'from': msg['from'],
-        'to': msg['to'],
-        'channel': str(msg['channel']),
-        'text': msg['payload']['text'],
-        'timestamp': msg['timestamp'],
-        'hops_away': msg['hops_away'] if 'hops_away' in msg else None,
-        'rssi': msg['rssi'] if 'rssi' in msg else None,
-        'snr': msg['snr'] if 'snr' in msg else None,
-    })
-    save()
-
-def handle_traceroute(client, userdata, msg):
-    global traceroutes
-    global traceroutes_by_node
-
-    msg['from'] = f'{msg["from"]:x}'
-    msg['to'] = f'{msg["to"]:x}'
-    if msg['sender'] and isinstance(msg['sender'], str):
-      msg['sender'] = msg['sender'].replace('!', '')
-    msg['route'] = msg['payload']['route']
-    if id in traceroutes_by_node:
-        traceroutes_by_node[id].insert(0, msg)
-    else:
-        traceroutes_by_node[id] = [msg]
-    traceroutes.insert(0, msg)
-    save()
-
 def load_nodes_from_file():
     n = {}
     if os.path.exists("output/data/nodes.json"):
@@ -290,216 +57,6 @@ def load_nodes_from_file():
             n = json.load(f, cls=_JSONDecoder)
     return n
 
-def _serialize_node(node):
-    """
-    Serialize a node object to a format suitable for saving to an HTML file.
-    """
-    global config
-    global nodes
-
-    last_seen = node["last_seen"] if isinstance(node["last_seen"], datetime.datetime) else datetime.datetime.fromisoformat(node["last_seen"])
-    id = node["id"].replace("!","") if isinstance(node["id"], str) else node["id"]
-    serialized = {
-        "id": id,
-        "shortname": node["shortname"],
-        "longname": node["longname"],
-        "hardware": node["hardware"],
-        "position": _serialize_position(node["position"]) if node["position"] else None,
-        "neighborinfo": _serialize_neighborinfo(node) if node['neighborinfo'] else None,
-        "telemetry": node["telemetry"],
-        "last_seen_human": last_seen.astimezone().isoformat(),
-        "last_seen": last_seen,
-        "since": datetime.datetime.now(ZoneInfo(config['server']['timezone'])) - last_seen,
-    }
-    server_node = nodes[f'{config["server"]["node_id"]}']
-    if server_node and 'position' in server_node and server_node["position"] and server_node["position"]["latitude_i"] != 0 and server_node["position"]["longitude_i"] != 0 and node["position"] and node["position"]["latitude_i"] != 0 and node["position"]["longitude_i"] != 0:
-        serialized["distance_from_host_node"] = round(distance_between_two_points(
-            node["position"]["latitude_i"] / 10000000,
-            node["position"]["longitude_i"] / 10000000,
-            server_node["position"]["latitude_i"] / 10000000,
-            server_node["position"]["longitude_i"] / 10000000
-          ), 2)
-    return serialized
-
-def _serialize_neighborinfo(node):
-    """
-    Serialize a neighborinfo object to a format suitable for saving to an HTML file.
-    """
-    ni = node['neighborinfo'].copy()
-    ni['neighbors'] = _serialize_neighborinfo_neighbors(node) if 'neighbors' in ni else None
-    return ni
-
-def _serialize_neighborinfo_neighbors(node):
-    """
-    Serialize a neighborinfo object to a format suitable for saving to an HTML file.
-    """
-    global nodes
-
-    from_node = nodes[node['id']]
-    ns = []
-    for n in node['neighborinfo']['neighbors']:
-      id = convert_node_id_from_int_to_hex(n["node_id"])
-      neighbor = {
-        "node_id": id,
-        "snr": n["snr"],
-      }
-      if id in nodes:
-        ni = nodes[id]
-        if from_node['position'] and ni['position']:
-          neighbor["distance"] = round(distance_between_two_points(
-            from_node["position"]["latitude_i"] / 10000000,
-            from_node["position"]["longitude_i"] / 10000000,
-            ni["position"]["latitude_i"] / 10000000,
-            ni["position"]["longitude_i"] / 10000000
-          ), 2)
-      ns.append(neighbor)
-    return ns
-
-def _serialize_position(position):
-    """
-    Serialize a position object to a format suitable for saving to an HTML file.
-    """
-    if "altitude" in position:
-        altitude = position["altitude"]
-    else:
-        altitude = None
-
-    return {
-        "altitude": altitude,
-        "latitude": position["latitude_i"] / 10000000,
-        "longitude": position["longitude_i"] / 10000000
-    }
-
-def sort_nodes_by_shortname():
-    global nodes
-    nodes = dict(sorted(nodes.items(), key=lambda item: item[1]["shortname"]))
-
-def render_static_html_files():
-    global config
-    global chat
-    global nodes
-    global telemetry
-    global traceroutes
-
-    print("Rendering static HTML files")
-
-    # index.html
-    print("Rendering index.html")
-    env = Environment(loader=FileSystemLoader('.'), autoescape=True)
-    template = env.get_template(f'{config["paths"]["templates"]}/static/index.html.j2')
-    rendered_html = template.render(config=config, nodes=nodes, active_nodes=nodes, datetime=datetime.datetime, zoneinfo=ZoneInfo(config['server']['timezone']), timestamp=datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
-    with open(f"{config['paths']['output']}/index.html", "w", encoding='utf-8') as f:
-        f.write(rendered_html)
-
-    # chat.html
-    print("Rendering chat.html")
-    save_chat_to_file(chat, "html", f"{config['paths']['output']}/chat.html")
-
-    # map.html
-    print("Rendering map.html")
-    server_node = nodes[f'{config["server"]["node_id"]}']
-    env = Environment(loader=FileSystemLoader('.'), autoescape=True)
-    template = env.get_template(f'{config["paths"]["templates"]}/static/map.html.j2')
-    rendered_html = template.render(config=config, server_node=server_node, nodes=nodes, calculate_distance_between_nodes=calculate_distance_between_nodes, convert_node_id_from_hex_to_int=convert_node_id_from_hex_to_int, convert_node_id_from_int_to_hex=convert_node_id_from_int_to_hex, datetime=datetime, zoneinfo=ZoneInfo(config['server']['timezone']), timestamp=datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
-    with open(f"{config['paths']['output']}/map.html", "w", encoding='utf-8') as f:
-        f.write(rendered_html)
-
-    # mesh_log.html
-    print("Rendering mesh_log.html")
-    env = Environment(loader=FileSystemLoader('.'), autoescape=True)
-    template = env.get_template(f'{config["paths"]["templates"]}/static/mesh_log.html.j2')
-    rendered_html = template.render(config=config, messages=messages, json=json, datetime=datetime.datetime, zoneinfo=ZoneInfo(config['server']['timezone']), timestamp=datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
-    with open(f"{config['paths']['output']}/mesh_log.html", "w", encoding='utf-8') as f:
-        f.write(rendered_html)
-
-    # mqtt_log.html
-    print("Rendering mqtt_log.html")
-    env = Environment(loader=FileSystemLoader('.'), autoescape=True)
-    template = env.get_template(f'{config["paths"]["templates"]}/static/mqtt_log.html.j2')
-    rendered_html = template.render(config=config, messages=mqtt_messages, mqtt_connect_time=mqtt_connect_time, json=json, datetime=datetime.datetime, zoneinfo=ZoneInfo(config['server']['timezone']), timestamp=datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
-    with open(f"{config['paths']['output']}/mqtt_log.html", "w", encoding='utf-8') as f:
-        f.write(rendered_html)
-
-    # node_{{id}}.html
-    for id, node in nodes.items():
-        id = id.replace('!', '')
-        print(f"Rendering node_{id}.html")
-        env = Environment(loader=FileSystemLoader('.'), autoescape=True)
-        template = env.get_template(f'{config["paths"]["templates"]}/static/node.html.j2')
-        rendered_html = template.render(config=config, nodes=nodes, node=node, hardware=HardwareModel, datetime=datetime.datetime, zoneinfo=ZoneInfo(config['server']['timezone']), timestamp=datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
-        with open(f"{config['paths']['output']}/node_{id}.html", "w", encoding='utf-8') as f:
-            f.write(rendered_html)
-
-    # nodes.html
-    print("Rendering nodes.html")
-    env = Environment(loader=FileSystemLoader('.'), autoescape=True)
-    template = env.get_template(f'{config["paths"]["templates"]}/static/nodes.html.j2')
-    active_nodes = {}
-    for id, node in nodes.items():
-        if 'active' in node and node['active']:
-            active_nodes[id] = _serialize_node(node)
-    rendered_html = template.render(config=config, nodes=nodes, active_nodes=active_nodes, hardware=HardwareModel, datetime=datetime.datetime, zoneinfo=ZoneInfo(config['server']['timezone']), timestamp=datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
-    with open(f"{config['paths']['output']}/nodes.html", "w", encoding='utf-8') as f:
-        f.write(rendered_html)
-
-    # neighbors.html
-    print("Rendering neighbors.html")
-    active_nodes_with_neighbors = {}
-    for id, node in nodes.items():
-        if 'active' in node and node['active'] and 'neighborinfo' in node and node['neighborinfo']:
-            active_nodes_with_neighbors[id] = _serialize_node(node)
-    env = Environment(loader=FileSystemLoader('.'), autoescape=True)
-    template = env.get_template(f'{config["paths"]["templates"]}/static/neighbors.html.j2')
-    rendered_html = template.render(config=config, nodes=nodes, active_nodes=active_nodes, active_nodes_with_neighbors=active_nodes_with_neighbors, calculate_distance_between_nodes=calculate_distance_between_nodes, convert_node_id_from_int_to_hex=convert_node_id_from_int_to_hex, datetime=datetime.datetime, zoneinfo=ZoneInfo(config['server']['timezone']), timestamp=datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
-    with open(f"{config['paths']['output']}/neighbors.html", "w", encoding='utf-8') as f:
-        f.write(rendered_html)
-
-    # routes.html
-    print("Rendering routes.html")
-    env = Environment(loader=FileSystemLoader('.'), autoescape=True)
-    template = env.get_template(f'{config["paths"]["templates"]}/static/routes.html.j2')
-    rendered_html = template.render(config=config, nodes=nodes, datetime=datetime.datetime, zoneinfo=ZoneInfo(config['server']['timezone']), timestamp=datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
-    with open(f"{config['paths']['output']}/routes.html", "w", encoding='utf-8') as f:
-        f.write(rendered_html)
-
-    # stats.html
-    print("Rendering stats.html")
-    stats = {
-        'active_nodes': 0,
-        'total_chat': len(chat['channels']['0']['messages']),
-        'total_nodes': len(nodes),
-        'total_messages': len(messages),
-        'total_mqtt_messages': len(mqtt_messages),
-        'total_telemetry': len(telemetry),
-        'total_traceroutes': len(traceroutes),
-    }
-    for _, node in nodes.items():
-        if 'active' in node and node['active']:
-            stats['active_nodes'] += 1
-    env = Environment(loader=FileSystemLoader('.'), autoescape=True)
-    template = env.get_template(f'{config["paths"]["templates"]}/static/stats.html.j2')
-    rendered_html = template.render(config=config, stats=stats, nodes=nodes, datetime=datetime.datetime, zoneinfo=ZoneInfo(config['server']['timezone']), timestamp=datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
-    with open(f"{config['paths']['output']}/stats.html", "w", encoding='utf-8') as f:
-        f.write(rendered_html)
-
-    # telemetry.html
-    print("Rendering telemetry.html")
-    env = Environment(loader=FileSystemLoader('.'), autoescape=True)
-    template = env.get_template(f'{config["paths"]["templates"]}/static/telemetry.html.j2')
-    rendered_html = template.render(config=config, nodes=nodes, telemetry=telemetry, datetime=datetime.datetime, zoneinfo=ZoneInfo(config['server']['timezone']), timestamp=datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
-    with open(f"{config['paths']['output']}/telemetry.html", "w", encoding='utf-8') as f:
-        f.write(rendered_html)
-
-    # traceroutes.html
-    print("Rendering traceroutes.html")
-    env = Environment(loader=FileSystemLoader('.'), autoescape=True)
-    template = env.get_template(f'{config["paths"]["templates"]}/static/traceroutes.html.j2')
-    rendered_html = template.render(config=config, nodes=nodes, traceroutes=traceroutes, datetime=datetime.datetime, zoneinfo=ZoneInfo(config['server']['timezone']), timestamp=datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
-    with open(f"{config['paths']['output']}/traceroutes.html", "w", encoding='utf-8') as f:
-        f.write(rendered_html)
-
-    print("Done rendering static files")
-
 def backfill_node_infos():
     global nodes
 
@@ -568,60 +125,24 @@ def save():
         print(f"Rendered in {round(end.timestamp() - save_start.timestamp(), 3)} seconds")
         config['server']['last_render'] = end
 
-def save_nodes_to_file():
-    global config
-    global chat
-    global nodes
-
-    old_nodes = nodes
-    nodes = {}
-    for id, node in old_nodes.items():
-        if id.startswith('!'):
-            id = id.replace('!', '')
-        nodes[id] = node
-    save_chat_to_file(chat, "json", f"{config['paths']['data']}/chat.json")
-    print(f"Saved {len(chat['channels']['0']['messages'])} chat messages to file ({config['paths']['data']}/chat.json)")
-    save_to_json_file(nodes, f"{config['paths']['data']}/nodes.json")
-    print(f"Saved {len(nodes)} nodes to file ({config['paths']['data']}/nodes.json)")
-    save_to_json_file(telemetry, f"{config['paths']['data']}/telemetry.json")
-    print(f"Saved {len(telemetry)} telemetry to file ({config['paths']['data']}/telemetry.json)")
-    save_to_json_file(traceroutes, f"{config['paths']['data']}/traceroutes.json")
-    print(f"Saved {len(traceroutes)} traceroutes to file ({config['paths']['data']}/traceroutes.json)")
-
-def save_node_infos_to_file(node_infos, type,path, config=config):
-    if type == "json":
-      with open(path, "w", encoding='utf-8') as f:
-          json.dump(node_infos, f, indent=2, sort_keys=True, cls=_JSONEncoder)
-    if type == "html":
-      env = Environment(loader=FileSystemLoader('.'), autoescape=True)
-      template = env.get_template(f'{config["paths"]["templates"]}/static/node_infos.html.j2')
-      rendered_html = template.render(node_infos=node_infos, datetime=datetime.datetime, timestamp=datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
-      with open(path, "w", encoding='utf-8') as f:
-          f.write(rendered_html)
-
 def load():
     global config
-    global chat
-    global nodes
-    global telemetry
-    global telemetry_by_node
-    global traceroutes
-    global traceroutes_by_node
+    global data
 
     try:
-      nodes = load_nodes_from_file()
-      print(f"Loaded {len(nodes)} existing nodes from file ({config['paths']['data']}/nodes.json)")
+      data.update('nodes', load_nodes_from_file())
+      print(f"Loaded {len(data.nodes)} existing nodes from file ({config['paths']['data']}/nodes.json)")
     except FileNotFoundError:
-      nodes = {}
-    if config['server']['node_id'] not in nodes:
-      nodes[config['server']['node_id']] = Node.default_node(config['server']['node_id'])
-    nodes['ffffffff'] = Node.default_node('ffffffff')
+      data.update('nodes', {})
+    if config['server']['node_id'] not in data.nodes:
+      data.nodes[config['server']['node_id']] = Node.default_node(config['server']['node_id'])
+    data.nodes['ffffffff'] = Node.default_node('ffffffff')
 
     try:
-      chat = load_chat_from_file("json", f"{config['paths']['data']}/chat.json")
-      print(f"Loaded {len(chat['channels']['0']['messages'])} chat messages from file ({config['paths']['data']}/chat.json)")
+      data.chat = load_chat_from_file("json", f"{config['paths']['data']}/chat.json")
+      print(f"Loaded {len(data.chat['channels']['0']['messages'])} chat messages from file ({config['paths']['data']}/chat.json)")
     except FileNotFoundError:
-      chat = {
+      data.chat = {
           'channels': {
               '0': {
                 'name': 'General',
@@ -631,38 +152,38 @@ def load():
       }
 
     try:
-      telemetry = load_from_json_file(f"{config['paths']['data']}/telemetry.json")
-      if telemetry is None or len(telemetry) == 0:
-        telemetry = []
-      if telemetry_by_node is None or len(telemetry_by_node) == 0:
-        telemetry_by_node = {}
-      for msg in telemetry:
+      data.telemetry = load_from_json_file(f"{config['paths']['data']}/telemetry.json")
+      if data.telemetry is None or len(data.telemetry) == 0:
+        data.telemetry = []
+      if data.telemetry_by_node is None or len(data.telemetry_by_node) == 0:
+        data.telemetry_by_node = {}
+      for msg in data.telemetry:
         id = msg['from']
-        if id not in telemetry_by_node:
-          telemetry_by_node[id] = []
-        telemetry_by_node[id].insert(0, msg)
-      print(f"Loaded {len(telemetry)} telemetry messages from file ({config['paths']['data']}/telemetry.json)")
-      print(f"Loaded telemetry data for {len(telemetry_by_node)} nodes")
+        if id not in data.telemetry_by_node:
+          data.telemetry_by_node[id] = []
+        data.telemetry_by_node[id].insert(0, msg)
+      print(f"Loaded {len(data.telemetry)} telemetry messages from file ({config['paths']['data']}/telemetry.json)")
+      print(f"Loaded telemetry data for {len(data.telemetry_by_node)} nodes")
     except FileNotFoundError:
-      telemetry = []
-      telemetry_by_node = {}
+      data.telemetry = []
+      data.telemetry_by_node = {}
 
     try:
-        traceroutes = load_from_json_file(f"{config['paths']['data']}/traceroutes.json")
-        if traceroutes is None or len(traceroutes) == 0:
-          traceroutes = []
-        if traceroutes_by_node is None or len(traceroutes_by_node) == 0:
-          traceroutes_by_node = {}
-        for msg in traceroutes:
+        data.traceroutes = load_from_json_file(f"{config['paths']['data']}/traceroutes.json")
+        if data.traceroutes is None or len(data.traceroutes) == 0:
+          data.traceroutes = []
+        if data.traceroutes_by_node is None or len(data.traceroutes_by_node) == 0:
+          data.traceroutes_by_node = {}
+        for msg in data.traceroutes:
           id = msg['from']
-          if id not in traceroutes_by_node:
-            traceroutes_by_node[id] = []
-          traceroutes_by_node[id].insert(0, msg)
-        print(f"Loaded {len(traceroutes)} traceroutes from file ({config['paths']['data']}/traceroutes.json)")
-        print(f"Loaded traceroutes data for {len(traceroutes_by_node)} nodes")
+          if id not in data.traceroutes_by_node:
+            data.traceroutes_by_node[id] = []
+          data.traceroutes_by_node[id].insert(0, msg)
+        print(f"Loaded {len(data.traceroutes)} traceroutes from file ({config['paths']['data']}/traceroutes.json)")
+        print(f"Loaded traceroutes data for {len(data.traceroutes_by_node)} nodes")
     except FileNotFoundError:
-        traceroutes = []
-        traceroutes_by_node = {}
+        data.traceroutes = []
+        data.traceroutes_by_node = {}
 
 def load_chat_from_file(type, path, config=config):
     global chat
@@ -676,22 +197,6 @@ def load_from_json_file(path, config=config):
     with open(path, "r", encoding='utf-8') as f:
         return json.load(f, cls=_JSONDecoder)
 
-def save_to_json_file(data, path, config=config):
-    with open(path, "w", encoding='utf-8') as f:
-        json.dump(data, f, indent=2, sort_keys=True, cls=_JSONEncoder)
-
-def save_chat_to_file(chat, type, path, config=config):
-    global nodes
-    if type == "json":
-      with open(path, "w", encoding='utf-8') as f:
-          json.dump(chat, f, indent=2, sort_keys=True, cls=_JSONEncoder)
-    if type == "html":
-      env = Environment(loader=FileSystemLoader('.'), autoescape=True)
-      template = env.get_template(f'{config["paths"]["templates"]}/static/chat.html.j2')
-      rendered_html = template.render(config=config, nodes=nodes, chat=chat, calculate_distance_between_nodes=calculate_distance_between_nodes, datetime=datetime.datetime, zoneinfo=ZoneInfo(config['server']['timezone']), timestamp=datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
-      with open(path, "w", encoding='utf-8') as f:
-          f.write(rendered_html)
-
 def run():
     global chat
     global config
@@ -716,9 +221,12 @@ def run():
     if os.environ.get('MQTT_TOPIC') is not None:
         config['broker']['topic'] = os.environ['MQTT_TOPIC']
 
-    client = connect_mqtt(config['broker']['host'], config['broker']['port'], config['broker']['client_id'], config['broker']['username'], config['broker']['password'])
-    subscribe(client, config['broker']['topic'])
+    if config['broker']['enabled'] is True:
+        print("Connecting to MQTT broker")
+        mqtt = MQTT(config, data)
+        mqtt.subscribe(config['broker']['topic'])
 
+    # discord
     # if os.environ.get('DISCORD_TOKEN') is not None:
     #     config['integrations']['discord']['token'] = os.environ['DISCORD_TOKEN']
     #     config['integrations']['discord']['channel_id'] = os.environ['DISCORD_CHANNEL_ID']
@@ -752,7 +260,5 @@ def run():
 
     #     discord_client.run(config['integrations']['discord']['token'])
 
-    client.loop_forever()
-
 if __name__ == "__main__":
     run()
diff --git a/memory_data_store.py b/memory_data_store.py
new file mode 100644
index 0000000..8674d78
--- /dev/null
+++ b/memory_data_store.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+from datetime import datetime
+from zoneinfo import ZoneInfo
+
+from static_html_renderer import StaticHTMLRenderer
+
+
+class MemoryDataStore:
+  def __init__(self, config):
+    self.config = config
+    self.chat: dict = {}
+    self.chat['channels'] = {
+        '0': {
+            'name': 'General',
+            'messages': []
+        }
+    }
+    self.messages: list = []
+    self.mqtt_messages: list = []
+    self.mqtt_connect_time: datetime = self.config['server']['start_time']
+    self.nodes: dict = {}
+    self.telemetry: list = []
+    self.telemetry_by_node: dict = {}
+    self.traceroutes: list = []
+    self.traceroutes_by_node: dict = {}
+
+    self.static_html_renderer = StaticHTMLRenderer(config, self)
+
+  def update(self, key, value):
+    self.__dict__[key] = value
+
+  def update_node(self, id: str, node):
+    node['active'] = True
+    node['last_seen'] = datetime.now().astimezone(ZoneInfo(self.config['server']['timezone']))
+    self.nodes[id] = node
+
+  def save(self):
+    save_start = datetime.now(ZoneInfo(self.config['server']['timezone']))
+    last_data = self.config['server']['last_data_save'] if 'last_data_save' in self.config['server'] else self.config['server']['start_time']
+    since_last_data = (save_start - last_data).total_seconds()
+    last_render = self.config['server']['last_render'] if 'last_render' in self.config['server'] else self.config['server']['start_time']
+    since_last_render = (save_start - last_render).total_seconds()
+    last_backfill = self.config['server']['last_backfill'] if 'last_backfill' in self.config['server'] else self.config['server']['start_time']
+    since_last_backfill = (save_start - last_backfill).total_seconds()
+    print(f"Since last - data save: {since_last_data}, render: {since_last_render}, backfill: {since_last_backfill}")
+
+    # if since_last_backfill >= 900:
+    #     self.backfill_node_infos()
+    #     end = datetime.now(ZoneInfo(self.config['server']['timezone']))
+    #     print(f"Backfilled in {round(end.timestamp() - save_start.timestamp(), 3)} seconds")
+    #     self.config['server']['last_backfill'] = end
+
+    # if since_last_data >= self.config['server']['intervals']['data_save']:
+    #     self.save_nodes_to_file()
+    #     end = datetime.now(ZoneInfo(self.config['server']['timezone']))
+    #     print(f"Saved json data in {round(end.timestamp() - save_start.timestamp(), 3)} seconds")
+    #     self.config['server']['last_data_save'] = end
+
+    if since_last_render >= self.config['server']['intervals']['render']:
+        self.static_html_renderer.render()
+        end = datetime.now(ZoneInfo(self.config['server']['timezone']))
+        print(f"Rendered in {round(end.timestamp() - save_start.timestamp(), 3)} seconds")
+        self.config['server']['last_render'] = end
diff --git a/mqtt.py b/mqtt.py
new file mode 100644
index 0000000..109b05c
--- /dev/null
+++ b/mqtt.py
@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+
+import datetime
+import json
+import traceback
+from zoneinfo import ZoneInfo
+import paho.mqtt.client as mqtt_client
+
+from encoders import _JSONDecoder
+from models.node import Node
+
+class MQTT:
+    def __init__(self, config, data):
+        self.config = config
+        self.data = data
+
+        self.host = config['broker']['host']
+        self.port = config['broker']['port']
+        self.client_id = config['broker']['client_id']
+        self.username = config['broker']['username']
+        self.password = config['broker']['password']
+
+        self.client = self.connect()
+        self.client.loop_forever()
+
+    ### actions
+
+    def connect(self):
+        ### paho callbacks
+
+        def on_connect(client, userdata, flags, rc, properties=None):
+            if rc == 0:
+                self.data.mqtt_connect_time = datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+                print("Connected to MQTT broker at %s:%d (as client_id %s)" % (self.host, self.port, self.client_id))
+                self.subscribe(self.config['broker']['topic'])
+            else:
+                print("Failed to connect, error: %s\n" % rc)
+
+        def on_message(client, userdata, msg, properties=None):
+            try:
+                decoded = msg.payload.decode("utf-8")
+                j = json.loads(decoded, cls=_JSONDecoder)
+                self.handle_log(msg)
+                if j['type'] == "neighborinfo":
+                    self.handle_neighborinfo(j)
+                if j['type'] == "nodeinfo":
+                    self.handle_nodeinfo(j)
+                if j['type'] == "position":
+                    self.handle_position(j)
+                if j['type'] == "telemetry":
+                    self.handle_telemetry(j)
+                if j['type'] == "text":
+                    self.handle_text(j)
+                if j['type'] == "traceroute":
+                    self.handle_traceroute(j)
+                self.prune_expired_nodes()
+            except Exception as e:
+                print(e)
+                traceback.print_exc()
+
+        client = mqtt_client.Client(client_id=self.client_id, callback_api_version=mqtt_client.CallbackAPIVersion.VERSION2)
+        client.on_connect = on_connect
+        client.on_message = on_message
+        client.username_pw_set(self.username, self.password)
+        client.connect(self.host, self.port)
+        return client
+
+    def publish(self, topic, msg):
+        result = self.client.publish(topic, msg)
+        status = result[0]
+        if status == 0:
+            print(f"Send `{msg}` to topic `{topic}`")
+            print("Done!")
+            return True
+        else:
+            print(f"Failed to send message to topic {topic}")
+            return False
+
+    def subscribe(self, topic):
+        self.client.subscribe(topic)
+        print(f"Subscribed to topic `{topic}`")
+
+    def unsubscribe(self, topic):
+        self.client.unsubscribe(topic)
+
+    ### message handlers
+
+    def handle_log(self, msg):
+        print(f"MQTT >> {msg.topic} -- {msg.payload.decode("utf-8")}")
+        self.data.messages.append(msg.payload.decode("utf-8"))
+        self.data.mqtt_messages.append(msg)
+        with open(f'{self.config["paths"]["data"]}/message-log.jsonl', 'a', encoding='utf-8') as f:
+            f.write(f"{msg.payload.decode("utf-8")}\n")
+
+    def handle_neighborinfo(self, msg):
+        msg['from'] = f'{msg["from"]:x}'
+        msg['to'] = f'{msg["to"]:x}'
+        if msg['sender'] and isinstance(msg['sender'], str):
+            msg['sender'] = msg['sender'].replace('!', '')
+
+        id = msg['from']
+        if id in self.data.nodes:
+            node = self.data.nodes[id]
+            node['neighborinfo'] = msg['payload']
+            self.data.update_node(id, node)
+            print(f"Node {id} updated with neighborinfo")
+        else:
+            node = Node.default_node(id)
+            node['neighborinfo'] = msg['payload']
+            self.data.update_node(id, node)
+            print(f"Node {id} skeleton added with neighborinfo")
+        self.data.save()
+
+    def handle_nodeinfo(self, msg):
+        msg['from'] = f'{msg["from"]:x}'
+        msg['to'] = f'{msg["to"]:x}'
+        if msg['sender'] and isinstance(msg['sender'], str):
+            msg['sender'] = msg['sender'].replace('!', '')
+
+        id = msg['payload']['id']
+        if id in self.data.nodes:
+            node = self.data.nodes[id]
+            node['hardware'] = msg['payload']['hardware']
+            node['longname'] = msg['payload']['longname']
+            node['shortname'] = msg['payload']['shortname']
+            self.data.update_node(id, node)
+            print(f"Node {id} updated")
+        else:
+            node = Node.default_node(id)
+            node['hardware'] = msg['payload']['hardware']
+            node['longname'] = msg['payload']['longname']
+            node['shortname'] = msg['payload']['shortname']
+            self.data.update_node(id, node)
+            print(f"Node {id} added")
+        self.sort_nodes_by_shortname()
+        self.data.save()
+
+    def handle_position(self, msg):
+        msg['from'] = f'{msg["from"]:x}'
+        msg['to'] = f'{msg["to"]:x}'
+        if msg['sender'] and isinstance(msg['sender'], str):
+            msg['sender'] = msg['sender'].replace('!', '')
+
+        id = msg['from']
+        if id in self.data.nodes:
+            node = self.data.nodes[id]
+            node['position'] = msg['payload'] if 'payload' in msg else None
+            self.data.update_node(id, node)
+            print(f"Node {id} updated with position")
+        else:
+            node = Node.default_node(id)
+            node['position'] = msg['payload'] if 'payload' in msg else None
+            self.data.update_node(id, node)
+            print(f"Node {id} skeleton added with position")
+        self.data.save()
+
+    def handle_telemetry(self, msg):
+        msg['from'] = f'{msg["from"]:x}'
+        msg['to'] = f'{msg["to"]:x}'
+        if msg['sender'] and isinstance(msg['sender'], str):
+            msg['sender'] = msg['sender'].replace('!', '')
+
+        id = msg['from']
+        if id in self.data.nodes:
+            node = self.data.nodes[id]
+            node['telemetry'] = msg['payload'] if 'payload' in msg else None
+            self.data.update_node(id, node)
+            print(f"Node {id} updated with telemetry")
+        else:
+            node = Node.default_node(id)
+            node['telemetry'] = msg['payload'] if 'payload' in msg else None
+            self.data.update_node(id, node)
+            print(f"Node {id} skeleton added with telemetry")
+
+        if id not in self.data.telemetry_by_node:
+            self.data.telemetry_by_node[id] = []
+
+        if 'payload' in msg:
+            self.data.telemetry.insert(0, msg)
+            self.data.telemetry_by_node[id].insert(0, msg)
+
+        self.data.save()
+
+    def handle_text(self, msg):
+        msg['from'] = f'{msg["from"]:x}'
+        msg['to'] = f'{msg["to"]:x}'
+        if msg['sender'] and isinstance(msg['sender'], str):
+            msg['sender'] = msg['sender'].replace('!', '')
+
+        self.data.chat['channels'][str(msg['channel'])]['messages'].insert(0, {
+            'id': msg['id'],
+            'sender': msg['sender'],
+            'from': msg['from'],
+            'to': msg['to'],
+            'channel': str(msg['channel']),
+            'text': msg['payload']['text'],
+            'timestamp': msg['timestamp'],
+            'hops_away': msg['hops_away'] if 'hops_away' in msg else None,
+            'rssi': msg['rssi'] if 'rssi' in msg else None,
+            'snr': msg['snr'] if 'snr' in msg else None,
+        })
+        self.data.save()
+
+    def handle_traceroute(self, msg):
+        msg['from'] = f'{msg["from"]:x}'
+        msg['to'] = f'{msg["to"]:x}'
+        if msg['sender'] and isinstance(msg['sender'], str):
+            msg['sender'] = msg['sender'].replace('!', '')
+            msg['route'] = msg['payload']['route']
+
+        if id in self.data.traceroutes_by_node:
+            self.data.traceroutes_by_node[id].insert(0, msg)
+        else:
+            self.data.traceroutes_by_node[id] = [msg]
+        self.data.traceroutes.insert(0, msg)
+        self.data.save()
+
+    ### helpers
+
+    # TODO: where should this really live?
+    def prune_expired_nodes(self):
+        now = datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+        ids_to_delete: list[str] = []
+        for id, node in self.data.nodes.items():
+            if node['last_seen'] is None:
+                ids_to_delete.append(node['id'])
+                continue
+            last_seen = datetime.datetime.fromisoformat(node['last_seen']).astimezone() if isinstance(node['last_seen'], str) else node['last_seen']
+            try:
+                since = (now - last_seen).seconds
+            except Exception:
+                print(f"Node {id} has invalid last_seen: {node['last_seen']}")
+                self.data.nodes[id]['last_seen'] = None
+                self.data.nodes[id]['active'] = False
+            if node['active'] and since >= self.config['server']['node_activity_prune_threshold']:
+                ids_to_delete.append(node['id'])
+                print(f"Node {id} pruned (last heard {since} seconds ago)")
+
+        for id in ids_to_delete:
+            self.data.nodes[id]['active'] = False
+
+    # TODO: where should this really live?
+    def sort_nodes_by_shortname(self):
+        self.data.nodes = dict(sorted(self.data.nodes.items(), key=lambda item: item[1]["shortname"]))
diff --git a/static_html_renderer.py b/static_html_renderer.py
new file mode 100644
index 0000000..4d04d58
--- /dev/null
+++ b/static_html_renderer.py
@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+
+import datetime
+import json
+from zoneinfo import ZoneInfo
+from jinja2 import Environment, FileSystemLoader
+
+import geo
+import meshtastic
+import utils
+
+class StaticHTMLRenderer:
+  def __init__(self, config, data):
+    self.config = config
+    self.data = data
+    self.output_path = self.config['paths']['output']
+    self.template_path = f"{self.config['paths']['templates']}/static"
+
+  def render(self):
+    self.render_index()
+    self.render_chat()
+    self.render_map()
+    self.render_mesh_log()
+    self.render_mqtt_log()
+    self.render_neighbors()
+    self.render_nodes_each()
+    self.render_nodes()
+    self.render_routes()
+    self.render_stats()
+    self.render_telemetry()
+    self.render_traceroutes()
+
+  def save_file(self, filename, content):
+    with open(f"{self.output_path}/{filename}", "w", encoding='utf-8') as f:
+      f.write(content)
+
+  def render_html(self, template_file, **kwargs):
+    env = Environment(loader=FileSystemLoader('.'), autoescape=True)
+    template_file = 'node.html' if template_file.startswith('node_') else f'{template_file}'
+    template = env.get_template(f'{self.template_path}/{template_file}.j2')
+    html = template.render(**kwargs)
+    return html
+
+  def render_html_and_save(self, filename, **kwargs):
+    print(f"Rendering {filename}")
+    html = self.render_html(filename, **kwargs)
+    self.save_file(filename, html)
+
+
+  ### Page Renderers
+
+  def render_chat(self):
+    self.render_html_and_save(
+      'chat.html',
+      config=self.config,
+      nodes=self.data.nodes,
+      chat=self.data.chat,
+      utils=utils,
+      datetime=datetime.datetime,
+      zoneinfo=ZoneInfo(self.config['server']['timezone']),
+      timestamp=datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+    )
+
+  def render_index(self):
+    self.render_html_and_save(
+      'index.html',
+      config=self.config,
+      nodes=self.data.nodes,
+      active_nodes=self.data.nodes,
+      datetime=datetime.datetime,
+      zoneinfo=ZoneInfo(self.config['server']['timezone']),
+      timestamp=datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+    )
+
+  def render_map(self):
+    server_node = self.data.nodes[self.config['server']['node_id']]
+    self.render_html_and_save(
+      'map.html',
+      config=self.config,
+      server_node=server_node,
+      nodes=self.data.nodes,
+      utils=utils,
+      datetime=datetime,
+      zoneinfo=ZoneInfo(self.config['server']['timezone']),
+      timestamp=datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+    )
+
+  def render_mesh_log(self):
+    self.render_html_and_save(
+      'mesh_log.html',
+      config=self.config,
+      messages=self.data.messages,
+      json=json,
+      datetime=datetime.datetime,
+      zoneinfo=ZoneInfo(self.config['server']['timezone']),
+      timestamp=datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+    )
+
+  def render_mqtt_log(self):
+    self.render_html_and_save(
+      'mqtt_log.html',
+      config=self.config,
+      messages=self.data.mqtt_messages,
+      mqtt_connect_time=self.data.mqtt_connect_time,
+      json=json,
+      datetime=datetime.datetime,
+      zoneinfo=ZoneInfo(self.config['server']['timezone']),
+      timestamp=datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+    )
+
+  def render_neighbors(self):
+    active_nodes_with_neighbors = {}
+    for id, node in self.data.nodes.items():
+        if 'active' in node and node['active'] and 'neighborinfo' in node and node['neighborinfo']:
+            active_nodes_with_neighbors[id] = self._serialize_node(node)
+
+    self.render_html_and_save(
+      'neighbors.html',
+      config=self.config,
+      nodes=self.data.nodes,
+      active_nodes_with_neighbors=active_nodes_with_neighbors,
+      geo=geo,
+      utils=utils,
+      datetime=datetime.datetime,
+      zoneinfo=ZoneInfo(self.config['server']['timezone']),
+      timestamp=datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+    )
+
+  def render_nodes(self):
+    active_nodes = {}
+    for id, node in self.data.nodes.items():
+        if 'active' in node and node['active']:
+            active_nodes[id] = self._serialize_node(node)
+
+    self.render_html_and_save(
+      'nodes.html',
+      config=self.config,
+      nodes=self.data.nodes,
+      active_nodes=active_nodes,
+      hardware=meshtastic.HardwareModel,
+      datetime=datetime.datetime,
+      zoneinfo=ZoneInfo(self.config['server']['timezone']),
+      timestamp=datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+    )
+
+  def render_nodes_each(self):
+    for id, node in self.data.nodes.items():
+      id = id.replace('!', '') # todo: remove this line
+      self.render_html_and_save(
+        f"node_{id}.html",
+        config=self.config,
+        node=node,
+        nodes=self.data.nodes,
+        hardware=meshtastic.HardwareModel,
+        datetime=datetime.datetime,
+        zoneinfo=ZoneInfo(self.config['server']['timezone']),
+        timestamp=datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+      )
+
+  def render_routes(self):
+    self.render_html_and_save(
+      'routes.html',
+      config=self.config,
+      nodes=self.data.nodes,
+      active_nodes=self.data.nodes,
+      datetime=datetime.datetime,
+      zoneinfo=ZoneInfo(self.config['server']['timezone']),
+      timestamp=datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+    )
+
+  def render_stats(self):
+    stats = {
+      'active_nodes': 0,
+      'total_chat': len(self.data.chat['channels']['0']['messages']),
+      'total_nodes': len(self.data.nodes),
+      'total_messages': len(self.data.messages),
+      'total_mqtt_messages': len(self.data.mqtt_messages),
+      'total_telemetry': len(self.data.telemetry),
+      'total_traceroutes': len(self.data.traceroutes),
+    }
+    for _, node in self.data.nodes.items():
+        if 'active' in node and node['active']:
+            stats['active_nodes'] += 1
+
+    self.render_html_and_save(
+      'stats.html',
+      config=self.config,
+      stats=stats,
+      nodes=self.data.nodes,
+      datetime=datetime.datetime,
+      zoneinfo=ZoneInfo(self.config['server']['timezone']),
+      timestamp=datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+    )
+
+  def render_telemetry(self):
+    self.render_html_and_save(
+      'telemetry.html',
+      config=self.config,
+      nodes=self.data.nodes,
+      telemetry=self.data.telemetry,
+      datetime=datetime.datetime,
+      zoneinfo=ZoneInfo(self.config['server']['timezone']),
+      timestamp=datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+    )
+
+  def render_traceroutes(self):
+    self.render_html_and_save(
+      'traceroutes.html',
+      config=self.config,
+      nodes=self.data.nodes,
+      traceroutes=self.data.traceroutes,
+      datetime=datetime.datetime,
+      zoneinfo=ZoneInfo(self.config['server']['timezone']),
+      timestamp=datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
+    )
+
+  # TODO: move to models
+  def _serialize_node(self, node):
+    """
+    Serialize a node object to a format suitable for saving to an HTML file.
+    """
+
+    last_seen = node["last_seen"] if isinstance(node["last_seen"], datetime.datetime) else datetime.datetime.fromisoformat(node["last_seen"])
+    id = node["id"].replace("!","") if isinstance(node["id"], str) else node["id"]
+    serialized = {
+        "id": id,
+        "shortname": node["shortname"],
+        "longname": node["longname"],
+        "hardware": node["hardware"],
+        "position": self._serialize_position(node["position"]) if node["position"] else None,
+        "neighborinfo": self._serialize_neighborinfo(node) if node['neighborinfo'] else None,
+        "telemetry": node["telemetry"],
+        "last_seen_human": last_seen.astimezone().isoformat(),
+        "last_seen": last_seen,
+        "since": datetime.datetime.now(ZoneInfo(self.config['server']['timezone'])) - last_seen,
+    }
+    server_node = self.data.nodes[f'{self.config["server"]["node_id"]}']
+    if server_node and 'position' in server_node and server_node["position"] and server_node["position"]["latitude_i"] != 0 and server_node["position"]["longitude_i"] != 0 and node["position"] and node["position"]["latitude_i"] != 0 and node["position"]["longitude_i"] != 0:
+        serialized["distance_from_host_node"] = round(geo.distance_between_two_points(
+            node["position"]["latitude_i"] / 10000000,
+            node["position"]["longitude_i"] / 10000000,
+            server_node["position"]["latitude_i"] / 10000000,
+            server_node["position"]["longitude_i"] / 10000000
+          ), 2)
+    return serialized
+
+  def _serialize_neighborinfo(self, node):
+    """
+    Serialize a neighborinfo object to a format suitable for saving to an HTML file.
+    """
+    ni = node['neighborinfo'].copy()
+    ni['neighbors'] = self._serialize_neighborinfo_neighbors(node) if 'neighbors' in ni else None
+    return ni
+
+  def _serialize_neighborinfo_neighbors(self, node):
+    """
+    Serialize a neighborinfo object to a format suitable for saving to an HTML file.
+    """
+    global nodes
+
+    from_node = self.data.nodes[node['id']]
+    ns = []
+    for n in node['neighborinfo']['neighbors']:
+      id = utils.convert_node_id_from_int_to_hex(n["node_id"])
+      neighbor = {
+        "node_id": id,
+        "snr": n["snr"],
+      }
+      if id in self.data.nodes:
+        ni = self.data.nodes[id]
+        if from_node['position'] and ni['position']:
+          neighbor["distance"] = round(geo.distance_between_two_points(
+            from_node["position"]["latitude_i"] / 10000000,
+            from_node["position"]["longitude_i"] / 10000000,
+            ni["position"]["latitude_i"] / 10000000,
+            ni["position"]["longitude_i"] / 10000000
+          ), 2)
+      ns.append(neighbor)
+    return ns
+
+  def _serialize_position(self, position):
+    """
+    Serialize a position object to a format suitable for saving to an HTML file.
+    """
+    if "altitude" in position:
+        altitude = position["altitude"]
+    else:
+        altitude = None
+
+    return {
+        "altitude": altitude,
+        "latitude": position["latitude_i"] / 10000000,
+        "longitude": position["longitude_i"] / 10000000
+    }
diff --git a/utils.py b/utils.py
new file mode 100644
index 0000000..fc1c9ac
--- /dev/null
+++ b/utils.py
@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+from geo import distance_between_two_points
+
+def calculate_distance_between_nodes(node1, node2):
+  if node1 is None or node2 is None:
+    return None
+  if node1["position"] is None or node2["position"] is None:
+    return None
+  return round(distance_between_two_points(
+    node1["position"]["latitude_i"] / 10000000,
+    node1["position"]["longitude_i"] / 10000000,
+    node2["position"]["latitude_i"] / 10000000,
+    node2["position"]["longitude_i"] / 10000000
+  ), 2)
+
+def convert_node_id_from_int_to_hex(id: int):
+  return f'{id:x}'
+
+def convert_node_id_from_hex_to_int(id: str):
+  if id.startswith('!'):
+      id = id.replace('!', '')
+  return int(id, 16)

From 4363987ad136613eac9f6bb1845ef9dffde7da4f Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Mon, 8 Jul 2024 16:35:45 -0700
Subject: [PATCH 03/20] update templates with new designs and support for
 refactor

---
 templates/static/chat.html.j2        |  62 ++++++------
 templates/static/layout.html.j2      |  68 ++++++++------
 templates/static/map.html.j2         |   2 +-
 templates/static/neighbors.html.j2   | 135 +++++++++++++++------------
 templates/static/node.html.j2        |   6 +-
 templates/static/telemetry.html.j2   |   9 +-
 templates/static/traceroutes.html.j2 |   9 +-
 7 files changed, 163 insertions(+), 128 deletions(-)

diff --git a/templates/static/chat.html.j2 b/templates/static/chat.html.j2
index 60f1248..7a4a084 100644
--- a/templates/static/chat.html.j2
+++ b/templates/static/chat.html.j2
@@ -3,54 +3,64 @@
 {% block title %}Chat{% endblock %}
 
 {% block content %}
-    <h1>Chat</h1>
-    <p>
+    <h5 class="mb-2 text-gray-500">Chat</h5>
+    <h1 class="mb-2 text-xl">Chat</h1>
+    <p class="mb-2">
       There are <b>{{ chat['channels']['0']['messages']|count }}</b> messages on channel 0 that have
       been heard by the mesh by <b>KE-R</b> (!4355f528).
     </p>
-    <p>Last updated: {{ timestamp.astimezone(zoneinfo).strftime('%Y-%m-%d %H:%M:%S %z') }}</p>
 
-    <h2>Channel 0</h2>
-    <table border=1 cellpadding=4>
+    <h2 class="text-lg font-bold">Channel 0</h2>
+    <table class="w-full max-w-full table-fixed border-collapse border border-gray-500 bg-gray-50">
+      <thead>
         <tr>
-            <th>Time</th>
-            <th>From</th>
-            <th>Via</th>
-            <th>To</th>
-            <th>Hops</th>
-            <th>DX</th>
-            <th width=65%>Message</th>
+            <th class="w-32 max-w-32 border border-gray-500 bg-gray-400">Time</th>
+            <th class="w-12 max-w-12 border border-gray-500 bg-gray-400">From</th>
+            <th class="w-12 max-w-12 border border-gray-500 bg-gray-400">Via</th>
+            <th class="w-12 max-w-12 border border-gray-500 bg-gray-400">To</th>
+            <th class="w-12 max-w-12 border border-gray-500 bg-gray-400">Hops</th>
+            <th class="w-20 max-w-20 border border-gray-500 bg-gray-400">DX</th>
+            <th class="p-1 text-wrap border border-gray-500 bg-gray-400">Message</th>
         </tr>
+      </thead>
+      <tbody>
         {% for message in chat['channels']['0']['messages'] %}
         {% set node_from = nodes[message.from] if message.from in nodes else None %}
         {% set node_sender = nodes[message.sender] if message.sender in nodes else None %}
         {% set node_to = nodes[message.to] if message.to in nodes else None %}
-        {% set distance_from_sender = calculate_distance_between_nodes(node_from, node_sender) if node_from and node_sender else None %}
+        {% set distance_from_sender = utils.calculate_distance_between_nodes(node_from, node_sender) if node_from and node_sender else None %}
         <tr>
-            <td>{{ datetime.fromtimestamp(message.timestamp).astimezone(zoneinfo).strftime('%Y-%m-%d %H:%M:%S %z') }}</td>
-            <td>
-              <span title="{{ message.from + " / " + nodes[message.from].longname if message.from in nodes else (message.from + ' / Unknown') }}">
+            <td class="p-1 border border-gray-400">{{ datetime.fromtimestamp(message.timestamp).astimezone(zoneinfo).strftime('%Y-%m-%d %H:%M:%S %z') }}</td>
+            <td class="p-1 text-center border border-gray-400">
+              <a href="node_{{ node_from.id }}.html" title="{{ message.from + " / " + nodes[message.from].longname if message.from in nodes else (message.from + ' / Unknown') }}">
                 {{ nodes[message.from].shortname if message.from in nodes else 'UNK' }}
-              </span>
+              </a>
             </td>
-            <td>
-              <span title="{{ message.sender + " / " + nodes[message.sender].longname if message.sender in nodes else (message.sender + ' / Unknown') }}">
+            <td class="p-1 text-center border border-gray-400">
+              <a href="node_{{ node_sender.id }}.html" title="{{ message.sender + " / " + nodes[message.sender].longname if message.sender in nodes else (message.sender + ' / Unknown') }}">
                 {{ nodes[message.sender].shortname if message.sender in nodes else 'UNK' }}
-              </span>
+              </a>
             </td>
-            <td>
-              <span title="{{ message.to + " / " + nodes[message.to].longname if message.to in nodes else (message.to + ' / Unknown') }}">
+            <td class="p-1 text-center border border-gray-400">
+              {% if node_to and node_to.id != 'ffffffff' %}
+              <a href="node_{{ node_to.id }}.html" title="{{ message.to + " / " + nodes[message.to].longname if message.to in nodes else (message.to + ' / Unknown') }}">
                 {{ nodes[message.to].shortname if message.to in nodes else 'UNK' }}
-              </span>
+              </a>
+              {% else %}
+              <span class="text-gray-500">ALL</span>
+              {% endif %}
             </td>
-            <td align=center>{{ message.hops_away }}</td>
-            <td align=right>
+            <td class="p-1 border border-gray-400" align=center>{{ message.hops_away }}</td>
+            <td class="p-1 text-nowrap border border-gray-400" align=right>
               {% if distance_from_sender %}
               {{ distance_from_sender }} km
               {% endif %}
             </td>
-            <td>{{ message.text }}</td>
+            <td class="p-1 text-wrap border border-gray-400">
+              {{ message.text }}
+            </td>
         </tr>
         {% endfor %}
+      </tbody>
     </table>
 {% endblock %}
diff --git a/templates/static/layout.html.j2 b/templates/static/layout.html.j2
index d9a4e39..3f300c5 100644
--- a/templates/static/layout.html.j2
+++ b/templates/static/layout.html.j2
@@ -7,7 +7,7 @@
     <script>
       tailwind.config = {
         corePlugins: {
-          preflight: false,
+          preflight: true,
         },
       }
     </script>
@@ -38,81 +38,89 @@
         {% endif %}
 
         <nav class="flex flex-col flex-1">
-          <h3>Mesh</h3>
-          <div class="mb-2">
+          <h3 class="font-bold">Mesh</h3>
+          <div class="mb-1">
             <a href="chat.html">
-              <img src="images/icons/chat.svg" width="20" height="20" align="middle" style="vertical-align: middle;">
+              <img src="images/icons/chat.svg" width="20" height="20" align="middle" class="inline-block" style="vertical-align: middle;">
               Chat
             </a>
           </div>
-          <div class="mb-2">
+          <div class="mb-1">
             <a href="map.html">
-              <img src="images/icons/map.svg" width="20" height="20" align="middle" style="vertical-align: middle;">
+              <img src="images/icons/map.svg" width="20" height="20" align="middle" class="inline-block" style="vertical-align: middle;">
               Map
             </a>
           </div>
-          <div class="mb-2">
+          <div class="mb-1">
             <a href="nodes.html">
-              <img src="images/icons/node.svg" width="20" height="20" align="middle" style="vertical-align: middle;">
+              <img src="images/icons/node.svg" width="20" height="20" align="middle" class="inline-block" style="vertical-align: middle;">
               Nodes
             </a>
           </div>
-          <div class="mb-2">
+          <div class="mb-1">
             <a href="neighbors.html">
-              <img src="images/icons/neighbors.svg" width="20" height="20" align="middle" style="vertical-align: middle;">
+              <img src="images/icons/neighbors.svg" width="20" height="20" align="middle" class="inline-block" style="vertical-align: middle;">
               Node Neighbors
             </a>
           </div>
-          <div class="mb-2">
+          <div class="mb-1">
             <a href="stats.html">
-              <img src="images/icons/stats.svg" width="20" height="20" align="middle" style="vertical-align: middle;">
+              <img src="images/icons/stats.svg" width="20" height="20" align="middle" class="inline-block" style="vertical-align: middle;">
               Stats
             </a>
           </div>
-          <div class="mb-2">
+          <div class="mb-1">
             <a href="telemetry.html">
-              <img src="images/icons/telemetry.svg" width="20" height="20" align="middle" style="vertical-align: middle;">
+              <img src="images/icons/telemetry.svg" width="20" height="20" align="middle" class="inline-block" style="vertical-align: middle;">
               Telemetry
             </a>
           </div>
-          <div class="mb-2">
+          <div class="mb-1">
             <a href="traceroutes.html">
-              <img src="images/icons/route2.svg" width="20" height="20" align="middle" style="vertical-align: middle;">
+              <img src="images/icons/route2.svg" width="20" height="20" align="middle" class="inline-block" style="vertical-align: middle;">
               Traceroutes
             </a>
           </div>
         </nav>
 
         <nav class="flex flex-col flex-1">
-          <h3>Logs</h3>
-          <div class="mb-2">
+          <h3 class="font-bold">Logs</h3>
+          <div class="mb-1">
             <a href="mesh_log.html">Mesh Messages</a>
           </div>
-          <div class="mb-2">
+          <div class="mb-1">
             <a href="mqtt_log.html">MQTT Messages</a>
           </div>
         </nav>
 
         <nav class="flex flex-col flex-1">
-          <h3>Other Tools</h3>
-          <div class="mb-2">
+          <h3 class="font-bold">Other Tools</h3>
+          <div class="mb-1">
             <a href="https://meshview.armooo.net" target="_blank">Armooo's MeshView</a>
           </div>
-          <div class="mb-2">
+          <div class="mb-1">
             <a href="https://meshtastic.liamcottle.net" target="_blank">Liam's Meshtastic Web</a>
           </div>
-          <div class="mb-2">
+          <div class="mb-1">
             <a href="https://meshmap.net" target="_blank">MeshMap</a>
           </div>
-          <div class="mb-2">
+          <div class="mb-1">
             <a href="https://app.bayme.sh" target="_blank">Bay Mesh Explorer</a>
           </div>
-          <div class="mb-2">
+          <div>
             <a href="https://heywhatsthat.com/profiler.html" target="_blank">HWT Path Profiler</a>
           </div>
         </nav>
 
         <div class="flex-grow"></div>
+
+        <div class="flex flex-col">
+          <div class="font-bold">Data Updated</div>
+          <div>{{ timestamp.astimezone(zoneinfo).strftime('%Y-%m-%d %H:%M:%S %z') }}</div>
+        </div>
+
+        <div class="flex-grow"></div>
+
         <div class="flex flex-col">
           <h5 class="mb-2">Powered by MeshInfo</h5>
           <a href="https://github.com/kevinelliott/meshinfo" class="text-xs text-gray-500">
@@ -124,7 +132,7 @@
 
     <div class="lg:pl-60">
       <main class="py-1">
-        <div class="px-4 sm:px-6 lg:px-8">
+        <div class="w-full px-4 py-2 sm:px-6 sm:py-2 lg:px-6 lg:py-2">
           {% block content %}{% endblock %}
         </div>
       </main>
@@ -133,10 +141,10 @@
 
   <style>
       body {
-          font-family: monospace;
-          font-size: 12px;
-          margin: 0;
-          padding: 0;
+        font-family: monospace;
+        font-size: 12px;
+        margin: 0;
+        padding: 0;
       }
       a {
         text-decoration: none;
diff --git a/templates/static/map.html.j2 b/templates/static/map.html.j2
index 2a0ceab..f55522f 100644
--- a/templates/static/map.html.j2
+++ b/templates/static/map.html.j2
@@ -126,7 +126,7 @@
       nodes['{{ id }}'].neighbors = [
         {% for neighbor in node.neighborinfo.neighbors %}
           {
-            id: '{{ convert_node_id_from_int_to_hex(neighbor.node_id) }}',
+            id: '{{ utils.convert_node_id_from_int_to_hex(neighbor.node_id) }}',
             snr: '{{ neighbor.snr }}',
             distance: '{{ neighbor.distance }}',
           },
diff --git a/templates/static/neighbors.html.j2 b/templates/static/neighbors.html.j2
index def09d5..ecab439 100644
--- a/templates/static/neighbors.html.j2
+++ b/templates/static/neighbors.html.j2
@@ -3,92 +3,103 @@
 {% block title %}Neighbors{% endblock %}
 
 {% block content %}
-    <h1>Neighbors</h1>
-    <p>
+    <h5 class="mb-2 text-gray-500">Neighbors</h5>
+    <h1 class="mb-2 text-xl">Neighbors</h1>
+    <p class="mb-2">
       There are <b>{{ active_nodes_with_neighbors|count }}</b> active nodes with neighbors
       that have been heard by the mesh by <b>KE-R</b> (!4355f528).
     </p>
-    <p>Last updated: {{ timestamp.astimezone(zoneinfo).strftime('%Y-%m-%d %H:%M:%S %z') }}</p>
-    <table border=1 cellspacing=2 cellpadding=4 width=100%>
-        <tr bgcolor=lightgray>
-            <th>ID</th>
-            <th colspan=2>Name</th>
-            <th colspan=3>Neighbors</th>
-            <th colspan=2>Seen</th>
+
+    <table class="w-full max-w-full table-auto border-collapse border border-gray-500 bg-gray-50">
+      <thead>
+        <tr>
+          <th class="w-20 max-w-20 border border-gray-500 bg-gray-400">ID</th>
+          <th class="border border-gray-500 bg-gray-400" colspan=2>Name</th>
+          <th class="border border-gray-500 bg-gray-400" colspan=3>Neighbors</th>
+          <th class="hidden xl:table-cell border border-gray-500 bg-gray-400" colspan=2>Seen</th>
         </tr>
-        <tr bgcolor=lightgray>
-            <th></th>
-            <th>Short</th>
-            <th>Long</th>
-            <th>Heard</th>
-            <th>Heard By</th>
-            <th>Interval</th>
-            <th>Last</th>
-            <th>Since</th>
+        <tr>
+          <th class="w-12 max-w-12 border border-gray-500 bg-gray-400"></th>
+          <th class="w-12 max-w-12 border border-gray-500 bg-gray-400">Short</th>
+          <th class="border border-gray-500 bg-gray-400">Long</th>
+          <th class="border border-gray-500 bg-gray-400">Heard</th>
+          <th class="border border-gray-500 bg-gray-400">Heard By</th>
+          <th class="border border-gray-500 bg-gray-400">Interval</th>
+          <th class="hidden xl:table-cell border border-gray-500 bg-gray-400">Last</th>
+          <th class="hidden xl:table-cell w-20 max-w-20 border border-gray-500 bg-gray-400">Since</th>
         </tr>
+      </thead>
+      <tbody>
         {% for id, node in active_nodes_with_neighbors.items() %}
         <tr>
-            <td>
-              {% if id %}
-              {% set id = id|replace('!', '') %}
-                <a href='node_{{ id }}.html'>{{ id }}</a>
-              {% else %}
+          <td class="p-1 border border-gray-400" align=center valign=middle>
+            {% if id %}
+            {% set id = id|replace('!', '') %}
+              <a href='node_{{ id }}.html'>
+                <img src="https://api.dicebear.com/9.x/bottts-neutral/svg?seed={{ id }}" alt="Avatar" class="w-16 h-16 mb-1 object-cover">
                 {{ id }}
-              {% endif %}
-            </td>
-            <td style="color: #{{ '777' if node.shortname == 'UNK' else '000' }}">
-              {% if id %}
-                <a href='node_{{ id }}.html'>{{ node.shortname }}</a>
-              {% else %}
-                <span class="text-gray-500">{{ node.shortname }}</span>
-              {% endif %}
-            </td>
-            <td style="color: #{{ '777' if node.shortname == 'UNK' else '000' }}">
-              {{ node.longname }}
-            </td>
-            {% if node.neighborinfo %}
-            <td align=left>
-              <table width=100%>
+              </a>
+            {% else %}
+              <img src="https://api.dicebear.com/9.x/bottts-neutral/svg?seed={{ id }}" alt="Avatar" class="w-16 h-16 mb-1 object-cover">
+              {{ id }}
+            {% endif %}
+          </td>
+          <td class="p-1 border border-gray-400" style="color: #{{ '777' if node.shortname == 'UNK' else '000' }}" align=center>
+            {% if id %}
+              <a href='node_{{ id }}.html'>{{ node.shortname }}</a>
+            {% else %}
+              <span class="text-gray-500">{{ node.shortname }}</span>
+            {% endif %}
+          </td>
+          <td class="p-1 border border-gray-400" style="color: #{{ '777' if node.shortname == 'UNK' else '000' }}">
+            {{ node.longname }}
+          </td>
+          {% if node.neighborinfo %}
+          <td class="p-0 border border-gray-400" valign=top>
+            <table class="table-auto min-w-full">
+              <tbody class="divide-y divide-dashed divide-gray-400">
               {% for neighbor in node.neighborinfo.neighbors %}
                 <tr>
-                  <td width=25%>
+                  <td class="w-1/3 p-1 text-nowrap">
                     {% if neighbor.node_id in nodes %}
-                    {{ nodes[neighbor.node_id].shortname }}
+                    <a href="node_{{ nodes[neighbor.node_id].id }}.html">{{ nodes[neighbor.node_id].shortname }}</a>
                     {% else %}
-                    UNK
+                    <span class="text-gray-500">UNK</span>
                     {% endif %}
                   </td>
-                  <td>
+                  <td class="p-1 text-nowrap">
                     SNR: {{ neighbor.snr }}
                   </td>
-                  <td align=right>
+                  <td class="p-1 text-nowrap" align=right>
                     {% if neighbor.distance %}
                       {{ neighbor.distance }} km
                     {% endif %}
                   </td>
                 </tr>
               {% endfor %}
-              </table>
-            </td>
-            <td align=left>
-              <table width=100%>
+              </tbody>
+            </table>
+          </td>
+          <td class="p-0 border border-gray-400" valign=top>
+            <table class="table-auto min-w-full">
+              <tbody class="divide-y divide-dashed divide-gray-400">
               {% for nid, nnode in nodes.items() %}
               {% if nnode.neighborinfo %}
               {% for neighbor in nnode.neighborinfo.neighbors %}
-              {% if convert_node_id_from_int_to_hex(neighbor.node_id) == id %}
+              {% if utils.convert_node_id_from_int_to_hex(neighbor.node_id) == id %}
                 <tr>
-                  <td width=25%>
+                  <td class="w-1/3 p-1 text-nowrap" width=25%>
                     {% if nid in nodes %}
-                      {{ nodes[nid].shortname }}
+                      <a href="node_{{ nid }}.html">{{ nodes[nid].shortname }}</a>
                     {% else %}
-                      UNK
+                      <span class="text-gray-500">UNK</span>
                     {% endif %}
                   </td>
-                  <td>
+                  <td class="p-1 text-nowrap">
                     SNR: {{ neighbor.snr }}
                   </td>
-                  <td align=right>
-                    {% set dist = calculate_distance_between_nodes(nodes[nid], nodes[id]) %}
+                  <td class="p-1 text-nowrap" align=right>
+                    {% set dist = utils.calculate_distance_between_nodes(nodes[nid], nodes[id]) %}
                     {% if dist %}
                     {{ dist }} km
                     {% endif %}
@@ -98,15 +109,17 @@
               {% endfor %}
               {% endif %}
               {% endfor %}
-              </table>
-            <td align=right>{{ node.neighborinfo.node_broadcast_interval_secs }}s</td>
-            {% else %}
-            <td></td>
-            {% endif %}
-            <td>{{ node.last_seen_human }}</td>
-            <td align=right>{{ node.since.seconds }} secs</td>
+              </tbody>
+            </table>
+          <td class="p-1 border border-gray-400" align=right>{{ node.neighborinfo.node_broadcast_interval_secs }}s</td>
+          {% else %}
+          <td class="p-1 border border-gray-400"></td>
+          {% endif %}
+          <td class="hidden xl:table-cell p-1 border border-gray-400">{{ node.last_seen_human }}</td>
+          <td class="hidden xl:table-cell p-1 text-nowrap border border-gray-400" align=right>{{ node.since.seconds }} secs</td>
         </tr>
         {% endfor %}
+      </tbody>
     </table>
     <br><br>
     <br><br>
diff --git a/templates/static/node.html.j2 b/templates/static/node.html.j2
index ce944bf..a7bf562 100644
--- a/templates/static/node.html.j2
+++ b/templates/static/node.html.j2
@@ -7,8 +7,10 @@
 {% endblock %}
 
 {% block content %}
-  <h5 class="text-gray-500">Node Detail</h5>
-  <h1>{{ node.shortname }} - {{ node.longname }}</h1>
+  <h5 class="mb-2 text-gray-500">
+    <a href="nodes.html">Nodes</a> &gt; {{ node.shortname }}
+  </h5>
+  <h1 class="mb-2 text-xl">{{ node.shortname }} - {{ node.longname }}</h1>
 
   <div class="flex align-middle mb-4">
   {% if node.telemetry %}
diff --git a/templates/static/telemetry.html.j2 b/templates/static/telemetry.html.j2
index 13a0612..152314c 100644
--- a/templates/static/telemetry.html.j2
+++ b/templates/static/telemetry.html.j2
@@ -3,11 +3,12 @@
 {% block title %}Telemetry{% endblock %}
 
 {% block content %}
-    <h1>Telemetry</h1>
-    <p>
-      Telemetry as seen by <b>KE-R</b> (!4355f528).
+    <h5 class="mb-2 text-gray-500">Telemetry</h5>
+    <h1 class="mb-2 text-xl">Telemetry</h1>
+    <p class="mb-2">
+      Telemetry as seen by <b>{{ config['server']['node_id'] }}</b> ({{ config['server']['node_id'] }}).
     </p>
-    <p>Last updated: {{ timestamp.astimezone(zoneinfo).strftime('%Y-%m-%d %H:%M:%S %z') }}</p>
+
     <table border=1 cellpadding=4>
       <tr>
         <td>Timestamp</td>
diff --git a/templates/static/traceroutes.html.j2 b/templates/static/traceroutes.html.j2
index 740a65e..a6e6b6e 100644
--- a/templates/static/traceroutes.html.j2
+++ b/templates/static/traceroutes.html.j2
@@ -3,11 +3,12 @@
 {% block title %}Traceroutes{% endblock %}
 
 {% block content %}
-    <h1>Traceroutes</h1>
-    <p>
-      Traceroutes as seen by <b>KE-R</b> (!4355f528).
+    <h5 class="mb-2 text-gray-500">Traceroutes</h5>
+    <h1 class="mb-2 text-xl">Traceroutes</h1>
+    <p class="mb-2">
+      Traceroutes as seen by <b>{{ config['server']['node_id'] }}</b> ({{ config['server']['node_id'] }}).
     </p>
-    <p>Last updated: {{ timestamp.astimezone(zoneinfo).strftime('%Y-%m-%d %H:%M:%S %z') }}</p>
+
     <table border=1 cellpadding=4>
       <tr>
         <td>Timestamp</td>

From ac56d6d2af57fc2c5befd7ff43017f8640b297f6 Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Mon, 8 Jul 2024 16:35:55 -0700
Subject: [PATCH 04/20] one more template

---
 templates/static/nodes.html.j2 | 214 +++++++++++++++++----------------
 1 file changed, 111 insertions(+), 103 deletions(-)

diff --git a/templates/static/nodes.html.j2 b/templates/static/nodes.html.j2
index f043949..308408c 100644
--- a/templates/static/nodes.html.j2
+++ b/templates/static/nodes.html.j2
@@ -3,122 +3,130 @@
 {% block title %}Nodes{% endblock %}
 
 {% block content %}
-    <h1>Nodes</h1>
-    <p>
+    <h5 class="mb-2 text-gray-500">Nodes</h5>
+    <h1 class="mb-2 text-xl">Nodes</h1>
+    <p class="mb-2">
       There are <b>{{ active_nodes|count }}</b> active out of a total of <b>{{ nodes|count }}</b> seen nodes
-      that have been heard by the mesh by <b>KE-R</b> (!4355f528).
+      that have been heard by the mesh by <b>{{ config['server']['node_id'] }}</b> ({{ config['server']['node_id'] }}).
     </p>
-    <p>Last updated: {{ timestamp.astimezone(zoneinfo).strftime('%Y-%m-%d %H:%M:%S %z') }}</p>
-    <table border=1 cellspacing=2 cellpadding=4>
-        <tr bgcolor=lightgray>
-            <th>ID</th>
-            <th colspan=2>Name</th>
-            <th>Hardware</th>
-            <th colspan=4>Last Position</th>
-            <th>Neighbors</th>
-            <th colspan=4>Telemetry</th>
-            <th colspan=1>Seen</th>
+    <table class="w-full max-w-full table-auto border-collapse border border-gray-500 bg-gray-50">
+      <thead>
+        <tr>
+          <th class="border border-gray-500 bg-gray-400" colspan=2>ID</th>
+          <th class="border border-gray-500 bg-gray-400" colspan=2>Name</th>
+          <th class="hidden sm:table-cell border border-gray-500 bg-gray-400">Hardware</th>
+          <th class="hidden xl:table-cell border border-gray-500 bg-gray-400" colspan=4>Last Position</th>
+          <th class="hidden md:table-cell border border-gray-500 bg-gray-400">Neighbors</th>
+          <th class="hidden 2xl:table-cell border border-gray-500 bg-gray-400" colspan=4>Telemetry</th>
+          <th class="border border-gray-500 bg-gray-400" colspan=1>Seen</th>
         </tr>
-        <tr bgcolor=lightgray>
-            <th></th>
-            <th>Short</th>
-            <th>Long</th>
-            <th></th>
-            <th>Altitude</th>
-            <th>Latitude</th>
-            <th>Longitude</th>
-            <th>DX</th>
-            <th>Count</th>
-            <th>
-              <img src="images/icons/battery.svg" class="w-4 h-4 inline-block" alt="Battery" title="Battery">
-            </th>
-            <th>
-              <img src="images/icons/voltage.svg" class="w-4 h-4 inline-block" alt="Voltage" title="Voltage">
-            </th>
-            <th>
-              <img src="images/icons/up.svg" alt="Air Util TX" class="w-4 h-4 inline-block" title="Air Util TX">
-            </th>
-            <th>
-              <img src="images/icons/down.svg" class="w-4 h-4 inline-block" alt="Channel Util" title="Channel Util">
-            </th>
-            <th>Since</th>
+        <tr>
+          <th class="w-8 min-w-8 max-w-8 h-8 min-h-8 max-h-8 border border-gray-500 bg-gray-400"></th>
+          <th class="border border-gray-500 bg-gray-400"></th>
+          <th class="border border-gray-500 bg-gray-400">Short</th>
+          <th class="border border-gray-500 bg-gray-400">Long</th>
+          <th class="hidden sm:table-cell border border-gray-500 bg-gray-400"></th>
+          <th class="hidden xl:table-cell border border-gray-500 bg-gray-400">Altitude</th>
+          <th class="hidden xl:table-cell border border-gray-500 bg-gray-400">Latitude</th>
+          <th class="hidden xl:table-cell border border-gray-500 bg-gray-400">Longitude</th>
+          <th class="hidden xl:table-cell border border-gray-500 bg-gray-400">DX</th>
+          <th class="hidden md:table-cell border border-gray-500 bg-gray-400">Count</th>
+          <th class="hidden 2xl:table-cell border border-gray-500 bg-gray-400">
+            <img src="images/icons/battery.svg" class="w-4 h-4 inline-block" alt="Battery" title="Battery">
+          </th>
+          <th class="hidden 2xl:table-cell border border-gray-500 bg-gray-400">
+            <img src="images/icons/voltage.svg" class="w-4 h-4 inline-block" alt="Voltage" title="Voltage">
+          </th>
+          <th class="hidden 2xl:table-cell border border-gray-500 bg-gray-400">
+            <img src="images/icons/up.svg" alt="Air Util TX" class="w-4 h-4 inline-block" title="Air Util TX">
+          </th>
+          <th class="hidden 2xl:table-cell border border-gray-500 bg-gray-400">
+            <img src="images/icons/down.svg" class="w-4 h-4 inline-block" alt="Channel Util" title="Channel Util">
+          </th>
+          <th class="border border-gray-500 bg-gray-400">Since</th>
         </tr>
+      </thead>
+      <tbody>
         {% for id, node in active_nodes.items() %}
         <tr>
-            <td>
-              {% if id %}
-              {% set id = id|replace('!', '') %}
-                <a href='node_{{ id }}.html'>{{ id }}</a>
-              {% else %}
-                {{ id }}
-              {% endif %}
-            </td>
-            <td style="color: #{{ '777' if node.shortname == 'UNK' else '000' }}">
-              {% if id %}
-              {% set id = id|replace('!', '') %}
-                <a href='node_{{ id }}.html'>{{ node.shortname }}</a>
-              {% else %}
-                {{ node.shortname }}
-              {% endif %}
-            </td>
-            <td style="color: #{{ '777' if node.shortname == 'UNK' else '000' }}">
-              {{ node.longname }}
-            </td>
-            <td>
-              {% if node.hardware %}
-              {{ hardware(node.hardware).name }}
-              {% endif %}
-            </td>
-            {% if node.position %}
-            <td align=right>
-              {% if node.position.altitude %}
-              {{ node.position.altitude }} m
-              {% endif %}
-            </td>
-            <td>{{ node.position.latitude }}</td>
-            <td>{{ node.position.longitude }}</td>
-            <td align=right>{{ node.distance_from_host_node }} km</td>
+          <td class="w-8 min-w-8 max-w-8 h-8 min-h-8 max-h-8 p-0 box-border border border-gray-400" align=center valign=middle>
+            <img src="https://api.dicebear.com/9.x/bottts-neutral/svg?seed={{ id }}" alt="Avatar" class="w-8 h-8 object-cover">
+          </td>
+          <td class="p-1 border border-gray-400">
+            {% if id %}
+            {% set id = id|replace('!', '') %}
+              <a href='node_{{ id }}.html'>{{ id }}</a>
             {% else %}
-            <td></td>
-            <td></td>
-            <td></td>
-            <td></td>
+              {{ id }}
             {% endif %}
-            {% if node.neighborinfo %}
-            <td align=center>{{ node.neighborinfo.neighbors_count }}</td>
+          </td>
+          <td class="p-1 border border-gray-400" style="color: #{{ '777' if node.shortname == 'UNK' else '000' }}">
+            {% if id %}
+            {% set id = id|replace('!', '') %}
+              <a href='node_{{ id }}.html'>{{ node.shortname }}</a>
             {% else %}
-            <td></td>
+              {{ node.shortname }}
             {% endif %}
-            {% if node.telemetry %}
-            <td align=right>
-              {% if 'battery_level' in node.telemetry %}
-              {{ node.telemetry.battery_level }}%
-              {% endif %}
-            </td>
-            <td align=right>
-              {% if 'voltage' in node.telemetry %}
-              {{ node.telemetry.voltage|round(2) }}V
-              {% endif %}
-            </td>
-            <td align=right>
-              {% if 'air_util_tx' in node.telemetry %}
-              {{ node.telemetry.air_util_tx|round(1) }}%
-              {% endif %}
-            </td>
-            <td align=right>
-              {% if 'channel_utilization' in node.telemetry %}
-              {{ node.telemetry.channel_utilization|round(1) }}%
-              {% endif %}
-            </td>
-            {% else %}
-            <td></td>
-            <td></td>
-            <td></td>
-            <td></td>
+          </td>
+          <td class="p-1 border border-gray-400 text-nowrap truncate" style="color: #{{ '777' if node.shortname == 'UNK' else '000' }}">
+            {{ node.longname }}
+          </td>
+          <td class="hidden sm:table-cell p-1 border border-gray-400 text-nowrap truncate">
+            {% if node.hardware %}
+            {{ hardware(node.hardware).name }}
+            {% endif %}
+          </td>
+          {% if node.position %}
+          <td class="hidden xl:table-cell p-1 border border-gray-400" align=right>
+            {% if node.position.altitude %}
+            {{ node.position.altitude }} m
+            {% endif %}
+          </td>
+          <td class="hidden xl:table-cell p-1 border border-gray-400">{{ node.position.latitude }}</td>
+          <td class="hidden xl:table-cell p-1 border border-gray-400">{{ node.position.longitude }}</td>
+          <td class="hidden xl:table-cell p-1 border border-gray-400 text-nowrap" align=right>{{ node.distance_from_host_node }} km</td>
+          {% else %}
+          <td class="hidden xl:table-cell p-1 border border-gray-400"></td>
+          <td class="hidden xl:table-cell p-1 border border-gray-400"></td>
+          <td class="hidden xl:table-cell p-1 border border-gray-400"></td>
+          <td class="hidden xl:table-cell p-1 border border-gray-400"></td>
+          {% endif %}
+          {% if node.neighborinfo %}
+          <td class="hidden md:table-cell p-1 border border-gray-400" align=center>{{ node.neighborinfo.neighbors_count }}</td>
+          {% else %}
+          <td class="hidden md:table-cell p-1 border border-gray-400"></td>
+          {% endif %}
+          {% if node.telemetry %}
+          <td class="hidden 2xl:table-cell p-1 border border-gray-400" align=right>
+            {% if 'battery_level' in node.telemetry %}
+            {{ node.telemetry.battery_level }}%
+            {% endif %}
+          </td>
+          <td class="hidden 2xl:table-cell p-1 border border-gray-400" align=right>
+            {% if 'voltage' in node.telemetry %}
+            {{ node.telemetry.voltage|round(2) }}V
+            {% endif %}
+          </td>
+          <td class="hidden 2xl:table-cell p-1 border border-gray-400" align=right>
+            {% if 'air_util_tx' in node.telemetry %}
+            {{ node.telemetry.air_util_tx|round(1) }}%
+            {% endif %}
+          </td>
+          <td class="hidden 2xl:table-cell p-1 border border-gray-400" align=right>
+            {% if 'channel_utilization' in node.telemetry %}
+            {{ node.telemetry.channel_utilization|round(1) }}%
             {% endif %}
-            <td align=right title="{{ node.last_seen_human }}">{{ node.since.seconds }} secs</td>
+          </td>
+          {% else %}
+          <td class="hidden 2xl:table-cell p-1 border border-gray-400"></td>
+          <td class="hidden 2xl:table-cell p-1 border border-gray-400"></td>
+          <td class="hidden 2xl:table-cell p-1 border border-gray-400"></td>
+          <td class="hidden 2xl:table-cell p-1 border border-gray-400"></td>
+          {% endif %}
+          <td class="p-1 border border-gray-400 text-nowrap" align=right title="{{ node.last_seen_human }}">{{ node.since.seconds }} secs</td>
         </tr>
         {% endfor %}
+      </tbody>
     </table>
     <br><br>
     <br><br>

From af0edc5101bf755d7c570da0c8342d176fbd099c Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:40:05 -0700
Subject: [PATCH 05/20] add a Caddyfile for dev

---
 Caddyfile.dev | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 Caddyfile.dev

diff --git a/Caddyfile.dev b/Caddyfile.dev
new file mode 100644
index 0000000..2df9580
--- /dev/null
+++ b/Caddyfile.dev
@@ -0,0 +1,12 @@
+{
+	debug
+}
+
+localhost {
+	root * /srv
+	encode gzip
+	file_server
+	log {
+		output file /var/log/caddy.log
+	}
+}

From b44ced561d11df06ee3123fca5eeffee9f36b0de Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:40:15 -0700
Subject: [PATCH 06/20] add a docker-compose file for dev

---
 docker-compose-dev.yml | 52 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 docker-compose-dev.yml

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
new file mode 100644
index 0000000..d4b79cc
--- /dev/null
+++ b/docker-compose-dev.yml
@@ -0,0 +1,52 @@
+services:
+  caddy:
+    image: caddy:latest
+    ports:
+      - 80:80
+      - 443:443
+    volumes:
+      - ./caddy/data:/data/caddy
+      - ./Caddyfile.dev:/etc/caddy/Caddyfile
+      - ./output/static-html:/srv
+      - ./public/images:/srv/images
+    environment:
+      - CADDY_AGREE=true
+    restart: always
+
+  postgres:
+    image: postgres:latest
+    volumes:
+      - ./postgres/data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=password
+    restart: always
+
+  mqtt:
+    container_name: mqtt
+    image: eclipse-mosquitto:latest
+    ports:
+      - 1883:1883
+    volumes:
+      - ./mosquitto/data:/mosquitto/data:rw
+      - ./mosquitto/config:/mosquitto/config:rw
+    restart: always
+
+  meshinfo:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ./config.json:/app/config.json
+      - ./output:/app/output
+      - ./templates:/app/templates
+    environment:
+      - PYTHONUNBUFFERED=1
+      - MQTT_HOST=mqtt
+      - MQTT_PORT=1883
+      - MQTT_USERNAME=meshinfo
+      - MQTT_PASSWORD=m3sht4st1c
+    restart: always
+    depends_on:
+      - caddy
+      - postgres
+      - mqtt

From 9de421b5466c0aad3d6d1937cc144dd5f6cffd00 Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:40:24 -0700
Subject: [PATCH 07/20] moar icons

---
 public/images/icons/current.svg           |  9 +++++++++
 public/images/icons/pressure.svg          |  7 +++++++
 public/images/icons/relative-humidity.svg | 11 +++++++++++
 public/images/icons/resistance.svg        |  8 ++++++++
 4 files changed, 35 insertions(+)
 create mode 100644 public/images/icons/current.svg
 create mode 100644 public/images/icons/pressure.svg
 create mode 100644 public/images/icons/relative-humidity.svg
 create mode 100644 public/images/icons/resistance.svg

diff --git a/public/images/icons/current.svg b/public/images/icons/current.svg
new file mode 100644
index 0000000..bcad84c
--- /dev/null
+++ b/public/images/icons/current.svg
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="1200pt" height="1200pt" version="1.1" viewBox="0 0 1200 1200" xmlns="http://www.w3.org/2000/svg">
+ <g>
+  <path d="m1042.8 758.4-96-69.602c-8.3984-6-19.199-7.1992-28.801-2.3984-9.6016 4.8008-15.602 14.398-15.602 25.199v42l-229.2 0.003906c-15.602 0-27.602 12-27.602 27.602s12 27.602 27.602 27.602h229.2v42c0 10.801 6 20.398 15.602 25.199 3.6016 2.3984 8.3984 3.6016 12 3.6016 6 0 12-1.1992 16.801-4.8008l96-69.602c7.1992-4.8008 12-13.199 12-22.801 0-9.6016-4.8008-18.004-12-24.004z"/>
+  <path d="m542.4 758.4-96-69.602c-8.3984-6-19.199-7.1992-28.801-2.3984-9.6016 4.8008-15.602 14.398-15.602 25.199v42l-228 0.003906c-15.602 0-27.602 12-27.602 27.602s12 27.602 27.602 27.602h229.2v42c0 10.801 6 20.398 15.602 25.199 3.6016 2.3984 8.3984 3.6016 12 3.6016 6 0 12-1.1992 16.801-4.8008l96-69.602c7.1992-4.8008 12-13.199 12-22.801 0-9.6016-6-18.004-13.203-24.004z"/>
+  <path d="m1144.8 544.8c-10.801-2.3984-19.199-8.3984-30-14.398-19.199-12-43.199-26.398-87.602-26.398-44.398 0-68.398 14.398-87.602 26.398-15.602 9.6016-27.602 16.801-54 16.801-26.398 0-38.398-7.1992-54-16.801-19.199-12-43.199-26.398-87.602-26.398-44.398 0-68.398 14.398-87.602 26.398-16.801 9.6016-27.602 16.801-54 16.801-26.398 0-38.398-7.1992-54-16.801-19.199-12-43.199-26.398-87.602-26.398-44.398 0-68.398 14.398-87.602 26.398-15.602 9.6016-27.602 16.801-54 16.801-26.398 0-38.398-7.1992-54-16.801-19.199-12-43.199-26.398-87.602-26.398-44.398 0-68.398 14.398-87.602 26.398-9.6016 6-18 10.801-28.801 13.199-12 3.6016-22.801 13.199-25.199 25.199-4.8008 24 16.801 43.199 39.602 37.199 21.602-4.8008 36-14.398 48-21.602 15.602-9.6016 27.602-16.801 54-16.801 26.398 0 38.398 7.1992 54 16.801 19.199 12 43.199 26.398 87.602 26.398 44.398 0 68.398-14.398 87.602-26.398 15.602-9.6016 27.602-16.801 54-16.801 26.398 0 38.398 7.1992 54 16.801 19.199 12 43.199 26.398 87.602 26.398 44.398 0 68.398-14.398 87.602-26.398 15.602-9.6016 27.602-16.801 54-16.801 26.398 0 38.398 7.1992 54 16.801 19.199 12 43.199 26.398 87.602 26.398 44.398 0 68.398-14.398 87.602-26.398 15.602-9.6016 27.602-16.801 54-16.801 26.398 0 38.398 7.1992 54 16.801 12 7.1992 26.398 15.602 46.801 21.602 20.398 4.8008 40.801-9.6016 40.801-31.199 1.1992-13.199-9.6016-26.402-24-30z"/>
+  <path d="m69.602 424.8c21.602-4.8008 36-14.398 48-21.602 15.602-9.6016 27.602-16.801 54-16.801 26.398 0 38.398 7.1992 54 16.801 19.199 12 43.199 26.398 87.602 26.398 44.398 0 68.398-14.398 87.602-26.398 15.602-9.6016 27.602-16.801 54-16.801 26.398 0 38.398 7.1992 54 16.801 19.199 12 43.199 26.398 87.602 26.398 44.398 0 68.398-14.398 87.602-26.398 15.602-9.6016 27.602-16.801 54-16.801 26.398 0 38.398 7.1992 54 16.801 19.199 12 43.199 26.398 87.602 26.398 44.398 0 68.398-14.398 87.602-26.398 15.602-9.6016 27.602-16.801 54-16.801 26.398 0 38.398 7.1992 54 16.801 12 7.1992 26.398 15.602 46.801 21.602 20.398 4.8008 40.801-9.6016 40.801-31.199 0-14.398-10.801-27.602-25.199-31.199-10.801-2.3984-19.199-8.3984-30-14.398-19.199-12-43.199-26.398-87.602-26.398-44.398 0-68.398 14.398-87.602 26.398-15.602 9.6016-27.602 16.801-54 16.801-26.398 0-38.398-7.1992-54-16.801-19.199-12-43.199-26.398-87.602-26.398-44.398 0-68.398 14.398-87.602 26.398-16.801 9.6016-27.602 16.801-54 16.801-26.398 0-38.398-7.1992-54-16.801-19.199-12-43.199-26.398-87.602-26.398-44.398 0-68.398 14.398-87.602 26.398-15.602 9.6016-27.602 16.801-54 16.801-26.398 0-38.398-7.1992-54-16.801-19.199-12-43.199-26.398-87.602-26.398-42.004-0.003907-67.203 14.395-86.402 26.395-9.6016 6-18 10.801-28.801 13.199-12 3.6016-22.801 13.199-25.199 25.199-3.6016 25.203 18 44.402 39.602 38.402z"/>
+ </g>
+</svg>
diff --git a/public/images/icons/pressure.svg b/public/images/icons/pressure.svg
new file mode 100644
index 0000000..98f8a78
--- /dev/null
+++ b/public/images/icons/pressure.svg
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="1200pt" height="1200pt" version="1.1" viewBox="0 0 1200 1200" xmlns="http://www.w3.org/2000/svg">
+ <g>
+  <path d="m993.75 450c0-217.5-176.25-393.75-393.75-393.75s-393.75 176.25-393.75 393.75c0 150 84.375 279.38 206.25 346.88v84.375c0 31.875 24.375 56.25 56.25 56.25h37.5v187.5c0 11.25 7.5 18.75 18.75 18.75h150c11.25 0 18.75-7.5 18.75-18.75v-187.5h37.5c31.875 0 56.25-24.375 56.25-56.25v-84.375c121.88-67.5 206.25-196.88 206.25-346.88zm-243.75 431.25c0 11.25-7.5 18.75-18.75 18.75h-262.5c-11.25 0-18.75-7.5-18.75-18.75v-67.5c46.875 18.75 97.5 30 150 30s103.12-11.25 150-30zm-150-75c-196.88 0-356.25-159.38-356.25-356.25s159.38-356.25 356.25-356.25 356.25 159.38 356.25 356.25-159.38 356.25-356.25 356.25z"/>
+  <path d="m600 131.25c-176.25 0-318.75 142.5-318.75 318.75s142.5 318.75 318.75 318.75 318.75-142.5 318.75-318.75-142.5-318.75-318.75-318.75zm183.75 275.62-127.5 144.38c11.25 13.125 18.75 30 18.75 48.75 0 41.25-33.75 75-75 75s-75-33.75-75-75 33.75-75 75-75c9.375 0 16.875 1.875 24.375 3.75l129.38-148.12c7.5-7.5 18.75-9.375 26.25-1.875 9.375 7.5 9.375 18.75 3.75 28.125zm41.25 61.875c-11.25 0-18.75-7.5-18.75-18.75 0-114.38-91.875-206.25-206.25-206.25s-206.25 91.875-206.25 206.25c0 11.25-7.5 18.75-18.75 18.75s-18.75-7.5-18.75-18.75c0-135 108.75-243.75 243.75-243.75s243.75 108.75 243.75 243.75c0 11.25-7.5 18.75-18.75 18.75z"/>
+ </g>
+</svg>
diff --git a/public/images/icons/relative-humidity.svg b/public/images/icons/relative-humidity.svg
new file mode 100644
index 0000000..7e2bab8
--- /dev/null
+++ b/public/images/icons/relative-humidity.svg
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="1200pt" height="1200pt" version="1.1" viewBox="0 0 1200 1200" xmlns="http://www.w3.org/2000/svg">
+ <g>
+  <path d="m262.5 579.38-161.02 260.16c-17.344 27.891-26.484 60-26.484 93.047v11.25c0 99.844 84.141 181.17 187.5 181.17s187.5-81.328 187.5-181.17v-11.25c0-32.812-9.1406-65.156-26.484-93.047zm136.88 414.14c-16.172 42.891-49.922 76.641-92.812 92.812l-13.125-35.156c62.578-23.438 94.453-93.281 70.781-155.86-1.875-5.1562-4.2188-10.078-6.7969-15l-55.078-102.66 33.047-17.812 55.312 102.66c21.562 40.312 24.844 88.125 8.6719 131.02z"/>
+  <path d="m806.25 504.84v-12.656c0-37.266-10.078-73.828-29.297-105.7l-176.95-293.91-176.95 293.91c-19.219 31.875-29.297 68.438-29.297 105.7v12.656c0 113.2 92.578 205.55 206.25 205.55 113.67-0.23438 206.25-92.344 206.25-205.55zm-127.27-131.72-37.031-44.531 28.828-23.906 37.031 44.531c74.766 89.766 62.812 223.36-27.188 298.12-10.547 8.9062-22.031 16.641-34.219 23.203l-18.047-32.812c84.375-46.172 115.55-151.88 69.609-236.48-5.3906-10.078-11.719-19.453-18.984-28.125z"/>
+  <path d="m777.89 837.42c-16.875-1.6406-33.516 4.2188-45.469 16.172l-62.812 62.812c-7.2656 7.2656-19.219 7.2656-26.719 0-22.031-22.031-57.656-22.031-79.688 0-1.4062 1.4062-2.8125 3.0469-4.2188 4.6875l-48.75 60.938 29.297 23.438 48.75-60.938c6.5625-8.2031 18.281-9.375 26.484-3.0469 0.46875 0.46875 1.1719 0.9375 1.6406 1.4062 22.031 22.031 57.656 22.031 79.688 0l61.641-61.641c4.4531-14.766 10.781-28.828 18.984-41.953z"/>
+  <path d="m1102.5 846.33 53.203-44.531-23.906-28.828-48.984 41.016 15.703 25.547c1.4062 2.1094 2.5781 4.4531 3.9844 6.7969z"/>
+  <path d="m828.98 865.55c7.5 5.625 18.047 4.9219 24.609-1.6406l46.875-46.875c20.391-20.391 52.5-22.031 75-4.2188l43.594 34.922c7.0312 5.625 16.875 5.3906 23.906-0.23438l39.844-33.281-145.31-234.84-159.61 258.05c10.312 0.9375 19.922 4.6875 28.125 10.781z"/>
+  <path d="m1066.9 876.09c-20.625 17.109-50.391 17.344-71.25 0.70312l-43.594-34.922c-7.5-6.0938-18.281-5.3906-25.078 1.4062l-46.875 46.875c-19.922 19.688-51.094 22.031-73.594 5.1562l-22.969-17.109c-7.5-5.625-18.047-4.9219-24.609 1.6406l-1.4062 1.4062c-5.1562 16.641-7.7344 33.75-7.7344 51.094v11.25c0 99.844 84.141 181.17 187.5 181.17s187.5-81.328 187.5-181.17v-11.25c0-30.234-7.7344-59.766-22.5-86.25zm-3.9844 141.8c-18.516 31.406-47.109 55.781-81.328 68.438l-13.125-35.156c53.203-19.688 85.547-73.828 77.578-129.84l37.266-5.1562c4.4531 35.156-2.5781 71.016-20.391 101.72z"/>
+ </g>
+</svg>
diff --git a/public/images/icons/resistance.svg b/public/images/icons/resistance.svg
new file mode 100644
index 0000000..6a0d437
--- /dev/null
+++ b/public/images/icons/resistance.svg
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="1200pt" height="1200pt" version="1.1" viewBox="0 0 1200 1200" xmlns="http://www.w3.org/2000/svg">
+ <g fill-rule="evenodd">
+  <path d="m175 487.5c62.102 0 112.5 50.398 112.5 112.5s-50.398 112.5-112.5 112.5-112.5-50.398-112.5-112.5 50.398-112.5 112.5-112.5zm0 75c-20.699 0-37.5 16.801-37.5 37.5s16.801 37.5 37.5 37.5 37.5-16.801 37.5-37.5-16.801-37.5-37.5-37.5z"/>
+  <path d="m1025 487.5c62.102 0 112.5 50.398 112.5 112.5s-50.398 112.5-112.5 112.5-112.5-50.398-112.5-112.5 50.398-112.5 112.5-112.5zm0 75c-20.699 0-37.5 16.801-37.5 37.5s16.801 37.5 37.5 37.5 37.5-16.801 37.5-37.5-16.801-37.5-37.5-37.5z"/>
+  <path d="m950 562.5c20.699 0 37.5 16.801 37.5 37.5s-16.801 37.5-37.5 37.5h-16.898l-20.898 167.15c-2.25 17.898-16.898 31.699-34.949 32.801-18 1.0508-34.25-10.852-38.648-28.352l-63.602-254.5-63.602 254.5c-4.1992 16.699-19.199 28.398-36.398 28.398-17.199 0-32.199-11.699-36.398-28.398l-63.602-254.5-63.602 254.5c-4.1992 16.699-19.199 28.398-36.398 28.398-17.199 0-32.199-11.699-36.398-28.398l-63.602-254.5-13.602 54.5c-4.1992 16.699-19.199 28.398-36.398 28.398h-75c-20.699 0-37.5-16.801-37.5-37.5 0-20.699 16.801-37.5 37.5-37.5h45.699l42.898-171.6c4.1992-16.699 19.199-28.398 36.398-28.398s32.199 11.699 36.398 28.398l63.602 254.5 63.602-254.5c4.1992-16.699 19.199-28.398 36.398-28.398s32.199 11.699 36.398 28.398l63.602 254.5 63.602-254.5c4.1992-16.699 19.199-28.398 36.398-28.398s32.199 11.699 36.398 28.398l51.301 205.25 0.10156-0.80078c2.3516-18.75 18.301-32.852 37.199-32.852h50z"/>
+ </g>
+</svg>

From 16df64c0942930b7d86b24a7b0b60af183b38140 Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:40:56 -0700
Subject: [PATCH 08/20] add caddy/ path to gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 00aeecf..8ecc2bb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,7 @@ node_modules/
 .env
 
 backups/*
+caddy
 output/*
 mosquitto/data/*
 postgres/data/*

From 7f9731c42e244883c59c1bc5f59a7bd378565fb5 Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:41:04 -0700
Subject: [PATCH 09/20] update encoders

---
 encoders.py | 60 ++++++++++++++++++++++++++---------------------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/encoders.py b/encoders.py
index c25892f..42449f1 100644
--- a/encoders.py
+++ b/encoders.py
@@ -4,36 +4,36 @@
 import json
 
 class _JSONEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, (datetime.date, datetime.datetime)):
-            return obj.astimezone().isoformat()
-        if isinstance(obj, str):
-            return obj.replace('!', '')
-        return obj
+  def default(self, obj):
+    if isinstance(obj, datetime.datetime):
+      return obj.astimezone().isoformat()
+    if isinstance(obj, datetime.timedelta):
+      return None
+    return obj
 
 class _JSONDecoder(json.JSONDecoder):
-    def __init__(self, *args, **kwargs):
-        json.JSONDecoder.__init__(
-            self, object_hook=self.object_hook, *args, **kwargs)
+  def __init__(self, *args, **kwargs):
+    json.JSONDecoder.__init__(
+      self, object_hook=self.object_hook, *args, **kwargs)
 
-    def object_hook(self, obj):
-        ret = {}
-        for key, value in obj.items():
-            if key in {'last_seen'}:
-                ret[key] = datetime.datetime.fromisoformat(value)
-            elif key in {'id'}:
-                if isinstance(value, str):
-                    ret[key] = value.replace('!', '')
-                else:
-                    ret[key] = value
-            elif key in {'sender'}:
-                if isinstance(value, str):
-                    ret[key] = value.replace('!', '')
-                else:
-                    ret[key] = value
-            else:
-                if isinstance(value, str):
-                    ret[key] = value.replace('!', '')
-                else:
-                    ret[key] = value
-        return ret
+  def object_hook(self, obj):
+    ret = {}
+    for key, value in obj.items():
+      if key in {'last_seen', 'last_geocoding'}:
+        ret[key] = datetime.datetime.fromisoformat(value)
+      elif key in {'id'}:
+        if isinstance(value, str):
+          ret[key] = value.replace('!', '')
+        else:
+          ret[key] = value
+      elif key in {'sender'}:
+        if isinstance(value, str):
+          ret[key] = value.replace('!', '')
+        else:
+          ret[key] = value
+      else:
+        if isinstance(value, str):
+          ret[key] = value.replace('!', '')
+        else:
+          ret[key] = value
+    return ret

From 8e7486956dc15e1a3092204a5615028f2611bdd4 Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:41:11 -0700
Subject: [PATCH 10/20] update README

---
 README.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/README.md b/README.md
index ab5e78b..cbfe7e7 100644
--- a/README.md
+++ b/README.md
@@ -106,6 +106,15 @@ Build. Be sure to specify a related version number and suffix (this example `dev
 scripts/docker-build.sh 0.0.1dev5
 ```
 
+### Running via Docker Compose while developing
+
+```sh
+docker compose -f docker-compose-dev.yml up --build --force-recreate
+```
+
+You will need to CTRL-C and run again if you make any changes to the python code, but not if you only make changes to
+the templates.
+
 ### Release
 
 Tag the release using git and push up the tag. The image will be build by GitHub automatically (see: https://github.com/MeshAddicts/meshinfo/actions/workflows/docker.yml).

From 391d7b9c59fe37569c24c307ed7aa936f2882a53 Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:41:22 -0700
Subject: [PATCH 11/20] update the config sample for new stuffs

---
 config.json.sample | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/config.json.sample b/config.json.sample
index a2ce89e..398857e 100644
--- a/config.json.sample
+++ b/config.json.sample
@@ -19,6 +19,7 @@
     "timezone": "America/Los_Angeles",
     "start_time": null,
     "intervals": {
+      "enrich": 60,
       "data_save": 60,
       "render": 5
     }
@@ -47,6 +48,14 @@
       "token": "token",
       "channel": "1247618108810596392",
       "webhook": "webhook"
+    },
+    "geocoding": {
+      "enabled": false,
+      "provider": "geocode.maps.co",
+      "geocode.maps.co": {
+        "api_key": "XXXXXXXXXXXXXXXX"
+      }
     }
-  }
+  },
+  "debug": false
 }

From b8e8aff678f51c4bd80b9ccef4989721c55ddf48 Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:41:45 -0700
Subject: [PATCH 12/20] add since to Node

---
 models/node.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/models/node.py b/models/node.py
index fabf4b0..d7cfb11 100644
--- a/models/node.py
+++ b/models/node.py
@@ -14,6 +14,7 @@ def default_node(id: str):
         'position': None,
         'telemetry': None,
         'active': False,
+        'since': datetime.datetime.now(datetime.timezone.utc).astimezone() - datetime.datetime.now(datetime.timezone.utc).astimezone(),
         'last_seen': datetime.datetime.now(datetime.timezone.utc).astimezone().isoformat()
       }
 
@@ -26,5 +27,6 @@ def default_node(id: str):
       'position': None,
       'telemetry': None,
       'active': True,
+      'since': datetime.datetime.now(datetime.timezone.utc).astimezone() - datetime.datetime.now(datetime.timezone.utc).astimezone(),
       'last_seen': datetime.datetime.now(datetime.timezone.utc).astimezone().isoformat()
     }

From 58bc3a8d939348c3903f1da04aa8bddb77becd2b Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:42:03 -0700
Subject: [PATCH 13/20] improve the data renderer

---
 data_renderer.py | 28 +++++++++++-----------------
 1 file changed, 11 insertions(+), 17 deletions(-)

diff --git a/data_renderer.py b/data_renderer.py
index 0d160bb..11b2895 100644
--- a/data_renderer.py
+++ b/data_renderer.py
@@ -1,40 +1,34 @@
 #!/usr/bin/env python3
 
-import datetime
 import json
-from zoneinfo import ZoneInfo
-from jinja2 import Environment, FileSystemLoader
 
 from encoders import _JSONEncoder
 
 class DataRenderer:
-  def __init__(self, config, nodes, chat, telemetry, traceroutes):
+  def __init__(self, config, data):
     self.config = config
-    self.nodes = nodes
-    self.chat = chat
-    self.telemetry = telemetry
-    self.traceroutes = traceroutes
+    self.data = data
 
   def render(self):
-    self.save_file(self.chat, "chat.json")
-    print(f"Saved {len(self.chat['channels']['0']['messages'])} chat messages to file ({self.config['paths']['data']}/chat.json)")
+    self.save_file("chat.json", self.data.chat)
+    print(f"Saved {len(self.data.chat['channels']['0']['messages'])} chat messages to file ({self.config['paths']['data']}/chat.json)")
 
-    old_nodes = nodes
     nodes = {}
-    for id, node in old_nodes.items():
+    for id, node in self.data.nodes.items():
         if id.startswith('!'):
             id = id.replace('!', '')
         nodes[id] = node
 
-    self.save_file(nodes, "nodes.json")
+    self.save_file("nodes.json", self.data.nodes)
     print(f"Saved {len(nodes)} nodes to file ({self.config['paths']['data']}/nodes.json)")
 
-    self.save_file(self.telemetry, "telemetry.json")
-    print(f"Saved {len(self.telemetry)} telemetry to file ({self.config['paths']['data']}/telemetry.json)")
+    self.save_file("telemetry.json", self.data.telemetry)
+    print(f"Saved {len(self.data.telemetry)} telemetry to file ({self.config['paths']['data']}/telemetry.json)")
 
-    self.save_file(self.traceroutes, "traceroutes.json")
-    print(f"Saved {len(self.traceroutes)} traceroutes to file ({self.config['paths']['data']}/traceroutes.json)")
+    self.save_file("traceroutes.json", self.data.traceroutes)
+    print(f"Saved {len(self.data.traceroutes)} traceroutes to file ({self.config['paths']['data']}/traceroutes.json)")
 
   def save_file(self, filename, data):
+    print(f"Saving {filename}")
     with open(f"{self.config['paths']['data']}/{filename}", "w", encoding='utf-8') as f:
       json.dump(data, f, indent=2, sort_keys=True, cls=_JSONEncoder)

From 57aaef3af1ec8859f99b394ca2e5c9ae19b529c3 Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:42:20 -0700
Subject: [PATCH 14/20] add geocding to utils

---
 utils.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/utils.py b/utils.py
index fc1c9ac..5f8b3d0 100644
--- a/utils.py
+++ b/utils.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import requests
 from geo import distance_between_two_points
 
 def calculate_distance_between_nodes(node1, node2):
@@ -7,6 +8,8 @@ def calculate_distance_between_nodes(node1, node2):
     return None
   if node1["position"] is None or node2["position"] is None:
     return None
+  if 'latitude_i' not in node1["position"] or 'longitude_i' not in node1["position"] or 'latitude_i' not in node2["position"] or 'longitude_i' not in node2["position"] or node1["position"]["latitude_i"] is None or node1["position"]["longitude_i"] is None or node2["position"]["latitude_i"] is None or node2["position"]["longitude_i"] is None:
+    return None
   return round(distance_between_two_points(
     node1["position"]["latitude_i"] / 10000000,
     node1["position"]["longitude_i"] / 10000000,
@@ -21,3 +24,14 @@ def convert_node_id_from_hex_to_int(id: str):
   if id.startswith('!'):
       id = id.replace('!', '')
   return int(id, 16)
+
+def geocode_position(api_key: str, latitude: float, longitude: float):
+  if latitude is None or longitude is None:
+    return None
+  print(f"Geocoding {latitude}, {longitude}")
+  url = f"https://geocode.maps.co/reverse?lat={latitude}&lon={longitude}&api_key={api_key}"
+  response = requests.get(url)
+  if response.status_code != 200:
+    return None
+  print(f"Geocoded {latitude}, {longitude} to {response.json()}")
+  return response.json()

From dcb6a5d3e5ad4398259784d1a6997207c25c7ca3 Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:42:44 -0700
Subject: [PATCH 15/20] improve StaticHTMLRenderer for some positional stuff

---
 static_html_renderer.py | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/static_html_renderer.py b/static_html_renderer.py
index 4d04d58..44915b6 100644
--- a/static_html_renderer.py
+++ b/static_html_renderer.py
@@ -29,6 +29,7 @@ def render(self):
     self.render_stats()
     self.render_telemetry()
     self.render_traceroutes()
+    print("Done rendering static HTML files")
 
   def save_file(self, filename, content):
     with open(f"{self.output_path}/{filename}", "w", encoding='utf-8') as f:
@@ -42,7 +43,8 @@ def render_html(self, template_file, **kwargs):
     return html
 
   def render_html_and_save(self, filename, **kwargs):
-    print(f"Rendering {filename}")
+    if self.config['debug']:
+      print(f"Rendering {filename}")
     html = self.render_html(filename, **kwargs)
     self.save_file(filename, html)
 
@@ -152,6 +154,7 @@ def render_nodes_each(self):
         node=node,
         nodes=self.data.nodes,
         hardware=meshtastic.HardwareModel,
+        utils=utils,
         datetime=datetime.datetime,
         zoneinfo=ZoneInfo(self.config['server']['timezone']),
         timestamp=datetime.datetime.now(ZoneInfo(self.config['server']['timezone']))
@@ -235,13 +238,15 @@ def _serialize_node(self, node):
         "since": datetime.datetime.now(ZoneInfo(self.config['server']['timezone'])) - last_seen,
     }
     server_node = self.data.nodes[f'{self.config["server"]["node_id"]}']
-    if server_node and 'position' in server_node and server_node["position"] and server_node["position"]["latitude_i"] != 0 and server_node["position"]["longitude_i"] != 0 and node["position"] and node["position"]["latitude_i"] != 0 and node["position"]["longitude_i"] != 0:
-        serialized["distance_from_host_node"] = round(geo.distance_between_two_points(
-            node["position"]["latitude_i"] / 10000000,
-            node["position"]["longitude_i"] / 10000000,
-            server_node["position"]["latitude_i"] / 10000000,
-            server_node["position"]["longitude_i"] / 10000000
-          ), 2)
+    if server_node and 'position' in server_node and node and 'position' in node:
+      if server_node["position"] and 'latitude_i' in server_node["position"] and 'longitude_i' in server_node["position"] and node["position"] and 'latitude_i' in node["position"] and 'longitude_i' in node["position"]:
+        if server_node["position"]["latitude_i"] != 0 and server_node["position"]["longitude_i"] != 0 and node["position"] and node["position"]["latitude_i"] != 0 and node["position"]["longitude_i"] != 0:
+          serialized["distance_from_host_node"] = round(geo.distance_between_two_points(
+              node["position"]["latitude_i"] / 10000000,
+              node["position"]["longitude_i"] / 10000000,
+              server_node["position"]["latitude_i"] / 10000000,
+              server_node["position"]["longitude_i"] / 10000000
+            ), 2)
     return serialized
 
   def _serialize_neighborinfo(self, node):
@@ -268,7 +273,7 @@ def _serialize_neighborinfo_neighbors(self, node):
       }
       if id in self.data.nodes:
         ni = self.data.nodes[id]
-        if from_node['position'] and ni['position']:
+        if from_node['position'] and ni['position'] and 'latitude_i' in from_node['position'] and 'longitude_i' in from_node['position'] and 'latitude_i' in ni['position'] and 'longitude_i' in ni['position']:
           neighbor["distance"] = round(geo.distance_between_two_points(
             from_node["position"]["latitude_i"] / 10000000,
             from_node["position"]["longitude_i"] / 10000000,
@@ -287,8 +292,15 @@ def _serialize_position(self, position):
     else:
         altitude = None
 
+    if "latitude_i" in position and "longitude_i" in position:
+       latitude = position["latitude_i"] / 10000000
+       longitude = position["longitude_i"] / 10000000
+    else:
+       latitude = None
+       longitude = None
+
     return {
         "altitude": altitude,
-        "latitude": position["latitude_i"] / 10000000,
-        "longitude": position["longitude_i"] / 10000000
+        "latitude": latitude,
+        "longitude": longitude
     }

From 7936e91fe726a259ab3b1201ce0b53604228c49d Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:43:02 -0700
Subject: [PATCH 16/20] significant MemoryDataStore improvements/additions

---
 memory_data_store.py | 185 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 171 insertions(+), 14 deletions(-)

diff --git a/memory_data_store.py b/memory_data_store.py
index 8674d78..cf68106 100644
--- a/memory_data_store.py
+++ b/memory_data_store.py
@@ -1,10 +1,16 @@
 #!/usr/bin/env python3
 
-from datetime import datetime
+from datetime import datetime, timedelta
+import json
+import os
 from zoneinfo import ZoneInfo
+import requests
 
+from data_renderer import DataRenderer
+from encoders import _JSONDecoder
+from models.node import Node
 from static_html_renderer import StaticHTMLRenderer
-
+import utils
 
 class MemoryDataStore:
   def __init__(self, config):
@@ -25,16 +31,117 @@ def __init__(self, config):
     self.traceroutes: list = []
     self.traceroutes_by_node: dict = {}
 
+    self.data_renderer = DataRenderer(config, self)
     self.static_html_renderer = StaticHTMLRenderer(config, self)
 
   def update(self, key, value):
     self.__dict__[key] = value
 
   def update_node(self, id: str, node):
+    if node['position'] is None:
+      node['position'] = {}
+
+    if self.config['integrations']['geocoding']['enabled']:
+      if 'geocoded' not in node['position']:
+        node['position']['geocoded'] = None
+      if 'latitude_i' in node['position'] and 'longitude_i' in node['position'] and node['position']['latitude_i'] is not None and node['position']['longitude_i'] is not None:
+        if node['position']['geocoded'] is None or node['position']['last_geocoding'] is None or node['position']['last_geocoding'] < datetime.now().astimezone(ZoneInfo(self.config['server']['timezone'])) - timedelta(minutes=60):
+          geocoded = utils.geocode_position(self.config['integrations']['geocoding']['geocode.maps.co']['api_key'], node['position']['latitude_i'] / 10000000, node['position']['longitude_i'] / 10000000)
+          if geocoded is not None:
+            node['position']['geocoded'] = geocoded
+            node['position']['last_geocoding'] = datetime.now().astimezone(ZoneInfo(self.config['server']['timezone']))
+
     node['active'] = True
+    if 'last_seen' in node and node['last_seen'] is not None and isinstance(node['last_seen'], str):
+      node['last_seen'] = datetime.fromisoformat(node['last_seen']).astimezone(ZoneInfo(self.config['server']['timezone']))
+    node['since'] = datetime.now().astimezone(ZoneInfo(self.config['server']['timezone'])) - node['last_seen']
     node['last_seen'] = datetime.now().astimezone(ZoneInfo(self.config['server']['timezone']))
     self.nodes[id] = node
 
+  def load(self):
+    try:
+      nodes = self.load_json_file(f"{self.config['paths']['data']}/nodes.json")
+      if nodes is not None:
+        for id, node in nodes.items():
+          if id.startswith('!'):
+            id = id.replace('!', '')
+          if node['active'] is None:
+            node['active'] = False
+          if 'last_seen' not in node:
+            node['last_seen'] = None
+          if 'since' not in node:
+            node['since'] = None
+          nodes[id] = node
+        self.nodes = nodes
+      print(f"Loaded {len(self.nodes)} existing nodes from file ({self.config['paths']['data']}/nodes.json)")
+    except FileNotFoundError:
+      self.nodes = {}
+    if self.config['server']['node_id'] not in self.nodes:
+      self.nodes[self.config['server']['node_id']] = Node.default_node(self.config['server']['node_id'])
+    self.nodes['ffffffff'] = Node.default_node('ffffffff')
+
+    try:
+      chat = self.load_json_file(f"{self.config['paths']['data']}/chat.json")
+      if chat is not None:
+        self.chat = chat
+      print(f"Loaded {len(self.chat['channels']['0']['messages'])} chat messages from file ({self.config['paths']['data']}/chat.json)")
+    except FileNotFoundError:
+      self.chat = {
+          'channels': {
+              '0': {
+                'name': 'General',
+                'messages': []
+              }
+          }
+      }
+
+    try:
+      telemetry = self.load_json_file(f"{self.config['paths']['data']}/telemetry.json")
+      if telemetry is not None:
+        self.telemetry = telemetry
+      else:
+        self.telemetry = []
+      if self.telemetry_by_node is None or len(self.telemetry_by_node) == 0:
+        self.telemetry_by_node = {}
+      for msg in self.telemetry:
+        id = msg['from']
+        if id not in self.telemetry_by_node:
+          self.telemetry_by_node[id] = []
+        self.telemetry_by_node[id].insert(0, msg)
+      print(f"Loaded {len(self.telemetry)} telemetry messages from file ({self.config['paths']['data']}/telemetry.json)")
+      print(f"Loaded telemetry data for {len(self.telemetry_by_node)} nodes")
+    except FileNotFoundError:
+      self.telemetry = []
+      self.telemetry_by_node = {}
+
+    try:
+        traceroutes = self.load_json_file(f"{self.config['paths']['data']}/traceroutes.json")
+        if traceroutes is not None:
+          self.traceroutes = traceroutes
+        else:
+          self.traceroutes = []
+        if self.traceroutes_by_node is None or len(self.traceroutes_by_node) == 0:
+          self.traceroutes_by_node = {}
+        for msg in self.traceroutes:
+          id = msg['from']
+          if id not in self.traceroutes_by_node:
+            self.traceroutes_by_node[id] = []
+          self.traceroutes_by_node[id].insert(0, msg)
+        print(f"Loaded {len(self.traceroutes)} traceroutes from file ({self.config['paths']['data']}/traceroutes.json)")
+        print(f"Loaded traceroutes data for {len(self.traceroutes_by_node)} nodes")
+    except FileNotFoundError:
+        self.traceroutes = []
+        self.traceroutes_by_node = {}
+
+  def load_json_file(self, filename):
+    if os.path.exists(filename):
+        with open(filename, "r", encoding='utf-8') as f:
+            n = json.load(f, cls=_JSONDecoder)
+            return n
+    else:
+        return None
+
+
   def save(self):
     save_start = datetime.now(ZoneInfo(self.config['server']['timezone']))
     last_data = self.config['server']['last_data_save'] if 'last_data_save' in self.config['server'] else self.config['server']['start_time']
@@ -43,22 +150,72 @@ def save(self):
     since_last_render = (save_start - last_render).total_seconds()
     last_backfill = self.config['server']['last_backfill'] if 'last_backfill' in self.config['server'] else self.config['server']['start_time']
     since_last_backfill = (save_start - last_backfill).total_seconds()
-    print(f"Since last - data save: {since_last_data}, render: {since_last_render}, backfill: {since_last_backfill}")
+    print(f"Save (since last): data: {since_last_data} (threshhold: {self.config['server']['intervals']['data_save']}), render: {since_last_render} (threshhold: {self.config['server']['intervals']['render']}), backfill: {since_last_backfill} (threshhold: {self.config['server']['intervals']['enrich']})")
 
-    # if since_last_backfill >= 900:
-    #     self.backfill_node_infos()
-    #     end = datetime.now(ZoneInfo(self.config['server']['timezone']))
-    #     print(f"Backfilled in {round(end.timestamp() - save_start.timestamp(), 3)} seconds")
-    #     self.config['server']['last_backfill'] = end
+    if since_last_backfill >= self.config['server']['intervals']['enrich']:
+        self.backfill_node_infos()
+        end = datetime.now(ZoneInfo(self.config['server']['timezone']))
+        print(f"Enriched in {round(end.timestamp() - save_start.timestamp(), 2)} seconds")
+        self.config['server']['last_backfill'] = end
 
-    # if since_last_data >= self.config['server']['intervals']['data_save']:
-    #     self.save_nodes_to_file()
-    #     end = datetime.now(ZoneInfo(self.config['server']['timezone']))
-    #     print(f"Saved json data in {round(end.timestamp() - save_start.timestamp(), 3)} seconds")
-    #     self.config['server']['last_data_save'] = end
+    if since_last_data >= self.config['server']['intervals']['data_save']:
+        self.data_renderer.render()
+        end = datetime.now(ZoneInfo(self.config['server']['timezone']))
+        print(f"Saved json data in {round(end.timestamp() - save_start.timestamp(), 2)} seconds")
+        self.config['server']['last_data_save'] = end
 
     if since_last_render >= self.config['server']['intervals']['render']:
         self.static_html_renderer.render()
         end = datetime.now(ZoneInfo(self.config['server']['timezone']))
-        print(f"Rendered in {round(end.timestamp() - save_start.timestamp(), 3)} seconds")
+        print(f"Rendered in {round(end.timestamp() - save_start.timestamp(), 2)} seconds")
         self.config['server']['last_render'] = end
+
+  ### helpers
+
+  def backfill_node_infos(self):
+    nodes_needing_enrichment = {}
+    for id, node in self.nodes.items():
+      if 'shortname' not in node or 'longname' not in node or node['shortname'] == 'UNK' or node['longname'] == 'Unknown':
+        nodes_needing_enrichment[id] = node
+    print(f"Nodes needing enrichment: {len(nodes_needing_enrichment)}")
+    if len(nodes_needing_enrichment) > 0:
+      self.enrich_nodes(nodes_needing_enrichment)
+
+  def enrich_nodes(self, node_to_enrich):
+    node_ids = list(node_to_enrich.keys())
+    print(f"Enriching nodes: {','.join(node_ids)}")
+    for node_id in node_ids:
+      print(f"Enriching {node_id}")
+      url = f"https://data.bayme.sh/api/node/infos?ids={node_id}"
+      response = requests.get(url)
+      # print(f"Response code: {response.status_code}")
+      if response.status_code == 200:
+        data = response.json()
+        for node_id, node_info in data.items():
+          print(f"Got info for {node_id}")
+          if node_id in self.nodes:
+            print(f"Enriched {node_id}")
+            node = self.nodes[node_id]
+            node['shortname'] = node_info['shortName']
+            node['longname'] = node_info['longName']
+            self.nodes[node_id] = node
+      else:
+          print(f"Failed to get info for {node_id}")
+
+  def find_node_by_int_id(self, id: int):
+    return self.nodes.get(utils.convert_node_id_from_int_to_hex(id), None)
+
+  def find_node_by_hex_id(self, id: str):
+    return self.nodes.get(id, None)
+
+  def find_node_by_short_name(self, sn: str):
+    for _id, node in self.nodes.items():
+      if node['shortname'] == sn:
+        return node
+    return None
+
+  def find_node_by_longname(self, ln: str):
+    for _id, node in self.nodes.items():
+      if node['longname'] == ln:
+        return node
+    return None

From 07fcf993f5b77965d11d9ce92ea4c336f06cdea9 Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:43:23 -0700
Subject: [PATCH 17/20] some route decoding for the mqtt handler

---
 mqtt.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/mqtt.py b/mqtt.py
index 109b05c..141affd 100644
--- a/mqtt.py
+++ b/mqtt.py
@@ -206,7 +206,14 @@ def handle_traceroute(self, msg):
         msg['to'] = f'{msg["to"]:x}'
         if msg['sender'] and isinstance(msg['sender'], str):
             msg['sender'] = msg['sender'].replace('!', '')
-            msg['route'] = msg['payload']['route']
+        msg['route'] = msg['payload']['route']
+        msg['route_ids'] = []
+        for r in msg['route']:
+            node = self.data.find_node_by_longname(r)
+            if node:
+                msg['route_ids'].append(node['id'])
+            else:
+                msg['route_ids'].append(r)
 
         if id in self.data.traceroutes_by_node:
             self.data.traceroutes_by_node[id].insert(0, msg)

From 61bf5f96b01ec3782b60e7ccfa821eb61fe9fcfa Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:43:41 -0700
Subject: [PATCH 18/20] yeah buddy, slimming down main significantly

---
 main.py | 189 +-------------------------------------------------------
 1 file changed, 3 insertions(+), 186 deletions(-)

diff --git a/main.py b/main.py
index 222f9d7..3d2f219 100644
--- a/main.py
+++ b/main.py
@@ -1,21 +1,13 @@
 #!/usr/bin/env python3
 
 import datetime
-import json
 from zoneinfo import ZoneInfo
 import os
-from jinja2 import Environment, FileSystemLoader
 from dotenv import load_dotenv
-import requests
 
 from config import Config
-from encoders import _JSONDecoder, _JSONEncoder
 from memory_data_store import MemoryDataStore
-from meshtastic import HardwareModel
-from models.node import Node
-import geo
 from mqtt import MQTT
-import utils
 
 load_dotenv()
 
@@ -23,184 +15,9 @@
 data = MemoryDataStore(config)
 data.update('mqtt_connect_time', datetime.datetime.now(ZoneInfo(config['server']['timezone'])))
 
-def find_node_by_int_id(id: int):
-    return nodes.get(utils.convert_node_id_from_int_to_hex(id), None)
-
-def find_node_by_hex_id(id: str):
-    return nodes.get(id, None)
-
-def find_node_by_short_name(sn: str):
-    for _id, node in nodes.items():
-        if node['shortname'] == sn:
-            return node
-    return None
-
-def prune_expired_nodes():
-    global config
-    global nodes
-
-    now = datetime.datetime.now(ZoneInfo(config['server']['timezone']))
-    ids_to_delete: list[str] = []
-    for id, node in nodes.items():
-        last_seen = datetime.datetime.fromisoformat(node['last_seen']).astimezone() if isinstance(node['last_seen'], str) else node['last_seen']
-        since = (now - last_seen).seconds
-        if node['active'] and since >= config['server']['node_activity_prune_threshold']:
-            ids_to_delete.append(node['id'])
-            print(f"Node {id} pruned (last heard {since} seconds ago)")
-    for id in ids_to_delete:
-        nodes[id]['active'] = False
-
-def load_nodes_from_file():
-    n = {}
-    if os.path.exists("output/data/nodes.json"):
-        with open("output/data/nodes.json", "r", encoding='utf-8') as f:
-            n = json.load(f, cls=_JSONDecoder)
-    return n
-
-def backfill_node_infos():
-    global nodes
-
-    # get nodes that need shortname and longname
-    nodes_needing_enrichment = {}
-    for id, node in nodes.items():
-        if 'shortname' not in node or 'longname' not in node or node['shortname'] == 'UNK' or node['longname'] == 'Unknown':
-            nodes_needing_enrichment[id] = node
-    print(f"Nodes needing enrichment: {len(nodes_needing_enrichment)}")
-    if len(nodes_needing_enrichment) > 0:
-        enrich_nodes(nodes_needing_enrichment)
-
-def enrich_nodes(node_to_enrich):
-    global nodes
-
-    node_ids = list(node_to_enrich.keys())
-    print(f"Enriching nodes: {','.join(node_ids)}")
-    for node_id in node_ids:
-        print(f"Enriching {node_id}")
-        url = f"https://data.bayme.sh/api/node/infos?ids={node_id}"
-        response = requests.get(url)
-        # print(f"Response code: {response.status_code}")
-        if response.status_code == 200:
-            data = response.json()
-            for node_id, node_info in data.items():
-                print(f"Got info for {node_id}")
-                if node_id in nodes:
-                    print(f"Enriched {node_id}")
-                    node = nodes[node_id]
-                    node['shortname'] = node_info['shortName']
-                    node['longname'] = node_info['longName']
-                    nodes[node_id] = node
-        else:
-            print(f"Failed to get info for {node_id}")
-
-def save():
-    global nodes
-    global chat
-    global config
-    global traceroutes
-
-    save_start = datetime.datetime.now(ZoneInfo(config['server']['timezone']))
-    last_data = config['server']['last_data_save'] if 'last_data_save' in config['server'] else config['server']['start_time']
-    since_last_data = (save_start - last_data).total_seconds()
-    last_render = config['server']['last_render'] if 'last_render' in config['server'] else config['server']['start_time']
-    since_last_render = (save_start - last_render).total_seconds()
-    last_backfill = config['server']['last_backfill'] if 'last_backfill' in config['server'] else config['server']['start_time']
-    since_last_backfill = (save_start - last_backfill).total_seconds()
-    print(f"Since last - data save: {since_last_data}, render: {since_last_render}, backfill: {since_last_backfill}")
-
-    if since_last_backfill >= 900:
-        backfill_node_infos()
-        end = datetime.datetime.now(ZoneInfo(config['server']['timezone']))
-        print(f"Backfilled in {round(end.timestamp() - save_start.timestamp(), 3)} seconds")
-        config['server']['last_backfill'] = end
-
-    if since_last_data >= config['server']['intervals']['data_save']:
-        save_nodes_to_file()
-        end = datetime.datetime.now(ZoneInfo(config['server']['timezone']))
-        print(f"Saved json data in {round(end.timestamp() - save_start.timestamp(), 3)} seconds")
-        config['server']['last_data_save'] = end
-
-    if since_last_render >= config['server']['intervals']['render']:
-        render_static_html_files()
-        end = datetime.datetime.now(ZoneInfo(config['server']['timezone']))
-        print(f"Rendered in {round(end.timestamp() - save_start.timestamp(), 3)} seconds")
-        config['server']['last_render'] = end
-
-def load():
-    global config
-    global data
-
-    try:
-      data.update('nodes', load_nodes_from_file())
-      print(f"Loaded {len(data.nodes)} existing nodes from file ({config['paths']['data']}/nodes.json)")
-    except FileNotFoundError:
-      data.update('nodes', {})
-    if config['server']['node_id'] not in data.nodes:
-      data.nodes[config['server']['node_id']] = Node.default_node(config['server']['node_id'])
-    data.nodes['ffffffff'] = Node.default_node('ffffffff')
-
-    try:
-      data.chat = load_chat_from_file("json", f"{config['paths']['data']}/chat.json")
-      print(f"Loaded {len(data.chat['channels']['0']['messages'])} chat messages from file ({config['paths']['data']}/chat.json)")
-    except FileNotFoundError:
-      data.chat = {
-          'channels': {
-              '0': {
-                'name': 'General',
-                'messages': []
-              }
-          }
-      }
-
-    try:
-      data.telemetry = load_from_json_file(f"{config['paths']['data']}/telemetry.json")
-      if data.telemetry is None or len(data.telemetry) == 0:
-        data.telemetry = []
-      if data.telemetry_by_node is None or len(data.telemetry_by_node) == 0:
-        data.telemetry_by_node = {}
-      for msg in data.telemetry:
-        id = msg['from']
-        if id not in data.telemetry_by_node:
-          data.telemetry_by_node[id] = []
-        data.telemetry_by_node[id].insert(0, msg)
-      print(f"Loaded {len(data.telemetry)} telemetry messages from file ({config['paths']['data']}/telemetry.json)")
-      print(f"Loaded telemetry data for {len(data.telemetry_by_node)} nodes")
-    except FileNotFoundError:
-      data.telemetry = []
-      data.telemetry_by_node = {}
-
-    try:
-        data.traceroutes = load_from_json_file(f"{config['paths']['data']}/traceroutes.json")
-        if data.traceroutes is None or len(data.traceroutes) == 0:
-          data.traceroutes = []
-        if data.traceroutes_by_node is None or len(data.traceroutes_by_node) == 0:
-          data.traceroutes_by_node = {}
-        for msg in data.traceroutes:
-          id = msg['from']
-          if id not in data.traceroutes_by_node:
-            data.traceroutes_by_node[id] = []
-          data.traceroutes_by_node[id].insert(0, msg)
-        print(f"Loaded {len(data.traceroutes)} traceroutes from file ({config['paths']['data']}/traceroutes.json)")
-        print(f"Loaded traceroutes data for {len(data.traceroutes_by_node)} nodes")
-    except FileNotFoundError:
-        data.traceroutes = []
-        data.traceroutes_by_node = {}
-
-def load_chat_from_file(type, path, config=config):
-    global chat
-    if type == "json":
-      with open(path, "r", encoding='utf-8') as f:
-          return json.load(f, cls=_JSONDecoder)
-    if type == "html":
-      pass
-
-def load_from_json_file(path, config=config):
-    with open(path, "r", encoding='utf-8') as f:
-        return json.load(f, cls=_JSONDecoder)
-
 def run():
-    global chat
     global config
-    global nodes
+    global data
 
     if not os.path.exists(config['paths']['output']):
         os.makedirs(config['paths']['output'])
@@ -209,8 +26,8 @@ def run():
 
     os.environ['TZ'] = config['server']['timezone']
 
-    load()
-    save()
+    data.load()
+    data.save()
 
     if os.environ.get('MQTT_BROKER') is not None:
         config['broker']['host'] = os.environ['MQTT_BROKER']

From 08eee805585f7565a475ad801c4d920f557b1418 Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:43:55 -0700
Subject: [PATCH 19/20] Major template improvements for map, node, stats,
 telemetry, and traceroutes

---
 .../static/layout-partial-sidebar.html.j2     |  72 ++++
 templates/static/map.html.j2                  |   2 +-
 templates/static/node.html.j2                 | 325 ++++++++++++------
 templates/static/stats.html.j2                |  45 +--
 templates/static/telemetry.html.j2            | 291 +++++++++-------
 templates/static/traceroutes.html.j2          |  68 +++-
 6 files changed, 531 insertions(+), 272 deletions(-)
 create mode 100644 templates/static/layout-partial-sidebar.html.j2

diff --git a/templates/static/layout-partial-sidebar.html.j2 b/templates/static/layout-partial-sidebar.html.j2
new file mode 100644
index 0000000..20d03c1
--- /dev/null
+++ b/templates/static/layout-partial-sidebar.html.j2
@@ -0,0 +1,72 @@
+<!-- Static sidebar for desktop -->
+<div class="hidden lg:fixed lg:inset-y-0 lg:z-50 lg:flex lg:w-60 lg:flex-col">
+  <!-- Sidebar component, swap this element with another sidebar if you like -->
+  <div class="flex flex-col px-6 pb-4 overflow-y-auto bg-gray-300 border-r-2 grow gap-y-5 border-r-cyan-600">
+    <div class="flex items-center h-24 mt-4 shrink-0">
+      <div class="text-2xl">
+        <div class="p-0 m-0">S<span class="text-gray-500">ACRAMENTO</span></div>
+        <div class="p-0 m-0">V<span class="text-gray-500">ALLEY</span></div>
+        <div class="p-0 m-0">M<span class="text-gray-500">ESH</span></div>
+      </div>
+    </div>
+
+    <div>
+      {{ config.mesh.description }}
+    </div>
+
+    {% if config.mesh.url %}
+      <div>
+        <a href="{{ config.mesh.url }}" class="text-xs text-gray-900">Website</a>
+      </div>
+    {% endif %}
+
+    <nav class="flex flex-col flex-1">
+      <h3>Mesh</h3>
+      <div class="mb-2">
+        <a href="chat.html">Chat</a>
+      </div>
+      <div class="mb-2">
+        <a href="map.html">Map</a>
+      </div>
+      <div class="mb-2">
+        <a href="nodes.html">Nodes</a>
+      </div>
+      <div class="mb-2">
+        <a href="neighbors.html">Node Neighbors</a>
+      </div>
+      <div class="mb-2">
+        <a href="network.html">Network</a>
+      </div>
+      <div class="mb-2">
+        <a href="routes.html">Routes</a>
+      </div>
+      <div class="mb-2">
+        <a href="stats.html">Stats</a>
+      </div>
+      <div class="mb-2">
+        <a href="telemetry.html">Telemetry</a>
+      </div>
+      <div class="mb-2">
+        <a href="traceroutes.html">Traceroutes</a>
+      </div>
+    </nav>
+
+    <nav class="flex flex-col flex-1">
+      <h3>Logs</h3>
+      <div class="mb-2">
+        <a href="mesh_log.html">Mesh Messages</a>
+      </div>
+      <div class="mb-2">
+        <a href="mqtt_log.html">MQTT Messages</a>
+      </div>
+    </nav>
+
+    <div class="flex-grow"></div>
+    <div class="flex flex-col">
+      <h5 class="mb-2">Powered by MeshInfo</h5>
+      <a href="https://github.com/kevinelliott/meshinfo" class="text-xs text-gray-500">
+        <img src="https://img.shields.io/github/stars/kevinelliott/meshinfo?style=social" alt="GitHub Stars">
+      </a>
+    </div>
+  </div>
+</div>
diff --git a/templates/static/map.html.j2 b/templates/static/map.html.j2
index f55522f..666193b 100644
--- a/templates/static/map.html.j2
+++ b/templates/static/map.html.j2
@@ -113,7 +113,7 @@
     const nodes = {};
     var neighborLayers = [];
     {% for id, node in nodes.items() %}
-      {% if node.position is defined and node.position %}
+      {% if node.position is defined and node.position and node.position.longitude_i %}
       nodes['{{ id }}'] = {
         id: '{{ id }}',
         shortname: '{{ node.shortname }}',
diff --git a/templates/static/node.html.j2 b/templates/static/node.html.j2
index a7bf562..3f812da 100644
--- a/templates/static/node.html.j2
+++ b/templates/static/node.html.j2
@@ -7,109 +7,234 @@
 {% endblock %}
 
 {% block content %}
-  <h5 class="mb-2 text-gray-500">
+  <h5 class="mb-4 text-gray-500">
     <a href="nodes.html">Nodes</a> &gt; {{ node.shortname }}
   </h5>
-  <h1 class="mb-2 text-xl">{{ node.shortname }} - {{ node.longname }}</h1>
-
-  <div class="flex align-middle mb-4">
-  {% if node.telemetry %}
-  {% if node.telemetry.air_util_tx %}
-      <span class="mr-4 align-middle" title="Air Util TX">
-        <img src="images/icons/up.svg" alt="Air Util TX" class="w-4 h-4 inline-block align-middle">
-        {{ node.telemetry.air_util_tx | round(1) }}%
-      </span>
-    {% endif %}
-    {% if node.telemetry.channel_utilization %}
-      <span class="mr-4 align-middle" title="Channel Util">
-        <img src="images/icons/down.svg" alt="Channel Util" class="w-4 h-4 inline-block align-middle">
-        {{ node.telemetry.channel_utilization | round(1) }}%
-      </span>
-    {% endif %}
-    {% if node.telemetry.battery_level %}
-      <span class="mr-4 align-middle" title="Battery Level">
-        <img src="images/icons/battery.svg" alt="Battery" class="w-4 h-4 inline-block align-middle">
-        {{ node.telemetry.battery_level | round(0) }}%
-      </span>
-    {% endif %}
-    {% if node.telemetry.temperature %}
-      <span class="mr-4 align-middle" title="Temperature">
-        <img src="images/icons/temperature.svg" alt="Temperature" class="w-4 h-4 inline-block align-middle">
-        {{ node.telemetry.temperature | round(1) }}&deg;C
-      </span>
-    {% endif %}
-    {% if node.telemetry.humidity %}
-      <span class="mr-4 align-middle" title="Humidity">
-        <img src="images/icons/humidity.svg" alt="Humidity" class="w-4 h-4 inline-block align-middle">
-        {{ node.telemetry.humidity }}%
-      </span>
-    {% endif %}
-    {% if node.telemetry.voltage %}
-      <span class="mr-4 align-middle" title="Voltage">
-        <img src="images/icons/voltage.svg" alt="Voltage" class="w-4 h-4 inline-block align-middle">
-        {{ node.telemetry.voltage | round(1) }}V
-      </span>
-    {% endif %}
-  {% endif %}
+
+  <div class="mb-4 flex flex-row">
+    <div class="mr-2">
+      <img src="https://api.dicebear.com/9.x/bottts-neutral/svg?seed={{ node.id }}" alt="Avatar" class="w-16 h-16 mb-1 object-cover">
+    </div>
+    <div>
+      <h1 class="mb-2 text-xl">{{ node.shortname }} - {{ node.longname }}</h1>
+      <div class="flex align-middle mb-4">
+      {% if node.telemetry %}
+      {% if node.telemetry.air_util_tx %}
+          <span class="mr-4 align-middle" title="Air Util TX">
+            <img src="images/icons/up.svg" alt="Air Util TX" class="w-4 h-4 inline-block align-middle">
+            {{ node.telemetry.air_util_tx | round(1) }}%
+          </span>
+        {% endif %}
+        {% if node.telemetry.channel_utilization %}
+          <span class="mr-4 align-middle" title="Channel Util">
+            <img src="images/icons/down.svg" alt="Channel Util" class="w-4 h-4 inline-block align-middle">
+            {{ node.telemetry.channel_utilization | round(1) }}%
+          </span>
+        {% endif %}
+        {% if node.telemetry.battery_level %}
+          <span class="mr-4 align-middle" title="Battery Level">
+            <img src="images/icons/battery.svg" alt="Battery" class="w-4 h-4 inline-block align-middle">
+            {{ node.telemetry.battery_level | round(0) }}%
+          </span>
+        {% endif %}
+        {% if node.telemetry.temperature %}
+          <span class="mr-4 align-middle" title="Temperature">
+            <img src="images/icons/temperature.svg" alt="Temperature" class="w-4 h-4 inline-block align-middle">
+            {{ node.telemetry.temperature | round(1) }}&deg;C
+          </span>
+        {% endif %}
+        {% if node.telemetry.humidity %}
+          <span class="mr-4 align-middle" title="Humidity">
+            <img src="images/icons/humidity.svg" alt="Humidity" class="w-4 h-4 inline-block align-middle">
+            {{ node.telemetry.humidity }}%
+          </span>
+        {% endif %}
+        {% if node.telemetry.voltage %}
+          <span class="mr-4 align-middle" title="Voltage">
+            <img src="images/icons/voltage.svg" alt="Voltage" class="w-4 h-4 inline-block align-middle">
+            {{ node.telemetry.voltage | round(1) }}V
+          </span>
+        {% endif %}
+      {% endif %}
+      </div>
+    </div>
   </div>
 
-  {% if node.position and node.position.latitude_i and node.position.longitude_i %}
-  <div id="map" class="map"></div>
-  {% endif %}
-
-  <p>ID: {{ node.id }}</p>
-  <p>
-    Type:
-    {% if node.hardware %}
-      {{ hardware(node.hardware).name }}
-    {% else %}
-      Unknown
-    {% endif %}
-  </p>
-  <p>
-    Status:
-    {% if node.active %}
-      <span class="text-green-500">Online</span>
-    {% else %}
-      <span class="text-gray-700">Offline</span>
-    {% endif %}
-  </p>
-  <p>Position:
-    {% if node.position and node.position.latitude_i and node.position.longitude_i %}
-      {{ node.position.longitude_i / 1e7 }}, {{ node.position.latitude_i / 1e7 }}
-    {% else %}
-      Unknown
-    {% endif %}
-  </p>
-  <p>Altitude:
-    {% if node.position and node.position.altitude %}
-      {{ node.position.altitude }}
-    {% else %}
-      Unknown
-    {% endif %}
-  </p>
-  <p>Last Seen: {{ node.last_seen }}</p>
-
-  <h3 class="text-gray-600">Heard</h3>
-  <p>Heard {{ node.neighborinfo.neighbors|count }} zero-hop nodes</p>
-  <ul>
-    {% for neighbor in node.neighborinfo.neighbors %}
-      <li>{{ neighbor }}</li>
-    {% endfor %}
-  </ul>
-
-  <h3 class="text-gray-600">Heard By</h3>
-  <ul>
-    {% for id, node in nodes.items() %}
-      {% if node.neighborinfo and node.neighborinfo.neighbors %}
-        {% for neighbor in node.neighborinfo.neighbors %}
-          {% if neighbor.node_id == node.id %}
-            <li>{{ id }}</li>
-          {% endif %}
-        {% endfor %}
+  <div class="grid grid-col-1 md:grid-flow-col md:auto-cols-max">
+    <div class="w-auto sm:w-96 mb-4 md:mr-4">
+      {% if node.position and node.position.latitude_i and node.position.longitude_i %}
+      <div class="w-auto sm:w-96 mb-4 md:mr-4">
+        <div id="map" class="map"></div>
+      </div>
       {% endif %}
-    {% endfor %}
-  </ul>
+
+      <div>
+        <h3 class="font-bold text-gray-600">Elsewhere</h3>
+        <div>
+          <a href="https://meshview.armooo.net/packet_list/{{ utils.convert_node_id_from_hex_to_int(node.id) }}" target="_blank">Armooo's MeshView</a><br/>
+          <a href="https://app.bayme.sh/node/{{ node.id }}" target="_blank">Bay Mesh Explorer</a><br/>
+          <a href="https://meshtastic.liamcottle.net/?node_id={{ utils.convert_node_id_from_hex_to_int(node.id) }}" target="_blank">Liam's Map</a><br/>
+          <a href="https://meshmap.net/#{{ utils.convert_node_id_from_hex_to_int(node.id) }}" target="_blank">MeshMap</a>
+        </div>
+      </div>
+    </div>
+
+    <div class="w-auto md:w-96">
+      <div class="mb-4">
+        <h3 class="mb-2 font-bold text-gray-600">Details</h3>
+        <table class="table-auto min-w-full border border-gray-200 bg-gray-50">
+          <tbody class="divide-y divide-dashed divide-gray-200">
+            <tr>
+              <th class="p-1" align=left>ID (hex)</th>
+              <td class="p-1">{{ node.id }}</td>
+            </tr>
+            <tr>
+              <th class="p-1" align=left>ID (int)</th>
+              <td class="p-1">{{ utils.convert_node_id_from_hex_to_int(node.id) }}</td>
+            </tr>
+            <tr>
+              <th class="p-1" align=left>Hardware</th>
+              <td class="p-1">
+              {% if node.hardware %}
+                {{ hardware(node.hardware).name }}
+              {% else %}
+                Unknown
+              {% endif %}
+              </td>
+            </tr>
+            <tr>
+              <th class="p-1" align=left>Position</th>
+              <td class="p-1">
+              {% if node.position and node.position.latitude_i and node.position.longitude_i %}
+                {{ node.position.longitude_i / 1e7 }}, {{ node.position.latitude_i / 1e7 }}
+              {% else %}
+                Unknown
+              {% endif %}
+            </tr>
+            <tr>
+              <th class="p-1" align=left>Location</th>
+              <td class="p-1">
+              {% if node.position and node.position.geocoded %}
+                {{ node.position.geocoded.display_name }}
+              {% else %}
+                Unknown
+              {% endif %}
+              </td>
+            </tr>
+            <tr>
+              <th class="p-1" align=left>Altitude</th>
+              <td class="p-1">
+              {% if node.position and node.position.altitude %}
+                {{ node.position.altitude }} m
+              {% else %}
+                Unknown
+              {% endif %}
+              </td>
+            </tr>
+            <tr>
+              {% set server_node = nodes[config['server']['node_id']] %}
+              {% set distance_from_server_node = utils.calculate_distance_between_nodes(server_node, node) %}
+              <th class="p-1 text-nowrap whitespace-nowrap" align=left>
+                Distance from {{ server_node.shortname }}
+              </th>
+              <td class="p-1">
+              {% if distance_from_server_node != None %}
+                {{ distance_from_server_node }} km
+              {% else %}
+                Unknown
+              {% endif %}
+              </td>
+            </tr>
+            <tr>
+              <th class="p-1" align=left>Status</th>
+              <td class="p-1">
+              {% if node.active %}
+                <span class="text-green-500">Online</span>
+              {% else %}
+                <span class="text-gray-700">Offline</span>
+              {% endif %}
+              </td>
+            </tr>
+            <tr>
+              <th class="p-1" align=left>Last Seen</th>
+              {% if node.last_seen %}
+              <td class="p-1 text-nowrap" title="{{ node.last_seen }}">{{ node.last_seen }}</td>
+              {% else %}
+              <td>Unknown</td>
+              {% endif %}
+            </tr>
+          </tbody>
+        </table>
+      </div>
+
+      <div class="mb-4">
+        <h3 class="mb-2 font-bold text-gray-600">Heard (zero hop)</h3>
+        <table class="table-auto min-w-full border border-gray-200 bg-gray-50">
+          <tbody class="divide-y divide-dashed divide-gray-200">
+          {% for neighbor in node.neighborinfo.neighbors %}
+            {% set nid = utils.convert_node_id_from_int_to_hex(neighbor.node_id) %}
+            {% set nnode = nodes[nid] if nid in nodes else None %}
+            <tr>
+              <td class="w-1/3 p-1 text-nowrap">
+                {% if nnode %}
+                <a href="node_{{ nnode.id }}.html">{{ nnode.shortname }}</a>
+                {% else %}
+                <span class="text-gray-500">UNK</span>
+                {% endif %}
+              </td>
+              <td class="p-1 text-nowrap">
+                SNR: {{ neighbor.snr }}
+              </td>
+              <td class="p-1 text-nowrap" align=right>
+                {% if nnode %}
+                {% set dist = utils.calculate_distance_between_nodes(nnode, node) %}
+                {% if dist %}
+                {{ dist }} km
+                {% endif %}
+                {% endif %}
+              </td>
+            </tr>
+          {% endfor %}
+          </tbody>
+        </table>
+      </div>
+
+      <div class="mb-4">
+        <h3 class="mb-2 font-bold text-gray-600">Heard By (zero hop)</h3>
+        <table class="table-auto min-w-full border border-gray-200 bg-gray-50">
+          <tbody class="divide-y divide-dashed divide-gray-200">
+          {% for id, nnode in nodes.items() %}
+            {% if nnode.neighborinfo and nnode.neighborinfo.neighbors %}
+              {% for neighbor in nnode.neighborinfo.neighbors %}
+                {% if utils.convert_node_id_from_int_to_hex(neighbor.node_id) == node.id %}
+                <tr>
+                  <td class="w-1/3 p-1 text-nowrap" width=25%>
+                    {% if id in nodes %}
+                      <a href="node_{{ id }}.html">{{ nodes[id].shortname }}</a>
+                    {% else %}
+                      <span class="text-gray-500">UNK</span>
+                    {% endif %}
+                  </td>
+                  <td class="p-1 text-nowrap">
+                    SNR: {{ neighbor.snr }}
+                  </td>
+                  <td class="p-1 text-nowrap" align=right>
+                    {% set dist = utils.calculate_distance_between_nodes(nodes[id], node) %}
+                    {% if dist %}
+                    {{ dist }} km
+                    {% endif %}
+                  </td>
+                </tr>
+                {% endif %}
+              {% endfor %}
+            {% endif %}
+          {% endfor %}
+          </tbody>
+        </table>
+      </div>
+    </div>
+  </div>
+
 
   <script type="module">
     var defaultStyle = new ol.style.Style({
@@ -210,8 +335,8 @@
   </script>
   <style>
     #map {
-      height: 400px;
-      width: 400px;
+      height: 300px;
+      width: 100%;
     }
   </style>
 {% endblock %}
diff --git a/templates/static/stats.html.j2 b/templates/static/stats.html.j2
index 77e0615..265b0be 100644
--- a/templates/static/stats.html.j2
+++ b/templates/static/stats.html.j2
@@ -3,62 +3,63 @@
 {% block title %}Stats{% endblock %}
 
 {% block content %}
-    <h1>Stats</h1>
-    <p>
-      Some revelations based on messages that have
-      been heard by the mesh by <b>KE-R</b> (!4355f528).
-    </p>
-    <p>Last updated: {{ timestamp.astimezone(zoneinfo).strftime('%Y-%m-%d %H:%M:%S %z') }}</p>
+  <h5 class="mb-2 text-gray-500">Stats</h5>
+  <h1 class="mb-2 text-xl">Stats</h1>
+  <p class="mb-2">
+    Some revelations based on messages that have
+    been heard by the mesh by <b>KE-R</b> (!4355f528).
+  </p>
 
-    <h3 class="text-xl">Current</h3>
+  <h3 class="text-lg">Current</h3>
+  <div class="mb-4">
+    <h3>Active Nodes</h3>
     <div>
-      <h3>Active Nodes</h3>
-      <div>
-        <div>{{ stats.active_nodes }}</div>
-      </div>
+      <div>{{ stats.active_nodes }}</div>
     </div>
+  </div>
 
-    <h3 class="text-xl">Persisted</h3>
-    <div>
+  <div class="mb-4">
+    <h3 class="text-lg">Persisted</h3>
+    <div class="mb-2">
       <h3>Known Nodes</h3>
       <div>
         <div>{{ stats.total_nodes }}</div>
       </div>
     </div>
-    <div>
+    <div class="mb-2">
       <h3>Chat Messages</h3>
       <div>
         <div>{{ stats.total_chat }}</div>
       </div>
     </div>
-    <div>
+    <div class="mb-2">
       <h3>Telemetry</h3>
       <div>
         <div>{{ stats.total_telemetry }}</div>
       </div>
     </div>
-    <div>
+    <div class="mb-2">
       <h3>Traceroutes</h3>
       <div>
         <div>{{ stats.total_traceroutes }}</div>
       </div>
     </div>
+  </div>
 
-    <h3 class="text-xl">Since Last Restart</h3>
-    <div>
+  <div class="mb-4">
+    <h3 class="text-lg">Since Last Restart</h3>
+    <div class="mb-2">
       <h3>Messages (Session Total)</h3>
       <div>
         <div>{{ stats.total_messages }}</div>
       </div>
     </div>
-
-    <div>
+    <div class="mb-2">
       <h3>Messages (Session MQTT)</h3>
       <div>
         <div>{{ stats.total_mqtt_messages }}</div>
       </div>
     </div>
-
-
+  </div>
 
 {% endblock %}
diff --git a/templates/static/telemetry.html.j2 b/templates/static/telemetry.html.j2
index 152314c..3a22873 100644
--- a/templates/static/telemetry.html.j2
+++ b/templates/static/telemetry.html.j2
@@ -3,138 +3,169 @@
 {% block title %}Telemetry{% endblock %}
 
 {% block content %}
-    <h5 class="mb-2 text-gray-500">Telemetry</h5>
-    <h1 class="mb-2 text-xl">Telemetry</h1>
-    <p class="mb-2">
-      Telemetry as seen by <b>{{ config['server']['node_id'] }}</b> ({{ config['server']['node_id'] }}).
-    </p>
+  <h5 class="mb-2 text-gray-500">Telemetry</h5>
+  <h1 class="mb-2 text-xl">Telemetry</h1>
+  <p class="mb-2">
+    Telemetry as seen by <b>{{ nodes[config['server']['node_id']].shortname }}</b> ({{ config['server']['node_id'] }}).
+  </p>
 
-    <table border=1 cellpadding=4>
+  <table class="w-full max-w-full table-auto border-collapse border border-gray-500 bg-gray-50">
+    <thead>
       <tr>
-        <td>Timestamp</td>
-        <td>Node</td>
-        <td>Air Util TX</td>
-        <td>Battery Level</td>
-        <td>Channel Utilization</td>
-        <td>Uptime</td>
-        <td>Voltage</td>
-        <td>Current</td>
-        <td>Barometric Pressure</td>
-        <td>Relative Humidity</td>
-        <td>Temperature</td>
-        <td>Gas Resistance</td>
+        <th class="p-1 border border-gray-500 bg-gray-400" align=left>Timestamp</th>
+        <th class="p-1 border border-gray-500 bg-gray-400" align=left>Node</th>
+        <th class="p-1 border border-gray-500 bg-gray-400" align=center>
+          <img src="images/icons/up.svg" alt="Air Util TX" class="w-4 h-4 inline-block" title="Air Util TX">
+        </th>
+        <th class="p-1 border border-gray-500 bg-gray-400" align=center>
+          <img src="images/icons/down.svg" class="w-4 h-4 inline-block" alt="Channel Util" title="Channel Util">
+        </th>
+        <th class="p-1 border border-gray-500 bg-gray-400" align=center>
+          <img src="images/icons/battery.svg" class="w-6 h-6 inline-block" alt="Battery" title="Battery">
+        </th>
+        <th class="p-1 border border-gray-500 bg-gray-400" align=center>Uptime</th>
+        <th class="p-1 border border-gray-500 bg-gray-400" align=center>
+          <img src="images/icons/voltage.svg" class="w-4 h-4 inline-block" alt="Voltage" title="Voltage">
+        </th>
+        <th class="p-1 hidden lg:table-cell border border-gray-500 bg-gray-400" align=center>
+          <img src="images/icons/current.svg" class="w-5 h-5 inline-block" alt="Current" title="Current">
+        </th>
+        <th class="p-1 hidden lg:table-cell border border-gray-500 bg-gray-400" align=center>
+          <img src="images/icons/pressure.svg" class="w-4 h-4 inline-block" alt="Barometric Pressure" title="Barometric Pressure">
+        </th>
+        <th class="p-1 hidden lg:table-cell border border-gray-500 bg-gray-400" align=center>
+          <img src="images/icons/relative-humidity.svg" class="w-4 h-4 inline-block" alt="Relative Humidity" title="Relative Humidity">
+        </th>
+        <th class="p-1 hidden lg:table-cell border border-gray-500 bg-gray-400" align=center>
+          <img src="images/icons/temperature.svg" class="w-4 h-4 inline-block" alt="Temperature" title="Temperature">
+        </th>
+        <th class="p-1 hidden lg:table-cell border border-gray-500 bg-gray-400" align=center>
+          <img src="images/icons/resistance.svg" class="w-5 h-5 inline-block" alt="Gas Resistance" title="Gas Resistance">
+        </th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for item in telemetry[0:1000] %}
+      {% set inode = nodes[item.from] %}
+      <tr>
+        <td class="p-1 border border-gray-400 text-nowrap">
+          {{ datetime.fromtimestamp(item.timestamp).astimezone(zoneinfo) }}
+        </td>
+        <td class="p-1 border border-gray-400">
+          {% if inode %}
+          <a href="node_{{ inode.id }}.html">{{ inode.shortname }}</a>
+          {% else %}
+          <span class="text-gray-500">UNK</span>
+          {% endif %}
+        </td>
+        <td class="p-1 border border-gray-400" align=right>
+          {% if item.payload.air_util_tx is defined %}
+          {{ item.payload.air_util_tx | round(2) }}%
+          {% endif %}
+        </td>
+        <td class="p-1 border border-gray-400" align=right>
+          {% if item.payload.channel_utilization is defined %}
+          {{ item.payload.channel_utilization | round(1) }}%
+          {% endif %}
+        </td>
+        <td class="p-1 border border-gray-400" align=right>
+          {% if item.payload.battery_level is defined %}
+          {{ item.payload.battery_level | round(2) }}%
+          {% endif %}
+        </td>
+        <td class="p-1 border border-gray-400" align=right>
+          {% if item.payload.uptime_seconds is defined %}
+          {{ item.payload.uptime_seconds }}
+          {% endif %}
+        </td>
+        <td class="p-1 border border-gray-400 text-nowrap" align=right>
+          {% if item.payload.voltage is defined %}
+          {{ item.payload.voltage | round(2) }} V
+          {% endif %}
+          {% if item.payload.voltage_ch1 is defined and item.payload.voltage_ch2 is defined and item.payload.voltage_ch3 is defined %}
+          <table>
+          <tr>
+            <td>
+              Ch1
+            </td>
+            <td align=right class="pl-2">
+              {{ item.payload.voltage_ch1 | round(2) }} V<br/>
+            </td>
+          </tr>
+          <tr>
+            <td>
+              Ch2
+            </td>
+            <td align=right class="pl-2">
+              {{ item.payload.voltage_ch2 | round(2) }} V<br/>
+            </td>
+          </tr>
+          <tr>
+            <td>
+              Ch3
+            </td>
+            <td align=right class="pl-2">
+              {{ item.payload.voltage_ch3 | round(2) }} V
+            </td>
+          </tr>
+          </table>
+          {% endif %}
+        </td>
+        <td class="hidden lg:table-cell p-1 border border-gray-400 text-nowrap" align=right>
+          {% if item.payload.current is defined %}
+          {{ item.payload.current | round(2) }} mA
+          {% endif %}
+          {% if item.payload.current_ch1 is defined and item.payload.current_ch2 is defined and item.payload.current_ch3 is defined %}
+          <table>
+          <tr>
+            <td>
+              Ch1
+            </td>
+            <td align=right class="pl-2">
+              {{ item.payload.current_ch1 | round(2) }} mA<br/>
+            </td>
+          </tr>
+          <tr>
+            <td>
+              Ch2
+            </td>
+            <td align=right class="pl-2">
+              {{ item.payload.current_ch2 | round(2) }} mA<br/>
+            </td>
+          </tr>
+          <tr>
+            <td>
+              Ch3
+            </td>
+            <td align=right class="pl-2">
+              {{ item.payload.current_ch3 | round(2) }} mA
+            </td>
+          </tr>
+          </table>
+          {% endif %}
+        </td>
+        <td class="hidden lg:table-cell p-1 border border-gray-400 text-nowrap" align=right>
+          {% if item.payload.barometric_pressure is defined %}
+          {{ item.payload.barometric_pressure | round(2) }} hPa
+          {% endif %}
+        </td>
+        <td class="hidden lg:table-cell p-1 border border-gray-400 text-nowrap" align=right>
+          {% if item.payload.relative_humidity is defined and item.payload.relative_humidity %}
+          {{ item.payload.relative_humidity | round(2) }}%
+          {% endif %}
+        </td>
+        <td class="hidden lg:table-cell p-1 border border-gray-400 text-nowrap" align=right>
+          {% if item.payload.temperature is defined %}
+          {{ item.payload.temperature | round(2) }} &deg;C
+          {% endif %}
+        </td>
+        <td class="hidden lg:table-cell p-1 border border-gray-400 text-nowrap" align=right>
+          {% if item.payload.gas_resistance is defined %}
+          {{ item.payload.gas_resistance | round(2) }} Ohm
+          {% endif %}
+        </td>
       </tr>
-      {% for item in telemetry[0:10000] %}
-        <tr>
-          <td style="white-space:nowrap">{{ datetime.fromtimestamp(item.timestamp).astimezone(zoneinfo) }}</td>
-          <td><b>{{ item.from }}</b></td>
-          <td align=right>
-            {% if item.payload.air_util_tx is defined %}
-            {{ item.payload.air_util_tx | round(2) }}%
-            {% endif %}
-          </td>
-          <td align=right>
-            {% if item.payload.battery_level is defined %}
-            {{ item.payload.battery_level | round(2) }}%
-            {% endif %}
-          </td>
-          <td align=right>
-            {% if item.payload.channel_utilization is defined %}
-            {{ item.payload.channel_utilization | round(1) }}%
-            {% endif %}
-          </td>
-          <td align=right>
-            {% if item.payload.uptime_seconds is defined %}
-            {{ item.payload.uptime_seconds }}
-            {% endif %}
-          </td>
-          <td align=right style="white-space:nowrap">
-            {% if item.payload.voltage is defined %}
-            {{ item.payload.voltage | round(2) }} V
-            {% endif %}
-            {% if item.payload.voltage_ch1 is defined and item.payload.voltage_ch2 is defined and item.payload.voltage_ch3 is defined %}
-            <table>
-            <tr>
-              <td>
-                Ch1
-              </td>
-              <td align=right class="pl-2">
-                {{ item.payload.voltage_ch1 | round(2) }} V<br/>
-              </td>
-            </tr>
-            <tr>
-              <td>
-                Ch2
-              </td>
-              <td align=right class="pl-2">
-                {{ item.payload.voltage_ch2 | round(2) }} V<br/>
-              </td>
-            </tr>
-            <tr>
-              <td>
-                Ch3
-              </td>
-              <td align=right class="pl-2">
-                {{ item.payload.voltage_ch3 | round(2) }} V
-              </td>
-            </tr>
-            </table>
-            {% endif %}
-          </td>
-          <td align=right style="white-space:nowrap">
-            {% if item.payload.current is defined %}
-            {{ item.payload.current | round(2) }} mA
-            {% endif %}
-            {% if item.payload.current_ch1 is defined and item.payload.current_ch2 is defined and item.payload.current_ch3 is defined %}
-            <table>
-            <tr>
-              <td>
-                Ch1
-              </td>
-              <td align=right class="pl-2">
-                {{ item.payload.current_ch1 | round(2) }} mA<br/>
-              </td>
-            </tr>
-            <tr>
-              <td>
-                Ch2
-              </td>
-              <td align=right class="pl-2">
-                {{ item.payload.current_ch2 | round(2) }} mA<br/>
-              </td>
-            </tr>
-            <tr>
-              <td>
-                Ch3
-              </td>
-              <td align=right class="pl-2">
-                {{ item.payload.current_ch3 | round(2) }} mA
-              </td>
-            </tr>
-            </table>
-            {% endif %}
-          </td>
-          <td align=right>
-            {% if item.payload.barometric_pressure is defined %}
-            {{ item.payload.barometric_pressure | round(2) }} hPa
-            {% endif %}
-          </td>
-          <td align=right>
-            {% if item.payload.relative_humidity is defined and item.payload.relative_humidity %}
-            {{ item.payload.relative_humidity | round(2) }}%
-            {% endif %}
-          </td>
-          <td align=right>
-            {% if item.payload.temperature is defined %}
-            {{ item.payload.temperature | round(2) }} &deg;C
-            {% endif %}
-          </td>
-          <td align=right>
-            {% if item.payload.gas_resistance is defined %}
-            {{ item.payload.gas_resistance | round(2) }} Ohm
-            {% endif %}
-          </td>
-        </tr>
       {% endfor %}
-    </table>
+    </tbody>
+  </table>
 {% endblock %}
diff --git a/templates/static/traceroutes.html.j2 b/templates/static/traceroutes.html.j2
index a6e6b6e..9b35050 100644
--- a/templates/static/traceroutes.html.j2
+++ b/templates/static/traceroutes.html.j2
@@ -3,30 +3,60 @@
 {% block title %}Traceroutes{% endblock %}
 
 {% block content %}
-    <h5 class="mb-2 text-gray-500">Traceroutes</h5>
-    <h1 class="mb-2 text-xl">Traceroutes</h1>
-    <p class="mb-2">
-      Traceroutes as seen by <b>{{ config['server']['node_id'] }}</b> ({{ config['server']['node_id'] }}).
-    </p>
+  <h5 class="mb-2 text-gray-500">Traceroutes</h5>
+  <h1 class="mb-2 text-xl">Traceroutes</h1>
+  <p class="mb-2">
+    Traceroutes as seen by <b>{{ nodes[config['server']['node_id']].shortname }}</b> ({{ config['server']['node_id'] }}).
+  </p>
 
-    <table border=1 cellpadding=4>
+  <table class="w-full max-w-full table-auto border-collapse border border-gray-500 bg-gray-50">
+    <thead>
       <tr>
-        <td>Timestamp</td>
-        <td>From</td>
-        <td>To</td>
-        <td>Hops</td>
-        <td>Route</td>
-        <td>Route Hops</td>
+        <th class="p-1 border border-gray-500 bg-gray-400" align=left>Timestamp</th>
+        <th class="p-1 border border-gray-500 bg-gray-400" align=left>From</th>
+        <th class="p-1 border border-gray-500 bg-gray-400" align=left>To</th>
+        <th class="p-1 border border-gray-500 bg-gray-400" align=left>Hops</th>
+        <th class="p-1 border border-gray-500 bg-gray-400" align=left>Route</th>
+        <th class="p-1 border border-gray-500 bg-gray-400 text-nowrap" align=left>Route Hops</th>
       </tr>
+    </thead>
+    <tbody>
       {% for item in traceroutes %}
         <tr>
-          <td style="white-space:nowrap">{{ datetime.fromtimestamp(item.timestamp).astimezone(zoneinfo) }}</td>
-          <td>{{ item.from }}</td>
-          <td>{{ item.to }}</td>
-          <td>{{ item.hops_away }}</td>
-          <td>{{ item.route | join('<span class="text-gray-300"> > </span>') | safe }}</td>
-          <td>{{ item.route | length }}</td>
+          <td class="p-1 border border-gray-400 text-nowrap">{{ datetime.fromtimestamp(item.timestamp).astimezone(zoneinfo) }}</td>
+          <td class="p-1 border border-gray-400">
+            {% set fnode = nodes[item.from] %}
+            {% if fnode %}
+              <a href="node_{{ item.from }}.html">{{ fnode.shortname }}</a>
+            {% else %}
+              <span class="text-gray-500">UNK</span>
+            {% endif %}
+          </td>
+          <td class="p-1 border border-gray-400">
+            {% set tnode = nodes[item.to] %}
+            {% if tnode %}
+              <a href="node_{{ item.to }}.html">{{ tnode.shortname }}</a>
+            {% else %}
+              <span class="text-gray-500">UNK</span>
+            {% endif %}
+          </td>
+          <td class="p-1 border border-gray-400" align=center>{{ item.hops_away }}</td>
+          <td class="p-1 border border-gray-400">
+            {% for hop in item.route_ids %}
+              {% set hnode = nodes[hop] %}
+              {% if hnode %}
+                <a href="node_{{ hop }}.html">{{ hnode.shortname }}</a>
+              {% else %}
+                <span class="text-gray-500">UNK</span>
+              {% endif %}
+              {% if not loop.last %}
+                <span class="text-gray-300"> > </span>
+              {% endif %}
+            {% endfor %}
+          </td>
+          <td class="p-1 border border-gray-400" align=center>{{ item.route | length }}</td>
         </tr>
       {% endfor %}
-    </table>
+    </tbody>
+  </table>
 {% endblock %}

From b36c7686caab0093434e7fb0bc5585db10931ec7 Mon Sep 17 00:00:00 2001
From: Kevin Elliott <kevin@welikeinc.com>
Date: Tue, 9 Jul 2024 20:44:25 -0700
Subject: [PATCH 20/20] removed partial sidebar template as it was a mistake to
 include

---
 .../static/layout-partial-sidebar.html.j2     | 72 -------------------
 1 file changed, 72 deletions(-)
 delete mode 100644 templates/static/layout-partial-sidebar.html.j2

diff --git a/templates/static/layout-partial-sidebar.html.j2 b/templates/static/layout-partial-sidebar.html.j2
deleted file mode 100644
index 20d03c1..0000000
--- a/templates/static/layout-partial-sidebar.html.j2
+++ /dev/null
@@ -1,72 +0,0 @@
-<!-- Static sidebar for desktop -->
-<div class="hidden lg:fixed lg:inset-y-0 lg:z-50 lg:flex lg:w-60 lg:flex-col">
-  <!-- Sidebar component, swap this element with another sidebar if you like -->
-  <div class="flex flex-col px-6 pb-4 overflow-y-auto bg-gray-300 border-r-2 grow gap-y-5 border-r-cyan-600">
-    <div class="flex items-center h-24 mt-4 shrink-0">
-      <div class="text-2xl">
-        <div class="p-0 m-0">S<span class="text-gray-500">ACRAMENTO</span></div>
-        <div class="p-0 m-0">V<span class="text-gray-500">ALLEY</span></div>
-        <div class="p-0 m-0">M<span class="text-gray-500">ESH</span></div>
-      </div>
-    </div>
-
-    <div>
-      {{ config.mesh.description }}
-    </div>
-
-    {% if config.mesh.url %}
-      <div>
-        <a href="{{ config.mesh.url }}" class="text-xs text-gray-900">Website</a>
-      </div>
-    {% endif %}
-
-    <nav class="flex flex-col flex-1">
-      <h3>Mesh</h3>
-      <div class="mb-2">
-        <a href="chat.html">Chat</a>
-      </div>
-      <div class="mb-2">
-        <a href="map.html">Map</a>
-      </div>
-      <div class="mb-2">
-        <a href="nodes.html">Nodes</a>
-      </div>
-      <div class="mb-2">
-        <a href="neighbors.html">Node Neighbors</a>
-      </div>
-      <div class="mb-2">
-        <a href="network.html">Network</a>
-      </div>
-      <div class="mb-2">
-        <a href="routes.html">Routes</a>
-      </div>
-      <div class="mb-2">
-        <a href="stats.html">Stats</a>
-      </div>
-      <div class="mb-2">
-        <a href="telemetry.html">Telemetry</a>
-      </div>
-      <div class="mb-2">
-        <a href="traceroutes.html">Traceroutes</a>
-      </div>
-    </nav>
-
-    <nav class="flex flex-col flex-1">
-      <h3>Logs</h3>
-      <div class="mb-2">
-        <a href="mesh_log.html">Mesh Messages</a>
-      </div>
-      <div class="mb-2">
-        <a href="mqtt_log.html">MQTT Messages</a>
-      </div>
-    </nav>
-
-    <div class="flex-grow"></div>
-    <div class="flex flex-col">
-      <h5 class="mb-2">Powered by MeshInfo</h5>
-      <a href="https://github.com/kevinelliott/meshinfo" class="text-xs text-gray-500">
-        <img src="https://img.shields.io/github/stars/kevinelliott/meshinfo?style=social" alt="GitHub Stars">
-      </a>
-    </div>
-  </div>
-</div>