Merge branch 'develop'

config.json.sample

@@ -30,6 +30,7 @@
     }
   },
   "paths": {
+    "backups": "output/backups",
     "data": "output/data",
     "output": "output/static-html",
     "templates": "templates"
@@ -42,9 +43,13 @@
       "data_save": 300,
       "render": 5
     },
+    "backups": {
+      "enabled": true,
+      "interval": 86400
+    },
     "enrich": {
       "enabled": false,
-      "interval": 900
+      "interval": 900,
       "provider": "world.meshinfo.network"
     },
     "graph": {

main.py

@@ -30,6 +30,7 @@ async def main():
 
     data.load()
     await data.save()
+    await data.backup()
 
     async with asyncio.TaskGroup() as tg:
         loop = asyncio.get_event_loop()

memory_data_store.py

@@ -4,6 +4,7 @@
 from datetime import datetime, timedelta
 import json
 import os
+import shutil
 from zoneinfo import ZoneInfo
 import aiohttp
 
@@ -164,9 +165,12 @@ async def save(self):
     since_last_render = (save_start - last_render).total_seconds()
     last_backfill = self.config['server']['last_backfill'] if 'last_backfill' in self.config['server'] else self.config['server']['start_time']
     since_last_backfill = (save_start - last_backfill).total_seconds()
-    print(f"Save (since last): data: {since_last_data} (threshhold: {self.config['server']['intervals']['data_save']}), render: {since_last_render} (threshhold: {self.config['server']['intervals']['render']}), enrich: {since_last_backfill} (threshhold: {self.config['server']['enrich']['interval']})")
+    last_backup = self.config['server']['last_backup'] if 'last_backup' in self.config['server'] else self.config['server']['start_time']
+    since_last_backup = (save_start - last_backup).total_seconds()
+    print(f"Save (since last): data: {since_last_data} (threshhold: {self.config['server']['intervals']['data_save']}), render: {since_last_render} (threshhold: {self.config['server']['intervals']['render']}), enrich: {since_last_backfill} (threshhold: {self.config['server']['enrich']['interval']}), backup: {since_last_backup} (threshhold: {self.config['server']['backups']['interval']})")
 
-    if self.config['server']['enrich']['enabled'] and since_last_backfill >= self.config['server']['enrich']['interval']:
+    if 'enrich' in self.config['server'] and self.config['server']['enrich']['enabled']:
+      if since_last_backfill >= self.config['server']['enrich']['interval']:
         await self.backfill_node_infos()
         end = datetime.now(ZoneInfo(self.config['server']['timezone']))
         print(f"Enriched in {round(end.timestamp() - save_start.timestamp(), 2)} seconds")
@@ -187,8 +191,33 @@ async def save(self):
         print(f"Rendered in {round(end.timestamp() - save_start.timestamp(), 2)} seconds")
         self.config['server']['last_render'] = end
 
+    if 'backups' in self.config['server'] and self.config['server']['backups']['enabled']:
+      if since_last_backup >= self.config['server']['backup']['interval']:
+        await self.backup()
+        end = datetime.now(ZoneInfo(self.config['server']['timezone']))
+        print(f"Backed up in {round(end.timestamp() - save_start.timestamp(), 2)} seconds")
+        self.config['server']['last_backup'] = end
+
   ### helpers
 
+  async def backup(self):
+    now = f"{datetime.now(ZoneInfo(self.config['server']['timezone'])).strftime('%Y%m%d-%H%M%S')}"
+    base_name = f"{self.config['paths']['backups']}/backup-{now}"
+    tmp_path = f"/tmp/meshinfo/backup-{now}"
+    os.makedirs(tmp_path, exist_ok=True)
+    shutil.copytree("output/data", f"{tmp_path}/data")
+    shutil.copytree("output/static-html", f"{tmp_path}/static-html")
+    shutil.copyfile("config.json", f"{tmp_path}/config.json")
+
+    shutil.make_archive(
+      base_name,
+      'bztar',
+      root_dir=tmp_path,
+      base_dir=".",
+      verbose=True)
+    print(f"Backed up to {base_name}.tar.bz2")
+    shutil.rmtree(tmp_path)
+
   async def backfill_node_infos(self):
     nodes_needing_enrichment = {}
     for id, node in self.nodes.items():