Add restic backups

Bootstrap-Academy · Sep 9, 2024 · 6f5e8ce · 6f5e8ce
1 parent f1fa11e
commit 6f5e8ce
Show file tree

Hide file tree

Showing 8 changed files with 143 additions and 8 deletions.
diff --git a/hosts/prod/default.nix b/hosts/prod/default.nix
@@ -1,4 +1,4 @@
-{
+{config, ...}: {
   imports = [
     ./backend
     ./dns.nix
@@ -24,5 +24,16 @@
     };
   };
 
-  sops.secrets."ssh/private-key".path = "/root/.ssh/id_ed25519";
+  backup.targets = {
+    box = {
+      repository = "sftp://[email protected]:23/backups/prod";
+      repositoryPasswordFile = config.sops.secrets."backup/box/repository-password".path;
+      sshKeyFile = config.sops.secrets."ssh/private-key".path;
+    };
+  };
+
+  sops.secrets = {
+    "ssh/private-key".path = "/root/.ssh/id_ed25519";
+    "backup/box/repository-password" = {};
+  };
 }
diff --git a/hosts/prod/morpheushelper/default.nix b/hosts/prod/morpheushelper/default.nix
@@ -91,6 +91,9 @@ in {
     "/var/lib/mysql"
   ];
 
+  backup.exclude = ["/var/lib/mysql"];
+  backup.prepare = "${config.services.mysql.package}/bin/mysqldump --all-databases > mysql-dump.sql";
+
   sops = {
     secrets = {
       "morpheushelper/discord-token" = {};

diff --git a/hosts/prod/secrets.yml b/hosts/prod/secrets.yml
@@ -39,6 +39,9 @@ academy-backend:
         calendar-secret: ENC[AES256_GCM,data:XNfylTKEGHul9x7hibd2N7u6BZftHsF8HwjYYsA7omOrEko+J6dPCxqZsIa1kV8xGmsCFszLC7ApVPCar9iUjg==,iv:76R0wCTQu59t8b9epVRY1u3ZvUjGgc+eIls+BPL2YJg=,tag:p/CDbYM8N6VvVL4f1Au6Jw==,type:str]
     challenges-ms:
         sentry-dsn: ENC[AES256_GCM,data:6uYzelb9WlvoiOlOuI+wOK5RwF2MHidAZWPNiR7KVmmr75xyRUFr2H5BzvpXsMfo3xiVKZ0jPhjf73jtMbA4oHjo/rjIiw4=,iv:UuaEqqtnNJ8H4x3wpmzrtB2N34F7+EWIHwLY0dn2wm8=,tag:SOdlA6qamJGLpOV6nhqjAQ==,type:str]
+backup:
+    box:
+        repository-password: ENC[AES256_GCM,data:+tuWs+H4m4XP+itLopN5UmM9pqs25WTTpLGgjg9D4P8wPN6xlMHC2/kNgWg6LtmDCHq5JbmBACzRwo/x3+DsXB++HErbbXo1ZnXoUTncB4lVgkeUYWjHM3WA9sBleTxSOoPfbr2gPYRlcP5tS7GMAK47vZ/3MC3j3qQHiAc6JTk=,iv:LBTQE/BTCVkKIriA+st6ak0zalKiiBCBWONyzaSVAN8=,tag:Cmi2FsZod5niytPw4DAWdw==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -54,8 +57,8 @@ sops:
             Tk96anAyOEFGd1plUlp4SU5LbE5TT1UK6SPKztdzU5K1FjQ5sFjUnF4HK8cAFqh1
             YR7o5tur1y/bLMESGS7/j7ofST96NuyU+EVgs/lt0Rd0Voh1Q8aKKQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-09-09T16:13:02Z"
-    mac: ENC[AES256_GCM,data:VUrrPbP8SDNdTA1zz/tpnwwQrs5kr6QdWsxheyD6m29wSOs3hKvLt4x9V/qadv6vRHuCBftbOsGpImP7wfr7z6uBstEKAZya+ObOlZShGhfDkvdECGoCQtA//AmtAYxJAoAuMVDonXasccRUYyZDtEAlWbHz40JyZdG+HqnnA3k=,iv:ILUCt1i63seMsOKYqFJ5L6O2oWY2Nk4Ju9O6W+piSkw=,tag:KUfiQL408aZuUk5NS6/C5w==,type:str]
+    lastmodified: "2024-09-09T20:17:59Z"
+    mac: ENC[AES256_GCM,data:2bOVdK08xiAB8cvhlGmvXlrGHS5hsd8Wx9KeNhewiK00RhD14dcBlquDoZVDS9aTsCMpPnme1C92Z1CAD0jFiH46JoEtp8SX98Sv+Lgsbp0B3c+3An8OP7yFodDhsZ51tOnTBtEKfMJ/gp4ZWHaTlsYzZPxS6uV+DDysD7YFSSY=,iv:aK9JJA5yynlZP4WbuqhRahUcxl72cB7RasJweG2JriQ=,tag:Y9dme55yjZyrPcYKu+1n5g==,type:str]
     pgp:
         - created_at: "2024-09-09T14:34:26Z"
           enc: |-

diff --git a/hosts/test/default.nix b/hosts/test/default.nix
@@ -1,4 +1,4 @@
-{
+{config, ...}: {
   imports = [
     ./backend
     ./firewall.nix
@@ -18,5 +18,16 @@
     };
   };
 
-  sops.secrets."ssh/private-key".path = "/root/.ssh/id_ed25519";
+  backup.targets = {
+    box = {
+      repository = "sftp://[email protected]:23/backups/test";
+      repositoryPasswordFile = config.sops.secrets."backup/box/repository-password".path;
+      sshKeyFile = config.sops.secrets."ssh/private-key".path;
+    };
+  };
+
+  sops.secrets = {
+    "ssh/private-key".path = "/root/.ssh/id_ed25519";
+    "backup/box/repository-password" = {};
+  };
 }
diff --git a/hosts/test/secrets.yml b/hosts/test/secrets.yml
@@ -24,6 +24,9 @@ academy-backend:
         sentry-dsn: ENC[AES256_GCM,data:Oc9PmF//SGKgCPQhDA+aNQf9W75ssomwQZIQLNJRoqLfhMlJfUsis+4gM7keGJQyo2teH6/xzvTL77xQC4TAe8GWTD+O9A==,iv:Tp2WVkc7nO/jJkwN1ZsLVw/h850iWIKnf5Svm1VNEq8=,tag:lThWcFwVlZuBuNm1JG3Hhg==,type:str]
 ssh:
     private-key: ENC[AES256_GCM,data:E2xba+ZPcix/pfKl/lU6KEHpvGpQZ1QSwCb3EtOr1QiwnxCX5fy//Jav+7sTnOtoNdiXQNLHp37WSa+y9KtU35mXf+TlU6kaAJItvn7VtrCjXWUUuFNujr3EYJN2Jq5dfrVl0Cf8r3DjwDoYLD3PVqOVcjDSiDI94lJ3prnlqt1UJRA/0O8d9X8iPx0Kr0yzxpOGOlbA3ESmHPMkSDMMiJUNJq5j0DKoX1KI3yuJ7goH3rKdPraVV44bvlL/EiJFT9G03Fup7VKUJTKR/h3y7oyH3ferOcECNQ6Jp6VsC1Q50gAlICXDsobZoVhTOW+ceGzQKFrOtugtWvlCTpIIeN6tDuP2/oG0+FjuaLPBJUuhmwI97eHOhux9KaZAPPsawaaEfo7kakcPdMbx5mjjiEmZTGEHaXoIVUuVWQzbpUlq4cyu7l6s+tlDOW4+krDwesMON0PqsBIKCRYA+sJ0HAtnjirqvs0Q0Hj6Fbh0HFPEsrA4HXEcyJnR0hgaKp19bDAG,iv:2owaBkw3yIpLw/L3n6qAS/sYEqj/fsxbyLYsmy5B3OM=,tag:0ba0ruf16TZFWdYNMFMOKQ==,type:str]
+backup:
+    box:
+        repository-password: ENC[AES256_GCM,data:iheyA6YgRbOn00D2SMip11U5DGGmU5bCSU+vYPQT6HqlpE+fv4oN/g2RMba4L8fURLsK+wS/MTvIG1eS1lFC7Bv+y8pK0BBPsWWsKWw78/x8CGOqG2DUem3Zm+XoK3We7QflprlExtGPJrOCIVvUN9duLlM8uMBUck8cQRpSGEs=,iv:NK08WFAN8rI/HjcmxtCQ9UsSPPbep2bKqHZmrGgt2jg=,tag:DSet/EbMveuvGisRlCCDyw==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -39,8 +42,8 @@ sops:
             SkpnU0VUUEpCTnQzcGV0VGUxemowdGMKazKmQYxq1mFF39tdqc456Acv/Mg5fYVg
             yBAAoulARJkRGHx0PlovW8E6vzgMfiKd1Alb9Z1C3ZV5caZZwADokA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-09-08T12:41:59Z"
-    mac: ENC[AES256_GCM,data:G0jWaF05nx4bvyHuCZ4Be25OMdIGLwZhARDsWc6edQXRwMzMoveAO52ScAldO41xeOmeAIa2twDB5ONYtA0gBr8kDynFXp8PaQbHt158I0PMiTkyou21gjs87q7yt8LYdVJxJMy1Dp2cZ3Vy2tM1GavigtzdAvLRU877ckz//EQ=,iv:dv/8lVHDbnNFlhjH4h6OymzSAw2BvlHhNKg3JOevG90=,tag:U9qQ9r1D34Ij3HMbTfZqIg==,type:str]
+    lastmodified: "2024-09-09T20:08:07Z"
+    mac: ENC[AES256_GCM,data:qic0760clJSg872EGAZ3RTgx9B9789TC9gV0qRo0k6EkTSD1uek5V030Wv9gbtEtciJkAXg5vzfenFRvBPVEVo3kG3UXMz1F4FV6FPP7IXlkwM90EbVLUwmUWC0Yw1SWDEa8XERj13YHm0sPv+mslJ/MDfVF8TBXrNDpjD2gkRo=,iv:6QxyE1IynV+QiJeMlr9E1tJtwv7sF5Lnp5WpdRP2MKE=,tag:jk2cV1U1ARerc31xVxQwJw==,type:str]
     pgp:
         - created_at: "2024-09-08T12:41:53Z"
           enc: |-

diff --git a/modules/backup.nix b/modules/backup.nix
@@ -0,0 +1,100 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: {
+  options.backup = {
+    targets = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.submodule {
+        options = {
+          repository = lib.mkOption {
+            type = lib.types.str;
+          };
+          repositoryPasswordFile = lib.mkOption {
+            type = lib.types.path;
+          };
+          environmentFile = lib.mkOption {
+            type = lib.types.nullOr lib.types.path;
+            default = null;
+          };
+          sshKeyFile = lib.mkOption {
+            type = lib.types.nullOr lib.types.path;
+            default = null;
+          };
+        };
+      });
+      default = {};
+    };
+
+    exclude = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [];
+    };
+
+    prepare = lib.mkOption {
+      type = lib.types.lines;
+      default = "";
+    };
+
+    schedule = lib.mkOption {
+      type = lib.types.str;
+      default = "hourly";
+    };
+  };
+
+  config = let
+    cfg = config.backup;
+    targets = cfg.targets;
+
+    targetConfig = target: {
+      repository,
+      repositoryPasswordFile,
+      environmentFile,
+      sshKeyFile,
+    }: {
+      inherit repository;
+      timerConfig = null;
+      passwordFile = repositoryPasswordFile;
+      environmentFile = lib.mkIf (environmentFile != null) environmentFile;
+      extraOptions = lib.mkIf (sshKeyFile != null) ["sftp.args='-i ${sshKeyFile}'"];
+
+      initialize = true;
+      paths = ["/persistent/data/.snapshots/backup"];
+      exclude = map (x:
+        if lib.hasPrefix "/" x
+        then "/persistent/data/.snapshots/backup${x}"
+        else throw "Invalid backup exclude path: ${x}")
+      cfg.exclude;
+    };
+  in
+    lib.mkIf (targets != {}) {
+      systemd.timers.prepare-backup.timerConfig.Persistent = true;
+      systemd.services.prepare-backup = {
+        startAt = cfg.schedule;
+        wants = ["network-online.target"];
+        after = ["network-online.target"];
+        onSuccess = lib.mapAttrsToList (target: _: "restic-backups-${target}.service") targets;
+        path = with pkgs; [coreutils btrfs-progs];
+        script = ''
+          set -e
+
+          if [[ -e /persistent/data/backup ]]; then
+            rm -rf /persistent/data/backup
+          fi
+
+          mkdir -m 700 /persistent/data/backup
+          cd /persistent/data/backup
+          ${cfg.prepare}
+          date --iso-8601=seconds > /persistent/data/backup/timestamp
+
+          if [[ -e /persistent/data/.snapshots/backup ]]; then
+            btrfs subvolume delete /persistent/data/.snapshots/backup
+          fi
+          btrfs subvolume snapshot -r /persistent/data /persistent/data/.snapshots/backup
+        '';
+      };
+
+      services.restic.backups = builtins.mapAttrs targetConfig targets;
+    };
+}
diff --git a/modules/default.nix b/modules/default.nix
@@ -2,6 +2,7 @@
   imports = [
     ./acme.nix
     ./backend
+    ./backup.nix
     ./boot.nix
     ./btrfs.nix
     ./containers.nix

diff --git a/modules/postgres.nix b/modules/postgres.nix
@@ -55,5 +55,8 @@
       environment.persistence = lib.mkIf config.filesystems.defaultLayout {
         "/persistent/data".directories = ["/var/lib/postgresql"];
       };
+
+      backup.exclude = ["/var/lib/postgresql"];
+      backup.prepare = "${pkgs.sudo}/bin/sudo -u postgres ${cfg.package}/bin/pg_dumpall > postgresql-dump.sql";
     };
 }