From 6f5e8ce49c896be9ea8141d70015547a52739b92 Mon Sep 17 00:00:00 2001 From: Defelo Date: Mon, 9 Sep 2024 22:23:04 +0200 Subject: [PATCH] Add restic backups --- hosts/prod/default.nix | 15 +++- hosts/prod/morpheushelper/default.nix | 3 + hosts/prod/secrets.yml | 7 +- hosts/test/default.nix | 15 +++- hosts/test/secrets.yml | 7 +- modules/backup.nix | 100 ++++++++++++++++++++++++++ modules/default.nix | 1 + modules/postgres.nix | 3 + 8 files changed, 143 insertions(+), 8 deletions(-) create mode 100644 modules/backup.nix diff --git a/hosts/prod/default.nix b/hosts/prod/default.nix index 3019878..7395073 100644 --- a/hosts/prod/default.nix +++ b/hosts/prod/default.nix @@ -1,4 +1,4 @@ -{ +{config, ...}: { imports = [ ./backend ./dns.nix @@ -24,5 +24,16 @@ }; }; - sops.secrets."ssh/private-key".path = "/root/.ssh/id_ed25519"; + backup.targets = { + box = { + repository = "sftp://u381435@u381435.your-storagebox.de:23/backups/prod"; + repositoryPasswordFile = config.sops.secrets."backup/box/repository-password".path; + sshKeyFile = config.sops.secrets."ssh/private-key".path; + }; + }; + + sops.secrets = { + "ssh/private-key".path = "/root/.ssh/id_ed25519"; + "backup/box/repository-password" = {}; + }; } diff --git a/hosts/prod/morpheushelper/default.nix b/hosts/prod/morpheushelper/default.nix index c40bd4b..0e03002 100644 --- a/hosts/prod/morpheushelper/default.nix +++ b/hosts/prod/morpheushelper/default.nix @@ -91,6 +91,9 @@ in { "/var/lib/mysql" ]; + backup.exclude = ["/var/lib/mysql"]; + backup.prepare = "${config.services.mysql.package}/bin/mysqldump --all-databases > mysql-dump.sql"; + sops = { secrets = { "morpheushelper/discord-token" = {}; diff --git a/hosts/prod/secrets.yml b/hosts/prod/secrets.yml index 16e0bdb..19f2328 100644 --- a/hosts/prod/secrets.yml +++ b/hosts/prod/secrets.yml @@ -39,6 +39,9 @@ academy-backend: calendar-secret: ENC[AES256_GCM,data:XNfylTKEGHul9x7hibd2N7u6BZftHsF8HwjYYsA7omOrEko+J6dPCxqZsIa1kV8xGmsCFszLC7ApVPCar9iUjg==,iv:76R0wCTQu59t8b9epVRY1u3ZvUjGgc+eIls+BPL2YJg=,tag:p/CDbYM8N6VvVL4f1Au6Jw==,type:str] challenges-ms: sentry-dsn: ENC[AES256_GCM,data:6uYzelb9WlvoiOlOuI+wOK5RwF2MHidAZWPNiR7KVmmr75xyRUFr2H5BzvpXsMfo3xiVKZ0jPhjf73jtMbA4oHjo/rjIiw4=,iv:UuaEqqtnNJ8H4x3wpmzrtB2N34F7+EWIHwLY0dn2wm8=,tag:SOdlA6qamJGLpOV6nhqjAQ==,type:str] +backup: + box: + repository-password: ENC[AES256_GCM,data:+tuWs+H4m4XP+itLopN5UmM9pqs25WTTpLGgjg9D4P8wPN6xlMHC2/kNgWg6LtmDCHq5JbmBACzRwo/x3+DsXB++HErbbXo1ZnXoUTncB4lVgkeUYWjHM3WA9sBleTxSOoPfbr2gPYRlcP5tS7GMAK47vZ/3MC3j3qQHiAc6JTk=,iv:LBTQE/BTCVkKIriA+st6ak0zalKiiBCBWONyzaSVAN8=,tag:Cmi2FsZod5niytPw4DAWdw==,type:str] sops: kms: [] gcp_kms: [] @@ -54,8 +57,8 @@ sops: Tk96anAyOEFGd1plUlp4SU5LbE5TT1UK6SPKztdzU5K1FjQ5sFjUnF4HK8cAFqh1 YR7o5tur1y/bLMESGS7/j7ofST96NuyU+EVgs/lt0Rd0Voh1Q8aKKQ== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-09-09T16:13:02Z" - mac: ENC[AES256_GCM,data:VUrrPbP8SDNdTA1zz/tpnwwQrs5kr6QdWsxheyD6m29wSOs3hKvLt4x9V/qadv6vRHuCBftbOsGpImP7wfr7z6uBstEKAZya+ObOlZShGhfDkvdECGoCQtA//AmtAYxJAoAuMVDonXasccRUYyZDtEAlWbHz40JyZdG+HqnnA3k=,iv:ILUCt1i63seMsOKYqFJ5L6O2oWY2Nk4Ju9O6W+piSkw=,tag:KUfiQL408aZuUk5NS6/C5w==,type:str] + lastmodified: "2024-09-09T20:17:59Z" + mac: ENC[AES256_GCM,data:2bOVdK08xiAB8cvhlGmvXlrGHS5hsd8Wx9KeNhewiK00RhD14dcBlquDoZVDS9aTsCMpPnme1C92Z1CAD0jFiH46JoEtp8SX98Sv+Lgsbp0B3c+3An8OP7yFodDhsZ51tOnTBtEKfMJ/gp4ZWHaTlsYzZPxS6uV+DDysD7YFSSY=,iv:aK9JJA5yynlZP4WbuqhRahUcxl72cB7RasJweG2JriQ=,tag:Y9dme55yjZyrPcYKu+1n5g==,type:str] pgp: - created_at: "2024-09-09T14:34:26Z" enc: |- diff --git a/hosts/test/default.nix b/hosts/test/default.nix index 2d2d46e..dae1b7d 100644 --- a/hosts/test/default.nix +++ b/hosts/test/default.nix @@ -1,4 +1,4 @@ -{ +{config, ...}: { imports = [ ./backend ./firewall.nix @@ -18,5 +18,16 @@ }; }; - sops.secrets."ssh/private-key".path = "/root/.ssh/id_ed25519"; + backup.targets = { + box = { + repository = "sftp://u381435@u381435.your-storagebox.de:23/backups/test"; + repositoryPasswordFile = config.sops.secrets."backup/box/repository-password".path; + sshKeyFile = config.sops.secrets."ssh/private-key".path; + }; + }; + + sops.secrets = { + "ssh/private-key".path = "/root/.ssh/id_ed25519"; + "backup/box/repository-password" = {}; + }; } diff --git a/hosts/test/secrets.yml b/hosts/test/secrets.yml index e81dfab..cd99b01 100644 --- a/hosts/test/secrets.yml +++ b/hosts/test/secrets.yml @@ -24,6 +24,9 @@ academy-backend: sentry-dsn: ENC[AES256_GCM,data:Oc9PmF//SGKgCPQhDA+aNQf9W75ssomwQZIQLNJRoqLfhMlJfUsis+4gM7keGJQyo2teH6/xzvTL77xQC4TAe8GWTD+O9A==,iv:Tp2WVkc7nO/jJkwN1ZsLVw/h850iWIKnf5Svm1VNEq8=,tag:lThWcFwVlZuBuNm1JG3Hhg==,type:str] ssh: private-key: ENC[AES256_GCM,data:E2xba+ZPcix/pfKl/lU6KEHpvGpQZ1QSwCb3EtOr1QiwnxCX5fy//Jav+7sTnOtoNdiXQNLHp37WSa+y9KtU35mXf+TlU6kaAJItvn7VtrCjXWUUuFNujr3EYJN2Jq5dfrVl0Cf8r3DjwDoYLD3PVqOVcjDSiDI94lJ3prnlqt1UJRA/0O8d9X8iPx0Kr0yzxpOGOlbA3ESmHPMkSDMMiJUNJq5j0DKoX1KI3yuJ7goH3rKdPraVV44bvlL/EiJFT9G03Fup7VKUJTKR/h3y7oyH3ferOcECNQ6Jp6VsC1Q50gAlICXDsobZoVhTOW+ceGzQKFrOtugtWvlCTpIIeN6tDuP2/oG0+FjuaLPBJUuhmwI97eHOhux9KaZAPPsawaaEfo7kakcPdMbx5mjjiEmZTGEHaXoIVUuVWQzbpUlq4cyu7l6s+tlDOW4+krDwesMON0PqsBIKCRYA+sJ0HAtnjirqvs0Q0Hj6Fbh0HFPEsrA4HXEcyJnR0hgaKp19bDAG,iv:2owaBkw3yIpLw/L3n6qAS/sYEqj/fsxbyLYsmy5B3OM=,tag:0ba0ruf16TZFWdYNMFMOKQ==,type:str] +backup: + box: + repository-password: ENC[AES256_GCM,data:iheyA6YgRbOn00D2SMip11U5DGGmU5bCSU+vYPQT6HqlpE+fv4oN/g2RMba4L8fURLsK+wS/MTvIG1eS1lFC7Bv+y8pK0BBPsWWsKWw78/x8CGOqG2DUem3Zm+XoK3We7QflprlExtGPJrOCIVvUN9duLlM8uMBUck8cQRpSGEs=,iv:NK08WFAN8rI/HjcmxtCQ9UsSPPbep2bKqHZmrGgt2jg=,tag:DSet/EbMveuvGisRlCCDyw==,type:str] sops: kms: [] gcp_kms: [] @@ -39,8 +42,8 @@ sops: SkpnU0VUUEpCTnQzcGV0VGUxemowdGMKazKmQYxq1mFF39tdqc456Acv/Mg5fYVg yBAAoulARJkRGHx0PlovW8E6vzgMfiKd1Alb9Z1C3ZV5caZZwADokA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-09-08T12:41:59Z" - mac: ENC[AES256_GCM,data:G0jWaF05nx4bvyHuCZ4Be25OMdIGLwZhARDsWc6edQXRwMzMoveAO52ScAldO41xeOmeAIa2twDB5ONYtA0gBr8kDynFXp8PaQbHt158I0PMiTkyou21gjs87q7yt8LYdVJxJMy1Dp2cZ3Vy2tM1GavigtzdAvLRU877ckz//EQ=,iv:dv/8lVHDbnNFlhjH4h6OymzSAw2BvlHhNKg3JOevG90=,tag:U9qQ9r1D34Ij3HMbTfZqIg==,type:str] + lastmodified: "2024-09-09T20:08:07Z" + mac: ENC[AES256_GCM,data:qic0760clJSg872EGAZ3RTgx9B9789TC9gV0qRo0k6EkTSD1uek5V030Wv9gbtEtciJkAXg5vzfenFRvBPVEVo3kG3UXMz1F4FV6FPP7IXlkwM90EbVLUwmUWC0Yw1SWDEa8XERj13YHm0sPv+mslJ/MDfVF8TBXrNDpjD2gkRo=,iv:6QxyE1IynV+QiJeMlr9E1tJtwv7sF5Lnp5WpdRP2MKE=,tag:jk2cV1U1ARerc31xVxQwJw==,type:str] pgp: - created_at: "2024-09-08T12:41:53Z" enc: |- diff --git a/modules/backup.nix b/modules/backup.nix new file mode 100644 index 0000000..ceb9b2f --- /dev/null +++ b/modules/backup.nix @@ -0,0 +1,100 @@ +{ + config, + lib, + pkgs, + ... +}: { + options.backup = { + targets = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule { + options = { + repository = lib.mkOption { + type = lib.types.str; + }; + repositoryPasswordFile = lib.mkOption { + type = lib.types.path; + }; + environmentFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + }; + sshKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + }; + }; + }); + default = {}; + }; + + exclude = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + }; + + prepare = lib.mkOption { + type = lib.types.lines; + default = ""; + }; + + schedule = lib.mkOption { + type = lib.types.str; + default = "hourly"; + }; + }; + + config = let + cfg = config.backup; + targets = cfg.targets; + + targetConfig = target: { + repository, + repositoryPasswordFile, + environmentFile, + sshKeyFile, + }: { + inherit repository; + timerConfig = null; + passwordFile = repositoryPasswordFile; + environmentFile = lib.mkIf (environmentFile != null) environmentFile; + extraOptions = lib.mkIf (sshKeyFile != null) ["sftp.args='-i ${sshKeyFile}'"]; + + initialize = true; + paths = ["/persistent/data/.snapshots/backup"]; + exclude = map (x: + if lib.hasPrefix "/" x + then "/persistent/data/.snapshots/backup${x}" + else throw "Invalid backup exclude path: ${x}") + cfg.exclude; + }; + in + lib.mkIf (targets != {}) { + systemd.timers.prepare-backup.timerConfig.Persistent = true; + systemd.services.prepare-backup = { + startAt = cfg.schedule; + wants = ["network-online.target"]; + after = ["network-online.target"]; + onSuccess = lib.mapAttrsToList (target: _: "restic-backups-${target}.service") targets; + path = with pkgs; [coreutils btrfs-progs]; + script = '' + set -e + + if [[ -e /persistent/data/backup ]]; then + rm -rf /persistent/data/backup + fi + + mkdir -m 700 /persistent/data/backup + cd /persistent/data/backup + ${cfg.prepare} + date --iso-8601=seconds > /persistent/data/backup/timestamp + + if [[ -e /persistent/data/.snapshots/backup ]]; then + btrfs subvolume delete /persistent/data/.snapshots/backup + fi + btrfs subvolume snapshot -r /persistent/data /persistent/data/.snapshots/backup + ''; + }; + + services.restic.backups = builtins.mapAttrs targetConfig targets; + }; +} diff --git a/modules/default.nix b/modules/default.nix index baa799e..a7a7ee8 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -2,6 +2,7 @@ imports = [ ./acme.nix ./backend + ./backup.nix ./boot.nix ./btrfs.nix ./containers.nix diff --git a/modules/postgres.nix b/modules/postgres.nix index 752a019..e6da53d 100644 --- a/modules/postgres.nix +++ b/modules/postgres.nix @@ -55,5 +55,8 @@ environment.persistence = lib.mkIf config.filesystems.defaultLayout { "/persistent/data".directories = ["/var/lib/postgresql"]; }; + + backup.exclude = ["/var/lib/postgresql"]; + backup.prepare = "${pkgs.sudo}/bin/sudo -u postgres ${cfg.package}/bin/pg_dumpall > postgresql-dump.sql"; }; }