From 431dd058c5b803ec42a6f40227d398e6f7539361 Mon Sep 17 00:00:00 2001 From: Pierre Penninckx Date: Tue, 12 Nov 2024 21:40:52 +0100 Subject: [PATCH] Backup contracts for files and databases (#344) This PR continues the work started in https://github.com/ibizaman/selfhostblocks/pull/314 I had to create my own PR since I couldn't add commits on the fork. --------- Co-authored-by: sivertism <10866270+sivertism@users.noreply.github.com> --- CHANGELOG.md | 7 + docs/blocks.md | 4 + docs/contracts.md | 11 +- docs/default.nix | 10 + flake.nix | 2 + lib/default.nix | 2 +- modules/blocks/ldap.nix | 11 +- modules/blocks/postgresql.nix | 35 ++ modules/blocks/postgresql/docs/default.md | 39 ++ modules/blocks/restic.nix | 390 ++++++++++++------ modules/blocks/restic/docs/default.md | 88 ++-- modules/blocks/restic/dummyModule.nix | 3 + modules/contracts/backup.nix | 80 ++-- modules/contracts/backup/docs/default.md | 63 +-- modules/contracts/backup/dummyModule.nix | 6 +- modules/contracts/backup/test.nix | 121 ++++++ modules/contracts/databasebackup.nix | 77 ++++ .../contracts/databasebackup/docs/default.md | 100 +++++ .../contracts/databasebackup/dummyModule.nix | 43 ++ modules/contracts/databasebackup/test.nix | 84 ++++ modules/contracts/default.nix | 3 + modules/services/arr.nix | 11 +- modules/services/audiobookshelf.nix | 11 +- modules/services/deluge.nix | 11 +- modules/services/forgejo.nix | 11 +- modules/services/forgejo/docs/default.md | 7 +- modules/services/grocy.nix | 11 +- modules/services/hledger.nix | 11 +- modules/services/home-assistant.nix | 11 +- modules/services/jellyfin.nix | 11 +- modules/services/nextcloud-server.nix | 11 +- .../services/nextcloud-server/docs/default.md | 7 +- modules/services/vaultwarden.nix | 11 +- modules/services/vaultwarden/docs/default.md | 7 +- test/blocks/restic.nix | 92 ++--- test/contracts/backup.nix | 43 ++ test/contracts/databasebackup.nix | 32 ++ test/services/vaultwarden.nix | 15 +- 38 files changed, 1128 insertions(+), 364 deletions(-) create mode 100644 modules/blocks/postgresql/docs/default.md create mode 100644 modules/blocks/restic/dummyModule.nix create mode 100644 modules/contracts/backup/test.nix create mode 100644 modules/contracts/databasebackup.nix create mode 100644 modules/contracts/databasebackup/docs/default.md create mode 100644 modules/contracts/databasebackup/dummyModule.nix create mode 100644 modules/contracts/databasebackup/test.nix create mode 100644 test/contracts/backup.nix create mode 100644 test/contracts/databasebackup.nix diff --git a/CHANGELOG.md b/CHANGELOG.md index d09ee594..5ac0224c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Upcoming Release +## New Features + +- Backup: + - Add feature to backup databases with the database backup contract, implemented with `shb.restic.databases`. + ## Breaking Changes - Remove dependency on `sops-nix`. @@ -33,6 +38,8 @@ - `shb.forgejo.sso.secretFileForAuthelia` -> `shb.forgejo.ldap.sharedSecretForAuthelia.result.path`. - `shb.forgejo.adminPasswordFile` -> `shb.forgejo.adminPassword.result.path`. - `shb.forgejo.databasePasswordFile` -> `shb.forgejo.databasePassword.result.path`. +- Backup: + - `shb.restic.instances` options has been split between `shb.restic.instances.request` and `shb.restic.instances.settings`, matching better with contracts. ## User Facing Backwards Compatible Changes diff --git a/docs/blocks.md b/docs/blocks.md index 9eff5964..e9781289 100644 --- a/docs/blocks.md +++ b/docs/blocks.md @@ -34,6 +34,10 @@ Not all blocks are yet documented. You can find all available blocks [in the rep modules/blocks/ssl/docs/default.md ``` +```{=include=} chapters html:into-file=//blocks-postgresql.html +modules/blocks/postgresql/docs/default.md +``` + ```{=include=} chapters html:into-file=//blocks-restic.html modules/blocks/restic/docs/default.md ``` diff --git a/docs/contracts.md b/docs/contracts.md index 919ef84a..31ed94ed 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -20,10 +20,13 @@ as possible, reducing the quite thick layer that it is now. Provided contracts are: - [SSL generator contract](contracts-ssl.html) to generate SSL certificates. - Two implementations are provided: self-signed and Let's Encrypt. + Two providers are implemented: self-signed and Let's Encrypt. - [Backup contract](contracts-backup.html) to backup directories. - This contract allows to backup multiple times the same directories for extra protection. + One provider is implemented: Restic. +- [Database Backup contract](contracts-databasebackup.html) to backup database dumps. + One provider is implemented: Restic. - [Secret contract](contracts-secret.html) to provide secrets that are deployed outside of the Nix store. + One provider is implemented: SOPS. ```{=include=} chapters html:into-file=//contracts-ssl.html modules/contracts/ssl/docs/default.md @@ -33,6 +36,10 @@ modules/contracts/ssl/docs/default.md modules/contracts/backup/docs/default.md ``` +```{=include=} chapters html:into-file=//contracts-databasebackup.html +modules/contracts/databasebackup/docs/default.md +``` + ```{=include=} chapters html:into-file=//contracts-secret.html modules/contracts/secret/docs/default.md ``` diff --git a/docs/default.nix b/docs/default.nix index b76fed54..410d5e4f 100644 --- a/docs/default.nix +++ b/docs/default.nix @@ -132,6 +132,11 @@ in stdenv.mkDerivation { '@OPTIONS_JSON@' \ ${individualModuleOptionsDocs [ ../modules/blocks/ssl.nix ]}/share/doc/nixos/options.json + substituteInPlace ./modules/blocks/postgresql/docs/default.md \ + --replace \ + '@OPTIONS_JSON@' \ + ${individualModuleOptionsDocs [ ../modules/blocks/postgresql.nix ]}/share/doc/nixos/options.json + substituteInPlace ./modules/blocks/restic/docs/default.md \ --replace \ '@OPTIONS_JSON@' \ @@ -160,6 +165,11 @@ in stdenv.mkDerivation { '@OPTIONS_JSON@' \ ${individualModuleOptionsDocs [ ../modules/contracts/backup/dummyModule.nix ]}/share/doc/nixos/options.json + substituteInPlace ./modules/contracts/databasebackup/docs/default.md \ + --replace \ + '@OPTIONS_JSON@' \ + ${individualModuleOptionsDocs [ ../modules/contracts/databasebackup/dummyModule.nix ]}/share/doc/nixos/options.json + substituteInPlace ./modules/contracts/secret/docs/default.md \ --replace \ '@OPTIONS_JSON@' \ diff --git a/flake.nix b/flake.nix index 9e7be65f..d72ff8c7 100644 --- a/flake.nix +++ b/flake.nix @@ -136,6 +136,8 @@ // (vm_test "restic" ./test/blocks/restic.nix) // (vm_test "ssl" ./test/blocks/ssl.nix) + // (vm_test "contracts-backup" ./test/contracts/backup.nix) + // (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix) // (vm_test "contracts-secret" ./test/contracts/secret.nix) )); } diff --git a/lib/default.nix b/lib/default.nix index b935a247..9350a525 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -248,7 +248,7 @@ rec { results = pkgs.lib.runTests tests; in if results != [ ] then - builtins.throw (concatStringsSep "\n" (map resultToString (lib.traceValSeqN 3 results))) + builtins.throw (concatStringsSep "\n" (map resultToString (lib.traceValSeq results))) else pkgs.runCommand "nix-flake-tests-success" { } "echo > $out"; diff --git a/modules/blocks/ldap.nix b/modules/blocks/ldap.nix index 48f9ca8d..6bee35b2 100644 --- a/modules/blocks/ldap.nix +++ b/modules/blocks/ldap.nix @@ -95,7 +95,7 @@ in }; backup = lib.mkOption { - type = contracts.backup; + type = contracts.backup.request; description = '' Backup configuration. This is an output option. @@ -104,10 +104,11 @@ in ``` shb.restic.instances."lldap" = { - enable = true; - - # Options specific to Restic. - } // config.shb.lldap.backup; + request = config.shb.lldap.backup; + settings = { + enable = true; + }; + }; ``` ''; readOnly = true; diff --git a/modules/blocks/postgresql.nix b/modules/blocks/postgresql.nix index 1c2bfd07..cf48ce41 100644 --- a/modules/blocks/postgresql.nix +++ b/modules/blocks/postgresql.nix @@ -1,6 +1,7 @@ { config, lib, pkgs, ... }: let cfg = config.shb.postgresql; + contracts = pkgs.callPackage ../contracts {}; upgrade-script = old: new: let @@ -49,6 +50,40 @@ in default = false; }; + databasebackup = lib.mkOption { + description = '' + Backup configuration. This is an output option. + + Use it to initialize a block implementing the "backup" contract. + For example, with the restic block: + + ``` + shb.restic.instances."postgresql" = { + request = config.shb.postgresl.backup; + settings = { + enable = true; + }; + }; + ``` + ''; + + type = contracts.databasebackup.requestType; + + default = { + user = "postgres"; + + backupFile = "postgres.sql"; + + backupCmd = '' + ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable + ''; + + restoreCmd = '' + ${pkgs.gzip}/bin/gunzip | ${pkgs.postgresql}/bin/psql postgres + ''; + }; + }; + ensures = lib.mkOption { description = "List of username, database and/or passwords that should be created."; type = lib.types.listOf (lib.types.submodule { diff --git a/modules/blocks/postgresql/docs/default.md b/modules/blocks/postgresql/docs/default.md new file mode 100644 index 00000000..f8dc9337 --- /dev/null +++ b/modules/blocks/postgresql/docs/default.md @@ -0,0 +1,39 @@ +# PostgreSQL Block {#blocks-postgresql} + +Defined in [`/modules/blocks/postgresql.nix`](@REPO@/modules/blocks/postgresql.nix). + +This block sets up a [PostgreSQL][] database. + +[postgresql]: https://www.postgresql.org/ + +## Tests {#blocks-postgresql-tests} + +Specific integration tests are defined in [`/test/blocks/postgresql.nix`](@REPO@/test/blocks/postgresql.nix). + +## Database Backup Requester Contracts {#blocks-postgresql-contract-databasebackup} + +This block can be backed up using the [database backup](contracts-databasebackup.html) contract. + +Contract integration tests are defined in [`/test/contracts/databasebackup.nix`](@REPO@/test/contracts/databasebackup.nix). + +### Backing up All Databases {#blocks-postgresql-contract-databasebackup-all} + +```nix +{ + my.backup.provider."postgresql" = { + request = config.shb.postgresql.databasebackup; + + settings = { + // Specific options for the backup provider. + }; + }; +} +``` + +## Options Reference {#blocks-postgresql-options} + +```{=include=} options +id-prefix: blocks-postgresql-options- +list-id: selfhostblocks-block-postgresql-options +source: @OPTIONS_JSON@ +``` diff --git a/modules/blocks/restic.nix b/modules/blocks/restic.nix index 5b0cbd2c..e9c33512 100644 --- a/modules/blocks/restic.nix +++ b/modules/blocks/restic.nix @@ -4,44 +4,36 @@ let cfg = config.shb.restic; shblib = pkgs.callPackage ../../lib {}; + contracts = pkgs.callPackage ../contracts {}; - instanceOptions = { - enable = lib.mkEnableOption "shb restic. A disabled instance will not backup data anymore but still provides the helper tool to introspect and rollback snapshots"; + inherit (lib) concatStringsSep filterAttrs flatten literalExpression optionals optionalString listToAttrs mapAttrsToList mkEnableOption mkOption mkMerge; + inherit (lib) generators hasPrefix mkIf nameValuePair optionalAttrs removePrefix; + inherit (lib.types) attrsOf enum int ints listOf oneOf nonEmptyListOf nonEmptyStr nullOr path str submodule; - passphraseFile = lib.mkOption { - description = "Encryption key for the backups."; - type = lib.types.path; - }; - - user = lib.mkOption { - description = '' - Unix user doing the backups. Must be the user owning the files to be backed up. - ''; - type = lib.types.str; - }; + commonOptions = { + enable = mkEnableOption '' + this backup intance. - sourceDirectories = lib.mkOption { - description = "Source directories."; - type = lib.types.nonEmptyListOf lib.types.str; - }; + A disabled instance will not backup data anymore + but still provides the helper tool to restore snapshots + ''; - excludePatterns = lib.mkOption { - description = "Exclude patterns."; - type = lib.types.listOf lib.types.str; - default = []; + passphraseFile = mkOption { + description = "Encryption key for the backups."; + type = path; }; - repositories = lib.mkOption { + repository = mkOption { description = "Repositories to back this instance to."; - type = lib.types.nonEmptyListOf (lib.types.submodule { + type = submodule { options = { - path = lib.mkOption { - type = lib.types.str; + path = mkOption { + type = str; description = "Repository location"; }; - secrets = lib.mkOption { - type = lib.types.attrsOf shblib.secretFileType; + secrets = mkOption { + type = attrsOf shblib.secretFileType; default = {}; description = '' Secrets needed to access the repository where the backups will be stored. @@ -49,17 +41,17 @@ let See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets. - ''; - example = lib.literalExpression '' + ''; + example = literalExpression '' { - AWS_ACCESS_KEY_ID = ; - AWS_SECRET_ACCESS_KEY = ; + AWS_ACCESS_KEY_ID.source = ; + AWS_SECRET_ACCESS_KEY.source = ; } - ''; + ''; }; - timerConfig = lib.mkOption { - type = lib.types.attrsOf utils.systemdUtils.unitOptions.unitOption; + timerConfig = mkOption { + type = attrsOf utils.systemdUtils.unitOptions.unitOption; default = { OnCalendar = "daily"; Persistent = true; @@ -72,12 +64,12 @@ let }; }; }; - }); + }; }; - retention = lib.mkOption { + retention = mkOption { description = "For how long to keep backup files."; - type = lib.types.attrsOf (lib.types.oneOf [ lib.types.int lib.types.nonEmptyStr ]); + type = attrsOf (oneOf [ int nonEmptyStr ]); default = { keep_within = "1d"; keep_hourly = 24; @@ -87,73 +79,139 @@ let }; }; - hooks = lib.mkOption { - description = "Hooks to run before or after the backup."; - default = {}; - type = lib.types.submodule { - options = { - before_backup = lib.mkOption { - description = "Hooks to run before backup"; - type = lib.types.listOf lib.types.str; - default = []; - }; - - after_backup = lib.mkOption { - description = "Hooks to run after backup"; - type = lib.types.listOf lib.types.str; - default = []; - }; - }; - }; - }; - - limitUploadKiBs = lib.mkOption { - type = lib.types.nullOr lib.types.int; + limitUploadKiBs = mkOption { + type = nullOr int; description = "Limit upload bandwidth to the given KiB/s amount."; default = null; example = 8000; }; - limitDownloadKiBs = lib.mkOption { - type = lib.types.nullOr lib.types.int; + limitDownloadKiBs = mkOption { + type = nullOr int; description = "Limit download bandwidth to the given KiB/s amount."; default = null; example = 8000; }; }; - repoSlugName = name: builtins.replaceStrings ["/" ":"] ["_" "_"] (lib.strings.removePrefix "/" name); + repoSlugName = name: builtins.replaceStrings ["/" ":"] ["_" "_"] (removePrefix "/" name); backupName = name: repository: "${name}_${repoSlugName repository.path}"; fullName = name: repository: "restic-backups-${name}_${repoSlugName repository.path}"; in { options.shb.restic = { - instances = lib.mkOption { - description = "Each instance is a backup setting"; + instances = mkOption { + description = "Each instance is backing up some directories to one repository."; default = {}; - type = lib.types.attrsOf (lib.types.submodule { - options = instanceOptions; - }); + type = attrsOf (submodule ({ name, options, ... }: { + options = { + request = mkOption { + description = '' + Request part of the backup contract. + + Accepts values from a requester. + ''; + + type = contracts.backup.request; + }; + + settings = mkOption { + description = '' + Settings specific to the Restic provider. + ''; + + type = submodule { + options = commonOptions; + }; + }; + + result = mkOption { + description = '' + Result part of the backup contract. + + Contains the output of the Restic provider. + ''; + type = lib.types.anything; # contracts.databasebackup.result; + default = { + restoreScript = fullName name options.settings.value.repository; + backupService = "${fullName name options.settings.value.repository}.service"; + }; + defaultText = literalExpression '' + { + restoreScript = "${fullName "" { path = "path/to/repository"; }}"; + backupService = "${fullName "" { path = "path/to/repository"; }}.service"; + } + ''; + }; + }; + })); + }; + + databases = mkOption { + description = "Databases to backup following the database backup contract."; + default = {}; + type = attrsOf (submodule ({ name, options, ... }: { + options = { + request = mkOption { + description = '' + Request part of the backup contract. + + Accepts values from a requester. + ''; + + type = contracts.databasebackup.requestType; + }; + + settings = mkOption { + description = '' + Settings specific to the Restic provider. + ''; + + type = submodule { + options = commonOptions; + }; + }; + + result = mkOption { + description = '' + Result part of the backup contract. + + Contains the output of the Restic provider. + ''; + type = contracts.databasebackup.resultType; + default = { + restoreScript = fullName name options.settings.value.repository; + backupService = "${fullName name options.settings.value.repository}.service"; + }; + defaultText = literalExpression '' + { + restoreScript = "${fullName "" { path = "path/to/repository"; }}"; + backupService = "${fullName "" { path = "path/to/repository"; }}.service"; + } + ''; + }; + }; + })); }; # Taken from https://github.com/HubbeKing/restic-kubernetes/blob/73bfbdb0ba76939a4c52173fa2dbd52070710008/README.md?plain=1#L23 - performance = lib.mkOption { + performance = mkOption { description = "Reduce performance impact of backup jobs."; default = {}; - type = lib.types.submodule { + type = submodule { options = { - niceness = lib.mkOption { - type = lib.types.ints.between (-20) 19; + niceness = mkOption { + type = ints.between (-20) 19; description = "nice priority adjustment, defaults to 15 for ~20% CPU time of normal-priority process"; default = 15; }; - ioSchedulingClass = lib.mkOption { - type = lib.types.enum [ "idle" "best-effort" "realtime" ]; + ioSchedulingClass = mkOption { + type = enum [ "idle" "best-effort" "realtime" ]; description = "ionice scheduling class, defaults to best-effort IO. Only used for `restic backup`, `restic forget` and `restic check` commands."; default = "best-effort"; }; - ioPriority = lib.mkOption { - type = lib.types.nullOr (lib.types.ints.between 0 7); + ioPriority = mkOption { + type = nullOr (ints.between 0 7); description = "ionice priority, defaults to 7 for lowest priority IO. Only used for `restic backup`, `restic forget` and `restic check` commands."; default = 7; }; @@ -162,80 +220,121 @@ in }; }; - config = lib.mkIf (cfg.instances != {}) ( + config = mkIf (cfg.instances != {} || cfg.databases != {}) ( let - enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances; - in lib.mkMerge [ + enabledInstances = filterAttrs (k: i: i.settings.enable) cfg.instances; + enabledDatabases = filterAttrs (k: i: i.settings.enable) cfg.databases; + in mkMerge [ { - environment.systemPackages = lib.optionals (enabledInstances != {}) [ pkgs.restic ]; - + environment.systemPackages = optionals (enabledInstances != {} || enabledDatabases != {}) [ pkgs.restic ]; + } + { + # Create repository if it is a local path. systemd.tmpfiles.rules = let - mkRepositorySettings = name: instance: repository: lib.optionals (lib.hasPrefix "/" repository.path) [ - "d '${repository.path}' 0750 ${instance.user} root - -" + mkSettings = name: instance: optionals (hasPrefix "/" instance.settings.repository.path) [ + "d '${instance.settings.repository.path}' 0750 ${instance.request.user} root - -" ]; - - mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories; in - lib.flatten (lib.attrsets.mapAttrsToList mkSettings cfg.instances); - + flatten (mapAttrsToList mkSettings (cfg.instances // cfg.databases)); + } + { services.restic.backups = let - mkRepositorySettings = name: instance: repository: { - "${name}_${repoSlugName repository.path}" = { - inherit (instance) user; + mkSettings = name: instance: { + "${name}_${repoSlugName instance.settings.repository.path}" = { + inherit (instance.request) user; - repository = repository.path; + repository = instance.settings.repository.path; - paths = instance.sourceDirectories; + paths = instance.request.sourceDirectories; - passwordFile = toString instance.passphraseFile; + passwordFile = toString instance.settings.passphraseFile; initialize = true; - inherit (repository) timerConfig; + inherit (instance.settings.repository) timerConfig; - pruneOpts = lib.mapAttrsToList (name: value: + pruneOpts = mapAttrsToList (name: value: "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}" - ) instance.retention; + ) instance.settings.retention; - backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup; + backupPrepareCommand = concatStringsSep "\n" instance.request.hooks.before_backup; - backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup; - } // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) { - exclude = instance.excludePatterns; + backupCleanupCommand = concatStringsSep "\n" instance.request.hooks.after_backup; extraBackupArgs = - (lib.optionals (instance.limitUploadKiBs != null) [ - "--limit-upload=${toString instance.limitUploadKiBs}" + (optionals (instance.settings.limitUploadKiBs != null) [ + "--limit-upload=${toString instance.settings.limitUploadKiBs}" ]) - ++ (lib.optionals (instance.limitDownloadKiBs != null) [ - "--limit-download=${toString instance.limitDownloadKiBs}" + ++ (optionals (instance.settings.limitDownloadKiBs != null) [ + "--limit-download=${toString instance.settings.limitDownloadKiBs}" ]); + } // optionalAttrs (builtins.length instance.request.excludePatterns > 0) { + exclude = instance.request.excludePatterns; }; }; - - mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories; in - lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings enabledInstances)); + mkMerge (flatten (mapAttrsToList mkSettings enabledInstances)); + } + { + services.restic.backups = + let + mkSettings = name: instance: { + "${name}_${repoSlugName instance.settings.repository.path}" = { + inherit (instance.request) user; + + repository = instance.settings.repository.path; + dynamicFilesFrom = "echo"; + + passwordFile = toString instance.settings.passphraseFile; + + initialize = true; + + inherit (instance.settings.repository) timerConfig; + + pruneOpts = mapAttrsToList (name: value: + "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}" + ) instance.settings.retention; + + extraBackupArgs = + (optionals (instance.settings.limitUploadKiBs != null) [ + "--limit-upload=${toString instance.settings.limitUploadKiBs}" + ]) + ++ (optionals (instance.settings.limitDownloadKiBs != null) [ + "--limit-download=${toString instance.settings.limitDownloadKiBs}" + ]) + ++ + (let + cmd = pkgs.writeShellScriptBin "dump.sh" instance.request.backupCmd; + in + [ + "--stdin-filename ${instance.request.backupFile} --stdin-from-command -- ${cmd}/bin/dump.sh" + ]); + }; + }; + in + mkMerge (flatten (mapAttrsToList mkSettings enabledDatabases)); + } + { systemd.services = let - mkRepositorySettings = name: instance: repository: + mkSettings = name: instance: let - serviceName = fullName name repository; + serviceName = fullName name instance.settings.repository; in { - ${serviceName} = lib.mkMerge [ + ${serviceName} = mkMerge [ { serviceConfig = { Nice = cfg.performance.niceness; IOSchedulingClass = cfg.performance.ioSchedulingClass; IOSchedulingPriority = cfg.performance.ioPriority; - BindReadOnlyPaths = instance.sourceDirectories; + # BindReadOnlyPaths = instance.sourceDirectories; }; } - (lib.attrsets.optionalAttrs (repository.secrets != {}) + (optionalAttrs (instance.settings.repository.secrets != {}) { serviceConfig.EnvironmentFile = [ "/run/secrets_restic/${serviceName}" @@ -245,13 +344,13 @@ in }) ]; - "${serviceName}-pre" = lib.mkIf (repository.secrets != {}) + "${serviceName}-pre" = mkIf (instance.settings.repository.secrets != {}) (let script = shblib.genConfigOutOfBandSystemd { - config = repository.secrets; + config = instance.settings.repository.secrets; configLocation = "/run/secrets_restic/${serviceName}"; - generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} { globalSection = v; }); - user = instance.user; + generator = name: v: pkgs.writeText "template" (generators.toINIWithGlobalSection {} { globalSection = v; }); + user = instance.request.user; }; in { @@ -260,37 +359,62 @@ in serviceConfig.LoadCredential = script.loadCredentials; }); }; - mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories; in - lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings enabledInstances)); + mkMerge (flatten (mapAttrsToList mkSettings (enabledInstances // enabledDatabases))); } { system.activationScripts = let - mkEnv = name: instance: repository: - lib.nameValuePair "${fullName name repository}_gen" + mkEnv = name: instance: + nameValuePair "${fullName name instance.settings.repository}_gen" (shblib.replaceSecrets { - userConfig = repository.secrets // { - RESTIC_PASSWORD_FILE = instance.passphraseFile; - RESTIC_REPOSITORY = repository.path; + userConfig = instance.settings.repository.secrets // { + RESTIC_PASSWORD_FILE = instance.settings.passphraseFile; + RESTIC_REPOSITORY = instance.settings.repository.path; }; - resultPath = "/run/secrets_restic_env/${fullName name repository}"; - generator = name: v: pkgs.writeText (fullName name repository) (lib.generators.toINIWithGlobalSection {} { globalSection = v; }); - user = instance.user; + resultPath = "/run/secrets_restic_env/${fullName name instance.settings.repository}"; + generator = name: v: pkgs.writeText (fullName name instance.settings.repository) (generators.toINIWithGlobalSection {} { globalSection = v; }); + user = instance.request.user; }); - mkSettings = name: instance: builtins.map (mkEnv name instance) instance.repositories; in - lib.listToAttrs (lib.flatten (lib.attrsets.mapAttrsToList mkSettings cfg.instances)); + listToAttrs (flatten (mapAttrsToList mkEnv (cfg.instances // cfg.databases))); + } + { + environment.systemPackages = let + mkResticBinary = name: instance: + pkgs.writeShellScriptBin (fullName name instance.settings.repository) '' + set -euo pipefail + export $(grep -v '^#' "/run/secrets_restic_env/${fullName name instance.settings.repository}" \ + | xargs -d '\n') + + if ! [ "$1" = "restore" ]; then + sudo --preserve-env -u ${instance.request.user} ${pkgs.restic}/bin/restic $@ + else + shift + sudo --preserve-env -u ${instance.request.user} sh -c "${pkgs.restic}/bin/restic restore $@ --target /" + fi + ''; + in + flatten (mapAttrsToList mkResticBinary cfg.instances); + } + { environment.systemPackages = let - mkResticBinary = name: instance: repository: - pkgs.writeShellScriptBin (fullName name repository) '' - export $(grep -v '^#' "/run/secrets_restic_env/${fullName name repository}" \ + mkResticBinary = name: instance: + pkgs.writeShellScriptBin (fullName name instance.settings.repository) '' + set -euo pipefail + + export $(grep -v '^#' "/run/secrets_restic_env/${fullName name instance.settings.repository}" \ | xargs -d '\n') - ${pkgs.restic}/bin/restic $@ + + if ! [ "$1" = "restore" ]; then + sudo --preserve-env -u ${instance.request.user} ${pkgs.restic}/bin/restic $@ + else + shift + sudo --preserve-env -u ${instance.request.user} sh -c "${pkgs.restic}/bin/restic dump $@ ${instance.request.backupFile} | ${instance.request.restoreCmd}" + fi ''; - mkSettings = name: instance: builtins.map (mkResticBinary name instance) instance.repositories; in - lib.flatten (lib.attrsets.mapAttrsToList mkSettings cfg.instances); + flatten (mapAttrsToList mkResticBinary cfg.databases); } ]); } diff --git a/modules/blocks/restic/docs/default.md b/modules/blocks/restic/docs/default.md index 6fb94791..2ff082c5 100644 --- a/modules/blocks/restic/docs/default.md +++ b/modules/blocks/restic/docs/default.md @@ -6,15 +6,17 @@ This block sets up a backup job using [Restic][restic]. [restic]: https://restic.net/ -## Contract {#blocks-restic-features} +## Tests {#blocks-restic-tests} -This block implements the [backup](contracts-backup.html) contract. +Specific integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix). -Integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix). +## Provider Contracts {#blocks-restic-contract-provider} -## Usage {#blocks-backup-usage} +This block implements the [backup](contracts-backup.html) and [database backup](contracts-databasebackup.html) contracts. -### One folder backed up to mounted hard drives {#blocks-backup-usage-one} +Contract integration tests are defined in [`/test/contracts/backup.nix`](@REPO@/test/contracts/backup.nix). + +### One folder backed up to mounted hard drives {#blocks-restic-contract-provider-one} The following snippet shows how to configure the backup of 1 folder to 1 repository. @@ -22,35 +24,39 @@ We assume that the folder is used by the `myservice` service and is owned by a u ```nix shb.restic.instances.myservice = { - enable = true; + request = { + user = "myservice"; - user = "myservice"; + sourceDirectories = [ + "/var/lib/myfolder" + ]; + }; - passphraseFile = ""; + settings = { + enable = true; - repositories = [{ - path = "/srv/backups/myservice"; - timerConfig = { - OnCalendar = "00:00:00"; - RandomizedDelaySec = "3h"; - }; - }]; + passphraseFile = ""; - sourceDirectories = [ - "/var/lib/myfolder" - ]; + repository = { + path = "/srv/backups/myservice"; + timerConfig = { + OnCalendar = "00:00:00"; + RandomizedDelaySec = "3h"; + }; + }; - retention = { - keep_within = "1d"; - keep_hourly = 24; - keep_daily = 7; - keep_weekly = 4; - keep_monthly = 6; + retention = { + keep_within = "1d"; + keep_hourly = 24; + keep_daily = 7; + keep_weekly = 4; + keep_monthly = 6; + }; }; }; ``` -### One folder backed up to S3 {#blocks-restic-usage-remote} +### One folder backed up to S3 {#blocks-restic-contract-provider-remote} Here we will only highlight the differences with the previous configuration. @@ -59,7 +65,7 @@ This assumes you have access to such a remote S3 store, for example by using [Ba ```diff shb.backup.instances.myservice = { - repositories = [{ + repository = { - path = "/srv/pool1/backups/myfolder"; + path = "s3:s3.us-west-000.backblazeb2.com/backups/myfolder"; timerConfig = { @@ -71,11 +77,11 @@ This assumes you have access to such a remote S3 store, for example by using [Ba + AWS_ACCESS_KEY_ID.source=""; + AWS_SECRET_ACCESS_KEY.source=""; + }; - }]; + }; } ``` -### Secrets {#blocks-restic-secrets} +## Secrets {#blocks-restic-secrets} To be secure, the secrets should deployed out of band, otherwise they will be world-readable in the nix store. @@ -84,15 +90,13 @@ The code to backup to Backblaze with secrets stored in Sops would look like so: ```nix shb.restic.instances.myfolder.passphraseFile = config.sops.secrets."myservice/backup/passphrase".path; -shb.restic.instances.myfolder.repositories = [ - { - path = "s3:s3.us-west-000.backblazeb2.com/"; - secrets = { - AWS_ACCESS_KEY_ID.source = config.sops.secrets."backup/b2/access_key_id".path; - AWS_SECRET_ACCESS_KEY.source = config.sops.secrets."backup/b2/secret_access_key".path; - }; - } -]; +shb.restic.instances.myfolder.repository = { + path = "s3:s3.us-west-000.backblazeb2.com/"; + secrets = { + AWS_ACCESS_KEY_ID.source = config.sops.secrets."backup/b2/access_key_id".path; + AWS_SECRET_ACCESS_KEY.source = config.sops.secrets."backup/b2/secret_access_key".path; + }; +}; sops.secrets."myservice/backup/passphrase" = { sopsFile = ./secrets.yaml; @@ -117,7 +121,7 @@ sops.secrets."backup/b2/secret_access_key" = { Pay attention that the owner must be the `myservice` user, the one owning the files to be backed up. A `secrets` contract is in progress that will allow one to not care about such details. -### Multiple directories to multiple destinations {#blocks-restic-usage-multiple} +## Multiple directories to multiple destinations {#blocks-restic-usage-multiple} The following snippet shows how to configure backup of any number of folders to 3 repositories, each happening at different times to avoid I/O contention. @@ -228,10 +232,10 @@ restic-myfolder2_s3_s3.us-west-000.backblazeb2.com_backups Discovering those is easy thanks to tab-completion. -One can then restore a backup with: +One can then restore a backup from a given repository with: ```bash -restic-myfolder1_srv_pool1_backups restore latest -t / +restic-myfolder1_srv_pool1_backups restore latest ``` ### Troubleshooting {#blocks-restic-maintenance-troubleshooting} @@ -241,7 +245,7 @@ In case something bad happens with a backup, the [official documentation](https: ## Options Reference {#blocks-restic-options} ```{=include=} options -id-prefix: blocks-backup-options- -list-id: selfhostblocks-block-backup-options +id-prefix: blocks-restic-options- +list-id: selfhostblocks-block-restic-options source: @OPTIONS_JSON@ ``` diff --git a/modules/blocks/restic/dummyModule.nix b/modules/blocks/restic/dummyModule.nix new file mode 100644 index 00000000..f0a8b23f --- /dev/null +++ b/modules/blocks/restic/dummyModule.nix @@ -0,0 +1,3 @@ +{ lib, ... }: +{ +} diff --git a/modules/contracts/backup.nix b/modules/contracts/backup.nix index 2cb6d102..3483d0be 100644 --- a/modules/contracts/backup.nix +++ b/modules/contracts/backup.nix @@ -1,42 +1,60 @@ { lib, ... }: -lib.types.submodule { - freeformType = lib.types.anything; - - options = { - user = lib.mkOption { - description = "Unix user doing the backups."; - type = lib.types.str; - }; +let + inherit (lib) mkOption; + inherit (lib.types) anything listOf nonEmptyListOf submodule str; +in +{ + request = submodule { + options = { + user = mkOption { + description = "Unix user doing the backups."; + type = str; + }; - sourceDirectories = lib.mkOption { - description = "Directories to backup."; - type = lib.types.nonEmptyListOf lib.types.str; - }; + sourceDirectories = mkOption { + description = "Directories to backup."; + type = nonEmptyListOf str; + }; - excludePatterns = lib.mkOption { - description = "Patterns to exclude."; - type = lib.types.listOf lib.types.str; - default = []; - }; + excludePatterns = mkOption { + description = "Patterns to exclude."; + type = listOf str; + default = []; + }; - hooks = lib.mkOption { - description = "Hooks to run around the backup."; - default = {}; - type = lib.types.submodule { - options = { - before_backup = lib.mkOption { - description = "Hooks to run before backup"; - type = lib.types.listOf lib.types.str; - default = []; - }; + hooks = mkOption { + description = "Hooks to run around the backup."; + default = {}; + type = submodule { + options = { + before_backup = mkOption { + description = "Hooks to run before backup"; + type = listOf str; + default = []; + }; - after_backup = lib.mkOption { - description = "Hooks to run after backup"; - type = lib.types.listOf lib.types.str; - default = []; + after_backup = mkOption { + description = "Hooks to run after backup"; + type = listOf str; + default = []; + }; }; }; }; }; }; + + result = submodule { + options = { + restoreScript = mkOption { + description = "Name of script that can restore the database."; + type = str; + }; + + backupService = mkOption { + description = "Name of service backing up the database."; + type = str; + }; + }; + }; } diff --git a/modules/contracts/backup/docs/default.md b/modules/contracts/backup/docs/default.md index 559f6c70..403797ea 100644 --- a/modules/contracts/backup/docs/default.md +++ b/modules/contracts/backup/docs/default.md @@ -1,15 +1,13 @@ -# Backup Contract {#backup-contract} +# Backup Contract {#contract-backup} This NixOS contract represents a backup job that will backup one or more files or directories -at a regular schedule. +on a regular schedule. It is a contract between a service that has files to be backed up and a service that backs up files. -All options in this contract should be set by the former. -The latter will then use the values of those options to know what to backup. -## Contract Reference {#backup-contract-options} +## Contract Reference {#contract-backup-options} These are all the options that are expected to exist for this contract to be respected. @@ -19,9 +17,11 @@ list-id: selfhostblocks-options source: @OPTIONS_JSON@ ``` -## Usage {#backup-contract-usage} +## Usage {#contract-backup-usage} A service that can be backed up will provide a `backup` option. +Such a service is a `requester` providing a `request` for a module `provider` of this contract. + What this option defines is, from the user perspective - that is _you_ - an implementation detail but it will at least define what directories to backup, the user to backup with @@ -32,9 +32,8 @@ Here is an example module defining such a `backup` option: ```nix { options = { - myservice.backup = lib.mkOption { - type = contracts.backup; - readOnly = true; + myservice.backup = mkOption { + type = contracts.backup.request; default = { user = "myservice"; sourceDirectories = [ @@ -46,23 +45,26 @@ Here is an example module defining such a `backup` option: }; ``` -As you can see, NixOS modules are a bit abused to make contracts work. -Default values are set as well as the `readOnly` attribute to ensure those values stay as defined. - Now, on the other side we have a service that uses this `backup` option and actually backs up files. +This service is a `provider` of this contract and will provide a `result` option. + Let's assume such a module is available under the `backupservice` option and that one can create multiple backup instances under `backupservice.instances`. Then, to actually backup the `myservice` service, one would write: ```nix -backupservice.instances.myservice = myservice.backup // { - enable = true; +backupservice.instances.myservice = { + request = myservice.backup; + + settings = { + enable = true; + + repository = { + path = "/srv/backup/myservice"; + }; - repository = { - path = "/srv/backup/myservice"; + # ... Other options specific to backupservice like scheduling. }; - - # ... Other options specific to backupservice like scheduling. }; ``` @@ -70,28 +72,27 @@ It is advised to backup files to different location, to improve redundancy. Thanks to using contracts, this can be made easily either with the same `backupservice`: ```nix -backupservice.instances.myservice_2 = myservice.backup // { - enable = true; - - repository = { - path = ""; +backupservice.instances.myservice_2 = { + request = myservice.backup; + + settings = { + enable = true; + + repository = { + path = ""; + }; }; }; ``` Or with another module `backupservice_2`! -## Provided Implementations {#backup-contract-impl} +## Providers of the Backup Contract {#contract-backup-providers} -An implementation here is a service that understands the `backup` contract -and will backup the files accordingly. - -One implementation is provided out of the box: - [Restic block](blocks-restic.html). +- [Borgbackup block](blocks-borgbackup.html) [WIP]. -A second one based on `borgbackup` is in progress. - -## Services Providing `backup` Option {#backup-contract-services} +## Requester Blocks and Services {#contract-backup-requesters} - Audiobookshelf (no manual yet) - Deluge (no manual yet) diff --git a/modules/contracts/backup/dummyModule.nix b/modules/contracts/backup/dummyModule.nix index 8a7d3a87..673c3d53 100644 --- a/modules/contracts/backup/dummyModule.nix +++ b/modules/contracts/backup/dummyModule.nix @@ -1,10 +1,12 @@ { pkgs, lib, ... }: let contracts = pkgs.callPackage ../. {}; + + inherit (lib) mkOption; in { - options.shb.contracts.backup = lib.mkOption { + options.shb.contracts.backup = mkOption { description = "Contract for backups."; - type = contracts.backup; + type = contracts.backup.request; }; } diff --git a/modules/contracts/backup/test.nix b/modules/contracts/backup/test.nix new file mode 100644 index 00000000..b8b59965 --- /dev/null +++ b/modules/contracts/backup/test.nix @@ -0,0 +1,121 @@ +{ pkgs, lib, ... }: +let + pkgs' = pkgs; + + testLib = pkgs.callPackage ../../../test/common.nix {}; + + inherit (lib) concatStringsSep concatMapStringsSep getAttrFromPath mkIf optionalAttrs setAttrByPath; + inherit (testLib) indent; +in +{ name, + providerRoot, + modules ? [], + username ? "me", + sourceDirectories ? [ + "/opt/files/A" + "/opt/files/B" + ], + settings, # repository -> attrset +}: pkgs.testers.runNixOSTest { + inherit name; + + nodes.machine = { config, ... }: { + imports = ( testLib.baseImports pkgs' ) ++ modules; + + config = lib.mkMerge [ + (setAttrByPath providerRoot { + request = { + inherit sourceDirectories; + user = username; + }; + settings = settings "/opt/repos/${name}"; + }) + (mkIf (username != "root") { + users.users.${username} = { + isSystemUser = true; + extraGroups = [ "sudoers" ]; + group = "root"; + }; + }) + ]; + }; + + extraPythonPackages = p: [ p.dictdiffer ]; + skipTypeCheck = true; + + testScript = { nodes, ... }: let + provider = getAttrFromPath providerRoot nodes.machine; + backupService = provider.result.backupService; + restoreScript = provider.result.restoreScript; + onAllSourceDirectories = f: concatMapStringsSep "\n" (path: indent 4 (f path)) sourceDirectories; + in '' + from dictdiffer import diff + + username = "${username}" + sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ] + + def list_files(dir): + files_and_content = {} + + files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1] + + for f in files: + content = machine.succeed(f"""cat {f}""").strip() + files_and_content[f] = content + + return files_and_content + + def assert_files(dir, files): + result = list(diff(list_files(dir), files)) + if len(result) > 0: + raise Exception("Unexpected files:", result) + + with subtest("Create initial content"): + for path in sourceDirectories: + machine.succeed(f""" + mkdir -p {path} + echo repo_fileA_1 > {path}/fileA + echo repo_fileB_1 > {path}/fileB + + chown {username}: -R {path} + chmod go-rwx -R {path} + """) + + for path in sourceDirectories: + assert_files(path, { + f'{path}/fileA': 'repo_fileA_1', + f'{path}/fileB': 'repo_fileB_1', + }) + + with subtest("First backup in repo"): + print(machine.succeed("systemctl cat ${backupService}")) + machine.succeed("systemctl start ${backupService}") + + with subtest("New content"): + for path in sourceDirectories: + machine.succeed(f""" + echo repo_fileA_2 > {path}/fileA + echo repo_fileB_2 > {path}/fileB + """) + + assert_files(path, { + f'{path}/fileA': 'repo_fileA_2', + f'{path}/fileB': 'repo_fileB_2', + }) + + with subtest("Delete content"): + for path in sourceDirectories: + machine.succeed(f"""rm -r {path}/*""") + + assert_files(path, {}) + + with subtest("Restore initial content from repo"): + machine.succeed("""${restoreScript} restore latest""") + + for path in sourceDirectories: + assert_files(path, { + f'{path}/fileA': 'repo_fileA_1', + f'{path}/fileB': 'repo_fileB_1', + }) + ''; +} diff --git a/modules/contracts/databasebackup.nix b/modules/contracts/databasebackup.nix new file mode 100644 index 00000000..4d1653ca --- /dev/null +++ b/modules/contracts/databasebackup.nix @@ -0,0 +1,77 @@ +{ lib, ... }: +let + inherit (lib) mkIf mkOption literalExpression; + inherit (lib.types) anything submodule str; +in +{ + requestType = submodule { + options = { + user = mkOption { + description = "Unix user doing the backups."; + type = str; + example = "postgres"; + }; + + backupFile = mkOption { + description = "Filename of the backup."; + type = str; + default = "dump"; + example = "postgresql.sql"; + }; + + backupCmd = mkOption { + description = "Command that produces the database dump on stdout."; + type = str; + example = literalExpression '' + ''${pkgs.postgresql}/bin/pg_dumpall | ''${pkgs.gzip}/bin/gzip --rsyncable + ''; + }; + + restoreCmd = mkOption { + description = "Command that reads the database dump on stdin and restores the database."; + type = str; + example = literalExpression '' + ''${pkgs.gzip}/bin/gunzip | ''${pkgs.postgresql}/bin/psql postgres + ''; + }; + }; + }; + + + resultType = submodule { + options = { + restoreScript = mkOption { + description = '' + Name of script that can restore the database. + One can then list snapshots with: + + ```bash + $ my_restore_script snapshots + ``` + + And restore the database with: + + ```bash + $ my_restore_script restore latest + ``` + ''; + type = str; + example = "my_restore_script"; + }; + + backupService = mkOption { + description = '' + Name of service backing up the database. + + This script can be ran manually to backup the database: + + ```bash + $ systemctl start my_backup_service + ``` + ''; + type = str; + example = "my_backup_service.service"; + }; + }; + }; +} diff --git a/modules/contracts/databasebackup/docs/default.md b/modules/contracts/databasebackup/docs/default.md new file mode 100644 index 00000000..f9f73842 --- /dev/null +++ b/modules/contracts/databasebackup/docs/default.md @@ -0,0 +1,100 @@ +# Database Backup Contract {#contract-databasebackup} + +This NixOS contract represents a backup job +that will backup everything in one database +on a regular schedule. + +It is a contract between a service that has database dumps to be backed up +and a service that backs up databases dumps. + +## Contract Reference {#contract-databasebackup-options} + +These are all the options that are expected to exist for this contract to be respected. + +```{=include=} options +id-prefix: contracts-databasebackup-options- +list-id: selfhostblocks-options +source: @OPTIONS_JSON@ +``` + +## Usage {#contract-databasebackup-usage} + +A database that can be backed up will provide a `databasebackup` option. +Such a service is a `requester` providing a `request` for a module `provider` of this contract. + +What this option defines is, from the user perspective - that is _you_ - an implementation detail +but it will at least define how to create a database dump, +the user to backup with +and how to restore from a database dump. + +Here is an example module defining such a `databasebackup` option: + +```nix +{ + options = { + myservice.databasebackup = mkOption { + type = contracts.databasebackup.request; + default = { + user = "myservice"; + backupCmd = '' + ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable + ''; + restoreCmd = '' + ${pkgs.gzip}/bin/gunzip | ${pkgs.postgresql}/bin/psql postgres + ''; + }; + }; + }; +}; +``` + +Now, on the other side we have a service that uses this `backup` option and actually backs up files. +This service is a `provider` of this contract and will provide a `result` option. + +Let's assume such a module is available under the `databasebackupservice` option +and that one can create multiple backup instances under `databasebackupservice.instances`. +Then, to actually backup the `myservice` service, one would write: + +```nix +databasebackupservice.instances.myservice = { + request = myservice.databasebackup; + + settings = { + enable = true; + + repository = { + path = "/srv/backup/myservice"; + }; + + # ... Other options specific to backupservice like scheduling. + }; +}; +``` + +It is advised to backup files to different location, to improve redundancy. +Thanks to using contracts, this can be made easily either with the same `databasebackupservice`: + +```nix +databasebackupservice.instances.myservice_2 = { + request = myservice.backup; + + settings = { + enable = true; + + repository = { + path = ""; + }; + }; +}; +``` + +Or with another module `databasebackupservice_2`! + +## Providers of the Database Backup Contract {#contract-databasebackup-providers} + +- [Restic block](blocks-restic.html). +- [Borgbackup block](blocks-borgbackup.html) [WIP]. + +## Requester Blocks and Services {#contract-databasebackup-requesters} + +- [PostgreSQL](blocks-postgresql.html#blocks-postgresql-contract-databasebackup). diff --git a/modules/contracts/databasebackup/dummyModule.nix b/modules/contracts/databasebackup/dummyModule.nix new file mode 100644 index 00000000..2671c52a --- /dev/null +++ b/modules/contracts/databasebackup/dummyModule.nix @@ -0,0 +1,43 @@ +{ pkgs, lib, ... }: +let + contracts = pkgs.callPackage ../. {}; + + inherit (lib) mkOption; + inherit (lib.types) anything submodule; +in +{ + options.shb.contracts.databasebackup = mkOption { + description = '' + Contract for database backup between a requester module + and a provider module. + + The requester communicates to the provider + how to backup the database + through the `request` options. + + The provider reads from the `request` options + and backs up the database as requested. + It communicates to the requester what script is used + to backup and restore the database + through the `result` options. + ''; + + type = submodule { + options = { + request = mkOption { + description = '' + Options set by a requester module of the database contract. + ''; + type = contracts.databasebackup.requestType; + }; + + result = mkOption { + description = '' + Options set by a provider module of the database contract. + ''; + type = contracts.databasebackup.resultType; + }; + }; + }; + }; +} diff --git a/modules/contracts/databasebackup/test.nix b/modules/contracts/databasebackup/test.nix new file mode 100644 index 00000000..34ce3830 --- /dev/null +++ b/modules/contracts/databasebackup/test.nix @@ -0,0 +1,84 @@ +{ pkgs, lib, ... }: +let + pkgs' = pkgs; + + testLib = pkgs.callPackage ../../../test/common.nix {}; + + inherit (lib) getAttrFromPath mkIf optionalAttrs setAttrByPath; +in +{ name, + requesterRoot, + providerRoot, + providerExtraConfig ? null, # { username, database } -> attrset + modules ? [], + username ? "me", + database ? "me", + settings, # repository -> attrset +}: pkgs.testers.runNixOSTest { + inherit name; + + nodes.machine = { config, ... }: { + imports = ( testLib.baseImports pkgs' ) ++ modules; + config = lib.mkMerge [ + (setAttrByPath providerRoot { + request = (getAttrFromPath requesterRoot config).databasebackup; + settings = settings "/opt/repos/database"; + }) + (mkIf (username != "root") { + users.users.${username} = { + isSystemUser = true; + extraGroups = [ "sudoers" ]; + group = "root"; + }; + }) + (optionalAttrs (providerExtraConfig != null) (providerExtraConfig { inherit username database; })) + ]; + }; + + testScript = { nodes, ... }: let + provider = (getAttrFromPath providerRoot nodes.machine).result; + in '' + import csv + + start_all() + machine.wait_for_unit("postgresql.service") + machine.wait_for_open_port(5432) + + def peer_cmd(cmd, db="me"): + return "sudo -u me psql -U me {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db) + + def query(query): + res = machine.succeed(peer_cmd(query)) + return list(dict(l) for l in csv.DictReader(res.splitlines())) + + def cmp_tables(a, b): + for i in range(max(len(a), len(b))): + diff = set(a[i]) ^ set(b[i]) + if len(diff) > 0: + raise Exception(i, diff) + + table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}] + + with subtest("create fixture"): + machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)")) + machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)")) + + res = query("SELECT * FROM test") + cmp_tables(res, table) + + with subtest("backup"): + print(machine.succeed("systemctl cat ${provider.backupService}")) + machine.succeed("systemctl start ${provider.backupService}") + + with subtest("drop database"): + machine.succeed(peer_cmd("DROP DATABASE me", db="postgres")) + + with subtest("restore"): + print(machine.succeed("readlink -f $(type ${provider.restoreScript})")) + machine.succeed("${provider.restoreScript} restore latest ") + + with subtest("check restoration"): + res = query("SELECT * FROM test") + cmp_tables(res, table) + ''; +} diff --git a/modules/contracts/default.nix b/modules/contracts/default.nix index c19f509d..d5a28732 100644 --- a/modules/contracts/default.nix +++ b/modules/contracts/default.nix @@ -1,10 +1,13 @@ { pkgs, lib }: { + databasebackup = import ./databasebackup.nix { inherit lib; }; backup = import ./backup.nix { inherit lib; }; mount = import ./mount.nix { inherit lib; }; secret = import ./secret.nix { inherit lib; }; ssl = import ./ssl.nix { inherit lib; }; test = { secret = import ./secret/test.nix { inherit pkgs lib; }; + databasebackup = import ./databasebackup/test.nix { inherit pkgs lib; }; + backup = import ./backup/test.nix { inherit pkgs lib; }; }; } diff --git a/modules/services/arr.nix b/modules/services/arr.nix index 2f59987e..ca17b31e 100644 --- a/modules/services/arr.nix +++ b/modules/services/arr.nix @@ -316,7 +316,7 @@ let }; backup = lib.mkOption { - type = contracts.backup; + type = contracts.backup.request; description = '' Backup configuration. This is an output option. @@ -325,10 +325,11 @@ let ``` shb.restic.instances."${name}" = { - enable = true; - - # Options specific to Restic. - } // config.shb.${name}.backup; + request = config.shb.${name}.backup; + settings = { + enable = true; + }; + } ``` ''; readOnly = true; diff --git a/modules/services/audiobookshelf.nix b/modules/services/audiobookshelf.nix index ffa06f5a..46428bcc 100644 --- a/modules/services/audiobookshelf.nix +++ b/modules/services/audiobookshelf.nix @@ -83,7 +83,7 @@ in }; backup = lib.mkOption { - type = contracts.backup; + type = contracts.backup.request; description = '' Backup configuration. This is an output option. @@ -92,10 +92,11 @@ in ``` shb.restic.instances."audiobookshelf" = { - enable = true; - - # Options specific to Restic. - } // config.shb.audiobookshelf.backup; + request = config.shb.audiobookshelf.backup; + settings = { + enable = true; + }; + }; ``` ''; readOnly = true; diff --git a/modules/services/deluge.nix b/modules/services/deluge.nix index 4244e78e..e3fa12d2 100644 --- a/modules/services/deluge.nix +++ b/modules/services/deluge.nix @@ -234,7 +234,7 @@ in }; backup = lib.mkOption { - type = contracts.backup; + type = contracts.backup.request; description = '' Backup configuration. This is an output option. @@ -243,10 +243,11 @@ in ``` shb.restic.instances."vaultwarden" = { - enable = true; - - # Options specific to Restic. - } // config.shb.vaultwarden.backup; + request = config.shb.vaultwarden.backup; + settings = { + enable = true; + }; + }; ``` ''; readOnly = true; diff --git a/modules/services/forgejo.nix b/modules/services/forgejo.nix index f78fb54b..50fafb6d 100644 --- a/modules/services/forgejo.nix +++ b/modules/services/forgejo.nix @@ -218,7 +218,7 @@ in }; backup = lib.mkOption { - type = contracts.backup; + type = contracts.backup.request; description = '' Backup configuration. This is an output option. @@ -227,10 +227,11 @@ in ``` shb.restic.instances."forgejo" = { - enable = true; - - # Options specific to Restic. - } // config.shb.forgejo.backup; + request = config.shb.forgejo.backup; + settings = { + enable = true; + }; + }; ``` ''; readOnly = true; diff --git a/modules/services/forgejo/docs/default.md b/modules/services/forgejo/docs/default.md index aa070c39..8ad3ffc7 100644 --- a/modules/services/forgejo/docs/default.md +++ b/modules/services/forgejo/docs/default.md @@ -213,8 +213,11 @@ twice with a future secrets SHB block. Backing up Forgejo using the [Restic block](blocks-restic.html) is done like so: ```nix -shb.restic.instances."forgejo" = config.shb.forgejo.backup // { - enable = true; +shb.restic.instances."forgejo" = { + request = config.shb.forgejo.backup; + settings = { + enable = true; + }; }; ``` diff --git a/modules/services/grocy.nix b/modules/services/grocy.nix index 92a5908d..e1dc3bb9 100644 --- a/modules/services/grocy.nix +++ b/modules/services/grocy.nix @@ -63,7 +63,7 @@ in }; backup = lib.mkOption { - type = contracts.backup; + type = contracts.backup.request; description = '' Backup configuration. This is an output option. @@ -72,10 +72,11 @@ in ``` shb.restic.instances."grocy" = { - enable = true; - - # Options specific to Restic. - } // config.shb.grocy.backup; + request = config.shb.grocy.backup; + settings = { + enable = true; + }; + }; ``` ''; readOnly = true; diff --git a/modules/services/hledger.nix b/modules/services/hledger.nix index 3c7521d8..ee4c3c9c 100644 --- a/modules/services/hledger.nix +++ b/modules/services/hledger.nix @@ -55,7 +55,7 @@ in }; backup = lib.mkOption { - type = contracts.backup; + type = contracts.backup.request; description = '' Backup configuration. This is an output option. @@ -64,10 +64,11 @@ in ``` shb.restic.instances."hledger" = { - enable = true; - - # Options specific to Restic. - } // config.shb.hledger.backup; + request = config.shb.hledger.backup; + settings = { + enable = true; + }; + }; ``` ''; readOnly = true; diff --git a/modules/services/home-assistant.nix b/modules/services/home-assistant.nix index d2cdc98d..979a845e 100644 --- a/modules/services/home-assistant.nix +++ b/modules/services/home-assistant.nix @@ -137,7 +137,7 @@ in }; backup = lib.mkOption { - type = contracts.backup; + type = contracts.backup.request; description = '' Backup configuration. This is an output option. @@ -146,10 +146,11 @@ in ``` shb.restic.instances."home-assistant" = { - enable = true; - - # Options specific to Restic. - } // config.shb.home-assistant.backup; + request = config.shb.home-assistant.backup; + settings = { + enable = true; + }; + }; ``` ''; readOnly = true; diff --git a/modules/services/jellyfin.nix b/modules/services/jellyfin.nix index c537d178..506340c8 100644 --- a/modules/services/jellyfin.nix +++ b/modules/services/jellyfin.nix @@ -139,7 +139,7 @@ in }; backup = lib.mkOption { - type = contracts.backup; + type = contracts.backup.request; description = '' Backup configuration. This is an output option. @@ -148,10 +148,11 @@ in ``` shb.restic.instances."jellyfin" = { - enable = true; - - # Options specific to Restic. - } // config.shb.jellyfin.backup; + request = config.shb.jellyfin.backup; + settings = { + enable = true; + }; + }; ``` ''; readOnly = true; diff --git a/modules/services/nextcloud-server.nix b/modules/services/nextcloud-server.nix index c224ca6f..e6625133 100644 --- a/modules/services/nextcloud-server.nix +++ b/modules/services/nextcloud-server.nix @@ -498,7 +498,7 @@ in backup = lib.mkOption { - type = contracts.backup; + type = contracts.backup.request; description = '' Backup configuration. This is an output option. @@ -507,10 +507,11 @@ in ``` shb.restic.instances."nextcloud" = { - enable = true; - - # Options specific to Restic. - } // config.shb.nextcloud.backup; + request = config.shb.nextcloud.backup; + settings = { + enable = true; + }; + }; ``` ''; readOnly = true; diff --git a/modules/services/nextcloud-server/docs/default.md b/modules/services/nextcloud-server/docs/default.md index a89a84d7..8422684a 100644 --- a/modules/services/nextcloud-server/docs/default.md +++ b/modules/services/nextcloud-server/docs/default.md @@ -281,8 +281,11 @@ shb.nextcloud.postgresSettings = { Backing up Nextcloud using the [Restic block](blocks-restic.html) is done like so: ```nix -shb.restic.instances."nextcloud" = config.shb.nextcloud.backup // { - enable = true; +shb.restic.instances."nextcloud" = { + request = config.shb.nextcloud.backup; + settings = { + enable = true; + }; }; ``` diff --git a/modules/services/vaultwarden.nix b/modules/services/vaultwarden.nix index 32d35730..710a9e6c 100644 --- a/modules/services/vaultwarden.nix +++ b/modules/services/vaultwarden.nix @@ -115,7 +115,7 @@ in }; backup = lib.mkOption { - type = contracts.backup; + type = contracts.backup.request; description = '' Backup configuration. This is an output option. @@ -124,10 +124,11 @@ in ``` shb.restic.instances."vaultwarden" = { - enable = true; - - # Options specific to Restic. - } // config.shb.vaultwarden.backup; + request = config.shb.vaultwarden.backup; + settings = { + enable = true; + }; + }; ``` ''; readOnly = true; diff --git a/modules/services/vaultwarden/docs/default.md b/modules/services/vaultwarden/docs/default.md index f2b130e4..4ad4a4fe 100644 --- a/modules/services/vaultwarden/docs/default.md +++ b/modules/services/vaultwarden/docs/default.md @@ -97,8 +97,11 @@ shb.zfs.datasets."postgresql".path = "/var/lib/postgresql"; Backing up Vaultwarden using the [Restic block](blocks-restic.html) is done like so: ```nix -shb.restic.instances."vaultwarden" = config.shb.vaultwarden.backup // { - enable = true; +shb.restic.instances."vaultwarden" = { + request = config.shb.vaultwarden.backup; + settings = { + enable = true; + }; }; ``` diff --git a/test/blocks/restic.nix b/test/blocks/restic.nix index 7fc2beab..d8c9ed23 100644 --- a/test/blocks/restic.nix +++ b/test/blocks/restic.nix @@ -32,19 +32,12 @@ let }; shb.restic.instances."testinstance" = { - enable = true; + settings = { + enable = true; - passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase"); + passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase"); - sourceDirectories = [ - "/opt/files/A" - "/opt/files/B" - ]; - - user = user; - - repositories = [ - { + repository = { path = "/opt/repos/A"; timerConfig = { OnCalendar = "00:00:00"; @@ -56,39 +49,45 @@ let A.source = config.shb.hardcodedsecret.A.path; B.source = config.shb.hardcodedsecret.B.path; }; - } - { - path = "/opt/repos/B"; - timerConfig = { - OnCalendar = "00:00:00"; - RandomizedDelaySec = "5h"; - }; - } - ]; - - hooks.before_backup = ['' - echo $RUNTIME_DIRECTORY - if [ "$RUNTIME_DIRECTORY" = /run/restic-backups-testinstance_opt_repos_A ]; then - if ! [ -f /run/secrets_restic/restic-backups-testinstance_opt_repos_A ]; then - exit 10 - fi - if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then - echo "A:$A" - exit 11 - fi - if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then - echo "B:$B" - exit 12 + }; + }; + + request = { + inherit user; + + sourceDirectories = [ + "/opt/files/A" + "/opt/files/B" + ]; + + hooks.before_backup = ['' + echo $RUNTIME_DIRECTORY + if [ "$RUNTIME_DIRECTORY" = /run/restic-backups-testinstance_opt_repos_A ]; then + if ! [ -f /run/secrets_restic/restic-backups-testinstance_opt_repos_A ]; then + exit 10 + fi + if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then + echo "A:$A" + exit 11 + fi + if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then + echo "B:$B" + exit 12 + fi fi - fi - '']; + '']; + }; }; }; extraPythonPackages = p: [ p.dictdiffer ]; skipTypeCheck = true; - testScript = '' + testScript = { nodes, ... }: let + provider = nodes.machine.shb.restic.instances."testinstance"; + backupService = provider.result.backupService; + restoreScript = provider.result.restoreScript; + in '' from dictdiffer import diff def list_files(dir): @@ -133,7 +132,7 @@ let }) with subtest("First backup in repo A"): - machine.succeed("systemctl start restic-backups-testinstance_opt_repos_A") + machine.succeed("systemctl start ${backupService}") with subtest("New content"): machine.succeed(""" @@ -150,9 +149,6 @@ let '/opt/files/A/fileB': 'repoA_fileB_2', }) - with subtest("Second backup in repo B"): - machine.succeed("systemctl start restic-backups-testinstance_opt_repos_B") - with subtest("Delete content"): machine.succeed(""" rm -r /opt/files/A /opt/files/B @@ -162,7 +158,7 @@ let with subtest("Restore initial content from repo A"): machine.succeed(""" - restic-testinstance_opt_repos_A restore latest -t / + ${restoreScript} restore latest """) assert_files("/opt/files", { @@ -171,18 +167,6 @@ let '/opt/files/A/fileA': 'repoA_fileA_1', '/opt/files/A/fileB': 'repoA_fileB_1', }) - - with subtest("Restore initial content from repo B"): - machine.succeed(""" - restic-testinstance_opt_repos_B restore latest -t / - """) - - assert_files("/opt/files", { - '/opt/files/B/fileA': 'repoB_fileA_2', - '/opt/files/B/fileB': 'repoB_fileB_2', - '/opt/files/A/fileA': 'repoA_fileA_2', - '/opt/files/A/fileB': 'repoA_fileB_2', - }) ''; }; diff --git a/test/contracts/backup.nix b/test/contracts/backup.nix new file mode 100644 index 00000000..3a401866 --- /dev/null +++ b/test/contracts/backup.nix @@ -0,0 +1,43 @@ +{ pkgs, ... }: +let + contracts = pkgs.callPackage ../../modules/contracts {}; +in +{ + restic_root = contracts.test.backup { + name = "restic_root"; + username = "root"; + providerRoot = [ "shb" "restic" "instances" "mytest" ]; + modules = [ + ../../modules/blocks/restic.nix + ]; + settings = repository: { + enable = true; + passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase"); + repository = { + path = repository; + timerConfig = { + OnCalendar = "00:00:00"; + }; + }; + }; + }; + + restic_me = contracts.test.backup { + name = "restic_me"; + username = "me"; + providerRoot = [ "shb" "restic" "instances" "mytest" ]; + modules = [ + ../../modules/blocks/restic.nix + ]; + settings = repository: { + enable = true; + passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase"); + repository = { + path = repository; + timerConfig = { + OnCalendar = "00:00:00"; + }; + }; + }; + }; +} diff --git a/test/contracts/databasebackup.nix b/test/contracts/databasebackup.nix new file mode 100644 index 00000000..b91f4e19 --- /dev/null +++ b/test/contracts/databasebackup.nix @@ -0,0 +1,32 @@ +{ pkgs, ... }: +let + contracts = pkgs.callPackage ../../modules/contracts {}; +in +{ + restic_postgres = contracts.test.databasebackup { + name = "restic_postgres"; + requesterRoot = [ "shb" "postgresql" ]; + providerRoot = [ "shb" "restic" "databases" "postgresql" ]; + modules = [ + ../../modules/blocks/postgresql.nix + ../../modules/blocks/restic.nix + ]; + settings = repository: { + enable = true; + passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase"); + repository = { + path = repository; + timerConfig = { + OnCalendar = "00:00:00"; + }; + }; + }; + providerExtraConfig = { username, database, ... }: { + shb.postgresql.ensures = [ + { + inherit username database; + } + ]; + }; + }; +} diff --git a/test/services/vaultwarden.nix b/test/services/vaultwarden.nix index 0439054b..ebe89c09 100644 --- a/test/services/vaultwarden.nix +++ b/test/services/vaultwarden.nix @@ -93,18 +93,19 @@ let imports = [ ../../modules/blocks/restic.nix ]; - shb.restic.instances."testinstance" = config.shb.vaultwarden.backup // { - enable = true; - passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase"); - repositories = [ - { + shb.restic.instances."testinstance" = { + request = config.shb.vaultwarden.backup; + settings = { + enable = true; + passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase"); + repository = { path = "/opt/repos/A"; timerConfig = { OnCalendar = "00:00:00"; RandomizedDelaySec = "5h"; }; - } - ]; + }; + }; }; }; in