From 1ed93ac4a17b68ebfb5c7284299a24808021293f Mon Sep 17 00:00:00 2001 From: mitchmindtree Date: Mon, 27 Sep 2021 17:04:29 +1000 Subject: [PATCH 1/6] nixos/nextcloud: Add option for using object storage as primary storage This allows to declaratively configure an S3 class object storage as the primary storage for the nextcloud service. Previously, this could only be achieved by manually editing the `config.php`. I've started testing this today with my own digitalocean nextcloud instance, which now points to my digitalocean S3-compatible "Space" and all appears to be working smoothly. My motivation for this change is my recent discovery of how much cheaper some S3-compatible object storage options are compared to digitalocean's "Volume" options. Implementation follows the "Simple Storage Service" instructions here: https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/primary_storage.html I have neglected to implement a submodule for the OpenStack Swift object storage as I don't personally have a use case for it or a method to test it, however the new `nextcloud.objectstore.s3` submodule should act as a useful guide for anyone who does wish to implement it. --- nixos/modules/services/web-apps/nextcloud.nix | 158 +++++++++++++++++- 1 file changed, 154 insertions(+), 4 deletions(-) diff --git a/nixos/modules/services/web-apps/nextcloud.nix b/nixos/modules/services/web-apps/nextcloud.nix index 3c952fd883aa..dd3cbdfc8104 100644 --- a/nixos/modules/services/web-apps/nextcloud.nix +++ b/nixos/modules/services/web-apps/nextcloud.nix @@ -312,6 +312,124 @@ in { phone-numbers. ''; }; + + objectstore = let + s3Arguments = { + bucket = mkOption { + type = types.str; + example = "nextcloud"; + description = '' + The name of the S3 bucket. + ''; + }; + autocreate = mkOption { + type = types.bool; + description = '' + Create the objectstore if it does not exist. + ''; + }; + key = mkOption { + type = types.str; + example = "EJ39ITYZEUH5BGWDRUFY"; + description = '' + The access key for the S3 bucket. + ''; + }; + secret = mkOption { + type = types.nullOr types.str; + default = null; + example = "M5MrXTRjkyMaxXPe2FRXMTfTfbKEnZCu+7uRTVSj"; + description = '' + The access secret for the S3 bucket. Use + secretFile to avoid this being world-readable + in the /nix/store + ''; + }; + secretFile = mkOption { + type = types.nullOr types.str; + default = null; + example = "/var/nextcloud-objectstore-s3-secret"; + description = '' + The full path to a file that contains the access secret. Must be + readable by user nextcloud. + ''; + }; + hostname = mkOption { + type = types.nullOr types.str; + default = null; + example = "example.com"; + description = '' + Required for some non-Amazon implementations. + ''; + }; + port = mkOption { + type = types.nullOr types.port; + default = null; + description = '' + Required for some non-Amazon implementations. + ''; + }; + useSsl = mkOption { + type = types.nullOr types.bool; + default = null; + description = '' + Use SSL for objectstore access. + ''; + }; + region = mkOption { + type = types.nullOr types.str; + default = null; + example = "REGION"; + description = '' + Required for some non-Amazon implementations. + ''; + }; + usePathStyle = mkOption { + type = types.bool; + default = false; + description = '' + Required for some non-Amazon S3 implementations. + + Ordinarily, requests will be made with + http://bucket.hostname.domain/, but with path style + enabled requests are made with + http://hostname.domain/bucket instead. + ''; + }; + }; + in mkOption { + type = types.nullOr (types.submodule { + options = { + s3 = mkOption { + type = types.submodule { + options = { + enable = mkEnableOption "S3 object storage as primary storage."; + arguments = mkOption { + type = types.submodule { + options = s3Arguments; + }; + description = '' + Configuration arguments for the object storage. + ''; + }; + }; + }; + description = '' + Mounts a bucket on an Amazon S3 object storage or compatible + implementation into the virtual filesystem. + ''; + }; + }; + }); + default = null; + description = '' + Options for configuring object storage as nextcloud's primary storage. + + See nextcloud's documentation on "Object Storage as Primary Storage" + for details on how to select the right class and argument set for + your needs. + ''; + }; }; enableImagemagick = mkEnableOption '' @@ -390,6 +508,14 @@ in { { assertion = versionOlder cfg.package.version "21" -> cfg.config.defaultPhoneRegion == null; message = "The `defaultPhoneRegion'-setting is only supported for Nextcloud >=21!"; } + { assertion = acfg.objectstore == null + || (lists.count (v: v.enable) (attrsets.attrValues acfg.objectstore)) == 1; + message = "If using objectstore class as primary storage exactly one class can be enabled."; + } + { assertion = let s3 = acfg.objectstore.s3; in acfg.objectstore == null + || (!s3.enable || ((s3.arguments.secret != null) != (s3.arguments.secretFile != null))); + message = "S3 storage requires specifying exactly one of secret or secretFile"; + } ]; warnings = let @@ -479,11 +605,34 @@ in { nextcloud-setup = let c = cfg.config; writePhpArrary = a: "[${concatMapStringsSep "," (val: ''"${toString val}"'') a}]"; + requiresReadSecretFunction = c.dbpassFile != null + || (c.objectstore != null && (c.objectstore.s3.enable && c.objectstore.s3.arguments.secretFile != null)); + objectstoreConfig = let + class = if c.objectstore.s3.enable then "S3" else ""; + args = if c.objectstore.s3.enable then c.objectstore.s3.arguments else {}; + classLine = '''class' => '\\OC\\Files\\ObjectStore\\${class}',''; + argumentLines = optionalString c.objectstore.s3.enable '' + 'bucket' => '${args.bucket}', + 'autocreate' => ${toString args.autocreate}, + 'key' => '${args.key}', + ${optionalString (args.secret != null) "'secret' => '${args.secret}',"} + ${optionalString (args.secretFile != null) "'secret' => nix_read_secret('${args.secretFile}'),"} + ${optionalString (args.hostname != null) "'hostname' => '${args.hostname}',"} + ${optionalString (args.port != null) "'port' => ${toString args.port},"} + ${optionalString (args.useSsl != null) "'use_ssl' => ${if args.useSsl then "true" else "false"},"} + ${optionalString (args.region != null) "'region' => '${args.region}',"} + 'use_path_style' => ${if args.usePathStyle then "true" else "false"}, + ''; + in optionalString (c.objectstore != null) '''objectstore' => [ + ${classLine} + 'arguments' => [ + ${argumentLines} + ], + ]''; overrideConfig = pkgs.writeText "nextcloud-config.php" '' '${c.dbuser}',"} ${optionalString (c.dbtableprefix != null) "'dbtableprefix' => '${toString c.dbtableprefix}',"} ${optionalString (c.dbpass != null) "'dbpassword' => '${c.dbpass}',"} - ${optionalString (c.dbpassFile != null) "'dbpassword' => nix_read_pwd(),"} + ${optionalString (c.dbpassFile != null) "'dbpassword' => nix_read_secret('${c.dbpassFile}'),"} 'dbtype' => '${c.dbtype}', 'trusted_domains' => ${writePhpArrary ([ cfg.hostName ] ++ c.extraTrustedDomains)}, 'trusted_proxies' => ${writePhpArrary (c.trustedProxies)}, ${optionalString (c.defaultPhoneRegion != null) "'default_phone_region' => '${c.defaultPhoneRegion}',"} + ${objectstoreConfig} ]; ''; occInstallCmd = let From 03171ae31abf0427467fea968c3fa465338535f3 Mon Sep 17 00:00:00 2001 From: mitchmindtree Date: Tue, 28 Sep 2021 12:09:05 +1000 Subject: [PATCH 2/6] nixos/nextcloud: Remove `objectstore.s3.secret` option We should discourage users from adding secrets in a way that allows for them to end up in the globally readable `/nix/store`. Users should use the `objectstore.s3.secretFile` option instead. --- nixos/modules/services/web-apps/nextcloud.nix | 22 +++---------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/nixos/modules/services/web-apps/nextcloud.nix b/nixos/modules/services/web-apps/nextcloud.nix index dd3cbdfc8104..e07c57b47bf4 100644 --- a/nixos/modules/services/web-apps/nextcloud.nix +++ b/nixos/modules/services/web-apps/nextcloud.nix @@ -335,19 +335,8 @@ in { The access key for the S3 bucket. ''; }; - secret = mkOption { - type = types.nullOr types.str; - default = null; - example = "M5MrXTRjkyMaxXPe2FRXMTfTfbKEnZCu+7uRTVSj"; - description = '' - The access secret for the S3 bucket. Use - secretFile to avoid this being world-readable - in the /nix/store - ''; - }; secretFile = mkOption { - type = types.nullOr types.str; - default = null; + type = types.str; example = "/var/nextcloud-objectstore-s3-secret"; description = '' The full path to a file that contains the access secret. Must be @@ -512,10 +501,6 @@ in { || (lists.count (v: v.enable) (attrsets.attrValues acfg.objectstore)) == 1; message = "If using objectstore class as primary storage exactly one class can be enabled."; } - { assertion = let s3 = acfg.objectstore.s3; in acfg.objectstore == null - || (!s3.enable || ((s3.arguments.secret != null) != (s3.arguments.secretFile != null))); - message = "S3 storage requires specifying exactly one of secret or secretFile"; - } ]; warnings = let @@ -606,7 +591,7 @@ in { c = cfg.config; writePhpArrary = a: "[${concatMapStringsSep "," (val: ''"${toString val}"'') a}]"; requiresReadSecretFunction = c.dbpassFile != null - || (c.objectstore != null && (c.objectstore.s3.enable && c.objectstore.s3.arguments.secretFile != null)); + || (c.objectstore != null && c.objectstore.s3.enable); objectstoreConfig = let class = if c.objectstore.s3.enable then "S3" else ""; args = if c.objectstore.s3.enable then c.objectstore.s3.arguments else {}; @@ -615,8 +600,7 @@ in { 'bucket' => '${args.bucket}', 'autocreate' => ${toString args.autocreate}, 'key' => '${args.key}', - ${optionalString (args.secret != null) "'secret' => '${args.secret}',"} - ${optionalString (args.secretFile != null) "'secret' => nix_read_secret('${args.secretFile}'),"} + 'secret' => nix_read_secret('${args.secretFile}'), ${optionalString (args.hostname != null) "'hostname' => '${args.hostname}',"} ${optionalString (args.port != null) "'port' => ${toString args.port},"} ${optionalString (args.useSsl != null) "'use_ssl' => ${if args.useSsl then "true" else "false"},"} From b23d6a4113bb087357d66e5073c1ee6883eb1744 Mon Sep 17 00:00:00 2001 From: mitchmindtree Date: Sun, 3 Oct 2021 13:41:02 +1000 Subject: [PATCH 3/6] nixos/nextcloud: Simplify `objectstore.s3` options, remove submodule Removes the submodule in favour of using an attrset. Also: - Makes better use of nix's laziness in config expansion. - Makes use of `boolToString` where applicable. --- nixos/modules/services/web-apps/nextcloud.nix | 82 ++++++------------- 1 file changed, 24 insertions(+), 58 deletions(-) diff --git a/nixos/modules/services/web-apps/nextcloud.nix b/nixos/modules/services/web-apps/nextcloud.nix index e07c57b47bf4..f2b50ec43a4c 100644 --- a/nixos/modules/services/web-apps/nextcloud.nix +++ b/nixos/modules/services/web-apps/nextcloud.nix @@ -313,8 +313,17 @@ in { ''; }; - objectstore = let - s3Arguments = { + objectstore = { + s3 = { + enable = mkEnableOption '' + S3 object storage as primary storage. + + This mounts a bucket on an Amazon S3 object storage or compatible + implementation into the virtual filesystem. + + See nextcloud's documentation on "Object Storage as Primary + Storage" for more details. + ''; bucket = mkOption { type = types.str; example = "nextcloud"; @@ -386,38 +395,6 @@ in { ''; }; }; - in mkOption { - type = types.nullOr (types.submodule { - options = { - s3 = mkOption { - type = types.submodule { - options = { - enable = mkEnableOption "S3 object storage as primary storage."; - arguments = mkOption { - type = types.submodule { - options = s3Arguments; - }; - description = '' - Configuration arguments for the object storage. - ''; - }; - }; - }; - description = '' - Mounts a bucket on an Amazon S3 object storage or compatible - implementation into the virtual filesystem. - ''; - }; - }; - }); - default = null; - description = '' - Options for configuring object storage as nextcloud's primary storage. - - See nextcloud's documentation on "Object Storage as Primary Storage" - for details on how to select the right class and argument set for - your needs. - ''; }; }; @@ -497,10 +474,6 @@ in { { assertion = versionOlder cfg.package.version "21" -> cfg.config.defaultPhoneRegion == null; message = "The `defaultPhoneRegion'-setting is only supported for Nextcloud >=21!"; } - { assertion = acfg.objectstore == null - || (lists.count (v: v.enable) (attrsets.attrValues acfg.objectstore)) == 1; - message = "If using objectstore class as primary storage exactly one class can be enabled."; - } ]; warnings = let @@ -590,29 +563,22 @@ in { nextcloud-setup = let c = cfg.config; writePhpArrary = a: "[${concatMapStringsSep "," (val: ''"${toString val}"'') a}]"; - requiresReadSecretFunction = c.dbpassFile != null - || (c.objectstore != null && c.objectstore.s3.enable); - objectstoreConfig = let - class = if c.objectstore.s3.enable then "S3" else ""; - args = if c.objectstore.s3.enable then c.objectstore.s3.arguments else {}; - classLine = '''class' => '\\OC\\Files\\ObjectStore\\${class}',''; - argumentLines = optionalString c.objectstore.s3.enable '' - 'bucket' => '${args.bucket}', - 'autocreate' => ${toString args.autocreate}, - 'key' => '${args.key}', - 'secret' => nix_read_secret('${args.secretFile}'), - ${optionalString (args.hostname != null) "'hostname' => '${args.hostname}',"} - ${optionalString (args.port != null) "'port' => ${toString args.port},"} - ${optionalString (args.useSsl != null) "'use_ssl' => ${if args.useSsl then "true" else "false"},"} - ${optionalString (args.region != null) "'region' => '${args.region}',"} - 'use_path_style' => ${if args.usePathStyle then "true" else "false"}, - ''; - in optionalString (c.objectstore != null) '''objectstore' => [ - ${classLine} + requiresReadSecretFunction = c.dbpassFile != null || c.objectstore.s3.enable; + objectstoreConfig = let s3 = c.objectstore.s3; in optionalString s3.enable '''objectstore' => [ + 'class' => '\\OC\\Files\\ObjectStore\\S3', 'arguments' => [ - ${argumentLines} + 'bucket' => '${s3.bucket}', + 'autocreate' => ${boolToString s3.autocreate}, + 'key' => '${s3.key}', + 'secret' => nix_read_secret('${s3.secretFile}'), + ${optionalString (s3.hostname != null) "'hostname' => '${s3.hostname}',"} + ${optionalString (s3.port != null) "'port' => ${toString s3.port},"} + ${optionalString (s3.useSsl != null) "'use_ssl' => ${boolToString s3.useSsl},"} + ${optionalString (s3.region != null) "'region' => '${s3.region}',"} + 'use_path_style' => ${boolToString s3.usePathStyle}, ], ]''; + overrideConfig = pkgs.writeText "nextcloud-config.php" '' Date: Sun, 3 Oct 2021 13:50:25 +1000 Subject: [PATCH 4/6] nixos/nextcloud: Make `objectstore.s3.useSsl` explicitly true by default This appears to match the nextcloud default behaviour observed here: https://github.com/nextcloud/server/blob/e2116e2fb226890341c548e3f7d79c0ac63c1b06/lib/private/Files/ObjectStore/S3ConnectionTrait.php#L83 --- nixos/modules/services/web-apps/nextcloud.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nixos/modules/services/web-apps/nextcloud.nix b/nixos/modules/services/web-apps/nextcloud.nix index f2b50ec43a4c..7794cf4518f1 100644 --- a/nixos/modules/services/web-apps/nextcloud.nix +++ b/nixos/modules/services/web-apps/nextcloud.nix @@ -368,8 +368,8 @@ in { ''; }; useSsl = mkOption { - type = types.nullOr types.bool; - default = null; + type = types.bool; + default = true; description = '' Use SSL for objectstore access. ''; @@ -573,7 +573,7 @@ in { 'secret' => nix_read_secret('${s3.secretFile}'), ${optionalString (s3.hostname != null) "'hostname' => '${s3.hostname}',"} ${optionalString (s3.port != null) "'port' => ${toString s3.port},"} - ${optionalString (s3.useSsl != null) "'use_ssl' => ${boolToString s3.useSsl},"} + 'use_ssl' => ${boolToString s3.useSsl}, ${optionalString (s3.region != null) "'region' => '${s3.region}',"} 'use_path_style' => ${boolToString s3.usePathStyle}, ], From a539a82707bad3c644fe5537300808e040540ae5 Mon Sep 17 00:00:00 2001 From: mitchmindtree Date: Sun, 3 Oct 2021 17:29:13 +1000 Subject: [PATCH 5/6] nixos/nextcloud: Account for nix_read_secret refactor in exception msg Previously, the `nix_read_pwd` function was only used for reading the `dbpassFile`, however it has since been refactored to handle reading other secret files too. This fixes the message of the exception that is thrown in the case that the file is not present so that it no longer refers specifically to the `dbpass` file. --- nixos/modules/services/web-apps/nextcloud.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/web-apps/nextcloud.nix b/nixos/modules/services/web-apps/nextcloud.nix index 7794cf4518f1..e9673f8367b6 100644 --- a/nixos/modules/services/web-apps/nextcloud.nix +++ b/nixos/modules/services/web-apps/nextcloud.nix @@ -585,7 +585,7 @@ in { function nix_read_secret($file) { if (!file_exists($file)) { throw new \RuntimeException(sprintf( - "Cannot start Nextcloud, dbpass file %s set by NixOS doesn't seem to " + "Cannot start Nextcloud, secret file %s set by NixOS doesn't seem to " . "exist! Please make sure that the file exists and has appropriate " . "permissions for user & group 'nextcloud'!", $file From c5d08ebee1805668b2a3945f0f5b2e06a1e03412 Mon Sep 17 00:00:00 2001 From: mitchmindtree Date: Tue, 5 Oct 2021 17:07:44 +1000 Subject: [PATCH 6/6] nixos/nextcloud: Fix ambiguity in `objectstoreConfig` string Previously this was a little tricky to read and had the potential to cause some ambiguity in string parsing. --- nixos/modules/services/web-apps/nextcloud.nix | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/nixos/modules/services/web-apps/nextcloud.nix b/nixos/modules/services/web-apps/nextcloud.nix index e9673f8367b6..a247c8b636d7 100644 --- a/nixos/modules/services/web-apps/nextcloud.nix +++ b/nixos/modules/services/web-apps/nextcloud.nix @@ -564,20 +564,22 @@ in { c = cfg.config; writePhpArrary = a: "[${concatMapStringsSep "," (val: ''"${toString val}"'') a}]"; requiresReadSecretFunction = c.dbpassFile != null || c.objectstore.s3.enable; - objectstoreConfig = let s3 = c.objectstore.s3; in optionalString s3.enable '''objectstore' => [ - 'class' => '\\OC\\Files\\ObjectStore\\S3', - 'arguments' => [ - 'bucket' => '${s3.bucket}', - 'autocreate' => ${boolToString s3.autocreate}, - 'key' => '${s3.key}', - 'secret' => nix_read_secret('${s3.secretFile}'), - ${optionalString (s3.hostname != null) "'hostname' => '${s3.hostname}',"} - ${optionalString (s3.port != null) "'port' => ${toString s3.port},"} - 'use_ssl' => ${boolToString s3.useSsl}, - ${optionalString (s3.region != null) "'region' => '${s3.region}',"} - 'use_path_style' => ${boolToString s3.usePathStyle}, - ], - ]''; + objectstoreConfig = let s3 = c.objectstore.s3; in optionalString s3.enable '' + 'objectstore' => [ + 'class' => '\\OC\\Files\\ObjectStore\\S3', + 'arguments' => [ + 'bucket' => '${s3.bucket}', + 'autocreate' => ${boolToString s3.autocreate}, + 'key' => '${s3.key}', + 'secret' => nix_read_secret('${s3.secretFile}'), + ${optionalString (s3.hostname != null) "'hostname' => '${s3.hostname}',"} + ${optionalString (s3.port != null) "'port' => ${toString s3.port},"} + 'use_ssl' => ${boolToString s3.useSsl}, + ${optionalString (s3.region != null) "'region' => '${s3.region}',"} + 'use_path_style' => ${boolToString s3.usePathStyle}, + ], + ] + ''; overrideConfig = pkgs.writeText "nextcloud-config.php" ''