From b7fdefc8a4182956ec114eeeaae029418a310c4d Mon Sep 17 00:00:00 2001 From: Markus Kowalewski Date: Thu, 28 Dec 2017 17:51:42 -0800 Subject: [PATCH] beegfs: init at 6.17 package, kernel module, nixos module, and nixos test --- nixos/modules/module-list.nix | 1 + .../services/network-filesystems/beegfs.nix | 343 ++++++++++++++++++ nixos/release.nix | 1 + nixos/tests/beegfs.nix | 115 ++++++ pkgs/os-specific/linux/beegfs/default.nix | 122 +++++++ .../linux/beegfs/kernel-module.nix | 43 +++ pkgs/top-level/all-packages.nix | 4 + 7 files changed, 629 insertions(+) create mode 100644 nixos/modules/services/network-filesystems/beegfs.nix create mode 100644 nixos/tests/beegfs.nix create mode 100644 pkgs/os-specific/linux/beegfs/default.nix create mode 100644 pkgs/os-specific/linux/beegfs/kernel-module.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 405dc3823d52..f32fb50368e3 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -404,6 +404,7 @@ ./services/monitoring/vnstat.nix ./services/monitoring/zabbix-agent.nix ./services/monitoring/zabbix-server.nix + ./services/network-filesystems/beegfs.nix ./services/network-filesystems/cachefilesd.nix ./services/network-filesystems/davfs2.nix ./services/network-filesystems/drbd.nix diff --git a/nixos/modules/services/network-filesystems/beegfs.nix b/nixos/modules/services/network-filesystems/beegfs.nix new file mode 100644 index 000000000000..58745efa1365 --- /dev/null +++ b/nixos/modules/services/network-filesystems/beegfs.nix @@ -0,0 +1,343 @@ +{ config, lib, pkgs, ...} : + +with lib; + +let + cfg = config.services.beegfs; + + # functions for the generations of config files + + configMgmtd = name: cfg: pkgs.writeText "mgmt-${name}.conf" '' + storeMgmtdDirectory = ${cfg.mgmtd.storeDir} + storeAllowFirstRunInit = false + connAuthFile = ${cfg.connAuthFile} + connPortShift = ${toString cfg.connPortShift} + + ${cfg.mgmtd.extraConfig} + ''; + + configAdmon = name: cfg: pkgs.writeText "admon-${name}.conf" '' + sysMgmtdHost = ${cfg.mgmtdHost} + connAuthFile = ${cfg.connAuthFile} + connPortShift = ${toString cfg.connPortShift} + + ${cfg.admon.extraConfig} + ''; + + configMeta = name: cfg: pkgs.writeText "meta-${name}.conf" '' + storeMetaDirectory = ${cfg.meta.storeDir} + sysMgmtdHost = ${cfg.mgmtdHost} + connAuthFile = ${cfg.connAuthFile} + connPortShift = ${toString cfg.connPortShift} + storeAllowFirstRunInit = false + + ${cfg.mgmtd.extraConfig} + ''; + + configStorage = name: cfg: pkgs.writeText "storage-${name}.conf" '' + storeStorageDirectory = ${cfg.storage.storeDir} + sysMgmtdHost = ${cfg.mgmtdHost} + connAuthFile = ${cfg.connAuthFile} + connPortShift = ${toString cfg.connPortShift} + storeAllowFirstRunInit = false + + ${cfg.storage.extraConfig} + ''; + + configHelperd = name: cfg: pkgs.writeText "helperd-${name}.conf" '' + connAuthFile = ${cfg.connAuthFile} + ${cfg.helperd.extraConfig} + ''; + + configClientFilename = name : "/etc/beegfs/client-${name}.conf"; + + configClient = name: cfg: '' + sysMgmtdHost = ${cfg.mgmtdHost} + connAuthFile = ${cfg.connAuthFile} + connPortShift = ${toString cfg.connPortShift} + + ${cfg.client.extraConfig} + ''; + + serviceList = [ + { service = "admon"; cfgFile = configAdmon; } + { service = "meta"; cfgFile = configMeta; } + { service = "mgmtd"; cfgFile = configMgmtd; } + { service = "storage"; cfgFile = configStorage; } + ]; + + # functions to generate systemd.service entries + + systemdEntry = service: cfgFile: (mapAttrs' ( name: cfg: + (nameValuePair "beegfs-${service}-${name}" (mkIf cfg."${service}".enable { + wantedBy = [ "multi-user.target" ]; + requires = [ "network-online.target" ]; + after = [ "network-online.target" ]; + serviceConfig = rec { + ExecStart = '' + ${pkgs.beegfs}/bin/beegfs-${service} \ + cfgFile=${cfgFile name cfg} \ + pidFile=${PIDFile} + ''; + PIDFile = "/run/beegfs-${service}-${name}.pid"; + TimeoutStopSec = "300"; + }; + }))) cfg); + + systemdHelperd = mapAttrs' ( name: cfg: + (nameValuePair "beegfs-helperd-${name}" (mkIf cfg.client.enable { + wantedBy = [ "multi-user.target" ]; + requires = [ "network-online.target" ]; + after = [ "network-online.target" ]; + serviceConfig = rec { + ExecStart = '' + ${pkgs.beegfs}/bin/beegfs-helperd \ + cfgFile=${configHelperd name cfg} \ + pidFile=${PIDFile} + ''; + PIDFile = "/run/beegfs-helperd-${name}.pid"; + TimeoutStopSec = "300"; + }; + }))) cfg; + + # wrappers to beegfs tools. Avoid typing path of config files + utilWrappers = mapAttrsToList ( name: cfg: + ( pkgs.runCommand "beegfs-utils-${name}" { nativeBuildInputs = [ pkgs.makeWrapper ]; } '' + mkdir -p $out/bin + + makeWrapper ${pkgs.beegfs}/bin/beegfs-check-servers \ + $out/bin/beegfs-check-servers-${name} \ + --add-flags "-c ${configClientFilename name}" \ + --prefix PATH : ${lib.makeBinPath [ pkgs.beegfs ]} + + makeWrapper ${pkgs.beegfs}/bin/beegfs-ctl \ + $out/bin/beegfs-ctl-${name} \ + --add-flags "--cfgFile=${configClientFilename name}" + + makeWrapper ${pkgs.beegfs}/bin/beegfs-ctl \ + $out/bin/beegfs-df-${name} \ + --add-flags "--cfgFile=${configClientFilename name}" \ + --add-flags --listtargets \ + --add-flags --hidenodeid \ + --add-flags --pools \ + --add-flags --spaceinfo + + makeWrapper ${pkgs.beegfs}/bin/beegfs-fsck \ + $out/bin/beegfs-fsck-${name} \ + --add-flags "--cfgFile=${configClientFilename name}" + '' + )) cfg; +in +{ + ###### interface + + options = { + services.beegfsEnable = mkEnableOption "BeeGFS"; + + services.beegfs = mkOption { + default = {}; + description = '' + BeeGFS configurations. Every mount point requires a separate configuration. + ''; + type = with types; attrsOf (submodule ({ config, ... } : { + options = { + mgmtdHost = mkOption { + type = types.str; + default = null; + example = "master"; + description = ''Hostname of managament host.''; + }; + + connAuthFile = mkOption { + type = types.str; + default = ""; + example = "/etc/my.key"; + description = "File containing shared secret authentication."; + }; + + connPortShift = mkOption { + type = types.int; + default = 0; + example = 5; + description = '' + For each additional beegfs configuration shift all + service TCP/UDP ports by at least 5. + ''; + }; + + client = { + enable = mkEnableOption "BeeGFS client"; + + mount = mkOption { + type = types.bool; + default = true; + description = "Create fstab entry automatically"; + }; + + mountPoint = mkOption { + type = types.str; + default = "/run/beegfs"; + description = '' + Mount point under which the beegfs filesytem should be mounted. + If mounted manually the mount option specifing the config file is needed: + cfgFile=/etc/beegfs/beegfs-client-.conf + ''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = '' + Additional lines for beegfs-client.conf. + See documentation for further details. + ''; + }; + }; + + helperd = { + extraConfig = mkOption { + type = types.lines; + default = ""; + description = '' + Additional lines for beegfs-helperd.conf. See documentation + for further details. + ''; + }; + }; + + mgmtd = { + enable = mkEnableOption "BeeGFS mgmtd daemon"; + + storeDir = mkOption { + type = types.path; + default = null; + example = "/data/beegfs-mgmtd"; + description = '' + Data directory for mgmtd. + Must not be shared with other beegfs daemons. + This directory must exist and it must be initialized + with beegfs-setup-mgmtd, e.g. "beegfs-setup-mgmtd -C -p " + ''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = '' + Additional lines for beegfs-mgmtd.conf. See documentation + for further details. + ''; + }; + }; + + admon = { + enable = mkEnableOption "BeeGFS admon daemon"; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = '' + Additional lines for beegfs-admon.conf. See documentation + for further details. + ''; + }; + }; + + meta = { + enable = mkEnableOption "BeeGFS meta data daemon"; + + storeDir = mkOption { + type = types.path; + default = null; + example = "/data/beegfs-meta"; + description = '' + Data directory for meta data service. + Must not be shared with other beegfs daemons. + The underlying filesystem must be mounted with xattr turned on. + This directory must exist and it must be initialized + with beegfs-setup-meta, e.g. + "beegfs-setup-meta -C -s -p " + ''; + }; + + extraConfig = mkOption { + type = types.str; + default = ""; + description = '' + Additional lines for beegfs-meta.conf. See documentation + for further details. + ''; + }; + }; + + storage = { + enable = mkEnableOption "BeeGFS storage daemon"; + + storeDir = mkOption { + type = types.path; + default = null; + example = "/data/beegfs-storage"; + description = '' + Data directories for storage service. + Must not be shared with other beegfs daemons. + The underlying filesystem must be mounted with xattr turned on. + This directory must exist and it must be initialized + with beegfs-setup-storage, e.g. + "beegfs-setup-storage -C -s -i -p " + ''; + }; + + extraConfig = mkOption { + type = types.str; + default = ""; + description = '' + Addional lines for beegfs-storage.conf. See documentation + for further details. + ''; + }; + }; + }; + })); + }; + }; + + ###### implementation + + config = + mkIf config.services.beegfsEnable { + + environment.systemPackages = utilWrappers; + + # Put the client.conf files in /etc since they are needed + # by the commandline tools + environment.etc = mapAttrs' ( name: cfg: + (nameValuePair "beegfs/client-${name}.conf" (mkIf (cfg.client.enable) + { + enable = true; + text = configClient name cfg; + }))) cfg; + + # Kernel module, we need it only once per host. + boot = mkIf ( + foldr (a: b: a || b) false + (map (x: x.client.enable) (collect (x: x ? client) cfg))) + { + kernelModules = [ "beegfs" ]; + extraModulePackages = [ pkgs.linuxPackages.beegfs-module ]; + }; + + # generate fstab entries + fileSystems = mapAttrs' (name: cfg: + (nameValuePair cfg.client.mountPoint (optionalAttrs cfg.client.mount (mkIf cfg.client.enable { + device = "beegfs_nodev"; + fsType = "beegfs"; + mountPoint = cfg.client.mountPoint; + options = [ "cfgFile=${configClientFilename name}" "_netdev" ]; + })))) cfg; + + # generate systemd services + systemd.services = systemdHelperd // + foldr (a: b: a // b) {} + (map (x: systemdEntry x.service x.cfgFile) serviceList); + }; +} diff --git a/nixos/release.nix b/nixos/release.nix index b7ec97bcf828..cf3fe6abd48c 100644 --- a/nixos/release.nix +++ b/nixos/release.nix @@ -232,6 +232,7 @@ in rec { tests.atd = callTest tests/atd.nix {}; tests.acme = callTest tests/acme.nix {}; tests.avahi = callTest tests/avahi.nix {}; + tests.beegfs = callTest tests/beegfs.nix {}; tests.bittorrent = callTest tests/bittorrent.nix {}; tests.blivet = callTest tests/blivet.nix {}; tests.boot = callSubTests tests/boot.nix {}; diff --git a/nixos/tests/beegfs.nix b/nixos/tests/beegfs.nix new file mode 100644 index 000000000000..433910feafe3 --- /dev/null +++ b/nixos/tests/beegfs.nix @@ -0,0 +1,115 @@ +import ./make-test.nix ({ pkgs, ... } : + +let + connAuthFile="beegfs/auth-def.key"; + + client = { config, pkgs, lib, ... } : { + networking.firewall.enable = false; + services.beegfsEnable = true; + services.beegfs.default = { + mgmtdHost = "mgmt"; + connAuthFile = "/etc/${connAuthFile}"; + client = { + mount = false; + enable = true; + }; + }; + + fileSystems = pkgs.lib.mkVMOverride # FIXME: this should be creatd by the module + [ { mountPoint = "/beegfs"; + device = "default"; + fsType = "beegfs"; + options = [ "cfgFile=/etc/beegfs/client-default.conf" "_netdev" ]; + } + ]; + + environment.etc."${connAuthFile}" = { + enable = true; + text = "ThisIsALousySecret"; + mode = "0600"; + }; + }; + + + server = service : { config, pkgs, lib, ... } : { + networking.firewall.enable = false; + boot.initrd.postDeviceCommands = '' + ${pkgs.e2fsprogs}/bin/mkfs.ext4 -L data /dev/vdb + ''; + + virtualisation.emptyDiskImages = [ 4096 ]; + + fileSystems = pkgs.lib.mkVMOverride + [ { mountPoint = "/data"; + device = "/dev/disk/by-label/data"; + fsType = "ext4"; + } + ]; + + environment.systemPackages = with pkgs; [ beegfs ]; + environment.etc."${connAuthFile}" = { + enable = true; + text = "ThisIsALousySecret"; + mode = "0600"; + }; + + services.beegfsEnable = true; + services.beegfs.default = { + mgmtdHost = "mgmt"; + connAuthFile = "/etc/${connAuthFile}"; + "${service}" = { + enable = true; + storeDir = "/data"; + }; + }; + }; + +in +{ + name = "beegfs"; + + nodes = { + meta = server "meta"; + mgmt = server "mgmtd"; + storage1 = server "storage"; + storage2 = server "storage"; + client1 = client; + client2 = client; + }; + + testScript = '' + # Initalize the data directories + $mgmt->waitForUnit("default.target"); + $mgmt->succeed("beegfs-setup-mgmtd -C -f -p /data"); + $mgmt->succeed("systemctl start beegfs-mgmtd-default"); + + $meta->waitForUnit("default.target"); + $meta->succeed("beegfs-setup-meta -C -f -s 1 -p /data"); + $meta->succeed("systemctl start beegfs-meta-default"); + + $storage1->waitForUnit("default.target"); + $storage1->succeed("beegfs-setup-storage -C -f -s 1 -i 1 -p /data"); + $storage1->succeed("systemctl start beegfs-storage-default"); + + $storage2->waitForUnit("default.target"); + $storage2->succeed("beegfs-setup-storage -C -f -s 2 -i 2 -p /data"); + $storage2->succeed("systemctl start beegfs-storage-default"); + + # + + # Basic test + $client1->waitForUnit("beegfs.mount"); + $client1->succeed("beegfs-check-servers-default"); + $client1->succeed("echo test > /beegfs/test"); + $client2->waitForUnit("beegfs.mount"); + $client2->succeed("test -e /beegfs/test"); + $client2->succeed("cat /beegfs/test | grep test"); + + # test raid0/stripping + $client1->succeed("dd if=/dev/urandom bs=1M count=10 of=/beegfs/striped"); + $client2->succeed("cat /beegfs/striped > /dev/null"); + + # check if fs is still healthy + $client1->succeed("beegfs-fsck-default --checkfs"); + ''; +}) diff --git a/pkgs/os-specific/linux/beegfs/default.nix b/pkgs/os-specific/linux/beegfs/default.nix new file mode 100644 index 000000000000..1bb5612ce92c --- /dev/null +++ b/pkgs/os-specific/linux/beegfs/default.nix @@ -0,0 +1,122 @@ +{ stdenv, fetchurl, pkgconfig, unzip, which +, libuuid, attr, xfsprogs, cppunit +, zlib, openssl, sqlite, jre, openjdk, ant +} : + +let + version = "6.17"; + + subdirs = [ + "beegfs_thirdparty/build" + "beegfs_opentk_lib/build" + "beegfs_common/build" + "beegfs_admon/build" + "beegfs_java_lib/build" + "beegfs_ctl/build" + "beegfs_fsck/build" + "beegfs_helperd/build" + "beegfs_meta/build" + "beegfs_mgmtd/build" + "beegfs_online_cfg/build" + "beegfs_storage/build" + "beegfs_utils/build" + ]; + +in stdenv.mkDerivation rec { + name = "beegfs-${version}"; + + src = fetchurl { + url = "https://git.beegfs.com/pub/v6/repository/archive.tar.bz2?ref=${version}"; + sha256 = "10xs7gzdmlg23k6zn1b7jij3lljn7rr1j6h476hq4lbg981qk3n3"; + }; + + nativeBuildInputs = [ which unzip pkgconfig cppunit openjdk ant]; + buildInputs = [ libuuid attr xfsprogs zlib openssl sqlite jre ]; + + postPatch = '' + patchShebangs ./ + find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \; + find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \; + find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \; + ''; + + buildPhase = '' + for i in ${toString subdirs}; do + make -C $i + done + make -C beegfs_admon/build admon_gui + ''; + + installPhase = '' + binDir=$out/bin + docDir=$out/share/doc/beegfs + includeDir=$out/include/beegfs + libDir=$out/lib + libDirPkg=$out/lib/beegfs + + mkdir -p $binDir $libDir $libDirPkg $docDir $includeDir + + cp beegfs_admon/build/beegfs-admon $binDir + cp beegfs_admon/build/dist/usr/bin/beegfs-admon-gui $binDir + cp beegfs_admon_gui/dist/beegfs-admon-gui.jar $libDirPkg + cp beegfs_admon/build/dist/etc/beegfs-admon.conf $docDir + + cp beegfs_java_lib/build/jbeegfs.jar $libDirPkg + cp beegfs_java_lib/build/libjbeegfs.so $libDir + + cp beegfs_ctl/build/beegfs-ctl $binDir + cp beegfs_fsck/build/beegfs-fsck $binDir + + cp beegfs_utils/scripts/beegfs-check-servers $binDir + cp beegfs_utils/scripts/beegfs-df $binDir + cp beegfs_utils/scripts/beegfs-net $binDir + + cp beegfs_helperd/build/beegfs-helperd $binDir + cp beegfs_helperd/build/dist/etc/beegfs-helperd.conf $docDir + + cp beegfs_client_module/build/dist/sbin/beegfs-setup-client $binDir + cp beegfs_client_module/build/dist/etc/beegfs-client.conf $docDir + + cp beegfs_meta/build/beegfs-meta $binDir + cp beegfs_meta/build/dist/sbin/beegfs-setup-meta $binDir + cp beegfs_meta/build/dist/etc/beegfs-meta.conf $docDir + + cp beegfs_mgmtd/build/beegfs-mgmtd $binDir + cp beegfs_mgmtd/build/dist/sbin/beegfs-setup-mgmtd $binDir + cp beegfs_mgmtd/build/dist/etc/beegfs-mgmtd.conf $docDir + + cp beegfs_storage/build/beegfs-storage $binDir + cp beegfs_storage/build/dist/sbin/beegfs-setup-storage $binDir + cp beegfs_storage/build/dist/etc/beegfs-storage.conf $docDir + + cp beegfs_opentk_lib/build/libbeegfs-opentk.so $libDir + + cp beegfs_client_devel/build/dist/usr/share/doc/beegfs-client-devel/examples/* $docDir + cp -r beegfs_client_devel/include/* $includeDir + ''; + + postFixup = '' + substituteInPlace $out/bin/beegfs-admon-gui \ + --replace " java " " ${jre}/bin/java " \ + --replace "/opt/beegfs/beegfs-admon-gui/beegfs-admon-gui.jar" \ + "$libDirPkg/beegfs-admon-gui.jar" + ''; + + doCheck = true; + + checkPhase = '' + beegfs_common/build/test-runner --text + ''; + + meta = with stdenv.lib; { + description = "High performance distributed filesystem with RDMA support"; + homepage = "https://www.beegfs.io"; + platforms = [ "i686-linux" "x86_64-linux" ]; + license = { + fullName = "BeeGFS_EULA"; + url = "https://www.beegfs.io/docs/BeeGFS_EULA.txt"; + free = false; + }; + maintainers = with maintainers; [ markuskowa ]; + }; +} diff --git a/pkgs/os-specific/linux/beegfs/kernel-module.nix b/pkgs/os-specific/linux/beegfs/kernel-module.nix new file mode 100644 index 000000000000..e68a9eab3785 --- /dev/null +++ b/pkgs/os-specific/linux/beegfs/kernel-module.nix @@ -0,0 +1,43 @@ +{ stdenv, fetchurl, which +, kmod, kernel +} : + +let + version = "6.17"; +in stdenv.mkDerivation { + name = "beegfs-module-${version}-${kernel.version}"; + + src = fetchurl { + url = "https://git.beegfs.com/pub/v6/repository/archive.tar.bz2?ref=${version}"; + sha256 = "10xs7gzdmlg23k6zn1b7jij3lljn7rr1j6h476hq4lbg981qk3n3"; + }; + + hardeningDisable = [ "fortify" "pic" "stackprotector" ]; + + nativeBuildInputs = [ which kmod ]; + + makeFlags = [ "KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build/" ]; + + postPatch = '' + patchShebangs ./ + find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \; + find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \; + find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \; + ''; + + preBuild = "cd beegfs_client_module/build"; + + installPhase = '' + instdir=$out/lib/modules/${kernel.modDirVersion}/extras/fs/beegfs + mkdir -p $instdir + cp beegfs.ko $instdir + ''; + + meta = with stdenv.lib; { + description = "High performance distributed filesystem with RDMA support"; + homepage = "https://www.beegfs.io"; + platforms = [ "i686-linux" "x86_64-linux" ]; + license = licenses.gpl2; + maintainers = with maintainers; [ markuskowa ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 63fe2f5a6d27..8f69c2e53a55 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1338,6 +1338,8 @@ with pkgs; beanstalkd = callPackage ../servers/beanstalkd { }; + beegfs = callPackage ../os-specific/linux/beegfs { }; + beets = callPackage ../tools/audio/beets { pythonPackages = python2Packages; }; @@ -12821,6 +12823,8 @@ with pkgs; bbswitch = callPackage ../os-specific/linux/bbswitch {}; + beegfs-module = callPackage ../os-specific/linux/beegfs/kernel-module.nix { }; + ati_drivers_x11 = callPackage ../os-specific/linux/ati-drivers { }; blcr = callPackage ../os-specific/linux/blcr { };