From 5a573862a25c5389944a5d8661a8c55172d50437 Mon Sep 17 00:00:00 2001 From: "William A. Kennington III" Date: Tue, 21 Apr 2015 14:57:48 -0700 Subject: [PATCH] ceph: Major refactoring --- pkgs/tools/filesystems/ceph/0.80.nix | 5 +- pkgs/tools/filesystems/ceph/0.94.nix | 2 + .../filesystems/ceph/fix-pgrefdebugging.patch | 17 ++ pkgs/tools/filesystems/ceph/generic.nix | 156 ++++++++++++++++-- pkgs/top-level/all-packages.nix | 2 +- 5 files changed, 168 insertions(+), 14 deletions(-) create mode 100644 pkgs/tools/filesystems/ceph/fix-pgrefdebugging.patch diff --git a/pkgs/tools/filesystems/ceph/0.80.nix b/pkgs/tools/filesystems/ceph/0.80.nix index d5fcfab46ab9..2faf30f570cf 100644 --- a/pkgs/tools/filesystems/ceph/0.80.nix +++ b/pkgs/tools/filesystems/ceph/0.80.nix @@ -9,5 +9,8 @@ callPackage ./generic.nix (args // rec { sha256 = "0c1hwl2jzghvbrnddwbp748b02jqznvigwriqx447iz2xyrz2w8q"; }; - patches = [ ./0001-Cleanup-boost-optionals.patch ]; + patches = [ + ./0001-Cleanup-boost-optionals.patch + ./fix-pgrefdebugging.patch + ]; }) diff --git a/pkgs/tools/filesystems/ceph/0.94.nix b/pkgs/tools/filesystems/ceph/0.94.nix index c6a394d783ad..0fc2b60748ab 100644 --- a/pkgs/tools/filesystems/ceph/0.94.nix +++ b/pkgs/tools/filesystems/ceph/0.94.nix @@ -8,4 +8,6 @@ callPackage ./generic.nix (args // rec { rev = "refs/tags/v${version}"; sha256 = "0rrl10vda4xv22al2c5ccd8v8drs26186dvkrxndvqz8p9999cjx"; }; + + patches = [ ./fix-pgrefdebugging.patch ]; }) diff --git a/pkgs/tools/filesystems/ceph/fix-pgrefdebugging.patch b/pkgs/tools/filesystems/ceph/fix-pgrefdebugging.patch new file mode 100644 index 000000000000..4d13a0572d87 --- /dev/null +++ b/pkgs/tools/filesystems/ceph/fix-pgrefdebugging.patch @@ -0,0 +1,17 @@ +diff --git a/configure.ac b/configure.ac +index 38ab682..febb58e 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -541,8 +541,10 @@ fi + #set pg ref debugging? + AC_ARG_ENABLE([pgrefdebugging], + [AS_HELP_STRING([--enable-pgrefdebugging], [enable pg ref debugging])], +- [AC_DEFINE([PG_DEBUG_REFS], [1], [Defined if you want pg ref debugging])], +- []) ++ [], [enable_pgrefdebugging=no]) ++AS_IF([test "x$enable_pgrefdebugging" = "xyes"], ++ [AC_DEFINE([PG_DEBUG_REFS], [1], [Defined if you want pg ref debugging])], ++ []) + + # + # Java is painful diff --git a/pkgs/tools/filesystems/ceph/generic.nix b/pkgs/tools/filesystems/ceph/generic.nix index 73097fecb36e..2ee8593ab7cc 100644 --- a/pkgs/tools/filesystems/ceph/generic.nix +++ b/pkgs/tools/filesystems/ceph/generic.nix @@ -1,7 +1,21 @@ { stdenv, autoconf, automake, makeWrapper, pkgconfig, libtool, which -, boost, btrfsProgs, cryptopp, curl, expat, fcgi, fuse, gperftools, keyutils -, leveldb, libaio, libatomic_ops, libedit, libuuid, linuxHeaders, openssl -, python, snappy, udev, xfsprogs, xz +, boost, python, pythonPackages, libxml2 + +# Optional Dependencies +, snappy ? null, leveldb ? null, yasm ? null, fcgi ? null, expat ? null +, curl ? null, fuse ? null, accelio ? null, libibverbs ? null, librdmacm ? null +, libedit ? null, libatomic_ops ? null, kinetic-cpp-client ? null +, rocksdb ? null, libs3 ? null + +# Mallocs +, jemalloc ? null, gperftools ? null + +# Crypto Dependencies +, cryptopp ? null +, nss ? null, nspr ? null + +# Linux Only Dependencies +, linuxHeaders, libuuid, udev, keyutils, libaio ? null, libxfs ? null , zfs ? null # Version specific arguments @@ -9,12 +23,51 @@ , ... }: +# We must have one crypto library +assert cryptopp != null || (nss != null && nspr != null); + with stdenv.lib; let - wrapArgs = "--prefix PYTHONPATH : \"$(toPythonPath $out)\"" + mkFlag = trueStr: falseStr: cond: name: val: + if cond == null then null else + "--${if cond != false then trueStr else falseStr}${name}${if val != null && cond != false then "=${val}" else ""}"; + mkEnable = mkFlag "enable-" "disable-"; + mkWith = mkFlag "with-" "without-"; + mkOther = mkFlag "" "" true; + + # TODO: Backport patches to support xio + accelio = null; + + hasServer = snappy != null && leveldb != null; + hasMon = hasServer; + hasMds = hasServer; + hasOsd = hasServer; + hasRadosgw = fcgi != null && expat != null && curl != null && libedit != null; + + hasXio = (stdenv.isLinux || stdenv.isFreebsd) && + accelio != null && libibverbs != null && librdmacm != null; + + # Malloc implementation (can be jemalloc, tcmalloc or null) + malloc = if jemalloc != null then jemalloc else gperftools; + + # We prefer nss over cryptopp + cryptoStr = if nss != null && nspr != null then "nss" else + if cryptopp != null then "cryptopp" else "none"; + cryptoLibsMap = { + nss = [ nss nspr ]; + cryptopp = [ cryptopp ]; + none = [ ]; + }; + + # TODO: Fix Rocksdb which is currently too new + rocksdb = null; + # TODO: Fix Kinetic which is also in a mismatched state + kinetic-cpp-client = null; + + wrapArgs = "--prefix PYTHONPATH : \"$(toPythonPath $lib)\"" + " --prefix PYTHONPATH : \"$(toPythonPath ${python.modules.readline})\"" - + " --prefix PATH : \"$out/bin\"" - + " --prefix LD_LIBRARY_PATH : \"$out/lib\""; + + " --prefix PYTHONPATH : \"$(toPythonPath ${pythonPackages.flask})\"" + + " --prefix PATH : \"$out/bin\""; in stdenv.mkDerivation { name="ceph-${version}"; @@ -26,30 +79,109 @@ stdenv.mkDerivation { ]; nativeBuildInputs = [ autoconf automake makeWrapper pkgconfig libtool which ]; - buildInputs = [ - boost btrfsProgs cryptopp curl expat fcgi fuse gperftools keyutils - libatomic_ops leveldb libaio libedit libuuid linuxHeaders openssl python - snappy udev xfsprogs.lib xz zfs + buildInputs = cryptoLibsMap.${cryptoStr} ++ [ + boost python libxml2 yasm libatomic_ops kinetic-cpp-client rocksdb libs3 malloc + ] ++ stdenv.lib.optional stdenv.isLinux [ + linuxHeaders libuuid udev keyutils libaio libxfs zfs + ] ++ stdenv.lib.optional hasServer [ + snappy leveldb + ] ++ stdenv.lib.optional hasRadosgw [ + fcgi expat curl fuse libedit + ] ++ stdenv.lib.optional hasXio [ + accelio libibverbs librdmacm ]; + postPatch = '' + # Fix zfs pkgconfig detection + sed -i 's,\[zfs\],\[libzfs\],g' configure.ac + + # Fix seagate kinetic linking + sed -i 's,libcrypto.a,-lcrypto,g' src/os/Makefile.am + ''; + preConfigure = '' + # Ceph expects the arch command to be usable during configure + # for detecting the assembly type + mkdir mybin + echo "#${stdenv.shell} -e" >> mybin/arch + echo "uname -m" >> mybin/arch + chmod +x mybin/arch + PATH="$PATH:$(pwd)/mybin" + ./autogen.sh + + # Fix the python site-packages install directory + sed -i "s,\(PYTHON\(\|_EXEC\)_PREFIX=\).*,\1'$lib',g" configure ''; configureFlags = [ "--exec_prefix=\${out}" "--sysconfdir=/etc" "--localstatedir=/var" - ] ++ optional (zfs != null) "--with-zfs=${zfs}"; + "--libdir=\${lib}/lib" + "--includedir=\${lib}/include" + + (mkWith true "rbd" null) + (mkWith true "cephfs" null) + (mkWith hasRadosgw "radosgw" null) + (mkWith true "radosstriper" null) + (mkWith hasServer "mon" null) + (mkWith hasServer "osd" null) + (mkWith hasServer "mds" null) + (mkEnable true "client" null) + (mkEnable hasServer "server" null) + (mkWith (cryptoStr == "cryptopp") "cryptopp" null) + (mkWith (cryptoStr == "nss") "nss" null) + (mkEnable false "root-make-check" null) + (mkWith false "profiler" null) + (mkWith false "debug" null) + (mkEnable false "coverage" null) + (mkWith (fuse != null) "fuse" null) + (mkWith (malloc == jemalloc) "jemalloc" null) + (mkWith (malloc == gperftools) "tcmalloc" null) + (mkEnable false "pgrefdebugging" null) + (mkEnable false "cephfs-java" null) + (mkEnable hasXio "xio" null) + (mkWith (libatomic_ops != null) "libatomic-ops" null) + (mkWith true "ocf" null) + (mkWith (kinetic-cpp-client != null) "kinetic" null) + (mkWith (rocksdb != null) "librocksdb" null) + (mkWith false "librocksdb-static" null) + (mkWith (libs3 != null) "system-libs3" null) + (mkWith true "rest-bench" null) + ] ++ optional stdenv.isLinux [ + (mkWith (libaio != null) "libaio" null) + (mkWith (libxfs != null) "libxfs" null) + (mkWith (zfs != null) "libzfs" null) + ]; installFlags = [ "sysconfdir=\${out}/etc" ]; + outputs = [ "out" "lib" ]; + postInstall = '' + # Wrap all of the python scripts wrapProgram $out/bin/ceph ${wrapArgs} wrapProgram $out/bin/ceph-brag ${wrapArgs} wrapProgram $out/bin/ceph-rest-api ${wrapArgs} wrapProgram $out/sbin/ceph-create-keys ${wrapArgs} wrapProgram $out/sbin/ceph-disk ${wrapArgs} + + # Bring in lib as a native build input + mkdir -p $out/nix-support + echo "$lib" > $out/nix-support/propagated-native-build-inputs + + # Fix the python library loading + find $lib/lib -name \*.pyc -or -name \*.pyd -exec rm {} \; + for PY in $(find $lib/lib -name \*.py); do + LIB="$(sed -n "s/.*find_library('\([^)]*\)').*/\1/p" "$PY")" + REALLIB="$lib/lib/lib$LIB.so" + sed -i "s,\(library_path[ ]*=[ ]*\).*,\1'$REALLIB',g" "$PY" + + # Reapply compilation optimizations + NAME=$(basename -s .py "$PY") + (cd "$(dirname $PY)"; python -c "import $NAME"; python -O -c "import $NAME") + done ''; enableParallelBuilding = true; @@ -59,7 +191,7 @@ stdenv.mkDerivation { description = "Distributed storage system"; license = licenses.lgpl21; maintainers = with maintainers; [ ak wkennington ]; - platforms = with platforms; linux; + platforms = with platforms; unix; }; passthru.version = version; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index af2c69ec5048..bfb8bfde67e9 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -937,7 +937,7 @@ let # Only ever add ceph LTS releases # The default should always be symlinked to the latest LTS # Dev should always point to the latest versioned release - libceph = ceph; + libceph = ceph.lib; ceph-0_80 = callPackage ../tools/filesystems/ceph/0.80.nix { }; ceph-0_94 = callPackage ../tools/filesystems/ceph/0.94.nix { }; ceph = callPackage ../tools/filesystems/ceph { };