;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2017, 2018 Marius Bakke <mbakke@fastmail.com> ;;; Copyright © 2017 Rutger Helling <rhelling@mykolab.com> ;;; ;;; This file is part of GNU Guix. ;;; ;;; GNU Guix is free software; you can redistribute it and/or modify it ;;; under the terms of the GNU General Public License as published by ;;; the Free Software Foundation; either version 3 of the License, or (at ;;; your option) any later version. ;;; ;;; GNU Guix is distributed in the hope that it will be useful, but ;;; WITHOUT ANY WARRANTY; without even the implied warranty of ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ;;; GNU General Public License for more details. ;;; ;;; You should have received a copy of the GNU General Public License ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. (define-module (gnu packages storage) #:use-module (guix download) #:use-module ((guix licenses) #:prefix license:) #:use-module (guix packages) #:use-module (guix utils) #:use-module (guix build-system cmake) #:use-module (gnu packages) #:use-module (gnu packages admin) #:use-module (gnu packages assembly) #:use-module (gnu packages bdw-gc) #:use-module (gnu packages boost) #:use-module (gnu packages check) #:use-module (gnu packages cmake) #:use-module (gnu packages compression) #:use-module (gnu packages crypto) #:use-module (gnu packages cryptsetup) #:use-module (gnu packages curl) #:use-module (gnu packages databases) #:use-module (gnu packages disk) #:use-module (gnu packages gnuzilla) #:use-module (gnu packages gperf) #:use-module (gnu packages jemalloc) #:use-module (gnu packages linux) #:use-module (gnu packages lua) #:use-module (gnu packages openldap) #:use-module (gnu packages perl) #:use-module (gnu packages pkg-config) #:use-module (gnu packages python) #:use-module (gnu packages tls) #:use-module (gnu packages web) #:use-module (gnu packages xml)) (define-public ceph (package (name "ceph") (version "12.2.5") (source (origin (method url-fetch) (uri (string-append "https://download.ceph.com/tarballs/ceph-" version ".tar.gz")) (sha256 (base32 "1ydc3mfvc0vpnpfnfmissvsrsj4jyxgzc2pcl1a4vdr3bwkcglp3")) (patches (search-patches "ceph-skip-unittest_blockdev.patch" "ceph-skip-collect-sys-info-test.patch" "ceph-rocksdb-compat.patch" "ceph-disable-cpu-optimizations.patch")) (modules '((guix build utils))) (snippet '(begin (for-each delete-file-recursively '(;; TODO: Unbundle these: ;"src/isa-l" ;"src/lua" ;"src/googletest" ;"src/xxHash" ;"src/zstd" ;"src/civetweb" "src/test/downloads" "src/dpdk" "src/spdk" "src/rocksdb" "src/boost")) #t)))) (build-system cmake-build-system) (arguments `(#:configure-flags (let* ((out (assoc-ref %outputs "out")) (lib (assoc-ref %outputs "lib")) (libdir (string-append lib "/lib"))) (list (string-append "-DCMAKE_INSTALL_PREFIX=" out) (string-append "-DCMAKE_INSTALL_LIBDIR=" libdir) ;; We need both libdir and libdir/ceph in RUNPATH. (string-append "-DCMAKE_INSTALL_RPATH=" libdir ";" libdir "/ceph") (string-append "-DCMAKE_INSTALL_SYSCONFDIR=" out "/etc") (string-append "-DCMAKE_INSTALL_MANDIR=" out "/share/man") (string-append "-DCMAKE_INSTALL_DOCDIR=" out "/share/ceph/doc") (string-append "-DCMAKE_INSTALL_LIBEXECDIR=" out "/libexec") (string-append "-DKEYUTILS_INCLUDE_DIR=" (assoc-ref %build-inputs "keyutils") "/include") "-DCMAKE_INSTALL_LOCALSTATEDIR=/var" "-DENABLE_SHARED=ON" "-DWITH_EMBEDDED=OFF" "-DWITH_SYSTEM_ROCKSDB=ON" "-DWITH_SYSTEM_BOOST=ON" "-DWITH_PYTHON3=ON" ;; TODO: Enable these when available in Guix. "-DWITH_BABELTRACE=OFF" "-DWITH_LTTNG=OFF" "-DWITH_XFS=OFF" "-DWITH_XIO=OFF" ;; Use jemalloc instead of tcmalloc. "-DALLOCATOR=jemalloc")) ;; FIXME: Some of the tests leak Btrfs subvolumes on Btrfs. See ;; <https://bugs.gnu.org/29674> for details. Disable tests until ;; resolved. #:tests? #f #:cmake ,cmake-3.11 #:phases (modify-phases %standard-phases (add-after 'unpack 'patch-source (lambda* (#:key outputs #:allow-other-keys) (let ((out (assoc-ref outputs "out")) (lib (assoc-ref outputs "lib"))) ;; Make header files follow the dynamic libraries. (substitute* "src/include/CMakeLists.txt" (("DESTINATION include") (string-append "DESTINATION " lib "/include"))) (substitute* "cmake/modules/Distutils.cmake" ;; Prevent creation of Python eggs. (("setup.py install") "setup.py install --single-version-externally-managed --root=/")) (substitute* (find-files "src/pybind" "^setup\\.py$") ;; Here we inject an extra line to the `setup.py' of the ;; Python C libraries so RUNPATH gets set up correctly. (("^([[:blank:]]+)extra_compile_args=(.*)$" _ indent args) (string-append indent "extra_compile_args=" args indent "extra_link_args=['-Wl,-rpath=" lib "/lib'],\n"))) (substitute* "src/ceph-disk/tox.ini" ;; Disable flake8 test since it complains about too long lines. (("envlist = flake8,py27") "envlist = py27")) (substitute* "src/ceph-detect-init/tox.ini" ;; Disable python3 tests until we at least get py2 working. (("envlist = pep8,py27,py3") "envlist = pep8,py27")) (substitute* "src/key_value_store/kv_flat_btree_async.cc" (("/usr/include/") "")) (substitute* "src/test/test_subprocess.cc" (("/bin/sh") (which "sh"))) (substitute* "qa/standalone/special/ceph_objectstore_tool.py" (("/bin/rm") (which "rm"))) (substitute* "src/ceph-disk/ceph_disk/main.py" (("/bin/mount") "mount") (("/bin/umount") "umount") (("/sbin/blkid") (which "blkid")) (("'cryptsetup'") (string-append "'" (which "cryptsetup") "'")) (("'sgdisk'") (string-append "'" (which "sgdisk") "'")) (("'parted'") (string-append "'" (which "parted") "'")) (("'udevadm'") (string-append "'" (which "udevadm") "'"))) (substitute* "udev/50-rbd.rules" (("/usr/bin/ceph-rbdnamer") (string-append out "/bin/ceph-rbdnamer"))) (substitute* "udev/60-ceph-by-parttypeuuid.rules" (("/sbin/blkid") (which "blkid"))) (substitute* "udev/95-ceph-osd.rules" (("/usr/sbin/ceph-disk") (string-append out "/bin/ceph-disk"))) (substitute* "src/test/run-cli-tests" ;; Use our python-cram instead of the (un)bundled one. (("CRAM_BIN=.*$") (string-append "CRAM_BIN=" (which "cram") "\n"))) ;; Disable tests that are known to fail. ;; TODO: The majority of these fail because ;; 'qa/workunits/ceph-helpers.sh' expects to find ;; /tmp/ceph-disk-virtualenv/bin/ceph-disk, but somehow ;; src/ceph-disk/CMakeLists.txt fails to create it. (substitute* "src/test/CMakeLists.txt" ;; FIXME: "create cannot load compressor of type zlib" ;; "libceph_zlib.so: undefined symbol: isal_deflate" (("^add_subdirectory\\(compressor\\)") "") ;; FIXME: These tests fails because `ceph-disk' ;; is not available. (("^add_ceph_test\\(test-ceph-helpers\\.sh.*$") "\n") (("^add_ceph_test\\(test_pidfile\\.sh.*$") "\n") ;; XXX Why does this fail. (("^add_ceph_test\\(cephtool-test-mon\\.sh.*$") "\n") ;; This fails due to missing '/etc/fstab'. (("^add_ceph_test\\(cephtool-test-rados\\.sh.*$") "\n") ;; `Bad messages to stderr: OSD has the store locked' (("^add_ceph_test\\(ceph_objectstore_tool\\.py.*$") "\n") ;; The bundled python-cram fork needs patching to work on ;; guix, and the system version does not support --error-dir. ;; https://bitbucket.org/brodie/cram/issues/9 (("^add_ceph_test\\(run-cli-tests.*$") "\n") ;; FIXME: tox/virtualenv/pip does not discover the ;; required packages and tries to go online. (("^add_test\\(NAME run-tox-ceph-disk.*$") "\n") (("^add_test\\(NAME run-tox-ceph-detect-init.*$") "\n") ;; Also remove from the set_property block. (("run-tox-ceph-disk") "") (("run-tox-ceph-detect-init") "")) ;; TODO: This also seems to fail because of /etc/os-release. ;; How to make src/common/util.cc behave without it. (substitute* "src/test/crush/CMakeLists.txt" (("^add_ceph_test\\(crush-classes\\.sh.*$") "\n")) ;; More 'ceph-disk' issues here.. :-( (substitute* "src/test/erasure-code/CMakeLists.txt" (("^add_ceph_test\\(test-erasure-code-plugins\\.sh.*$") "\n") (("^add_ceph_test\\(test-erasure-code\\.sh.*$") "\n") (("^add_ceph_test\\(test-erasure-eio\\.sh.*$") "\n")) (substitute* "src/test/libradosstriper/CMakeLists.txt" (("^add_ceph_test\\(rados-striper\\.sh.*$") "\n")) (substitute* "src/test/mon/CMakeLists.txt" (("^add_ceph_test\\(osd-crush\\.sh.*$") "\n") (("^add_ceph_test\\(test_pool_quota\\.sh.*$") "\n") (("^add_ceph_test\\(osd-pool-create\\.sh.*$") "\n")) (substitute* "src/test/osd/CMakeLists.txt" (("^add_ceph_test\\(osd-bench\\.sh.*$") "\n") (("^add_ceph_test\\(osd-config\\.sh.*$") "\n") (("add_ceph_test\\(osd-dup\\.sh.*$") "\n") (("^add_ceph_test\\(osd-markdown\\.sh.*$") "\n") (("^add_ceph_test\\(osd-reactivate\\.sh.*$") "\n") (("^add_ceph_test\\(osd-reuse-id\\.sh.*$") "\n") (("^add_ceph_test\\(osd-scrub-repair\\.sh.*$") "\n") (("^add_ceph_test\\(osd-scrub-snaps\\.sh.*$") "\n") (("^add_ceph_test\\(osd-copy-from\\.sh.*$") "\n") (("^add_ceph_test\\(osd-fast-mark-down\\.sh.*$") "\n")) #t))) (add-before 'check 'set-check-environment (lambda _ ;; Run tests in parallel. (setenv "CTEST_PARALLEL_LEVEL" (number->string (parallel-job-count))) ;; `pip' requires write access in $HOME. (setenv "HOME" "/tmp") #t)) (add-before 'install 'set-install-environment (lambda* (#:key outputs #:allow-other-keys) (let* ((out (assoc-ref outputs "out")) (py2sitedir (string-append out "/lib/python2.7/site-packages")) (py3sitedir (string-append out "/lib/python" ,(version-major+minor (package-version python)) "/site-packages"))) ;; The Python install scripts refuses to function if ;; the install directory is not on PYTHONPATH. (setenv "PYTHONPATH" (string-append py2sitedir ":" py3sitedir ":" (getenv "PYTHONPATH"))) #t))) (add-after 'install 'wrap-python-scripts (lambda* (#:key inputs outputs #:allow-other-keys) (let* ((out (assoc-ref outputs "out")) (ceph (string-append out "/bin/ceph")) (ceph-disk (string-append out "/bin/ceph-disk")) (ceph-detect-init (string-append out "/bin/ceph-detect-init")) (prettytable (assoc-ref inputs "python2-prettytable")) (sitedir (lambda (package) (string-append package "/lib/python2.7/site-packages"))) (PYTHONPATH (string-append (sitedir out) ":" (sitedir prettytable)))) (for-each (lambda (executable) (wrap-program executable `("PYTHONPATH" ":" prefix (,PYTHONPATH)))) (list ceph ceph-disk ceph-detect-init)) #t)))))) (outputs '("out" "lib")) (native-inputs `(("gperf" ,gperf) ("pkg-config" ,pkg-config) ("python-cython" ,python-cython) ("python-sphinx" ,python-sphinx) ("yasm" ,yasm) ;; For tests. ("inetutils" ,inetutils) ("jq" ,jq) ("perl" ,perl) ("xmlstarlet" ,xmlstarlet) ("python2-cram" ,python2-cram) ("python2-virtualenv" ,python2-virtualenv) ;; These dependencies are taken from test-requirements.txt ;; of ceph-disk and ceph-detect-init. The latter can also ;; test against python3, but let's try to get python2 tests ;; working first since that is the default. ("python2-configobj" ,python2-configobj) ("python2-coverage" ,python2-coverage) ("python2-discover" ,python2-discover) ("python2-fixtures" ,python2-fixtures) ("python2-flake8" ,python2-flake8) ("python2-mock" ,python2-mock) ("python2-nose" ,python2-nose) ("python2-pip" ,python2-pip) ("python2-pytest" ,python2-pytest) ("python2-subunit" ,python2-subunit) ("python2-testrepository" ,python2-testrepository) ("python2-testtools" ,python2-testtools) ("python2-tox" ,python2-tox))) (inputs `(("boost" ,boost-1.66) ("curl" ,curl) ("cryptsetup" ,cryptsetup) ("expat" ,expat) ("fcgi" ,fcgi) ("fuse" ,fuse) ("gptfdisk" ,gptfdisk) ("jemalloc" ,jemalloc) ("keyutils" ,keyutils) ("leveldb" ,leveldb) ("libaio" ,libaio) ("libatomic-ops" ,libatomic-ops) ("lua" ,lua) ("lz4" ,lz4) ("openldap" ,openldap) ("openssl" ,openssl) ("nss" ,nss) ("parted" ,parted) ("python@2" ,python-2) ("python2-prettytable" ,python2-prettytable) ;used by ceph_daemon.py ("python@3" ,python-3) ("rocksdb" ,rocksdb) ("snappy" ,snappy) ("udev" ,eudev) ("util-linux" ,util-linux) ("zlib" ,zlib))) (home-page "https://ceph.com/") (synopsis "Distributed object store and file system") (description "Ceph is a distributed storage system designed for reliability and performance. It provides network-based block devices (RBD), a POSIX compliant file system (CephFS), and offers compatibility with various storage protocols (S3, NFS, and others) through the RADOS gateway.") ;; The Ceph libraries are LGPL2.1 and most of the utilities fall under ;; GPL2. The installed erasure code plugins are BSD-3 licensed and do ;; not use the GPL code. The source archive includes a number of files ;; carrying other licenses; consult COPYING for more information. Note ;; that COPYING does not cover third-party bundled software. (license (list license:lgpl2.1 license:gpl2 ;some files are 'or later' license:cc-by-sa3.0 ;documentation license:bsd-3 ;isa-l,jerasure,++ license:expat)))) ;civetweb,java bindings