diff options
| -rw-r--r-- | gnu/local.mk | 4 | ||||
| -rw-r--r-- | gnu/packages/patches/ceph-disable-cpu-optimizations.patch | 46 | ||||
| -rw-r--r-- | gnu/packages/patches/ceph-skip-collect-sys-info-test.patch | 29 | ||||
| -rw-r--r-- | gnu/packages/patches/ceph-skip-unittest_blockdev.patch | 28 | ||||
| -rw-r--r-- | gnu/packages/storage.scm | 341 |
5 files changed, 448 insertions, 0 deletions
diff --git a/gnu/local.mk b/gnu/local.mk index cc187e2d20..2466f57dee 100644 --- a/gnu/local.mk +++ b/gnu/local.mk @@ -353,6 +353,7 @@ GNU_SYSTEM_MODULES = \ %D%/packages/ssh.scm \ %D%/packages/stalonetray.scm \ %D%/packages/statistics.scm \ + %D%/packages/storage.scm \ %D%/packages/suckless.scm \ %D%/packages/swig.scm \ %D%/packages/sync.scm \ @@ -508,6 +509,9 @@ dist_patch_DATA = \ %D%/packages/patches/calibre-drop-unrar.patch \ %D%/packages/patches/calibre-no-updates-dialog.patch \ %D%/packages/patches/cdparanoia-fpic.patch \ + %D%/packages/patches/ceph-disable-cpu-optimizations.patch \ + %D%/packages/patches/ceph-skip-collect-sys-info-test.patch \ + %D%/packages/patches/ceph-skip-unittest_blockdev.patch \ %D%/packages/patches/chicken-CVE-2017-6949.patch \ %D%/packages/patches/chmlib-inttypes.patch \ %D%/packages/patches/clang-libc-search-path.patch \ diff --git a/gnu/packages/patches/ceph-disable-cpu-optimizations.patch b/gnu/packages/patches/ceph-disable-cpu-optimizations.patch new file mode 100644 index 0000000000..f4d258b602 --- /dev/null +++ b/gnu/packages/patches/ceph-disable-cpu-optimizations.patch @@ -0,0 +1,46 @@ +Disable CPU optimizations not supported by the vast majority of +x86_64 systems. Also don't add anything for i686. + +--- a/cmake/modules/SIMDExt.cmake 2017-03-23 22:22:58.254071694 +0100 ++++ b/cmake/modules/SIMDExt.cmake 2017-03-23 22:23:22.446848845 +0100 +@@ -6,10 +6,6 @@ + # HAVE_INTEL_SSE + # HAVE_INTEL_SSE2 + # HAVE_INTEL_SSE3 +-# HAVE_INTEL_SSSE3 +-# HAVE_INTEL_PCLMUL +-# HAVE_INTEL_SSE4_1 +-# HAVE_INTEL_SSE4_2 + # + # SIMD_COMPILE_FLAGS + # +@@ -41,7 +37,7 @@ + if(HAVE_ARM_NEON) + set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -mfpu=neon") + endif() +-elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "i386|i686|amd64|x86_64|AMD64") ++elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64|AMD64") + set(HAVE_INTEL 1) + CHECK_C_COMPILER_FLAG(-msse HAVE_INTEL_SSE) + if(HAVE_INTEL_SSE) +@@ -55,20 +51,4 @@ + if(HAVE_INTEL_SSE3) + set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -msse3") + endif() +- CHECK_C_COMPILER_FLAG(-mssse3 HAVE_INTEL_SSSE3) +- if(HAVE_INTEL_SSSE3) +- set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -mssse3") +- endif() +- CHECK_C_COMPILER_FLAG(-mpclmul HAVE_INTEL_PCLMUL) +- if(HAVE_INTEL_PCLMUL) +- set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -mpclmul") +- endif() +- CHECK_C_COMPILER_FLAG(-msse4.1 HAVE_INTEL_SSE4_1) +- if(HAVE_INTEL_SSE4_1) +- set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -msse4.1") +- endif() +- CHECK_C_COMPILER_FLAG(-msse4.2 HAVE_INTEL_SSE4_2) +- if(HAVE_INTEL_SSE4_2) +- set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -msse4.2") +- endif() + endif() diff --git a/gnu/packages/patches/ceph-skip-collect-sys-info-test.patch b/gnu/packages/patches/ceph-skip-collect-sys-info-test.patch new file mode 100644 index 0000000000..f210799254 --- /dev/null +++ b/gnu/packages/patches/ceph-skip-collect-sys-info-test.patch @@ -0,0 +1,29 @@ +This test fails because /etc/os-release is not available. + +diff --git a/src/test/common/test_util.cc b/src/test/common/test_util.cc +index 288dd37db1..95b6cd4b30 100644 +--- a/src/test/common/test_util.cc ++++ b/src/test/common/test_util.cc +@@ -31,19 +31,3 @@ TEST(util, unit_to_bytesize) + + ASSERT_EQ(65536ll, unit_to_bytesize(" 64K", &cerr)); + } +- +-#if defined(__linux__) +-TEST(util, collect_sys_info) +-{ +- map<string, string> sys_info; +- +- CephContext *cct = (new CephContext(CEPH_ENTITY_TYPE_CLIENT))->get(); +- collect_sys_info(&sys_info, cct); +- +- ASSERT_TRUE(sys_info.find("distro") != sys_info.end()); +- ASSERT_TRUE(sys_info.find("distro_version") != sys_info.end()); +- ASSERT_TRUE(sys_info.find("distro_description") != sys_info.end()); +- +- cct->put(); +-} +-#endif +-- +2.11.1 + diff --git a/gnu/packages/patches/ceph-skip-unittest_blockdev.patch b/gnu/packages/patches/ceph-skip-unittest_blockdev.patch new file mode 100644 index 0000000000..43b9984862 --- /dev/null +++ b/gnu/packages/patches/ceph-skip-unittest_blockdev.patch @@ -0,0 +1,28 @@ +This test tries to walk a sysfs path and hits a null pointer exception. + +/tmp/guix-build-ceph-12.0.0.drv-0/ceph-12.0.0/src/test/common/test_blkdev.cc:32: Failure +Expected: (dir) != (nullptr), actual: NULL vs 8-byte object <00-00 00-00 00-00 00-00> + +diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt +index 5172663898..ac84085eaa 100644 +--- a/src/test/common/CMakeLists.txt ++++ b/src/test/common/CMakeLists.txt +@@ -12,15 +12,6 @@ target_link_libraries(get_command_descriptions + ${CMAKE_DL_LIBS} + ) + +-if(HAVE_BLKID) +-# unittest_blkdev +-add_executable(unittest_blkdev +- test_blkdev.cc +- ) +-add_ceph_unittest(unittest_blkdev ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/unittest_blkdev) +-target_link_libraries(unittest_blkdev global ${BLKID_LIBRARIES}) +-endif(HAVE_BLKID) +- + # unittest_bloom_filter + add_executable(unittest_bloom_filter + test_bloom_filter.cc +-- +2.11.1 + diff --git a/gnu/packages/storage.scm b/gnu/packages/storage.scm new file mode 100644 index 0000000000..8207c2b388 --- /dev/null +++ b/gnu/packages/storage.scm @@ -0,0 +1,341 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2017 Marius Bakke <mbakke@fastmail.com> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (gnu packages storage) + #:use-module (guix download) + #:use-module ((guix licenses) #:prefix license:) + #:use-module (guix packages) + #:use-module (guix utils) + #:use-module (guix build-system cmake) + #:use-module (gnu packages) + #:use-module (gnu packages admin) + #:use-module (gnu packages assembly) + #:use-module (gnu packages bdw-gc) + #:use-module (gnu packages boost) + #:use-module (gnu packages compression) + #:use-module (gnu packages crypto) + #:use-module (gnu packages cryptsetup) + #:use-module (gnu packages curl) + #:use-module (gnu packages databases) + #:use-module (gnu packages disk) + #:use-module (gnu packages gnuzilla) + #:use-module (gnu packages jemalloc) + #:use-module (gnu packages linux) + #:use-module (gnu packages lua) + #:use-module (gnu packages openldap) + #:use-module (gnu packages perl) + #:use-module (gnu packages pkg-config) + #:use-module (gnu packages python) + #:use-module (gnu packages tls) + #:use-module (gnu packages web) + #:use-module (gnu packages xml)) + +(define-public ceph + (package + (name "ceph") + (version "12.0.0") + (source (origin + (method url-fetch) + (uri (string-append "https://download.ceph.com/tarballs/ceph-" + version ".tar.gz")) + (sha256 + (base32 + "0yzvwlwg85w04q4d9ac73vfpxjnl72dl6kc857ihv5k1lqbpp5d0")) + (patches + (search-patches "ceph-skip-unittest_blockdev.patch" + "ceph-skip-collect-sys-info-test.patch" + "ceph-disable-cpu-optimizations.patch")) + (modules '((guix build utils))) + (snippet + '(begin + (for-each delete-file-recursively + '(;; TODO: Unbundle these: + ;"src/isa-l" + ;"src/lua" + ;"src/googletest" + ;"src/xxHash" + ;"src/zstd" + ;"src/civetweb" + ;"src/Beast" + "src/test/downloads" + "src/dpdk" + "src/spdk" + "src/rocksdb" + "src/boost")) + #t)))) + (build-system cmake-build-system) + (arguments + `(#:configure-flags + (let* ((out (assoc-ref %outputs "out")) + (lib (assoc-ref %outputs "lib")) + (libdir (string-append lib "/lib"))) + (list (string-append "-DCMAKE_INSTALL_PREFIX=" out) + (string-append "-DCMAKE_INSTALL_LIBDIR=" libdir) + ;; We need both libdir and libdir/ceph in RUNPATH. + (string-append "-DCMAKE_INSTALL_RPATH=" + libdir ";" libdir "/ceph") + (string-append "-DCMAKE_INSTALL_SYSCONFDIR=" out "/etc") + (string-append "-DCMAKE_INSTALL_MANDIR=" out "/share/man") + (string-append "-DCMAKE_INSTALL_DOCDIR=" out "/share/ceph/doc") + (string-append "-DCMAKE_INSTALL_LIBEXECDIR=" out "/libexec") + (string-append "-DKEYUTILS_INCLUDE_DIR=" + (assoc-ref %build-inputs "keyutils") "/include") + "-DCMAKE_INSTALL_LOCALSTATEDIR=/var" + "-DENABLE_SHARED=ON" + "-DWITH_EMBEDDED=OFF" + "-DWITH_SYSTEM_ROCKSDB=ON" + "-DWITH_SYSTEM_BOOST=ON" + "-DWITH_PYTHON3=ON" + ;; TODO: Enable these when available in Guix. + "-DWITH_LTTNG=OFF" + "-DWITH_XFS=OFF" + "-DWITH_XIO=OFF" + ;; Use jemalloc instead of tcmalloc. + "-DALLOCATOR=jemalloc")) + #:phases + (modify-phases %standard-phases + (add-after 'unpack 'patch-source + (lambda* (#:key outputs #:allow-other-keys) + (let ((out (assoc-ref outputs "out")) + (lib (assoc-ref outputs "lib"))) + + ;; Make header files follow the dynamic libraries. + (substitute* "src/include/CMakeLists.txt" + (("DESTINATION include") + (string-append "DESTINATION " lib "/include"))) + + (substitute* "cmake/modules/Distutils.cmake" + ;; Prevent creation of Python eggs. + (("setup.py install") + "setup.py install --single-version-externally-managed --root=/")) + + (substitute* (find-files "src/pybind" "^setup\\.py$") + ;; Here we inject an extra line to the `setup.py' of the + ;; Python C libraries so RUNPATH gets set up correctly. + (("^([[:blank:]]+)extra_compile_args=(.*)$" _ indent args) + (string-append indent "extra_compile_args=" args + indent "extra_link_args=['-Wl,-rpath=" + lib "/lib'],\n"))) + + (substitute* "src/ceph-disk/tox.ini" + ;; Disable flake8 test since it complains about too long lines. + (("envlist = flake8,py27") "envlist = py27")) + + (substitute* "src/ceph-detect-init/tox.ini" + ;; Disable python3 tests until we at least get py2 working. + (("envlist = pep8,py27,py3") "envlist = pep8,py27")) + + (substitute* "src/key_value_store/kv_flat_btree_async.cc" + (("/usr/include/") "")) + + (substitute* "src/test/test_subprocess.cc" + (("/bin/sh") (which "sh"))) + (substitute* "src/test/ceph_objectstore_tool.py" + (("/bin/rm") (which "rm"))) + (substitute* "src/ceph-disk/ceph_disk/main.py" + (("/bin/mount") "mount") + (("/bin/umount") "umount") + (("/sbin/blkid") (which "blkid")) + (("'sgdisk'") (string-append "'" (which "sgdisk") "'")) + (("'parted'") (string-append "'" (which "parted") "'")) + (("'udevadm'") (string-append "'" (which "udevadm") "'"))) + + (substitute* "src/ceph-disk-udev" + (("/sbin/cryptsetup") (which "cryptsetup")) + (("/usr/sbin/sgdisk") (which "sgdisk")) + (("/usr/sbin/ceph-disk") + (string-append out "/bin/ceph-disk"))) + (substitute* "udev/50-rbd.rules" + (("/usr/bin/ceph-rbdnamer") + (string-append out "/bin/ceph-rbdnamer"))) + (substitute* "udev/60-ceph-by-parttypeuuid.rules" + (("/sbin/blkid") (which "blkid"))) + (substitute* "udev/95-ceph-osd.rules" + (("/usr/sbin/ceph-disk") + (string-append out "/bin/ceph-disk"))) + + (substitute* "src/test/run-cli-tests" + ;; Use our python-cram instead of the (un)bundled one. + (("CRAM_BIN=.*$") + (string-append "CRAM_BIN=" (which "cram") "\n"))) + + ;; Disable tests that are known to fail. + ;; TODO: The majority of these fail because + ;; 'qa/workunits/ceph-helpers.sh' expects to find + ;; /tmp/ceph-disk-virtualenv/bin/ceph-disk, but somehow + ;; src/ceph-disk/CMakeLists.txt fails to create it. + (substitute* "src/test/CMakeLists.txt" + ;; FIXME: "create cannot load compressor of type zlib" + ;; "libceph_zlib.so: undefined symbol: isal_deflate" + (("^add_subdirectory\\(compressor\\)") "") + ;; FIXME: These tests fails because `ceph-disk' + ;; is not available. + (("^add_ceph_test\\(test-ceph-helpers\\.sh.*$") "\n") + (("^add_ceph_test\\(test_pidfile\\.sh.*$") "\n") + ;; XXX Why does this fail. + (("^add_ceph_test\\(cephtool-test-mon\\.sh.*$") "\n") + ;; This fails due to missing '/etc/fstab'. + (("^add_ceph_test\\(cephtool-test-rados\\.sh.*$") "\n") + ;; `Bad messages to stderr: OSD has the store locked' + (("^add_ceph_test\\(ceph_objectstore_tool\\.py.*$") "\n") + ;; The bundled python-cram fork needs patching to work on + ;; guix, and the system version does not support --error-dir. + ;; https://bitbucket.org/brodie/cram/issues/9 + (("^add_ceph_test\\(run-cli-tests.*$") "\n") + ;; FIXME: tox/virtualenv/pip does not discover the + ;; required packages and tries to go online. + (("^add_test\\(NAME run-tox-ceph-disk.*$") "\n") + (("^add_test\\(NAME run-tox-ceph-detect-init.*$") "\n") + ;; Also remove from the set_property block. + (("run-tox-ceph-disk") "") + (("run-tox-ceph-detect-init") "")) + ;; More 'ceph-disk' issues here.. :-( + (substitute* "src/test/erasure-code/CMakeLists.txt" + (("^add_ceph_test\\(test-erasure-code-plugins\\.sh.*$") "\n") + (("^add_ceph_test\\(test-erasure-code\\.sh.*$") "\n") + (("^add_ceph_test\\(test-erasure-eio\\.sh.*$") "\n")) + (substitute* "src/test/libradosstriper/CMakeLists.txt" + (("^add_ceph_test\\(rados-striper\\.sh.*$") "\n")) + (substitute* "src/test/mon/CMakeLists.txt" + (("^add_ceph_test\\(osd-crush\\.sh.*$") "\n") + (("^add_ceph_test\\(test_pool_quota\\.sh.*$") "\n") + (("^add_ceph_test\\(osd-pool-create\\.sh.*$") "\n")) + (substitute* "src/test/osd/CMakeLists.txt" + (("^add_ceph_test\\(osd-bench\\.sh.*$") "\n") + (("^add_ceph_test\\(osd-config\\.sh.*$") "\n") + (("^add_ceph_test\\(osd-markdown\\.sh.*$") "\n") + (("^add_ceph_test\\(osd-reactivate\\.sh.*$") "\n") + (("^add_ceph_test\\(osd-reuse-id\\.sh.*$") "\n") + (("^add_ceph_test\\(osd-scrub-repair\\.sh.*$") "\n") + (("^add_ceph_test\\(osd-scrub-snaps\\.sh.*$") "\n") + (("^add_ceph_test\\(osd-copy-from\\.sh.*$") "\n") + (("^add_ceph_test\\(osd-fast-mark-down\\.sh.*$") "\n")) + #t))) + (add-before 'check 'set-check-environment + (lambda _ + ;; Run tests in parallel. + (setenv "CTEST_PARALLEL_LEVEL" + (number->string (parallel-job-count))) + ;; `pip' requires write access in $HOME. + (setenv "HOME" "/tmp") + #t)) + (add-before 'install 'set-install-environment + (lambda* (#:key outputs #:allow-other-keys) + (let* ((out (assoc-ref outputs "out")) + (py2sitedir + (string-append out "/lib/python2.7/site-packages")) + (py3sitedir + (string-append out "/lib/python" + ,(version-major+minor + (package-version python)) + "/site-packages"))) + ;; The Python install scripts refuses to function if + ;; the install directory is not on PYTHONPATH. + (setenv "PYTHONPATH" + (string-append py2sitedir ":" py3sitedir ":" + (getenv "PYTHONPATH"))) + #t))) + (add-after 'install 'wrap-python-scripts + (lambda* (#:key outputs #:allow-other-keys) + (let* ((out (assoc-ref outputs "out")) + (ceph-disk (string-append out "/bin/ceph-disk")) + (ceph-detect-init (string-append + out "/bin/ceph-detect-init")) + (PYTHONPATH (string-append + out "/lib/python2.7/site-packages"))) + + (for-each (lambda (executable) + (wrap-program executable + `("PYTHONPATH" ":" prefix (,PYTHONPATH)))) + (list ceph-disk ceph-detect-init)) + #t)))))) + (outputs + '("out" "lib")) + (native-inputs + `(("pkg-config" ,pkg-config) + ("python-cython" ,python-cython) + ("python-sphinx" ,python-sphinx) + ("yasm" ,yasm) + + ;; For tests. + ("inetutils" ,inetutils) + ("jq" ,jq) + ("perl" ,perl) + ("xmlstarlet" ,xmlstarlet) + ("python2-cram" ,python2-cram) + ("python2-virtualenv" ,python2-virtualenv) + + ;; These dependencies are taken from test-requirements.txt + ;; of ceph-disk and ceph-detect-init. The latter can also + ;; test against python3, but let's try to get python2 tests + ;; working first since that is the default. + ("python2-configobj" ,python2-configobj) + ("python2-coverage" ,python2-coverage) + ("python2-discover" ,python2-discover) + ("python2-fixtures" ,python2-fixtures) + ("python2-flake8" ,python2-flake8) + ("python2-mock" ,python2-mock) + ("python2-nose" ,python2-nose) + ("python2-pip" ,python2-pip) + ("python2-pytest" ,python2-pytest) + ("python2-subunit" ,python2-subunit) + ("python2-testrepository" ,python2-testrepository) + ("python2-testtools" ,python2-testtools) + ("python2-tox" ,python2-tox))) + (inputs + `(("boost" ,boost) + ("curl" ,curl) + ("cryptsetup" ,cryptsetup) + ("expat" ,expat) + ("fcgi" ,fcgi) + ("fuse" ,fuse) + ("gptfdisk" ,gptfdisk) + ("jemalloc" ,jemalloc) + ("keyutils" ,keyutils) + ("leveldb" ,leveldb) + ("libaio" ,libaio) + ("libatomic-ops" ,libatomic-ops) + ("lua" ,lua) + ("lz4" ,lz4) + ("openldap" ,openldap) + ("openssl" ,openssl) + ("nss" ,nss) + ("parted" ,parted) + ("python@2" ,python-2) + ("python@3" ,python-3) + ("rocksdb" ,rocksdb) + ("snappy" ,snappy) + ("udev" ,eudev) + ("util-linux" ,util-linux) + ("zlib" ,zlib))) + (home-page "https://ceph.com/") + (synopsis "Distributed object store and file system") + (description + "Ceph is a distributed storage system designed for reliability and +performance. It provides network-based block devices (RBD), a POSIX +compliant filesystem (CephFS), and offers compatibility with various +storage protocols (S3, NFS, and others) through the RADOS gateway.") + ;; The Ceph libraries are LGPL2.1 and most of the utilities fall under + ;; GPL2. The installed erasure code plugins are BSD-3 licensed and do + ;; not use the GPL code. The source archive includes a number of files + ;; carrying other licenses; consult COPYING for more information. Note + ;; that COPYING does not cover third-party bundled software. + (license (list license:lgpl2.1 license:gpl2 ;some files are 'or later' + license:cc-by-sa3.0 ;documentation + license:bsd-3 ;isa-l,jerasure,++ + license:expat)))) ;civetweb,java bindings |