diff options
Diffstat (limited to 'guix')
48 files changed, 2865 insertions, 656 deletions
diff --git a/guix/build-system/cargo.scm b/guix/build-system/cargo.scm new file mode 100644 index 0000000000..3582f0e328 --- /dev/null +++ b/guix/build-system/cargo.scm @@ -0,0 +1,149 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2013, 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2013 Andreas Enge <andreas@enge.fr> +;;; Copyright © 2013 Nikita Karetnikov <nikita@karetnikov.org> +;;; Copyright © 2016 David Craven <david@craven.ch> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix build-system cargo) + #:use-module (guix search-paths) + #:use-module (guix store) + #:use-module (guix utils) + #:use-module (guix derivations) + #:use-module (guix packages) + #:use-module (guix build-system) + #:use-module (guix build-system gnu) + #:use-module (ice-9 match) + #:use-module (srfi srfi-26) + #:export (cargo-build-system + crate-url + crate-url? + crate-uri)) + +(define crate-url "https://crates.io/api/v1/crates/") +(define crate-url? (cut string-prefix? crate-url <>)) + +(define (crate-uri name version) + "Return a URI string for the crate package hosted at crates.io corresponding +to NAME and VERSION." + (string-append crate-url name "/" version "/download")) + +(define (default-cargo) + "Return the default Cargo package." + ;; Lazily resolve the binding to avoid a circular dependency. + (let ((rust (resolve-interface '(gnu packages rust)))) + (module-ref rust 'cargo))) + +(define (default-rustc) + "Return the default Rustc package." + ;; Lazily resolve the binding to avoid a circular dependency. + (let ((rust (resolve-interface '(gnu packages rust)))) + (module-ref rust 'rustc))) + +(define %cargo-build-system-modules + ;; Build-side modules imported by default. + `((guix build cargo-build-system) + ,@%gnu-build-system-modules)) + +(define* (cargo-build store name inputs + #:key + (tests? #t) + (test-target #f) + (configure-flags #f) + (phases '(@ (guix build cargo-build-system) + %standard-phases)) + (outputs '("out")) + (search-paths '()) + (system (%current-system)) + (guile #f) + (imported-modules %cargo-build-system-modules) + (modules '((guix build cargo-build-system) + (guix build utils)))) + "Build SOURCE using CARGO, and with INPUTS." + + (define builder + `(begin + (use-modules ,@modules) + (cargo-build #:name ,name + #:source ,(match (assoc-ref inputs "source") + (((? derivation? source)) + (derivation->output-path source)) + ((source) + source) + (source + source)) + #:system ,system + #:test-target ,test-target + #:tests? ,tests? + #:phases ,phases + #:outputs %outputs + #:search-paths ',(map search-path-specification->sexp + search-paths) + #:inputs %build-inputs))) + + (define guile-for-build + (match guile + ((? package?) + (package-derivation store guile system #:graft? #f)) + (#f ; the default + (let* ((distro (resolve-interface '(gnu packages commencement))) + (guile (module-ref distro 'guile-final))) + (package-derivation store guile system #:graft? #f))))) + + (build-expression->derivation store name builder + #:inputs inputs + #:system system + #:modules imported-modules + #:outputs (cons "src" outputs) + #:guile-for-build guile-for-build)) + +(define* (lower name + #:key source inputs native-inputs outputs system target + (cargo (default-cargo)) + (rustc (default-rustc)) + #:allow-other-keys + #:rest arguments) + "Return a bag for NAME." + + (define private-keywords + '(#:source #:target #:cargo #:rustc #:inputs #:native-inputs #:outputs)) + + (and (not target) ;; TODO: support cross-compilation + (bag + (name name) + (system system) + (target target) + (host-inputs `(,@(if source + `(("source" ,source)) + '()) + ,@inputs + + ;; Keep the standard inputs of 'gnu-build-system' + ,@(standard-packages))) + (build-inputs `(("cargo" ,cargo) + ("rustc" ,rustc) + ,@native-inputs)) + (outputs outputs) + (build cargo-build) + (arguments (strip-keyword-arguments private-keywords arguments))))) + +(define cargo-build-system + (build-system + (name 'cargo) + (description + "Cargo build system, to build Rust crates") + (lower lower))) diff --git a/guix/build-system/ocaml.scm b/guix/build-system/ocaml.scm new file mode 100644 index 0000000000..f4f57b5ad5 --- /dev/null +++ b/guix/build-system/ocaml.scm @@ -0,0 +1,181 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2016, 2017 Julien Lepiller <julien@lepiller.eu> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix build-system ocaml) + #:use-module (guix store) + #:use-module (guix utils) + #:use-module (guix derivations) + #:use-module (guix search-paths) + #:use-module (guix build-system) + #:use-module (guix build-system gnu) + #:use-module (guix packages) + #:use-module (ice-9 match) + #:export (%ocaml-build-system-modules + ocaml-build + ocaml-build-system)) + +;; Commentary: +;; +;; Standard build procedure for packages using ocaml. This is implemented as an +;; extension of `gnu-build-system'. +;; +;; OCaml packages don't use a single standard for their build system. Some use +;; autotools, other use custom configure scripts with Makefiles, others use +;; oasis to generate the configure script and Makefile and lastly, some use +;; custom ocaml scripts. +;; +;; Each phase in the build system will try to figure out what the build system +;; is for that package. Most packages come with a custom configure script and +;; a Makefile that in turn call custom build tools. Packages built with oasis +;; will have a `setup.ml' file in the top directory, that can be used for all +;; phases. In that case the Makefile is here only to call that script. In case +;; the setup.ml do not work as expected, the @var{use-make} argument can be +;; used to ignore the setup.ml file and run make instead. +;; +;; Some packages use their own custom scripts, `pkg/pkg.ml' or +;; `pkg/build.ml'. They can be used here too. +;; +;; Code: + +(define %ocaml-build-system-modules + ;; Build-side modules imported by default. + `((guix build ocaml-build-system) + ,@%gnu-build-system-modules)) + +(define (default-ocaml) + "Return the default OCaml package." + + ;; Do not use `@' to avoid introducing circular dependencies. + (let ((module (resolve-interface '(gnu packages ocaml)))) + (module-ref module 'ocaml))) + +(define (default-findlib) + "Return the default OCaml-findlib package." + + ;; Do not use `@' to avoid introducing circular dependencies. + (let ((module (resolve-interface '(gnu packages ocaml)))) + (module-ref module 'ocaml-findlib))) + +(define* (lower name + #:key source inputs native-inputs outputs system target + (ocaml (default-ocaml)) + (findlib (default-findlib)) + #:allow-other-keys + #:rest arguments) + "Return a bag for NAME." + (define private-keywords + '(#:source #:target #:ocaml #:findlib #:inputs #:native-inputs)) + + (and (not target) ;XXX: no cross-compilation + (bag + (name name) + (system system) + (host-inputs `(,@(if source + `(("source" ,source)) + '()) + ,@inputs + + ;; Keep the standard inputs of 'gnu-build-system'. + ,@(standard-packages))) + (build-inputs `(("ocaml" ,ocaml) + ("findlib" ,findlib) + ,@native-inputs)) + (outputs outputs) + (build ocaml-build) + (arguments (strip-keyword-arguments private-keywords arguments))))) + +(define* (ocaml-build store name inputs + #:key (guile #f) + (outputs '("out")) (configure-flags ''()) + (search-paths '()) + (make-flags ''()) + (build-flags ''()) + (out-of-source? #t) + (use-make? #f) + (tests? #t) + (test-flags ''("--enable-tests")) + (test-target "test") + (install-target "install") + (validate-runpath? #t) + (patch-shebangs? #t) + (strip-binaries? #t) + (strip-flags ''("--strip-debug")) + (strip-directories ''("lib" "lib64" "libexec" + "bin" "sbin")) + (phases '(@ (guix build ocaml-build-system) + %standard-phases)) + (system (%current-system)) + (imported-modules %ocaml-build-system-modules) + (modules '((guix build ocaml-build-system) + (guix build utils)))) + "Build SOURCE using OCAML, and with INPUTS. This assumes that SOURCE +provides a 'setup.ml' file as its build system." + (define builder + `(begin + (use-modules ,@modules) + (ocaml-build #:source ,(match (assoc-ref inputs "source") + (((? derivation? source)) + (derivation->output-path source)) + ((source) + source) + (source + source)) + #:system ,system + #:outputs %outputs + #:inputs %build-inputs + #:search-paths ',(map search-path-specification->sexp + search-paths) + #:phases ,phases + #:configure-flags ,configure-flags + #:test-flags ,test-flags + #:make-flags ,make-flags + #:build-flags ,build-flags + #:out-of-source? ,out-of-source? + #:use-make? ,use-make? + #:tests? ,tests? + #:test-target ,test-target + #:install-target ,install-target + #:validate-runpath? ,validate-runpath? + #:patch-shebangs? ,patch-shebangs? + #:strip-binaries? ,strip-binaries? + #:strip-flags ,strip-flags + #:strip-directories ,strip-directories))) + + (define guile-for-build + (match guile + ((? package?) + (package-derivation store guile system #:graft? #f)) + (#f ; the default + (let* ((distro (resolve-interface '(gnu packages commencement))) + (guile (module-ref distro 'guile-final))) + (package-derivation store guile system #:graft? #f))))) + + (build-expression->derivation store name builder + #:system system + #:inputs inputs + #:modules imported-modules + #:outputs outputs + #:guile-for-build guile-for-build)) + +(define ocaml-build-system + (build-system + (name 'ocaml) + (description "The standard OCaml build system") + (lower lower))) + +;;; ocaml.scm ends here diff --git a/guix/build-system/python.scm b/guix/build-system/python.scm index adeceb4a89..d4d3d28f2a 100644 --- a/guix/build-system/python.scm +++ b/guix/build-system/python.scm @@ -177,6 +177,7 @@ pre-defined variants." #:key (tests? #t) (test-target "test") + (use-setuptools? #t) (configure-flags ''()) (phases '(@ (guix build python-build-system) %standard-phases)) @@ -204,6 +205,7 @@ provides a 'setup.py' file as its build system." #:system ,system #:test-target ,test-target #:tests? ,tests? + #:use-setuptools? ,use-setuptools? #:phases ,phases #:outputs %outputs #:search-paths ',(map search-path-specification->sexp diff --git a/guix/build/cargo-build-system.scm b/guix/build/cargo-build-system.scm new file mode 100644 index 0000000000..7d656a8d58 --- /dev/null +++ b/guix/build/cargo-build-system.scm @@ -0,0 +1,110 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2016 David Craven <david@craven.ch> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix build cargo-build-system) + #:use-module ((guix build gnu-build-system) #:prefix gnu:) + #:use-module (guix build utils) + #:use-module (ice-9 ftw) + #:use-module (ice-9 format) + #:use-module (ice-9 match) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-26) + #:export (%standard-phases + cargo-build)) + +;; Commentary: +;; +;; Builder-side code of the standard Rust package build procedure. +;; +;; Code: + +;; FIXME: Needs to be parsed from url not package name. +(define (package-name->crate-name name) + "Return the crate name of NAME." + (match (string-split name #\-) + (("rust" rest ...) + (string-join rest "-")) + (_ #f))) + +(define* (configure #:key inputs #:allow-other-keys) + "Replace Cargo.toml [dependencies] section with guix inputs." + ;; Make sure Cargo.toml is writeable when the crate uses git-fetch. + (chmod "Cargo.toml" #o644) + (let ((port (open-file "Cargo.toml" "a" #:encoding "utf-8"))) + (format port "~%[replace]~%") + (for-each + (match-lambda + ((name . path) + (let ((crate (package-name->crate-name name))) + (when (and crate path) + (match (string-split (basename path) #\-) + ((_ ... version) + (format port "\"~a:~a\" = { path = \"~a/share/rust-source\" }~%" + crate version path))))))) + inputs) + (close-port port)) + #t) + +(define* (build #:key (cargo-build-flags '("--release" "--frozen")) + #:allow-other-keys) + "Build a given Cargo package." + (if (file-exists? "Cargo.lock") + (zero? (apply system* `("cargo" "build" ,@cargo-build-flags))) + #t)) + +(define* (check #:key tests? #:allow-other-keys) + "Run tests for a given Cargo package." + (if (and tests? (file-exists? "Cargo.lock")) + (zero? (system* "cargo" "test")) + #t)) + +(define* (install #:key inputs outputs #:allow-other-keys) + "Install a given Cargo package." + (let* ((out (assoc-ref outputs "out")) + (src (assoc-ref inputs "source")) + (rsrc (string-append (assoc-ref outputs "src") + "/share/rust-source"))) + (mkdir-p rsrc) + ;; Rust doesn't have a stable ABI yet. Because of this + ;; Cargo doesn't have a search path for binaries yet. + ;; Until this changes we are working around this by + ;; distributing crates as source and replacing + ;; references in Cargo.toml with store paths. + (copy-recursively "src" (string-append rsrc "/src")) + (install-file "Cargo.toml" rsrc) + ;; When the package includes executables we install + ;; it using cargo install. This fails when the crate + ;; doesn't contain an executable. + (if (file-exists? "Cargo.lock") + (system* "cargo" "install" "--root" out) + (mkdir out)))) + +(define %standard-phases + ;; 'configure' phase is not needed. + (modify-phases gnu:%standard-phases + (replace 'configure configure) + (replace 'build build) + (replace 'check check) + (replace 'install install))) + +(define* (cargo-build #:key inputs (phases %standard-phases) + #:allow-other-keys #:rest args) + "Build the given Cargo package, applying all of PHASES in order." + (apply gnu:gnu-build #:inputs inputs #:phases phases args)) + +;;; cargo-build-system.scm ends here diff --git a/guix/build/cmake-build-system.scm b/guix/build/cmake-build-system.scm index f57622e0f4..27f2b5c872 100644 --- a/guix/build/cmake-build-system.scm +++ b/guix/build/cmake-build-system.scm @@ -66,6 +66,7 @@ (define* (check #:key (tests? #t) (parallel-tests? #t) (test-target "test") #:allow-other-keys) (let ((gnu-check (assoc-ref gnu:%standard-phases 'check))) + (setenv "CTEST_OUTPUT_ON_FAILURE" "1") (gnu-check #:tests? tests? #:test-target test-target #:parallel-tests? parallel-tests?))) diff --git a/guix/build/download.scm b/guix/build/download.scm index 8e32b3d7ff..203338b527 100644 --- a/guix/build/download.scm +++ b/guix/build/download.scm @@ -289,9 +289,12 @@ DIRECTORY. Those authority certificates are checked when (string-suffix? ".pem" file))) '()))) (for-each (lambda (file) - (set-certificate-credentials-x509-trust-file! - cred (string-append directory "/" file) - x509-certificate-format/pem)) + (let ((file (string-append directory "/" file))) + ;; Protect against dangling symlinks. + (when (file-exists? file) + (set-certificate-credentials-x509-trust-file! + cred file + x509-certificate-format/pem)))) (or files '())) cred)) diff --git a/guix/build/make-bootstrap.scm b/guix/build/make-bootstrap.scm new file mode 100644 index 0000000000..21c78cc8f5 --- /dev/null +++ b/guix/build/make-bootstrap.scm @@ -0,0 +1,85 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2015, 2017 Manolis Fragkiskos Ragkousis <manolis837@gmail.com> +;;; Copyright © 2015 Ludovic Courtès <ludo@gnu.org> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix build make-bootstrap) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-19) + #:use-module (srfi srfi-26) + #:use-module (guix build utils) + #:export (make-stripped-libc)) + +;; Commentary: +;; +;; This module provides facilities to build the bootstrap binaries. +;; +;; Code: + +(define (make-stripped-libc output libc kernel-headers) + "Copy to OUTPUT the subset of LIBC and KERNEL-HEADERS that is needed +when producing a bootstrap libc." + + (define (copy-mach-headers output kernel-headers) + (let* ((incdir (string-append output "/include"))) + (copy-recursively (string-append libc "/include") incdir) + + (copy-recursively (string-append kernel-headers "/include/mach") + (string-append incdir "/mach")) + #t)) + + (define (copy-linux-headers output kernel-headers) + (let* ((incdir (string-append output "/include"))) + (copy-recursively (string-append libc "/include") incdir) + + ;; Copy some of the Linux-Libre headers that glibc headers + ;; refer to. + (mkdir (string-append incdir "/linux")) + (for-each (lambda (file) + (install-file (string-append kernel-headers "/include/linux/" file) + (string-append incdir "/linux"))) + '("limits.h" "errno.h" "socket.h" "kernel.h" + "sysctl.h" "param.h" "ioctl.h" "types.h" + "posix_types.h" "stddef.h")) + + (copy-recursively (string-append kernel-headers "/include/asm") + (string-append incdir "/asm")) + (copy-recursively (string-append kernel-headers "/include/asm-generic") + (string-append incdir "/asm-generic")) + #t)) + + (define %libc-object-files-rx "^(crt.*|ld.*|lib(c|m|dl|rt|pthread|nsl|\ +util).*\\.so(\\..*)?|lib(machuser|hurduser).so.*|(libc(rt|)|libpthread)\ +_nonshared\\.a)$") + + (setvbuf (current-output-port) _IOLBF) + (let* ((libdir (string-append output "/lib"))) + (mkdir-p libdir) + (for-each (lambda (file) + (let ((target (string-append libdir "/" + (basename file)))) + (copy-file file target) + (remove-store-references target))) + (find-files (string-append libc "/lib") %libc-object-files-rx)) + #t) + + (if (directory-exists? (string-append kernel-headers "/include/mach")) + (copy-mach-headers output kernel-headers) + (copy-linux-headers output kernel-headers))) + + diff --git a/guix/build/ocaml-build-system.scm b/guix/build/ocaml-build-system.scm new file mode 100644 index 0000000000..f77251ca09 --- /dev/null +++ b/guix/build/ocaml-build-system.scm @@ -0,0 +1,119 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2016, 2017 Julien Lepiller <julien@lepiller.eu> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix build ocaml-build-system) + #:use-module ((guix build gnu-build-system) #:prefix gnu:) + #:use-module (guix build utils) + #:use-module (ice-9 match) + #:export (%standard-phases + ocaml-build)) + +;; Commentary: +;; +;; Builder-side code of the standard ocaml build procedure. +;; +;; Code: + +(define* (ocaml-findlib-environment #:key outputs #:allow-other-keys) + (let* ((out (assoc-ref outputs "out"))) + (setenv "OCAMLFIND_DESTDIR" (string-append out "/lib/ocaml/site-lib")) + (setenv "OCAMLFIND_LDCONF" "ignore")) + #t) + +(define* (configure #:key outputs (configure-flags '()) + (test-flags '("--enable-tests")) tests? + #:allow-other-keys) + "Configure the given package." + (let* ((out (assoc-ref outputs "out"))) + (format #t "build directory: ~s~%" (getcwd)) + (if (file-exists? "setup.ml") + (let ((args `("-configure" + "--prefix" ,out + ,@(if tests? + test-flags + '()) + ,@configure-flags))) + (format #t "running 'setup.ml' with arguments ~s~%" args) + (zero? (apply system* "ocaml" "setup.ml" args))) + (let ((args `("-prefix" ,out ,@configure-flags))) + (format #t "running 'configure' with arguments ~s~%" args) + (zero? (apply system* "./configure" args)))))) + +(define* (build #:key inputs outputs (build-flags '()) (make-flags '()) + (use-make? #f) #:allow-other-keys) + "Build the given package." + (if (and (file-exists? "setup.ml") (not use-make?)) + (zero? (apply system* "ocaml" "setup.ml" "-build" build-flags)) + (if (file-exists? "Makefile") + (zero? (apply system* "make" make-flags)) + (let ((file (if (file-exists? "pkg/pkg.ml") "pkg/pkg.ml" "pkg/build.ml"))) + (zero? (apply system* "ocaml" "-I" + (string-append (assoc-ref inputs "findlib") + "/lib/ocaml/site-lib") + file build-flags)))))) + +(define* (check #:key inputs outputs (make-flags '()) (test-target "test") tests? + (use-make? #f) #:allow-other-keys) + "Install the given package." + (when tests? + (if (and (file-exists? "setup.ml") (not use-make?)) + (zero? (system* "ocaml" "setup.ml" (string-append "-" test-target))) + (if (file-exists? "Makefile") + (zero? (apply system* "make" test-target make-flags)) + (let ((file (if (file-exists? "pkg/pkg.ml") "pkg/pkg.ml" "pkg/build.ml"))) + (zero? (system* "ocaml" "-I" + (string-append (assoc-ref inputs "findlib") + "/lib/ocaml/site-lib") + file test-target))))))) + +(define* (install #:key outputs (build-flags '()) (make-flags '()) (use-make? #f) + (install-target "install") + #:allow-other-keys) + "Install the given package." + (let ((out (assoc-ref outputs "out"))) + (if (and (file-exists? "setup.ml") (not use-make?)) + (zero? (apply system* "ocaml" "setup.ml" + (string-append "-" install-target) build-flags)) + (if (file-exists? "Makefile") + (zero? (apply system* "make" install-target make-flags)) + (zero? (system* "opam-installer" "-i" (string-append "--prefix=" out) + (string-append "--libdir=" out "/lib/ocaml/site-lib"))))))) + +(define* (prepare-install #:key outputs #:allow-other-keys) + "Prepare for building the given package." + (mkdir-p (string-append (assoc-ref outputs "out") "/lib/ocaml/site-lib")) + (mkdir-p (string-append (assoc-ref outputs "out") "/bin"))) + +(define %standard-phases + ;; Everything is as with the GNU Build System except for the `configure' + ;; , `build', `check' and `install' phases. + (modify-phases gnu:%standard-phases + (add-before 'configure 'ocaml-findlib-environment + ocaml-findlib-environment) + (add-before 'install 'prepare-install prepare-install) + (replace 'configure configure) + (replace 'build build) + (replace 'check check) + (replace 'install install))) + +(define* (ocaml-build #:key inputs (phases %standard-phases) + #:allow-other-keys #:rest args) + "Build the given package, applying all of PHASES in order." + (apply gnu:gnu-build #:inputs inputs #:phases phases args)) + +;;; ocaml-build-system.scm ends here diff --git a/guix/build/pull.scm b/guix/build/pull.scm index 871bf6f535..6034e93cbf 100644 --- a/guix/build/pull.scm +++ b/guix/build/pull.scm @@ -84,7 +84,7 @@ containing the source code. Write any debugging output to DEBUG-PORT." (("@GZIP@") (string-append gzip "/bin/gzip")) (("@BZIP2@") (string-append bzip2 "/bin/bzip2")) (("@XZ@") (string-append xz "/bin/xz")) - (("@NIX_INSTANTIATE@") "")) ;remnants from the past + (("@NIX_INSTANTIATE@") "nix-instantiate")) ;for (guix import nix) ;; Augment the search path so Scheme code can be compiled. (set! %load-path (cons out %load-path)) diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 9109fb4ac7..3f280b0ac0 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -3,6 +3,7 @@ ;;; Copyright © 2013 Andreas Enge <andreas@enge.fr> ;;; Copyright © 2013 Nikita Karetnikov <nikita@karetnikov.org> ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org> +;;; Copyright © 2016 Hartmut Goebel <h.goebel@crazy-compilers.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -27,31 +28,119 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) #:export (%standard-phases + add-installed-pythonpath + site-packages python-build)) ;; Commentary: ;; ;; Builder-side code of the standard Python package build procedure. ;; -;; Code: +;; +;; Backgound about the Python installation methods +;; +;; In Python there are different ways to install packages: distutils, +;; setuptools, easy_install and pip. All of these are sharing the file +;; setup.py, introduced with distutils in Python 2.0. The setup.py file can be +;; considered as a kind of Makefile accepting targets (or commands) like +;; "build" and "install". As of autumn 2016 the recommended way to install +;; Python packages is using pip. +;; +;; For both distutils and setuptools, running "python setup.py install" is the +;; way to install Python packages. With distutils the "install" command +;; basically copies all packages into <prefix>/lib/pythonX.Y/site-packages. +;; +;; Some time later "setuptools" was established to enhance distutils. To use +;; setuptools, the developer imports setuptools in setup.py. When importing +;; setuptools, the original "install" command gets overwritten by setuptools' +;; "install" command. +;; +;; The command-line tools easy_install and pip are both capable of finding and +;; downloading the package source from PyPI (the Python Package Index). Both +;; of them import setuptools and execute the "setup.py" file under their +;; control. Thus the "setup.py" behaves as if the developer had imported +;; setuptools within setup.py - even is still using only distutils. +;; +;; Setuptools' "install" command (to be more precise: the "easy_install" +;; command which is called by "install") will put the path of the currently +;; installed version of each package and it's dependencies (as declared in +;; setup.py) into an "easy-install.pth" file. In Guix each packages gets its +;; own "site-packages" directory and thus an "easy-install.pth" of its own. +;; To avoid conflicts, the python build system renames the file to +;; <packagename>.pth in the phase rename-pth-file. To ensure that Python will +;; process the .pth file, easy_install also creates a basic "site.py" in each +;; "site-packages" directory. The file is the same for all packages, thus +;; there is no need to rename it. For more information about .pth files and +;; the site module, please refere to +;; https://docs.python.org/3/library/site.html. +;; +;; The .pth files contain the file-system paths (pointing to the store) of all +;; dependencies. So the dependency is hidden in the .pth file but is not +;; visible in the file-system. Now if packages A and B both required packages +;; P, but in different versions, Guix will not detect this when installing +;; both A and B to a profile. (For details and example see +;; https://lists.gnu.org/archive/html/guix-devel/2016-10/msg01233.html.) +;; +;; Pip behaves a bit different then easy_install: it always executes +;; "setup.py" with the option "--single-version-externally-managed" set. This +;; makes setuptools' "install" command run the original "install" command +;; instead of the "easy_install" command, so no .pth file (and no site.py) +;; will be created. The "site-packages" directory only contains the package +;; and the related .egg-info directory. +;; +;; This is exactly what we need for Guix and this is what we mimic in the +;; install phase below. +;; +;; As a draw back, the magic of the .pth file of linking to the other required +;; packages is gone and these packages have now to be declared as +;; "propagated-inputs". +;; +;; Note: Importing setuptools also adds two sub-commands: "install_egg_info" +;; and "install_scripts". These sub-commands are executed even if +;; "--single-version-externally-managed" is set, thus the .egg-info directory +;; and the scripts defined in entry-points will always be created. + +(define setuptools-shim + ;; Run setup.py with "setuptools" being imported, which will patch + ;; "distutils". This is needed for packages using "distutils" instead of + ;; "setuptools" since the former does not understand the + ;; "--single-version-externally-managed" flag. + ;; Python code taken from pip 9.0.1 pip/utils/setuptools_build.py + (string-append + "import setuptools, tokenize;__file__='setup.py';" + "f=getattr(tokenize, 'open', open)(__file__);" + "code=f.read().replace('\\r\\n', '\\n');" + "f.close();" + "exec(compile(code, __file__, 'exec'))")) -(define (call-setuppy command params) +(define (call-setuppy command params use-setuptools?) (if (file-exists? "setup.py") (begin (format #t "running \"python setup.py\" with command ~s and parameters ~s~%" command params) - (zero? (apply system* "python" "setup.py" command params))) + (if use-setuptools? + (zero? (apply system* "python" "-c" setuptools-shim + command params)) + (zero? (apply system* "python" "./setup.py" command params)))) (error "no setup.py found"))) -(define* (build #:rest empty) +(define* (build #:key use-setuptools? #:allow-other-keys) "Build a given Python package." - (call-setuppy "build" '())) + (call-setuppy "build" '() use-setuptools?)) -(define* (check #:key tests? test-target #:allow-other-keys) +(define* (check #:key tests? test-target use-setuptools? #:allow-other-keys) "Run the test suite of a given Python package." (if tests? - (call-setuppy test-target '()) + ;; Running `setup.py test` creates an additional .egg-info directory in + ;; build/lib in some cases, e.g. if the source is in a sub-directory + ;; (given with `package_dir`). This will by copied to the output, too, + ;; so we need to remove. + (let ((before (find-files "build" "\\.egg-info$" #:directories? #t))) + (call-setuppy test-target '() use-setuptools?) + (let* ((after (find-files "build" "\\.egg-info$" #:directories? #t)) + (inter (lset-difference eqv? after before))) + (for-each delete-file-recursively inter))) #t)) (define (get-python-version python) @@ -60,25 +149,36 @@ (major+minor (take components 2))) (string-join major+minor "."))) -(define* (install #:key outputs inputs (configure-flags '()) +(define (site-packages inputs outputs) + "Return the path of the current output's Python site-package." + (let* ((out (assoc-ref outputs "out")) + (python (assoc-ref inputs "python"))) + (string-append out "/lib/python" + (get-python-version python) + "/site-packages/"))) + +(define (add-installed-pythonpath inputs outputs) + "Prepend the Python site-package of OUTPUT to PYTHONPATH. This is useful +when running checks after installing the package." + (let ((old-path (getenv "PYTHONPATH")) + (add-path (site-packages inputs outputs))) + (setenv "PYTHONPATH" + (string-append add-path + (if old-path (string-append ":" old-path) ""))) + #t)) + +(define* (install #:key outputs (configure-flags '()) use-setuptools? #:allow-other-keys) "Install a given Python package." (let* ((out (assoc-ref outputs "out")) (params (append (list (string-append "--prefix=" out)) - configure-flags)) - (python-version (get-python-version (assoc-ref inputs "python"))) - (old-path (getenv "PYTHONPATH")) - (add-path (string-append out "/lib/python" python-version - "/site-packages/"))) - ;; create the module installation directory and add it to PYTHONPATH - ;; to make setuptools happy - (mkdir-p add-path) - (setenv "PYTHONPATH" - (string-append (if old-path - (string-append old-path ":") - "") - add-path)) - (call-setuppy "install" params))) + (if use-setuptools? + ;; distutils does not accept these flags + (list "--single-version-externally-managed" + "--root=/") + '()) + configure-flags))) + (call-setuppy "install" params use-setuptools?))) (define* (wrap #:key inputs outputs #:allow-other-keys) (define (list-of-files dir) @@ -112,6 +212,9 @@ (define* (rename-pth-file #:key name inputs outputs #:allow-other-keys) "Rename easy-install.pth to NAME.pth to avoid conflicts between packages installed with setuptools." + ;; Even if the "easy-install.pth" is not longer created, we kept this phase. + ;; There still may be packages creating an "easy-install.pth" manually for + ;; some good reason. (let* ((out (assoc-ref outputs "out")) (python (assoc-ref inputs "python")) (site-packages (string-append out "/lib/python" @@ -137,8 +240,7 @@ installed with setuptools." #t)) (define %standard-phases - ;; 'configure' and 'build' phases are not needed. Everything is done during - ;; 'install'. + ;; 'configure' phase is not needed. (modify-phases gnu:%standard-phases (add-after 'unpack 'ensure-no-mtimes-pre-1980 ensure-no-mtimes-pre-1980) (delete 'configure) diff --git a/guix/build/syscalls.scm b/guix/build/syscalls.scm index 9386c0f5d0..2e37846ff0 100644 --- a/guix/build/syscalls.scm +++ b/guix/build/syscalls.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2015 David Thompson <davet@gnu.org> ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org> ;;; @@ -1474,7 +1474,9 @@ always a positive integer." ;; ENOTTY is what we're after but 2012-and-earlier Linux versions ;; would return EINVAL instead in some cases: ;; <https://bugs.ruby-lang.org/issues/10494>. - (if (or (= errno ENOTTY) (= errno EINVAL)) + ;; Furthermore, some FUSE file systems like unionfs return ENOSYS for + ;; that ioctl. + (if (memv errno (list ENOTTY EINVAL ENOSYS)) (fall-back) (apply throw args)))))) diff --git a/guix/derivations.scm b/guix/derivations.scm index 7ed9bd61d3..d5e4b5730b 100644 --- a/guix/derivations.scm +++ b/guix/derivations.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -453,19 +453,22 @@ one-argument procedure similar to that returned by 'substitution-oracle'." (loop (read drv-port) (cons (ununquote exp) result)))))) -(define read-derivation - (let ((cache (make-weak-value-hash-table 200))) - (lambda (drv-port) - "Read the derivation from DRV-PORT and return the corresponding +(define %derivation-cache + ;; Maps derivation file names to <derivation> objects. + ;; XXX: This is redundant with 'atts-cache' in the store. + (make-weak-value-hash-table 200)) + +(define (read-derivation drv-port) + "Read the derivation from DRV-PORT and return the corresponding <derivation> object." - ;; Memoize that operation because `%read-derivation' is quite expensive, - ;; and because the same argument is read more than 15 times on average - ;; during something like (package-derivation s gdb). - (let ((file (and=> (port-filename drv-port) basename))) - (or (and file (hash-ref cache file)) - (let ((drv (%read-derivation drv-port))) - (hash-set! cache file drv) - drv)))))) + ;; Memoize that operation because `%read-derivation' is quite expensive, + ;; and because the same argument is read more than 15 times on average + ;; during something like (package-derivation s gdb). + (let ((file (port-filename drv-port))) + (or (and file (hash-ref %derivation-cache file)) + (let ((drv (%read-derivation drv-port))) + (hash-set! %derivation-cache file drv) + drv)))) (define-inlinable (write-sequence lst write-item port) ;; Write each element of LST with WRITE-ITEM to PORT, separating them with a @@ -520,9 +523,9 @@ that form." (define (write-input input port) (match input (($ <derivation-input> path sub-drvs) - (display "(" port) - (write path port) - (display "," port) + (display "(\"" port) + (display path port) + (display "\"," port) (write-string-list sub-drvs) (display ")" port)))) @@ -545,7 +548,7 @@ that form." (write-list inputs write-input port) (display "," port) (write-string-list sources) - (format port ",~s,~s," system builder) + (simple-format port ",\"~a\",\"~a\"," system builder) (write-string-list args) (display "," port) (write-list env-vars write-env-var port) @@ -866,10 +869,12 @@ output should not be used." system builder args env-vars #f)) (drv (add-output-paths drv-masked))) - (let ((file (add-text-to-store store (string-append name ".drv") - (derivation->string drv) - (map derivation-input-path inputs)))) - (set-file-name drv file)))) + (let* ((file (add-text-to-store store (string-append name ".drv") + (derivation->string drv) + (map derivation-input-path inputs))) + (drv (set-file-name drv file))) + (hash-set! %derivation-cache file drv) + drv))) (define* (map-derivation store drv mapping #:key (system (%current-system))) diff --git a/guix/docker.scm b/guix/docker.scm new file mode 100644 index 0000000000..dbe1e5351c --- /dev/null +++ b/guix/docker.scm @@ -0,0 +1,127 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix docker) + #:use-module (guix hash) + #:use-module (guix store) + #:use-module (guix utils) + #:use-module ((guix build utils) + #:select (delete-file-recursively + with-directory-excursion)) + #:use-module (json) + #:use-module (rnrs bytevectors) + #:use-module (ice-9 match) + #:export (build-docker-image)) + +;; Generate a 256-bit identifier in hexadecimal encoding for the Docker image +;; containing the closure at PATH. +(define docker-id + (compose bytevector->base16-string sha256 string->utf8)) + +(define (layer-diff-id layer) + "Generate a layer DiffID for the given LAYER archive." + (string-append "sha256:" (bytevector->base16-string (file-sha256 layer)))) + +;; This is the semantic version of the JSON metadata schema according to +;; https://github.com/docker/docker/blob/master/image/spec/v1.2.md +;; It is NOT the version of the image specification. +(define schema-version "1.0") + +(define (image-description id time) + "Generate a simple image description." + `((id . ,id) + (created . ,time) + (container_config . #nil))) + +(define (generate-tag path) + "Generate an image tag for the given PATH." + (match (string-split (basename path) #\-) + ((hash name . rest) (string-append name ":" hash)))) + +(define (manifest path id) + "Generate a simple image manifest." + `(((Config . "config.json") + (RepoTags . (,(generate-tag path))) + (Layers . (,(string-append id "/layer.tar")))))) + +;; According to the specifications this is required for backwards +;; compatibility. It duplicates information provided by the manifest. +(define (repositories path id) + "Generate a repositories file referencing PATH and the image ID." + `((,(generate-tag path) . ((latest . ,id))))) + +;; See https://github.com/opencontainers/image-spec/blob/master/config.md +(define (config layer time arch) + "Generate a minimal image configuration for the given LAYER file." + ;; "architecture" must be values matching "platform.arch" in the + ;; runtime-spec at + ;; https://github.com/opencontainers/runtime-spec/blob/v1.0.0-rc2/config.md#platform + `((architecture . ,arch) + (comment . "Generated by GNU Guix") + (created . ,time) + (config . #nil) + (container_config . #nil) + (os . "linux") + (rootfs . ((type . "layers") + (diff_ids . (,(layer-diff-id layer))))))) + +(define* (build-docker-image path #:key system) + "Generate a Docker image archive from the given store PATH. The image +contains the closure of the given store item." + (let ((id (docker-id path)) + (time (strftime "%FT%TZ" (localtime (current-time)))) + (name (string-append (getcwd) + "/docker-image-" (basename path) ".tar")) + (arch (match system + ("x86_64-linux" "amd64") + ("i686-linux" "386") + ("armhf-linux" "arm") + ("mips64el-linux" "mips64le")))) + (and (call-with-temporary-directory + (lambda (directory) + (with-directory-excursion directory + ;; Add symlink from /bin to /gnu/store/.../bin + (symlink (string-append path "/bin") "bin") + + (mkdir id) + (with-directory-excursion id + (with-output-to-file "VERSION" + (lambda () (display schema-version))) + (with-output-to-file "json" + (lambda () (scm->json (image-description id time)))) + + ;; Wrap it up + (let ((items (with-store store + (requisites store (list path))))) + (and (zero? (apply system* "tar" "-cf" "layer.tar" + (cons "../bin" items))) + (delete-file "../bin")))) + + (with-output-to-file "config.json" + (lambda () + (scm->json (config (string-append id "/layer.tar") + time arch)))) + (with-output-to-file "manifest.json" + (lambda () + (scm->json (manifest path id)))) + (with-output-to-file "repositories" + (lambda () + (scm->json (repositories path id))))) + (and (zero? (system* "tar" "-C" directory "-cf" name ".")) + (begin (delete-file-recursively directory) #t)))) + name))) diff --git a/guix/gexp.scm b/guix/gexp.scm index fd5dc49233..1f7fbef0a0 100644 --- a/guix/gexp.scm +++ b/guix/gexp.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -669,41 +669,34 @@ references; otherwise, return only non-native references." result) result)) (($ <gexp-input> (? gexp? exp) _ #f) - (if native? - (append (gexp-inputs exp #:native? #t) - result) - (append (gexp-inputs exp) - result))) + (append (gexp-inputs exp #:native? native?) + result)) (($ <gexp-input> (? string? str)) (if (direct-store-path? str) (cons `(,str) result) result)) - (($ <gexp-input> (? struct? thing) output) - (if (lookup-compiler thing) + (($ <gexp-input> (? struct? thing) output n?) + (if (and (eqv? n? native?) (lookup-compiler thing)) ;; THING is a derivation, or a package, or an origin, etc. (cons `(,thing ,output) result) result)) (($ <gexp-input> (lst ...) output n?) - (fold-right add-reference-inputs result - ;; XXX: For now, automatically convert LST to a list of - ;; gexp-inputs. - (map (match-lambda - ((? gexp-input? x) x) - (x (%gexp-input x "out" (or n? native?)))) - lst))) + (if (eqv? native? n?) + (fold-right add-reference-inputs result + ;; XXX: For now, automatically convert LST to a list of + ;; gexp-inputs. + (map (match-lambda + ((? gexp-input? x) x) + (x (%gexp-input x "out" (or n? native?)))) + lst)) + result)) (_ ;; Ignore references to other kinds of objects. result))) - (define (native-input? x) - (and (gexp-input? x) - (gexp-input-native? x))) - (fold-right add-reference-inputs '() - (if native? - (filter native-input? (gexp-references exp)) - (remove native-input? (gexp-references exp))))) + (gexp-references exp))) (define gexp-native-inputs (cut gexp-inputs <> #:native? #t)) @@ -819,9 +812,9 @@ environment." (cons exp result)) ((ungexp-native-splicing _ ...) (cons exp result)) - ((exp0 exp ...) + ((exp0 . exp) (let ((result (loop #'exp0 result))) - (fold loop result #'(exp ...)))) + (loop #'exp result))) (_ result)))) @@ -853,9 +846,9 @@ environment." (match (assoc exp substs) ((_ id) id) - (_ - #'(syntax-error "error: no 'ungexp' substitution" - #'ref)))) + (_ ;internal error + (with-syntax ((exp exp)) + #'(syntax-error "error: no 'ungexp' substitution" exp))))) (define (substitute-ungexp-splicing exp substs) (syntax-case exp () @@ -867,7 +860,7 @@ environment." #,(substitute-references #'(rest ...) substs)))) (_ #'(syntax-error "error: no 'ungexp-splicing' substitution" - #'ref)))))) + exp)))))) (define (substitute-references exp substs) ;; Return a variant of EXP where all the cars of SUBSTS have been @@ -882,9 +875,9 @@ environment." (substitute-ungexp-splicing exp substs)) (((ungexp-native-splicing _ ...) rest ...) (substitute-ungexp-splicing exp substs)) - ((exp0 exp ...) + ((exp0 . exp) #`(cons #,(substitute-references #'exp0 substs) - #,(substitute-references #'(exp ...) substs))) + #,(substitute-references #'exp substs))) (x #''x))) (syntax-case s (ungexp output) diff --git a/guix/git-download.scm b/guix/git-download.scm index 9cc6dd3d94..fca44f552a 100644 --- a/guix/git-download.scm +++ b/guix/git-download.scm @@ -30,7 +30,9 @@ git-reference-commit git-reference-recursive? - git-fetch)) + git-fetch + git-version + git-file-name)) ;;; Commentary: ;;; @@ -82,14 +84,26 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." (((names dirs) ...) dirs))) - (git-fetch '#$(git-reference-url ref) - '#$(git-reference-commit ref) + (git-fetch (getenv "git url") (getenv "git commit") #$output - #:recursive? '#$(git-reference-recursive? ref) + #:recursive? (call-with-input-string + (getenv "git recursive?") + read) #:git-command (string-append #+git "/bin/git"))))) (mlet %store-monad ((guile (package->derivation guile system))) (gexp->derivation (or name "git-checkout") build + + ;; Use environment variables and a fixed script name so + ;; there's only one script in store for all the + ;; downloads. + #:script-name "git-download" + #:env-vars + `(("git url" . ,(git-reference-url ref)) + ("git commit" . ,(git-reference-commit ref)) + ("git recursive?" . ,(object->string + (git-reference-recursive? ref)))) + #:system system #:local-build? #t ;don't offload repo cloning #:hash-algo hash-algo @@ -98,4 +112,12 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." #:guile-for-build guile #:local-build? #t))) +(define (git-version version revision commit) + "Return the version string for packages using git-download." + (string-append version "-" revision "." (string-take commit 7))) + +(define (git-file-name name version) + "Return the file-name for packages using git-download." + (string-append name "-" version "-checkout")) + ;;; git-download.scm ends here diff --git a/guix/gnu-maintenance.scm b/guix/gnu-maintenance.scm index 78392c9a11..789724c8c0 100644 --- a/guix/gnu-maintenance.scm +++ b/guix/gnu-maintenance.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2012, 2013 Nikita Karetnikov <nikita@karetnikov.org> ;;; ;;; This file is part of GNU Guix. @@ -60,7 +60,8 @@ %gnu-updater %gnome-updater %kde-updater - %xorg-updater)) + %xorg-updater + %kernel.org-updater)) ;;; Commentary: ;;; @@ -75,17 +76,17 @@ ;;; (define %gnumaint-base-url - "http://cvs.savannah.gnu.org/viewvc/*checkout*/gnumaint/") + "http://cvs.savannah.gnu.org/viewvc/*checkout*/womb/gnumaint/") (define %package-list-url (string->uri - (string-append %gnumaint-base-url "gnupackages.txt?root=womb"))) + (string-append %gnumaint-base-url "gnupackages.txt"))) (define %package-description-url ;; This file contains package descriptions in recutils format. ;; See <https://lists.gnu.org/archive/html/guix-devel/2013-10/msg00071.html>. (string->uri - (string-append %gnumaint-base-url "pkgblurbs.txt?root=womb"))) + (string-append %gnumaint-base-url "pkgblurbs.txt"))) (define-record-type* <gnu-package-descriptor> gnu-package-descriptor @@ -448,21 +449,26 @@ elpa.gnu.org, and all the GNOME packages." (not (gnome-package? package)) (gnu-package? package))) -(define (gnome-package? package) - "Return true if PACKAGE is a GNOME package, hosted on gnome.org." - (define gnome-uri? - (match-lambda - ((? string? uri) - (string-prefix? "mirror://gnome/" uri)) - (_ - #f))) - - (match (package-source package) - ((? origin? origin) - (match (origin-uri origin) - ((? gnome-uri?) #t) - (_ #f))) - (_ #f))) +(define (url-prefix-predicate prefix) + "Return a predicate that returns true when passed a package where one of its +source URLs starts with PREFIX." + (lambda (package) + (define matching-uri? + (match-lambda + ((? string? uri) + (string-prefix? prefix uri)) + (_ + #f))) + + (match (package-source package) + ((? origin? origin) + (match (origin-uri origin) + ((? matching-uri?) #t) + (_ #f))) + (_ #f)))) + +(define gnome-package? + (url-prefix-predicate "mirror://gnome/")) (define (latest-gnome-release package) "Return the latest release of PACKAGE, the name of a GNOME package." @@ -504,49 +510,19 @@ elpa.gnu.org, and all the GNOME packages." ;; checksums. #:file->signature (const #f)))) -(define (kde-package? package) - "Return true if PACKAGE is a KDE package, developed by KDE.org." - (define kde-uri? - (match-lambda - ((? string? uri) - (string-prefix? "mirror://kde/" uri)) - (_ - #f))) - - (match (package-source package) - ((? origin? origin) - (match (origin-uri origin) - ((? kde-uri?) #t) - (_ #f))) - (_ #f))) (define (latest-kde-release package) "Return the latest release of PACKAGE, the name of an KDE.org package." (let ((uri (string->uri (origin-uri (package-source package))))) (false-if-ftp-error (latest-ftp-release - (package-name package) + (or (assoc-ref (package-properties package) 'upstream-name) + (package-name package)) #:server "mirrors.mit.edu" #:directory (string-append "/kde" (dirname (dirname (uri-path uri)))) #:file->signature (const #f))))) -(define (xorg-package? package) - "Return true if PACKAGE is an X.org package, developed by X.org." - (define xorg-uri? - (match-lambda - ((? string? uri) - (string-prefix? "mirror://xorg/" uri)) - (_ - #f))) - - (match (package-source package) - ((? origin? origin) - (match (origin-uri origin) - ((? xorg-uri?) #t) - (_ #f))) - (_ #f))) - (define (latest-xorg-release package) "Return the latest release of PACKAGE, the name of an X.org package." (let ((uri (string->uri (origin-uri (package-source package))))) @@ -557,6 +533,22 @@ elpa.gnu.org, and all the GNOME packages." #:directory (string-append "/pub/xorg/" (dirname (uri-path uri))))))) +(define (latest-kernel.org-release package) + "Return the latest release of PACKAGE, the name of a kernel.org package." + (let ((uri (string->uri (origin-uri (package-source package))))) + (false-if-ftp-error + (latest-ftp-release + (package-name package) + #:server "ftp.free.fr" ;a mirror reachable over FTP + #:directory (string-append "/mirrors/ftp.kernel.org" + (dirname (uri-path uri))) + + ;; kernel.org provides "foo-x.y.tar.sign" files, which are signatures of + ;; the uncompressed tarball. + #:file->signature (lambda (tarball) + (string-append (file-sans-extension tarball) + ".sign")))))) + (define %gnu-updater (upstream-updater (name 'gnu) @@ -575,14 +567,21 @@ elpa.gnu.org, and all the GNOME packages." (upstream-updater (name 'kde) (description "Updater for KDE packages") - (pred kde-package?) + (pred (url-prefix-predicate "mirror://kde/")) (latest latest-kde-release))) (define %xorg-updater (upstream-updater (name 'xorg) (description "Updater for X.org packages") - (pred xorg-package?) + (pred (url-prefix-predicate "mirror://xorg/")) (latest latest-xorg-release))) +(define %kernel.org-updater + (upstream-updater + (name 'kernel.org) + (description "Updater for packages hosted on kernel.org") + (pred (url-prefix-predicate "mirror://kernel.org/")) + (latest latest-kernel.org-release))) + ;;; gnu-maintenance.scm ends here diff --git a/guix/grafts.scm b/guix/grafts.scm index dda7c1d235..2006d3908e 100644 --- a/guix/grafts.scm +++ b/guix/grafts.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -214,6 +214,17 @@ available." (delete-duplicates (concatenate refs) string=?)) result)))))) +(define-syntax-rule (with-cache key exp ...) + "Cache the value of monadic expression EXP under KEY." + (mlet %state-monad ((cache (current-state))) + (match (vhash-assq key cache) + ((_ . result) ;cache hit + (return result)) + (#f ;cache miss + (mlet %state-monad ((result (begin exp ...))) + (set-current-state (vhash-consq key result cache)) + (return result)))))) + (define* (cumulative-grafts store drv grafts references #:key @@ -252,48 +263,39 @@ derivations to the corresponding set of grafts." #:system system)) (state-return grafts)))) - (define (return/cache cache value) - (mbegin %state-monad - (set-current-state (vhash-consq drv value cache)) - (return value))) - - (mlet %state-monad ((cache (current-state))) - (match (vhash-assq drv cache) - ((_ . grafts) ;hit + (with-cache drv + (match (non-self-references references drv outputs) + (() ;no dependencies (return grafts)) - (#f ;miss - (match (non-self-references references drv outputs) - (() ;no dependencies - (return/cache cache grafts)) - (deps ;one or more dependencies - (mlet %state-monad ((grafts (mapm %state-monad dependency-grafts deps))) - (let ((grafts (delete-duplicates (concatenate grafts) equal?))) - (match (filter (lambda (graft) - (member (graft-origin-file-name graft) deps)) - grafts) - (() - (return/cache cache grafts)) - ((applicable ..1) - ;; Use APPLICABLE, the subset of GRAFTS that is really - ;; applicable to DRV, to avoid creating several identical - ;; grafted variants of DRV. - (let* ((new (graft-derivation/shallow store drv applicable - #:guile guile - #:system system)) - - ;; Replace references to any of the outputs of DRV, - ;; even if that's more than needed. This is so that - ;; the result refers only to the outputs of NEW and - ;; not to those of DRV. - (grafts (append (map (lambda (output) - (graft - (origin drv) - (origin-output output) - (replacement new) - (replacement-output output))) - (derivation-output-names drv)) - grafts))) - (return/cache cache grafts)))))))))))) + (deps ;one or more dependencies + (mlet %state-monad ((grafts (mapm %state-monad dependency-grafts deps))) + (let ((grafts (delete-duplicates (concatenate grafts) equal?))) + (match (filter (lambda (graft) + (member (graft-origin-file-name graft) deps)) + grafts) + (() + (return grafts)) + ((applicable ..1) + ;; Use APPLICABLE, the subset of GRAFTS that is really + ;; applicable to DRV, to avoid creating several identical + ;; grafted variants of DRV. + (let* ((new (graft-derivation/shallow store drv applicable + #:guile guile + #:system system)) + + ;; Replace references to any of the outputs of DRV, + ;; even if that's more than needed. This is so that + ;; the result refers only to the outputs of NEW and + ;; not to those of DRV. + (grafts (append (map (lambda (output) + (graft + (origin drv) + (origin-output output) + (replacement new) + (replacement-output output))) + (derivation-output-names drv)) + grafts))) + (return grafts)))))))))) (define* (graft-derivation store drv grafts #:key (guile (%guile-for-build)) @@ -333,4 +335,8 @@ it otherwise. It returns the previous setting." (lambda (store) (values (%graft? enable?) store))) +;; Local Variables: +;; eval: (put 'with-cache 'scheme-indent-function 1) +;; End: + ;;; grafts.scm ends here diff --git a/guix/graph.scm b/guix/graph.scm index 735d340c2c..7af2cd3b80 100644 --- a/guix/graph.scm +++ b/guix/graph.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015, 2016 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2016 Ricardo Wurmus <rekado@elephly.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -21,6 +22,7 @@ #:use-module (guix monads) #:use-module (guix records) #:use-module (guix sets) + #:use-module (rnrs io ports) #:use-module (srfi srfi-1) #:use-module (srfi srfi-9) #:use-module (srfi srfi-26) @@ -41,9 +43,13 @@ node-transitive-edges node-reachable-count + %graph-backends + %d3js-backend %graphviz-backend graph-backend? graph-backend + graph-backend-name + graph-backend-description export-graph)) @@ -140,12 +146,14 @@ typically returned by 'node-edges' or 'node-back-edges'." ;;; (define-record-type <graph-backend> - (graph-backend prologue epilogue node edge) + (graph-backend name description prologue epilogue node edge) graph-backend? - (prologue graph-backend-prologue) - (epilogue graph-backend-epilogue) - (node graph-backend-node) - (edge graph-backend-edge)) + (name graph-backend-name) + (description graph-backend-description) + (prologue graph-backend-prologue) + (epilogue graph-backend-epilogue) + (node graph-backend-node) + (edge graph-backend-edge)) (define %colors ;; See colortbl.h in Graphviz. @@ -170,9 +178,66 @@ typically returned by 'node-edges' or 'node-back-edges'." id1 id2 (pop-color id1))) (define %graphviz-backend - (graph-backend emit-prologue emit-epilogue + (graph-backend "graphviz" + "Generate graph in DOT format for use with Graphviz." + emit-prologue emit-epilogue emit-node emit-edge)) + +;;; +;;; d3js export. +;;; + +(define (emit-d3js-prologue name port) + (format port "\ +<!DOCTYPE html> +<html> + <head> + <meta charset=\"utf-8\"> + <style> +text { + font: 10px sans-serif; + pointer-events: none; +} + </style> + <script type=\"text/javascript\" src=\"~a\"></script> + </head> + <body> + <script type=\"text/javascript\"> +var nodes = {}, + nodeArray = [], + links = []; +" (search-path %load-path "d3.v3.js"))) + +(define (emit-d3js-epilogue port) + (format port "</script><script type=\"text/javascript\" src=\"~a\"></script></body></html>" + (search-path %load-path "graph.js"))) + +(define (emit-d3js-node id label port) + (format port "\ +nodes[\"~a\"] = {\"id\": \"~a\", \"label\": \"~a\", \"index\": nodeArray.length}; +nodeArray.push(nodes[\"~a\"]);~%" + id id label id)) + +(define (emit-d3js-edge id1 id2 port) + (format port "links.push({\"source\": \"~a\", \"target\": \"~a\"});~%" + id1 id2)) + +(define %d3js-backend + (graph-backend "d3js" + "Generate chord diagrams with d3js." + emit-d3js-prologue emit-d3js-epilogue + emit-d3js-node emit-d3js-edge)) + + +;;; +;;; Shared. +;;; + +(define %graph-backends + (list %graphviz-backend + %d3js-backend)) + (define* (export-graph sinks port #:key reverse-edges? node-type @@ -181,7 +246,7 @@ typically returned by 'node-edges' or 'node-back-edges'." given BACKEND. Use NODE-TYPE to traverse the DAG. When REVERSE-EDGES? is true, draw reverse arrows." (match backend - (($ <graph-backend> emit-prologue emit-epilogue emit-node emit-edge) + (($ <graph-backend> _ _ emit-prologue emit-epilogue emit-node emit-edge) (emit-prologue (node-type-name node-type) port) (match node-type diff --git a/guix/import/cpan.scm b/guix/import/cpan.scm index d244969c9e..b19d56ddcf 100644 --- a/guix/import/cpan.scm +++ b/guix/import/cpan.scm @@ -24,18 +24,23 @@ #:use-module ((ice-9 popen) #:select (open-pipe* close-pipe)) #:use-module ((ice-9 rdelim) #:select (read-line)) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-26) #:use-module (json) #:use-module (guix hash) #:use-module (guix store) #:use-module (guix utils) #:use-module (guix base32) - #:use-module ((guix download) #:select (download-to-store)) - #:use-module (guix import utils) + #:use-module (guix ui) + #:use-module ((guix download) #:select (download-to-store url-fetch)) + #:use-module ((guix import utils) #:select (factorize-uri + flatten assoc-ref*)) #:use-module (guix import json) #:use-module (guix packages) + #:use-module (guix upstream) #:use-module (guix derivations) #:use-module (gnu packages perl) - #:export (cpan->guix-package)) + #:export (cpan->guix-package + %cpan-updater)) ;;; Commentary: ;;; @@ -84,28 +89,49 @@ module is distributed with 'Test::Simple', so (module->dist-name \"ok\") would return \"Test-Simple\"" (assoc-ref (json-fetch (string-append "https://api.metacpan.org/module/" - module)) + module + "?fields=distribution")) "distribution")) -(define (cpan-fetch module) +(define (package->upstream-name package) + "Return the CPAN name of PACKAGE." + (let* ((properties (package-properties package)) + (upstream-name (and=> properties + (cut assoc-ref <> 'upstream-name)))) + (or upstream-name + (match (package-source package) + ((? origin? origin) + (match (origin-uri origin) + ((or (? string? url) (url _ ...)) + (match (string-match (string-append "([^/]*)-v?[0-9\\.]+") url) + (#f #f) + (m (match:substring m 1)))) + (_ #f))) + (_ #f))))) + +(define (cpan-fetch name) "Return an alist representation of the CPAN metadata for the perl module MODULE, or #f on failure. MODULE should be e.g. \"Test::Script\"" ;; This API always returns the latest release of the module. - (json-fetch (string-append "https://api.metacpan.org/release/" - ;; XXX: The 'release' api requires the "release" - ;; name of the package. This substitution seems - ;; reasonably consistent across packages. - (module->name module)))) + (json-fetch (string-append "https://api.metacpan.org/release/" name))) (define (cpan-home name) (string-append "http://search.cpan.org/dist/" name)) -(define (fix-source-url download-url) - "Return a new download URL based on DOWNLOAD-URL which now uses our mirrors, -if the original's domain was metacpan." - (regexp-substitute/global #f "http[s]?://cpan.metacpan.org" download-url +(define (cpan-source-url meta) + "Return the download URL for a module's source tarball." + (regexp-substitute/global #f "http[s]?://cpan.metacpan.org" + (assoc-ref meta "download_url") 'pre "mirror://cpan" 'post)) +(define (cpan-version meta) + "Return the version number from META." + (match (assoc-ref meta "version") + ((? number? version) + ;; version is sometimes not quoted in the module json, so it gets + ;; imported into Guile as a number, so convert it to a string. + (number->string version)) + (version version))) (define %corelist (delay @@ -116,6 +142,31 @@ if the original's domain was metacpan." (and (access? core X_OK) core)))) +(define core-module? + (let ((perl-version (package-version perl)) + (rx (make-regexp + (string-append "released with perl v?([0-9\\.]*)" + "(.*and removed from v?([0-9\\.]*))?")))) + (lambda (name) + (define (version-between? lower version upper) + (and (version>=? version lower) + (or (not upper) + (version>? upper version)))) + (and (force %corelist) + (parameterize ((current-error-port (%make-void-port "w"))) + (let* ((corelist (open-pipe* OPEN_READ (force %corelist) name))) + (let loop () + (let ((line (read-line corelist))) + (if (eof-object? line) + (begin (close-pipe corelist) #f) + (or (and=> (regexp-exec rx line) + (lambda (m) + (let ((first (match:substring m 1)) + (last (match:substring m 3))) + (version-between? + first perl-version last)))) + (loop))))))))))) + (define (cpan-module->sexp meta) "Return the `package' s-expression for a CPAN module from the metadata in META." @@ -127,35 +178,8 @@ META." (string-downcase name) (string-append "perl-" (string-downcase name)))) - (define version - (match (assoc-ref meta "version") - ((? number? vrs) (number->string vrs)) - ((? string? vrs) vrs))) - - (define core-module? - (let ((perl-version (package-version perl)) - (rx (make-regexp - (string-append "released with perl v?([0-9\\.]*)" - "(.*and removed from v?([0-9\\.]*))?")))) - (lambda (name) - (define (version-between? lower version upper) - (and (version>=? version lower) - (or (not upper) - (version>? upper version)))) - (and (force %corelist) - (parameterize ((current-error-port (%make-void-port "w"))) - (let* ((corelist (open-pipe* OPEN_READ (force %corelist) name))) - (let loop () - (let ((line (read-line corelist))) - (if (eof-object? line) - (begin (close-pipe corelist) #f) - (or (and=> (regexp-exec rx line) - (lambda (m) - (let ((first (match:substring m 1)) - (last (match:substring m 3))) - (version-between? - first perl-version last)))) - (loop))))))))))) + (define version (cpan-version meta)) + (define source-url (cpan-source-url meta)) (define (convert-inputs phases) ;; Convert phase dependencies into a list of name/variable pairs. @@ -193,8 +217,6 @@ META." (list (list guix-name (list 'quasiquote inputs)))))) - (define source-url (fix-source-url (assoc-ref meta "download_url"))) - (let ((tarball (with-store store (download-to-store store source-url)))) `(package @@ -224,5 +246,61 @@ META." (define (cpan->guix-package module-name) "Fetch the metadata for PACKAGE-NAME from metacpan.org, and return the `package' s-expression corresponding to that package, or #f on failure." - (let ((module-meta (cpan-fetch module-name))) + (let ((module-meta (cpan-fetch (module->name module-name)))) (and=> module-meta cpan-module->sexp))) + +(define (cpan-package? package) + "Return #t if PACKAGE is a package from CPAN." + (define cpan-url? + (let ((cpan-rx (make-regexp (string-append "(" + "mirror://cpan" "|" + "https?://www.cpan.org" "|" + "https?://cpan.metacpan.org" + ")")))) + (lambda (url) + (regexp-exec cpan-rx url)))) + + (let ((source-url (and=> (package-source package) origin-uri)) + (fetch-method (and=> (package-source package) origin-method))) + (and (eq? fetch-method url-fetch) + (match source-url + ((? string?) + (cpan-url? source-url)) + ((source-url ...) + (any cpan-url? source-url)))))) + +(define (latest-release package) + "Return an <upstream-source> for the latest release of PACKAGE." + (match (cpan-fetch (package->upstream-name package)) + (#f #f) + (meta + (let ((core-inputs + (match (package-direct-inputs package) + (((_ inputs _ ...) ...) + (filter-map (match-lambda + ((and (? package?) + (? cpan-package?) + (= package->upstream-name + (? core-module? name))) + name) + (else #f)) + inputs))))) + ;; Warn about inputs that are part of perl's core + (unless (null? core-inputs) + (for-each (lambda (module) + (warning (_ "input '~a' of ~a is in Perl core~%") + module (package-name package))) + core-inputs))) + (let ((version (cpan-version meta)) + (url (cpan-source-url meta))) + (upstream-source + (package (package-name package)) + (version version) + (urls url)))))) + +(define %cpan-updater + (upstream-updater + (name 'cpan) + (description "Updater for CPAN packages") + (pred cpan-package?) + (latest latest-release))) diff --git a/guix/import/cran.scm b/guix/import/cran.scm index 3fb2e213b0..463a25514e 100644 --- a/guix/import/cran.scm +++ b/guix/import/cran.scm @@ -23,6 +23,11 @@ #:use-module ((ice-9 rdelim) #:select (read-string)) #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) + #:use-module (srfi srfi-34) + #:use-module (srfi srfi-41) + #:use-module (ice-9 receive) + #:use-module (web uri) + #:use-module (guix combinators) #:use-module (guix http-client) #:use-module (guix hash) #:use-module (guix store) @@ -32,8 +37,10 @@ #:use-module ((guix build-system r) #:select (cran-uri bioconductor-uri)) #:use-module (guix upstream) #:use-module (guix packages) + #:use-module (gnu packages) #:export (cran->guix-package bioconductor->guix-package + recursive-import %cran-updater %bioconductor-updater)) @@ -51,19 +58,21 @@ ("Artistic-2.0" 'artistic2.0) ("Apache License 2.0" 'asl2.0) ("BSD_2_clause" 'bsd-2) + ("BSD_2_clause + file LICENSE" 'bsd-2) ("BSD_3_clause" 'bsd-3) + ("BSD_3_clause + file LICENSE" 'bsd-3) ("GPL" (list 'gpl2+ 'gpl3+)) ("GPL (>= 2)" 'gpl2+) ("GPL (>= 3)" 'gpl3+) - ("GPL-2" 'gpl2+) - ("GPL-3" 'gpl3+) - ("LGPL-2" 'lgpl2.0+) - ("LGPL-2.1" 'lgpl2.1+) - ("LGPL-3" 'lgpl3+) + ("GPL-2" 'gpl2) + ("GPL-3" 'gpl3) + ("LGPL-2" 'lgpl2.0) + ("LGPL-2.1" 'lgpl2.1) + ("LGPL-3" 'lgpl3) ("LGPL (>= 2)" 'lgpl2.0+) ("LGPL (>= 3)" 'lgpl3+) - ("MIT" 'x11) - ("MIT + file LICENSE" 'x11) + ("MIT" 'expat) + ("MIT + file LICENSE" 'expat) ((x) (string->license x)) ((lst ...) `(list ,@(map string->license lst))) (_ #f))) @@ -121,10 +130,18 @@ package definition." (define (fetch-description base-url name) "Return an alist of the contents of the DESCRIPTION file for the R package -NAME, or #f on failure. NAME is case-sensitive." +NAME, or #f in case of failure. NAME is case-sensitive." ;; This API always returns the latest release of the module. (let ((url (string-append base-url name "/DESCRIPTION"))) - (description->alist (read-string (http-fetch url))))) + (guard (c ((http-get-error? c) + (format (current-error-port) + "error: failed to retrieve package information \ +from ~s: ~a (~s)~%" + (uri->string (http-get-error-uri c)) + (http-get-error-code c) + (http-get-error-reason c)) + #f)) + (description->alist (read-string (http-fetch url)))))) (define (listify meta field) "Look up FIELD in the alist META. If FIELD contains a comma-separated @@ -146,14 +163,49 @@ empty list when the FIELD cannot be found." (string-any char-set:whitespace item))) (map string-trim-both items)))))) +(define default-r-packages + (list "KernSmooth" + "MASS" + "Matrix" + "base" + "boot" + "class" + "cluster" + "codetools" + "compiler" + "datasets" + "foreign" + "grDevices" + "graphics" + "grid" + "lattice" + "methods" + "mgcv" + "nlme" + "nnet" + "parallel" + "rpart" + "spatial" + "splines" + "stats" + "stats4" + "survival" + "tcltk" + "tools" + "translations" + "utils")) + +(define (guix-name name) + "Return a Guix package name for a given R package name." + (string-append "r-" (string-map (match-lambda + (#\_ #\-) + (#\. #\-) + (chr (char-downcase chr))) + name))) + (define (description->package repository meta) "Return the `package' s-expression for an R package published on REPOSITORY from the alist META, which was derived from the R package's DESCRIPTION file." - (define (guix-name name) - (if (string-prefix? "r-" name) - (string-downcase name) - (string-append "r-" (string-downcase name)))) - (let* ((base-url (case repository ((cran) %cran-url) ((bioconductor) %bioconductor-url))) @@ -174,42 +226,107 @@ from the alist META, which was derived from the R package's DESCRIPTION file." (_ #f))) (tarball (with-store store (download-to-store store source-url))) (sysdepends (map string-downcase (listify meta "SystemRequirements"))) - (propagate (map guix-name (lset-union equal? - (listify meta "Imports") - (listify meta "LinkingTo") - (delete "R" - (listify meta "Depends")))))) - `(package - (name ,(guix-name name)) - (version ,version) - (source (origin - (method url-fetch) - (uri (,(procedure-name uri-helper) ,name version)) - (sha256 - (base32 - ,(bytevector->nix-base32-string (file-sha256 tarball)))))) - ,@(if (not (equal? (string-append "r-" name) - (guix-name name))) - `((properties ,`(,'quasiquote ((,'upstream-name . ,name))))) - '()) - (build-system r-build-system) - ,@(maybe-inputs sysdepends) - ,@(maybe-inputs propagate 'propagated-inputs) - (home-page ,(if (string-null? home-page) - (string-append base-url name) - home-page)) - (synopsis ,synopsis) - (description ,(beautify-description (assoc-ref meta "Description"))) - (license ,license)))) - -(define* (cran->guix-package package-name #:optional (repo 'cran)) - "Fetch the metadata for PACKAGE-NAME from REPO and return the `package' + (propagate (filter (lambda (name) + (not (member name default-r-packages))) + (lset-union equal? + (listify meta "Imports") + (listify meta "LinkingTo") + (delete "R" + (listify meta "Depends")))))) + (values + `(package + (name ,(guix-name name)) + (version ,version) + (source (origin + (method url-fetch) + (uri (,(procedure-name uri-helper) ,name version)) + (sha256 + (base32 + ,(bytevector->nix-base32-string (file-sha256 tarball)))))) + ,@(if (not (equal? (string-append "r-" name) + (guix-name name))) + `((properties ,`(,'quasiquote ((,'upstream-name . ,name))))) + '()) + (build-system r-build-system) + ,@(maybe-inputs sysdepends) + ,@(maybe-inputs (map guix-name propagate) 'propagated-inputs) + (home-page ,(if (string-null? home-page) + (string-append base-url name) + home-page)) + (synopsis ,synopsis) + (description ,(beautify-description (or (assoc-ref meta "Description") + ""))) + (license ,license)) + propagate))) + +(define cran->guix-package + (memoize + (lambda* (package-name #:optional (repo 'cran)) + "Fetch the metadata for PACKAGE-NAME from REPO and return the `package' s-expression corresponding to that package, or #f on failure." - (let* ((url (case repo - ((cran) %cran-url) - ((bioconductor) %bioconductor-svn-url))) - (module-meta (fetch-description url package-name))) - (and=> module-meta (cut description->package repo <>)))) + (let* ((url (case repo + ((cran) %cran-url) + ((bioconductor) %bioconductor-svn-url))) + (module-meta (fetch-description url package-name))) + (and=> module-meta (cut description->package repo <>)))))) + +(define* (recursive-import package-name #:optional (repo 'cran)) + "Generate a stream of package expressions for PACKAGE-NAME and all its +dependencies." + (receive (package . dependencies) + (cran->guix-package package-name repo) + (if (not package) + stream-null + + ;; Generate a lazy stream of package expressions for all unknown + ;; dependencies in the graph. + (let* ((make-state (lambda (queue done) + (cons queue done))) + (next (match-lambda + (((next . rest) . done) next))) + (imported (match-lambda + ((queue . done) done))) + (done? (match-lambda + ((queue . done) + (zero? (length queue))))) + (unknown? (lambda* (dependency #:optional (done '())) + (and (not (member dependency + done)) + (null? (find-packages-by-name + (guix-name dependency)))))) + (update (lambda (state new-queue) + (match state + (((head . tail) . done) + (make-state (lset-difference + equal? + (lset-union equal? new-queue tail) + done) + (cons head done))))))) + (stream-cons + package + (stream-unfold + ;; map: produce a stream element + (lambda (state) + (cran->guix-package (next state) repo)) + + ;; predicate + (compose not done?) + + ;; generator: update the queue + (lambda (state) + (receive (package . dependencies) + (cran->guix-package (next state) repo) + (if package + (update state (filter (cut unknown? <> + (cons (next state) + (imported state))) + (car dependencies))) + ;; TODO: Try the other archives before giving up + (update state (imported state))))) + + ;; initial state + (make-state (filter unknown? (car dependencies)) + (list package-name)))))))) ;;; diff --git a/guix/import/crate.scm b/guix/import/crate.scm new file mode 100644 index 0000000000..233a20e983 --- /dev/null +++ b/guix/import/crate.scm @@ -0,0 +1,165 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2016 David Craven <david@craven.ch> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix import crate) + #:use-module (guix base32) + #:use-module (guix build-system cargo) + #:use-module ((guix download) #:prefix download:) + #:use-module (guix hash) + #:use-module (guix http-client) + #:use-module (guix import json) + #:use-module (guix import utils) + #:use-module ((guix licenses) #:prefix license:) + #:use-module (guix monads) + #:use-module (guix packages) + #:use-module (guix upstream) + #:use-module (guix utils) + #:use-module (ice-9 match) + #:use-module (ice-9 pretty-print) ; recursive + #:use-module (json) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-2) + #:use-module (srfi srfi-26) + #:export (crate->guix-package + guix-package->crate-name + %crate-updater)) + +(define (crate-fetch crate-name callback) + "Fetch the metadata for CRATE-NAME from crates.io and call the callback." + + (define (crates->inputs crates) + (sort (map (cut assoc-ref <> "crate_id") crates) string-ci<?)) + + (define (string->license string) + (map spdx-string->license (string-split string #\/))) + + (define (crate-kind-predicate kind) + (lambda (dep) (string=? (assoc-ref dep "kind") kind))) + + (and-let* ((crate-json (json-fetch (string-append crate-url crate-name))) + (crate (assoc-ref crate-json "crate")) + (name (assoc-ref crate "name")) + (version (assoc-ref crate "max_version")) + (homepage (assoc-ref crate "homepage")) + (repository (assoc-ref crate "repository")) + (synopsis (assoc-ref crate "description")) + (description (assoc-ref crate "description")) + (license (string->license (assoc-ref crate "license"))) + (path (string-append "/" version "/dependencies")) + (deps-json (json-fetch (string-append crate-url name path))) + (deps (assoc-ref deps-json "dependencies")) + (input-crates (filter (crate-kind-predicate "normal") deps)) + (native-input-crates + (filter (lambda (dep) + (not ((crate-kind-predicate "normal") dep))) deps)) + (inputs (crates->inputs input-crates)) + (native-inputs (crates->inputs native-input-crates)) + (home-page (match homepage + (() repository) + (_ homepage)))) + (callback #:name name #:version version + #:inputs inputs #:native-inputs native-inputs + #:home-page home-page #:synopsis synopsis + #:description description #:license license))) + +(define* (make-crate-sexp #:key name version inputs native-inputs + home-page synopsis description license + #:allow-other-keys) + "Return the `package' s-expression for a rust package with the given NAME, +VERSION, INPUTS, NATIVE-INPUTS, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." + (let* ((port (http-fetch (crate-uri name version))) + (guix-name (crate-name->package-name name)) + (inputs (map crate-name->package-name inputs)) + (native-inputs (map crate-name->package-name native-inputs)) + (pkg `(package + (name ,guix-name) + (version ,version) + (source (origin + (method url-fetch) + (uri (crate-uri ,name version)) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + ,(bytevector->nix-base32-string (port-sha256 port)))))) + (build-system cargo-build-system) + ,@(maybe-native-inputs native-inputs "src") + ,@(maybe-inputs inputs "src") + (home-page ,(match home-page + (() "") + (_ home-page))) + (synopsis ,synopsis) + (description ,(beautify-description description)) + (license ,(match license + (() #f) + ((license) license) + (_ `(list ,@license))))))) + (close-port port) + pkg)) + +(define (crate->guix-package crate-name) + "Fetch the metadata for CRATE-NAME from crates.io, and return the +`package' s-expression corresponding to that package, or #f on failure." + (crate-fetch crate-name make-crate-sexp)) + +(define (guix-package->crate-name package) + "Return the crate name of PACKAGE." + (and-let* ((origin (package-source package)) + (uri (origin-uri origin)) + (crate-url? uri) + (len (string-length crate-url)) + (path (xsubstring uri len)) + (parts (string-split path #\/))) + (match parts + ((name _ ...) name)))) + +(define (crate-name->package-name name) + (string-append "rust-" (string-join (string-split name #\_) "-"))) + +;;; +;;; Updater +;;; + +(define (crate-package? package) + "Return true if PACKAGE is a Rust crate from crates.io." + (let ((source-url (and=> (package-source package) origin-uri)) + (fetch-method (and=> (package-source package) origin-method))) + (and (eq? fetch-method download:url-fetch) + (match source-url + ((? string?) + (crate-url? source-url)) + ((source-url ...) + (any crate-url? source-url)))))) + +(define (latest-release package) + "Return an <upstream-source> for the latest release of PACKAGE." + (let* ((crate-name (guix-package->crate-name package)) + (callback (lambda* (#:key version #:allow-other-keys) version)) + (version (crate-fetch crate-name callback)) + (url (crate-uri crate-name version))) + (upstream-source + (package (package-name package)) + (version version) + (urls (list url))))) + +(define %crate-updater + (upstream-updater + (name 'crates) + (description "Updater for crates.io packages") + (pred crate-package?) + (latest latest-release))) + diff --git a/guix/import/elpa.scm b/guix/import/elpa.scm index 320a09e8c6..96cf5bbae6 100644 --- a/guix/import/elpa.scm +++ b/guix/import/elpa.scm @@ -89,7 +89,13 @@ NAMES (strings)." "Fetch URL, store the content in a temporary file and call PROC with that file. Returns the value returned by PROC. On error call ERROR-THUNK and return its value or leave if it's false." - (proc (http-fetch/cached (string->uri url)))) + (catch #t + (lambda () + (proc (http-fetch/cached (string->uri url)))) + (lambda (key . args) + (if error-thunk + (error-thunk) + (leave (_ "~A: download failed~%") url))))) (define (is-elpa-package? name elpa-pkg-spec) "Return true if the string NAME corresponds to the name of the package @@ -222,7 +228,7 @@ type '<elpa-package>'." (bytevector->nix-base32-string (file-sha256 tarball)) "failed to download package"))))) (build-system emacs-build-system) - ,@(maybe-inputs 'inputs dependencies) + ,@(maybe-inputs 'propagated-inputs dependencies) (home-page ,(elpa-package-home-page pkg)) (synopsis ,(elpa-package-synopsis pkg)) (description ,(elpa-package-description pkg)) diff --git a/guix/import/gem.scm b/guix/import/gem.scm index 3d0c190656..3ad7facc7f 100644 --- a/guix/import/gem.scm +++ b/guix/import/gem.scm @@ -38,14 +38,8 @@ (define (rubygems-fetch name) "Return an alist representation of the RubyGems metadata for the package NAME, or #f on failure." - ;; XXX: We want to silence the download progress report, which is especially - ;; annoying for 'guix refresh', but we have to use a file port. - (call-with-output-file "/dev/null" - (lambda (null) - (with-error-to-port null - (lambda () - (json-fetch - (string-append "https://rubygems.org/api/v1/gems/" name ".json"))))))) + (json-fetch + (string-append "https://rubygems.org/api/v1/gems/" name ".json"))) (define (ruby-package-name name) "Given the NAME of a package on RubyGems, return a Guix-compliant name for diff --git a/guix/import/github.scm b/guix/import/github.scm index 0843ddeefd..01452b12e3 100644 --- a/guix/import/github.scm +++ b/guix/import/github.scm @@ -23,23 +23,12 @@ #:use-module (guix utils) #:use-module ((guix download) #:prefix download:) #:use-module (guix import utils) + #:use-module (guix import json) #:use-module (guix packages) #:use-module (guix upstream) #:use-module (web uri) #:export (%github-updater)) -(define (json-fetch* url) - "Return a list/hash representation of the JSON resource URL, or #f on -failure." - (call-with-output-file "/dev/null" - (lambda (null) - (with-error-to-port null - (lambda () - (call-with-temporary-output-file - (lambda (temp port) - (and (url-fetch url temp) - (call-with-input-file temp json->scm))))))))) - (define (find-extension url) "Return the extension of the archive e.g. '.tar.gz' given a URL, or false if none is recognized" @@ -136,7 +125,7 @@ the package e.g. 'bedtools2'. Return #f if there is no releases" "https://api.github.com/repos/" (github-user-slash-repository url) "/releases")) - (json (json-fetch* + (json (json-fetch (if token (string-append api-url "?access_token=" token) api-url)))) diff --git a/guix/import/json.scm b/guix/import/json.scm index c3092a5a9d..5940f5e48f 100644 --- a/guix/import/json.scm +++ b/guix/import/json.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 David Thompson <davet@gnu.org> -;;; Copyright © 2015 Eric Bavier <bavier@member.fsf.org> +;;; Copyright © 2015, 2016 Eric Bavier <bavier@member.fsf.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -19,14 +19,17 @@ (define-module (guix import json) #:use-module (json) - #:use-module (guix utils) + #:use-module (guix http-client) #:use-module (guix import utils) + #:use-module (srfi srfi-34) #:export (json-fetch)) (define (json-fetch url) "Return an alist representation of the JSON resource URL, or #f on failure." - (call-with-temporary-output-file - (lambda (temp port) - (and (url-fetch url temp) - (hash-table->alist - (call-with-input-file temp json->scm)))))) + (guard (c ((and (http-get-error? c) + (= 404 (http-get-error-code c))) + #f)) ;"expected" if package is unknown + (let* ((port (http-fetch url)) + (result (hash-table->alist (json->scm port)))) + (close-port port) + result))) diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 68153d5ab1..7cce0fc594 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -51,14 +51,8 @@ (define (pypi-fetch name) "Return an alist representation of the PyPI metadata for the package NAME, or #f on failure." - ;; XXX: We want to silence the download progress report, which is especially - ;; annoying for 'guix refresh', but we have to use a file port. - (call-with-output-file "/dev/null" - (lambda (null) - (with-error-to-port null - (lambda () - (json-fetch (string-append "https://pypi.python.org/pypi/" - name "/json"))))))) + (json-fetch (string-append "https://pypi.python.org/pypi/" + name "/json"))) ;; For packages found on PyPI that lack a source distribution. (define-condition-type &missing-source-error &error @@ -309,7 +303,8 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." "Return true if PACKAGE is a Python package from PyPI." (define (pypi-url? url) - (string-prefix? "https://pypi.python.org/" url)) + (or (string-prefix? "https://pypi.python.org/" url) + (string-prefix? "https://pypi.io/packages" url))) (let ((source-url (and=> (package-source package) origin-uri)) (fetch-method (and=> (package-source package) origin-method))) diff --git a/guix/import/utils.scm b/guix/import/utils.scm index 057c2d9c7d..be1980d08f 100644 --- a/guix/import/utils.scm +++ b/guix/import/utils.scm @@ -22,6 +22,7 @@ #:use-module (guix base32) #:use-module ((guix build download) #:prefix build:) #:use-module (guix hash) + #:use-module (guix http-client) #:use-module ((guix licenses) #:prefix license:) #:use-module (guix utils) #:use-module (ice-9 match) @@ -36,6 +37,10 @@ url-fetch guix-hash-url + maybe-inputs + maybe-native-inputs + package->definition + spdx-string->license license->symbol @@ -205,3 +210,34 @@ into a proper sentence and by using two spaces between sentences." ;; Use double spacing between sentences (regexp-substitute/global #f "\\. \\b" cleaned 'pre ". " 'post))) + +(define* (package-names->package-inputs names #:optional (output #f)) + (map (lambda (input) + (cons* input (list 'unquote (string->symbol input)) + (or (and output (list output)) + '()))) + names)) + +(define* (maybe-inputs package-names #:optional (output #f)) + "Given a list of PACKAGE-NAMES, tries to generate the 'inputs' field of a +package definition." + (match (package-names->package-inputs package-names output) + (() + '()) + ((package-inputs ...) + `((inputs (,'quasiquote ,package-inputs)))))) + +(define* (maybe-native-inputs package-names #:optional (output #f)) + "Given a list of PACKAGE-NAMES, tries to generate the 'inputs' field of a +package definition." + (match (package-names->package-inputs package-names output) + (() + '()) + ((package-inputs ...) + `((native-inputs (,'quasiquote ,package-inputs)))))) + +(define (package->definition guix-package) + (match guix-package + (('package ('name (? string? name)) _ ...) + `(define-public ,(string->symbol name) + ,guix-package)))) diff --git a/guix/profiles.scm b/guix/profiles.scm index 0b317ef51e..e7707b6543 100644 --- a/guix/profiles.scm +++ b/guix/profiles.scm @@ -919,10 +919,14 @@ files for the truetype fonts of the @var{manifest} entries." (define* (profile-derivation manifest #:key (hooks %default-profile-hooks) + (locales? #t) system) "Return a derivation that builds a profile (aka. 'user environment') with the given MANIFEST. The profile includes additional derivations returned by -the monadic procedures listed in HOOKS--such as an Info 'dir' file, etc." +the monadic procedures listed in HOOKS--such as an Info 'dir' file, etc. + +When LOCALES? is true, the build is performed under a UTF-8 locale; this adds +a dependency on the 'glibc-utf8-locales' package." (mlet %store-monad ((system (if system (return system) (current-system))) @@ -939,6 +943,19 @@ the monadic procedures listed in HOOKS--such as an Info 'dir' file, etc." extras) (manifest-inputs manifest))) + (define glibc-utf8-locales ;lazy reference + (module-ref (resolve-interface '(gnu packages base)) + 'glibc-utf8-locales)) + + (define set-utf8-locale + ;; Some file names (e.g., in 'nss-certs') are UTF-8 encoded so + ;; install a UTF-8 locale. + #~(begin + (setenv "LOCPATH" + #$(file-append glibc-utf8-locales "/lib/locale/" + (package-version glibc-utf8-locales))) + (setlocale LC_ALL "en_US.utf8"))) + (define builder (with-imported-modules '((guix build profiles) (guix build union) @@ -953,6 +970,8 @@ the monadic procedures listed in HOOKS--such as an Info 'dir' file, etc." (setvbuf (current-output-port) _IOLBF) (setvbuf (current-error-port) _IOLBF) + #+(if locales? set-utf8-locale #t) + (define search-paths ;; Search paths of MANIFEST's packages, converted back to their ;; record form. @@ -1099,7 +1118,8 @@ case when generations have been deleted (there are \"holes\")." "Link GENERATION, a string, to the empty profile. An error is raised if that fails." (let* ((drv (run-with-store store - (profile-derivation (manifest '())))) + (profile-derivation (manifest '()) + #:locales? #f))) (prof (derivation->output-path drv "out"))) (build-derivations store (list drv)) (switch-symlinks generation prof))) diff --git a/guix/scripts/archive.scm b/guix/scripts/archive.scm index 400353247c..6eba9e0008 100644 --- a/guix/scripts/archive.scm +++ b/guix/scripts/archive.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2013, 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -30,6 +31,7 @@ #:use-module (guix ui) #:use-module (guix pki) #:use-module (guix pk-crypto) + #:use-module (guix docker) #:use-module (guix scripts) #:use-module (guix scripts build) #:use-module (gnu packages) @@ -41,7 +43,8 @@ #:use-module (srfi srfi-26) #:use-module (srfi srfi-37) #:use-module (ice-9 binary-ports) - #:export (guix-archive)) + #:export (guix-archive + options->derivations+files)) ;;; @@ -62,6 +65,8 @@ Export/import one or more packages from/to the store.\n")) (display (_ " --export export the specified files/packages to stdout")) (display (_ " + --format=FMT export files/packages in the specified format FMT")) + (display (_ " -r, --recursive combined with '--export', include dependencies")) (display (_ " --import import from the archive passed on stdin")) @@ -116,6 +121,9 @@ Export/import one or more packages from/to the store.\n")) (option '("export") #f #f (lambda (opt name arg result) (alist-cons 'export #t result))) + (option '(#\f "format") #t #f + (lambda (opt name arg result . rest) + (alist-cons 'format arg result))) (option '(#\r "recursive") #f #f (lambda (opt name arg result) (alist-cons 'export-recursive? #t result))) @@ -330,7 +338,15 @@ the input port." (else (with-store store (cond ((assoc-ref opts 'export) - (export-from-store store opts)) + (cond ((equal? (assoc-ref opts 'format) "docker") + (match (car opts) + (('argument . (? store-path? item)) + (format #t "~a\n" + (build-docker-image + item + #:system (assoc-ref opts 'system)))) + (_ (leave (_ "argument must be a direct store path~%"))))) + (_ (export-from-store store opts)))) ((assoc-ref opts 'import) (import-paths store (current-input-port))) ((assoc-ref opts 'missing) diff --git a/guix/scripts/build.scm b/guix/scripts/build.scm index 8c2c4902fc..ccb4c275fc 100644 --- a/guix/scripts/build.scm +++ b/guix/scripts/build.scm @@ -151,7 +151,11 @@ the new package's version number from URI." ;; Use #:recursive? #t to allow for directories. (source (download-to-store store uri - #:recursive? #t)))))) + #:recursive? #t)) + + ;; Override the replacement, otherwise '--with-source' would + ;; have no effect. + (replacement #f))))) ;;; diff --git a/guix/scripts/copy.scm b/guix/scripts/copy.scm new file mode 100644 index 0000000000..9ae204e6c6 --- /dev/null +++ b/guix/scripts/copy.scm @@ -0,0 +1,207 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2016 Ludovic Courtès <ludo@gnu.org> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix scripts copy) + #:use-module (guix ui) + #:use-module (guix scripts) + #:use-module (guix ssh) + #:use-module (guix store) + #:use-module (guix utils) + #:use-module (guix derivations) + #:use-module (guix scripts build) + #:use-module ((guix scripts archive) #:select (options->derivations+files)) + #:use-module (ssh session) + #:use-module (ssh auth) + #:use-module (ssh key) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-37) + #:use-module (ice-9 match) + #:use-module (ice-9 format) + #:export (guix-copy)) + + +;;; +;;; Exchanging store items over SSH. +;;; + +(define %compression + "zlib@openssh.com,zlib") + +(define* (open-ssh-session host #:key user port) + "Open an SSH session for HOST and return it. When USER and PORT are #f, use +default values or whatever '~/.ssh/config' specifies; otherwise use them. +Throw an error on failure." + (let ((session (make-session #:user user + #:host host + #:port port + #:timeout 10 ;seconds + ;; #:log-verbosity 'protocol + + ;; We need lightweight compression when + ;; exchanging full archives. + #:compression %compression + #:compression-level 3))) + + ;; Honor ~/.ssh/config. + (session-parse-config! session) + + (match (connect! session) + ('ok + ;; Let the SSH agent authenticate us to the server. + (match (userauth-agent! session) + ('success + session) + (x + (disconnect! session) + (leave (_ "SSH authentication failed for '~a': ~a~%") + host (get-error session))))) + (x + ;; Connection failed or timeout expired. + (leave (_ "SSH connection to '~a' failed: ~a~%") + host (get-error session)))))) + +(define (ssh-spec->user+host+port spec) + "Parse SPEC, a string like \"user@host:port\" or just \"host\", and return +three values: the user name (or #f), the host name, and the TCP port +number (or #f) corresponding to SPEC." + (define tokens + (char-set #\@ #\:)) + + (match (string-tokenize spec (char-set-complement tokens)) + ((host) + (values #f host #f)) + ((left right) + (if (string-index spec #\@) + (values left right #f) + (values #f left (string->number right)))) + ((user host port) + (match (string->number port) + ((? integer? port) + (values user host port)) + (x + (leave (_ "~a: invalid TCP port number~%") port)))) + (x + (leave (_ "~a: invalid SSH specification~%") spec)))) + +(define (send-to-remote-host target opts) + "Send ITEMS to TARGET. ITEMS is a list of store items or package names; for ; +package names, build the underlying packages before sending them." + (with-store local + (set-build-options-from-command-line local opts) + (let-values (((user host port) + (ssh-spec->user+host+port target)) + ((drv items) + (options->derivations+files local opts))) + (show-what-to-build local drv + #:use-substitutes? (assoc-ref opts 'substitutes?) + #:dry-run? (assoc-ref opts 'dry-run?)) + + (and (or (assoc-ref opts 'dry-run?) + (build-derivations local drv)) + (let* ((session (open-ssh-session host #:user user #:port port)) + (sent (send-files local items + (connect-to-remote-daemon session) + #:recursive? #t))) + (format #t "~{~a~%~}" sent) + sent))))) + +(define (retrieve-from-remote-host source opts) + "Retrieve ITEMS from SOURCE." + (with-store local + (let*-values (((user host port) + (ssh-spec->user+host+port source)) + ((session) + (open-ssh-session host #:user user #:port port)) + ((remote) + (connect-to-remote-daemon session))) + (set-build-options-from-command-line local opts) + ;; TODO: Here we could to compute and build the derivations on REMOTE + ;; rather than on LOCAL (one-off offloading) but that is currently too + ;; slow due to the many RPC round trips. So we just assume that REMOTE + ;; contains ITEMS. + (let*-values (((drv items) + (options->derivations+files local opts)) + ((retrieved) + (retrieve-files local items remote #:recursive? #t))) + (format #t "~{~a~%~}" retrieved) + retrieved)))) + + +;;; +;;; Options. +;;; + +(define (show-help) + (display (_ "Usage: guix copy [OPTION]... ITEMS... +Copy ITEMS to or from the specified host over SSH.\n")) + (display (_ " + --to=HOST send ITEMS to HOST")) + (display (_ " + --from=HOST receive ITEMS from HOST")) + (newline) + (show-build-options-help) + (newline) + (display (_ " + -h, --help display this help and exit")) + (display (_ " + -V, --version display version information and exit")) + (newline) + (show-bug-report-information)) + +(define %options + ;; Specifications of the command-line options. + (cons* (option '("to") #t #f + (lambda (opt name arg result) + (alist-cons 'destination arg result))) + (option '("from") #t #f + (lambda (opt name arg result) + (alist-cons 'source arg result))) + (option '(#\h "help") #f #f + (lambda args + (show-help) + (exit 0))) + (option '(#\V "version") #f #f + (lambda args + (show-version-and-exit "guix copy"))) + (option '(#\s "system") #t #f + (lambda (opt name arg result) + (alist-cons 'system arg + (alist-delete 'system result eq?)))) + %standard-build-options)) + +(define %default-options + `((system . ,(%current-system)) + (substitutes? . #t) + (graft? . #t) + (max-silent-time . 3600) + (verbosity . 0))) + + +;;; +;;; Entry point. +;;; + +(define (guix-copy . args) + (with-error-handling + (let* ((opts (parse-command-line args %options (list %default-options))) + (source (assoc-ref opts 'source)) + (target (assoc-ref opts 'destination))) + (cond (target (send-to-remote-host target opts)) + (source (retrieve-from-remote-host source opts)) + (else (leave (_ "use '--to' or '--from'~%"))))))) diff --git a/guix/scripts/environment.scm b/guix/scripts/environment.scm index 6dea67ca22..1d3be6a84f 100644 --- a/guix/scripts/environment.scm +++ b/guix/scripts/environment.scm @@ -155,6 +155,9 @@ COMMAND or an interactive shell in that environment.\n")) (display (_ " -s, --system=SYSTEM attempt to build for SYSTEM--e.g., \"i686-linux\"")) (display (_ " + -r, --root=FILE make FILE a symlink to the result, and register it + as a garbage collector root")) + (display (_ " -C, --container run command within an isolated container")) (display (_ " -N, --network allow containers to access the network")) @@ -247,6 +250,9 @@ COMMAND or an interactive shell in that environment.\n")) (alist-cons 'file-system-mapping (specification->file-system-mapping arg #f) result))) + (option '(#\r "root") #t #f + (lambda (opt name arg result) + (alist-cons 'gc-root arg result))) (option '("bootstrap") #f #f (lambda (opt name arg result) (alist-cons 'bootstrap? #t result))) @@ -323,7 +329,8 @@ profile." #:system system #:hooks (if bootstrap? '() - %default-profile-hooks))) + %default-profile-hooks) + #:locales? (not bootstrap?))) (define requisites* (store-lift requisites)) @@ -522,7 +529,26 @@ message if any test fails." (report-error (_ "cannot create container: /proc/self/setgroups does not exist\n")) (leave (_ "is your kernel version < 3.19?\n")))) -;; Entry point. +(define (register-gc-root target root) + "Make ROOT an indirect root to TARGET. This is procedure is idempotent." + (let* ((root (string-append (canonicalize-path (dirname root)) + "/" root))) + (catch 'system-error + (lambda () + (symlink target root) + ((store-lift add-indirect-root) root)) + (lambda args + (if (and (= EEXIST (system-error-errno args)) + (equal? (false-if-exception (readlink root)) target)) + (with-monad %store-monad + (return #t)) + (apply throw args)))))) + + +;;; +;;; Entry point. +;;; + (define (guix-environment . args) (with-error-handling (let* ((opts (parse-args args)) @@ -578,7 +604,9 @@ message if any test fails." system)) (prof-drv (inputs->profile-derivation inputs system bootstrap?)) - (profile -> (derivation->output-path prof-drv))) + (profile -> (derivation->output-path prof-drv)) + (gc-root -> (assoc-ref opts 'gc-root))) + ;; First build the inputs. This is necessary even for ;; --search-paths. Additionally, we might need to build bash for ;; a container. @@ -587,6 +615,9 @@ message if any test fails." (list prof-drv bash) (list prof-drv)) opts) + (mwhen gc-root + (register-gc-root profile gc-root)) + (cond ((assoc-ref opts 'dry-run?) (return #t)) diff --git a/guix/scripts/graph.scm b/guix/scripts/graph.scm index 2f70d64c90..79ce503a2e 100644 --- a/guix/scripts/graph.scm +++ b/guix/scripts/graph.scm @@ -37,6 +37,7 @@ #:use-module (srfi srfi-37) #:use-module (ice-9 match) #:export (%package-node-type + %reverse-package-node-type %bag-node-type %bag-with-origins-node-type %bag-emerged-node-type @@ -103,6 +104,25 @@ name." ;;; +;;; Reverse package DAG. +;;; + +(define %reverse-package-node-type + ;; For this node type we first need to compute the list of packages and the + ;; list of back-edges. Since we want to do it only once, we use the + ;; promises below. + (let* ((packages (delay (fold-packages cons '()))) + (back-edges (delay (run-with-store #f ;store not actually needed + (node-back-edges %package-node-type + (force packages)))))) + (node-type + (inherit %package-node-type) + (name "reverse-package") + (description "the reverse DAG of packages") + (edges (lift1 (force back-edges) %store-monad))))) + + +;;; ;;; Package DAG using bags. ;;; @@ -323,6 +343,7 @@ substitutes." (define %node-types ;; List of all the node types. (list %package-node-type + %reverse-package-node-type %bag-node-type %bag-with-origins-node-type %bag-emerged-node-type @@ -337,6 +358,13 @@ substitutes." %node-types) (leave (_ "~a: unknown node type~%") name))) +(define (lookup-backend name) + "Return the graph backend called NAME. Raise an error if it is not found." + (or (find (lambda (backend) + (string=? (graph-backend-name backend) name)) + %graph-backends) + (leave (_ "~a: unknown backend~%") name))) + (define (list-node-types) "Print the available node types along with their synopsis." (display (_ "The available node types are:\n")) @@ -347,6 +375,16 @@ substitutes." (node-type-description type))) %node-types)) +(define (list-backends) + "Print the available backends along with their synopsis." + (display (_ "The available backend types are:\n")) + (newline) + (for-each (lambda (backend) + (format #t " - ~a: ~a~%" + (graph-backend-name backend) + (graph-backend-description backend))) + %graph-backends)) + ;;; ;;; Command-line options. @@ -361,6 +399,14 @@ substitutes." (lambda (opt name arg result) (list-node-types) (exit 0))) + (option '(#\b "backend") #t #f + (lambda (opt name arg result) + (alist-cons 'backend (lookup-backend arg) + result))) + (option '("list-backends") #f #f + (lambda (opt name arg result) + (list-backends) + (exit 0))) (option '(#\e "expression") #t #f (lambda (opt name arg result) (alist-cons 'expression arg result))) @@ -378,6 +424,10 @@ substitutes." (display (_ "Usage: guix graph PACKAGE... Emit a Graphviz (dot) representation of the dependencies of PACKAGE...\n")) (display (_ " + -b, --backend=TYPE produce a graph with the given backend TYPE")) + (display (_ " + --list-backends list the available graph backends")) + (display (_ " -t, --type=TYPE represent nodes of the given TYPE")) (display (_ " --list-types list the available graph types")) @@ -392,7 +442,8 @@ Emit a Graphviz (dot) representation of the dependencies of PACKAGE...\n")) (show-bug-report-information)) (define %default-options - `((node-type . ,%package-node-type))) + `((node-type . ,%package-node-type) + (backend . ,%graphviz-backend))) ;;; @@ -407,6 +458,7 @@ Emit a Graphviz (dot) representation of the dependencies of PACKAGE...\n")) (lambda (arg result) (alist-cons 'argument arg result)) %default-options)) + (backend (assoc-ref opts 'backend)) (type (assoc-ref opts 'node-type)) (items (filter-map (match-lambda (('argument . (? store-path? item)) @@ -429,7 +481,8 @@ Emit a Graphviz (dot) representation of the dependencies of PACKAGE...\n")) items))) (export-graph (concatenate nodes) (current-output-port) - #:node-type type))))))) + #:node-type type + #:backend backend))))))) #t) ;;; graph.scm ends here diff --git a/guix/scripts/hash.scm b/guix/scripts/hash.scm index a339a8556b..640b2417d2 100644 --- a/guix/scripts/hash.scm +++ b/guix/scripts/hash.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2016 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2016, 2017 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2013 Nikita Karetnikov <nikita@karetnikov.org> ;;; Copyright © 2016 Jan Nieuwenhuizen <janneke@gnu.org> ;;; @@ -116,6 +116,9 @@ and 'hexadecimal' can be used as well).\n")) (case (stat:type stat) ((directory) (member (basename file) '(".bzr" ".git" ".hg" ".svn" "CVS"))) + ((regular) + ;; Git sub-modules have a '.git' file that is a regular text file. + (string=? (basename file) ".git")) (else #f))) diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm index e54744feca..4d07e0fd69 100644 --- a/guix/scripts/import.scm +++ b/guix/scripts/import.scm @@ -73,7 +73,7 @@ rather than \\n." ;;; Entry point. ;;; -(define importers '("gnu" "nix" "pypi" "cpan" "hackage" "elpa" "gem" "cran")) +(define importers '("gnu" "nix" "pypi" "cpan" "hackage" "elpa" "gem" "cran" "crate")) (define (resolve-importer name) (let ((module (resolve-interface @@ -107,10 +107,17 @@ Run IMPORTER with ARGS.\n")) (show-version-and-exit "guix import")) ((importer args ...) (if (member importer importers) - (match (apply (resolve-importer importer) args) - ((and expr ('package _ ...)) - (pretty-print expr (newline-rewriting-port - (current-output-port)))) - (x - (leave (_ "'~a' import failed~%") importer))) + (let ((print (lambda (expr) + (pretty-print expr (newline-rewriting-port + (current-output-port)))))) + (match (apply (resolve-importer importer) args) + ((and expr ('package _ ...)) + (print expr)) + ((? list? expressions) + (for-each (lambda (expr) + (print expr) + (newline)) + expressions)) + (x + (leave (_ "'~a' import failed~%") importer)))) (leave (_ "~a: invalid importer~%") importer))))) diff --git a/guix/scripts/import/cran.scm b/guix/scripts/import/cran.scm index ace1123b90..66c660ae14 100644 --- a/guix/scripts/import/cran.scm +++ b/guix/scripts/import/cran.scm @@ -26,6 +26,7 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-37) + #:use-module (srfi srfi-41) #:use-module (ice-9 match) #:use-module (ice-9 format) #:export (guix-import-cran)) @@ -63,6 +64,9 @@ Import and convert the CRAN package for PACKAGE-NAME.\n")) (lambda (opt name arg result) (alist-cons 'repo (string->symbol arg) (alist-delete 'repo result)))) + (option '(#\r "recursive") #f #f + (lambda (opt name arg result) + (alist-cons 'recursive #t result))) %standard-import-options)) @@ -88,12 +92,22 @@ Import and convert the CRAN package for PACKAGE-NAME.\n")) (reverse opts)))) (match args ((package-name) - (let ((sexp (cran->guix-package package-name - (or (assoc-ref opts 'repo) 'cran)))) - (unless sexp - (leave (_ "failed to download description for package '~a'~%") - package-name)) - sexp)) + (if (assoc-ref opts 'recursive) + ;; Recursive import + (map (match-lambda + ((and ('package ('name name) . rest) pkg) + `(define-public ,(string->symbol name) + ,pkg)) + (_ #f)) + (stream->list (recursive-import package-name + (or (assoc-ref opts 'repo) 'cran)))) + ;; Single import + (let ((sexp (cran->guix-package package-name + (or (assoc-ref opts 'repo) 'cran)))) + (unless sexp + (leave (_ "failed to download description for package '~a'~%") + package-name)) + sexp))) (() (leave (_ "too few arguments~%"))) ((many ...) diff --git a/guix/scripts/import/crate.scm b/guix/scripts/import/crate.scm new file mode 100644 index 0000000000..4337a0b623 --- /dev/null +++ b/guix/scripts/import/crate.scm @@ -0,0 +1,94 @@ + +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2014 David Thompson <davet@gnu.org> +;;; Copyright © 2016 David Craven <david@craven.ch> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix scripts import crate) + #:use-module (guix ui) + #:use-module (guix utils) + #:use-module (guix scripts) + #:use-module (guix import crate) + #:use-module (guix scripts import) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-37) + #:use-module (ice-9 match) + #:use-module (ice-9 format) + #:export (guix-import-crate)) + + +;;; +;;; Command-line options. +;;; + +(define %default-options + '()) + +(define (show-help) + (display (_ "Usage: guix import crate PACKAGE-NAME +Import and convert the crate.io package for PACKAGE-NAME.\n")) + (display (_ " + -h, --help display this help and exit")) + (display (_ " + -V, --version display version information and exit")) + (newline) + (show-bug-report-information)) + +(define %options + ;; Specification of the command-line options. + (cons* (option '(#\h "help") #f #f + (lambda args + (show-help) + (exit 0))) + (option '(#\V "version") #f #f + (lambda args + (show-version-and-exit "guix import crate"))) + %standard-import-options)) + + +;;; +;;; Entry point. +;;; + +(define (guix-import-crate . args) + (define (parse-options) + ;; Return the alist of option values. + (args-fold* args %options + (lambda (opt name arg result) + (leave (_ "~A: unrecognized option~%") name)) + (lambda (arg result) + (alist-cons 'argument arg result)) + %default-options)) + + (let* ((opts (parse-options)) + (args (filter-map (match-lambda + (('argument . value) + value) + (_ #f)) + (reverse opts)))) + (match args + ((package-name) + (let ((sexp (crate->guix-package package-name))) + (unless sexp + (leave (_ "failed to download meta-data for package '~a'~%") + package-name)) + sexp)) + (() + (leave (_ "too few arguments~%"))) + ((many ...) + (leave (_ "too many arguments~%")))))) diff --git a/guix/scripts/lint.scm b/guix/scripts/lint.scm index 9641d3926a..9b991786c3 100644 --- a/guix/scripts/lint.scm +++ b/guix/scripts/lint.scm @@ -4,6 +4,7 @@ ;;; Copyright © 2013, 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2015, 2016 Mathieu Lirzin <mthl@gnu.org> ;;; Copyright © 2016 Danny Milosavljevic <dannym+a@scratchpost.org> +;;; Copyright © 2016 Hartmut Goebel <h.goebel@crazy-compilers.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -59,6 +60,7 @@ #:export (guix-lint check-description-style check-inputs-should-be-native + check-inputs-should-not-be-an-input-at-all check-patch-file-names check-synopsis-style check-derivation @@ -229,34 +231,65 @@ by two spaces; possible infraction~p at ~{~a~^, ~}") (format #f (_ "invalid description: ~s") description) 'description)))) +(define (warn-if-package-has-input linted inputs-to-check input-names message) + ;; Emit a warning MESSAGE if some of the inputs named in INPUT-NAMES are + ;; contained in INPUTS-TO-CHECK, which are assumed to be inputs of package + ;; LINTED. + (match inputs-to-check + (((labels packages . outputs) ...) + (for-each (lambda (package output) + (when (package? package) + (let ((input (string-append + (package-name package) + (if (> (length output) 0) + (string-append ":" (car output)) + "")))) + (when (member input input-names) + (emit-warning linted + (format #f (_ message) input) + 'inputs-to-check))))) + packages outputs)))) + (define (check-inputs-should-be-native package) ;; Emit a warning if some inputs of PACKAGE are likely to belong to its ;; native inputs. - (let ((linted package) + (let ((message "'~a' should probably be a native input") (inputs (package-inputs package)) - (native-inputs + (input-names '("pkg-config" "extra-cmake-modules" "glib:bin" "intltool" "itstool" - "qttools"))) - (match inputs - (((labels packages . outputs) ...) - (for-each (lambda (package output) - (when (package? package) - (let ((input (string-append - (package-name package) - (if (> (length output) 0) - (string-append ":" (car output)) - "")))) - (when (member input native-inputs) - (emit-warning linted - (format #f (_ "'~a' should probably \ -be a native input") - input) - 'inputs))))) - packages outputs))))) + "qttools" + "python-coverage" "python2-coverage" + "python-cython" "python2-cython" + "python-docutils" "python2-docutils" + "python-mock" "python2-mock" + "python-nose" "python2-nose" + "python-pbr" "python2-pbr" + "python-pytest" "python2-pytest" + "python-pytest-cov" "python2-pytest-cov" + "python-setuptools-scm" "python2-setuptools-scm" + "python-sphinx" "python2-sphinx"))) + (warn-if-package-has-input package inputs input-names message))) + +(define (check-inputs-should-not-be-an-input-at-all package) + ;; Emit a warning if some inputs of PACKAGE are likely to should not be + ;; an input at all. + (let ((message "'~a' should probably not be an input at all") + (inputs (package-inputs package)) + (input-names + '("python-setuptools" + "python2-setuptools" + "python-pip" + "python2-pip"))) + (warn-if-package-has-input package (package-inputs package) + input-names message) + (warn-if-package-has-input package (package-native-inputs package) + input-names message) + (warn-if-package-has-input package (package-propagated-inputs package) + input-names message))) (define (package-name-regexp package) "Return a regexp that matches PACKAGE's name as a word at the beginning of a @@ -876,6 +909,10 @@ them for PACKAGE." (description "Identify inputs that should be native inputs") (check check-inputs-should-be-native)) (lint-checker + (name 'inputs-should-not-be-input) + (description "Identify inputs that should be inputs at all") + (check check-inputs-should-not-be-an-input-at-all)) + (lint-checker (name 'patch-file-names) (description "Validate file names and availability of patches") (check check-patch-file-names)) diff --git a/guix/scripts/offload.scm b/guix/scripts/offload.scm index 2e0268020c..6a4ae28689 100644 --- a/guix/scripts/offload.scm +++ b/guix/scripts/offload.scm @@ -24,8 +24,10 @@ #:use-module (ssh popen) #:use-module (ssh dist) #:use-module (ssh dist node) + #:use-module (ssh version) #:use-module (guix config) #:use-module (guix records) + #:use-module (guix ssh) #:use-module (guix store) #:use-module (guix derivations) #:use-module ((guix serialization) @@ -74,6 +76,10 @@ (private-key build-machine-private-key ; file name (default (user-openssh-private-key))) (host-key build-machine-host-key) ; string + (compression build-machine-compression ; string + (default "zlib@openssh.com,zlib")) + (compression-level build-machine-compression-level ;integer + (default 3)) (daemon-socket build-machine-daemon-socket ; string (default "/var/guix/daemon-socket/socket")) (parallel-builds build-machine-parallel-builds ; number @@ -168,86 +174,53 @@ private key from '~a': ~a") (session (make-session #:user (build-machine-user machine) #:host (build-machine-name machine) #:port (build-machine-port machine) - #:timeout 5 ;seconds + #:timeout 10 ;seconds ;; #:log-verbosity 'protocol #:identity (build-machine-private-key machine) + ;; By default libssh reads ~/.ssh/known_hosts + ;; and uses that to adjust its choice of cipher + ;; suites, which changes the type of host key + ;; that the server sends (RSA vs. Ed25519, + ;; etc.). Opt for something reproducible and + ;; stateless instead. + #:knownhosts "/dev/null" + ;; We need lightweight compression when ;; exchanging full archives. - #:compression "zlib" - #:compression-level 3))) - (connect! session) - - ;; Authenticate the server. XXX: Guile-SSH 0.10.1 doesn't know about - ;; ed25519 keys and 'get-key-type' returns #f in that case. - (let-values (((server) (get-server-public-key session)) - ((type key) (host-key->type+key - (build-machine-host-key machine)))) - (unless (and (or (not (get-key-type server)) - (eq? (get-key-type server) type)) - (string=? (public-key->string server) key)) - ;; Key mismatch: something's wrong. XXX: It could be that the server - ;; provided its Ed25519 key when we where expecting its RSA key. - (leave (_ "server at '~a' returned host key '~a' of type '~a' \ + #:compression + (build-machine-compression machine) + #:compression-level + (build-machine-compression-level machine)))) + (match (connect! session) + ('ok + ;; Authenticate the server. XXX: Guile-SSH 0.10.1 doesn't know about + ;; ed25519 keys and 'get-key-type' returns #f in that case. + (let-values (((server) (get-server-public-key session)) + ((type key) (host-key->type+key + (build-machine-host-key machine)))) + (unless (and (or (not (get-key-type server)) + (eq? (get-key-type server) type)) + (string=? (public-key->string server) key)) + ;; Key mismatch: something's wrong. XXX: It could be that the server + ;; provided its Ed25519 key when we where expecting its RSA key. + (leave (_ "server at '~a' returned host key '~a' of type '~a' \ instead of '~a' of type '~a'~%") - (build-machine-name machine) - (public-key->string server) (get-key-type server) - key type))) - - (let ((auth (userauth-public-key! session private))) - (unless (eq? 'success auth) - (disconnect! session) - (leave (_ "SSH public key authentication failed for '~a': ~a~%") - (build-machine-name machine) (get-error session)))) - - session)) - -(define* (connect-to-remote-daemon session - #:optional - (socket-name "/var/guix/daemon-socket/socket")) - "Connect to the remote build daemon listening on SOCKET-NAME over SESSION, -an SSH session. Return a <nix-server> object." - (define redirect - ;; Code run in SESSION to redirect the remote process' stdin/stdout to the - ;; daemon's socket, à la socat. The SSH protocol supports forwarding to - ;; Unix-domain sockets but libssh doesn't have an API for that, hence this - ;; hack. - `(begin - (use-modules (ice-9 match) (rnrs io ports)) - - (let ((sock (socket AF_UNIX SOCK_STREAM 0)) - (stdin (current-input-port)) - (stdout (current-output-port))) - (setvbuf stdin _IONBF) - (setvbuf stdout _IONBF) - (connect sock AF_UNIX ,socket-name) - - (let loop () - (match (select (list stdin sock) '() (list stdin stdout sock)) - ((reads writes ()) - (when (memq stdin reads) - (match (get-bytevector-some stdin) - ((? eof-object?) - (primitive-exit 0)) - (bv - (put-bytevector sock bv)))) - (when (memq sock reads) - (match (get-bytevector-some sock) - ((? eof-object?) - (primitive-exit 0)) - (bv - (put-bytevector stdout bv)))) - (loop)) - (_ - (primitive-exit 1))))))) - - (let ((channel - (open-remote-pipe* session OPEN_BOTH - ;; Sort-of shell-quote REDIRECT. - "guile" "-c" - (object->string - (object->string redirect))))) - (open-connection #:port channel))) + (build-machine-name machine) + (public-key->string server) (get-key-type server) + key type))) + + (let ((auth (userauth-public-key! session private))) + (unless (eq? 'success auth) + (disconnect! session) + (leave (_ "SSH public key authentication failed for '~a': ~a~%") + (build-machine-name machine) (get-error session)))) + + session) + (x + ;; Connection failed or timeout expired. + (leave (_ "failed to connect to '~a': ~a~%") + (build-machine-name machine) (get-error session)))))) ;;; @@ -363,8 +336,9 @@ MACHINE." ;; Protect DRV from garbage collection. (add-temp-root store (derivation-file-name drv)) - (send-files (cons (derivation-file-name drv) inputs) - store) + (with-store local + (send-files local (cons (derivation-file-name drv) inputs) store + #:log-port (current-output-port))) (format (current-error-port) "offloading '~a' to '~a'...~%" (derivation-file-name drv) (build-machine-name machine)) (format (current-error-port) "@ build-remote ~a ~a~%" @@ -379,92 +353,20 @@ MACHINE." ;; Use exit code 100 for a permanent build failure. The daemon ;; interprets other non-zero codes as transient build failures. (primitive-exit 100))) - (build-derivations store (list drv))) + (parameterize ((current-build-output-port (build-log-port))) + (build-derivations store (list drv)))) - (retrieve-files outputs store) + (retrieve-files* outputs store) (format (current-error-port) "done with offloaded '~a'~%" (derivation-file-name drv))) -(define (store-import-channel session) - "Return an output port to which archives to be exported to SESSION's store -can be written." - ;; Using the 'import-paths' RPC on a remote store would be slow because it - ;; makes a round trip every time 32 KiB have been transferred. This - ;; procedure instead opens a separate channel to use the remote - ;; 'import-paths' procedure, which consumes all the data in a single round - ;; trip. - (define import - `(begin - (use-modules (guix)) - - (with-store store - (setvbuf (current-input-port) _IONBF) - (import-paths store (current-input-port))))) - - (open-remote-output-pipe session - (string-join - `("guile" "-c" - ,(object->string - (object->string import)))))) - -(define (store-export-channel session files) - "Return an input port from which an export of FILES from SESSION's store can -be read." - ;; Same as above: this is more efficient than calling 'export-paths' on a - ;; remote store. - (define export - `(begin - (use-modules (guix)) - - (with-store store - (setvbuf (current-output-port) _IONBF) - (export-paths store ',files (current-output-port))))) - - (open-remote-input-pipe session - (string-join - `("guile" "-c" - ,(object->string - (object->string export)))))) - -(define (send-files files remote) - "Send the subset of FILES that's missing to REMOTE, a remote store." - (with-store store - ;; Compute the subset of FILES missing on SESSION, and send them in - ;; topologically sorted order so that they can actually be imported. - (let* ((sorted (topologically-sorted store files)) - (session (channel-get-session (nix-server-socket remote))) - (node (make-node session)) - (missing (node-eval node - `(begin - (use-modules (guix) - (srfi srfi-1) (srfi srfi-26)) - - (with-store store - (remove (cut valid-path? store <>) - ',sorted))))) - (port (store-import-channel session))) - (format #t (_ "sending ~a store files to '~a'...~%") - (length missing) (session-get session 'host)) - - (export-paths store missing port) - - ;; Tell the remote process that we're done. (In theory the - ;; end-of-archive mark of 'export-paths' would be enough, but in - ;; practice it's not.) - (channel-send-eof port) - - ;; Wait for completion of the remote process. - (let ((result (zero? (channel-get-exit-status port)))) - (close-port port) - result)))) - -(define (retrieve-files files remote) - "Retrieve FILES from SESSION's store, and import them." - (let* ((session (channel-get-session (nix-server-socket remote))) - (host (session-get session 'host)) - (port (store-export-channel session files))) - (format #t (_ "retrieving ~a files from '~a'...~%") - (length files) host) +(define (retrieve-files* files remote) + "Retrieve FILES from REMOTE and import them using 'restore-file-set'." + (let-values (((port count) + (file-retrieval-port files remote))) + (format #t (N_ "retrieving ~a store item from '~a'...~%" + "retrieving ~a store items from '~a'...~%" count) + count (remote-store-host remote)) ;; We cannot use the 'import-paths' RPC here because we already ;; hold the locks for FILES. @@ -489,37 +391,30 @@ be read." (define (machine-load machine) "Return the load of MACHINE, divided by the number of parallel builds -allowed on MACHINE." - (let* ((session (open-ssh-session machine)) - (pipe (open-remote-pipe* session OPEN_READ +allowed on MACHINE. Return +∞ if MACHINE is unreachable." + ;; Note: This procedure is costly since it creates a new SSH session. + (match (false-if-exception (open-ssh-session machine)) + ((? session? session) + (let* ((pipe (open-remote-pipe* session OPEN_READ "cat" "/proc/loadavg")) - (line (read-line pipe))) - (close-port pipe) - - (if (eof-object? line) - +inf.0 ;MACHINE does not respond, so assume it is infinitely loaded - (match (string-tokenize line) - ((one five fifteen . _) - (let* ((raw (string->number five)) - (jobs (build-machine-parallel-builds machine)) - (normalized (/ raw jobs))) - (format (current-error-port) "load on machine '~a' is ~s\ + (line (read-line pipe))) + (close-port pipe) + + (if (eof-object? line) + +inf.0 ;MACHINE does not respond, so assume it is infinitely loaded + (match (string-tokenize line) + ((one five fifteen . _) + (let* ((raw (string->number five)) + (jobs (build-machine-parallel-builds machine)) + (normalized (/ raw jobs))) + (format (current-error-port) "load on machine '~a' is ~s\ (normalized: ~s)~%" - (build-machine-name machine) raw normalized) - normalized)) - (_ - +inf.0))))) ;something's fishy about MACHINE, so avoid it - -(define (machine-power-factor m) - "Return a factor that aggregates the speed and load of M. The higher the -better." - (/ (build-machine-speed m) - (+ 1 (machine-load m)))) - -(define (machine-less-loaded-or-faster? m1 m2) - "Return #t if M1 is either less loaded or faster than M2. (This relation -defines a total order on machines.)" - (> (machine-power-factor m1) (machine-power-factor m2))) + (build-machine-name machine) raw normalized) + normalized)) + (_ + +inf.0))))) ;something's fishy about MACHINE, so avoid it + (_ + +inf.0))) ;failed to connect to MACHINE, so avoid it (define (machine-lock-file machine hint) "Return the name of MACHINE's lock file for HINT." @@ -548,29 +443,39 @@ defines a total order on machines.)" ;; 5. Release the global machine-choice lock. (with-file-lock (machine-choice-lock-file) - (define machines+slots + (define machines+slots+loads (filter-map (lambda (machine) + ;; Call 'machine-load' from here to make sure it is called + ;; only once per machine (it is expensive). (let ((slot (acquire-build-slot machine))) - (and slot (list machine slot)))) + (and slot + (list machine slot (machine-load machine))))) machines)) (define (undecorate pred) (lambda (a b) (match a - ((machine1 slot1) + ((machine1 slot1 load1) (match b - ((machine2 slot2) - (pred machine1 machine2))))))) - - (let loop ((machines+slots - (sort machines+slots + ((machine2 slot2 load2) + (pred machine1 load1 machine2 load2))))))) + + (define (machine-less-loaded-or-faster? m1 l1 m2 l2) + ;; Return #t if M1 is either less loaded or faster than M2, with L1 + ;; being the load of M1 and L2 the load of M2. (This relation defines a + ;; total order on machines.) + (> (/ (build-machine-speed m1) (+ 1 l1)) + (/ (build-machine-speed m2) (+ 1 l2)))) + + (let loop ((machines+slots+loads + (sort machines+slots+loads (undecorate machine-less-loaded-or-faster?)))) - (match machines+slots - (((best slot) others ...) + (match machines+slots+loads + (((best slot load) others ...) ;; Return the best machine unless it's already overloaded. - (if (< (machine-load best) 2.) + (if (< load 2.) (match others - (((machines slots) ...) + (((machines slots loads) ...) ;; Release slots from the uninteresting machines. (for-each release-build-slot slots) @@ -618,6 +523,96 @@ defines a total order on machines.)" ;;; +;;; Installation tests. +;;; + +(define (assert-node-repl node name) + "Bail out if NODE is not running Guile." + (match (node-guile-version node) + (#f + (leave (_ "Guile could not be started on '~a'~%") + name)) + ((? string? version) + ;; Note: The version string already contains the word "Guile". + (info (_ "'~a' is running ~a~%") + name (node-guile-version node))))) + +(define (assert-node-has-guix node name) + "Bail out if NODE lacks the (guix) module, or if its daemon is not running." + (match (node-eval node + '(begin + (use-modules (guix)) + (with-store store + (add-text-to-store store "test" + "Hello, build machine!")))) + ((? string? str) + (info (_ "Guix is usable on '~a' (test returned ~s)~%") + name str)) + (x + (leave (_ "failed to use Guix module on '~a' (test returned ~s)~%") + name x)))) + +(define %random-state + (delay + (seed->random-state (logxor (getpid) (car (gettimeofday)))))) + +(define* (nonce #:optional (name (gethostname))) + (string-append name "-" + (number->string (random 1000000 (force %random-state))))) + +(define (assert-node-can-import node name daemon-socket) + "Bail out if NODE refuses to import our archives." + (let ((session (node-session node))) + (with-store store + (let* ((item (add-text-to-store store "export-test" (nonce))) + (remote (connect-to-remote-daemon session daemon-socket))) + (with-store local + (send-files local (list item) remote)) + + (if (valid-path? remote item) + (info (_ "'~a' successfully imported '~a'~%") + name item) + (leave (_ "'~a' was not properly imported on '~a'~%") + item name)))))) + +(define (assert-node-can-export node name daemon-socket) + "Bail out if we cannot import signed archives from NODE." + (let* ((session (node-session node)) + (remote (connect-to-remote-daemon session daemon-socket)) + (item (add-text-to-store remote "import-test" (nonce name)))) + (with-store store + (if (and (retrieve-files store (list item) remote) + (valid-path? store item)) + (info (_ "successfully imported '~a' from '~a'~%") + item name) + (leave (_ "failed to import '~a' from '~a'~%") + item name))))) + +(define (check-machine-availability machine-file pred) + "Check that each machine matching PRED in MACHINE-FILE is usable as a build +machine." + (define (build-machine=? m1 m2) + (and (string=? (build-machine-name m1) (build-machine-name m2)) + (= (build-machine-port m1) (build-machine-port m2)))) + + ;; A given build machine may appear several times (e.g., once for + ;; "x86_64-linux" and a second time for "i686-linux"); test them only once. + (let ((machines (filter pred + (delete-duplicates (build-machines machine-file) + build-machine=?)))) + (info (_ "testing ~a build machines defined in '~a'...~%") + (length machines) machine-file) + (let* ((names (map build-machine-name machines)) + (sockets (map build-machine-daemon-socket machines)) + (sessions (map open-ssh-session machines)) + (nodes (map make-node sessions))) + (for-each assert-node-repl nodes names) + (for-each assert-node-has-guix nodes names) + (for-each assert-node-can-import nodes names sockets) + (for-each assert-node-can-export nodes names sockets)))) + + +;;; ;;; Entry point. ;;; @@ -635,6 +630,12 @@ defines a total order on machines.)" (and=> (passwd:dir (getpw (getuid))) (cut setenv "HOME" <>)) + ;; We rely on protocol-level compression from libssh to optimize large data + ;; transfers. Warn if it's missing. + (unless (zlib-support?) + (warning (_ "Guile-SSH lacks zlib support")) + (warning (_ "data transfers will *not* be compressed!"))) + (match args ((system max-silent-time print-build-trace? build-timeout) (let ((max-silent-time (string->number max-silent-time)) @@ -660,6 +661,18 @@ defines a total order on machines.)" (else (leave (_ "invalid request line: ~s~%") line))) (loop (read-line))))))) + (("test" rest ...) + (with-error-handling + (let-values (((file pred) + (match rest + ((file regexp) + (values file + (compose (cut string-match regexp <>) + build-machine-name))) + ((file) (values file (const #t))) + (() (values %machine-file (const #t))) + (_ (leave (_ "wrong number of arguments~%")))))) + (check-machine-availability (or file %machine-file) pred)))) (("--version") (show-version-and-exit "guix offload")) (("--help") diff --git a/guix/scripts/package.scm b/guix/scripts/package.scm index 96a22f6fab..90e7fa2298 100644 --- a/guix/scripts/package.scm +++ b/guix/scripts/package.scm @@ -200,7 +200,8 @@ specified in MANIFEST, a manifest object." (profile-derivation manifest #:hooks (if bootstrap? '() - %default-profile-hooks)))) + %default-profile-hooks) + #:locales? (not bootstrap?)))) (prof (derivation->output-path prof-drv))) (show-what-to-build store (list prof-drv) #:use-substitutes? use-substitutes? diff --git a/guix/scripts/publish.scm b/guix/scripts/publish.scm index 1b32f639ea..33a7b3bd42 100644 --- a/guix/scripts/publish.scm +++ b/guix/scripts/publish.scm @@ -365,6 +365,14 @@ example: \"/foo/bar\" yields '(\"foo\" \"bar\")." (response-headers response) eq?))) +(define (with-content-length response length) + "Return RESPONSE with a 'content-length' header set to LENGTH." + (set-field response (response-headers) + (alist-cons 'content-length length + (alist-delete 'content-length + (response-headers response) + eq?)))) + (define-syntax-rule (swallow-EPIPE exp ...) "Swallow EPIPE errors raised by EXP..." (catch 'system-error @@ -432,13 +440,8 @@ blocking." (call-with-input-file (utf8->string body) (lambda (input) (let* ((size (stat:size (stat input))) - (headers (alist-cons 'content-length size - (alist-delete 'content-length - (response-headers response) - eq?))) - (response (write-response (set-field response - (response-headers) - headers) + (response (write-response (with-content-length response + size) client)) (output (response-port response))) (dump-port input output) diff --git a/guix/scripts/refresh.scm b/guix/scripts/refresh.scm index b81c69f9fe..0dd7eee974 100644 --- a/guix/scripts/refresh.scm +++ b/guix/scripts/refresh.scm @@ -35,7 +35,8 @@ #:select (%gnu-updater %gnome-updater %kde-updater - %xorg-updater)) + %xorg-updater + %kernel.org-updater)) #:use-module (guix import elpa) #:use-module (guix import cran) #:use-module (guix import hackage) @@ -118,7 +119,7 @@ (show-version-and-exit "guix refresh"))))) (define (show-help) - (display (_ "Usage: guix refresh [OPTION]... PACKAGE... + (display (_ "Usage: guix refresh [OPTION]... [PACKAGE]... Update package definitions to match the latest upstream version. When PACKAGE... is given, update only the specified packages. Otherwise @@ -200,15 +201,18 @@ unavailable optional dependencies such as Guile-JSON." %gnome-updater %kde-updater %xorg-updater + %kernel.org-updater %elpa-updater %cran-updater %bioconductor-updater %hackage-updater + ((guix import cpan) => %cpan-updater) ((guix import pypi) => %pypi-updater) ((guix import gem) => %gem-updater) - ((guix import github) => %github-updater))) + ((guix import github) => %github-updater) + ((guix import crate) => %crate-updater))) -(define (lookup-updater name) +(define (lookup-updater-by-name name) "Return the updater called NAME." (or (find (lambda (updater) (eq? name (upstream-updater-name updater))) @@ -218,38 +222,84 @@ unavailable optional dependencies such as Guile-JSON." (define (list-updaters-and-exit) "Display available updaters and exit." (format #t (_ "Available updaters:~%")) - (for-each (lambda (updater) - (format #t "- ~a: ~a~%" - (upstream-updater-name updater) - (_ (upstream-updater-description updater)))) - %updaters) + (newline) + + (let* ((packages (fold-packages cons '())) + (total (length packages))) + (define covered + (fold (lambda (updater covered) + (let ((matches (count (upstream-updater-predicate updater) + packages))) + ;; TRANSLATORS: The parenthetical expression here is rendered + ;; like "(42% coverage)" and denotes the fraction of packages + ;; covered by the given updater. + (format #t (_ " - ~a: ~a (~2,1f% coverage)~%") + (upstream-updater-name updater) + (_ (upstream-updater-description updater)) + (* 100. (/ matches total))) + (+ covered matches))) + 0 + %updaters)) + + (newline) + (format #t (_ "~2,1f% of the packages are covered by these updaters.~%") + (* 100. (/ covered total)))) (exit 0)) +(define (warn-no-updater package) + (format (current-error-port) + (_ "~a: warning: no updater for ~a~%") + (location->string (package-location package)) + (package-name package))) + (define* (update-package store package updaters - #:key (key-download 'interactive)) + #:key (key-download 'interactive) warn?) "Update the source file that defines PACKAGE with the new version. KEY-DOWNLOAD specifies a download policy for missing OpenPGP keys; allowed -values: 'interactive' (default), 'always', and 'never'." - (let-values (((version tarball) - (package-update store package updaters - #:key-download key-download)) - ((loc) - (or (package-field-location package 'version) - (package-location package)))) - (when version - (if (and=> tarball file-exists?) - (begin - (format (current-error-port) - (_ "~a: ~a: updating from version ~a to version ~a...~%") - (location->string loc) - (package-name package) - (package-version package) version) - (let ((hash (call-with-input-file tarball - port-sha256))) - (update-package-source package version hash))) - (warning (_ "~a: version ~a could not be \ +values: 'interactive' (default), 'always', and 'never'. When WARN? is true, +warn about packages that have no matching updater." + (if (lookup-updater package updaters) + (let-values (((version tarball) + (package-update store package updaters + #:key-download key-download)) + ((loc) + (or (package-field-location package 'version) + (package-location package)))) + (when version + (if (and=> tarball file-exists?) + (begin + (format (current-error-port) + (_ "~a: ~a: updating from version ~a to version ~a...~%") + (location->string loc) + (package-name package) + (package-version package) version) + (let ((hash (call-with-input-file tarball + port-sha256))) + (update-package-source package version hash))) + (warning (_ "~a: version ~a could not be \ downloaded and authenticated; not updating~%") - (package-name package) version))))) + (package-name package) version)))) + (when warn? + (warn-no-updater package)))) + +(define* (check-for-package-update package updaters #:key warn?) + "Check whether an update is available for PACKAGE and print a message. When +WARN? is true and no updater exists for PACKAGE, print a warning." + (match (package-latest-release package updaters) + ((? upstream-source? source) + (when (version>? (upstream-source-version source) + (package-version package)) + (let ((loc (or (package-field-location package 'version) + (package-location package)))) + (format (current-error-port) + (_ "~a: ~a would be upgraded from ~a to ~a~%") + (location->string loc) + (package-name package) (package-version package) + (upstream-source-version source))))) + (#f + (when warn? + (warn-no-updater package))))) + ;;; @@ -312,7 +362,7 @@ dependent packages are rebuilt: ~{~a~^ ~}~%" ;; Return the list of updaters to use. (match (filter-map (match-lambda (('updaters . names) - (map lookup-updater names)) + (map lookup-updater-by-name names)) (_ #f)) opts) (() @@ -360,6 +410,12 @@ update would trigger a complete rebuild." (updaters (options->updaters opts)) (list-dependent? (assoc-ref opts 'list-dependent?)) (key-download (assoc-ref opts 'key-download)) + + ;; Warn about missing updaters when a package is explicitly given on + ;; the command line. + (warn? (or (assoc-ref opts 'argument) + (assoc-ref opts 'expression))) + (packages (match (filter-map (match-lambda (('argument . spec) @@ -397,22 +453,14 @@ update would trigger a complete rebuild." (%gpg-command)))) (for-each (cut update-package store <> updaters - #:key-download key-download) + #:key-download key-download + #:warn? warn?) packages) (with-monad %store-monad (return #t)))) (else - (for-each (lambda (package) - (match (package-update-path package updaters) - ((? upstream-source? source) - (let ((loc (or (package-field-location package 'version) - (package-location package)))) - (format (current-error-port) - (_ "~a: ~a would be upgraded from ~a to ~a~%") - (location->string loc) - (package-name package) (package-version package) - (upstream-source-version source)))) - (#f #f))) + (for-each (cut check-for-package-update <> updaters + #:warn? warn?) packages) (with-monad %store-monad (return #t))))))))) diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index bb373a6726..144a7fd377 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -326,7 +326,7 @@ it atomically, and then run OS's activation script." (let* ((system (derivation->output-path drv)) (number (+ 1 (generation-number profile))) (generation (generation-file-name profile number))) - (symlink system generation) + (switch-symlinks generation system) (switch-symlinks profile generation) (format #t (_ "activating system...~%")) diff --git a/guix/ssh.scm b/guix/ssh.scm new file mode 100644 index 0000000000..3548243839 --- /dev/null +++ b/guix/ssh.scm @@ -0,0 +1,226 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2016 Ludovic Courtès <ludo@gnu.org> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix ssh) + #:use-module (guix store) + #:autoload (guix ui) (N_) + #:use-module (ssh channel) + #:use-module (ssh popen) + #:use-module (ssh session) + #:use-module (ssh dist) + #:use-module (ssh dist node) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-34) + #:use-module (srfi srfi-35) + #:use-module (ice-9 match) + #:use-module (ice-9 binary-ports) + #:export (connect-to-remote-daemon + send-files + retrieve-files + remote-store-host + + file-retrieval-port)) + +;;; Commentary: +;;; +;;; This module provides tools to support communication with remote stores +;;; over SSH, using Guile-SSH. +;;; +;;; Code: + +(define* (connect-to-remote-daemon session + #:optional + (socket-name "/var/guix/daemon-socket/socket")) + "Connect to the remote build daemon listening on SOCKET-NAME over SESSION, +an SSH session. Return a <nix-server> object." + (define redirect + ;; Code run in SESSION to redirect the remote process' stdin/stdout to the + ;; daemon's socket, à la socat. The SSH protocol supports forwarding to + ;; Unix-domain sockets but libssh doesn't have an API for that, hence this + ;; hack. + `(begin + (use-modules (ice-9 match) (rnrs io ports)) + + (let ((sock (socket AF_UNIX SOCK_STREAM 0)) + (stdin (current-input-port)) + (stdout (current-output-port))) + (setvbuf stdin _IONBF) + (setvbuf stdout _IONBF) + (connect sock AF_UNIX ,socket-name) + + (let loop () + (match (select (list stdin sock) '() (list stdin stdout sock)) + ((reads writes ()) + (when (memq stdin reads) + (match (get-bytevector-some stdin) + ((? eof-object?) + (primitive-exit 0)) + (bv + (put-bytevector sock bv)))) + (when (memq sock reads) + (match (get-bytevector-some sock) + ((? eof-object?) + (primitive-exit 0)) + (bv + (put-bytevector stdout bv)))) + (loop)) + (_ + (primitive-exit 1))))))) + + (let ((channel + (open-remote-pipe* session OPEN_BOTH + ;; Sort-of shell-quote REDIRECT. + "guile" "-c" + (object->string + (object->string redirect))))) + (open-connection #:port channel))) + +(define (store-import-channel session) + "Return an output port to which archives to be exported to SESSION's store +can be written." + ;; Using the 'import-paths' RPC on a remote store would be slow because it + ;; makes a round trip every time 32 KiB have been transferred. This + ;; procedure instead opens a separate channel to use the remote + ;; 'import-paths' procedure, which consumes all the data in a single round + ;; trip. + (define import + `(begin + (use-modules (guix)) + + (with-store store + (setvbuf (current-input-port) _IONBF) + + ;; FIXME: Exceptions are silently swallowed. We should report them + ;; somehow. + (import-paths store (current-input-port))))) + + (open-remote-output-pipe session + (string-join + `("guile" "-c" + ,(object->string + (object->string import)))))) + +(define* (store-export-channel session files + #:key recursive?) + "Return an input port from which an export of FILES from SESSION's store can +be read. When RECURSIVE? is true, the closure of FILES is exported." + ;; Same as above: this is more efficient than calling 'export-paths' on a + ;; remote store. + (define export + `(begin + (use-modules (guix)) + + (with-store store + (setvbuf (current-output-port) _IONBF) + + ;; FIXME: Exceptions are silently swallowed. We should report them + ;; somehow. + (export-paths store ',files (current-output-port) + #:recursive? ,recursive?)))) + + (open-remote-input-pipe session + (string-join + `("guile" "-c" + ,(object->string + (object->string export)))))) + +(define* (send-files local files remote + #:key + recursive? + (log-port (current-error-port))) + "Send the subset of FILES from LOCAL (a local store) that's missing to +REMOTE, a remote store. When RECURSIVE? is true, send the closure of FILES. +Return the list of store items actually sent." + ;; Compute the subset of FILES missing on SESSION and send them. + (let* ((files (if recursive? (requisites local files) files)) + (session (channel-get-session (nix-server-socket remote))) + (node (make-node session)) + (missing (node-eval node + `(begin + (use-modules (guix) + (srfi srfi-1) (srfi srfi-26)) + + (with-store store + (remove (cut valid-path? store <>) + ',files))))) + (count (length missing)) + (port (store-import-channel session))) + (format log-port (N_ "sending ~a store item to '~a'...~%" + "sending ~a store items to '~a'...~%" count) + count (session-get session 'host)) + + ;; Send MISSING in topological order. + (export-paths local missing port) + + ;; Tell the remote process that we're done. (In theory the end-of-archive + ;; mark of 'export-paths' would be enough, but in practice it's not.) + (channel-send-eof port) + + ;; Wait for completion of the remote process. + (let ((result (zero? (channel-get-exit-status port)))) + (close-port port) + missing))) + +(define (remote-store-session remote) + "Return the SSH channel beneath REMOTE, a remote store as returned by +'connect-to-remote-daemon', or #f." + (channel-get-session (nix-server-socket remote))) + +(define (remote-store-host remote) + "Return the name of the host REMOTE is connected to, where REMOTE is a +remote store as returned by 'connect-to-remote-daemon'." + (match (remote-store-session remote) + (#f #f) + ((? session? session) + (session-get session 'host)))) + +(define* (file-retrieval-port files remote + #:key recursive?) + "Return an input port from which to retrieve FILES (a list of store items) +from REMOTE, along with the number of items to retrieve (lower than or equal +to the length of FILES.)" + (values (store-export-channel (remote-store-session remote) files + #:recursive? recursive?) + (length files))) ;XXX: inaccurate when RECURSIVE? is true + +(define* (retrieve-files local files remote + #:key recursive? (log-port (current-error-port))) + "Retrieve FILES from REMOTE and import them using the 'import-paths' RPC on +LOCAL. When RECURSIVE? is true, retrieve the closure of FILES." + (let-values (((port count) + (file-retrieval-port files remote + #:recursive? recursive?))) + (format #t (N_ "retrieving ~a store item from '~a'...~%" + "retrieving ~a store items from '~a'...~%" count) + count (remote-store-host remote)) + (when (eof-object? (lookahead-u8 port)) + ;; The failure could be because one of the requested store items is not + ;; valid on REMOTE, or because Guile or Guix is improperly installed. + ;; TODO: Improve error reporting. + (raise (condition + (&message + (message + (format #f + (_ "failed to retrieve store items from '~a'") + (remote-store-host remote))))))) + + (let ((result (import-paths local port))) + (close-port port) + result))) + +;;; ssh.scm ends here diff --git a/guix/store.scm b/guix/store.scm index 689a94c636..49549d0771 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -374,29 +374,36 @@ space on the file system so that the garbage collector can still operate, should the disk become full. When CPU-AFFINITY is true, it must be an integer corresponding to an OS-level CPU number to which the daemon's worker process for this connection will be pinned. Return a server object." - (let ((port (or port (open-unix-domain-socket file)))) - (write-int %worker-magic-1 port) - (let ((r (read-int port))) - (and (eqv? r %worker-magic-2) - (let ((v (read-int port))) - (and (eqv? (protocol-major %protocol-version) - (protocol-major v)) - (begin - (write-int %protocol-version port) - (when (>= (protocol-minor v) 14) - (write-int (if cpu-affinity 1 0) port) - (when cpu-affinity - (write-int cpu-affinity port))) - (when (>= (protocol-minor v) 11) - (write-int (if reserve-space? 1 0) port)) - (let ((conn (%make-nix-server port - (protocol-major v) - (protocol-minor v) - (make-hash-table 100) - (make-hash-table 100)))) - (let loop ((done? (process-stderr conn))) - (or done? (process-stderr conn))) - conn)))))))) + (guard (c ((nar-error? c) + ;; One of the 'write-' or 'read-' calls below failed, but this is + ;; really a connection error. + (raise (condition + (&nix-connection-error (file (or port file)) + (errno EPROTO)) + (&message (message "build daemon handshake failed")))))) + (let ((port (or port (open-unix-domain-socket file)))) + (write-int %worker-magic-1 port) + (let ((r (read-int port))) + (and (eqv? r %worker-magic-2) + (let ((v (read-int port))) + (and (eqv? (protocol-major %protocol-version) + (protocol-major v)) + (begin + (write-int %protocol-version port) + (when (>= (protocol-minor v) 14) + (write-int (if cpu-affinity 1 0) port) + (when cpu-affinity + (write-int cpu-affinity port))) + (when (>= (protocol-minor v) 11) + (write-int (if reserve-space? 1 0) port)) + (let ((conn (%make-nix-server port + (protocol-major v) + (protocol-minor v) + (make-hash-table 100) + (make-hash-table 100)))) + (let loop ((done? (process-stderr conn))) + (or done? (process-stderr conn))) + conn))))))))) (define (close-connection server) "Close the connection to SERVER." @@ -470,7 +477,8 @@ encoding conversion errors." ;; Write a byte stream to USER-PORT. (let* ((len (read-int p)) (m (modulo len 8))) - (dump-port p user-port len) + (dump-port p user-port len + #:buffer-size (if (<= len 16384) 16384 65536)) (unless (zero? m) ;; Consume padding, as for strings. (get-bytevector-n p (- 8 m)))) diff --git a/guix/ui.scm b/guix/ui.scm index cafb3c6705..7d4c437354 100644 --- a/guix/ui.scm +++ b/guix/ui.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2013 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2013 Nikita Karetnikov <nikita@karetnikov.org> ;;; Copyright © 2014 Cyril Roelandt <tipecaml@gmail.com> @@ -306,7 +306,13 @@ exiting. ARGS is the list of arguments received by the 'throw' handler." "Display version information for COMMAND and `(exit 0)'." (simple-format #t "~a (~a) ~a~%" command %guix-package-name %guix-version) - (display (_ "Copyright (C) 2016 the Guix authors + (format #t "Copyright ~a 2017 ~a" + ;; TRANSLATORS: Translate "(C)" to the copyright symbol + ;; (C-in-a-circle), if this symbol is available in the user's + ;; locale. Otherwise, do not translate "(C)"; leave it as-is. */ + (_ "(C)") + (_ "the Guix authors\n")) + (display (_"\ License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. diff --git a/guix/upstream.scm b/guix/upstream.scm index 18157376d2..2334c4c0a6 100644 --- a/guix/upstream.scm +++ b/guix/upstream.scm @@ -26,6 +26,11 @@ #:use-module (guix packages) #:use-module (guix ui) #:use-module (guix base32) + #:use-module (guix gexp) + #:use-module (guix store) + #:use-module ((guix derivations) + #:select (built-derivations derivation->output-path)) + #:use-module (guix monads) #:use-module (srfi srfi-1) #:use-module (srfi srfi-9) #:use-module (srfi srfi-11) @@ -49,8 +54,11 @@ upstream-updater-predicate upstream-updater-latest + lookup-updater + download-tarball - package-update-path + package-latest-release + package-latest-release* package-update update-package-source)) @@ -127,17 +135,50 @@ them matches." (and (pred package) latest))) updaters)) -(define (package-update-path package updaters) +(define (package-latest-release package updaters) "Return an upstream source to update PACKAGE, a <package> object, or #f if -no update is needed or known." +none of UPDATERS matches PACKAGE. It is the caller's responsibility to ensure +that the returned source is newer than the current one." (match (lookup-updater package updaters) ((? procedure? latest-release) - (match (latest-release package) - ((and source ($ <upstream-source> name version)) - (and (version>? version (package-version package)) - source)) - (_ #f))) - (#f #f))) + (latest-release package)) + (_ #f))) + +(define (package-latest-release* package updaters) + "Like 'package-latest-release', but ensure that the return source is newer +than that of PACKAGE." + (match (package-latest-release package updaters) + ((and source ($ <upstream-source> name version)) + (and (version>? version (package-version package)) + source)) + (_ + #f))) + +(define (uncompressed-tarball name tarball) + "Return a derivation that decompresses TARBALL." + (define (ref package) + (module-ref (resolve-interface '(gnu packages compression)) + package)) + + (define compressor + (cond ((or (string-suffix? ".gz" tarball) + (string-suffix? ".tgz" tarball)) + (file-append (ref 'gzip) "/bin/gzip")) + ((string-suffix? ".bz2" tarball) + (file-append (ref 'bzip2) "/bin/bzip2")) + ((string-suffix? ".xz" tarball) + (file-append (ref 'xz) "/bin/xz")) + ((string-suffix? ".lz" tarball) + (file-append (ref 'lzip) "/bin/lzip")) + (else + (error "unknown archive type" tarball)))) + + (gexp->derivation (file-sans-extension name) + #~(begin + (copy-file #+tarball #+name) + (and (zero? (system* #+compressor "-d" #+name)) + (copy-file #+(file-sans-extension name) + #$output))))) (define* (download-tarball store url signature-url #:key (key-download 'interactive)) @@ -149,8 +190,22 @@ values: 'interactive' (default), 'always', and 'never'." (let ((tarball (download-to-store store url))) (if (not signature-url) tarball - (let* ((sig (download-to-store store signature-url)) - (ret (gnupg-verify* sig tarball #:key-download key-download))) + (let* ((sig (download-to-store store signature-url)) + + ;; Sometimes we get a signature over the uncompressed tarball. + ;; In that case, decompress the tarball in the store so that we + ;; can check the signature. + (data (if (string-prefix? (basename url) + (basename signature-url)) + tarball + (run-with-store store + (mlet %store-monad ((drv (uncompressed-tarball + (basename url) tarball))) + (mbegin %store-monad + (built-derivations (list drv)) + (return (derivation->output-path drv))))))) + + (ret (gnupg-verify* sig data #:key-download key-download))) (if ret tarball (begin @@ -179,19 +234,23 @@ values: the item from LST1 and the item from LST2 that match PRED." PACKAGE, or #f and #f when PACKAGE is up-to-date. KEY-DOWNLOAD specifies a download policy for missing OpenPGP keys; allowed values: 'always', 'never', and 'interactive' (default)." - (match (package-update-path package updaters) + (match (package-latest-release* package updaters) (($ <upstream-source> _ version urls signature-urls) (let*-values (((name) (package-name package)) ((archive-type) (match (and=> (package-source package) origin-uri) ((? string? uri) - (or (file-extension uri) "gz")) + (file-extension (basename uri))) (_ "gz"))) ((url signature-url) (find2 (lambda (url sig-url) - (string-suffix? archive-type url)) + ;; Some URIs lack a file extension, like + ;; 'https://crates.io/???/0.1/download'. In that + ;; case, pick the first URL. + (or (not archive-type) + (string-suffix? archive-type url))) urls (or signature-urls (circular-list #f))))) (let ((tarball (download-tarball store url signature-url diff --git a/guix/utils.scm b/guix/utils.scm index 65a2baa0a2..06f49daca8 100644 --- a/guix/utils.scm +++ b/guix/utils.scm @@ -70,6 +70,7 @@ %current-system %current-target-system package-name->name+version + target-mingw? version-compare version>? version>=? @@ -508,6 +509,10 @@ returned. Both parts must not contain any '@'." (idx (values (substring spec 0 idx) (substring spec (1+ idx)))))) +(define* (target-mingw? #:optional (target (%current-target-system))) + (and target + (string-suffix? "-mingw32" target))) + (define version-compare (let ((strverscmp (let ((sym (or (dynamic-func "strverscmp" (dynamic-link)) |