diff options
Diffstat (limited to 'guix')
| -rw-r--r-- | guix/build-system/cargo.scm | 21 | ||||
| -rw-r--r-- | guix/build/union.scm | 40 | ||||
| -rw-r--r-- | guix/ci.scm | 78 | ||||
| -rw-r--r-- | guix/discovery.scm | 31 | ||||
| -rw-r--r-- | guix/docker.scm | 200 | ||||
| -rw-r--r-- | guix/gexp.scm | 101 | ||||
| -rw-r--r-- | guix/git.scm | 82 | ||||
| -rw-r--r-- | guix/modules.scm | 10 | ||||
| -rw-r--r-- | guix/scripts/build.scm | 16 | ||||
| -rw-r--r-- | guix/scripts/environment.scm | 2 | ||||
| -rw-r--r-- | guix/scripts/gc.scm | 10 | ||||
| -rw-r--r-- | guix/scripts/graph.scm | 38 | ||||
| -rw-r--r-- | guix/scripts/pack.scm | 73 | ||||
| -rw-r--r-- | guix/scripts/package.scm | 17 | ||||
| -rw-r--r-- | guix/scripts/system.scm | 12 | ||||
| -rw-r--r-- | guix/scripts/weather.scm | 109 | ||||
| -rw-r--r-- | guix/self.scm | 610 | ||||
| -rw-r--r-- | guix/upstream.scm | 5 |
18 files changed, 1230 insertions, 225 deletions
diff --git a/guix/build-system/cargo.scm b/guix/build-system/cargo.scm index c637fbb162..4a1eb0cfa0 100644 --- a/guix/build-system/cargo.scm +++ b/guix/build-system/cargo.scm @@ -43,17 +43,11 @@ to NAME and VERSION." (string-append crate-url name "/" version "/download")) -(define (default-cargo) - "Return the default Cargo package." +(define (default-rust) + "Return the default Rust package." ;; Lazily resolve the binding to avoid a circular dependency. (let ((rust (resolve-interface '(gnu packages rust)))) - (module-ref rust 'cargo))) - -(define (default-rustc) - "Return the default Rustc package." - ;; Lazily resolve the binding to avoid a circular dependency. - (let ((rust (resolve-interface '(gnu packages rust)))) - (module-ref rust 'rustc))) + (module-ref rust 'rust))) (define %cargo-build-system-modules ;; Build-side modules imported by default. @@ -115,14 +109,13 @@ to NAME and VERSION." (define* (lower name #:key source inputs native-inputs outputs system target - (cargo (default-cargo)) - (rustc (default-rustc)) + (rust (default-rust)) #:allow-other-keys #:rest arguments) "Return a bag for NAME." (define private-keywords - '(#:source #:target #:cargo #:rustc #:inputs #:native-inputs #:outputs)) + '(#:source #:target #:rust #:inputs #:native-inputs #:outputs)) (and (not target) ;; TODO: support cross-compilation (bag @@ -136,8 +129,8 @@ to NAME and VERSION." ;; Keep the standard inputs of 'gnu-build-system' ,@(standard-packages))) - (build-inputs `(("cargo" ,cargo) - ("rustc" ,rustc) + (build-inputs `(("cargo" ,rust "cargo") + ("rustc" ,rust) ,@native-inputs)) (outputs outputs) (build cargo-build) diff --git a/guix/build/union.scm b/guix/build/union.scm index d46b750035..1179f1234b 100644 --- a/guix/build/union.scm +++ b/guix/build/union.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2016, 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2014 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2017 Huang Ying <huang.ying.caritas@gmail.com> ;;; @@ -25,7 +25,9 @@ #:use-module (srfi srfi-26) #:use-module (rnrs bytevectors) #:use-module (rnrs io ports) - #:export (union-build)) + #:export (union-build + + warn-about-collision)) ;;; Commentary: ;;; @@ -76,14 +78,29 @@ identical, #f otherwise." (or (eof-object? n1) (loop)))))))))))))) +(define (warn-about-collision files) + "Handle the collision among FILES by emitting a warning and choosing the +first one of THEM." + (format (current-error-port) + "~%warning: collision encountered:~%~{ ~a~%~}" + files) + (let ((file (first files))) + (format (current-error-port) "warning: choosing ~a~%" file) + file)) + (define* (union-build output inputs #:key (log-port (current-error-port)) (create-all-directories? #f) - (symlink symlink)) + (symlink symlink) + (resolve-collision warn-about-collision)) "Build in the OUTPUT directory a symlink tree that is the union of all the INPUTS, using SYMLINK to create symlinks. As a special case, if CREATE-ALL-DIRECTORIES?, creates the subdirectories in the output directory to -make sure the caller can modify them later." +make sure the caller can modify them later. + +When two or more regular files collide, call RESOLVE-COLLISION with the list +of colliding files and use the one that it returns; or, if RESOLVE-COLLISION +returns #f, skip the faulty file altogether." (define (symlink* input output) (format log-port "`~a' ~~> `~a'~%" input output) @@ -92,17 +109,10 @@ make sure the caller can modify them later." (define (resolve-collisions output dirs files) (cond ((null? dirs) ;; The inputs are all files. - (format (current-error-port) - "~%warning: collision encountered:~%~{~a~%~}" - files) - - (let ((file (first files))) - ;; TODO: Implement smarter strategies. - (format (current-error-port) - "warning: arbitrarily choosing ~a~%" - file) - - (symlink* file output))) + (match (resolve-collision files) + (#f #f) + ((? string? file) + (symlink* file output)))) (else ;; The inputs are a mixture of files and directories diff --git a/guix/ci.scm b/guix/ci.scm new file mode 100644 index 0000000000..881f3d3927 --- /dev/null +++ b/guix/ci.scm @@ -0,0 +1,78 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2018 Ludovic Courtès <ludo@gnu.org> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix ci) + #:use-module (guix http-client) + #:autoload (json parser) (json->scm) + #:use-module (srfi srfi-9) + #:export (build? + build-id + build-derivation + build-system + build-status + build-timestamp + + %query-limit + queued-builds + latest-builds)) + +;;; Commentary: +;;; +;;; This module provides a client to the HTTP interface of the Hydra and +;;; Cuirass continuous integration (CI) tools. +;;; +;;; Code: + +(define-record-type <build> + (make-build id derivation system status timestamp) + build? + (id build-id) ;integer + (derivation build-derivation) ;string | #f + (system build-system) ;string + (status build-status) ;integer + (timestamp build-timestamp)) ;integer + +(define %query-limit + ;; Max number of builds requested in queries. + 1000) + +(define (json-fetch url) + (let* ((port (http-fetch url)) + (json (json->scm port))) + (close-port port) + json)) + +(define (json->build json) + (make-build (hash-ref json "id") + (hash-ref json "derivation") + (hash-ref json "system") + (hash-ref json "buildstatus") + (hash-ref json "timestamp"))) + +(define* (queued-builds url #:optional (limit %query-limit)) + "Return the list of queued derivations on URL." + (let ((queue (json-fetch (string-append url "/api/queue?nr=" + (number->string limit))))) + (map json->build queue))) + +(define* (latest-builds url #:optional (limit %query-limit)) + (let ((latest (json-fetch (string-append url "/api/latestbuilds?nr=" + (number->string limit))))) + ;; Note: Hydra does not provide a "derivation" field for entries in + ;; 'latestbuilds', but Cuirass does. + (map json->build latest))) diff --git a/guix/discovery.scm b/guix/discovery.scm index 7b57579023..2b627d108e 100644 --- a/guix/discovery.scm +++ b/guix/discovery.scm @@ -17,7 +17,7 @@ ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. (define-module (guix discovery) - #:use-module (guix ui) + #:use-module (guix i18n) #:use-module (guix modules) #:use-module (guix combinators) #:use-module (guix build syscalls) @@ -25,7 +25,8 @@ #:use-module (ice-9 match) #:use-module (ice-9 vlist) #:use-module (ice-9 ftw) - #:export (scheme-modules + #:export (scheme-files + scheme-modules fold-modules all-modules fold-module-public-variables)) @@ -85,13 +86,18 @@ DIRECTORY is not accessible." (lambda args (let ((errno (system-error-errno args))) (unless (= errno ENOENT) - (warning (G_ "cannot access `~a': ~a~%") - directory (strerror errno))) + (format (current-error-port) ;XXX + (G_ "cannot access `~a': ~a~%") + directory (strerror errno))) '()))))) -(define* (scheme-modules directory #:optional sub-directory) +(define* (scheme-modules directory #:optional sub-directory + #:key (warn (const #f))) "Return the list of Scheme modules available under DIRECTORY. -Optionally, narrow the search to SUB-DIRECTORY." +Optionally, narrow the search to SUB-DIRECTORY. + +WARN is called when a module could not be loaded. It is passed the module +name and the exception key and arguments." (define prefix-len (string-length directory)) @@ -103,31 +109,32 @@ Optionally, narrow the search to SUB-DIRECTORY." (resolve-interface module)) (lambda args ;; Report the error, but keep going. - (warn-about-load-error module args) + (warn module args) #f)))) (scheme-files (if sub-directory (string-append directory "/" sub-directory) directory)))) -(define (fold-modules proc init path) +(define* (fold-modules proc init path #:key (warn (const #f))) "Fold over all the Scheme modules present in PATH, a list of directories. Call (PROC MODULE RESULT) for each module that is found." (fold (lambda (spec result) (match spec ((? string? directory) - (fold proc result (scheme-modules directory))) + (fold proc result (scheme-modules directory #:warn warn))) ((directory . sub-directory) (fold proc result - (scheme-modules directory sub-directory))))) + (scheme-modules directory sub-directory + #:warn warn))))) '() path)) -(define (all-modules path) +(define* (all-modules path #:key (warn (const #f))) "Return the list of package modules found in PATH, a list of directories to search. Entries in PATH can be directory names (strings) or (DIRECTORY . SUB-DIRECTORY) pairs, in which case modules are searched for beneath SUB-DIRECTORY." - (fold-modules cons '() path)) + (fold-modules cons '() path #:warn warn)) (define (fold-module-public-variables proc init modules) "Call (PROC OBJECT RESULT) for each variable exported by one of MODULES, diff --git a/guix/docker.scm b/guix/docker.scm index 060232148e..a75534c33b 100644 --- a/guix/docker.scm +++ b/guix/docker.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2018 Chris Marusich <cmmarusich@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -23,9 +24,12 @@ #:use-module ((guix build utils) #:select (mkdir-p delete-file-recursively - with-directory-excursion)) - #:use-module (guix build store-copy) + with-directory-excursion + invoke)) #:use-module (srfi srfi-19) + #:use-module (srfi srfi-26) + #:use-module ((texinfo string-utils) + #:select (escape-special-chars)) #:use-module (rnrs bytevectors) #:use-module (ice-9 match) #:export (build-docker-image)) @@ -33,8 +37,7 @@ ;; Load Guile-JSON at run time to simplify the job of 'imported-modules' & co. (module-use! (current-module) (resolve-interface '(json))) -;; Generate a 256-bit identifier in hexadecimal encoding for the Docker image -;; containing the closure at PATH. +;; Generate a 256-bit identifier in hexadecimal encoding for the Docker image. (define docker-id (compose bytevector->base16-string sha256 string->utf8)) @@ -102,82 +105,123 @@ return \"a\"." ((first rest ...) first))) -(define* (build-docker-image image path - #:key closure compressor +(define* (build-docker-image image paths prefix + #:key (symlinks '()) + (transformations '()) (system (utsname:machine (uname))) + compressor (creation-time (current-time time-utc))) - "Write to IMAGE a Docker image archive from the given store PATH. The image -contains the closure of PATH, as specified in CLOSURE (a file produced by -#:references-graphs). SYMLINKS must be a list of (SOURCE -> TARGET) tuples -describing symlinks to be created in the image, where each TARGET is relative -to PATH. SYSTEM is a GNU triplet (or prefix thereof) of the system the -binaries at PATH are for; it is used to produce metadata in the image. - -Use COMPRESSOR, a command such as '(\"gzip\" \"-9n\"), to compress IMAGE. Use -CREATION-TIME, a SRFI-19 time-utc object, as the creation time in metadata." - (let ((directory "/tmp/docker-image") ;temporary working directory - (closure (canonicalize-path closure)) - (id (docker-id path)) - (time (date->string (time-utc->date creation-time) "~4")) - (arch (let-syntax ((cond* (syntax-rules () - ((_ (pattern clause) ...) - (cond ((string-prefix? pattern system) - clause) - ... - (else - (error "unsupported system" - system))))))) - (cond* ("x86_64" "amd64") - ("i686" "386") - ("arm" "arm") - ("mips64" "mips64le"))))) + "Write to IMAGE a Docker image archive containing the given PATHS. PREFIX +must be a store path that is a prefix of any store paths in PATHS. + +SYMLINKS must be a list of (SOURCE -> TARGET) tuples describing symlinks to be +created in the image, where each TARGET is relative to PREFIX. +TRANSFORMATIONS must be a list of (OLD -> NEW) tuples describing how to +transform the PATHS. Any path in PATHS that begins with OLD will be rewritten +in the Docker image so that it begins with NEW instead. If a path is a +non-empty directory, then its contents will be recursively added, as well. + +SYSTEM is a GNU triplet (or prefix thereof) of the system the binaries in +PATHS are for; it is used to produce metadata in the image. Use COMPRESSOR, a +command such as '(\"gzip\" \"-9n\"), to compress IMAGE. Use CREATION-TIME, a +SRFI-19 time-utc object, as the creation time in metadata." + (define (sanitize path-fragment) + (escape-special-chars + ;; GNU tar strips the leading slash off of absolute paths before applying + ;; the transformations, so we need to do the same, or else our + ;; replacements won't match any paths. + (string-trim path-fragment #\/) + ;; Escape the basic regexp special characters (see: "(sed) BRE syntax"). + ;; We also need to escape "/" because we use it as a delimiter. + "/*.^$[]\\" + #\\)) + (define transformation->replacement + (match-lambda + ((old '-> new) + ;; See "(tar) transform" for details on the expression syntax. + (string-append "s/^" (sanitize old) "/" (sanitize new) "/")))) + (define (transformations->expression transformations) + (let ((replacements (map transformation->replacement transformations))) + (string-append + ;; Avoid transforming link targets, since that would break some links + ;; (e.g., symlinks that point to an absolute store path). + "flags=rSH;" + (string-join replacements ";") + ;; Some paths might still have a leading path delimiter even after tar + ;; transforms them (e.g., "/a/b" might be transformed into "/b"), so + ;; strip any leading path delimiters that remain. + ";s,^//*,,"))) + (define transformation-options + (if (eq? '() transformations) + '() + `("--transform" ,(transformations->expression transformations)))) + (let* ((directory "/tmp/docker-image") ;temporary working directory + (id (docker-id prefix)) + (time (date->string (time-utc->date creation-time) "~4")) + (arch (let-syntax ((cond* (syntax-rules () + ((_ (pattern clause) ...) + (cond ((string-prefix? pattern system) + clause) + ... + (else + (error "unsupported system" + system))))))) + (cond* ("x86_64" "amd64") + ("i686" "386") + ("arm" "arm") + ("mips64" "mips64le"))))) ;; Make sure we start with a fresh, empty working directory. (mkdir directory) - - (and (with-directory-excursion directory - (mkdir id) - (with-directory-excursion id - (with-output-to-file "VERSION" - (lambda () (display schema-version))) - (with-output-to-file "json" - (lambda () (scm->json (image-description id time)))) - - ;; Wrap it up. - (let ((items (call-with-input-file closure - read-reference-graph))) - ;; Create SYMLINKS. - (for-each (match-lambda - ((source '-> target) - (let ((source (string-trim source #\/))) - (mkdir-p (dirname source)) - (symlink (string-append path "/" target) - source)))) - symlinks) - - (and (zero? (apply system* "tar" "-cf" "layer.tar" - (append %tar-determinism-options - items - (map symlink-source symlinks)))) - (for-each delete-file-recursively - (map (compose topmost-component symlink-source) - symlinks))))) - - (with-output-to-file "config.json" - (lambda () - (scm->json (config (string-append id "/layer.tar") - time arch)))) - (with-output-to-file "manifest.json" - (lambda () - (scm->json (manifest path id)))) - (with-output-to-file "repositories" - (lambda () - (scm->json (repositories path id))))) - - (and (zero? (apply system* "tar" "-C" directory "-cf" image - `(,@%tar-determinism-options - ,@(if compressor - (list "-I" (string-join compressor)) - '()) - "."))) - (begin (delete-file-recursively directory) #t))))) + (with-directory-excursion directory + (mkdir id) + (with-directory-excursion id + (with-output-to-file "VERSION" + (lambda () (display schema-version))) + (with-output-to-file "json" + (lambda () (scm->json (image-description id time)))) + + ;; Create SYMLINKS. + (for-each (match-lambda + ((source '-> target) + (let ((source (string-trim source #\/))) + (mkdir-p (dirname source)) + (symlink (string-append prefix "/" target) + source)))) + symlinks) + + (apply invoke "tar" "-cf" "layer.tar" + `(,@transformation-options + ,@%tar-determinism-options + ,@paths + ,@(map symlink-source symlinks))) + ;; It is possible for "/" to show up in the archive, especially when + ;; applying transformations. For example, the transformation + ;; "s,^/a,," will (perhaps surprisingly) cause GNU tar to transform + ;; the path "/a" into "/". The presence of "/" in the archive is + ;; probably benign, but it is definitely safe to remove it, so let's + ;; do that. This fails when "/" is not in the archive, so use system* + ;; instead of invoke to avoid an exception in that case. + (system* "tar" "--delete" "/" "-f" "layer.tar") + (for-each delete-file-recursively + (map (compose topmost-component symlink-source) + symlinks))) + + (with-output-to-file "config.json" + (lambda () + (scm->json (config (string-append id "/layer.tar") + time arch)))) + (with-output-to-file "manifest.json" + (lambda () + (scm->json (manifest prefix id)))) + (with-output-to-file "repositories" + (lambda () + (scm->json (repositories prefix id))))) + + (apply invoke "tar" "-cf" image "-C" directory + `(,@%tar-determinism-options + ,@(if compressor + (list "-I" (string-join compressor)) + '()) + ".")) + (delete-file-recursively directory))) diff --git a/guix/gexp.scm b/guix/gexp.scm index 8dea022e04..d26fad7e0b 100644 --- a/guix/gexp.scm +++ b/guix/gexp.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org> ;;; ;;; This file is part of GNU Guix. @@ -60,6 +60,7 @@ program-file-name program-file-gexp program-file-guile + program-file-module-path scheme-file scheme-file? @@ -380,45 +381,49 @@ This is the declarative counterpart of 'gexp->derivation'." (apply gexp->derivation name gexp options))))) (define-record-type <program-file> - (%program-file name gexp guile) + (%program-file name gexp guile path) program-file? (name program-file-name) ;string (gexp program-file-gexp) ;gexp - (guile program-file-guile)) ;package + (guile program-file-guile) ;package + (path program-file-module-path)) ;list of strings -(define* (program-file name gexp #:key (guile #f)) +(define* (program-file name gexp #:key (guile #f) (module-path %load-path)) "Return an object representing the executable store item NAME that runs -GEXP. GUILE is the Guile package used to execute that script. +GEXP. GUILE is the Guile package used to execute that script. Imported +modules of GEXP are looked up in MODULE-PATH. This is the declarative counterpart of 'gexp->script'." - (%program-file name gexp guile)) + (%program-file name gexp guile module-path)) (define-gexp-compiler (program-file-compiler (file <program-file>) system target) ;; Compile FILE by returning a derivation that builds the script. (match file - (($ <program-file> name gexp guile) + (($ <program-file> name gexp guile module-path) (gexp->script name gexp + #:module-path module-path #:guile (or guile (default-guile)))))) (define-record-type <scheme-file> - (%scheme-file name gexp) + (%scheme-file name gexp splice?) scheme-file? (name scheme-file-name) ;string - (gexp scheme-file-gexp)) ;gexp + (gexp scheme-file-gexp) ;gexp + (splice? scheme-file-splice?)) ;Boolean -(define* (scheme-file name gexp) +(define* (scheme-file name gexp #:key splice?) "Return an object representing the Scheme file NAME that contains GEXP. This is the declarative counterpart of 'gexp->file'." - (%scheme-file name gexp)) + (%scheme-file name gexp splice?)) (define-gexp-compiler (scheme-file-compiler (file <scheme-file>) system target) ;; Compile FILE by returning a derivation that builds the file. (match file - (($ <scheme-file> name gexp) - (gexp->file name gexp)))) + (($ <scheme-file> name gexp splice?) + (gexp->file name gexp #:splice? splice?)))) ;; Appending SUFFIX to BASE's output file name. (define-record-type <file-append> @@ -1116,11 +1121,14 @@ they can refer to each other." (module-ref (resolve-interface '(gnu packages guile)) 'guile-2.2)) -(define (load-path-expression modules) +(define* (load-path-expression modules #:optional (path %load-path)) "Return as a monadic value a gexp that sets '%load-path' and -'%load-compiled-path' to point to MODULES, a list of module names." - (mlet %store-monad ((modules (imported-modules modules)) - (compiled (compiled-modules modules))) +'%load-compiled-path' to point to MODULES, a list of module names. MODULES +are searched for in PATH." + (mlet %store-monad ((modules (imported-modules modules + #:module-path path)) + (compiled (compiled-modules modules + #:module-path path))) (return (gexp (eval-when (expand load eval) (set! %load-path (cons (ungexp modules) %load-path)) @@ -1129,11 +1137,13 @@ they can refer to each other." %load-compiled-path))))))) (define* (gexp->script name exp - #:key (guile (default-guile))) + #:key (guile (default-guile)) + (module-path %load-path)) "Return an executable script NAME that runs EXP using GUILE, with EXP's -imported modules in its search path." +imported modules in its search path. Look up EXP's modules in MODULE-PATH." (mlet %store-monad ((set-load-path - (load-path-expression (gexp-modules exp)))) + (load-path-expression (gexp-modules exp) + module-path))) (gexp->derivation name (gexp (call-with-output-file (ungexp output) @@ -1148,29 +1158,47 @@ imported modules in its search path." (write '(ungexp set-load-path) port) (write '(ungexp exp) port) - (chmod port #o555))))))) - -(define* (gexp->file name exp #:key (set-load-path? #t)) - "Return a derivation that builds a file NAME containing EXP. When -SET-LOAD-PATH? is true, emit code in the resulting file to set '%load-path' -and '%load-compiled-path' to honor EXP's imported modules." + (chmod port #o555)))) + #:module-path module-path))) + +(define* (gexp->file name exp #:key + (set-load-path? #t) + (module-path %load-path) + (splice? #f)) + "Return a derivation that builds a file NAME containing EXP. When SPLICE? +is true, EXP is considered to be a list of expressions that will be spliced in +the resulting file. + +When SET-LOAD-PATH? is true, emit code in the resulting file to set +'%load-path' and '%load-compiled-path' to honor EXP's imported modules. +Lookup EXP's modules in MODULE-PATH." (match (if set-load-path? (gexp-modules exp) '()) (() ;zero modules (gexp->derivation name (gexp (call-with-output-file (ungexp output) (lambda (port) - (write '(ungexp exp) port)))) + (for-each (lambda (exp) + (write exp port)) + '(ungexp (if splice? + exp + (gexp ((ungexp exp))))))))) #:local-build? #t #:substitutable? #f)) ((modules ...) - (mlet %store-monad ((set-load-path (load-path-expression modules))) + (mlet %store-monad ((set-load-path (load-path-expression modules + module-path))) (gexp->derivation name (gexp (call-with-output-file (ungexp output) (lambda (port) (write '(ungexp set-load-path) port) - (write '(ungexp exp) port)))) + (for-each (lambda (exp) + (write exp port)) + '(ungexp (if splice? + exp + (gexp ((ungexp exp))))))))) + #:module-path module-path #:local-build? #t #:substitutable? #f))))) @@ -1235,7 +1263,8 @@ This yields an 'etc' directory containing these two files." files)))))) (define* (directory-union name things - #:key (copy? #f) (quiet? #f)) + #:key (copy? #f) (quiet? #f) + (resolve-collision 'warn-about-collision)) "Return a directory that is the union of THINGS, where THINGS is a list of file-like objects denoting directories. For example: @@ -1243,6 +1272,10 @@ file-like objects denoting directories. For example: yields a directory that is the union of the 'guile' and 'emacs' packages. +Call RESOLVE-COLLISION when several files collide, passing it the list of +colliding files. RESOLVE-COLLISION must return the chosen file or #f, in +which case the colliding entry is skipped altogether. + When HARD-LINKS? is true, create hard links instead of symlinks. When QUIET? is true, the derivation will not print anything." (define symlink @@ -1266,12 +1299,16 @@ is true, the derivation will not print anything." (computed-file name (with-imported-modules '((guix build union)) (gexp (begin - (use-modules (guix build union)) + (use-modules (guix build union) + (srfi srfi-1)) ;for 'first' and 'last' + (union-build (ungexp output) '(ungexp things) #:log-port (ungexp log-port) - #:symlink (ungexp symlink))))))))) + #:symlink (ungexp symlink) + #:resolve-collision + (ungexp resolve-collision))))))))) ;;; diff --git a/guix/git.scm b/guix/git.scm index d31c35f64f..9e89cc0062 100644 --- a/guix/git.scm +++ b/guix/git.scm @@ -28,9 +28,11 @@ #:use-module (rnrs bytevectors) #:use-module (ice-9 match) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) #:export (%repository-cache-directory + update-cached-checkout latest-repository-commit)) (define %repository-cache-directory @@ -68,11 +70,6 @@ make sure no empty directory is left behind." (lambda _ (false-if-exception (rmdir directory))))) -(define (repository->head-sha1 repo) - "Return the sha1 of the HEAD commit in REPOSITORY as a string." - (let ((oid (reference-target (repository-head repo)))) - (oid->string (commit-id (commit-lookup repo oid))))) - (define (url+commit->name url sha1) "Return the string \"<REPO-NAME>-<SHA1:7>\" where REPO-NAME is the name of the git repository, extracted from URL and SHA1:7 the seven first digits @@ -82,21 +79,9 @@ of SHA1 string." (last (string-split url #\/)) ".git" "") "-" (string-take sha1 7))) -(define* (copy-to-store store cache-directory #:key url repository) - "Copy CACHE-DIRECTORY recursively to STORE. URL and REPOSITORY are used to -create the store directory name." - (define (dot-git? file stat) - (and (string=? (basename file) ".git") - (eq? 'directory (stat:type stat)))) - - (let* ((commit (repository->head-sha1 repository)) - (name (url+commit->name url commit))) - (values (add-to-store store name #t "sha256" cache-directory - #:select? (negate dot-git?)) - commit))) - (define (switch-to-ref repository ref) - "Switch to REPOSITORY's branch, commit or tag specified by REF." + "Switch to REPOSITORY's branch, commit or tag specified by REF. Return the +OID (roughly the commit hash) corresponding to REF." (define obj (match ref (('branch . branch) @@ -122,7 +107,38 @@ create the store directory name." (string-append "refs/tags/" tag)))) (object-lookup repository oid))))) - (reset repository obj RESET_HARD)) + (reset repository obj RESET_HARD) + (object-id obj)) + +(define* (update-cached-checkout url + #:key + (ref '(branch . "origin/master")) + (cache-directory + (%repository-cache-directory))) + "Update the cached checkout of URL to REF in CACHE-DIRECTORY. Return two +values: the cache directory name, and the SHA1 commit (a string) corresponding +to REF. + +REF is pair whose key is [branch | commit | tag] and value the associated +data, respectively [<branch name> | <sha1> | <tag name>]." + (with-libgit2 + (let* ((cache-dir (url-cache-directory url cache-directory)) + (cache-exists? (openable-repository? cache-dir)) + (repository (if cache-exists? + (repository-open cache-dir) + (clone* url cache-dir)))) + ;; Only fetch remote if it has not been cloned just before. + (when cache-exists? + (remote-fetch (remote-lookup repository "origin"))) + (let ((oid (switch-to-ref repository ref))) + + ;; Reclaim file descriptors and memory mappings associated with + ;; REPOSITORY as soon as possible. + (when (module-defined? (resolve-interface '(git repository)) + 'repository-close!) + (repository-close! repository)) + + (values cache-dir (oid->string oid)))))) (define* (latest-repository-commit store url #:key @@ -137,16 +153,16 @@ data, respectively [<branch name> | <sha1> | <tag name>]. Git repositories are kept in the cache directory specified by %repository-cache-directory parameter." - (with-libgit2 - (let* ((cache-dir (url-cache-directory url cache-directory)) - (cache-exists? (openable-repository? cache-dir)) - (repository (if cache-exists? - (repository-open cache-dir) - (clone* url cache-dir)))) - ;; Only fetch remote if it has not been cloned just before. - (when cache-exists? - (remote-fetch (remote-lookup repository "origin"))) - (switch-to-ref repository ref) - (copy-to-store store cache-dir - #:url url - #:repository repository)))) + (define (dot-git? file stat) + (and (string=? (basename file) ".git") + (eq? 'directory (stat:type stat)))) + + (let*-values (((checkout commit) + (update-cached-checkout url + #:ref ref + #:cache-directory cache-directory)) + ((name) + (url+commit->name url commit))) + (values (add-to-store store name #t "sha256" checkout + #:select? (negate dot-git?)) + commit))) diff --git a/guix/modules.scm b/guix/modules.scm index 6c602eda48..65928f67f2 100644 --- a/guix/modules.scm +++ b/guix/modules.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2016, 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -25,10 +25,12 @@ #:use-module (ice-9 match) #:export (missing-dependency-error? missing-dependency-module + missing-dependency-search-path file-name->module-name module-name->file-name + source-module-dependencies source-module-closure live-module-closure guix-module-name?)) @@ -46,7 +48,8 @@ ;; The error corresponding to a missing module. (define-condition-type &missing-dependency-error &error missing-dependency-error? - (module missing-dependency-module)) + (module missing-dependency-module) + (search-path missing-dependency-search-path)) (define (colon-symbol? obj) "Return true if OBJ is a symbol that starts with a colon." @@ -131,7 +134,8 @@ depends on." (module-file-dependencies file)) (#f (raise (condition (&missing-dependency-error - (module module)))))))) + (module module) + (search-path load-path)))))))) (define* (module-closure modules #:key diff --git a/guix/scripts/build.scm b/guix/scripts/build.scm index 57f2d82c5c..401087e830 100644 --- a/guix/scripts/build.scm +++ b/guix/scripts/build.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2013 Mark H Weaver <mhw@netris.org> ;;; ;;; This file is part of GNU Guix. @@ -69,13 +69,21 @@ found. Return #f if no build log was found." (define (valid-url? url) ;; Probe URL and return #t if it is accessible. - (catch 'getaddrinfo-error + (catch #t (lambda () (guard (c ((http-get-error? c) #f)) (close-port (http-fetch url #:buffered? #f)) #t)) - (lambda _ - #f))) + (match-lambda* + (('getaddrinfo-error . _) + #f) + (('tls-certificate-error args ...) + (report-error (G_ "cannot access build log at '~a':~%") url) + (print-exception (current-error-port) #f + 'tls-certificate-error args) + (exit 1)) + ((key . args) + (apply throw key args))))) (define (find-url file) (let ((base (basename file))) diff --git a/guix/scripts/environment.scm b/guix/scripts/environment.scm index 4f88c513c0..f8a9702b30 100644 --- a/guix/scripts/environment.scm +++ b/guix/scripts/environment.scm @@ -332,7 +332,7 @@ packages." (let ((module (make-user-module '()))) (packages->outputs (load* file module) mode))) (('manifest . file) - (let ((module (make-user-module '()))) + (let ((module (make-user-module '((guix profiles) (gnu))))) (manifest->outputs (load* file module)))) (_ '(#f))) opts))) diff --git a/guix/scripts/gc.scm b/guix/scripts/gc.scm index a31d2236b0..e4ed7227ff 100644 --- a/guix/scripts/gc.scm +++ b/guix/scripts/gc.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -61,6 +61,8 @@ Invoke the garbage collector.\n")) -R, --requisites list the requisites of PATHS")) (display (G_ " --referrers list the referrers of PATHS")) + (display (G_ " + --derivers list the derivers of PATHS")) (newline) (display (G_ " --verify[=OPTS] verify the integrity of the store; OPTS is a @@ -153,6 +155,10 @@ Invoke the garbage collector.\n")) (lambda (opt name arg result) (alist-cons 'action 'list-referrers (alist-delete 'action result)))) + (option '("derivers") #f #f + (lambda (opt name arg result) + (alist-cons 'action 'list-derivers + (alist-delete 'action result)))) (option '("list-failures") #f #f (lambda (opt name arg result) (alist-cons 'action 'list-failures @@ -241,6 +247,8 @@ Invoke the garbage collector.\n")) (requisites store (list item))))) ((list-referrers) (list-relatives referrers)) + ((list-derivers) + (list-relatives valid-derivers)) ((optimize) (assert-no-extra-arguments) (optimize-store store)) diff --git a/guix/scripts/graph.scm b/guix/scripts/graph.scm index 78f09f181b..346ca4ea88 100644 --- a/guix/scripts/graph.scm +++ b/guix/scripts/graph.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -27,9 +27,11 @@ #:use-module (guix gexp) #:use-module (guix derivations) #:use-module (guix memoization) + #:use-module (guix modules) #:use-module ((guix build-system gnu) #:select (standard-packages)) #:use-module (gnu packages) #:use-module (guix sets) + #:use-module ((guix utils) #:select (location-file)) #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) @@ -44,6 +46,7 @@ %derivation-node-type %reference-node-type %referrer-node-type + %module-node-type %node-types guix-graph)) @@ -332,6 +335,36 @@ substitutes." ;;; +;;; Scheme modules. +;;; + +(define (module-from-package package) + (file-name->module-name (location-file (package-location package)))) + +(define (source-module-dependencies* module) + "Like 'source-module-dependencies' but filter out modules that are not +package modules, while attempting to retain user package modules." + (remove (match-lambda + (('guix _ ...) #t) + (('system _ ...) #t) + (('language _ ...) #t) + (('ice-9 _ ...) #t) + (('srfi _ ...) #t) + (_ #f)) + (source-module-dependencies module))) + +(define %module-node-type + ;; Show the graph of package modules. + (node-type + (name "module") + (description "the graph of package modules") + (convert (lift1 (compose list module-from-package) %store-monad)) + (identifier (lift1 identity %store-monad)) + (label object->string) + (edges (lift1 source-module-dependencies* %store-monad)))) + + +;;; ;;; List of node types. ;;; @@ -344,7 +377,8 @@ substitutes." %bag-emerged-node-type %derivation-node-type %reference-node-type - %referrer-node-type)) + %referrer-node-type + %module-node-type)) (define (lookup-node-type name) "Return the node type called NAME. Raise an error if it is not found." diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index 59dd117edb..488638adc5 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -3,6 +3,7 @@ ;;; Copyright © 2017 Efraim Flashner <efraim@flashner.co.il> ;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2018 Konrad Hinsen <konrad.hinsen@fastmail.net> +;;; Copyright © 2018 Chris Marusich <cmmarusich@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -33,7 +34,9 @@ #:use-module (guix derivations) #:use-module (guix scripts build) #:use-module (gnu packages) + #:use-module (gnu packages bootstrap) #:use-module (gnu packages compression) + #:use-module (gnu packages guile) #:autoload (gnu packages base) (tar) #:autoload (gnu packages package-management) (guix) #:autoload (gnu packages gnupg) (libgcrypt) @@ -67,6 +70,11 @@ #~(#+(file-append bzip2 "/bin/bzip2") "-9")) (compressor "none" "" #f))) +;; This one is only for use in this module, so don't put it in %compressors. +(define bootstrap-xz + (compressor "bootstrap-xz" ".xz" + #~(#+(file-append %bootstrap-coreutils&co "/bin/xz") "-e -T0"))) + (define (lookup-compressor name) "Return the compressor object called NAME. Error out if it could not be found." @@ -230,6 +238,7 @@ the image." (define build (with-imported-modules `(,@(source-module-closure '((guix docker)) #:select? not-config?) + (guix build store-copy) ((guix config) => ,config)) #~(begin ;; Guile-JSON is required by (guix docker). @@ -237,13 +246,15 @@ the image." (string-append #+json "/share/guile/site/" (effective-version))) - (use-modules (guix docker) (srfi srfi-19)) + (use-modules (guix docker) (srfi srfi-19) (guix build store-copy)) (setenv "PATH" (string-append #$tar "/bin")) - (build-docker-image #$output #$profile + (build-docker-image #$output + (call-with-input-file "profile" + read-reference-graph) + #$profile #:system (or #$target (utsname:machine (uname))) - #:closure "profile" #:symlinks '#$symlinks #:compressor '#$(compressor-command compressor) #:creation-time (make-time time-utc 0 1))))) @@ -325,6 +336,9 @@ the image." (option '("localstatedir") #f #f (lambda (opt name arg result) (alist-cons 'localstatedir? #t result))) + (option '("bootstrap") #f #f + (lambda (opt name arg result) + (alist-cons 'bootstrap? #t result))) (append %transformation-options %standard-build-options))) @@ -352,6 +366,8 @@ Create a bundle of PACKAGE.\n")) -m, --manifest=FILE create a pack with the manifest from FILE")) (display (G_ " --localstatedir include /var/guix in the resulting pack")) + (display (G_ " + --bootstrap use the bootstrap binaries to build the pack")) (newline) (display (G_ " -h, --help display this help and exit")) @@ -393,28 +409,43 @@ Create a bundle of PACKAGE.\n")) (else (packages->manifest packages))))) (with-error-handling - (parameterize ((%graft? (assoc-ref opts 'graft?))) - (let* ((dry-run? (assoc-ref opts 'dry-run?)) - (manifest (manifest-from-args opts)) - (pack-format (assoc-ref opts 'format)) - (name (string-append (symbol->string pack-format) - "-pack")) - (target (assoc-ref opts 'target)) - (compressor (assoc-ref opts 'compressor)) - (symlinks (assoc-ref opts 'symlinks)) - (build-image (match (assq-ref %formats pack-format) - ((? procedure? proc) proc) - (#f - (leave (G_ "~a: unknown pack format") - format)))) - (localstatedir? (assoc-ref opts 'localstatedir?))) - (with-store store + (let* ((dry-run? (assoc-ref opts 'dry-run?)) + (manifest (manifest-from-args opts)) + (pack-format (assoc-ref opts 'format)) + (name (string-append (symbol->string pack-format) + "-pack")) + (target (assoc-ref opts 'target)) + (bootstrap? (assoc-ref opts 'bootstrap?)) + (compressor (if bootstrap? + bootstrap-xz + (assoc-ref opts 'compressor))) + (tar (if bootstrap? + %bootstrap-coreutils&co + tar)) + (symlinks (assoc-ref opts 'symlinks)) + (build-image (match (assq-ref %formats pack-format) + ((? procedure? proc) proc) + (#f + (leave (G_ "~a: unknown pack format") + format)))) + (localstatedir? (assoc-ref opts 'localstatedir?))) + (with-store store + (parameterize ((%graft? (assoc-ref opts 'graft?)) + (%guile-for-build (package-derivation + store + (if (assoc-ref opts 'bootstrap?) + %bootstrap-guile + (canonical-package guile-2.2))))) ;; Set the build options before we do anything else. (set-build-options-from-command-line store opts) (run-with-store store (mlet* %store-monad ((profile (profile-derivation manifest + #:hooks (if bootstrap? + '() + %default-profile-hooks) + #:locales? (not bootstrap?) #:target target)) (drv (build-image name profile #:target @@ -424,7 +455,9 @@ Create a bundle of PACKAGE.\n")) #:symlinks symlinks #:localstatedir? - localstatedir?))) + localstatedir? + #:tar + tar))) (mbegin %store-monad (show-what-to-build* (list drv) #:use-substitutes? diff --git a/guix/scripts/package.scm b/guix/scripts/package.scm index d8b80efe8e..4f519e6f33 100644 --- a/guix/scripts/package.scm +++ b/guix/scripts/package.scm @@ -194,15 +194,18 @@ denote ranges as interpreted by 'matching-generations'." (define* (build-and-use-profile store profile manifest #:key + allow-collisions? bootstrap? use-substitutes? dry-run?) "Build a new generation of PROFILE, a file name, using the packages -specified in MANIFEST, a manifest object." +specified in MANIFEST, a manifest object. When ALLOW-COLLISIONS? is true, +do not treat collisions in MANIFEST as an error." (when (equal? profile %current-profile) (ensure-default-profile)) (let* ((prof-drv (run-with-store store (profile-derivation manifest + #:allow-collisions? allow-collisions? #:hooks (if bootstrap? '() %default-profile-hooks) @@ -408,6 +411,8 @@ Install, remove, or upgrade packages in a single transaction.\n")) -p, --profile=PROFILE use PROFILE instead of the user's default profile")) (newline) (display (G_ " + --allow-collisions do not treat collisions in the profile as an error")) + (display (G_ " --bootstrap use the bootstrap Guile to build the profile")) (display (G_ " --verbose produce verbose output")) @@ -544,6 +549,10 @@ kind of search path~%") (lambda (opt name arg result arg-handler) (values (alist-cons 'verbose? #t result) #f))) + (option '("allow-collisions") #f #f + (lambda (opt name arg result arg-handler) + (values (alist-cons 'allow-collisions? #t result) + #f))) (option '(#\s "search") #t #f (lambda (opt name arg result arg-handler) (values (cons `(query search ,(or arg "")) @@ -831,13 +840,15 @@ processed, #f otherwise." (let* ((user-module (make-user-module '((guix profiles) (gnu)))) (manifest (load* file user-module)) (bootstrap? (assoc-ref opts 'bootstrap?)) - (substitutes? (assoc-ref opts 'substitutes?))) + (substitutes? (assoc-ref opts 'substitutes?)) + (allow-collisions? (assoc-ref opts 'allow-collisions?))) (if dry-run? (format #t (G_ "would install new manifest from '~a' with ~d entries~%") file (length (manifest-entries manifest))) (format #t (G_ "installing new manifest from '~a' with ~d entries~%") file (length (manifest-entries manifest)))) (build-and-use-profile store profile manifest + #:allow-collisions? allow-collisions? #:bootstrap? bootstrap? #:use-substitutes? substitutes? #:dry-run? dry-run?))) @@ -856,6 +867,7 @@ processed, #f otherwise." (define dry-run? (assoc-ref opts 'dry-run?)) (define bootstrap? (assoc-ref opts 'bootstrap?)) (define substitutes? (assoc-ref opts 'substitutes?)) + (define allow-collisions? (assoc-ref opts 'allow-collisions?)) (define profile (or (assoc-ref opts 'profile) %current-profile)) (define transform (options->transformation opts)) @@ -894,6 +906,7 @@ processed, #f otherwise." (show-manifest-transaction store manifest step3 #:dry-run? dry-run?) (build-and-use-profile store profile new + #:allow-collisions? allow-collisions? #:bootstrap? bootstrap? #:use-substitutes? substitutes? #:dry-run? dry-run?)))) diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index f0c4a2ba1b..b50cabcd1a 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -1,7 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2016 Alex Kost <alezost@gmail.com> -;;; Copyright © 2016, 2017 Chris Marusich <cmmarusich@gmail.com> +;;; Copyright © 2016, 2017, 2018 Chris Marusich <cmmarusich@gmail.com> ;;; Copyright © 2017 Mathieu Othacehe <m.othacehe@gmail.com> ;;; ;;; This file is part of GNU Guix. @@ -701,7 +701,9 @@ checking this by themselves in their 'check' procedure." ("iso9660" "image.iso") (_ "disk-image")) #:disk-image-size image-size - #:file-system-type file-system-type)))) + #:file-system-type file-system-type)) + ((docker-image) + (system-docker-image os #:register-closures? #t)))) (define (maybe-suggest-running-guix-pull) "Suggest running 'guix pull' if this has never been done before." @@ -905,6 +907,8 @@ Some ACTIONS support additional ARGS.\n")) (display (G_ "\ disk-image build a disk image, suitable for a USB stick\n")) (display (G_ "\ + docker-image build a Docker image\n")) + (display (G_ "\ init initialize a root file system to run GNU\n")) (display (G_ "\ extension-graph emit the service extension graph in Dot format\n")) @@ -1142,7 +1146,7 @@ argument list and OPTS is the option alist." (case action ((build container vm vm-image disk-image reconfigure init extension-graph shepherd-graph list-generations roll-back - switch-generation search) + switch-generation search docker-image) (alist-cons 'action action result)) (else (leave (G_ "~a: unknown action~%") action)))))) @@ -1171,7 +1175,7 @@ argument list and OPTS is the option alist." (exit 1)) (case action - ((build container vm vm-image disk-image reconfigure) + ((build container vm vm-image disk-image docker-image reconfigure) (unless (or (= count 1) (and expr (= count 0))) (fail))) diff --git a/guix/scripts/weather.scm b/guix/scripts/weather.scm index 2e782e36ce..5c934abaef 100644 --- a/guix/scripts/weather.scm +++ b/guix/scripts/weather.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> ;;; ;;; This file is part of GNU Guix. @@ -29,11 +29,14 @@ #:use-module (guix grafts) #:use-module ((guix build syscalls) #:select (terminal-columns)) #:use-module (guix scripts substitute) + #:use-module (guix http-client) + #:use-module (guix ci) #:use-module (gnu packages) #:use-module (web uri) #:use-module (srfi srfi-1) #:use-module (srfi srfi-19) #:use-module (srfi srfi-26) + #:use-module (srfi srfi-34) #:use-module (srfi srfi-37) #:use-module (ice-9 match) #:use-module (ice-9 format) @@ -100,6 +103,57 @@ values." (define-syntax-rule (let/time ((time result exp)) body ...) (call-with-time (lambda () exp) (lambda (time result) body ...))) +(define (histogram field proc seed lst) + "Return an alist giving a histogram of all the values of FIELD for elements +of LST. FIELD must be a one element procedure that returns a field's value. +For each FIELD value, call PROC with the previous field-specific result. +Example: + + (histogram car (lambda (x n) (+ 1 n)) 0 '((a . x)(b . y)(a . z))) + => ((a . 2) (b . 1)) + +meaning that we have two a's and one b." + (let loop ((lst lst) + (result '())) + (match lst + (() + result) + ((head . tail) + (let ((value (field head))) + (loop tail + (match (assoc-ref result value) + (#f + `((,value . ,(proc head seed)) ,@result)) + (previous + `((,value . ,(proc head previous)) + ,@(alist-delete value result)))))))))) + +(define (throughput lst timestamp) + "Return the throughput, in items per second, given the elements of LST, +calling TIMESTAMP to get the \"timestamp\" of each item." + (let ((oldest (reduce min +inf.0 (map build-timestamp lst))) + (now (time-second (current-time time-utc)))) + (/ (length lst) (- now oldest) 1.))) + +(define (queued-subset queue items) + "Return the subset of ITEMS, a list of store file names, that appears in +QUEUE, a list of builds. Return #f if elements in QUEUE lack information +about the derivations queued, as is the case with Hydra." + (define queued + (append-map (lambda (build) + (match (false-if-exception + (read-derivation-from-file (build-derivation build))) + (#f + '()) + (drv + (match (derivation->output-paths drv) + (((names . items) ...) items))))) + queue)) + + (if (any (negate build-derivation) queue) + #f ;no derivation information + (lset-intersection string=? queued items))) + (define (report-server-coverage server items) "Report the subset of ITEMS available as substitutes on SERVER." (define MiB (* (expt 2 20) 1.)) @@ -111,6 +165,8 @@ values." (format #t "~a~%" server) (let ((obtained (length narinfos)) (requested (length items)) + (missing (lset-difference string=? + items (map narinfo-path narinfos))) (sizes (filter-map narinfo-file-size narinfos)) (time (+ (time-second time) (/ (time-nanosecond time) 1e9)))) @@ -131,7 +187,56 @@ values." (format #t (G_ " ~,3h seconds per request (~,1h seconds in total)~%") (/ time requested 1.) time) (format #t (G_ " ~,1h requests per second~%") - (/ requested time 1.))))) + (/ requested time 1.)) + + (guard (c ((http-get-error? c) + (if (= 404 (http-get-error-code c)) + (format (current-error-port) + (G_ " (continuous integration information \ +unavailable)~%")) + (format (current-error-port) + (G_ " '~a' returned ~a (~s)~%") + (uri->string (http-get-error-uri c)) + (http-get-error-code c) + (http-get-error-reason c))))) + (let* ((max %query-limit) + (queue (queued-builds server max)) + (len (length queue)) + (histo (histogram build-system + (lambda (build count) + (+ 1 count)) + 0 queue))) + (newline) + (unless (null? missing) + (let ((missing (length missing))) + (match (queued-subset queue missing) + (#f #f) + ((= length queued) + (format #t (G_ " ~,1f% (~h out of ~h) of the missing items \ +are queued~%") + (* 100. (/ queued missing)) + queued missing))))) + + (if (>= len max) + (format #t (G_ " at least ~h queued builds~%") len) + (format #t (G_ " ~h queued builds~%") len)) + (for-each (match-lambda + ((system . count) + (format #t (G_ " ~a: ~a (~0,1f%)~%") + system count (* 100. (/ count len))))) + histo)) + + (let* ((latest (latest-builds server)) + (builds/sec (throughput latest build-timestamp))) + (format #t (G_ " build rate: ~1,2f builds per hour~%") + (* builds/sec 3600.)) + (for-each (match-lambda + ((system . builds) + (format #t (G_ " ~a: ~,2f builds per hour~%") + system + (* (throughput builds build-timestamp) + 3600.)))) + (histogram build-system cons '() latest))))))) ;;; diff --git a/guix/self.scm b/guix/self.scm new file mode 100644 index 0000000000..6220efb397 --- /dev/null +++ b/guix/self.scm @@ -0,0 +1,610 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2017, 2018 Ludovic Courtès <ludo@gnu.org> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix self) + #:use-module (guix config) + #:use-module (guix i18n) + #:use-module (guix modules) + #:use-module (guix gexp) + #:use-module (guix store) + #:use-module (guix monads) + #:use-module (guix discovery) + #:use-module (guix packages) + #:use-module (guix sets) + #:use-module (guix utils) + #:use-module (guix modules) + #:use-module (guix build utils) + #:use-module ((guix build compile) #:select (%lightweight-optimizations)) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-9) + #:use-module (ice-9 match) + #:export (make-config.scm + compiled-guix + guix-derivation + reload-guix)) + + +;;; +;;; Dependency handling. +;;; + +(define* (false-if-wrong-guile package + #:optional (guile-version (effective-version))) + "Return #f if PACKAGE depends on the \"wrong\" major version of Guile (e.g., +2.0 instead of 2.2), otherwise return PACKAGE." + (let ((guile (any (match-lambda + ((label (? package? dep) _ ...) + (and (string=? (package-name dep) "guile") + dep))) + (package-direct-inputs package)))) + (and (or (not guile) + (string-prefix? guile-version + (package-version guile))) + package))) + +(define (package-for-guile guile-version . names) + "Return the package with one of the given NAMES that depends on +GUILE-VERSION (\"2.0\" or \"2.2\"), or #f if none of the packages matches." + (let loop ((names names)) + (match names + (() + #f) + ((name rest ...) + (match (specification->package name) + (#f + (loop rest)) + ((? package? package) + (or (false-if-wrong-guile package guile-version) + (loop rest)))))))) + +(define specification->package + ;; Use our own variant of that procedure because that of (gnu packages) + ;; would traverse all the .scm files, which is wasteful. + (let ((ref (lambda (module variable) + (module-ref (resolve-interface module) variable)))) + (match-lambda + ("guile" (ref '(gnu packages commencement) 'guile-final)) + ("guile-json" (ref '(gnu packages guile) 'guile-json)) + ("guile-ssh" (ref '(gnu packages ssh) 'guile-ssh)) + ("guile-git" (ref '(gnu packages guile) 'guile-git)) + ("libgcrypt" (ref '(gnu packages gnupg) 'libgcrypt)) + ("zlib" (ref '(gnu packages compression) 'zlib)) + ("gzip" (ref '(gnu packages compression) 'gzip)) + ("bzip2" (ref '(gnu packages compression) 'bzip2)) + ("xz" (ref '(gnu packages compression) 'xz)) + ("guix" (ref '(gnu packages package-management) + 'guix-register)) + ("guile2.0-json" (ref '(gnu packages guile) 'guile2.0-json)) + ("guile2.0-ssh" (ref '(gnu packages ssh) 'guile2.0-ssh)) + ("guile2.0-git" (ref '(gnu packages guile) 'guile2.0-git)) + (_ #f)))) ;no such package + + +;;; +;;; Derivations. +;;; + +;; Node in a DAG of build tasks. Each node maps to a derivation, but it's +;; easier to express things this way. +(define-record-type <node> + (node name modules source dependencies compiled) + node? + (name node-name) ;string + (modules node-modules) ;list of module names + (source node-source) ;list of source files + (dependencies node-dependencies) ;list of nodes + (compiled node-compiled)) ;node -> lowerable object + +(define (node-fold proc init nodes) + (let loop ((nodes nodes) + (visited (setq)) + (result init)) + (match nodes + (() result) + ((head tail ...) + (if (set-contains? visited head) + (loop tail visited result) + (loop tail (set-insert head visited) + (proc head result))))))) + +(define (node-modules/recursive nodes) + (node-fold (lambda (node modules) + (append (node-modules node) modules)) + '() + nodes)) + +(define* (closure modules #:optional (except '())) + (source-module-closure modules + #:select? + (match-lambda + (('guix 'config) + #f) + ((and module + (or ('guix _ ...) ('gnu _ ...))) + (not (member module except))) + (rest #f)))) + +(define module->import + ;; Return a file-name/file-like object pair for the specified module and + ;; suitable for 'imported-files'. + (match-lambda + ((module '=> thing) + (let ((file (module-name->file-name module))) + (list file thing))) + (module + (let ((file (module-name->file-name module))) + (list file + (local-file (search-path %load-path file))))))) + +(define* (scheme-node name modules #:optional (dependencies '()) + #:key (extra-modules '()) (extra-files '()) + (extensions '()) + parallel? guile-for-build) + "Return a node that builds the given Scheme MODULES, and depends on +DEPENDENCIES (a list of nodes). EXTRA-MODULES is a list of additional modules +added to the source, and EXTRA-FILES is a list of additional files. +EXTENSIONS is a set of full-blown Guile packages (e.g., 'guile-json') that +must be present in the search path." + (let* ((modules (append extra-modules + (closure modules + (node-modules/recursive dependencies)))) + (module-files (map module->import modules)) + (source (imported-files (string-append name "-source") + (append module-files extra-files)))) + (node name modules source dependencies + (compiled-modules name source modules + (map node-source dependencies) + (map node-compiled dependencies) + #:extensions extensions + #:parallel? parallel? + #:guile-for-build guile-for-build)))) + +(define (file-imports directory sub-directory pred) + "List all the files matching PRED under DIRECTORY/SUB-DIRECTORY. Return a +list of file-name/file-like objects suitable as inputs to 'imported-files'." + (map (lambda (file) + (list (string-drop file (+ 1 (string-length directory))) + (local-file file #:recursive? #t))) + (find-files (string-append directory "/" sub-directory) pred))) + +(define (scheme-modules* directory sub-directory) + "Return the list of module names found under SUB-DIRECTORY in DIRECTORY." + (let ((prefix (string-length directory))) + (map (lambda (file) + (file-name->module-name (string-drop file prefix))) + (scheme-files (string-append directory "/" sub-directory))))) + +(define* (compiled-guix source #:key (version %guix-version) + (name (string-append "guix-" version)) + (guile-version (effective-version)) + (guile-for-build (guile-for-build guile-version)) + (libgcrypt (specification->package "libgcrypt")) + (zlib (specification->package "zlib")) + (gzip (specification->package "gzip")) + (bzip2 (specification->package "bzip2")) + (xz (specification->package "xz")) + (guix (specification->package "guix"))) + "Return a file-like object that contains a compiled Guix." + (define guile-json + (package-for-guile guile-version + "guile-json" + "guile2.0-json")) + + (define guile-ssh + (package-for-guile guile-version + "guile-ssh" + "guile2.0-ssh")) + + (define guile-git + (package-for-guile guile-version + "guile-git" + "guile2.0-git")) + + + (define dependencies + (match (append-map (lambda (package) + (cons (list "x" package) + (package-transitive-inputs package))) + (list guile-git guile-json guile-ssh)) + (((labels packages _ ...) ...) + packages))) + + (define *core-modules* + (scheme-node "guix-core" + '((guix) + (guix monad-repl) + (guix packages) + (guix download) + (guix discovery) + (guix profiles) + (guix build-system gnu) + (guix build-system trivial) + (guix build profiles) + (guix build gnu-build-system)) + + ;; Provide a dummy (guix config) with the default version + ;; number, storedir, etc. This is so that "guix-core" is the + ;; same across all installations and doesn't need to be + ;; rebuilt when the version changes, which in turn means we + ;; can have substitutes for it. + #:extra-modules + `(((guix config) + => ,(make-config.scm #:libgcrypt + (specification->package + "libgcrypt")))) + + #:guile-for-build guile-for-build)) + + (define *extra-modules* + (scheme-node "guix-extra" + (filter-map (match-lambda + (('guix 'scripts _ ..1) #f) + (name name)) + (scheme-modules* source "guix")) + (list *core-modules*) + #:extensions dependencies + #:guile-for-build guile-for-build)) + + (define *package-modules* + (scheme-node "guix-packages" + `((gnu packages) + ,@(scheme-modules* source "gnu/packages")) + (list *core-modules* *extra-modules*) + #:extensions dependencies + #:extra-files ;all the non-Scheme files + (file-imports source "gnu/packages" + (lambda (file stat) + (and (eq? 'regular (stat:type stat)) + (not (string-suffix? ".scm" file)) + (not (string-suffix? ".go" file)) + (not (string-prefix? ".#" file)) + (not (string-suffix? "~" file))))) + #:guile-for-build guile-for-build)) + + (define *system-modules* + (scheme-node "guix-system" + `((gnu system) + (gnu services) + ,@(scheme-modules* source "gnu/system") + ,@(scheme-modules* source "gnu/services")) + (list *package-modules* *extra-modules* *core-modules*) + #:extensions dependencies + #:extra-files + (file-imports source "gnu/system/examples" (const #t)) + #:guile-for-build + guile-for-build)) + + (define *cli-modules* + (scheme-node "guix-cli" + (scheme-modules* source "/guix/scripts") + (list *core-modules* *extra-modules* *package-modules* + *system-modules*) + #:extensions dependencies + #:guile-for-build guile-for-build)) + + (define *config* + (scheme-node "guix-config" + '() + #:extra-modules + `(((guix config) + => ,(make-config.scm #:libgcrypt libgcrypt + #:zlib zlib + #:gzip gzip + #:bzip2 bzip2 + #:xz xz + #:guix guix + #:package-name + %guix-package-name + #:package-version + version + #:bug-report-address + %guix-bug-report-address + #:home-page-url + %guix-home-page-url))) + #:guile-for-build guile-for-build)) + + (directory-union name + (append-map (lambda (node) + (list (node-source node) + (node-compiled node))) + + ;; Note: *CONFIG* comes first so that it + ;; overrides the (guix config) module that + ;; comes with *CORE-MODULES*. + (list *config* + *cli-modules* + *system-modules* + *package-modules* + *extra-modules* + *core-modules*)) + + ;; Silently choose the first entry upon collision so that + ;; we choose *CONFIG*. + #:resolve-collision 'first + + ;; When we do (add-to-store "utils.scm"), "utils.scm" must + ;; be a regular file, not a symlink. Thus, arrange so that + ;; regular files appear as regular files in the final + ;; output. + #:copy? #t + #:quiet? #t)) + + +;;; +;;; Generating (guix config). +;;; + +(define %dependency-variables + ;; (guix config) variables corresponding to dependencies. + '(%libgcrypt %libz %xz %gzip %bzip2 %nix-instantiate + %sbindir %guix-register-program)) + +(define %persona-variables + ;; (guix config) variables that define Guix's persona. + '(%guix-package-name + %guix-version + %guix-bug-report-address + %guix-home-page-url)) + +(define %config-variables + ;; (guix config) variables corresponding to Guix configuration (storedir, + ;; localstatedir, etc.) + (sort (filter pair? + (module-map (lambda (name var) + (and (not (memq name %dependency-variables)) + (not (memq name %persona-variables)) + (cons name (variable-ref var)))) + (resolve-interface '(guix config)))) + (lambda (name+value1 name+value2) + (string<? (symbol->string (car name+value1)) + (symbol->string (car name+value2)))))) + +(define* (make-config.scm #:key libgcrypt zlib gzip xz bzip2 guix + (package-name "GNU Guix") + (package-version "0") + (bug-report-address "bug-guix@gnu.org") + (home-page-url "https://gnu.org/s/guix")) + + ;; Hack so that Geiser is not confused. + (define defmod 'define-module) + + (scheme-file "config.scm" + #~(;; The following expressions get spliced. + (#$defmod (guix config) + #:export (%guix-package-name + %guix-version + %guix-bug-report-address + %guix-home-page-url + %sbindir + %guix-register-program + %libgcrypt + %libz + %gzip + %bzip2 + %xz + %nix-instantiate)) + + #$@(map (match-lambda + ((name . value) + #~(define-public #$name #$value))) + %config-variables) + + (define %guix-package-name #$package-name) + (define %guix-version #$package-version) + (define %guix-bug-report-address #$bug-report-address) + (define %guix-home-page-url #$home-page-url) + + (define %sbindir + ;; This is used to define '%guix-register-program'. + ;; TODO: Use a derivation that builds nothing but the + ;; C++ part. + #+(and guix (file-append guix "/sbin"))) + + (define %guix-register-program + (or (getenv "GUIX_REGISTER") + (and %sbindir + (string-append %sbindir "/guix-register")))) + + (define %gzip + #+(and gzip (file-append gzip "/bin/gzip"))) + (define %bzip2 + #+(and bzip2 (file-append bzip2 "/bin/bzip2"))) + (define %xz + #+(and xz (file-append xz "/bin/xz"))) + + (define %libgcrypt + #+(and libgcrypt + (file-append libgcrypt "/lib/libgcrypt"))) + (define %libz + #+(and zlib + (file-append zlib "/lib/libz"))) + + (define %nix-instantiate ;for (guix import snix) + "nix-instantiate")) + + ;; Guile 2.0 *requires* the 'define-module' to be at the + ;; top-level or it 'toplevel-ref' in the resulting .go file are + ;; made relative to a nonexistent anonymous module. + #:splice? #t)) + + + +;;; +;;; Building. +;;; + +(define (imported-files name files) + ;; This is a non-monadic, simplified version of 'imported-files' from (guix + ;; gexp). + (define build + (with-imported-modules (source-module-closure + '((guix build utils))) + #~(begin + (use-modules (ice-9 match) + (guix build utils)) + + (mkdir (ungexp output)) (chdir (ungexp output)) + (for-each (match-lambda + ((final-path store-path) + (mkdir-p (dirname final-path)) + + ;; Note: We need regular files to be regular files, not + ;; symlinks, as this makes a difference for + ;; 'add-to-store'. + (copy-file store-path final-path))) + '#$files)))) + + ;; We're just copying files around, no need to substitute or offload it. + (computed-file name build + #:options '(#:local-build? #t + #:substitutable? #f))) + +(define* (compiled-modules name module-tree modules + #:optional + (dependencies '()) + (dependencies-compiled '()) + #:key + (extensions '()) ;full-blown Guile packages + parallel? + guile-for-build) + ;; This is a non-monadic, enhanced version of 'compiled-file' from (guix + ;; gexp). + (define build + (with-imported-modules (source-module-closure + '((guix build compile) + (guix build utils))) + #~(begin + (use-modules (srfi srfi-26) + (ice-9 match) + (ice-9 format) + (ice-9 threads) + (guix build compile) + (guix build utils)) + + (define (regular? file) + (not (member file '("." "..")))) + + (define (report-load file total completed) + (display #\cr) + (format #t + "loading...\t~5,1f% of ~d files" ;FIXME: i18n + (* 100. (/ completed total)) total) + (force-output)) + + (define (report-compilation file total completed) + (display #\cr) + (format #t "compiling...\t~5,1f% of ~d files" ;FIXME: i18n + (* 100. (/ completed total)) total) + (force-output)) + + (define (process-directory directory output) + (let ((files (find-files directory "\\.scm$")) + (prefix (+ 1 (string-length directory)))) + ;; Hide compilation warnings. + (parameterize ((current-warning-port (%make-void-port "w"))) + (compile-files directory #$output + (map (cut string-drop <> prefix) files) + #:workers (parallel-job-count) + #:report-load report-load + #:report-compilation report-compilation)))) + + (setvbuf (current-output-port) _IONBF) + (setvbuf (current-error-port) _IONBF) + + (set! %load-path (cons #+module-tree %load-path)) + (set! %load-path + (append '#+dependencies + (map (lambda (extension) + (string-append extension "/share/guile/site/" + (effective-version))) + '#+extensions) + %load-path)) + + (set! %load-compiled-path + (append '#+dependencies-compiled + (map (lambda (extension) + (string-append extension "/lib/guile/" + (effective-version) + "/site-ccache")) + '#+extensions) + %load-compiled-path)) + + ;; Load the compiler modules upfront. + (compile #f) + + (mkdir #$output) + (chdir #+module-tree) + (process-directory "." #$output) + (newline)))) + + (computed-file name build + #:guile guile-for-build + #:options + `(#:local-build? #f ;allow substitutes + + ;; Don't annoy people about _IONBF deprecation. + #:env-vars (("GUILE_WARN_DEPRECATED" . "no"))))) + + +;;; +;;; Building. +;;; + +(define (guile-for-build version) + "Return a derivation for Guile 2.0 or 2.2, whichever matches the currently +running Guile." + (define canonical-package ;soft reference + (module-ref (resolve-interface '(gnu packages base)) + 'canonical-package)) + + (match version + ("2.2.2" + ;; Gross hack to avoid ABI incompatibilities (see + ;; <https://bugs.gnu.org/29570>.) + (module-ref (resolve-interface '(gnu packages guile)) + 'guile-2.2.2)) + ("2.2" + (canonical-package (module-ref (resolve-interface '(gnu packages guile)) + 'guile-2.2/fixed))) + ("2.0" + (module-ref (resolve-interface '(gnu packages guile)) + 'guile-2.0)))) + +(define* (guix-derivation source version + #:optional (guile-version (effective-version))) + "Return, as a monadic value, the derivation to build the Guix from SOURCE +for GUILE-VERSION. Use VERSION as the version string." + (define (shorten version) + (if (and (string-every char-set:hex-digit version) + (> (string-length version) 9)) + (string-take version 9) ;Git commit + version)) + + (define guile + (guile-for-build guile-version)) + + (mbegin %store-monad + (set-guile-for-build guile) + (lower-object (compiled-guix source + #:version version + #:name (string-append "guix-" + (shorten version)) + #:guile-version (match guile-version + ("2.2.2" "2.2") + (version version)) + #:guile-for-build guile)))) diff --git a/guix/upstream.scm b/guix/upstream.scm index caaa0e44e4..9e1056f7a7 100644 --- a/guix/upstream.scm +++ b/guix/upstream.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2015 Alex Kost <alezost@gmail.com> ;;; ;;; This file is part of GNU Guix. @@ -153,7 +153,8 @@ correspond to the same version." (cons (resolve-interface '(guix gnu-maintenance)) (all-modules (map (lambda (entry) `(,entry . "guix/import")) - %load-path)))) + %load-path) + #:warn warn-about-load-error))) (define %updaters ;; The list of publically-known updaters. |