diff options
Diffstat (limited to 'guix/scripts')
| -rw-r--r-- | guix/scripts/build.scm | 16 | ||||
| -rw-r--r-- | guix/scripts/environment.scm | 2 | ||||
| -rw-r--r-- | guix/scripts/gc.scm | 10 | ||||
| -rw-r--r-- | guix/scripts/graph.scm | 38 | ||||
| -rw-r--r-- | guix/scripts/pack.scm | 73 | ||||
| -rw-r--r-- | guix/scripts/package.scm | 17 | ||||
| -rw-r--r-- | guix/scripts/system.scm | 12 | ||||
| -rw-r--r-- | guix/scripts/weather.scm | 109 |
8 files changed, 241 insertions, 36 deletions
diff --git a/guix/scripts/build.scm b/guix/scripts/build.scm index 57f2d82c5c..401087e830 100644 --- a/guix/scripts/build.scm +++ b/guix/scripts/build.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2013 Mark H Weaver <mhw@netris.org> ;;; ;;; This file is part of GNU Guix. @@ -69,13 +69,21 @@ found. Return #f if no build log was found." (define (valid-url? url) ;; Probe URL and return #t if it is accessible. - (catch 'getaddrinfo-error + (catch #t (lambda () (guard (c ((http-get-error? c) #f)) (close-port (http-fetch url #:buffered? #f)) #t)) - (lambda _ - #f))) + (match-lambda* + (('getaddrinfo-error . _) + #f) + (('tls-certificate-error args ...) + (report-error (G_ "cannot access build log at '~a':~%") url) + (print-exception (current-error-port) #f + 'tls-certificate-error args) + (exit 1)) + ((key . args) + (apply throw key args))))) (define (find-url file) (let ((base (basename file))) diff --git a/guix/scripts/environment.scm b/guix/scripts/environment.scm index 4f88c513c0..f8a9702b30 100644 --- a/guix/scripts/environment.scm +++ b/guix/scripts/environment.scm @@ -332,7 +332,7 @@ packages." (let ((module (make-user-module '()))) (packages->outputs (load* file module) mode))) (('manifest . file) - (let ((module (make-user-module '()))) + (let ((module (make-user-module '((guix profiles) (gnu))))) (manifest->outputs (load* file module)))) (_ '(#f))) opts))) diff --git a/guix/scripts/gc.scm b/guix/scripts/gc.scm index a31d2236b0..e4ed7227ff 100644 --- a/guix/scripts/gc.scm +++ b/guix/scripts/gc.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -61,6 +61,8 @@ Invoke the garbage collector.\n")) -R, --requisites list the requisites of PATHS")) (display (G_ " --referrers list the referrers of PATHS")) + (display (G_ " + --derivers list the derivers of PATHS")) (newline) (display (G_ " --verify[=OPTS] verify the integrity of the store; OPTS is a @@ -153,6 +155,10 @@ Invoke the garbage collector.\n")) (lambda (opt name arg result) (alist-cons 'action 'list-referrers (alist-delete 'action result)))) + (option '("derivers") #f #f + (lambda (opt name arg result) + (alist-cons 'action 'list-derivers + (alist-delete 'action result)))) (option '("list-failures") #f #f (lambda (opt name arg result) (alist-cons 'action 'list-failures @@ -241,6 +247,8 @@ Invoke the garbage collector.\n")) (requisites store (list item))))) ((list-referrers) (list-relatives referrers)) + ((list-derivers) + (list-relatives valid-derivers)) ((optimize) (assert-no-extra-arguments) (optimize-store store)) diff --git a/guix/scripts/graph.scm b/guix/scripts/graph.scm index 78f09f181b..346ca4ea88 100644 --- a/guix/scripts/graph.scm +++ b/guix/scripts/graph.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -27,9 +27,11 @@ #:use-module (guix gexp) #:use-module (guix derivations) #:use-module (guix memoization) + #:use-module (guix modules) #:use-module ((guix build-system gnu) #:select (standard-packages)) #:use-module (gnu packages) #:use-module (guix sets) + #:use-module ((guix utils) #:select (location-file)) #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) @@ -44,6 +46,7 @@ %derivation-node-type %reference-node-type %referrer-node-type + %module-node-type %node-types guix-graph)) @@ -332,6 +335,36 @@ substitutes." ;;; +;;; Scheme modules. +;;; + +(define (module-from-package package) + (file-name->module-name (location-file (package-location package)))) + +(define (source-module-dependencies* module) + "Like 'source-module-dependencies' but filter out modules that are not +package modules, while attempting to retain user package modules." + (remove (match-lambda + (('guix _ ...) #t) + (('system _ ...) #t) + (('language _ ...) #t) + (('ice-9 _ ...) #t) + (('srfi _ ...) #t) + (_ #f)) + (source-module-dependencies module))) + +(define %module-node-type + ;; Show the graph of package modules. + (node-type + (name "module") + (description "the graph of package modules") + (convert (lift1 (compose list module-from-package) %store-monad)) + (identifier (lift1 identity %store-monad)) + (label object->string) + (edges (lift1 source-module-dependencies* %store-monad)))) + + +;;; ;;; List of node types. ;;; @@ -344,7 +377,8 @@ substitutes." %bag-emerged-node-type %derivation-node-type %reference-node-type - %referrer-node-type)) + %referrer-node-type + %module-node-type)) (define (lookup-node-type name) "Return the node type called NAME. Raise an error if it is not found." diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index 59dd117edb..488638adc5 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -3,6 +3,7 @@ ;;; Copyright © 2017 Efraim Flashner <efraim@flashner.co.il> ;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2018 Konrad Hinsen <konrad.hinsen@fastmail.net> +;;; Copyright © 2018 Chris Marusich <cmmarusich@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -33,7 +34,9 @@ #:use-module (guix derivations) #:use-module (guix scripts build) #:use-module (gnu packages) + #:use-module (gnu packages bootstrap) #:use-module (gnu packages compression) + #:use-module (gnu packages guile) #:autoload (gnu packages base) (tar) #:autoload (gnu packages package-management) (guix) #:autoload (gnu packages gnupg) (libgcrypt) @@ -67,6 +70,11 @@ #~(#+(file-append bzip2 "/bin/bzip2") "-9")) (compressor "none" "" #f))) +;; This one is only for use in this module, so don't put it in %compressors. +(define bootstrap-xz + (compressor "bootstrap-xz" ".xz" + #~(#+(file-append %bootstrap-coreutils&co "/bin/xz") "-e -T0"))) + (define (lookup-compressor name) "Return the compressor object called NAME. Error out if it could not be found." @@ -230,6 +238,7 @@ the image." (define build (with-imported-modules `(,@(source-module-closure '((guix docker)) #:select? not-config?) + (guix build store-copy) ((guix config) => ,config)) #~(begin ;; Guile-JSON is required by (guix docker). @@ -237,13 +246,15 @@ the image." (string-append #+json "/share/guile/site/" (effective-version))) - (use-modules (guix docker) (srfi srfi-19)) + (use-modules (guix docker) (srfi srfi-19) (guix build store-copy)) (setenv "PATH" (string-append #$tar "/bin")) - (build-docker-image #$output #$profile + (build-docker-image #$output + (call-with-input-file "profile" + read-reference-graph) + #$profile #:system (or #$target (utsname:machine (uname))) - #:closure "profile" #:symlinks '#$symlinks #:compressor '#$(compressor-command compressor) #:creation-time (make-time time-utc 0 1))))) @@ -325,6 +336,9 @@ the image." (option '("localstatedir") #f #f (lambda (opt name arg result) (alist-cons 'localstatedir? #t result))) + (option '("bootstrap") #f #f + (lambda (opt name arg result) + (alist-cons 'bootstrap? #t result))) (append %transformation-options %standard-build-options))) @@ -352,6 +366,8 @@ Create a bundle of PACKAGE.\n")) -m, --manifest=FILE create a pack with the manifest from FILE")) (display (G_ " --localstatedir include /var/guix in the resulting pack")) + (display (G_ " + --bootstrap use the bootstrap binaries to build the pack")) (newline) (display (G_ " -h, --help display this help and exit")) @@ -393,28 +409,43 @@ Create a bundle of PACKAGE.\n")) (else (packages->manifest packages))))) (with-error-handling - (parameterize ((%graft? (assoc-ref opts 'graft?))) - (let* ((dry-run? (assoc-ref opts 'dry-run?)) - (manifest (manifest-from-args opts)) - (pack-format (assoc-ref opts 'format)) - (name (string-append (symbol->string pack-format) - "-pack")) - (target (assoc-ref opts 'target)) - (compressor (assoc-ref opts 'compressor)) - (symlinks (assoc-ref opts 'symlinks)) - (build-image (match (assq-ref %formats pack-format) - ((? procedure? proc) proc) - (#f - (leave (G_ "~a: unknown pack format") - format)))) - (localstatedir? (assoc-ref opts 'localstatedir?))) - (with-store store + (let* ((dry-run? (assoc-ref opts 'dry-run?)) + (manifest (manifest-from-args opts)) + (pack-format (assoc-ref opts 'format)) + (name (string-append (symbol->string pack-format) + "-pack")) + (target (assoc-ref opts 'target)) + (bootstrap? (assoc-ref opts 'bootstrap?)) + (compressor (if bootstrap? + bootstrap-xz + (assoc-ref opts 'compressor))) + (tar (if bootstrap? + %bootstrap-coreutils&co + tar)) + (symlinks (assoc-ref opts 'symlinks)) + (build-image (match (assq-ref %formats pack-format) + ((? procedure? proc) proc) + (#f + (leave (G_ "~a: unknown pack format") + format)))) + (localstatedir? (assoc-ref opts 'localstatedir?))) + (with-store store + (parameterize ((%graft? (assoc-ref opts 'graft?)) + (%guile-for-build (package-derivation + store + (if (assoc-ref opts 'bootstrap?) + %bootstrap-guile + (canonical-package guile-2.2))))) ;; Set the build options before we do anything else. (set-build-options-from-command-line store opts) (run-with-store store (mlet* %store-monad ((profile (profile-derivation manifest + #:hooks (if bootstrap? + '() + %default-profile-hooks) + #:locales? (not bootstrap?) #:target target)) (drv (build-image name profile #:target @@ -424,7 +455,9 @@ Create a bundle of PACKAGE.\n")) #:symlinks symlinks #:localstatedir? - localstatedir?))) + localstatedir? + #:tar + tar))) (mbegin %store-monad (show-what-to-build* (list drv) #:use-substitutes? diff --git a/guix/scripts/package.scm b/guix/scripts/package.scm index d8b80efe8e..4f519e6f33 100644 --- a/guix/scripts/package.scm +++ b/guix/scripts/package.scm @@ -194,15 +194,18 @@ denote ranges as interpreted by 'matching-generations'." (define* (build-and-use-profile store profile manifest #:key + allow-collisions? bootstrap? use-substitutes? dry-run?) "Build a new generation of PROFILE, a file name, using the packages -specified in MANIFEST, a manifest object." +specified in MANIFEST, a manifest object. When ALLOW-COLLISIONS? is true, +do not treat collisions in MANIFEST as an error." (when (equal? profile %current-profile) (ensure-default-profile)) (let* ((prof-drv (run-with-store store (profile-derivation manifest + #:allow-collisions? allow-collisions? #:hooks (if bootstrap? '() %default-profile-hooks) @@ -408,6 +411,8 @@ Install, remove, or upgrade packages in a single transaction.\n")) -p, --profile=PROFILE use PROFILE instead of the user's default profile")) (newline) (display (G_ " + --allow-collisions do not treat collisions in the profile as an error")) + (display (G_ " --bootstrap use the bootstrap Guile to build the profile")) (display (G_ " --verbose produce verbose output")) @@ -544,6 +549,10 @@ kind of search path~%") (lambda (opt name arg result arg-handler) (values (alist-cons 'verbose? #t result) #f))) + (option '("allow-collisions") #f #f + (lambda (opt name arg result arg-handler) + (values (alist-cons 'allow-collisions? #t result) + #f))) (option '(#\s "search") #t #f (lambda (opt name arg result arg-handler) (values (cons `(query search ,(or arg "")) @@ -831,13 +840,15 @@ processed, #f otherwise." (let* ((user-module (make-user-module '((guix profiles) (gnu)))) (manifest (load* file user-module)) (bootstrap? (assoc-ref opts 'bootstrap?)) - (substitutes? (assoc-ref opts 'substitutes?))) + (substitutes? (assoc-ref opts 'substitutes?)) + (allow-collisions? (assoc-ref opts 'allow-collisions?))) (if dry-run? (format #t (G_ "would install new manifest from '~a' with ~d entries~%") file (length (manifest-entries manifest))) (format #t (G_ "installing new manifest from '~a' with ~d entries~%") file (length (manifest-entries manifest)))) (build-and-use-profile store profile manifest + #:allow-collisions? allow-collisions? #:bootstrap? bootstrap? #:use-substitutes? substitutes? #:dry-run? dry-run?))) @@ -856,6 +867,7 @@ processed, #f otherwise." (define dry-run? (assoc-ref opts 'dry-run?)) (define bootstrap? (assoc-ref opts 'bootstrap?)) (define substitutes? (assoc-ref opts 'substitutes?)) + (define allow-collisions? (assoc-ref opts 'allow-collisions?)) (define profile (or (assoc-ref opts 'profile) %current-profile)) (define transform (options->transformation opts)) @@ -894,6 +906,7 @@ processed, #f otherwise." (show-manifest-transaction store manifest step3 #:dry-run? dry-run?) (build-and-use-profile store profile new + #:allow-collisions? allow-collisions? #:bootstrap? bootstrap? #:use-substitutes? substitutes? #:dry-run? dry-run?)))) diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index f0c4a2ba1b..b50cabcd1a 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -1,7 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2016 Alex Kost <alezost@gmail.com> -;;; Copyright © 2016, 2017 Chris Marusich <cmmarusich@gmail.com> +;;; Copyright © 2016, 2017, 2018 Chris Marusich <cmmarusich@gmail.com> ;;; Copyright © 2017 Mathieu Othacehe <m.othacehe@gmail.com> ;;; ;;; This file is part of GNU Guix. @@ -701,7 +701,9 @@ checking this by themselves in their 'check' procedure." ("iso9660" "image.iso") (_ "disk-image")) #:disk-image-size image-size - #:file-system-type file-system-type)))) + #:file-system-type file-system-type)) + ((docker-image) + (system-docker-image os #:register-closures? #t)))) (define (maybe-suggest-running-guix-pull) "Suggest running 'guix pull' if this has never been done before." @@ -905,6 +907,8 @@ Some ACTIONS support additional ARGS.\n")) (display (G_ "\ disk-image build a disk image, suitable for a USB stick\n")) (display (G_ "\ + docker-image build a Docker image\n")) + (display (G_ "\ init initialize a root file system to run GNU\n")) (display (G_ "\ extension-graph emit the service extension graph in Dot format\n")) @@ -1142,7 +1146,7 @@ argument list and OPTS is the option alist." (case action ((build container vm vm-image disk-image reconfigure init extension-graph shepherd-graph list-generations roll-back - switch-generation search) + switch-generation search docker-image) (alist-cons 'action action result)) (else (leave (G_ "~a: unknown action~%") action)))))) @@ -1171,7 +1175,7 @@ argument list and OPTS is the option alist." (exit 1)) (case action - ((build container vm vm-image disk-image reconfigure) + ((build container vm vm-image disk-image docker-image reconfigure) (unless (or (= count 1) (and expr (= count 0))) (fail))) diff --git a/guix/scripts/weather.scm b/guix/scripts/weather.scm index 2e782e36ce..5c934abaef 100644 --- a/guix/scripts/weather.scm +++ b/guix/scripts/weather.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> ;;; ;;; This file is part of GNU Guix. @@ -29,11 +29,14 @@ #:use-module (guix grafts) #:use-module ((guix build syscalls) #:select (terminal-columns)) #:use-module (guix scripts substitute) + #:use-module (guix http-client) + #:use-module (guix ci) #:use-module (gnu packages) #:use-module (web uri) #:use-module (srfi srfi-1) #:use-module (srfi srfi-19) #:use-module (srfi srfi-26) + #:use-module (srfi srfi-34) #:use-module (srfi srfi-37) #:use-module (ice-9 match) #:use-module (ice-9 format) @@ -100,6 +103,57 @@ values." (define-syntax-rule (let/time ((time result exp)) body ...) (call-with-time (lambda () exp) (lambda (time result) body ...))) +(define (histogram field proc seed lst) + "Return an alist giving a histogram of all the values of FIELD for elements +of LST. FIELD must be a one element procedure that returns a field's value. +For each FIELD value, call PROC with the previous field-specific result. +Example: + + (histogram car (lambda (x n) (+ 1 n)) 0 '((a . x)(b . y)(a . z))) + => ((a . 2) (b . 1)) + +meaning that we have two a's and one b." + (let loop ((lst lst) + (result '())) + (match lst + (() + result) + ((head . tail) + (let ((value (field head))) + (loop tail + (match (assoc-ref result value) + (#f + `((,value . ,(proc head seed)) ,@result)) + (previous + `((,value . ,(proc head previous)) + ,@(alist-delete value result)))))))))) + +(define (throughput lst timestamp) + "Return the throughput, in items per second, given the elements of LST, +calling TIMESTAMP to get the \"timestamp\" of each item." + (let ((oldest (reduce min +inf.0 (map build-timestamp lst))) + (now (time-second (current-time time-utc)))) + (/ (length lst) (- now oldest) 1.))) + +(define (queued-subset queue items) + "Return the subset of ITEMS, a list of store file names, that appears in +QUEUE, a list of builds. Return #f if elements in QUEUE lack information +about the derivations queued, as is the case with Hydra." + (define queued + (append-map (lambda (build) + (match (false-if-exception + (read-derivation-from-file (build-derivation build))) + (#f + '()) + (drv + (match (derivation->output-paths drv) + (((names . items) ...) items))))) + queue)) + + (if (any (negate build-derivation) queue) + #f ;no derivation information + (lset-intersection string=? queued items))) + (define (report-server-coverage server items) "Report the subset of ITEMS available as substitutes on SERVER." (define MiB (* (expt 2 20) 1.)) @@ -111,6 +165,8 @@ values." (format #t "~a~%" server) (let ((obtained (length narinfos)) (requested (length items)) + (missing (lset-difference string=? + items (map narinfo-path narinfos))) (sizes (filter-map narinfo-file-size narinfos)) (time (+ (time-second time) (/ (time-nanosecond time) 1e9)))) @@ -131,7 +187,56 @@ values." (format #t (G_ " ~,3h seconds per request (~,1h seconds in total)~%") (/ time requested 1.) time) (format #t (G_ " ~,1h requests per second~%") - (/ requested time 1.))))) + (/ requested time 1.)) + + (guard (c ((http-get-error? c) + (if (= 404 (http-get-error-code c)) + (format (current-error-port) + (G_ " (continuous integration information \ +unavailable)~%")) + (format (current-error-port) + (G_ " '~a' returned ~a (~s)~%") + (uri->string (http-get-error-uri c)) + (http-get-error-code c) + (http-get-error-reason c))))) + (let* ((max %query-limit) + (queue (queued-builds server max)) + (len (length queue)) + (histo (histogram build-system + (lambda (build count) + (+ 1 count)) + 0 queue))) + (newline) + (unless (null? missing) + (let ((missing (length missing))) + (match (queued-subset queue missing) + (#f #f) + ((= length queued) + (format #t (G_ " ~,1f% (~h out of ~h) of the missing items \ +are queued~%") + (* 100. (/ queued missing)) + queued missing))))) + + (if (>= len max) + (format #t (G_ " at least ~h queued builds~%") len) + (format #t (G_ " ~h queued builds~%") len)) + (for-each (match-lambda + ((system . count) + (format #t (G_ " ~a: ~a (~0,1f%)~%") + system count (* 100. (/ count len))))) + histo)) + + (let* ((latest (latest-builds server)) + (builds/sec (throughput latest build-timestamp))) + (format #t (G_ " build rate: ~1,2f builds per hour~%") + (* builds/sec 3600.)) + (for-each (match-lambda + ((system . builds) + (format #t (G_ " ~a: ~,2f builds per hour~%") + system + (* (throughput builds build-timestamp) + 3600.)))) + (histogram build-system cons '() latest))))))) ;;; |