From fe17037b387c6eca0c45f0526d2761e982a192bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Court=C3=A8s?= Date: Tue, 9 Oct 2018 09:53:02 +0200 Subject: status: Gracefully handle invalid UTF-8 in build logs. * guix/status.scm (maybe-utf8->string): New procedure. (build-event-output-port): Use it in lieu of 'utf8->string'. * tests/status.scm ("build-output-port, UTF-8") ("current-build-output-port, UTF-8 + garbage"): New tests. --- tests/status.scm | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/status.scm b/tests/status.scm index 04dedb702c..486ad04dd2 100644 --- a/tests/status.scm +++ b/tests/status.scm @@ -20,7 +20,9 @@ #:use-module (guix status) #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) - #:use-module (srfi srfi-64)) + #:use-module (srfi srfi-64) + #:use-module (rnrs bytevectors) + #:use-module (rnrs io ports)) (test-begin "status") @@ -112,4 +114,22 @@ (display "@ substituter-succeeded baz\n" port) (list first (get-status))))) +(test-equal "build-output-port, UTF-8" + '((build-log "lambda is λ!\n")) + (let-values (((port get-status) (build-event-output-port cons '())) + ((bv) (string->utf8 "lambda is λ!\n"))) + (put-bytevector port bv) + (force-output port) + (get-status))) + +(test-equal "current-build-output-port, UTF-8 + garbage" + ;; What about a mixture of UTF-8 + garbage? + '((build-log "garbage: �lambda: λ\n")) + (let-values (((port get-status) (build-event-output-port cons '()))) + (display "garbage: " port) + (put-bytevector port #vu8(128)) + (put-bytevector port (string->utf8 "lambda: λ\n")) + (force-output port) + (get-status))) + (test-end "status") -- cgit v1.2.3 From b33e191c86b7638517ea838b63a54d031a033554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Court=C3=A8s?= Date: Tue, 9 Oct 2018 18:52:37 +0200 Subject: guix build: '-f' accepts file-like objects. * guix/scripts/build.scm (options->things-to-build)[validate-type]: Check for 'file-like?'. (options->derivations): Accept 'file-like?'. * tests/guix-build.sh: Add a test with 'computed-file'. * doc/guix.texi (Additional Build Options): Mention file-like objects. --- tests/guix-build.sh | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'tests') diff --git a/tests/guix-build.sh b/tests/guix-build.sh index 92e7299321..7842ce87c6 100644 --- a/tests/guix-build.sh +++ b/tests/guix-build.sh @@ -221,6 +221,10 @@ guix build -e "(begin guix build -e '#~(mkdir #$output)' -d guix build -e '#~(mkdir #$output)' -d | grep 'gexp\.drv' +# Same with a file-like object. +guix build -e '(computed-file "foo" #~(mkdir #$output))' -d +guix build -e '(computed-file "foo" #~(mkdir #$output))' -d | grep 'foo\.drv' + # Building from a package file. cat > "$module_dir/package.scm"< Date: Mon, 15 Oct 2018 22:40:35 +0200 Subject: daemon: Support multiplexed build output. This allows clients to tell whether output comes from the daemon or, if it comes from a builder, from which builder it comes. The latter is particularly useful when MAX-BUILD-JOBS > 1. * nix/libstore/build.cc (DerivationGoal::tryBuildHook) (DerivationGoal::startBuilder): Print the child's PID in "@ build-started" traces. (DerivationGoal::handleChildOutput): Define 'prefix', pass it to 'writeToStderr'. * nix/libstore/globals.cc (Settings:Settings): Initialize 'multiplexedBuildOutput'. (Settings::update): Likewise. * nix/libstore/globals.hh (Settings)[multiplexedBuildOutput]: New field. Update 'printBuildTrace' documentation. * nix/libstore/worker-protocol.hh (PROTOCOL_VERSION): Bump to 0.163. * nix/nix-daemon/nix-daemon.cc (performOp) : Special-case "multiplexed-build-output" and remove "use-ssh-substituter". * guix/store.scm (set-build-options): Add #:multiplexed-build-output? and honor it. (%protocol-version): Bump to #x163. * tests/store.scm ("multiplexed-build-output"): New test. fixlet --- tests/store.scm | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) (limited to 'tests') diff --git a/tests/store.scm b/tests/store.scm index 2858369706..3ff526cdcf 100644 --- a/tests/store.scm +++ b/tests/store.scm @@ -31,6 +31,7 @@ #:use-module (gnu packages) #:use-module (gnu packages bootstrap) #:use-module (ice-9 match) + #:use-module (ice-9 regex) #:use-module (rnrs bytevectors) #:use-module (rnrs io ports) #:use-module (web uri) @@ -1021,4 +1022,66 @@ (call-with-input-file (derivation->output-path drv2) read)))))) +(test-equal "multiplexed-build-output" + '("Hello from first." "Hello from second.") + (with-store store + (let* ((build (add-text-to-store store "build.sh" + "echo Hello from $NAME.; echo > $out")) + (bash (add-to-store store "bash" #t "sha256" + (search-bootstrap-binary "bash" + (%current-system)))) + (drv1 (derivation store "one" bash + `("-e" ,build) + #:inputs `((,bash) (,build)) + #:env-vars `(("NAME" . "first") + ("x" . ,(random-text))))) + (drv2 (derivation store "two" bash + `("-e" ,build) + #:inputs `((,bash) (,build)) + #:env-vars `(("NAME" . "second") + ("x" . ,(random-text)))))) + (set-build-options store + #:print-build-trace #t + #:multiplexed-build-output? #t + #:max-build-jobs 10) + (let ((port (open-output-string))) + ;; Send the build log to PORT. + (parameterize ((current-build-output-port port)) + (build-derivations store (list drv1 drv2))) + + ;; Retrieve the build log; make sure it contains valid "@ build-log" + ;; traces that allow us to retrieve each builder's output (we assume + ;; there's exactly one "build-output" trace for each builder, which is + ;; reasonable.) + (let* ((log (get-output-string port)) + (started (fold-matches + (make-regexp "@ build-started ([^ ]+) - ([^ ]+) ([^ ]+) ([0-9]+)") + log '() cons)) + (done (fold-matches + (make-regexp "@ build-succeeded (.*) - (.*) (.*) (.*)") + log '() cons)) + (output (fold-matches + (make-regexp "@ build-log ([[:digit:]]+) ([[:digit:]]+)\n([A-Za-z .*]+)\n") + log '() cons)) + (drv-pid (lambda (name) + (lambda (m) + (let ((drv (match:substring m 1)) + (pid (string->number + (match:substring m 4)))) + (and (string-suffix? name drv) pid))))) + (pid-log (lambda (pid) + (lambda (m) + (let ((n (string->number + (match:substring m 1))) + (len (string->number + (match:substring m 2))) + (str (match:substring m 3))) + (and (= pid n) + (= (string-length str) (- len 1)) + str))))) + (pid1 (any (drv-pid "one.drv") started)) + (pid2 (any (drv-pid "two.drv") started))) + (list (any (pid-log pid1) output) + (any (pid-log pid2) output))))))) + (test-end "store") -- cgit v1.2.3 From f9a8fce10f2d99efec7cb1dd0f6c5f0df9d1b2df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Court=C3=A8s?= Date: Mon, 15 Oct 2018 23:06:55 +0200 Subject: status: Build upon multiplexed build output. This allows for more accurate status tracking and parsing of extended build traces. * guix/status.scm (multiplexed-output-supported?): New procedure. (print-build-event): Don't print \r when PRINT-LOG? is true. Adjust 'build-log' handling for when 'multiplexed-output-supported?' returns true. (bytevector-index, split-lines): New procedures. (build-event-output-port)[%build-output-pid, %build-output] [%build-output-left]: New variables. [process-line]: Handle "@ build-output" traces. [process-build-output]: New procedure. [write!]: Add case for when %BUILD-OUTPUT-PID is true. Use 'bytevector-index' rather than 'string-index'. (compute-status): Add #:derivation-path->output-path. Use it. * tests/status.scm ("compute-status, multiplexed build output"): New test. ("build-output-port, UTF-8") ("current-build-output-port, UTF-8 + garbage"): Adjust to new 'build-log' output. * guix/scripts/build.scm (set-build-options-from-command-line): Pass #:multiplexed-build-output?. (%default-options): Add 'multiplexed-build-output?'. * guix/scripts/environment.scm (%default-options): Likewise. * guix/scripts/pack.scm (%default-options): Likewise. * guix/scripts/package.scm (%default-options): Likewise. * guix/scripts/pull.scm (%default-options): Likewise. * guix/scripts/system.scm (%default-options): Likewise. --- tests/status.scm | 51 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/status.scm b/tests/status.scm index 486ad04dd2..3b74946673 100644 --- a/tests/status.scm +++ b/tests/status.scm @@ -22,7 +22,8 @@ #:use-module (srfi srfi-11) #:use-module (srfi srfi-64) #:use-module (rnrs bytevectors) - #:use-module (rnrs io ports)) + #:use-module (rnrs io ports) + #:use-module (ice-9 match)) (test-begin "status") @@ -115,7 +116,7 @@ (list first (get-status))))) (test-equal "build-output-port, UTF-8" - '((build-log "lambda is λ!\n")) + '((build-log #f "lambda is λ!\n")) (let-values (((port get-status) (build-event-output-port cons '())) ((bv) (string->utf8 "lambda is λ!\n"))) (put-bytevector port bv) @@ -124,7 +125,7 @@ (test-equal "current-build-output-port, UTF-8 + garbage" ;; What about a mixture of UTF-8 + garbage? - '((build-log "garbage: �lambda: λ\n")) + '((build-log #f "garbage: �lambda: λ\n")) (let-values (((port get-status) (build-event-output-port cons '()))) (display "garbage: " port) (put-bytevector port #vu8(128)) @@ -132,4 +133,48 @@ (force-output port) (get-status))) +(test-equal "compute-status, multiplexed build output" + (list (build-status + (building '("foo.drv")) + (downloading (list (download "bar" "http://example.org/bar" + #:size 999 + #:start 'now)))) + (build-status + (building '("foo.drv")) + (downloading (list (download "bar" "http://example.org/bar" + #:size 999 + #:transferred 42 + #:start 'now)))) + (build-status + ;; XXX: Should "bar.drv" be present twice? + (builds-completed '("bar.drv" "foo.drv")) + (downloads-completed (list (download "bar" "http://example.org/bar" + #:size 999 + #:transferred 999 + #:start 'now + #:end 'now))))) + (let-values (((port get-status) + (build-event-output-port (lambda (event status) + (compute-status event status + #:current-time + (const 'now) + #:derivation-path->output-path + (match-lambda + ("bar.drv" "bar"))))))) + (display "@ build-started foo.drv 121\n" port) + (display "@ build-started bar.drv 144\n" port) + (display "@ build-log 121 6\nHello!" port) + (display "@ build-log 144 50 +@ download-started bar http://example.org/bar 999\n" port) + (let ((first (get-status))) + (display "@ build-log 121 30\n@ build-started FAKE!.drv 555\n") + (display "@ build-log 144 54 +@ download-progress bar http://example.org/bar 999 42\n" + port) + (let ((second (get-status))) + (display "@ download-succeeded bar http://example.org/bar 999\n" port) + (display "@ build-succeeded foo.drv\n" port) + (display "@ build-succeeded bar.drv\n" port) + (list first second (get-status)))))) + (test-end "status") -- cgit v1.2.3 From 278f86a43f1561b1c064ce88da012db414ec7efc Mon Sep 17 00:00:00 2001 From: Eric Bavier Date: Wed, 10 Oct 2018 16:42:02 -0500 Subject: ui: Fix port-buffering with guile@2.0. * guix/status.scm (build-event-output-port)[guile@2.0]: Do not call 'setvbuf' on custom binary port. * tests/status.scm (current-build-output-port, UTF-8 + garbage)[guile@2.0]: Use "?" in place of REPLACEMENT CHARACTER. --- tests/status.scm | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/status.scm b/tests/status.scm index 3b74946673..99abb41c8b 100644 --- a/tests/status.scm +++ b/tests/status.scm @@ -125,7 +125,10 @@ (test-equal "current-build-output-port, UTF-8 + garbage" ;; What about a mixture of UTF-8 + garbage? - '((build-log #f "garbage: �lambda: λ\n")) + (let ((replacement (cond-expand + ((and guile-2 (not guile-2.2)) "?") + (else "�")))) + `((build-log #f ,(string-append "garbage: " replacement "lambda: λ\n")))) (let-values (((port get-status) (build-event-output-port cons '()))) (display "garbage: " port) (put-bytevector port #vu8(128)) -- cgit v1.2.3