From dc0f74e5fc26977a3ee6c4f2aa74a141f4359982 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 18 Jan 2017 23:21:29 +0100 Subject: Add (guix status) and use it for pretty colored output. * guix/progress.scm (progress-reporter/trace): New procedure. (%progress-interval): New variable. (progress-reporter/file): Use it. * guix/scripts/build.scm (set-build-options-from-command-line): Pass #:print-extended-build-trace?. (%default-options): Add 'print-extended-build-trace?'. (guix-build): Parameterize CURRENT-TERMINAL-COLUMNS. Use 'build-status-updater'. * guix/scripts/environment.scm (%default-options): Add 'print-extended-build-trace?'. (guix-environment): Wrap body in 'with-status-report'. * guix/scripts/pack.scm (%default-options): Add 'print-build-trace?' and 'print-extended-build-trace?'. (guix-pack): Wrap body in 'with-status-report'. * guix/scripts/package.scm (%default-options, guix-package): Likewise. * guix/scripts/system.scm (%default-options, guix-system): Likewise. * guix/scripts/pull.scm (%default-options, guix-pull): Likewise. * guix/scripts/substitute.scm (progress-report-port): Don't call STOP when TOTAL is zero. (process-substitution): Add #:print-build-trace? and honor it. (guix-substitute)[print-build-trace?]: New variable. Pass #:print-build-trace? to 'process-substitution'. * guix/status.scm: New file. * guix/store.scm (set-build-options): Add #:print-extended-build-trace?; pass it into PAIRS. (%protocol-version): Bump. (protocol-version, nix-server-version): New procedures. (current-store-protocol-version): New variable. (with-store, build-things): Parameterize it. * guix/ui.scm (build-output-port): Remove. (colorize-string): Export. * po/guix/POTFILES.in: Add guix/status.scm. * tests/status.scm: New file. * Makefile.am (SCM_TESTS): Add it. * nix/libstore/worker-protocol.hh (PROTOCOL_VERSION): Bump to 0x162. * nix/libstore/build.cc (DerivationGoal::registerOutputs) (SubstitutionGoal::finished): Print a "@ hash-mismatch" trace before throwing. --- nix/libstore/build.cc | 27 +++++++++++++++------------ nix/libstore/worker-protocol.hh | 2 +- 2 files changed, 16 insertions(+), 13 deletions(-) (limited to 'nix') diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc index c7f32494d0..b2c319f00b 100644 --- a/nix/libstore/build.cc +++ b/nix/libstore/build.cc @@ -2466,13 +2466,13 @@ void DerivationGoal::registerOutputs() /* Check the hash. */ Hash h2 = recursive ? hashPath(ht, actualPath).first : hashFile(ht, actualPath); - if (h != h2) - throw BuildError( - format("%1% hash mismatch for output path `%2%'\n" - " expected: %3%\n" - " actual: %4%") - % i->second.hashAlgo % path - % printHash16or32(h) % printHash16or32(h2)); + if (h != h2) { + if (settings.printBuildTrace) + printMsg(lvlError, format("@ hash-mismatch %1% %2% %3% %4%") + % path % i->second.hashAlgo + % printHash16or32(h) % printHash16or32(h2)); + throw BuildError(format("hash mismatch for store item '%1%'") % path); + } } /* Get rid of all weird permissions. This also checks that @@ -3157,11 +3157,14 @@ void SubstitutionGoal::finished() throw Error(format("unknown hash algorithm in `%1%'") % expectedHashStr); Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1)); Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first; - if (expectedHash != actualHash) - throw SubstError(format("hash mismatch in downloaded path `%1%'\n" - " expected: %2%\n" - " actual: %3%") - % storePath % printHash(expectedHash) % printHash(actualHash)); + if (expectedHash != actualHash) { + if (settings.printBuildTrace) + printMsg(lvlError, format("@ hash-mismatch %1% %2% %3% %4%") + % storePath % "sha256" + % printHash16or32(expectedHash) + % printHash16or32(actualHash)); + throw SubstError(format("hash mismatch for substituted item `%1%'") % storePath); + } } } catch (SubstError & e) { diff --git a/nix/libstore/worker-protocol.hh b/nix/libstore/worker-protocol.hh index efe9eadf23..103d60a8c2 100644 --- a/nix/libstore/worker-protocol.hh +++ b/nix/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x161 +#define PROTOCOL_VERSION 0x162 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) -- cgit v1.2.3 From 240a9c69a6064544a616acc521c993542c364948 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 12 Sep 2018 15:08:38 +0200 Subject: perform-download: Optionally report a "download-progress" trace. * guix/scripts/perform-download.scm (perform-download): Add #:print-build-trace? and pass it to 'url-fetch'. (guix-perform-download): Define 'print-build-trace?' and pass it to 'perform-download'. * guix/build/download.scm (ftp-fetch): Add #:print-build-trace? and honor it. (url-fetch): Likewise. * nix/libstore/builtins.cc (builtinDownload): Set _NIX_OPTIONS environment variable. --- guix/build/download.scm | 33 ++++++++++++++++++++++----------- guix/scripts/perform-download.scm | 18 +++++++++++++----- nix/libstore/builtins.cc | 5 ++++- 3 files changed, 39 insertions(+), 17 deletions(-) (limited to 'nix') diff --git a/guix/build/download.scm b/guix/build/download.scm index 315a3554ec..54163849a2 100644 --- a/guix/build/download.scm +++ b/guix/build/download.scm @@ -115,7 +115,7 @@ (define (nar-uri-abbreviation uri) (string-drop path 33) path))) -(define* (ftp-fetch uri file #:key timeout) +(define* (ftp-fetch uri file #:key timeout print-build-trace?) "Fetch data from URI and write it to FILE. Return FILE on success. Bail out if the connection could not be established in less than TIMEOUT seconds." (let* ((conn (match (and=> (uri-userinfo uri) @@ -136,12 +136,17 @@ (define* (ftp-fetch uri file #:key timeout) (lambda (out) (dump-port* in out #:buffer-size %http-receive-buffer-size - #:reporter (progress-reporter/file - (uri-abbreviation uri) size)))) - - (ftp-close conn)) - (newline) - file) + #:reporter + (if print-build-trace? + (progress-reporter/trace + file (uri->string uri) size) + (progress-reporter/file + (uri-abbreviation uri) size))))) + + (ftp-close conn) + (unless print-build-trace? + (newline)) + file)) ;; Autoload GnuTLS so that this module can be used even when GnuTLS is ;; not available. At compile time, this yields "possibly unbound @@ -723,7 +728,8 @@ (define* (url-fetch url file #:key (timeout 10) (verify-certificate? #t) (mirrors '()) (content-addressed-mirrors '()) - (hashes '())) + (hashes '()) + print-build-trace?) "Fetch FILE from URL; URL may be either a single string, or a list of string denoting alternate URLs for FILE. Return #f on failure, and FILE on success. @@ -759,13 +765,18 @@ (define (fetch uri file) (lambda (output) (dump-port* port output #:buffer-size %http-receive-buffer-size - #:reporter (progress-reporter/file - (uri-abbreviation uri) size)) + #:reporter (if print-build-trace? + (progress-reporter/trace + file (uri->string uri) size) + (progress-reporter/file + (uri-abbreviation uri) size))) (newline))) file))) ((ftp) (false-if-exception* (ftp-fetch uri file - #:timeout timeout))) + #:timeout timeout + #:print-build-trace? + print-build-trace?))) (else (format #t "skipping URI with unsupported scheme: ~s~%" uri) diff --git a/guix/scripts/perform-download.scm b/guix/scripts/perform-download.scm index 9f6ecc00d2..df787a9940 100644 --- a/guix/scripts/perform-download.scm +++ b/guix/scripts/perform-download.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2016, 2017 Ludovic Courtès +;;; Copyright © 2016, 2017, 2018 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -41,14 +41,14 @@ (define %user-module (module-use! module (resolve-interface '(guix base32))) module)) -(define* (perform-download drv #:optional output) +(define* (perform-download drv #:optional output + #:key print-build-trace?) "Perform the download described by DRV, a fixed-output derivation, to OUTPUT. Note: Unless OUTPUT is #f, we don't read the value of 'out' in DRV since the actual output is different from that when we're doing a 'bmCheck' or 'bmRepair' build." - ;; TODO: Use 'trace-progress-proc' when possible. (derivation-let drv ((url "url") (output* "out") (executable "executable") @@ -68,6 +68,7 @@ (define* (perform-download drv #:optional output) ;; We're invoked by the daemon, which gives us write access to OUTPUT. (when (url-fetch url output + #:print-build-trace? print-build-trace? #:mirrors (if mirrors (call-with-input-file mirrors read) '()) @@ -99,6 +100,11 @@ (define (guix-perform-download . args) of GnuTLS over HTTPS, before we have built GnuTLS. See ." + (define print-build-trace? + (match (getenv "_NIX_OPTIONS") + (#f #f) + (str (string-contains str "print-extended-build-trace=1")))) + ;; This program must be invoked by guix-daemon under an unprivileged UID to ;; prevent things downloading from 'file:///etc/shadow' or arbitrary code ;; execution via the content-addressed mirror procedures. (That means we @@ -108,10 +114,12 @@ (define (guix-perform-download . args) (((? derivation-path? drv) (? store-path? output)) (assert-low-privileges) (perform-download (read-derivation-from-file drv) - output)) + output + #:print-build-trace? print-build-trace?)) (((? derivation-path? drv)) ;backward compatibility (assert-low-privileges) - (perform-download (read-derivation-from-file drv))) + (perform-download (read-derivation-from-file drv) + #:print-build-trace? print-build-trace?)) (("--version") (show-version-and-exit)) (x diff --git a/nix/libstore/builtins.cc b/nix/libstore/builtins.cc index a5ebb47737..1f52511c80 100644 --- a/nix/libstore/builtins.cc +++ b/nix/libstore/builtins.cc @@ -1,5 +1,5 @@ /* GNU Guix --- Functional package management for GNU - Copyright (C) 2016, 2017 Ludovic Courtès + Copyright (C) 2016, 2017, 2018 Ludovic Courtès This file is part of GNU Guix. @@ -47,6 +47,9 @@ static void builtinDownload(const Derivation &drv, content-addressed mirrors) works correctly. */ setenv("NIX_STORE", settings.nixStore.c_str(), 1); + /* Tell it about options such as "print-extended-build-trace". */ + setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); + /* XXX: Hack our way to use the 'download' script from 'LIBEXECDIR/guix' or just 'LIBEXECDIR', depending on whether we're running uninstalled or not. */ -- cgit v1.2.3 From 7eca7892014d173ff523417138b90aeff5b7dec1 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Tue, 2 Oct 2018 22:53:47 +0200 Subject: daemon: Remove "case hack" for nars. This code has never been of any use in Guix. * nix/libutil/archive.cc (useCaseHack): Remove. (parse): Keep only the alternate branch in "if (useCaseHack)". --- nix/libutil/archive.cc | 31 +------------------------------ nix/libutil/archive.hh | 5 ----- 2 files changed, 1 insertion(+), 35 deletions(-) (limited to 'nix') diff --git a/nix/libutil/archive.cc b/nix/libutil/archive.cc index 4e3d99356a..2599030454 100644 --- a/nix/libutil/archive.cc +++ b/nix/libutil/archive.cc @@ -21,14 +21,6 @@ namespace nix { - -bool useCaseHack = -#if __APPLE__ - true; -#else - false; -#endif - static string archiveVersion1 = "nix-archive-1"; static string caseHackSuffix = "~nix~case~hack~"; @@ -85,19 +77,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) the case hack applied by restorePath(). */ std::map unhacked; for (auto & i : readDirectory(path)) - if (useCaseHack) { - string name(i.name); - size_t pos = i.name.find(caseHackSuffix); - if (pos != string::npos) { - printMsg(lvlDebug, format("removing case hack suffix from `%1%'") % (path + "/" + i.name)); - name.erase(pos); - } - if (unhacked.find(name) != unhacked.end()) - throw Error(format("file name collision in between `%1%' and `%2%'") - % (path + "/" + unhacked[name]) % (path + "/" + i.name)); - unhacked[name] = i.name; - } else - unhacked[i.name] = i.name; + unhacked[i.name] = i.name; for (auto & i : unhacked) if (filter(path + "/" + i.first)) { @@ -251,15 +231,6 @@ static void parse(ParseSink & sink, Source & source, const Path & path) if (name <= prevName) throw Error("NAR directory is not sorted"); prevName = name; - if (useCaseHack) { - auto i = names.find(name); - if (i != names.end()) { - printMsg(lvlDebug, format("case collision between `%1%' and `%2%'") % i->first % name); - name += caseHackSuffix; - name += std::to_string(++i->second); - } else - names[name] = 0; - } } else if (s == "node") { if (s.empty()) throw badArchive("entry name missing"); parse(sink, source, path + "/" + name); diff --git a/nix/libutil/archive.hh b/nix/libutil/archive.hh index c216e9768f..9b83a5f288 100644 --- a/nix/libutil/archive.hh +++ b/nix/libutil/archive.hh @@ -71,9 +71,4 @@ void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); - -// FIXME: global variables are bad m'kay. -extern bool useCaseHack; - - } -- cgit v1.2.3 From c577ac194afb4c9df76f9955aeb6118aafac6c34 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Tue, 2 Oct 2018 22:57:13 +0200 Subject: daemon: Remove unused 'Settings' fields. * nix/libstore/globals.hh (Settings)[sshSubstituterHosts] [useSshSubstituter, logServers, enableImportNative]: Remove fields. * nix/libstore/globals.cc (Settings::Settings, Settings::update): Remove references to these. --- nix/libstore/globals.cc | 9 --------- nix/libstore/globals.hh | 12 ------------ 2 files changed, 21 deletions(-) (limited to 'nix') diff --git a/nix/libstore/globals.cc b/nix/libstore/globals.cc index fcafac2df6..94c2e516f8 100644 --- a/nix/libstore/globals.cc +++ b/nix/libstore/globals.cc @@ -42,7 +42,6 @@ Settings::Settings() syncBeforeRegistering = false; useSubstitutes = true; useChroot = false; - useSshSubstituter = false; impersonateLinux26 = false; keepLog = true; #if HAVE_BZLIB_H @@ -60,7 +59,6 @@ Settings::Settings() envKeepDerivations = false; lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; showTrace = false; - enableImportNative = false; } @@ -142,11 +140,6 @@ void Settings::update() _get(gcKeepDerivations, "gc-keep-derivations"); _get(autoOptimiseStore, "auto-optimise-store"); _get(envKeepDerivations, "env-keep-derivations"); - _get(sshSubstituterHosts, "ssh-substituter-hosts"); - _get(useSshSubstituter, "use-ssh-substituter"); - _get(logServers, "log-servers"); - _get(enableImportNative, "allow-unsafe-native-code-during-evaluation"); - _get(useCaseHack, "use-case-hack"); string subs = getEnv("NIX_SUBSTITUTERS", "default"); if (subs == "default") { @@ -157,8 +150,6 @@ void Settings::update() #endif substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl"); - if (useSshSubstituter) - substituters.push_back(nixLibexecDir + "/nix/substituters/download-via-ssh"); } else substituters = tokenizeString(subs, ":"); } diff --git a/nix/libstore/globals.hh b/nix/libstore/globals.hh index 1293625e1f..4c142e6933 100644 --- a/nix/libstore/globals.hh +++ b/nix/libstore/globals.hh @@ -161,12 +161,6 @@ struct Settings { /* Whether to build in chroot. */ bool useChroot; - /* Set of ssh connection strings for the ssh substituter */ - Strings sshSubstituterHosts; - - /* Whether to use the ssh substituter at all */ - bool useSshSubstituter; - /* Whether to impersonate a Linux 2.6 machine on newer kernels. */ bool impersonateLinux26; @@ -212,12 +206,6 @@ struct Settings { /* Whether to show a stack trace if Nix evaluation fails. */ bool showTrace; - /* A list of URL prefixes that can return Nix build logs. */ - Strings logServers; - - /* Whether the importNative primop should be enabled */ - bool enableImportNative; - private: SettingsMap settings, overrides; -- cgit v1.2.3 From 6ef61cc4c30e94acbd7437f19c893f63a7112267 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 15 Oct 2018 22:40:35 +0200 Subject: daemon: Support multiplexed build output. This allows clients to tell whether output comes from the daemon or, if it comes from a builder, from which builder it comes. The latter is particularly useful when MAX-BUILD-JOBS > 1. * nix/libstore/build.cc (DerivationGoal::tryBuildHook) (DerivationGoal::startBuilder): Print the child's PID in "@ build-started" traces. (DerivationGoal::handleChildOutput): Define 'prefix', pass it to 'writeToStderr'. * nix/libstore/globals.cc (Settings:Settings): Initialize 'multiplexedBuildOutput'. (Settings::update): Likewise. * nix/libstore/globals.hh (Settings)[multiplexedBuildOutput]: New field. Update 'printBuildTrace' documentation. * nix/libstore/worker-protocol.hh (PROTOCOL_VERSION): Bump to 0.163. * nix/nix-daemon/nix-daemon.cc (performOp) : Special-case "multiplexed-build-output" and remove "use-ssh-substituter". * guix/store.scm (set-build-options): Add #:multiplexed-build-output? and honor it. (%protocol-version): Bump to #x163. * tests/store.scm ("multiplexed-build-output"): New test. fixlet --- guix/store.scm | 15 +++++++++- nix/libstore/build.cc | 25 ++++++++++++---- nix/libstore/globals.cc | 2 ++ nix/libstore/globals.hh | 9 +++++- nix/libstore/worker-protocol.hh | 2 +- nix/nix-daemon/nix-daemon.cc | 2 +- tests/store.scm | 63 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 108 insertions(+), 10 deletions(-) (limited to 'nix') diff --git a/guix/store.scm b/guix/store.scm index 8b35fc8d7a..b1bdbf3813 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -155,7 +155,7 @@ (define-module (guix store) derivation-log-file log-file)) -(define %protocol-version #x162) +(define %protocol-version #x163) (define %worker-magic-1 #x6e697863) ; "nixc" (define %worker-magic-2 #x6478696f) ; "dxio" @@ -709,6 +709,15 @@ (define* (set-build-options server ;; disabled by default. print-extended-build-trace? + ;; When true, the daemon prefixes builder output + ;; with "@ build-log" traces so we can + ;; distinguish it from daemon output, and we can + ;; distinguish each builder's output + ;; (PRINT-BUILD-TRACE must be true as well.) The + ;; latter is particularly useful when + ;; MAX-BUILD-JOBS > 1. + multiplexed-build-output? + build-cores (use-substitutes? #t) @@ -757,6 +766,10 @@ (define socket `(("print-extended-build-trace" . ,(if print-extended-build-trace? "1" "0"))) '()) + ,@(if multiplexed-build-output? + `(("multiplexed-build-output" + . ,(if multiplexed-build-output? "true" "false"))) + '()) ,@(if timeout `(("build-timeout" . ,(number->string timeout))) '()) diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc index b2c319f00b..d7b8b0f0ca 100644 --- a/nix/libstore/build.cc +++ b/nix/libstore/build.cc @@ -1652,8 +1652,8 @@ HookReply DerivationGoal::tryBuildHook() worker.childStarted(shared_from_this(), hook->pid, fds, false, false); if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-started %1% - %2% %3%") - % drvPath % drv.platform % logFile); + printMsg(lvlError, format("@ build-started %1% - %2% %3% %4%") + % drvPath % drv.platform % logFile % hook->pid); return rpAccept; } @@ -2038,8 +2038,8 @@ void DerivationGoal::startBuilder() if (!msg.empty()) throw Error(msg); if (settings.printBuildTrace) { - printMsg(lvlError, format("@ build-started %1% - %2% %3%") - % drvPath % drv.platform % logFile); + printMsg(lvlError, format("@ build-started %1% - %2% %3% %4%") + % drvPath % drv.platform % logFile % pid); } } @@ -2736,6 +2736,19 @@ void DerivationGoal::deleteTmpDir(bool force) void DerivationGoal::handleChildOutput(int fd, const string & data) { + string prefix; + + if (settings.multiplexedBuildOutput) { + /* Print a prefix that allows clients to determine whether a message + comes from the daemon or from a build process, and in the latter + case, which build process it comes from. The PID here matches the + one given in "@ build-started" traces; it's shorter that the + derivation file name, hence this choice. */ + prefix = "@ build-log " + + std::to_string(pid < 0 ? hook->pid : pid) + + " " + std::to_string(data.size()) + "\n"; + } + if ((hook && fd == hook->builderOut.readSide) || (!hook && fd == builderOut.readSide)) { @@ -2748,7 +2761,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) return; } if (verbosity >= settings.buildVerbosity) - writeToStderr(data); + writeToStderr(prefix + data); if (gzLogFile) { if (data.size() > 0) { @@ -2767,7 +2780,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) } if (hook && fd == hook->fromHook.readSide) - writeToStderr(data); + writeToStderr(prefix + data); } diff --git a/nix/libstore/globals.cc b/nix/libstore/globals.cc index 94c2e516f8..4b5b485e65 100644 --- a/nix/libstore/globals.cc +++ b/nix/libstore/globals.cc @@ -36,6 +36,7 @@ Settings::Settings() buildTimeout = 0; useBuildHook = true; printBuildTrace = false; + multiplexedBuildOutput = false; reservedSize = 8 * 1024 * 1024; fsyncMetadata = true; useSQLiteWAL = true; @@ -120,6 +121,7 @@ void Settings::update() _get(maxBuildJobs, "build-max-jobs"); _get(buildCores, "build-cores"); _get(thisSystem, "system"); + _get(multiplexedBuildOutput, "multiplexed-build-output"); _get(maxSilentTime, "build-max-silent-time"); _get(buildTimeout, "build-timeout"); _get(reservedSize, "gc-reserved-space"); diff --git a/nix/libstore/globals.hh b/nix/libstore/globals.hh index 4c142e6933..a6935c3337 100644 --- a/nix/libstore/globals.hh +++ b/nix/libstore/globals.hh @@ -127,7 +127,7 @@ struct Settings { a fixed format to allow its progress to be monitored. Each line starts with a "@". The following are defined: - @ build-started + @ build-started @ build-failed @ build-succeeded @ substituter-started @@ -139,6 +139,13 @@ struct Settings { builders. */ bool printBuildTrace; + /* When true, 'buildDerivations' prefixes lines coming from builders so + that clients know exactly which line comes from which builder, and + which line comes from the daemon itself. The prefix for data coming + from builders is "log:PID:LEN:DATA" where PID uniquely identifies the + builder (PID is given in "build-started" traces.) */ + bool multiplexedBuildOutput; + /* Amount of reserved space for the garbage collector (/nix/var/nix/db/reserved). */ off_t reservedSize; diff --git a/nix/libstore/worker-protocol.hh b/nix/libstore/worker-protocol.hh index 103d60a8c2..ea67b10a5b 100644 --- a/nix/libstore/worker-protocol.hh +++ b/nix/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x162 +#define PROTOCOL_VERSION 0x163 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/nix/nix-daemon/nix-daemon.cc b/nix/nix-daemon/nix-daemon.cc index 782e4acfc5..2939422172 100644 --- a/nix/nix-daemon/nix-daemon.cc +++ b/nix/nix-daemon/nix-daemon.cc @@ -594,7 +594,7 @@ static void performOp(bool trusted, unsigned int clientVersion, if (name == "build-timeout" || name == "build-max-silent-time" || name == "build-max-jobs" || name == "build-cores" || name == "build-repeat" - || name == "use-ssh-substituter") + || name == "multiplexed-build-output") settings.set(name, value); else settings.set(trusted ? name : "untrusted-" + name, value); diff --git a/tests/store.scm b/tests/store.scm index 2858369706..3ff526cdcf 100644 --- a/tests/store.scm +++ b/tests/store.scm @@ -31,6 +31,7 @@ (define-module (test-store) #:use-module (gnu packages) #:use-module (gnu packages bootstrap) #:use-module (ice-9 match) + #:use-module (ice-9 regex) #:use-module (rnrs bytevectors) #:use-module (rnrs io ports) #:use-module (web uri) @@ -1021,4 +1022,66 @@ (define ref-hash (call-with-input-file (derivation->output-path drv2) read)))))) +(test-equal "multiplexed-build-output" + '("Hello from first." "Hello from second.") + (with-store store + (let* ((build (add-text-to-store store "build.sh" + "echo Hello from $NAME.; echo > $out")) + (bash (add-to-store store "bash" #t "sha256" + (search-bootstrap-binary "bash" + (%current-system)))) + (drv1 (derivation store "one" bash + `("-e" ,build) + #:inputs `((,bash) (,build)) + #:env-vars `(("NAME" . "first") + ("x" . ,(random-text))))) + (drv2 (derivation store "two" bash + `("-e" ,build) + #:inputs `((,bash) (,build)) + #:env-vars `(("NAME" . "second") + ("x" . ,(random-text)))))) + (set-build-options store + #:print-build-trace #t + #:multiplexed-build-output? #t + #:max-build-jobs 10) + (let ((port (open-output-string))) + ;; Send the build log to PORT. + (parameterize ((current-build-output-port port)) + (build-derivations store (list drv1 drv2))) + + ;; Retrieve the build log; make sure it contains valid "@ build-log" + ;; traces that allow us to retrieve each builder's output (we assume + ;; there's exactly one "build-output" trace for each builder, which is + ;; reasonable.) + (let* ((log (get-output-string port)) + (started (fold-matches + (make-regexp "@ build-started ([^ ]+) - ([^ ]+) ([^ ]+) ([0-9]+)") + log '() cons)) + (done (fold-matches + (make-regexp "@ build-succeeded (.*) - (.*) (.*) (.*)") + log '() cons)) + (output (fold-matches + (make-regexp "@ build-log ([[:digit:]]+) ([[:digit:]]+)\n([A-Za-z .*]+)\n") + log '() cons)) + (drv-pid (lambda (name) + (lambda (m) + (let ((drv (match:substring m 1)) + (pid (string->number + (match:substring m 4)))) + (and (string-suffix? name drv) pid))))) + (pid-log (lambda (pid) + (lambda (m) + (let ((n (string->number + (match:substring m 1))) + (len (string->number + (match:substring m 2))) + (str (match:substring m 3))) + (and (= pid n) + (= (string-length str) (- len 1)) + str))))) + (pid1 (any (drv-pid "one.drv") started)) + (pid2 (any (drv-pid "two.drv") started))) + (list (any (pid-log pid1) output) + (any (pid-log pid2) output))))))) + (test-end "store") -- cgit v1.2.3 From 0fe1fba4af41f267c4bb2c006fb37f42422ab703 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 14 Nov 2018 18:11:58 +0100 Subject: daemon: Install 'authenticate' script under LIBEXECDIR/guix. That way it is handled in the same way as other helper scripts. * nix/scripts/guix-authenticate.in: Rename to... * nix/scripts/authenticate.in: ... this. * config-daemon.ac: Adjust accordingly. * nix/local.mk (libstore_a_CPPFLAGS): Remove -DOPENSSL_PATH. (nodist_libexec_SCRIPTS): Remove. (nodist_pkglibexec_SCRIPTS): New variable. * nix/nix-daemon/guix-daemon.cc (main): Remove 'setenv' call for "PATH". * nix/libstore/local-store.cc (runAuthenticationProgram): New function. (LocalStore::exportPath, LocalStore::importPath): Use it instead of 'runProgram' and OPENSSL_PATH. --- config-daemon.ac | 4 ++-- nix/libstore/local-store.cc | 17 +++++++++++++++-- nix/local.mk | 7 ++----- nix/nix-daemon/guix-daemon.cc | 12 ------------ nix/scripts/authenticate.in | 11 +++++++++++ nix/scripts/guix-authenticate.in | 11 ----------- 6 files changed, 30 insertions(+), 32 deletions(-) create mode 100644 nix/scripts/authenticate.in delete mode 100644 nix/scripts/guix-authenticate.in (limited to 'nix') diff --git a/config-daemon.ac b/config-daemon.ac index 04f0fde3b2..aac46817f5 100644 --- a/config-daemon.ac +++ b/config-daemon.ac @@ -163,8 +163,8 @@ if test "x$guix_build_daemon" = "xyes"; then [chmod +x nix/scripts/download]) AC_CONFIG_FILES([nix/scripts/substitute], [chmod +x nix/scripts/substitute]) - AC_CONFIG_FILES([nix/scripts/guix-authenticate], - [chmod +x nix/scripts/guix-authenticate]) + AC_CONFIG_FILES([nix/scripts/authenticate], + [chmod +x nix/scripts/authenticate]) AC_CONFIG_FILES([nix/scripts/offload], [chmod +x nix/scripts/offload]) fi diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc index 4c55c6ea0d..0aed59710f 100644 --- a/nix/libstore/local-store.cc +++ b/nix/libstore/local-store.cc @@ -1222,6 +1222,18 @@ static void checkSecrecy(const Path & path) } +static std::string runAuthenticationProgram(const Strings & args) +{ + /* Use the 'authenticate' script from 'LIBEXECDIR/guix' or just + 'LIBEXECDIR', depending on whether we're uninstalled or not. */ + const bool installed = getenv("GUIX_UNINSTALLED") == NULL; + const string program = settings.nixLibexecDir + + (installed ? "/guix" : "") + + "/authenticate"; + + return runProgram(program, false, args); +} + void LocalStore::exportPath(const Path & path, bool sign, Sink & sink) { @@ -1276,7 +1288,8 @@ void LocalStore::exportPath(const Path & path, bool sign, args.push_back(secretKey); args.push_back("-in"); args.push_back(hashFile); - string signature = runProgram(OPENSSL_PATH, true, args); + + string signature = runAuthenticationProgram(args); writeString(signature, hashAndWriteSink); @@ -1366,7 +1379,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) args.push_back("-pubin"); args.push_back("-in"); args.push_back(sigFile); - string hash2 = runProgram(OPENSSL_PATH, true, args); + string hash2 = runAuthenticationProgram(args); /* Note: runProgram() throws an exception if the signature is invalid. */ diff --git a/nix/local.mk b/nix/local.mk index 7d45f200b8..fe45c344f0 100644 --- a/nix/local.mk +++ b/nix/local.mk @@ -113,7 +113,6 @@ libstore_a_CPPFLAGS = \ -DGUIX_CONFIGURATION_DIRECTORY=\"$(sysconfdir)/guix\" \ -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ -DNIX_BIN_DIR=\"$(bindir)\" \ - -DOPENSSL_PATH="\"guix-authenticate\"" \ -DDEFAULT_CHROOT_DIRS="\"\"" libstore_a_CXXFLAGS = $(AM_CXXFLAGS) \ @@ -168,10 +167,8 @@ nodist_pkglibexec_SCRIPTS += \ endif BUILD_DAEMON_OFFLOAD - -# XXX: It'd be better to hide it in $(pkglibexecdir). -nodist_libexec_SCRIPTS = \ - %D%/scripts/guix-authenticate +nodist_pkglibexec_SCRIPTS += \ + %D%/scripts/authenticate # The '.service' files for systemd. systemdservicedir = $(libdir)/systemd/system diff --git a/nix/nix-daemon/guix-daemon.cc b/nix/nix-daemon/guix-daemon.cc index b71b100f6c..8fdab2d116 100644 --- a/nix/nix-daemon/guix-daemon.cc +++ b/nix/nix-daemon/guix-daemon.cc @@ -466,18 +466,6 @@ main (int argc, char *argv[]) { settings.processEnvironment (); - /* Hackily help 'local-store.cc' find our 'guix-authenticate' program, which - is known as 'OPENSSL_PATH' here. */ - std::string search_path; - search_path = settings.nixLibexecDir; - if (getenv ("PATH") != NULL) - { - search_path += ":"; - search_path += getenv ("PATH"); - } - - setenv ("PATH", search_path.c_str (), 1); - /* Use our substituter by default. */ settings.substituters.clear (); settings.set ("build-use-substitutes", "true"); diff --git a/nix/scripts/authenticate.in b/nix/scripts/authenticate.in new file mode 100644 index 0000000000..5ce57915f0 --- /dev/null +++ b/nix/scripts/authenticate.in @@ -0,0 +1,11 @@ +#!@SHELL@ +# A shorthand for "guix authenticate", for use by the daemon. + +if test "x$GUIX_UNINSTALLED" = "x" +then + prefix="@prefix@" + exec_prefix="@exec_prefix@" + exec "@bindir@/guix" authenticate "$@" +else + exec guix authenticate "$@" +fi diff --git a/nix/scripts/guix-authenticate.in b/nix/scripts/guix-authenticate.in deleted file mode 100644 index 5ce57915f0..0000000000 --- a/nix/scripts/guix-authenticate.in +++ /dev/null @@ -1,11 +0,0 @@ -#!@SHELL@ -# A shorthand for "guix authenticate", for use by the daemon. - -if test "x$GUIX_UNINSTALLED" = "x" -then - prefix="@prefix@" - exec_prefix="@exec_prefix@" - exec "@bindir@/guix" authenticate "$@" -else - exec guix authenticate "$@" -fi -- cgit v1.2.3 From bb640d6133dea0ede7e80db9ca64a9b2e451317a Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sat, 24 Nov 2018 15:59:26 +0100 Subject: daemon: Ignore '--keep-failed' for TCP/IP clients. * nix/nix-daemon/nix-daemon.cc (performOp) : When 'isRemoteConnection' is true, set 'settings.keepFailed' to zero. * doc/guix.texi (Common Build Options): Document this behavior. --- doc/guix.texi | 4 ++++ nix/nix-daemon/nix-daemon.cc | 6 ++++++ 2 files changed, 10 insertions(+) (limited to 'nix') diff --git a/doc/guix.texi b/doc/guix.texi index e783c16365..a2af1e9cbd 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -6229,6 +6229,10 @@ the end of the build log. This is useful when debugging build issues. @xref{Debugging Build Failures}, for tips and tricks on how to debug build issues. +This option has no effect when connecting to a remote daemon with a +@code{guix://} URI (@pxref{The Store, the @code{GUIX_DAEMON_SOCKET} +variable}). + @item --keep-going @itemx -k Keep going when some of the derivations fail to build; return only once diff --git a/nix/nix-daemon/nix-daemon.cc b/nix/nix-daemon/nix-daemon.cc index 2939422172..6ce475a26c 100644 --- a/nix/nix-daemon/nix-daemon.cc +++ b/nix/nix-daemon/nix-daemon.cc @@ -565,6 +565,12 @@ static void performOp(bool trusted, unsigned int clientVersion, case wopSetOptions: { settings.keepFailed = readInt(from) != 0; + if (isRemoteConnection) + /* When the client is remote, don't keep the failed build tree as + it is presumably inaccessible to the client and could fill up + our disk. */ + settings.keepFailed = 0; + settings.keepGoing = readInt(from) != 0; settings.set("build-fallback", readInt(from) ? "true" : "false"); verbosity = (Verbosity) readInt(from); -- cgit v1.2.3