Merge branch 'master' into staging

12 files changed, 74 insertions, 97 deletions

diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc
index c7f32494d0..d7b8b0f0ca 100644
--- a/nix/libstore/build.cc
+++ b/nix/libstore/build.cc
@@ -1652,8 +1652,8 @@ HookReply DerivationGoal::tryBuildHook()
     worker.childStarted(shared_from_this(), hook->pid, fds, false, false);
 
     if (settings.printBuildTrace)
-        printMsg(lvlError, format("@ build-started %1% - %2% %3%")
-            % drvPath % drv.platform % logFile);
+        printMsg(lvlError, format("@ build-started %1% - %2% %3% %4%")
+            % drvPath % drv.platform % logFile % hook->pid);
 
     return rpAccept;
 }
@@ -2038,8 +2038,8 @@ void DerivationGoal::startBuilder()
     if (!msg.empty()) throw Error(msg);
 
     if (settings.printBuildTrace) {
-        printMsg(lvlError, format("@ build-started %1% - %2% %3%")
-            % drvPath % drv.platform % logFile);
+        printMsg(lvlError, format("@ build-started %1% - %2% %3% %4%")
+            % drvPath % drv.platform % logFile % pid);
     }
 
 }
@@ -2466,13 +2466,13 @@ void DerivationGoal::registerOutputs()
 
             /* Check the hash. */
             Hash h2 = recursive ? hashPath(ht, actualPath).first : hashFile(ht, actualPath);
-            if (h != h2)
-                throw BuildError(
-                    format("%1% hash mismatch for output path `%2%'\n"
-			   "  expected: %3%\n"
-			   "  actual:   %4%")
-                    % i->second.hashAlgo % path
-		    % printHash16or32(h) % printHash16or32(h2));
+            if (h != h2) {
+		if (settings.printBuildTrace)
+		    printMsg(lvlError, format("@ hash-mismatch %1% %2% %3% %4%")
+			     % path % i->second.hashAlgo
+			     % printHash16or32(h) % printHash16or32(h2));
+                throw BuildError(format("hash mismatch for store item '%1%'") % path);
+	    }
         }
 
         /* Get rid of all weird permissions.  This also checks that
@@ -2736,6 +2736,19 @@ void DerivationGoal::deleteTmpDir(bool force)
 
 void DerivationGoal::handleChildOutput(int fd, const string & data)
 {
+    string prefix;
+
+    if (settings.multiplexedBuildOutput) {
+	/* Print a prefix that allows clients to determine whether a message
+	   comes from the daemon or from a build process, and in the latter
+	   case, which build process it comes from.  The PID here matches the
+	   one given in "@ build-started" traces; it's shorter that the
+	   derivation file name, hence this choice.  */
+	prefix = "@ build-log "
+	    + std::to_string(pid < 0 ? hook->pid : pid)
+	    + " " + std::to_string(data.size()) + "\n";
+    }
+
     if ((hook && fd == hook->builderOut.readSide) ||
         (!hook && fd == builderOut.readSide))
     {
@@ -2748,7 +2761,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
             return;
         }
         if (verbosity >= settings.buildVerbosity)
-            writeToStderr(data);
+            writeToStderr(prefix + data);
 
 	if (gzLogFile) {
 	    if (data.size() > 0) {
@@ -2767,7 +2780,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
     }
 
     if (hook && fd == hook->fromHook.readSide)
-        writeToStderr(data);
+        writeToStderr(prefix + data);
 }
 
 
@@ -3157,11 +3170,14 @@ void SubstitutionGoal::finished()
                 throw Error(format("unknown hash algorithm in `%1%'") % expectedHashStr);
             Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1));
             Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first;
-            if (expectedHash != actualHash)
-                throw SubstError(format("hash mismatch in downloaded path `%1%'\n"
-					"  expected: %2%\n"
-					"  actual:   %3%")
-                    % storePath % printHash(expectedHash) % printHash(actualHash));
+            if (expectedHash != actualHash) {
+		if (settings.printBuildTrace)
+		    printMsg(lvlError, format("@ hash-mismatch %1% %2% %3% %4%")
+			     % storePath % "sha256"
+			     % printHash16or32(expectedHash)
+			     % printHash16or32(actualHash));
+                throw SubstError(format("hash mismatch for substituted item `%1%'") % storePath);
+	    }
         }
 
     } catch (SubstError & e) {
diff --git a/nix/libstore/builtins.cc b/nix/libstore/builtins.cc
index a5ebb47737..1f52511c80 100644
--- a/nix/libstore/builtins.cc
+++ b/nix/libstore/builtins.cc
@@ -1,5 +1,5 @@
 /* GNU Guix --- Functional package management for GNU
-   Copyright (C) 2016, 2017 Ludovic Courtès <ludo@gnu.org>
+   Copyright (C) 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
 
    This file is part of GNU Guix.
 
@@ -47,6 +47,9 @@ static void builtinDownload(const Derivation &drv,
        content-addressed mirrors) works correctly.  */
     setenv("NIX_STORE", settings.nixStore.c_str(), 1);
 
+    /* Tell it about options such as "print-extended-build-trace".  */
+    setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
+
     /* XXX: Hack our way to use the 'download' script from 'LIBEXECDIR/guix'
        or just 'LIBEXECDIR', depending on whether we're running uninstalled or
        not.  */
diff --git a/nix/libstore/globals.cc b/nix/libstore/globals.cc
index fcafac2df6..4b5b485e65 100644
--- a/nix/libstore/globals.cc
+++ b/nix/libstore/globals.cc
@@ -36,13 +36,13 @@ Settings::Settings()
     buildTimeout = 0;
     useBuildHook = true;
     printBuildTrace = false;
+    multiplexedBuildOutput = false;
     reservedSize = 8 * 1024 * 1024;
     fsyncMetadata = true;
     useSQLiteWAL = true;
     syncBeforeRegistering = false;
     useSubstitutes = true;
     useChroot = false;
-    useSshSubstituter = false;
     impersonateLinux26 = false;
     keepLog = true;
 #if HAVE_BZLIB_H
@@ -60,7 +60,6 @@ Settings::Settings()
     envKeepDerivations = false;
     lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
     showTrace = false;
-    enableImportNative = false;
 }
 
 
@@ -122,6 +121,7 @@ void Settings::update()
     _get(maxBuildJobs, "build-max-jobs");
     _get(buildCores, "build-cores");
     _get(thisSystem, "system");
+    _get(multiplexedBuildOutput, "multiplexed-build-output");
     _get(maxSilentTime, "build-max-silent-time");
     _get(buildTimeout, "build-timeout");
     _get(reservedSize, "gc-reserved-space");
@@ -142,11 +142,6 @@ void Settings::update()
     _get(gcKeepDerivations, "gc-keep-derivations");
     _get(autoOptimiseStore, "auto-optimise-store");
     _get(envKeepDerivations, "env-keep-derivations");
-    _get(sshSubstituterHosts, "ssh-substituter-hosts");
-    _get(useSshSubstituter, "use-ssh-substituter");
-    _get(logServers, "log-servers");
-    _get(enableImportNative, "allow-unsafe-native-code-during-evaluation");
-    _get(useCaseHack, "use-case-hack");
 
     string subs = getEnv("NIX_SUBSTITUTERS", "default");
     if (subs == "default") {
@@ -157,8 +152,6 @@ void Settings::update()
 #endif
         substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
         substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
-        if (useSshSubstituter)
-            substituters.push_back(nixLibexecDir + "/nix/substituters/download-via-ssh");
     } else
         substituters = tokenizeString<Strings>(subs, ":");
 }
diff --git a/nix/libstore/globals.hh b/nix/libstore/globals.hh
index 1293625e1f..a6935c3337 100644
--- a/nix/libstore/globals.hh
+++ b/nix/libstore/globals.hh
@@ -127,7 +127,7 @@ struct Settings {
        a fixed format to allow its progress to be monitored.  Each
        line starts with a "@".  The following are defined:
 
-       @ build-started <drvpath> <outpath> <system> <logfile>
+       @ build-started <drvpath> <outpath> <system> <logfile> <pid>
        @ build-failed <drvpath> <outpath> <exitcode> <error text>
        @ build-succeeded <drvpath> <outpath>
        @ substituter-started <outpath> <substituter>
@@ -139,6 +139,13 @@ struct Settings {
        builders. */
     bool printBuildTrace;
 
+    /* When true, 'buildDerivations' prefixes lines coming from builders so
+       that clients know exactly which line comes from which builder, and
+       which line comes from the daemon itself.  The prefix for data coming
+       from builders is "log:PID:LEN:DATA" where PID uniquely identifies the
+       builder (PID is given in "build-started" traces.)  */
+    bool multiplexedBuildOutput;
+
     /* Amount of reserved space for the garbage collector
        (/nix/var/nix/db/reserved). */
     off_t reservedSize;
@@ -161,12 +168,6 @@ struct Settings {
     /* Whether to build in chroot. */
     bool useChroot;
 
-    /* Set of ssh connection strings for the ssh substituter */
-    Strings sshSubstituterHosts;
-
-    /* Whether to use the ssh substituter at all */
-    bool useSshSubstituter;
-
     /* Whether to impersonate a Linux 2.6 machine on newer kernels. */
     bool impersonateLinux26;
 
@@ -212,12 +213,6 @@ struct Settings {
     /* Whether to show a stack trace if Nix evaluation fails. */
     bool showTrace;
 
-    /* A list of URL prefixes that can return Nix build logs. */
-    Strings logServers;
-
-    /* Whether the importNative primop should be enabled */
-    bool enableImportNative;
-
 private:
     SettingsMap settings, overrides;
 
diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc
index 4c55c6ea0d..0aed59710f 100644
--- a/nix/libstore/local-store.cc
+++ b/nix/libstore/local-store.cc
@@ -1222,6 +1222,18 @@ static void checkSecrecy(const Path & path)
 }
 
 
+static std::string runAuthenticationProgram(const Strings & args)
+{
+    /* Use the 'authenticate' script from 'LIBEXECDIR/guix' or just
+       'LIBEXECDIR', depending on whether we're uninstalled or not.  */
+    const bool installed = getenv("GUIX_UNINSTALLED") == NULL;
+    const string program = settings.nixLibexecDir
+	+ (installed ? "/guix" : "")
+	+ "/authenticate";
+
+    return runProgram(program, false, args);
+}
+
 void LocalStore::exportPath(const Path & path, bool sign,
     Sink & sink)
 {
@@ -1276,7 +1288,8 @@ void LocalStore::exportPath(const Path & path, bool sign,
         args.push_back(secretKey);
         args.push_back("-in");
         args.push_back(hashFile);
-        string signature = runProgram(OPENSSL_PATH, true, args);
+
+        string signature = runAuthenticationProgram(args);
 
         writeString(signature, hashAndWriteSink);
 
@@ -1366,7 +1379,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
             args.push_back("-pubin");
             args.push_back("-in");
             args.push_back(sigFile);
-            string hash2 = runProgram(OPENSSL_PATH, true, args);
+            string hash2 = runAuthenticationProgram(args);
 
             /* Note: runProgram() throws an exception if the signature
                is invalid. */
diff --git a/nix/libstore/worker-protocol.hh b/nix/libstore/worker-protocol.hh
index efe9eadf23..ea67b10a5b 100644
--- a/nix/libstore/worker-protocol.hh
+++ b/nix/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x161
+#define PROTOCOL_VERSION 0x163
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
diff --git a/nix/libutil/archive.cc b/nix/libutil/archive.cc
index 4e3d99356a..2599030454 100644
--- a/nix/libutil/archive.cc
+++ b/nix/libutil/archive.cc
@@ -21,14 +21,6 @@
 
 namespace nix {
 
-
-bool useCaseHack =
-#if __APPLE__
-    true;
-#else
-    false;
-#endif
-
 static string archiveVersion1 = "nix-archive-1";
 
 static string caseHackSuffix = "~nix~case~hack~";
@@ -85,19 +77,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
            the case hack applied by restorePath(). */
         std::map<string, string> unhacked;
         for (auto & i : readDirectory(path))
-            if (useCaseHack) {
-                string name(i.name);
-                size_t pos = i.name.find(caseHackSuffix);
-                if (pos != string::npos) {
-                    printMsg(lvlDebug, format("removing case hack suffix from `%1%'") % (path + "/" + i.name));
-                    name.erase(pos);
-                }
-                if (unhacked.find(name) != unhacked.end())
-                    throw Error(format("file name collision in between `%1%' and `%2%'")
-                        % (path + "/" + unhacked[name]) % (path + "/" + i.name));
-                unhacked[name] = i.name;
-            } else
-                unhacked[i.name] = i.name;
+	    unhacked[i.name] = i.name;
 
         for (auto & i : unhacked)
             if (filter(path + "/" + i.first)) {
@@ -251,15 +231,6 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
                     if (name <= prevName)
                         throw Error("NAR directory is not sorted");
                     prevName = name;
-                    if (useCaseHack) {
-                        auto i = names.find(name);
-                        if (i != names.end()) {
-                            printMsg(lvlDebug, format("case collision between `%1%' and `%2%'") % i->first % name);
-                            name += caseHackSuffix;
-                            name += std::to_string(++i->second);
-                        } else
-                            names[name] = 0;
-                    }
                 } else if (s == "node") {
                     if (s.empty()) throw badArchive("entry name missing");
                     parse(sink, source, path + "/" + name);
diff --git a/nix/libutil/archive.hh b/nix/libutil/archive.hh
index c216e9768f..9b83a5f288 100644
--- a/nix/libutil/archive.hh
+++ b/nix/libutil/archive.hh
@@ -71,9 +71,4 @@ void parseDump(ParseSink & sink, Source & source);
 
 void restorePath(const Path & path, Source & source);
 
-
-// FIXME: global variables are bad m'kay.
-extern bool useCaseHack;
-
-
 }
diff --git a/nix/local.mk b/nix/local.mk
index 7d45f200b8..fe45c344f0 100644
--- a/nix/local.mk
+++ b/nix/local.mk
@@ -113,7 +113,6 @@ libstore_a_CPPFLAGS =				\
   -DGUIX_CONFIGURATION_DIRECTORY=\"$(sysconfdir)/guix\"		\
   -DNIX_LIBEXEC_DIR=\"$(libexecdir)\"		\
   -DNIX_BIN_DIR=\"$(bindir)\"			\
-  -DOPENSSL_PATH="\"guix-authenticate\""	\
   -DDEFAULT_CHROOT_DIRS="\"\""
 
 libstore_a_CXXFLAGS = $(AM_CXXFLAGS)		\
@@ -168,10 +167,8 @@ nodist_pkglibexec_SCRIPTS +=			\
 
 endif BUILD_DAEMON_OFFLOAD
 
-
-# XXX: It'd be better to hide it in $(pkglibexecdir).
-nodist_libexec_SCRIPTS =			\
-  %D%/scripts/guix-authenticate
+nodist_pkglibexec_SCRIPTS +=			\
+  %D%/scripts/authenticate
 
 # The '.service' files for systemd.
 systemdservicedir = $(libdir)/systemd/system
diff --git a/nix/nix-daemon/guix-daemon.cc b/nix/nix-daemon/guix-daemon.cc
index b71b100f6c..8fdab2d116 100644
--- a/nix/nix-daemon/guix-daemon.cc
+++ b/nix/nix-daemon/guix-daemon.cc
@@ -466,18 +466,6 @@ main (int argc, char *argv[])
     {
       settings.processEnvironment ();
 
-      /* Hackily help 'local-store.cc' find our 'guix-authenticate' program, which
-	 is known as 'OPENSSL_PATH' here.  */
-      std::string search_path;
-      search_path = settings.nixLibexecDir;
-      if (getenv ("PATH") != NULL)
-	{
-	  search_path += ":";
-	  search_path += getenv ("PATH");
-	}
-
-      setenv ("PATH", search_path.c_str (), 1);
-
       /* Use our substituter by default.  */
       settings.substituters.clear ();
       settings.set ("build-use-substitutes", "true");
diff --git a/nix/nix-daemon/nix-daemon.cc b/nix/nix-daemon/nix-daemon.cc
index 782e4acfc5..6ce475a26c 100644
--- a/nix/nix-daemon/nix-daemon.cc
+++ b/nix/nix-daemon/nix-daemon.cc
@@ -565,6 +565,12 @@ static void performOp(bool trusted, unsigned int clientVersion,
 
     case wopSetOptions: {
         settings.keepFailed = readInt(from) != 0;
+	if (isRemoteConnection)
+	    /* When the client is remote, don't keep the failed build tree as
+	       it is presumably inaccessible to the client and could fill up
+	       our disk.  */
+	    settings.keepFailed = 0;
+
         settings.keepGoing = readInt(from) != 0;
         settings.set("build-fallback", readInt(from) ? "true" : "false");
         verbosity = (Verbosity) readInt(from);
@@ -594,7 +600,7 @@ static void performOp(bool trusted, unsigned int clientVersion,
                 if (name == "build-timeout" || name == "build-max-silent-time"
                     || name == "build-max-jobs" || name == "build-cores"
                     || name == "build-repeat"
-                    || name == "use-ssh-substituter")
+                    || name == "multiplexed-build-output")
                     settings.set(name, value);
                 else
                     settings.set(trusted ? name : "untrusted-" + name, value);
diff --git a/nix/scripts/guix-authenticate.in b/nix/scripts/authenticate.in
index 5ce57915f0..5ce57915f0 100644
--- a/nix/scripts/guix-authenticate.in
+++ b/nix/scripts/authenticate.in