aboutsummaryrefslogtreecommitdiff
path: root/nix/libstore
diff options
context:
space:
mode:
authorLudovic Courtès <ludo@gnu.org>2014-12-17 23:00:42 +0100
committerLudovic Courtès <ludo@gnu.org>2014-12-19 22:47:37 +0100
commit36457566f9917dc7c0c348d012816a2ca333ef1b (patch)
tree6f1d22a195ea2483b9ce539227d65e8e2a9c137d /nix/libstore
parent2c7ee1672029aa43afb509af5b5f7261244fa2d1 (diff)
downloadgnu-guix-36457566f9917dc7c0c348d012816a2ca333ef1b.tar
gnu-guix-36457566f9917dc7c0c348d012816a2ca333ef1b.tar.gz
Merge branch 'nix' into 'master'.
Diffstat (limited to 'nix/libstore')
-rw-r--r--nix/libstore/build.cc3351
-rw-r--r--nix/libstore/derivations.cc278
-rw-r--r--nix/libstore/derivations.hh93
-rw-r--r--nix/libstore/gc.cc748
-rw-r--r--nix/libstore/globals.cc240
-rw-r--r--nix/libstore/globals.hh218
-rw-r--r--nix/libstore/local-store.cc2010
-rw-r--r--nix/libstore/local-store.hh333
-rw-r--r--nix/libstore/misc.cc220
-rw-r--r--nix/libstore/misc.hh38
-rw-r--r--nix/libstore/optimise-store.cc180
-rw-r--r--nix/libstore/pathlocks.cc199
-rw-r--r--nix/libstore/pathlocks.hh45
-rw-r--r--nix/libstore/references.cc122
-rw-r--r--nix/libstore/references.hh11
-rw-r--r--nix/libstore/remote-store.cc602
-rw-r--r--nix/libstore/remote-store.hh104
-rw-r--r--nix/libstore/schema.sql44
-rw-r--r--nix/libstore/store-api.cc331
-rw-r--r--nix/libstore/store-api.hh366
-rw-r--r--nix/libstore/worker-protocol.hh60
21 files changed, 9593 insertions, 0 deletions
diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc
new file mode 100644
index 0000000000..2e2f92fadf
--- /dev/null
+++ b/nix/libstore/build.cc
@@ -0,0 +1,3351 @@
+#include "config.h"
+
+#include "references.hh"
+#include "pathlocks.hh"
+#include "misc.hh"
+#include "globals.hh"
+#include "local-store.hh"
+#include "util.hh"
+#include "archive.hh"
+#include "affinity.hh"
+
+#include <map>
+#include <sstream>
+#include <algorithm>
+
+#include <time.h>
+#include <sys/time.h>
+#include <sys/wait.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/utsname.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+#include <cstring>
+
+#include <pwd.h>
+#include <grp.h>
+
+#include <bzlib.h>
+
+/* Includes required for chroot support. */
+#if HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+#if HAVE_SYS_MOUNT_H
+#include <sys/mount.h>
+#endif
+#if HAVE_SCHED_H
+#include <sched.h>
+#endif
+
+/* In GNU libc 2.11, <sys/mount.h> does not define `MS_PRIVATE', but
+ <linux/fs.h> does. */
+#if !defined MS_PRIVATE && defined HAVE_LINUX_FS_H
+#include <linux/fs.h>
+#endif
+
+#define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS)
+
+#if CHROOT_ENABLED
+#include <sys/socket.h>
+#include <sys/ioctl.h>
+#include <net/if.h>
+#include <netinet/ip.h>
+#endif
+
+#if HAVE_SYS_PERSONALITY_H
+#include <sys/personality.h>
+#define CAN_DO_LINUX32_BUILDS
+#endif
+
+#if HAVE_STATVFS
+#include <sys/statvfs.h>
+#endif
+
+
+namespace nix {
+
+using std::map;
+
+
+static string pathNullDevice = "/dev/null";
+
+
+/* Forward definition. */
+class Worker;
+struct HookInstance;
+
+
+/* A pointer to a goal. */
+class Goal;
+typedef std::shared_ptr<Goal> GoalPtr;
+typedef std::weak_ptr<Goal> WeakGoalPtr;
+
+/* Set of goals. */
+typedef set<GoalPtr> Goals;
+typedef list<WeakGoalPtr> WeakGoals;
+
+/* A map of paths to goals (and the other way around). */
+typedef map<Path, WeakGoalPtr> WeakGoalMap;
+
+
+
+class Goal : public std::enable_shared_from_this<Goal>
+{
+public:
+ typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
+
+protected:
+
+ /* Backlink to the worker. */
+ Worker & worker;
+
+ /* Goals that this goal is waiting for. */
+ Goals waitees;
+
+ /* Goals waiting for this one to finish. Must use weak pointers
+ here to prevent cycles. */
+ WeakGoals waiters;
+
+ /* Number of goals we are/were waiting for that have failed. */
+ unsigned int nrFailed;
+
+ /* Number of substitution goals we are/were waiting for that
+ failed because there are no substituters. */
+ unsigned int nrNoSubstituters;
+
+ /* Number of substitution goals we are/were waiting for that
+ failed because othey had unsubstitutable references. */
+ unsigned int nrIncompleteClosure;
+
+ /* Name of this goal for debugging purposes. */
+ string name;
+
+ /* Whether the goal is finished. */
+ ExitCode exitCode;
+
+ Goal(Worker & worker) : worker(worker)
+ {
+ nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
+ exitCode = ecBusy;
+ }
+
+ virtual ~Goal()
+ {
+ trace("goal destroyed");
+ }
+
+public:
+ virtual void work() = 0;
+
+ void addWaitee(GoalPtr waitee);
+
+ virtual void waiteeDone(GoalPtr waitee, ExitCode result);
+
+ virtual void handleChildOutput(int fd, const string & data)
+ {
+ abort();
+ }
+
+ virtual void handleEOF(int fd)
+ {
+ abort();
+ }
+
+ void trace(const format & f);
+
+ string getName()
+ {
+ return name;
+ }
+
+ ExitCode getExitCode()
+ {
+ return exitCode;
+ }
+
+ /* Cancel the goal. It should wake up its waiters, get rid of any
+ running child processes that are being monitored by the worker
+ (important!), etc. */
+ virtual void cancel(bool timeout) = 0;
+
+protected:
+ void amDone(ExitCode result);
+};
+
+
+/* A mapping used to remember for each child process to what goal it
+ belongs, and file descriptors for receiving log data and output
+ path creation commands. */
+struct Child
+{
+ WeakGoalPtr goal;
+ set<int> fds;
+ bool respectTimeouts;
+ bool inBuildSlot;
+ time_t lastOutput; /* time we last got output on stdout/stderr */
+ time_t timeStarted;
+};
+
+typedef map<pid_t, Child> Children;
+
+
+/* The worker class. */
+class Worker
+{
+private:
+
+ /* Note: the worker should only have strong pointers to the
+ top-level goals. */
+
+ /* The top-level goals of the worker. */
+ Goals topGoals;
+
+ /* Goals that are ready to do some work. */
+ WeakGoals awake;
+
+ /* Goals waiting for a build slot. */
+ WeakGoals wantingToBuild;
+
+ /* Child processes currently running. */
+ Children children;
+
+ /* Number of build slots occupied. This includes local builds and
+ substitutions but not remote builds via the build hook. */
+ unsigned int nrLocalBuilds;
+
+ /* Maps used to prevent multiple instantiations of a goal for the
+ same derivation / path. */
+ WeakGoalMap derivationGoals;
+ WeakGoalMap substitutionGoals;
+
+ /* Goals waiting for busy paths to be unlocked. */
+ WeakGoals waitingForAnyGoal;
+
+ /* Goals sleeping for a few seconds (polling a lock). */
+ WeakGoals waitingForAWhile;
+
+ /* Last time the goals in `waitingForAWhile' where woken up. */
+ time_t lastWokenUp;
+
+public:
+
+ /* Set if at least one derivation had a BuildError (i.e. permanent
+ failure). */
+ bool permanentFailure;
+
+ LocalStore & store;
+
+ std::shared_ptr<HookInstance> hook;
+
+ Worker(LocalStore & store);
+ ~Worker();
+
+ /* Make a goal (with caching). */
+ GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
+ GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false);
+
+ /* Remove a dead goal. */
+ void removeGoal(GoalPtr goal);
+
+ /* Wake up a goal (i.e., there is something for it to do). */
+ void wakeUp(GoalPtr goal);
+
+ /* Return the number of local build and substitution processes
+ currently running (but not remote builds via the build
+ hook). */
+ unsigned int getNrLocalBuilds();
+
+ /* Registers a running child process. `inBuildSlot' means that
+ the process counts towards the jobs limit. */
+ void childStarted(GoalPtr goal, pid_t pid,
+ const set<int> & fds, bool inBuildSlot, bool respectTimeouts);
+
+ /* Unregisters a running child process. `wakeSleepers' should be
+ false if there is no sense in waking up goals that are sleeping
+ because they can't run yet (e.g., there is no free build slot,
+ or the hook would still say `postpone'). */
+ void childTerminated(pid_t pid, bool wakeSleepers = true);
+
+ /* Put `goal' to sleep until a build slot becomes available (which
+ might be right away). */
+ void waitForBuildSlot(GoalPtr goal);
+
+ /* Wait for any goal to finish. Pretty indiscriminate way to
+ wait for some resource that some other goal is holding. */
+ void waitForAnyGoal(GoalPtr goal);
+
+ /* Wait for a few seconds and then retry this goal. Used when
+ waiting for a lock held by another process. This kind of
+ polling is inefficient, but POSIX doesn't really provide a way
+ to wait for multiple locks in the main select() loop. */
+ void waitForAWhile(GoalPtr goal);
+
+ /* Loop until the specified top-level goals have finished. */
+ void run(const Goals & topGoals);
+
+ /* Wait for input to become available. */
+ void waitForInput();
+
+ unsigned int exitStatus();
+};
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+void addToWeakGoals(WeakGoals & goals, GoalPtr p)
+{
+ // FIXME: necessary?
+ foreach (WeakGoals::iterator, i, goals)
+ if (i->lock() == p) return;
+ goals.push_back(p);
+}
+
+
+void Goal::addWaitee(GoalPtr waitee)
+{
+ waitees.insert(waitee);
+ addToWeakGoals(waitee->waiters, shared_from_this());
+}
+
+
+void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
+{
+ assert(waitees.find(waitee) != waitees.end());
+ waitees.erase(waitee);
+
+ trace(format("waitee `%1%' done; %2% left") %
+ waitee->name % waitees.size());
+
+ if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
+
+ if (result == ecNoSubstituters) ++nrNoSubstituters;
+
+ if (result == ecIncompleteClosure) ++nrIncompleteClosure;
+
+ if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
+
+ /* If we failed and keepGoing is not set, we remove all
+ remaining waitees. */
+ foreach (Goals::iterator, i, waitees) {
+ GoalPtr goal = *i;
+ WeakGoals waiters2;
+ foreach (WeakGoals::iterator, j, goal->waiters)
+ if (j->lock() != shared_from_this()) waiters2.push_back(*j);
+ goal->waiters = waiters2;
+ }
+ waitees.clear();
+
+ worker.wakeUp(shared_from_this());
+ }
+}
+
+
+void Goal::amDone(ExitCode result)
+{
+ trace("done");
+ assert(exitCode == ecBusy);
+ assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
+ exitCode = result;
+ foreach (WeakGoals::iterator, i, waiters) {
+ GoalPtr goal = i->lock();
+ if (goal) goal->waiteeDone(shared_from_this(), result);
+ }
+ waiters.clear();
+ worker.removeGoal(shared_from_this());
+}
+
+
+void Goal::trace(const format & f)
+{
+ debug(format("%1%: %2%") % name % f);
+}
+
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+/* Common initialisation performed in child processes. */
+static void commonChildInit(Pipe & logPipe)
+{
+ restoreAffinity();
+
+ /* Put the child in a separate session (and thus a separate
+ process group) so that it has no controlling terminal (meaning
+ that e.g. ssh cannot open /dev/tty) and it doesn't receive
+ terminal signals. */
+ if (setsid() == -1)
+ throw SysError(format("creating a new session"));
+
+ /* Dup the write side of the logger pipe into stderr. */
+ if (dup2(logPipe.writeSide, STDERR_FILENO) == -1)
+ throw SysError("cannot pipe standard error into log file");
+
+ /* Dup stderr to stdout. */
+ if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
+ throw SysError("cannot dup stderr into stdout");
+
+ /* Reroute stdin to /dev/null. */
+ int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
+ if (fdDevNull == -1)
+ throw SysError(format("cannot open `%1%'") % pathNullDevice);
+ if (dup2(fdDevNull, STDIN_FILENO) == -1)
+ throw SysError("cannot dup null device into stdin");
+ close(fdDevNull);
+}
+
+
+/* Convert a string list to an array of char pointers. Careful: the
+ string list should outlive the array. */
+const char * * strings2CharPtrs(const Strings & ss)
+{
+ const char * * arr = new const char * [ss.size() + 1];
+ const char * * p = arr;
+ foreach (Strings::const_iterator, i, ss) *p++ = i->c_str();
+ *p = 0;
+ return arr;
+}
+
+
+/* Restore default handling of SIGPIPE, otherwise some programs will
+ randomly say "Broken pipe". */
+static void restoreSIGPIPE()
+{
+ struct sigaction act, oact;
+ act.sa_handler = SIG_DFL;
+ act.sa_flags = 0;
+ sigemptyset(&act.sa_mask);
+ if (sigaction(SIGPIPE, &act, &oact)) throw SysError("resetting SIGPIPE");
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+class UserLock
+{
+private:
+ /* POSIX locks suck. If we have a lock on a file, and we open and
+ close that file again (without closing the original file
+ descriptor), we lose the lock. So we have to be *very* careful
+ not to open a lock file on which we are holding a lock. */
+ static PathSet lockedPaths; /* !!! not thread-safe */
+
+ Path fnUserLock;
+ AutoCloseFD fdUserLock;
+
+ string user;
+ uid_t uid;
+ gid_t gid;
+
+public:
+ UserLock();
+ ~UserLock();
+
+ void acquire();
+ void release();
+
+ void kill();
+
+ string getUser() { return user; }
+ uid_t getUID() { return uid; }
+ uid_t getGID() { return gid; }
+
+ bool enabled() { return uid != 0; }
+
+};
+
+
+PathSet UserLock::lockedPaths;
+
+
+UserLock::UserLock()
+{
+ uid = gid = 0;
+}
+
+
+UserLock::~UserLock()
+{
+ release();
+}
+
+
+void UserLock::acquire()
+{
+ assert(uid == 0);
+
+ assert(settings.buildUsersGroup != "");
+
+ /* Get the members of the build-users-group. */
+ struct group * gr = getgrnam(settings.buildUsersGroup.c_str());
+ if (!gr)
+ throw Error(format("the group `%1%' specified in `build-users-group' does not exist")
+ % settings.buildUsersGroup);
+ gid = gr->gr_gid;
+
+ /* Copy the result of getgrnam. */
+ Strings users;
+ for (char * * p = gr->gr_mem; *p; ++p) {
+ debug(format("found build user `%1%'") % *p);
+ users.push_back(*p);
+ }
+
+ if (users.empty())
+ throw Error(format("the build users group `%1%' has no members")
+ % settings.buildUsersGroup);
+
+ /* Find a user account that isn't currently in use for another
+ build. */
+ foreach (Strings::iterator, i, users) {
+ debug(format("trying user `%1%'") % *i);
+
+ struct passwd * pw = getpwnam(i->c_str());
+ if (!pw)
+ throw Error(format("the user `%1%' in the group `%2%' does not exist")
+ % *i % settings.buildUsersGroup);
+
+ createDirs(settings.nixStateDir + "/userpool");
+
+ fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
+
+ if (lockedPaths.find(fnUserLock) != lockedPaths.end())
+ /* We already have a lock on this one. */
+ continue;
+
+ AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT, 0600);
+ if (fd == -1)
+ throw SysError(format("opening user lock `%1%'") % fnUserLock);
+ closeOnExec(fd);
+
+ if (lockFile(fd, ltWrite, false)) {
+ fdUserLock = fd.borrow();
+ lockedPaths.insert(fnUserLock);
+ user = *i;
+ uid = pw->pw_uid;
+
+ /* Sanity check... */
+ if (uid == getuid() || uid == geteuid())
+ throw Error(format("the Nix user should not be a member of `%1%'")
+ % settings.buildUsersGroup);
+
+ return;
+ }
+ }
+
+ throw Error(format("all build users are currently in use; "
+ "consider creating additional users and adding them to the `%1%' group")
+ % settings.buildUsersGroup);
+}
+
+
+void UserLock::release()
+{
+ if (uid == 0) return;
+ fdUserLock.close(); /* releases lock */
+ assert(lockedPaths.find(fnUserLock) != lockedPaths.end());
+ lockedPaths.erase(fnUserLock);
+ fnUserLock = "";
+ uid = 0;
+}
+
+
+void UserLock::kill()
+{
+ assert(enabled());
+ killUser(uid);
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+struct HookInstance
+{
+ /* Pipes for talking to the build hook. */
+ Pipe toHook;
+
+ /* Pipe for the hook's standard output/error. */
+ Pipe fromHook;
+
+ /* Pipe for the builder's standard output/error. */
+ Pipe builderOut;
+
+ /* The process ID of the hook. */
+ Pid pid;
+
+ HookInstance();
+
+ ~HookInstance();
+};
+
+
+HookInstance::HookInstance()
+{
+ debug("starting build hook");
+
+ Path buildHook = absPath(getEnv("NIX_BUILD_HOOK"));
+
+ /* Create a pipe to get the output of the child. */
+ fromHook.create();
+
+ /* Create the communication pipes. */
+ toHook.create();
+
+ /* Create a pipe to get the output of the builder. */
+ builderOut.create();
+
+ /* Fork the hook. */
+ pid = maybeVfork();
+ switch (pid) {
+
+ case -1:
+ throw SysError("unable to fork");
+
+ case 0:
+ try { /* child */
+
+ commonChildInit(fromHook);
+
+ if (chdir("/") == -1) throw SysError("changing into `/");
+
+ /* Dup the communication pipes. */
+ if (dup2(toHook.readSide, STDIN_FILENO) == -1)
+ throw SysError("dupping to-hook read side");
+
+ /* Use fd 4 for the builder's stdout/stderr. */
+ if (dup2(builderOut.writeSide, 4) == -1)
+ throw SysError("dupping builder's stdout/stderr");
+
+ execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(),
+ (format("%1%") % settings.maxSilentTime).str().c_str(),
+ (format("%1%") % settings.printBuildTrace).str().c_str(),
+ (format("%1%") % settings.buildTimeout).str().c_str(),
+ NULL);
+
+ throw SysError(format("executing `%1%'") % buildHook);
+
+ } catch (std::exception & e) {
+ writeToStderr("build hook error: " + string(e.what()) + "\n");
+ }
+ _exit(1);
+ }
+
+ /* parent */
+ pid.setSeparatePG(true);
+ pid.setKillSignal(SIGTERM);
+ fromHook.writeSide.close();
+ toHook.readSide.close();
+}
+
+
+HookInstance::~HookInstance()
+{
+ try {
+ pid.kill();
+ } catch (...) {
+ ignoreException();
+ }
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+typedef map<string, string> HashRewrites;
+
+
+string rewriteHashes(string s, const HashRewrites & rewrites)
+{
+ foreach (HashRewrites::const_iterator, i, rewrites) {
+ assert(i->first.size() == i->second.size());
+ size_t j = 0;
+ while ((j = s.find(i->first, j)) != string::npos) {
+ debug(format("rewriting @ %1%") % j);
+ s.replace(j, i->second.size(), i->second);
+ }
+ }
+ return s;
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
+
+class SubstitutionGoal;
+
+class DerivationGoal : public Goal
+{
+private:
+ /* The path of the derivation. */
+ Path drvPath;
+
+ /* The specific outputs that we need to build. Empty means all of
+ them. */
+ StringSet wantedOutputs;
+
+ /* Whether additional wanted outputs have been added. */
+ bool needRestart;
+
+ /* Whether to retry substituting the outputs after building the
+ inputs. */
+ bool retrySubstitution;
+
+ /* The derivation stored at drvPath. */
+ Derivation drv;
+
+ /* The remainder is state held during the build. */
+
+ /* Locks on the output paths. */
+ PathLocks outputLocks;
+
+ /* All input paths (that is, the union of FS closures of the
+ immediate input paths). */
+ PathSet inputPaths;
+
+ /* Referenceable paths (i.e., input and output paths). */
+ PathSet allPaths;
+
+ /* Outputs that are already valid. If we're repairing, these are
+ the outputs that are valid *and* not corrupt. */
+ PathSet validPaths;
+
+ /* Outputs that are corrupt or not valid. */
+ PathSet missingPaths;
+
+ /* User selected for running the builder. */
+ UserLock buildUser;
+
+ /* The process ID of the builder. */
+ Pid pid;
+
+ /* The temporary directory. */
+ Path tmpDir;
+
+ /* File descriptor for the log file. */
+ FILE * fLogFile;
+ BZFILE * bzLogFile;
+ AutoCloseFD fdLogFile;
+
+ /* Number of bytes received from the builder's stdout/stderr. */
+ unsigned long logSize;
+
+ /* Pipe for the builder's standard output/error. */
+ Pipe builderOut;
+
+ /* The build hook. */
+ std::shared_ptr<HookInstance> hook;
+
+ /* Whether we're currently doing a chroot build. */
+ bool useChroot;
+
+ Path chrootRootDir;
+
+ /* RAII object to delete the chroot directory. */
+ std::shared_ptr<AutoDelete> autoDelChroot;
+
+ /* All inputs that are regular files. */
+ PathSet regularInputPaths;
+
+ /* Whether this is a fixed-output derivation. */
+ bool fixedOutput;
+
+ typedef void (DerivationGoal::*GoalState)();
+ GoalState state;
+
+ /* Stuff we need to pass to initChild(). */
+ typedef map<Path, Path> DirsInChroot; // maps target path to source path
+ DirsInChroot dirsInChroot;
+ typedef map<string, string> Environment;
+ Environment env;
+
+ /* Hash rewriting. */
+ HashRewrites rewritesToTmp, rewritesFromTmp;
+ typedef map<Path, Path> RedirectedOutputs;
+ RedirectedOutputs redirectedOutputs;
+
+ BuildMode buildMode;
+
+ /* If we're repairing without a chroot, there may be outputs that
+ are valid but corrupt. So we redirect these outputs to
+ temporary paths. */
+ PathSet redirectedBadOutputs;
+
+ /* Set of inodes seen during calls to canonicalisePathMetaData()
+ for this build's outputs. This needs to be shared between
+ outputs to allow hard links between outputs. */
+ InodesSeen inodesSeen;
+
+ /* Magic exit code denoting that setting up the child environment
+ failed. (It's possible that the child actually returns the
+ exit code, but ah well.) */
+ const static int childSetupFailed = 189;
+
+public:
+ DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal);
+ ~DerivationGoal();
+
+ void cancel(bool timeout);
+
+ void work();
+
+ Path getDrvPath()
+ {
+ return drvPath;
+ }
+
+ /* Add wanted outputs to an already existing derivation goal. */
+ void addWantedOutputs(const StringSet & outputs);
+
+private:
+ /* The states. */
+ void init();
+ void haveDerivation();
+ void outputsSubstituted();
+ void closureRepaired();
+ void inputsRealised();
+ void tryToBuild();
+ void buildDone();
+
+ /* Is the build hook willing to perform the build? */
+ HookReply tryBuildHook();
+
+ /* Start building a derivation. */
+ void startBuilder();
+
+ /* Initialise the builder's process. */
+ void initChild();
+
+ friend int childEntry(void *);
+
+ /* Check that the derivation outputs all exist and register them
+ as valid. */
+ void registerOutputs();
+
+ /* Open a log file and a pipe to it. */
+ Path openLogFile();
+
+ /* Close the log file. */
+ void closeLogFile();
+
+ /* Delete the temporary directory, if we have one. */
+ void deleteTmpDir(bool force);
+
+ /* Callback used by the worker to write to the log. */
+ void handleChildOutput(int fd, const string & data);
+ void handleEOF(int fd);
+
+ /* Return the set of (in)valid paths. */
+ PathSet checkPathValidity(bool returnValid, bool checkHash);
+
+ /* Abort the goal if `path' failed to build. */
+ bool pathFailed(const Path & path);
+
+ /* Forcibly kill the child process, if any. */
+ void killChild();
+
+ Path addHashRewrite(const Path & path);
+
+ void repairClosure();
+};
+
+
+DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
+ : Goal(worker)
+ , wantedOutputs(wantedOutputs)
+ , needRestart(false)
+ , retrySubstitution(false)
+ , fLogFile(0)
+ , bzLogFile(0)
+ , useChroot(false)
+ , buildMode(buildMode)
+{
+ this->drvPath = drvPath;
+ state = &DerivationGoal::init;
+ name = (format("building of `%1%'") % drvPath).str();
+ trace("created");
+}
+
+
+DerivationGoal::~DerivationGoal()
+{
+ /* Careful: we should never ever throw an exception from a
+ destructor. */
+ try {
+ killChild();
+ deleteTmpDir(false);
+ closeLogFile();
+ } catch (...) {
+ ignoreException();
+ }
+}
+
+
+void DerivationGoal::killChild()
+{
+ if (pid != -1) {
+ worker.childTerminated(pid);
+
+ if (buildUser.enabled()) {
+ /* If we're using a build user, then there is a tricky
+ race condition: if we kill the build user before the
+ child has done its setuid() to the build user uid, then
+ it won't be killed, and we'll potentially lock up in
+ pid.wait(). So also send a conventional kill to the
+ child. */
+ ::kill(-pid, SIGKILL); /* ignore the result */
+ buildUser.kill();
+ pid.wait(true);
+ } else
+ pid.kill();
+
+ assert(pid == -1);
+ }
+
+ hook.reset();
+}
+
+
+void DerivationGoal::cancel(bool timeout)
+{
+ if (settings.printBuildTrace && timeout)
+ printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath);
+ killChild();
+ amDone(ecFailed);
+}
+
+
+void DerivationGoal::work()
+{
+ (this->*state)();
+}
+
+
+void DerivationGoal::addWantedOutputs(const StringSet & outputs)
+{
+ /* If we already want all outputs, there is nothing to do. */
+ if (wantedOutputs.empty()) return;
+
+ if (outputs.empty()) {
+ wantedOutputs.clear();
+ needRestart = true;
+ } else
+ foreach (StringSet::const_iterator, i, outputs)
+ if (wantedOutputs.find(*i) == wantedOutputs.end()) {
+ wantedOutputs.insert(*i);
+ needRestart = true;
+ }
+}
+
+
+void DerivationGoal::init()
+{
+ trace("init");
+
+ if (settings.readOnlyMode)
+ throw Error(format("cannot build derivation `%1%' - no write access to the Nix store") % drvPath);
+
+ /* The first thing to do is to make sure that the derivation
+ exists. If it doesn't, it may be created through a
+ substitute. */
+ addWaitee(worker.makeSubstitutionGoal(drvPath));
+
+ state = &DerivationGoal::haveDerivation;
+}
+
+
+void DerivationGoal::haveDerivation()
+{
+ trace("loading derivation");
+
+ if (nrFailed != 0) {
+ printMsg(lvlError, format("cannot build missing derivation `%1%'") % drvPath);
+ amDone(ecFailed);
+ return;
+ }
+
+ /* `drvPath' should already be a root, but let's be on the safe
+ side: if the user forgot to make it a root, we wouldn't want
+ things being garbage collected while we're busy. */
+ worker.store.addTempRoot(drvPath);
+
+ assert(worker.store.isValidPath(drvPath));
+
+ /* Get the derivation. */
+ drv = derivationFromPath(worker.store, drvPath);
+
+ foreach (DerivationOutputs::iterator, i, drv.outputs)
+ worker.store.addTempRoot(i->second.path);
+
+ /* Check what outputs paths are not already valid. */
+ PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair);
+
+ /* If they are all valid, then we're done. */
+ if (invalidOutputs.size() == 0 && buildMode == bmNormal) {
+ amDone(ecSuccess);
+ return;
+ }
+
+ /* Check whether any output previously failed to build. If so,
+ don't bother. */
+ foreach (PathSet::iterator, i, invalidOutputs)
+ if (pathFailed(*i)) return;
+
+ /* We are first going to try to create the invalid output paths
+ through substitutes. If that doesn't work, we'll build
+ them. */
+ if (settings.useSubstitutes && !willBuildLocally(drv))
+ foreach (PathSet::iterator, i, invalidOutputs)
+ addWaitee(worker.makeSubstitutionGoal(*i, buildMode == bmRepair));
+
+ if (waitees.empty()) /* to prevent hang (no wake-up event) */
+ outputsSubstituted();
+ else
+ state = &DerivationGoal::outputsSubstituted;
+}
+
+
+void DerivationGoal::outputsSubstituted()
+{
+ trace("all outputs substituted (maybe)");
+
+ if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback)
+ throw Error(format("some substitutes for the outputs of derivation `%1%' failed (usually happens due to networking issues); try `--fallback' to build derivation from source ") % drvPath);
+
+ /* If the substitutes form an incomplete closure, then we should
+ build the dependencies of this derivation, but after that, we
+ can still use the substitutes for this derivation itself. */
+ if (nrIncompleteClosure > 0 && !retrySubstitution) retrySubstitution = true;
+
+ nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
+
+ if (needRestart) {
+ needRestart = false;
+ haveDerivation();
+ return;
+ }
+
+ unsigned int nrInvalid = checkPathValidity(false, buildMode == bmRepair).size();
+ if (buildMode == bmNormal && nrInvalid == 0) {
+ amDone(ecSuccess);
+ return;
+ }
+ if (buildMode == bmRepair && nrInvalid == 0) {
+ repairClosure();
+ return;
+ }
+ if (buildMode == bmCheck && nrInvalid > 0)
+ throw Error(format("some outputs of `%1%' are not valid, so checking is not possible") % drvPath);
+
+ /* Otherwise, at least one of the output paths could not be
+ produced using a substitute. So we have to build instead. */
+
+ /* Make sure checkPathValidity() from now on checks all
+ outputs. */
+ wantedOutputs = PathSet();
+
+ /* The inputs must be built before we can build this goal. */
+ foreach (DerivationInputs::iterator, i, drv.inputDrvs)
+ addWaitee(worker.makeDerivationGoal(i->first, i->second, buildMode == bmRepair ? bmRepair : bmNormal));
+
+ foreach (PathSet::iterator, i, drv.inputSrcs)
+ addWaitee(worker.makeSubstitutionGoal(*i));
+
+ if (waitees.empty()) /* to prevent hang (no wake-up event) */
+ inputsRealised();
+ else
+ state = &DerivationGoal::inputsRealised;
+}
+
+
+void DerivationGoal::repairClosure()
+{
+ /* If we're repairing, we now know that our own outputs are valid.
+ Now check whether the other paths in the outputs closure are
+ good. If not, then start derivation goals for the derivations
+ that produced those outputs. */
+
+ /* Get the output closure. */
+ PathSet outputClosure;
+ foreach (DerivationOutputs::iterator, i, drv.outputs)
+ computeFSClosure(worker.store, i->second.path, outputClosure);
+
+ /* Filter out our own outputs (which we have already checked). */
+ foreach (DerivationOutputs::iterator, i, drv.outputs)
+ outputClosure.erase(i->second.path);
+
+ /* Get all dependencies of this derivation so that we know which
+ derivation is responsible for which path in the output
+ closure. */
+ PathSet inputClosure;
+ computeFSClosure(worker.store, drvPath, inputClosure);
+ std::map<Path, Path> outputsToDrv;
+ foreach (PathSet::iterator, i, inputClosure)
+ if (isDerivation(*i)) {
+ Derivation drv = derivationFromPath(worker.store, *i);
+ foreach (DerivationOutputs::iterator, j, drv.outputs)
+ outputsToDrv[j->second.path] = *i;
+ }
+
+ /* Check each path (slow!). */
+ PathSet broken;
+ foreach (PathSet::iterator, i, outputClosure) {
+ if (worker.store.pathContentsGood(*i)) continue;
+ printMsg(lvlError, format("found corrupted or missing path `%1%' in the output closure of `%2%'") % *i % drvPath);
+ Path drvPath2 = outputsToDrv[*i];
+ if (drvPath2 == "")
+ addWaitee(worker.makeSubstitutionGoal(*i, true));
+ else
+ addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair));
+ }
+
+ if (waitees.empty()) {
+ amDone(ecSuccess);
+ return;
+ }
+
+ state = &DerivationGoal::closureRepaired;
+}
+
+
+void DerivationGoal::closureRepaired()
+{
+ trace("closure repaired");
+ if (nrFailed > 0)
+ throw Error(format("some paths in the output closure of derivation `%1%' could not be repaired") % drvPath);
+ amDone(ecSuccess);
+}
+
+
+void DerivationGoal::inputsRealised()
+{
+ trace("all inputs realised");
+
+ if (nrFailed != 0) {
+ printMsg(lvlError,
+ format("cannot build derivation `%1%': %2% dependencies couldn't be built")
+ % drvPath % nrFailed);
+ amDone(ecFailed);
+ return;
+ }
+
+ if (retrySubstitution) {
+ haveDerivation();
+ return;
+ }
+
+ /* Gather information necessary for computing the closure and/or
+ running the build hook. */
+
+ /* The outputs are referenceable paths. */
+ foreach (DerivationOutputs::iterator, i, drv.outputs) {
+ debug(format("building path `%1%'") % i->second.path);
+ allPaths.insert(i->second.path);
+ }
+
+ /* Determine the full set of input paths. */
+
+ /* First, the input derivations. */
+ foreach (DerivationInputs::iterator, i, drv.inputDrvs) {
+ /* Add the relevant output closures of the input derivation
+ `*i' as input paths. Only add the closures of output paths
+ that are specified as inputs. */
+ assert(worker.store.isValidPath(i->first));
+ Derivation inDrv = derivationFromPath(worker.store, i->first);
+ foreach (StringSet::iterator, j, i->second)
+ if (inDrv.outputs.find(*j) != inDrv.outputs.end())
+ computeFSClosure(worker.store, inDrv.outputs[*j].path, inputPaths);
+ else
+ throw Error(
+ format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'")
+ % drvPath % *j % i->first);
+ }
+
+ /* Second, the input sources. */
+ foreach (PathSet::iterator, i, drv.inputSrcs)
+ computeFSClosure(worker.store, *i, inputPaths);
+
+ debug(format("added input paths %1%") % showPaths(inputPaths));
+
+ allPaths.insert(inputPaths.begin(), inputPaths.end());
+
+ /* Is this a fixed-output derivation? */
+ fixedOutput = true;
+ foreach (DerivationOutputs::iterator, i, drv.outputs)
+ if (i->second.hash == "") fixedOutput = false;
+
+ /* Okay, try to build. Note that here we don't wait for a build
+ slot to become available, since we don't need one if there is a
+ build hook. */
+ state = &DerivationGoal::tryToBuild;
+ worker.wakeUp(shared_from_this());
+}
+
+
+PathSet outputPaths(const DerivationOutputs & outputs)
+{
+ PathSet paths;
+ foreach (DerivationOutputs::const_iterator, i, outputs)
+ paths.insert(i->second.path);
+ return paths;
+}
+
+
+static string get(const StringPairs & map, const string & key)
+{
+ StringPairs::const_iterator i = map.find(key);
+ return i == map.end() ? (string) "" : i->second;
+}
+
+
+static bool canBuildLocally(const string & platform)
+{
+ return platform == settings.thisSystem
+#ifdef CAN_DO_LINUX32_BUILDS
+ || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux")
+#endif
+ ;
+}
+
+
+bool willBuildLocally(const Derivation & drv)
+{
+ return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform);
+}
+
+
+void DerivationGoal::tryToBuild()
+{
+ trace("trying to build");
+
+ /* Check for the possibility that some other goal in this process
+ has locked the output since we checked in haveDerivation().
+ (It can't happen between here and the lockPaths() call below
+ because we're not allowing multi-threading.) If so, put this
+ goal to sleep until another goal finishes, then try again. */
+ foreach (DerivationOutputs::iterator, i, drv.outputs)
+ if (pathIsLockedByMe(i->second.path)) {
+ debug(format("putting derivation `%1%' to sleep because `%2%' is locked by another goal")
+ % drvPath % i->second.path);
+ worker.waitForAnyGoal(shared_from_this());
+ return;
+ }
+
+ /* Obtain locks on all output paths. The locks are automatically
+ released when we exit this function or Nix crashes. If we
+ can't acquire the lock, then continue; hopefully some other
+ goal can start a build, and if not, the main loop will sleep a
+ few seconds and then retry this goal. */
+ if (!outputLocks.lockPaths(outputPaths(drv.outputs), "", false)) {
+ worker.waitForAWhile(shared_from_this());
+ return;
+ }
+
+ /* Now check again whether the outputs are valid. This is because
+ another process may have started building in parallel. After
+ it has finished and released the locks, we can (and should)
+ reuse its results. (Strictly speaking the first check can be
+ omitted, but that would be less efficient.) Note that since we
+ now hold the locks on the output paths, no other process can
+ build this derivation, so no further checks are necessary. */
+ validPaths = checkPathValidity(true, buildMode == bmRepair);
+ assert(buildMode != bmCheck || validPaths.size() == drv.outputs.size());
+ if (buildMode != bmCheck && validPaths.size() == drv.outputs.size()) {
+ debug(format("skipping build of derivation `%1%', someone beat us to it") % drvPath);
+ outputLocks.setDeletion(true);
+ amDone(ecSuccess);
+ return;
+ }
+
+ missingPaths = outputPaths(drv.outputs);
+ if (buildMode != bmCheck)
+ foreach (PathSet::iterator, i, validPaths) missingPaths.erase(*i);
+
+ /* If any of the outputs already exist but are not valid, delete
+ them. */
+ foreach (DerivationOutputs::iterator, i, drv.outputs) {
+ Path path = i->second.path;
+ if (worker.store.isValidPath(path)) continue;
+ if (!pathExists(path)) continue;
+ debug(format("removing invalid path `%1%'") % path);
+ deletePath(path);
+ }
+
+ /* Check again whether any output previously failed to build,
+ because some other process may have tried and failed before we
+ acquired the lock. */
+ foreach (DerivationOutputs::iterator, i, drv.outputs)
+ if (pathFailed(i->second.path)) return;
+
+ /* Don't do a remote build if the derivation has the attribute
+ `preferLocalBuild' set. Also, check and repair modes are only
+ supported for local builds. */
+ bool buildLocally = buildMode != bmNormal || willBuildLocally(drv);
+
+ /* Is the build hook willing to accept this job? */
+ if (!buildLocally) {
+ switch (tryBuildHook()) {
+ case rpAccept:
+ /* Yes, it has started doing so. Wait until we get
+ EOF from the hook. */
+ state = &DerivationGoal::buildDone;
+ return;
+ case rpPostpone:
+ /* Not now; wait until at least one child finishes or
+ the wake-up timeout expires. */
+ worker.waitForAWhile(shared_from_this());
+ outputLocks.unlock();
+ return;
+ case rpDecline:
+ /* We should do it ourselves. */
+ break;
+ }
+ }
+
+ /* Make sure that we are allowed to start a build. If this
+ derivation prefers to be done locally, do it even if
+ maxBuildJobs is 0. */
+ unsigned int curBuilds = worker.getNrLocalBuilds();
+ if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
+ worker.waitForBuildSlot(shared_from_this());
+ outputLocks.unlock();
+ return;
+ }
+
+ try {
+
+ /* Okay, we have to build. */
+ startBuilder();
+
+ } catch (BuildError & e) {
+ printMsg(lvlError, e.msg());
+ outputLocks.unlock();
+ buildUser.release();
+ if (settings.printBuildTrace)
+ printMsg(lvlError, format("@ build-failed %1% - %2% %3%")
+ % drvPath % 0 % e.msg());
+ worker.permanentFailure = true;
+ amDone(ecFailed);
+ return;
+ }
+
+ /* This state will be reached when we get EOF on the child's
+ log pipe. */
+ state = &DerivationGoal::buildDone;
+}
+
+
+void replaceValidPath(const Path & storePath, const Path tmpPath)
+{
+ /* We can't atomically replace storePath (the original) with
+ tmpPath (the replacement), so we have to move it out of the
+ way first. We'd better not be interrupted here, because if
+ we're repairing (say) Glibc, we end up with a broken system. */
+ Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % rand()).str();
+ if (pathExists(storePath))
+ rename(storePath.c_str(), oldPath.c_str());
+ if (rename(tmpPath.c_str(), storePath.c_str()) == -1)
+ throw SysError(format("moving `%1%' to `%2%'") % tmpPath % storePath);
+ if (pathExists(oldPath))
+ deletePath(oldPath);
+}
+
+
+void DerivationGoal::buildDone()
+{
+ trace("build done");
+
+ /* Since we got an EOF on the logger pipe, the builder is presumed
+ to have terminated. In fact, the builder could also have
+ simply have closed its end of the pipe --- just don't do that
+ :-) */
+ int status;
+ pid_t savedPid;
+ if (hook) {
+ savedPid = hook->pid;
+ status = hook->pid.wait(true);
+ } else {
+ /* !!! this could block! security problem! solution: kill the
+ child */
+ savedPid = pid;
+ status = pid.wait(true);
+ }
+
+ debug(format("builder process for `%1%' finished") % drvPath);
+
+ /* So the child is gone now. */
+ worker.childTerminated(savedPid);
+
+ /* Close the read side of the logger pipe. */
+ if (hook) {
+ hook->builderOut.readSide.close();
+ hook->fromHook.readSide.close();
+ }
+ else builderOut.readSide.close();
+
+ /* Close the log file. */
+ closeLogFile();
+
+ /* When running under a build user, make sure that all processes
+ running under that uid are gone. This is to prevent a
+ malicious user from leaving behind a process that keeps files
+ open and modifies them after they have been chown'ed to
+ root. */
+ if (buildUser.enabled()) buildUser.kill();
+
+ bool diskFull = false;
+
+ try {
+
+ /* Check the exit status. */
+ if (!statusOk(status)) {
+
+ /* Heuristically check whether the build failure may have
+ been caused by a disk full condition. We have no way
+ of knowing whether the build actually got an ENOSPC.
+ So instead, check if the disk is (nearly) full now. If
+ so, we don't mark this build as a permanent failure. */
+#if HAVE_STATVFS
+ unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable
+ struct statvfs st;
+ if (statvfs(settings.nixStore.c_str(), &st) == 0 &&
+ (unsigned long long) st.f_bavail * st.f_bsize < required)
+ diskFull = true;
+ if (statvfs(tmpDir.c_str(), &st) == 0 &&
+ (unsigned long long) st.f_bavail * st.f_bsize < required)
+ diskFull = true;
+#endif
+
+ deleteTmpDir(false);
+
+ /* Move paths out of the chroot for easier debugging of
+ build failures. */
+ if (useChroot && buildMode == bmNormal)
+ foreach (PathSet::iterator, i, missingPaths)
+ if (pathExists(chrootRootDir + *i))
+ rename((chrootRootDir + *i).c_str(), i->c_str());
+
+ if (WIFEXITED(status) && WEXITSTATUS(status) == childSetupFailed)
+ throw Error(format("failed to set up the build environment for `%1%'") % drvPath);
+
+ if (diskFull)
+ printMsg(lvlError, "note: build failure may have been caused by lack of free disk space");
+
+ throw BuildError(format("builder for `%1%' %2%")
+ % drvPath % statusToString(status));
+ }
+
+ /* Compute the FS closure of the outputs and register them as
+ being valid. */
+ registerOutputs();
+
+ if (buildMode == bmCheck) {
+ amDone(ecSuccess);
+ return;
+ }
+
+ /* Delete unused redirected outputs (when doing hash rewriting). */
+ foreach (RedirectedOutputs::iterator, i, redirectedOutputs)
+ if (pathExists(i->second)) deletePath(i->second);
+
+ /* Delete the chroot (if we were using one). */
+ autoDelChroot.reset(); /* this runs the destructor */
+
+ deleteTmpDir(true);
+
+ /* It is now safe to delete the lock files, since all future
+ lockers will see that the output paths are valid; they will
+ not create new lock files with the same names as the old
+ (unlinked) lock files. */
+ outputLocks.setDeletion(true);
+ outputLocks.unlock();
+
+ } catch (BuildError & e) {
+ printMsg(lvlError, e.msg());
+ outputLocks.unlock();
+ buildUser.release();
+
+ /* When using a build hook, the hook will return a remote
+ build failure using exit code 100. Anything else is a hook
+ problem. */
+ bool hookError = hook &&
+ (!WIFEXITED(status) || WEXITSTATUS(status) != 100);
+
+ if (settings.printBuildTrace) {
+ if (hook && hookError)
+ printMsg(lvlError, format("@ hook-failed %1% - %2% %3%")
+ % drvPath % status % e.msg());
+ else
+ printMsg(lvlError, format("@ build-failed %1% - %2% %3%")
+ % drvPath % 1 % e.msg());
+ }
+
+ /* Register the outputs of this build as "failed" so we won't
+ try to build them again (negative caching). However, don't
+ do this for fixed-output derivations, since they're likely
+ to fail for transient reasons (e.g., fetchurl not being
+ able to access the network). Hook errors (like
+ communication problems with the remote machine) shouldn't
+ be cached either. */
+ if (settings.cacheFailure && !hookError && !fixedOutput)
+ foreach (DerivationOutputs::iterator, i, drv.outputs)
+ worker.store.registerFailedPath(i->second.path);
+
+ worker.permanentFailure = !hookError && !fixedOutput && !diskFull;
+ amDone(ecFailed);
+ return;
+ }
+
+ /* Release the build user, if applicable. */
+ buildUser.release();
+
+ if (settings.printBuildTrace)
+ printMsg(lvlError, format("@ build-succeeded %1% -") % drvPath);
+
+ amDone(ecSuccess);
+}
+
+
+HookReply DerivationGoal::tryBuildHook()
+{
+ if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline;
+
+ if (!worker.hook)
+ worker.hook = std::shared_ptr<HookInstance>(new HookInstance);
+
+ /* Tell the hook about system features (beyond the system type)
+ required from the build machine. (The hook could parse the
+ drv file itself, but this is easier.) */
+ Strings features = tokenizeString<Strings>(get(drv.env, "requiredSystemFeatures"));
+ foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */
+
+ /* Send the request to the hook. */
+ writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%")
+ % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0")
+ % drv.platform % drvPath % concatStringsSep(",", features)).str());
+
+ /* Read the first line of input, which should be a word indicating
+ whether the hook wishes to perform the build. */
+ string reply;
+ while (true) {
+ string s = readLine(worker.hook->fromHook.readSide);
+ if (string(s, 0, 2) == "# ") {
+ reply = string(s, 2);
+ break;
+ }
+ s += "\n";
+ writeToStderr(s);
+ }
+
+ debug(format("hook reply is `%1%'") % reply);
+
+ if (reply == "decline" || reply == "postpone")
+ return reply == "decline" ? rpDecline : rpPostpone;
+ else if (reply != "accept")
+ throw Error(format("bad hook reply `%1%'") % reply);
+
+ printMsg(lvlTalkative, format("using hook to build path(s) %1%") % showPaths(missingPaths));
+
+ hook = worker.hook;
+ worker.hook.reset();
+
+ /* Tell the hook all the inputs that have to be copied to the
+ remote system. This unfortunately has to contain the entire
+ derivation closure to ensure that the validity invariant holds
+ on the remote system. (I.e., it's unfortunate that we have to
+ list it since the remote system *probably* already has it.) */
+ PathSet allInputs;
+ allInputs.insert(inputPaths.begin(), inputPaths.end());
+ computeFSClosure(worker.store, drvPath, allInputs);
+
+ string s;
+ foreach (PathSet::iterator, i, allInputs) { s += *i; s += ' '; }
+ writeLine(hook->toHook.writeSide, s);
+
+ /* Tell the hooks the missing outputs that have to be copied back
+ from the remote system. */
+ s = "";
+ foreach (PathSet::iterator, i, missingPaths) { s += *i; s += ' '; }
+ writeLine(hook->toHook.writeSide, s);
+
+ hook->toHook.writeSide.close();
+
+ /* Create the log file and pipe. */
+ Path logFile = openLogFile();
+
+ set<int> fds;
+ fds.insert(hook->fromHook.readSide);
+ fds.insert(hook->builderOut.readSide);
+ worker.childStarted(shared_from_this(), hook->pid, fds, false, false);
+
+ if (settings.printBuildTrace)
+ printMsg(lvlError, format("@ build-started %1% - %2% %3%")
+ % drvPath % drv.platform % logFile);
+
+ return rpAccept;
+}
+
+
+void chmod_(const Path & path, mode_t mode)
+{
+ if (chmod(path.c_str(), mode) == -1)
+ throw SysError(format("setting permissions on `%1%'") % path);
+}
+
+
+int childEntry(void * arg)
+{
+ ((DerivationGoal *) arg)->initChild();
+ return 1;
+}
+
+
+void DerivationGoal::startBuilder()
+{
+ startNest(nest, lvlInfo, format(
+ buildMode == bmRepair ? "repairing path(s) %1%" :
+ buildMode == bmCheck ? "checking path(s) %1%" :
+ "building path(s) %1%") % showPaths(missingPaths));
+
+ /* Right platform? */
+ if (!canBuildLocally(drv.platform)) {
+ if (settings.printBuildTrace)
+ printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv.platform);
+ throw Error(
+ format("a `%1%' is required to build `%3%', but I am a `%2%'")
+ % drv.platform % settings.thisSystem % drvPath);
+ }
+
+ /* Construct the environment passed to the builder. */
+
+ /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
+ PATH is not set. We don't want this, so we fill it in with some dummy
+ value. */
+ env["PATH"] = "/path-not-set";
+
+ /* Set HOME to a non-existing path to prevent certain programs from using
+ /etc/passwd (or NIS, or whatever) to locate the home directory (for
+ example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd
+ if HOME is not set, but they will just assume that the settings file
+ they are looking for does not exist if HOME is set but points to some
+ non-existing path. */
+ Path homeDir = "/homeless-shelter";
+ env["HOME"] = homeDir;
+
+ /* Tell the builder where the Nix store is. Usually they
+ shouldn't care, but this is useful for purity checking (e.g.,
+ the compiler or linker might only want to accept paths to files
+ in the store or in the build directory). */
+ env["NIX_STORE"] = settings.nixStore;
+
+ /* The maximum number of cores to utilize for parallel building. */
+ env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
+
+ /* Add all bindings specified in the derivation. */
+ foreach (StringPairs::iterator, i, drv.env)
+ env[i->first] = i->second;
+
+ /* Create a temporary directory where the build will take
+ place. */
+ tmpDir = createTempDir("", "nix-build-" + storePathToName(drvPath), false, false, 0700);
+
+ /* For convenience, set an environment pointing to the top build
+ directory. */
+ env["NIX_BUILD_TOP"] = tmpDir;
+
+ /* Also set TMPDIR and variants to point to this directory. */
+ env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir;
+
+ /* Explicitly set PWD to prevent problems with chroot builds. In
+ particular, dietlibc cannot figure out the cwd because the
+ inode of the current directory doesn't appear in .. (because
+ getdents returns the inode of the mount point). */
+ env["PWD"] = tmpDir;
+
+ /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
+ derivation, tell the builder, so that for instance `fetchurl'
+ can skip checking the output. On older Nixes, this environment
+ variable won't be set, so `fetchurl' will do the check. */
+ if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1";
+
+ /* *Only* if this is a fixed-output derivation, propagate the
+ values of the environment variables specified in the
+ `impureEnvVars' attribute to the builder. This allows for
+ instance environment variables for proxy configuration such as
+ `http_proxy' to be easily passed to downloaders like
+ `fetchurl'. Passing such environment variables from the caller
+ to the builder is generally impure, but the output of
+ fixed-output derivations is by definition pure (since we
+ already know the cryptographic hash of the output). */
+ if (fixedOutput) {
+ Strings varNames = tokenizeString<Strings>(get(drv.env, "impureEnvVars"));
+ foreach (Strings::iterator, i, varNames) env[*i] = getEnv(*i);
+ }
+
+ /* The `exportReferencesGraph' feature allows the references graph
+ to be passed to a builder. This attribute should be a list of
+ pairs [name1 path1 name2 path2 ...]. The references graph of
+ each `pathN' will be stored in a text file `nameN' in the
+ temporary build directory. The text files have the format used
+ by `nix-store --register-validity'. However, the deriver
+ fields are left empty. */
+ string s = get(drv.env, "exportReferencesGraph");
+ Strings ss = tokenizeString<Strings>(s);
+ if (ss.size() % 2 != 0)
+ throw BuildError(format("odd number of tokens in `exportReferencesGraph': `%1%'") % s);
+ for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
+ string fileName = *i++;
+ checkStoreName(fileName); /* !!! abuse of this function */
+
+ /* Check that the store path is valid. */
+ Path storePath = *i++;
+ if (!isInStore(storePath))
+ throw BuildError(format("`exportReferencesGraph' contains a non-store path `%1%'")
+ % storePath);
+ storePath = toStorePath(storePath);
+ if (!worker.store.isValidPath(storePath))
+ throw BuildError(format("`exportReferencesGraph' contains an invalid path `%1%'")
+ % storePath);
+
+ /* If there are derivations in the graph, then include their
+ outputs as well. This is useful if you want to do things
+ like passing all build-time dependencies of some path to a
+ derivation that builds a NixOS DVD image. */
+ PathSet paths, paths2;
+ computeFSClosure(worker.store, storePath, paths);
+ paths2 = paths;
+
+ foreach (PathSet::iterator, j, paths2) {
+ if (isDerivation(*j)) {
+ Derivation drv = derivationFromPath(worker.store, *j);
+ foreach (DerivationOutputs::iterator, k, drv.outputs)
+ computeFSClosure(worker.store, k->second.path, paths);
+ }
+ }
+
+ /* Write closure info to `fileName'. */
+ writeFile(tmpDir + "/" + fileName,
+ worker.store.makeValidityRegistration(paths, false, false));
+ }
+
+
+ /* If `build-users-group' is not empty, then we have to build as
+ one of the members of that group. */
+ if (settings.buildUsersGroup != "") {
+ buildUser.acquire();
+ assert(buildUser.getUID() != 0);
+ assert(buildUser.getGID() != 0);
+
+ /* Make sure that no other processes are executing under this
+ uid. */
+ buildUser.kill();
+
+ /* Change ownership of the temporary build directory. */
+ if (chown(tmpDir.c_str(), buildUser.getUID(), buildUser.getGID()) == -1)
+ throw SysError(format("cannot change ownership of `%1%'") % tmpDir);
+
+ /* Check that the Nix store has the appropriate permissions,
+ i.e., owned by root and mode 1775 (sticky bit on so that
+ the builder can create its output but not mess with the
+ outputs of other processes). */
+ struct stat st;
+ if (stat(settings.nixStore.c_str(), &st) == -1)
+ throw SysError(format("cannot stat `%1%'") % settings.nixStore);
+ if (!(st.st_mode & S_ISVTX) ||
+ ((st.st_mode & S_IRWXG) != S_IRWXG) ||
+ (st.st_gid != buildUser.getGID()))
+ throw Error(format(
+ "builder does not have write permission to `%2%'; "
+ "try `chgrp %1% %2%; chmod 1775 %2%'")
+ % buildUser.getGID() % settings.nixStore);
+ }
+
+
+ /* Are we doing a chroot build? Note that fixed-output
+ derivations are never done in a chroot, mainly so that
+ functions like fetchurl (which needs a proper /etc/resolv.conf)
+ work properly. Purity checking for fixed-output derivations
+ is somewhat pointless anyway. */
+ useChroot = settings.useChroot;
+
+ if (fixedOutput) useChroot = false;
+
+ /* Hack to allow derivations to disable chroot builds. */
+ if (get(drv.env, "__noChroot") == "1") useChroot = false;
+
+ if (useChroot) {
+#if CHROOT_ENABLED
+ /* Create a temporary directory in which we set up the chroot
+ environment using bind-mounts. We put it in the Nix store
+ to ensure that we can create hard-links to non-directory
+ inputs in the fake Nix store in the chroot (see below). */
+ chrootRootDir = drvPath + ".chroot";
+ if (pathExists(chrootRootDir)) deletePath(chrootRootDir);
+
+ /* Clean up the chroot directory automatically. */
+ autoDelChroot = std::shared_ptr<AutoDelete>(new AutoDelete(chrootRootDir));
+
+ printMsg(lvlChatty, format("setting up chroot environment in `%1%'") % chrootRootDir);
+
+ /* Create a writable /tmp in the chroot. Many builders need
+ this. (Of course they should really respect $TMPDIR
+ instead.) */
+ Path chrootTmpDir = chrootRootDir + "/tmp";
+ createDirs(chrootTmpDir);
+ chmod_(chrootTmpDir, 01777);
+
+ /* Create a /etc/passwd with entries for the build user and the
+ nobody account. The latter is kind of a hack to support
+ Samba-in-QEMU. */
+ createDirs(chrootRootDir + "/etc");
+
+ writeFile(chrootRootDir + "/etc/passwd",
+ (format(
+ "nixbld:x:%1%:%2%:Nix build user:/:/noshell\n"
+ "nobody:x:65534:65534:Nobody:/:/noshell\n")
+ % (buildUser.enabled() ? buildUser.getUID() : getuid())
+ % (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
+
+ /* Declare the build user's group so that programs get a consistent
+ view of the system (e.g., "id -gn"). */
+ writeFile(chrootRootDir + "/etc/group",
+ (format("nixbld:!:%1%:\n")
+ % (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
+
+ /* Create /etc/hosts with localhost entry. */
+ writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n");
+
+ /* Bind-mount a user-configurable set of directories from the
+ host file system. */
+ foreach (StringSet::iterator, i, settings.dirsInChroot) {
+ size_t p = i->find('=');
+ if (p == string::npos)
+ dirsInChroot[*i] = *i;
+ else
+ dirsInChroot[string(*i, 0, p)] = string(*i, p + 1);
+ }
+ dirsInChroot[tmpDir] = tmpDir;
+
+ /* Make the closure of the inputs available in the chroot,
+ rather than the whole Nix store. This prevents any access
+ to undeclared dependencies. Directories are bind-mounted,
+ while other inputs are hard-linked (since only directories
+ can be bind-mounted). !!! As an extra security
+ precaution, make the fake Nix store only writable by the
+ build user. */
+ createDirs(chrootRootDir + settings.nixStore);
+ chmod_(chrootRootDir + settings.nixStore, 01777);
+
+ foreach (PathSet::iterator, i, inputPaths) {
+ struct stat st;
+ if (lstat(i->c_str(), &st))
+ throw SysError(format("getting attributes of path `%1%'") % *i);
+ if (S_ISDIR(st.st_mode))
+ dirsInChroot[*i] = *i;
+ else {
+ Path p = chrootRootDir + *i;
+ if (link(i->c_str(), p.c_str()) == -1) {
+ /* Hard-linking fails if we exceed the maximum
+ link count on a file (e.g. 32000 of ext3),
+ which is quite possible after a `nix-store
+ --optimise'. */
+ if (errno != EMLINK)
+ throw SysError(format("linking `%1%' to `%2%'") % p % *i);
+ StringSink sink;
+ dumpPath(*i, sink);
+ StringSource source(sink.s);
+ restorePath(p, source);
+ }
+
+ regularInputPaths.insert(*i);
+ }
+ }
+
+ /* If we're repairing or checking, it's possible that we're
+ rebuilding a path that is in settings.dirsInChroot
+ (typically the dependencies of /bin/sh). Throw them
+ out. */
+ if (buildMode != bmNormal)
+ foreach (DerivationOutputs::iterator, i, drv.outputs)
+ dirsInChroot.erase(i->second.path);
+
+#else
+ throw Error("chroot builds are not supported on this platform");
+#endif
+ }
+
+ else {
+
+ if (pathExists(homeDir))
+ throw Error(format("directory `%1%' exists; please remove it") % homeDir);
+
+ /* We're not doing a chroot build, but we have some valid
+ output paths. Since we can't just overwrite or delete
+ them, we have to do hash rewriting: i.e. in the
+ environment/arguments passed to the build, we replace the
+ hashes of the valid outputs with unique dummy strings;
+ after the build, we discard the redirected outputs
+ corresponding to the valid outputs, and rewrite the
+ contents of the new outputs to replace the dummy strings
+ with the actual hashes. */
+ if (validPaths.size() > 0)
+ foreach (PathSet::iterator, i, validPaths)
+ addHashRewrite(*i);
+
+ /* If we're repairing, then we don't want to delete the
+ corrupt outputs in advance. So rewrite them as well. */
+ if (buildMode == bmRepair)
+ foreach (PathSet::iterator, i, missingPaths)
+ if (worker.store.isValidPath(*i) && pathExists(*i)) {
+ addHashRewrite(*i);
+ redirectedBadOutputs.insert(*i);
+ }
+ }
+
+
+ /* Run the builder. */
+ printMsg(lvlChatty, format("executing builder `%1%'") % drv.builder);
+
+ /* Create the log file. */
+ Path logFile = openLogFile();
+
+ /* Create a pipe to get the output of the builder. */
+ builderOut.create();
+
+ /* Fork a child to build the package. Note that while we
+ currently use forks to run and wait for the children, it
+ shouldn't be hard to use threads for this on systems where
+ fork() is unavailable or inefficient.
+
+ If we're building in a chroot, then also set up private
+ namespaces for the build:
+
+ - The PID namespace causes the build to start as PID 1.
+ Processes outside of the chroot are not visible to those on
+ the inside, but processes inside the chroot are visible from
+ the outside (though with different PIDs).
+
+ - The private mount namespace ensures that all the bind mounts
+ we do will only show up in this process and its children, and
+ will disappear automatically when we're done.
+
+ - The private network namespace ensures that the builder cannot
+ talk to the outside world (or vice versa). It only has a
+ private loopback interface.
+
+ - The IPC namespace prevents the builder from communicating
+ with outside processes using SysV IPC mechanisms (shared
+ memory, message queues, semaphores). It also ensures that
+ all IPC objects are destroyed when the builder exits.
+
+ - The UTS namespace ensures that builders see a hostname of
+ localhost rather than the actual hostname.
+ */
+#if CHROOT_ENABLED
+ if (useChroot) {
+ char stack[32 * 1024];
+ pid = clone(childEntry, stack + sizeof(stack) - 8,
+ CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWNET | CLONE_NEWIPC | CLONE_NEWUTS | SIGCHLD, this);
+ } else
+#endif
+ {
+ pid = fork();
+ if (pid == 0) initChild();
+ }
+
+ if (pid == -1) throw SysError("unable to fork");
+
+ /* parent */
+ pid.setSeparatePG(true);
+ builderOut.writeSide.close();
+ worker.childStarted(shared_from_this(), pid,
+ singleton<set<int> >(builderOut.readSide), true, true);
+
+ if (settings.printBuildTrace) {
+ printMsg(lvlError, format("@ build-started %1% - %2% %3%")
+ % drvPath % drv.platform % logFile);
+ }
+}
+
+
+void DerivationGoal::initChild()
+{
+ /* Warning: in the child we should absolutely not make any SQLite
+ calls! */
+
+ bool inSetup = true;
+
+ try { /* child */
+
+#if CHROOT_ENABLED
+ if (useChroot) {
+ /* Initialise the loopback interface. */
+ AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
+ if (fd == -1) throw SysError("cannot open IP socket");
+
+ struct ifreq ifr;
+ strcpy(ifr.ifr_name, "lo");
+ ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
+ if (ioctl(fd, SIOCSIFFLAGS, &ifr) == -1)
+ throw SysError("cannot set loopback interface flags");
+
+ fd.close();
+
+ /* Set the hostname etc. to fixed values. */
+ char hostname[] = "localhost";
+ sethostname(hostname, sizeof(hostname));
+ char domainname[] = "(none)"; // kernel default
+ setdomainname(domainname, sizeof(domainname));
+
+ /* Make all filesystems private. This is necessary
+ because subtrees may have been mounted as "shared"
+ (MS_SHARED). (Systemd does this, for instance.) Even
+ though we have a private mount namespace, mounting
+ filesystems on top of a shared subtree still propagates
+ outside of the namespace. Making a subtree private is
+ local to the namespace, though, so setting MS_PRIVATE
+ does not affect the outside world. */
+ Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n");
+ foreach (Strings::iterator, i, mounts) {
+ vector<string> fields = tokenizeString<vector<string> >(*i, " ");
+ string fs = decodeOctalEscaped(fields.at(4));
+ if (mount(0, fs.c_str(), 0, MS_PRIVATE, 0) == -1)
+ throw SysError(format("unable to make filesystem `%1%' private") % fs);
+ }
+
+ /* Set up a nearly empty /dev, unless the user asked to
+ bind-mount the host /dev. */
+ if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
+ createDirs(chrootRootDir + "/dev/shm");
+ createDirs(chrootRootDir + "/dev/pts");
+ Strings ss;
+ ss.push_back("/dev/full");
+#ifdef __linux__
+ if (pathExists("/dev/kvm"))
+ ss.push_back("/dev/kvm");
+#endif
+ ss.push_back("/dev/null");
+ ss.push_back("/dev/random");
+ ss.push_back("/dev/tty");
+ ss.push_back("/dev/urandom");
+ ss.push_back("/dev/zero");
+ foreach (Strings::iterator, i, ss) dirsInChroot[*i] = *i;
+ createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
+ createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
+ createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
+ createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
+ }
+
+ /* Bind-mount all the directories from the "host"
+ filesystem that we want in the chroot
+ environment. */
+ foreach (DirsInChroot::iterator, i, dirsInChroot) {
+ struct stat st;
+ Path source = i->second;
+ Path target = chrootRootDir + i->first;
+ if (source == "/proc") continue; // backwards compatibility
+ debug(format("bind mounting `%1%' to `%2%'") % source % target);
+ if (stat(source.c_str(), &st) == -1)
+ throw SysError(format("getting attributes of path `%1%'") % source);
+ if (S_ISDIR(st.st_mode))
+ createDirs(target);
+ else {
+ createDirs(dirOf(target));
+ writeFile(target, "");
+ }
+ if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
+ throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target);
+ }
+
+ /* Bind a new instance of procfs on /proc to reflect our
+ private PID namespace. */
+ createDirs(chrootRootDir + "/proc");
+ if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
+ throw SysError("mounting /proc");
+
+ /* Mount a new tmpfs on /dev/shm to ensure that whatever
+ the builder puts in /dev/shm is cleaned up automatically. */
+ if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, 0) == -1)
+ throw SysError("mounting /dev/shm");
+
+ /* Mount a new devpts on /dev/pts. Note that this
+ requires the kernel to be compiled with
+ CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
+ if /dev/ptx/ptmx exists). */
+ if (pathExists("/dev/pts/ptmx") &&
+ !pathExists(chrootRootDir + "/dev/ptmx")
+ && dirsInChroot.find("/dev/pts") == dirsInChroot.end())
+ {
+ if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == -1)
+ throw SysError("mounting /dev/pts");
+ createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
+
+ /* Make sure /dev/pts/ptmx is world-writable. With some
+ Linux versions, it is created with permissions 0. */
+ chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
+ }
+
+ /* Do the chroot(). Below we do a chdir() to the
+ temporary build directory to make sure the current
+ directory is in the chroot. (Actually the order
+ doesn't matter, since due to the bind mount tmpDir and
+ tmpRootDit/tmpDir are the same directories.) */
+ if (chroot(chrootRootDir.c_str()) == -1)
+ throw SysError(format("cannot change root directory to `%1%'") % chrootRootDir);
+ }
+#endif
+
+ commonChildInit(builderOut);
+
+ if (chdir(tmpDir.c_str()) == -1)
+ throw SysError(format("changing into `%1%'") % tmpDir);
+
+ /* Close all other file descriptors. */
+ closeMostFDs(set<int>());
+
+#ifdef CAN_DO_LINUX32_BUILDS
+ /* Change the personality to 32-bit if we're doing an
+ i686-linux build on an x86_64-linux machine. */
+ struct utsname utsbuf;
+ uname(&utsbuf);
+ if (drv.platform == "i686-linux" &&
+ (settings.thisSystem == "x86_64-linux" ||
+ (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
+ if (personality(0x0008 | 0x8000000 /* == PER_LINUX32_3GB */) == -1)
+ throw SysError("cannot set i686-linux personality");
+ }
+
+ /* Impersonate a Linux 2.6 machine to get some determinism in
+ builds that depend on the kernel version. */
+ if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && settings.impersonateLinux26) {
+ int cur = personality(0xffffffff);
+ if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
+ }
+#endif
+
+ /* Fill in the environment. */
+ Strings envStrs;
+ foreach (Environment::const_iterator, i, env)
+ envStrs.push_back(rewriteHashes(i->first + "=" + i->second, rewritesToTmp));
+ const char * * envArr = strings2CharPtrs(envStrs);
+
+ Path program = drv.builder.c_str();
+ std::vector<const char *> args; /* careful with c_str()! */
+ string user; /* must be here for its c_str()! */
+
+ /* If we are running in `build-users' mode, then switch to the
+ user we allocated above. Make sure that we drop all root
+ privileges. Note that above we have closed all file
+ descriptors except std*, so that's safe. Also note that
+ setuid() when run as root sets the real, effective and
+ saved UIDs. */
+ if (buildUser.enabled()) {
+ printMsg(lvlChatty, format("switching to user `%1%'") % buildUser.getUser());
+
+ if (setgroups(0, 0) == -1)
+ throw SysError("cannot clear the set of supplementary groups");
+
+ if (setgid(buildUser.getGID()) == -1 ||
+ getgid() != buildUser.getGID() ||
+ getegid() != buildUser.getGID())
+ throw SysError("setgid failed");
+
+ if (setuid(buildUser.getUID()) == -1 ||
+ getuid() != buildUser.getUID() ||
+ geteuid() != buildUser.getUID())
+ throw SysError("setuid failed");
+ }
+
+ /* Fill in the arguments. */
+ string builderBasename = baseNameOf(drv.builder);
+ args.push_back(builderBasename.c_str());
+ foreach (Strings::iterator, i, drv.args)
+ args.push_back(rewriteHashes(*i, rewritesToTmp).c_str());
+ args.push_back(0);
+
+ restoreSIGPIPE();
+
+ /* Execute the program. This should not return. */
+ inSetup = false;
+ execve(program.c_str(), (char * *) &args[0], (char * *) envArr);
+
+ throw SysError(format("executing `%1%'") % drv.builder);
+
+ } catch (std::exception & e) {
+ writeToStderr("build error: " + string(e.what()) + "\n");
+ _exit(inSetup ? childSetupFailed : 1);
+ }
+
+ abort(); /* never reached */
+}
+
+
+/* Parse a list of reference specifiers. Each element must either be
+ a store path, or the symbolic name of the output of the derivation
+ (such as `out'). */
+PathSet parseReferenceSpecifiers(const Derivation & drv, string attr)
+{
+ PathSet result;
+ Paths paths = tokenizeString<Paths>(attr);
+ foreach (Strings::iterator, i, paths) {
+ if (isStorePath(*i))
+ result.insert(*i);
+ else if (drv.outputs.find(*i) != drv.outputs.end())
+ result.insert(drv.outputs.find(*i)->second.path);
+ else throw BuildError(
+ format("derivation contains an illegal reference specifier `%1%'")
+ % *i);
+ }
+ return result;
+}
+
+
+void DerivationGoal::registerOutputs()
+{
+ /* When using a build hook, the build hook can register the output
+ as valid (by doing `nix-store --import'). If so we don't have
+ to do anything here. */
+ if (hook) {
+ bool allValid = true;
+ foreach (DerivationOutputs::iterator, i, drv.outputs)
+ if (!worker.store.isValidPath(i->second.path)) allValid = false;
+ if (allValid) return;
+ }
+
+ ValidPathInfos infos;
+
+ /* Check whether the output paths were created, and grep each
+ output path to determine what other paths it references. Also make all
+ output paths read-only. */
+ foreach (DerivationOutputs::iterator, i, drv.outputs) {
+ Path path = i->second.path;
+ if (missingPaths.find(path) == missingPaths.end()) continue;
+
+ Path actualPath = path;
+ if (useChroot) {
+ actualPath = chrootRootDir + path;
+ if (pathExists(actualPath)) {
+ /* Move output paths from the chroot to the Nix store. */
+ if (buildMode == bmRepair)
+ replaceValidPath(path, actualPath);
+ else
+ if (buildMode != bmCheck && rename(actualPath.c_str(), path.c_str()) == -1)
+ throw SysError(format("moving build output `%1%' from the chroot to the Nix store") % path);
+ }
+ if (buildMode != bmCheck) actualPath = path;
+ } else {
+ Path redirected = redirectedOutputs[path];
+ if (buildMode == bmRepair
+ && redirectedBadOutputs.find(path) != redirectedBadOutputs.end()
+ && pathExists(redirected))
+ replaceValidPath(path, redirected);
+ if (buildMode == bmCheck)
+ actualPath = redirected;
+ }
+
+ struct stat st;
+ if (lstat(actualPath.c_str(), &st) == -1) {
+ if (errno == ENOENT)
+ throw BuildError(
+ format("builder for `%1%' failed to produce output path `%2%'")
+ % drvPath % path);
+ throw SysError(format("getting attributes of path `%1%'") % actualPath);
+ }
+
+#ifndef __CYGWIN__
+ /* Check that the output is not group or world writable, as
+ that means that someone else can have interfered with the
+ build. Also, the output should be owned by the build
+ user. */
+ if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
+ (buildUser.enabled() && st.st_uid != buildUser.getUID()))
+ throw BuildError(format("suspicious ownership or permission on `%1%'; rejecting this build output") % path);
+#endif
+
+ /* Apply hash rewriting if necessary. */
+ bool rewritten = false;
+ if (!rewritesFromTmp.empty()) {
+ printMsg(lvlError, format("warning: rewriting hashes in `%1%'; cross fingers") % path);
+
+ /* Canonicalise first. This ensures that the path we're
+ rewriting doesn't contain a hard link to /etc/shadow or
+ something like that. */
+ canonicalisePathMetaData(actualPath, buildUser.enabled() ? buildUser.getUID() : -1, inodesSeen);
+
+ /* FIXME: this is in-memory. */
+ StringSink sink;
+ dumpPath(actualPath, sink);
+ deletePath(actualPath);
+ sink.s = rewriteHashes(sink.s, rewritesFromTmp);
+ StringSource source(sink.s);
+ restorePath(actualPath, source);
+
+ rewritten = true;
+ }
+
+ startNest(nest, lvlTalkative,
+ format("scanning for references inside `%1%'") % path);
+
+ /* Check that fixed-output derivations produced the right
+ outputs (i.e., the content hash should match the specified
+ hash). */
+ if (i->second.hash != "") {
+
+ bool recursive; HashType ht; Hash h;
+ i->second.parseHashInfo(recursive, ht, h);
+
+ if (!recursive) {
+ /* The output path should be a regular file without
+ execute permission. */
+ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
+ throw BuildError(
+ format("output path `%1% should be a non-executable regular file") % path);
+ }
+
+ /* Check the hash. */
+ Hash h2 = recursive ? hashPath(ht, actualPath).first : hashFile(ht, actualPath);
+ if (h != h2)
+ throw BuildError(
+ format("output path `%1%' should have %2% hash `%3%', instead has `%4%'")
+ % path % i->second.hashAlgo % printHash16or32(h) % printHash16or32(h2));
+ }
+
+ /* Get rid of all weird permissions. This also checks that
+ all files are owned by the build user, if applicable. */
+ canonicalisePathMetaData(actualPath,
+ buildUser.enabled() && !rewritten ? buildUser.getUID() : -1, inodesSeen);
+
+ /* For this output path, find the references to other paths
+ contained in it. Compute the SHA-256 NAR hash at the same
+ time. The hash is stored in the database so that we can
+ verify later on whether nobody has messed with the store. */
+ HashResult hash;
+ PathSet references = scanForReferences(actualPath, allPaths, hash);
+
+ if (buildMode == bmCheck) {
+ ValidPathInfo info = worker.store.queryPathInfo(path);
+ if (hash.first != info.hash)
+ throw Error(format("derivation `%2%' may not be deterministic: hash mismatch in output `%1%'") % drvPath % path);
+ continue;
+ }
+
+ /* For debugging, print out the referenced and unreferenced
+ paths. */
+ foreach (PathSet::iterator, i, inputPaths) {
+ PathSet::iterator j = references.find(*i);
+ if (j == references.end())
+ debug(format("unreferenced input: `%1%'") % *i);
+ else
+ debug(format("referenced input: `%1%'") % *i);
+ }
+
+ /* If the derivation specifies an `allowedReferences'
+ attribute (containing a list of paths that the output may
+ refer to), check that all references are in that list. !!!
+ allowedReferences should really be per-output. */
+ if (drv.env.find("allowedReferences") != drv.env.end()) {
+ PathSet allowed = parseReferenceSpecifiers(drv, get(drv.env, "allowedReferences"));
+ foreach (PathSet::iterator, i, references)
+ if (allowed.find(*i) == allowed.end())
+ throw BuildError(format("output is not allowed to refer to path `%1%'") % *i);
+ }
+
+ worker.store.optimisePath(path); // FIXME: combine with scanForReferences()
+
+ worker.store.markContentsGood(path);
+
+ ValidPathInfo info;
+ info.path = path;
+ info.hash = hash.first;
+ info.narSize = hash.second;
+ info.references = references;
+ info.deriver = drvPath;
+ infos.push_back(info);
+ }
+
+ if (buildMode == bmCheck) return;
+
+ /* Register each output path as valid, and register the sets of
+ paths referenced by each of them. If there are cycles in the
+ outputs, this will fail. */
+ worker.store.registerValidPaths(infos);
+}
+
+
+string drvsLogDir = "drvs";
+
+
+Path DerivationGoal::openLogFile()
+{
+ logSize = 0;
+
+ if (!settings.keepLog) return "";
+
+ string baseName = baseNameOf(drvPath);
+
+ /* Create a log file. */
+ Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str();
+ createDirs(dir);
+
+ if (settings.compressLog) {
+
+ Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName, 2)).str();
+ AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
+ if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName);
+ closeOnExec(fd);
+
+ if (!(fLogFile = fdopen(fd.borrow(), "w")))
+ throw SysError(format("opening file `%1%'") % logFileName);
+
+ int err;
+ if (!(bzLogFile = BZ2_bzWriteOpen(&err, fLogFile, 9, 0, 0)))
+ throw Error(format("cannot open compressed log file `%1%'") % logFileName);
+
+ return logFileName;
+
+ } else {
+ Path logFileName = (format("%1%/%2%") % dir % string(baseName, 2)).str();
+ fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
+ if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName);
+ closeOnExec(fdLogFile);
+ return logFileName;
+ }
+}
+
+
+void DerivationGoal::closeLogFile()
+{
+ if (bzLogFile) {
+ int err;
+ BZ2_bzWriteClose(&err, bzLogFile, 0, 0, 0);
+ bzLogFile = 0;
+ if (err != BZ_OK) throw Error(format("cannot close compressed log file (BZip2 error = %1%)") % err);
+ }
+
+ if (fLogFile) {
+ fclose(fLogFile);
+ fLogFile = 0;
+ }
+
+ fdLogFile.close();
+}
+
+
+void DerivationGoal::deleteTmpDir(bool force)
+{
+ if (tmpDir != "") {
+ if (settings.keepFailed && !force) {
+ printMsg(lvlError,
+ format("note: keeping build directory `%2%'")
+ % drvPath % tmpDir);
+ chmod(tmpDir.c_str(), 0755);
+ }
+ else
+ deletePath(tmpDir);
+ tmpDir = "";
+ }
+}
+
+
+void DerivationGoal::handleChildOutput(int fd, const string & data)
+{
+ if ((hook && fd == hook->builderOut.readSide) ||
+ (!hook && fd == builderOut.readSide))
+ {
+ logSize += data.size();
+ if (settings.maxLogSize && logSize > settings.maxLogSize) {
+ printMsg(lvlError,
+ format("%1% killed after writing more than %2% bytes of log output")
+ % getName() % settings.maxLogSize);
+ cancel(true); // not really a timeout, but close enough
+ return;
+ }
+ if (verbosity >= settings.buildVerbosity)
+ writeToStderr(data);
+ if (bzLogFile) {
+ int err;
+ BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size());
+ if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err);
+ } else if (fdLogFile != -1)
+ writeFull(fdLogFile, (unsigned char *) data.data(), data.size());
+ }
+
+ if (hook && fd == hook->fromHook.readSide)
+ writeToStderr(data);
+}
+
+
+void DerivationGoal::handleEOF(int fd)
+{
+ worker.wakeUp(shared_from_this());
+}
+
+
+PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
+{
+ PathSet result;
+ foreach (DerivationOutputs::iterator, i, drv.outputs) {
+ if (!wantOutput(i->first, wantedOutputs)) continue;
+ bool good =
+ worker.store.isValidPath(i->second.path) &&
+ (!checkHash || worker.store.pathContentsGood(i->second.path));
+ if (good == returnValid) result.insert(i->second.path);
+ }
+ return result;
+}
+
+
+bool DerivationGoal::pathFailed(const Path & path)
+{
+ if (!settings.cacheFailure) return false;
+
+ if (!worker.store.hasPathFailed(path)) return false;
+
+ printMsg(lvlError, format("builder for `%1%' failed previously (cached)") % path);
+
+ if (settings.printBuildTrace)
+ printMsg(lvlError, format("@ build-failed %1% - cached") % drvPath);
+
+ worker.permanentFailure = true;
+ amDone(ecFailed);
+
+ return true;
+}
+
+
+Path DerivationGoal::addHashRewrite(const Path & path)
+{
+ string h1 = string(path, settings.nixStore.size() + 1, 32);
+ string h2 = string(printHash32(hashString(htSHA256, "rewrite:" + drvPath + ":" + path)), 0, 32);
+ Path p = settings.nixStore + "/" + h2 + string(path, settings.nixStore.size() + 33);
+ if (pathExists(p)) deletePath(p);
+ assert(path.size() == p.size());
+ rewritesToTmp[h1] = h2;
+ rewritesFromTmp[h2] = h1;
+ redirectedOutputs[path] = p;
+ return p;
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+class SubstitutionGoal : public Goal
+{
+ friend class Worker;
+
+private:
+ /* The store path that should be realised through a substitute. */
+ Path storePath;
+
+ /* The remaining substituters. */
+ Paths subs;
+
+ /* The current substituter. */
+ Path sub;
+
+ /* Whether any substituter can realise this path */
+ bool hasSubstitute;
+
+ /* Path info returned by the substituter's query info operation. */
+ SubstitutablePathInfo info;
+
+ /* Pipe for the substituter's standard output. */
+ Pipe outPipe;
+
+ /* Pipe for the substituter's standard error. */
+ Pipe logPipe;
+
+ /* The process ID of the builder. */
+ Pid pid;
+
+ /* Lock on the store path. */
+ std::shared_ptr<PathLocks> outputLock;
+
+ /* Whether to try to repair a valid path. */
+ bool repair;
+
+ /* Location where we're downloading the substitute. Differs from
+ storePath when doing a repair. */
+ Path destPath;
+
+ typedef void (SubstitutionGoal::*GoalState)();
+ GoalState state;
+
+public:
+ SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false);
+ ~SubstitutionGoal();
+
+ void cancel(bool timeout);
+
+ void work();
+
+ /* The states. */
+ void init();
+ void tryNext();
+ void gotInfo();
+ void referencesValid();
+ void tryToRun();
+ void finished();
+
+ /* Callback used by the worker to write to the log. */
+ void handleChildOutput(int fd, const string & data);
+ void handleEOF(int fd);
+
+ Path getStorePath() { return storePath; }
+};
+
+
+SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool repair)
+ : Goal(worker)
+ , hasSubstitute(false)
+ , repair(repair)
+{
+ this->storePath = storePath;
+ state = &SubstitutionGoal::init;
+ name = (format("substitution of `%1%'") % storePath).str();
+ trace("created");
+}
+
+
+SubstitutionGoal::~SubstitutionGoal()
+{
+ if (pid != -1) worker.childTerminated(pid);
+}
+
+
+void SubstitutionGoal::cancel(bool timeout)
+{
+ if (settings.printBuildTrace && timeout)
+ printMsg(lvlError, format("@ substituter-failed %1% timeout") % storePath);
+ if (pid != -1) {
+ pid_t savedPid = pid;
+ pid.kill();
+ worker.childTerminated(savedPid);
+ }
+ amDone(ecFailed);
+}
+
+
+void SubstitutionGoal::work()
+{
+ (this->*state)();
+}
+
+
+void SubstitutionGoal::init()
+{
+ trace("init");
+
+ worker.store.addTempRoot(storePath);
+
+ /* If the path already exists we're done. */
+ if (!repair && worker.store.isValidPath(storePath)) {
+ amDone(ecSuccess);
+ return;
+ }
+
+ if (settings.readOnlyMode)
+ throw Error(format("cannot substitute path `%1%' - no write access to the Nix store") % storePath);
+
+ subs = settings.substituters;
+
+ tryNext();
+}
+
+
+void SubstitutionGoal::tryNext()
+{
+ trace("trying next substituter");
+
+ if (subs.size() == 0) {
+ /* None left. Terminate this goal and let someone else deal
+ with it. */
+ debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath);
+ /* Hack: don't indicate failure if there were no substituters.
+ In that case the calling derivation should just do a
+ build. */
+ amDone(hasSubstitute ? ecFailed : ecNoSubstituters);
+ return;
+ }
+
+ sub = subs.front();
+ subs.pop_front();
+
+ SubstitutablePathInfos infos;
+ PathSet dummy(singleton<PathSet>(storePath));
+ worker.store.querySubstitutablePathInfos(sub, dummy, infos);
+ SubstitutablePathInfos::iterator k = infos.find(storePath);
+ if (k == infos.end()) { tryNext(); return; }
+ info = k->second;
+ hasSubstitute = true;
+
+ /* To maintain the closure invariant, we first have to realise the
+ paths referenced by this one. */
+ foreach (PathSet::iterator, i, info.references)
+ if (*i != storePath) /* ignore self-references */
+ addWaitee(worker.makeSubstitutionGoal(*i));
+
+ if (waitees.empty()) /* to prevent hang (no wake-up event) */
+ referencesValid();
+ else
+ state = &SubstitutionGoal::referencesValid;
+}
+
+
+void SubstitutionGoal::referencesValid()
+{
+ trace("all references realised");
+
+ if (nrFailed > 0) {
+ debug(format("some references of path `%1%' could not be realised") % storePath);
+ amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
+ return;
+ }
+
+ foreach (PathSet::iterator, i, info.references)
+ if (*i != storePath) /* ignore self-references */
+ assert(worker.store.isValidPath(*i));
+
+ state = &SubstitutionGoal::tryToRun;
+ worker.wakeUp(shared_from_this());
+}
+
+
+void SubstitutionGoal::tryToRun()
+{
+ trace("trying to run");
+
+ /* Make sure that we are allowed to start a build. Note that even
+ is maxBuildJobs == 0 (no local builds allowed), we still allow
+ a substituter to run. This is because substitutions cannot be
+ distributed to another machine via the build hook. */
+ if (worker.getNrLocalBuilds() >= (settings.maxBuildJobs == 0 ? 1 : settings.maxBuildJobs)) {
+ worker.waitForBuildSlot(shared_from_this());
+ return;
+ }
+
+ /* Maybe a derivation goal has already locked this path
+ (exceedingly unlikely, since it should have used a substitute
+ first, but let's be defensive). */
+ outputLock.reset(); // make sure this goal's lock is gone
+ if (pathIsLockedByMe(storePath)) {
+ debug(format("restarting substitution of `%1%' because it's locked by another goal")
+ % storePath);
+ worker.waitForAnyGoal(shared_from_this());
+ return; /* restart in the tryToRun() state when another goal finishes */
+ }
+
+ /* Acquire a lock on the output path. */
+ outputLock = std::shared_ptr<PathLocks>(new PathLocks);
+ if (!outputLock->lockPaths(singleton<PathSet>(storePath), "", false)) {
+ worker.waitForAWhile(shared_from_this());
+ return;
+ }
+
+ /* Check again whether the path is invalid. */
+ if (!repair && worker.store.isValidPath(storePath)) {
+ debug(format("store path `%1%' has become valid") % storePath);
+ outputLock->setDeletion(true);
+ amDone(ecSuccess);
+ return;
+ }
+
+ printMsg(lvlInfo, format("fetching path `%1%'...") % storePath);
+
+ outPipe.create();
+ logPipe.create();
+
+ destPath = repair ? storePath + ".tmp" : storePath;
+
+ /* Remove the (stale) output path if it exists. */
+ if (pathExists(destPath))
+ deletePath(destPath);
+
+ worker.store.setSubstituterEnv();
+
+ /* Fill in the arguments. */
+ Strings args;
+ args.push_back(baseNameOf(sub));
+ args.push_back("--substitute");
+ args.push_back(storePath);
+ args.push_back(destPath);
+ const char * * argArr = strings2CharPtrs(args);
+
+ /* Fork the substitute program. */
+ pid = maybeVfork();
+
+ switch (pid) {
+
+ case -1:
+ throw SysError("unable to fork");
+
+ case 0:
+ try { /* child */
+
+ commonChildInit(logPipe);
+
+ if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
+ throw SysError("cannot dup output pipe into stdout");
+
+ execv(sub.c_str(), (char * *) argArr);
+
+ throw SysError(format("executing `%1%'") % sub);
+
+ } catch (std::exception & e) {
+ writeToStderr("substitute error: " + string(e.what()) + "\n");
+ }
+ _exit(1);
+ }
+
+ /* parent */
+ pid.setSeparatePG(true);
+ pid.setKillSignal(SIGTERM);
+ outPipe.writeSide.close();
+ logPipe.writeSide.close();
+ worker.childStarted(shared_from_this(),
+ pid, singleton<set<int> >(logPipe.readSide), true, true);
+
+ state = &SubstitutionGoal::finished;
+
+ if (settings.printBuildTrace)
+ printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub);
+}
+
+
+void SubstitutionGoal::finished()
+{
+ trace("substitute finished");
+
+ /* Since we got an EOF on the logger pipe, the substitute is
+ presumed to have terminated. */
+ pid_t savedPid = pid;
+ int status = pid.wait(true);
+
+ /* So the child is gone now. */
+ worker.childTerminated(savedPid);
+
+ /* Close the read side of the logger pipe. */
+ logPipe.readSide.close();
+
+ /* Get the hash info from stdout. */
+ string dummy = readLine(outPipe.readSide);
+ string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : "";
+ outPipe.readSide.close();
+
+ /* Check the exit status and the build result. */
+ HashResult hash;
+ try {
+
+ if (!statusOk(status))
+ throw SubstError(format("fetching path `%1%' %2%")
+ % storePath % statusToString(status));
+
+ if (!pathExists(destPath))
+ throw SubstError(format("substitute did not produce path `%1%'") % destPath);
+
+ hash = hashPath(htSHA256, destPath);
+
+ /* Verify the expected hash we got from the substituer. */
+ if (expectedHashStr != "") {
+ size_t n = expectedHashStr.find(':');
+ if (n == string::npos)
+ throw Error(format("bad hash from substituter: %1%") % expectedHashStr);
+ HashType hashType = parseHashType(string(expectedHashStr, 0, n));
+ if (hashType == htUnknown)
+ throw Error(format("unknown hash algorithm in `%1%'") % expectedHashStr);
+ Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1));
+ Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first;
+ if (expectedHash != actualHash)
+ throw SubstError(format("hash mismatch in downloaded path `%1%': expected %2%, got %3%")
+ % storePath % printHash(expectedHash) % printHash(actualHash));
+ }
+
+ } catch (SubstError & e) {
+
+ printMsg(lvlInfo, e.msg());
+
+ if (settings.printBuildTrace) {
+ printMsg(lvlError, format("@ substituter-failed %1% %2% %3%")
+ % storePath % status % e.msg());
+ }
+
+ /* Try the next substitute. */
+ state = &SubstitutionGoal::tryNext;
+ worker.wakeUp(shared_from_this());
+ return;
+ }
+
+ if (repair) replaceValidPath(storePath, destPath);
+
+ canonicalisePathMetaData(storePath, -1);
+
+ worker.store.optimisePath(storePath); // FIXME: combine with hashPath()
+
+ ValidPathInfo info2;
+ info2.path = storePath;
+ info2.hash = hash.first;
+ info2.narSize = hash.second;
+ info2.references = info.references;
+ info2.deriver = info.deriver;
+ worker.store.registerValidPath(info2);
+
+ outputLock->setDeletion(true);
+ outputLock.reset();
+
+ worker.store.markContentsGood(storePath);
+
+ printMsg(lvlChatty,
+ format("substitution of path `%1%' succeeded") % storePath);
+
+ if (settings.printBuildTrace)
+ printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath);
+
+ amDone(ecSuccess);
+}
+
+
+void SubstitutionGoal::handleChildOutput(int fd, const string & data)
+{
+ assert(fd == logPipe.readSide);
+ if (verbosity >= settings.buildVerbosity) writeToStderr(data);
+ /* Don't write substitution output to a log file for now. We
+ probably should, though. */
+}
+
+
+void SubstitutionGoal::handleEOF(int fd)
+{
+ if (fd == logPipe.readSide) worker.wakeUp(shared_from_this());
+}
+
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+static bool working = false;
+
+
+Worker::Worker(LocalStore & store)
+ : store(store)
+{
+ /* Debugging: prevent recursive workers. */
+ if (working) abort();
+ working = true;
+ nrLocalBuilds = 0;
+ lastWokenUp = 0;
+ permanentFailure = false;
+}
+
+
+Worker::~Worker()
+{
+ working = false;
+
+ /* Explicitly get rid of all strong pointers now. After this all
+ goals that refer to this worker should be gone. (Otherwise we
+ are in trouble, since goals may call childTerminated() etc. in
+ their destructors). */
+ topGoals.clear();
+}
+
+
+GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOutputs, BuildMode buildMode)
+{
+ GoalPtr goal = derivationGoals[path].lock();
+ if (!goal) {
+ goal = GoalPtr(new DerivationGoal(path, wantedOutputs, *this, buildMode));
+ derivationGoals[path] = goal;
+ wakeUp(goal);
+ } else
+ (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs);
+ return goal;
+}
+
+
+GoalPtr Worker::makeSubstitutionGoal(const Path & path, bool repair)
+{
+ GoalPtr goal = substitutionGoals[path].lock();
+ if (!goal) {
+ goal = GoalPtr(new SubstitutionGoal(path, *this, repair));
+ substitutionGoals[path] = goal;
+ wakeUp(goal);
+ }
+ return goal;
+}
+
+
+static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
+{
+ /* !!! inefficient */
+ for (WeakGoalMap::iterator i = goalMap.begin();
+ i != goalMap.end(); )
+ if (i->second.lock() == goal) {
+ WeakGoalMap::iterator j = i; ++j;
+ goalMap.erase(i);
+ i = j;
+ }
+ else ++i;
+}
+
+
+void Worker::removeGoal(GoalPtr goal)
+{
+ nix::removeGoal(goal, derivationGoals);
+ nix::removeGoal(goal, substitutionGoals);
+ if (topGoals.find(goal) != topGoals.end()) {
+ topGoals.erase(goal);
+ /* If a top-level goal failed, then kill all other goals
+ (unless keepGoing was set). */
+ if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing)
+ topGoals.clear();
+ }
+
+ /* Wake up goals waiting for any goal to finish. */
+ foreach (WeakGoals::iterator, i, waitingForAnyGoal) {
+ GoalPtr goal = i->lock();
+ if (goal) wakeUp(goal);
+ }
+
+ waitingForAnyGoal.clear();
+}
+
+
+void Worker::wakeUp(GoalPtr goal)
+{
+ goal->trace("woken up");
+ addToWeakGoals(awake, goal);
+}
+
+
+unsigned Worker::getNrLocalBuilds()
+{
+ return nrLocalBuilds;
+}
+
+
+void Worker::childStarted(GoalPtr goal,
+ pid_t pid, const set<int> & fds, bool inBuildSlot,
+ bool respectTimeouts)
+{
+ Child child;
+ child.goal = goal;
+ child.fds = fds;
+ child.timeStarted = child.lastOutput = time(0);
+ child.inBuildSlot = inBuildSlot;
+ child.respectTimeouts = respectTimeouts;
+ children[pid] = child;
+ if (inBuildSlot) nrLocalBuilds++;
+}
+
+
+void Worker::childTerminated(pid_t pid, bool wakeSleepers)
+{
+ assert(pid != -1); /* common mistake */
+
+ Children::iterator i = children.find(pid);
+ assert(i != children.end());
+
+ if (i->second.inBuildSlot) {
+ assert(nrLocalBuilds > 0);
+ nrLocalBuilds--;
+ }
+
+ children.erase(pid);
+
+ if (wakeSleepers) {
+
+ /* Wake up goals waiting for a build slot. */
+ foreach (WeakGoals::iterator, i, wantingToBuild) {
+ GoalPtr goal = i->lock();
+ if (goal) wakeUp(goal);
+ }
+
+ wantingToBuild.clear();
+ }
+}
+
+
+void Worker::waitForBuildSlot(GoalPtr goal)
+{
+ debug("wait for build slot");
+ if (getNrLocalBuilds() < settings.maxBuildJobs)
+ wakeUp(goal); /* we can do it right away */
+ else
+ addToWeakGoals(wantingToBuild, goal);
+}
+
+
+void Worker::waitForAnyGoal(GoalPtr goal)
+{
+ debug("wait for any goal");
+ addToWeakGoals(waitingForAnyGoal, goal);
+}
+
+
+void Worker::waitForAWhile(GoalPtr goal)
+{
+ debug("wait for a while");
+ addToWeakGoals(waitingForAWhile, goal);
+}
+
+
+void Worker::run(const Goals & _topGoals)
+{
+ foreach (Goals::iterator, i, _topGoals) topGoals.insert(*i);
+
+ startNest(nest, lvlDebug, format("entered goal loop"));
+
+ while (1) {
+
+ checkInterrupt();
+
+ /* Call every wake goal. */
+ while (!awake.empty() && !topGoals.empty()) {
+ WeakGoals awake2(awake);
+ awake.clear();
+ foreach (WeakGoals::iterator, i, awake2) {
+ checkInterrupt();
+ GoalPtr goal = i->lock();
+ if (goal) goal->work();
+ if (topGoals.empty()) break;
+ }
+ }
+
+ if (topGoals.empty()) break;
+
+ /* Wait for input. */
+ if (!children.empty() || !waitingForAWhile.empty())
+ waitForInput();
+ else {
+ if (awake.empty() && settings.maxBuildJobs == 0) throw Error(
+ "unable to start any build; either increase `--max-jobs' "
+ "or enable distributed builds");
+ assert(!awake.empty());
+ }
+ }
+
+ /* If --keep-going is not set, it's possible that the main goal
+ exited while some of its subgoals were still active. But if
+ --keep-going *is* set, then they must all be finished now. */
+ assert(!settings.keepGoing || awake.empty());
+ assert(!settings.keepGoing || wantingToBuild.empty());
+ assert(!settings.keepGoing || children.empty());
+}
+
+
+void Worker::waitForInput()
+{
+ printMsg(lvlVomit, "waiting for children");
+
+ /* Process output from the file descriptors attached to the
+ children, namely log output and output path creation commands.
+ We also use this to detect child termination: if we get EOF on
+ the logger pipe of a build, we assume that the builder has
+ terminated. */
+
+ bool useTimeout = false;
+ struct timeval timeout;
+ timeout.tv_usec = 0;
+ time_t before = time(0);
+
+ /* If we're monitoring for silence on stdout/stderr, or if there
+ is a build timeout, then wait for input until the first
+ deadline for any child. */
+ assert(sizeof(time_t) >= sizeof(long));
+ time_t nearest = LONG_MAX; // nearest deadline
+ foreach (Children::iterator, i, children) {
+ if (!i->second.respectTimeouts) continue;
+ if (settings.maxSilentTime != 0)
+ nearest = std::min(nearest, i->second.lastOutput + settings.maxSilentTime);
+ if (settings.buildTimeout != 0)
+ nearest = std::min(nearest, i->second.timeStarted + settings.buildTimeout);
+ }
+ if (nearest != LONG_MAX) {
+ timeout.tv_sec = std::max((time_t) 1, nearest - before);
+ useTimeout = true;
+ printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
+ }
+
+ /* If we are polling goals that are waiting for a lock, then wake
+ up after a few seconds at most. */
+ if (!waitingForAWhile.empty()) {
+ useTimeout = true;
+ if (lastWokenUp == 0)
+ printMsg(lvlError, "waiting for locks or build slots...");
+ if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before;
+ timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before));
+ } else lastWokenUp = 0;
+
+ using namespace std;
+ /* Use select() to wait for the input side of any logger pipe to
+ become `available'. Note that `available' (i.e., non-blocking)
+ includes EOF. */
+ fd_set fds;
+ FD_ZERO(&fds);
+ int fdMax = 0;
+ foreach (Children::iterator, i, children) {
+ foreach (set<int>::iterator, j, i->second.fds) {
+ FD_SET(*j, &fds);
+ if (*j >= fdMax) fdMax = *j + 1;
+ }
+ }
+
+ if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) {
+ if (errno == EINTR) return;
+ throw SysError("waiting for input");
+ }
+
+ time_t after = time(0);
+
+ /* Process all available file descriptors. */
+
+ /* Since goals may be canceled from inside the loop below (causing
+ them go be erased from the `children' map), we have to be
+ careful that we don't keep iterators alive across calls to
+ cancel(). */
+ set<pid_t> pids;
+ foreach (Children::iterator, i, children) pids.insert(i->first);
+
+ foreach (set<pid_t>::iterator, i, pids) {
+ checkInterrupt();
+ Children::iterator j = children.find(*i);
+ if (j == children.end()) continue; // child destroyed
+ GoalPtr goal = j->second.goal.lock();
+ assert(goal);
+
+ set<int> fds2(j->second.fds);
+ foreach (set<int>::iterator, k, fds2) {
+ if (FD_ISSET(*k, &fds)) {
+ unsigned char buffer[4096];
+ ssize_t rd = read(*k, buffer, sizeof(buffer));
+ if (rd == -1) {
+ if (errno != EINTR)
+ throw SysError(format("reading from %1%")
+ % goal->getName());
+ } else if (rd == 0) {
+ debug(format("%1%: got EOF") % goal->getName());
+ goal->handleEOF(*k);
+ j->second.fds.erase(*k);
+ } else {
+ printMsg(lvlVomit, format("%1%: read %2% bytes")
+ % goal->getName() % rd);
+ string data((char *) buffer, rd);
+ j->second.lastOutput = after;
+ goal->handleChildOutput(*k, data);
+ }
+ }
+ }
+
+ if (goal->getExitCode() == Goal::ecBusy &&
+ settings.maxSilentTime != 0 &&
+ j->second.respectTimeouts &&
+ after - j->second.lastOutput >= (time_t) settings.maxSilentTime)
+ {
+ printMsg(lvlError,
+ format("%1% timed out after %2% seconds of silence")
+ % goal->getName() % settings.maxSilentTime);
+ goal->cancel(true);
+ }
+
+ else if (goal->getExitCode() == Goal::ecBusy &&
+ settings.buildTimeout != 0 &&
+ j->second.respectTimeouts &&
+ after - j->second.timeStarted >= (time_t) settings.buildTimeout)
+ {
+ printMsg(lvlError,
+ format("%1% timed out after %2% seconds")
+ % goal->getName() % settings.buildTimeout);
+ goal->cancel(true);
+ }
+ }
+
+ if (!waitingForAWhile.empty() && lastWokenUp + settings.pollInterval <= after) {
+ lastWokenUp = after;
+ foreach (WeakGoals::iterator, i, waitingForAWhile) {
+ GoalPtr goal = i->lock();
+ if (goal) wakeUp(goal);
+ }
+ waitingForAWhile.clear();
+ }
+}
+
+
+unsigned int Worker::exitStatus()
+{
+ return permanentFailure ? 100 : 1;
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
+{
+ startNest(nest, lvlDebug,
+ format("building %1%") % showPaths(drvPaths));
+
+ Worker worker(*this);
+
+ Goals goals;
+ foreach (PathSet::const_iterator, i, drvPaths) {
+ DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i);
+ if (isDerivation(i2.first))
+ goals.insert(worker.makeDerivationGoal(i2.first, i2.second, buildMode));
+ else
+ goals.insert(worker.makeSubstitutionGoal(*i, buildMode));
+ }
+
+ worker.run(goals);
+
+ PathSet failed;
+ foreach (Goals::iterator, i, goals)
+ if ((*i)->getExitCode() == Goal::ecFailed) {
+ DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i->get());
+ if (i2) failed.insert(i2->getDrvPath());
+ else failed.insert(dynamic_cast<SubstitutionGoal *>(i->get())->getStorePath());
+ }
+
+ if (!failed.empty())
+ throw Error(format("build of %1% failed") % showPaths(failed), worker.exitStatus());
+}
+
+
+void LocalStore::ensurePath(const Path & path)
+{
+ /* If the path is already valid, we're done. */
+ if (isValidPath(path)) return;
+
+ Worker worker(*this);
+ GoalPtr goal = worker.makeSubstitutionGoal(path);
+ Goals goals = singleton<Goals>(goal);
+
+ worker.run(goals);
+
+ if (goal->getExitCode() != Goal::ecSuccess)
+ throw Error(format("path `%1%' does not exist and cannot be created") % path, worker.exitStatus());
+}
+
+
+void LocalStore::repairPath(const Path & path)
+{
+ Worker worker(*this);
+ GoalPtr goal = worker.makeSubstitutionGoal(path, true);
+ Goals goals = singleton<Goals>(goal);
+
+ worker.run(goals);
+
+ if (goal->getExitCode() != Goal::ecSuccess)
+ throw Error(format("cannot repair path `%1%'") % path, worker.exitStatus());
+}
+
+
+}
diff --git a/nix/libstore/derivations.cc b/nix/libstore/derivations.cc
new file mode 100644
index 0000000000..d91e42784c
--- /dev/null
+++ b/nix/libstore/derivations.cc
@@ -0,0 +1,278 @@
+#include "derivations.hh"
+#include "store-api.hh"
+#include "globals.hh"
+#include "util.hh"
+#include "misc.hh"
+
+
+namespace nix {
+
+
+void DerivationOutput::parseHashInfo(bool & recursive, HashType & hashType, Hash & hash) const
+{
+ recursive = false;
+ string algo = hashAlgo;
+
+ if (string(algo, 0, 2) == "r:") {
+ recursive = true;
+ algo = string(algo, 2);
+ }
+
+ hashType = parseHashType(algo);
+ if (hashType == htUnknown)
+ throw Error(format("unknown hash algorithm `%1%'") % algo);
+
+ hash = parseHash(hashType, this->hash);
+}
+
+
+Path writeDerivation(StoreAPI & store,
+ const Derivation & drv, const string & name, bool repair)
+{
+ PathSet references;
+ references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
+ foreach (DerivationInputs::const_iterator, i, drv.inputDrvs)
+ references.insert(i->first);
+ /* Note that the outputs of a derivation are *not* references
+ (that can be missing (of course) and should not necessarily be
+ held during a garbage collection). */
+ string suffix = name + drvExtension;
+ string contents = unparseDerivation(drv);
+ return settings.readOnlyMode
+ ? computeStorePathForText(suffix, contents, references)
+ : store.addTextToStore(suffix, contents, references, repair);
+}
+
+
+static Path parsePath(std::istream & str)
+{
+ string s = parseString(str);
+ if (s.size() == 0 || s[0] != '/')
+ throw Error(format("bad path `%1%' in derivation") % s);
+ return s;
+}
+
+
+static StringSet parseStrings(std::istream & str, bool arePaths)
+{
+ StringSet res;
+ while (!endOfList(str))
+ res.insert(arePaths ? parsePath(str) : parseString(str));
+ return res;
+}
+
+
+Derivation parseDerivation(const string & s)
+{
+ Derivation drv;
+ std::istringstream str(s);
+ expect(str, "Derive([");
+
+ /* Parse the list of outputs. */
+ while (!endOfList(str)) {
+ DerivationOutput out;
+ expect(str, "("); string id = parseString(str);
+ expect(str, ","); out.path = parsePath(str);
+ expect(str, ","); out.hashAlgo = parseString(str);
+ expect(str, ","); out.hash = parseString(str);
+ expect(str, ")");
+ drv.outputs[id] = out;
+ }
+
+ /* Parse the list of input derivations. */
+ expect(str, ",[");
+ while (!endOfList(str)) {
+ expect(str, "(");
+ Path drvPath = parsePath(str);
+ expect(str, ",[");
+ drv.inputDrvs[drvPath] = parseStrings(str, false);
+ expect(str, ")");
+ }
+
+ expect(str, ",["); drv.inputSrcs = parseStrings(str, true);
+ expect(str, ","); drv.platform = parseString(str);
+ expect(str, ","); drv.builder = parseString(str);
+
+ /* Parse the builder arguments. */
+ expect(str, ",[");
+ while (!endOfList(str))
+ drv.args.push_back(parseString(str));
+
+ /* Parse the environment variables. */
+ expect(str, ",[");
+ while (!endOfList(str)) {
+ expect(str, "("); string name = parseString(str);
+ expect(str, ","); string value = parseString(str);
+ expect(str, ")");
+ drv.env[name] = value;
+ }
+
+ expect(str, ")");
+ return drv;
+}
+
+
+static void printString(string & res, const string & s)
+{
+ res += '"';
+ for (const char * i = s.c_str(); *i; i++)
+ if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; }
+ else if (*i == '\n') res += "\\n";
+ else if (*i == '\r') res += "\\r";
+ else if (*i == '\t') res += "\\t";
+ else res += *i;
+ res += '"';
+}
+
+
+template<class ForwardIterator>
+static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
+{
+ res += '[';
+ bool first = true;
+ for ( ; i != j; ++i) {
+ if (first) first = false; else res += ',';
+ printString(res, *i);
+ }
+ res += ']';
+}
+
+
+string unparseDerivation(const Derivation & drv)
+{
+ string s;
+ s.reserve(65536);
+ s += "Derive([";
+
+ bool first = true;
+ foreach (DerivationOutputs::const_iterator, i, drv.outputs) {
+ if (first) first = false; else s += ',';
+ s += '('; printString(s, i->first);
+ s += ','; printString(s, i->second.path);
+ s += ','; printString(s, i->second.hashAlgo);
+ s += ','; printString(s, i->second.hash);
+ s += ')';
+ }
+
+ s += "],[";
+ first = true;
+ foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) {
+ if (first) first = false; else s += ',';
+ s += '('; printString(s, i->first);
+ s += ','; printStrings(s, i->second.begin(), i->second.end());
+ s += ')';
+ }
+
+ s += "],";
+ printStrings(s, drv.inputSrcs.begin(), drv.inputSrcs.end());
+
+ s += ','; printString(s, drv.platform);
+ s += ','; printString(s, drv.builder);
+ s += ','; printStrings(s, drv.args.begin(), drv.args.end());
+
+ s += ",[";
+ first = true;
+ foreach (StringPairs::const_iterator, i, drv.env) {
+ if (first) first = false; else s += ',';
+ s += '('; printString(s, i->first);
+ s += ','; printString(s, i->second);
+ s += ')';
+ }
+
+ s += "])";
+
+ return s;
+}
+
+
+bool isDerivation(const string & fileName)
+{
+ return hasSuffix(fileName, drvExtension);
+}
+
+
+bool isFixedOutputDrv(const Derivation & drv)
+{
+ return drv.outputs.size() == 1 &&
+ drv.outputs.begin()->first == "out" &&
+ drv.outputs.begin()->second.hash != "";
+}
+
+
+DrvHashes drvHashes;
+
+
+/* Returns the hash of a derivation modulo fixed-output
+ subderivations. A fixed-output derivation is a derivation with one
+ output (`out') for which an expected hash and hash algorithm are
+ specified (using the `outputHash' and `outputHashAlgo'
+ attributes). We don't want changes to such derivations to
+ propagate upwards through the dependency graph, changing output
+ paths everywhere.
+
+ For instance, if we change the url in a call to the `fetchurl'
+ function, we do not want to rebuild everything depending on it
+ (after all, (the hash of) the file being downloaded is unchanged).
+ So the *output paths* should not change. On the other hand, the
+ *derivation paths* should change to reflect the new dependency
+ graph.
+
+ That's what this function does: it returns a hash which is just the
+ hash of the derivation ATerm, except that any input derivation
+ paths have been replaced by the result of a recursive call to this
+ function, and that for fixed-output derivations we return a hash of
+ its output path. */
+Hash hashDerivationModulo(StoreAPI & store, Derivation drv)
+{
+ /* Return a fixed hash for fixed-output derivations. */
+ if (isFixedOutputDrv(drv)) {
+ DerivationOutputs::const_iterator i = drv.outputs.begin();
+ return hashString(htSHA256, "fixed:out:"
+ + i->second.hashAlgo + ":"
+ + i->second.hash + ":"
+ + i->second.path);
+ }
+
+ /* For other derivations, replace the inputs paths with recursive
+ calls to this function.*/
+ DerivationInputs inputs2;
+ foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) {
+ Hash h = drvHashes[i->first];
+ if (h.type == htUnknown) {
+ assert(store.isValidPath(i->first));
+ Derivation drv2 = parseDerivation(readFile(i->first));
+ h = hashDerivationModulo(store, drv2);
+ drvHashes[i->first] = h;
+ }
+ inputs2[printHash(h)] = i->second;
+ }
+ drv.inputDrvs = inputs2;
+
+ return hashString(htSHA256, unparseDerivation(drv));
+}
+
+
+DrvPathWithOutputs parseDrvPathWithOutputs(const string & s)
+{
+ size_t n = s.find("!");
+ return n == s.npos
+ ? DrvPathWithOutputs(s, std::set<string>())
+ : DrvPathWithOutputs(string(s, 0, n), tokenizeString<std::set<string> >(string(s, n + 1), ","));
+}
+
+
+Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs)
+{
+ return outputs.empty()
+ ? drvPath
+ : drvPath + "!" + concatStringsSep(",", outputs);
+}
+
+
+bool wantOutput(const string & output, const std::set<string> & wanted)
+{
+ return wanted.empty() || wanted.find(output) != wanted.end();
+}
+
+
+}
diff --git a/nix/libstore/derivations.hh b/nix/libstore/derivations.hh
new file mode 100644
index 0000000000..703410b925
--- /dev/null
+++ b/nix/libstore/derivations.hh
@@ -0,0 +1,93 @@
+#pragma once
+
+#include "types.hh"
+#include "hash.hh"
+
+#include <map>
+
+
+namespace nix {
+
+
+/* Extension of derivations in the Nix store. */
+const string drvExtension = ".drv";
+
+
+/* Abstract syntax of derivations. */
+
+struct DerivationOutput
+{
+ Path path;
+ string hashAlgo; /* hash used for expected hash computation */
+ string hash; /* expected hash, may be null */
+ DerivationOutput()
+ {
+ }
+ DerivationOutput(Path path, string hashAlgo, string hash)
+ {
+ this->path = path;
+ this->hashAlgo = hashAlgo;
+ this->hash = hash;
+ }
+ void parseHashInfo(bool & recursive, HashType & hashType, Hash & hash) const;
+};
+
+typedef std::map<string, DerivationOutput> DerivationOutputs;
+
+/* For inputs that are sub-derivations, we specify exactly which
+ output IDs we are interested in. */
+typedef std::map<Path, StringSet> DerivationInputs;
+
+typedef std::map<string, string> StringPairs;
+
+struct Derivation
+{
+ DerivationOutputs outputs; /* keyed on symbolic IDs */
+ DerivationInputs inputDrvs; /* inputs that are sub-derivations */
+ PathSet inputSrcs; /* inputs that are sources */
+ string platform;
+ Path builder;
+ Strings args;
+ StringPairs env;
+};
+
+
+class StoreAPI;
+
+
+/* Write a derivation to the Nix store, and return its path. */
+Path writeDerivation(StoreAPI & store,
+ const Derivation & drv, const string & name, bool repair = false);
+
+/* Parse a derivation. */
+Derivation parseDerivation(const string & s);
+
+/* Print a derivation. */
+string unparseDerivation(const Derivation & drv);
+
+/* Check whether a file name ends with the extensions for
+ derivations. */
+bool isDerivation(const string & fileName);
+
+/* Return true iff this is a fixed-output derivation. */
+bool isFixedOutputDrv(const Derivation & drv);
+
+Hash hashDerivationModulo(StoreAPI & store, Derivation drv);
+
+/* Memoisation of hashDerivationModulo(). */
+typedef std::map<Path, Hash> DrvHashes;
+
+extern DrvHashes drvHashes;
+
+/* Split a string specifying a derivation and a set of outputs
+ (/nix/store/hash-foo!out1,out2,...) into the derivation path and
+ the outputs. */
+typedef std::pair<string, std::set<string> > DrvPathWithOutputs;
+DrvPathWithOutputs parseDrvPathWithOutputs(const string & s);
+
+Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs);
+
+bool wantOutput(const string & output, const std::set<string> & wanted);
+
+
+}
diff --git a/nix/libstore/gc.cc b/nix/libstore/gc.cc
new file mode 100644
index 0000000000..f90edac1cd
--- /dev/null
+++ b/nix/libstore/gc.cc
@@ -0,0 +1,748 @@
+#include "globals.hh"
+#include "misc.hh"
+#include "local-store.hh"
+
+#include <functional>
+#include <queue>
+#include <algorithm>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+
+namespace nix {
+
+
+static string gcLockName = "gc.lock";
+static string tempRootsDir = "temproots";
+static string gcRootsDir = "gcroots";
+
+
+/* Acquire the global GC lock. This is used to prevent new Nix
+ processes from starting after the temporary root files have been
+ read. To be precise: when they try to create a new temporary root
+ file, they will block until the garbage collector has finished /
+ yielded the GC lock. */
+int LocalStore::openGCLock(LockType lockType)
+{
+ Path fnGCLock = (format("%1%/%2%")
+ % settings.nixStateDir % gcLockName).str();
+
+ debug(format("acquiring global GC lock `%1%'") % fnGCLock);
+
+ AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT, 0600);
+ if (fdGCLock == -1)
+ throw SysError(format("opening global GC lock `%1%'") % fnGCLock);
+ closeOnExec(fdGCLock);
+
+ if (!lockFile(fdGCLock, lockType, false)) {
+ printMsg(lvlError, format("waiting for the big garbage collector lock..."));
+ lockFile(fdGCLock, lockType, true);
+ }
+
+ /* !!! Restrict read permission on the GC root. Otherwise any
+ process that can open the file for reading can DoS the
+ collector. */
+
+ return fdGCLock.borrow();
+}
+
+
+static void makeSymlink(const Path & link, const Path & target)
+{
+ /* Create directories up to `gcRoot'. */
+ createDirs(dirOf(link));
+
+ /* Create the new symlink. */
+ Path tempLink = (format("%1%.tmp-%2%-%3%")
+ % link % getpid() % rand()).str();
+ createSymlink(target, tempLink);
+
+ /* Atomically replace the old one. */
+ if (rename(tempLink.c_str(), link.c_str()) == -1)
+ throw SysError(format("cannot rename `%1%' to `%2%'")
+ % tempLink % link);
+}
+
+
+void LocalStore::syncWithGC()
+{
+ AutoCloseFD fdGCLock = openGCLock(ltRead);
+}
+
+
+void LocalStore::addIndirectRoot(const Path & path)
+{
+ string hash = printHash32(hashString(htSHA1, path));
+ Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
+ % settings.nixStateDir % gcRootsDir % hash).str());
+ makeSymlink(realRoot, path);
+}
+
+
+Path addPermRoot(StoreAPI & store, const Path & _storePath,
+ const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir)
+{
+ Path storePath(canonPath(_storePath));
+ Path gcRoot(canonPath(_gcRoot));
+ assertStorePath(storePath);
+
+ if (isInStore(gcRoot))
+ throw Error(format(
+ "creating a garbage collector root (%1%) in the Nix store is forbidden "
+ "(are you running nix-build inside the store?)") % gcRoot);
+
+ if (indirect) {
+ /* Don't clobber the the link if it already exists and doesn't
+ point to the Nix store. */
+ if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
+ throw Error(format("cannot create symlink `%1%'; already exists") % gcRoot);
+ makeSymlink(gcRoot, storePath);
+ store.addIndirectRoot(gcRoot);
+ }
+
+ else {
+ if (!allowOutsideRootsDir) {
+ Path rootsDir = canonPath((format("%1%/%2%") % settings.nixStateDir % gcRootsDir).str());
+
+ if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
+ throw Error(format(
+ "path `%1%' is not a valid garbage collector root; "
+ "it's not in the directory `%2%'")
+ % gcRoot % rootsDir);
+ }
+
+ makeSymlink(gcRoot, storePath);
+ }
+
+ /* Check that the root can be found by the garbage collector.
+ !!! This can be very slow on machines that have many roots.
+ Instead of reading all the roots, it would be more efficient to
+ check if the root is in a directory in or linked from the
+ gcroots directory. */
+ if (settings.checkRootReachability) {
+ Roots roots = store.findRoots();
+ if (roots.find(gcRoot) == roots.end())
+ printMsg(lvlError,
+ format(
+ "warning: `%1%' is not in a directory where the garbage collector looks for roots; "
+ "therefore, `%2%' might be removed by the garbage collector")
+ % gcRoot % storePath);
+ }
+
+ /* Grab the global GC root, causing us to block while a GC is in
+ progress. This prevents the set of permanent roots from
+ increasing while a GC is in progress. */
+ store.syncWithGC();
+
+ return gcRoot;
+}
+
+
+/* The file to which we write our temporary roots. */
+static Path fnTempRoots;
+static AutoCloseFD fdTempRoots;
+
+
+void LocalStore::addTempRoot(const Path & path)
+{
+ /* Create the temporary roots file for this process. */
+ if (fdTempRoots == -1) {
+
+ while (1) {
+ Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str();
+ createDirs(dir);
+
+ fnTempRoots = (format("%1%/%2%")
+ % dir % getpid()).str();
+
+ AutoCloseFD fdGCLock = openGCLock(ltRead);
+
+ if (pathExists(fnTempRoots))
+ /* It *must* be stale, since there can be no two
+ processes with the same pid. */
+ unlink(fnTempRoots.c_str());
+
+ fdTempRoots = openLockFile(fnTempRoots, true);
+
+ fdGCLock.close();
+
+ debug(format("acquiring read lock on `%1%'") % fnTempRoots);
+ lockFile(fdTempRoots, ltRead, true);
+
+ /* Check whether the garbage collector didn't get in our
+ way. */
+ struct stat st;
+ if (fstat(fdTempRoots, &st) == -1)
+ throw SysError(format("statting `%1%'") % fnTempRoots);
+ if (st.st_size == 0) break;
+
+ /* The garbage collector deleted this file before we could
+ get a lock. (It won't delete the file after we get a
+ lock.) Try again. */
+ }
+
+ }
+
+ /* Upgrade the lock to a write lock. This will cause us to block
+ if the garbage collector is holding our lock. */
+ debug(format("acquiring write lock on `%1%'") % fnTempRoots);
+ lockFile(fdTempRoots, ltWrite, true);
+
+ string s = path + '\0';
+ writeFull(fdTempRoots, (const unsigned char *) s.data(), s.size());
+
+ /* Downgrade to a read lock. */
+ debug(format("downgrading to read lock on `%1%'") % fnTempRoots);
+ lockFile(fdTempRoots, ltRead, true);
+}
+
+
+void removeTempRoots()
+{
+ if (fdTempRoots != -1) {
+ fdTempRoots.close();
+ unlink(fnTempRoots.c_str());
+ }
+}
+
+
+/* Automatically clean up the temporary roots file when we exit. */
+struct RemoveTempRoots
+{
+ ~RemoveTempRoots()
+ {
+ removeTempRoots();
+ }
+};
+
+static RemoveTempRoots autoRemoveTempRoots __attribute__((unused));
+
+
+typedef std::shared_ptr<AutoCloseFD> FDPtr;
+typedef list<FDPtr> FDs;
+
+
+static void readTempRoots(PathSet & tempRoots, FDs & fds)
+{
+ /* Read the `temproots' directory for per-process temporary root
+ files. */
+ Strings tempRootFiles = readDirectory(
+ (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str());
+
+ foreach (Strings::iterator, i, tempRootFiles) {
+ Path path = (format("%1%/%2%/%3%") % settings.nixStateDir % tempRootsDir % *i).str();
+
+ debug(format("reading temporary root file `%1%'") % path);
+ FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
+ if (*fd == -1) {
+ /* It's okay if the file has disappeared. */
+ if (errno == ENOENT) continue;
+ throw SysError(format("opening temporary roots file `%1%'") % path);
+ }
+
+ /* This should work, but doesn't, for some reason. */
+ //FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
+ //if (*fd == -1) continue;
+
+ /* Try to acquire a write lock without blocking. This can
+ only succeed if the owning process has died. In that case
+ we don't care about its temporary roots. */
+ if (lockFile(*fd, ltWrite, false)) {
+ printMsg(lvlError, format("removing stale temporary roots file `%1%'") % path);
+ unlink(path.c_str());
+ writeFull(*fd, (const unsigned char *) "d", 1);
+ continue;
+ }
+
+ /* Acquire a read lock. This will prevent the owning process
+ from upgrading to a write lock, therefore it will block in
+ addTempRoot(). */
+ debug(format("waiting for read lock on `%1%'") % path);
+ lockFile(*fd, ltRead, true);
+
+ /* Read the entire file. */
+ string contents = readFile(*fd);
+
+ /* Extract the roots. */
+ string::size_type pos = 0, end;
+
+ while ((end = contents.find((char) 0, pos)) != string::npos) {
+ Path root(contents, pos, end - pos);
+ debug(format("got temporary root `%1%'") % root);
+ assertStorePath(root);
+ tempRoots.insert(root);
+ pos = end + 1;
+ }
+
+ fds.push_back(fd); /* keep open */
+ }
+}
+
+
+static void foundRoot(StoreAPI & store,
+ const Path & path, const Path & target, Roots & roots)
+{
+ Path storePath = toStorePath(target);
+ if (store.isValidPath(storePath))
+ roots[path] = storePath;
+ else
+ printMsg(lvlInfo, format("skipping invalid root from `%1%' to `%2%'") % path % storePath);
+}
+
+
+static void findRoots(StoreAPI & store, const Path & path, Roots & roots)
+{
+ try {
+
+ struct stat st = lstat(path);
+
+ if (S_ISDIR(st.st_mode)) {
+ Strings names = readDirectory(path);
+ foreach (Strings::iterator, i, names)
+ findRoots(store, path + "/" + *i, roots);
+ }
+
+ else if (S_ISLNK(st.st_mode)) {
+ Path target = readLink(path);
+ if (isInStore(target))
+ foundRoot(store, path, target, roots);
+
+ /* Handle indirect roots. */
+ else {
+ target = absPath(target, dirOf(path));
+ if (!pathExists(target)) {
+ if (isInDir(path, settings.nixStateDir + "/" + gcRootsDir + "/auto")) {
+ printMsg(lvlInfo, format("removing stale link from `%1%' to `%2%'") % path % target);
+ unlink(path.c_str());
+ }
+ } else {
+ struct stat st2 = lstat(target);
+ if (!S_ISLNK(st2.st_mode)) return;
+ Path target2 = readLink(target);
+ if (isInStore(target2)) foundRoot(store, target, target2, roots);
+ }
+ }
+ }
+
+ }
+
+ catch (SysError & e) {
+ /* We only ignore permanent failures. */
+ if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR)
+ printMsg(lvlInfo, format("cannot read potential root `%1%'") % path);
+ else
+ throw;
+ }
+}
+
+
+Roots LocalStore::findRoots()
+{
+ Roots roots;
+
+ /* Process direct roots in {gcroots,manifests,profiles}. */
+ nix::findRoots(*this, settings.nixStateDir + "/" + gcRootsDir, roots);
+ nix::findRoots(*this, settings.nixStateDir + "/manifests", roots);
+ nix::findRoots(*this, settings.nixStateDir + "/profiles", roots);
+
+ return roots;
+}
+
+
+static void addAdditionalRoots(StoreAPI & store, PathSet & roots)
+{
+ Path rootFinder = getEnv("NIX_ROOT_FINDER",
+ settings.nixLibexecDir + "/guix/list-runtime-roots");
+
+ if (rootFinder.empty()) return;
+
+ debug(format("executing `%1%' to find additional roots") % rootFinder);
+
+ string result = runProgram(rootFinder);
+
+ StringSet paths = tokenizeString<StringSet>(result, "\n");
+
+ foreach (StringSet::iterator, i, paths) {
+ if (isInStore(*i)) {
+ Path path = toStorePath(*i);
+ if (roots.find(path) == roots.end() && store.isValidPath(path)) {
+ debug(format("got additional root `%1%'") % path);
+ roots.insert(path);
+ }
+ }
+ }
+}
+
+
+struct GCLimitReached { };
+
+
+struct LocalStore::GCState
+{
+ GCOptions options;
+ GCResults & results;
+ PathSet roots;
+ PathSet tempRoots;
+ PathSet dead;
+ PathSet alive;
+ bool gcKeepOutputs;
+ bool gcKeepDerivations;
+ unsigned long long bytesInvalidated;
+ Path trashDir;
+ bool shouldDelete;
+ GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { }
+};
+
+
+bool LocalStore::isActiveTempFile(const GCState & state,
+ const Path & path, const string & suffix)
+{
+ return hasSuffix(path, suffix)
+ && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end();
+}
+
+
+void LocalStore::deleteGarbage(GCState & state, const Path & path)
+{
+ unsigned long long bytesFreed;
+ deletePath(path, bytesFreed);
+ state.results.bytesFreed += bytesFreed;
+}
+
+
+void LocalStore::deletePathRecursive(GCState & state, const Path & path)
+{
+ checkInterrupt();
+
+ unsigned long long size = 0;
+
+ if (isValidPath(path)) {
+ PathSet referrers;
+ queryReferrers(path, referrers);
+ foreach (PathSet::iterator, i, referrers)
+ if (*i != path) deletePathRecursive(state, *i);
+ size = queryPathInfo(path).narSize;
+ invalidatePathChecked(path);
+ }
+
+ struct stat st;
+ if (lstat(path.c_str(), &st)) {
+ if (errno == ENOENT) return;
+ throw SysError(format("getting status of %1%") % path);
+ }
+
+ printMsg(lvlInfo, format("deleting `%1%'") % path);
+
+ state.results.paths.insert(path);
+
+ /* If the path is not a regular file or symlink, move it to the
+ trash directory. The move is to ensure that later (when we're
+ not holding the global GC lock) we can delete the path without
+ being afraid that the path has become alive again. Otherwise
+ delete it right away. */
+ if (S_ISDIR(st.st_mode)) {
+ // Estimate the amount freed using the narSize field. FIXME:
+ // if the path was not valid, need to determine the actual
+ // size.
+ state.bytesInvalidated += size;
+ // Mac OS X cannot rename directories if they are read-only.
+ if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
+ throw SysError(format("making `%1%' writable") % path);
+ Path tmp = state.trashDir + "/" + baseNameOf(path);
+ if (rename(path.c_str(), tmp.c_str()))
+ throw SysError(format("unable to rename `%1%' to `%2%'") % path % tmp);
+ } else
+ deleteGarbage(state, path);
+
+ if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
+ printMsg(lvlInfo, format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
+ throw GCLimitReached();
+ }
+}
+
+
+bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path)
+{
+ if (visited.find(path) != visited.end()) return false;
+
+ if (state.alive.find(path) != state.alive.end()) {
+ return true;
+ }
+
+ if (state.dead.find(path) != state.dead.end()) {
+ return false;
+ }
+
+ if (state.roots.find(path) != state.roots.end()) {
+ printMsg(lvlDebug, format("cannot delete `%1%' because it's a root") % path);
+ state.alive.insert(path);
+ return true;
+ }
+
+ visited.insert(path);
+
+ if (!isValidPath(path)) return false;
+
+ PathSet incoming;
+
+ /* Don't delete this path if any of its referrers are alive. */
+ queryReferrers(path, incoming);
+
+ /* If gc-keep-derivations is set and this is a derivation, then
+ don't delete the derivation if any of the outputs are alive. */
+ if (state.gcKeepDerivations && isDerivation(path)) {
+ PathSet outputs = queryDerivationOutputs(path);
+ foreach (PathSet::iterator, i, outputs)
+ if (isValidPath(*i) && queryDeriver(*i) == path)
+ incoming.insert(*i);
+ }
+
+ /* If gc-keep-outputs is set, then don't delete this path if there
+ are derivers of this path that are not garbage. */
+ if (state.gcKeepOutputs) {
+ PathSet derivers = queryValidDerivers(path);
+ foreach (PathSet::iterator, i, derivers)
+ incoming.insert(*i);
+ }
+
+ foreach (PathSet::iterator, i, incoming)
+ if (*i != path)
+ if (canReachRoot(state, visited, *i)) {
+ state.alive.insert(path);
+ return true;
+ }
+
+ return false;
+}
+
+
+void LocalStore::tryToDelete(GCState & state, const Path & path)
+{
+ checkInterrupt();
+
+ if (path == linksDir || path == state.trashDir) return;
+
+ startNest(nest, lvlDebug, format("considering whether to delete `%1%'") % path);
+
+ if (!isValidPath(path)) {
+ /* A lock file belonging to a path that we're building right
+ now isn't garbage. */
+ if (isActiveTempFile(state, path, ".lock")) return;
+
+ /* Don't delete .chroot directories for derivations that are
+ currently being built. */
+ if (isActiveTempFile(state, path, ".chroot")) return;
+ }
+
+ PathSet visited;
+
+ if (canReachRoot(state, visited, path)) {
+ printMsg(lvlDebug, format("cannot delete `%1%' because it's still reachable") % path);
+ } else {
+ /* No path we visited was a root, so everything is garbage.
+ But we only delete ‘path’ and its referrers here so that
+ ‘nix-store --delete’ doesn't have the unexpected effect of
+ recursing into derivations and outputs. */
+ state.dead.insert(visited.begin(), visited.end());
+ if (state.shouldDelete)
+ deletePathRecursive(state, path);
+ }
+}
+
+
+/* Unlink all files in /nix/store/.links that have a link count of 1,
+ which indicates that there are no other links and so they can be
+ safely deleted. FIXME: race condition with optimisePath(): we
+ might see a link count of 1 just before optimisePath() increases
+ the link count. */
+void LocalStore::removeUnusedLinks(const GCState & state)
+{
+ AutoCloseDir dir = opendir(linksDir.c_str());
+ if (!dir) throw SysError(format("opening directory `%1%'") % linksDir);
+
+ long long actualSize = 0, unsharedSize = 0;
+
+ struct dirent * dirent;
+ while (errno = 0, dirent = readdir(dir)) {
+ checkInterrupt();
+ string name = dirent->d_name;
+ if (name == "." || name == "..") continue;
+ Path path = linksDir + "/" + name;
+
+ struct stat st;
+ if (lstat(path.c_str(), &st) == -1)
+ throw SysError(format("statting `%1%'") % path);
+
+ if (st.st_nlink != 1) {
+ unsigned long long size = st.st_blocks * 512ULL;
+ actualSize += size;
+ unsharedSize += (st.st_nlink - 1) * size;
+ continue;
+ }
+
+ printMsg(lvlTalkative, format("deleting unused link `%1%'") % path);
+
+ if (unlink(path.c_str()) == -1)
+ throw SysError(format("deleting `%1%'") % path);
+
+ state.results.bytesFreed += st.st_blocks * 512;
+ }
+
+ struct stat st;
+ if (stat(linksDir.c_str(), &st) == -1)
+ throw SysError(format("statting `%1%'") % linksDir);
+ long long overhead = st.st_blocks * 512ULL;
+
+ printMsg(lvlInfo, format("note: currently hard linking saves %.2f MiB")
+ % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
+}
+
+
+void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
+{
+ GCState state(results);
+ state.options = options;
+ state.trashDir = settings.nixStore + "/trash";
+ state.gcKeepOutputs = settings.gcKeepOutputs;
+ state.gcKeepDerivations = settings.gcKeepDerivations;
+
+ /* Using `--ignore-liveness' with `--delete' can have unintended
+ consequences if `gc-keep-outputs' or `gc-keep-derivations' are
+ true (the garbage collector will recurse into deleting the
+ outputs or derivers, respectively). So disable them. */
+ if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) {
+ state.gcKeepOutputs = false;
+ state.gcKeepDerivations = false;
+ }
+
+ state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
+
+ /* Acquire the global GC root. This prevents
+ a) New roots from being added.
+ b) Processes from creating new temporary root files. */
+ AutoCloseFD fdGCLock = openGCLock(ltWrite);
+
+ /* Find the roots. Since we've grabbed the GC lock, the set of
+ permanent roots cannot increase now. */
+ printMsg(lvlError, format("finding garbage collector roots..."));
+ Roots rootMap = options.ignoreLiveness ? Roots() : findRoots();
+
+ foreach (Roots::iterator, i, rootMap) state.roots.insert(i->second);
+
+ /* Add additional roots returned by the program specified by the
+ NIX_ROOT_FINDER environment variable. This is typically used
+ to add running programs to the set of roots (to prevent them
+ from being garbage collected). */
+ if (!options.ignoreLiveness)
+ addAdditionalRoots(*this, state.roots);
+
+ /* Read the temporary roots. This acquires read locks on all
+ per-process temporary root files. So after this point no paths
+ can be added to the set of temporary roots. */
+ FDs fds;
+ readTempRoots(state.tempRoots, fds);
+ state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
+
+ /* After this point the set of roots or temporary roots cannot
+ increase, since we hold locks on everything. So everything
+ that is not reachable from `roots'. */
+
+ if (state.shouldDelete) {
+ if (pathExists(state.trashDir)) deleteGarbage(state, state.trashDir);
+ createDirs(state.trashDir);
+ }
+
+ /* Now either delete all garbage paths, or just the specified
+ paths (for gcDeleteSpecific). */
+
+ if (options.action == GCOptions::gcDeleteSpecific) {
+
+ foreach (PathSet::iterator, i, options.pathsToDelete) {
+ assertStorePath(*i);
+ tryToDelete(state, *i);
+ if (state.dead.find(*i) == state.dead.end())
+ throw Error(format("cannot delete path `%1%' since it is still alive") % *i);
+ }
+
+ } else if (options.maxFreed > 0) {
+
+ if (state.shouldDelete)
+ printMsg(lvlError, format("deleting garbage..."));
+ else
+ printMsg(lvlError, format("determining live/dead paths..."));
+
+ try {
+
+ AutoCloseDir dir = opendir(settings.nixStore.c_str());
+ if (!dir) throw SysError(format("opening directory `%1%'") % settings.nixStore);
+
+ /* Read the store and immediately delete all paths that
+ aren't valid. When using --max-freed etc., deleting
+ invalid paths is preferred over deleting unreachable
+ paths, since unreachable paths could become reachable
+ again. We don't use readDirectory() here so that GCing
+ can start faster. */
+ Paths entries;
+ struct dirent * dirent;
+ while (errno = 0, dirent = readdir(dir)) {
+ checkInterrupt();
+ string name = dirent->d_name;
+ if (name == "." || name == "..") continue;
+ Path path = settings.nixStore + "/" + name;
+ if (isValidPath(path))
+ entries.push_back(path);
+ else
+ tryToDelete(state, path);
+ }
+
+ dir.close();
+
+ /* Now delete the unreachable valid paths. Randomise the
+ order in which we delete entries to make the collector
+ less biased towards deleting paths that come
+ alphabetically first (e.g. /nix/store/000...). This
+ matters when using --max-freed etc. */
+ vector<Path> entries_(entries.begin(), entries.end());
+ random_shuffle(entries_.begin(), entries_.end());
+
+ foreach (vector<Path>::iterator, i, entries_)
+ tryToDelete(state, *i);
+
+ } catch (GCLimitReached & e) {
+ }
+ }
+
+ if (state.options.action == GCOptions::gcReturnLive) {
+ state.results.paths = state.alive;
+ return;
+ }
+
+ if (state.options.action == GCOptions::gcReturnDead) {
+ state.results.paths = state.dead;
+ return;
+ }
+
+ /* Allow other processes to add to the store from here on. */
+ fdGCLock.close();
+ fds.clear();
+
+ /* Delete the trash directory. */
+ printMsg(lvlInfo, format("deleting `%1%'") % state.trashDir);
+ deleteGarbage(state, state.trashDir);
+
+ /* Clean up the links directory. */
+ if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
+ printMsg(lvlError, format("deleting unused links..."));
+ removeUnusedLinks(state);
+ }
+
+ /* While we're at it, vacuum the database. */
+ if (options.action == GCOptions::gcDeleteDead) vacuumDB();
+}
+
+
+}
diff --git a/nix/libstore/globals.cc b/nix/libstore/globals.cc
new file mode 100644
index 0000000000..86fa56739c
--- /dev/null
+++ b/nix/libstore/globals.cc
@@ -0,0 +1,240 @@
+#include "config.h"
+
+#include "globals.hh"
+#include "util.hh"
+
+#include <map>
+#include <algorithm>
+
+
+namespace nix {
+
+
+/* The default location of the daemon socket, relative to nixStateDir.
+ The socket is in a directory to allow you to control access to the
+ Nix daemon by setting the mode/ownership of the directory
+ appropriately. (This wouldn't work on the socket itself since it
+ must be deleted and recreated on startup.) */
+#define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
+
+
+Settings settings;
+
+
+Settings::Settings()
+{
+ keepFailed = false;
+ keepGoing = false;
+ tryFallback = false;
+ buildVerbosity = lvlError;
+ maxBuildJobs = 1;
+ buildCores = 1;
+ readOnlyMode = false;
+ thisSystem = SYSTEM;
+ maxSilentTime = 0;
+ buildTimeout = 0;
+ useBuildHook = true;
+ printBuildTrace = false;
+ reservedSize = 1024 * 1024;
+ fsyncMetadata = true;
+ useSQLiteWAL = true;
+ syncBeforeRegistering = false;
+ useSubstitutes = true;
+ useChroot = false;
+ useSshSubstituter = false;
+ impersonateLinux26 = false;
+ keepLog = true;
+ compressLog = true;
+ maxLogSize = 0;
+ cacheFailure = false;
+ pollInterval = 5;
+ checkRootReachability = false;
+ gcKeepOutputs = false;
+ gcKeepDerivations = true;
+ autoOptimiseStore = false;
+ envKeepDerivations = false;
+ lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
+ showTrace = false;
+}
+
+
+void Settings::processEnvironment()
+{
+ nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
+ nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
+ nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
+ nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
+ nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
+ nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
+ nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
+ nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
+ nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH);
+}
+
+
+void Settings::loadConfFile()
+{
+ Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str();
+ if (!pathExists(settingsFile)) return;
+ string contents = readFile(settingsFile);
+
+ unsigned int pos = 0;
+
+ while (pos < contents.size()) {
+ string line;
+ while (pos < contents.size() && contents[pos] != '\n')
+ line += contents[pos++];
+ pos++;
+
+ string::size_type hash = line.find('#');
+ if (hash != string::npos)
+ line = string(line, 0, hash);
+
+ vector<string> tokens = tokenizeString<vector<string> >(line);
+ if (tokens.empty()) continue;
+
+ if (tokens.size() < 2 || tokens[1] != "=")
+ throw Error(format("illegal configuration line `%1%' in `%2%'") % line % settingsFile);
+
+ string name = tokens[0];
+
+ vector<string>::iterator i = tokens.begin();
+ advance(i, 2);
+ settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow
+ };
+}
+
+
+void Settings::set(const string & name, const string & value)
+{
+ settings[name] = value;
+ overrides[name] = value;
+}
+
+
+void Settings::update()
+{
+ get(tryFallback, "build-fallback");
+ get(maxBuildJobs, "build-max-jobs");
+ get(buildCores, "build-cores");
+ get(thisSystem, "system");
+ get(maxSilentTime, "build-max-silent-time");
+ get(buildTimeout, "build-timeout");
+ get(reservedSize, "gc-reserved-space");
+ get(fsyncMetadata, "fsync-metadata");
+ get(useSQLiteWAL, "use-sqlite-wal");
+ get(syncBeforeRegistering, "sync-before-registering");
+ get(useSubstitutes, "build-use-substitutes");
+ get(buildUsersGroup, "build-users-group");
+ get(useChroot, "build-use-chroot");
+ get(dirsInChroot, "build-chroot-dirs");
+ get(impersonateLinux26, "build-impersonate-linux-26");
+ get(keepLog, "build-keep-log");
+ get(compressLog, "build-compress-log");
+ get(maxLogSize, "build-max-log-size");
+ get(cacheFailure, "build-cache-failure");
+ get(pollInterval, "build-poll-interval");
+ get(checkRootReachability, "gc-check-reachability");
+ get(gcKeepOutputs, "gc-keep-outputs");
+ get(gcKeepDerivations, "gc-keep-derivations");
+ get(autoOptimiseStore, "auto-optimise-store");
+ get(envKeepDerivations, "env-keep-derivations");
+ get(sshSubstituterHosts, "ssh-substituter-hosts");
+ get(useSshSubstituter, "use-ssh-substituter");
+
+ string subs = getEnv("NIX_SUBSTITUTERS", "default");
+ if (subs == "default") {
+ substituters.clear();
+#if 0
+ if (getEnv("NIX_OTHER_STORES") != "")
+ substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
+#endif
+ substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
+ substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
+ if (useSshSubstituter)
+ substituters.push_back(nixLibexecDir + "/nix/substituters/download-via-ssh");
+ } else
+ substituters = tokenizeString<Strings>(subs, ":");
+}
+
+
+void Settings::get(string & res, const string & name)
+{
+ SettingsMap::iterator i = settings.find(name);
+ if (i == settings.end()) return;
+ res = i->second;
+}
+
+
+void Settings::get(bool & res, const string & name)
+{
+ SettingsMap::iterator i = settings.find(name);
+ if (i == settings.end()) return;
+ if (i->second == "true") res = true;
+ else if (i->second == "false") res = false;
+ else throw Error(format("configuration option `%1%' should be either `true' or `false', not `%2%'")
+ % name % i->second);
+}
+
+
+void Settings::get(StringSet & res, const string & name)
+{
+ SettingsMap::iterator i = settings.find(name);
+ if (i == settings.end()) return;
+ res.clear();
+ Strings ss = tokenizeString<Strings>(i->second);
+ res.insert(ss.begin(), ss.end());
+}
+
+void Settings::get(Strings & res, const string & name)
+{
+ SettingsMap::iterator i = settings.find(name);
+ if (i == settings.end()) return;
+ res = tokenizeString<Strings>(i->second);
+}
+
+
+template<class N> void Settings::get(N & res, const string & name)
+{
+ SettingsMap::iterator i = settings.find(name);
+ if (i == settings.end()) return;
+ if (!string2Int(i->second, res))
+ throw Error(format("configuration setting `%1%' should have an integer value") % name);
+}
+
+
+string Settings::pack()
+{
+ string s;
+ foreach (SettingsMap::iterator, i, settings) {
+ if (i->first.find('\n') != string::npos ||
+ i->first.find('=') != string::npos ||
+ i->second.find('\n') != string::npos)
+ throw Error("illegal option name/value");
+ s += i->first; s += '='; s += i->second; s += '\n';
+ }
+ return s;
+}
+
+
+void Settings::unpack(const string & pack) {
+ Strings lines = tokenizeString<Strings>(pack, "\n");
+ foreach (Strings::iterator, i, lines) {
+ string::size_type eq = i->find('=');
+ if (eq == string::npos)
+ throw Error("illegal option name/value");
+ set(i->substr(0, eq), i->substr(eq + 1));
+ }
+}
+
+
+Settings::SettingsMap Settings::getOverrides()
+{
+ return overrides;
+}
+
+
+const string nixVersion = PACKAGE_VERSION;
+
+
+}
diff --git a/nix/libstore/globals.hh b/nix/libstore/globals.hh
new file mode 100644
index 0000000000..711c365294
--- /dev/null
+++ b/nix/libstore/globals.hh
@@ -0,0 +1,218 @@
+#pragma once
+
+#include "types.hh"
+
+#include <map>
+#include <sys/types.h>
+
+
+namespace nix {
+
+
+struct Settings {
+
+ typedef std::map<string, string> SettingsMap;
+
+ Settings();
+
+ void processEnvironment();
+
+ void loadConfFile();
+
+ void set(const string & name, const string & value);
+
+ void update();
+
+ string pack();
+
+ void unpack(const string & pack);
+
+ SettingsMap getOverrides();
+
+ /* The directory where we store sources and derived files. */
+ Path nixStore;
+
+ Path nixDataDir; /* !!! fix */
+
+ /* The directory where we log various operations. */
+ Path nixLogDir;
+
+ /* The directory where state is stored. */
+ Path nixStateDir;
+
+ /* The directory where we keep the SQLite database. */
+ Path nixDBPath;
+
+ /* The directory where configuration files are stored. */
+ Path nixConfDir;
+
+ /* The directory where internal helper programs are stored. */
+ Path nixLibexecDir;
+
+ /* The directory where the main programs are stored. */
+ Path nixBinDir;
+
+ /* File name of the socket the daemon listens to. */
+ Path nixDaemonSocketFile;
+
+ /* Whether to keep temporary directories of failed builds. */
+ bool keepFailed;
+
+ /* Whether to keep building subgoals when a sibling (another
+ subgoal of the same goal) fails. */
+ bool keepGoing;
+
+ /* Whether, if we cannot realise the known closure corresponding
+ to a derivation, we should try to normalise the derivation
+ instead. */
+ bool tryFallback;
+
+ /* Verbosity level for build output. */
+ Verbosity buildVerbosity;
+
+ /* Maximum number of parallel build jobs. 0 means unlimited. */
+ unsigned int maxBuildJobs;
+
+ /* Number of CPU cores to utilize in parallel within a build,
+ i.e. by passing this number to Make via '-j'. 0 means that the
+ number of actual CPU cores on the local host ought to be
+ auto-detected. */
+ unsigned int buildCores;
+
+ /* Read-only mode. Don't copy stuff to the store, don't change
+ the database. */
+ bool readOnlyMode;
+
+ /* The canonical system name, as returned by config.guess. */
+ string thisSystem;
+
+ /* The maximum time in seconds that a builer can go without
+ producing any output on stdout/stderr before it is killed. 0
+ means infinity. */
+ time_t maxSilentTime;
+
+ /* The maximum duration in seconds that a builder can run. 0
+ means infinity. */
+ time_t buildTimeout;
+
+ /* The substituters. There are programs that can somehow realise
+ a store path without building, e.g., by downloading it or
+ copying it from a CD. */
+ Paths substituters;
+
+ /* Whether to use build hooks (for distributed builds). Sometimes
+ users want to disable this from the command-line. */
+ bool useBuildHook;
+
+ /* Whether buildDerivations() should print out lines on stderr in
+ a fixed format to allow its progress to be monitored. Each
+ line starts with a "@". The following are defined:
+
+ @ build-started <drvpath> <outpath> <system> <logfile>
+ @ build-failed <drvpath> <outpath> <exitcode> <error text>
+ @ build-succeeded <drvpath> <outpath>
+ @ substituter-started <outpath> <substituter>
+ @ substituter-failed <outpath> <exitcode> <error text>
+ @ substituter-succeeded <outpath>
+
+ Best combined with --no-build-output, otherwise stderr might
+ conceivably contain lines in this format printed by the
+ builders. */
+ bool printBuildTrace;
+
+ /* Amount of reserved space for the garbage collector
+ (/nix/var/nix/db/reserved). */
+ off_t reservedSize;
+
+ /* Whether SQLite should use fsync. */
+ bool fsyncMetadata;
+
+ /* Whether SQLite should use WAL mode. */
+ bool useSQLiteWAL;
+
+ /* Whether to call sync() before registering a path as valid. */
+ bool syncBeforeRegistering;
+
+ /* Whether to use substitutes. */
+ bool useSubstitutes;
+
+ /* The Unix group that contains the build users. */
+ string buildUsersGroup;
+
+ /* Whether to build in chroot. */
+ bool useChroot;
+
+ /* The directories from the host filesystem to be included in the
+ chroot. */
+ StringSet dirsInChroot;
+
+ /* Set of ssh connection strings for the ssh substituter */
+ Strings sshSubstituterHosts;
+
+ /* Whether to use the ssh substituter at all */
+ bool useSshSubstituter;
+
+ /* Whether to impersonate a Linux 2.6 machine on newer kernels. */
+ bool impersonateLinux26;
+
+ /* Whether to store build logs. */
+ bool keepLog;
+
+ /* Whether to compress logs. */
+ bool compressLog;
+
+ /* Maximum number of bytes a builder can write to stdout/stderr
+ before being killed (0 means no limit). */
+ unsigned long maxLogSize;
+
+ /* Whether to cache build failures. */
+ bool cacheFailure;
+
+ /* How often (in seconds) to poll for locks. */
+ unsigned int pollInterval;
+
+ /* Whether to check if new GC roots can in fact be found by the
+ garbage collector. */
+ bool checkRootReachability;
+
+ /* Whether the garbage collector should keep outputs of live
+ derivations. */
+ bool gcKeepOutputs;
+
+ /* Whether the garbage collector should keep derivers of live
+ paths. */
+ bool gcKeepDerivations;
+
+ /* Whether to automatically replace files with identical contents
+ with hard links. */
+ bool autoOptimiseStore;
+
+ /* Whether to add derivations as a dependency of user environments
+ (to prevent them from being GCed). */
+ bool envKeepDerivations;
+
+ /* Whether to lock the Nix client and worker to the same CPU. */
+ bool lockCPU;
+
+ /* Whether to show a stack trace if Nix evaluation fails. */
+ bool showTrace;
+
+private:
+ SettingsMap settings, overrides;
+
+ void get(string & res, const string & name);
+ void get(bool & res, const string & name);
+ void get(StringSet & res, const string & name);
+ void get(Strings & res, const string & name);
+ template<class N> void get(N & res, const string & name);
+};
+
+
+// FIXME: don't use a global variable.
+extern Settings settings;
+
+
+extern const string nixVersion;
+
+
+}
diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc
new file mode 100644
index 0000000000..1293a6e8f2
--- /dev/null
+++ b/nix/libstore/local-store.cc
@@ -0,0 +1,2010 @@
+#include "config.h"
+#include "local-store.hh"
+#include "globals.hh"
+#include "archive.hh"
+#include "pathlocks.hh"
+#include "worker-protocol.hh"
+#include "derivations.hh"
+#include "affinity.hh"
+
+#include <iostream>
+#include <algorithm>
+#include <cstring>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <utime.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <stdio.h>
+#include <time.h>
+
+#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H
+#include <sched.h>
+#include <sys/statvfs.h>
+#include <sys/mount.h>
+#endif
+
+#if HAVE_LINUX_FS_H
+#include <linux/fs.h>
+#include <sys/ioctl.h>
+#include <errno.h>
+#endif
+
+#include <sqlite3.h>
+
+
+namespace nix {
+
+
+MakeError(SQLiteError, Error);
+MakeError(SQLiteBusy, SQLiteError);
+
+
+static void throwSQLiteError(sqlite3 * db, const format & f)
+ __attribute__ ((noreturn));
+
+static void throwSQLiteError(sqlite3 * db, const format & f)
+{
+ int err = sqlite3_errcode(db);
+ if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) {
+ if (err == SQLITE_PROTOCOL)
+ printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)");
+ else {
+ static bool warned = false;
+ if (!warned) {
+ printMsg(lvlError, "warning: SQLite database is busy");
+ warned = true;
+ }
+ }
+ /* Sleep for a while since retrying the transaction right away
+ is likely to fail again. */
+#if HAVE_NANOSLEEP
+ struct timespec t;
+ t.tv_sec = 0;
+ t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */
+ nanosleep(&t, 0);
+#else
+ sleep(1);
+#endif
+ throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db));
+ }
+ else
+ throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db));
+}
+
+
+/* Convenience macros for retrying a SQLite transaction. */
+#define retry_sqlite while (1) { try {
+#define end_retry_sqlite break; } catch (SQLiteBusy & e) { } }
+
+
+SQLite::~SQLite()
+{
+ try {
+ if (db && sqlite3_close(db) != SQLITE_OK)
+ throwSQLiteError(db, "closing database");
+ } catch (...) {
+ ignoreException();
+ }
+}
+
+
+void SQLiteStmt::create(sqlite3 * db, const string & s)
+{
+ checkInterrupt();
+ assert(!stmt);
+ if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK)
+ throwSQLiteError(db, "creating statement");
+ this->db = db;
+}
+
+
+void SQLiteStmt::reset()
+{
+ assert(stmt);
+ /* Note: sqlite3_reset() returns the error code for the most
+ recent call to sqlite3_step(). So ignore it. */
+ sqlite3_reset(stmt);
+ curArg = 1;
+}
+
+
+SQLiteStmt::~SQLiteStmt()
+{
+ try {
+ if (stmt && sqlite3_finalize(stmt) != SQLITE_OK)
+ throwSQLiteError(db, "finalizing statement");
+ } catch (...) {
+ ignoreException();
+ }
+}
+
+
+void SQLiteStmt::bind(const string & value)
+{
+ if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
+ throwSQLiteError(db, "binding argument");
+}
+
+
+void SQLiteStmt::bind(int value)
+{
+ if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK)
+ throwSQLiteError(db, "binding argument");
+}
+
+
+void SQLiteStmt::bind64(long long value)
+{
+ if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK)
+ throwSQLiteError(db, "binding argument");
+}
+
+
+void SQLiteStmt::bind()
+{
+ if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK)
+ throwSQLiteError(db, "binding argument");
+}
+
+
+/* Helper class to ensure that prepared statements are reset when
+ leaving the scope that uses them. Unfinished prepared statements
+ prevent transactions from being aborted, and can cause locks to be
+ kept when they should be released. */
+struct SQLiteStmtUse
+{
+ SQLiteStmt & stmt;
+ SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt)
+ {
+ stmt.reset();
+ }
+ ~SQLiteStmtUse()
+ {
+ try {
+ stmt.reset();
+ } catch (...) {
+ ignoreException();
+ }
+ }
+};
+
+
+struct SQLiteTxn
+{
+ bool active;
+ sqlite3 * db;
+
+ SQLiteTxn(sqlite3 * db) : active(false) {
+ this->db = db;
+ if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK)
+ throwSQLiteError(db, "starting transaction");
+ active = true;
+ }
+
+ void commit()
+ {
+ if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK)
+ throwSQLiteError(db, "committing transaction");
+ active = false;
+ }
+
+ ~SQLiteTxn()
+ {
+ try {
+ if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK)
+ throwSQLiteError(db, "aborting transaction");
+ } catch (...) {
+ ignoreException();
+ }
+ }
+};
+
+
+void checkStoreNotSymlink()
+{
+ if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return;
+ Path path = settings.nixStore;
+ struct stat st;
+ while (path != "/") {
+ if (lstat(path.c_str(), &st))
+ throw SysError(format("getting status of `%1%'") % path);
+ if (S_ISLNK(st.st_mode))
+ throw Error(format(
+ "the path `%1%' is a symlink; "
+ "this is not allowed for the Nix store and its parent directories")
+ % path);
+ path = dirOf(path);
+ }
+}
+
+
+LocalStore::LocalStore(bool reserveSpace)
+ : didSetSubstituterEnv(false)
+{
+ schemaPath = settings.nixDBPath + "/schema";
+
+ if (settings.readOnlyMode) {
+ openDB(false);
+ return;
+ }
+
+ /* Create missing state directories if they don't already exist. */
+ createDirs(settings.nixStore);
+ makeStoreWritable();
+ createDirs(linksDir = settings.nixStore + "/.links");
+ Path profilesDir = settings.nixStateDir + "/profiles";
+ createDirs(settings.nixStateDir + "/profiles");
+ createDirs(settings.nixStateDir + "/temproots");
+ createDirs(settings.nixDBPath);
+ Path gcRootsDir = settings.nixStateDir + "/gcroots";
+ if (!pathExists(gcRootsDir)) {
+ createDirs(gcRootsDir);
+ createSymlink(profilesDir, gcRootsDir + "/profiles");
+ }
+
+ checkStoreNotSymlink();
+
+ /* We can't open a SQLite database if the disk is full. Since
+ this prevents the garbage collector from running when it's most
+ needed, we reserve some dummy space that we can free just
+ before doing a garbage collection. */
+ try {
+ Path reservedPath = settings.nixDBPath + "/reserved";
+ if (reserveSpace) {
+ struct stat st;
+ if (stat(reservedPath.c_str(), &st) == -1 ||
+ st.st_size != settings.reservedSize)
+ writeFile(reservedPath, string(settings.reservedSize, 'X'));
+ }
+ else
+ deletePath(reservedPath);
+ } catch (SysError & e) { /* don't care about errors */
+ }
+
+ /* Acquire the big fat lock in shared mode to make sure that no
+ schema upgrade is in progress. */
+ try {
+ Path globalLockPath = settings.nixDBPath + "/big-lock";
+ globalLock = openLockFile(globalLockPath.c_str(), true);
+ } catch (SysError & e) {
+ if (e.errNo != EACCES) throw;
+ settings.readOnlyMode = true;
+ openDB(false);
+ return;
+ }
+
+ if (!lockFile(globalLock, ltRead, false)) {
+ printMsg(lvlError, "waiting for the big Nix store lock...");
+ lockFile(globalLock, ltRead, true);
+ }
+
+ /* Check the current database schema and if necessary do an
+ upgrade. */
+ int curSchema = getSchema();
+ if (curSchema > nixSchemaVersion)
+ throw Error(format("current Nix store schema is version %1%, but I only support %2%")
+ % curSchema % nixSchemaVersion);
+
+ else if (curSchema == 0) { /* new store */
+ curSchema = nixSchemaVersion;
+ openDB(true);
+ writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
+ }
+
+ else if (curSchema < nixSchemaVersion) {
+ if (curSchema < 5)
+ throw Error(
+ "Your Nix store has a database in Berkeley DB format,\n"
+ "which is no longer supported. To convert to the new format,\n"
+ "please upgrade Nix to version 0.12 first.");
+
+ if (!lockFile(globalLock, ltWrite, false)) {
+ printMsg(lvlError, "waiting for exclusive access to the Nix store...");
+ lockFile(globalLock, ltWrite, true);
+ }
+
+ /* Get the schema version again, because another process may
+ have performed the upgrade already. */
+ curSchema = getSchema();
+
+ if (curSchema < 6) upgradeStore6();
+ else if (curSchema < 7) { upgradeStore7(); openDB(true); }
+
+ writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
+
+ lockFile(globalLock, ltRead, true);
+ }
+
+ else openDB(false);
+}
+
+
+LocalStore::~LocalStore()
+{
+ try {
+ foreach (RunningSubstituters::iterator, i, runningSubstituters) {
+ if (i->second.disabled) continue;
+ i->second.to.close();
+ i->second.from.close();
+ i->second.error.close();
+ i->second.pid.wait(true);
+ }
+ } catch (...) {
+ ignoreException();
+ }
+}
+
+
+int LocalStore::getSchema()
+{
+ int curSchema = 0;
+ if (pathExists(schemaPath)) {
+ string s = readFile(schemaPath);
+ if (!string2Int(s, curSchema))
+ throw Error(format("`%1%' is corrupt") % schemaPath);
+ }
+ return curSchema;
+}
+
+
+void LocalStore::openDB(bool create)
+{
+ if (access(settings.nixDBPath.c_str(), R_OK | W_OK))
+ throw SysError(format("Nix database directory `%1%' is not writable") % settings.nixDBPath);
+
+ /* Open the Nix database. */
+ string dbPath = settings.nixDBPath + "/db.sqlite";
+ if (sqlite3_open_v2(dbPath.c_str(), &db.db,
+ SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
+ throw Error(format("cannot open Nix database `%1%'") % dbPath);
+
+ if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
+ throwSQLiteError(db, "setting timeout");
+
+ if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK)
+ throwSQLiteError(db, "enabling foreign keys");
+
+ /* !!! check whether sqlite has been built with foreign key
+ support */
+
+ /* Whether SQLite should fsync(). "Normal" synchronous mode
+ should be safe enough. If the user asks for it, don't sync at
+ all. This can cause database corruption if the system
+ crashes. */
+ string syncMode = settings.fsyncMetadata ? "normal" : "off";
+ if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
+ throwSQLiteError(db, "setting synchronous mode");
+
+ /* Set the SQLite journal mode. WAL mode is fastest, so it's the
+ default. */
+ string mode = settings.useSQLiteWAL ? "wal" : "truncate";
+ string prevMode;
+ {
+ SQLiteStmt stmt;
+ stmt.create(db, "pragma main.journal_mode;");
+ if (sqlite3_step(stmt) != SQLITE_ROW)
+ throwSQLiteError(db, "querying journal mode");
+ prevMode = string((const char *) sqlite3_column_text(stmt, 0));
+ }
+ if (prevMode != mode &&
+ sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
+ throwSQLiteError(db, "setting journal mode");
+
+ /* Increase the auto-checkpoint interval to 40000 pages. This
+ seems enough to ensure that instantiating the NixOS system
+ derivation is done in a single fsync(). */
+ if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, 0, 0) != SQLITE_OK)
+ throwSQLiteError(db, "setting autocheckpoint interval");
+
+ /* Initialise the database schema, if necessary. */
+ if (create) {
+ const char * schema =
+#include "schema.sql.hh"
+ ;
+ if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK)
+ throwSQLiteError(db, "initialising database schema");
+ }
+
+ /* Prepare SQL statements. */
+ stmtRegisterValidPath.create(db,
+ "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);");
+ stmtUpdatePathInfo.create(db,
+ "update ValidPaths set narSize = ?, hash = ? where path = ?;");
+ stmtAddReference.create(db,
+ "insert or replace into Refs (referrer, reference) values (?, ?);");
+ stmtQueryPathInfo.create(db,
+ "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;");
+ stmtQueryReferences.create(db,
+ "select path from Refs join ValidPaths on reference = id where referrer = ?;");
+ stmtQueryReferrers.create(db,
+ "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
+ stmtInvalidatePath.create(db,
+ "delete from ValidPaths where path = ?;");
+ stmtRegisterFailedPath.create(db,
+ "insert or ignore into FailedPaths (path, time) values (?, ?);");
+ stmtHasPathFailed.create(db,
+ "select time from FailedPaths where path = ?;");
+ stmtQueryFailedPaths.create(db,
+ "select path from FailedPaths;");
+ // If the path is a derivation, then clear its outputs.
+ stmtClearFailedPath.create(db,
+ "delete from FailedPaths where ?1 = '*' or path = ?1 "
+ "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);");
+ stmtAddDerivationOutput.create(db,
+ "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
+ stmtQueryValidDerivers.create(db,
+ "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
+ stmtQueryDerivationOutputs.create(db,
+ "select id, path from DerivationOutputs where drv = ?;");
+ // Use "path >= ?" with limit 1 rather than "path like '?%'" to
+ // ensure efficient lookup.
+ stmtQueryPathFromHashPart.create(db,
+ "select path from ValidPaths where path >= ? limit 1;");
+}
+
+
+/* To improve purity, users may want to make the Nix store a read-only
+ bind mount. So make the Nix store writable for this process. */
+void LocalStore::makeStoreWritable()
+{
+#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_REMOUNT)
+ if (getuid() != 0) return;
+ /* Check if /nix/store is on a read-only mount. */
+ struct statvfs stat;
+ if (statvfs(settings.nixStore.c_str(), &stat) != 0)
+ throw SysError("getting info about the Nix store mount point");
+
+ if (stat.f_flag & ST_RDONLY) {
+ if (unshare(CLONE_NEWNS) == -1)
+ throw SysError("setting up a private mount namespace");
+
+ if (mount(0, settings.nixStore.c_str(), 0, MS_REMOUNT | MS_BIND, 0) == -1)
+ throw SysError(format("remounting %1% writable") % settings.nixStore);
+ }
+#endif
+}
+
+
+const time_t mtimeStore = 1; /* 1 second into the epoch */
+
+
+static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st)
+{
+ if (!S_ISLNK(st.st_mode)) {
+
+ /* Mask out all type related bits. */
+ mode_t mode = st.st_mode & ~S_IFMT;
+
+ if (mode != 0444 && mode != 0555) {
+ mode = (st.st_mode & S_IFMT)
+ | 0444
+ | (st.st_mode & S_IXUSR ? 0111 : 0);
+ if (chmod(path.c_str(), mode) == -1)
+ throw SysError(format("changing mode of `%1%' to %2$o") % path % mode);
+ }
+
+ }
+
+ if (st.st_mtime != mtimeStore) {
+ struct timeval times[2];
+ times[0].tv_sec = st.st_atime;
+ times[0].tv_usec = 0;
+ times[1].tv_sec = mtimeStore;
+ times[1].tv_usec = 0;
+#if HAVE_LUTIMES
+ if (lutimes(path.c_str(), times) == -1)
+ if (errno != ENOSYS ||
+ (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1))
+#else
+ if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
+#endif
+ throw SysError(format("changing modification time of `%1%'") % path);
+ }
+}
+
+
+void canonicaliseTimestampAndPermissions(const Path & path)
+{
+ struct stat st;
+ if (lstat(path.c_str(), &st))
+ throw SysError(format("getting attributes of path `%1%'") % path);
+ canonicaliseTimestampAndPermissions(path, st);
+}
+
+
+static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen)
+{
+ checkInterrupt();
+
+ struct stat st;
+ if (lstat(path.c_str(), &st))
+ throw SysError(format("getting attributes of path `%1%'") % path);
+
+ /* Really make sure that the path is of a supported type. This
+ has already been checked in dumpPath(). */
+ assert(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode));
+
+ /* Fail if the file is not owned by the build user. This prevents
+ us from messing up the ownership/permissions of files
+ hard-linked into the output (e.g. "ln /etc/shadow $out/foo").
+ However, ignore files that we chown'ed ourselves previously to
+ ensure that we don't fail on hard links within the same build
+ (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
+ if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
+ assert(!S_ISDIR(st.st_mode));
+ if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
+ throw BuildError(format("invalid ownership on file `%1%'") % path);
+ mode_t mode = st.st_mode & ~S_IFMT;
+ assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
+ return;
+ }
+
+ inodesSeen.insert(Inode(st.st_dev, st.st_ino));
+
+ canonicaliseTimestampAndPermissions(path, st);
+
+ /* Change ownership to the current uid. If it's a symlink, use
+ lchown if available, otherwise don't bother. Wrong ownership
+ of a symlink doesn't matter, since the owning user can't change
+ the symlink and can't delete it because the directory is not
+ writable. The only exception is top-level paths in the Nix
+ store (since that directory is group-writable for the Nix build
+ users group); we check for this case below. */
+ if (st.st_uid != geteuid()) {
+#if HAVE_LCHOWN
+ if (lchown(path.c_str(), geteuid(), (gid_t) -1) == -1)
+#else
+ if (!S_ISLNK(st.st_mode) &&
+ chown(path.c_str(), geteuid(), (gid_t) -1) == -1)
+#endif
+ throw SysError(format("changing owner of `%1%' to %2%")
+ % path % geteuid());
+ }
+
+ if (S_ISDIR(st.st_mode)) {
+ Strings names = readDirectory(path);
+ foreach (Strings::iterator, i, names)
+ canonicalisePathMetaData_(path + "/" + *i, fromUid, inodesSeen);
+ }
+}
+
+
+void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen)
+{
+ canonicalisePathMetaData_(path, fromUid, inodesSeen);
+
+ /* On platforms that don't have lchown(), the top-level path can't
+ be a symlink, since we can't change its ownership. */
+ struct stat st;
+ if (lstat(path.c_str(), &st))
+ throw SysError(format("getting attributes of path `%1%'") % path);
+
+ if (st.st_uid != geteuid()) {
+ assert(S_ISLNK(st.st_mode));
+ throw Error(format("wrong ownership of top-level store path `%1%'") % path);
+ }
+}
+
+
+void canonicalisePathMetaData(const Path & path, uid_t fromUid)
+{
+ InodesSeen inodesSeen;
+ canonicalisePathMetaData(path, fromUid, inodesSeen);
+}
+
+
+void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv)
+{
+ string drvName = storePathToName(drvPath);
+ assert(isDerivation(drvName));
+ drvName = string(drvName, 0, drvName.size() - drvExtension.size());
+
+ if (isFixedOutputDrv(drv)) {
+ DerivationOutputs::const_iterator out = drv.outputs.find("out");
+ if (out == drv.outputs.end())
+ throw Error(format("derivation `%1%' does not have an output named `out'") % drvPath);
+
+ bool recursive; HashType ht; Hash h;
+ out->second.parseHashInfo(recursive, ht, h);
+ Path outPath = makeFixedOutputPath(recursive, ht, h, drvName);
+
+ StringPairs::const_iterator j = drv.env.find("out");
+ if (out->second.path != outPath || j == drv.env.end() || j->second != outPath)
+ throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'")
+ % drvPath % out->second.path % outPath);
+ }
+
+ else {
+ Derivation drvCopy(drv);
+ foreach (DerivationOutputs::iterator, i, drvCopy.outputs) {
+ i->second.path = "";
+ drvCopy.env[i->first] = "";
+ }
+
+ Hash h = hashDerivationModulo(*this, drvCopy);
+
+ foreach (DerivationOutputs::const_iterator, i, drv.outputs) {
+ Path outPath = makeOutputPath(i->first, h, drvName);
+ StringPairs::const_iterator j = drv.env.find(i->first);
+ if (i->second.path != outPath || j == drv.env.end() || j->second != outPath)
+ throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'")
+ % drvPath % i->second.path % outPath);
+ }
+ }
+}
+
+
+unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
+{
+ SQLiteStmtUse use(stmtRegisterValidPath);
+ stmtRegisterValidPath.bind(info.path);
+ stmtRegisterValidPath.bind("sha256:" + printHash(info.hash));
+ stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime);
+ if (info.deriver != "")
+ stmtRegisterValidPath.bind(info.deriver);
+ else
+ stmtRegisterValidPath.bind(); // null
+ if (info.narSize != 0)
+ stmtRegisterValidPath.bind64(info.narSize);
+ else
+ stmtRegisterValidPath.bind(); // null
+ if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE)
+ throwSQLiteError(db, format("registering valid path `%1%' in database") % info.path);
+ unsigned long long id = sqlite3_last_insert_rowid(db);
+
+ /* If this is a derivation, then store the derivation outputs in
+ the database. This is useful for the garbage collector: it can
+ efficiently query whether a path is an output of some
+ derivation. */
+ if (isDerivation(info.path)) {
+ Derivation drv = parseDerivation(readFile(info.path));
+
+ /* Verify that the output paths in the derivation are correct
+ (i.e., follow the scheme for computing output paths from
+ derivations). Note that if this throws an error, then the
+ DB transaction is rolled back, so the path validity
+ registration above is undone. */
+ if (checkOutputs) checkDerivationOutputs(info.path, drv);
+
+ foreach (DerivationOutputs::iterator, i, drv.outputs) {
+ SQLiteStmtUse use(stmtAddDerivationOutput);
+ stmtAddDerivationOutput.bind(id);
+ stmtAddDerivationOutput.bind(i->first);
+ stmtAddDerivationOutput.bind(i->second.path);
+ if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE)
+ throwSQLiteError(db, format("adding derivation output for `%1%' in database") % info.path);
+ }
+ }
+
+ return id;
+}
+
+
+void LocalStore::addReference(unsigned long long referrer, unsigned long long reference)
+{
+ SQLiteStmtUse use(stmtAddReference);
+ stmtAddReference.bind(referrer);
+ stmtAddReference.bind(reference);
+ if (sqlite3_step(stmtAddReference) != SQLITE_DONE)
+ throwSQLiteError(db, "adding reference to database");
+}
+
+
+void LocalStore::registerFailedPath(const Path & path)
+{
+ retry_sqlite {
+ SQLiteStmtUse use(stmtRegisterFailedPath);
+ stmtRegisterFailedPath.bind(path);
+ stmtRegisterFailedPath.bind(time(0));
+ if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE)
+ throwSQLiteError(db, format("registering failed path `%1%'") % path);
+ } end_retry_sqlite;
+}
+
+
+bool LocalStore::hasPathFailed(const Path & path)
+{
+ retry_sqlite {
+ SQLiteStmtUse use(stmtHasPathFailed);
+ stmtHasPathFailed.bind(path);
+ int res = sqlite3_step(stmtHasPathFailed);
+ if (res != SQLITE_DONE && res != SQLITE_ROW)
+ throwSQLiteError(db, "querying whether path failed");
+ return res == SQLITE_ROW;
+ } end_retry_sqlite;
+}
+
+
+PathSet LocalStore::queryFailedPaths()
+{
+ retry_sqlite {
+ SQLiteStmtUse use(stmtQueryFailedPaths);
+
+ PathSet res;
+ int r;
+ while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) {
+ const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0);
+ assert(s);
+ res.insert(s);
+ }
+
+ if (r != SQLITE_DONE)
+ throwSQLiteError(db, "error querying failed paths");
+
+ return res;
+ } end_retry_sqlite;
+}
+
+
+void LocalStore::clearFailedPaths(const PathSet & paths)
+{
+ retry_sqlite {
+ SQLiteTxn txn(db);
+
+ foreach (PathSet::const_iterator, i, paths) {
+ SQLiteStmtUse use(stmtClearFailedPath);
+ stmtClearFailedPath.bind(*i);
+ if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE)
+ throwSQLiteError(db, format("clearing failed path `%1%' in database") % *i);
+ }
+
+ txn.commit();
+ } end_retry_sqlite;
+}
+
+
+Hash parseHashField(const Path & path, const string & s)
+{
+ string::size_type colon = s.find(':');
+ if (colon == string::npos)
+ throw Error(format("corrupt hash `%1%' in valid-path entry for `%2%'")
+ % s % path);
+ HashType ht = parseHashType(string(s, 0, colon));
+ if (ht == htUnknown)
+ throw Error(format("unknown hash type `%1%' in valid-path entry for `%2%'")
+ % string(s, 0, colon) % path);
+ return parseHash(ht, string(s, colon + 1));
+}
+
+
+ValidPathInfo LocalStore::queryPathInfo(const Path & path)
+{
+ ValidPathInfo info;
+ info.path = path;
+
+ assertStorePath(path);
+
+ retry_sqlite {
+
+ /* Get the path info. */
+ SQLiteStmtUse use1(stmtQueryPathInfo);
+
+ stmtQueryPathInfo.bind(path);
+
+ int r = sqlite3_step(stmtQueryPathInfo);
+ if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path);
+ if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database");
+
+ info.id = sqlite3_column_int(stmtQueryPathInfo, 0);
+
+ const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1);
+ assert(s);
+ info.hash = parseHashField(path, s);
+
+ info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2);
+
+ s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3);
+ if (s) info.deriver = s;
+
+ /* Note that narSize = NULL yields 0. */
+ info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4);
+
+ /* Get the references. */
+ SQLiteStmtUse use2(stmtQueryReferences);
+
+ stmtQueryReferences.bind(info.id);
+
+ while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) {
+ s = (const char *) sqlite3_column_text(stmtQueryReferences, 0);
+ assert(s);
+ info.references.insert(s);
+ }
+
+ if (r != SQLITE_DONE)
+ throwSQLiteError(db, format("error getting references of `%1%'") % path);
+
+ return info;
+ } end_retry_sqlite;
+}
+
+
+/* Update path info in the database. Currently only updates the
+ narSize field. */
+void LocalStore::updatePathInfo(const ValidPathInfo & info)
+{
+ SQLiteStmtUse use(stmtUpdatePathInfo);
+ if (info.narSize != 0)
+ stmtUpdatePathInfo.bind64(info.narSize);
+ else
+ stmtUpdatePathInfo.bind(); // null
+ stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash));
+ stmtUpdatePathInfo.bind(info.path);
+ if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE)
+ throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path);
+}
+
+
+unsigned long long LocalStore::queryValidPathId(const Path & path)
+{
+ SQLiteStmtUse use(stmtQueryPathInfo);
+ stmtQueryPathInfo.bind(path);
+ int res = sqlite3_step(stmtQueryPathInfo);
+ if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0);
+ if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path);
+ throwSQLiteError(db, "querying path in database");
+}
+
+
+bool LocalStore::isValidPath_(const Path & path)
+{
+ SQLiteStmtUse use(stmtQueryPathInfo);
+ stmtQueryPathInfo.bind(path);
+ int res = sqlite3_step(stmtQueryPathInfo);
+ if (res != SQLITE_DONE && res != SQLITE_ROW)
+ throwSQLiteError(db, "querying path in database");
+ return res == SQLITE_ROW;
+}
+
+
+bool LocalStore::isValidPath(const Path & path)
+{
+ retry_sqlite {
+ return isValidPath_(path);
+ } end_retry_sqlite;
+}
+
+
+PathSet LocalStore::queryValidPaths(const PathSet & paths)
+{
+ retry_sqlite {
+ PathSet res;
+ foreach (PathSet::const_iterator, i, paths)
+ if (isValidPath_(*i)) res.insert(*i);
+ return res;
+ } end_retry_sqlite;
+}
+
+
+PathSet LocalStore::queryAllValidPaths()
+{
+ retry_sqlite {
+ SQLiteStmt stmt;
+ stmt.create(db, "select path from ValidPaths");
+
+ PathSet res;
+ int r;
+ while ((r = sqlite3_step(stmt)) == SQLITE_ROW) {
+ const char * s = (const char *) sqlite3_column_text(stmt, 0);
+ assert(s);
+ res.insert(s);
+ }
+
+ if (r != SQLITE_DONE)
+ throwSQLiteError(db, "error getting valid paths");
+
+ return res;
+ } end_retry_sqlite;
+}
+
+
+void LocalStore::queryReferences(const Path & path,
+ PathSet & references)
+{
+ ValidPathInfo info = queryPathInfo(path);
+ references.insert(info.references.begin(), info.references.end());
+}
+
+
+void LocalStore::queryReferrers_(const Path & path, PathSet & referrers)
+{
+ SQLiteStmtUse use(stmtQueryReferrers);
+
+ stmtQueryReferrers.bind(path);
+
+ int r;
+ while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) {
+ const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0);
+ assert(s);
+ referrers.insert(s);
+ }
+
+ if (r != SQLITE_DONE)
+ throwSQLiteError(db, format("error getting references of `%1%'") % path);
+}
+
+
+void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
+{
+ assertStorePath(path);
+ retry_sqlite {
+ queryReferrers_(path, referrers);
+ } end_retry_sqlite;
+}
+
+
+Path LocalStore::queryDeriver(const Path & path)
+{
+ return queryPathInfo(path).deriver;
+}
+
+
+PathSet LocalStore::queryValidDerivers(const Path & path)
+{
+ assertStorePath(path);
+
+ retry_sqlite {
+ SQLiteStmtUse use(stmtQueryValidDerivers);
+ stmtQueryValidDerivers.bind(path);
+
+ PathSet derivers;
+ int r;
+ while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) {
+ const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1);
+ assert(s);
+ derivers.insert(s);
+ }
+
+ if (r != SQLITE_DONE)
+ throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path);
+
+ return derivers;
+ } end_retry_sqlite;
+}
+
+
+PathSet LocalStore::queryDerivationOutputs(const Path & path)
+{
+ retry_sqlite {
+ SQLiteStmtUse use(stmtQueryDerivationOutputs);
+ stmtQueryDerivationOutputs.bind(queryValidPathId(path));
+
+ PathSet outputs;
+ int r;
+ while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) {
+ const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1);
+ assert(s);
+ outputs.insert(s);
+ }
+
+ if (r != SQLITE_DONE)
+ throwSQLiteError(db, format("error getting outputs of `%1%'") % path);
+
+ return outputs;
+ } end_retry_sqlite;
+}
+
+
+StringSet LocalStore::queryDerivationOutputNames(const Path & path)
+{
+ retry_sqlite {
+ SQLiteStmtUse use(stmtQueryDerivationOutputs);
+ stmtQueryDerivationOutputs.bind(queryValidPathId(path));
+
+ StringSet outputNames;
+ int r;
+ while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) {
+ const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0);
+ assert(s);
+ outputNames.insert(s);
+ }
+
+ if (r != SQLITE_DONE)
+ throwSQLiteError(db, format("error getting output names of `%1%'") % path);
+
+ return outputNames;
+ } end_retry_sqlite;
+}
+
+
+Path LocalStore::queryPathFromHashPart(const string & hashPart)
+{
+ if (hashPart.size() != 32) throw Error("invalid hash part");
+
+ Path prefix = settings.nixStore + "/" + hashPart;
+
+ retry_sqlite {
+ SQLiteStmtUse use(stmtQueryPathFromHashPart);
+ stmtQueryPathFromHashPart.bind(prefix);
+
+ int res = sqlite3_step(stmtQueryPathFromHashPart);
+ if (res == SQLITE_DONE) return "";
+ if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database");
+
+ const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0);
+ return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
+ } end_retry_sqlite;
+}
+
+
+void LocalStore::setSubstituterEnv()
+{
+ if (didSetSubstituterEnv) return;
+
+ /* Pass configuration options (including those overridden with
+ --option) to substituters. */
+ setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
+
+ didSetSubstituterEnv = true;
+}
+
+
+void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run)
+{
+ if (run.disabled || run.pid != -1) return;
+
+ debug(format("starting substituter program `%1%'") % substituter);
+
+ Pipe toPipe, fromPipe, errorPipe;
+
+ toPipe.create();
+ fromPipe.create();
+ errorPipe.create();
+
+ setSubstituterEnv();
+
+ run.pid = maybeVfork();
+
+ switch (run.pid) {
+
+ case -1:
+ throw SysError("unable to fork");
+
+ case 0: /* child */
+ try {
+ restoreAffinity();
+ if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
+ throw SysError("dupping stdin");
+ if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
+ throw SysError("dupping stdout");
+ if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
+ throw SysError("dupping stderr");
+ execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
+ throw SysError(format("executing `%1%'") % substituter);
+ } catch (std::exception & e) {
+ std::cerr << "error: " << e.what() << std::endl;
+ }
+ _exit(1);
+ }
+
+ /* Parent. */
+
+ run.program = baseNameOf(substituter);
+ run.to = toPipe.writeSide.borrow();
+ run.from = run.fromBuf.fd = fromPipe.readSide.borrow();
+ run.error = errorPipe.readSide.borrow();
+
+ toPipe.readSide.close();
+ fromPipe.writeSide.close();
+ errorPipe.writeSide.close();
+
+ /* The substituter may exit right away if it's disabled in any way
+ (e.g. copy-from-other-stores.pl will exit if no other stores
+ are configured). */
+ try {
+ getLineFromSubstituter(run);
+ } catch (EndOfFile & e) {
+ run.to.close();
+ run.from.close();
+ run.error.close();
+ run.disabled = true;
+ if (run.pid.wait(true) != 0) throw;
+ }
+}
+
+
+/* Read a line from the substituter's stdout, while also processing
+ its stderr. */
+string LocalStore::getLineFromSubstituter(RunningSubstituter & run)
+{
+ string res, err;
+
+ /* We might have stdout data left over from the last time. */
+ if (run.fromBuf.hasData()) goto haveData;
+
+ while (1) {
+ checkInterrupt();
+
+ fd_set fds;
+ FD_ZERO(&fds);
+ FD_SET(run.from, &fds);
+ FD_SET(run.error, &fds);
+
+ /* Wait for data to appear on the substituter's stdout or
+ stderr. */
+ if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) {
+ if (errno == EINTR) continue;
+ throw SysError("waiting for input from the substituter");
+ }
+
+ /* Completely drain stderr before dealing with stdout. */
+ if (FD_ISSET(run.error, &fds)) {
+ char buf[4096];
+ ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf));
+ if (n == -1) {
+ if (errno == EINTR) continue;
+ throw SysError("reading from substituter's stderr");
+ }
+ if (n == 0) throw EndOfFile(format("substituter `%1%' died unexpectedly") % run.program);
+ err.append(buf, n);
+ string::size_type p;
+ while ((p = err.find('\n')) != string::npos) {
+ printMsg(lvlError, run.program + ": " + string(err, 0, p));
+ err = string(err, p + 1);
+ }
+ }
+
+ /* Read from stdout until we get a newline or the buffer is empty. */
+ else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) {
+ haveData:
+ do {
+ unsigned char c;
+ run.fromBuf(&c, 1);
+ if (c == '\n') {
+ if (!err.empty()) printMsg(lvlError, run.program + ": " + err);
+ return res;
+ }
+ res += c;
+ } while (run.fromBuf.hasData());
+ }
+ }
+}
+
+
+template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run)
+{
+ string s = getLineFromSubstituter(run);
+ T res;
+ if (!string2Int(s, res)) throw Error("integer expected from stream");
+ return res;
+}
+
+
+PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
+{
+ PathSet res;
+ foreach (Paths::iterator, i, settings.substituters) {
+ if (res.size() == paths.size()) break;
+ RunningSubstituter & run(runningSubstituters[*i]);
+ startSubstituter(*i, run);
+ if (run.disabled) continue;
+ string s = "have ";
+ foreach (PathSet::const_iterator, j, paths)
+ if (res.find(*j) == res.end()) { s += *j; s += " "; }
+ writeLine(run.to, s);
+ while (true) {
+ /* FIXME: we only read stderr when an error occurs, so
+ substituters should only write (short) messages to
+ stderr when they fail. I.e. they shouldn't write debug
+ output. */
+ Path path = getLineFromSubstituter(run);
+ if (path == "") break;
+ res.insert(path);
+ }
+ }
+ return res;
+}
+
+
+void LocalStore::querySubstitutablePathInfos(const Path & substituter,
+ PathSet & paths, SubstitutablePathInfos & infos)
+{
+ RunningSubstituter & run(runningSubstituters[substituter]);
+ startSubstituter(substituter, run);
+ if (run.disabled) return;
+
+ string s = "info ";
+ foreach (PathSet::const_iterator, i, paths)
+ if (infos.find(*i) == infos.end()) { s += *i; s += " "; }
+ writeLine(run.to, s);
+
+ while (true) {
+ Path path = getLineFromSubstituter(run);
+ if (path == "") break;
+ if (paths.find(path) == paths.end())
+ throw Error(format("got unexpected path `%1%' from substituter") % path);
+ paths.erase(path);
+ SubstitutablePathInfo & info(infos[path]);
+ info.deriver = getLineFromSubstituter(run);
+ if (info.deriver != "") assertStorePath(info.deriver);
+ int nrRefs = getIntLineFromSubstituter<int>(run);
+ while (nrRefs--) {
+ Path p = getLineFromSubstituter(run);
+ assertStorePath(p);
+ info.references.insert(p);
+ }
+ info.downloadSize = getIntLineFromSubstituter<long long>(run);
+ info.narSize = getIntLineFromSubstituter<long long>(run);
+ }
+}
+
+
+void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos)
+{
+ PathSet todo = paths;
+ foreach (Paths::iterator, i, settings.substituters) {
+ if (todo.empty()) break;
+ querySubstitutablePathInfos(*i, todo, infos);
+ }
+}
+
+
+Hash LocalStore::queryPathHash(const Path & path)
+{
+ return queryPathInfo(path).hash;
+}
+
+
+void LocalStore::registerValidPath(const ValidPathInfo & info)
+{
+ ValidPathInfos infos;
+ infos.push_back(info);
+ registerValidPaths(infos);
+}
+
+
+void LocalStore::registerValidPaths(const ValidPathInfos & infos)
+{
+ /* SQLite will fsync by default, but the new valid paths may not be fsync-ed.
+ * So some may want to fsync them before registering the validity, at the
+ * expense of some speed of the path registering operation. */
+ if (settings.syncBeforeRegistering) sync();
+
+ retry_sqlite {
+ SQLiteTxn txn(db);
+ PathSet paths;
+
+ foreach (ValidPathInfos::const_iterator, i, infos) {
+ assert(i->hash.type == htSHA256);
+ if (isValidPath_(i->path))
+ updatePathInfo(*i);
+ else
+ addValidPath(*i, false);
+ paths.insert(i->path);
+ }
+
+ foreach (ValidPathInfos::const_iterator, i, infos) {
+ unsigned long long referrer = queryValidPathId(i->path);
+ foreach (PathSet::iterator, j, i->references)
+ addReference(referrer, queryValidPathId(*j));
+ }
+
+ /* Check that the derivation outputs are correct. We can't do
+ this in addValidPath() above, because the references might
+ not be valid yet. */
+ foreach (ValidPathInfos::const_iterator, i, infos)
+ if (isDerivation(i->path)) {
+ // FIXME: inefficient; we already loaded the
+ // derivation in addValidPath().
+ Derivation drv = parseDerivation(readFile(i->path));
+ checkDerivationOutputs(i->path, drv);
+ }
+
+ /* Do a topological sort of the paths. This will throw an
+ error if a cycle is detected and roll back the
+ transaction. Cycles can only occur when a derivation
+ has multiple outputs. */
+ topoSortPaths(*this, paths);
+
+ txn.commit();
+ } end_retry_sqlite;
+}
+
+
+/* Invalidate a path. The caller is responsible for checking that
+ there are no referrers. */
+void LocalStore::invalidatePath(const Path & path)
+{
+ debug(format("invalidating path `%1%'") % path);
+
+ drvHashes.erase(path);
+
+ SQLiteStmtUse use(stmtInvalidatePath);
+
+ stmtInvalidatePath.bind(path);
+
+ if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE)
+ throwSQLiteError(db, format("invalidating path `%1%' in database") % path);
+
+ /* Note that the foreign key constraints on the Refs table take
+ care of deleting the references entries for `path'. */
+}
+
+
+Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
+ bool recursive, HashType hashAlgo, bool repair)
+{
+ Hash h = hashString(hashAlgo, dump);
+
+ Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name);
+
+ addTempRoot(dstPath);
+
+ if (repair || !isValidPath(dstPath)) {
+
+ /* The first check above is an optimisation to prevent
+ unnecessary lock acquisition. */
+
+ PathLocks outputLock(singleton<PathSet, Path>(dstPath));
+
+ if (repair || !isValidPath(dstPath)) {
+
+ if (pathExists(dstPath)) deletePath(dstPath);
+
+ if (recursive) {
+ StringSource source(dump);
+ restorePath(dstPath, source);
+ } else
+ writeFile(dstPath, dump);
+
+ canonicalisePathMetaData(dstPath, -1);
+
+ /* Register the SHA-256 hash of the NAR serialisation of
+ the path in the database. We may just have computed it
+ above (if called with recursive == true and hashAlgo ==
+ sha256); otherwise, compute it here. */
+ HashResult hash;
+ if (recursive) {
+ hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
+ hash.second = dump.size();
+ } else
+ hash = hashPath(htSHA256, dstPath);
+
+ optimisePath(dstPath); // FIXME: combine with hashPath()
+
+ ValidPathInfo info;
+ info.path = dstPath;
+ info.hash = hash.first;
+ info.narSize = hash.second;
+ registerValidPath(info);
+ }
+
+ outputLock.setDeletion(true);
+ }
+
+ return dstPath;
+}
+
+
+Path LocalStore::addToStore(const Path & _srcPath,
+ bool recursive, HashType hashAlgo, PathFilter & filter, bool repair)
+{
+ Path srcPath(absPath(_srcPath));
+ debug(format("adding `%1%' to the store") % srcPath);
+
+ /* Read the whole path into memory. This is not a very scalable
+ method for very large paths, but `copyPath' is mainly used for
+ small files. */
+ StringSink sink;
+ if (recursive)
+ dumpPath(srcPath, sink, filter);
+ else
+ sink.s = readFile(srcPath);
+
+ return addToStoreFromDump(sink.s, baseNameOf(srcPath), recursive, hashAlgo, repair);
+}
+
+
+Path LocalStore::addTextToStore(const string & name, const string & s,
+ const PathSet & references, bool repair)
+{
+ Path dstPath = computeStorePathForText(name, s, references);
+
+ addTempRoot(dstPath);
+
+ if (repair || !isValidPath(dstPath)) {
+
+ PathLocks outputLock(singleton<PathSet, Path>(dstPath));
+
+ if (repair || !isValidPath(dstPath)) {
+
+ if (pathExists(dstPath)) deletePath(dstPath);
+
+ writeFile(dstPath, s);
+
+ canonicalisePathMetaData(dstPath, -1);
+
+ HashResult hash = hashPath(htSHA256, dstPath);
+
+ optimisePath(dstPath);
+
+ ValidPathInfo info;
+ info.path = dstPath;
+ info.hash = hash.first;
+ info.narSize = hash.second;
+ info.references = references;
+ registerValidPath(info);
+ }
+
+ outputLock.setDeletion(true);
+ }
+
+ return dstPath;
+}
+
+
+struct HashAndWriteSink : Sink
+{
+ Sink & writeSink;
+ HashSink hashSink;
+ HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
+ {
+ }
+ virtual void operator () (const unsigned char * data, size_t len)
+ {
+ writeSink(data, len);
+ hashSink(data, len);
+ }
+ Hash currentHash()
+ {
+ return hashSink.currentHash().first;
+ }
+};
+
+
+#define EXPORT_MAGIC 0x4558494e
+
+
+static void checkSecrecy(const Path & path)
+{
+ struct stat st;
+ if (stat(path.c_str(), &st))
+ throw SysError(format("getting status of `%1%'") % path);
+ if ((st.st_mode & (S_IRWXG | S_IRWXO)) != 0)
+ throw Error(format("file `%1%' should be secret (inaccessible to everybody else)!") % path);
+}
+
+
+void LocalStore::exportPath(const Path & path, bool sign,
+ Sink & sink)
+{
+ assertStorePath(path);
+
+ addTempRoot(path);
+ if (!isValidPath(path))
+ throw Error(format("path `%1%' is not valid") % path);
+
+ HashAndWriteSink hashAndWriteSink(sink);
+
+ dumpPath(path, hashAndWriteSink);
+
+ /* Refuse to export paths that have changed. This prevents
+ filesystem corruption from spreading to other machines.
+ Don't complain if the stored hash is zero (unknown). */
+ Hash hash = hashAndWriteSink.currentHash();
+ Hash storedHash = queryPathHash(path);
+ if (hash != storedHash && storedHash != Hash(storedHash.type))
+ throw Error(format("hash of path `%1%' has changed from `%2%' to `%3%'!") % path
+ % printHash(storedHash) % printHash(hash));
+
+ writeInt(EXPORT_MAGIC, hashAndWriteSink);
+
+ writeString(path, hashAndWriteSink);
+
+ PathSet references;
+ queryReferences(path, references);
+ writeStrings(references, hashAndWriteSink);
+
+ Path deriver = queryDeriver(path);
+ writeString(deriver, hashAndWriteSink);
+
+ if (sign) {
+ Hash hash = hashAndWriteSink.currentHash();
+
+ writeInt(1, hashAndWriteSink);
+
+ Path tmpDir = createTempDir();
+ AutoDelete delTmp(tmpDir);
+ Path hashFile = tmpDir + "/hash";
+ writeFile(hashFile, printHash(hash));
+
+ Path secretKey = settings.nixConfDir + "/signing-key.sec";
+ checkSecrecy(secretKey);
+
+ Strings args;
+ args.push_back("rsautl");
+ args.push_back("-sign");
+ args.push_back("-inkey");
+ args.push_back(secretKey);
+ args.push_back("-in");
+ args.push_back(hashFile);
+ string signature = runProgram(OPENSSL_PATH, true, args);
+
+ writeString(signature, hashAndWriteSink);
+
+ } else
+ writeInt(0, hashAndWriteSink);
+}
+
+
+struct HashAndReadSource : Source
+{
+ Source & readSource;
+ HashSink hashSink;
+ bool hashing;
+ HashAndReadSource(Source & readSource) : readSource(readSource), hashSink(htSHA256)
+ {
+ hashing = true;
+ }
+ size_t read(unsigned char * data, size_t len)
+ {
+ size_t n = readSource.read(data, len);
+ if (hashing) hashSink(data, n);
+ return n;
+ }
+};
+
+
+/* Create a temporary directory in the store that won't be
+ garbage-collected. */
+Path LocalStore::createTempDirInStore()
+{
+ Path tmpDir;
+ do {
+ /* There is a slight possibility that `tmpDir' gets deleted by
+ the GC between createTempDir() and addTempRoot(), so repeat
+ until `tmpDir' exists. */
+ tmpDir = createTempDir(settings.nixStore);
+ addTempRoot(tmpDir);
+ } while (!pathExists(tmpDir));
+ return tmpDir;
+}
+
+
+Path LocalStore::importPath(bool requireSignature, Source & source)
+{
+ HashAndReadSource hashAndReadSource(source);
+
+ /* We don't yet know what store path this archive contains (the
+ store path follows the archive data proper), and besides, we
+ don't know yet whether the signature is valid. */
+ Path tmpDir = createTempDirInStore();
+ AutoDelete delTmp(tmpDir);
+ Path unpacked = tmpDir + "/unpacked";
+
+ restorePath(unpacked, hashAndReadSource);
+
+ unsigned int magic = readInt(hashAndReadSource);
+ if (magic != EXPORT_MAGIC)
+ throw Error("Nix archive cannot be imported; wrong format");
+
+ Path dstPath = readStorePath(hashAndReadSource);
+
+ printMsg(lvlInfo, format("importing path `%1%'") % dstPath);
+
+ PathSet references = readStorePaths<PathSet>(hashAndReadSource);
+
+ Path deriver = readString(hashAndReadSource);
+ if (deriver != "") assertStorePath(deriver);
+
+ Hash hash = hashAndReadSource.hashSink.finish().first;
+ hashAndReadSource.hashing = false;
+
+ bool haveSignature = readInt(hashAndReadSource) == 1;
+
+ if (requireSignature && !haveSignature)
+ throw Error(format("imported archive of `%1%' lacks a signature") % dstPath);
+
+ if (haveSignature) {
+ string signature = readString(hashAndReadSource);
+
+ if (requireSignature) {
+ Path sigFile = tmpDir + "/sig";
+ writeFile(sigFile, signature);
+
+ Strings args;
+ args.push_back("rsautl");
+ args.push_back("-verify");
+ args.push_back("-inkey");
+ args.push_back(settings.nixConfDir + "/signing-key.pub");
+ args.push_back("-pubin");
+ args.push_back("-in");
+ args.push_back(sigFile);
+ string hash2 = runProgram(OPENSSL_PATH, true, args);
+
+ /* Note: runProgram() throws an exception if the signature
+ is invalid. */
+
+ if (printHash(hash) != hash2)
+ throw Error(
+ "signed hash doesn't match actual contents of imported "
+ "archive; archive could be corrupt, or someone is trying "
+ "to import a Trojan horse");
+ }
+ }
+
+ /* Do the actual import. */
+
+ /* !!! way too much code duplication with addTextToStore() etc. */
+ addTempRoot(dstPath);
+
+ if (!isValidPath(dstPath)) {
+
+ PathLocks outputLock;
+
+ /* Lock the output path. But don't lock if we're being called
+ from a build hook (whose parent process already acquired a
+ lock on this path). */
+ Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS"));
+ if (find(locksHeld.begin(), locksHeld.end(), dstPath) == locksHeld.end())
+ outputLock.lockPaths(singleton<PathSet, Path>(dstPath));
+
+ if (!isValidPath(dstPath)) {
+
+ if (pathExists(dstPath)) deletePath(dstPath);
+
+ if (rename(unpacked.c_str(), dstPath.c_str()) == -1)
+ throw SysError(format("cannot move `%1%' to `%2%'")
+ % unpacked % dstPath);
+
+ canonicalisePathMetaData(dstPath, -1);
+
+ /* !!! if we were clever, we could prevent the hashPath()
+ here. */
+ HashResult hash = hashPath(htSHA256, dstPath);
+
+ optimisePath(dstPath); // FIXME: combine with hashPath()
+
+ ValidPathInfo info;
+ info.path = dstPath;
+ info.hash = hash.first;
+ info.narSize = hash.second;
+ info.references = references;
+ info.deriver = deriver != "" && isValidPath(deriver) ? deriver : "";
+ registerValidPath(info);
+ }
+
+ outputLock.setDeletion(true);
+ }
+
+ return dstPath;
+}
+
+
+Paths LocalStore::importPaths(bool requireSignature, Source & source)
+{
+ Paths res;
+ while (true) {
+ unsigned long long n = readLongLong(source);
+ if (n == 0) break;
+ if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'");
+ res.push_back(importPath(requireSignature, source));
+ }
+ return res;
+}
+
+
+void LocalStore::invalidatePathChecked(const Path & path)
+{
+ assertStorePath(path);
+
+ retry_sqlite {
+ SQLiteTxn txn(db);
+
+ if (isValidPath_(path)) {
+ PathSet referrers; queryReferrers_(path, referrers);
+ referrers.erase(path); /* ignore self-references */
+ if (!referrers.empty())
+ throw PathInUse(format("cannot delete path `%1%' because it is in use by %2%")
+ % path % showPaths(referrers));
+ invalidatePath(path);
+ }
+
+ txn.commit();
+ } end_retry_sqlite;
+}
+
+
+bool LocalStore::verifyStore(bool checkContents, bool repair)
+{
+ printMsg(lvlError, format("reading the Nix store..."));
+
+ bool errors = false;
+
+ /* Acquire the global GC lock to prevent a garbage collection. */
+ AutoCloseFD fdGCLock = openGCLock(ltWrite);
+
+ Paths entries = readDirectory(settings.nixStore);
+ PathSet store(entries.begin(), entries.end());
+
+ /* Check whether all valid paths actually exist. */
+ printMsg(lvlInfo, "checking path existence...");
+
+ PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
+
+ foreach (PathSet::iterator, i, validPaths2)
+ verifyPath(*i, store, done, validPaths, repair, errors);
+
+ /* Release the GC lock so that checking content hashes (which can
+ take ages) doesn't block the GC or builds. */
+ fdGCLock.close();
+
+ /* Optionally, check the content hashes (slow). */
+ if (checkContents) {
+ printMsg(lvlInfo, "checking hashes...");
+
+ Hash nullHash(htSHA256);
+
+ foreach (PathSet::iterator, i, validPaths) {
+ try {
+ ValidPathInfo info = queryPathInfo(*i);
+
+ /* Check the content hash (optionally - slow). */
+ printMsg(lvlTalkative, format("checking contents of `%1%'") % *i);
+ HashResult current = hashPath(info.hash.type, *i);
+
+ if (info.hash != nullHash && info.hash != current.first) {
+ printMsg(lvlError, format("path `%1%' was modified! "
+ "expected hash `%2%', got `%3%'")
+ % *i % printHash(info.hash) % printHash(current.first));
+ if (repair) repairPath(*i); else errors = true;
+ } else {
+
+ bool update = false;
+
+ /* Fill in missing hashes. */
+ if (info.hash == nullHash) {
+ printMsg(lvlError, format("fixing missing hash on `%1%'") % *i);
+ info.hash = current.first;
+ update = true;
+ }
+
+ /* Fill in missing narSize fields (from old stores). */
+ if (info.narSize == 0) {
+ printMsg(lvlError, format("updating size field on `%1%' to %2%") % *i % current.second);
+ info.narSize = current.second;
+ update = true;
+ }
+
+ if (update) updatePathInfo(info);
+
+ }
+
+ } catch (Error & e) {
+ /* It's possible that the path got GC'ed, so ignore
+ errors on invalid paths. */
+ if (isValidPath(*i))
+ printMsg(lvlError, format("error: %1%") % e.msg());
+ else
+ printMsg(lvlError, format("warning: %1%") % e.msg());
+ errors = true;
+ }
+ }
+ }
+
+ return errors;
+}
+
+
+void LocalStore::verifyPath(const Path & path, const PathSet & store,
+ PathSet & done, PathSet & validPaths, bool repair, bool & errors)
+{
+ checkInterrupt();
+
+ if (done.find(path) != done.end()) return;
+ done.insert(path);
+
+ if (!isStorePath(path)) {
+ printMsg(lvlError, format("path `%1%' is not in the Nix store") % path);
+ invalidatePath(path);
+ return;
+ }
+
+ if (store.find(baseNameOf(path)) == store.end()) {
+ /* Check any referrers first. If we can invalidate them
+ first, then we can invalidate this path as well. */
+ bool canInvalidate = true;
+ PathSet referrers; queryReferrers(path, referrers);
+ foreach (PathSet::iterator, i, referrers)
+ if (*i != path) {
+ verifyPath(*i, store, done, validPaths, repair, errors);
+ if (validPaths.find(*i) != validPaths.end())
+ canInvalidate = false;
+ }
+
+ if (canInvalidate) {
+ printMsg(lvlError, format("path `%1%' disappeared, removing from database...") % path);
+ invalidatePath(path);
+ } else {
+ printMsg(lvlError, format("path `%1%' disappeared, but it still has valid referrers!") % path);
+ if (repair)
+ try {
+ repairPath(path);
+ } catch (Error & e) {
+ printMsg(lvlError, format("warning: %1%") % e.msg());
+ errors = true;
+ }
+ else errors = true;
+ }
+
+ return;
+ }
+
+ validPaths.insert(path);
+}
+
+
+bool LocalStore::pathContentsGood(const Path & path)
+{
+ std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path);
+ if (i != pathContentsGoodCache.end()) return i->second;
+ printMsg(lvlInfo, format("checking path `%1%'...") % path);
+ ValidPathInfo info = queryPathInfo(path);
+ bool res;
+ if (!pathExists(path))
+ res = false;
+ else {
+ HashResult current = hashPath(info.hash.type, path);
+ Hash nullHash(htSHA256);
+ res = info.hash == nullHash || info.hash == current.first;
+ }
+ pathContentsGoodCache[path] = res;
+ if (!res) printMsg(lvlError, format("path `%1%' is corrupted or missing!") % path);
+ return res;
+}
+
+
+void LocalStore::markContentsGood(const Path & path)
+{
+ pathContentsGoodCache[path] = true;
+}
+
+
+/* Functions for upgrading from the pre-SQLite database. */
+
+PathSet LocalStore::queryValidPathsOld()
+{
+ PathSet paths;
+ Strings entries = readDirectory(settings.nixDBPath + "/info");
+ foreach (Strings::iterator, i, entries)
+ if (i->at(0) != '.') paths.insert(settings.nixStore + "/" + *i);
+ return paths;
+}
+
+
+ValidPathInfo LocalStore::queryPathInfoOld(const Path & path)
+{
+ ValidPathInfo res;
+ res.path = path;
+
+ /* Read the info file. */
+ string baseName = baseNameOf(path);
+ Path infoFile = (format("%1%/info/%2%") % settings.nixDBPath % baseName).str();
+ if (!pathExists(infoFile))
+ throw Error(format("path `%1%' is not valid") % path);
+ string info = readFile(infoFile);
+
+ /* Parse it. */
+ Strings lines = tokenizeString<Strings>(info, "\n");
+
+ foreach (Strings::iterator, i, lines) {
+ string::size_type p = i->find(':');
+ if (p == string::npos)
+ throw Error(format("corrupt line in `%1%': %2%") % infoFile % *i);
+ string name(*i, 0, p);
+ string value(*i, p + 2);
+ if (name == "References") {
+ Strings refs = tokenizeString<Strings>(value, " ");
+ res.references = PathSet(refs.begin(), refs.end());
+ } else if (name == "Deriver") {
+ res.deriver = value;
+ } else if (name == "Hash") {
+ res.hash = parseHashField(path, value);
+ } else if (name == "Registered-At") {
+ int n = 0;
+ string2Int(value, n);
+ res.registrationTime = n;
+ }
+ }
+
+ return res;
+}
+
+
+/* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */
+void LocalStore::upgradeStore6()
+{
+ printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
+
+ openDB(true);
+
+ PathSet validPaths = queryValidPathsOld();
+
+ SQLiteTxn txn(db);
+
+ foreach (PathSet::iterator, i, validPaths) {
+ addValidPath(queryPathInfoOld(*i), false);
+ std::cerr << ".";
+ }
+
+ std::cerr << "|";
+
+ foreach (PathSet::iterator, i, validPaths) {
+ ValidPathInfo info = queryPathInfoOld(*i);
+ unsigned long long referrer = queryValidPathId(*i);
+ foreach (PathSet::iterator, j, info.references)
+ addReference(referrer, queryValidPathId(*j));
+ std::cerr << ".";
+ }
+
+ std::cerr << "\n";
+
+ txn.commit();
+}
+
+
+#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL)
+
+static void makeMutable(const Path & path)
+{
+ checkInterrupt();
+
+ struct stat st = lstat(path);
+
+ if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return;
+
+ if (S_ISDIR(st.st_mode)) {
+ Strings names = readDirectory(path);
+ foreach (Strings::iterator, i, names)
+ makeMutable(path + "/" + *i);
+ }
+
+ /* The O_NOFOLLOW is important to prevent us from changing the
+ mutable bit on the target of a symlink (which would be a
+ security hole). */
+ AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW);
+ if (fd == -1) {
+ if (errno == ELOOP) return; // it's a symlink
+ throw SysError(format("opening file `%1%'") % path);
+ }
+
+ unsigned int flags = 0, old;
+
+ /* Silently ignore errors getting/setting the immutable flag so
+ that we work correctly on filesystems that don't support it. */
+ if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return;
+ old = flags;
+ flags &= ~FS_IMMUTABLE_FL;
+ if (old == flags) return;
+ if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return;
+}
+
+/* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */
+void LocalStore::upgradeStore7()
+{
+ if (getuid() != 0) return;
+ printMsg(lvlError, "removing immutable bits from the Nix store (this may take a while)...");
+ makeMutable(settings.nixStore);
+}
+
+#else
+
+void LocalStore::upgradeStore7()
+{
+}
+
+#endif
+
+
+void LocalStore::vacuumDB()
+{
+ if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK)
+ throwSQLiteError(db, "vacuuming SQLite database");
+}
+
+
+}
diff --git a/nix/libstore/local-store.hh b/nix/libstore/local-store.hh
new file mode 100644
index 0000000000..09639e74cf
--- /dev/null
+++ b/nix/libstore/local-store.hh
@@ -0,0 +1,333 @@
+#pragma once
+
+#include <string>
+
+#include "store-api.hh"
+#include "util.hh"
+#include "pathlocks.hh"
+
+
+class sqlite3;
+class sqlite3_stmt;
+
+
+namespace nix {
+
+
+/* Nix store and database schema version. Version 1 (or 0) was Nix <=
+ 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10.
+ Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is
+ Nix 1.0. Version 7 is Nix 1.3. */
+const int nixSchemaVersion = 7;
+
+
+extern string drvsLogDir;
+
+
+struct Derivation;
+
+
+struct OptimiseStats
+{
+ unsigned long totalFiles;
+ unsigned long sameContents;
+ unsigned long filesLinked;
+ unsigned long long bytesFreed;
+ unsigned long long blocksFreed;
+ OptimiseStats()
+ {
+ totalFiles = sameContents = filesLinked = 0;
+ bytesFreed = blocksFreed = 0;
+ }
+};
+
+
+struct RunningSubstituter
+{
+ Path program;
+ Pid pid;
+ AutoCloseFD to, from, error;
+ FdSource fromBuf;
+ bool disabled;
+ RunningSubstituter() : disabled(false) { };
+};
+
+
+/* Wrapper object to close the SQLite database automatically. */
+struct SQLite
+{
+ sqlite3 * db;
+ SQLite() { db = 0; }
+ ~SQLite();
+ operator sqlite3 * () { return db; }
+};
+
+
+/* Wrapper object to create and destroy SQLite prepared statements. */
+struct SQLiteStmt
+{
+ sqlite3 * db;
+ sqlite3_stmt * stmt;
+ unsigned int curArg;
+ SQLiteStmt() { stmt = 0; }
+ void create(sqlite3 * db, const string & s);
+ void reset();
+ ~SQLiteStmt();
+ operator sqlite3_stmt * () { return stmt; }
+ void bind(const string & value);
+ void bind(int value);
+ void bind64(long long value);
+ void bind();
+};
+
+
+class LocalStore : public StoreAPI
+{
+private:
+ typedef std::map<Path, RunningSubstituter> RunningSubstituters;
+ RunningSubstituters runningSubstituters;
+
+ Path linksDir;
+
+public:
+
+ /* Initialise the local store, upgrading the schema if
+ necessary. */
+ LocalStore(bool reserveSpace = true);
+
+ ~LocalStore();
+
+ /* Implementations of abstract store API methods. */
+
+ bool isValidPath(const Path & path);
+
+ PathSet queryValidPaths(const PathSet & paths);
+
+ PathSet queryAllValidPaths();
+
+ ValidPathInfo queryPathInfo(const Path & path);
+
+ Hash queryPathHash(const Path & path);
+
+ void queryReferences(const Path & path, PathSet & references);
+
+ void queryReferrers(const Path & path, PathSet & referrers);
+
+ Path queryDeriver(const Path & path);
+
+ PathSet queryValidDerivers(const Path & path);
+
+ PathSet queryDerivationOutputs(const Path & path);
+
+ StringSet queryDerivationOutputNames(const Path & path);
+
+ Path queryPathFromHashPart(const string & hashPart);
+
+ PathSet querySubstitutablePaths(const PathSet & paths);
+
+ void querySubstitutablePathInfos(const Path & substituter,
+ PathSet & paths, SubstitutablePathInfos & infos);
+
+ void querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos);
+
+ Path addToStore(const Path & srcPath,
+ bool recursive = true, HashType hashAlgo = htSHA256,
+ PathFilter & filter = defaultPathFilter, bool repair = false);
+
+ /* Like addToStore(), but the contents of the path are contained
+ in `dump', which is either a NAR serialisation (if recursive ==
+ true) or simply the contents of a regular file (if recursive ==
+ false). */
+ Path addToStoreFromDump(const string & dump, const string & name,
+ bool recursive = true, HashType hashAlgo = htSHA256, bool repair = false);
+
+ Path addTextToStore(const string & name, const string & s,
+ const PathSet & references, bool repair = false);
+
+ void exportPath(const Path & path, bool sign,
+ Sink & sink);
+
+ Paths importPaths(bool requireSignature, Source & source);
+
+ void buildPaths(const PathSet & paths, BuildMode buildMode);
+
+ void ensurePath(const Path & path);
+
+ void addTempRoot(const Path & path);
+
+ void addIndirectRoot(const Path & path);
+
+ void syncWithGC();
+
+ Roots findRoots();
+
+ void collectGarbage(const GCOptions & options, GCResults & results);
+
+ /* Optimise the disk space usage of the Nix store by hard-linking
+ files with the same contents. */
+ void optimiseStore(OptimiseStats & stats);
+
+ /* Optimise a single store path. */
+ void optimisePath(const Path & path);
+
+ /* Check the integrity of the Nix store. Returns true if errors
+ remain. */
+ bool verifyStore(bool checkContents, bool repair);
+
+ /* Register the validity of a path, i.e., that `path' exists, that
+ the paths referenced by it exists, and in the case of an output
+ path of a derivation, that it has been produced by a successful
+ execution of the derivation (or something equivalent). Also
+ register the hash of the file system contents of the path. The
+ hash must be a SHA-256 hash. */
+ void registerValidPath(const ValidPathInfo & info);
+
+ void registerValidPaths(const ValidPathInfos & infos);
+
+ /* Register that the build of a derivation with output `path' has
+ failed. */
+ void registerFailedPath(const Path & path);
+
+ /* Query whether `path' previously failed to build. */
+ bool hasPathFailed(const Path & path);
+
+ PathSet queryFailedPaths();
+
+ void clearFailedPaths(const PathSet & paths);
+
+ void vacuumDB();
+
+ /* Repair the contents of the given path by redownloading it using
+ a substituter (if available). */
+ void repairPath(const Path & path);
+
+ /* Check whether the given valid path exists and has the right
+ contents. */
+ bool pathContentsGood(const Path & path);
+
+ void markContentsGood(const Path & path);
+
+ void setSubstituterEnv();
+
+private:
+
+ Path schemaPath;
+
+ /* Lock file used for upgrading. */
+ AutoCloseFD globalLock;
+
+ /* The SQLite database object. */
+ SQLite db;
+
+ /* Some precompiled SQLite statements. */
+ SQLiteStmt stmtRegisterValidPath;
+ SQLiteStmt stmtUpdatePathInfo;
+ SQLiteStmt stmtAddReference;
+ SQLiteStmt stmtQueryPathInfo;
+ SQLiteStmt stmtQueryReferences;
+ SQLiteStmt stmtQueryReferrers;
+ SQLiteStmt stmtInvalidatePath;
+ SQLiteStmt stmtRegisterFailedPath;
+ SQLiteStmt stmtHasPathFailed;
+ SQLiteStmt stmtQueryFailedPaths;
+ SQLiteStmt stmtClearFailedPath;
+ SQLiteStmt stmtAddDerivationOutput;
+ SQLiteStmt stmtQueryValidDerivers;
+ SQLiteStmt stmtQueryDerivationOutputs;
+ SQLiteStmt stmtQueryPathFromHashPart;
+
+ /* Cache for pathContentsGood(). */
+ std::map<Path, bool> pathContentsGoodCache;
+
+ bool didSetSubstituterEnv;
+
+ int getSchema();
+
+ void openDB(bool create);
+
+ void makeStoreWritable();
+
+ unsigned long long queryValidPathId(const Path & path);
+
+ unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true);
+
+ void addReference(unsigned long long referrer, unsigned long long reference);
+
+ void appendReferrer(const Path & from, const Path & to, bool lock);
+
+ void rewriteReferrers(const Path & path, bool purge, PathSet referrers);
+
+ void invalidatePath(const Path & path);
+
+ /* Delete a path from the Nix store. */
+ void invalidatePathChecked(const Path & path);
+
+ void verifyPath(const Path & path, const PathSet & store,
+ PathSet & done, PathSet & validPaths, bool repair, bool & errors);
+
+ void updatePathInfo(const ValidPathInfo & info);
+
+ void upgradeStore6();
+ void upgradeStore7();
+ PathSet queryValidPathsOld();
+ ValidPathInfo queryPathInfoOld(const Path & path);
+
+ struct GCState;
+
+ void deleteGarbage(GCState & state, const Path & path);
+
+ void tryToDelete(GCState & state, const Path & path);
+
+ bool canReachRoot(GCState & state, PathSet & visited, const Path & path);
+
+ void deletePathRecursive(GCState & state, const Path & path);
+
+ bool isActiveTempFile(const GCState & state,
+ const Path & path, const string & suffix);
+
+ int openGCLock(LockType lockType);
+
+ void removeUnusedLinks(const GCState & state);
+
+ void startSubstituter(const Path & substituter,
+ RunningSubstituter & runningSubstituter);
+
+ string getLineFromSubstituter(RunningSubstituter & run);
+
+ template<class T> T getIntLineFromSubstituter(RunningSubstituter & run);
+
+ Path createTempDirInStore();
+
+ Path importPath(bool requireSignature, Source & source);
+
+ void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
+
+ void optimisePath_(OptimiseStats & stats, const Path & path);
+
+ // Internal versions that are not wrapped in retry_sqlite.
+ bool isValidPath_(const Path & path);
+ void queryReferrers_(const Path & path, PathSet & referrers);
+};
+
+
+typedef std::pair<dev_t, ino_t> Inode;
+typedef set<Inode> InodesSeen;
+
+
+/* "Fix", or canonicalise, the meta-data of the files in a store path
+ after it has been built. In particular:
+ - the last modification date on each file is set to 1 (i.e.,
+ 00:00:01 1/1/1970 UTC)
+ - the permissions are set of 444 or 555 (i.e., read-only with or
+ without execute permission; setuid bits etc. are cleared)
+ - the owner and group are set to the Nix user and group, if we're
+ running as root. */
+void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen);
+void canonicalisePathMetaData(const Path & path, uid_t fromUid);
+
+void canonicaliseTimestampAndPermissions(const Path & path);
+
+MakeError(PathInUse, Error);
+
+}
diff --git a/nix/libstore/misc.cc b/nix/libstore/misc.cc
new file mode 100644
index 0000000000..1bf3f93782
--- /dev/null
+++ b/nix/libstore/misc.cc
@@ -0,0 +1,220 @@
+#include "misc.hh"
+#include "store-api.hh"
+#include "local-store.hh"
+#include "globals.hh"
+
+
+namespace nix {
+
+
+Derivation derivationFromPath(StoreAPI & store, const Path & drvPath)
+{
+ assertStorePath(drvPath);
+ store.ensurePath(drvPath);
+ return parseDerivation(readFile(drvPath));
+}
+
+
+void computeFSClosure(StoreAPI & store, const Path & path,
+ PathSet & paths, bool flipDirection, bool includeOutputs, bool includeDerivers)
+{
+ if (paths.find(path) != paths.end()) return;
+ paths.insert(path);
+
+ PathSet edges;
+
+ if (flipDirection) {
+ store.queryReferrers(path, edges);
+
+ if (includeOutputs) {
+ PathSet derivers = store.queryValidDerivers(path);
+ foreach (PathSet::iterator, i, derivers)
+ edges.insert(*i);
+ }
+
+ if (includeDerivers && isDerivation(path)) {
+ PathSet outputs = store.queryDerivationOutputs(path);
+ foreach (PathSet::iterator, i, outputs)
+ if (store.isValidPath(*i) && store.queryDeriver(*i) == path)
+ edges.insert(*i);
+ }
+
+ } else {
+ store.queryReferences(path, edges);
+
+ if (includeOutputs && isDerivation(path)) {
+ PathSet outputs = store.queryDerivationOutputs(path);
+ foreach (PathSet::iterator, i, outputs)
+ if (store.isValidPath(*i)) edges.insert(*i);
+ }
+
+ if (includeDerivers) {
+ Path deriver = store.queryDeriver(path);
+ if (store.isValidPath(deriver)) edges.insert(deriver);
+ }
+ }
+
+ foreach (PathSet::iterator, i, edges)
+ computeFSClosure(store, *i, paths, flipDirection, includeOutputs, includeDerivers);
+}
+
+
+Path findOutput(const Derivation & drv, string id)
+{
+ foreach (DerivationOutputs::const_iterator, i, drv.outputs)
+ if (i->first == id) return i->second.path;
+ throw Error(format("derivation has no output `%1%'") % id);
+}
+
+
+void queryMissing(StoreAPI & store, const PathSet & targets,
+ PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+ unsigned long long & downloadSize, unsigned long long & narSize)
+{
+ downloadSize = narSize = 0;
+
+ PathSet todo(targets.begin(), targets.end()), done;
+
+ /* Getting substitute info has high latency when using the binary
+ cache substituter. Thus it's essential to do substitute
+ queries in parallel as much as possible. To accomplish this
+ we do the following:
+
+ - For all paths still to be processed (‘todo’), we add all
+ paths for which we need info to the set ‘query’. For an
+ unbuilt derivation this is the output paths; otherwise, it's
+ the path itself.
+
+ - We get info about all paths in ‘query’ in parallel.
+
+ - We process the results and add new items to ‘todo’ if
+ necessary. E.g. if a path is substitutable, then we need to
+ get info on its references.
+
+ - Repeat until ‘todo’ is empty.
+ */
+
+ while (!todo.empty()) {
+
+ PathSet query, todoDrv, todoNonDrv;
+
+ foreach (PathSet::iterator, i, todo) {
+ if (done.find(*i) != done.end()) continue;
+ done.insert(*i);
+
+ DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i);
+
+ if (isDerivation(i2.first)) {
+ if (!store.isValidPath(i2.first)) {
+ // FIXME: we could try to substitute p.
+ unknown.insert(*i);
+ continue;
+ }
+ Derivation drv = derivationFromPath(store, i2.first);
+
+ PathSet invalid;
+ foreach (DerivationOutputs::iterator, j, drv.outputs)
+ if (wantOutput(j->first, i2.second)
+ && !store.isValidPath(j->second.path))
+ invalid.insert(j->second.path);
+ if (invalid.empty()) continue;
+
+ todoDrv.insert(*i);
+ if (settings.useSubstitutes && !willBuildLocally(drv))
+ query.insert(invalid.begin(), invalid.end());
+ }
+
+ else {
+ if (store.isValidPath(*i)) continue;
+ query.insert(*i);
+ todoNonDrv.insert(*i);
+ }
+ }
+
+ todo.clear();
+
+ SubstitutablePathInfos infos;
+ store.querySubstitutablePathInfos(query, infos);
+
+ foreach (PathSet::iterator, i, todoDrv) {
+ DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i);
+
+ // FIXME: cache this
+ Derivation drv = derivationFromPath(store, i2.first);
+
+ PathSet outputs;
+ bool mustBuild = false;
+ if (settings.useSubstitutes && !willBuildLocally(drv)) {
+ foreach (DerivationOutputs::iterator, j, drv.outputs) {
+ if (!wantOutput(j->first, i2.second)) continue;
+ if (!store.isValidPath(j->second.path)) {
+ if (infos.find(j->second.path) == infos.end())
+ mustBuild = true;
+ else
+ outputs.insert(j->second.path);
+ }
+ }
+ } else
+ mustBuild = true;
+
+ if (mustBuild) {
+ willBuild.insert(i2.first);
+ todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
+ foreach (DerivationInputs::iterator, j, drv.inputDrvs)
+ todo.insert(makeDrvPathWithOutputs(j->first, j->second));
+ } else
+ todoNonDrv.insert(outputs.begin(), outputs.end());
+ }
+
+ foreach (PathSet::iterator, i, todoNonDrv) {
+ done.insert(*i);
+ SubstitutablePathInfos::iterator info = infos.find(*i);
+ if (info != infos.end()) {
+ willSubstitute.insert(*i);
+ downloadSize += info->second.downloadSize;
+ narSize += info->second.narSize;
+ todo.insert(info->second.references.begin(), info->second.references.end());
+ } else
+ unknown.insert(*i);
+ }
+ }
+}
+
+
+static void dfsVisit(StoreAPI & store, const PathSet & paths,
+ const Path & path, PathSet & visited, Paths & sorted,
+ PathSet & parents)
+{
+ if (parents.find(path) != parents.end())
+ throw BuildError(format("cycle detected in the references of `%1%'") % path);
+
+ if (visited.find(path) != visited.end()) return;
+ visited.insert(path);
+ parents.insert(path);
+
+ PathSet references;
+ if (store.isValidPath(path))
+ store.queryReferences(path, references);
+
+ foreach (PathSet::iterator, i, references)
+ /* Don't traverse into paths that don't exist. That can
+ happen due to substitutes for non-existent paths. */
+ if (*i != path && paths.find(*i) != paths.end())
+ dfsVisit(store, paths, *i, visited, sorted, parents);
+
+ sorted.push_front(path);
+ parents.erase(path);
+}
+
+
+Paths topoSortPaths(StoreAPI & store, const PathSet & paths)
+{
+ Paths sorted;
+ PathSet visited, parents;
+ foreach (PathSet::const_iterator, i, paths)
+ dfsVisit(store, paths, *i, visited, sorted, parents);
+ return sorted;
+}
+
+
+}
diff --git a/nix/libstore/misc.hh b/nix/libstore/misc.hh
new file mode 100644
index 0000000000..144cb7f457
--- /dev/null
+++ b/nix/libstore/misc.hh
@@ -0,0 +1,38 @@
+#pragma once
+
+#include "derivations.hh"
+
+
+namespace nix {
+
+
+/* Read a derivation, after ensuring its existence through
+ ensurePath(). */
+Derivation derivationFromPath(StoreAPI & store, const Path & drvPath);
+
+/* Place in `paths' the set of all store paths in the file system
+ closure of `storePath'; that is, all paths than can be directly or
+ indirectly reached from it. `paths' is not cleared. If
+ `flipDirection' is true, the set of paths that can reach
+ `storePath' is returned; that is, the closures under the
+ `referrers' relation instead of the `references' relation is
+ returned. */
+void computeFSClosure(StoreAPI & store, const Path & path,
+ PathSet & paths, bool flipDirection = false,
+ bool includeOutputs = false, bool includeDerivers = false);
+
+/* Return the path corresponding to the output identifier `id' in the
+ given derivation. */
+Path findOutput(const Derivation & drv, string id);
+
+/* Given a set of paths that are to be built, return the set of
+ derivations that will be built, and the set of output paths that
+ will be substituted. */
+void queryMissing(StoreAPI & store, const PathSet & targets,
+ PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+ unsigned long long & downloadSize, unsigned long long & narSize);
+
+bool willBuildLocally(const Derivation & drv);
+
+
+}
diff --git a/nix/libstore/optimise-store.cc b/nix/libstore/optimise-store.cc
new file mode 100644
index 0000000000..d833f3aa05
--- /dev/null
+++ b/nix/libstore/optimise-store.cc
@@ -0,0 +1,180 @@
+#include "config.h"
+
+#include "util.hh"
+#include "local-store.hh"
+#include "globals.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+
+
+namespace nix {
+
+
+static void makeWritable(const Path & path)
+{
+ struct stat st;
+ if (lstat(path.c_str(), &st))
+ throw SysError(format("getting attributes of path `%1%'") % path);
+ if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
+ throw SysError(format("changing writability of `%1%'") % path);
+}
+
+
+struct MakeReadOnly
+{
+ Path path;
+ MakeReadOnly(const Path & path) : path(path) { }
+ ~MakeReadOnly()
+ {
+ try {
+ /* This will make the path read-only. */
+ if (path != "") canonicaliseTimestampAndPermissions(path);
+ } catch (...) {
+ ignoreException();
+ }
+ }
+};
+
+
+void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
+{
+ checkInterrupt();
+
+ struct stat st;
+ if (lstat(path.c_str(), &st))
+ throw SysError(format("getting attributes of path `%1%'") % path);
+
+ if (S_ISDIR(st.st_mode)) {
+ Strings names = readDirectory(path);
+ foreach (Strings::iterator, i, names)
+ optimisePath_(stats, path + "/" + *i);
+ return;
+ }
+
+ /* We can hard link regular files and maybe symlinks. */
+ if (!S_ISREG(st.st_mode)
+#if CAN_LINK_SYMLINK
+ && !S_ISLNK(st.st_mode)
+#endif
+ ) return;
+
+ /* Sometimes SNAFUs can cause files in the Nix store to be
+ modified, in particular when running programs as root under
+ NixOS (example: $fontconfig/var/cache being modified). Skip
+ those files. FIXME: check the modification time. */
+ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
+ printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path);
+ return;
+ }
+
+ /* Hash the file. Note that hashPath() returns the hash over the
+ NAR serialisation, which includes the execute bit on the file.
+ Thus, executable and non-executable files with the same
+ contents *won't* be linked (which is good because otherwise the
+ permissions would be screwed up).
+
+ Also note that if `path' is a symlink, then we're hashing the
+ contents of the symlink (i.e. the result of readlink()), not
+ the contents of the target (which may not even exist). */
+ Hash hash = hashPath(htSHA256, path).first;
+ stats.totalFiles++;
+ printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
+
+ /* Check if this is a known hash. */
+ Path linkPath = linksDir + "/" + printHash32(hash);
+
+ if (!pathExists(linkPath)) {
+ /* Nope, create a hard link in the links directory. */
+ if (link(path.c_str(), linkPath.c_str()) == 0) return;
+ if (errno != EEXIST)
+ throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
+ /* Fall through if another process created ‘linkPath’ before
+ we did. */
+ }
+
+ /* Yes! We've seen a file with the same contents. Replace the
+ current file with a hard link to that file. */
+ struct stat stLink;
+ if (lstat(linkPath.c_str(), &stLink))
+ throw SysError(format("getting attributes of path `%1%'") % linkPath);
+
+ stats.sameContents++;
+ if (st.st_ino == stLink.st_ino) {
+ printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath);
+ return;
+ }
+
+ printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath);
+
+ /* Make the containing directory writable, but only if it's not
+ the store itself (we don't want or need to mess with its
+ permissions). */
+ bool mustToggle = !isStorePath(path);
+ if (mustToggle) makeWritable(dirOf(path));
+
+ /* When we're done, make the directory read-only again and reset
+ its timestamp back to 0. */
+ MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
+
+ Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
+ % settings.nixStore % getpid() % rand()).str();
+
+ if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
+ if (errno == EMLINK) {
+ /* Too many links to the same file (>= 32000 on most file
+ systems). This is likely to happen with empty files.
+ Just shrug and ignore. */
+ if (st.st_size)
+ printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
+ return;
+ }
+ throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath);
+ }
+
+ /* Atomically replace the old file with the new hard link. */
+ if (rename(tempLink.c_str(), path.c_str()) == -1) {
+ if (unlink(tempLink.c_str()) == -1)
+ printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
+ if (errno == EMLINK) {
+ /* Some filesystems generate too many links on the rename,
+ rather than on the original link. (Probably it
+ temporarily increases the st_nlink field before
+ decreasing it again.) */
+ if (st.st_size)
+ printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
+ return;
+ }
+ throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path);
+ }
+
+ stats.filesLinked++;
+ stats.bytesFreed += st.st_size;
+ stats.blocksFreed += st.st_blocks;
+}
+
+
+void LocalStore::optimiseStore(OptimiseStats & stats)
+{
+ PathSet paths = queryAllValidPaths();
+
+ foreach (PathSet::iterator, i, paths) {
+ addTempRoot(*i);
+ if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
+ startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
+ optimisePath_(stats, *i);
+ }
+}
+
+
+void LocalStore::optimisePath(const Path & path)
+{
+ OptimiseStats stats;
+ if (settings.autoOptimiseStore) optimisePath_(stats, path);
+}
+
+
+}
diff --git a/nix/libstore/pathlocks.cc b/nix/libstore/pathlocks.cc
new file mode 100644
index 0000000000..b858ed238d
--- /dev/null
+++ b/nix/libstore/pathlocks.cc
@@ -0,0 +1,199 @@
+#include "pathlocks.hh"
+#include "util.hh"
+
+#include <cerrno>
+#include <cstdlib>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+
+namespace nix {
+
+
+int openLockFile(const Path & path, bool create)
+{
+ AutoCloseFD fd;
+
+ fd = open(path.c_str(), O_RDWR | (create ? O_CREAT : 0), 0600);
+ if (fd == -1 && (create || errno != ENOENT))
+ throw SysError(format("opening lock file `%1%'") % path);
+
+ closeOnExec(fd);
+
+ return fd.borrow();
+}
+
+
+void deleteLockFile(const Path & path, int fd)
+{
+ /* Get rid of the lock file. Have to be careful not to introduce
+ races. Write a (meaningless) token to the file to indicate to
+ other processes waiting on this lock that the lock is stale
+ (deleted). */
+ unlink(path.c_str());
+ writeFull(fd, (const unsigned char *) "d", 1);
+ /* Note that the result of unlink() is ignored; removing the lock
+ file is an optimisation, not a necessity. */
+}
+
+
+bool lockFile(int fd, LockType lockType, bool wait)
+{
+ struct flock lock;
+ if (lockType == ltRead) lock.l_type = F_RDLCK;
+ else if (lockType == ltWrite) lock.l_type = F_WRLCK;
+ else if (lockType == ltNone) lock.l_type = F_UNLCK;
+ else abort();
+ lock.l_whence = SEEK_SET;
+ lock.l_start = 0;
+ lock.l_len = 0; /* entire file */
+
+ if (wait) {
+ while (fcntl(fd, F_SETLKW, &lock) != 0) {
+ checkInterrupt();
+ if (errno != EINTR)
+ throw SysError(format("acquiring/releasing lock"));
+ }
+ } else {
+ while (fcntl(fd, F_SETLK, &lock) != 0) {
+ checkInterrupt();
+ if (errno == EACCES || errno == EAGAIN) return false;
+ if (errno != EINTR)
+ throw SysError(format("acquiring/releasing lock"));
+ }
+ }
+
+ return true;
+}
+
+
+/* This enables us to check whether are not already holding a lock on
+ a file ourselves. POSIX locks (fcntl) suck in this respect: if we
+ close a descriptor, the previous lock will be closed as well. And
+ there is no way to query whether we already have a lock (F_GETLK
+ only works on locks held by other processes). */
+static StringSet lockedPaths; /* !!! not thread-safe */
+
+
+PathLocks::PathLocks()
+ : deletePaths(false)
+{
+}
+
+
+PathLocks::PathLocks(const PathSet & paths, const string & waitMsg)
+ : deletePaths(false)
+{
+ lockPaths(paths, waitMsg);
+}
+
+
+bool PathLocks::lockPaths(const PathSet & _paths,
+ const string & waitMsg, bool wait)
+{
+ assert(fds.empty());
+
+ /* Note that `fds' is built incrementally so that the destructor
+ will only release those locks that we have already acquired. */
+
+ /* Sort the paths. This assures that locks are always acquired in
+ the same order, thus preventing deadlocks. */
+ Paths paths(_paths.begin(), _paths.end());
+ paths.sort();
+
+ /* Acquire the lock for each path. */
+ foreach (Paths::iterator, i, paths) {
+ checkInterrupt();
+ Path path = *i;
+ Path lockPath = path + ".lock";
+
+ debug(format("locking path `%1%'") % path);
+
+ if (lockedPaths.find(lockPath) != lockedPaths.end())
+ throw Error("deadlock: trying to re-acquire self-held lock");
+
+ AutoCloseFD fd;
+
+ while (1) {
+
+ /* Open/create the lock file. */
+ fd = openLockFile(lockPath, true);
+
+ /* Acquire an exclusive lock. */
+ if (!lockFile(fd, ltWrite, false)) {
+ if (wait) {
+ if (waitMsg != "") printMsg(lvlError, waitMsg);
+ lockFile(fd, ltWrite, true);
+ } else {
+ /* Failed to lock this path; release all other
+ locks. */
+ unlock();
+ return false;
+ }
+ }
+
+ debug(format("lock acquired on `%1%'") % lockPath);
+
+ /* Check that the lock file hasn't become stale (i.e.,
+ hasn't been unlinked). */
+ struct stat st;
+ if (fstat(fd, &st) == -1)
+ throw SysError(format("statting lock file `%1%'") % lockPath);
+ if (st.st_size != 0)
+ /* This lock file has been unlinked, so we're holding
+ a lock on a deleted file. This means that other
+ processes may create and acquire a lock on
+ `lockPath', and proceed. So we must retry. */
+ debug(format("open lock file `%1%' has become stale") % lockPath);
+ else
+ break;
+ }
+
+ /* Use borrow so that the descriptor isn't closed. */
+ fds.push_back(FDPair(fd.borrow(), lockPath));
+ lockedPaths.insert(lockPath);
+ }
+
+ return true;
+}
+
+
+PathLocks::~PathLocks()
+{
+ unlock();
+}
+
+
+void PathLocks::unlock()
+{
+ foreach (list<FDPair>::iterator, i, fds) {
+ if (deletePaths) deleteLockFile(i->second, i->first);
+
+ lockedPaths.erase(i->second);
+ if (close(i->first) == -1)
+ printMsg(lvlError,
+ format("error (ignored): cannot close lock file on `%1%'") % i->second);
+
+ debug(format("lock released on `%1%'") % i->second);
+ }
+
+ fds.clear();
+}
+
+
+void PathLocks::setDeletion(bool deletePaths)
+{
+ this->deletePaths = deletePaths;
+}
+
+
+bool pathIsLockedByMe(const Path & path)
+{
+ Path lockPath = path + ".lock";
+ return lockedPaths.find(lockPath) != lockedPaths.end();
+}
+
+
+}
diff --git a/nix/libstore/pathlocks.hh b/nix/libstore/pathlocks.hh
new file mode 100644
index 0000000000..8a6b1450da
--- /dev/null
+++ b/nix/libstore/pathlocks.hh
@@ -0,0 +1,45 @@
+#pragma once
+
+#include "types.hh"
+
+
+namespace nix {
+
+
+/* Open (possibly create) a lock file and return the file descriptor.
+ -1 is returned if create is false and the lock could not be opened
+ because it doesn't exist. Any other error throws an exception. */
+int openLockFile(const Path & path, bool create);
+
+/* Delete an open lock file. */
+void deleteLockFile(const Path & path, int fd);
+
+enum LockType { ltRead, ltWrite, ltNone };
+
+bool lockFile(int fd, LockType lockType, bool wait);
+
+
+class PathLocks
+{
+private:
+ typedef std::pair<int, Path> FDPair;
+ list<FDPair> fds;
+ bool deletePaths;
+
+public:
+ PathLocks();
+ PathLocks(const PathSet & paths,
+ const string & waitMsg = "");
+ bool lockPaths(const PathSet & _paths,
+ const string & waitMsg = "",
+ bool wait = true);
+ ~PathLocks();
+ void unlock();
+ void setDeletion(bool deletePaths);
+};
+
+
+bool pathIsLockedByMe(const Path & path);
+
+
+}
diff --git a/nix/libstore/references.cc b/nix/libstore/references.cc
new file mode 100644
index 0000000000..282b848938
--- /dev/null
+++ b/nix/libstore/references.cc
@@ -0,0 +1,122 @@
+#include "references.hh"
+#include "hash.hh"
+#include "util.hh"
+#include "archive.hh"
+
+#include <map>
+#include <cstdlib>
+
+
+namespace nix {
+
+
+static unsigned int refLength = 32; /* characters */
+
+
+static void search(const unsigned char * s, unsigned int len,
+ StringSet & hashes, StringSet & seen)
+{
+ static bool initialised = false;
+ static bool isBase32[256];
+ if (!initialised) {
+ for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false;
+ for (unsigned int i = 0; i < base32Chars.size(); ++i)
+ isBase32[(unsigned char) base32Chars[i]] = true;
+ initialised = true;
+ }
+
+ for (unsigned int i = 0; i + refLength <= len; ) {
+ int j;
+ bool match = true;
+ for (j = refLength - 1; j >= 0; --j)
+ if (!isBase32[(unsigned char) s[i + j]]) {
+ i += j + 1;
+ match = false;
+ break;
+ }
+ if (!match) continue;
+ string ref((const char *) s + i, refLength);
+ if (hashes.find(ref) != hashes.end()) {
+ debug(format("found reference to `%1%' at offset `%2%'")
+ % ref % i);
+ seen.insert(ref);
+ hashes.erase(ref);
+ }
+ ++i;
+ }
+}
+
+
+struct RefScanSink : Sink
+{
+ HashSink hashSink;
+ StringSet hashes;
+ StringSet seen;
+
+ string tail;
+
+ RefScanSink() : hashSink(htSHA256) { }
+
+ void operator () (const unsigned char * data, size_t len);
+};
+
+
+void RefScanSink::operator () (const unsigned char * data, size_t len)
+{
+ hashSink(data, len);
+
+ /* It's possible that a reference spans the previous and current
+ fragment, so search in the concatenation of the tail of the
+ previous fragment and the start of the current fragment. */
+ string s = tail + string((const char *) data, len > refLength ? refLength : len);
+ search((const unsigned char *) s.data(), s.size(), hashes, seen);
+
+ search(data, len, hashes, seen);
+
+ unsigned int tailLen = len <= refLength ? len : refLength;
+ tail =
+ string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) +
+ string((const char *) data + len - tailLen, tailLen);
+}
+
+
+PathSet scanForReferences(const string & path,
+ const PathSet & refs, HashResult & hash)
+{
+ RefScanSink sink;
+ std::map<string, Path> backMap;
+
+ /* For efficiency (and a higher hit rate), just search for the
+ hash part of the file name. (This assumes that all references
+ have the form `HASH-bla'). */
+ foreach (PathSet::const_iterator, i, refs) {
+ string baseName = baseNameOf(*i);
+ string::size_type pos = baseName.find('-');
+ if (pos == string::npos)
+ throw Error(format("bad reference `%1%'") % *i);
+ string s = string(baseName, 0, pos);
+ assert(s.size() == refLength);
+ assert(backMap.find(s) == backMap.end());
+ // parseHash(htSHA256, s);
+ sink.hashes.insert(s);
+ backMap[s] = *i;
+ }
+
+ /* Look for the hashes in the NAR dump of the path. */
+ dumpPath(path, sink);
+
+ /* Map the hashes found back to their store paths. */
+ PathSet found;
+ foreach (StringSet::iterator, i, sink.seen) {
+ std::map<string, Path>::iterator j;
+ if ((j = backMap.find(*i)) == backMap.end()) abort();
+ found.insert(j->second);
+ }
+
+ hash = sink.hashSink.finish();
+
+ return found;
+}
+
+
+}
diff --git a/nix/libstore/references.hh b/nix/libstore/references.hh
new file mode 100644
index 0000000000..013809d122
--- /dev/null
+++ b/nix/libstore/references.hh
@@ -0,0 +1,11 @@
+#pragma once
+
+#include "types.hh"
+#include "hash.hh"
+
+namespace nix {
+
+PathSet scanForReferences(const Path & path, const PathSet & refs,
+ HashResult & hash);
+
+}
diff --git a/nix/libstore/remote-store.cc b/nix/libstore/remote-store.cc
new file mode 100644
index 0000000000..4619206932
--- /dev/null
+++ b/nix/libstore/remote-store.cc
@@ -0,0 +1,602 @@
+#include "serialise.hh"
+#include "util.hh"
+#include "remote-store.hh"
+#include "worker-protocol.hh"
+#include "archive.hh"
+#include "affinity.hh"
+#include "globals.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <fcntl.h>
+
+#include <iostream>
+#include <unistd.h>
+#include <cstring>
+
+namespace nix {
+
+
+Path readStorePath(Source & from)
+{
+ Path path = readString(from);
+ assertStorePath(path);
+ return path;
+}
+
+
+template<class T> T readStorePaths(Source & from)
+{
+ T paths = readStrings<T>(from);
+ foreach (typename T::iterator, i, paths) assertStorePath(*i);
+ return paths;
+}
+
+template PathSet readStorePaths(Source & from);
+
+
+RemoteStore::RemoteStore()
+{
+ initialised = false;
+}
+
+
+void RemoteStore::openConnection(bool reserveSpace)
+{
+ if (initialised) return;
+ initialised = true;
+
+ string remoteMode = getEnv("NIX_REMOTE");
+
+ if (remoteMode == "daemon")
+ /* Connect to a daemon that does the privileged work for
+ us. */
+ connectToDaemon();
+ else
+ throw Error(format("invalid setting for NIX_REMOTE, `%1%'") % remoteMode);
+
+ from.fd = fdSocket;
+ to.fd = fdSocket;
+
+ /* Send the magic greeting, check for the reply. */
+ try {
+ writeInt(WORKER_MAGIC_1, to);
+ to.flush();
+ unsigned int magic = readInt(from);
+ if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
+
+ daemonVersion = readInt(from);
+ if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
+ throw Error("Nix daemon protocol version not supported");
+ writeInt(PROTOCOL_VERSION, to);
+
+ if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) {
+ int cpu = settings.lockCPU ? lockToCurrentCPU() : -1;
+ if (cpu != -1) {
+ writeInt(1, to);
+ writeInt(cpu, to);
+ } else
+ writeInt(0, to);
+ }
+
+ if (GET_PROTOCOL_MINOR(daemonVersion) >= 11)
+ writeInt(reserveSpace, to);
+
+ processStderr();
+ }
+ catch (Error & e) {
+ throw Error(format("cannot start worker (%1%)")
+ % e.msg());
+ }
+
+ setOptions();
+}
+
+
+void RemoteStore::connectToDaemon()
+{
+ fdSocket = socket(PF_UNIX, SOCK_STREAM, 0);
+ if (fdSocket == -1)
+ throw SysError("cannot create Unix domain socket");
+ closeOnExec(fdSocket);
+
+ string socketPath = settings.nixDaemonSocketFile;
+
+ /* Urgh, sockaddr_un allows path names of only 108 characters. So
+ chdir to the socket directory so that we can pass a relative
+ path name. !!! this is probably a bad idea in multi-threaded
+ applications... */
+ AutoCloseFD fdPrevDir = open(".", O_RDONLY);
+ if (fdPrevDir == -1) throw SysError("couldn't open current directory");
+ chdir(dirOf(socketPath).c_str());
+ Path socketPathRel = "./" + baseNameOf(socketPath);
+
+ struct sockaddr_un addr;
+ addr.sun_family = AF_UNIX;
+ if (socketPathRel.size() >= sizeof(addr.sun_path))
+ throw Error(format("socket path `%1%' is too long") % socketPathRel);
+ using namespace std;
+ strcpy(addr.sun_path, socketPathRel.c_str());
+
+ if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+ throw SysError(format("cannot connect to daemon at `%1%'") % socketPath);
+
+ if (fchdir(fdPrevDir) == -1)
+ throw SysError("couldn't change back to previous directory");
+}
+
+
+RemoteStore::~RemoteStore()
+{
+ try {
+ to.flush();
+ fdSocket.close();
+ if (child != -1)
+ child.wait(true);
+ } catch (...) {
+ ignoreException();
+ }
+}
+
+
+void RemoteStore::setOptions()
+{
+ writeInt(wopSetOptions, to);
+
+ writeInt(settings.keepFailed, to);
+ writeInt(settings.keepGoing, to);
+ writeInt(settings.tryFallback, to);
+ writeInt(verbosity, to);
+ writeInt(settings.maxBuildJobs, to);
+ writeInt(settings.maxSilentTime, to);
+ if (GET_PROTOCOL_MINOR(daemonVersion) >= 2)
+ writeInt(settings.useBuildHook, to);
+ if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) {
+ writeInt(settings.buildVerbosity, to);
+ writeInt(logType, to);
+ writeInt(settings.printBuildTrace, to);
+ }
+ if (GET_PROTOCOL_MINOR(daemonVersion) >= 6)
+ writeInt(settings.buildCores, to);
+ if (GET_PROTOCOL_MINOR(daemonVersion) >= 10)
+ writeInt(settings.useSubstitutes, to);
+
+ if (GET_PROTOCOL_MINOR(daemonVersion) >= 12) {
+ Settings::SettingsMap overrides = settings.getOverrides();
+ writeInt(overrides.size(), to);
+ foreach (Settings::SettingsMap::iterator, i, overrides) {
+ writeString(i->first, to);
+ writeString(i->second, to);
+ }
+ }
+
+ processStderr();
+}
+
+
+bool RemoteStore::isValidPath(const Path & path)
+{
+ openConnection();
+ writeInt(wopIsValidPath, to);
+ writeString(path, to);
+ processStderr();
+ unsigned int reply = readInt(from);
+ return reply != 0;
+}
+
+
+PathSet RemoteStore::queryValidPaths(const PathSet & paths)
+{
+ openConnection();
+ if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
+ PathSet res;
+ foreach (PathSet::const_iterator, i, paths)
+ if (isValidPath(*i)) res.insert(*i);
+ return res;
+ } else {
+ writeInt(wopQueryValidPaths, to);
+ writeStrings(paths, to);
+ processStderr();
+ return readStorePaths<PathSet>(from);
+ }
+}
+
+
+PathSet RemoteStore::queryAllValidPaths()
+{
+ openConnection();
+ writeInt(wopQueryAllValidPaths, to);
+ processStderr();
+ return readStorePaths<PathSet>(from);
+}
+
+
+PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
+{
+ openConnection();
+ if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
+ PathSet res;
+ foreach (PathSet::const_iterator, i, paths) {
+ writeInt(wopHasSubstitutes, to);
+ writeString(*i, to);
+ processStderr();
+ if (readInt(from)) res.insert(*i);
+ }
+ return res;
+ } else {
+ writeInt(wopQuerySubstitutablePaths, to);
+ writeStrings(paths, to);
+ processStderr();
+ return readStorePaths<PathSet>(from);
+ }
+}
+
+
+void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos)
+{
+ if (paths.empty()) return;
+
+ openConnection();
+
+ if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return;
+
+ if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
+
+ foreach (PathSet::const_iterator, i, paths) {
+ SubstitutablePathInfo info;
+ writeInt(wopQuerySubstitutablePathInfo, to);
+ writeString(*i, to);
+ processStderr();
+ unsigned int reply = readInt(from);
+ if (reply == 0) continue;
+ info.deriver = readString(from);
+ if (info.deriver != "") assertStorePath(info.deriver);
+ info.references = readStorePaths<PathSet>(from);
+ info.downloadSize = readLongLong(from);
+ info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0;
+ infos[*i] = info;
+ }
+
+ } else {
+
+ writeInt(wopQuerySubstitutablePathInfos, to);
+ writeStrings(paths, to);
+ processStderr();
+ unsigned int count = readInt(from);
+ for (unsigned int n = 0; n < count; n++) {
+ Path path = readStorePath(from);
+ SubstitutablePathInfo & info(infos[path]);
+ info.deriver = readString(from);
+ if (info.deriver != "") assertStorePath(info.deriver);
+ info.references = readStorePaths<PathSet>(from);
+ info.downloadSize = readLongLong(from);
+ info.narSize = readLongLong(from);
+ }
+
+ }
+}
+
+
+ValidPathInfo RemoteStore::queryPathInfo(const Path & path)
+{
+ openConnection();
+ writeInt(wopQueryPathInfo, to);
+ writeString(path, to);
+ processStderr();
+ ValidPathInfo info;
+ info.path = path;
+ info.deriver = readString(from);
+ if (info.deriver != "") assertStorePath(info.deriver);
+ info.hash = parseHash(htSHA256, readString(from));
+ info.references = readStorePaths<PathSet>(from);
+ info.registrationTime = readInt(from);
+ info.narSize = readLongLong(from);
+ return info;
+}
+
+
+Hash RemoteStore::queryPathHash(const Path & path)
+{
+ openConnection();
+ writeInt(wopQueryPathHash, to);
+ writeString(path, to);
+ processStderr();
+ string hash = readString(from);
+ return parseHash(htSHA256, hash);
+}
+
+
+void RemoteStore::queryReferences(const Path & path,
+ PathSet & references)
+{
+ openConnection();
+ writeInt(wopQueryReferences, to);
+ writeString(path, to);
+ processStderr();
+ PathSet references2 = readStorePaths<PathSet>(from);
+ references.insert(references2.begin(), references2.end());
+}
+
+
+void RemoteStore::queryReferrers(const Path & path,
+ PathSet & referrers)
+{
+ openConnection();
+ writeInt(wopQueryReferrers, to);
+ writeString(path, to);
+ processStderr();
+ PathSet referrers2 = readStorePaths<PathSet>(from);
+ referrers.insert(referrers2.begin(), referrers2.end());
+}
+
+
+Path RemoteStore::queryDeriver(const Path & path)
+{
+ openConnection();
+ writeInt(wopQueryDeriver, to);
+ writeString(path, to);
+ processStderr();
+ Path drvPath = readString(from);
+ if (drvPath != "") assertStorePath(drvPath);
+ return drvPath;
+}
+
+
+PathSet RemoteStore::queryValidDerivers(const Path & path)
+{
+ openConnection();
+ writeInt(wopQueryValidDerivers, to);
+ writeString(path, to);
+ processStderr();
+ return readStorePaths<PathSet>(from);
+}
+
+
+PathSet RemoteStore::queryDerivationOutputs(const Path & path)
+{
+ openConnection();
+ writeInt(wopQueryDerivationOutputs, to);
+ writeString(path, to);
+ processStderr();
+ return readStorePaths<PathSet>(from);
+}
+
+
+PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
+{
+ openConnection();
+ writeInt(wopQueryDerivationOutputNames, to);
+ writeString(path, to);
+ processStderr();
+ return readStrings<PathSet>(from);
+}
+
+
+Path RemoteStore::queryPathFromHashPart(const string & hashPart)
+{
+ openConnection();
+ writeInt(wopQueryPathFromHashPart, to);
+ writeString(hashPart, to);
+ processStderr();
+ Path path = readString(from);
+ if (!path.empty()) assertStorePath(path);
+ return path;
+}
+
+
+Path RemoteStore::addToStore(const Path & _srcPath,
+ bool recursive, HashType hashAlgo, PathFilter & filter, bool repair)
+{
+ if (repair) throw Error("repairing is not supported when building through the Nix daemon");
+
+ openConnection();
+
+ Path srcPath(absPath(_srcPath));
+
+ writeInt(wopAddToStore, to);
+ writeString(baseNameOf(srcPath), to);
+ /* backwards compatibility hack */
+ writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to);
+ writeInt(recursive ? 1 : 0, to);
+ writeString(printHashType(hashAlgo), to);
+ dumpPath(srcPath, to, filter);
+ processStderr();
+ return readStorePath(from);
+}
+
+
+Path RemoteStore::addTextToStore(const string & name, const string & s,
+ const PathSet & references, bool repair)
+{
+ if (repair) throw Error("repairing is not supported when building through the Nix daemon");
+
+ openConnection();
+ writeInt(wopAddTextToStore, to);
+ writeString(name, to);
+ writeString(s, to);
+ writeStrings(references, to);
+
+ processStderr();
+ return readStorePath(from);
+}
+
+
+void RemoteStore::exportPath(const Path & path, bool sign,
+ Sink & sink)
+{
+ openConnection();
+ writeInt(wopExportPath, to);
+ writeString(path, to);
+ writeInt(sign ? 1 : 0, to);
+ processStderr(&sink); /* sink receives the actual data */
+ readInt(from);
+}
+
+
+Paths RemoteStore::importPaths(bool requireSignature, Source & source)
+{
+ openConnection();
+ writeInt(wopImportPaths, to);
+ /* We ignore requireSignature, since the worker forces it to true
+ anyway. */
+ processStderr(0, &source);
+ return readStorePaths<Paths>(from);
+}
+
+
+void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
+{
+ if (buildMode != bmNormal) throw Error("repairing or checking is not supported when building through the Nix daemon");
+ openConnection();
+ writeInt(wopBuildPaths, to);
+ if (GET_PROTOCOL_MINOR(daemonVersion) >= 13)
+ writeStrings(drvPaths, to);
+ else {
+ /* For backwards compatibility with old daemons, strip output
+ identifiers. */
+ PathSet drvPaths2;
+ foreach (PathSet::const_iterator, i, drvPaths)
+ drvPaths2.insert(string(*i, 0, i->find('!')));
+ writeStrings(drvPaths2, to);
+ }
+ processStderr();
+ readInt(from);
+}
+
+
+void RemoteStore::ensurePath(const Path & path)
+{
+ openConnection();
+ writeInt(wopEnsurePath, to);
+ writeString(path, to);
+ processStderr();
+ readInt(from);
+}
+
+
+void RemoteStore::addTempRoot(const Path & path)
+{
+ openConnection();
+ writeInt(wopAddTempRoot, to);
+ writeString(path, to);
+ processStderr();
+ readInt(from);
+}
+
+
+void RemoteStore::addIndirectRoot(const Path & path)
+{
+ openConnection();
+ writeInt(wopAddIndirectRoot, to);
+ writeString(path, to);
+ processStderr();
+ readInt(from);
+}
+
+
+void RemoteStore::syncWithGC()
+{
+ openConnection();
+ writeInt(wopSyncWithGC, to);
+ processStderr();
+ readInt(from);
+}
+
+
+Roots RemoteStore::findRoots()
+{
+ openConnection();
+ writeInt(wopFindRoots, to);
+ processStderr();
+ unsigned int count = readInt(from);
+ Roots result;
+ while (count--) {
+ Path link = readString(from);
+ Path target = readStorePath(from);
+ result[link] = target;
+ }
+ return result;
+}
+
+
+void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
+{
+ openConnection(false);
+
+ writeInt(wopCollectGarbage, to);
+ writeInt(options.action, to);
+ writeStrings(options.pathsToDelete, to);
+ writeInt(options.ignoreLiveness, to);
+ writeLongLong(options.maxFreed, to);
+ writeInt(0, to);
+ if (GET_PROTOCOL_MINOR(daemonVersion) >= 5) {
+ /* removed options */
+ writeInt(0, to);
+ writeInt(0, to);
+ }
+
+ processStderr();
+
+ results.paths = readStrings<PathSet>(from);
+ results.bytesFreed = readLongLong(from);
+ readLongLong(from); // obsolete
+}
+
+
+PathSet RemoteStore::queryFailedPaths()
+{
+ openConnection();
+ writeInt(wopQueryFailedPaths, to);
+ processStderr();
+ return readStorePaths<PathSet>(from);
+}
+
+
+void RemoteStore::clearFailedPaths(const PathSet & paths)
+{
+ openConnection();
+ writeInt(wopClearFailedPaths, to);
+ writeStrings(paths, to);
+ processStderr();
+ readInt(from);
+}
+
+
+void RemoteStore::processStderr(Sink * sink, Source * source)
+{
+ to.flush();
+ unsigned int msg;
+ while ((msg = readInt(from)) == STDERR_NEXT
+ || msg == STDERR_READ || msg == STDERR_WRITE) {
+ if (msg == STDERR_WRITE) {
+ string s = readString(from);
+ if (!sink) throw Error("no sink");
+ (*sink)((const unsigned char *) s.data(), s.size());
+ }
+ else if (msg == STDERR_READ) {
+ if (!source) throw Error("no source");
+ size_t len = readInt(from);
+ unsigned char * buf = new unsigned char[len];
+ AutoDeleteArray<unsigned char> d(buf);
+ writeString(buf, source->read(buf, len), to);
+ to.flush();
+ }
+ else {
+ string s = readString(from);
+ writeToStderr(s);
+ }
+ }
+ if (msg == STDERR_ERROR) {
+ string error = readString(from);
+ unsigned int status = GET_PROTOCOL_MINOR(daemonVersion) >= 8 ? readInt(from) : 1;
+ throw Error(format("%1%") % error, status);
+ }
+ else if (msg != STDERR_LAST)
+ throw Error("protocol error processing standard error");
+}
+
+
+}
diff --git a/nix/libstore/remote-store.hh b/nix/libstore/remote-store.hh
new file mode 100644
index 0000000000..04b60fce4b
--- /dev/null
+++ b/nix/libstore/remote-store.hh
@@ -0,0 +1,104 @@
+#pragma once
+
+#include <string>
+
+#include "store-api.hh"
+
+
+namespace nix {
+
+
+class Pipe;
+class Pid;
+struct FdSink;
+struct FdSource;
+
+
+class RemoteStore : public StoreAPI
+{
+public:
+
+ RemoteStore();
+
+ ~RemoteStore();
+
+ /* Implementations of abstract store API methods. */
+
+ bool isValidPath(const Path & path);
+
+ PathSet queryValidPaths(const PathSet & paths);
+
+ PathSet queryAllValidPaths();
+
+ ValidPathInfo queryPathInfo(const Path & path);
+
+ Hash queryPathHash(const Path & path);
+
+ void queryReferences(const Path & path, PathSet & references);
+
+ void queryReferrers(const Path & path, PathSet & referrers);
+
+ Path queryDeriver(const Path & path);
+
+ PathSet queryValidDerivers(const Path & path);
+
+ PathSet queryDerivationOutputs(const Path & path);
+
+ StringSet queryDerivationOutputNames(const Path & path);
+
+ Path queryPathFromHashPart(const string & hashPart);
+
+ PathSet querySubstitutablePaths(const PathSet & paths);
+
+ void querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos);
+
+ Path addToStore(const Path & srcPath,
+ bool recursive = true, HashType hashAlgo = htSHA256,
+ PathFilter & filter = defaultPathFilter, bool repair = false);
+
+ Path addTextToStore(const string & name, const string & s,
+ const PathSet & references, bool repair = false);
+
+ void exportPath(const Path & path, bool sign,
+ Sink & sink);
+
+ Paths importPaths(bool requireSignature, Source & source);
+
+ void buildPaths(const PathSet & paths, BuildMode buildMode);
+
+ void ensurePath(const Path & path);
+
+ void addTempRoot(const Path & path);
+
+ void addIndirectRoot(const Path & path);
+
+ void syncWithGC();
+
+ Roots findRoots();
+
+ void collectGarbage(const GCOptions & options, GCResults & results);
+
+ PathSet queryFailedPaths();
+
+ void clearFailedPaths(const PathSet & paths);
+
+private:
+ AutoCloseFD fdSocket;
+ FdSink to;
+ FdSource from;
+ Pid child;
+ unsigned int daemonVersion;
+ bool initialised;
+
+ void openConnection(bool reserveSpace = true);
+
+ void processStderr(Sink * sink = 0, Source * source = 0);
+
+ void connectToDaemon();
+
+ void setOptions();
+};
+
+
+}
diff --git a/nix/libstore/schema.sql b/nix/libstore/schema.sql
new file mode 100644
index 0000000000..c1b4a689af
--- /dev/null
+++ b/nix/libstore/schema.sql
@@ -0,0 +1,44 @@
+create table if not exists ValidPaths (
+ id integer primary key autoincrement not null,
+ path text unique not null,
+ hash text not null,
+ registrationTime integer not null,
+ deriver text,
+ narSize integer
+);
+
+create table if not exists Refs (
+ referrer integer not null,
+ reference integer not null,
+ primary key (referrer, reference),
+ foreign key (referrer) references ValidPaths(id) on delete cascade,
+ foreign key (reference) references ValidPaths(id) on delete restrict
+);
+
+create index if not exists IndexReferrer on Refs(referrer);
+create index if not exists IndexReference on Refs(reference);
+
+-- Paths can refer to themselves, causing a tuple (N, N) in the Refs
+-- table. This causes a deletion of the corresponding row in
+-- ValidPaths to cause a foreign key constraint violation (due to `on
+-- delete restrict' on the `reference' column). Therefore, explicitly
+-- get rid of self-references.
+create trigger if not exists DeleteSelfRefs before delete on ValidPaths
+ begin
+ delete from Refs where referrer = old.id and reference = old.id;
+ end;
+
+create table if not exists DerivationOutputs (
+ drv integer not null,
+ id text not null, -- symbolic output id, usually "out"
+ path text not null,
+ primary key (drv, id),
+ foreign key (drv) references ValidPaths(id) on delete cascade
+);
+
+create index if not exists IndexDerivationOutputs on DerivationOutputs(path);
+
+create table if not exists FailedPaths (
+ path text primary key not null,
+ time integer not null
+);
diff --git a/nix/libstore/store-api.cc b/nix/libstore/store-api.cc
new file mode 100644
index 0000000000..0238e5b0b6
--- /dev/null
+++ b/nix/libstore/store-api.cc
@@ -0,0 +1,331 @@
+#include "store-api.hh"
+#include "globals.hh"
+#include "util.hh"
+
+#include <climits>
+
+
+namespace nix {
+
+
+GCOptions::GCOptions()
+{
+ action = gcDeleteDead;
+ ignoreLiveness = false;
+ maxFreed = ULLONG_MAX;
+}
+
+
+bool isInStore(const Path & path)
+{
+ return isInDir(path, settings.nixStore);
+}
+
+
+bool isStorePath(const Path & path)
+{
+ return isInStore(path)
+ && path.find('/', settings.nixStore.size() + 1) == Path::npos;
+}
+
+
+void assertStorePath(const Path & path)
+{
+ if (!isStorePath(path))
+ throw Error(format("path `%1%' is not in the Nix store") % path);
+}
+
+
+Path toStorePath(const Path & path)
+{
+ if (!isInStore(path))
+ throw Error(format("path `%1%' is not in the Nix store") % path);
+ Path::size_type slash = path.find('/', settings.nixStore.size() + 1);
+ if (slash == Path::npos)
+ return path;
+ else
+ return Path(path, 0, slash);
+}
+
+
+Path followLinksToStore(const Path & _path)
+{
+ Path path = absPath(_path);
+ while (!isInStore(path)) {
+ if (!isLink(path)) break;
+ string target = readLink(path);
+ path = absPath(target, dirOf(path));
+ }
+ if (!isInStore(path))
+ throw Error(format("path `%1%' is not in the Nix store") % path);
+ return path;
+}
+
+
+Path followLinksToStorePath(const Path & path)
+{
+ return toStorePath(followLinksToStore(path));
+}
+
+
+string storePathToName(const Path & path)
+{
+ assertStorePath(path);
+ return string(path, settings.nixStore.size() + 34);
+}
+
+
+void checkStoreName(const string & name)
+{
+ string validChars = "+-._?=";
+ /* Disallow names starting with a dot for possible security
+ reasons (e.g., "." and ".."). */
+ if (string(name, 0, 1) == ".")
+ throw Error(format("illegal name: `%1%'") % name);
+ foreach (string::const_iterator, i, name)
+ if (!((*i >= 'A' && *i <= 'Z') ||
+ (*i >= 'a' && *i <= 'z') ||
+ (*i >= '0' && *i <= '9') ||
+ validChars.find(*i) != string::npos))
+ {
+ throw Error(format("invalid character `%1%' in name `%2%'")
+ % *i % name);
+ }
+}
+
+
+/* Store paths have the following form:
+
+ <store>/<h>-<name>
+
+ where
+
+ <store> = the location of the Nix store, usually /nix/store
+
+ <name> = a human readable name for the path, typically obtained
+ from the name attribute of the derivation, or the name of the
+ source file from which the store path is created. For derivation
+ outputs other than the default "out" output, the string "-<id>"
+ is suffixed to <name>.
+
+ <h> = base-32 representation of the first 160 bits of a SHA-256
+ hash of <s>; the hash part of the store name
+
+ <s> = the string "<type>:sha256:<h2>:<store>:<name>";
+ note that it includes the location of the store as well as the
+ name to make sure that changes to either of those are reflected
+ in the hash (e.g. you won't get /nix/store/<h>-name1 and
+ /nix/store/<h>-name2 with equal hash parts).
+
+ <type> = one of:
+ "text:<r1>:<r2>:...<rN>"
+ for plain text files written to the store using
+ addTextToStore(); <r1> ... <rN> are the references of the
+ path.
+ "source"
+ for paths copied to the store using addToStore() when recursive
+ = true and hashAlgo = "sha256"
+ "output:<id>"
+ for either the outputs created by derivations, OR paths copied
+ to the store using addToStore() with recursive != true or
+ hashAlgo != "sha256" (in that case "source" is used; it's
+ silly, but it's done that way for compatibility). <id> is the
+ name of the output (usually, "out").
+
+ <h2> = base-16 representation of a SHA-256 hash of:
+ if <type> = "text:...":
+ the string written to the resulting store path
+ if <type> = "source":
+ the serialisation of the path from which this store path is
+ copied, as returned by hashPath()
+ if <type> = "output:out":
+ for non-fixed derivation outputs:
+ the derivation (see hashDerivationModulo() in
+ primops.cc)
+ for paths copied by addToStore() or produced by fixed-output
+ derivations:
+ the string "fixed:out:<rec><algo>:<hash>:", where
+ <rec> = "r:" for recursive (path) hashes, or "" or flat
+ (file) hashes
+ <algo> = "md5", "sha1" or "sha256"
+ <hash> = base-16 representation of the path or flat hash of
+ the contents of the path (or expected contents of the
+ path for fixed-output derivations)
+
+ It would have been nicer to handle fixed-output derivations under
+ "source", e.g. have something like "source:<rec><algo>", but we're
+ stuck with this for now...
+
+ The main reason for this way of computing names is to prevent name
+ collisions (for security). For instance, it shouldn't be feasible
+ to come up with a derivation whose output path collides with the
+ path for a copied source. The former would have a <s> starting with
+ "output:out:", while the latter would have a <2> starting with
+ "source:".
+*/
+
+
+Path makeStorePath(const string & type,
+ const Hash & hash, const string & name)
+{
+ /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
+ string s = type + ":sha256:" + printHash(hash) + ":"
+ + settings.nixStore + ":" + name;
+
+ checkStoreName(name);
+
+ return settings.nixStore + "/"
+ + printHash32(compressHash(hashString(htSHA256, s), 20))
+ + "-" + name;
+}
+
+
+Path makeOutputPath(const string & id,
+ const Hash & hash, const string & name)
+{
+ return makeStorePath("output:" + id, hash,
+ name + (id == "out" ? "" : "-" + id));
+}
+
+
+Path makeFixedOutputPath(bool recursive,
+ HashType hashAlgo, Hash hash, string name)
+{
+ return hashAlgo == htSHA256 && recursive
+ ? makeStorePath("source", hash, name)
+ : makeStorePath("output:out", hashString(htSHA256,
+ "fixed:out:" + (recursive ? (string) "r:" : "") +
+ printHashType(hashAlgo) + ":" + printHash(hash) + ":"),
+ name);
+}
+
+
+std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
+ bool recursive, HashType hashAlgo, PathFilter & filter)
+{
+ HashType ht(hashAlgo);
+ Hash h = recursive ? hashPath(ht, srcPath, filter).first : hashFile(ht, srcPath);
+ string name = baseNameOf(srcPath);
+ Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name);
+ return std::pair<Path, Hash>(dstPath, h);
+}
+
+
+Path computeStorePathForText(const string & name, const string & s,
+ const PathSet & references)
+{
+ Hash hash = hashString(htSHA256, s);
+ /* Stuff the references (if any) into the type. This is a bit
+ hacky, but we can't put them in `s' since that would be
+ ambiguous. */
+ string type = "text";
+ foreach (PathSet::const_iterator, i, references) {
+ type += ":";
+ type += *i;
+ }
+ return makeStorePath(type, hash, name);
+}
+
+
+/* Return a string accepted by decodeValidPathInfo() that
+ registers the specified paths as valid. Note: it's the
+ responsibility of the caller to provide a closure. */
+string StoreAPI::makeValidityRegistration(const PathSet & paths,
+ bool showDerivers, bool showHash)
+{
+ string s = "";
+
+ foreach (PathSet::iterator, i, paths) {
+ s += *i + "\n";
+
+ ValidPathInfo info = queryPathInfo(*i);
+
+ if (showHash) {
+ s += printHash(info.hash) + "\n";
+ s += (format("%1%\n") % info.narSize).str();
+ }
+
+ Path deriver = showDerivers ? info.deriver : "";
+ s += deriver + "\n";
+
+ s += (format("%1%\n") % info.references.size()).str();
+
+ foreach (PathSet::iterator, j, info.references)
+ s += *j + "\n";
+ }
+
+ return s;
+}
+
+
+ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
+{
+ ValidPathInfo info;
+ getline(str, info.path);
+ if (str.eof()) { info.path = ""; return info; }
+ if (hashGiven) {
+ string s;
+ getline(str, s);
+ info.hash = parseHash(htSHA256, s);
+ getline(str, s);
+ if (!string2Int(s, info.narSize)) throw Error("number expected");
+ }
+ getline(str, info.deriver);
+ string s; int n;
+ getline(str, s);
+ if (!string2Int(s, n)) throw Error("number expected");
+ while (n--) {
+ getline(str, s);
+ info.references.insert(s);
+ }
+ if (!str || str.eof()) throw Error("missing input");
+ return info;
+}
+
+
+string showPaths(const PathSet & paths)
+{
+ string s;
+ foreach (PathSet::const_iterator, i, paths) {
+ if (s.size() != 0) s += ", ";
+ s += "`" + *i + "'";
+ }
+ return s;
+}
+
+
+void exportPaths(StoreAPI & store, const Paths & paths,
+ bool sign, Sink & sink)
+{
+ foreach (Paths::const_iterator, i, paths) {
+ writeInt(1, sink);
+ store.exportPath(*i, sign, sink);
+ }
+ writeInt(0, sink);
+}
+
+
+}
+
+
+#include "local-store.hh"
+#include "serialise.hh"
+#include "remote-store.hh"
+
+
+namespace nix {
+
+
+std::shared_ptr<StoreAPI> store;
+
+
+std::shared_ptr<StoreAPI> openStore(bool reserveSpace)
+{
+ if (getEnv("NIX_REMOTE") == "")
+ return std::shared_ptr<StoreAPI>(new LocalStore(reserveSpace));
+ else
+ return std::shared_ptr<StoreAPI>(new RemoteStore());
+}
+
+
+}
diff --git a/nix/libstore/store-api.hh b/nix/libstore/store-api.hh
new file mode 100644
index 0000000000..b635fee2cf
--- /dev/null
+++ b/nix/libstore/store-api.hh
@@ -0,0 +1,366 @@
+#pragma once
+
+#include "hash.hh"
+#include "serialise.hh"
+
+#include <string>
+#include <map>
+#include <memory>
+
+
+namespace nix {
+
+
+typedef std::map<Path, Path> Roots;
+
+
+struct GCOptions
+{
+ /* Garbage collector operation:
+
+ - `gcReturnLive': return the set of paths reachable from
+ (i.e. in the closure of) the roots.
+
+ - `gcReturnDead': return the set of paths not reachable from
+ the roots.
+
+ - `gcDeleteDead': actually delete the latter set.
+
+ - `gcDeleteSpecific': delete the paths listed in
+ `pathsToDelete', insofar as they are not reachable.
+ */
+ typedef enum {
+ gcReturnLive,
+ gcReturnDead,
+ gcDeleteDead,
+ gcDeleteSpecific,
+ } GCAction;
+
+ GCAction action;
+
+ /* If `ignoreLiveness' is set, then reachability from the roots is
+ ignored (dangerous!). However, the paths must still be
+ unreferenced *within* the store (i.e., there can be no other
+ store paths that depend on them). */
+ bool ignoreLiveness;
+
+ /* For `gcDeleteSpecific', the paths to delete. */
+ PathSet pathsToDelete;
+
+ /* Stop after at least `maxFreed' bytes have been freed. */
+ unsigned long long maxFreed;
+
+ GCOptions();
+};
+
+
+struct GCResults
+{
+ /* Depending on the action, the GC roots, or the paths that would
+ be or have been deleted. */
+ PathSet paths;
+
+ /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
+ number of bytes that would be or was freed. */
+ unsigned long long bytesFreed;
+
+ GCResults()
+ {
+ bytesFreed = 0;
+ }
+};
+
+
+struct SubstitutablePathInfo
+{
+ Path deriver;
+ PathSet references;
+ unsigned long long downloadSize; /* 0 = unknown or inapplicable */
+ unsigned long long narSize; /* 0 = unknown */
+};
+
+typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
+
+
+struct ValidPathInfo
+{
+ Path path;
+ Path deriver;
+ Hash hash;
+ PathSet references;
+ time_t registrationTime;
+ unsigned long long narSize; // 0 = unknown
+ unsigned long long id; // internal use only
+ ValidPathInfo() : registrationTime(0), narSize(0) { }
+};
+
+typedef list<ValidPathInfo> ValidPathInfos;
+
+
+enum BuildMode { bmNormal, bmRepair, bmCheck };
+
+
+class StoreAPI
+{
+public:
+
+ virtual ~StoreAPI() { }
+
+ /* Check whether a path is valid. */
+ virtual bool isValidPath(const Path & path) = 0;
+
+ /* Query which of the given paths is valid. */
+ virtual PathSet queryValidPaths(const PathSet & paths) = 0;
+
+ /* Query the set of all valid paths. */
+ virtual PathSet queryAllValidPaths() = 0;
+
+ /* Query information about a valid path. */
+ virtual ValidPathInfo queryPathInfo(const Path & path) = 0;
+
+ /* Query the hash of a valid path. */
+ virtual Hash queryPathHash(const Path & path) = 0;
+
+ /* Query the set of outgoing FS references for a store path. The
+ result is not cleared. */
+ virtual void queryReferences(const Path & path,
+ PathSet & references) = 0;
+
+ /* Queries the set of incoming FS references for a store path.
+ The result is not cleared. */
+ virtual void queryReferrers(const Path & path,
+ PathSet & referrers) = 0;
+
+ /* Query the deriver of a store path. Return the empty string if
+ no deriver has been set. */
+ virtual Path queryDeriver(const Path & path) = 0;
+
+ /* Return all currently valid derivations that have `path' as an
+ output. (Note that the result of `queryDeriver()' is the
+ derivation that was actually used to produce `path', which may
+ not exist anymore.) */
+ virtual PathSet queryValidDerivers(const Path & path) = 0;
+
+ /* Query the outputs of the derivation denoted by `path'. */
+ virtual PathSet queryDerivationOutputs(const Path & path) = 0;
+
+ /* Query the output names of the derivation denoted by `path'. */
+ virtual StringSet queryDerivationOutputNames(const Path & path) = 0;
+
+ /* Query the full store path given the hash part of a valid store
+ path, or "" if the path doesn't exist. */
+ virtual Path queryPathFromHashPart(const string & hashPart) = 0;
+
+ /* Query which of the given paths have substitutes. */
+ virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0;
+
+ /* Query substitute info (i.e. references, derivers and download
+ sizes) of a set of paths. If a path does not have substitute
+ info, it's omitted from the resulting ‘infos’ map. */
+ virtual void querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos) = 0;
+
+ /* Copy the contents of a path to the store and register the
+ validity the resulting path. The resulting path is returned.
+ The function object `filter' can be used to exclude files (see
+ libutil/archive.hh). */
+ virtual Path addToStore(const Path & srcPath,
+ bool recursive = true, HashType hashAlgo = htSHA256,
+ PathFilter & filter = defaultPathFilter, bool repair = false) = 0;
+
+ /* Like addToStore, but the contents written to the output path is
+ a regular file containing the given string. */
+ virtual Path addTextToStore(const string & name, const string & s,
+ const PathSet & references, bool repair = false) = 0;
+
+ /* Export a store path, that is, create a NAR dump of the store
+ path and append its references and its deriver. Optionally, a
+ cryptographic signature (created by OpenSSL) of the preceding
+ data is attached. */
+ virtual void exportPath(const Path & path, bool sign,
+ Sink & sink) = 0;
+
+ /* Import a sequence of NAR dumps created by exportPaths() into
+ the Nix store. */
+ virtual Paths importPaths(bool requireSignature, Source & source) = 0;
+
+ /* For each path, if it's a derivation, build it. Building a
+ derivation means ensuring that the output paths are valid. If
+ they are already valid, this is a no-op. Otherwise, validity
+ can be reached in two ways. First, if the output paths is
+ substitutable, then build the path that way. Second, the
+ output paths can be created by running the builder, after
+ recursively building any sub-derivations. For inputs that are
+ not derivations, substitute them. */
+ virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) = 0;
+
+ /* Ensure that a path is valid. If it is not currently valid, it
+ may be made valid by running a substitute (if defined for the
+ path). */
+ virtual void ensurePath(const Path & path) = 0;
+
+ /* Add a store path as a temporary root of the garbage collector.
+ The root disappears as soon as we exit. */
+ virtual void addTempRoot(const Path & path) = 0;
+
+ /* Add an indirect root, which is merely a symlink to `path' from
+ /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed
+ to be a symlink to a store path. The garbage collector will
+ automatically remove the indirect root when it finds that
+ `path' has disappeared. */
+ virtual void addIndirectRoot(const Path & path) = 0;
+
+ /* Acquire the global GC lock, then immediately release it. This
+ function must be called after registering a new permanent root,
+ but before exiting. Otherwise, it is possible that a running
+ garbage collector doesn't see the new root and deletes the
+ stuff we've just built. By acquiring the lock briefly, we
+ ensure that either:
+
+ - The collector is already running, and so we block until the
+ collector is finished. The collector will know about our
+ *temporary* locks, which should include whatever it is we
+ want to register as a permanent lock.
+
+ - The collector isn't running, or it's just started but hasn't
+ acquired the GC lock yet. In that case we get and release
+ the lock right away, then exit. The collector scans the
+ permanent root and sees our's.
+
+ In either case the permanent root is seen by the collector. */
+ virtual void syncWithGC() = 0;
+
+ /* Find the roots of the garbage collector. Each root is a pair
+ (link, storepath) where `link' is the path of the symlink
+ outside of the Nix store that point to `storePath'. */
+ virtual Roots findRoots() = 0;
+
+ /* Perform a garbage collection. */
+ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
+
+ /* Return the set of paths that have failed to build.*/
+ virtual PathSet queryFailedPaths() = 0;
+
+ /* Clear the "failed" status of the given paths. The special
+ value `*' causes all failed paths to be cleared. */
+ virtual void clearFailedPaths(const PathSet & paths) = 0;
+
+ /* Return a string representing information about the path that
+ can be loaded into the database using `nix-store --load-db' or
+ `nix-store --register-validity'. */
+ string makeValidityRegistration(const PathSet & paths,
+ bool showDerivers, bool showHash);
+};
+
+
+/* !!! These should be part of the store API, I guess. */
+
+/* Throw an exception if `path' is not directly in the Nix store. */
+void assertStorePath(const Path & path);
+
+bool isInStore(const Path & path);
+bool isStorePath(const Path & path);
+
+/* Extract the name part of the given store path. */
+string storePathToName(const Path & path);
+
+void checkStoreName(const string & name);
+
+
+/* Chop off the parts after the top-level store name, e.g.,
+ /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
+Path toStorePath(const Path & path);
+
+
+/* Follow symlinks until we end up with a path in the Nix store. */
+Path followLinksToStore(const Path & path);
+
+
+/* Same as followLinksToStore(), but apply toStorePath() to the
+ result. */
+Path followLinksToStorePath(const Path & path);
+
+
+/* Constructs a unique store path name. */
+Path makeStorePath(const string & type,
+ const Hash & hash, const string & name);
+
+Path makeOutputPath(const string & id,
+ const Hash & hash, const string & name);
+
+Path makeFixedOutputPath(bool recursive,
+ HashType hashAlgo, Hash hash, string name);
+
+
+/* This is the preparatory part of addToStore() and addToStoreFixed();
+ it computes the store path to which srcPath is to be copied.
+ Returns the store path and the cryptographic hash of the
+ contents of srcPath. */
+std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
+ bool recursive = true, HashType hashAlgo = htSHA256,
+ PathFilter & filter = defaultPathFilter);
+
+/* Preparatory part of addTextToStore().
+
+ !!! Computation of the path should take the references given to
+ addTextToStore() into account, otherwise we have a (relatively
+ minor) security hole: a caller can register a source file with
+ bogus references. If there are too many references, the path may
+ not be garbage collected when it has to be (not really a problem,
+ the caller could create a root anyway), or it may be garbage
+ collected when it shouldn't be (more serious).
+
+ Hashing the references would solve this (bogus references would
+ simply yield a different store path, so other users wouldn't be
+ affected), but it has some backwards compatibility issues (the
+ hashing scheme changes), so I'm not doing that for now. */
+Path computeStorePathForText(const string & name, const string & s,
+ const PathSet & references);
+
+
+/* Remove the temporary roots file for this process. Any temporary
+ root becomes garbage after this point unless it has been registered
+ as a (permanent) root. */
+void removeTempRoots();
+
+
+/* Register a permanent GC root. */
+Path addPermRoot(StoreAPI & store, const Path & storePath,
+ const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false);
+
+
+/* Sort a set of paths topologically under the references relation.
+ If p refers to q, then p preceeds q in this list. */
+Paths topoSortPaths(StoreAPI & store, const PathSet & paths);
+
+
+/* For now, there is a single global store API object, but we'll
+ purify that in the future. */
+extern std::shared_ptr<StoreAPI> store;
+
+
+/* Factory method: open the Nix database, either through the local or
+ remote implementation. */
+std::shared_ptr<StoreAPI> openStore(bool reserveSpace = true);
+
+
+/* Display a set of paths in human-readable form (i.e., between quotes
+ and separated by commas). */
+string showPaths(const PathSet & paths);
+
+
+ValidPathInfo decodeValidPathInfo(std::istream & str,
+ bool hashGiven = false);
+
+
+/* Export multiple paths in the format expected by ‘nix-store
+ --import’. */
+void exportPaths(StoreAPI & store, const Paths & paths,
+ bool sign, Sink & sink);
+
+
+MakeError(SubstError, Error)
+MakeError(BuildError, Error) /* denotes a permanent build failure */
+
+
+}
diff --git a/nix/libstore/worker-protocol.hh b/nix/libstore/worker-protocol.hh
new file mode 100644
index 0000000000..9317f89c37
--- /dev/null
+++ b/nix/libstore/worker-protocol.hh
@@ -0,0 +1,60 @@
+#pragma once
+
+namespace nix {
+
+
+#define WORKER_MAGIC_1 0x6e697863
+#define WORKER_MAGIC_2 0x6478696f
+
+#define PROTOCOL_VERSION 0x10e
+#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
+#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
+
+
+typedef enum {
+ wopQuit = 0,
+ wopIsValidPath = 1,
+ wopHasSubstitutes = 3,
+ wopQueryPathHash = 4,
+ wopQueryReferences = 5,
+ wopQueryReferrers = 6,
+ wopAddToStore = 7,
+ wopAddTextToStore = 8,
+ wopBuildPaths = 9,
+ wopEnsurePath = 10,
+ wopAddTempRoot = 11,
+ wopAddIndirectRoot = 12,
+ wopSyncWithGC = 13,
+ wopFindRoots = 14,
+ wopExportPath = 16,
+ wopQueryDeriver = 18,
+ wopSetOptions = 19,
+ wopCollectGarbage = 20,
+ wopQuerySubstitutablePathInfo = 21,
+ wopQueryDerivationOutputs = 22,
+ wopQueryAllValidPaths = 23,
+ wopQueryFailedPaths = 24,
+ wopClearFailedPaths = 25,
+ wopQueryPathInfo = 26,
+ wopImportPaths = 27,
+ wopQueryDerivationOutputNames = 28,
+ wopQueryPathFromHashPart = 29,
+ wopQuerySubstitutablePathInfos = 30,
+ wopQueryValidPaths = 31,
+ wopQuerySubstitutablePaths = 32,
+ wopQueryValidDerivers = 33,
+} WorkerOp;
+
+
+#define STDERR_NEXT 0x6f6c6d67
+#define STDERR_READ 0x64617461 // data needed from source
+#define STDERR_WRITE 0x64617416 // data for sink
+#define STDERR_LAST 0x616c7473
+#define STDERR_ERROR 0x63787470
+
+
+Path readStorePath(Source & from);
+template<class T> T readStorePaths(Source & from);
+
+
+}