aboutsummaryrefslogtreecommitdiff
path: root/nix/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'nix/libstore')
-rw-r--r--nix/libstore/build.cc313
-rw-r--r--nix/libstore/gc.cc69
-rw-r--r--nix/libstore/globals.cc92
-rw-r--r--nix/libstore/globals.hh26
-rw-r--r--nix/libstore/local-store.cc81
-rw-r--r--nix/libstore/local-store.hh19
-rw-r--r--nix/libstore/optimise-store.cc16
-rw-r--r--nix/libstore/remote-store.cc31
-rw-r--r--nix/libstore/remote-store.hh5
-rw-r--r--nix/libstore/store-api.hh4
-rw-r--r--nix/libstore/worker-protocol.hh2
11 files changed, 379 insertions, 279 deletions
diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc
index f38cd29940..009fcb2c0c 100644
--- a/nix/libstore/build.cc
+++ b/nix/libstore/build.cc
@@ -57,9 +57,8 @@
#include <netinet/ip.h>
#endif
-#if HAVE_SYS_PERSONALITY_H
+#if __linux__
#include <sys/personality.h>
-#define CAN_DO_LINUX32_BUILDS
#endif
#if HAVE_STATVFS
@@ -85,8 +84,12 @@ class Goal;
typedef std::shared_ptr<Goal> GoalPtr;
typedef std::weak_ptr<Goal> WeakGoalPtr;
+struct CompareGoalPtrs {
+ bool operator() (const GoalPtr & a, const GoalPtr & b);
+};
+
/* Set of goals. */
-typedef set<GoalPtr> Goals;
+typedef set<GoalPtr, CompareGoalPtrs> Goals;
typedef list<WeakGoalPtr> WeakGoals;
/* A map of paths to goals (and the other way around). */
@@ -173,11 +176,20 @@ public:
(important!), etc. */
virtual void cancel(bool timeout) = 0;
+ virtual string key() = 0;
+
protected:
void amDone(ExitCode result);
};
+bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) {
+ string s1 = a->key();
+ string s2 = b->key();
+ return s1 < s2;
+}
+
+
/* A mapping used to remember for each child process to what goal it
belongs, and file descriptors for receiving log data and output
path creation commands. */
@@ -238,6 +250,9 @@ public:
failure). */
bool permanentFailure;
+ /* Set if at least one derivation had a timeout. */
+ bool timedOut;
+
LocalStore & store;
std::shared_ptr<HookInstance> hook;
@@ -301,6 +316,7 @@ public:
void addToWeakGoals(WeakGoals & goals, GoalPtr p)
{
// FIXME: necessary?
+ // FIXME: O(n)
foreach (WeakGoals::iterator, i, goals)
if (i->lock() == p) return;
goals.push_back(p);
@@ -374,8 +390,6 @@ void Goal::trace(const format & f)
/* Common initialisation performed in child processes. */
static void commonChildInit(Pipe & logPipe)
{
- restoreAffinity();
-
/* Put the child in a separate session (and thus a separate
process group) so that it has no controlling terminal (meaning
that e.g. ssh cannot open /dev/tty) and it doesn't receive
@@ -590,7 +604,9 @@ HookInstance::HookInstance()
{
debug("starting build hook");
- Path buildHook = absPath(getEnv("NIX_BUILD_HOOK"));
+ Path buildHook = getEnv("NIX_BUILD_HOOK");
+ if (string(buildHook, 0, 1) != "/") buildHook = settings.nixLibexecDir + "/nix/" + buildHook;
+ buildHook = canonPath(buildHook);
/* Create a pipe to get the output of the child. */
fromHook.create();
@@ -602,44 +618,30 @@ HookInstance::HookInstance()
builderOut.create();
/* Fork the hook. */
- pid = maybeVfork();
- switch (pid) {
-
- case -1:
- throw SysError("unable to fork");
+ pid = startProcess([&]() {
- case 0:
- try { /* child */
+ commonChildInit(fromHook);
- commonChildInit(fromHook);
+ if (chdir("/") == -1) throw SysError("changing into `/");
- if (chdir("/") == -1) throw SysError("changing into `/");
+ /* Dup the communication pipes. */
+ if (dup2(toHook.readSide, STDIN_FILENO) == -1)
+ throw SysError("dupping to-hook read side");
- /* Dup the communication pipes. */
- if (dup2(toHook.readSide, STDIN_FILENO) == -1)
- throw SysError("dupping to-hook read side");
+ /* Use fd 4 for the builder's stdout/stderr. */
+ if (dup2(builderOut.writeSide, 4) == -1)
+ throw SysError("dupping builder's stdout/stderr");
- /* Use fd 4 for the builder's stdout/stderr. */
- if (dup2(builderOut.writeSide, 4) == -1)
- throw SysError("dupping builder's stdout/stderr");
+ execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(),
+ (format("%1%") % settings.maxSilentTime).str().c_str(),
+ (format("%1%") % settings.printBuildTrace).str().c_str(),
+ (format("%1%") % settings.buildTimeout).str().c_str(),
+ NULL);
- execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(),
- (format("%1%") % settings.maxSilentTime).str().c_str(),
- (format("%1%") % settings.printBuildTrace).str().c_str(),
- (format("%1%") % settings.buildTimeout).str().c_str(),
- NULL);
+ throw SysError(format("executing `%1%'") % buildHook);
+ });
- throw SysError(format("executing `%1%'") % buildHook);
-
- } catch (std::exception & e) {
- writeToStderr("build hook error: " + string(e.what()) + "\n");
- }
- _exit(1);
- }
-
- /* parent */
pid.setSeparatePG(true);
- pid.setKillSignal(SIGTERM);
fromHook.writeSide.close();
toHook.readSide.close();
}
@@ -648,7 +650,8 @@ HookInstance::HookInstance()
HookInstance::~HookInstance()
{
try {
- pid.kill();
+ toHook.writeSide.close();
+ pid.kill(true);
} catch (...) {
ignoreException();
}
@@ -784,17 +787,21 @@ private:
outputs to allow hard links between outputs. */
InodesSeen inodesSeen;
- /* Magic exit code denoting that setting up the child environment
- failed. (It's possible that the child actually returns the
- exit code, but ah well.) */
- const static int childSetupFailed = 189;
-
public:
DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal);
~DerivationGoal();
void cancel(bool timeout);
+ string key()
+ {
+ /* Ensure that derivations get built in order of their name,
+ i.e. a derivation named "aardvark" always comes before
+ "baboon". And substitution goals always happen before
+ derivation goals (due to "b$"). */
+ return "b$" + storePathToName(drvPath) + "$" + drvPath;
+ }
+
void work();
Path getDrvPath()
@@ -879,13 +886,9 @@ DerivationGoal::~DerivationGoal()
{
/* Careful: we should never ever throw an exception from a
destructor. */
- try {
- killChild();
- deleteTmpDir(false);
- closeLogFile();
- } catch (...) {
- ignoreException();
- }
+ try { killChild(); } catch (...) { ignoreException(); }
+ try { deleteTmpDir(false); } catch (...) { ignoreException(); }
+ try { closeLogFile(); } catch (...) { ignoreException(); }
}
@@ -956,6 +959,11 @@ void DerivationGoal::init()
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
+ if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
+ haveDerivation();
+ return;
+ }
+
addWaitee(worker.makeSubstitutionGoal(drvPath));
state = &DerivationGoal::haveDerivation;
@@ -1209,7 +1217,7 @@ static string get(const StringPairs & map, const string & key)
static bool canBuildLocally(const string & platform)
{
return platform == settings.thisSystem
-#ifdef CAN_DO_LINUX32_BUILDS
+#if __linux__
|| (platform == "i686-linux" && settings.thisSystem == "x86_64-linux")
#endif
;
@@ -1433,9 +1441,6 @@ void DerivationGoal::buildDone()
if (pathExists(chrootRootDir + *i))
rename((chrootRootDir + *i).c_str(), i->c_str());
- if (WIFEXITED(status) && WEXITSTATUS(status) == childSetupFailed)
- throw Error(format("failed to set up the build environment for `%1%'") % drvPath);
-
if (diskFull)
printMsg(lvlError, "note: build failure may have been caused by lack of free disk space");
@@ -1469,37 +1474,41 @@ void DerivationGoal::buildDone()
outputLocks.unlock();
} catch (BuildError & e) {
- printMsg(lvlError, e.msg());
+ if (!hook)
+ printMsg(lvlError, e.msg());
outputLocks.unlock();
buildUser.release();
- /* When using a build hook, the hook will return a remote
- build failure using exit code 100. Anything else is a hook
- problem. */
- bool hookError = hook &&
- (!WIFEXITED(status) || WEXITSTATUS(status) != 100);
+ if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) {
+ if (settings.printBuildTrace)
+ printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath);
+ worker.timedOut = true;
+ }
- if (settings.printBuildTrace) {
- if (hook && hookError)
+ else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
+ if (settings.printBuildTrace)
printMsg(lvlError, format("@ hook-failed %1% - %2% %3%")
% drvPath % status % e.msg());
- else
+ }
+
+ else {
+ if (settings.printBuildTrace)
printMsg(lvlError, format("@ build-failed %1% - %2% %3%")
% drvPath % 1 % e.msg());
+ worker.permanentFailure = !fixedOutput && !diskFull;
+
+ /* Register the outputs of this build as "failed" so we
+ won't try to build them again (negative caching).
+ However, don't do this for fixed-output derivations,
+ since they're likely to fail for transient reasons
+ (e.g., fetchurl not being able to access the network).
+ Hook errors (like communication problems with the
+ remote machine) shouldn't be cached either. */
+ if (settings.cacheFailure && !fixedOutput && !diskFull)
+ foreach (DerivationOutputs::iterator, i, drv.outputs)
+ worker.store.registerFailedPath(i->second.path);
}
- /* Register the outputs of this build as "failed" so we won't
- try to build them again (negative caching). However, don't
- do this for fixed-output derivations, since they're likely
- to fail for transient reasons (e.g., fetchurl not being
- able to access the network). Hook errors (like
- communication problems with the remote machine) shouldn't
- be cached either. */
- if (settings.cacheFailure && !hookError && !fixedOutput)
- foreach (DerivationOutputs::iterator, i, drv.outputs)
- worker.store.registerFailedPath(i->second.path);
-
- worker.permanentFailure = !hookError && !fixedOutput && !diskFull;
amDone(ecFailed);
return;
}
@@ -1825,12 +1834,15 @@ void DerivationGoal::startBuilder()
/* Bind-mount a user-configurable set of directories from the
host file system. */
- foreach (StringSet::iterator, i, settings.dirsInChroot) {
- size_t p = i->find('=');
+ PathSet dirs = tokenizeString<StringSet>(settings.get("build-chroot-dirs", string(DEFAULT_CHROOT_DIRS)));
+ PathSet dirs2 = tokenizeString<StringSet>(settings.get("build-extra-chroot-dirs", string("")));
+ dirs.insert(dirs2.begin(), dirs2.end());
+ for (auto & i : dirs) {
+ size_t p = i.find('=');
if (p == string::npos)
- dirsInChroot[*i] = *i;
+ dirsInChroot[i] = i;
else
- dirsInChroot[string(*i, 0, p)] = string(*i, p + 1);
+ dirsInChroot[string(i, 0, p)] = string(i, p + 1);
}
dirsInChroot[tmpDir] = tmpDir;
@@ -1969,10 +1981,15 @@ void DerivationGoal::startBuilder()
worker.childStarted(shared_from_this(), pid,
singleton<set<int> >(builderOut.readSide), true, true);
+ /* Check if setting up the build environment failed. */
+ string msg = readLine(builderOut.readSide);
+ if (!msg.empty()) throw Error(msg);
+
if (settings.printBuildTrace) {
printMsg(lvlError, format("@ build-started %1% - %2% %3%")
% drvPath % drv.platform % logFile);
}
+
}
@@ -1981,10 +1998,14 @@ void DerivationGoal::initChild()
/* Warning: in the child we should absolutely not make any SQLite
calls! */
- bool inSetup = true;
-
try { /* child */
+ _writeToStderr = 0;
+
+ restoreAffinity();
+
+ commonChildInit(builderOut);
+
#if CHROOT_ENABLED
if (useChroot) {
/* Initialise the loopback interface. */
@@ -2088,9 +2109,9 @@ void DerivationGoal::initChild()
throw SysError("mounting /dev/pts");
createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
- /* Make sure /dev/pts/ptmx is world-writable. With some
- Linux versions, it is created with permissions 0. */
- chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
+ /* Make sure /dev/pts/ptmx is world-writable. With some
+ Linux versions, it is created with permissions 0. */
+ chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
}
/* Do the chroot(). Below we do a chdir() to the
@@ -2103,15 +2124,13 @@ void DerivationGoal::initChild()
}
#endif
- commonChildInit(builderOut);
-
if (chdir(tmpDir.c_str()) == -1)
throw SysError(format("changing into `%1%'") % tmpDir);
/* Close all other file descriptors. */
closeMostFDs(set<int>());
-#ifdef CAN_DO_LINUX32_BUILDS
+#if __linux__
/* Change the personality to 32-bit if we're doing an
i686-linux build on an x86_64-linux machine. */
struct utsname utsbuf;
@@ -2119,7 +2138,7 @@ void DerivationGoal::initChild()
if (drv.platform == "i686-linux" &&
(settings.thisSystem == "x86_64-linux" ||
(!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
- if (personality(0x0008 | 0x8000000 /* == PER_LINUX32_3GB */) == -1)
+ if (personality(PER_LINUX32) == -1)
throw SysError("cannot set i686-linux personality");
}
@@ -2129,6 +2148,11 @@ void DerivationGoal::initChild()
int cur = personality(0xffffffff);
if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
}
+
+ /* Disable address space randomization for improved
+ determinism. */
+ int cur = personality(0xffffffff);
+ if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
#endif
/* Fill in the environment. */
@@ -2167,21 +2191,28 @@ void DerivationGoal::initChild()
/* Fill in the arguments. */
string builderBasename = baseNameOf(drv.builder);
args.push_back(builderBasename.c_str());
- foreach (Strings::iterator, i, drv.args)
- args.push_back(rewriteHashes(*i, rewritesToTmp).c_str());
+ foreach (Strings::iterator, i, drv.args) {
+ auto re = rewriteHashes(*i, rewritesToTmp);
+ auto cstr = new char[re.length()+1];
+ std::strcpy(cstr, re.c_str());
+
+ args.push_back(cstr);
+ }
args.push_back(0);
restoreSIGPIPE();
+ /* Indicate that we managed to set up the build environment. */
+ writeToStderr("\n");
+
/* Execute the program. This should not return. */
- inSetup = false;
execve(program.c_str(), (char * *) &args[0], (char * *) envArr);
throw SysError(format("executing `%1%'") % drv.builder);
} catch (std::exception & e) {
- writeToStderr("build error: " + string(e.what()) + "\n");
- _exit(inSetup ? childSetupFailed : 1);
+ writeToStderr("while setting up the build environment: " + string(e.what()) + "\n");
+ _exit(1);
}
abort(); /* never reached */
@@ -2333,7 +2364,7 @@ void DerivationGoal::registerOutputs()
if (buildMode == bmCheck) {
ValidPathInfo info = worker.store.queryPathInfo(path);
if (hash.first != info.hash)
- throw Error(format("derivation `%2%' may not be deterministic: hash mismatch in output `%1%'") % drvPath % path);
+ throw Error(format("derivation `%1%' may not be deterministic: hash mismatch in output `%2%'") % drvPath % path);
continue;
}
@@ -2347,16 +2378,36 @@ void DerivationGoal::registerOutputs()
debug(format("referenced input: `%1%'") % *i);
}
- /* If the derivation specifies an `allowedReferences'
- attribute (containing a list of paths that the output may
- refer to), check that all references are in that list. !!!
- allowedReferences should really be per-output. */
- if (drv.env.find("allowedReferences") != drv.env.end()) {
- PathSet allowed = parseReferenceSpecifiers(drv, get(drv.env, "allowedReferences"));
- foreach (PathSet::iterator, i, references)
- if (allowed.find(*i) == allowed.end())
- throw BuildError(format("output is not allowed to refer to path `%1%'") % *i);
- }
+ /* Enforce `allowedReferences' and friends. */
+ auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) {
+ if (drv.env.find(attrName) == drv.env.end()) return;
+
+ PathSet spec = parseReferenceSpecifiers(drv, get(drv.env, attrName));
+
+ PathSet used;
+ if (recursive) {
+ /* Our requisites are the union of the closures of our references. */
+ for (auto & i : references)
+ /* Don't call computeFSClosure on ourselves. */
+ if (actualPath != i)
+ computeFSClosure(worker.store, i, used);
+ } else
+ used = references;
+
+ for (auto & i : used)
+ if (allowed) {
+ if (spec.find(i) == spec.end())
+ throw BuildError(format("output (`%1%') is not allowed to refer to path `%2%'") % actualPath % i);
+ } else {
+ if (spec.find(i) != spec.end())
+ throw BuildError(format("output (`%1%') is not allowed to refer to path `%2%'") % actualPath % i);
+ }
+ };
+
+ checkRefs("allowedReferences", true, false);
+ checkRefs("allowedRequisites", true, true);
+ checkRefs("disallowedReferences", false, false);
+ checkRefs("disallowedRequisites", false, true);
worker.store.optimisePath(path); // FIXME: combine with scanForReferences()
@@ -2586,6 +2637,13 @@ public:
void cancel(bool timeout);
+ string key()
+ {
+ /* "a$" ensures substitution goals happen before derivation
+ goals. */
+ return "a$" + storePathToName(storePath) + "$" + storePath;
+ }
+
void work();
/* The states. */
@@ -2781,32 +2839,18 @@ void SubstitutionGoal::tryToRun()
const char * * argArr = strings2CharPtrs(args);
/* Fork the substitute program. */
- pid = maybeVfork();
-
- switch (pid) {
+ pid = startProcess([&]() {
- case -1:
- throw SysError("unable to fork");
+ commonChildInit(logPipe);
- case 0:
- try { /* child */
+ if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
+ throw SysError("cannot dup output pipe into stdout");
- commonChildInit(logPipe);
+ execv(sub.c_str(), (char * *) argArr);
- if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
- throw SysError("cannot dup output pipe into stdout");
+ throw SysError(format("executing `%1%'") % sub);
+ });
- execv(sub.c_str(), (char * *) argArr);
-
- throw SysError(format("executing `%1%'") % sub);
-
- } catch (std::exception & e) {
- writeToStderr("substitute error: " + string(e.what()) + "\n");
- }
- _exit(1);
- }
-
- /* parent */
pid.setSeparatePG(true);
pid.setKillSignal(SIGTERM);
outPipe.writeSide.close();
@@ -2944,6 +2988,7 @@ Worker::Worker(LocalStore & store)
nrLocalBuilds = 0;
lastWokenUp = 0;
permanentFailure = false;
+ timedOut = false;
}
@@ -3109,15 +3154,19 @@ void Worker::run(const Goals & _topGoals)
checkInterrupt();
- /* Call every wake goal. */
+ /* Call every wake goal (in the ordering established by
+ CompareGoalPtrs). */
while (!awake.empty() && !topGoals.empty()) {
- WeakGoals awake2(awake);
+ Goals awake2;
+ for (auto & i : awake) {
+ GoalPtr goal = i.lock();
+ if (goal) awake2.insert(goal);
+ }
awake.clear();
- foreach (WeakGoals::iterator, i, awake2) {
+ for (auto & goal : awake2) {
checkInterrupt();
- GoalPtr goal = i->lock();
- if (goal) goal->work();
- if (topGoals.empty()) break;
+ goal->work();
+ if (topGoals.empty()) break; // stuff may have been cancelled
}
}
@@ -3255,6 +3304,7 @@ void Worker::waitForInput()
format("%1% timed out after %2% seconds of silence")
% goal->getName() % settings.maxSilentTime);
goal->cancel(true);
+ timedOut = true;
}
else if (goal->getExitCode() == Goal::ecBusy &&
@@ -3266,6 +3316,7 @@ void Worker::waitForInput()
format("%1% timed out after %2% seconds")
% goal->getName() % settings.buildTimeout);
goal->cancel(true);
+ timedOut = true;
}
}
@@ -3282,7 +3333,7 @@ void Worker::waitForInput()
unsigned int Worker::exitStatus()
{
- return permanentFailure ? 100 : 1;
+ return timedOut ? 101 : (permanentFailure ? 100 : 1);
}
diff --git a/nix/libstore/gc.cc b/nix/libstore/gc.cc
index f90edac1cd..f98e02c1e2 100644
--- a/nix/libstore/gc.cc
+++ b/nix/libstore/gc.cc
@@ -115,7 +115,10 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath,
% gcRoot % rootsDir);
}
- makeSymlink(gcRoot, storePath);
+ if (baseNameOf(gcRoot) == baseNameOf(storePath))
+ writeFile(gcRoot, "");
+ else
+ makeSymlink(gcRoot, storePath);
}
/* Check that the root can be found by the garbage collector.
@@ -142,11 +145,6 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath,
}
-/* The file to which we write our temporary roots. */
-static Path fnTempRoots;
-static AutoCloseFD fdTempRoots;
-
-
void LocalStore::addTempRoot(const Path & path)
{
/* Create the temporary roots file for this process. */
@@ -201,27 +199,6 @@ void LocalStore::addTempRoot(const Path & path)
}
-void removeTempRoots()
-{
- if (fdTempRoots != -1) {
- fdTempRoots.close();
- unlink(fnTempRoots.c_str());
- }
-}
-
-
-/* Automatically clean up the temporary roots file when we exit. */
-struct RemoveTempRoots
-{
- ~RemoveTempRoots()
- {
- removeTempRoots();
- }
-};
-
-static RemoveTempRoots autoRemoveTempRoots __attribute__((unused));
-
-
typedef std::shared_ptr<AutoCloseFD> FDPtr;
typedef list<FDPtr> FDs;
@@ -230,11 +207,11 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
{
/* Read the `temproots' directory for per-process temporary root
files. */
- Strings tempRootFiles = readDirectory(
+ DirEntries tempRootFiles = readDirectory(
(format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str());
- foreach (Strings::iterator, i, tempRootFiles) {
- Path path = (format("%1%/%2%/%3%") % settings.nixStateDir % tempRootsDir % *i).str();
+ for (auto & i : tempRootFiles) {
+ Path path = (format("%1%/%2%/%3%") % settings.nixStateDir % tempRootsDir % i.name).str();
debug(format("reading temporary root file `%1%'") % path);
FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
@@ -294,19 +271,19 @@ static void foundRoot(StoreAPI & store,
}
-static void findRoots(StoreAPI & store, const Path & path, Roots & roots)
+static void findRoots(StoreAPI & store, const Path & path, unsigned char type, Roots & roots)
{
try {
- struct stat st = lstat(path);
+ if (type == DT_UNKNOWN)
+ type = getFileType(path);
- if (S_ISDIR(st.st_mode)) {
- Strings names = readDirectory(path);
- foreach (Strings::iterator, i, names)
- findRoots(store, path + "/" + *i, roots);
+ if (type == DT_DIR) {
+ for (auto & i : readDirectory(path))
+ findRoots(store, path + "/" + i.name, i.type, roots);
}
- else if (S_ISLNK(st.st_mode)) {
+ else if (type == DT_LNK) {
Path target = readLink(path);
if (isInStore(target))
foundRoot(store, path, target, roots);
@@ -328,6 +305,12 @@ static void findRoots(StoreAPI & store, const Path & path, Roots & roots)
}
}
+ else if (type == DT_REG) {
+ Path storePath = settings.nixStore + "/" + baseNameOf(path);
+ if (store.isValidPath(storePath))
+ roots[path] = storePath;
+ }
+
}
catch (SysError & e) {
@@ -345,9 +328,10 @@ Roots LocalStore::findRoots()
Roots roots;
/* Process direct roots in {gcroots,manifests,profiles}. */
- nix::findRoots(*this, settings.nixStateDir + "/" + gcRootsDir, roots);
- nix::findRoots(*this, settings.nixStateDir + "/manifests", roots);
- nix::findRoots(*this, settings.nixStateDir + "/profiles", roots);
+ nix::findRoots(*this, settings.nixStateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
+ if (pathExists(settings.nixStateDir + "/manifests"))
+ nix::findRoots(*this, settings.nixStateDir + "/manifests", DT_UNKNOWN, roots);
+ nix::findRoots(*this, settings.nixStateDir + "/profiles", DT_UNKNOWN, roots);
return roots;
}
@@ -449,7 +433,6 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
// if the path was not valid, need to determine the actual
// size.
state.bytesInvalidated += size;
- // Mac OS X cannot rename directories if they are read-only.
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError(format("making `%1%' writable") % path);
Path tmp = state.trashDir + "/" + baseNameOf(path);
@@ -649,7 +632,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
/* After this point the set of roots or temporary roots cannot
increase, since we hold locks on everything. So everything
- that is not reachable from `roots'. */
+ that is not reachable from `roots' is garbage. */
if (state.shouldDelete) {
if (pathExists(state.trashDir)) deleteGarbage(state, state.trashDir);
@@ -741,7 +724,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
}
/* While we're at it, vacuum the database. */
- if (options.action == GCOptions::gcDeleteDead) vacuumDB();
+ //if (options.action == GCOptions::gcDeleteDead) vacuumDB();
}
diff --git a/nix/libstore/globals.cc b/nix/libstore/globals.cc
index 86fa56739c..bb08a7d0b0 100644
--- a/nix/libstore/globals.cc
+++ b/nix/libstore/globals.cc
@@ -2,6 +2,7 @@
#include "globals.hh"
#include "util.hh"
+#include "archive.hh"
#include <map>
#include <algorithm>
@@ -55,6 +56,7 @@ Settings::Settings()
envKeepDerivations = false;
lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
showTrace = false;
+ enableImportNative = false;
}
@@ -112,35 +114,61 @@ void Settings::set(const string & name, const string & value)
}
+string Settings::get(const string & name, const string & def)
+{
+ auto i = settings.find(name);
+ if (i == settings.end()) return def;
+ return i->second;
+}
+
+
+Strings Settings::get(const string & name, const Strings & def)
+{
+ auto i = settings.find(name);
+ if (i == settings.end()) return def;
+ return tokenizeString<Strings>(i->second);
+}
+
+
+bool Settings::get(const string & name, bool def)
+{
+ bool res = def;
+ _get(res, name);
+ return res;
+}
+
+
void Settings::update()
{
- get(tryFallback, "build-fallback");
- get(maxBuildJobs, "build-max-jobs");
- get(buildCores, "build-cores");
- get(thisSystem, "system");
- get(maxSilentTime, "build-max-silent-time");
- get(buildTimeout, "build-timeout");
- get(reservedSize, "gc-reserved-space");
- get(fsyncMetadata, "fsync-metadata");
- get(useSQLiteWAL, "use-sqlite-wal");
- get(syncBeforeRegistering, "sync-before-registering");
- get(useSubstitutes, "build-use-substitutes");
- get(buildUsersGroup, "build-users-group");
- get(useChroot, "build-use-chroot");
- get(dirsInChroot, "build-chroot-dirs");
- get(impersonateLinux26, "build-impersonate-linux-26");
- get(keepLog, "build-keep-log");
- get(compressLog, "build-compress-log");
- get(maxLogSize, "build-max-log-size");
- get(cacheFailure, "build-cache-failure");
- get(pollInterval, "build-poll-interval");
- get(checkRootReachability, "gc-check-reachability");
- get(gcKeepOutputs, "gc-keep-outputs");
- get(gcKeepDerivations, "gc-keep-derivations");
- get(autoOptimiseStore, "auto-optimise-store");
- get(envKeepDerivations, "env-keep-derivations");
- get(sshSubstituterHosts, "ssh-substituter-hosts");
- get(useSshSubstituter, "use-ssh-substituter");
+ _get(tryFallback, "build-fallback");
+ _get(maxBuildJobs, "build-max-jobs");
+ _get(buildCores, "build-cores");
+ _get(thisSystem, "system");
+ _get(maxSilentTime, "build-max-silent-time");
+ _get(buildTimeout, "build-timeout");
+ _get(reservedSize, "gc-reserved-space");
+ _get(fsyncMetadata, "fsync-metadata");
+ _get(useSQLiteWAL, "use-sqlite-wal");
+ _get(syncBeforeRegistering, "sync-before-registering");
+ _get(useSubstitutes, "build-use-substitutes");
+ _get(buildUsersGroup, "build-users-group");
+ _get(useChroot, "build-use-chroot");
+ _get(impersonateLinux26, "build-impersonate-linux-26");
+ _get(keepLog, "build-keep-log");
+ _get(compressLog, "build-compress-log");
+ _get(maxLogSize, "build-max-log-size");
+ _get(cacheFailure, "build-cache-failure");
+ _get(pollInterval, "build-poll-interval");
+ _get(checkRootReachability, "gc-check-reachability");
+ _get(gcKeepOutputs, "gc-keep-outputs");
+ _get(gcKeepDerivations, "gc-keep-derivations");
+ _get(autoOptimiseStore, "auto-optimise-store");
+ _get(envKeepDerivations, "env-keep-derivations");
+ _get(sshSubstituterHosts, "ssh-substituter-hosts");
+ _get(useSshSubstituter, "use-ssh-substituter");
+ _get(logServers, "log-servers");
+ _get(enableImportNative, "allow-unsafe-native-code-during-evaluation");
+ _get(useCaseHack, "use-case-hack");
string subs = getEnv("NIX_SUBSTITUTERS", "default");
if (subs == "default") {
@@ -158,7 +186,7 @@ void Settings::update()
}
-void Settings::get(string & res, const string & name)
+void Settings::_get(string & res, const string & name)
{
SettingsMap::iterator i = settings.find(name);
if (i == settings.end()) return;
@@ -166,7 +194,7 @@ void Settings::get(string & res, const string & name)
}
-void Settings::get(bool & res, const string & name)
+void Settings::_get(bool & res, const string & name)
{
SettingsMap::iterator i = settings.find(name);
if (i == settings.end()) return;
@@ -177,7 +205,7 @@ void Settings::get(bool & res, const string & name)
}
-void Settings::get(StringSet & res, const string & name)
+void Settings::_get(StringSet & res, const string & name)
{
SettingsMap::iterator i = settings.find(name);
if (i == settings.end()) return;
@@ -186,7 +214,7 @@ void Settings::get(StringSet & res, const string & name)
res.insert(ss.begin(), ss.end());
}
-void Settings::get(Strings & res, const string & name)
+void Settings::_get(Strings & res, const string & name)
{
SettingsMap::iterator i = settings.find(name);
if (i == settings.end()) return;
@@ -194,7 +222,7 @@ void Settings::get(Strings & res, const string & name)
}
-template<class N> void Settings::get(N & res, const string & name)
+template<class N> void Settings::_get(N & res, const string & name)
{
SettingsMap::iterator i = settings.find(name);
if (i == settings.end()) return;
diff --git a/nix/libstore/globals.hh b/nix/libstore/globals.hh
index 711c365294..c17e10d7c3 100644
--- a/nix/libstore/globals.hh
+++ b/nix/libstore/globals.hh
@@ -21,6 +21,12 @@ struct Settings {
void set(const string & name, const string & value);
+ string get(const string & name, const string & def);
+
+ Strings get(const string & name, const Strings & def);
+
+ bool get(const string & name, bool def);
+
void update();
string pack();
@@ -142,10 +148,6 @@ struct Settings {
/* Whether to build in chroot. */
bool useChroot;
- /* The directories from the host filesystem to be included in the
- chroot. */
- StringSet dirsInChroot;
-
/* Set of ssh connection strings for the ssh substituter */
Strings sshSubstituterHosts;
@@ -197,14 +199,20 @@ struct Settings {
/* Whether to show a stack trace if Nix evaluation fails. */
bool showTrace;
+ /* A list of URL prefixes that can return Nix build logs. */
+ Strings logServers;
+
+ /* Whether the importNative primop should be enabled */
+ bool enableImportNative;
+
private:
SettingsMap settings, overrides;
- void get(string & res, const string & name);
- void get(bool & res, const string & name);
- void get(StringSet & res, const string & name);
- void get(Strings & res, const string & name);
- template<class N> void get(N & res, const string & name);
+ void _get(string & res, const string & name);
+ void _get(bool & res, const string & name);
+ void _get(StringSet & res, const string & name);
+ void _get(Strings & res, const string & name);
+ template<class N> void _get(N & res, const string & name);
};
diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc
index 2c3d65215c..a890ab56b3 100644
--- a/nix/libstore/local-store.cc
+++ b/nix/libstore/local-store.cc
@@ -358,7 +358,17 @@ LocalStore::~LocalStore()
i->second.to.close();
i->second.from.close();
i->second.error.close();
- i->second.pid.wait(true);
+ if (i->second.pid != -1)
+ i->second.pid.wait(true);
+ }
+ } catch (...) {
+ ignoreException();
+ }
+
+ try {
+ if (fdTempRoots != -1) {
+ fdTempRoots.close();
+ unlink(fnTempRoots.c_str());
}
} catch (...) {
ignoreException();
@@ -551,9 +561,9 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
- /* Really make sure that the path is of a supported type. This
- has already been checked in dumpPath(). */
- assert(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode));
+ /* Really make sure that the path is of a supported type. */
+ if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
+ throw Error(format("file ‘%1%’ has an unsupported type") % path);
/* Fail if the file is not owned by the build user. This prevents
us from messing up the ownership/permissions of files
@@ -593,9 +603,9 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
}
if (S_ISDIR(st.st_mode)) {
- Strings names = readDirectory(path);
- foreach (Strings::iterator, i, names)
- canonicalisePathMetaData_(path + "/" + *i, fromUid, inodesSeen);
+ DirEntries entries = readDirectory(path);
+ for (auto & i : entries)
+ canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen);
}
}
@@ -1083,31 +1093,16 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
setSubstituterEnv();
- run.pid = maybeVfork();
-
- switch (run.pid) {
-
- case -1:
- throw SysError("unable to fork");
-
- case 0: /* child */
- try {
- restoreAffinity();
- if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
- throw SysError("dupping stdin");
- if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
- throw SysError("dupping stdout");
- if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
- throw SysError("dupping stderr");
- execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
- throw SysError(format("executing `%1%'") % substituter);
- } catch (std::exception & e) {
- std::cerr << "error: " << e.what() << std::endl;
- }
- _exit(1);
- }
-
- /* Parent. */
+ run.pid = startProcess([&]() {
+ if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
+ throw SysError("dupping stdin");
+ if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
+ throw SysError("dupping stdout");
+ if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
+ throw SysError("dupping stderr");
+ execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
+ throw SysError(format("executing `%1%'") % substituter);
+ });
run.program = baseNameOf(substituter);
run.to = toPipe.writeSide.borrow();
@@ -1170,8 +1165,7 @@ string LocalStore::getLineFromSubstituter(RunningSubstituter & run)
string::size_type p;
while (((p = err.find('\n')) != string::npos)
|| ((p = err.find('\r')) != string::npos)) {
- string thing(err, 0, p + 1);
- writeToStderr(run.program + ": " + thing);
+ printMsg(lvlError, run.program + ": " + string(err, 0, p));
err = string(err, p + 1);
}
}
@@ -1503,7 +1497,8 @@ void LocalStore::exportPath(const Path & path, bool sign,
{
assertStorePath(path);
- addTempRoot(path);
+ printMsg(lvlInfo, format("exporting path `%1%'") % path);
+
if (!isValidPath(path))
throw Error(format("path `%1%' is not valid") % path);
@@ -1613,8 +1608,6 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
Path dstPath = readStorePath(hashAndReadSource);
- printMsg(lvlInfo, format("importing path `%1%'") % dstPath);
-
PathSet references = readStorePaths<PathSet>(hashAndReadSource);
Path deriver = readString(hashAndReadSource);
@@ -1747,8 +1740,8 @@ bool LocalStore::verifyStore(bool checkContents, bool repair)
/* Acquire the global GC lock to prevent a garbage collection. */
AutoCloseFD fdGCLock = openGCLock(ltWrite);
- Paths entries = readDirectory(settings.nixStore);
- PathSet store(entries.begin(), entries.end());
+ PathSet store;
+ for (auto & i : readDirectory(settings.nixStore)) store.insert(i.name);
/* Check whether all valid paths actually exist. */
printMsg(lvlInfo, "checking path existence...");
@@ -1898,9 +1891,8 @@ void LocalStore::markContentsGood(const Path & path)
PathSet LocalStore::queryValidPathsOld()
{
PathSet paths;
- Strings entries = readDirectory(settings.nixDBPath + "/info");
- foreach (Strings::iterator, i, entries)
- if (i->at(0) != '.') paths.insert(settings.nixStore + "/" + *i);
+ for (auto & i : readDirectory(settings.nixDBPath + "/info"))
+ if (i.name.at(0) != '.') paths.insert(settings.nixStore + "/" + i.name);
return paths;
}
@@ -1987,9 +1979,8 @@ static void makeMutable(const Path & path)
if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return;
if (S_ISDIR(st.st_mode)) {
- Strings names = readDirectory(path);
- foreach (Strings::iterator, i, names)
- makeMutable(path + "/" + *i);
+ for (auto & i : readDirectory(path))
+ makeMutable(path + "/" + i.name);
}
/* The O_NOFOLLOW is important to prevent us from changing the
diff --git a/nix/libstore/local-store.hh b/nix/libstore/local-store.hh
index 54331e448a..e0aabdba42 100644
--- a/nix/libstore/local-store.hh
+++ b/nix/libstore/local-store.hh
@@ -1,16 +1,12 @@
#pragma once
#include <string>
+#include <unordered_set>
#include "store-api.hh"
#include "util.hh"
#include "pathlocks.hh"
-#if HAVE_TR1_UNORDERED_SET
-#include <tr1/unordered_set>
-#endif
-
-
class sqlite3;
class sqlite3_stmt;
@@ -171,6 +167,9 @@ public:
files with the same contents. */
void optimiseStore(OptimiseStats & stats);
+ /* Generic variant of the above method. */
+ void optimiseStore();
+
/* Optimise a single store path. */
void optimisePath(const Path & path);
@@ -245,6 +244,10 @@ private:
bool didSetSubstituterEnv;
+ /* The file to which we write our temporary roots. */
+ Path fnTempRoots;
+ AutoCloseFD fdTempRoots;
+
int getSchema();
void openDB(bool create);
@@ -306,11 +309,7 @@ private:
void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
-#if HAVE_TR1_UNORDERED_SET
- typedef std::tr1::unordered_set<ino_t> InodeHash;
-#else
- typedef std::set<ino_t> InodeHash;
-#endif
+ typedef std::unordered_set<ino_t> InodeHash;
InodeHash loadInodeHash();
Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
diff --git a/nix/libstore/optimise-store.cc b/nix/libstore/optimise-store.cc
index 67ee94a4bd..8ba9d1a263 100644
--- a/nix/libstore/optimise-store.cc
+++ b/nix/libstore/optimise-store.cc
@@ -225,6 +225,22 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
}
}
+static string showBytes(unsigned long long bytes)
+{
+ return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
+}
+
+void LocalStore::optimiseStore()
+{
+ OptimiseStats stats;
+
+ optimiseStore(stats);
+
+ printMsg(lvlError,
+ format("%1% freed by hard-linking %2% files")
+ % showBytes(stats.bytesFreed)
+ % stats.filesLinked);
+}
void LocalStore::optimisePath(const Path & path)
{
diff --git a/nix/libstore/remote-store.cc b/nix/libstore/remote-store.cc
index 4619206932..448d9b6bc1 100644
--- a/nix/libstore/remote-store.cc
+++ b/nix/libstore/remote-store.cc
@@ -87,8 +87,7 @@ void RemoteStore::openConnection(bool reserveSpace)
processStderr();
}
catch (Error & e) {
- throw Error(format("cannot start worker (%1%)")
- % e.msg());
+ throw Error(format("cannot start daemon worker: %1%") % e.msg());
}
setOptions();
@@ -133,8 +132,6 @@ RemoteStore::~RemoteStore()
try {
to.flush();
fdSocket.close();
- if (child != -1)
- child.wait(true);
} catch (...) {
ignoreException();
}
@@ -402,8 +399,23 @@ Path RemoteStore::addToStore(const Path & _srcPath,
writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to);
writeInt(recursive ? 1 : 0, to);
writeString(printHashType(hashAlgo), to);
- dumpPath(srcPath, to, filter);
- processStderr();
+
+ try {
+ to.written = 0;
+ to.warn = true;
+ dumpPath(srcPath, to, filter);
+ to.warn = false;
+ processStderr();
+ } catch (SysError & e) {
+ /* Daemon closed while we were sending the path. Probably OOM
+ or I/O error. */
+ if (e.errNo == EPIPE)
+ try {
+ processStderr();
+ } catch (EndOfFile & e) { }
+ throw;
+ }
+
return readStorePath(from);
}
@@ -564,6 +576,13 @@ void RemoteStore::clearFailedPaths(const PathSet & paths)
readInt(from);
}
+void RemoteStore::optimiseStore()
+{
+ openConnection();
+ writeInt(wopOptimiseStore, to);
+ processStderr();
+ readInt(from);
+}
void RemoteStore::processStderr(Sink * sink, Source * source)
{
diff --git a/nix/libstore/remote-store.hh b/nix/libstore/remote-store.hh
index 04b60fce4b..98774c10b3 100644
--- a/nix/libstore/remote-store.hh
+++ b/nix/libstore/remote-store.hh
@@ -82,12 +82,13 @@ public:
PathSet queryFailedPaths();
void clearFailedPaths(const PathSet & paths);
-
+
+ void optimiseStore();
+
private:
AutoCloseFD fdSocket;
FdSink to;
FdSource from;
- Pid child;
unsigned int daemonVersion;
bool initialised;
diff --git a/nix/libstore/store-api.hh b/nix/libstore/store-api.hh
index b635fee2cf..3109f100ef 100644
--- a/nix/libstore/store-api.hh
+++ b/nix/libstore/store-api.hh
@@ -250,6 +250,10 @@ public:
`nix-store --register-validity'. */
string makeValidityRegistration(const PathSet & paths,
bool showDerivers, bool showHash);
+
+ /* Optimise the disk space usage of the Nix store by hard-linking files
+ with the same contents. */
+ virtual void optimiseStore() = 0;
};
diff --git a/nix/libstore/worker-protocol.hh b/nix/libstore/worker-protocol.hh
index 9317f89c37..4b040b77ce 100644
--- a/nix/libstore/worker-protocol.hh
+++ b/nix/libstore/worker-protocol.hh
@@ -12,7 +12,6 @@ namespace nix {
typedef enum {
- wopQuit = 0,
wopIsValidPath = 1,
wopHasSubstitutes = 3,
wopQueryPathHash = 4,
@@ -43,6 +42,7 @@ typedef enum {
wopQueryValidPaths = 31,
wopQuerySubstitutablePaths = 32,
wopQueryValidDerivers = 33,
+ wopOptimiseStore = 34
} WorkerOp;
generated by cgit v1.2.3 (git 2.45.0) at 2024-11-18 13:20:48 +0000