Merge branch 'master' into core-updates

6 files changed, 392 insertions, 413 deletions

diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc
index 347e8a703f..882bce1f40 100644
--- a/nix/libstore/local-store.cc
+++ b/nix/libstore/local-store.cc
@@ -40,171 +40,6 @@
 namespace nix {
 
 
-MakeError(SQLiteError, Error);
-MakeError(SQLiteBusy, SQLiteError);
-
-
-static void throwSQLiteError(sqlite3 * db, const format & f)
-    __attribute__ ((noreturn));
-
-static void throwSQLiteError(sqlite3 * db, const format & f)
-{
-    int err = sqlite3_errcode(db);
-    if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) {
-        if (err == SQLITE_PROTOCOL)
-            printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)");
-        else {
-            static bool warned = false;
-            if (!warned) {
-                printMsg(lvlError, "warning: SQLite database is busy");
-                warned = true;
-            }
-        }
-        /* Sleep for a while since retrying the transaction right away
-           is likely to fail again. */
-#if HAVE_NANOSLEEP
-        struct timespec t;
-        t.tv_sec = 0;
-        t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */
-        nanosleep(&t, 0);
-#else
-        sleep(1);
-#endif
-        throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db));
-    }
-    else
-        throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db));
-}
-
-
-/* Convenience macros for retrying a SQLite transaction. */
-#define retry_sqlite while (1) { try {
-#define end_retry_sqlite break; } catch (SQLiteBusy & e) { } }
-
-
-SQLite::~SQLite()
-{
-    try {
-        if (db && sqlite3_close(db) != SQLITE_OK)
-            throwSQLiteError(db, "closing database");
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-void SQLiteStmt::create(sqlite3 * db, const string & s)
-{
-    checkInterrupt();
-    assert(!stmt);
-    if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK)
-        throwSQLiteError(db, "creating statement");
-    this->db = db;
-}
-
-
-void SQLiteStmt::reset()
-{
-    assert(stmt);
-    /* Note: sqlite3_reset() returns the error code for the most
-       recent call to sqlite3_step().  So ignore it. */
-    sqlite3_reset(stmt);
-    curArg = 1;
-}
-
-
-SQLiteStmt::~SQLiteStmt()
-{
-    try {
-        if (stmt && sqlite3_finalize(stmt) != SQLITE_OK)
-            throwSQLiteError(db, "finalizing statement");
-    } catch (...) {
-        ignoreException();
-    }
-}
-
-
-void SQLiteStmt::bind(const string & value)
-{
-    if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
-        throwSQLiteError(db, "binding argument");
-}
-
-
-void SQLiteStmt::bind(int value)
-{
-    if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK)
-        throwSQLiteError(db, "binding argument");
-}
-
-
-void SQLiteStmt::bind64(long long value)
-{
-    if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK)
-        throwSQLiteError(db, "binding argument");
-}
-
-
-void SQLiteStmt::bind()
-{
-    if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK)
-        throwSQLiteError(db, "binding argument");
-}
-
-
-/* Helper class to ensure that prepared statements are reset when
-   leaving the scope that uses them.  Unfinished prepared statements
-   prevent transactions from being aborted, and can cause locks to be
-   kept when they should be released. */
-struct SQLiteStmtUse
-{
-    SQLiteStmt & stmt;
-    SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt)
-    {
-        stmt.reset();
-    }
-    ~SQLiteStmtUse()
-    {
-        try {
-            stmt.reset();
-        } catch (...) {
-            ignoreException();
-        }
-    }
-};
-
-
-struct SQLiteTxn
-{
-    bool active;
-    sqlite3 * db;
-
-    SQLiteTxn(sqlite3 * db) : active(false) {
-        this->db = db;
-        if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK)
-            throwSQLiteError(db, "starting transaction");
-        active = true;
-    }
-
-    void commit()
-    {
-        if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK)
-            throwSQLiteError(db, "committing transaction");
-        active = false;
-    }
-
-    ~SQLiteTxn()
-    {
-        try {
-            if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK)
-                throwSQLiteError(db, "aborting transaction");
-        } catch (...) {
-            ignoreException();
-        }
-    }
-};
-
-
 void checkStoreNotSymlink()
 {
     if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return;
@@ -494,6 +329,7 @@ void LocalStore::openDB(bool create)
     // ensure efficient lookup.
     stmtQueryPathFromHashPart.create(db,
         "select path from ValidPaths where path >= ? limit 1;");
+    stmtQueryValidPaths.create(db, "select path from ValidPaths");
 }
 
 
@@ -688,23 +524,16 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation &
 }
 
 
-unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
+uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
 {
-    SQLiteStmtUse use(stmtRegisterValidPath);
-    stmtRegisterValidPath.bind(info.path);
-    stmtRegisterValidPath.bind("sha256:" + printHash(info.hash));
-    stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime);
-    if (info.deriver != "")
-        stmtRegisterValidPath.bind(info.deriver);
-    else
-        stmtRegisterValidPath.bind(); // null
-    if (info.narSize != 0)
-        stmtRegisterValidPath.bind64(info.narSize);
-    else
-        stmtRegisterValidPath.bind(); // null
-    if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE)
-        throwSQLiteError(db, format("registering valid path `%1%' in database") % info.path);
-    unsigned long long id = sqlite3_last_insert_rowid(db);
+    stmtRegisterValidPath.use()
+        (info.path)
+        ("sha256:" + printHash(info.hash))
+        (info.registrationTime == 0 ? time(0) : info.registrationTime)
+        (info.deriver, info.deriver != "")
+        (info.narSize, info.narSize != 0)
+        .exec();
+    uint64_t id = sqlite3_last_insert_rowid(db);
 
     /* If this is a derivation, then store the derivation outputs in
        the database.  This is useful for the garbage collector: it can
@@ -720,13 +549,12 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che
            registration above is undone. */
         if (checkOutputs) checkDerivationOutputs(info.path, drv);
 
-        foreach (DerivationOutputs::iterator, i, drv.outputs) {
-            SQLiteStmtUse use(stmtAddDerivationOutput);
-            stmtAddDerivationOutput.bind(id);
-            stmtAddDerivationOutput.bind(i->first);
-            stmtAddDerivationOutput.bind(i->second.path);
-            if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE)
-                throwSQLiteError(db, format("adding derivation output for `%1%' in database") % info.path);
+        for (auto & i : drv.outputs) {
+            stmtAddDerivationOutput.use()
+                (id)
+                (i.first)
+                (i.second.path)
+                .exec();
         }
     }
 
@@ -734,76 +562,52 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che
 }
 
 
-void LocalStore::addReference(unsigned long long referrer, unsigned long long reference)
+void LocalStore::addReference(uint64_t referrer, uint64_t reference)
 {
-    SQLiteStmtUse use(stmtAddReference);
-    stmtAddReference.bind(referrer);
-    stmtAddReference.bind(reference);
-    if (sqlite3_step(stmtAddReference) != SQLITE_DONE)
-        throwSQLiteError(db, "adding reference to database");
+    stmtAddReference.use()(referrer)(reference).exec();
 }
 
 
 void LocalStore::registerFailedPath(const Path & path)
 {
-    retry_sqlite {
-        SQLiteStmtUse use(stmtRegisterFailedPath);
-        stmtRegisterFailedPath.bind(path);
-        stmtRegisterFailedPath.bind(time(0));
-        if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE)
-            throwSQLiteError(db, format("registering failed path `%1%'") % path);
-    } end_retry_sqlite;
+    retrySQLite<void>([&]() {
+        stmtRegisterFailedPath.use()(path)(time(0)).step();
+    });
 }
 
 
 bool LocalStore::hasPathFailed(const Path & path)
 {
-    retry_sqlite {
-        SQLiteStmtUse use(stmtHasPathFailed);
-        stmtHasPathFailed.bind(path);
-        int res = sqlite3_step(stmtHasPathFailed);
-        if (res != SQLITE_DONE && res != SQLITE_ROW)
-            throwSQLiteError(db, "querying whether path failed");
-        return res == SQLITE_ROW;
-    } end_retry_sqlite;
+    return retrySQLite<bool>([&]() {
+        return stmtHasPathFailed.use()(path).next();
+    });
 }
 
 
 PathSet LocalStore::queryFailedPaths()
 {
-    retry_sqlite {
-        SQLiteStmtUse use(stmtQueryFailedPaths);
+    return retrySQLite<PathSet>([&]() {
+        auto useQueryFailedPaths(stmtQueryFailedPaths.use());
 
         PathSet res;
-        int r;
-        while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) {
-            const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0);
-            assert(s);
-            res.insert(s);
-        }
-
-        if (r != SQLITE_DONE)
-            throwSQLiteError(db, "error querying failed paths");
+        while (useQueryFailedPaths.next())
+            res.insert(useQueryFailedPaths.getStr(0));
 
         return res;
-    } end_retry_sqlite;
+    });
 }
 
 
 void LocalStore::clearFailedPaths(const PathSet & paths)
 {
-    retry_sqlite {
+    retrySQLite<void>([&]() {
         SQLiteTxn txn(db);
 
-        foreach (PathSet::const_iterator, i, paths) {
-            SQLiteStmtUse use(stmtClearFailedPath);
-            stmtClearFailedPath.bind(*i);
-            if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE)
-                throwSQLiteError(db, format("clearing failed path `%1%' in database") % *i);
-        }
+        for (auto & path : paths)
+            stmtClearFailedPath.use()(path).exec();
 
         txn.commit();
-    } end_retry_sqlite;
+    });
 }
 
 
@@ -828,47 +632,34 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
 
     assertStorePath(path);
 
-    retry_sqlite {
+    return retrySQLite<ValidPathInfo>([&]() {
 
         /* Get the path info. */
-        SQLiteStmtUse use1(stmtQueryPathInfo);
+        auto useQueryPathInfo(stmtQueryPathInfo.use()(path));
 
-        stmtQueryPathInfo.bind(path);
+        if (!useQueryPathInfo.next())
+            throw Error(format("path `%1%' is not valid") % path);
 
-        int r = sqlite3_step(stmtQueryPathInfo);
-        if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path);
-        if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database");
+        info.id = useQueryPathInfo.getInt(0);
 
-        info.id = sqlite3_column_int(stmtQueryPathInfo, 0);
+        info.hash = parseHashField(path, useQueryPathInfo.getStr(1));
 
-        const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1);
-        assert(s);
-        info.hash = parseHashField(path, s);
+        info.registrationTime = useQueryPathInfo.getInt(2);
 
-        info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2);
-
-        s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3);
+        auto s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3);
         if (s) info.deriver = s;
 
         /* Note that narSize = NULL yields 0. */
-        info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4);
+        info.narSize = useQueryPathInfo.getInt(4);
 
         /* Get the references. */
-        SQLiteStmtUse use2(stmtQueryReferences);
-
-        stmtQueryReferences.bind(info.id);
-
-        while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) {
-            s = (const char *) sqlite3_column_text(stmtQueryReferences, 0);
-            assert(s);
-            info.references.insert(s);
-        }
+        auto useQueryReferences(stmtQueryReferences.use()(info.id));
 
-        if (r != SQLITE_DONE)
-            throwSQLiteError(db, format("error getting references of `%1%'") % path);
+        while (useQueryReferences.next())
+            info.references.insert(useQueryReferences.getStr(0));
 
         return info;
-    } end_retry_sqlite;
+    });
 }
 
 
@@ -876,78 +667,56 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
    narSize field. */
 void LocalStore::updatePathInfo(const ValidPathInfo & info)
 {
-    SQLiteStmtUse use(stmtUpdatePathInfo);
-    if (info.narSize != 0)
-        stmtUpdatePathInfo.bind64(info.narSize);
-    else
-        stmtUpdatePathInfo.bind(); // null
-    stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash));
-    stmtUpdatePathInfo.bind(info.path);
-    if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE)
-        throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path);
+    stmtUpdatePathInfo.use()
+        (info.narSize, info.narSize != 0)
+        ("sha256:" + printHash(info.hash))
+        (info.path)
+        .exec();
 }
 
 
-unsigned long long LocalStore::queryValidPathId(const Path & path)
+uint64_t LocalStore::queryValidPathId(const Path & path)
 {
-    SQLiteStmtUse use(stmtQueryPathInfo);
-    stmtQueryPathInfo.bind(path);
-    int res = sqlite3_step(stmtQueryPathInfo);
-    if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0);
-    if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path);
-    throwSQLiteError(db, "querying path in database");
+    auto use(stmtQueryPathInfo.use()(path));
+    if (!use.next())
+        throw Error(format("path ‘%1%’ is not valid") % path);
+    return use.getInt(0);
 }
 
 
 bool LocalStore::isValidPath_(const Path & path)
 {
-    SQLiteStmtUse use(stmtQueryPathInfo);
-    stmtQueryPathInfo.bind(path);
-    int res = sqlite3_step(stmtQueryPathInfo);
-    if (res != SQLITE_DONE && res != SQLITE_ROW)
-        throwSQLiteError(db, "querying path in database");
-    return res == SQLITE_ROW;
+    return stmtQueryPathInfo.use()(path).next();
 }
 
 
 bool LocalStore::isValidPath(const Path & path)
 {
-    retry_sqlite {
+    return retrySQLite<bool>([&]() {
         return isValidPath_(path);
-    } end_retry_sqlite;
+    });
 }
 
 
 PathSet LocalStore::queryValidPaths(const PathSet & paths)
 {
-    retry_sqlite {
+    return retrySQLite<PathSet>([&]() {
         PathSet res;
         foreach (PathSet::const_iterator, i, paths)
             if (isValidPath_(*i)) res.insert(*i);
         return res;
-    } end_retry_sqlite;
+    });
 }
 
 
 PathSet LocalStore::queryAllValidPaths()
 {
-    retry_sqlite {
-        SQLiteStmt stmt;
-        stmt.create(db, "select path from ValidPaths");
-
+    return retrySQLite<PathSet>([&]() {
+        auto use(stmtQueryValidPaths.use());
         PathSet res;
-        int r;
-        while ((r = sqlite3_step(stmt)) == SQLITE_ROW) {
-            const char * s = (const char *) sqlite3_column_text(stmt, 0);
-            assert(s);
-            res.insert(s);
-        }
-
-        if (r != SQLITE_DONE)
-            throwSQLiteError(db, "error getting valid paths");
-
+        while (use.next()) res.insert(use.getStr(0));
         return res;
-    } end_retry_sqlite;
+    });
 }
 
 
@@ -961,28 +730,19 @@ void LocalStore::queryReferences(const Path & path,
 
 void LocalStore::queryReferrers_(const Path & path, PathSet & referrers)
 {
-    SQLiteStmtUse use(stmtQueryReferrers);
-
-    stmtQueryReferrers.bind(path);
+    auto useQueryReferrers(stmtQueryReferrers.use()(path));
 
-    int r;
-    while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) {
-        const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0);
-        assert(s);
-        referrers.insert(s);
-    }
-
-    if (r != SQLITE_DONE)
-        throwSQLiteError(db, format("error getting references of `%1%'") % path);
+    while (useQueryReferrers.next())
+        referrers.insert(useQueryReferrers.getStr(0));
 }
 
 
 void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
 {
     assertStorePath(path);
-    retry_sqlite {
+    return retrySQLite<void>([&]() {
         queryReferrers_(path, referrers);
-    } end_retry_sqlite;
+    });
 }
 
 
@@ -996,67 +756,43 @@ PathSet LocalStore::queryValidDerivers(const Path & path)
 {
     assertStorePath(path);
 
-    retry_sqlite {
-        SQLiteStmtUse use(stmtQueryValidDerivers);
-        stmtQueryValidDerivers.bind(path);
+    return retrySQLite<PathSet>([&]() {
+        auto useQueryValidDerivers(stmtQueryValidDerivers.use()(path));
 
         PathSet derivers;
-        int r;
-        while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) {
-            const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1);
-            assert(s);
-            derivers.insert(s);
-        }
-
-        if (r != SQLITE_DONE)
-            throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path);
+        while (useQueryValidDerivers.next())
+            derivers.insert(useQueryValidDerivers.getStr(1));
 
         return derivers;
-    } end_retry_sqlite;
+    });
 }
 
 
 PathSet LocalStore::queryDerivationOutputs(const Path & path)
 {
-    retry_sqlite {
-        SQLiteStmtUse use(stmtQueryDerivationOutputs);
-        stmtQueryDerivationOutputs.bind(queryValidPathId(path));
+    return retrySQLite<PathSet>([&]() {
+        auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));
 
         PathSet outputs;
-        int r;
-        while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) {
-            const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1);
-            assert(s);
-            outputs.insert(s);
-        }
-
-        if (r != SQLITE_DONE)
-            throwSQLiteError(db, format("error getting outputs of `%1%'") % path);
+        while (useQueryDerivationOutputs.next())
+            outputs.insert(useQueryDerivationOutputs.getStr(1));
 
         return outputs;
-    } end_retry_sqlite;
+    });
 }
 
 
 StringSet LocalStore::queryDerivationOutputNames(const Path & path)
 {
-    retry_sqlite {
-        SQLiteStmtUse use(stmtQueryDerivationOutputs);
-        stmtQueryDerivationOutputs.bind(queryValidPathId(path));
+    return retrySQLite<StringSet>([&]() {
+        auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));
 
         StringSet outputNames;
-        int r;
-        while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) {
-            const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0);
-            assert(s);
-            outputNames.insert(s);
-        }
-
-        if (r != SQLITE_DONE)
-            throwSQLiteError(db, format("error getting output names of `%1%'") % path);
+        while (useQueryDerivationOutputs.next())
+            outputNames.insert(useQueryDerivationOutputs.getStr(0));
 
         return outputNames;
-    } end_retry_sqlite;
+    });
 }
 
 
@@ -1066,17 +802,14 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart)
 
     Path prefix = settings.nixStore + "/" + hashPart;
 
-    retry_sqlite {
-        SQLiteStmtUse use(stmtQueryPathFromHashPart);
-        stmtQueryPathFromHashPart.bind(prefix);
+    return retrySQLite<Path>([&]() -> Path {
+        auto useQueryPathFromHashPart(stmtQueryPathFromHashPart.use()(prefix));
 
-        int res = sqlite3_step(stmtQueryPathFromHashPart);
-        if (res == SQLITE_DONE) return "";
-        if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database");
+        if (!useQueryPathFromHashPart.next()) return "";
 
         const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0);
         return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
-    } end_retry_sqlite;
+    });
 }
 
 
@@ -1306,7 +1039,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
      * expense of some speed of the path registering operation. */
     if (settings.syncBeforeRegistering) sync();
 
-    retry_sqlite {
+    return retrySQLite<void>([&]() {
         SQLiteTxn txn(db);
         PathSet paths;
 
@@ -1319,10 +1052,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
             paths.insert(i->path);
         }
 
-        foreach (ValidPathInfos::const_iterator, i, infos) {
-            unsigned long long referrer = queryValidPathId(i->path);
-            foreach (PathSet::iterator, j, i->references)
-                addReference(referrer, queryValidPathId(*j));
+        for (auto & i : infos) {
+            auto referrer = queryValidPathId(i.path);
+            for (auto & j : i.references)
+                addReference(referrer, queryValidPathId(j));
         }
 
         /* Check that the derivation outputs are correct.  We can't do
@@ -1343,7 +1076,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         topoSortPaths(*this, paths);
 
         txn.commit();
-    } end_retry_sqlite;
+    });
 }
 
 
@@ -1355,12 +1088,7 @@ void LocalStore::invalidatePath(const Path & path)
 
     drvHashes.erase(path);
 
-    SQLiteStmtUse use(stmtInvalidatePath);
-
-    stmtInvalidatePath.bind(path);
-
-    if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE)
-        throwSQLiteError(db, format("invalidating path `%1%' in database") % path);
+    stmtInvalidatePath.use()(path).exec();
 
     /* Note that the foreign key constraints on the Refs table take
        care of deleting the references entries for `path'. */
@@ -1733,7 +1461,7 @@ void LocalStore::invalidatePathChecked(const Path & path)
 {
     assertStorePath(path);
 
-    retry_sqlite {
+    retrySQLite<void>([&]() {
         SQLiteTxn txn(db);
 
         if (isValidPath_(path)) {
@@ -1746,7 +1474,7 @@ void LocalStore::invalidatePathChecked(const Path & path)
         }
 
         txn.commit();
-    } end_retry_sqlite;
+    });
 }
 
 
diff --git a/nix/libstore/local-store.hh b/nix/libstore/local-store.hh
index 819f59327a..6110468498 100644
--- a/nix/libstore/local-store.hh
+++ b/nix/libstore/local-store.hh
@@ -1,15 +1,12 @@
 #pragma once
 
+#include "sqlite.hh"
 #include <string>
 #include <unordered_set>
 
+#include "pathlocks.hh"
 #include "store-api.hh"
 #include "util.hh"
-#include "pathlocks.hh"
-
-
-class sqlite3;
-class sqlite3_stmt;
 
 
 namespace nix {
@@ -52,34 +49,6 @@ struct RunningSubstituter
 };
 
 
-/* Wrapper object to close the SQLite database automatically. */
-struct SQLite
-{
-    sqlite3 * db;
-    SQLite() { db = 0; }
-    ~SQLite();
-    operator sqlite3 * () { return db; }
-};
-
-
-/* Wrapper object to create and destroy SQLite prepared statements. */
-struct SQLiteStmt
-{
-    sqlite3 * db;
-    sqlite3_stmt * stmt;
-    unsigned int curArg;
-    SQLiteStmt() { stmt = 0; }
-    void create(sqlite3 * db, const string & s);
-    void reset();
-    ~SQLiteStmt();
-    operator sqlite3_stmt * () { return stmt; }
-    void bind(const string & value);
-    void bind(int value);
-    void bind64(long long value);
-    void bind();
-};
-
-
 class LocalStore : public StoreAPI
 {
 private:
@@ -238,6 +207,7 @@ private:
     SQLiteStmt stmtQueryValidDerivers;
     SQLiteStmt stmtQueryDerivationOutputs;
     SQLiteStmt stmtQueryPathFromHashPart;
+    SQLiteStmt stmtQueryValidPaths;
 
     /* Cache for pathContentsGood(). */
     std::map<Path, bool> pathContentsGoodCache;
@@ -254,11 +224,11 @@ private:
 
     void makeStoreWritable();
 
-    unsigned long long queryValidPathId(const Path & path);
+    uint64_t queryValidPathId(const Path & path);
 
-    unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true);
+    uint64_t addValidPath(const ValidPathInfo & info, bool checkOutputs = true);
 
-    void addReference(unsigned long long referrer, unsigned long long reference);
+    void addReference(uint64_t referrer, uint64_t reference);
 
     void appendReferrer(const Path & from, const Path & to, bool lock);
 
diff --git a/nix/libstore/optimise-store.cc b/nix/libstore/optimise-store.cc
index d7508b025e..565c62ca83 100644
--- a/nix/libstore/optimise-store.cc
+++ b/nix/libstore/optimise-store.cc
@@ -148,10 +148,23 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
             inodeHash.insert(st.st_ino);
             return;
         }
-        if (errno != EEXIST)
+
+	switch (errno) {
+	case EEXIST:
+	    /* Fall through if another process created ‘linkPath’ before
+	       we did. */
+	    break;
+
+	case ENOSPC:
+	    /* On ext4, that probably means the directory index is full.  When
+	       that happens, it's fine to ignore it: we just effectively
+	       disable deduplication of this file.  */
+	    printMsg(lvlInfo, format("cannot link `%1%' to `%2%': %m") % linkPath % path);
+	    return;
+
+	default:
             throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
-        /* Fall through if another process created ‘linkPath’ before
-           we did. */
+	}
     }
 
     /* Yes!  We've seen a file with the same contents.  Replace the
@@ -195,8 +208,8 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
                 printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
             return;
         }
-        throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath);
-    }
+	    throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath);
+	}
 
     /* Atomically replace the old file with the new hard link. */
     if (rename(tempLink.c_str(), path.c_str()) == -1) {
diff --git a/nix/libstore/sqlite.cc b/nix/libstore/sqlite.cc
new file mode 100644
index 0000000000..e08c67f40e
--- /dev/null
+++ b/nix/libstore/sqlite.cc
@@ -0,0 +1,166 @@
+#include "sqlite.hh"
+#include "util.hh"
+
+#include <sqlite3.h>
+
+namespace nix {
+
+[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f)
+{
+    int err = sqlite3_errcode(db);
+    if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) {
+        if (err == SQLITE_PROTOCOL)
+            printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)");
+        else {
+            static bool warned = false;
+            if (!warned) {
+                printMsg(lvlError, "warning: SQLite database is busy");
+                warned = true;
+            }
+        }
+        /* Sleep for a while since retrying the transaction right away
+           is likely to fail again. */
+#if HAVE_NANOSLEEP
+        struct timespec t;
+        t.tv_sec = 0;
+        t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */
+        nanosleep(&t, 0);
+#else
+        sleep(1);
+#endif
+        throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db));
+    }
+    else
+        throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db));
+}
+
+SQLite::~SQLite()
+{
+    try {
+        if (db && sqlite3_close(db) != SQLITE_OK)
+            throwSQLiteError(db, "closing database");
+    } catch (...) {
+        ignoreException();
+    }
+}
+
+void SQLiteStmt::create(sqlite3 * db, const string & s)
+{
+    checkInterrupt();
+    assert(!stmt);
+    if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK)
+        throwSQLiteError(db, "creating statement");
+    this->db = db;
+}
+
+SQLiteStmt::~SQLiteStmt()
+{
+    try {
+        if (stmt && sqlite3_finalize(stmt) != SQLITE_OK)
+            throwSQLiteError(db, "finalizing statement");
+    } catch (...) {
+        ignoreException();
+    }
+}
+
+SQLiteStmt::Use::Use(SQLiteStmt & stmt)
+    : stmt(stmt)
+{
+    assert(stmt.stmt);
+    /* Note: sqlite3_reset() returns the error code for the most
+       recent call to sqlite3_step().  So ignore it. */
+    sqlite3_reset(stmt);
+}
+
+SQLiteStmt::Use::~Use()
+{
+    sqlite3_reset(stmt);
+}
+
+SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull)
+{
+    if (notNull) {
+        if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
+            throwSQLiteError(stmt.db, "binding argument");
+    } else
+        bind();
+    return *this;
+}
+
+SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
+{
+    if (notNull) {
+        if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK)
+            throwSQLiteError(stmt.db, "binding argument");
+    } else
+        bind();
+    return *this;
+}
+
+SQLiteStmt::Use & SQLiteStmt::Use::bind()
+{
+    if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK)
+        throwSQLiteError(stmt.db, "binding argument");
+    return *this;
+}
+
+int SQLiteStmt::Use::step()
+{
+    return sqlite3_step(stmt);
+}
+
+void SQLiteStmt::Use::exec()
+{
+    int r = step();
+    assert(r != SQLITE_ROW);
+    if (r != SQLITE_DONE)
+        throwSQLiteError(stmt.db, "executing SQLite statement");
+}
+
+bool SQLiteStmt::Use::next()
+{
+    int r = step();
+    if (r != SQLITE_DONE && r != SQLITE_ROW)
+        throwSQLiteError(stmt.db, "executing SQLite query");
+    return r == SQLITE_ROW;
+}
+
+std::string SQLiteStmt::Use::getStr(int col)
+{
+    auto s = (const char *) sqlite3_column_text(stmt, col);
+    assert(s);
+    return s;
+}
+
+int64_t SQLiteStmt::Use::getInt(int col)
+{
+    // FIXME: detect nulls?
+    return sqlite3_column_int64(stmt, col);
+}
+
+SQLiteTxn::SQLiteTxn(sqlite3 * db)
+{
+    this->db = db;
+    if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK)
+        throwSQLiteError(db, "starting transaction");
+    active = true;
+}
+
+void SQLiteTxn::commit()
+{
+    if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK)
+        throwSQLiteError(db, "committing transaction");
+    active = false;
+}
+
+SQLiteTxn::~SQLiteTxn()
+{
+    try {
+        if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK)
+            throwSQLiteError(db, "aborting transaction");
+    } catch (...) {
+        ignoreException();
+    }
+}
+
+}
diff --git a/nix/libstore/sqlite.hh b/nix/libstore/sqlite.hh
new file mode 100644
index 0000000000..326e4a4855
--- /dev/null
+++ b/nix/libstore/sqlite.hh
@@ -0,0 +1,102 @@
+#pragma once
+
+#include <functional>
+#include <string>
+
+#include "types.hh"
+
+class sqlite3;
+class sqlite3_stmt;
+
+namespace nix {
+
+/* RAII wrapper to close a SQLite database automatically. */
+struct SQLite
+{
+    sqlite3 * db;
+    SQLite() { db = 0; }
+    ~SQLite();
+    operator sqlite3 * () { return db; }
+};
+
+/* RAII wrapper to create and destroy SQLite prepared statements. */
+struct SQLiteStmt
+{
+    sqlite3 * db = 0;
+    sqlite3_stmt * stmt = 0;
+    SQLiteStmt() { }
+    void create(sqlite3 * db, const std::string & s);
+    ~SQLiteStmt();
+    operator sqlite3_stmt * () { return stmt; }
+
+    /* Helper for binding / executing statements. */
+    class Use
+    {
+        friend struct SQLiteStmt;
+    private:
+        SQLiteStmt & stmt;
+        unsigned int curArg = 1;
+        Use(SQLiteStmt & stmt);
+
+    public:
+
+        ~Use();
+
+        /* Bind the next parameter. */
+        Use & operator () (const std::string & value, bool notNull = true);
+        Use & operator () (int64_t value, bool notNull = true);
+        Use & bind(); // null
+
+        int step();
+
+        /* Execute a statement that does not return rows. */
+        void exec();
+
+        /* For statements that return 0 or more rows. Returns true iff
+           a row is available. */
+        bool next();
+
+        std::string getStr(int col);
+        int64_t getInt(int col);
+    };
+
+    Use use()
+    {
+        return Use(*this);
+    }
+};
+
+/* RAII helper that ensures transactions are aborted unless explicitly
+   committed. */
+struct SQLiteTxn
+{
+    bool active = false;
+    sqlite3 * db;
+
+    SQLiteTxn(sqlite3 * db);
+
+    void commit();
+
+    ~SQLiteTxn();
+};
+
+
+MakeError(SQLiteError, Error);
+MakeError(SQLiteBusy, SQLiteError);
+
+[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f);
+
+/* Convenience function for retrying a SQLite transaction when the
+   database is busy. */
+template<typename T>
+T retrySQLite(std::function<T()> fun)
+{
+    while (true) {
+        try {
+            return fun();
+        } catch (SQLiteBusy & e) {
+        }
+    }
+}
+
+}
diff --git a/nix/libstore/store-api.hh b/nix/libstore/store-api.hh
index 3e982f6dd3..fa78d595f2 100644
--- a/nix/libstore/store-api.hh
+++ b/nix/libstore/store-api.hh
@@ -89,8 +89,8 @@ struct ValidPathInfo
     Hash hash;
     PathSet references;
     time_t registrationTime = 0;
-    unsigned long long narSize = 0; // 0 = unknown
-    unsigned long long id; // internal use only
+    uint64_t narSize = 0; // 0 = unknown
+    uint64_t id; // internal use only
 
     bool operator == (const ValidPathInfo & i) const
     {