From 517ce0c15bdf802b3c378fff48b1e5be4167a4fa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 30 Mar 2016 12:04:27 +0200 Subject: daemon: Turn retrying SQLite transactions into a higher-order function. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * nix/libstore/local-store.cc (retry_sqlite, end_retry_sqlite): Remove. (retrySQLite): New template. (LocalStore::registerFailedPath, LocalStore::hasPathFailed) (LocalStore::queryFailedPaths, LocalStore::clearFailedPaths) (LocalStore::queryPathInfo, LocalStore::isValidPath_) (LocalStore::queryValidPaths, LocalStore::queryAllValidPaths) (LocalStore::queryReferrers, LocalStore::queryValidDerivers) (LocalStore::queryDerivationOutputs) (LocalStore::queryDerivationOutputNames) (LocalStore::queryPathFromHashPart, LocalStore::registerValidPaths) (LocalStore::invalidatePathChecked): Use it. Co-authored-by: Ludovic Courtès --- nix/libstore/local-store.cc | 75 +++++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 33 deletions(-) (limited to 'nix/libstore') diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc index 347e8a703f..9fdd094cbf 100644 --- a/nix/libstore/local-store.cc +++ b/nix/libstore/local-store.cc @@ -77,9 +77,18 @@ static void throwSQLiteError(sqlite3 * db, const format & f) } -/* Convenience macros for retrying a SQLite transaction. */ -#define retry_sqlite while (1) { try { -#define end_retry_sqlite break; } catch (SQLiteBusy & e) { } } +/* Convenience function for retrying a SQLite transaction when the + database is busy. */ +template +T retrySQLite(std::function fun) +{ + while (true) { + try { + return fun(); + } catch (SQLiteBusy & e) { + } + } +} SQLite::~SQLite() @@ -746,32 +755,32 @@ void LocalStore::addReference(unsigned long long referrer, unsigned long long re void LocalStore::registerFailedPath(const Path & path) { - retry_sqlite { + retrySQLite([&]() { SQLiteStmtUse use(stmtRegisterFailedPath); stmtRegisterFailedPath.bind(path); stmtRegisterFailedPath.bind(time(0)); if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) throwSQLiteError(db, format("registering failed path `%1%'") % path); - } end_retry_sqlite; + }); } bool LocalStore::hasPathFailed(const Path & path) { - retry_sqlite { + return retrySQLite([&]() { SQLiteStmtUse use(stmtHasPathFailed); stmtHasPathFailed.bind(path); int res = sqlite3_step(stmtHasPathFailed); if (res != SQLITE_DONE && res != SQLITE_ROW) throwSQLiteError(db, "querying whether path failed"); return res == SQLITE_ROW; - } end_retry_sqlite; + }); } PathSet LocalStore::queryFailedPaths() { - retry_sqlite { + return retrySQLite([&]() { SQLiteStmtUse use(stmtQueryFailedPaths); PathSet res; @@ -786,13 +795,13 @@ PathSet LocalStore::queryFailedPaths() throwSQLiteError(db, "error querying failed paths"); return res; - } end_retry_sqlite; + }); } void LocalStore::clearFailedPaths(const PathSet & paths) { - retry_sqlite { + retrySQLite([&]() { SQLiteTxn txn(db); foreach (PathSet::const_iterator, i, paths) { @@ -803,7 +812,7 @@ void LocalStore::clearFailedPaths(const PathSet & paths) } txn.commit(); - } end_retry_sqlite; + }); } @@ -828,7 +837,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) assertStorePath(path); - retry_sqlite { + return retrySQLite([&]() { /* Get the path info. */ SQLiteStmtUse use1(stmtQueryPathInfo); @@ -868,7 +877,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) throwSQLiteError(db, format("error getting references of `%1%'") % path); return info; - } end_retry_sqlite; + }); } @@ -912,26 +921,26 @@ bool LocalStore::isValidPath_(const Path & path) bool LocalStore::isValidPath(const Path & path) { - retry_sqlite { + return retrySQLite([&]() { return isValidPath_(path); - } end_retry_sqlite; + }); } PathSet LocalStore::queryValidPaths(const PathSet & paths) { - retry_sqlite { + return retrySQLite([&]() { PathSet res; foreach (PathSet::const_iterator, i, paths) if (isValidPath_(*i)) res.insert(*i); return res; - } end_retry_sqlite; + }); } PathSet LocalStore::queryAllValidPaths() { - retry_sqlite { + return retrySQLite([&]() { SQLiteStmt stmt; stmt.create(db, "select path from ValidPaths"); @@ -947,7 +956,7 @@ PathSet LocalStore::queryAllValidPaths() throwSQLiteError(db, "error getting valid paths"); return res; - } end_retry_sqlite; + }); } @@ -980,9 +989,9 @@ void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) void LocalStore::queryReferrers(const Path & path, PathSet & referrers) { assertStorePath(path); - retry_sqlite { + return retrySQLite([&]() { queryReferrers_(path, referrers); - } end_retry_sqlite; + }); } @@ -996,7 +1005,7 @@ PathSet LocalStore::queryValidDerivers(const Path & path) { assertStorePath(path); - retry_sqlite { + return retrySQLite([&]() { SQLiteStmtUse use(stmtQueryValidDerivers); stmtQueryValidDerivers.bind(path); @@ -1012,13 +1021,13 @@ PathSet LocalStore::queryValidDerivers(const Path & path) throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path); return derivers; - } end_retry_sqlite; + }); } PathSet LocalStore::queryDerivationOutputs(const Path & path) { - retry_sqlite { + return retrySQLite([&]() { SQLiteStmtUse use(stmtQueryDerivationOutputs); stmtQueryDerivationOutputs.bind(queryValidPathId(path)); @@ -1034,13 +1043,13 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path) throwSQLiteError(db, format("error getting outputs of `%1%'") % path); return outputs; - } end_retry_sqlite; + }); } StringSet LocalStore::queryDerivationOutputNames(const Path & path) { - retry_sqlite { + return retrySQLite([&]() { SQLiteStmtUse use(stmtQueryDerivationOutputs); stmtQueryDerivationOutputs.bind(queryValidPathId(path)); @@ -1056,7 +1065,7 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path) throwSQLiteError(db, format("error getting output names of `%1%'") % path); return outputNames; - } end_retry_sqlite; + }); } @@ -1066,7 +1075,7 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart) Path prefix = settings.nixStore + "/" + hashPart; - retry_sqlite { + return retrySQLite([&]() -> Path { SQLiteStmtUse use(stmtQueryPathFromHashPart); stmtQueryPathFromHashPart.bind(prefix); @@ -1076,7 +1085,7 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart) const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; - } end_retry_sqlite; + }); } @@ -1306,7 +1315,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) * expense of some speed of the path registering operation. */ if (settings.syncBeforeRegistering) sync(); - retry_sqlite { + return retrySQLite([&]() { SQLiteTxn txn(db); PathSet paths; @@ -1343,7 +1352,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSortPaths(*this, paths); txn.commit(); - } end_retry_sqlite; + }); } @@ -1733,7 +1742,7 @@ void LocalStore::invalidatePathChecked(const Path & path) { assertStorePath(path); - retry_sqlite { + retrySQLite([&]() { SQLiteTxn txn(db); if (isValidPath_(path)) { @@ -1746,7 +1755,7 @@ void LocalStore::invalidatePathChecked(const Path & path) } txn.commit(); - } end_retry_sqlite; + }); } -- cgit v1.2.3 From 7bed5d91dea6ecff565d51a021e6f99718d04f86 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 30 Mar 2016 13:27:25 +0200 Subject: daemon: Factor out SQLite handling. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * nix/libstore/local-store.cc: Move SQLite code to... * nix/libstore/sqlite.cc, nix/libstore/sqlite.hh: ... here. New files. * nix/local.mk (libstore_a_SOURCES): Add sqlite.cc. (libstore_headers): Add sqlite.hh. Co-authored-by: Ludovic Courtès --- nix/libstore/local-store.cc | 174 -------------------------------------------- nix/libstore/local-store.hh | 35 +-------- nix/libstore/sqlite.cc | 139 +++++++++++++++++++++++++++++++++++ nix/libstore/sqlite.hh | 83 +++++++++++++++++++++ nix/local.mk | 4 +- 5 files changed, 227 insertions(+), 208 deletions(-) create mode 100644 nix/libstore/sqlite.cc create mode 100644 nix/libstore/sqlite.hh (limited to 'nix/libstore') diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc index 9fdd094cbf..6d298cda43 100644 --- a/nix/libstore/local-store.cc +++ b/nix/libstore/local-store.cc @@ -40,180 +40,6 @@ namespace nix { -MakeError(SQLiteError, Error); -MakeError(SQLiteBusy, SQLiteError); - - -static void throwSQLiteError(sqlite3 * db, const format & f) - __attribute__ ((noreturn)); - -static void throwSQLiteError(sqlite3 * db, const format & f) -{ - int err = sqlite3_errcode(db); - if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - if (err == SQLITE_PROTOCOL) - printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); - else { - static bool warned = false; - if (!warned) { - printMsg(lvlError, "warning: SQLite database is busy"); - warned = true; - } - } - /* Sleep for a while since retrying the transaction right away - is likely to fail again. */ -#if HAVE_NANOSLEEP - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ - nanosleep(&t, 0); -#else - sleep(1); -#endif - throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); - } - else - throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); -} - - -/* Convenience function for retrying a SQLite transaction when the - database is busy. */ -template -T retrySQLite(std::function fun) -{ - while (true) { - try { - return fun(); - } catch (SQLiteBusy & e) { - } - } -} - - -SQLite::~SQLite() -{ - try { - if (db && sqlite3_close(db) != SQLITE_OK) - throwSQLiteError(db, "closing database"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::create(sqlite3 * db, const string & s) -{ - checkInterrupt(); - assert(!stmt); - if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, "creating statement"); - this->db = db; -} - - -void SQLiteStmt::reset() -{ - assert(stmt); - /* Note: sqlite3_reset() returns the error code for the most - recent call to sqlite3_step(). So ignore it. */ - sqlite3_reset(stmt); - curArg = 1; -} - - -SQLiteStmt::~SQLiteStmt() -{ - try { - if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, "finalizing statement"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::bind(const string & value) -{ - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind(int value) -{ - if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind64(long long value) -{ - if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind() -{ - if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -/* Helper class to ensure that prepared statements are reset when - leaving the scope that uses them. Unfinished prepared statements - prevent transactions from being aborted, and can cause locks to be - kept when they should be released. */ -struct SQLiteStmtUse -{ - SQLiteStmt & stmt; - SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt) - { - stmt.reset(); - } - ~SQLiteStmtUse() - { - try { - stmt.reset(); - } catch (...) { - ignoreException(); - } - } -}; - - -struct SQLiteTxn -{ - bool active; - sqlite3 * db; - - SQLiteTxn(sqlite3 * db) : active(false) { - this->db = db; - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "starting transaction"); - active = true; - } - - void commit() - { - if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "committing transaction"); - active = false; - } - - ~SQLiteTxn() - { - try { - if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "aborting transaction"); - } catch (...) { - ignoreException(); - } - } -}; - - void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; diff --git a/nix/libstore/local-store.hh b/nix/libstore/local-store.hh index 819f59327a..f591589838 100644 --- a/nix/libstore/local-store.hh +++ b/nix/libstore/local-store.hh @@ -1,15 +1,12 @@ #pragma once +#include "sqlite.hh" #include #include +#include "pathlocks.hh" #include "store-api.hh" #include "util.hh" -#include "pathlocks.hh" - - -class sqlite3; -class sqlite3_stmt; namespace nix { @@ -52,34 +49,6 @@ struct RunningSubstituter }; -/* Wrapper object to close the SQLite database automatically. */ -struct SQLite -{ - sqlite3 * db; - SQLite() { db = 0; } - ~SQLite(); - operator sqlite3 * () { return db; } -}; - - -/* Wrapper object to create and destroy SQLite prepared statements. */ -struct SQLiteStmt -{ - sqlite3 * db; - sqlite3_stmt * stmt; - unsigned int curArg; - SQLiteStmt() { stmt = 0; } - void create(sqlite3 * db, const string & s); - void reset(); - ~SQLiteStmt(); - operator sqlite3_stmt * () { return stmt; } - void bind(const string & value); - void bind(int value); - void bind64(long long value); - void bind(); -}; - - class LocalStore : public StoreAPI { private: diff --git a/nix/libstore/sqlite.cc b/nix/libstore/sqlite.cc new file mode 100644 index 0000000000..8646ff5b12 --- /dev/null +++ b/nix/libstore/sqlite.cc @@ -0,0 +1,139 @@ +#include "sqlite.hh" +#include "util.hh" + +#include + +namespace nix { + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f) +{ + int err = sqlite3_errcode(db); + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { + if (err == SQLITE_PROTOCOL) + printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); + else { + static bool warned = false; + if (!warned) { + printMsg(lvlError, "warning: SQLite database is busy"); + warned = true; + } + } + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ +#if HAVE_NANOSLEEP + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ + nanosleep(&t, 0); +#else + sleep(1); +#endif + throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); + } + else + throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); +} + +SQLite::~SQLite() +{ + try { + if (db && sqlite3_close(db) != SQLITE_OK) + throwSQLiteError(db, "closing database"); + } catch (...) { + ignoreException(); + } +} + +void SQLiteStmt::create(sqlite3 * db, const string & s) +{ + checkInterrupt(); + assert(!stmt); + if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) + throwSQLiteError(db, "creating statement"); + this->db = db; +} + +void SQLiteStmt::reset() +{ + assert(stmt); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); + curArg = 1; +} + +SQLiteStmt::~SQLiteStmt() +{ + try { + if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) + throwSQLiteError(db, "finalizing statement"); + } catch (...) { + ignoreException(); + } +} + +void SQLiteStmt::bind(const string & value) +{ + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(db, "binding argument"); +} + +void SQLiteStmt::bind(int value) +{ + if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(db, "binding argument"); +} + +void SQLiteStmt::bind64(long long value) +{ + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(db, "binding argument"); +} + +void SQLiteStmt::bind() +{ + if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) + throwSQLiteError(db, "binding argument"); +} + +SQLiteStmtUse::SQLiteStmtUse(SQLiteStmt & stmt) + : stmt(stmt) +{ + stmt.reset(); +} + +SQLiteStmtUse::~SQLiteStmtUse() +{ + try { + stmt.reset(); + } catch (...) { + ignoreException(); + } +} + +SQLiteTxn::SQLiteTxn(sqlite3 * db) +{ + this->db = db; + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "starting transaction"); + active = true; +} + +void SQLiteTxn::commit() +{ + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "committing transaction"); + active = false; +} + +SQLiteTxn::~SQLiteTxn() +{ + try { + if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "aborting transaction"); + } catch (...) { + ignoreException(); + } +} + +} diff --git a/nix/libstore/sqlite.hh b/nix/libstore/sqlite.hh new file mode 100644 index 0000000000..0abdb74631 --- /dev/null +++ b/nix/libstore/sqlite.hh @@ -0,0 +1,83 @@ +#pragma once + +#include +#include + +#include "types.hh" + +class sqlite3; +class sqlite3_stmt; + +namespace nix { + +/* RAII wrapper to close a SQLite database automatically. */ +struct SQLite +{ + sqlite3 * db; + SQLite() { db = 0; } + ~SQLite(); + operator sqlite3 * () { return db; } +}; + +/* RAII wrapper to create and destroy SQLite prepared statements. */ +struct SQLiteStmt +{ + sqlite3 * db; + sqlite3_stmt * stmt; + unsigned int curArg; + SQLiteStmt() { stmt = 0; } + void create(sqlite3 * db, const std::string & s); + void reset(); + ~SQLiteStmt(); + operator sqlite3_stmt * () { return stmt; } + void bind(const std::string & value); + void bind(int value); + void bind64(long long value); + void bind(); +}; + +/* Helper class to ensure that prepared statements are reset when + leaving the scope that uses them. Unfinished prepared statements + prevent transactions from being aborted, and can cause locks to be + kept when they should be released. */ +struct SQLiteStmtUse +{ + SQLiteStmt & stmt; + SQLiteStmtUse(SQLiteStmt & stmt); + ~SQLiteStmtUse(); +}; + +/* RAII helper that ensures transactions are aborted unless explicitly + committed. */ +struct SQLiteTxn +{ + bool active = false; + sqlite3 * db; + + SQLiteTxn(sqlite3 * db); + + void commit(); + + ~SQLiteTxn(); +}; + + +MakeError(SQLiteError, Error); +MakeError(SQLiteBusy, SQLiteError); + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f); + +/* Convenience function for retrying a SQLite transaction when the + database is busy. */ +template +T retrySQLite(std::function fun) +{ + while (true) { + try { + return fun(); + } catch (SQLiteBusy & e) { + } + } +} + +} diff --git a/nix/local.mk b/nix/local.mk index b0e9bc1a2b..c666edd033 100644 --- a/nix/local.mk +++ b/nix/local.mk @@ -86,7 +86,8 @@ libstore_a_SOURCES = \ %D%/libstore/local-store.cc \ %D%/libstore/build.cc \ %D%/libstore/pathlocks.cc \ - %D%/libstore/derivations.cc + %D%/libstore/derivations.cc \ + %D%/libstore/sqlite.cc libstore_headers = \ %D%/libstore/references.hh \ @@ -96,6 +97,7 @@ libstore_headers = \ %D%/libstore/derivations.hh \ %D%/libstore/misc.hh \ %D%/libstore/local-store.hh \ + %D%/libstore/sqlite.hh \ %D%/libstore/store-api.hh libstore_a_CPPFLAGS = \ -- cgit v1.2.3 From b1fd0ab73425e682a0a1404da6963f47a033bb34 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 30 Mar 2016 15:50:45 +0200 Subject: daemon: Improve the SQLite wrapper API. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In particular, this eliminates a bunch of boilerplate code. Also integrates these Nix commits: 80da7a6 Probably fix SQLITE_BUSY errors 37a337b throwSQLiteError(): Check for SIGINT so we don't loop forever Co-authored-by: Ludovic Courtès --- nix/libstore/local-store.cc | 243 +++++++++++++------------------------------- nix/libstore/local-store.hh | 7 +- nix/libstore/sqlite.cc | 85 ++++++++++------ nix/libstore/sqlite.hh | 57 +++++++---- nix/libstore/store-api.hh | 4 +- 5 files changed, 168 insertions(+), 228 deletions(-) (limited to 'nix/libstore') diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc index 6d298cda43..882bce1f40 100644 --- a/nix/libstore/local-store.cc +++ b/nix/libstore/local-store.cc @@ -329,6 +329,7 @@ void LocalStore::openDB(bool create) // ensure efficient lookup. stmtQueryPathFromHashPart.create(db, "select path from ValidPaths where path >= ? limit 1;"); + stmtQueryValidPaths.create(db, "select path from ValidPaths"); } @@ -523,23 +524,16 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & } -unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) +uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) { - SQLiteStmtUse use(stmtRegisterValidPath); - stmtRegisterValidPath.bind(info.path); - stmtRegisterValidPath.bind("sha256:" + printHash(info.hash)); - stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); - if (info.deriver != "") - stmtRegisterValidPath.bind(info.deriver); - else - stmtRegisterValidPath.bind(); // null - if (info.narSize != 0) - stmtRegisterValidPath.bind64(info.narSize); - else - stmtRegisterValidPath.bind(); // null - if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering valid path `%1%' in database") % info.path); - unsigned long long id = sqlite3_last_insert_rowid(db); + stmtRegisterValidPath.use() + (info.path) + ("sha256:" + printHash(info.hash)) + (info.registrationTime == 0 ? time(0) : info.registrationTime) + (info.deriver, info.deriver != "") + (info.narSize, info.narSize != 0) + .exec(); + uint64_t id = sqlite3_last_insert_rowid(db); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -555,13 +549,12 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che registration above is undone. */ if (checkOutputs) checkDerivationOutputs(info.path, drv); - foreach (DerivationOutputs::iterator, i, drv.outputs) { - SQLiteStmtUse use(stmtAddDerivationOutput); - stmtAddDerivationOutput.bind(id); - stmtAddDerivationOutput.bind(i->first); - stmtAddDerivationOutput.bind(i->second.path); - if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) - throwSQLiteError(db, format("adding derivation output for `%1%' in database") % info.path); + for (auto & i : drv.outputs) { + stmtAddDerivationOutput.use() + (id) + (i.first) + (i.second.path) + .exec(); } } @@ -569,24 +562,16 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che } -void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) +void LocalStore::addReference(uint64_t referrer, uint64_t reference) { - SQLiteStmtUse use(stmtAddReference); - stmtAddReference.bind(referrer); - stmtAddReference.bind(reference); - if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throwSQLiteError(db, "adding reference to database"); + stmtAddReference.use()(referrer)(reference).exec(); } void LocalStore::registerFailedPath(const Path & path) { retrySQLite([&]() { - SQLiteStmtUse use(stmtRegisterFailedPath); - stmtRegisterFailedPath.bind(path); - stmtRegisterFailedPath.bind(time(0)); - if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering failed path `%1%'") % path); + stmtRegisterFailedPath.use()(path)(time(0)).step(); }); } @@ -594,12 +579,7 @@ void LocalStore::registerFailedPath(const Path & path) bool LocalStore::hasPathFailed(const Path & path) { return retrySQLite([&]() { - SQLiteStmtUse use(stmtHasPathFailed); - stmtHasPathFailed.bind(path); - int res = sqlite3_step(stmtHasPathFailed); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying whether path failed"); - return res == SQLITE_ROW; + return stmtHasPathFailed.use()(path).next(); }); } @@ -607,18 +587,11 @@ bool LocalStore::hasPathFailed(const Path & path) PathSet LocalStore::queryFailedPaths() { return retrySQLite([&]() { - SQLiteStmtUse use(stmtQueryFailedPaths); + auto useQueryFailedPaths(stmtQueryFailedPaths.use()); PathSet res; - int r; - while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error querying failed paths"); + while (useQueryFailedPaths.next()) + res.insert(useQueryFailedPaths.getStr(0)); return res; }); @@ -630,12 +603,8 @@ void LocalStore::clearFailedPaths(const PathSet & paths) retrySQLite([&]() { SQLiteTxn txn(db); - foreach (PathSet::const_iterator, i, paths) { - SQLiteStmtUse use(stmtClearFailedPath); - stmtClearFailedPath.bind(*i); - if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("clearing failed path `%1%' in database") % *i); - } + for (auto & path : paths) + stmtClearFailedPath.use()(path).exec(); txn.commit(); }); @@ -666,41 +635,28 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) return retrySQLite([&]() { /* Get the path info. */ - SQLiteStmtUse use1(stmtQueryPathInfo); - - stmtQueryPathInfo.bind(path); + auto useQueryPathInfo(stmtQueryPathInfo.use()(path)); - int r = sqlite3_step(stmtQueryPathInfo); - if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); - if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); + if (!useQueryPathInfo.next()) + throw Error(format("path `%1%' is not valid") % path); - info.id = sqlite3_column_int(stmtQueryPathInfo, 0); + info.id = useQueryPathInfo.getInt(0); - const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); - assert(s); - info.hash = parseHashField(path, s); + info.hash = parseHashField(path, useQueryPathInfo.getStr(1)); - info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); + info.registrationTime = useQueryPathInfo.getInt(2); - s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); + auto s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); if (s) info.deriver = s; /* Note that narSize = NULL yields 0. */ - info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); + info.narSize = useQueryPathInfo.getInt(4); /* Get the references. */ - SQLiteStmtUse use2(stmtQueryReferences); - - stmtQueryReferences.bind(info.id); + auto useQueryReferences(stmtQueryReferences.use()(info.id)); - while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { - s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); - assert(s); - info.references.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of `%1%'") % path); + while (useQueryReferences.next()) + info.references.insert(useQueryReferences.getStr(0)); return info; }); @@ -711,37 +667,26 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) narSize field. */ void LocalStore::updatePathInfo(const ValidPathInfo & info) { - SQLiteStmtUse use(stmtUpdatePathInfo); - if (info.narSize != 0) - stmtUpdatePathInfo.bind64(info.narSize); - else - stmtUpdatePathInfo.bind(); // null - stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash)); - stmtUpdatePathInfo.bind(info.path); - if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) - throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path); + stmtUpdatePathInfo.use() + (info.narSize, info.narSize != 0) + ("sha256:" + printHash(info.hash)) + (info.path) + .exec(); } -unsigned long long LocalStore::queryValidPathId(const Path & path) +uint64_t LocalStore::queryValidPathId(const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); - if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); - throwSQLiteError(db, "querying path in database"); + auto use(stmtQueryPathInfo.use()(path)); + if (!use.next()) + throw Error(format("path ‘%1%’ is not valid") % path); + return use.getInt(0); } bool LocalStore::isValidPath_(const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying path in database"); - return res == SQLITE_ROW; + return stmtQueryPathInfo.use()(path).next(); } @@ -767,20 +712,9 @@ PathSet LocalStore::queryValidPaths(const PathSet & paths) PathSet LocalStore::queryAllValidPaths() { return retrySQLite([&]() { - SQLiteStmt stmt; - stmt.create(db, "select path from ValidPaths"); - + auto use(stmtQueryValidPaths.use()); PathSet res; - int r; - while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmt, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error getting valid paths"); - + while (use.next()) res.insert(use.getStr(0)); return res; }); } @@ -796,19 +730,10 @@ void LocalStore::queryReferences(const Path & path, void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) { - SQLiteStmtUse use(stmtQueryReferrers); - - stmtQueryReferrers.bind(path); - - int r; - while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); - assert(s); - referrers.insert(s); - } + auto useQueryReferrers(stmtQueryReferrers.use()(path)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of `%1%'") % path); + while (useQueryReferrers.next()) + referrers.insert(useQueryReferrers.getStr(0)); } @@ -832,19 +757,11 @@ PathSet LocalStore::queryValidDerivers(const Path & path) assertStorePath(path); return retrySQLite([&]() { - SQLiteStmtUse use(stmtQueryValidDerivers); - stmtQueryValidDerivers.bind(path); + auto useQueryValidDerivers(stmtQueryValidDerivers.use()(path)); PathSet derivers; - int r; - while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); - assert(s); - derivers.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path); + while (useQueryValidDerivers.next()) + derivers.insert(useQueryValidDerivers.getStr(1)); return derivers; }); @@ -854,19 +771,11 @@ PathSet LocalStore::queryValidDerivers(const Path & path) PathSet LocalStore::queryDerivationOutputs(const Path & path) { return retrySQLite([&]() { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path))); PathSet outputs; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); - assert(s); - outputs.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting outputs of `%1%'") % path); + while (useQueryDerivationOutputs.next()) + outputs.insert(useQueryDerivationOutputs.getStr(1)); return outputs; }); @@ -876,19 +785,11 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path) StringSet LocalStore::queryDerivationOutputNames(const Path & path) { return retrySQLite([&]() { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path))); StringSet outputNames; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0); - assert(s); - outputNames.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting output names of `%1%'") % path); + while (useQueryDerivationOutputs.next()) + outputNames.insert(useQueryDerivationOutputs.getStr(0)); return outputNames; }); @@ -902,12 +803,9 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart) Path prefix = settings.nixStore + "/" + hashPart; return retrySQLite([&]() -> Path { - SQLiteStmtUse use(stmtQueryPathFromHashPart); - stmtQueryPathFromHashPart.bind(prefix); + auto useQueryPathFromHashPart(stmtQueryPathFromHashPart.use()(prefix)); - int res = sqlite3_step(stmtQueryPathFromHashPart); - if (res == SQLITE_DONE) return ""; - if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database"); + if (!useQueryPathFromHashPart.next()) return ""; const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; @@ -1154,10 +1052,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) paths.insert(i->path); } - foreach (ValidPathInfos::const_iterator, i, infos) { - unsigned long long referrer = queryValidPathId(i->path); - foreach (PathSet::iterator, j, i->references) - addReference(referrer, queryValidPathId(*j)); + for (auto & i : infos) { + auto referrer = queryValidPathId(i.path); + for (auto & j : i.references) + addReference(referrer, queryValidPathId(j)); } /* Check that the derivation outputs are correct. We can't do @@ -1190,12 +1088,7 @@ void LocalStore::invalidatePath(const Path & path) drvHashes.erase(path); - SQLiteStmtUse use(stmtInvalidatePath); - - stmtInvalidatePath.bind(path); - - if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) - throwSQLiteError(db, format("invalidating path `%1%' in database") % path); + stmtInvalidatePath.use()(path).exec(); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ diff --git a/nix/libstore/local-store.hh b/nix/libstore/local-store.hh index f591589838..6110468498 100644 --- a/nix/libstore/local-store.hh +++ b/nix/libstore/local-store.hh @@ -207,6 +207,7 @@ private: SQLiteStmt stmtQueryValidDerivers; SQLiteStmt stmtQueryDerivationOutputs; SQLiteStmt stmtQueryPathFromHashPart; + SQLiteStmt stmtQueryValidPaths; /* Cache for pathContentsGood(). */ std::map pathContentsGoodCache; @@ -223,11 +224,11 @@ private: void makeStoreWritable(); - unsigned long long queryValidPathId(const Path & path); + uint64_t queryValidPathId(const Path & path); - unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true); + uint64_t addValidPath(const ValidPathInfo & info, bool checkOutputs = true); - void addReference(unsigned long long referrer, unsigned long long reference); + void addReference(uint64_t referrer, uint64_t reference); void appendReferrer(const Path & from, const Path & to, bool lock); diff --git a/nix/libstore/sqlite.cc b/nix/libstore/sqlite.cc index 8646ff5b12..e08c67f40e 100644 --- a/nix/libstore/sqlite.cc +++ b/nix/libstore/sqlite.cc @@ -53,15 +53,6 @@ void SQLiteStmt::create(sqlite3 * db, const string & s) this->db = db; } -void SQLiteStmt::reset() -{ - assert(stmt); - /* Note: sqlite3_reset() returns the error code for the most - recent call to sqlite3_step(). So ignore it. */ - sqlite3_reset(stmt); - curArg = 1; -} - SQLiteStmt::~SQLiteStmt() { try { @@ -72,43 +63,79 @@ SQLiteStmt::~SQLiteStmt() } } -void SQLiteStmt::bind(const string & value) +SQLiteStmt::Use::Use(SQLiteStmt & stmt) + : stmt(stmt) +{ + assert(stmt.stmt); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use::~Use() { - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); + sqlite3_reset(stmt); } -void SQLiteStmt::bind(int value) +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) { - if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); + if (notNull) { + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; } -void SQLiteStmt::bind64(long long value) +SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) { - if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); + if (notNull) { + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; } -void SQLiteStmt::bind() +SQLiteStmt::Use & SQLiteStmt::Use::bind() { if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); + throwSQLiteError(stmt.db, "binding argument"); + return *this; } -SQLiteStmtUse::SQLiteStmtUse(SQLiteStmt & stmt) - : stmt(stmt) +int SQLiteStmt::Use::step() { - stmt.reset(); + return sqlite3_step(stmt); } -SQLiteStmtUse::~SQLiteStmtUse() +void SQLiteStmt::Use::exec() { - try { - stmt.reset(); - } catch (...) { - ignoreException(); - } + int r = step(); + assert(r != SQLITE_ROW); + if (r != SQLITE_DONE) + throwSQLiteError(stmt.db, "executing SQLite statement"); +} + +bool SQLiteStmt::Use::next() +{ + int r = step(); + if (r != SQLITE_DONE && r != SQLITE_ROW) + throwSQLiteError(stmt.db, "executing SQLite query"); + return r == SQLITE_ROW; +} + +std::string SQLiteStmt::Use::getStr(int col) +{ + auto s = (const char *) sqlite3_column_text(stmt, col); + assert(s); + return s; +} + +int64_t SQLiteStmt::Use::getInt(int col) +{ + // FIXME: detect nulls? + return sqlite3_column_int64(stmt, col); } SQLiteTxn::SQLiteTxn(sqlite3 * db) diff --git a/nix/libstore/sqlite.hh b/nix/libstore/sqlite.hh index 0abdb74631..326e4a4855 100644 --- a/nix/libstore/sqlite.hh +++ b/nix/libstore/sqlite.hh @@ -22,29 +22,48 @@ struct SQLite /* RAII wrapper to create and destroy SQLite prepared statements. */ struct SQLiteStmt { - sqlite3 * db; - sqlite3_stmt * stmt; - unsigned int curArg; - SQLiteStmt() { stmt = 0; } + sqlite3 * db = 0; + sqlite3_stmt * stmt = 0; + SQLiteStmt() { } void create(sqlite3 * db, const std::string & s); - void reset(); ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } - void bind(const std::string & value); - void bind(int value); - void bind64(long long value); - void bind(); -}; -/* Helper class to ensure that prepared statements are reset when - leaving the scope that uses them. Unfinished prepared statements - prevent transactions from being aborted, and can cause locks to be - kept when they should be released. */ -struct SQLiteStmtUse -{ - SQLiteStmt & stmt; - SQLiteStmtUse(SQLiteStmt & stmt); - ~SQLiteStmtUse(); + /* Helper for binding / executing statements. */ + class Use + { + friend struct SQLiteStmt; + private: + SQLiteStmt & stmt; + unsigned int curArg = 1; + Use(SQLiteStmt & stmt); + + public: + + ~Use(); + + /* Bind the next parameter. */ + Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (int64_t value, bool notNull = true); + Use & bind(); // null + + int step(); + + /* Execute a statement that does not return rows. */ + void exec(); + + /* For statements that return 0 or more rows. Returns true iff + a row is available. */ + bool next(); + + std::string getStr(int col); + int64_t getInt(int col); + }; + + Use use() + { + return Use(*this); + } }; /* RAII helper that ensures transactions are aborted unless explicitly diff --git a/nix/libstore/store-api.hh b/nix/libstore/store-api.hh index 3e982f6dd3..fa78d595f2 100644 --- a/nix/libstore/store-api.hh +++ b/nix/libstore/store-api.hh @@ -89,8 +89,8 @@ struct ValidPathInfo Hash hash; PathSet references; time_t registrationTime = 0; - unsigned long long narSize = 0; // 0 = unknown - unsigned long long id; // internal use only + uint64_t narSize = 0; // 0 = unknown + uint64_t id; // internal use only bool operator == (const ValidPathInfo & i) const { -- cgit v1.2.3 From 12b6c951cf5ca6055a22a2eec85665353f5510e5 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 28 Oct 2016 20:34:15 +0200 Subject: daemon: Do not error out when deduplication fails due to ENOSPC. This solves a problem whereby if /gnu/store/.links had enough entries, ext4's directory index would be full, leading to link(2) returning ENOSPC. * nix/libstore/optimise-store.cc (LocalStore::optimisePath_): Upon ENOSPC from link(2), print a message and return instead of throwing a 'SysError'. --- nix/libstore/optimise-store.cc | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) (limited to 'nix/libstore') diff --git a/nix/libstore/optimise-store.cc b/nix/libstore/optimise-store.cc index d7508b025e..565c62ca83 100644 --- a/nix/libstore/optimise-store.cc +++ b/nix/libstore/optimise-store.cc @@ -148,10 +148,23 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa inodeHash.insert(st.st_ino); return; } - if (errno != EEXIST) + + switch (errno) { + case EEXIST: + /* Fall through if another process created ‘linkPath’ before + we did. */ + break; + + case ENOSPC: + /* On ext4, that probably means the directory index is full. When + that happens, it's fine to ignore it: we just effectively + disable deduplication of this file. */ + printMsg(lvlInfo, format("cannot link `%1%' to `%2%': %m") % linkPath % path); + return; + + default: throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path); - /* Fall through if another process created ‘linkPath’ before - we did. */ + } } /* Yes! We've seen a file with the same contents. Replace the @@ -195,8 +208,8 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); return; } - throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath); - } + throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath); + } /* Atomically replace the old file with the new hard link. */ if (rename(tempLink.c_str(), path.c_str()) == -1) { -- cgit v1.2.3