From e393cbf8a33debd96fb1c0509fdd12fd197b57b6 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Fri, 13 Nov 2020 22:51:04 +0000 Subject: Revert "Remove redundant sqlite-reset calls" I do not understand why, but I think the removal of the "redundant" reset statements has broken the database in some way I haven't been able to pin down. Either writes aren't working, or some reads are returning stale data. I'm also seeing errors about locked tables :( This reverts commit 049334e423241426c0eef526eda8818e96f5b3ca. --- guix-build-coordinator/datastore/sqlite.scm | 646 +++++++++++++++++----------- 1 file changed, 397 insertions(+), 249 deletions(-) diff --git a/guix-build-coordinator/datastore/sqlite.scm b/guix-build-coordinator/datastore/sqlite.scm index 82d7003..5e05ada 100644 --- a/guix-build-coordinator/datastore/sqlite.scm +++ b/guix-build-coordinator/datastore/sqlite.scm @@ -176,13 +176,17 @@ SELECT description FROM agents WHERE id = :id" statement #:id uuid) - (match (sqlite-map - (match-lambda - (#(description) - `((description . ,description)))) - statement) - (() #f) - ((agent) agent)))))) + (let ((result + (match (sqlite-map + (match-lambda + (#(description) + `((description . ,description)))) + statement) + (() #f) + ((agent) agent)))) + (sqlite-reset statement) + + result))))) (define-method (datastore-new-agent (datastore ) @@ -206,12 +210,15 @@ SELECT description FROM agents WHERE id = :id" SELECT id, description FROM agents ORDER BY id" #:cache? #t))) - (sqlite-map - (match-lambda - (#(id description) - `((uuid . ,id) - (description . ,description)))) - statement))))) + (let ((agents (sqlite-map + (match-lambda + (#(id description) + `((uuid . ,id) + (description . ,description)))) + statement))) + (sqlite-reset statement) + + agents))))) (define-method (datastore-new-agent-password (datastore ) @@ -243,9 +250,13 @@ WHERE agent_id = :agent_id AND password = :password" #:agent_id uuid #:password password) - (match (sqlite-step statement) - (#f #f) - (#(1) #t)))))) + (let ((result + (match (sqlite-step statement) + (#f #f) + (#(1) #t)))) + (sqlite-reset statement) + + result))))) (define-method (datastore-store-derivation (datastore ) @@ -302,10 +313,13 @@ WHERE related_derivations.name != :derivation statement #:derivation derivation) - (sqlite-map - (match-lambda - (#(derivation) derivation)) - statement))))) + (let ((result (sqlite-map + (match-lambda + (#(derivation) derivation)) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-list-failed-builds-with-blocking-count (datastore )) @@ -355,13 +369,16 @@ WHERE builds.processed = 1 ) ORDER BY 3 DESC, 2, 1" #:cache? #t))) - (sqlite-map - (match-lambda - (#(uuid derivation-name blocked-count) - `((uuid . ,uuid) - (derivation_name . ,derivation-name) - (blocked_count . ,blocked-count)))) - statement))))) + (let ((result (sqlite-map + (match-lambda + (#(uuid derivation-name blocked-count) + `((uuid . ,uuid) + (derivation_name . ,derivation-name) + (blocked_count . ,blocked-count)))) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-list-builds-for-derivation-recursive-inputs (datastore ) @@ -393,10 +410,13 @@ INNER JOIN related_derivations statement #:derivation derivation) - (sqlite-map - (match-lambda - (#(uuid) uuid)) - statement))))) + (let ((result (sqlite-map + (match-lambda + (#(uuid) uuid)) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-store-build (datastore ) @@ -476,11 +496,15 @@ INSERT INTO build_tags (build_id, tag_id) VALUES (:build_id, :tag_id)" SELECT agent_id, result, COUNT(*) FROM build_results GROUP BY agent_id, result" #:cache? #t))) - (sqlite-map - (match-lambda - (#(agent_id result count) - (cons (list agent_id result) count))) - statement))))) + (let ((result + (sqlite-map + (match-lambda + (#(agent_id result count) + (cons (list agent_id result) count))) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-store-build-result (datastore ) @@ -573,7 +597,10 @@ WHERE builds.uuid = :build_id AND derivation_outputs.name = :name" #:name name) (match (sqlite-step statement) - (#(id) id)))) + (#(id) + (sqlite-reset statement) + + id)))) (sqlite-exec db @@ -670,13 +697,17 @@ ORDER BY start_time DESC" statement #:build_id build-id) - (sqlite-map - (match-lambda - (#(start_time agent_id) - `((start-time . ,(match (strptime "%F %T" start_time) - ((parts . _) parts))) - (agent-id . ,agent_id)))) - statement))))) + (let ((result + (sqlite-map + (match-lambda + (#(start_time agent_id) + `((start-time . ,(match (strptime "%F %T" start_time) + ((parts . _) parts))) + (agent-id . ,agent_id)))) + statement))) + (sqlite-reset statement) + + result))))) (define (insert-setup-failure-and-remove-allocation db @@ -767,10 +798,13 @@ WHERE setup_failure_id = :id" statement #:id setup-failure-id) - (sqlite-map - (match-lambda - (#(missing-input) missing-input)) - statement))))) + (let ((result (sqlite-map + (match-lambda + (#(missing-input) missing-input)) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-store-setup-failure (datastore ) @@ -801,11 +835,15 @@ INNER JOIN derivations GROUP BY derivations.system" #:cache? #t))) - (sqlite-map - (match-lambda - (#(system count) - (cons system count))) - statement))))) + (let ((result + (sqlite-map + (match-lambda + (#(system count) + (cons system count))) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-for-each-build (datastore ) @@ -826,7 +864,10 @@ SELECT uuid FROM builds ORDER BY uuid" (proc uuid) (loop (sqlite-step statement))) - (#f #t))))))) + (#f + (sqlite-reset statement) + + #t))))))) (define-method (datastore-find-build (datastore ) @@ -847,24 +888,28 @@ WHERE uuid = :uuid" statement #:uuid uuid) - (match (sqlite-step statement) - (#(uuid derivation_name priority processed created_at end_time) - `((uuid . ,uuid) - (derivation-name . ,derivation_name) - (priority . ,priority) - (processed . ,(cond - ((= 0 processed) #f) - ((= 1 processed) #t) - (else - (error "unknown processed value")))) - (created-at . ,(if (string? created_at) - (match (strptime "%F %T" created_at) - ((parts . _) parts)) - #f)) - (end-time . ,(if (string? end_time) - (match (strptime "%F %T" end_time) - ((parts . _) parts)) - #f))))))))) + (let ((result + (match (sqlite-step statement) + (#(uuid derivation_name priority processed created_at end_time) + `((uuid . ,uuid) + (derivation-name . ,derivation_name) + (priority . ,priority) + (processed . ,(cond + ((= 0 processed) #f) + ((= 1 processed) #t) + (else + (error "unknown processed value")))) + (created-at . ,(if (string? created_at) + (match (strptime "%F %T" created_at) + ((parts . _) parts)) + #f)) + (end-time . ,(if (string? end_time) + (match (strptime "%F %T" end_time) + ((parts . _) parts)) + #f))))))) + (sqlite-reset statement) + + result))))) (define-method (datastore-find-build-result (datastore ) @@ -885,12 +930,16 @@ WHERE build_id = :build_id" statement #:build_id build-id) - (match (sqlite-step statement) - (#(agent_id result failure_reason) - `((agent_id . ,agent_id) - (result . ,result) - (failure_reason . ,failure_reason))) - (#f #f)))))) + (let ((result + (match (sqlite-step statement) + (#(agent_id result failure_reason) + `((agent_id . ,agent_id) + (result . ,result) + (failure_reason . ,failure_reason))) + (#f #f)))) + (sqlite-reset statement) + + result))))) (define-method (datastore-find-build-derivation-system (datastore ) @@ -912,8 +961,12 @@ WHERE builds.uuid = :build_id" statement #:build_id build-id) - (match (sqlite-step statement) - (#(system) system)))))) + (let ((system + (match (sqlite-step statement) + (#(system) system)))) + (sqlite-reset statement) + + system))))) (define-method (datastore-list-builds-for-output (datastore ) @@ -942,15 +995,19 @@ WHERE derivation_outputs.output = :output" statement #:output output) - (sqlite-map - (match-lambda - (#(uuid derivation priority processed result) - `((uuid . ,uuid) - (derivation . ,derivation) - (priority . ,priority) - (processed . ,processed) - (result . ,result)))) - statement))))))) + (let ((result + (sqlite-map + (match-lambda + (#(uuid derivation priority processed result) + `((uuid . ,uuid) + (derivation . ,derivation) + (priority . ,priority) + (processed . ,processed) + (result . ,result)))) + statement))) + (sqlite-reset statement) + + result))))))) (define-method (datastore-list-builds-for-output-and-system (datastore ) @@ -978,12 +1035,16 @@ WHERE derivation_outputs.output = :output #:output output #:system system) - (sqlite-map - (match-lambda - (#(uuid derivation) - `((uuid . ,uuid) - (derivation . ,derivation)))) - statement))))) + (let ((result + (sqlite-map + (match-lambda + (#(uuid derivation) + `((uuid . ,uuid) + (derivation . ,derivation)))) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-list-builds-for-derivation (datastore ) @@ -1002,11 +1063,15 @@ SELECT uuid FROM builds WHERE derivation_name = :derivation" statement #:derivation derivation) - (sqlite-map - (match-lambda - (#(uuid) - `((uuid . ,uuid)))) - statement))))) + (let ((result + (sqlite-map + (match-lambda + (#(uuid) + `((uuid . ,uuid)))) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-update (datastore )) @@ -1029,11 +1094,15 @@ FROM setup_failures GROUP BY agent_id, failure_reason" #:cache? #t))) - (sqlite-map - (match-lambda - (#(agent_id failure_reason count) - (cons (list agent_id failure_reason) count))) - statement))))) + (let ((result + (sqlite-map + (match-lambda + (#(agent_id failure_reason count) + (cons (list agent_id failure_reason) count))) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-list-setup-failures-for-build (datastore ) @@ -1054,13 +1123,16 @@ WHERE build_id = :build_id" statement #:build_id build-id) - (sqlite-map - (match-lambda - (#(id agent-id failure-reason) - `((id . ,id) - (agent-id . ,agent-id) - (failure-reason . ,failure-reason)))) - statement))))) + (let ((result (sqlite-map + (match-lambda + (#(id agent-id failure-reason) + `((id . ,id) + (agent-id . ,agent-id) + (failure-reason . ,failure-reason)))) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-fetch-setup-failures (datastore )) @@ -1078,23 +1150,26 @@ INNER JOIN builds WHERE builds.processed = 0" #:cache? #t))) - (sqlite-fold - (lambda (row result) - (match row - (#(id build-id agent-id failure-reason) - (let ((failures-for-build-id - (or (hash-ref result build-id) - '()))) - (hash-set! - result - build-id - (cons `((id . ,id) - (agent-id . ,agent-id) - (failure-reason . ,failure-reason)) - failures-for-build-id))))) - result) - (make-hash-table) - statement))))) + (let ((result (sqlite-fold + (lambda (row result) + (match row + (#(id build-id agent-id failure-reason) + (let ((failures-for-build-id + (or (hash-ref result build-id) + '()))) + (hash-set! + result + build-id + (cons `((id . ,id) + (agent-id . ,agent-id) + (failure-reason . ,failure-reason)) + failures-for-build-id))))) + result) + (make-hash-table) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-list-processed-builds (datastore )) @@ -1108,13 +1183,16 @@ WHERE builds.processed = 0" SELECT uuid, derivation_name, priority FROM builds WHERE processed = 1" #:cache? #t))) - (sqlite-map - (match-lambda - (#(uuid derivation_name priority) - `((uuid . ,uuid) - (derivation-name . ,derivation_name) - (priority . ,priority)))) - statement))))) + (let ((builds (sqlite-map + (match-lambda + (#(uuid derivation_name priority) + `((uuid . ,uuid) + (derivation-name . ,derivation_name) + (priority . ,priority)))) + statement))) + (sqlite-reset statement) + + builds))))) (define-method (datastore-list-unprocessed-builds (datastore )) @@ -1131,13 +1209,16 @@ WHERE processed = 0 ORDER BY priority DESC" #:cache? #t))) - (sqlite-map - (match-lambda - (#(uuid derivation_name priority) - `((uuid . ,uuid) - (derivation-name . ,derivation_name) - (priority . ,priority)))) - statement))))) + (let ((builds (sqlite-map + (match-lambda + (#(uuid derivation_name priority) + `((uuid . ,uuid) + (derivation-name . ,derivation_name) + (priority . ,priority)))) + statement))) + (sqlite-reset statement) + + builds))))) (define-method (datastore-fetch-build-ids-and-propagated-priorities-for-unprocessed-builds (datastore ) @@ -1227,14 +1308,16 @@ WHERE builds.processed = 0 GROUP BY builds_with_derived_priority.uuid" #:cache? #t))) - (sqlite-fold - (lambda (row result) - (match row - (#(uuid derived-priority) - (hash-set! result uuid derived-priority))) - result) - (make-hash-table 10000) - statement))) + (let ((result (sqlite-fold + (lambda (row result) + (match row + (#(uuid derived-priority) + (hash-set! result uuid derived-priority))) + result) + (make-hash-table 10000) + statement))) + + result))) (datastore-call-with-transaction datastore @@ -1269,7 +1352,8 @@ VALUES (:event, :arguments)" (lambda (port) (write arguments port)))) - (sqlite-step statement)) + (sqlite-step statement) + (sqlite-reset statement)) #t) (define-method (datastore-count-unprocessed-hook-events @@ -1284,12 +1368,15 @@ VALUES (:event, :arguments)" SELECT event, COUNT(*) FROM unprocessed_hook_events GROUP BY event" #:cache? #t))) - (sqlite-map - (match-lambda - (#(event count) - `((event . ,event) - (count . ,count)))) - statement))))) + (let ((counts (sqlite-map + (match-lambda + (#(event count) + `((event . ,event) + (count . ,count)))) + statement))) + (sqlite-reset statement) + + counts))))) (define-method (datastore-list-unprocessed-hook-events (datastore ) @@ -1313,15 +1400,18 @@ LIMIT :limit" #:event (symbol->string event) #:limit limit) - (sqlite-map - (match-lambda - (#(id event arguments) - (list id - (string->symbol event) - (call-with-input-string arguments - (lambda (port) - (read port)))))) - statement))))) + (let ((events (sqlite-map + (match-lambda + (#(id event arguments) + (list id + (string->symbol event) + (call-with-input-string arguments + (lambda (port) + (read port)))))) + statement))) + (sqlite-reset statement) + + events))))) (define-method (datastore-delete-unprocessed-hook-event (datastore ) @@ -1350,11 +1440,15 @@ FROM build_allocation_plan GROUP BY agent_id" #:cache? #t))) - (sqlite-map - (match-lambda - (#(agent_id count) - (cons agent_id count))) - statement))))) + (let ((result + (sqlite-map + (match-lambda + (#(agent_id count) + (cons agent_id count))) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-replace-build-allocation-plan (datastore ) @@ -1407,11 +1501,15 @@ INSERT INTO build_allocation_plan (build_id, agent_id, ordering) VALUES " SELECT agent_id, COUNT(*) FROM allocated_builds GROUP BY agent_id" #:cache? #t))) - (sqlite-map - (match-lambda - (#(agent_id count) - (cons agent_id count))) - statement))))) + (let ((result + (sqlite-map + (match-lambda + (#(agent_id count) + (cons agent_id count))) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-agent-requested-systems (datastore ) @@ -1433,9 +1531,13 @@ ORDER BY system ASC" statement #:agent_id agent-id) - (sqlite-map - (match-lambda (#(system) system)) - statement))))) + (let ((result + (sqlite-map + (match-lambda (#(system) system)) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-update-agent-requested-systems (datastore ) @@ -1618,12 +1720,15 @@ LIMIT :limit" #:agent_id agent-id #:limit limit) - (sqlite-map - (match-lambda - (#(uuid derivation_name) - `((uuid . ,uuid) - (derivation-name . ,derivation_name)))) - statement))))) + (let ((builds (sqlite-map + (match-lambda + (#(uuid derivation_name) + `((uuid . ,uuid) + (derivation-name . ,derivation_name)))) + statement))) + (sqlite-reset statement) + + builds))))) (define-method (datastore-list-agent-builds (datastore ) @@ -1646,13 +1751,16 @@ WHERE allocated_builds.agent_id = :agent_id" statement #:agent_id agent-id) - (sqlite-map - (match-lambda - (#(uuid derivation_name priority) - `((uuid . ,uuid) - (derivation-name . ,derivation_name) - (priority . ,priority)))) - statement))))) + (let ((builds (sqlite-map + (match-lambda + (#(uuid derivation_name priority) + `((uuid . ,uuid) + (derivation-name . ,derivation_name) + (priority . ,priority)))) + statement))) + (sqlite-reset statement) + + builds))))) (define-method (datastore-agent-for-build (datastore ) @@ -1676,9 +1784,13 @@ WHERE build_results.build_id = :build_id" statement #:build_id build-id) - (match (sqlite-step statement) - (#(agent-id) agent-id) - (#f #f)))))) + (let ((result + (match (sqlite-step statement) + (#(agent-id) agent-id) + (#f #f)))) + (sqlite-reset statement) + + result))))) (define (db-open database) (define flags @@ -1725,9 +1837,13 @@ WHERE build_results.build_id = :build_id" db "SELECT changes();" #:cache? #t))) + (let ((count + (vector-ref (sqlite-step statement) + 0))) - (vector-ref (sqlite-step statement) - 0))) + (sqlite-reset statement) + + count))) (define (last-insert-rowid db) (let ((statement @@ -1735,9 +1851,13 @@ WHERE build_results.build_id = :build_id" db "SELECT last_insert_rowid();" #:cache? #t))) + (let ((id + (vector-ref (sqlite-step statement) + 0))) + + (sqlite-reset statement) - (vector-ref (sqlite-step statement) - 0))) + id))) (define (select-derivation-outputs db derivation-name) (let ((statement @@ -1751,11 +1871,14 @@ SELECT name, id FROM derivation_outputs WHERE derivation_name = :derivation_name statement #:derivation_name derivation-name) - (sqlite-map - (match-lambda - (#(name output-id) - (cons name output-id))) - statement))) + (let ((outputs (sqlite-map + (match-lambda + (#(name output-id) + (cons name output-id))) + statement))) + (sqlite-reset statement) + + outputs))) (define (db-find-derivation db name) (let ((statement @@ -1771,14 +1894,18 @@ WHERE name = :name" statement #:name name) - (match (sqlite-step statement) - (#f #f) - (#(system fixed_output) - `((system . ,system) - (fixed-output? . ,(cond - ((eq? fixed_output 0) #f) - ((eq? fixed_output 1) #t) - (else fixed_output)))))))) + (let ((result + (match (sqlite-step statement) + (#f #f) + (#(system fixed_output) + `((system . ,system) + (fixed-output? . ,(cond + ((eq? fixed_output 0) #f) + ((eq? fixed_output 1) #t) + (else fixed_output)))))))) + (sqlite-reset statement) + + result))) (define-method (datastore-find-derivation (datastore ) @@ -1807,12 +1934,16 @@ WHERE derivation_name = :derivation_name" statement #:derivation_name derivation-name) - (sqlite-map - (match-lambda - (#(name output) - `((name . ,name) - (output . ,output)))) - statement))))) + (let ((result + (sqlite-map + (match-lambda + (#(name output) + `((name . ,name) + (output . ,output)))) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-list-build-outputs (datastore ) @@ -1838,18 +1969,22 @@ WHERE builds.uuid = :build_id" statement #:build_id build-id) - (sqlite-map - (match-lambda - (#(name output hash size store_references) - `((name . ,name) - (output . ,output) - (hash . ,hash) - (size . ,size) - (references . ,(and store_references - (list->vector - (string-split store_references - #\space))))))) - statement))))) + (let ((result + (sqlite-map + (match-lambda + (#(name output hash size store_references) + `((name . ,name) + (output . ,output) + (hash . ,hash) + (size . ,size) + (references . ,(and store_references + (list->vector + (string-split store_references + #\space))))))) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-find-derivation-system (datastore ) @@ -1870,8 +2005,12 @@ WHERE name = :name" statement #:name derivation-name) - (match (sqlite-step statement) - (#(system) system)))))) + (let ((system + (match (sqlite-step statement) + (#(system) system)))) + (sqlite-reset statement) + + system))))) (define-method (datastore-find-derivation-inputs (datastore ) @@ -1895,13 +2034,17 @@ WHERE derivation_inputs.derivation_name = :derivation_name" statement #:derivation_name derivation-name) - (sqlite-map - (match-lambda - (#(derivation output-name output) - `((derivation . ,derivation) - (output_name . ,output-name) - (output . ,output)))) - statement))))) + (let ((result + (sqlite-map + (match-lambda + (#(derivation output-name output) + `((derivation . ,derivation) + (output_name . ,output-name) + (output . ,output)))) + statement))) + (sqlite-reset statement) + + result))))) (define-method (datastore-find-derivation-for-output (datastore ) @@ -1937,9 +2080,13 @@ WHERE output = :output #:derivation start-derivation-name #:output output) - (match (sqlite-step statement) - (#f #f) - (#(derivation) derivation)))))) + (let ((result + (match (sqlite-step statement) + (#f #f) + (#(derivation) derivation)))) + (sqlite-reset statement) + + result))))) (define (insert-derivation-and-return-outputs db derivation) (define derivation-name @@ -1975,6 +2122,7 @@ INSERT OR IGNORE INTO derivations (name, system, fixed_output) #:fixed_output (if fixed-output? 1 0)) (sqlite-step statement) + (sqlite-reset statement) (changes-count db))))) @@ -2083,8 +2231,8 @@ VALUES (:uuid, :derivation_name, :priority, datetime('now'))" #:derivation_name derivation-name #:priority priority) - (sqlite-step statement)) - #t) + (sqlite-step statement) + (sqlite-reset statement))) (define (insert-agent db uuid description) (let ((statement @@ -2100,8 +2248,8 @@ VALUES (:id, :description)" #:id uuid #:description description) - (sqlite-step statement)) - #t) + (sqlite-step statement) + (sqlite-reset statement))) (define (insert-agent-password db uuid password) (let ((statement @@ -2117,5 +2265,5 @@ VALUES (:agent_id, :password)" #:agent_id uuid #:password password) - (sqlite-step statement)) - #t) + (sqlite-step statement) + (sqlite-reset statement))) -- cgit v1.2.3