diff options
author | Christopher Baines <mail@cbaines.net> | 2021-12-30 22:02:34 +0000 |
---|---|---|
committer | Christopher Baines <mail@cbaines.net> | 2021-12-30 22:17:08 +0000 |
commit | dce619ed3bc6bf051885ff1ef0ac35090a12f704 (patch) | |
tree | e95f8cf2da5d0d4553fa67b3f3ed3ccd6c97ee0e /guix-build-coordinator/datastore/sqlite.scm | |
parent | 70a5c0459e593187b5925bd5bea0078f435613f6 (diff) | |
download | build-coordinator-dce619ed3bc6bf051885ff1ef0ac35090a12f704.tar build-coordinator-dce619ed3bc6bf051885ff1ef0ac35090a12f704.tar.gz |
Optimise inserting derivations
Avoid some queries by just moving around the values being queried for
better. Also cache the outputs for a derivation so that doesn't have to be
looked up.
Diffstat (limited to 'guix-build-coordinator/datastore/sqlite.scm')
-rw-r--r-- | guix-build-coordinator/datastore/sqlite.scm | 131 |
1 files changed, 76 insertions, 55 deletions
diff --git a/guix-build-coordinator/datastore/sqlite.scm b/guix-build-coordinator/datastore/sqlite.scm index 7cc66fe..e64b6c4 100644 --- a/guix-build-coordinator/datastore/sqlite.scm +++ b/guix-build-coordinator/datastore/sqlite.scm @@ -769,13 +769,17 @@ INSERT INTO agent_tags (agent_id, tag_id) VALUES (:agent_id, :tag_id)" tags))))) #t) +(define %derivation-outputs-cache + (make-hash-table)) + (define-method (datastore-store-derivation (datastore <sqlite-datastore>) derivation) (datastore-call-with-transaction datastore (lambda (db) - (insert-derivation-and-return-outputs db derivation)) + (insert-derivation-and-return-outputs db derivation) + (hash-clear! %derivation-outputs-cache)) #:duration-metric-name "store_derivation") #t) @@ -3335,62 +3339,78 @@ SELECT system FROM systems WHERE id = :id" (define derivation-name (derivation-file-name derivation)) - (define (insert-derivation) - (let ((derivation-details (db-find-derivation db derivation-name)) - (fixed-output? (fixed-output-derivation? derivation))) - (if derivation-details - (begin - (unless (equal? (assq-ref derivation-details 'fixed-output?) - fixed-output?) - (let ((statement (sqlite-prepare - db - " + (define (maybe-fix-fixed-output-field derivation-details) + (let ((fixed-output? (fixed-output-derivation? derivation))) + (unless (equal? (assq-ref derivation-details 'fixed-output?) + fixed-output?) + (let ((statement (sqlite-prepare + db + " UPDATE derivations SET fixed_output = :fixed_output WHERE name = :name" - #:cache? #t))) - (sqlite-bind-arguments statement - #:name derivation-name - #:fixed_output (if fixed-output? 1 0)))) - 0) - (let ((statement - (sqlite-prepare - db - " -INSERT OR IGNORE INTO derivations (name, system_id, fixed_output) + #:cache? #t))) + (sqlite-bind-arguments statement + #:name derivation-name + #:fixed_output (if fixed-output? 1 0)))))) + + (define (insert-derivation) + (let ((statement + (sqlite-prepare + db + " +INSERT INTO derivations (name, system_id, fixed_output) VALUES (:name, :system_id, :fixed_output)" - #:cache? #t))) - - (sqlite-bind-arguments - statement - #:name derivation-name - #:system_id - (let ((system (derivation-system derivation))) - (or - (db-system->system-id db system) - (db-insert-system db system))) - #:fixed_output (if fixed-output? 1 0)) - - (sqlite-step statement) - (sqlite-reset statement) + #:cache? #t))) + + (sqlite-bind-arguments + statement + #:name derivation-name + #:system_id + (let ((system (derivation-system derivation))) + (or + (db-system->system-id db system) + (db-insert-system db system))) + #:fixed_output (if (fixed-output-derivation? derivation) 1 0)) - (changes-count db))))) + (sqlite-step statement) + (sqlite-reset statement) - (call-with-delay-logging - (lambda () - (let ((changes (insert-derivation))) - (unless (eq? changes 0) - (insert-derivation-inputs - db - derivation-name - (derivation-inputs derivation)) + (last-insert-rowid db))) - (insert-derivation-outputs - db - derivation-name - (derivation-outputs derivation))) - (select-derivation-outputs db derivation-name))))) + (or (hash-ref %derivation-outputs-cache + derivation-name) + (let ((derivation-details + (db-find-derivation db derivation-name))) + (if derivation-details + (begin + (maybe-fix-fixed-output-field derivation-details) + + (let ((derivation-outputs + (select-derivation-outputs db derivation-name))) + + (hash-set! %derivation-outputs-cache + derivation-name + derivation-outputs) + + derivation-outputs)) + (let ((derivation-id (insert-derivation))) + (insert-derivation-inputs + db + derivation-id + (derivation-inputs derivation)) + + (let ((derivation-outputs + (insert-derivation-outputs + db + derivation-id + (derivation-outputs derivation)))) + (hash-set! %derivation-outputs-cache + derivation-name + derivation-outputs) -(define (insert-derivation-inputs db derivation-name derivation-inputs) + derivation-outputs)))))) + +(define (insert-derivation-inputs db derivation-id derivation-inputs) (unless (null? derivation-inputs) (let ((derivation-output-ids (append-map @@ -3414,7 +3434,7 @@ VALUES (:derivation_id, :derivation_output_id)" (for-each (lambda (derivation-output-id) (sqlite-bind-arguments statement - #:derivation_id (db-find-derivation-id db derivation-name) + #:derivation_id derivation-id #:derivation_output_id derivation-output-id) (sqlite-step statement) (sqlite-reset statement)) @@ -3453,7 +3473,7 @@ SELECT id FROM outputs WHERE output = :output" result))) -(define (insert-derivation-outputs db derivation-name derivation-outputs) +(define (insert-derivation-outputs db derivation-id derivation-outputs) (define output-has-successful-build? (let ((statement (sqlite-prepare @@ -3520,7 +3540,7 @@ INSERT INTO derivation_outputs (derivation_id, name, output_id) ((name . output-id) (sqlite-bind-arguments statement - #:derivation_id (db-find-derivation-id db derivation-name) + #:derivation_id derivation-id #:name name #:output_id output-id) (sqlite-step statement) @@ -3530,8 +3550,9 @@ INSERT INTO derivation_outputs (derivation_id, name, output_id) (for-each (lambda (output-id) (unless (output-has-successful-build? output-id) (insert-into-unbuilt-outputs output-id))) - (map cdr derivation-outputs-with-ids))) - #t) + (map cdr derivation-outputs-with-ids)) + + derivation-outputs-with-ids)) (define-method (datastore-insert-build (datastore <sqlite-datastore>) |