Optimise inserting derivations

Avoid some queries by just moving around the values being queried for
better. Also cache the outputs for a derivation so that doesn't have to be
looked up.

1 files changed, 76 insertions, 55 deletions

diff --git a/guix-build-coordinator/datastore/sqlite.scm b/guix-build-coordinator/datastore/sqlite.scm
index 7cc66fe..e64b6c4 100644
--- a/guix-build-coordinator/datastore/sqlite.scm
+++ b/guix-build-coordinator/datastore/sqlite.scm
@@ -769,13 +769,17 @@ INSERT INTO agent_tags (agent_id, tag_id) VALUES (:agent_id, :tag_id)"
             tags)))))
   #t)
 
+(define %derivation-outputs-cache
+  (make-hash-table))
+
 (define-method (datastore-store-derivation
                 (datastore <sqlite-datastore>)
                 derivation)
   (datastore-call-with-transaction
    datastore
    (lambda (db)
-     (insert-derivation-and-return-outputs db derivation))
+     (insert-derivation-and-return-outputs db derivation)
+     (hash-clear! %derivation-outputs-cache))
    #:duration-metric-name "store_derivation")
 
   #t)
@@ -3335,62 +3339,78 @@ SELECT system FROM systems WHERE id = :id"
   (define derivation-name
     (derivation-file-name derivation))
 
-  (define (insert-derivation)
-    (let ((derivation-details (db-find-derivation db derivation-name))
-          (fixed-output? (fixed-output-derivation? derivation)))
-      (if derivation-details
-          (begin
-            (unless (equal? (assq-ref derivation-details 'fixed-output?)
-                            fixed-output?)
-              (let ((statement (sqlite-prepare
-                                db
-                                "
+  (define (maybe-fix-fixed-output-field derivation-details)
+    (let ((fixed-output? (fixed-output-derivation? derivation)))
+      (unless (equal? (assq-ref derivation-details 'fixed-output?)
+                      fixed-output?)
+        (let ((statement (sqlite-prepare
+                          db
+                          "
 UPDATE derivations SET fixed_output = :fixed_output WHERE name = :name"
-                                #:cache? #t)))
-                (sqlite-bind-arguments statement
-                                       #:name derivation-name
-                                       #:fixed_output (if fixed-output? 1 0))))
-            0)
-          (let ((statement
-                 (sqlite-prepare
-                  db
-                  "
-INSERT OR IGNORE INTO derivations (name, system_id, fixed_output)
+                          #:cache? #t)))
+          (sqlite-bind-arguments statement
+                                 #:name derivation-name
+                                 #:fixed_output (if fixed-output? 1 0))))))
+
+  (define (insert-derivation)
+    (let ((statement
+           (sqlite-prepare
+            db
+            "
+INSERT INTO derivations (name, system_id, fixed_output)
   VALUES (:name, :system_id, :fixed_output)"
-                  #:cache? #t)))
-
-            (sqlite-bind-arguments
-             statement
-             #:name derivation-name
-             #:system_id
-             (let ((system (derivation-system derivation)))
-               (or
-                (db-system->system-id db system)
-                (db-insert-system db system)))
-             #:fixed_output (if fixed-output? 1 0))
-
-            (sqlite-step statement)
-            (sqlite-reset statement)
+            #:cache? #t)))
+
+      (sqlite-bind-arguments
+       statement
+       #:name derivation-name
+       #:system_id
+       (let ((system (derivation-system derivation)))
+         (or
+          (db-system->system-id db system)
+          (db-insert-system db system)))
+       #:fixed_output (if (fixed-output-derivation? derivation) 1 0))
 
-            (changes-count db)))))
+      (sqlite-step statement)
+      (sqlite-reset statement)
 
-  (call-with-delay-logging
-   (lambda ()
-     (let ((changes (insert-derivation)))
-       (unless (eq? changes 0)
-         (insert-derivation-inputs
-          db
-          derivation-name
-          (derivation-inputs derivation))
+      (last-insert-rowid db)))
 
-         (insert-derivation-outputs
-          db
-          derivation-name
-          (derivation-outputs derivation)))
 
-       (select-derivation-outputs db derivation-name)))))
+  (or (hash-ref %derivation-outputs-cache
+                derivation-name)
+      (let ((derivation-details
+             (db-find-derivation db derivation-name)))
+        (if derivation-details
+            (begin
+              (maybe-fix-fixed-output-field derivation-details)
+
+              (let ((derivation-outputs
+                     (select-derivation-outputs db derivation-name)))
+
+                (hash-set! %derivation-outputs-cache
+                           derivation-name
+                           derivation-outputs)
+
+                derivation-outputs))
+            (let ((derivation-id (insert-derivation)))
+              (insert-derivation-inputs
+               db
+               derivation-id
+               (derivation-inputs derivation))
+
+              (let ((derivation-outputs
+                     (insert-derivation-outputs
+                      db
+                      derivation-id
+                      (derivation-outputs derivation))))
+                (hash-set! %derivation-outputs-cache
+                           derivation-name
+                           derivation-outputs)
 
-(define (insert-derivation-inputs db derivation-name derivation-inputs)
+                derivation-outputs))))))
+
+(define (insert-derivation-inputs db derivation-id derivation-inputs)
   (unless (null? derivation-inputs)
     (let ((derivation-output-ids
            (append-map
@@ -3414,7 +3434,7 @@ VALUES (:derivation_id, :derivation_output_id)"
         (for-each (lambda (derivation-output-id)
                     (sqlite-bind-arguments
                      statement
-                     #:derivation_id (db-find-derivation-id db derivation-name)
+                     #:derivation_id derivation-id
                      #:derivation_output_id derivation-output-id)
                     (sqlite-step statement)
                     (sqlite-reset statement))
@@ -3453,7 +3473,7 @@ SELECT id FROM outputs WHERE output = :output"
 
       result)))
 
-(define (insert-derivation-outputs db derivation-name derivation-outputs)
+(define (insert-derivation-outputs db derivation-id derivation-outputs)
   (define output-has-successful-build?
     (let ((statement
            (sqlite-prepare
@@ -3520,7 +3540,7 @@ INSERT INTO derivation_outputs (derivation_id, name, output_id)
                 ((name . output-id)
                  (sqlite-bind-arguments
                   statement
-                  #:derivation_id (db-find-derivation-id db derivation-name)
+                  #:derivation_id derivation-id
                   #:name name
                   #:output_id output-id)
                  (sqlite-step statement)
@@ -3530,8 +3550,9 @@ INSERT INTO derivation_outputs (derivation_id, name, output_id)
     (for-each (lambda (output-id)
                 (unless (output-has-successful-build? output-id)
                   (insert-into-unbuilt-outputs output-id)))
-              (map cdr derivation-outputs-with-ids)))
-  #t)
+              (map cdr derivation-outputs-with-ids))
+
+    derivation-outputs-with-ids))
 
 (define-method (datastore-insert-build
                 (datastore <sqlite-datastore>)