From b26ba82dadf3f1b9fee1498eeca263940cb71ce4 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Sat, 4 Jan 2020 11:20:02 +0000 Subject: Add more timing and debugging output to the load process To help identify what's taking the most time. --- guix-data-service/jobs/load-new-guix-revision.scm | 9 +++-- guix-data-service/model/derivation.scm | 48 ++++++++++++----------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/guix-data-service/jobs/load-new-guix-revision.scm b/guix-data-service/jobs/load-new-guix-revision.scm index 9c1405e..038d74f 100644 --- a/guix-data-service/jobs/load-new-guix-revision.scm +++ b/guix-data-service/jobs/load-new-guix-revision.scm @@ -916,9 +916,12 @@ WHERE job_id = $1" guix-revision-id lint-warning-ids))) (let ((package-derivation-ids - (inferior-data->package-derivation-ids - conn inf inferior-package-id->package-database-id - inferior-data-4-tuples))) + (log-time + "inferior-data->package-derivation-ids" + (lambda () + (inferior-data->package-derivation-ids + conn inf inferior-package-id->package-database-id + inferior-data-4-tuples))))) (update-builds-derivation-output-details-set-id conn (map fourth inferior-data-4-tuples)) diff --git a/guix-data-service/model/derivation.scm b/guix-data-service/model/derivation.scm index d2ed2fa..8ee1dd4 100644 --- a/guix-data-service/model/derivation.scm +++ b/guix-data-service/model/derivation.scm @@ -1104,24 +1104,29 @@ LIMIT $1" (simple-format #t "debug: ensure-input-derivations-exist: processing ~A derivations\n" (length input-derivation-file-names)) - (let* ((existing-derivation-entries - (derivation-file-names->vhash conn - derivation-ids-hash-table - input-derivation-file-names)) - - (missing-derivations-filenames - (filter (lambda (derivation-file-name) - (not (vhash-assoc derivation-file-name - existing-derivation-entries))) - input-derivation-file-names))) - - (unless (null? missing-derivations-filenames) - ;; Ensure all the input derivations exist - (insert-missing-derivations - conn - derivation-ids-hash-table - (map read-derivation-from-file - missing-derivations-filenames)))))) + (let ((existing-derivation-entries + (derivation-file-names->vhash conn + derivation-ids-hash-table + input-derivation-file-names))) + (simple-format + #t + "debug: ensure-input-derivations-exist: checking for missing input derivations\n") + (let ((missing-derivations-filenames + (filter (lambda (derivation-file-name) + (not (vhash-assoc derivation-file-name + existing-derivation-entries))) + input-derivation-file-names))) + + (unless (null? missing-derivations-filenames) + (simple-format + #f + "debug: ensure-input-derivations-exist: inserting missing input derivations\n") + ;; Ensure all the input derivations exist + (insert-missing-derivations + conn + derivation-ids-hash-table + (map read-derivation-from-file + missing-derivations-filenames))))))) (define (insert-into-derivations) (string-append @@ -1332,13 +1337,10 @@ WHERE " criteria ";")) (simple-format #t "debug: derivation-file-names->vhash: adding ~A entries to the cache\n" (vlist-length result-for-missing-file-names)) - (vhash-fold (lambda (key value _) - (hash-set! derivation-ids-hash-table key value)) - '() - result-for-missing-file-names) - (vhash-fold (lambda (key value combined) + ;; Update the derivation-ids-hash-table as we go through the vhash + (hash-set! derivation-ids-hash-table key value) (vhash-cons key value combined)) result result-for-missing-file-names))))) -- cgit v1.2.3