diff options
| author | Christopher Baines <mail@cbaines.net> | 2019-03-14 08:18:01 +0000 |
|---|---|---|
| committer | Christopher Baines <mail@cbaines.net> | 2019-03-14 08:18:01 +0000 |
| commit | d2697d8dd2b7394049b129225f7ba6ddf791bf32 (patch) | |
| tree | 7e42dcf6ca7b88c1b154450856d9b90c7735f617 | |
| parent | a0dd2982393fccdb6e04d16d701c8c44235d0b54 (diff) | |
| download | data-service-d2697d8dd2b7394049b129225f7ba6ddf791bf32.tar data-service-d2697d8dd2b7394049b129225f7ba6ddf791bf32.tar.gz | |
Avoid loading in all derivations
Just load in the ones that are not already in the database, as this
will hopefully be faster and require less memory.
| -rw-r--r-- | guix-data-service/jobs/load-new-guix-revision.scm | 7 | ||||
| -rw-r--r-- | guix-data-service/model/derivation.scm | 26 |
2 files changed, 14 insertions, 19 deletions
diff --git a/guix-data-service/jobs/load-new-guix-revision.scm b/guix-data-service/jobs/load-new-guix-revision.scm index 6b54147..e7fe308 100644 --- a/guix-data-service/jobs/load-new-guix-revision.scm +++ b/guix-data-service/jobs/load-new-guix-revision.scm @@ -82,12 +82,9 @@ (close-inferior inf) (let ((derivation-ids - (derivations->derivation-ids + (derivation-file-names->derivation-ids conn - (map (lambda (tuple) - (read-derivation-from-file - (fourth tuple))) - inferior-data-4-tuples))) + (map fourth inferior-data-4-tuples))) (flat-package-ids-systems-and-targets (map (match-lambda diff --git a/guix-data-service/model/derivation.scm b/guix-data-service/model/derivation.scm index b38efc7..3104059 100644 --- a/guix-data-service/model/derivation.scm +++ b/guix-data-service/model/derivation.scm @@ -18,7 +18,7 @@ select-derivations-by-id select-derivations-and-build-status-by-file-name insert-into-derivations - derivations->derivation-ids)) + derivation-file-names->derivation-ids)) (define (select-existing-derivations file-names) (string-append "SELECT id, file_name " @@ -558,28 +558,26 @@ ORDER BY derivations.system DESC, result result-for-missing-file-names))))) -(define (derivations->derivation-ids conn derivations) - (if (null? derivations) +(define (derivation-file-names->derivation-ids conn derivation-file-names) + (if (null? derivation-file-names) '() - (let* ((derivations-count (length derivations)) + (let* ((derivations-count (length derivation-file-names)) (derivation-ids-hash-table (make-hash-table derivations-count))) (simple-format - #t "debug: derivations->derivation-ids: processing ~A derivations\n" + #t "debug: derivation-file-names->derivation-ids: processing ~A derivations\n" derivations-count) - (let* ((derivation-file-names (map derivation-file-name - derivations)) - - (existing-derivation-entries + (let* ((existing-derivation-entries (derivation-file-names->vhash conn derivation-ids-hash-table derivation-file-names)) (missing-derivations - (deduplicate-derivations - (filter (lambda (derivation) - (not (vhash-assoc (derivation-file-name derivation) - existing-derivation-entries))) - derivations))) + (map read-derivation-from-file + (deduplicate-strings + (filter (lambda (derivation-file-name) + (not (vhash-assoc derivation-file-name + existing-derivation-entries))) + derivation-file-names)))) (new-derivation-entries (if (null? missing-derivations) |