diff options
| author | Christopher Baines <mail@cbaines.net> | 2022-01-14 15:25:53 +0000 |
|---|---|---|
| committer | Christopher Baines <mail@cbaines.net> | 2022-01-14 15:25:53 +0000 |
| commit | a7c9daab6ace33858be1dcde60f147c3c84afa81 (patch) | |
| tree | 0ed614a193b3c4057882b11b640a9ee923d13931 /guix-data-service/model/derivation.scm | |
| parent | 5a1b6d41eb35711ef89531e0cbd73dd7223419ec (diff) | |
| download | data-service-a7c9daab6ace33858be1dcde60f147c3c84afa81.tar data-service-a7c9daab6ace33858be1dcde60f147c3c84afa81.tar.gz | |
Process derivations in chunks
Which should reduce the peak memory usage.
Diffstat (limited to 'guix-data-service/model/derivation.scm')
| -rw-r--r-- | guix-data-service/model/derivation.scm | 42 |
1 files changed, 28 insertions, 14 deletions
diff --git a/guix-data-service/model/derivation.scm b/guix-data-service/model/derivation.scm index fddf0e3..a1ae821 100644 --- a/guix-data-service/model/derivation.scm +++ b/guix-data-service/model/derivation.scm @@ -1801,20 +1801,34 @@ INNER JOIN derivation_source_files derivation-ids-hash-table derivation-file-names) - (let ((missing-derivations - (with-time-logging "reading missing derivations" - (map read-derivation-from-file - (deduplicate-strings - (filter (lambda (derivation-file-name) - (not (hash-ref derivation-ids-hash-table - derivation-file-name))) - derivation-file-names)))))) - - (unless (null? missing-derivations) - (insert-missing-derivations conn - derivation-ids-hash-table - missing-derivations)) - + (let ((missing-derivation-filenames + (deduplicate-strings + (filter (lambda (derivation-file-name) + (not (hash-ref derivation-ids-hash-table + derivation-file-name))) + derivation-file-names)))) + + (chunk-for-each! + (lambda (missing-derivation-filenames-chunk) + (let ((missing-derivations-chunk + (with-time-logging + (simple-format #f "reading ~A missing derivations" + (length missing-derivation-filenames-chunk)) + (map read-derivation-from-file + ;; Do the filter again, since processing the last + ;; chunk might have inserted some of the + ;; derivations in this chunk + (filter (lambda (derivation-file-name) + (not (hash-ref derivation-ids-hash-table + derivation-file-name))) + missing-derivation-filenames-chunk))))) + + (unless (null? missing-derivations-chunk) + (insert-missing-derivations conn + derivation-ids-hash-table + missing-derivations-chunk)))) + 1000 + missing-derivation-filenames) (let ((all-ids (map (lambda (derivation-file-name) |