diff options
author | Christopher Baines <mail@cbaines.net> | 2022-09-17 00:41:14 +0200 |
---|---|---|
committer | Christopher Baines <mail@cbaines.net> | 2022-09-17 08:53:23 +0200 |
commit | 78a5abee21d9e507fbacf443c411cfe095889b03 (patch) | |
tree | 4135f2ff17080f2addc63e25597c6af7a5959a17 | |
parent | 7050ea749f8ac6e7c75ab20057d6871d0ff9db49 (diff) | |
download | data-service-78a5abee21d9e507fbacf443c411cfe095889b03.tar data-service-78a5abee21d9e507fbacf443c411cfe095889b03.tar.gz |
Improve chunking when inserting derivation inputs
Chunk the values inserted in the query, rather than the derivations involved,
as this is more consistent.
-rw-r--r-- | guix-data-service/model/derivation.scm | 73 |
1 files changed, 35 insertions, 38 deletions
diff --git a/guix-data-service/model/derivation.scm b/guix-data-service/model/derivation.scm index f037b50..7a4b214 100644 --- a/guix-data-service/model/derivation.scm +++ b/guix-data-service/model/derivation.scm @@ -1381,51 +1381,48 @@ WHERE derivation_source_files.store_path = $1" #f))) (define (insert-derivation-inputs conn derivation-ids derivations) - (define (process-chunk derivation-ids derivations) - (let ((query-parts - (append-map! - (lambda (derivation-id derivation) - (append-map! - (match-lambda - (($ <derivation-input> derivation-or-path sub-derivations) - (let ((path - (match derivation-or-path - ((? derivation? d) - ;; The first field changed to a derivation (from the file - ;; name) in 5cf4b26d52bcea382d98fb4becce89be9ee37b55 - (derivation-file-name d)) - ((? string? s) - s)))) - (map (lambda (sub-derivation) - (string-append "(" - (number->string derivation-id) - ", '" path - "', '" sub-derivation "')")) - sub-derivations)))) - (derivation-inputs derivation))) - derivation-ids - derivations))) - - (unless (null? query-parts) - (exec-query - conn - (string-append - " + (let ((query-parts + (append-map! + (lambda (derivation-id derivation) + (append-map! + (match-lambda + (($ <derivation-input> derivation-or-path sub-derivations) + (let ((path + (match derivation-or-path + ((? derivation? d) + ;; The first field changed to a derivation (from the file + ;; name) in 5cf4b26d52bcea382d98fb4becce89be9ee37b55 + (derivation-file-name d)) + ((? string? s) + s)))) + (map (lambda (sub-derivation) + (string-append "(" + (number->string derivation-id) + ", '" path + "', '" sub-derivation "')")) + sub-derivations)))) + (derivation-inputs derivation))) + derivation-ids + derivations))) + + (chunk-for-each! + (lambda (query-parts-chunk) + (exec-query + conn + (string-append + " INSERT INTO derivation_inputs (derivation_id, derivation_output_id) SELECT vals.derivation_id, derivation_outputs.id FROM (VALUES " - (string-join query-parts ", ") - ") AS vals (derivation_id, file_name, output_name) + (string-join query-parts-chunk ", ") + ") AS vals (derivation_id, file_name, output_name) INNER JOIN derivations ON derivations.file_name = vals.file_name INNER JOIN derivation_outputs ON derivation_outputs.derivation_id = derivations.id - AND vals.output_name = derivation_outputs.name"))))) - - (chunk-for-each! process-chunk - 200 - (list-copy derivation-ids) - (list-copy derivations))) + AND vals.output_name = derivation_outputs.name"))) + 1000 + query-parts))) (define (select-from-derivation-source-files store-paths) (string-append |