diff options
| author | Christopher Baines <mail@cbaines.net> | 2024-01-18 14:44:04 +0000 |
|---|---|---|
| committer | Christopher Baines <mail@cbaines.net> | 2024-01-18 15:34:40 +0000 |
| commit | c1d2f3a1b7ba20ad3f56840a160ee1ec08cd3a64 (patch) | |
| tree | a8ac56ddf1ff543a380938d7c9c3173c854fce6b /scripts | |
| parent | 6842a432d668f3e42de0b416813f359beef9ae6f (diff) | |
| download | data-service-c1d2f3a1b7ba20ad3f56840a160ee1ec08cd3a64.tar data-service-c1d2f3a1b7ba20ad3f56840a160ee1ec08cd3a64.tar.gz | |
Add meaningful parallelism to processing jobs
Make parallel use of inferiors when computing channel instance derivations,
and when extracting information about a revision. This should allow for some
horizontal scalability, reducing the impact of additional systems for which
derivations need computing.
This commit also fixes an apparent issue with package replacements, as
previously the wrong id was used, and this hid some issues around
deduplication.
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/guix-data-service-process-job.in | 5 | ||||
| -rw-r--r-- | scripts/guix-data-service-process-jobs.in | 14 |
2 files changed, 14 insertions, 5 deletions
diff --git a/scripts/guix-data-service-process-job.in b/scripts/guix-data-service-process-job.in index fd7ab3f..633d8db 100644 --- a/scripts/guix-data-service-process-job.in +++ b/scripts/guix-data-service-process-job.in @@ -78,7 +78,8 @@ (lambda () (process-load-new-guix-revision-job job - #:skip-system-tests? (assq-ref opts 'skip-system-tests))) + #:skip-system-tests? (assq-ref opts 'skip-system-tests) + #:parallelism (assq-ref opts 'parallelism))) #:hz 0 - #:parallelism (assq-ref opts 'parallelism) + #:parallelism 1 #:drain? #t))))) diff --git a/scripts/guix-data-service-process-jobs.in b/scripts/guix-data-service-process-jobs.in index 6ad1ec9..da4f614 100644 --- a/scripts/guix-data-service-process-jobs.in +++ b/scripts/guix-data-service-process-jobs.in @@ -44,11 +44,17 @@ result))) (option '("skip-system-tests") #f #f (lambda (opt name _ result) - (alist-cons 'skip-system-tests #t result))))) + (alist-cons 'skip-system-tests #t result))) + (option '("per-job-parallelism") #t #f + (lambda (opt name arg result) + (alist-cons 'per-job-parallelism + (string->number arg) + result))))) (define %default-options ;; Alist of default option values - `((max-processes . ,default-max-processes))) + `((max-processes . ,default-max-processes) + (per-job-parallelism . 1))) (define (parse-options args) (args-fold @@ -77,4 +83,6 @@ (or (assq-ref opts 'latest-branch-revision-max-processes) (* 2 (assq-ref opts 'max-processes))) #:skip-system-tests? - (assq-ref opts 'skip-system-tests))))) + (assq-ref opts 'skip-system-tests) + #:per-job-parallelism + (assq-ref opts 'per-job-parallelism))))) |