diff options
author | Christopher Baines <mail@cbaines.net> | 2023-07-09 16:52:35 +0100 |
---|---|---|
committer | Christopher Baines <mail@cbaines.net> | 2023-07-10 18:56:31 +0100 |
commit | 7251c7d653de29f36d50b33badf05a5db983b8e7 (patch) | |
tree | 3f74252cf1f0d13d35dc1253406d9a3b92b67f7e /guix-data-service/web/compare | |
parent | 672ee6216e1d15f7f550f53017323b59f05303cb (diff) | |
download | data-service-7251c7d653de29f36d50b33badf05a5db983b8e7.tar data-service-7251c7d653de29f36d50b33badf05a5db983b8e7.tar.gz |
Stop using a pool of threads for database operations
Now that squee cooperates with suspendable ports, this is unnecessary. Use a
connection pool to still support running queries in parallel using multiple
connections.
Diffstat (limited to 'guix-data-service/web/compare')
-rw-r--r-- | guix-data-service/web/compare/controller.scm | 512 |
1 files changed, 235 insertions, 277 deletions
diff --git a/guix-data-service/web/compare/controller.scm b/guix-data-service/web/compare/controller.scm index 3d96aa4..6380651 100644 --- a/guix-data-service/web/compare/controller.scm +++ b/guix-data-service/web/compare/controller.scm @@ -30,6 +30,7 @@ #:use-module (guix-data-service web util) #:use-module (guix-data-service web render) #:use-module (guix-data-service web query-parameters) + #:use-module (guix-data-service web controller) #:use-module (guix-data-service model utils) #:use-module (guix-data-service comparison) #:use-module (guix-data-service jobs load-new-guix-revision) @@ -55,42 +56,38 @@ s) (define (parse-commit s) - (parallel-via-thread-pool-channel - (with-thread-postgresql-connection - (lambda (conn) - (let* ((job-details - (select-job-for-commit conn s)) - (job-state - (assq-ref job-details 'state))) - (if job-details - (cond - ((string=? job-state "succeeded") - s) - ((string=? job-state "queued") - (make-invalid-query-parameter - s - `("data unavailable, " - (a (@ (href ,(string-append - "/revision/" s))) - "yet to process revision")))) - ((string=? job-state "failed") - (make-invalid-query-parameter - s - `("data unavailable, " - (a (@ (href ,(string-append - "/revision/" s))) - "failed to process revision")))) - (else - (make-invalid-query-parameter - s "unknown job state"))) + (with-resource-from-pool (connection-pool) conn + (let* ((job-details + (select-job-for-commit conn s)) + (job-state + (assq-ref job-details 'state))) + (if job-details + (cond + ((string=? job-state "succeeded") + s) + ((string=? job-state "queued") (make-invalid-query-parameter - s "unknown commit"))))))) + s + `("data unavailable, " + (a (@ (href ,(string-append + "/revision/" s))) + "yet to process revision")))) + ((string=? job-state "failed") + (make-invalid-query-parameter + s + `("data unavailable, " + (a (@ (href ,(string-append + "/revision/" s))) + "failed to process revision")))) + (else + (make-invalid-query-parameter + s "unknown job state"))) + (make-invalid-query-parameter + s "unknown commit"))))) (define (parse-derivation file-name) - (if (parallel-via-thread-pool-channel - (with-thread-postgresql-connection - (lambda (conn) - (select-derivation-by-file-name conn file-name)))) + (if (with-resource-from-pool (connection-pool) conn + (select-derivation-by-file-name conn file-name)) file-name (make-invalid-query-parameter file-name "unknown derivation"))) @@ -235,18 +232,16 @@ (letpar& ((base-job (match (assq-ref query-parameters 'base_commit) (($ <invalid-query-parameter> value) - (with-thread-postgresql-connection - (lambda (conn) - (and (string? value) - (select-job-for-commit conn value))))) + (with-resource-from-pool (connection-pool) conn + (and (string? value) + (select-job-for-commit conn value)))) (_ #f))) (target-job (match (assq-ref query-parameters 'target_commit) (($ <invalid-query-parameter> value) - (with-thread-postgresql-connection - (lambda (conn) - (and (string? value) - (select-job-for-commit conn value))))) + (with-resource-from-pool (connection-pool) conn + (and (string? value) + (select-job-for-commit conn value)))) (_ #f)))) (case (most-appropriate-mime-type '(application/json text/html) @@ -281,28 +276,24 @@ #f #f))))) (letpar& ((base-revision-id - (with-thread-postgresql-connection - (lambda (conn) - (commit->revision-id - conn - (assq-ref query-parameters 'base_commit))))) + (with-resource-from-pool (connection-pool) conn + (commit->revision-id + conn + (assq-ref query-parameters 'base_commit)))) (target-revision-id - (with-thread-postgresql-connection - (lambda (conn) - (commit->revision-id - conn - (assq-ref query-parameters 'target_commit))))) + (with-resource-from-pool (connection-pool) conn + (commit->revision-id + conn + (assq-ref query-parameters 'target_commit)))) (locale (assq-ref query-parameters 'locale))) (let-values (((base-packages-vhash target-packages-vhash) (package-data->package-data-vhashes - (parallel-via-thread-pool-channel - (with-thread-postgresql-connection - (lambda (conn) - (package-differences-data conn - base-revision-id - target-revision-id))))))) + (with-resource-from-pool (connection-pool) conn + (package-differences-data conn + base-revision-id + target-revision-id))))) (let ((new-packages (package-data-vhashes->new-packages base-packages-vhash target-packages-vhash)) @@ -313,20 +304,18 @@ (package-data-version-changes base-packages-vhash target-packages-vhash))) (letpar& ((lint-warnings-data - (with-thread-postgresql-connection - (lambda (conn) - (group-list-by-first-n-fields - 2 - (lint-warning-differences-data conn - base-revision-id - target-revision-id - locale))))) - (channel-news-data - (with-thread-postgresql-connection - (lambda (conn) - (channel-news-differences-data conn + (with-resource-from-pool (connection-pool) conn + (group-list-by-first-n-fields + 2 + (lint-warning-differences-data conn base-revision-id - target-revision-id))))) + target-revision-id + locale)))) + (channel-news-data + (with-resource-from-pool (connection-pool) conn + (channel-news-differences-data conn + base-revision-id + target-revision-id)))) (case (most-appropriate-mime-type '(application/json text/html) mime-types) @@ -412,18 +401,16 @@ (match-lambda ((locale) locale)) - (with-thread-postgresql-connection - (lambda (conn) - (lint-warning-message-locales-for-revision - conn - (assq-ref query-parameters 'target_commit)))))) - (cgit-url-bases - (with-thread-postgresql-connection - (lambda (conn) - (guix-revisions-cgit-url-bases + (with-resource-from-pool (connection-pool) conn + (lint-warning-message-locales-for-revision conn - (list base-revision-id - target-revision-id)))))) + (assq-ref query-parameters 'target_commit))))) + (cgit-url-bases + (with-resource-from-pool (connection-pool) conn + (guix-revisions-cgit-url-bases + conn + (list base-revision-id + target-revision-id))))) (render-html #:sxml (compare query-parameters 'revision @@ -463,29 +450,26 @@ (target-datetime (assq-ref query-parameters 'target_datetime)) (locale (assq-ref query-parameters 'locale))) (letpar& ((base-revision-details - (with-thread-postgresql-connection - (lambda (conn) - (select-guix-revision-for-branch-and-datetime - conn - base-branch - base-datetime)))) + (with-resource-from-pool (connection-pool) conn + (select-guix-revision-for-branch-and-datetime + conn + base-branch + base-datetime))) (target-revision-details - (with-thread-postgresql-connection - (lambda (conn) - (select-guix-revision-for-branch-and-datetime - conn - target-branch - target-datetime))))) - (letpar& ((lint-warnings-locale-options - (map - (match-lambda - ((locale) - locale)) - (with-thread-postgresql-connection - (lambda (conn) - (lint-warning-message-locales-for-revision - conn - (second base-revision-details))))))) + (with-resource-from-pool (connection-pool) conn + (select-guix-revision-for-branch-and-datetime + conn + target-branch + target-datetime)))) + (let ((lint-warnings-locale-options + (map + (match-lambda + ((locale) + locale)) + (with-resource-from-pool (connection-pool) conn + (lint-warning-message-locales-for-revision + conn + (second base-revision-details)))))) (let ((base-revision-id (first base-revision-details)) (target-revision-id @@ -493,12 +477,10 @@ (let-values (((base-packages-vhash target-packages-vhash) (package-data->package-data-vhashes - (parallel-via-thread-pool-channel - (with-thread-postgresql-connection - (lambda (conn) - (package-differences-data conn - base-revision-id - target-revision-id))))))) + (with-resource-from-pool (connection-pool) conn + (package-differences-data conn + base-revision-id + target-revision-id))))) (let* ((new-packages (package-data-vhashes->new-packages base-packages-vhash target-packages-vhash)) @@ -509,12 +491,10 @@ (package-data-version-changes base-packages-vhash target-packages-vhash)) (channel-news-data - (parallel-via-thread-pool-channel - (with-thread-postgresql-connection - (lambda (conn) - (channel-news-differences-data conn - base-revision-id - target-revision-id)))))) + (with-resource-from-pool (connection-pool) conn + (channel-news-differences-data conn + base-revision-id + target-revision-id)))) (case (most-appropriate-mime-type '(application/json text/html) mime-types) @@ -567,32 +547,29 @@ #:extra-headers http-headers-for-unchanging-content)) (else (render-html - #:sxml (compare `(,@query-parameters - (base_commit . ,(second base-revision-details)) - (target_commit . ,(second target-revision-details))) - 'datetime - (parallel-via-thread-pool-channel - (with-thread-postgresql-connection - (lambda (conn) - (guix-revisions-cgit-url-bases - conn - (list base-revision-id - target-revision-id))))) - new-packages - removed-packages - version-changes - (parallel-via-thread-pool-channel - (group-list-by-first-n-fields - 2 - (with-thread-postgresql-connection - (lambda (conn) - (lint-warning-differences-data - conn - base-revision-id - target-revision-id - locale))))) - lint-warnings-locale-options - channel-news-data) + #:sxml (compare + `(,@query-parameters + (base_commit . ,(second base-revision-details)) + (target_commit . ,(second target-revision-details))) + 'datetime + (with-resource-from-pool (connection-pool) conn + (guix-revisions-cgit-url-bases + conn + (list base-revision-id + target-revision-id))) + new-packages + removed-packages + version-changes + (group-list-by-first-n-fields + 2 + (with-resource-from-pool (connection-pool) conn + (lint-warning-differences-data + conn + base-revision-id + target-revision-id + locale))) + lint-warnings-locale-options + channel-news-data) #:extra-headers http-headers-for-unchanging-content))))))))))) (define (render-compare/derivation mime-types @@ -612,12 +589,11 @@ (let ((base-derivation (assq-ref query-parameters 'base_derivation)) (target-derivation (assq-ref query-parameters 'target_derivation))) - (letpar& ((data - (with-thread-postgresql-connection - (lambda (conn) - (derivation-differences-data conn - base-derivation - target-derivation))))) + (let ((data + (with-resource-from-pool (connection-pool) conn + (derivation-differences-data conn + base-derivation + target-derivation)))) (case (most-appropriate-mime-type '(application/json text/html) mime-types) @@ -655,9 +631,8 @@ ((? string? value) value) (_ #f)) (lambda (commit) - (with-thread-postgresql-connection - (lambda (conn) - (select-job-for-commit conn commit)))))) + (with-resource-from-pool (connection-pool) conn + (select-job-for-commit conn commit))))) (target-job (and=> (match (assq-ref query-parameters 'target_commit) (($ <invalid-query-parameter> value) @@ -665,9 +640,8 @@ ((? string? value) value) (_ #f)) (lambda (commit) - (with-thread-postgresql-connection - (lambda (conn) - (select-job-for-commit conn commit))))))) + (with-resource-from-pool (connection-pool) conn + (select-job-for-commit conn commit)))))) (render-json `((error . "invalid query") (query_parameters @@ -690,14 +664,14 @@ (target_job . ,target-job))))) (else (letpar& ((systems - (with-thread-postgresql-connection - list-systems)) + (call-with-resource-from-pool (connection-pool) + list-systems)) (targets - (with-thread-postgresql-connection - valid-targets)) + (call-with-resource-from-pool (connection-pool) + valid-targets)) (build-server-urls - (with-thread-postgresql-connection - select-build-server-urls-by-id))) + (call-with-resource-from-pool (connection-pool) + select-build-server-urls-by-id))) (render-html #:sxml (compare/package-derivations query-parameters @@ -718,19 +692,18 @@ (after-name (assq-ref query-parameters 'after_name)) (limit-results (assq-ref query-parameters 'limit_results))) (letpar& ((data - (with-thread-postgresql-connection - (lambda (conn) - (package-derivation-differences-data - conn - (commit->revision-id conn base-commit) - (commit->revision-id conn target-commit) - #:systems systems - #:targets targets - #:build-change build-change - #:after-name after-name - #:limit-results limit-results)))) + (with-resource-from-pool (connection-pool) conn + (package-derivation-differences-data + conn + (commit->revision-id conn base-commit) + (commit->revision-id conn target-commit) + #:systems systems + #:targets targets + #:build-change build-change + #:after-name after-name + #:limit-results limit-results))) (build-server-urls - (with-thread-postgresql-connection + (with-resource-from-pool (connection-pool) conn select-build-server-urls-by-id))) (let ((names-and-versions (package-derivation-data->names-and-versions data))) @@ -755,11 +728,11 @@ . ,derivation-changes)))) (else (letpar& ((systems - (with-thread-postgresql-connection - list-systems)) + (call-with-resource-from-pool (connection-pool) + list-systems)) (targets - (with-thread-postgresql-connection - valid-targets))) + (call-with-resource-from-pool (connection-pool) + valid-targets))) (render-html #:sxml (compare/package-derivations query-parameters @@ -784,11 +757,11 @@ #:sxml (compare/package-derivations query-parameters 'datetime - (parallel-via-thread-pool-channel - (with-thread-postgresql-connection list-systems)) + (call-with-resource-from-pool (connection-pool) + list-systems) (valid-targets->options - (parallel-via-thread-pool-channel - (with-thread-postgresql-connection valid-targets))) + (call-with-resource-from-pool (connection-pool) + valid-targets)) build-status-strings '() '() @@ -807,30 +780,27 @@ (limit-results (assq-ref query-parameters 'limit_results))) (letpar& ((base-revision-details - (with-thread-postgresql-connection - (lambda (conn) - (select-guix-revision-for-branch-and-datetime conn - base-branch - base-datetime)))) + (with-resource-from-pool (connection-pool) conn + (select-guix-revision-for-branch-and-datetime conn + base-branch + base-datetime))) (target-revision-details - (with-thread-postgresql-connection - (lambda (conn) - (select-guix-revision-for-branch-and-datetime conn - target-branch - target-datetime))))) + (with-resource-from-pool (connection-pool) conn + (select-guix-revision-for-branch-and-datetime conn + target-branch + target-datetime)))) (letpar& ((data - (with-thread-postgresql-connection - (lambda (conn) - (package-derivation-differences-data - conn - (first base-revision-details) - (first target-revision-details) - #:systems systems - #:targets targets - #:build-change build-change - #:after-name after-name - #:limit-results limit-results))))) + (with-resource-from-pool (connection-pool) conn + (package-derivation-differences-data + conn + (first base-revision-details) + (first target-revision-details) + #:systems systems + #:targets targets + #:build-change build-change + #:after-name after-name + #:limit-results limit-results)))) (let ((names-and-versions (package-derivation-data->names-and-versions data))) (let-values @@ -859,15 +829,17 @@ #:sxml (compare/package-derivations query-parameters 'datetime - (parallel-via-thread-pool-channel - (with-thread-postgresql-connection list-systems)) + (call-with-resource-from-pool + (connection-pool) + list-systems) (valid-targets->options - (parallel-via-thread-pool-channel - (with-thread-postgresql-connection valid-targets))) + (call-with-resource-from-pool + (connection-pool) + valid-targets)) build-status-strings - (parallel-via-thread-pool-channel - (with-thread-postgresql-connection - select-build-server-urls-by-id)) + (call-with-resource-from-pool + (connection-pool) + select-build-server-urls-by-id) derivation-changes base-revision-details target-revision-details)))))))))))) @@ -894,16 +866,14 @@ (letpar& ((base-job (match (assq-ref query-parameters 'base_commit) (($ <invalid-query-parameter> value) - (with-thread-postgresql-connection - (lambda (conn) - (select-job-for-commit conn value)))) + (with-resource-from-pool (connection-pool) conn + (select-job-for-commit conn value))) (_ #f))) (target-job (match (assq-ref query-parameters 'target_commit) (($ <invalid-query-parameter> value) - (with-thread-postgresql-connection - (lambda (conn) - (select-job-for-commit conn value)))) + (with-resource-from-pool (connection-pool) conn + (select-job-for-commit conn value))) (_ #f)))) (render-html #:sxml (compare-invalid-parameters @@ -914,26 +884,22 @@ (let ((base-commit (assq-ref query-parameters 'base_commit)) (target-commit (assq-ref query-parameters 'target_commit))) (letpar& ((base-revision-id - (with-thread-postgresql-connection - (lambda (conn) - (commit->revision-id - conn - base-commit)))) + (with-resource-from-pool (connection-pool) conn + (commit->revision-id + conn + base-commit))) (target-revision-id - (with-thread-postgresql-connection - (lambda (conn) - (commit->revision-id - conn - target-commit))))) + (with-resource-from-pool (connection-pool) conn + (commit->revision-id + conn + target-commit)))) (let-values (((base-packages-vhash target-packages-vhash) (package-data->package-data-vhashes - (parallel-via-thread-pool-channel - (with-thread-postgresql-connection - (lambda (conn) - (package-differences-data conn - base-revision-id - target-revision-id))))))) + (with-resource-from-pool (connection-pool) conn + (package-differences-data conn + base-revision-id + target-revision-id))))) (case (most-appropriate-mime-type '(application/json text/html) mime-types) @@ -967,10 +933,10 @@ '((error . "invalid query")))) (else (letpar& ((systems - (with-thread-postgresql-connection + (with-resource-from-pool (connection-pool) conn list-systems)) (build-server-urls - (with-thread-postgresql-connection + (with-resource-from-pool (connection-pool) conn select-build-server-urls-by-id))) (render-html #:sxml (compare/system-test-derivations @@ -986,26 +952,23 @@ (target-commit (assq-ref query-parameters 'target_commit)) (system (assq-ref query-parameters 'system))) (letpar& ((data - (with-thread-postgresql-connection - (lambda (conn) - (system-test-derivations-differences-data - conn - (commit->revision-id conn base-commit) - (commit->revision-id conn target-commit) - system)))) + (with-resource-from-pool (connection-pool) conn + (system-test-derivations-differences-data + conn + (commit->revision-id conn base-commit) + (commit->revision-id conn target-commit) + system))) (build-server-urls - (with-thread-postgresql-connection + (with-resource-from-pool (connection-pool) conn select-build-server-urls-by-id)) (base-git-repositories - (with-thread-postgresql-connection - (lambda (conn) - (git-repositories-containing-commit conn base-commit)))) + (with-resource-from-pool (connection-pool) conn + (git-repositories-containing-commit conn base-commit))) (target-git-repositories - (with-thread-postgresql-connection - (lambda (conn) - (git-repositories-containing-commit conn target-commit)))) + (with-resource-from-pool (connection-pool) conn + (git-repositories-containing-commit conn target-commit))) (systems - (with-thread-postgresql-connection + (with-resource-from-pool (connection-pool) conn list-systems))) (case (most-appropriate-mime-type '(application/json text/html) @@ -1040,10 +1003,10 @@ '((error . "invalid query")))) (else (letpar& ((systems - (with-thread-postgresql-connection + (with-resource-from-pool (connection-pool) conn list-systems)) (build-server-urls - (with-thread-postgresql-connection + (with-resource-from-pool (connection-pool) conn select-build-server-urls-by-id))) (render-html #:sxml (compare/system-test-derivations @@ -1062,42 +1025,37 @@ (system (assq-ref query-parameters 'system))) (letpar& ((base-revision-details - (with-thread-postgresql-connection - (lambda (conn) - (select-guix-revision-for-branch-and-datetime conn - base-branch - base-datetime)))) + (with-resource-from-pool (connection-pool) conn + (select-guix-revision-for-branch-and-datetime conn + base-branch + base-datetime))) (target-revision-details - (with-thread-postgresql-connection - (lambda (conn) - (select-guix-revision-for-branch-and-datetime conn - target-branch - target-datetime))))) + (with-resource-from-pool (connection-pool) conn + (select-guix-revision-for-branch-and-datetime conn + target-branch + target-datetime)))) (letpar& ((data - (with-thread-postgresql-connection - (lambda (conn) - (system-test-derivations-differences-data - conn - (first base-revision-details) - (first target-revision-details) - system)))) + (with-resource-from-pool (connection-pool) conn + (system-test-derivations-differences-data + conn + (first base-revision-details) + (first target-revision-details) + system))) (build-server-urls - (with-thread-postgresql-connection + (with-resource-from-pool (connection-pool) conn select-build-server-urls-by-id)) (base-git-repositories - (with-thread-postgresql-connection - (lambda (conn) - (git-repositories-containing-commit - conn - (second base-revision-details))))) + (with-resource-from-pool (connection-pool) conn + (git-repositories-containing-commit + conn + (second base-revision-details)))) (target-git-repositories - (with-thread-postgresql-connection - (lambda (conn) - (git-repositories-containing-commit - conn - (second target-revision-details))))) + (with-resource-from-pool (connection-pool) conn + (git-repositories-containing-commit + conn + (second target-revision-details)))) (systems - (with-thread-postgresql-connection + (with-resource-from-pool (connection-pool) conn list-systems))) (case (most-appropriate-mime-type '(application/json text/html) |