aboutsummaryrefslogtreecommitdiff
path: root/guix-data-service/web/revision
diff options
context:
space:
mode:
authorChristopher Baines <mail@cbaines.net>2020-10-03 21:35:31 +0100
committerChristopher Baines <mail@cbaines.net>2020-10-03 21:35:31 +0100
commitc3c9c07f9a208633882a21004d30c5ee29026cb1 (patch)
tree8cc2d34ee6e3be5600200b79ac2487d66c8c32f4 /guix-data-service/web/revision
parente2e55c69de1eceb77998ab059a943711ef7779fd (diff)
downloaddata-service-c3c9c07f9a208633882a21004d30c5ee29026cb1.tar
data-service-c3c9c07f9a208633882a21004d30c5ee29026cb1.tar.gz
Completely rework the way db connections are handled during requests
Previously, a connection was passed through the code handling the request. When queries were performed, this could block the thread though, potentially leaving the server unable to serve other requests. Instead, this now runs queries in a pool of threads. This should remove the possibility of blocking the threads used by the web server, and in doing so, some of the queries have been parallelised. I''m still not sure about the naming and syntax, but I think the functionality is a sort of step forward.
Diffstat (limited to 'guix-data-service/web/revision')
-rw-r--r--guix-data-service/web/revision/controller.scm1131
1 files changed, 630 insertions, 501 deletions
diff --git a/guix-data-service/web/revision/controller.scm b/guix-data-service/web/revision/controller.scm
index be6a4d0..d5049e0 100644
--- a/guix-data-service/web/revision/controller.scm
+++ b/guix-data-service/web/revision/controller.scm
@@ -24,6 +24,8 @@
#:use-module (texinfo html)
#:use-module (texinfo plain-text)
#:use-module (json)
+ #:use-module (guix-data-service utils)
+ #:use-module (guix-data-service database)
#:use-module (guix-data-service web render)
#:use-module (guix-data-service web sxml)
#:use-module (guix-data-service web query-parameters)
@@ -75,52 +77,57 @@
(string-append "unknown build status: "
status))))
-(define (parse-build-server conn)
- (lambda (v)
- (let ((build-servers (select-build-servers conn)))
- (or (any (match-lambda
- ((id url lookup-all-derivations? lookup-builds?)
- (if (eq? (string->number v)
- id)
- id
- #f)))
- build-servers)
- (make-invalid-query-parameter
- v
- "unknown build server")))))
+(define (parse-build-server v)
+ (letpar& ((build-servers
+ (with-thread-postgresql-connection select-build-servers)))
+ (or (any (match-lambda
+ ((id url lookup-all-derivations? lookup-builds?)
+ (if (eq? (string->number v)
+ id)
+ id
+ #f)))
+ build-servers)
+ (make-invalid-query-parameter
+ v
+ "unknown build server"))))
(define (revision-controller request
method-and-path-components
mime-types
- body
- conn)
+ body)
(define path
(uri-path (request-uri request)))
(match method-and-path-components
- (('GET "revision" commit-hash) (if (guix-commit-exists? conn commit-hash)
- (render-view-revision mime-types
- conn
- commit-hash
- #:path-base path)
- (render-unknown-revision mime-types
- conn
- commit-hash)))
+ (('GET "revision" commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
+ (render-view-revision mime-types
+ commit-hash
+ #:path-base path)
+ (render-unknown-revision mime-types
+ commit-hash)))
(('GET "revision" commit-hash "news")
- (if (guix-commit-exists? conn commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
(let ((parsed-query-parameters
(parse-query-parameters
request
`((lang ,identity #:multi-value)))))
(render-revision-news mime-types
- conn
commit-hash
parsed-query-parameters))
(render-unknown-revision mime-types
- conn
commit-hash)))
(('GET "revision" commit-hash "packages")
- (if (guix-commit-exists? conn commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
(let ((parsed-query-parameters
(guard-against-mutually-exclusive-query-parameters
(parse-query-parameters
@@ -140,48 +147,52 @@
(limit_results all_results)))))
(render-revision-packages mime-types
- conn
commit-hash
parsed-query-parameters
#:path-base path))
(render-unknown-revision mime-types
- conn
commit-hash)))
(('GET "revision" commit-hash "packages-translation-availability")
- (if (guix-commit-exists? conn commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
(render-revision-packages-translation-availability mime-types
- conn
commit-hash
#:path-base path)
(render-unknown-revision mime-types
- conn
commit-hash)))
(('GET "revision" commit-hash "package" name)
- (if (guix-commit-exists? conn commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
(render-revision-package mime-types
- conn
commit-hash
name)
(render-unknown-revision mime-types
- conn
commit-hash)))
(('GET "revision" commit-hash "package" name version)
- (if (guix-commit-exists? conn commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
(let ((parsed-query-parameters
(parse-query-parameters
request
`((locale ,identity #:default "en_US.UTF-8")))))
(render-revision-package-version mime-types
- conn
commit-hash
name
version
parsed-query-parameters))
(render-unknown-revision mime-types
- conn
commit-hash)))
(('GET "revision" commit-hash "package-derivations")
- (if (guix-commit-exists? conn commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
(let ((parsed-query-parameters
(guard-against-mutually-exclusive-query-parameters
(parse-query-parameters
@@ -201,15 +212,16 @@
'((limit_results all_results)))))
(render-revision-package-derivations mime-types
- conn
commit-hash
parsed-query-parameters
#:path-base path))
(render-unknown-revision mime-types
- conn
commit-hash)))
(('GET "revision" commit-hash "package-derivation-outputs")
- (if (guix-commit-exists? conn commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
(let ((parsed-query-parameters
(guard-against-mutually-exclusive-query-parameters
(parse-query-parameters
@@ -231,62 +243,67 @@
'((limit_results all_results)))))
(render-revision-package-derivation-outputs mime-types
- conn
commit-hash
parsed-query-parameters
#:path-base path))
(render-unknown-revision mime-types
- conn
commit-hash)))
(('GET "revision" commit-hash "system-tests")
- (if (guix-commit-exists? conn commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
(let ((parsed-query-parameters
(parse-query-parameters
request
`((system ,parse-system #:default "x86_64-linux")))))
(render-revision-system-tests mime-types
- conn
commit-hash
parsed-query-parameters
#:path-base path))
(render-unknown-revision mime-types
- conn
commit-hash)))
(('GET "revision" commit-hash "channel-instances")
- (if (guix-commit-exists? conn commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
(render-revision-channel-instances mime-types
- conn
commit-hash
#:path-base path)
(render-unknown-revision mime-types
- conn
commit-hash)))
(('GET "revision" commit-hash "package-substitute-availability")
- (if (guix-commit-exists? conn commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
(render-revision-package-substitute-availability mime-types
- conn
commit-hash
#:path-base path)
(render-unknown-revision mime-types
- conn
commit-hash)))
(('GET "revision" commit-hash "package-reproducibility")
- (if (guix-commit-exists? conn commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
(render-revision-package-reproduciblity mime-types
- conn
commit-hash
#:path-base path)
(render-unknown-revision mime-types
- conn
commit-hash)))
(('GET "revision" commit-hash "builds")
- (if (guix-commit-exists? conn commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
(let ((parsed-query-parameters
(guard-against-mutually-exclusive-query-parameters
(parse-query-parameters
request
`((build_status ,parse-build-status #:multi-value)
- (build_server ,(parse-build-server conn) #:multi-value)
+ (build_server ,parse-build-server #:multi-value)
(system ,parse-system #:default "x86_64-linux")
(target ,parse-target #:default "")
(limit_results ,parse-result-limit
@@ -296,15 +313,16 @@
'((limit_results all_results)))))
(render-revision-builds mime-types
- conn
commit-hash
parsed-query-parameters
#:path-base path))
(render-unknown-revision mime-types
- conn
commit-hash)))
(('GET "revision" commit-hash "lint-warnings")
- (if (guix-commit-exists? conn commit-hash)
+ (if (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (guix-commit-exists? conn commit-hash))))
(let ((parsed-query-parameters
(parse-query-parameters
request
@@ -318,12 +336,10 @@
"location"))))))
(render-revision-lint-warnings mime-types
- conn
commit-hash
parsed-query-parameters
#:path-base path))
(render-unknown-revision mime-types
- conn
commit-hash)))
(_ #f)))
@@ -336,7 +352,7 @@
(plain . ,(stexi->plain-text stexi))
(locale . ,locale))))
-(define (render-unknown-revision mime-types conn commit-hash)
+(define (render-unknown-revision mime-types commit-hash)
(case (most-appropriate-mime-type
'(application/json text/html)
mime-types)
@@ -345,31 +361,55 @@
'((unknown_commit . ,commit-hash))
#:code 404))
(else
+ (letpar& ((job
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-job-for-commit conn commit-hash))))
+ (git-repositories-and-branches
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (git-branches-with-repository-details-for-commit conn
+ commit-hash))))
+ (jobs-and-events
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-jobs-and-events-for-commit conn commit-hash)))))
+
(render-html
#:code 404
#:sxml (unknown-revision
commit-hash
- (select-job-for-commit
- conn commit-hash)
- (git-branches-with-repository-details-for-commit conn commit-hash)
- (select-jobs-and-events-for-commit conn commit-hash))))))
+ job
+ git-repositories-and-branches
+ jobs-and-events))))))
(define* (render-view-revision mime-types
- conn
commit-hash
#:key path-base
(header-text
`("Revision " (samp ,commit-hash))))
- (let ((packages-count
- (count-packages-in-revision conn commit-hash))
- (git-repositories-and-branches
- (git-branches-with-repository-details-for-commit conn commit-hash))
- (derivations-counts
- (count-packages-derivations-in-revision conn commit-hash))
- (jobs-and-events
- (select-jobs-and-events-for-commit conn commit-hash))
- (lint-warning-counts
- (lint-warning-count-by-lint-checker-for-revision conn commit-hash)))
+ (letpar& ((packages-count
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (count-packages-in-revision conn commit-hash))))
+ (git-repositories-and-branches
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (git-branches-with-repository-details-for-commit conn
+ commit-hash))))
+ (derivations-counts
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (count-packages-derivations-in-revision conn commit-hash))))
+ (jobs-and-events
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-jobs-and-events-for-commit conn commit-hash))))
+ (lint-warning-counts
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (lint-warning-count-by-lint-checker-for-revision conn
+ commit-hash)))))
(case (most-appropriate-mime-type
'(application/json text/html)
mime-types)
@@ -404,7 +444,6 @@
#:extra-headers http-headers-for-unchanging-content)))))
(define* (render-revision-system-tests mime-types
- conn
commit-hash
query-parameters
#:key
@@ -413,11 +452,13 @@
`("Revision " (samp ,commit-hash)))
(header-link
(string-append "/revision/" commit-hash)))
- (let ((system-tests
- (select-system-tests-for-guix-revision
- conn
- (assq-ref query-parameters 'system)
- commit-hash)))
+ (letpar& ((system-tests
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-system-tests-for-guix-revision
+ conn
+ (assq-ref query-parameters 'system)
+ commit-hash)))))
(case (most-appropriate-mime-type
'(application/json text/html)
mime-types)
@@ -440,20 +481,25 @@
(builds . ,(list->vector builds)))))
system-tests))))))
(else
- (render-html
- #:sxml (view-revision-system-tests
- commit-hash
- system-tests
- (git-repositories-containing-commit conn
- commit-hash)
- (valid-systems conn)
- query-parameters
- #:path-base path-base
- #:header-text header-text
- #:header-link header-link))))))
+ (letpar& ((git-repositories
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (git-repositories-containing-commit conn
+ commit-hash))))
+ (systems
+ (with-thread-postgresql-connection valid-systems)))
+ (render-html
+ #:sxml (view-revision-system-tests
+ commit-hash
+ system-tests
+ git-repositories
+ systems
+ query-parameters
+ #:path-base path-base
+ #:header-text header-text
+ #:header-link header-link)))))))
(define* (render-revision-channel-instances mime-types
- conn
commit-hash
#:key
(path-base "/revision/")
@@ -462,8 +508,10 @@
(header-link
(string-append "/revision/"
commit-hash)))
- (let ((channel-instances
- (select-channel-instances-for-guix-revision conn commit-hash)))
+ (letpar& ((channel-instances
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-channel-instances-for-guix-revision conn commit-hash)))))
(case (most-appropriate-mime-type
'(application/json text/html)
mime-types)
@@ -487,13 +535,16 @@
#:header-link header-link))))))
(define* (render-revision-package-substitute-availability mime-types
- conn
commit-hash
#:key path-base)
- (let ((substitute-availability
- (select-package-output-availability-for-revision conn commit-hash))
- (build-server-urls
- (select-build-server-urls-by-id conn)))
+ (letpar& ((substitute-availability
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-package-output-availability-for-revision conn
+ commit-hash))))
+ (build-server-urls
+ (with-thread-postgresql-connection
+ select-build-server-urls-by-id)))
(case (most-appropriate-mime-type
'(application/json text/html)
mime-types)
@@ -508,11 +559,12 @@
build-server-urls))))))
(define* (render-revision-package-reproduciblity mime-types
- conn
commit-hash
#:key path-base)
- (let ((output-consistency
- (select-output-consistency-for-revision conn commit-hash)))
+ (letpar& ((output-consistency
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-output-consistency-for-revision conn commit-hash)))))
(case (most-appropriate-mime-type
'(application/json text/html)
mime-types)
@@ -526,7 +578,6 @@
output-consistency))))))
(define (render-revision-news mime-types
- conn
commit-hash
query-parameters)
(if (any-invalid-query-parameters? query-parameters)
@@ -541,9 +592,12 @@
#:sxml (view-revision-news commit-hash
query-parameters
'()))))
- (let ((news-entries
- (select-channel-news-entries-contained-in-guix-revision conn
- commit-hash)))
+ (letpar& ((news-entries
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-channel-news-entries-contained-in-guix-revision
+ conn
+ commit-hash)))))
(case (most-appropriate-mime-type
'(application/json text/html)
mime-types)
@@ -558,7 +612,6 @@
#:extra-headers http-headers-for-unchanging-content))))))
(define* (render-revision-packages mime-types
- conn
commit-hash
query-parameters
#:key
@@ -589,101 +642,109 @@
'()
#f
#f
+ #f
#:path-base path-base
#:header-text header-text
#:header-link header-link))))
- (let* ((search-query (assq-ref query-parameters 'search_query))
- (limit-results (or (assq-ref query-parameters 'limit_results)
- 99999)) ; TODO There shouldn't be a limit
- (fields (assq-ref query-parameters 'field))
- (locale (assq-ref query-parameters 'locale))
- (packages
- (if search-query
- (search-packages-in-revision
- conn
- commit-hash
- search-query
- #:limit-results limit-results
- #:locale locale)
- (select-packages-in-revision
- conn
- commit-hash
- #:limit-results limit-results
- #:after-name (assq-ref query-parameters 'after_name)
- #:locale (assq-ref query-parameters 'locale))))
+ (let ((search-query (assq-ref query-parameters 'search_query))
+ (limit-results (or (assq-ref query-parameters 'limit_results)
+ 99999)) ; TODO There shouldn't be a limit
+ (fields (assq-ref query-parameters 'field))
+ (locale (assq-ref query-parameters 'locale)))
+ (letpar&
+ ((packages
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (if search-query
+ (search-packages-in-revision
+ conn
+ commit-hash
+ search-query
+ #:limit-results limit-results
+ #:locale locale)
+ (select-packages-in-revision
+ conn
+ commit-hash
+ #:limit-results limit-results
+ #:after-name (assq-ref query-parameters 'after_name)
+ #:locale (assq-ref query-parameters 'locale))))))
(git-repositories
- (git-repositories-containing-commit conn
- commit-hash))
- (show-next-page?
- (and (not search-query)
- (>= (length packages)
- limit-results)))
- (any-translations? (any-package-synopsis-or-descriptions-translations?
- packages locale)))
- (case (most-appropriate-mime-type
- '(application/json text/html)
- mime-types)
- ((application/json)
- (render-json
- `((revision
- . ((commit . ,commit-hash)))
- (packages
- . ,(list->vector
- (map (match-lambda
- ((name version synopsis synopsis-locale description description-locale home-page
- location-file location-line
- location-column-number licenses)
- `((name . ,name)
- ,@(if (member "version" fields)
- `((version . ,version))
- '())
- ,@(if (member "synopsis" fields)
- `((synopsis
- . ,(texinfo->variants-alist synopsis synopsis-locale)))
- '())
- ,@(if (member "description" fields)
- `((description
- . ,(texinfo->variants-alist description description-locale)))
- '())
- ,@(if (member "home-page" fields)
- `((home-page . ,home-page))
- '())
- ,@(if (member "location" fields)
- `((location
- . ((file . ,location-file)
- (line . ,location-line)
- (column . ,location-column-number))))
- '())
- ,@(if (member "licenses" fields)
- `((licenses
- . ,(if (string-null? licenses)
- #()
- (json-string->scm licenses))))
- '()))))
- packages))))
- #:extra-headers http-headers-for-unchanging-content))
- (else
- (let ((locale-options
- (description-and-synopsis-locale-options
- (package-description-and-synopsis-locale-options-guix-revision
- conn
- (commit->revision-id conn commit-hash)))))
- (render-html
- #:sxml (view-revision-packages commit-hash
- query-parameters
- packages
- git-repositories
- show-next-page?
- locale-options
- any-translations?
- #:path-base path-base
- #:header-text header-text
- #:header-link header-link)
- #:extra-headers http-headers-for-unchanging-content)))))))
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (git-repositories-containing-commit conn
+ commit-hash)))))
+ (let ((show-next-page?
+ (and (not search-query)
+ (>= (length packages)
+ limit-results)))
+ (any-translations? (any-package-synopsis-or-descriptions-translations?
+ packages locale)))
+ (case (most-appropriate-mime-type
+ '(application/json text/html)
+ mime-types)
+ ((application/json)
+ (render-json
+ `((revision
+ . ((commit . ,commit-hash)))
+ (packages
+ . ,(list->vector
+ (map (match-lambda
+ ((name version synopsis synopsis-locale description description-locale home-page
+ location-file location-line
+ location-column-number licenses)
+ `((name . ,name)
+ ,@(if (member "version" fields)
+ `((version . ,version))
+ '())
+ ,@(if (member "synopsis" fields)
+ `((synopsis
+ . ,(texinfo->variants-alist synopsis synopsis-locale)))
+ '())
+ ,@(if (member "description" fields)
+ `((description
+ . ,(texinfo->variants-alist description description-locale)))
+ '())
+ ,@(if (member "home-page" fields)
+ `((home-page . ,home-page))
+ '())
+ ,@(if (member "location" fields)
+ `((location
+ . ((file . ,location-file)
+ (line . ,location-line)
+ (column . ,location-column-number))))
+ '())
+ ,@(if (member "licenses" fields)
+ `((licenses
+ . ,(if (string-null? licenses)
+ #()
+ (json-string->scm licenses))))
+ '()))))
+ packages))))
+ #:extra-headers http-headers-for-unchanging-content))
+ (else
+ (letpar&
+ ((locale-options
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (description-and-synopsis-locale-options
+ (package-description-and-synopsis-locale-options-guix-revision
+ conn
+ (commit->revision-id conn commit-hash)))))))
+ (render-html
+ #:sxml (view-revision-packages commit-hash
+ query-parameters
+ packages
+ git-repositories
+ show-next-page?
+ locale-options
+ any-translations?
+ #:path-base path-base
+ #:header-text header-text
+ #:header-link header-link)
+ #:extra-headers http-headers-for-unchanging-content)))))))))
(define* (render-revision-packages-translation-availability mime-types
- conn
commit-hash
#:key
path-base
@@ -692,14 +753,20 @@
"/revision/" commit-hash))
(header-text
`("Revision " (samp ,commit-hash))))
- (let ((package-synopsis-counts
- (synopsis-counts-by-locale conn
- (commit->revision-id conn
- commit-hash)))
- (package-description-counts
- (description-counts-by-locale conn
- (commit->revision-id conn
- commit-hash))))
+ (letpar& ((package-synopsis-counts
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (synopsis-counts-by-locale conn
+ (commit->revision-id
+ conn
+ commit-hash)))))
+ (package-description-counts
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (description-counts-by-locale conn
+ (commit->revision-id
+ conn
+ commit-hash))))))
(case (most-appropriate-mime-type
'(application/json text/html)
mime-types)
@@ -718,7 +785,6 @@
#:header-text header-text))))))
(define* (render-revision-package mime-types
- conn
commit-hash
name
#:key
@@ -729,13 +795,17 @@
(header-link
(string-append
"/revision/" commit-hash)))
- (let ((package-versions
- (select-package-versions-for-revision conn
- commit-hash
- name))
- (git-repositories-and-branches
- (git-branches-with-repository-details-for-commit conn
- commit-hash)))
+ (letpar& ((package-versions
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-package-versions-for-revision conn
+ commit-hash
+ name))))
+ (git-repositories-and-branches
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (git-branches-with-repository-details-for-commit conn
+ commit-hash)))))
(case (most-appropriate-mime-type
'(application/json text/html)
mime-types)
@@ -755,7 +825,6 @@
#:extra-headers http-headers-for-unchanging-content)))))
(define* (render-revision-package-version mime-types
- conn
commit-hash
name
version
@@ -774,36 +843,48 @@
(match-lambda
((locale)
locale))
- (delete-duplicates
- (append
- (package-description-and-synopsis-locale-options-guix-revision
- conn (commit->revision-id conn commit-hash))
- (lint-warning-message-locales-for-revision conn commit-hash)))))
+ (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (delete-duplicates
+ (append
+ (package-description-and-synopsis-locale-options-guix-revision
+ conn (commit->revision-id conn commit-hash))
+ (lint-warning-message-locales-for-revision conn commit-hash))))))))
- (let* ((locale (assq-ref query-parameters 'locale))
- (metadata
- (select-package-metadata-by-revision-name-and-version
- conn
- commit-hash
- name
- version
- locale))
- (derivations
- (select-derivations-by-revision-name-and-version
- conn
- commit-hash
- name
- version))
- (git-repositories
- (git-repositories-containing-commit conn
- commit-hash))
- (lint-warnings
- (select-lint-warnings-by-revision-package-name-and-version
- conn
- commit-hash
- name
- version
- #:locale locale)))
+ (define locale (assq-ref query-parameters 'locale))
+
+ (letpar& ((metadata
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-package-metadata-by-revision-name-and-version
+ conn
+ commit-hash
+ name
+ version
+ locale))))
+ (derivations
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-derivations-by-revision-name-and-version
+ conn
+ commit-hash
+ name
+ version))))
+ (git-repositories
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (git-repositories-containing-commit conn
+ commit-hash))))
+ (lint-warnings
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-lint-warnings-by-revision-package-name-and-version
+ conn
+ commit-hash
+ name
+ version
+ #:locale locale)))))
(case (most-appropriate-mime-type
'(application/json text/html)
mime-types)
@@ -843,7 +924,6 @@
#:extra-headers http-headers-for-unchanging-content)))))
(define* (render-revision-package-derivations mime-types
- conn
commit-hash
query-parameters
#:key
@@ -861,100 +941,110 @@
(render-json
`((error . "invalid query"))))
(else
- (render-html
- #:sxml (view-revision-package-derivations commit-hash
- query-parameters
- (valid-systems conn)
- (valid-targets->options
- (valid-targets conn))
- '()
- '()
- #f
- #:path-base path-base
- #:header-text header-text
- #:header-link header-link))))
- (let* ((limit-results
- (assq-ref query-parameters 'limit_results))
- (all-results
- (assq-ref query-parameters 'all_results))
- (search-query
- (assq-ref query-parameters 'search_query))
- (fields
- (assq-ref query-parameters 'field))
- (derivations
- (if search-query
- (search-package-derivations-in-revision
- conn
- commit-hash
- search-query
- #:systems (assq-ref query-parameters 'system)
- #:targets (assq-ref query-parameters 'target)
- #:maximum-builds (assq-ref query-parameters 'maximum_builds)
- #:minimum-builds (assq-ref query-parameters 'minimum_builds)
- #:limit-results limit-results
- #:after-name (assq-ref query-parameters 'after_name)
- #:include-builds? (member "builds" fields))
- (select-package-derivations-in-revision
- conn
- commit-hash
- #:systems (assq-ref query-parameters 'system)
- #:targets (assq-ref query-parameters 'target)
- #:maximum-builds (assq-ref query-parameters 'maximum_builds)
- #:minimum-builds (assq-ref query-parameters 'minimum_builds)
- #:limit-results limit-results
- #:after-name (assq-ref query-parameters 'after_name)
- #:include-builds? (member "builds" fields))))
- (build-server-urls
- (select-build-server-urls-by-id conn))
- (show-next-page?
- (if all-results
- #f
- (and (not (null? derivations))
- (>= (length derivations)
- limit-results)))))
- (case (most-appropriate-mime-type
- '(application/json text/html)
- mime-types)
- ((application/json)
- (render-json
- `((derivations . ,(list->vector
- (map (match-lambda
- ((derivation system target)
- `((derivation . ,derivation)
- ,@(if (member "system" fields)
- `((system . ,system))
- '())
- ,@(if (member "target" fields)
- `((target . ,target))
- '())))
- ((derivation system target builds)
- `((derivation . ,derivation)
- ,@(if (member "system" fields)
- `((system . ,system))
- '())
- ,@(if (member "target" fields)
- `((target . ,target))
- '())
- (builds . ,builds))))
- derivations))))))
- (else
+ (letpar& ((systems
+ (with-thread-postgresql-connection valid-systems))
+ (targets
+ (with-thread-postgresql-connection valid-targets)))
(render-html
- #:sxml (view-revision-package-derivations
- commit-hash
- query-parameters
- (valid-systems conn)
- (valid-targets->options
- (valid-targets conn))
- derivations
- build-server-urls
- show-next-page?
- #:path-base path-base
- #:header-text header-text
- #:header-link header-link)))))))
+ #:sxml (view-revision-package-derivations commit-hash
+ query-parameters
+ systems
+ (valid-targets->options
+ targets)
+ '()
+ '()
+ #f
+ #:path-base path-base
+ #:header-text header-text
+ #:header-link header-link)))))
+ (let ((limit-results
+ (assq-ref query-parameters 'limit_results))
+ (all-results
+ (assq-ref query-parameters 'all_results))
+ (search-query
+ (assq-ref query-parameters 'search_query))
+ (fields
+ (assq-ref query-parameters 'field)))
+ (letpar&
+ ((derivations
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (if search-query
+ (search-package-derivations-in-revision
+ conn
+ commit-hash
+ search-query
+ #:systems (assq-ref query-parameters 'system)
+ #:targets (assq-ref query-parameters 'target)
+ #:maximum-builds (assq-ref query-parameters 'maximum_builds)
+ #:minimum-builds (assq-ref query-parameters 'minimum_builds)
+ #:limit-results limit-results
+ #:after-name (assq-ref query-parameters 'after_name)
+ #:include-builds? (member "builds" fields))
+ (select-package-derivations-in-revision
+ conn
+ commit-hash
+ #:systems (assq-ref query-parameters 'system)
+ #:targets (assq-ref query-parameters 'target)
+ #:maximum-builds (assq-ref query-parameters 'maximum_builds)
+ #:minimum-builds (assq-ref query-parameters 'minimum_builds)
+ #:limit-results limit-results
+ #:after-name (assq-ref query-parameters 'after_name)
+ #:include-builds? (member "builds" fields))))))
+ (build-server-urls
+ (with-thread-postgresql-connection
+ select-build-server-urls-by-id)))
+ (let ((show-next-page?
+ (if all-results
+ #f
+ (and (not (null? derivations))
+ (>= (length derivations)
+ limit-results)))))
+ (case (most-appropriate-mime-type
+ '(application/json text/html)
+ mime-types)
+ ((application/json)
+ (render-json
+ `((derivations . ,(list->vector
+ (map (match-lambda
+ ((derivation system target)
+ `((derivation . ,derivation)
+ ,@(if (member "system" fields)
+ `((system . ,system))
+ '())
+ ,@(if (member "target" fields)
+ `((target . ,target))
+ '())))
+ ((derivation system target builds)
+ `((derivation . ,derivation)
+ ,@(if (member "system" fields)
+ `((system . ,system))
+ '())
+ ,@(if (member "target" fields)
+ `((target . ,target))
+ '())
+ (builds . ,builds))))
+ derivations))))))
+ (else
+ (letpar& ((systems
+ (with-thread-postgresql-connection valid-systems))
+ (targets
+ (with-thread-postgresql-connection valid-targets)))
+ (render-html
+ #:sxml (view-revision-package-derivations
+ commit-hash
+ query-parameters
+ systems
+ (valid-targets->options targets)
+ derivations
+ build-server-urls
+ show-next-page?
+ #:path-base path-base
+ #:header-text header-text
+ #:header-link header-link))))))))))
(define* (render-revision-package-derivation-outputs
mime-types
- conn
commit-hash
query-parameters
#:key
@@ -964,7 +1054,8 @@
(header-link
(string-append "/revision/" commit-hash)))
(define build-server-urls
- (select-build-server-urls-by-id conn))
+ (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection select-build-server-urls-by-id)))
(if (any-invalid-query-parameters? query-parameters)
(case (most-appropriate-mime-type
@@ -974,66 +1065,74 @@
(render-json
`((error . "invalid query"))))
(else
- (render-html
- #:sxml (view-revision-package-derivation-outputs
- commit-hash
- query-parameters
- '()
- build-server-urls
- (valid-systems conn)
- (valid-targets->options
- (valid-targets conn))
- #f
- #:path-base path-base
- #:header-text header-text
- #:header-link header-link))))
- (let* ((limit-results
- (assq-ref query-parameters 'limit_results))
- (all-results
- (assq-ref query-parameters 'all_results))
- (derivation-outputs
- (select-derivation-outputs-in-revision
- conn
- commit-hash
- #:search-query (assq-ref query-parameters 'search_query)
- #:nars-from-build-servers
- (assq-ref query-parameters 'substitutes_available_from)
- #:no-nars-from-build-servers
- (assq-ref query-parameters 'substitutes_not_available_from)
- #:output-consistency
- (assq-ref query-parameters 'output_consistency)
- #:system (assq-ref query-parameters 'system)
- #:target (assq-ref query-parameters 'target)
- #:limit-results limit-results
- #:after-path (assq-ref query-parameters 'after_path)))
- (show-next-page?
- (if all-results
- #f
- (>= (length derivation-outputs)
- limit-results))))
- (case (most-appropriate-mime-type
- '(application/json text/html)
- mime-types)
- ((application/json)
- (render-json
- `()))
- (else
+ (letpar& ((systems
+ (with-thread-postgresql-connection valid-systems))
+ (targets
+ (with-thread-postgresql-connection valid-targets)))
(render-html
#:sxml (view-revision-package-derivation-outputs
commit-hash
query-parameters
- derivation-outputs
+ '()
build-server-urls
- (valid-systems conn)
- (valid-targets->options
- (valid-targets conn))
- show-next-page?
+ systems
+ (valid-targets->options targets)
+ #f
#:path-base path-base
#:header-text header-text
- #:header-link header-link)))))))
+ #:header-link header-link)))))
+ (let ((limit-results
+ (assq-ref query-parameters 'limit_results))
+ (all-results
+ (assq-ref query-parameters 'all_results)))
+ (letpar&
+ ((derivation-outputs
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-derivation-outputs-in-revision
+ conn
+ commit-hash
+ #:search-query (assq-ref query-parameters 'search_query)
+ #:nars-from-build-servers
+ (assq-ref query-parameters 'substitutes_available_from)
+ #:no-nars-from-build-servers
+ (assq-ref query-parameters 'substitutes_not_available_from)
+ #:output-consistency
+ (assq-ref query-parameters 'output_consistency)
+ #:system (assq-ref query-parameters 'system)
+ #:target (assq-ref query-parameters 'target)
+ #:limit-results limit-results
+ #:after-path (assq-ref query-parameters 'after_path))))))
+ (let ((show-next-page?
+ (if all-results
+ #f
+ (>= (length derivation-outputs)
+ limit-results))))
+ (case (most-appropriate-mime-type
+ '(application/json text/html)
+ mime-types)
+ ((application/json)
+ (render-json
+ `()))
+ (else
+ (letpar& ((systems
+ (with-thread-postgresql-connection valid-systems))
+ (targets
+ (with-thread-postgresql-connection valid-targets)))
+ (render-html
+ #:sxml (view-revision-package-derivation-outputs
+ commit-hash
+ query-parameters
+ derivation-outputs
+ build-server-urls
+ systems
+ (valid-targets->options targets)
+ show-next-page?
+ #:path-base path-base
+ #:header-text header-text
+ #:header-link header-link))))))))))
(define* (render-revision-builds mime-types
- conn
commit-hash
query-parameters
#:key
@@ -1043,51 +1142,69 @@
(header-link
(string-append "/revision/" commit-hash)))
(if (any-invalid-query-parameters? query-parameters)
- (render-html
- #:sxml (view-revision-builds query-parameters
- commit-hash
- build-status-strings
- (valid-systems conn)
- (valid-targets->options
- (valid-targets conn))
- '()
- '()
- '()))
+ (letpar& ((systems
+ (with-thread-postgresql-connection valid-systems))
+ (targets
+ (with-thread-postgresql-connection valid-targets)))
+ (render-html
+ #:sxml
+ (view-revision-builds query-parameters
+ commit-hash
+ build-status-strings
+ systems
+ (valid-targets->options targets)
+ '()
+ '()
+ '())))
(let ((system (assq-ref query-parameters 'system))
(target (assq-ref query-parameters 'target)))
- (render-html
- #:sxml (view-revision-builds query-parameters
- commit-hash
- build-status-strings
- (valid-systems conn)
- (valid-targets->options
- (valid-targets conn))
- (map (match-lambda
- ((id url lookup-all-derivations
- lookup-builds)
- (cons url id)))
- (select-build-servers conn))
- (select-build-stats
- conn
- (assq-ref query-parameters
- 'build_server)
- #:revision-commit commit-hash
- #:system system
- #:target target)
- (select-builds-with-context
- conn
- (assq-ref query-parameters
- 'build_status)
- (assq-ref query-parameters
- 'build_server)
- #:revision-commit commit-hash
- #:system system
- #:target target
- #:limit (assq-ref query-parameters
- 'limit_results)))))))
+ (letpar& ((systems
+ (with-thread-postgresql-connection valid-systems))
+ (targets
+ (with-thread-postgresql-connection valid-targets))
+ (build-server-options
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (map (match-lambda
+ ((id url lookup-all-derivations
+ lookup-builds)
+ (cons url id)))
+ (select-build-servers conn)))))
+ (stats
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-build-stats
+ conn
+ (assq-ref query-parameters
+ 'build_server)
+ #:revision-commit commit-hash
+ #:system system
+ #:target target))))
+ (builds
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (select-builds-with-context
+ conn
+ (assq-ref query-parameters
+ 'build_status)
+ (assq-ref query-parameters
+ 'build_server)
+ #:revision-commit commit-hash
+ #:system system
+ #:target target
+ #:limit (assq-ref query-parameters
+ 'limit_results))))))
+ (render-html
+ #:sxml (view-revision-builds query-parameters
+ commit-hash
+ build-status-strings
+ systems
+ (valid-targets->options targets)
+ build-server-options
+ stats
+ builds))))))
(define* (render-revision-lint-warnings mime-types
- conn
commit-hash
query-parameters
#:key
@@ -1097,18 +1214,24 @@
(header-link
(string-append "/revision/" commit-hash)))
(define lint-checker-options
- (map (match-lambda
- ((name description network-dependent)
- (cons (string-append name ": " description )
- name)))
- (lint-checkers-for-revision conn commit-hash)))
+ (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (map (match-lambda
+ ((name description network-dependent)
+ (cons (string-append name ": " description )
+ name)))
+ (lint-checkers-for-revision conn commit-hash))))))
(define lint-warnings-locale-options
- (map
- (match-lambda
- ((locale)
- locale))
- (lint-warning-message-locales-for-revision conn commit-hash)))
+ (parallel-via-thread-pool-channel
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (map
+ (match-lambda
+ ((locale)
+ locale))
+ (lint-warning-message-locales-for-revision conn commit-hash))))))
(if (any-invalid-query-parameters? query-parameters)
(case (most-appropriate-mime-type
@@ -1125,69 +1248,75 @@
'()
lint-checker-options
lint-warnings-locale-options
+ #t ; any-translated-lint-warnings?
#:path-base path-base
#:header-text header-text
#:header-link header-link))))
- (let* ((locale (assq-ref query-parameters 'locale))
- (package-query (assq-ref query-parameters 'package_query))
- (linters (assq-ref query-parameters 'linter))
- (message-query (assq-ref query-parameters 'message_query))
- (fields (assq-ref query-parameters 'field))
- (git-repositories
- (git-repositories-containing-commit conn
- commit-hash))
+ (let ((locale (assq-ref query-parameters 'locale))
+ (package-query (assq-ref query-parameters 'package_query))
+ (linters (assq-ref query-parameters 'linter))
+ (message-query (assq-ref query-parameters 'message_query))
+ (fields (assq-ref query-parameters 'field)))
+ (letpar&
+ ((git-repositories
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (git-repositories-containing-commit conn
+ commit-hash))))
(lint-warnings
- (lint-warnings-for-guix-revision conn commit-hash
- #:locale locale
- #:package-query package-query
- #:linters linters
- #:message-query message-query))
- (any-translated-lint-warnings?
- (any-translated-lint-warnings? lint-warnings locale)))
- (case (most-appropriate-mime-type
- '(application/json text/html)
- mime-types)
- ((application/json)
- (render-json
- `((revision
- . ((commit . ,commit-hash)))
- (lint_warnings
- . ,(list->vector
- (map (match-lambda
- ((id lint-checker-name lint-checker-description
- lint-checker-description-locale
- lint-checker-network-dependent
- package-name package-version
- file line-number column-number
- message message-locale)
- `((package . ((name . ,package-name)
- (version . ,package-version)))
- ,@(if (member "message" fields)
- `((message . ,message)
- (message-locale . ,message-locale))
- '())
- ,@(if (member "linter" fields)
- `((lint-checker-description . ,lint-checker-description)
- (lint-checker-description-locale . ,lint-checker-description-locale))
- '())
- ,@(if (member "location" fields)
- `((location . ((file . ,file)
- (line-number . ,line-number)
- (column-number . ,column-number))))
- '()))))
- lint-warnings))))
- #:extra-headers http-headers-for-unchanging-content))
- (else
- (render-html
- #:sxml (view-revision-lint-warnings commit-hash
- query-parameters
- lint-warnings
- git-repositories
- lint-checker-options
- lint-warnings-locale-options
- any-translated-lint-warnings?
- #:path-base path-base
- #:header-text header-text
- #:header-link header-link)
- #:extra-headers http-headers-for-unchanging-content))))))
+ (with-thread-postgresql-connection
+ (lambda (conn)
+ (lint-warnings-for-guix-revision conn commit-hash
+ #:locale locale
+ #:package-query package-query
+ #:linters linters
+ #:message-query message-query)))))
+ (let ((any-translated-lint-warnings?
+ (any-translated-lint-warnings? lint-warnings locale)))
+ (case (most-appropriate-mime-type
+ '(application/json text/html)
+ mime-types)
+ ((application/json)
+ (render-json
+ `((revision
+ . ((commit . ,commit-hash)))
+ (lint_warnings
+ . ,(list->vector
+ (map (match-lambda
+ ((id lint-checker-name lint-checker-description
+ lint-checker-description-locale
+ lint-checker-network-dependent
+ package-name package-version
+ file line-number column-number
+ message message-locale)
+ `((package . ((name . ,package-name)
+ (version . ,package-version)))
+ ,@(if (member "message" fields)
+ `((message . ,message)
+ (message-locale . ,message-locale))
+ '())
+ ,@(if (member "linter" fields)
+ `((lint-checker-description . ,lint-checker-description)
+ (lint-checker-description-locale . ,lint-checker-description-locale))
+ '())
+ ,@(if (member "location" fields)
+ `((location . ((file . ,file)
+ (line-number . ,line-number)
+ (column-number . ,column-number))))
+ '()))))
+ lint-warnings))))
+ #:extra-headers http-headers-for-unchanging-content))
+ (else
+ (render-html
+ #:sxml (view-revision-lint-warnings commit-hash
+ query-parameters
+ lint-warnings
+ git-repositories
+ lint-checker-options
+ lint-warnings-locale-options
+ any-translated-lint-warnings?
+ #:path-base path-base
+ #:header-text header-text
+ #:header-link header-link)
+ #:extra-headers http-headers-for-unchanging-content))))))))