diff options
| author | Christopher Baines <mail@cbaines.net> | 2023-01-30 12:22:34 +0000 |
|---|---|---|
| committer | Christopher Baines <mail@cbaines.net> | 2023-01-30 13:21:38 +0000 |
| commit | 620ebbba55bfe8c001caf03aac74ed298f1f9669 (patch) | |
| tree | 80c56e883301f88926d96e841f90bd2f639425f3 | |
| parent | 751910162c54d0bf85fa5a21c25ad229cb12828d (diff) | |
| download | build-coordinator-620ebbba55bfe8c001caf03aac74ed298f1f9669.tar build-coordinator-620ebbba55bfe8c001caf03aac74ed298f1f9669.tar.gz | |
Try to improve protections against getting stuck when substituting
I think the agent and coordinator can get stuck when talking to the
guix-daemon to substitute things, potentially because the Guile code involved
on the daemon side isn't guarding against getting stuck. Try to work around
this here by setting timeouts on the connection to the daemon.
| -rw-r--r-- | guix-build-coordinator/agent.scm | 2 | ||||
| -rw-r--r-- | guix-build-coordinator/utils.scm | 16 |
2 files changed, 17 insertions, 1 deletions
diff --git a/guix-build-coordinator/agent.scm b/guix-build-coordinator/agent.scm index c5aeeb9..77e0c73 100644 --- a/guix-build-coordinator/agent.scm +++ b/guix-build-coordinator/agent.scm @@ -677,6 +677,8 @@ but the guix-daemon claims it's unavailable" (lambda () (with-port-timeouts (lambda () + (set-store-connection-timeout + fetch-substitute-store) (parameterize ((current-build-output-port log-port)) (build-things fetch-substitute-store diff --git a/guix-build-coordinator/utils.scm b/guix-build-coordinator/utils.scm index 8067e88..e64842d 100644 --- a/guix-build-coordinator/utils.scm +++ b/guix-build-coordinator/utils.scm @@ -54,6 +54,8 @@ with-port-timeouts + set-store-connection-timeout + request-query-parameters call-with-streaming-http-request @@ -330,6 +332,14 @@ upcoming chunk." (make-port-timeout-error)))))) (thunk))) +(define* (set-store-connection-timeout store #:key (timeout 120)) + (define raw-port (store-connection-socket store)) + + (when (defined? 'SO_RCVTIMEO) + ;; This is only supported on Guile 3.0.9 and later + (setsockopt raw-port SOL_SOCKET SO_RCVTIMEO `(,timeout . 0)) + (setsockopt raw-port SOL_SOCKET SO_SNDTIMEO `(,timeout . 0)))) + (define* (make-chunked-output-port* port #:key (keep-alive? #f) (buffering 1200) report-bytes-sent) @@ -531,7 +541,11 @@ upcoming chunk." #:timeout ,(* 10 60))) (parameterize ((current-build-output-port log-port)) - (ensure-path store derivation-name))) + (with-port-timeouts + (lambda () + (set-store-connection-timeout store) + (ensure-path store derivation-name)) + #:timeout (* 120 1000)))) (with-store store (apply set-build-options store |