Retry file uploads to the coordinator

Previously SIGPIPE killed the agent. This should avoid that and have the agent
retry.

2 files changed, 30 insertions, 16 deletions

diff --git a/guix-build-coordinator/agent-messaging/http.scm b/guix-build-coordinator/agent-messaging/http.scm
index 4b569f1..a4be0b5 100644
--- a/guix-build-coordinator/agent-messaging/http.scm
+++ b/guix-build-coordinator/agent-messaging/http.scm
@@ -498,14 +498,18 @@ port. Also, the port used can be changed by passing the --port option.\n"
      coordinator-uri
      (string-append "/build/" build-id "/output/" output-name)))
 
-  (call-with-streaming-http-request
-   uri
-   (lambda (port)
-     (call-with-lzip-output-port port
-       (lambda (port)
-         (write-file file port))
-       #:level 9))
-   #:headers `((Authorization . ,auth-value))))
+  (retry-on-error
+   (lambda ()
+     (call-with-streaming-http-request
+      uri
+      (lambda (port)
+        (call-with-lzip-output-port port
+          (lambda (port)
+            (write-file file port))
+          #:level 9))
+      #:headers `((Authorization . ,auth-value))))
+   #:times 3
+   #:delay 30))
 
 (define (submit-log-file coordinator-uri agent-uuid password
                          build-id file)
@@ -528,14 +532,18 @@ port. Also, the port used can be changed by passing the --port option.\n"
      coordinator-uri
      (string-append "/build/" build-id "/log/" format)))
 
-  (call-with-streaming-http-request
-   uri
-   (lambda (request-port)
-     (call-with-input-file file
-       (lambda (file-port)
-         (dump-port file-port request-port))
-       #:binary #t))
-   #:headers `((Authorization . ,auth-value))))
+  (retry-on-error
+   (lambda ()
+     (call-with-streaming-http-request
+      uri
+      (lambda (request-port)
+        (call-with-input-file file
+          (lambda (file-port)
+            (dump-port file-port request-port))
+          #:binary #t))
+      #:headers `((Authorization . ,auth-value))))
+   #:times 3
+   #:delay 30))
 
 (define (submit-build-result coordinator-uri agent-uuid password
                              build-id result)
diff --git a/guix-build-coordinator/utils.scm b/guix-build-coordinator/utils.scm
index 550bd2f..585e33b 100644
--- a/guix-build-coordinator/utils.scm
+++ b/guix-build-coordinator/utils.scm
@@ -258,6 +258,12 @@ upcoming chunk."
                #:keep-alive? #t))
              (base64-output-port
               (make-base64-output-port chunked-output-port)))
+
+        ;; A SIGPIPE will kill Guile, so ignore it
+        (sigaction SIGPIPE
+          (lambda (arg)
+            (simple-format (current-error-port) "warning: SIGPIPE\n")))
+
         (callback base64-output-port)
         (close-port base64-output-port)
         (close-port chunked-output-port)