Support compressing data upfront to reduce transfer times

For sending larger outputs back to the coordinator.

1 files changed, 44 insertions, 10 deletions

diff --git a/guix-build-coordinator/agent-messaging/http.scm b/guix-build-coordinator/agent-messaging/http.scm
index cc09b95..12b3be7 100644
--- a/guix-build-coordinator/agent-messaging/http.scm
+++ b/guix-build-coordinator/agent-messaging/http.scm
@@ -35,6 +35,7 @@
   #:use-module (web response)
   #:use-module (web uri)
   #:use-module (fibers channels)
+  #:use-module (guix store)
   #:use-module (guix lzlib)
   #:use-module (guix base64)
   #:use-module (guix serialization)
@@ -499,18 +500,51 @@ port. Also, the port used can be changed by passing the --port option.\n"
      coordinator-uri
      (string-append "/build/" build-id "/output/" output-name)))
 
-  (retry-on-error
-   (lambda ()
-     (call-with-streaming-http-request
-      uri
-      (lambda (port)
-        (call-with-lzip-output-port port
+  (define path-info
+    (with-store store
+      (query-path-info store file)))
+
+  ;; For small outputs, compress while sending the data, but for bigger store
+  ;; items, do all the compression up front to hopefully reduce the time to
+  ;; send them.
+  (if (< (path-info-nar-size path-info)
+         250000000)                     ; .25GB
+      (retry-on-error
+       (lambda ()
+         (call-with-streaming-http-request
+          uri
+          (lambda (port)
+            (call-with-lzip-output-port port
+              (lambda (port)
+                (write-file file port))
+              #:level 9))
+          #:headers `((Authorization . ,auth-value))))
+       #:times 3
+       #:delay 30)
+      (let* ((directory (or (getenv "TMPDIR") "/tmp"))
+             (template  (string-append directory
+                                       "/guix-build-coordinator-file.XXXXXX"))
+             (out       (mkstemp! template)))
+
+        (call-with-lzip-output-port out
           (lambda (port)
             (write-file file port))
-          #:level 9))
-      #:headers `((Authorization . ,auth-value))))
-   #:times 3
-   #:delay 30))
+          #:level 9)
+        (close-port out)
+
+        (retry-on-error
+         (lambda ()
+           (call-with-input-file template
+             (lambda (file-port)
+               (call-with-streaming-http-request
+                uri
+                (lambda (port)
+                  (dump-port file-port port))
+                #:headers `((Authorization . ,auth-value))))))
+         #:times 3
+         #:delay 30)
+
+        (delete-file template))))
 
 (define (submit-log-file coordinator-uri agent-uuid password
                          build-id file)