Track errors when reading/substituting derivations

1 files changed, 14 insertions, 2 deletions

diff --git a/guix-build-coordinator/client-communication.scm b/guix-build-coordinator/client-communication.scm
index bffec12..47a289d 100644
--- a/guix-build-coordinator/client-communication.scm
+++ b/guix-build-coordinator/client-communication.scm
@@ -32,6 +32,7 @@
   #:use-module (json)
   #:use-module (logging logger)
   #:use-module (gcrypt random)
+  #:use-module (prometheus)
   #:use-module (web uri)
   #:use-module (web client)
   #:use-module (web request)
@@ -100,6 +101,14 @@
         (json-string->scm (utf8->string raw-body))
         '()))
 
+  (define read-drv-error-count-metric
+    (or (metrics-registry-fetch-metric
+         (build-coordinator-metrics-registry build-coordinator)
+         "read_derivation_error_total")
+        (make-counter-metric
+         (build-coordinator-metrics-registry build-coordinator)
+         "read_derivation_error_total")))
+
   (define (controller-thunk)
     (match method-and-path-components
       (('GET "build" uuid)
@@ -514,8 +523,11 @@
                               (retry-on-error
                                (lambda ()
                                  (read-drv/substitute derivation-file))
-                               #:times 5
-                               #:delay 5))
+                               #:times 2
+                               #:delay 3
+                               #:error-hook
+                               (lambda _
+                                 (metric-increment read-drv-error-count-metric))))
                             (lambda args
                               (backtrace))))
                         #:unwind? #t))