Start using Prometheus metrics with the agent

Rather than having the agent run a webserver, use the textfile collector from
the node exporter.

1 files changed, 50 insertions, 8 deletions

diff --git a/guix-build-coordinator/agent.scm b/guix-build-coordinator/agent.scm
index d3b5b4a..64d672a 100644
--- a/guix-build-coordinator/agent.scm
+++ b/guix-build-coordinator/agent.scm
@@ -32,6 +32,7 @@
   #:use-module (oop goops)
   #:use-module (logging logger)
   #:use-module (logging port-log)
+  #:use-module (prometheus)
   #:use-module (guix store)
   #:use-module (guix derivations)
   #:use-module (guix base32)
@@ -43,7 +44,8 @@
                    systems
                    max-parallel-builds
                    derivation-substitute-urls
-                   non-derivation-substitute-urls)
+                   non-derivation-substitute-urls
+                   metrics-file)
   (define lgr (make <logger>))
   (define port-log (make <port-log>
                      #:port (current-output-port)
@@ -54,6 +56,40 @@
                                lvl
                                str))))
 
+  (define metrics-enabled?
+    (and (not (string-null? metrics-file))
+         (let ((directory (dirname metrics-file)))
+           (or (file-exists? directory)
+               (begin
+                 (simple-format (current-error-port)
+                                "skipping writing metrics as ~A does not exist\n"
+                                directory)
+                 #f)))
+         (with-exception-handler
+             (lambda (exn)
+               (simple-format
+                (current-error-port)
+                "skipping writing metrics, encountered exception ~A\n"
+                exn)
+               #f)
+           (lambda ()
+             (let ((test-file (string-append metrics-file "-tmp")))
+               (call-with-output-file test-file
+                 (lambda (port)
+                   (display "test" port)))
+               (delete-file test-file)
+               #t))
+           #:unwind? #t)))
+
+  (define metrics-registry
+    (make-metrics-registry
+     #:namespace "guixbuildcoordinator_agent"))
+
+  (define (write-metrics)
+    (when metrics-enabled?
+      (write-textfile metrics-registry
+                      metrics-file)))
+
   (define (process-job build)
     (let ((build-id (assoc-ref build "uuid"))
           (derivation-name (assoc-ref build "derivation-name")))
@@ -62,13 +98,19 @@
                ": setting up to build: "
                derivation-name)
       (with-store store
-        (let ((pre-build-status (pre-build-process
-                                 lgr
-                                 store
-                                 build-id
-                                 derivation-substitute-urls
-                                 non-derivation-substitute-urls
-                                 derivation-name)))
+        (let ((pre-build-status
+               (call-with-duration-metric
+                metrics-registry
+                "pre_build_duration_seconds"
+                (lambda ()
+                  (pre-build-process lgr
+                                     store
+                                     build-id
+                                     derivation-substitute-urls
+                                     non-derivation-substitute-urls
+                                     derivation-name))
+                #:buckets (list 1 2.5 5 10 25 50 100 200 500 1000 (inf)))))
+          (write-metrics)
           (if (eq? (assq-ref pre-build-status 'result) 'success)
               (begin
                 (log-msg lgr 'INFO