diff options
| -rw-r--r-- | bffe.scm | 47 | ||||
| -rw-r--r-- | bffe/config.scm.in | 1 | ||||
| -rw-r--r-- | bffe/manage-builds.scm | 30 | ||||
| -rw-r--r-- | bffe/server.scm | 12 | ||||
| -rw-r--r-- | bffe/view/activity.scm | 1 | ||||
| -rw-r--r-- | bffe/view/agent.scm | 4 | ||||
| -rw-r--r-- | bffe/view/build.scm | 4 | ||||
| -rw-r--r-- | bffe/view/home.scm | 4 | ||||
| -rw-r--r-- | bffe/view/util.scm | 8 |
9 files changed, 48 insertions, 63 deletions
@@ -17,6 +17,7 @@ ;;; <http://www.gnu.org/licenses/>. (define-module (bffe) + #:use-module (srfi srfi-1) #:use-module (ice-9 format) #:use-module (oop goops) #:use-module (prometheus) @@ -45,11 +46,14 @@ (port-log (make <port-log> #:port (current-output-port) #:formatter - (lambda (lvl time str) + ;; In guile-lib v0.2.8 onwards, the formatter is + ;; called with more arguments + (lambda args ; lvl, time, str (format #f "~a (~5a): ~a~%" - (strftime "%F %H:%M:%S" (localtime time)) - lvl - str))))) + (strftime "%F %H:%M:%S" (localtime + (second args))) + (first args) + (third args)))))) (setvbuf (current-output-port) 'line) @@ -57,20 +61,23 @@ (open-log! lgr) (set-default-logger! lgr) - (let ((finished? (make-condition))) - (call-with-sigint - (lambda () - (run-fibers - (lambda () - (for-each start-submit-builds-fibers - build) + (with-fluids ((%file-port-name-canonicalization 'none)) + (let ((finished? (make-condition))) + (call-with-sigint + (lambda () + (run-fibers + (lambda () + (for-each + (lambda (spec) + (start-submit-builds-fibers metrics-registry spec)) + build) - (when web-server-args - (apply start-bffe-web-server - `(,@web-server-args - #:pid-file ,pid-file - #:metrics-registry ,metrics-registry))) - (wait finished?)) - #:hz 0 - #:parallelism 1)) - finished?)))) + (when web-server-args + (apply start-bffe-web-server + `(,@web-server-args + #:pid-file ,pid-file + #:metrics-registry ,metrics-registry))) + (wait finished?)) + #:hz 0 + #:parallelism 1)) + finished?))))) diff --git a/bffe/config.scm.in b/bffe/config.scm.in index 19e5832..7af9d4d 100644 --- a/bffe/config.scm.in +++ b/bffe/config.scm.in @@ -17,7 +17,6 @@ ;;; <http://www.gnu.org/licenses/>. (define-module (bffe config) - #:use-module (guix store) #:export (%config)) (define %config diff --git a/bffe/manage-builds.scm b/bffe/manage-builds.scm index 16484c3..b5d95bd 100644 --- a/bffe/manage-builds.scm +++ b/bffe/manage-builds.scm @@ -21,20 +21,17 @@ #:use-module (srfi srfi-71) #:use-module (ice-9 match) #:use-module (ice-9 exceptions) - #:use-module (rnrs bytevectors) #:use-module (ice-9 textual-ports) #:use-module (logging logger) - #:use-module (logging port-log) + #:use-module (prometheus) #:use-module (zlib) #:use-module (json) #:use-module (fibers) #:use-module (fibers channels) #:use-module (guix records) #:use-module (web uri) - #:use-module (web client) #:use-module (web response) #:use-module (guix-build-coordinator client-communication) - #:use-module ((guix-build-coordinator utils) #:select (create-work-queue)) #:use-module ((guix-build-coordinator utils fibers) #:select (retry-on-error)) #:use-module (bffe server) #:export (build-from-guix-data-service @@ -67,7 +64,9 @@ (default #f)) (revision-parameters build-from-guix-data-service-revision-parameters - (default '()))) + (default '())) + (parallelism build-from-guix-data-service-parallelism + (default 2))) (define-exception-type &guix-data-service-error &error make-guix-data-service-error @@ -418,12 +417,19 @@ " package builds for " log-suffix)))) (define (submit-builds-pass + metrics-registry submit-build/fiberized guix-data-service specification processed-commits-hash record-revision-as-processed systems-and-targets) + (define unseen-revisions-metric + (or (metrics-registry-fetch-metric + metrics-registry "unseen_revisions_total") + (make-gauge-metric metrics-registry + "unseen_revisions_total"))) + (fibers-for-each (lambda (repository-id) (fibers-for-each @@ -451,6 +457,7 @@ "revisions"))))) (log-msg 'DEBUG (length unseen-revisions) " unseen revisions") + (metric-set unseen-revisions-metric (length unseen-revisions)) (for-each (lambda (commit) (log-msg 'INFO "looking at revision " commit) @@ -492,6 +499,7 @@ systems-and-targets) (log-msg 'INFO "finished looking at revision " commit) + (metric-decrement unseen-revisions-metric) (record-revision-as-processed commit)) (reverse unseen-revisions)))) (let ((all-branches (all-repository-branches guix-data-service @@ -505,7 +513,8 @@ (sleep 60)) -(define* (start-submit-builds-fibers specification +(define* (start-submit-builds-fibers metrics-registry + specification #:key processed-commits-file) (define processed-commits-hash @@ -519,7 +528,7 @@ (simple-format port "~A\n" commit) (close-port port)))) -(define* (submit-build coordinator guix-data-service derivation + (define* (submit-build coordinator guix-data-service derivation #:key (priority 0) (log-prefix "") (tags '())) (retry-on-error @@ -572,8 +581,8 @@ (let ((submit-build/fiberized (fiberize submit-build - #:parallelism 8))) - + #:parallelism (build-from-guix-data-service-parallelism + specification)))) (spawn-fiber (lambda () (while #t @@ -587,7 +596,8 @@ (lambda () (with-throw-handler #t (lambda () - (submit-builds-pass submit-build/fiberized + (submit-builds-pass metrics-registry + submit-build/fiberized guix-data-service specification processed-commits-hash diff --git a/bffe/server.scm b/bffe/server.scm index dc9a06d..eae7291 100644 --- a/bffe/server.scm +++ b/bffe/server.scm @@ -24,12 +24,9 @@ #:use-module (srfi srfi-19) #:use-module (srfi srfi-71) #:use-module (ice-9 vlist) - #:use-module (ice-9 threads) #:use-module (ice-9 match) #:use-module (ice-9 atomic) #:use-module (ice-9 textual-ports) - #:use-module (rnrs bytevectors) - #:use-module (web http) #:use-module (web client) #:use-module (web request) #:use-module (web response) @@ -38,18 +35,11 @@ #:use-module (prometheus) #:use-module (system repl error-handling) #:use-module (fibers) - #:use-module (fibers scheduler) #:use-module (fibers channels) - #:use-module (fibers conditions) - #:use-module (fibers web server) #:use-module ((guix store) #:select (%store-prefix)) - #:use-module ((guix build utils) #:select (dump-port)) #:use-module (guix-data-service web util) - #:use-module ((guix-build-coordinator utils) - #:select (with-time-logging call-with-delay-logging)) #:use-module ((guix-build-coordinator utils fibers) #:select (run-server/patched retry-on-error)) - #:use-module (guix-build-coordinator client-communication) #:use-module (bffe config) #:use-module (bffe view util) #:use-module (bffe view home) @@ -154,7 +144,7 @@ (lambda () (with-throw-handler #t (lambda () - (let ((response + (let ((_ body (http-get* (string->uri diff --git a/bffe/view/activity.scm b/bffe/view/activity.scm index 1621872..5a97bd2 100644 --- a/bffe/view/activity.scm +++ b/bffe/view/activity.scm @@ -2,7 +2,6 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-19) #:use-module (ice-9 match) - #:use-module (ice-9 format) #:use-module (bffe view util) #:use-module ((guix store) #:select (%store-prefix)) #:export (activity)) diff --git a/bffe/view/agent.scm b/bffe/view/agent.scm index e210402..1de2f71 100644 --- a/bffe/view/agent.scm +++ b/bffe/view/agent.scm @@ -1,10 +1,6 @@ (define-module (bffe view agent) #:use-module (srfi srfi-1) - #:use-module (srfi srfi-19) - #:use-module (ice-9 match) - #:use-module (ice-9 format) #:use-module (bffe view util) - #:use-module ((guix store) #:select (%store-prefix)) #:export (agent)) (define (agent title agent-details) diff --git a/bffe/view/build.scm b/bffe/view/build.scm index ded00ef..c061668 100644 --- a/bffe/view/build.scm +++ b/bffe/view/build.scm @@ -1,10 +1,6 @@ (define-module (bffe view build) #:use-module (srfi srfi-1) - #:use-module (srfi srfi-19) - #:use-module (ice-9 match) - #:use-module (ice-9 format) #:use-module (bffe view util) - #:use-module ((guix store) #:select (%store-prefix)) #:export (build)) (define (build title build-details derivation-link-target tag-link-target) diff --git a/bffe/view/home.scm b/bffe/view/home.scm index 820f75f..3f3b1de 100644 --- a/bffe/view/home.scm +++ b/bffe/view/home.scm @@ -1,8 +1,4 @@ (define-module (bffe view home) - #:use-module (srfi srfi-1) - #:use-module (srfi srfi-19) - #:use-module (ice-9 match) - #:use-module (ice-9 format) #:use-module (bffe view util) #:export (home)) diff --git a/bffe/view/util.scm b/bffe/view/util.scm index 9b4a676..c6b89b0 100644 --- a/bffe/view/util.scm +++ b/bffe/view/util.scm @@ -17,23 +17,15 @@ ;;; <http://www.gnu.org/licenses/>. (define-module (bffe view util) - #:use-module (guix-data-service config) - #:use-module (guix-data-service web query-parameters) #:use-module (guix-data-service web util) - #:use-module (guix-data-service web html-utils) #:use-module ((guix-data-service web render) #:prefix guix-data-service:) #:use-module (ice-9 ftw) - #:use-module (ice-9 vlist) #:use-module (ice-9 match) #:use-module (ice-9 binary-ports) #:use-module (srfi srfi-1) #:use-module (srfi srfi-19) #:use-module (srfi srfi-26) - #:use-module (web uri) #:use-module (web response) - #:use-module (texinfo) - #:use-module (texinfo html) - #:use-module (json) #:export (layout header form-horizontal-control |