aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristopher Baines <mail@cbaines.net>2019-07-14 14:16:05 +0100
committerChristopher Baines <mail@cbaines.net>2019-07-14 14:43:47 +0100
commit6904547bf2dc9dae516af3911b1dd967ae7d47a6 (patch)
tree2d3b5cffb5e23652e7c70e0b945e97fa1a0825af
parent3dfa9212f1a492103e3c639ab16e7ccfdddc9134 (diff)
downloaddata-service-6904547bf2dc9dae516af3911b1dd967ae7d47a6.tar
data-service-6904547bf2dc9dae516af3911b1dd967ae7d47a6.tar.gz
Record jobs as failed when they are killed
So that they aren't retried again and again.
-rw-r--r--guix-data-service/jobs.scm18
-rw-r--r--guix-data-service/jobs/load-new-guix-revision.scm14
2 files changed, 30 insertions, 2 deletions
diff --git a/guix-data-service/jobs.scm b/guix-data-service/jobs.scm
index a63821f..dd680d0 100644
--- a/guix-data-service/jobs.scm
+++ b/guix-data-service/jobs.scm
@@ -13,8 +13,17 @@
"guix-data-service-process-job"
job-id))
+ (define (handle-job-failure job-id)
+ (record-job-event conn job-id "failure")
+ (display (simple-format #f "recording failure for job ~A\n" job-id)
+ (current-error-port))
+ (when (> (count-log-parts conn job-id)
+ 0)
+ (combine-log-parts! conn job-id)))
+
(process-jobs-concurrently fetch-new-jobs
- process-job))
+ process-job
+ handle-job-failure))
(define default-max-processes
(max (round (/ (current-processor-count)
@@ -27,6 +36,7 @@
(define* (process-jobs-concurrently fetch-new-jobs
process-job
+ handle-job-failure
#:key (max-processes
default-max-processes)
(timeout default-timeout))
@@ -88,7 +98,11 @@
#f "sending SIGTERM to pid ~A started at ~A, now running for ~A\n"
pid start-time running-for)
(current-error-port))
- (kill pid SIGTERM)))))
+ (kill pid SIGTERM)
+
+ (match job-args
+ ((id)
+ (handle-job-failure id)))))))
processes))
(define (fork-and-process-job job-args)
diff --git a/guix-data-service/jobs/load-new-guix-revision.scm b/guix-data-service/jobs/load-new-guix-revision.scm
index a1e22ba..24fada7 100644
--- a/guix-data-service/jobs/load-new-guix-revision.scm
+++ b/guix-data-service/jobs/load-new-guix-revision.scm
@@ -24,10 +24,13 @@
#:use-module (guix-data-service model package-metadata)
#:use-module (guix-data-service model derivation)
#:export (log-for-job
+ count-log-parts
+ combine-log-parts!
fetch-unlocked-jobs
process-load-new-guix-revision-job
select-job-for-commit
select-jobs-and-events
+ record-job-event
enqueue-load-new-guix-revision-job
most-recent-n-load-new-guix-revision-jobs))
@@ -122,6 +125,17 @@
($1, NULL) ON CONFLICT DO NOTHING"
(list job-id)))
+(define (count-log-parts conn job-id)
+ (match (exec-query
+ conn
+ "
+SELECT COUNT(*)
+FROM load_new_guix_revision_job_log_parts
+WHERE job_id = $1"
+ (list job-id))
+ (((id))
+ (string->number id))))
+
(define (combine-log-parts! conn job-id)
(with-postgresql-transaction
conn