Record jobs as failed when they are killed

So that they aren't retried again and again.

2 files changed, 30 insertions, 2 deletions

diff --git a/guix-data-service/jobs.scm b/guix-data-service/jobs.scm
index a63821f..dd680d0 100644
--- a/guix-data-service/jobs.scm
+++ b/guix-data-service/jobs.scm
@@ -13,8 +13,17 @@
             "guix-data-service-process-job"
             job-id))
 
+  (define (handle-job-failure job-id)
+    (record-job-event conn job-id "failure")
+    (display (simple-format #f "recording failure for job ~A\n" job-id)
+             (current-error-port))
+    (when (> (count-log-parts conn job-id)
+             0)
+      (combine-log-parts! conn job-id)))
+
   (process-jobs-concurrently fetch-new-jobs
-                             process-job))
+                             process-job
+                             handle-job-failure))
 
 (define default-max-processes
   (max (round (/ (current-processor-count)
@@ -27,6 +36,7 @@
 
 (define* (process-jobs-concurrently fetch-new-jobs
                                     process-job
+                                    handle-job-failure
                                     #:key (max-processes
                                            default-max-processes)
                                     (timeout default-timeout))
@@ -88,7 +98,11 @@
               #f "sending SIGTERM to pid ~A started at ~A, now running for ~A\n"
               pid start-time running-for)
              (current-error-port))
-            (kill pid SIGTERM)))))
+            (kill pid SIGTERM)
+
+            (match job-args
+              ((id)
+               (handle-job-failure id)))))))
      processes))
 
   (define (fork-and-process-job job-args)
diff --git a/guix-data-service/jobs/load-new-guix-revision.scm b/guix-data-service/jobs/load-new-guix-revision.scm
index a1e22ba..24fada7 100644
--- a/guix-data-service/jobs/load-new-guix-revision.scm
+++ b/guix-data-service/jobs/load-new-guix-revision.scm
@@ -24,10 +24,13 @@
   #:use-module (guix-data-service model package-metadata)
   #:use-module (guix-data-service model derivation)
   #:export (log-for-job
+            count-log-parts
+            combine-log-parts!
             fetch-unlocked-jobs
             process-load-new-guix-revision-job
             select-job-for-commit
             select-jobs-and-events
+            record-job-event
             enqueue-load-new-guix-revision-job
             most-recent-n-load-new-guix-revision-jobs))
 
@@ -122,6 +125,17 @@
 ($1, NULL) ON CONFLICT DO NOTHING"
    (list job-id)))
 
+(define (count-log-parts conn job-id)
+  (match (exec-query
+          conn
+          "
+SELECT COUNT(*)
+FROM load_new_guix_revision_job_log_parts
+WHERE job_id = $1"
+          (list job-id))
+    (((id))
+     (string->number id))))
+
 (define (combine-log-parts! conn job-id)
   (with-postgresql-transaction
    conn