From 9e80bda4f91e06e3e1d60cf48d33534b7cabb626 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Mon, 5 Aug 2019 20:59:48 +0100 Subject: Avoid erroring when processing emails again These changes allow processing emails again, and just creating job and branch entries where data is missing. --- guix-data-service/jobs/load-new-guix-revision.scm | 11 ++++++---- ...emove_duplicate_load_new_guix_revision_jobs.sql | 25 ++++++++++++++++++++++ ...emove_duplicate_load_new_guix_revision_jobs.sql | 7 ++++++ sqitch/sqitch.plan | 1 + ...emove_duplicate_load_new_guix_revision_jobs.sql | 7 ++++++ tests/jobs-load-new-guix-revision.scm | 19 +++++++++++++++- tests/model-git-branch.scm | 19 ++++++++++++++++ 7 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 sqitch/deploy/remove_duplicate_load_new_guix_revision_jobs.sql create mode 100644 sqitch/revert/remove_duplicate_load_new_guix_revision_jobs.sql create mode 100644 sqitch/verify/remove_duplicate_load_new_guix_revision_jobs.sql diff --git a/guix-data-service/jobs/load-new-guix-revision.scm b/guix-data-service/jobs/load-new-guix-revision.scm index 2dd3907..393ef06 100644 --- a/guix-data-service/jobs/load-new-guix-revision.scm +++ b/guix-data-service/jobs/load-new-guix-revision.scm @@ -630,12 +630,15 @@ WHERE job_id = $1" " INSERT INTO load_new_guix_revision_jobs (git_repository_id, commit, source) VALUES ($1, $2, $3) +ON CONFLICT DO NOTHING RETURNING id;") - (first - (exec-query conn - query - (list git-repository-id commit source)))) + (match (exec-query conn + query + (list git-repository-id commit source)) + ((result) + result) + (() #f))) (define (select-job-for-commit conn commit) (let ((result diff --git a/sqitch/deploy/remove_duplicate_load_new_guix_revision_jobs.sql b/sqitch/deploy/remove_duplicate_load_new_guix_revision_jobs.sql new file mode 100644 index 0000000..4a5110e --- /dev/null +++ b/sqitch/deploy/remove_duplicate_load_new_guix_revision_jobs.sql @@ -0,0 +1,25 @@ +-- Deploy guix-data-service:remove_duplicate_load_new_guix_revision_jobs to pg + +BEGIN; + +DELETE FROM load_new_guix_revision_job_events WHERE job_id NOT IN ( + SELECT MIN(id) FROM load_new_guix_revision_jobs + GROUP BY commit, git_repository_id +); + +DELETE FROM load_new_guix_revision_job_logs WHERE job_id NOT IN ( + SELECT MIN(id) FROM load_new_guix_revision_jobs + GROUP BY commit, git_repository_id +); + +DELETE FROM load_new_guix_revision_jobs WHERE id NOT IN ( + SELECT MIN(id) FROM load_new_guix_revision_jobs + GROUP BY commit, git_repository_id +); + +CREATE UNIQUE INDEX ON load_new_guix_revision_jobs ( + commit, + git_repository_id +); + +COMMIT; diff --git a/sqitch/revert/remove_duplicate_load_new_guix_revision_jobs.sql b/sqitch/revert/remove_duplicate_load_new_guix_revision_jobs.sql new file mode 100644 index 0000000..f35efed --- /dev/null +++ b/sqitch/revert/remove_duplicate_load_new_guix_revision_jobs.sql @@ -0,0 +1,7 @@ +-- Revert guix-data-service:remove_duplicate_load_new_guix_revision_jobs from pg + +BEGIN; + +-- XXX Add DDLs here. + +COMMIT; diff --git a/sqitch/sqitch.plan b/sqitch/sqitch.plan index e78dbac..541074c 100644 --- a/sqitch/sqitch.plan +++ b/sqitch/sqitch.plan @@ -17,3 +17,4 @@ load_new_guix_revision_job_logs 2019-06-21T14:33:09Z chris # Add l change_load_new_guix_revision_job_logs_contents_to_be_nullable 2019-07-07T20:10:54Z Christopher Baines # Change the contents field in the load_new_guix_revision_job_logs table\nto be nullable.\n\nwith '#' will # be ignored, and an empty message aborts the add. #\nChange to add: # #\nchange_load_new_guix_revision_job_logs_contents_to_be_nullable #\nsqitch/deploy/change_load_new_guix_revision_job_logs_contents_to_be_nullable.sql\nsqitch/revert/change_load_new_guix_revision_job_logs_contents_to_be_nullable.sql\nsqitch/verify/change_load_new_guix_revision_job_logs_contents_to_be_nullable.sql fix_duplicated_licenses 2019-07-30T05:48:17Z Christopher Baines # Fix duplicated licenses, and add constraints change_git_branches_primary_key 2019-08-05T18:57:41Z Christopher Baines # Change the git_branches primary key to include the git_repository_id,\nas this will allow having the same branch in different repositories. +remove_duplicate_load_new_guix_revision_jobs 2019-08-05T19:06:36Z Christopher Baines # Remove duplicate load_new_guix_revision_jobs diff --git a/sqitch/verify/remove_duplicate_load_new_guix_revision_jobs.sql b/sqitch/verify/remove_duplicate_load_new_guix_revision_jobs.sql new file mode 100644 index 0000000..3122c74 --- /dev/null +++ b/sqitch/verify/remove_duplicate_load_new_guix_revision_jobs.sql @@ -0,0 +1,7 @@ +-- Verify guix-data-service:remove_duplicate_load_new_guix_revision_jobs on pg + +BEGIN; + +-- XXX Add verifications here. + +ROLLBACK; diff --git a/tests/jobs-load-new-guix-revision.scm b/tests/jobs-load-new-guix-revision.scm index 14d0e33..cbd9adc 100644 --- a/tests/jobs-load-new-guix-revision.scm +++ b/tests/jobs-load-new-guix-revision.scm @@ -82,7 +82,24 @@ ((id) (process-load-new-guix-revision-job id)))))) - (exec-query conn "TRUNCATE load_new_guix_revision_jobs CASCADE"))) + (exec-query conn "TRUNCATE load_new_guix_revision_jobs CASCADE") + + (test-assert "test duplicate job handling" + (with-postgresql-transaction + conn + (lambda (conn) + (enqueue-load-new-guix-revision-job + conn + (git-repository-url->git-repository-id conn "test-url") + "test-commit" + "test-source") + (enqueue-load-new-guix-revision-job + conn + (git-repository-url->git-repository-id conn "test-url") + "test-commit" + "test-source") + #t) + #:always-rollback? #t)))) (test-end) diff --git a/tests/model-git-branch.scm b/tests/model-git-branch.scm index a879408..c69f69d 100644 --- a/tests/model-git-branch.scm +++ b/tests/model-git-branch.scm @@ -22,6 +22,25 @@ id (current-date))) #t) + #:always-rollback? #t)) + + (test-assert "insert-git-branch-entry works twice" + (with-postgresql-transaction + conn + (lambda (conn) + (let* ((url "test-url") + (id (git-repository-url->git-repository-id conn url))) + (insert-git-branch-entry conn + "master" + "test-commit" + id + (current-date)) + (insert-git-branch-entry conn + "master" + "test-commit" + id + (current-date))) + #t) #:always-rollback? #t)))) (test-end) -- cgit v1.2.3