Handle conflicts in insert-missing-data-and-return-all-ids

1 files changed, 25 insertions, 28 deletions

diff --git a/guix-data-service/model/utils.scm b/guix-data-service/model/utils.scm
index 8721c52..3fda78f 100644
--- a/guix-data-service/model/utils.scm
+++ b/guix-data-service/model/utils.scm
@@ -323,7 +323,7 @@ WHERE table_name = $1"
               ")"))
            missing-data)
       ", ")
-     " RETURNING id"))
+     " ON CONFLICT DO NOTHING"))
 
   (define (format-json json)
     ;; PostgreSQL formats JSON strings differently to guile-json, so use
@@ -355,16 +355,17 @@ WHERE table_name = $1"
             (error (simple-format #f "normalise-values: error: ~A\n" unknown))))
          data))
 
-  (let* ((existing-entries
+  (let* ((flattened-deduplicated-data
+          (if sets-of-data?
+              (delete-duplicates*
+               (concatenate data))
+              (if delete-duplicates?
+                  (delete-duplicates* data)
+                  data)))
+         (existing-entries
           (if use-temporary-table?
               (let ((temp-table-name
-                     (string-append "temp_" table-name))
-                    (data
-                     (if sets-of-data?
-                         (delete-duplicates* (concatenate data))
-                         (if delete-duplicates?
-                             (delete-duplicates* data)
-                             data))))
+                     (string-append "temp_" table-name)))
                 ;; Create a temporary table to store the data
                 (exec-query
                  conn
@@ -379,12 +380,11 @@ WHERE table_name = $1"
                   "ANALYZE " temp-table-name))
 
                 ;; Populate the temporary table
-                (if (null? data)
-                    '()
-                    (with-time-logging (string-append "populating " temp-table-name)
-                      (exec-query conn
-                                  (insert-sql data
-                                              #:table-name temp-table-name))))
+                (unless (null? flattened-deduplicated-data)
+                  (with-time-logging (string-append "populating " temp-table-name)
+                    (exec-query conn
+                                (insert-sql flattened-deduplicated-data
+                                            #:table-name temp-table-name))))
                 ;; Use the temporary table to find the existing values
                 (let ((result
                        (with-time-logging
@@ -400,7 +400,7 @@ WHERE table_name = $1"
                   result))
 
               ;; If not using a temporary table, just do a single SELECT query
-              (if (null? data)
+              (if (null? flattened-deduplicated-data)
                   '()
                   (fold
                    (lambda (data-chunk result)
@@ -411,15 +411,10 @@ WHERE table_name = $1"
                                           (string->number (first result)))
                                         #:vhash result))
                    vlist-null
-                   (chunk (if sets-of-data?
-                              (delete-duplicates*
-                               (concatenate data))
-                              data)
+                   (chunk flattened-deduplicated-data
                           3000)))))
          (missing-entries
-          (let loop ((lst (if sets-of-data?
-                              (concatenate data)
-                              data))
+          (let loop ((lst flattened-deduplicated-data)
                      (result '()))
             (if (null? lst)
                 (if delete-duplicates?
@@ -440,10 +435,12 @@ WHERE table_name = $1"
               '()
               (append-map!
                (lambda (missing-entries-chunk)
-                 (map (lambda (result)
-                        (string->number (first result)))
-                      (exec-query conn
-                                  (insert-sql missing-entries-chunk))))
+                 (exec-query conn
+                             (insert-sql missing-entries-chunk))
+
+                 (map (lambda (row)
+                        (string->number (first row)))
+                      (exec-query conn (select-query missing-entries-chunk))))
                (chunk missing-entries 3000))))
 
          (new-entries-lookup-vhash
@@ -474,4 +471,4 @@ WHERE table_name = $1"
                           (error "missing entry" field-values))))
                    data))))
     (values all-ids
-            new-entries)))
+            (delete-duplicates/sort! new-entries <))))