Change the locale codeset representation

From the normalized one, to the one actually contained within glibc. Recent
versions of glibc also contain symlinks linking the normalized codeset to the
locales with the .UTF-8 ending, but older ones do not.

Maybe handling codeset normalisation for queries would be good, but the locale
values ending in .UTF-8 are more compatible and allow the code to be
simplified. For querying, maybe there should be a locales table which handles
different representations.

17 files changed, 98 insertions, 82 deletions

diff --git a/guix-data-service/comparison.scm b/guix-data-service/comparison.scm
index cd55a28..e066278 100644
--- a/guix-data-service/comparison.scm
+++ b/guix-data-service/comparison.scm
@@ -634,7 +634,7 @@ WITH base_lint_warnings AS (
     ORDER BY lint_checkers.id,
              CASE
                WHEN lint_checker_descriptions.locale = $3 THEN 2
-               WHEN lint_checker_descriptions.locale = 'en_US.utf8' THEN 1
+               WHEN lint_checker_descriptions.locale = 'en_US.UTF-8' THEN 1
                ELSE 0
              END DESC
   ) AS translated_lint_checker_descriptions
@@ -653,7 +653,7 @@ WITH base_lint_warnings AS (
   ORDER BY lint_warnings.id,
            CASE
              WHEN lint_warning_messages.locale = $3 THEN 2
-             WHEN lint_warning_messages.locale = 'en_US.utf8' THEN 1
+             WHEN lint_warning_messages.locale = 'en_US.UTF-8' THEN 1
            ELSE 0 END DESC
 ), target_lint_warnings AS (
   SELECT DISTINCT ON (lint_warnings.id) lint_warnings.id,
@@ -684,7 +684,7 @@ WITH base_lint_warnings AS (
     ORDER BY lint_checkers.id,
              CASE
                WHEN lint_checker_descriptions.locale = $3 THEN 2
-               WHEN lint_checker_descriptions.locale = 'en_US.utf8' THEN 1
+               WHEN lint_checker_descriptions.locale = 'en_US.UTF-8' THEN 1
                ELSE 0
              END DESC
   ) AS translated_lint_checker_descriptions
@@ -703,7 +703,7 @@ WITH base_lint_warnings AS (
   ORDER BY lint_warnings.id,
           CASE
             WHEN lint_warning_messages.locale = $3 THEN 2
-            WHEN lint_warning_messages.locale = 'en_US.utf8' THEN 1
+            WHEN lint_warning_messages.locale = 'en_US.UTF-8' THEN 1
           ELSE 0 END DESC
 )
 SELECT coalesce(
diff --git a/guix-data-service/jobs/load-new-guix-revision.scm b/guix-data-service/jobs/load-new-guix-revision.scm
index 1e4a17e..2d97d65 100644
--- a/guix-data-service/jobs/load-new-guix-revision.scm
+++ b/guix-data-service/jobs/load-new-guix-revision.scm
@@ -366,19 +366,19 @@ WHERE job_id = $1"
 
 (define (all-inferior-lint-warnings inf store packages)
   (define locales
-    '("cs_CZ.utf8"
-      "da_DK.utf8"
-      "de_DE.utf8"
-      "eo_EO.utf8"
-      "es_ES.utf8"
-      "fr_FR.utf8"
-      "hu_HU.utf8"
-      "pl_PL.utf8"
-      "pt_BR.utf8"
-      ;;"sr_SR.utf8"
-      "sv_SE.utf8"
-      "vi_VN.utf8"
-      "zh_CN.utf8"))
+    '("cs_CZ.UTF-8"
+      "da_DK.UTF-8"
+      "de_DE.UTF-8"
+      "eo_EO.UTF-8"
+      "es_ES.UTF-8"
+      "fr_FR.UTF-8"
+      "hu_HU.UTF-8"
+      "pl_PL.UTF-8"
+      "pt_BR.UTF-8"
+      ;;"sr_SR.UTF-8"
+      "sv_SE.UTF-8"
+      "vi_VN.UTF-8"
+      "zh_CN.UTF-8"))
 
   (define (lint-warnings-for-checker checker-name)
     `(lambda (store)
@@ -406,7 +406,7 @@ WHERE job_id = $1"
                          file)
                      line
                      column)))
-            (let* ((source-locale "en_US.utf8")
+            (let* ((source-locale "en_US.UTF-8")
                    (source-message
                     (begin
                       (setlocale LC_MESSAGES source-locale)
@@ -464,7 +464,7 @@ WHERE job_id = $1"
           (inferior-eval
            `(begin
               (define (lint-descriptions-by-locale checker)
-                (let* ((source-locale "en_US.utf8")
+                (let* ((source-locale "en_US.UTF-8")
                        (source-description
                         (begin
                           (setlocale LC_MESSAGES source-locale)
@@ -1125,24 +1125,14 @@ WHERE job_id = $1"
       (error "error: inferior is #f"))
 
     ;; Normalise the locale for the inferior process
-    (catch
-      #t
+    (with-exception-handler
+        (lambda (key . args)
+          (simple-format
+           (current-error-port)
+           "warning: failed to set locale to en_US.UTF-8: ~A ~A\n"
+           key args))
       (lambda ()
-        (inferior-eval '(setlocale LC_ALL "en_US.utf8") inf))
-      (lambda (key . args)
-        (simple-format (current-error-port)
-                       "warning: failed to set locale to en_US.utf8: ~A ~A\n"
-                       key args)
-        (display "trying to setlocale to en_US.UTF-8 instead\n"
-                 (current-error-port))
-        (with-exception-handler
-            (lambda (key . args)
-              (simple-format
-               (current-error-port)
-               "warning: failed to set locale to en_US.UTF-8: ~A ~A\n"
-               key args))
-          (lambda ()
-            (inferior-eval '(setlocale LC_ALL "en_US.UTF-8") inf)))))
+        (inferior-eval '(setlocale LC_ALL "en_US.UTF-8") inf)))
 
     (inferior-eval '(use-modules (srfi srfi-1)
                                  (srfi srfi-34)
diff --git a/guix-data-service/model/lint-checker.scm b/guix-data-service/model/lint-checker.scm
index 6603bbb..be3cfc6 100644
--- a/guix-data-service/model/lint-checker.scm
+++ b/guix-data-service/model/lint-checker.scm
@@ -93,7 +93,7 @@ INNER JOIN lint_checker_description_sets
   ON lint_checkers.lint_checker_description_set_id = lint_checker_description_sets.id
 INNER JOIN lint_checker_descriptions
   ON lint_checker_descriptions.id = ANY (lint_checker_description_sets.description_ids)
-  AND lint_checker_descriptions.locale = 'en_US.utf8'
+  AND lint_checker_descriptions.locale = 'en_US.UTF-8'
 INNER JOIN (
   SELECT lint_checker_id, COUNT(*)
   FROM lint_warnings
@@ -138,7 +138,7 @@ INNER JOIN lint_checker_description_sets
   ON lint_checkers.lint_checker_description_set_id = lint_checker_description_sets.id
 INNER JOIN lint_checker_descriptions
   ON lint_checker_descriptions.id = ANY (lint_checker_description_sets.description_ids)
-  AND lint_checker_descriptions.locale = 'en_US.utf8'
+  AND lint_checker_descriptions.locale = 'en_US.UTF-8'
 WHERE lint_checkers.id IN (
   SELECT lint_checker_id
   FROM guix_revision_lint_checkers
diff --git a/guix-data-service/model/lint-warning.scm b/guix-data-service/model/lint-warning.scm
index 856f3e6..eff332f 100644
--- a/guix-data-service/model/lint-warning.scm
+++ b/guix-data-service/model/lint-warning.scm
@@ -90,7 +90,7 @@ INNER JOIN (
   ORDER BY lint_checkers.id,
            CASE
              WHEN lint_checker_descriptions.locale = $2 THEN 2
-             WHEN lint_checker_descriptions.locale = 'en_US.utf8' THEN 1
+             WHEN lint_checker_descriptions.locale = 'en_US.UTF-8' THEN 1
              ELSE 0
            END DESC
 ) AS translated_lint_checker_descriptions
@@ -131,7 +131,7 @@ INNER JOIN lint_warning_messages
                    " ORDER BY lint_warnings.id,
                               CASE
                                 WHEN lint_warning_messages.locale = $2 THEN 2
-                                WHEN lint_warning_messages.locale = 'en_US.utf8' THEN 1
+                                WHEN lint_warning_messages.locale = 'en_US.UTF-8' THEN 1
                                 ELSE 0
                               END DESC"))
 
@@ -174,7 +174,7 @@ INNER JOIN (
   ORDER BY lint_checkers.id,
            CASE
              WHEN lint_checker_descriptions.locale = $4 THEN 2
-             WHEN lint_checker_descriptions.locale = 'en_US.utf8' THEN 1
+             WHEN lint_checker_descriptions.locale = 'en_US.UTF-8' THEN 1
              ELSE 0
            END DESC
 ) AS translated_lint_checker_descriptions
@@ -202,7 +202,7 @@ WHERE packages.id IN (
   ORDER BY lint_warnings.id,
            CASE
              WHEN lint_warning_messages.locale = $4 THEN 2
-             WHEN lint_warning_messages.locale = 'en_US.utf8' THEN 1
+             WHEN lint_warning_messages.locale = 'en_US.UTF-8' THEN 1
              ELSE 0
            END DESC
 ")
diff --git a/guix-data-service/model/package-metadata.scm b/guix-data-service/model/package-metadata.scm
index 0598fc7..d7de893 100644
--- a/guix-data-service/model/package-metadata.scm
+++ b/guix-data-service/model/package-metadata.scm
@@ -42,19 +42,19 @@
             description-counts-by-locale))
 
 (define locales
-  '("cs_CZ.utf8"
-    "da_DK.utf8"
-    "de_DE.utf8"
-    "eo_EO.utf8"
-    "es_ES.utf8"
-    "fr_FR.utf8"
-    "hu_HU.utf8"
-    "pl_PL.utf8"
-    "pt_BR.utf8"
-    ;;"sr_SR.utf8"
-    "sv_SE.utf8"
-    "vi_VN.utf8"
-    "zh_CN.utf8"))
+  '("cs_CZ.UTF-8"
+    "da_DK.UTF-8"
+    "de_DE.UTF-8"
+    "eo_EO.UTF-8"
+    "es_ES.UTF-8"
+    "fr_FR.UTF-8"
+    "hu_HU.UTF-8"
+    "pl_PL.UTF-8"
+    "pt_BR.UTF-8"
+    ;;"sr_SR.UTF-8"
+    "sv_SE.UTF-8"
+    "vi_VN.UTF-8"
+    "zh_CN.UTF-8"))
 
 (define inferior-package-id
   (@@ (guix inferior) inferior-package-id))
@@ -144,7 +144,7 @@ INNER JOIN (
     AND packages.version = $3
   ORDER BY package_description_sets.id,
            CASE WHEN package_descriptions.locale = $4 THEN 2
-                WHEN package_descriptions.locale = 'en_US.utf8' THEN 1
+                WHEN package_descriptions.locale = 'en_US.UTF-8' THEN 1
                 ELSE 0
           END DESC
 ) AS translated_package_descriptions
@@ -163,7 +163,7 @@ INNER JOIN (
     AND packages.version = $3
   ORDER BY package_synopsis_sets.id,
            CASE WHEN package_synopsis.locale = $4 THEN 2
-                WHEN package_synopsis.locale = 'en_US.utf8' THEN 1
+                WHEN package_synopsis.locale = 'en_US.UTF-8' THEN 1
                 ELSE 0
           END DESC
 ) AS translated_package_synopsis
@@ -217,7 +217,7 @@ WHERE packages.id IN (
 
   (define (translate inferior-package-id)
     `(let* ((package (hashv-ref %package-table ,inferior-package-id))
-            (source-locale "en_US.utf8")
+            (source-locale "en_US.UTF-8")
             (source-synopsis
               (begin
                 (setlocale LC_MESSAGES source-locale)
@@ -505,7 +505,7 @@ INSERT INTO package_metadata_tsvectors (package_metadata_id, locale,
             synopsis_and_description, package_synopsis_id, package_description_id)
 SELECT DISTINCT ON (package_metadata.id, locale)
          package_metadata.id,
-         CASE WHEN translated_package_synopsis.locale != 'en_US.utf8'
+         CASE WHEN translated_package_synopsis.locale != 'en_US.UTF-8'
               THEN translated_package_synopsis.locale
               ELSE translated_package_descriptions.locale
          END AS locale,
@@ -524,7 +524,7 @@ INNER JOIN (
   INNER JOIN package_descriptions
     ON package_descriptions.id = ANY (package_description_sets.description_ids)
   ORDER BY package_description_sets.id,
-           CASE WHEN package_descriptions.locale = 'en_US.utf8' THEN 1
+           CASE WHEN package_descriptions.locale = 'en_US.UTF-8' THEN 1
                 ELSE 2
            END DESC
 ) AS translated_package_descriptions
@@ -537,7 +537,7 @@ INNER JOIN (
   INNER JOIN package_synopsis
     ON package_synopsis.id = ANY (package_synopsis_sets.synopsis_ids)
   ORDER BY package_synopsis_sets.id,
-           CASE WHEN package_synopsis.locale = 'en_US.utf8' THEN 1
+           CASE WHEN package_synopsis.locale = 'en_US.UTF-8' THEN 1
                 ELSE 2
            END DESC
 ) AS translated_package_synopsis
@@ -545,7 +545,7 @@ INNER JOIN (
      translated_package_synopsis.package_synopsis_set_id
   AND (translated_package_descriptions.locale =
        translated_package_synopsis.locale
-       OR translated_package_descriptions.locale = 'en_US.utf8')
+       OR translated_package_descriptions.locale = 'en_US.UTF-8')
 WHERE package_metadata.id IN ("
          (string-join
           (map number->string
diff --git a/guix-data-service/model/package.scm b/guix-data-service/model/package.scm
index a232aa7..c1de63b 100644
--- a/guix-data-service/model/package.scm
+++ b/guix-data-service/model/package.scm
@@ -86,7 +86,7 @@ WITH data AS (
       ON package_synopsis.id = ANY (package_synopsis_sets.synopsis_ids)
     ORDER BY package_synopsis_sets.id,
              CASE WHEN package_synopsis.locale = $2 THEN 2
-                  WHEN package_synopsis.locale = 'en_US.utf8' THEN 1
+                  WHEN package_synopsis.locale = 'en_US.UTF-8' THEN 1
                   ELSE 0
              END DESC
   ) AS translated_package_synopsis
@@ -99,7 +99,7 @@ WITH data AS (
       ON package_descriptions.id = ANY (package_description_sets.description_ids)
     ORDER BY package_description_sets.id,
              CASE WHEN package_descriptions.locale = $2 THEN 2
-                  WHEN package_descriptions.locale = 'en_US.utf8' THEN 1
+                  WHEN package_descriptions.locale = 'en_US.UTF-8' THEN 1
                   ELSE 0
              END DESC
   ) AS translated_package_descriptions
@@ -184,7 +184,7 @@ WITH search_results AS (
   )
   AND package_metadata_tsvectors.synopsis_and_description @@ plainto_tsquery($2)
   ORDER BY name,
-    CASE WHEN package_metadata_tsvectors.locale = 'en_US.utf8' THEN 2
+    CASE WHEN package_metadata_tsvectors.locale = 'en_US.UTF-8' THEN 2
          WHEN package_metadata_tsvectors.locale = $3 THEN 1
          ELSE 0
     END DESC
diff --git a/guix-data-service/web/compare/controller.scm b/guix-data-service/web/compare/controller.scm
index 98c74fa..c3db5e2 100644
--- a/guix-data-service/web/compare/controller.scm
+++ b/guix-data-service/web/compare/controller.scm
@@ -74,7 +74,7 @@
               request
               `((base_commit   ,(parse-commit conn) #:required)
                 (target_commit ,(parse-commit conn) #:required)
-                (locale        ,identity #:default "en_US.utf8")))))
+                (locale        ,identity #:default "en_US.UTF-8")))))
        (render-compare mime-types
                        conn
                        parsed-query-parameters)))
@@ -86,7 +86,7 @@
                 (base_datetime   ,parse-datetime #:required)
                 (target_branch   ,identity #:required)
                 (target_datetime ,parse-datetime #:required)
-                (locale          ,identity #:default "en_US.utf8")))))
+                (locale          ,identity #:default "en_US.UTF-8")))))
        (render-compare-by-datetime mime-types
                                    conn
                                    parsed-query-parameters)))
diff --git a/guix-data-service/web/repository/controller.scm b/guix-data-service/web/repository/controller.scm
index 2575170..7b2cb86 100644
--- a/guix-data-service/web/repository/controller.scm
+++ b/guix-data-service/web/repository/controller.scm
@@ -172,7 +172,7 @@
                   (guard-against-mutually-exclusive-query-parameters
                    (parse-query-parameters
                     request
-                    `((locale         ,identity #:default "en_US.utf8")
+                    `((locale         ,identity #:default "en_US.UTF-8")
                       (after_name     ,identity)
                       (field          ,identity #:multi-value
                                       #:default ("version" "synopsis"))
@@ -262,7 +262,7 @@
            (let ((parsed-query-parameters
                   (parse-query-parameters
                    request
-                   `((locale         ,identity #:default "en_US.utf8")
+                   `((locale         ,identity #:default "en_US.UTF-8")
                      (package_query  ,identity)
                      (linter         ,identity #:multi-value)
                      (message_query  ,identity)
@@ -293,7 +293,7 @@
            (parsed-query-parameters
             (parse-query-parameters
              request
-             `((locale ,identity #:default "en_US.utf8")))))
+             `((locale ,identity #:default "en_US.UTF-8")))))
        (if commit-hash
            (render-revision-package-version mime-types
                                             conn
diff --git a/guix-data-service/web/revision/controller.scm b/guix-data-service/web/revision/controller.scm
index 8338a00..ec93c04 100644
--- a/guix-data-service/web/revision/controller.scm
+++ b/guix-data-service/web/revision/controller.scm
@@ -125,7 +125,7 @@
                 (guard-against-mutually-exclusive-query-parameters
                  (parse-query-parameters
                   request
-                  `((locale         ,identity #:default "en_US.utf8")
+                  `((locale         ,identity #:default "en_US.UTF-8")
                     (after_name     ,identity)
                     (field          ,identity #:multi-value
                                     #:default ("version" "synopsis"))
@@ -170,7 +170,7 @@
          (let ((parsed-query-parameters
                 (parse-query-parameters
                  request
-                 `((locale ,identity #:default "en_US.utf8")))))
+                 `((locale ,identity #:default "en_US.UTF-8")))))
            (render-revision-package-version mime-types
                                                conn
                                                commit-hash
@@ -308,7 +308,7 @@
          (let ((parsed-query-parameters
                 (parse-query-parameters
                  request
-                 `((locale         ,identity #:default "en_US.utf8")
+                 `((locale         ,identity #:default "en_US.UTF-8")
                    (package_query  ,identity)
                    (linter         ,identity #:multi-value)
                    (message_query  ,identity)
diff --git a/guix-data-service/web/revision/html.scm b/guix-data-service/web/revision/html.scm
index 543fba8..5917d50 100644
--- a/guix-data-service/web/revision/html.scm
+++ b/guix-data-service/web/revision/html.scm
@@ -725,13 +725,13 @@
                                                          path-base header-link
                                                          header-text)
   (define total-package-synopsis
-    (assoc-ref package-synopsis-counts "en_US.utf8"))
+    (assoc-ref package-synopsis-counts "en_US.UTF-8"))
 
   (define total-package-descriptions
-    (assoc-ref package-description-counts "en_US.utf8"))
+    (assoc-ref package-description-counts "en_US.UTF-8"))
 
-  (assoc-remove! package-synopsis-counts "en_US.utf8")
-  (assoc-remove! package-description-counts "en_US.utf8")
+  (assoc-remove! package-synopsis-counts "en_US.UTF-8")
+  (assoc-remove! package-description-counts "en_US.UTF-8")
 
   (define synopsis-percentages
     (map
diff --git a/scripts/guix-data-service-process-job.in b/scripts/guix-data-service-process-job.in
index 995d464..0aec7be 100644
--- a/scripts/guix-data-service-process-job.in
+++ b/scripts/guix-data-service-process-job.in
@@ -31,7 +31,7 @@
 (setvbuf (current-output-port) 'line)
 (setvbuf (current-error-port) 'line)
 
-(setlocale LC_ALL "en_US.utf8")
+(setlocale LC_ALL "en_US.UTF-8")
 
 (debug-set! width 256)
 
diff --git a/sqitch/deploy/change_locale_values.sql b/sqitch/deploy/change_locale_values.sql
new file mode 100644
index 0000000..9069c61
--- /dev/null
+++ b/sqitch/deploy/change_locale_values.sql
@@ -0,0 +1,11 @@
+-- Deploy guix-data-service:change_locale_values to pg
+
+BEGIN;
+
+UPDATE lint_checker_descriptions SET locale = REPLACE(locale, '.utf8', '.UTF-8');
+UPDATE package_metadata_tsvectors SET locale = REPLACE(locale, '.utf8', '.UTF-8');
+UPDATE package_descriptions SET locale = REPLACE(locale, '.utf8', '.UTF-8');
+UPDATE package_synopsis SET locale = REPLACE(locale, '.utf8', '.UTF-8');
+UPDATE lint_warning_messages SET locale = REPLACE(locale, '.utf8', '.UTF-8');
+
+COMMIT;
diff --git a/sqitch/revert/change_locale_values.sql b/sqitch/revert/change_locale_values.sql
new file mode 100644
index 0000000..1771cc9
--- /dev/null
+++ b/sqitch/revert/change_locale_values.sql
@@ -0,0 +1,7 @@
+-- Revert guix-data-service:change_locale_values from pg
+
+BEGIN;
+
+-- XXX Add DDLs here.
+
+COMMIT;
diff --git a/sqitch/sqitch.plan b/sqitch/sqitch.plan
index 477f869..4d2b101 100644
--- a/sqitch/sqitch.plan
+++ b/sqitch/sqitch.plan
@@ -68,3 +68,4 @@ builds_remove_build_server_id_derivation_unique_constraint 2020-06-30T20:17:48Z
 derivation_output_details_sets_derivation_output_details_ids_index 2020-07-04T07:56:49Z Christopher Baines <mail@cbaines.net> # Add GIN index on derivation_output_details_ids
 add-tsvectors-per-locale 2020-07-16T18:19:44Z daniela <daniela@linux-ijv5> # Add tsvectors per locale
 change_package_descriptions_index 2020-08-23T11:56:00Z Christopher Baines <mail@cbaines.net> # Change the package_descriptions index
+change_locale_values 2020-09-26T10:20:22Z Christopher Baines <mail@cbaines.net> # Change locale values
diff --git a/sqitch/verify/change_locale_values.sql b/sqitch/verify/change_locale_values.sql
new file mode 100644
index 0000000..c435a50
--- /dev/null
+++ b/sqitch/verify/change_locale_values.sql
@@ -0,0 +1,7 @@
+-- Verify guix-data-service:change_locale_values on pg
+
+BEGIN;
+
+-- XXX Add verifications here.
+
+ROLLBACK;
diff --git a/tests/model-package-metadata.scm b/tests/model-package-metadata.scm
index 2b97465..2e67233 100644
--- a/tests/model-package-metadata.scm
+++ b/tests/model-package-metadata.scm
@@ -43,8 +43,8 @@
  ((guix-data-service model package-metadata)
   inferior-packages->translated-package-descriptions-and-synopsis
   (lambda (inferior inferior-package)
-    (cons `(("en_US.utf8" . "Fake synopsis"))
-          `(("en_US.utf8" . "Fake description")))))
+    (cons `(("en_US.UTF-8" . "Fake synopsis"))
+          `(("en_US.UTF-8" . "Fake description")))))
  (with-mock-inferior-packages
   (lambda ()
     (use-modules (guix-data-service model package)
diff --git a/tests/model-package.scm b/tests/model-package.scm
index 3999230..e953645 100644
--- a/tests/model-package.scm
+++ b/tests/model-package.scm
@@ -49,8 +49,8 @@
  ((guix-data-service model package-metadata)
   inferior-packages->translated-package-descriptions-and-synopsis
   (lambda (inferior inferior-package)
-    (cons `(("en_US.utf8" . "Fake synopsis"))
-          `(("en_US.utf8" . "Fake description")))))
+    (cons `(("en_US.UTF-8" . "Fake synopsis"))
+          `(("en_US.UTF-8" . "Fake description")))))
 (with-mock-inferior-packages
  (lambda ()
    (use-modules (guix-data-service model package)