diff options
author | Liliana Marie Prikler <liliana.prikler@gmail.com> | 2023-07-23 10:11:29 +0200 |
---|---|---|
committer | Liliana Marie Prikler <liliana.prikler@gmail.com> | 2023-07-23 10:11:29 +0200 |
commit | 4c204d01d57ac7da11a5772d5d4e3254d1c2408f (patch) | |
tree | c7e5cb013abc742734acd9613674df4ebddfdeef /guix | |
parent | 82bdb77082fa4e100761f70086b745dfb280c3ac (diff) | |
parent | 445a0359083388b5ee686e6e855f94a3aac5f79c (diff) | |
download | guix-4c204d01d57ac7da11a5772d5d4e3254d1c2408f.tar guix-4c204d01d57ac7da11a5772d5d4e3254d1c2408f.tar.gz |
Merge branch 'master' into gnome-team
Diffstat (limited to 'guix')
-rw-r--r-- | guix/build-system/gnu.scm | 6 | ||||
-rw-r--r-- | guix/build-system/texlive.scm | 28 | ||||
-rw-r--r-- | guix/build/pack.scm | 111 | ||||
-rw-r--r-- | guix/build/texlive-build-system.scm | 265 | ||||
-rw-r--r-- | guix/download.scm | 10 | ||||
-rw-r--r-- | guix/import/texlive.scm | 328 | ||||
-rw-r--r-- | guix/import/utils.scm | 16 | ||||
-rw-r--r-- | guix/licenses.scm | 6 | ||||
-rw-r--r-- | guix/lint.scm | 1 | ||||
-rw-r--r-- | guix/platform.scm | 12 | ||||
-rw-r--r-- | guix/profiles.scm | 36 | ||||
-rw-r--r-- | guix/scripts/pack.scm | 355 | ||||
-rw-r--r-- | guix/scripts/refresh.scm | 6 | ||||
-rw-r--r-- | guix/scripts/shell.scm | 4 | ||||
-rw-r--r-- | guix/svn-download.scm | 119 |
15 files changed, 897 insertions, 406 deletions
diff --git a/guix/build-system/gnu.scm b/guix/build-system/gnu.scm index 3308302472..c1aa187c42 100644 --- a/guix/build-system/gnu.scm +++ b/guix/build-system/gnu.scm @@ -266,13 +266,13 @@ listed in REFS." p)) -(define (standard-packages) +(define* (standard-packages #:optional (system (%current-system))) "Return the list of (NAME PACKAGE OUTPUT) or (NAME PACKAGE) tuples of standard packages used as implicit inputs of the GNU build system." ;; Resolve (gnu packages commencement) lazily to hide circular dependency. (let ((distro (resolve-module '(gnu packages commencement)))) - (module-ref distro '%final-inputs))) + ((module-ref distro '%final-inputs) system))) (define* (lower name #:key source inputs native-inputs outputs target @@ -303,7 +303,7 @@ standard packages used as implicit inputs of the GNU build system." (standard-cross-packages target 'host) '()) ,@(if implicit-inputs? - (standard-packages) + (standard-packages system) '()))) (host-inputs (if target inputs '())) diff --git a/guix/build-system/texlive.scm b/guix/build-system/texlive.scm index aefd573d11..88372faa58 100644 --- a/guix/build-system/texlive.scm +++ b/guix/build-system/texlive.scm @@ -34,8 +34,7 @@ texlive-ref texlive-origin %texlive-tag - %texlive-revision - %texlive-date)) + %texlive-revision)) ;; Commentary: ;; @@ -45,9 +44,8 @@ ;; These variables specify the SVN tag and the matching SVN revision. They ;; are taken from https://www.tug.org/svn/texlive/tags/ -(define %texlive-tag "texlive-2021.3") -(define %texlive-revision 59745) -(define %texlive-date "2021-06-28 21:59:21Z") +(define %texlive-tag "texlive-2023.0") +(define %texlive-revision 66594) (define (texlive-origin name version locations hash) "Return an <origin> object for a TeX Live package consisting of multiple @@ -88,24 +86,24 @@ level package ID." (let ((tex-mod (resolve-interface '(gnu packages tex)))) (module-ref tex-mod 'texlive-bin))) -(define (default-texlive-latex-base) - "Return the default texlive-latex-base package." +(define (texlive-latex-bin) + "Return the default texlive-latex-bin package." ;; Lazily resolve the binding to avoid a circular dependency. (let ((tex-mod (resolve-interface '(gnu packages tex)))) - (module-ref tex-mod 'texlive-latex-base))) + (module-ref tex-mod 'texlive-latex-bin))) (define* (lower name #:key source inputs native-inputs outputs system target - (texlive-latex-base (default-texlive-latex-base)) + (texlive-latex-bin? #true) (texlive-bin (default-texlive-bin)) #:allow-other-keys #:rest arguments) "Return a bag for NAME." (define private-keywords '(#:target #:inputs #:native-inputs - #:texlive-latex-base #:texlive-bin)) + #:texlive-latex-bin? #:texlive-bin)) (bag (name name) @@ -118,8 +116,8 @@ level package ID." ;; Keep the standard inputs of 'gnu-build-system'. ,@(standard-packages))) (build-inputs `(("texlive-bin" ,texlive-bin) - ,@(if texlive-latex-base - `(("texlive-latex-base" ,texlive-latex-base)) + ,@(if texlive-latex-bin? + `(("texlive-latex-bin" ,(texlive-latex-bin))) '()) ,@native-inputs)) (outputs outputs) @@ -130,8 +128,9 @@ level package ID." #:key source (tests? #f) - tex-directory (build-targets #f) + (create-formats #f) + (link-scripts #f) (tex-engine #f) ;; FIXME: This would normally default to "luatex" but @@ -161,8 +160,9 @@ level package ID." #$(with-build-variables inputs outputs #~(texlive-build #:name #$name #:source #+source - #:tex-directory #$tex-directory #:build-targets #$build-targets + #:create-formats #$create-formats + #:link-scripts #$link-scripts #:tex-engine #$(if tex-engine tex-engine tex-format) diff --git a/guix/build/pack.scm b/guix/build/pack.scm index 3b73d1b227..fcb1da2a6c 100644 --- a/guix/build/pack.scm +++ b/guix/build/pack.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> +;;; Copyright © 2021, 2023 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -17,8 +17,25 @@ ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. (define-module (guix build pack) + #:use-module (gnu build install) #:use-module (guix build utils) - #:export (tar-base-options)) + #:use-module (guix build store-copy) + #:use-module ((guix build union) #:select (relative-file-name)) + #:use-module (ice-9 match) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-26) + #:export (tar-base-options + populate-profile-root + build-self-contained-tarball)) + +;;; Commentary: + +;;; This module contains build-side common procedures used by the host-side +;;; (guix scripts pack) module, mostly to allow for code reuse. Due to making +;;; use of the (guix build store-copy) module, it transitively requires the +;;; sqlite and gcrypt extensions to be available. + +;;; Code: (define* (tar-base-options #:key tar compressor) "Return the base GNU tar options required to produce deterministic archives @@ -52,3 +69,93 @@ the `-I' option." ;; process. Use '--hard-dereference' to eliminate it. "--hard-dereference" "--check-links")) + +(define (assert-utf8-locale) + "Verify the current process is using the en_US.utf8 locale." + (unless (string=? "unset for tests" (getenv "GUIX_LOCPATH")) + (unless (false-if-exception (setlocale LC_ALL "en_US.utf8")) + (error "environment not configured for en_US.utf8 locale")))) + +(define* (populate-profile-root profile + #:key (profile-name "guix-profile") + localstatedir? + store-database + deduplicate? + (symlinks '())) + "Populate the root profile directory with SYMLINKS and a Guix database, when +LOCALSTATEDIR? is set, and a pre-computed STORE-DATABASE is provided. The +directory is created as \"root\" in the current working directory. When +DEDUPLICATE? is true, deduplicate the store items, which relies on hard +links. It needs to run in an environment where " + (define symlink->directives + ;; Return "populate directives" to make the given symlink and its + ;; parent directories. + (match-lambda + ((source '-> target) + (let ((target (string-append profile "/" target)) + (parent (dirname source))) + ;; Never add a 'directory' directive for "/" so as to + ;; preserve its ownership when extracting the archive (see + ;; below), and also because this would lead to adding the + ;; same entries twice in the tarball. + `(,@(if (string=? parent "/") + '() + `((directory ,parent))) + ;; Use a relative file name for compatibility with + ;; relocatable packs. + (,source -> ,(relative-file-name parent target))))))) + + (define directives + ;; Fully-qualified symlinks. + (append-map symlink->directives symlinks)) + + (define %root "root") + + (when localstatedir? + (unless store-database + (error "missing STORE-DATABASE argument"))) + + (assert-utf8-locale) + + ;; Note: there is not much to gain here with deduplication and there + ;; is the overhead of the '.links' directory, so turn it off by + ;; default. Furthermore GNU tar < 1.30 sometimes fails to extract + ;; tarballs with hard links: + ;; <http://lists.gnu.org/archive/html/bug-tar/2017-11/msg00009.html>. + (populate-store (list "profile") %root #:deduplicate? deduplicate?) + + (when localstatedir? + (install-database-and-gc-roots %root store-database + profile #:profile-name profile-name)) + + ;; Create SYMLINKS. + (for-each (cut evaluate-populate-directive <> %root) directives)) + +(define* (build-self-contained-tarball profile + tarball-file-name + #:key (profile-name "guix-profile") + localstatedir? + store-database + deduplicate? + symlinks + compressor-command) + "Create a self-contained tarball TARBALL-FILE-NAME from PROFILE, optionally +compressing it with COMPRESSOR-COMMAND, the complete command-line string to +use for the compressor." + (populate-profile-root profile + #:profile-name profile-name + #:localstatedir? localstatedir? + #:store-database store-database + #:deduplicate? deduplicate? + #:symlinks symlinks) + + (assert-utf8-locale) + + ;; GNU Tar recurses directories by default. Simply add the whole root + ;; directory, which contains all the files to be archived. This avoids + ;; creating duplicate files in the archives that would be stored as hard + ;; links by GNU Tar. + (apply invoke "tar" "-cvf" tarball-file-name "-C" "root" "." + (tar-base-options + #:tar "tar" + #:compressor compressor-command))) diff --git a/guix/build/texlive-build-system.scm b/guix/build/texlive-build-system.scm index 353fb934a6..a9fe9c80cc 100644 --- a/guix/build/texlive-build-system.scm +++ b/guix/build/texlive-build-system.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; Copyright © 2021 Thiago Jung Bauermann <bauermann@kolabnow.com> +;;; Copyright © 2023 Nicolas Goaziou <mail@nicolasgoaziou.fr> ;;; ;;; This file is part of GNU Guix. ;;; @@ -22,9 +23,11 @@ #:use-module ((guix build gnu-build-system) #:prefix gnu:) #:use-module (guix build utils) #:use-module (guix build union) - #:use-module (ice-9 match) + #:use-module (ice-9 format) #:use-module (ice-9 ftw) + #:use-module (ice-9 match) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-2) #:use-module (srfi srfi-26) #:export (%standard-phases texlive-build)) @@ -35,36 +38,264 @@ ;; ;; Code: -(define (compile-with-latex engine format file) +(define (runfiles-root-directories) + "Return list of root directories containing runfiles." + (scandir "." + (negate + (cut member <> '("." ".." "build" "doc" "source"))))) + +(define (texlive-input? input) + "Return #t if INPUT is a texlive input, #f otherwise." + (match input + (((or "source" (? (cut string-prefix? "texlive-" <>))) . _) #t) + (_ #f))) + +(define (install-as-runfiles dir regexp) + "Install files under DIR matching REGEXP on top of existing runfiles in the +current tree. Sub-directories below DIR are preserved when looking for the +runfile to replace. If a file has no matching runfile, it is ignored." + (let ((runfiles (append-map (cut find-files <>) + (runfiles-root-directories)))) + (for-each (lambda (file) + (match (filter + (cut string-suffix? + (string-drop file (string-length dir)) + <>) + runfiles) + ;; Current file is not a runfile. Ignore it. + (() #f) + ;; One candidate only. Replace it with the one from DIR. + ((destination) + (let ((target (dirname destination))) + (install-file file target) + (format #t "re-generated file ~s in ~s~%" + (basename file) + target))) + ;; Multiple candidates! Not much can be done. Hopefully, + ;; this should never happen. + (_ + (format (current-error-port) + "warning: ambiguous location for file ~s; ignoring it~%" + (basename file))))) + (find-files dir regexp)))) + +(define* (patch-shell-scripts #:rest _) + "Expand filenames for usual tools in shell scripts." + (when (file-exists? "scripts") + (let* ((commands '("awk" "basename" "cat" "grep" "mkdir" "rm" "sed" "sort" + "uname")) + (command-regexp (format #f + "\\b(~a)\\b" + (string-join commands "|")))) + (substitute* (find-files "scripts" "\\.sh$") + ((command-regexp _ command) + (which command)))))) + +(define* (delete-drv-files #:rest _) + "Delete pre-generated \".drv\" files in order to prevent build failures." + (when (file-exists? "source") + (for-each delete-file (find-files "source" "\\.drv$")))) + +(define* (generate-font-metrics #:key native-inputs inputs #:allow-other-keys) + ;; Decide what Metafont files to build by comparing them to the expected + ;; font metrics base names. Keep only files for which the two base names + ;; do match. + (define (font-metrics root) + (and (file-exists? root) + (map (cut basename <> ".tfm") (find-files root "\\.tfm$")))) + (define (font-files directory metrics) + (if (file-exists? directory) + (delete-duplicates + (filter (lambda (f) + (or (not metrics) + (member (basename f ".mf") metrics))) + (find-files directory "\\.mf$"))) + '())) + ;; Metafont files could be scattered across multiple directories. Treat + ;; each sub-directory as a separate font source. + (define (font-sources root metrics) + (delete-duplicates (map dirname (font-files root metrics)))) + (and-let* ((local-metrics (font-metrics "fonts/tfm")) + (local-sources (font-sources "fonts/source" local-metrics)) + ((not (null? local-sources))) ;nothing to generate: bail out + (root (getcwd)) + (metafont + (cond ((assoc-ref (or native-inputs inputs) "texlive-metafont") => + (cut string-append <> "/share/texmf-dist")) + (else + (error "Missing 'texlive-metafont' native input")))) + ;; Collect all font source files from texlive (native-)inputs so + ;; "mf" can know where to look for them. + (font-inputs + (delete-duplicates + (append-map (match-lambda + ((? (negate texlive-input?)) '()) + (("texlive-bin" . _) '()) + (("texlive-metafont" . _) + (list (string-append metafont "/metafont/base"))) + ((_ . input) + (font-sources input #f))) + (or native-inputs inputs))))) + ;; Tell mf where to find "mf.base". + (setenv "MFBASES" (string-append metafont "/web2c/")) + (mkdir-p "build") + (for-each + (lambda (source) + ;; Tell "mf" where are the font source files. In case current package + ;; provides multiple sources, treat them separately. + (setenv "MFINPUTS" + (string-join (cons (string-append root "/" source) + font-inputs) + ":")) + ;; Build font metrics (tfm). + (with-directory-excursion source + (for-each (lambda (font) + (format #t "building font ~a~%" font) + (invoke "mf" "-progname=mf" + (string-append "-output-directory=" + root "/build") + (string-append "\\" + "mode:=ljfour; " + "mag:=1; " + "batchmode; " + "input " + (basename font ".mf")))) + (font-files "." local-metrics))) + ;; Refresh font metrics at the appropriate location. + (install-as-runfiles "build" "\\.tfm$")) + local-sources))) + +(define* (create-formats #:key create-formats inputs #:allow-other-keys) + (define (collect-locations inputs pred) + (delete-duplicates + (append-map (match-lambda + ((? (negate texlive-input?)) '()) + ((_ . dir) + (if pred + (map dirname (find-files dir pred)) + (list dir)))) + inputs))) + (when create-formats + (setenv "TFMFONTS" + (string-join (collect-locations inputs "\\.tfm$") ":")) + (setenv "TEXINPUTS" + (string-join (collect-locations inputs #f) "//:" 'suffix)) + (setenv "LUAINPUTS" + (string-join (collect-locations inputs "\\.lua$") ":")) + (mkdir-p "web2c") + (for-each (cut invoke "fmtutil-sys" "--byfmt" <> "--fmtdir=web2c") + create-formats) + ;; Remove cruft. + (for-each delete-file (find-files "web2c" "\\.log$")))) + +(define (compile-with-latex engine format output file) (invoke engine "-interaction=nonstopmode" - "-output-directory=build" + (string-append "-output-directory=" output) (if format (string-append "&" format) "-ini") file)) (define* (build #:key inputs build-targets tex-engine tex-format #:allow-other-keys) - (mkdir "build") - (for-each (cut compile-with-latex tex-engine tex-format <>) - (if build-targets build-targets - (scandir "." (cut string-suffix? ".ins" <>))))) - -(define* (install #:key outputs tex-directory #:allow-other-keys) - (let* ((out (assoc-ref outputs "out")) - (target (string-append - out "/share/texmf-dist/tex/" tex-directory))) - (mkdir-p target) - (for-each delete-file (find-files "." "\\.(log|aux)$")) - (for-each (cut install-file <> target) - (find-files "build" ".*")))) + (let ((targets + (cond + (build-targets + ;; Collect the relative file names of all the specified targets. + (append-map (lambda (target) + (find-files "source" + (lambda (f _) + (string-suffix? (string-append "/" target) + f)))) + build-targets)) + ((directory-exists? "source") + ;; Prioritize ".ins" files over ".dtx" files. There's no + ;; scientific reasoning here; it just seems to work better. + (match (find-files "source" "\\.ins$") + (() (find-files "source" "\\.dtx$")) + (files files))) + (else '())))) + (unless (null? targets) + (let ((output (string-append (getcwd) "/build"))) + (mkdir-p output) + (for-each (lambda (target) + (with-directory-excursion (dirname target) + (compile-with-latex tex-engine + tex-format + output + (basename target)))) + targets)) + ;; Now move generated files from the "build" directory into the rest of + ;; the source tree, effectively replacing downloaded files. + ;; + ;; Documentation may have been generated, but replace only runfiles, + ;; i.e., files that belong neither to "doc" nor "source" trees. + ;; + ;; In TeX Live, all packages are fully pre-generated. As a consequence, + ;; a generated file from the "build" top directory absent from the rest of + ;; the tree is deemed unnecessary and can safely be ignored. + (install-as-runfiles "build" ".")))) + +(define* (install #:key outputs #:allow-other-keys) + (let ((out (assoc-ref outputs "out")) + (doc (assoc-ref outputs "doc"))) + ;; Take care of documentation. + (when (directory-exists? "doc") + (unless doc + (format (current-error-port) + "warning: missing 'doc' output for package documentation~%")) + (let ((doc-dir (string-append (or doc out) "/share/texmf-dist/doc"))) + (mkdir-p doc-dir) + (copy-recursively "doc" doc-dir))) + ;; Install runfiles. The package may not contain any, though. Create + ;; #$output anyway to handle this situation gracefully. + (mkdir-p out) + (let ((texmf (string-append out "/share/texmf-dist"))) + (for-each (lambda (root) + (let ((destination (string-append texmf "/" root))) + (mkdir-p destination) + (copy-recursively root destination))) + (runfiles-root-directories))))) + +(define* (link-scripts #:key link-scripts outputs #:allow-other-keys) + (when (pair? link-scripts) + (unless (file-exists? "scripts") + (error "missing \"scripts\" directory: no script to link")) + (let ((bin (string-append (assoc-ref outputs "out") "/bin")) + (filenames + (filter (lambda (f) (any (cut string-suffix? <> f) link-scripts)) + (find-files "scripts")))) + ;; Sanity check: make sure no provided script is ignored. + (let ((unknown (lset-difference string=? + (map basename link-scripts) + (map basename filenames)))) + (when (pair? unknown) + (error (format #f "cannot find script(s): ~a~%" + (string-join unknown))))) + ;; All lights are green. Create "bin/" and the symlinks. + (mkdir-p bin) + (for-each + (lambda (script) + ;; Remove extension, if any. + (let ((name (match (string-split (basename script) #\.) + ((name) name) + (tokens (string-join (drop-right tokens 1))))) + (origin (string-append "../share/texmf-dist/" script))) + (format #t "linking bin/~s to ~s~%" name origin) + (symlink origin (string-append bin "/" name)))) + filenames)))) (define %standard-phases (modify-phases gnu:%standard-phases (delete 'bootstrap) (delete 'configure) + (add-after 'unpack 'patch-shell-scripts patch-shell-scripts) + (add-before 'build 'delete-drv-files delete-drv-files) + (add-after 'delete-drv-files 'generate-font-metrics generate-font-metrics) (replace 'build build) + (add-after 'build 'create-formats create-formats) (delete 'check) - (replace 'install install))) + (replace 'install install) + (add-after 'install 'link-scripts link-scripts))) (define* (texlive-build #:key inputs (phases %standard-phases) #:allow-other-keys #:rest args) diff --git a/guix/download.scm b/guix/download.scm index 561a893eee..30d7c5a86e 100644 --- a/guix/download.scm +++ b/guix/download.scm @@ -91,8 +91,7 @@ "ftp://ftp.gnupg.org/gcrypt/") (gnome "https://download.gnome.org/" - "http://ftp.gnome.org/pub/GNOME/" - "http://mirror.yandex.ru/mirrors/ftp.gnome.org/") + "http://ftp.gnome.org/pub/GNOME/") (hackage "http://hackage.haskell.org/") (savannah ; http://download0.savannah.gnu.org/mirmon/savannah/ @@ -103,7 +102,6 @@ "https://mirror.csclub.uwaterloo.ca/nongnu/" "https://nongnu.askapache.com/" "https://savannah.c3sl.ufpr.br/" - "http://download.savannah.gnu.org/releases-noredirect/" "https://download-mirror.savannah.gnu.org/releases/" "ftp://ftp.twaren.net/Unix/NonGNU/" "ftp://mirror.csclub.uwaterloo.ca/nongnu/" @@ -129,9 +127,7 @@ "ftp://www.lt.netfilter.org/pub/") (kernel.org "http://linux-kernel.uio.no/pub/" - "http://kernel.osuosl.org/pub/" "http://ftp.be.debian.org/pub/" - "http://mirror.linux.org.au/" "https://mirrors.edge.kernel.org/pub/" "ftp://ftp.funet.fi/pub/mirrors/ftp.kernel.org/pub/") (apache ; from http://www.apache.org/mirrors/dist.html @@ -151,7 +147,6 @@ "http://www.x.org/releases/" ; main mirrors "http://mirror.csclub.uwaterloo.ca/x.org/" ; North America "http://xorg.mirrors.pair.com/" - "http://mirror.us.leaseweb.net/xorg/" "ftp://mirror.csclub.uwaterloo.ca/x.org/" "ftp://xorg.mirrors.pair.com/" "ftp://artfiles.org/x.org/" ; Europe @@ -168,8 +163,7 @@ "ftp://mirror.switch.ch/mirror/X11/" "ftp://mirrors.ircam.fr/pub/x.org/" "ftp://x.mirrors.skynet.be/pub/ftp.x.org/" - "http://x.cs.pu.edu.tw/" ; East Asia - "ftp://ftp.cs.cuhk.edu.hk/pub/X11" + "ftp://ftp.cs.cuhk.edu.hk/pub/X11" ; East Asia "ftp://ftp.u-aizu.ac.jp/pub/x11/x.org/" "ftp://ftp.yz.yamagata-u.ac.jp/pub/X11/x.org/" "ftp://ftp.kaist.ac.kr/x.org/" diff --git a/guix/import/texlive.scm b/guix/import/texlive.scm index 086cd363a9..b5a812b34e 100644 --- a/guix/import/texlive.scm +++ b/guix/import/texlive.scm @@ -48,9 +48,26 @@ ;;; ;;; Code: +;; Generic locations are parts of the tree shared by multiple packages. +;; Package definitions should single out files stored there, or all files in +;; the directory from all involved packages would be downloaded. +(define texlive-generic-locations + (list "doc/info/" + "doc/man/man1/" + "doc/man/man5/" + "doc/web2c/" + "scripts/context/lua/" + "scripts/context/perl/" + "scripts/texlive/" + "scripts/texlive-extra/" + "tex/generic/config/" + "tex/generic/hyphen/" + "web2c/")) + (define string->license (match-lambda - ("artistic2" 'gpl3+) + ("artistic2" 'artistic2.0) + ("apache2" 'asl2.0) ("gpl" 'gpl3+) ("gpl1" 'gpl1) ("gpl1+" 'gpl1+) @@ -70,19 +87,27 @@ ("lpplgpl" `(list lppl gpl1+)) ("lppl" 'lppl) - ("lppl1" 'lppl1.0+) ; usually means "or later" - ("lppl1.2" 'lppl1.2+) ; usually means "or later" - ("lppl1.3" 'lppl1.3+) ; usually means "or later" + ("lppl1" 'lppl1.0+) ; usually means "or later" + ("lppl1.2" 'lppl1.2+) ; usually means "or later" + ("lppl1.3" 'lppl1.3+) ; usually means "or later" ("lppl1.3a" 'lppl1.3a) ("lppl1.3b" 'lppl1.3b) ("lppl1.3c" 'lppl1.3c) - ("cc-by-2" 'cc-by-2.0) - ("cc-by-3" 'cc-by-3.0) + ("cc0" 'cc0) + ("cc-by-2" 'cc-by2.0) + ("cc-by-3" 'cc-by3.0) + ("cc-by-4" 'cc-by4.0) ("cc-by-sa-2" 'cc-by-sa2.0) ("cc-by-sa-3" 'cc-by-sa3.0) + ("cc-by-sa-4" 'cc-by-sa4.0) ("mit" 'expat) ("fdl" 'fdl1.3+) - ("gfl" 'gfl1.0) + ;; The GUST Font Nosource License, which is legally equivalent to + ;; lppl1.3c+, is no longer in use (per + ;; <https://www.gust.org.pl/projects/e-foundry/licenses>). It has de + ;; facto become GUST Font License 1.0. + ((or "gfl" "gfsl") 'gfl1.0) + ("isc" 'isc) ;; These are known non-free licenses ("noinfo" 'unknown) @@ -95,7 +120,8 @@ ("cc-by-nc-nd-2.5" 'non-free) ("cc-by-nc-nd-3" 'non-free) ("cc-by-nc-nd-4" 'non-free) - ((x) (string->license x)) + ((? string? x) (string->license (string-split x #\space))) + ((x) `(error unknown-license ,x)) ((lst ...) `(list ,@(map string->license lst))) (x `(error unknown-license ,x)))) @@ -108,21 +134,55 @@ (chr (char-downcase chr))) name))) +(define* (translate-depends depends #:optional texlive-only) + "Translate TeX Live packages DEPENDS into their equivalent Guix names +in `(gnu packages tex)' module, without \"texlive-\" prefix. The function +also removes packages not necessary in Guix. + +When TEXLIVE-ONLY is true, only TeX Live packages are returned." + (delete-duplicates + (filter-map (match-lambda + ;; Hyphenation. Every TeX Live package is replaced with + ;; "hyphen-complete", unless "hyphen-base" is the sole + ;; dependency. + ("hyphen-base" + (and (not (member "hyph-utf8" depends)) + "hyphen-base")) + ((or (? (cut string-prefix? "hyphen-" <>)) + "hyph-utf8" "dehyph" "dehyph-exptl" "ruhyphen" "ukrhyph") + (and (not texlive-only) "hyphen-complete")) + ;; Binaries placeholders are ignored. + ((? (cut string-suffix? ".ARCH" <>)) #f) + ;; So are TeX Live specific packages. + ((or (? (cut string-prefix? "texlive-" <>)) + "tlshell" "texlive.infra") + #f) + ;; And also development packages, which should inherit from + ;; the current package anyway. + ((? (cut string-suffix? "-dev" <>)) #f) + ;; Guix does not use Asymptote from TeX Live. Ignore it. + ("asymptote" #f) + ;; TeXworks in TeX Live is only for Windows. Don't bother. + ((or "texworks" "collection-texworks") #f) + ;; Others. + (name name)) + depends))) + (define (tlpdb-file) - (define texlive-bin + (define texlive-scripts ;; Resolve this variable lazily so that (gnu packages ...) does not end up ;; in the closure of this module. (module-ref (resolve-interface '(gnu packages tex)) - 'texlive-bin)) + 'texlive-scripts)) (with-store store (run-with-store store (mlet* %store-monad - ((drv (lower-object texlive-bin)) + ((drv (lower-object texlive-scripts)) (built (built-derivations (list drv)))) (match (derivation->output-paths drv) (((names . items) ...) - (return (string-append (first items) + (return (string-append (second items) ;"out" "/share/tlpkg/texlive.tlpdb")))))))) (define tlpdb @@ -133,12 +193,15 @@ '((name . string) (shortdesc . string) (longdesc . string) + (catalogue . string) (catalogue-license . string) (catalogue-ctan . string) (srcfiles . list) (runfiles . list) (docfiles . list) - (depend . simple-list))) + (binfiles . list) + (depend . simple-list) + (execute . simple-list))) (record (lambda* (key value alist #:optional (type 'string)) (let ((new @@ -195,6 +258,70 @@ (loop all (record key value current field-type) key)))) (loop all current #false)))))))))))) +;; Packages listed below are used to build "latex-bin" package, and therefore +;; cannot provide it automatically as a native input. Consequently, the +;; importer sets TEXLIVE-LATEX-BIN? argument to #F for all of them. +(define latex-bin-dependency-tree + (memoize + (lambda (package-database) + ;; Start out with "latex-bin", but also provide native inputs, which do + ;; not appear as dependents, as roots for the search. + (let loop ((packages + (list "latex-bin" "metafont" "modes" "tex")) + (deps '())) + (if (null? packages) + ;; `translate-depends' will always translate "hyphen-base" into + ;; "hyphen-complete". Make sure plain hyphen-base appears in the + ;; dependency tree. + (cons "hyphen-base" (translate-depends deps)) + (loop (append-map (lambda (name) + (let ((data (assoc-ref package-database name))) + (or (assoc-ref data 'depend) + '()))) + packages) + (append packages deps))))))) + +(define (formats package-data) + "Return a list of formats to build according to PACKAGE-DATA." + (and=> (assoc-ref package-data 'execute) + (lambda (actions) + (delete-duplicates + (filter-map + (lambda (action) + (match (string-split action #\space) + (("AddFormat" fmt . _) + (string-drop fmt (string-length "name="))) + (_ #f))) + ;; Get the right (alphabetic) order. + (reverse actions)))))) + +(define (linked-scripts name package-database) + "Return a list of script names to symlink from \"bin/\" directory for +package NAME according to PACKAGE-DATABASE. Consider as scripts files with +\".lua\", \".pl\", \".py\", \".sh\", \".tcl\", \".texlua\", \".tlu\" +extensions, and files without extension." + (and-let* ((data (assoc-ref package-database name)) + ;; Check if binaries are associated to the package. + (depend (assoc-ref data 'depend)) + ((member (string-append name ".ARCH") depend)) + ;; List those binaries. + (bin-data (assoc-ref package-database + ;; Any *nix-like architecture will do. + (string-append name ".x86_64-linux"))) + (binaries (map basename (assoc-ref bin-data 'binfiles))) + ;; List scripts candidates. Bail out if there are none. + (runfiles (assoc-ref data 'runfiles)) + (scripts (filter (cut string-prefix? "texmf-dist/scripts/" <>) + runfiles)) + ((pair? scripts))) + (filter-map (lambda (script) + (and (any (lambda (ext) + (member (basename script ext) binaries)) + '(".lua" ".pl" ".py" ".sh" ".tcl" ".texlua" ".tlu")) + (basename script))) + ;; Get the right (alphabetic) order. + (reverse scripts)))) + (define* (files-differ? directory package-name #:key (package-database tlpdb) @@ -233,28 +360,38 @@ of those files are returned that are unexpectedly installed." (lset-difference string=? (map strip-directory-prefix existing) files)))) -(define (files->directories files) - (define name->parts (cut string-split <> #\/)) - (map (cut string-join <> "/" 'suffix) - (delete-duplicates (map (lambda (file) - (drop-right (name->parts file) 1)) - (sort files string<)) - ;; Remove sub-directories, i.e. more specific - ;; entries with the same prefix. - (lambda (x y) (every equal? x y))))) +(define (files->locations files) + (define (trim-filename entry) + (string-join (drop-right (string-split entry #\/) 1) "/" 'suffix)) + ;; Generic locations are shared by multiple packages. Provide the full file + ;; name to make so as to extract only the files related to the package being + ;; imported. + (let-values (((generic specific) + (partition (lambda (f) + ;; Only grab files from generic locations, not + ;; sub-directories. + (any (cut string=? <> (trim-filename f)) + texlive-generic-locations)) + files))) + (append generic + ;; Remove sub-directories, i.e., more specific entries with the + ;; same prefix. + (delete-duplicates (sort (map trim-filename specific) string<) + string-prefix?)))) (define (tlpdb->package name version package-database) (and-let* ((data (assoc-ref package-database name)) - (dirs (files->directories - (filter-map (lambda (dir) + (locs (files->locations + (filter-map (lambda (file) ;; Ignore any file not starting with the ;; expected prefix. Nothing good can come ;; from this. - (and (string-prefix? "texmf-dist/" dir) - (string-drop dir (string-length "texmf-dist/")))) + (and (string-prefix? "texmf-dist/" file) + (string-drop file (string-length "texmf-dist/")))) (append (or (assoc-ref data 'docfiles) (list)) (or (assoc-ref data 'runfiles) (list)) (or (assoc-ref data 'srcfiles) (list)))))) + (texlive-name name) (name (guix-name name)) ;; TODO: we're ignoring the VERSION argument because that ;; information is distributed across %texlive-tag and @@ -262,51 +399,110 @@ of those files are returned that are unexpectedly installed." (ref (svn-multi-reference (url (string-append "svn://www.tug.org/texlive/tags/" %texlive-tag "/Master/texmf-dist")) - (locations dirs) + (locations locs) (revision %texlive-revision))) ;; Ignore arch-dependent packages. - (filtered-depends - (or (and=> (assoc-ref data 'depend) - (lambda (inputs) - (remove (cut string-suffix? ".ARCH" <>) inputs))) - '())) + (depends (or (assoc-ref data 'depend) '())) (source (with-store store (download-multi-svn-to-store store ref (string-append name "-svn-multi-checkout"))))) - (values - `(package - (inherit (simple-texlive-package - ,name - (list ,@dirs) - (base32 - ,(bytevector->nix-base32-string - (let-values (((port get-hash) (open-sha256-port))) - (write-file source port) - (force-output port) - (get-hash)))) - ,@(if (assoc-ref data 'srcfiles) '() '(#:trivial? #true)))) - ;; package->definition in (guix import utils) expects to see a - ;; version field. - (version ,version) - ,@(match filtered-depends - (() '()) - (inputs - `((propagated-inputs - (list ,@(map - (lambda (tex-name) - (let ((name (guix-name tex-name))) - (string->symbol name))) - inputs)))))) - ,@(or (and=> (assoc-ref data 'name) - (lambda (name) - `((home-page ,(string-append "https://ctan.org/pkg/" - name))))) - '((home-page "https://www.tug.org/texlive/"))) - (synopsis ,(assoc-ref data 'shortdesc)) - (description ,(and=> (assoc-ref data 'longdesc) beautify-description)) - (license ,(and=> (assoc-ref data 'catalogue-license) - string->license))) - filtered-depends))) + (let* ((scripts (linked-scripts texlive-name package-database)) + (tex-formats (formats data)) + (meta-package? (null? locs)) + (empty-package? (and meta-package? (not (pair? tex-formats))))) + (values + `(package + (name ,name) + (version (number->string %texlive-revision)) + (source ,(and (not meta-package?) + `(texlive-origin + name version + (list ,@(sort locs string<)) + (base32 + ,(bytevector->nix-base32-string + (let-values (((port get-hash) (open-sha256-port))) + (write-file source port) + (force-output port) + (get-hash))))))) + ,@(if (assoc-ref data 'docfiles) + '((outputs '("out" "doc"))) + '()) + ;; Set build-system. + ;; + ;; Use trivial build system only when the package contains no files, + ;; and no TeX format file is expected to be built. + (build-system ,(if empty-package? + 'trivial-build-system + 'texlive-build-system)) + ;; Generate arguments field. + ,@(let* ((latex-bin-dependency? + (member texlive-name + (latex-bin-dependency-tree package-database))) + (arguments + (append (if empty-package? + '(#:builder #~(mkdir #$output)) + '()) + (if latex-bin-dependency? + '(#:texlive-latex-bin? #f) + '()) + (if (pair? scripts) + `(#:link-scripts #~(list ,@scripts)) + '()) + (if (pair? tex-formats) + `(#:create-formats #~(list ,@tex-formats)) + '())))) + (if (pair? arguments) + `((arguments (list ,@arguments))) + '())) + ;; Native inputs. + ;; + ;; Texlive build system generates font metrics whenever a font + ;; metrics file has the same base name as a Metafont file. In this + ;; case, provide `texlive-metafont'. + ,@(or (and-let* ((runfiles (assoc-ref data 'runfiles)) + (metrics + (filter-map (lambda (f) + (and (string-suffix? ".tfm" f) + (basename f ".tfm"))) + runfiles)) + ((not (null? metrics))) + ((any (lambda (f) + (and (string-suffix? ".mf" f) + (member (basename f ".mf") metrics))) + runfiles))) + '((native-inputs (list texlive-metafont)))) + '()) + ;; Inputs. + ,@(match (append-map (lambda (s) + (cond ((string-suffix? ".pl" s) '(perl)) + ((string-suffix? ".py" s) '(python)) + ((string-suffix? ".tcl" s) '(tcl tk)) + (else '()))) + (or scripts '())) + (() '()) + (inputs `((inputs (list ,@(delete-duplicates inputs eq?)))))) + ;; Propagated inputs. + ,@(match (translate-depends depends) + (() '()) + (inputs + `((propagated-inputs + (list ,@(map (compose string->symbol guix-name) + (sort inputs string<?))))))) + (home-page + ,(cond + (meta-package? "https://www.tug.org/texlive/") + ((or (assoc-ref data 'catalogue) (assoc-ref data 'name)) => + (cut string-append "https://ctan.org/pkg/" <>)) + (else "https://www.tug.org/texlive/"))) + (synopsis ,(assoc-ref data 'shortdesc)) + (description ,(and=> (assoc-ref data 'longdesc) beautify-description)) + (license + ,(cond + (meta-package? + '(license:fsf-free "https://www.tug.org/texlive/copying.html")) + ((assoc-ref data 'catalogue-license) => string->license) + (else #f)))) + (translate-depends depends #t))))) (define texlive->guix-package (memoize diff --git a/guix/import/utils.scm b/guix/import/utils.scm index e9a0a7ecd7..257570e95b 100644 --- a/guix/import/utils.scm +++ b/guix/import/utils.scm @@ -2,7 +2,7 @@ ;;; Copyright © 2012, 2013, 2018, 2019, 2020, 2023 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2016 Jelle Licht <jlicht@fsfe.org> ;;; Copyright © 2016 David Craven <david@craven.ch> -;;; Copyright © 2017, 2019, 2020, 2022 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2017, 2019, 2020, 2022, 2023 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2018 Oleg Pykhalov <go.wigust@gmail.com> ;;; Copyright © 2019 Robert Vollmert <rob@vllmrt.net> ;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com> @@ -335,14 +335,21 @@ LENGTH characters." (cut string-trim-both <> #\') ;; Escape single @ to prevent it from being understood as ;; invalid Texinfo syntax. - (cut regexp-substitute/global #f "@" <> 'pre "@@" 'post))))) + (cut regexp-substitute/global #f "@" <> 'pre "@@" 'post) + ;; Wrap camelCase or PascalCase words in @code{...}. + (lambda (word) + (let ((pattern (make-regexp "([A-Z][a-z]+[A-Z]|[a-z]+[A-Z])"))) + (match (list-matches pattern word) + (() word) + (_ (string-append "@code{" word "}"))))))))) (words (string-tokenize (string-trim-both description) (char-set-complement (char-set #\space #\newline)))) (new-words (match words - (((and (or "A" "Functions" "Methods") first) . rest) + (((and (or "A" "Classes" "Functions" "Methods" "Tools") + first) . rest) (cons* "This" "package" "provides" (string-downcase first) rest)) (((and (or "Contains" @@ -436,10 +443,7 @@ APPEND-VERSION?/string is a string, append this string." (match guix-package ((or ('package ('name name) ('version version) . rest) - ('package ('inherit ('simple-texlive-package name . _)) - ('version version) . rest) ('let _ ('package ('name name) ('version version) . rest))) - `(define-public ,(string->symbol (cond ((string? append-version?/string) diff --git a/guix/licenses.scm b/guix/licenses.scm index e7e6ef3545..10f36b02f9 100644 --- a/guix/licenses.scm +++ b/guix/licenses.scm @@ -39,6 +39,7 @@ (define-module (guix licenses) #:use-module (srfi srfi-9) #:export (license? license-name license-uri license-comment + afl2.1 agpl1 agpl3 agpl3+ apsl2 asl1.1 asl2.0 @@ -171,6 +172,11 @@ cases, reduces to #t at macro-expansion time." (begin-license-definitions license? +(define afl2.1 + (license "AFL 2.1" + "https://spdx.org/licenses/AFL-2.1.html" + "https://www.gnu.org/licenses/license-list#AcademicFreeLicense")) + (define agpl1 (license "AGPL 1" "https://gnu.org/licenses/agpl.html" diff --git a/guix/lint.scm b/guix/lint.scm index 72b3f4e7b1..d173563e51 100644 --- a/guix/lint.scm +++ b/guix/lint.scm @@ -518,6 +518,7 @@ of a package, and INPUT-NAMES, a list of package specifications such as "qmake" "qttools-5" "texinfo" + "texlive-updmap.cfg" "xorg-server-for-tests" "yelp-tools"))) (map (lambda (input) diff --git a/guix/platform.scm b/guix/platform.scm index a2d95ab507..55917ca308 100644 --- a/guix/platform.scm +++ b/guix/platform.scm @@ -120,8 +120,8 @@ exception." (let ((s (platform-system platform))) (and (string? s) (string=? s system)))) (platforms)) - (raise-exception (condition (&platform-not-found-error - (target-or-system system)))))) + (raise (condition (&platform-not-found-error + (target-or-system system)))))) (define (lookup-platform-by-target target) "Return the platform corresponding to the given TARGET. Raise @@ -130,16 +130,16 @@ exception." (let ((t (platform-target platform))) (and (string? t) (string=? t target)))) (platforms)) - (raise-exception (condition (&platform-not-found-error - (target-or-system target)))))) + (raise (condition (&platform-not-found-error + (target-or-system target)))))) (define (lookup-platform-by-target-or-system target-or-system) "Return the platform corresponding to the given TARGET or SYSTEM. Raise &PLATFORM-NOT-FOUND-ERROR when no platform could be found." (or (false-if-platform-not-found (lookup-platform-by-target target-or-system)) (false-if-platform-not-found (lookup-platform-by-system target-or-system)) - (raise-exception (condition (&platform-not-found-error - (target-or-system target-or-system)))))) + (raise (condition (&platform-not-found-error + (target-or-system target-or-system)))))) (define (platform-system->target system) "Return the target matching the given SYSTEM if it exists or false diff --git a/guix/profiles.scm b/guix/profiles.scm index 2e2466ccbc..605d43f111 100644 --- a/guix/profiles.scm +++ b/guix/profiles.scm @@ -1789,17 +1789,16 @@ MANIFEST." '())))) (define texlive-inputs (append-map entry->texlive-input (manifest-entries manifest))) - (define texlive-bin - (module-ref (resolve-interface '(gnu packages tex)) 'texlive-bin)) + (define texlive-scripts + (module-ref (resolve-interface '(gnu packages tex)) 'texlive-scripts)) + (define texlive-libkpathsea + (module-ref (resolve-interface '(gnu packages tex)) 'texlive-libkpathsea)) (define coreutils (module-ref (resolve-interface '(gnu packages base)) 'coreutils)) (define grep (module-ref (resolve-interface '(gnu packages base)) 'grep)) (define sed (module-ref (resolve-interface '(gnu packages base)) 'sed)) - (define updmap.cfg - (module-ref (resolve-interface '(gnu packages tex)) - 'texlive-default-updmap.cfg)) (define build (with-imported-modules '((guix build utils) (guix build union)) @@ -1816,26 +1815,29 @@ MANIFEST." #:create-all-directories? #t #:log-port (%make-void-port "w")) - ;; XXX: This is annoying, but it's necessary because texlive-bin - ;; does not provide wrapped executables. + ;; XXX: This is annoying, but it's necessary because + ;; texlive-libkpathsea does not provide wrapped executables. (setenv "PATH" (string-append #$(file-append coreutils "/bin") ":" #$(file-append grep "/bin") ":" - #$(file-append sed "/bin"))) - (setenv "PERL5LIB" #$(file-append texlive-bin "/share/tlpkg")) + #$(file-append sed "/bin") + ":" + #$(file-append texlive-libkpathsea "/bin"))) + (setenv "PERL5LIB" #$(file-append texlive-scripts "/share/tlpkg")) (setenv "GUIX_TEXMF" "/tmp/texlive/share/texmf-dist") ;; Remove invalid maps from config file. (let* ((web2c (string-append #$output "/share/texmf-dist/web2c/")) (maproot (string-append #$output "/share/texmf-dist/fonts/map/")) (updmap.cfg (string-append web2c "updmap.cfg"))) - (mkdir-p web2c) - (copy-file #$updmap.cfg updmap.cfg) + (install-file #$(file-append texlive-scripts + "/share/texmf-dist/web2c/updmap.cfg") + web2c) (make-file-writable updmap.cfg) (let* ((port (open-pipe* OPEN_WRITE - #$(file-append texlive-bin "/bin/updmap-sys") + #$(file-append texlive-scripts "/bin/updmap-sys") "--syncwithtrees" "--nohash" "--force" @@ -1845,7 +1847,7 @@ MANIFEST." (error "failed to filter updmap.cfg"))) ;; Generate font maps. - (invoke #$(file-append texlive-bin "/bin/updmap-sys") + (invoke #$(file-append texlive-scripts "/bin/updmap-sys") (string-append "--cnffile=" updmap.cfg) (string-append "--dvipdfmxoutputdir=" maproot "dvipdfmx/updmap") @@ -1863,13 +1865,15 @@ MANIFEST." ;; to /tmp and run mktexlsr only once. (let ((a (string-append #$output "/share/texmf-dist")) (b "/tmp/texlive/share/texmf-dist") - (mktexlsr #$(file-append texlive-bin "/bin/mktexlsr"))) + (mktexlsr #$(file-append texlive-scripts "/bin/mktexlsr"))) + ;; Ignore original "updmap.cfg" from texlive-scripts input. + (delete-file "/tmp/texlive/share/texmf-dist/web2c/updmap.cfg") (copy-recursively a b) (invoke mktexlsr b) (install-file (string-append b "/ls-R") a)))))) - (mlet %store-monad ((texlive-base (manifest-lookup-package manifest "texlive-base"))) - (if (and texlive-base (pair? texlive-inputs)) + (with-monad %store-monad + (if (pair? texlive-inputs) (gexp->derivation "texlive-font-maps" build #:substitutable? #f #:local-build? #t diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index 0dc9979194..01995c48b7 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -72,6 +72,14 @@ %formats guix-pack)) +;;; Commentary: + +;;; This module implements the 'guix pack' command and the various supported +;;; formats. Where feasible, the builders of the packs should be implemented +;;; as single derivations to minimize storage requirements. + +;;; Code: + ;; This one is only for use in this module, so don't put it in %compressors. (define bootstrap-xz (compressor "bootstrap-xz" ".xz" @@ -197,153 +205,18 @@ target the profile's @file{bin/env} file: "Configure the environment to use the \"en_US.utf8\" locale provided by the GLIBC-UT8-LOCALES package." ;; Arrange to not depend on 'glibc-utf8-locales' when using '--bootstrap'. - (and (or (not (profile? profile)) - (profile-locales? profile)) - #~(begin - (setenv "GUIX_LOCPATH" - #+(file-append glibc-utf8-locales "/lib/locale")) - (setlocale LC_ALL "en_US.utf8")))) - -(define* (populate-profile-root profile - #:key (profile-name "guix-profile") - target - localstatedir? - deduplicate? - (symlinks '())) - "Populate the root profile directory with SYMLINKS and a Guix database, when -LOCALSTATEDIR? is set. When DEDUPLICATE? is true, deduplicate the store -items, which relies on hard links." - (define database - (and localstatedir? - (file-append (store-database (list profile)) - "/db/db.sqlite"))) - - (define bootstrap? - ;; Whether a '--bootstrap' environment is needed, for testing purposes. - ;; XXX: Infer that from available info. - (and (not database) (not (profile-locales? profile)))) - - (define (import-module? module) - ;; Since we don't use deduplication support in 'populate-store', don't - ;; import (guix store deduplication) and its dependencies, which includes - ;; Guile-Gcrypt, unless DEDUPLICATE? is #t. This makes it possible to run - ;; tests with '--bootstrap'. - (and (not-config? module) - (or deduplicate? (not (equal? '(guix store deduplication) module))))) - - (computed-file "profile-directory" - (with-imported-modules (source-module-closure - `((guix build pack) - (guix build store-copy) - (guix build utils) - (guix build union) - (gnu build install)) - #:select? import-module?) + (if (or (not (profile? profile)) + (profile-locales? profile)) #~(begin - (use-modules (guix build pack) - (guix build store-copy) - (guix build utils) - ((guix build union) #:select (relative-file-name)) - (gnu build install) - (srfi srfi-1) - (srfi srfi-26) - (ice-9 match)) - - (define symlink->directives - ;; Return "populate directives" to make the given symlink and its - ;; parent directories. - (match-lambda - ((source '-> target) - (let ((target (string-append #$profile "/" target)) - (parent (dirname source))) - ;; Never add a 'directory' directive for "/" so as to - ;; preserve its ownership when extracting the archive (see - ;; below), and also because this would lead to adding the - ;; same entries twice in the tarball. - `(,@(if (string=? parent "/") - '() - `((directory ,parent))) - ;; Use a relative file name for compatibility with - ;; relocatable packs. - (,source -> ,(relative-file-name parent target))))))) - - (define directives - ;; Fully-qualified symlinks. - (append-map symlink->directives '#$symlinks)) - - ;; Make sure non-ASCII file names are properly handled. - #+(set-utf8-locale profile) - - ;; Note: there is not much to gain here with deduplication and there - ;; is the overhead of the '.links' directory, so turn it off by - ;; default. Furthermore GNU tar < 1.30 sometimes fails to extract - ;; tarballs with hard links: - ;; <http://lists.gnu.org/archive/html/bug-tar/2017-11/msg00009.html>. - (populate-store (list "profile") #$output - #:deduplicate? #$deduplicate?) - - (when #+localstatedir? - (install-database-and-gc-roots #$output #+database #$profile - #:profile-name #$profile-name)) - - ;; Create SYMLINKS. - (for-each (cut evaluate-populate-directive <> #$output) - directives))) - #:local-build? #f - #:guile (if bootstrap? %bootstrap-guile (default-guile)) - #:options (list #:references-graphs `(("profile" ,profile)) - #:target target))) + (setenv "GUIX_LOCPATH" + #+(file-append glibc-utf8-locales "/lib/locale")) + (setlocale LC_ALL "en_US.utf8")) + #~(setenv "GUIX_LOCPATH" "unset for tests"))) ;;; ;;; Tarball format. ;;; -(define* (self-contained-tarball/builder profile - #:key (profile-name "guix-profile") - target - localstatedir? - deduplicate? - symlinks - compressor - archiver) - "Return a GEXP that can build a self-contained tarball." - - (define root (populate-profile-root profile - #:profile-name profile-name - #:target target - #:localstatedir? localstatedir? - #:deduplicate? deduplicate? - #:symlinks symlinks)) - - (with-imported-modules (source-module-closure '((guix build pack) - (guix build utils))) - #~(begin - (use-modules (guix build pack) - (guix build utils)) - - ;; Make sure non-ASCII file names are properly handled. - #+(set-utf8-locale profile) - - (define tar #+(file-append archiver "/bin/tar")) - - (define %root (if #$localstatedir? "." #$root)) - - (when #$localstatedir? - ;; Fix the permission of the Guix database file, which was made - ;; read-only when copied to the store in populate-profile-root. - (copy-recursively #$root %root) - (chmod (string-append %root "/var/guix/db/db.sqlite") #o644)) - - (with-directory-excursion %root - ;; GNU Tar recurses directories by default. Simply add the whole - ;; current directory, which contains all the files to be archived. - ;; This avoids creating duplicate files in the archives that would - ;; be stored as hard links by GNU Tar. - (apply invoke tar "-cvf" #$output "." - (tar-base-options - #:tar tar - #:compressor #+(and=> compressor compressor-command))))))) - (define* (self-contained-tarball name profile #:key target (profile-name "guix-profile") @@ -365,16 +238,48 @@ added to the pack." (warning (G_ "entry point not supported in the '~a' format~%") 'tarball)) - (gexp->derivation (string-append name ".tar" - (compressor-extension compressor)) - (self-contained-tarball/builder profile - #:profile-name profile-name - #:target target - #:localstatedir? localstatedir? - #:deduplicate? deduplicate? - #:symlinks symlinks - #:compressor compressor - #:archiver archiver))) + (define database + (and localstatedir? + (file-append (store-database (list profile)) + "/db/db.sqlite"))) + + (gexp->derivation + (string-append name ".tar" (compressor-extension compressor)) + ;; XXX: The conditional around deduplicate? is to allow the test to run + ;; without an external store. + (with-extensions (if deduplicate? (list guile-gcrypt) '()) + (with-imported-modules (let ((lst (source-module-closure + '((guix build pack) + (guix build utils)) + #:select? not-config?))) + (if deduplicate? + lst + (delete '(guix store deduplication) lst))) + + (source-module-closure '((guix build pack) + (guix build utils)) + #:select? not-config?) + #~(begin + (use-modules (guix build pack) + (guix build utils)) + + ;; Make sure non-ASCII file names are properly handled. + #+(set-utf8-locale profile) + + (setenv "PATH" #+(file-append archiver "/bin")) + + (build-self-contained-tarball #$profile + #$output + #:profile-name #$profile-name + #:localstatedir? #$localstatedir? + #:store-database #+database + #:deduplicate? #$deduplicate? + #:symlinks '#$symlinks + #:compressor-command + #+(and=> compressor + compressor-command))))) + #:target target + #:references-graphs `(("profile" ,profile)))) ;;; @@ -719,20 +624,10 @@ Valid compressors are: ~a~%") compressor-name %valid-compressors))) (warning (G_ "entry point not supported in the '~a' format~%") 'deb)) - (define data-tarball - (computed-file (string-append "data.tar" (compressor-extension - compressor)) - (self-contained-tarball/builder profile - #:target target - #:profile-name profile-name - #:localstatedir? localstatedir? - #:deduplicate? deduplicate? - #:symlinks symlinks - #:compressor compressor - #:archiver archiver) - #:local-build? #f ;allow offloading - #:options (list #:references-graphs `(("profile" ,profile)) - #:target target))) + (define database + (and localstatedir? + (file-append (store-database (list profile)) + "/db/db.sqlite"))) (define build (with-extensions (list guile-gcrypt) @@ -750,6 +645,9 @@ Valid compressors are: ~a~%") compressor-name %valid-compressors))) (ice-9 optargs) (srfi srfi-1)) + ;; Make sure non-ASCII file names are properly handled. + #+(set-utf8-locale profile) + (define machine-type ;; Extract the machine type from the specified target, else from the ;; current system. @@ -803,10 +701,26 @@ Valid compressors are: ~a~%") compressor-name %valid-compressors))) (lambda (port) (format port "~a~%" debian-format-version))) - (define data-tarball-file-name (strip-store-file-name - #+data-tarball)) + (define compressor-command + #+(and=> compressor compressor-command)) - (copy-file #+data-tarball data-tarball-file-name) + (define compressor-extension + #+(compressor-extension compressor)) + + (define data-tarball-file-name + (string-append "data.tar" compressor-extension)) + + (setenv "PATH" #+(file-append archiver "/bin")) + + (build-self-contained-tarball #$profile + data-tarball-file-name + #:profile-name #$profile-name + #:localstatedir? #$localstatedir? + #:store-database #+database + #:deduplicate? #$deduplicate? + #:symlinks '#$symlinks + #:compressor-command + compressor-command) ;; Generate the control archive. (let-keywords '#$extra-options #f @@ -815,8 +729,7 @@ Valid compressors are: ~a~%") compressor-name %valid-compressors))) (triggers-file #f)) (define control-tarball-file-name - (string-append "control.tar" - #$(compressor-extension compressor))) + (string-append "control.tar" compressor-extension)) ;; Write the compressed control tarball. Only the control file is ;; mandatory (see: 'man deb' and 'man deb-control'). @@ -846,7 +759,7 @@ Section: misc (apply invoke tar `(,@(tar-base-options #:tar tar - #:compressor #+(and=> compressor compressor-command)) + #:compressor compressor-command) "-cvf" ,control-tarball-file-name "control" ,@(if postinst-file '("postinst") '()) @@ -857,7 +770,9 @@ Section: misc "debian-binary" control-tarball-file-name data-tarball-file-name)))))) - (gexp->derivation (string-append name ".deb") build)) + (gexp->derivation (string-append name ".deb") build + #:target target + #:references-graphs `(("profile" ,profile)))) ;;; @@ -881,66 +796,27 @@ PREUN-FILE and POSTUN-FILE can be provided via EXTRA-OPTIONS." (when entry-point (warning (G_ "entry point not supported in the '~a' format~%") 'rpm)) - (define root (populate-profile-root profile - #:profile-name profile-name - #:target target - #:localstatedir? localstatedir? - #:deduplicate? deduplicate? - #:symlinks symlinks)) - - (define payload - (let* ((raw-cpio-file-name "payload.cpio") - (compressed-cpio-file-name (string-append raw-cpio-file-name - (compressor-extension - compressor)))) - (computed-file compressed-cpio-file-name - (with-imported-modules (source-module-closure - '((guix build utils) - (guix cpio) - (guix rpm))) - #~(begin - (use-modules (guix build utils) - (guix cpio) - (guix rpm) - (srfi srfi-1)) - - ;; Make sure non-ASCII file names are properly handled. - #+(set-utf8-locale profile) - - (define %root (if #$localstatedir? "." #$root)) - - (when #$localstatedir? - ;; Fix the permission of the Guix database file, which was made - ;; read-only when copied to the store in populate-profile-root. - (copy-recursively #$root %root) - (chmod (string-append %root "/var/guix/db/db.sqlite") #o644)) - - (call-with-output-file #$raw-cpio-file-name - (lambda (port) - (with-directory-excursion %root - ;; The first "." entry is discarded. - (write-cpio-archive - (remove fhs-directory? - (cdr (find-files "." #:directories? #t))) - port)))) - (when #+(compressor-command compressor) - (apply invoke (append #+(compressor-command compressor) - (list #$raw-cpio-file-name)))) - (copy-file #$compressed-cpio-file-name #$output))) - #:local-build? #f))) ;allow offloading + (define database + (and localstatedir? + (file-append (store-database (list profile)) + "/db/db.sqlite"))) (define build (with-extensions (list guile-gcrypt) (with-imported-modules `(((guix config) => ,(make-config.scm)) ,@(source-module-closure `((gcrypt hash) + (guix build pack) (guix build utils) + (guix cpio) (guix profiles) (guix rpm)) #:select? not-config?)) #~(begin (use-modules (gcrypt hash) + (guix build pack) (guix build utils) + (guix cpio) (guix profiles) (guix rpm) (ice-9 binary-ports) @@ -952,6 +828,35 @@ PREUN-FILE and POSTUN-FILE can be provided via EXTRA-OPTIONS." ;; Make sure non-ASCII file names are properly handled. #+(set-utf8-locale profile) + (define %root "root") + + (populate-profile-root #$profile + #:profile-name #$profile-name + #:localstatedir? #$localstatedir? + #:store-database #+database + #:deduplicate? #$deduplicate? + #:symlinks '#$symlinks) + + (define raw-cpio-file-name "payload.cpio") + + ;; Generate CPIO payload. + (call-with-output-file raw-cpio-file-name + (lambda (port) + (with-directory-excursion %root + ;; The first "." entry is discarded. + (write-cpio-archive + (remove fhs-directory? + (cdr (find-files "." #:directories? #t))) + port)))) + + (when #+(compressor-command compressor) + (apply invoke (append #+(compressor-command compressor) + (list raw-cpio-file-name)))) + + (define cpio-file-name + (string-append "payload.cpio" + #$(compressor-extension compressor))) + (define machine-type (and=> (or #$target %host-type) (lambda (triplet) @@ -979,7 +884,7 @@ PREUN-FILE and POSTUN-FILE can be provided via EXTRA-OPTIONS." #:target (or #$target %host-type))) (define payload-digest - (bytevector->hex-string (file-sha256 #$payload))) + (bytevector->hex-string (file-sha256 cpio-file-name))) (let-keywords '#$extra-options #f ((relocatable? #f) (prein-file #f) @@ -989,7 +894,7 @@ PREUN-FILE and POSTUN-FILE can be provided via EXTRA-OPTIONS." (let ((header (generate-header name version payload-digest - #$root + %root #$(compressor-name compressor) #:target (or #$target %host-type) #:relocatable? relocatable? @@ -1001,7 +906,7 @@ PREUN-FILE and POSTUN-FILE can be provided via EXTRA-OPTIONS." (define header-sha256 (bytevector->hex-string (sha256 (u8-list->bytevector header)))) - (define payload-size (stat:size (stat #$payload))) + (define payload-size (stat:size (stat cpio-file-name))) (define header+compressed-payload-size (+ (length header) payload-size)) @@ -1011,7 +916,7 @@ PREUN-FILE and POSTUN-FILE can be provided via EXTRA-OPTIONS." header+compressed-payload-size)) ;; Serialize the archive components to a file. - (call-with-input-file #$payload + (call-with-input-file cpio-file-name (lambda (in) (call-with-output-file #$output (lambda (out) @@ -1020,7 +925,9 @@ PREUN-FILE and POSTUN-FILE can be provided via EXTRA-OPTIONS." header)) (sendfile out in payload-size))))))))))) - (gexp->derivation (string-append name ".rpm") build)) + (gexp->derivation (string-append name ".rpm") build + #:target target + #:references-graphs `(("profile" ,profile)))) ;;; diff --git a/guix/scripts/refresh.scm b/guix/scripts/refresh.scm index f5cb18af22..a9241aa20d 100644 --- a/guix/scripts/refresh.scm +++ b/guix/scripts/refresh.scm @@ -228,7 +228,8 @@ options like '--recursive'." (let* ((input->package (match-lambda ((name (? package? package) _ ...) package) (_ #f))) - (final-inputs (map input->package %final-inputs)) + (final-inputs (map input->package + (%final-inputs (%current-system)))) (core (append final-inputs (append-map (compose (cut filter-map input->package <>) package-transitive-inputs) @@ -590,7 +591,8 @@ all are dependent packages: ~{~a~^ ~}~%") (string-append (config-directory) "/upstream/trustedkeys.kbx")))) (let* ((spec-line - (compose location->string + (compose (cut string-trim-right <> char-set:digit) + location->string package-location update-spec-package)) ;; Sort the specs so that we update packages from the diff --git a/guix/scripts/shell.scm b/guix/scripts/shell.scm index 1b42cc2af0..d67152cef7 100644 --- a/guix/scripts/shell.scm +++ b/guix/scripts/shell.scm @@ -374,7 +374,9 @@ return #f and #f." (define (key->file key) (string-append (%profile-cache-directory) "/" key)) - (let loop ((opts opts) + ;; A given key such as 'system might appear more than once in OPTS, so + ;; process it backwards so the last occurrence "wins". + (let loop ((opts (reverse opts)) (system (%current-system)) (file #f) (specs '())) diff --git a/guix/svn-download.scm b/guix/svn-download.scm index 769571b5f6..c6688908de 100644 --- a/guix/svn-download.scm +++ b/guix/svn-download.scm @@ -23,6 +23,7 @@ #:use-module (guix gexp) #:use-module (guix store) #:use-module (guix monads) + #:use-module (guix modules) #:use-module (guix packages) #:use-module (guix utils) #:use-module ((guix build svn) #:prefix build:) @@ -79,22 +80,38 @@ "Return a fixed-output derivation that fetches REF, a <svn-reference> object. The output is expected to have recursive hash HASH of type HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." + + (define guile-json + (module-ref (resolve-interface '(gnu packages guile)) 'guile-json-4)) + + (define guile-lzlib + (module-ref (resolve-interface '(gnu packages guile)) 'guile-lzlib)) + + (define guile-gnutls + (module-ref (resolve-interface '(gnu packages tls)) 'guile-gnutls)) + (define build - (with-imported-modules '((guix build svn) - (guix build utils)) - #~(begin - (use-modules (guix build svn) - (ice-9 match)) - - (svn-fetch (getenv "svn url") - (string->number (getenv "svn revision")) - #$output - #:svn-command #+(file-append svn "/bin/svn") - #:recursive? (match (getenv "svn recursive?") - ("yes" #t) - (_ #f)) - #:user-name (getenv "svn user name") - #:password (getenv "svn password"))))) + (with-imported-modules + (source-module-closure '((guix build svn) + (guix build download-nar) + (guix build utils))) + (with-extensions (list guile-json guile-gnutls ;for (guix swh) + guile-lzlib) + #~(begin + (use-modules (guix build svn) + (guix build download-nar) + (ice-9 match)) + + (or (svn-fetch (getenv "svn url") + (string->number (getenv "svn revision")) + #$output + #:svn-command #+(file-append svn "/bin/svn") + #:recursive? (match (getenv "svn recursive?") + ("yes" #t) + (_ #f)) + #:user-name (getenv "svn user name") + #:password (getenv "svn password")) + (download-nar #$output)))))) (mlet %store-monad ((guile (package->derivation guile system))) (gexp->derivation (or name "svn-checkout") build @@ -143,33 +160,53 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." "Return a fixed-output derivation that fetches REF, a <svn-multi-reference> object. The output is expected to have recursive hash HASH of type HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." + + (define guile-json + (module-ref (resolve-interface '(gnu packages guile)) 'guile-json-4)) + + (define guile-lzlib + (module-ref (resolve-interface '(gnu packages guile)) 'guile-lzlib)) + + (define guile-gnutls + (module-ref (resolve-interface '(gnu packages tls)) 'guile-gnutls)) + (define build - (with-imported-modules '((guix build svn) - (guix build utils)) - #~(begin - (use-modules (guix build svn) - (guix build utils) - (srfi srfi-1) - (ice-9 match)) - - (for-each (lambda (location) - ;; The directory must exist if we are to fetch only a - ;; single file. - (unless (string-suffix? "/" location) - (mkdir-p (string-append #$output "/" (dirname location)))) - (svn-fetch (string-append (getenv "svn url") "/" location) - (string->number (getenv "svn revision")) - (if (string-suffix? "/" location) - (string-append #$output "/" location) - (string-append #$output "/" (dirname location))) - #:svn-command #+(file-append svn "/bin/svn") - #:recursive? (match (getenv "svn recursive?") - ("yes" #t) - (_ #f)) - #:user-name (getenv "svn user name") - #:password (getenv "svn password"))) - (call-with-input-string (getenv "svn locations") - read))))) + (with-imported-modules + (source-module-closure '((guix build svn) + (guix build download-nar) + (guix build utils))) + (with-extensions (list guile-json guile-gnutls ;for (guix swh) + guile-lzlib) + #~(begin + (use-modules (guix build svn) + (guix build utils) + (guix build download-nar) + (srfi srfi-1) + (ice-9 match)) + + (or (every + (lambda (location) + ;; The directory must exist if we are to fetch only a + ;; single file. + (unless (string-suffix? "/" location) + (mkdir-p (string-append #$output "/" (dirname location)))) + (svn-fetch (string-append (getenv "svn url") "/" location) + (string->number (getenv "svn revision")) + (if (string-suffix? "/" location) + (string-append #$output "/" location) + (string-append #$output "/" (dirname location))) + #:svn-command #+(file-append svn "/bin/svn") + #:recursive? (match (getenv "svn recursive?") + ("yes" #t) + (_ #f)) + #:user-name (getenv "svn user name") + #:password (getenv "svn password"))) + (call-with-input-string (getenv "svn locations") + read)) + (begin + (when (file-exists? #$output) + (delete-file-recursively #$output)) + (download-nar #$output))))))) (mlet %store-monad ((guile (package->derivation guile system))) (gexp->derivation (or name "svn-checkout") build |