diff options
Diffstat (limited to 'guix')
| -rw-r--r-- | guix/build-system/cargo.scm | 2 | ||||
| -rw-r--r-- | guix/build-system/gnu.scm | 6 | ||||
| -rw-r--r-- | guix/build/copy-build-system.scm | 2 | ||||
| -rw-r--r-- | guix/build/gnu-build-system.scm | 2 | ||||
| -rw-r--r-- | guix/build/gremlin.scm | 76 | ||||
| -rw-r--r-- | guix/build/lisp-utils.scm | 2 | ||||
| -rw-r--r-- | guix/build/maven/pom.scm | 14 | ||||
| -rw-r--r-- | guix/build/python-build-system.scm | 24 | ||||
| -rw-r--r-- | guix/build/rpath.scm | 59 | ||||
| -rw-r--r-- | guix/build/ruby-build-system.scm | 25 | ||||
| -rw-r--r-- | guix/build/utils.scm | 73 | ||||
| -rw-r--r-- | guix/gexp.scm | 29 | ||||
| -rw-r--r-- | guix/packages.scm | 31 | ||||
| -rw-r--r-- | guix/scripts/pack.scm | 27 | ||||
| -rw-r--r-- | guix/store/roots.scm | 2 | ||||
| -rw-r--r-- | guix/utils.scm | 33 |
16 files changed, 249 insertions, 158 deletions
diff --git a/guix/build-system/cargo.scm b/guix/build-system/cargo.scm index 6c8edf6bac..ed69746a3b 100644 --- a/guix/build-system/cargo.scm +++ b/guix/build-system/cargo.scm @@ -217,7 +217,7 @@ any dependent crates. This can be a benefits: - It avoids waiting for quadratic builds from source: cargo always builds dependencies within the current workspace. This is largely due to Rust not having a stable ABI and other resolutions that cargo applies. This means that - if we have a depencency chain of X -> Y -> Z and we build each definition + if we have a dependency chain of X -> Y -> Z and we build each definition independently the following will happen: * Cargo will build and test crate Z * Cargo will build crate Z in Y's workspace, then build and test Y diff --git a/guix/build-system/gnu.scm b/guix/build-system/gnu.scm index 6b481ad45c..2c23197e77 100644 --- a/guix/build-system/gnu.scm +++ b/guix/build-system/gnu.scm @@ -215,7 +215,7 @@ use `--strip-all' as the arguments to `strip'." (arguments (let ((a (default-keyword-arguments (package-arguments p) '(#:configure-flags '() - #:strip-flags '("--strip-debug"))))) + #:strip-flags '("--strip-unneeded"))))) (substitute-keyword-arguments a ((#:configure-flags flags) `(cons* "--disable-shared" "LDFLAGS=-static" ,flags)) @@ -337,7 +337,7 @@ standard packages used as implicit inputs of the GNU build system." (parallel-tests? #t) (patch-shebangs? #t) (strip-binaries? #t) - (strip-flags ''("--strip-debug" + (strip-flags ''("--strip-unneeded" "--enable-deterministic-archives")) (strip-directories ''("lib" "lib64" "libexec" "bin" "sbin")) @@ -492,7 +492,7 @@ is one of `host' or `target'." (parallel-build? #t) (parallel-tests? #t) (patch-shebangs? #t) (strip-binaries? #t) - (strip-flags ''("--strip-debug" + (strip-flags ''("--strip-unneeded" "--enable-deterministic-archives")) (strip-directories ''("lib" "lib64" "libexec" "bin" "sbin")) diff --git a/guix/build/copy-build-system.scm b/guix/build/copy-build-system.scm index a86f0cde29..ac4a62a074 100644 --- a/guix/build/copy-build-system.scm +++ b/guix/build/copy-build-system.scm @@ -58,7 +58,7 @@ In the above, FILTERS are optional. one of the elements in the list. - With `#:include-regexp`, install subpaths matching the regexps in the list. - The `#:exclude*` FILTERS work similarly. Without `#:include*` flags, - install every subpath but the files matching the `#:exlude*` filters. + install every subpath but the files matching the `#:exclude*` filters. If both `#:include*` and `#:exclude*` are specified, the exclusion is done on the inclusion list. diff --git a/guix/build/gnu-build-system.scm b/guix/build/gnu-build-system.scm index 2e7dff2034..d3347c9518 100644 --- a/guix/build/gnu-build-system.scm +++ b/guix/build/gnu-build-system.scm @@ -425,7 +425,7 @@ makefiles." (objcopy-command (if target (string-append target "-objcopy") "objcopy")) - (strip-flags '("--strip-debug" + (strip-flags '("--strip-unneeded" "--enable-deterministic-archives")) (strip-directories '("lib" "lib64" "libexec" "bin" "sbin")) diff --git a/guix/build/gremlin.scm b/guix/build/gremlin.scm index e8ea66dfb3..6857e47b99 100644 --- a/guix/build/gremlin.scm +++ b/guix/build/gremlin.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2015, 2018 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2015, 2018, 2020 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -41,6 +41,16 @@ elf-dynamic-info-runpath expand-origin + file-dynamic-info + file-runpath + file-needed + + missing-runpath-error? + missing-runpath-error-file + runpath-too-long-error? + runpath-too-long-error-file + set-file-runpath + validate-needed-in-runpath strip-runpath)) @@ -232,6 +242,23 @@ string table if the type is a string." dynamic-entry-value)) '())))))) +(define (file-dynamic-info file) + "Return the <elf-dynamic-info> record of FILE, or #f if FILE lacks dynamic +info." + (call-with-input-file file + (lambda (port) + (elf-dynamic-info (parse-elf (get-bytevector-all port)))))) + +(define (file-runpath file) + "Return the DT_RUNPATH dynamic entry of FILE as a list of string, or #f if +FILE lacks dynamic info." + (and=> (file-dynamic-info file) elf-dynamic-info-runpath)) + +(define (file-needed file) + "Return the list of DT_NEEDED dynamic entries of FILE, or #f if FILE lacks +dynamic info." + (and=> (file-dynamic-info file) elf-dynamic-info-needed)) + (define %libc-libraries ;; List of libraries as of glibc 2.21 (there are more but those are ;; typically mean to be LD_PRELOADed and thus do not appear as NEEDED.) @@ -364,4 +391,49 @@ according to DT_NEEDED." (false-if-exception (close-port port)) (apply throw key args)))) -;;; gremlin.scm ends here + +(define-condition-type &missing-runpath-error &elf-error + missing-runpath-error? + (file missing-runpath-error-file)) + +(define-condition-type &runpath-too-long-error &elf-error + runpath-too-long-error? + (file runpath-too-long-error-file)) + +(define (set-file-runpath file path) + "Set the value of the DT_RUNPATH dynamic entry of FILE, which must name an +ELF file, to PATH, a list of strings. Raise a &missing-runpath-error or +&runpath-too-long-error when appropriate." + (define (call-with-input+output-file file proc) + (let ((port (open-file file "r+b"))) + (guard (c (#t (close-port port) (raise c))) + (proc port) + (close-port port)))) + + (call-with-input+output-file file + (lambda (port) + (let* ((elf (parse-elf (get-bytevector-all port))) + (entries (dynamic-entries elf (dynamic-link-segment elf))) + (runpath (find (lambda (entry) + (= DT_RUNPATH (dynamic-entry-type entry))) + entries)) + (path (string->utf8 (string-join path ":")))) + (unless runpath + (raise (condition (&missing-runpath-error (elf elf) + (file file))))) + + ;; There might be padding left beyond RUNPATH in the string table, but + ;; we don't know, so assume there's no padding. + (unless (<= (bytevector-length path) + (bytevector-length + (string->utf8 (dynamic-entry-value runpath)))) + (raise (condition (&runpath-too-long-error (elf #f #;elf) + (file file))))) + + (seek port (dynamic-entry-offset runpath) SEEK_SET) + (put-bytevector port path) + (put-u8 port 0))))) + +;;; Local Variables: +;;; eval: (put 'call-with-input+output-file 'scheme-indent-function 1) +;;; End: diff --git a/guix/build/lisp-utils.scm b/guix/build/lisp-utils.scm index 8a02cb68dd..17d2637f87 100644 --- a/guix/build/lisp-utils.scm +++ b/guix/build/lisp-utils.scm @@ -281,7 +281,7 @@ DEPENDENCY-PREFIXES to ensure references to those libraries are retained." type compress? #:allow-other-keys) - "Generate an executable by using asdf operation TYPE, containing whithin the + "Generate an executable by using asdf operation TYPE, containing within the image all DEPENDENCIES, and running ENTRY-PROGRAM in the case of an executable. Link in any asd files from DEPENDENCY-PREFIXES to ensure references to those libraries are retained." diff --git a/guix/build/maven/pom.scm b/guix/build/maven/pom.scm index aa60af2afa..dd61f659c2 100644 --- a/guix/build/maven/pom.scm +++ b/guix/build/maven/pom.scm @@ -59,7 +59,7 @@ represents a @file{.pom} file content, or parts of it." (pom-ref content "parent")) (define* (find-parent content inputs #:optional local-packages) - "Find the parent pom for the pom file whith @var{content} in a package's + "Find the parent pom for the pom file with @var{content} in a package's @var{inputs}. When the parent pom cannot be found in @var{inputs}, but @var{local-packages} is defined, the parent pom is looked up in it. @@ -243,17 +243,17 @@ to re-declare the namespaces in the top-level element." (define* (fix-pom-dependencies pom-file inputs #:key with-plugins? with-build-dependencies? (excludes '()) (local-packages '())) - "Open @var{pom-file}, and override its content, rewritting its dependencies + "Open @var{pom-file}, and override its content, rewriting its dependencies to set their version to the latest version available in the @var{inputs}. -@var{#:with-plugins?} controls whether plugins are also overiden. +@var{#:with-plugins?} controls whether plugins are also overridden. @var{#:with-build-dependencies?} controls whether build dependencies (whose -scope is not empty) are also overiden. By default build dependencies and -plugins are not overiden. +scope is not empty) are also overridden. By default build dependencies and +plugins are not overridden. @var{#:excludes} is an association list of groupID to a list of artifactIDs. When a pair (groupID, artifactID) is present in the list, its entry is -removed instead of being overiden. If the entry is ignored because of the +removed instead of being overridden. If the entry is ignored because of the previous arguments, the entry is not removed. @var{#:local-packages} is an association list that contains additional version @@ -262,7 +262,7 @@ not found in @var{inputs}, information from this list is used instead to determi the latest version of the package. This is an association list of group IDs to another association list of artifact IDs to a version number. -Returns nothing, but overides the @var{pom-file} as a side-effect." +Returns nothing, but overrides the @var{pom-file} as a side-effect." (define pom (get-pom pom-file)) (define (ls dir) diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 09bd8465c8..62e7a7b305 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -6,6 +6,7 @@ ;;; Copyright © 2016 Hartmut Goebel <h.goebel@crazy-compilers.com> ;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2018 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2020 Jakub Kądziołka <kuba@kadziolka.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -154,9 +155,14 @@ (major+minor (take components 2))) (string-join major+minor "."))) +(define (python-output outputs) + "Return the path of the python output, if there is one, or fall-back to out." + (or (assoc-ref outputs "python") + (assoc-ref outputs "out"))) + (define (site-packages inputs outputs) "Return the path of the current output's Python site-package." - (let* ((out (assoc-ref outputs "out")) + (let* ((out (python-output outputs)) (python (assoc-ref inputs "python"))) (string-append out "/lib/python" (python-version python) @@ -175,7 +181,7 @@ when running checks after installing the package." (define* (install #:key outputs (configure-flags '()) use-setuptools? #:allow-other-keys) "Install a given Python package." - (let* ((out (assoc-ref outputs "out")) + (let* ((out (python-output outputs)) (params (append (list (string-append "--prefix=" out)) (if use-setuptools? ;; distutils does not accept these flags @@ -199,12 +205,8 @@ when running checks after installing the package." (string-append dir "/sbin")))) outputs)) - (let* ((out (assoc-ref outputs "out")) - (python (assoc-ref inputs "python")) - (var `("PYTHONPATH" prefix - ,(cons (string-append out "/lib/python" - (python-version python) - "/site-packages") + (let* ((var `("PYTHONPATH" prefix + ,(cons (site-packages inputs outputs) (search-path-as-string->list (or (getenv "PYTHONPATH") "")))))) (for-each (lambda (dir) @@ -220,11 +222,7 @@ installed with setuptools." ;; Even if the "easy-install.pth" is not longer created, we kept this phase. ;; There still may be packages creating an "easy-install.pth" manually for ;; some good reason. - (let* ((out (assoc-ref outputs "out")) - (python (assoc-ref inputs "python")) - (site-packages (string-append out "/lib/python" - (python-version python) - "/site-packages")) + (let* ((site-packages (site-packages inputs outputs)) (easy-install-pth (string-append site-packages "/easy-install.pth")) (new-pth (string-append site-packages "/" name ".pth"))) (when (file-exists? easy-install-pth) diff --git a/guix/build/rpath.scm b/guix/build/rpath.scm deleted file mode 100644 index 75a1fef5ef..0000000000 --- a/guix/build/rpath.scm +++ /dev/null @@ -1,59 +0,0 @@ -;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013 Ludovic Courtès <ludo@gnu.org> -;;; -;;; This file is part of GNU Guix. -;;; -;;; GNU Guix is free software; you can redistribute it and/or modify it -;;; under the terms of the GNU General Public License as published by -;;; the Free Software Foundation; either version 3 of the License, or (at -;;; your option) any later version. -;;; -;;; GNU Guix is distributed in the hope that it will be useful, but -;;; WITHOUT ANY WARRANTY; without even the implied warranty of -;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -;;; GNU General Public License for more details. -;;; -;;; You should have received a copy of the GNU General Public License -;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. - -(define-module (guix build rpath) - #:use-module (ice-9 popen) - #:use-module (ice-9 rdelim) - #:export (%patchelf - file-rpath - augment-rpath)) - -;;; Commentary: -;;; -;;; Tools to manipulate the RPATH and RUNPATH of ELF binaries. Currently they -;;; rely on PatchELF. -;;; -;;; Code: - -(define %patchelf - ;; The `patchelf' command. - (make-parameter "patchelf")) - -(define %not-colon - (char-set-complement (char-set #\:))) - -(define (file-rpath file) - "Return the RPATH (or RUNPATH) of FILE as a list of directory names, or #f -on failure." - (let* ((p (open-pipe* OPEN_READ (%patchelf) "--print-rpath" file)) - (l (read-line p))) - (and (zero? (close-pipe p)) - (string-tokenize l %not-colon)))) - -(define (augment-rpath file dir) - "Add DIR to the front of the RPATH and RUNPATH of FILE. Return the new -RPATH as a list, or #f on failure." - (let* ((rpath (or (file-rpath file) '())) - (rpath* (cons dir rpath))) - (format #t "~a: changing RPATH from ~s to ~s~%" - file rpath rpath*) - (and (zero? (system* (%patchelf) "--set-rpath" - (string-join rpath* ":") file)) - rpath*))) - -;;; rpath.scm ends here diff --git a/guix/build/ruby-build-system.scm b/guix/build/ruby-build-system.scm index c957a61115..9aceb187a4 100644 --- a/guix/build/ruby-build-system.scm +++ b/guix/build/ruby-build-system.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2015 David Thompson <davet@gnu.org> ;;; Copyright © 2015 Pjotr Prins <pjotr.public01@thebird.nl> ;;; Copyright © 2015, 2016 Ben Woodcroft <donttrustben@gmail.com> +;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -73,13 +74,19 @@ directory." (define* (replace-git-ls-files #:key source #:allow-other-keys) "Many gemspec files downloaded from outside rubygems.org use `git ls-files` -to list of the files to be included in the built gem. However, since this +to list the files to be included in the built gem. However, since this operation is not deterministic, we replace it with `find`." - (when (not (gem-archive? source)) + (unless (gem-archive? source) (let ((gemspec (first-gemspec))) + ;; Do not include the freshly built .gem itself as it causes problems. + ;; Strip the first 2 characters ("./") to more exactly match the output + ;; given by 'git ls-files'. This is useful to prevent breaking regexps + ;; that could be used to filter the list of files. (substitute* gemspec - (("`git ls-files`") "`find . -type f |sort`") - (("`git ls-files -z`") "`find . -type f -print0 |sort -z`")))) + (("`git ls-files`") + "`find . -type f -not -regex '.*\\.gem$' | sort | cut -c3-`") + (("`git ls-files -z`") + "`find . -type f -not -regex '.*\\.gem$' -print0 | sort -z | cut -zc3-`")))) #t) (define* (extract-gemspec #:key source #:allow-other-keys) @@ -129,11 +136,7 @@ is #f." #:allow-other-keys) "Install the gem archive SOURCE to the output store item. Additional GEM-FLAGS are passed to the 'gem' invocation, if present." - (let* ((ruby-version - (match:substring (string-match "ruby-(.*)\\.[0-9]$" - (assoc-ref inputs "ruby")) - 1)) - (out (assoc-ref outputs "out")) + (let* ((out (assoc-ref outputs "out")) (vendor-dir (string-append out "/lib/ruby/vendor_ruby")) (gem-file (first-matching-file "\\.gem$")) (gem-file-basename (basename gem-file)) @@ -144,8 +147,8 @@ GEM-FLAGS are passed to the 'gem' invocation, if present." (setenv "GEM_VENDOR" vendor-dir) (or (zero? - ;; 'zero? system*' allows the custom error handling to function as - ;; expected, while 'invoke' raises its own exception. + ;; 'zero? system*' allows the custom error handling to function as + ;; expected, while 'invoke' raises its own exception. (apply system* "gem" "install" gem-file "--verbose" "--local" "--ignore-dependencies" "--vendor" diff --git a/guix/build/utils.scm b/guix/build/utils.scm index 419c10195b..fe2d82c99e 100644 --- a/guix/build/utils.scm +++ b/guix/build/utils.scm @@ -1,10 +1,12 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2013 Andreas Enge <andreas@enge.fr> ;;; Copyright © 2013 Nikita Karetnikov <nikita@karetnikov.org> ;;; Copyright © 2015, 2018 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2018 Arun Isaac <arunisaac@systemreboot.net> ;;; Copyright © 2018, 2019 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il> +;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -52,6 +54,7 @@ directory-exists? executable-file? symbolic-link? + call-with-temporary-output-file call-with-ascii-input-file elf-file? ar-file? @@ -110,7 +113,9 @@ make-desktop-entry-file - locale-category->string)) + locale-category->string + + %xz-parallel-args)) ;;; @@ -197,6 +202,22 @@ introduce the version part." "Return #t if FILE is a symbolic link (aka. \"symlink\".)" (eq? (stat:type (lstat file)) 'symlink)) +(define (call-with-temporary-output-file proc) + "Call PROC with a name of a temporary file and open output port to that +file; close the file and delete it when leaving the dynamic extent of this +call." + (let* ((directory (or (getenv "TMPDIR") "/tmp")) + (template (string-append directory "/guix-file.XXXXXX")) + (out (mkstemp! template))) + (dynamic-wind + (lambda () + #t) + (lambda () + (proc template out)) + (lambda () + (false-if-exception (close out)) + (false-if-exception (delete-file template)))))) + (define (call-with-ascii-input-file file proc) "Open FILE as an ASCII or binary file, and pass the resulting port to PROC. FILE is closed when PROC's dynamic extent is left. Return the @@ -365,6 +386,16 @@ verbose output to the LOG port." stat lstat))) +(define-syntax-rule (warn-on-error expr file) + (catch 'system-error + (lambda () + expr) + (lambda args + (format (current-error-port) + "warning: failed to delete ~a: ~a~%" + file (strerror + (system-error-errno args)))))) + (define* (delete-file-recursively dir #:key follow-mounts?) "Delete DIR recursively, like `rm -rf', without following symlinks. Don't @@ -375,10 +406,10 @@ errors." (or follow-mounts? (= dev (stat:dev stat)))) (lambda (file stat result) ; leaf - (delete-file file)) + (warn-on-error (delete-file file) file)) (const #t) ; down (lambda (dir stat result) ; up - (rmdir dir)) + (warn-on-error (rmdir dir) dir)) (const #t) ; skip (lambda (file stat errno result) (format (current-error-port) @@ -800,7 +831,7 @@ sub-expression. For example: ((\"hello\") \"good morning\\n\") ((\"foo([a-z]+)bar(.*)$\" all letters end) - (string-append \"baz\" letter end))) + (string-append \"baz\" letters end))) Here, anytime a line of FILE contains \"hello\", it is replaced by \"good morning\". Anytime a line of FILE matches the second regexp, ALL is bound to @@ -853,29 +884,38 @@ match the terminating newline of a line." ;;; (define* (dump-port in out + #:optional len #:key (buffer-size 16384) (progress (lambda (t k) (k)))) - "Read as much data as possible from IN and write it to OUT, using chunks of -BUFFER-SIZE bytes. Call PROGRESS at the beginning and after each successful -transfer of BUFFER-SIZE bytes or less, passing it the total number of bytes -transferred and the continuation of the transfer as a thunk." + "Read LEN bytes from IN or as much data as possible if LEN is #f, and write +it to OUT, using chunks of BUFFER-SIZE bytes. Call PROGRESS at the beginning +and after each successful transfer of BUFFER-SIZE bytes or less, passing it +the total number of bytes transferred and the continuation of the transfer as +a thunk." (define buffer (make-bytevector buffer-size)) (define (loop total bytes) (or (eof-object? bytes) + (and len (= total len)) (let ((total (+ total bytes))) (put-bytevector out buffer 0 bytes) (progress total (lambda () (loop total - (get-bytevector-n! in buffer 0 buffer-size))))))) + (get-bytevector-n! in buffer 0 + (if len + (min (- len total) buffer-size) + buffer-size)))))))) ;; Make sure PROGRESS is called when we start so that it can measure ;; throughput. (progress 0 (lambda () - (loop 0 (get-bytevector-n! in buffer 0 buffer-size))))) + (loop 0 (get-bytevector-n! in buffer 0 + (if len + (min len buffer-size) + buffer-size)))))) (define (set-file-time file stat) "Set the atime/mtime of FILE to that specified by STAT." @@ -1446,6 +1486,17 @@ returned." LC_NAME LC_NUMERIC LC_PAPER LC_TELEPHONE LC_TIME))) + +;;; +;;; Others. +;;; + +(define (%xz-parallel-args) + "The xz arguments required to enable bit-reproducible, multi-threaded +compression." + (list "--memlimit=50%" + (format #f "--threads=~a" (max 2 (parallel-job-count))))) + ;;; Local Variables: ;;; eval: (put 'call-with-output-file/atomic 'scheme-indent-function 1) ;;; eval: (put 'call-with-ascii-input-file 'scheme-indent-function 1) diff --git a/guix/gexp.scm b/guix/gexp.scm index 9339b226b7..b8c831ccc3 100644 --- a/guix/gexp.scm +++ b/guix/gexp.scm @@ -1535,7 +1535,8 @@ last one is created from the given <scheme-file> object." (guile (%guile-for-build)) (module-path %load-path) (extensions '()) - (deprecation-warnings #f)) + (deprecation-warnings #f) + (optimization-level 1)) "Return a derivation that builds a tree containing the `.go' files corresponding to MODULES. All the MODULES are built in a context where they can refer to each other. When TARGET is true, cross-compile MODULES for @@ -1559,6 +1560,13 @@ TARGET, a GNU triplet." (system base target) (system base compile)) + (define optimizations-for-level + (or (and=> (false-if-exception + (resolve-interface '(system base optimize))) + (lambda (iface) + (module-ref iface 'optimizations-for-level))) ;Guile 3.0 + (const '()))) + (define (regular? file) (not (member file '("." "..")))) @@ -1574,17 +1582,14 @@ TARGET, a GNU triplet." (ungexp (* total 2)) entry) - (ungexp-splicing - (if target - (gexp ((with-target (ungexp target) - (lambda () - (compile-file entry - #:output-file output - #:opts - %auto-compilation-options))))) - (gexp ((compile-file entry - #:output-file output - #:opts %auto-compilation-options))))) + (with-target (ungexp (or target (gexp %host-type))) + (lambda () + (compile-file entry + #:output-file output + #:opts + `(,@%auto-compilation-options + ,@(optimizations-for-level + (ungexp optimization-level)))))) (+ 1 processed)))) diff --git a/guix/packages.scm b/guix/packages.scm index 24d6417065..bdd03a6d91 100644 --- a/guix/packages.scm +++ b/guix/packages.scm @@ -5,6 +5,7 @@ ;;; Copyright © 2016 Alex Kost <alezost@gmail.com> ;;; Copyright © 2017, 2019, 2020 Efraim Flashner <efraim@flashner.co.il> ;;; Copyright © 2019 Marius Bakke <mbakke@fastmail.com> +;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -704,6 +705,8 @@ specifies modules in scope when evaluating SNIPPET." (setenv "PATH" (string-append #+xz "/bin" ":" #+decomp "/bin")) + (setenv "XZ_DEFAULTS" (string-join (%xz-parallel-args))) + ;; SOURCE may be either a directory or a tarball. (if (file-is-directory? #+source) (let* ((store (%store-directory)) @@ -1393,6 +1396,22 @@ TARGET." (bag (package->bag package system target))) (bag-grafts store bag))) +(define-inlinable (derivation=? drv1 drv2) + "Return true if DRV1 and DRV2 are equal." + (or (eq? drv1 drv2) + (string=? (derivation-file-name drv1) + (derivation-file-name drv2)))) + +(define (input=? input1 input2) + "Return true if INPUT1 and INPUT2 are equivalent." + (match input1 + ((label1 drv1 . outputs1) + (match input2 + ((label2 drv2 . outputs2) + (and (string=? label1 label2) + (equal? outputs1 outputs2) + (derivation=? drv1 drv2))))))) + (define* (bag->derivation store bag #:optional context) "Return the derivation to build BAG for SYSTEM. Optionally, CONTEXT can be @@ -1411,9 +1430,12 @@ error reporting." p)) (_ '())) inputs)))) - + ;; It's possible that INPUTS contains packages that are not 'eq?' but + ;; that lead to the same derivation. Delete those duplicates to avoid + ;; issues down the road, such as duplicate entries in '%build-inputs'. (apply (bag-build bag) - store (bag-name bag) input-drvs + store (bag-name bag) + (delete-duplicates input-drvs input=?) #:search-paths paths #:outputs (bag-outputs bag) #:system system (bag-arguments bag))))) @@ -1451,8 +1473,9 @@ This is an internal procedure." (apply (bag-build bag) store (bag-name bag) - #:native-drvs build-drvs - #:target-drvs (append host-drvs target-drvs) + #:native-drvs (delete-duplicates build-drvs input=?) + #:target-drvs (delete-duplicates (append host-drvs target-drvs) + input=?) #:search-paths paths #:native-search-paths npaths #:outputs (bag-outputs bag) diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index 0b66da01f9..ea2a96d5a1 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -5,6 +5,7 @@ ;;; Copyright © 2018 Chris Marusich <cmmarusich@gmail.com> ;;; Copyright © 2018 Efraim Flashner <efraim@flashner.co.il> ;;; Copyright © 2020 Tobias Geerinckx-Rice <me@tobias.gr> +;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -25,6 +26,7 @@ #:use-module (guix scripts) #:use-module (guix ui) #:use-module (guix gexp) + #:use-module ((guix build utils) #:select (%xz-parallel-args)) #:use-module (guix utils) #:use-module (guix store) #:use-module ((guix status) #:select (with-status-verbosity)) @@ -70,29 +72,34 @@ compressor? (name compressor-name) ;string (e.g., "gzip") (extension compressor-extension) ;string (e.g., ".lz") - (command compressor-command)) ;gexp (e.g., #~("/gnu/store/…/gzip" "-9n")) + (command compressor-command)) ;gexp (e.g., #~(list "/gnu/store/…/gzip" + ; "-9n" )) (define %compressors ;; Available compression tools. (list (compressor "gzip" ".gz" - #~(#+(file-append gzip "/bin/gzip") "-9n")) + #~(list #+(file-append gzip "/bin/gzip") "-9n")) (compressor "lzip" ".lz" - #~(#+(file-append lzip "/bin/lzip") "-9")) + #~(list #+(file-append lzip "/bin/lzip") "-9")) (compressor "xz" ".xz" - #~(#+(file-append xz "/bin/xz") "-e")) + #~(append (list #+(file-append xz "/bin/xz") + "-e") + (%xz-parallel-args))) (compressor "bzip2" ".bz2" - #~(#+(file-append bzip2 "/bin/bzip2") "-9")) + #~(list #+(file-append bzip2 "/bin/bzip2") "-9")) (compressor "zstd" ".zst" ;; The default level 3 compresses better than gzip in a ;; fraction of the time, while the highest level 19 ;; (de)compresses more slowly and worse than xz. - #~(#+(file-append zstd "/bin/zstd") "-3")) + #~(list #+(file-append zstd "/bin/zstd") "-3")) (compressor "none" "" #f))) ;; This one is only for use in this module, so don't put it in %compressors. (define bootstrap-xz (compressor "bootstrap-xz" ".xz" - #~(#+(file-append %bootstrap-coreutils&co "/bin/xz") "-e"))) + #~(append (list #+(file-append %bootstrap-coreutils&co "/bin/xz") + "-e") + (%xz-parallel-args)))) (define (lookup-compressor name) "Return the compressor object called NAME. Error out if it could not be @@ -269,7 +276,7 @@ added to the pack." #+@(if (compressor-command compressor) #~("-I" (string-join - '#+(compressor-command compressor))) + #+(compressor-command compressor))) #~()) "--format=gnu" @@ -541,11 +548,13 @@ the image." ,@(source-module-closure `((guix docker) (guix build store-copy) + (guix build utils) ;for %xz-parallel-args (guix profiles) (guix search-paths)) #:select? not-config?)) #~(begin (use-modules (guix docker) (guix build store-copy) + (guix build utils) (guix profiles) (guix search-paths) (srfi srfi-1) (srfi srfi-19) (ice-9 match)) @@ -602,7 +611,7 @@ the image." #~(list (string-append #$profile "/" #$entry-point))) #:extra-files directives - #:compressor '#+(compressor-command compressor) + #:compressor #+(compressor-command compressor) #:creation-time (make-time time-utc 0 1)))))) (gexp->derivation (string-append name ".tar" diff --git a/guix/store/roots.scm b/guix/store/roots.scm index 58653507f8..222f69c5c0 100644 --- a/guix/store/roots.scm +++ b/guix/store/roots.scm @@ -50,7 +50,7 @@ (define (gc-roots) "Return the list of garbage collector roots (\"GC roots\"). This includes -\"regular\" roots fount in %GC-ROOTS-DIRECTORY as well as indirect roots that +\"regular\" roots found in %GC-ROOTS-DIRECTORY as well as indirect roots that are user-controlled symlinks stored anywhere on the file system." (define (regular? file) (match file diff --git a/guix/utils.scm b/guix/utils.scm index b816c355dc..ba896623f4 100644 --- a/guix/utils.scm +++ b/guix/utils.scm @@ -8,6 +8,7 @@ ;;; Copyright © 2017 Mathieu Othacehe <m.othacehe@gmail.com> ;;; Copyright © 2018, 2020 Marius Bakke <marius@gnu.org> ;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il> +;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -35,7 +36,9 @@ #:use-module (rnrs io ports) ;need 'port-position' etc. #:use-module ((rnrs bytevectors) #:select (bytevector-u8-set!)) #:use-module (guix memoization) - #:use-module ((guix build utils) #:select (dump-port mkdir-p delete-file-recursively)) + #:use-module ((guix build utils) + #:select (dump-port mkdir-p delete-file-recursively + call-with-temporary-output-file %xz-parallel-args)) #:use-module ((guix build syscalls) #:select (mkdtemp! fdatasync)) #:use-module (guix diagnostics) ;<location>, &error-location, etc. #:use-module (ice-9 format) @@ -59,7 +62,9 @@ &fix-hint fix-hint? - condition-fix-hint) + condition-fix-hint + + call-with-temporary-output-file) #:export (strip-keyword-arguments default-keyword-arguments substitute-keyword-arguments @@ -94,7 +99,6 @@ tarball-sans-extension compressed-file? switch-symlinks - call-with-temporary-output-file call-with-temporary-directory with-atomic-file-output @@ -217,7 +221,7 @@ a symbol such as 'xz." (match compression ((or #f 'none) (values input '())) ('bzip2 (filtered-port `(,%bzip2 "-dc") input)) - ('xz (filtered-port `(,%xz "-dc") input)) + ('xz (filtered-port `(,%xz "-dc" ,@(%xz-parallel-args)) input)) ('gzip (filtered-port `(,%gzip "-dc") input)) ('lzip (values (lzip-port 'make-lzip-input-port input) '())) @@ -229,7 +233,7 @@ a symbol such as 'xz." (match compression ((or #f 'none) (values input '())) ('bzip2 (filtered-port `(,%bzip2 "-c") input)) - ('xz (filtered-port `(,%xz "-c") input)) + ('xz (filtered-port `(,%xz "-c" ,@(%xz-parallel-args)) input)) ('gzip (filtered-port `(,%gzip "-c") input)) ('lzip (values (lzip-port 'make-lzip-input-port/compressed input) '())) @@ -288,7 +292,8 @@ program--e.g., '(\"--fast\")." (match compression ((or #f 'none) (values output '())) ('bzip2 (filtered-output-port `(,%bzip2 "-c" ,@options) output)) - ('xz (filtered-output-port `(,%xz "-c" ,@options) output)) + ('xz (filtered-output-port `(,%xz "-c" ,@(%xz-parallel-args) + ,@options) output)) ('gzip (filtered-output-port `(,%gzip "-c" ,@options) output)) ('lzip (values (lzip-port 'make-lzip-output-port output) '())) @@ -677,22 +682,6 @@ REPLACEMENT." (substring str start index) pieces)))))))) -(define (call-with-temporary-output-file proc) - "Call PROC with a name of a temporary file and open output port to that -file; close the file and delete it when leaving the dynamic extent of this -call." - (let* ((directory (or (getenv "TMPDIR") "/tmp")) - (template (string-append directory "/guix-file.XXXXXX")) - (out (mkstemp! template))) - (dynamic-wind - (lambda () - #t) - (lambda () - (proc template out)) - (lambda () - (false-if-exception (close out)) - (false-if-exception (delete-file template)))))) - (define (call-with-temporary-directory proc) "Call PROC with a name of a temporary directory; close the directory and delete it when leaving the dynamic extent of this call." |