From ec4c81fe32a90890a6190443248078ce7366503f Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sat, 27 Oct 2018 23:47:59 +0200 Subject: pack: Move store database creation to a separate derivation. * guix/scripts/pack.scm (store-database): New procedure. (self-contained-tarball): Use it when LOCALSTATEDIR? is true. Remove 'schema' and add 'database'. [build]: Pass DATABASE to 'populate-single-profile-directory'. (squashfs-image): Remove #:deduplicate? parameter. [build]: Remove (gnu build install) and (guix config) from the imported modules. Remove 'with-extensions'. * gnu/build/install.scm (populate-single-profile-directory): Remove #:deduplicate?, #:register?, and #:schema; add #:database. Remove call to 'register-closure' and simply copy DATABASE instead. --- gnu/build/install.scm | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'gnu/build') diff --git a/gnu/build/install.scm b/gnu/build/install.scm index 98c547f2e4..9f9a6aba0f 100644 --- a/gnu/build/install.scm +++ b/gnu/build/install.scm @@ -161,14 +161,13 @@ deduplicates files common to CLOSURE and the rest of PREFIX." (define* (populate-single-profile-directory directory #:key profile closure (profile-name "guix-profile") - deduplicate? - register? schema) + database) "Populate DIRECTORY with a store containing PROFILE, whose closure is given in the file called CLOSURE (as generated by #:references-graphs.) DIRECTORY is initialized to contain a single profile under /root pointing to PROFILE. -When REGISTER? is true, initialize DIRECTORY/var/guix/db to reflect the -contents of the store; DEDUPLICATE? determines whether to deduplicate files in -the store. + +When DATABASE is true, copy it to DIRECTORY/var/guix/db and create +DIRECTORY/var/guix/gcroots and friends. PROFILE-NAME is the name of the profile being created under /var/guix/profiles, typically either \"guix-profile\" or \"current-guix\". @@ -189,11 +188,9 @@ This is used to create the self-contained tarballs with 'guix pack'." ;; Populate the store. (populate-store (list closure) directory) - (when register? - (register-closure (canonicalize-path directory) closure - #:deduplicate? deduplicate? - #:schema schema) - + (when database + (install-file database (scope "/var/guix/db/")) + (chmod (scope "/var/guix/db/db.sqlite") #o644) (mkdir-p* "/var/guix/profiles") (mkdir-p* "/var/guix/gcroots") (symlink* "/var/guix/profiles" -- cgit v1.2.3 From b27ef1d46cfdc3c994b106241f99cd7142083d13 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 28 Oct 2018 00:17:08 +0200 Subject: pack: Import (guix store database) only when '--localstatedir' is passed. This is another way to address , which was previously addressed in commit 19c924af4f3726688ca155a905ebf1cb9acdfca2. * gnu/build/install.scm (register-closure): Move to... * gnu/build/vm.scm (register-closure): ... here. New procedure. * guix/scripts/pack.scm (self-contained-tarball)[build]: Remove now unneeded 'with-extensions' form and custom (guix config) module. * tests/guix-pack.sh: Revert the strategy from commit 19c924af4f3726688ca155a905ebf1cb9acdfca2. * tests/pack.scm ("self-contained-tarball"): Likewise. --- gnu/build/install.scm | 18 ----- gnu/build/vm.scm | 19 ++++- guix/scripts/pack.scm | 211 +++++++++++++++++++++++++------------------------- tests/guix-pack.sh | 26 ++----- tests/pack.scm | 64 +++++++-------- 5 files changed, 159 insertions(+), 179 deletions(-) (limited to 'gnu/build') diff --git a/gnu/build/install.scm b/gnu/build/install.scm index 9f9a6aba0f..a31e1945d6 100644 --- a/gnu/build/install.scm +++ b/gnu/build/install.scm @@ -18,7 +18,6 @@ ;;; along with GNU Guix. If not, see . (define-module (gnu build install) - #:use-module (guix store database) #:use-module (guix build utils) #:use-module (guix build store-copy) #:use-module (srfi srfi-26) @@ -141,23 +140,6 @@ includes /etc, /var, /run, /bin/sh, etc., and all the symlinks to SYSTEM." (try)) (apply throw args))))))) -(define* (register-closure prefix closure - #:key - (deduplicate? #t) (reset-timestamps? #t) - (schema (sql-schema))) - "Register CLOSURE in PREFIX, where PREFIX is the directory name of the -target store and CLOSURE is the name of a file containing a reference graph as -produced by #:references-graphs.. As a side effect, if RESET-TIMESTAMPS? is -true, reset timestamps on store files and, if DEDUPLICATE? is true, -deduplicates files common to CLOSURE and the rest of PREFIX." - (let ((items (call-with-input-file closure read-reference-graph))) - (register-items items - #:prefix prefix - #:deduplicate? deduplicate? - #:reset-timestamps? reset-timestamps? - #:registration-time %epoch - #:schema schema))) - (define* (populate-single-profile-directory directory #:key profile closure (profile-name "guix-profile") diff --git a/gnu/build/vm.scm b/gnu/build/vm.scm index 5579886264..746808515f 100644 --- a/gnu/build/vm.scm +++ b/gnu/build/vm.scm @@ -25,7 +25,7 @@ #:use-module (guix build utils) #:use-module (guix build store-copy) #:use-module (guix build syscalls) - #:use-module ((guix store database) #:select (reset-timestamps)) + #:use-module (guix store database) #:use-module (gnu build linux-boot) #:use-module (gnu build install) #:use-module (gnu system uuid) @@ -191,6 +191,23 @@ the #:references-graphs parameter of 'derivation'." (mkdir output) (copy-recursively "xchg" output))))) +(define* (register-closure prefix closure + #:key + (deduplicate? #t) (reset-timestamps? #t) + (schema (sql-schema))) + "Register CLOSURE in PREFIX, where PREFIX is the directory name of the +target store and CLOSURE is the name of a file containing a reference graph as +produced by #:references-graphs.. As a side effect, if RESET-TIMESTAMPS? is +true, reset timestamps on store files and, if DEDUPLICATE? is true, +deduplicates files common to CLOSURE and the rest of PREFIX." + (let ((items (call-with-input-file closure read-reference-graph))) + (register-items items + #:prefix prefix + #:deduplicate? deduplicate? + #:reset-timestamps? reset-timestamps? + #:registration-time %epoch + #:schema schema))) + ;;; ;;; Partitions. diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index faeea68426..3e6430bcce 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -164,113 +164,110 @@ added to the pack." "/db/db.sqlite"))) (define build - (with-imported-modules `(((guix config) => ,(make-config.scm)) - ,@(source-module-closure - `((guix build utils) - (guix build union) - (guix build store-copy) - (gnu build install)) - #:select? not-config?)) - (with-extensions gcrypt-sqlite3&co - #~(begin - (use-modules (guix build utils) - ((guix build union) #:select (relative-file-name)) - (gnu build install) - (srfi srfi-1) - (srfi srfi-26) - (ice-9 match)) - - (define %root "root") - - (define symlink->directives - ;; Return "populate directives" to make the given symlink and its - ;; parent directories. - (match-lambda - ((source '-> target) - (let ((target (string-append #$profile "/" target)) - (parent (dirname source))) - ;; Never add a 'directory' directive for "/" so as to - ;; preserve its ownnership when extracting the archive (see - ;; below), and also because this would lead to adding the - ;; same entries twice in the tarball. - `(,@(if (string=? parent "/") - '() - `((directory ,parent))) - (,source - -> ,(relative-file-name parent target))))))) - - (define directives - ;; Fully-qualified symlinks. - (append-map symlink->directives '#$symlinks)) - - ;; The --sort option was added to GNU tar in version 1.28, released - ;; 2014-07-28. For testing, we use the bootstrap tar, which is - ;; older and doesn't support it. - (define tar-supports-sort? - (zero? (system* (string-append #+archiver "/bin/tar") - "cf" "/dev/null" "--files-from=/dev/null" - "--sort=name"))) - - ;; Add 'tar' to the search path. - (setenv "PATH" #+(file-append archiver "/bin")) - - ;; Note: there is not much to gain here with deduplication and there - ;; is the overhead of the '.links' directory, so turn it off. - ;; Furthermore GNU tar < 1.30 sometimes fails to extract tarballs - ;; with hard links: - ;; . - (populate-single-profile-directory %root - #:profile #$profile - #:closure "profile" - #:database #+database) - - ;; Create SYMLINKS. - (for-each (cut evaluate-populate-directive <> %root) - directives) - - ;; Create the tarball. Use GNU format so there's no file name - ;; length limitation. - (with-directory-excursion %root - (exit - (zero? (apply system* "tar" - #+@(if (compressor-command compressor) - #~("-I" - (string-join - '#+(compressor-command compressor))) - #~()) - "--format=gnu" - - ;; Avoid non-determinism in the archive. Use - ;; mtime = 1, not zero, because that is what the - ;; daemon does for files in the store (see the - ;; 'mtimeStore' constant in local-store.cc.) - (if tar-supports-sort? "--sort=name" "--mtime=@1") - "--mtime=@1" ;for files in /var/guix - "--owner=root:0" - "--group=root:0" - - "--check-links" - "-cvf" #$output - ;; Avoid adding / and /var to the tarball, so - ;; that the ownership and permissions of those - ;; directories will not be overwritten when - ;; extracting the archive. Do not include /root - ;; because the root account might have a - ;; different home directory. - #$@(if localstatedir? - '("./var/guix") - '()) - - (string-append "." (%store-directory)) - - (delete-duplicates - (filter-map (match-lambda - (('directory directory) - (string-append "." directory)) - ((source '-> _) - (string-append "." source)) - (_ #f)) - directives)))))))))) + (with-imported-modules (source-module-closure + `((guix build utils) + (guix build union) + (gnu build install)) + #:select? not-config?) + #~(begin + (use-modules (guix build utils) + ((guix build union) #:select (relative-file-name)) + (gnu build install) + (srfi srfi-1) + (srfi srfi-26) + (ice-9 match)) + + (define %root "root") + + (define symlink->directives + ;; Return "populate directives" to make the given symlink and its + ;; parent directories. + (match-lambda + ((source '-> target) + (let ((target (string-append #$profile "/" target)) + (parent (dirname source))) + ;; Never add a 'directory' directive for "/" so as to + ;; preserve its ownnership when extracting the archive (see + ;; below), and also because this would lead to adding the + ;; same entries twice in the tarball. + `(,@(if (string=? parent "/") + '() + `((directory ,parent))) + (,source + -> ,(relative-file-name parent target))))))) + + (define directives + ;; Fully-qualified symlinks. + (append-map symlink->directives '#$symlinks)) + + ;; The --sort option was added to GNU tar in version 1.28, released + ;; 2014-07-28. For testing, we use the bootstrap tar, which is + ;; older and doesn't support it. + (define tar-supports-sort? + (zero? (system* (string-append #+archiver "/bin/tar") + "cf" "/dev/null" "--files-from=/dev/null" + "--sort=name"))) + + ;; Add 'tar' to the search path. + (setenv "PATH" #+(file-append archiver "/bin")) + + ;; Note: there is not much to gain here with deduplication and there + ;; is the overhead of the '.links' directory, so turn it off. + ;; Furthermore GNU tar < 1.30 sometimes fails to extract tarballs + ;; with hard links: + ;; . + (populate-single-profile-directory %root + #:profile #$profile + #:closure "profile" + #:database #+database) + + ;; Create SYMLINKS. + (for-each (cut evaluate-populate-directive <> %root) + directives) + + ;; Create the tarball. Use GNU format so there's no file name + ;; length limitation. + (with-directory-excursion %root + (exit + (zero? (apply system* "tar" + #+@(if (compressor-command compressor) + #~("-I" + (string-join + '#+(compressor-command compressor))) + #~()) + "--format=gnu" + + ;; Avoid non-determinism in the archive. Use + ;; mtime = 1, not zero, because that is what the + ;; daemon does for files in the store (see the + ;; 'mtimeStore' constant in local-store.cc.) + (if tar-supports-sort? "--sort=name" "--mtime=@1") + "--mtime=@1" ;for files in /var/guix + "--owner=root:0" + "--group=root:0" + + "--check-links" + "-cvf" #$output + ;; Avoid adding / and /var to the tarball, so + ;; that the ownership and permissions of those + ;; directories will not be overwritten when + ;; extracting the archive. Do not include /root + ;; because the root account might have a + ;; different home directory. + #$@(if localstatedir? + '("./var/guix") + '()) + + (string-append "." (%store-directory)) + + (delete-duplicates + (filter-map (match-lambda + (('directory directory) + (string-append "." directory)) + ((source '-> _) + (string-append "." source)) + (_ #f)) + directives))))))))) (gexp->derivation (string-append name ".tar" (compressor-extension compressor)) diff --git a/tests/guix-pack.sh b/tests/guix-pack.sh index cd721a60e9..8c1f556426 100644 --- a/tests/guix-pack.sh +++ b/tests/guix-pack.sh @@ -29,33 +29,21 @@ fi guix pack --version -# Starting from commit 66e9944e078cbb9e0d618377dd6df6e639640efa, 'guix pack' -# produces derivations that refer to guile-sqlite3 and libgcrypt. To make -# that relatively inexpensive, run the test in the user's global store if -# possible, on the grounds that binaries may already be there or can be built -# or downloaded inexpensively. - -NIX_STORE_DIR="`guile -c '(use-modules (guix config))(display %storedir)'`" -localstatedir="`guile -c '(use-modules (guix config))(display %localstatedir)'`" -GUIX_DAEMON_SOCKET="$localstatedir/guix/daemon-socket/socket" -export NIX_STORE_DIR GUIX_DAEMON_SOCKET - -if ! guile -c '(use-modules (guix)) (exit (false-if-exception (open-connection)))' -then - exit 77 -fi +# Use --no-substitutes because we need to verify we can do this ourselves. +GUIX_BUILD_OPTIONS="--no-substitutes" +export GUIX_BUILD_OPTIONS # Build a tarball with no compression. -guix pack --compression=none guile-bootstrap +guix pack --compression=none --bootstrap guile-bootstrap # Build a tarball (with compression). Check that '-e' works as well. -out1="`guix pack guile-bootstrap`" -out2="`guix pack -e '(@ (gnu packages bootstrap) %bootstrap-guile)'`" +out1="`guix pack --bootstrap guile-bootstrap`" +out2="`guix pack --bootstrap -e '(@ (gnu packages bootstrap) %bootstrap-guile)'`" test -n "$out1" test "$out1" = "$out2" # Build a tarball with a symlink. -the_pack="`guix pack -S /opt/gnu/bin=bin guile-bootstrap`" +the_pack="`guix pack --bootstrap -S /opt/gnu/bin=bin guile-bootstrap`" # Try to extract it. Note: we cannot test whether /opt/gnu/bin/guile itself # exists because /opt/gnu/bin may be an absolute symlink to a store item that diff --git a/tests/pack.scm b/tests/pack.scm index 4eb5be92ff..6bd18bdee2 100644 --- a/tests/pack.scm +++ b/tests/pack.scm @@ -29,6 +29,9 @@ #:use-module (gnu packages bootstrap) #:use-module (srfi srfi-64)) +(define %store + (open-connection-for-tests)) + ;; Globally disable grafts because they can trigger early builds. (%graft? #f) @@ -48,40 +51,33 @@ (test-begin "pack") -;; The following test needs guile-sqlite3, libgcrypt, etc. as a consequence of -;; commit c45477d2a1a651485feede20fe0f3d15aec48b39 and related changes. Thus, -;; run it on the user's store, if it's available, on the grounds that these -;; dependencies may be already there, or we can get substitutes or build them -;; quite inexpensively; see . - -(with-external-store store - (unless store (test-skip 1)) - (test-assertm "self-contained-tarball" store - (mlet* %store-monad - ((profile (profile-derivation (packages->manifest - (list %bootstrap-guile)) - #:hooks '() - #:locales? #f)) - (tarball (self-contained-tarball "pack" profile - #:symlinks '(("/bin/Guile" - -> "bin/guile")) - #:compressor %gzip-compressor - #:archiver %tar-bootstrap)) - (check (gexp->derivation - "check-tarball" - #~(let ((bin (string-append "." #$profile "/bin"))) - (setenv "PATH" - (string-append #$%tar-bootstrap "/bin")) - (system* "tar" "xvf" #$tarball) - (mkdir #$output) - (exit - (and (file-exists? (string-append bin "/guile")) - (string=? (string-append #$%bootstrap-guile "/bin") - (readlink bin)) - (string=? (string-append ".." #$profile - "/bin/guile") - (readlink "bin/Guile")))))))) - (built-derivations (list check))))) +(unless (network-reachable?) (test-skip 1)) +(test-assertm "self-contained-tarball" %store + (mlet* %store-monad + ((profile (profile-derivation (packages->manifest + (list %bootstrap-guile)) + #:hooks '() + #:locales? #f)) + (tarball (self-contained-tarball "pack" profile + #:symlinks '(("/bin/Guile" + -> "bin/guile")) + #:compressor %gzip-compressor + #:archiver %tar-bootstrap)) + (check (gexp->derivation + "check-tarball" + #~(let ((bin (string-append "." #$profile "/bin"))) + (setenv "PATH" + (string-append #$%tar-bootstrap "/bin")) + (system* "tar" "xvf" #$tarball) + (mkdir #$output) + (exit + (and (file-exists? (string-append bin "/guile")) + (string=? (string-append #$%bootstrap-guile "/bin") + (readlink bin)) + (string=? (string-append ".." #$profile + "/bin/guile") + (readlink "bin/Guile")))))))) + (built-derivations (list check)))) (test-end) -- cgit v1.2.3 From c5ce2db56909e7dd3fdcd30fa453272d56b07451 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sat, 3 Nov 2018 23:01:44 +0100 Subject: install: Add 'install-database-and-gc-roots'. * gnu/build/install.scm (%root-profile): New variable. (install-database-and-gc-roots): New procedure. (populate-single-profile-directory): Replace inline code with a call to 'install-database-and-gc-roots'. --- gnu/build/install.scm | 48 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 16 deletions(-) (limited to 'gnu/build') diff --git a/gnu/build/install.scm b/gnu/build/install.scm index a31e1945d6..c9ebe124fe 100644 --- a/gnu/build/install.scm +++ b/gnu/build/install.scm @@ -26,6 +26,7 @@ evaluate-populate-directive populate-root-file-system register-closure + install-database-and-gc-roots populate-single-profile-directory)) ;;; Commentary: @@ -140,6 +141,35 @@ includes /etc, /var, /run, /bin/sh, etc., and all the symlinks to SYSTEM." (try)) (apply throw args))))))) +(define %root-profile + "/var/guix/profiles/per-user/root") + +(define* (install-database-and-gc-roots root database profile + #:key (profile-name "guix-profile")) + "Install DATABASE, the store database, under directory ROOT. Create +PROFILE-NAME and have it link to PROFILE, a store item." + (define (scope file) + (string-append root "/" file)) + + (define (mkdir-p* dir) + (mkdir-p (scope dir))) + + (define (symlink* old new) + (symlink old (scope new))) + + (install-file database (scope "/var/guix/db/")) + (chmod (scope "/var/guix/db/db.sqlite") #o644) + (mkdir-p* "/var/guix/profiles") + (mkdir-p* "/var/guix/gcroots") + (symlink* "/var/guix/profiles" "/var/guix/gcroots/profiles") + + ;; Make root's profile, which makes it a GC root. + (mkdir-p* %root-profile) + (symlink* profile + (string-append %root-profile "/" profile-name "-1-link")) + (symlink* (string-append profile-name "-1-link") + (string-append %root-profile "/" profile-name))) + (define* (populate-single-profile-directory directory #:key profile closure (profile-name "guix-profile") @@ -158,9 +188,6 @@ This is used to create the self-contained tarballs with 'guix pack'." (define (scope file) (string-append directory "/" file)) - (define %root-profile - "/var/guix/profiles/per-user/root") - (define (mkdir-p* dir) (mkdir-p (scope dir))) @@ -171,19 +198,8 @@ This is used to create the self-contained tarballs with 'guix pack'." (populate-store (list closure) directory) (when database - (install-file database (scope "/var/guix/db/")) - (chmod (scope "/var/guix/db/db.sqlite") #o644) - (mkdir-p* "/var/guix/profiles") - (mkdir-p* "/var/guix/gcroots") - (symlink* "/var/guix/profiles" - "/var/guix/gcroots/profiles")) - - ;; Make root's profile, which makes it a GC root. - (mkdir-p* %root-profile) - (symlink* profile - (string-append %root-profile "/" profile-name "-1-link")) - (symlink* (string-append profile-name "-1-link") - (string-append %root-profile "/" profile-name)) + (install-database-and-gc-roots directory database profile + #:profile-name profile-name)) (match profile-name ("guix-profile" -- cgit v1.2.3 From 970c9993f124789cb181f399d4981cdcf5d3fc26 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 7 Nov 2018 15:51:38 +0100 Subject: linux-initrd: Make sure 'build-initrd' can delete files. Fixes . Reported by Mark H Weaver . This fixes a regression introduced in 72dc64f8f720268930eed448abfc15d2a0eca3cf, which made files read-only. * gnu/build/linux-initrd.scm (build-initrd): Call 'make-file-writable' on all the files under contents/. --- gnu/build/linux-initrd.scm | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'gnu/build') diff --git a/gnu/build/linux-initrd.scm b/gnu/build/linux-initrd.scm index c65b5aacfa..fb8a1f5b2b 100644 --- a/gnu/build/linux-initrd.scm +++ b/gnu/build/linux-initrd.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013, 2014, 2015 Ludovic Courtès +;;; Copyright © 2013, 2014, 2015, 2018 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -139,6 +139,12 @@ REFERENCES-GRAPHS." (write-cpio-archive output "." #:gzip gzip)) + ;; Make sure directories are writable so we can delete files. + (for-each make-file-writable + (find-files "contents" + (lambda (file stat) + (eq? 'directory (stat:type stat))) + #:directories? #t)) (delete-file-recursively "contents")) ;;; linux-initrd.scm ends here -- cgit v1.2.3