From c423ae89185abab9ca6381a12285b85079367072 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 13 May 2018 18:46:13 +0200 Subject: packages: Add 'package-patched-vulnerabilities'. * guix/packages.scm (patch-file-name): New procedure. (%vulnerability-regexp): New variable. (package-patched-vulnerabilities): New procedure. * guix/scripts/lint.scm (patch-file-name): Remove. (check-vulnerabilities): Adjust to use 'package-patched-vulnerabilities'. * tests/packages.scm ("package-patched-vulnerabilities"): New test. --- tests/packages.scm | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'tests') diff --git a/tests/packages.scm b/tests/packages.scm index f1e7d3119b..65ccb14889 100644 --- a/tests/packages.scm +++ b/tests/packages.scm @@ -959,6 +959,21 @@ (define read-at ((("x" dep)) (eq? dep findutils))))))))) +(test-equal "package-patched-vulnerabilities" + '(("CVE-2015-1234") + ("CVE-2016-1234" "CVE-2018-4567") + ()) + (let ((p1 (dummy-package "pi" + (source (dummy-origin + (patches (list "/a/b/pi-CVE-2015-1234.patch")))))) + (p2 (dummy-package "pi" + (source (dummy-origin + (patches (list + "/a/b/pi-CVE-2016-1234-CVE-2018-4567.patch")))))) + (p3 (dummy-package "pi" (source (dummy-origin))))) + (map package-patched-vulnerabilities + (list p1 p2 p3)))) + (test-eq "fold-packages" hello (fold-packages (lambda (p r) (if (string=? (package-name p) "hello") -- cgit v1.2.3 From 3931c76154d4f418d5ea9acc5e47bf911d371c24 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 4 Jun 2018 15:40:09 +0200 Subject: database: 'with-database' can now initialize new databases. * nix/libstore/schema.sql: Rename to... * guix/store/schema.sql: ... this. * Makefile.am (nobase_dist_guilemodule_DATA): Add it. * nix/local.mk (%D%/libstore/schema.sql.hh): Adjust accordingly. * guix/store/database.scm (sql-schema): New variable. (sqlite-exec, initialize-database, call-with-database): New procedures. (with-database): Rewrite in terms of 'call-with-database'. * tests/store-database.scm ("new database"): New test. * guix/self.scm (compiled-guix)[*core-modules*]: Add 'schema.sql' to #:extra-files. --- Makefile.am | 1 + guix/self.scm | 4 +++- guix/store/database.scm | 50 +++++++++++++++++++++++++++++++++++++++++++----- guix/store/schema.sql | 44 ++++++++++++++++++++++++++++++++++++++++++ nix/libstore/schema.sql | 44 ------------------------------------------ nix/local.mk | 2 +- tests/store-database.scm | 23 ++++++++++++++++++++++ 7 files changed, 117 insertions(+), 51 deletions(-) create mode 100644 guix/store/schema.sql delete mode 100644 nix/libstore/schema.sql (limited to 'tests') diff --git a/Makefile.am b/Makefile.am index 7898a3648a..0267e8fe50 100644 --- a/Makefile.am +++ b/Makefile.am @@ -300,6 +300,7 @@ EXAMPLES = \ GOBJECTS = $(MODULES:%.scm=%.go) guix/config.go $(dist_noinst_DATA:%.scm=%.go) nobase_dist_guilemodule_DATA = \ + guix/store/schema.sql \ $(MODULES) $(MODULES_NOT_COMPILED) $(AUX_FILES) $(EXAMPLES) \ $(MISC_DISTRO_FILES) nobase_nodist_guilemodule_DATA = guix/config.scm diff --git a/guix/self.scm b/guix/self.scm index e71e086cdc..ed3f31cdbc 100644 --- a/guix/self.scm +++ b/guix/self.scm @@ -482,7 +482,9 @@ (define *core-modules* ;; but we don't need to compile it; not compiling it allows ;; us to avoid an extra dependency on guile-gdbm-ffi. #:extra-files - `(("guix/man-db.scm" ,(local-file "../guix/man-db.scm"))) + `(("guix/man-db.scm" ,(local-file "../guix/man-db.scm")) + ("guix/store/schema.sql" + ,(local-file "../guix/store/schema.sql"))) #:guile-for-build guile-for-build)) diff --git a/guix/store/database.scm b/guix/store/database.scm index 3623c0e7a0..e81ab3dc99 100644 --- a/guix/store/database.scm +++ b/guix/store/database.scm @@ -24,25 +24,65 @@ (define-module (guix store database) #:use-module (guix store deduplication) #:use-module (guix base16) #:use-module (guix build syscalls) + #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-19) + #:use-module (rnrs io ports) #:use-module (ice-9 match) - #:export (sqlite-register + #:use-module (system foreign) + #:export (sql-schema + with-database + sqlite-register register-path reset-timestamps)) ;;; Code for working with the store database directly. +(define sql-schema + ;; Name of the file containing the SQL scheme or #f. + (make-parameter #f)) -(define-syntax-rule (with-database file db exp ...) - "Open DB from FILE and close it when the dynamic extent of EXP... is left." - (let ((db (sqlite-open file))) +(define sqlite-exec + ;; XXX: This is was missing from guile-sqlite3 until + ;; . + (let ((exec (pointer->procedure + int + (dynamic-func "sqlite3_exec" (@@ (sqlite3) libsqlite3)) + '(* * * * *)))) + (lambda (db text) + (let ((ret (exec ((@@ (sqlite3) db-pointer) db) + (string->pointer text) + %null-pointer %null-pointer %null-pointer))) + (unless (zero? ret) + ((@@ (sqlite3) sqlite-error) db "sqlite-exec" ret)))))) + +(define (initialize-database db) + "Initializing DB, an empty database, by creating all the tables and indexes +as specified by SQL-SCHEMA." + (define schema + (or (sql-schema) + (search-path %load-path "guix/store/schema.sql"))) + + (sqlite-exec db (call-with-input-file schema get-string-all))) + +(define (call-with-database file proc) + "Pass PROC a database record corresponding to FILE. If FILE doesn't exist, +create it and initialize it as a new database." + (let ((new? (not (file-exists? file))) + (db (sqlite-open file))) (dynamic-wind noop (lambda () - exp ...) + (when new? + (initialize-database db)) + (proc db)) (lambda () (sqlite-close db))))) +(define-syntax-rule (with-database file db exp ...) + "Open DB from FILE and close it when the dynamic extent of EXP... is left. +If FILE doesn't exist, create it and initialize it as a new database." + (call-with-database file (lambda (db) exp ...))) + (define (last-insert-row-id db) ;; XXX: (sqlite3) currently lacks bindings for 'sqlite3_last_insert_rowid'. ;; Work around that. diff --git a/guix/store/schema.sql b/guix/store/schema.sql new file mode 100644 index 0000000000..c1b4a689af --- /dev/null +++ b/guix/store/schema.sql @@ -0,0 +1,44 @@ +create table if not exists ValidPaths ( + id integer primary key autoincrement not null, + path text unique not null, + hash text not null, + registrationTime integer not null, + deriver text, + narSize integer +); + +create table if not exists Refs ( + referrer integer not null, + reference integer not null, + primary key (referrer, reference), + foreign key (referrer) references ValidPaths(id) on delete cascade, + foreign key (reference) references ValidPaths(id) on delete restrict +); + +create index if not exists IndexReferrer on Refs(referrer); +create index if not exists IndexReference on Refs(reference); + +-- Paths can refer to themselves, causing a tuple (N, N) in the Refs +-- table. This causes a deletion of the corresponding row in +-- ValidPaths to cause a foreign key constraint violation (due to `on +-- delete restrict' on the `reference' column). Therefore, explicitly +-- get rid of self-references. +create trigger if not exists DeleteSelfRefs before delete on ValidPaths + begin + delete from Refs where referrer = old.id and reference = old.id; + end; + +create table if not exists DerivationOutputs ( + drv integer not null, + id text not null, -- symbolic output id, usually "out" + path text not null, + primary key (drv, id), + foreign key (drv) references ValidPaths(id) on delete cascade +); + +create index if not exists IndexDerivationOutputs on DerivationOutputs(path); + +create table if not exists FailedPaths ( + path text primary key not null, + time integer not null +); diff --git a/nix/libstore/schema.sql b/nix/libstore/schema.sql deleted file mode 100644 index c1b4a689af..0000000000 --- a/nix/libstore/schema.sql +++ /dev/null @@ -1,44 +0,0 @@ -create table if not exists ValidPaths ( - id integer primary key autoincrement not null, - path text unique not null, - hash text not null, - registrationTime integer not null, - deriver text, - narSize integer -); - -create table if not exists Refs ( - referrer integer not null, - reference integer not null, - primary key (referrer, reference), - foreign key (referrer) references ValidPaths(id) on delete cascade, - foreign key (reference) references ValidPaths(id) on delete restrict -); - -create index if not exists IndexReferrer on Refs(referrer); -create index if not exists IndexReference on Refs(reference); - --- Paths can refer to themselves, causing a tuple (N, N) in the Refs --- table. This causes a deletion of the corresponding row in --- ValidPaths to cause a foreign key constraint violation (due to `on --- delete restrict' on the `reference' column). Therefore, explicitly --- get rid of self-references. -create trigger if not exists DeleteSelfRefs before delete on ValidPaths - begin - delete from Refs where referrer = old.id and reference = old.id; - end; - -create table if not exists DerivationOutputs ( - drv integer not null, - id text not null, -- symbolic output id, usually "out" - path text not null, - primary key (drv, id), - foreign key (drv) references ValidPaths(id) on delete cascade -); - -create index if not exists IndexDerivationOutputs on DerivationOutputs(path); - -create table if not exists FailedPaths ( - path text primary key not null, - time integer not null -); diff --git a/nix/local.mk b/nix/local.mk index 39717711f8..b4c6ba61a4 100644 --- a/nix/local.mk +++ b/nix/local.mk @@ -163,7 +163,7 @@ noinst_HEADERS = \ $(libformat_headers) $(libutil_headers) $(libstore_headers) \ $(guix_daemon_headers) -%D%/libstore/schema.sql.hh: %D%/libstore/schema.sql +%D%/libstore/schema.sql.hh: guix/store/schema.sql $(AM_V_GEN)$(GUILE) --no-auto-compile -c \ "(use-modules (rnrs io ports)) \ (call-with-output-file \"$@\" \ diff --git a/tests/store-database.scm b/tests/store-database.scm index 1348a75c26..7947368595 100644 --- a/tests/store-database.scm +++ b/tests/store-database.scm @@ -20,6 +20,7 @@ (define-module (test-store-database) #:use-module (guix tests) #:use-module ((guix store) #:hide (register-path)) #:use-module (guix store database) + #:use-module ((guix utils) #:select (call-with-temporary-output-file)) #:use-module (srfi srfi-26) #:use-module (srfi srfi-64)) @@ -51,4 +52,26 @@ (define %store (null? (valid-derivers %store file)) (null? (referrers %store file)))))) +(test-equal "new database" + (list 1 2) + (call-with-temporary-output-file + (lambda (db-file port) + (delete-file db-file) + (sqlite-register #:db-file db-file + #:path "/gnu/foo" + #:references '() + #:deriver "/gnu/foo.drv" + #:hash (string-append "sha256:" (make-string 64 #\e)) + #:nar-size 1234) + (sqlite-register #:db-file db-file + #:path "/gnu/bar" + #:references '("/gnu/foo") + #:deriver "/gnu/bar.drv" + #:hash (string-append "sha256:" (make-string 64 #\a)) + #:nar-size 4321) + (let ((path-id (@@ (guix store database) path-id))) + (with-database db-file db + (list (path-id db "/gnu/foo") + (path-id db "/gnu/bar"))))))) + (test-end "store-database") -- cgit v1.2.3 From f8f9f7cabca3f0ea1f8b8cb4fecfc45889bdfb94 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 4 Jun 2018 18:33:19 +0200 Subject: database: Fail registration when encountering unregistered references. * guix/store/database.scm (add-reference-sql): Remove nested SELECT. (add-references): Expect REFERENCES to be a list of ids. (sqlite-register): Call 'path-id' for each of REFERENCES and pass it to 'add-references'. * tests/store-database.scm ("register-path with unregistered references"): New test. --- guix/store/database.scm | 18 +++++++++++------- tests/store-database.scm | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/guix/store/database.scm b/guix/store/database.scm index e81ab3dc99..d5e34ef044 100644 --- a/guix/store/database.scm +++ b/guix/store/database.scm @@ -27,6 +27,7 @@ (define-module (guix store database) #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-19) + #:use-module (srfi srfi-26) #:use-module (rnrs io ports) #:use-module (ice-9 match) #:use-module (system foreign) @@ -139,13 +140,11 @@ (define* (update-or-insert db #:key path deriver hash nar-size time) (last-insert-row-id db))))) (define add-reference-sql - "INSERT OR IGNORE INTO Refs (referrer, reference) SELECT :referrer, id -FROM ValidPaths WHERE path = :reference") + "INSERT INTO Refs (referrer, reference) VALUES (:referrer, :reference);") (define (add-references db referrer references) "REFERRER is the id of the referring store item, REFERENCES is a list -containing store items being referred to. Note that all of the store items in -REFERENCES must already be registered." +ids of items referred to." (let ((stmt (sqlite-prepare db add-reference-sql #:cache? #t))) (for-each (lambda (reference) (sqlite-reset stmt) @@ -164,15 +163,20 @@ (define* (sqlite-register #:key db-file path (references '()) item PATH refers to (they need to be already registered!), DERIVER is a string path of the derivation that created the store item PATH, HASH is the base16-encoded sha256 hash of the store item denoted by PATH (prefixed with -\"sha256:\") after being converted to nar form, and nar-size is the size in -bytes of the store item denoted by PATH after being converted to nar form." +\"sha256:\") after being converted to nar form, and NAR-SIZE is the size in +bytes of the store item denoted by PATH after being converted to nar form. + +Every store item in REFERENCES must already be registered." (with-database db-file db (let ((id (update-or-insert db #:path path #:deriver deriver #:hash hash #:nar-size nar-size #:time (time-second (current-time time-utc))))) - (add-references db id references)))) + ;; Call 'path-id' on each of REFERENCES. This ensures we get a + ;; "non-NULL constraint" failure if one of REFERENCES is unregistered. + (add-references db id + (map (cut path-id db <>) references))))) ;;; diff --git a/tests/store-database.scm b/tests/store-database.scm index 7947368595..9562055fd1 100644 --- a/tests/store-database.scm +++ b/tests/store-database.scm @@ -74,4 +74,24 @@ (define %store (list (path-id db "/gnu/foo") (path-id db "/gnu/bar"))))))) +(test-assert "register-path with unregistered references" + ;; Make sure we get a "NOT NULL constraint failed: Refs.reference" error + ;; when we try to add references that are not registered yet. Better safe + ;; than sorry. + (call-with-temporary-output-file + (lambda (db-file port) + (delete-file db-file) + (catch 'sqlite-error + (lambda () + (sqlite-register #:db-file db-file + #:path "/gnu/foo" + #:references '("/gnu/bar") + #:deriver "/gnu/foo.drv" + #:hash (string-append "sha256:" (make-string 64 #\e)) + #:nar-size 1234) + #f) + (lambda args + (pk 'welcome-exception! args) + #t))))) + (test-end "store-database") -- cgit v1.2.3 From 6892f0a247a06ac12c8c462692f8b3f93e872911 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 4 Jun 2018 22:06:34 +0200 Subject: store-copy: 'read-reference-graph' returns a list of records. The previous implementation of 'read-reference-graph' was good enough for many use cases, but it discarded the graph structure, which is useful information in some cases. * guix/build/store-copy.scm (): New record type. (read-reference-graph): Rewrite to return a list of . (closure-size, populate-store): Adjust accordingly. * gnu/services/base.scm (references-file): Adjust accordingly. * gnu/system/vm.scm (system-docker-image): Likewise. * guix/scripts/pack.scm (squashfs-image, docker-image): Likewise. * tests/gexp.scm ("gexp->derivation #:references-graphs"): Likewise. --- gnu/services/base.scm | 5 +- gnu/system/vm.scm | 6 ++- guix/build/store-copy.scm | 120 +++++++++++++++++++++++++++++++++++++++------- guix/scripts/pack.scm | 10 ++-- tests/gexp.scm | 17 ++++--- 5 files changed, 128 insertions(+), 30 deletions(-) (limited to 'tests') diff --git a/gnu/services/base.scm b/gnu/services/base.scm index b34bb7132b..68411439db 100644 --- a/gnu/services/base.scm +++ b/gnu/services/base.scm @@ -1592,8 +1592,9 @@ (define* (references-file item #:optional (name "references")) (call-with-output-file #$output (lambda (port) - (write (call-with-input-file "graph" - read-reference-graph) + (write (map store-info-item + (call-with-input-file "graph" + read-reference-graph)) port))))) #:options `(#:local-build? #f #:references-graphs (("graph" ,item)))) diff --git a/gnu/system/vm.scm b/gnu/system/vm.scm index 544c0e294d..4aea53d1cd 100644 --- a/gnu/system/vm.scm +++ b/gnu/system/vm.scm @@ -466,8 +466,10 @@ (define build (build-docker-image (string-append "/xchg/" #$name) ;; The output file. (cons* root-directory - (call-with-input-file (string-append "/xchg/" #$graph) - read-reference-graph)) + (map store-info-item + (call-with-input-file + (string-append "/xchg/" #$graph) + read-reference-graph))) #$os-drv #:compressor '(#+(file-append gzip "/bin/gzip") "-9n") #:creation-time (make-time time-utc 0 1) diff --git a/guix/build/store-copy.scm b/guix/build/store-copy.scm index fe2eb6f69a..bad1c09cba 100644 --- a/guix/build/store-copy.scm +++ b/guix/build/store-copy.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013, 2014, 2017 Ludovic Courtès +;;; Copyright © 2013, 2014, 2017, 2018 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -18,10 +18,21 @@ (define-module (guix build store-copy) #:use-module (guix build utils) + #:use-module (guix sets) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-9) + #:use-module (srfi srfi-26) + #:use-module (ice-9 match) #:use-module (ice-9 rdelim) #:use-module (ice-9 ftw) - #:export (read-reference-graph + #:use-module (ice-9 vlist) + #:export (store-info? + store-info-item + store-info-deriver + store-info-references + + read-reference-graph + closure-size populate-store)) @@ -34,19 +45,94 @@ (define-module (guix build store-copy) ;;; ;;; Code: +;; Information about a store item as produced by #:references-graphs. +(define-record-type + (store-info item deriver references) + store-info? + (item store-info-item) ;string + (deriver store-info-deriver) ;#f | string + (references store-info-references)) ;? + +;; TODO: Factorize with that in (guix store). +(define (topological-sort nodes edges) + "Return NODES in topological order according to EDGES. EDGES must be a +one-argument procedure that takes a node and returns the nodes it is connected +to." + (define (traverse) + ;; Do a simple depth-first traversal of all of PATHS. + (let loop ((nodes nodes) + (visited (setq)) + (result '())) + (match nodes + ((head tail ...) + (if (set-contains? visited head) + (loop tail visited result) + (call-with-values + (lambda () + (loop (edges head) + (set-insert head visited) + result)) + (lambda (visited result) + (loop tail visited (cons head result)))))) + (() + (values visited result))))) + + (call-with-values traverse + (lambda (_ result) + (reverse result)))) + (define (read-reference-graph port) - "Return a list of store paths from the reference graph at PORT. -The data at PORT is the format produced by #:references-graphs." - (let loop ((line (read-line port)) - (result '())) - (cond ((eof-object? line) - (delete-duplicates result)) - ((string-prefix? "/" line) - (loop (read-line port) - (cons line result))) - (else - (loop (read-line port) - result))))) + "Read the reference graph as produced by #:references-graphs from PORT and +return it as a list of records in topological order--i.e., leaves +come first. IOW, store items in the resulting list can be registered in the +order in which they appear. + +The reference graph format consists of sequences of lines like this: + + FILE + DERIVER + NUMBER-OF-REFERENCES + REF1 + ... + REFN + +It is meant as an internal format." + (let loop ((result '()) + (table vlist-null) + (referrers vlist-null)) + (match (read-line port) + ((? eof-object?) + ;; 'guix-daemon' gives us something that's in "reverse topological + ;; order"--i.e., leaves (items with zero references) come last. Here + ;; we compute the topological order that we want: leaves come first. + (let ((unreferenced? (lambda (item) + (let ((referrers (vhash-fold* cons '() + (store-info-item item) + referrers))) + (or (null? referrers) + (equal? (list item) referrers)))))) + (topological-sort (filter unreferenced? result) + (lambda (item) + (map (lambda (item) + (match (vhash-assoc item table) + ((_ . node) node))) + (store-info-references item)))))) + (item + (let* ((deriver (match (read-line port) + ("" #f) + (line line))) + (count (string->number (read-line port))) + (refs (unfold-right (cut >= <> count) + (lambda (n) + (read-line port)) + 1+ + 0)) + (item (store-info item deriver refs))) + (loop (cons item result) + (vhash-cons (store-info-item item) item table) + (fold (cut vhash-cons <> item <>) + referrers + refs))))))) (define (file-size file) "Return the size of bytes of FILE, entering it if FILE is a directory." @@ -72,7 +158,8 @@ (define (closure-size reference-graphs) "Return an estimate of the size of the closure described by REFERENCE-GRAPHS, a list of reference-graph files." (define (graph-from-file file) - (call-with-input-file file read-reference-graph)) + (map store-info-item + (call-with-input-file file read-reference-graph))) (define items (delete-duplicates (append-map graph-from-file reference-graphs))) @@ -88,7 +175,8 @@ (define store (define (things-to-copy) ;; Return the list of store files to copy to the image. (define (graph-from-file file) - (call-with-input-file file read-reference-graph)) + (map store-info-item + (call-with-input-file file read-reference-graph))) (delete-duplicates (append-map graph-from-file reference-graphs))) diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index 76729d8e10..78bfd01eff 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -251,8 +251,9 @@ (define build ;; ancestor directories and only keeps the basename. We fix this ;; in the following invocations of mksquashfs. (apply invoke "mksquashfs" - `(,@(call-with-input-file "profile" - read-reference-graph) + `(,@(map store-info-item + (call-with-input-file "profile" + read-reference-graph)) ,#$output ;; Do not perform duplicate checking because we @@ -352,8 +353,9 @@ (define build (setenv "PATH" (string-append #$archiver "/bin")) (build-docker-image #$output - (call-with-input-file "profile" - read-reference-graph) + (map store-info-item + (call-with-input-file "profile" + read-reference-graph)) #$profile #:system (or #$target (utsname:machine (uname))) #:symlinks '#$symlinks diff --git a/tests/gexp.scm b/tests/gexp.scm index a560adfc5c..83fe811546 100644 --- a/tests/gexp.scm +++ b/tests/gexp.scm @@ -615,6 +615,7 @@ (define guile ,guile) `(("graph" ,two)) #:modules '((guix build store-copy) + (guix sets) (guix build utils)))) (ok? (built-derivations (list drv))) (out -> (derivation->output-path drv))) @@ -815,21 +816,25 @@ (define (multiply x) (two (gexp->derivation "two" #~(symlink #$one #$output:chbouib))) (build -> (with-imported-modules '((guix build store-copy) + (guix sets) (guix build utils)) #~(begin (use-modules (guix build store-copy)) (with-output-to-file #$output (lambda () - (write (call-with-input-file "guile" - read-reference-graph)))) + (write (map store-info-item + (call-with-input-file "guile" + read-reference-graph))))) (with-output-to-file #$output:one (lambda () - (write (call-with-input-file "one" - read-reference-graph)))) + (write (map store-info-item + (call-with-input-file "one" + read-reference-graph))))) (with-output-to-file #$output:two (lambda () - (write (call-with-input-file "two" - read-reference-graph))))))) + (write (map store-info-item + (call-with-input-file "two" + read-reference-graph)))))))) (drv (gexp->derivation "ref-graphs" build #:references-graphs `(("one" ,one) ("two" ,two "chbouib") -- cgit v1.2.3 From 0d0438ed8cb744bffa8c7e0a8d60165ce604939f Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 6 Jun 2018 16:36:01 +0200 Subject: deduplicate: Fix a couple of thinkos. * guix/store/deduplication.scm (get-temp-link): Turn 'args' in the 'catch' handler into a rest argument. (deduplicate): Use 'lstat' instead of 'file-is-directory?' to properly handle symlinks. When iterating over the result of 'scandir', exclude the ".links" sub-directory. * tests/store-deduplication.scm ("deduplicate"): Create sub-directories and call 'deduplicate' directly on STORE. --- guix/store/deduplication.scm | 13 ++++++++----- tests/store-deduplication.scm | 9 ++++----- 2 files changed, 12 insertions(+), 10 deletions(-) (limited to 'tests') diff --git a/guix/store/deduplication.scm b/guix/store/deduplication.scm index 4b4ac01f64..d3139eb904 100644 --- a/guix/store/deduplication.scm +++ b/guix/store/deduplication.scm @@ -85,7 +85,7 @@ (define* (get-temp-link target #:optional (link-prefix (dirname target))) (lambda () (link target tempname) tempname) - (lambda (args) + (lambda args (if (= (system-error-errno args) EEXIST) (try (tempname-in link-prefix)) (throw 'system-error args)))))) @@ -120,12 +120,15 @@ (define* (deduplicate path hash #:key (store %store-directory)) (link-file (string-append links-directory "/" (bytevector->base16-string hash)))) (mkdir-p links-directory) - (if (file-is-directory? path) + (if (eq? 'directory (stat:type (lstat path))) ;; Can't hardlink directories, so hardlink their atoms. (for-each (lambda (file) - (unless (member file '("." "..")) - (deduplicate file (nar-sha256 file) - #:store store))) + (unless (or (member file '("." "..")) + (and (string=? path store) + (string=? file ".links"))) + (let ((file (string-append path "/" file))) + (deduplicate file (nar-sha256 file) + #:store store)))) (scandir path)) (if (file-exists? link-file) (false-if-system-error (EMLINK) diff --git a/tests/store-deduplication.scm b/tests/store-deduplication.scm index 04817a193a..2361723199 100644 --- a/tests/store-deduplication.scm +++ b/tests/store-deduplication.scm @@ -37,10 +37,12 @@ (define-module (test-store-deduplication) (lambda (store) (let ((data (string->utf8 "Hello, world!")) (identical (map (lambda (n) - (string-append store "/" (number->string n))) + (string-append store "/" (number->string n) + "/a/b/c")) (iota 5))) (unique (string-append store "/unique"))) (for-each (lambda (file) + (mkdir-p (dirname file)) (call-with-output-file file (lambda (port) (put-bytevector port data)))) @@ -49,10 +51,7 @@ (define-module (test-store-deduplication) (lambda (port) (put-bytevector port (string->utf8 "This is unique.")))) - (for-each (lambda (file) - (deduplicate file (sha256 data) #:store store)) - identical) - (deduplicate unique (nar-sha256 unique) #:store store) + (deduplicate store (nar-sha256 store) #:store store) ;; (system (string-append "ls -lRia " store)) (cons* (apply = (map (compose stat:ino stat) identical)) -- cgit v1.2.3 From ef1297e8c74a0358d2538a5dd43d50cde7bf14a3 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 7 Jun 2018 21:55:15 +0200 Subject: database: 'sqlite-register' takes a database, not a file name. * guix/store/database.scm (sqlite-register): Remove #:db-file and add 'db' parameter. Remove #:schema and 'parameterize'. (register-path): Wrap 'sqlite-register' call in 'with-database' and in 'parameterize'. * tests/store-database.scm ("new database") ("register-path with unregistered references"): Adjust accordingly. --- guix/store/database.scm | 57 ++++++++++++++++++++++-------------------------- tests/store-database.scm | 40 ++++++++++++++++----------------- 2 files changed, 46 insertions(+), 51 deletions(-) (limited to 'tests') diff --git a/guix/store/database.scm b/guix/store/database.scm index 67dfb8b0ee..1e5e3bcc71 100644 --- a/guix/store/database.scm +++ b/guix/store/database.scm @@ -157,30 +157,24 @@ (define (add-references db referrer references) (last-insert-row-id db)) references))) -;; XXX figure out caching of statement and database objects... later -(define* (sqlite-register #:key db-file path (references '()) - deriver hash nar-size - (schema (sql-schema))) - "Registers this stuff in a database specified by DB-FILE. PATH is the string -path of some store item, REFERENCES is a list of string paths which the store -item PATH refers to (they need to be already registered!), DERIVER is a string -path of the derivation that created the store item PATH, HASH is the -base16-encoded sha256 hash of the store item denoted by PATH (prefixed with -\"sha256:\") after being converted to nar form, and NAR-SIZE is the size in -bytes of the store item denoted by PATH after being converted to nar form. +(define* (sqlite-register db #:key path (references '()) + deriver hash nar-size) + "Registers this stuff in DB. PATH is the store item to register and +REFERENCES is the list of store items PATH refers to; DERIVER is the '.drv' +that produced PATH, HASH is the base16-encoded Nix sha256 hash of +PATH (prefixed with \"sha256:\"), and NAR-SIZE is the size in bytes PATH after +being converted to nar form. Every store item in REFERENCES must already be registered." - (parameterize ((sql-schema schema)) - (with-database db-file db - (let ((id (update-or-insert db #:path path - #:deriver deriver - #:hash hash - #:nar-size nar-size - #:time (time-second (current-time time-utc))))) - ;; Call 'path-id' on each of REFERENCES. This ensures we get a - ;; "non-NULL constraint" failure if one of REFERENCES is unregistered. - (add-references db id - (map (cut path-id db <>) references)))))) + (let ((id (update-or-insert db #:path path + #:deriver deriver + #:hash hash + #:nar-size nar-size + #:time (time-second (current-time time-utc))))) + ;; Call 'path-id' on each of REFERENCES. This ensures we get a + ;; "non-NULL constraint" failure if one of REFERENCES is unregistered. + (add-references db id + (map (cut path-id db <>) references)))) ;;; @@ -267,15 +261,16 @@ (define* (register-path path (when reset-timestamps? (reset-timestamps real-path)) (mkdir-p db-dir) - (sqlite-register - #:db-file (string-append db-dir "/db.sqlite") - #:schema schema - #:path to-register - #:references references - #:deriver deriver - #:hash (string-append "sha256:" - (bytevector->base16-string hash)) - #:nar-size nar-size) + (parameterize ((sql-schema schema)) + (with-database (string-append db-dir "/db.sqlite") db + (sqlite-register + db + #:path to-register + #:references references + #:deriver deriver + #:hash (string-append "sha256:" + (bytevector->base16-string hash)) + #:nar-size nar-size))) (when deduplicate? (deduplicate real-path hash #:store store-dir))))) diff --git a/tests/store-database.scm b/tests/store-database.scm index 9562055fd1..22c356679b 100644 --- a/tests/store-database.scm +++ b/tests/store-database.scm @@ -57,20 +57,20 @@ (define %store (call-with-temporary-output-file (lambda (db-file port) (delete-file db-file) - (sqlite-register #:db-file db-file - #:path "/gnu/foo" - #:references '() - #:deriver "/gnu/foo.drv" - #:hash (string-append "sha256:" (make-string 64 #\e)) - #:nar-size 1234) - (sqlite-register #:db-file db-file - #:path "/gnu/bar" - #:references '("/gnu/foo") - #:deriver "/gnu/bar.drv" - #:hash (string-append "sha256:" (make-string 64 #\a)) - #:nar-size 4321) - (let ((path-id (@@ (guix store database) path-id))) - (with-database db-file db + (with-database db-file db + (sqlite-register db + #:path "/gnu/foo" + #:references '() + #:deriver "/gnu/foo.drv" + #:hash (string-append "sha256:" (make-string 64 #\e)) + #:nar-size 1234) + (sqlite-register db + #:path "/gnu/bar" + #:references '("/gnu/foo") + #:deriver "/gnu/bar.drv" + #:hash (string-append "sha256:" (make-string 64 #\a)) + #:nar-size 4321) + (let ((path-id (@@ (guix store database) path-id))) (list (path-id db "/gnu/foo") (path-id db "/gnu/bar"))))))) @@ -83,12 +83,12 @@ (define %store (delete-file db-file) (catch 'sqlite-error (lambda () - (sqlite-register #:db-file db-file - #:path "/gnu/foo" - #:references '("/gnu/bar") - #:deriver "/gnu/foo.drv" - #:hash (string-append "sha256:" (make-string 64 #\e)) - #:nar-size 1234) + (with-database db-file db + (sqlite-register db #:path "/gnu/foo" + #:references '("/gnu/bar") + #:deriver "/gnu/foo.drv" + #:hash (string-append "sha256:" (make-string 64 #\e)) + #:nar-size 1234)) #f) (lambda args (pk 'welcome-exception! args) -- cgit v1.2.3 From df2f6400b1fbc282ef4d6dd7124ea1c17adc23c2 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 8 Jun 2018 10:56:19 +0200 Subject: store: Remove 'register-path'. * guix/store.scm (register-path): Remove. * guix/nar.scm: Use (guix store database). * guix/scripts/system.scm: Likewise. * tests/store-database.scm: Remove #:hide (register-path). * tests/store.scm ("register-path"): Remove. --- guix/nar.scm | 3 ++- guix/scripts/system.scm | 1 + guix/store.scm | 29 ----------------------------- tests/store-database.scm | 2 +- tests/store.scm | 22 +--------------------- 5 files changed, 5 insertions(+), 52 deletions(-) (limited to 'tests') diff --git a/guix/nar.scm b/guix/nar.scm index 9b4c608238..3556de1379 100644 --- a/guix/nar.scm +++ b/guix/nar.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2018 Ludovic Courtès ;;; Copyright © 2014 Mark H Weaver ;;; ;;; This file is part of GNU Guix. @@ -23,6 +23,7 @@ (define-module (guix nar) #:use-module ((guix build utils) #:select (delete-file-recursively with-directory-excursion)) #:use-module (guix store) + #:use-module (guix store database) #:use-module (guix ui) ; for '_' #:use-module (guix hash) #:use-module (guix pki) diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index 14be8ff8cf..9112177bfb 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -23,6 +23,7 @@ (define-module (guix scripts system) #:use-module (guix config) #:use-module (guix ui) #:use-module (guix store) + #:autoload (guix store database) (register-path) #:use-module (guix grafts) #:use-module (guix gexp) #:use-module (guix derivations) diff --git a/guix/store.scm b/guix/store.scm index 6742611c6f..773d53e82b 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -122,8 +122,6 @@ (define-module (guix store) current-build-output-port - register-path - %store-monad store-bind store-return @@ -1301,33 +1299,6 @@ (define-operation (clear-failed-paths (store-path-list items)) This makes sense only when the daemon was started with '--cache-failures'." boolean) -(define* (register-path path - #:key (references '()) deriver prefix - state-directory) - "Register PATH as a valid store file, with REFERENCES as its list of -references, and DERIVER as its deriver (.drv that led to it.) If PREFIX is -not #f, it must be the name of the directory containing the new store to -initialize; if STATE-DIRECTORY is not #f, it must be a string containing the -absolute file name to the state directory of the store being initialized. -Return #t on success. - -Use with care as it directly modifies the store! This is primarily meant to -be used internally by the daemon's build hook." - ;; Currently this is implemented by calling out to the fine C++ blob. - (let ((pipe (apply open-pipe* OPEN_WRITE %guix-register-program - `(,@(if prefix - `("--prefix" ,prefix) - '()) - ,@(if state-directory - `("--state-directory" ,state-directory) - '()))))) - (and pipe - (begin - (format pipe "~a~%~a~%~a~%" - path (or deriver "") (length references)) - (for-each (cut format pipe "~a~%" <>) references) - (zero? (close-pipe pipe)))))) - ;;; ;;; Store monad. diff --git a/tests/store-database.scm b/tests/store-database.scm index 22c356679b..fcae66e2de 100644 --- a/tests/store-database.scm +++ b/tests/store-database.scm @@ -18,7 +18,7 @@ (define-module (test-store-database) #:use-module (guix tests) - #:use-module ((guix store) #:hide (register-path)) + #:use-module (guix store) #:use-module (guix store database) #:use-module ((guix utils) #:select (call-with-temporary-output-file)) #:use-module (srfi srfi-26) diff --git a/tests/store.scm b/tests/store.scm index fdf3be33f6..afecec940a 100644 --- a/tests/store.scm +++ b/tests/store.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -777,26 +777,6 @@ (define ref-hash (pk 'corrupt-imported imported) #f))))) -(test-assert "register-path" - (let ((file (string-append (%store-prefix) "/" (make-string 32 #\f) - "-fake"))) - (when (valid-path? %store file) - (delete-paths %store (list file))) - (false-if-exception (delete-file file)) - - (let ((ref (add-text-to-store %store "ref-of-fake" (random-text))) - (drv (string-append file ".drv"))) - (call-with-output-file file - (cut display "This is a fake store item.\n" <>)) - (register-path file - #:references (list ref) - #:deriver drv) - - (and (valid-path? %store file) - (equal? (references %store file) (list ref)) - (null? (valid-derivers %store file)) - (null? (referrers %store file)))))) - (test-assert "verify-store" (let* ((text (random-text)) (file1 (add-text-to-store %store "foo" text)) -- cgit v1.2.3 From ea0a06cee2ba05451f94714a4f913db02efbe92c Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 8 Jun 2018 11:03:31 +0200 Subject: Remove 'guix-register' and its traces. * Makefile.am (SH_TESTS): Remove tests/guix-register.sh. * build-aux/pre-inst-env.in (GUIX_REGISTER): Remove. * gnu/build/install.scm (directives): Remove outdated comment. * gnu/build/vm.scm (root-partition-initializer): Update comment. * gnu/packages/package-management.scm (guix-register): Remove. * guix/config.scm.in (%sbindir, %guix-register-program): Remove. * guix/scripts/system.scm (install): Adjust docstring. * guix/self.scm (make-config.scm): Remove #:guix. Do not generate %sbindir and %guix-register-program. (specification->package): Remove "guix". * nix/guix-register/guix-register.cc: Remove. * nix/libstore/store-api.cc (decodeValidPathInfo): Remove. * nix/libstore/store-api.hh (decodeValidPathInfo): Remove declaration. * nix/local.mk (sbin_PROGRAMS, guix_register_SOURCES) (guix_register_CPPFLAGS, guix_register_LDFLAGS): Remove. * tests/guix-register.sh: Remove. --- .gitignore | 1 - Makefile.am | 7 - build-aux/pre-inst-env.in | 6 +- gnu/build/install.scm | 3 - gnu/build/vm.scm | 4 +- gnu/packages/package-management.scm | 36 ----- guix/config.scm.in | 12 +- guix/scripts/system.scm | 2 +- guix/self.scm | 21 +-- nix/guix-register/guix-register.cc | 254 ------------------------------------ nix/libstore/store-api.cc | 26 ---- nix/libstore/store-api.hh | 4 - nix/local.mk | 16 --- tests/guix-register.sh | 191 --------------------------- 14 files changed, 7 insertions(+), 576 deletions(-) delete mode 100644 nix/guix-register/guix-register.cc delete mode 100644 tests/guix-register.sh (limited to 'tests') diff --git a/.gitignore b/.gitignore index e2568ed5fe..35d50b35af 100644 --- a/.gitignore +++ b/.gitignore @@ -69,7 +69,6 @@ /etc/guix-publish.conf /etc/guix-publish.service /guix-daemon -/guix-register /guix/config.scm /libformat.a /libstore.a diff --git a/Makefile.am b/Makefile.am index f4cdba94a2..61a19b6b9e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -405,13 +405,6 @@ SH_TESTS = \ tests/guix-graph.sh \ tests/guix-lint.sh -if BUILD_DAEMON - -SH_TESTS += tests/guix-register.sh - -endif BUILD_DAEMON - - TESTS = $(SCM_TESTS) $(SH_TESTS) AM_TESTS_ENVIRONMENT = abs_top_srcdir="$(abs_top_srcdir)" GUILE_AUTO_COMPILE=0 diff --git a/build-aux/pre-inst-env.in b/build-aux/pre-inst-env.in index 14315d40d4..286a81591c 100644 --- a/build-aux/pre-inst-env.in +++ b/build-aux/pre-inst-env.in @@ -1,7 +1,7 @@ #!/bin/sh # GNU Guix --- Functional package management for GNU -# Copyright © 2012, 2013, 2014, 2015, 2017 Ludovic Courtès +# Copyright © 2012, 2013, 2014, 2015, 2017, 2018 Ludovic Courtès # Copyright © 2017 Eric Bavier # # This file is part of GNU Guix. @@ -55,10 +55,6 @@ NIX_BUILD_HOOK="$abs_top_builddir/nix/scripts/offload" @BUILD_DAEMON_OFFLOAD_FALSE@# No offloading support. @BUILD_DAEMON_OFFLOAD_FALSE@unset NIX_BUILD_HOOK -# The 'guix-register' program. -GUIX_REGISTER="$abs_top_builddir/guix-register" -export GUIX_REGISTER - # The following variables need only be defined when compiling Guix # modules, but we define them to be on the safe side in case of # auto-compilation. diff --git a/gnu/build/install.scm b/gnu/build/install.scm index 06ecb39952..5a5e703872 100644 --- a/gnu/build/install.scm +++ b/gnu/build/install.scm @@ -110,9 +110,6 @@ (define (directives store) ("/var/guix/gcroots/booted-system" -> "/run/booted-system") ("/var/guix/gcroots/current-system" -> "/run/current-system") - - ;; XXX: 'guix-register' creates this symlink with a wrong target, so - ;; create it upfront to be sure. ("/var/guix/gcroots/profiles" -> "/var/guix/profiles") (directory "/bin") diff --git a/gnu/build/vm.scm b/gnu/build/vm.scm index 803cd5996a..73d0191de7 100644 --- a/gnu/build/vm.scm +++ b/gnu/build/vm.scm @@ -346,7 +346,7 @@ (define target-store ;; Optionally, register the inputs in the image's store. (when register-closures? (unless copy-closures? - ;; XXX: 'guix-register' wants to palpate the things it registers, so + ;; XXX: 'register-closure' wants to palpate the things it registers, so ;; bind-mount the store on the target. (mkdir-p target-store) (mount (%store-directory) target-store "" MS_BIND)) @@ -365,7 +365,7 @@ (define target-store (display "populating...\n") (populate-root-file-system system-directory target) - ;; 'guix-register' resets timestamps and everything, so no need to do it + ;; 'register-closure' resets timestamps and everything, so no need to do it ;; once more in that case. (unless register-closures? (reset-timestamps target)))) diff --git a/gnu/packages/package-management.scm b/gnu/packages/package-management.scm index 786d2a53e9..24cf3ad015 100644 --- a/gnu/packages/package-management.scm +++ b/gnu/packages/package-management.scm @@ -294,42 +294,6 @@ (define (intern tarball) ;; Alias for backward compatibility. (define-public guix-devel guix) -(define-public guix-register - ;; This package is for internal consumption: it allows us to quickly build - ;; the 'guix-register' program, which is referred to by (guix config). - ;; TODO: Remove this hack when 'guix-register' has been superseded by Scheme - ;; code. - (package - (inherit guix) - (properties `((hidden? . #t))) - (name "guix-register") - - ;; Use a minimum set of dependencies. - (native-inputs - (fold alist-delete (package-native-inputs guix) - '("po4a" "graphviz" "help2man"))) - (propagated-inputs - `(("gnutls" ,gnutls) - ("guile-git" ,guile-git))) - - (arguments - (substitute-keyword-arguments (package-arguments guix) - ((#:tests? #f #f) - #f) - ((#:phases phases '%standard-phases) - `(modify-phases ,phases - (replace 'build - (lambda _ - (invoke "make" "nix/libstore/schema.sql.hh") - (invoke "make" "-j" (number->string - (parallel-job-count)) - "guix-register"))) - (delete 'copy-bootstrap-guile) - (replace 'install - (lambda _ - (invoke "make" "install-sbinPROGRAMS"))) - (delete 'wrap-program))))))) - (define-public guile2.0-guix (package (inherit guix) diff --git a/guix/config.scm.in b/guix/config.scm.in index aeea81bd3f..4490112e07 100644 --- a/guix/config.scm.in +++ b/guix/config.scm.in @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2018 Ludovic Courtès ;;; Copyright © 2017 Caleb Ristvedt ;;; ;;; This file is part of GNU Guix. @@ -26,13 +26,11 @@ (define-module (guix config) %storedir %localstatedir %sysconfdir - %sbindir %store-directory %state-directory %store-database-directory %config-directory - %guix-register-program %system %libgcrypt @@ -70,9 +68,6 @@ (define %localstatedir (define %sysconfdir "@guix_sysconfdir@") -(define %sbindir - "@guix_sbindir@") - (define %store-directory (or (and=> (getenv "NIX_STORE_DIR") canonicalize-path) %storedir)) @@ -91,11 +86,6 @@ (define %config-directory (or (getenv "GUIX_CONFIGURATION_DIRECTORY") (string-append %sysconfdir "/guix"))) -(define %guix-register-program - ;; The 'guix-register' program. - (or (getenv "GUIX_REGISTER") - (string-append %sbindir "/guix-register"))) - (define %system "@guix_system@") diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index 9112177bfb..727f1ac55f 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -198,7 +198,7 @@ (define* (install os-drv target bootcfg bootcfg-file) "Copy the closure of BOOTCFG, which includes the output of OS-DRV, to directory TARGET. TARGET must be an absolute directory name since that's what -'guix-register' expects. +'register-path' expects. When INSTALL-BOOTLOADER? is true, install bootloader using BOOTCFG." (define (maybe-copy to-copy) diff --git a/guix/self.scm b/guix/self.scm index ed3f31cdbc..3023ae379b 100644 --- a/guix/self.scm +++ b/guix/self.scm @@ -89,8 +89,6 @@ (define specification->package ("gzip" (ref '(gnu packages compression) 'gzip)) ("bzip2" (ref '(gnu packages compression) 'bzip2)) ("xz" (ref '(gnu packages compression) 'xz)) - ("guix" (ref '(gnu packages package-management) - 'guix-register)) ("guile2.0-json" (ref '(gnu packages guile) 'guile2.0-json)) ("guile2.0-ssh" (ref '(gnu packages ssh) 'guile2.0-ssh)) ("guile2.0-git" (ref '(gnu packages guile) 'guile2.0-git)) @@ -565,7 +563,6 @@ (define *config* #:gzip gzip #:bzip2 bzip2 #:xz xz - #:guix guix #:package-name %guix-package-name #:package-version @@ -630,8 +627,7 @@ (define built-modules (define %dependency-variables ;; (guix config) variables corresponding to dependencies. - '(%libgcrypt %libz %xz %gzip %bzip2 %nix-instantiate - %sbindir %guix-register-program)) + '(%libgcrypt %libz %xz %gzip %bzip2 %nix-instantiate)) (define %persona-variables ;; (guix config) variables that define Guix's persona. @@ -653,7 +649,7 @@ (define %config-variables (stringstring (car name+value1)) (symbol->string (car name+value2)))))) -(define* (make-config.scm #:key libgcrypt zlib gzip xz bzip2 guix +(define* (make-config.scm #:key libgcrypt zlib gzip xz bzip2 (package-name "GNU Guix") (package-version "0") (bug-report-address "bug-guix@gnu.org") @@ -669,8 +665,6 @@ (define defmod 'define-module) %guix-version %guix-bug-report-address %guix-home-page-url - %sbindir - %guix-register-program %libgcrypt %libz %gzip @@ -688,17 +682,6 @@ (define %guix-version #$package-version) (define %guix-bug-report-address #$bug-report-address) (define %guix-home-page-url #$home-page-url) - (define %sbindir - ;; This is used to define '%guix-register-program'. - ;; TODO: Use a derivation that builds nothing but the - ;; C++ part. - #+(and guix (file-append guix "/sbin"))) - - (define %guix-register-program - (or (getenv "GUIX_REGISTER") - (and %sbindir - (string-append %sbindir "/guix-register")))) - (define %gzip #+(and gzip (file-append gzip "/bin/gzip"))) (define %bzip2 diff --git a/nix/guix-register/guix-register.cc b/nix/guix-register/guix-register.cc deleted file mode 100644 index 16dae62b3d..0000000000 --- a/nix/guix-register/guix-register.cc +++ /dev/null @@ -1,254 +0,0 @@ -/* GNU Guix --- Functional package management for GNU - Copyright (C) 2013, 2014, 2015 Ludovic Courtès - Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, - 2013 Eelco Dolstra - - This file is part of GNU Guix. - - GNU Guix is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, or (at - your option) any later version. - - GNU Guix is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNU Guix. If not, see . */ - -/* This file derives from the implementation of 'nix-store - --register-validity', by Eelco Dolstra, as found in the Nix package - manager's src/nix-store/nix-store.cc. */ - -#include - -#include -#include - -#include -#include -#include -#include - -#include -#include - -using namespace nix; - -/* Input stream where we read closure descriptions. */ -static std::istream *input = &std::cin; - - - -/* Command-line options. */ - -const char *argp_program_version = - "guix-register (" PACKAGE_NAME ") " PACKAGE_VERSION; -const char *argp_program_bug_address = PACKAGE_BUGREPORT; - -static char doc[] = -"guix-register -- register a closure as valid in a store\ -\v\ -This program is used internally when populating a store with data \ -from an existing store. It updates the new store's database with \ -information about which store files are valid, and what their \ -references are."; - -#define GUIX_OPT_STATE_DIRECTORY 1 -#define GUIX_OPT_DEDUPLICATE 2 - -static const struct argp_option options[] = - { - { "prefix", 'p', "DIRECTORY", 0, - "Open the store that lies under DIRECTORY" }, - { "state-directory", GUIX_OPT_STATE_DIRECTORY, "DIRECTORY", 0, - "Use DIRECTORY as the state directory of the target store" }, - { "no-deduplication", GUIX_OPT_DEDUPLICATE, 0, 0, - "Disable automatic deduplication of registered store items" }, - { 0, 0, 0, 0, 0 } - }; - - -/* Prefix of the store being populated. */ -static std::string prefix; - -/* Whether to deduplicate the registered store items. */ -static bool deduplication = true; - -/* Parse a single option. */ -static error_t -parse_opt (int key, char *arg, struct argp_state *state) -{ - switch (key) - { - case 'p': - { - prefix = canonPath (arg); - settings.nixStore = prefix + NIX_STORE_DIR; - settings.nixDataDir = prefix + NIX_DATA_DIR; - settings.nixLogDir = prefix + NIX_LOG_DIR; - settings.nixStateDir = prefix + NIX_STATE_DIR; - settings.nixDBPath = settings.nixStateDir + "/db"; - break; - } - - case GUIX_OPT_STATE_DIRECTORY: - { - string state_dir = canonPath (arg); - - settings.nixStateDir = state_dir; - settings.nixDBPath = state_dir + "/db"; - break; - } - - case GUIX_OPT_DEDUPLICATE: - deduplication = false; - break; - - case ARGP_KEY_ARG: - { - std::ifstream *file; - - if (state->arg_num >= 2) - /* Too many arguments. */ - argp_usage (state); - - file = new std::ifstream (); - file->open (arg); - - input = file; - } - break; - - default: - return (error_t) ARGP_ERR_UNKNOWN; - } - - return (error_t) 0; -} - -/* Argument parsing. */ -static struct argp argp = { options, parse_opt, 0, doc }; - - -/* Read from INPUT the description of a closure, and register it as valid in - STORE. The expected format on INPUT is that used by #:references-graphs: - - FILE - DERIVER - NUMBER-OF-REFERENCES - REF1 - ... - REFN - - This is really meant as an internal format. */ -static void -register_validity (LocalStore *store, std::istream &input, - bool optimize = true, - bool reregister = true, bool hashGiven = false, - bool canonicalise = true) -{ - ValidPathInfos infos; - - while (1) - { - ValidPathInfo info = decodeValidPathInfo (input, hashGiven); - if (info.path == "") - break; - - if (!prefix.empty ()) - { - /* Rewrite the input to refer to the final name, as if we were in a - chroot under PREFIX. */ - std::string final_prefix (NIX_STORE_DIR "/"); - info.path = final_prefix + baseNameOf (info.path); - } - - /* Keep its real path to canonicalize it and compute its hash. */ - std::string real_path; - real_path = prefix + "/" + settings.nixStore + "/" + baseNameOf (info.path); - - if (!store->isValidPath (info.path) || reregister) - { - /* !!! races */ - if (canonicalise) - canonicalisePathMetaData (real_path, -1); - - if (!hashGiven) - { - HashResult hash = hashPath (htSHA256, real_path); - info.hash = hash.first; - info.narSize = hash.second; - } - infos.push_back (info); - } - } - - store->registerValidPaths (infos); - - /* XXX: When PREFIX is non-empty, store->linksDir points to the original - store's '.links' directory, which means 'optimisePath' would try to link - to that instead of linking to the target store. Thus, disable - deduplication in this case. */ - if (optimize) - { - /* Make sure deduplication is enabled. */ - settings.autoOptimiseStore = true; - - std::string store_dir = settings.nixStore; - - /* 'optimisePath' creates temporary links under 'settings.nixStore' and - this must be the real target store, under PREFIX, to avoid - cross-device links. Thus, temporarily switch the value of - 'settings.nixStore'. */ - settings.nixStore = prefix + store_dir; - for (auto&& i: infos) - store->optimisePath (prefix + i.path); - settings.nixStore = store_dir; - } -} - - -int -main (int argc, char *argv[]) -{ - /* Initialize libgcrypt, which is indirectly used. */ - if (!gcry_check_version (GCRYPT_VERSION)) - { - fprintf (stderr, "error: libgcrypt version mismatch\n"); - exit (EXIT_FAILURE); - } - - /* Tell Libgcrypt that initialization has completed, as per the Libgcrypt - 1.6.0 manual (although this does not appear to be strictly needed.) */ - gcry_control (GCRYCTL_INITIALIZATION_FINISHED, 0); - - /* Honor the environment variables, and initialize the settings. */ - settings.processEnvironment (); - - try - { - argp_parse (&argp, argc, argv, 0, 0, 0); - - /* Instantiate the store. This creates any missing directories among - 'settings.nixStore', 'settings.nixDBPath', etc. */ - LocalStore store; - - if (!prefix.empty ()) - /* Under the --prefix tree, the final name of the store will be - NIX_STORE_DIR. Set it here so that the database uses file names - prefixed by NIX_STORE_DIR and not PREFIX + NIX_STORE_DIR. */ - settings.nixStore = NIX_STORE_DIR; - - register_validity (&store, *input, deduplication); - } - catch (std::exception &e) - { - fprintf (stderr, "error: %s\n", e.what ()); - return EXIT_FAILURE; - } - - return EXIT_SUCCESS; -} diff --git a/nix/libstore/store-api.cc b/nix/libstore/store-api.cc index 6742d2ed49..9e07c67e97 100644 --- a/nix/libstore/store-api.cc +++ b/nix/libstore/store-api.cc @@ -226,32 +226,6 @@ string StoreAPI::makeValidityRegistration(const PathSet & paths, return s; } - -ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) -{ - ValidPathInfo info; - getline(str, info.path); - if (str.eof()) { info.path = ""; return info; } - if (hashGiven) { - string s; - getline(str, s); - info.hash = parseHash(htSHA256, s); - getline(str, s); - if (!string2Int(s, info.narSize)) throw Error("number expected"); - } - getline(str, info.deriver); - string s; int n; - getline(str, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { - getline(str, s); - info.references.insert(s); - } - if (!str || str.eof()) throw Error("missing input"); - return info; -} - - string showPaths(const PathSet & paths) { string s; diff --git a/nix/libstore/store-api.hh b/nix/libstore/store-api.hh index e957cedebc..2d9dcbd573 100644 --- a/nix/libstore/store-api.hh +++ b/nix/libstore/store-api.hh @@ -371,10 +371,6 @@ std::shared_ptr openStore(bool reserveSpace = true); string showPaths(const PathSet & paths); -ValidPathInfo decodeValidPathInfo(std::istream & str, - bool hashGiven = false); - - /* Export multiple paths in the format expected by ‘nix-store --import’. */ void exportPaths(StoreAPI & store, const Paths & paths, diff --git a/nix/local.mk b/nix/local.mk index b4c6ba61a4..140c78df37 100644 --- a/nix/local.mk +++ b/nix/local.mk @@ -120,7 +120,6 @@ libstore_a_CXXFLAGS = $(AM_CXXFLAGS) \ $(SQLITE3_CFLAGS) $(LIBGCRYPT_CFLAGS) bin_PROGRAMS = guix-daemon -sbin_PROGRAMS = guix-register guix_daemon_SOURCES = \ %D%/nix-daemon/nix-daemon.cc \ @@ -138,24 +137,9 @@ guix_daemon_LDADD = \ guix_daemon_headers = \ %D%/nix-daemon/shared.hh - -guix_register_SOURCES = \ - %D%/guix-register/guix-register.cc - -guix_register_CPPFLAGS = \ - $(libutil_a_CPPFLAGS) \ - $(libstore_a_CPPFLAGS) \ - -I$(top_srcdir)/%D%/libstore - -# XXX: Should we start using shared libs? -guix_register_LDADD = \ - libstore.a libutil.a libformat.a -lz \ - $(SQLITE3_LIBS) $(LIBGCRYPT_LIBS) - if HAVE_LIBBZ2 guix_daemon_LDADD += -lbz2 -guix_register_LDADD += -lbz2 endif HAVE_LIBBZ2 diff --git a/tests/guix-register.sh b/tests/guix-register.sh deleted file mode 100644 index 521735b8a4..0000000000 --- a/tests/guix-register.sh +++ /dev/null @@ -1,191 +0,0 @@ -# GNU Guix --- Functional package management for GNU -# Copyright © 2013, 2014, 2015, 2016 Ludovic Courtès -# -# This file is part of GNU Guix. -# -# GNU Guix is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or (at -# your option) any later version. -# -# GNU Guix is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with GNU Guix. If not, see . - -# -# Test the 'guix-register' command-line utility. -# - -guix-register --version - -new_store="t-register-$$" -closure="t-register-closure-$$" -rm -rf "$new_store" - -exit_hook=":" -trap "chmod -R +w $new_store ; rm -rf $new_store $closure ; \$exit_hook" EXIT - -# -# Registering items in the current store---i.e., without '--prefix'. -# - -new_file="$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-guix-register-$$" -echo "Fake store file to test registration." > "$new_file" - -# Register the file with zero references and no deriver. -guix-register < "$new_file2" -guix-register <> "$closure" < Date: Thu, 21 Jun 2018 23:50:47 +0200 Subject: system: Mapped devices needed for boot do not yield Shepherd services. Fixes . Reported by Taylan Kammer . * gnu/system.scm (non-boot-file-system-service)[mapped-devices-for-boot]: New variable. Remove dependencies of FS that are members of MAPPED-DEVICES-FOR-BOOT. (mapped-device-user): Rename to... (mapped-device-users): ... this. Use 'filter' instead of 'find'. (operating-system-user-mapped-devices) (operating-system-boot-mapped-devices): Use 'any file-system-needed-for-boot?' instead of looking at the first user. * tests/system.scm ("non-boot-file-system-service"): New test. --- gnu/system.scm | 34 +++++++++++++++++++--------------- tests/system.scm | 23 +++++++++++++++++++++++ 2 files changed, 42 insertions(+), 15 deletions(-) (limited to 'tests') diff --git a/gnu/system.scm b/gnu/system.scm index 84eab5f84f..e4a57475a9 100644 --- a/gnu/system.scm +++ b/gnu/system.scm @@ -359,6 +359,9 @@ (define file-systems (remove file-system-needed-for-boot? (operating-system-file-systems os))) + (define mapped-devices-for-boot + (operating-system-boot-mapped-devices os)) + (define (device-mappings fs) (let ((device (file-system-device fs))) (if (string? device) ;title is 'device @@ -374,21 +377,23 @@ (define (add-dependencies fs) (file-system (inherit fs) (dependencies - (delete-duplicates (append (device-mappings fs) - (file-system-dependencies fs)) - eq?)))) + (delete-duplicates + (remove (cut member <> mapped-devices-for-boot) + (append (device-mappings fs) + (file-system-dependencies fs))) + eq?)))) (service file-system-service-type (map add-dependencies file-systems))) -(define (mapped-device-user device file-systems) - "Return a file system among FILE-SYSTEMS that uses DEVICE, or #f." +(define (mapped-device-users device file-systems) + "Return the subset of FILE-SYSTEMS that use DEVICE." (let ((target (string-append "/dev/mapper/" (mapped-device-target device)))) - (find (lambda (fs) - (or (member device (file-system-dependencies fs)) - (and (string? (file-system-device fs)) - (string=? (file-system-device fs) target)))) - file-systems))) + (filter (lambda (fs) + (or (member device (file-system-dependencies fs)) + (and (string? (file-system-device fs)) + (string=? (file-system-device fs) target)))) + file-systems))) (define (operating-system-user-mapped-devices os) "Return the subset of mapped devices that can be installed in @@ -396,9 +401,8 @@ (define (operating-system-user-mapped-devices os) (let ((devices (operating-system-mapped-devices os)) (file-systems (operating-system-file-systems os))) (filter (lambda (md) - (let ((user (mapped-device-user md file-systems))) - (or (not user) - (not (file-system-needed-for-boot? user))))) + (let ((users (mapped-device-users md file-systems))) + (not (any file-system-needed-for-boot? users)))) devices))) (define (operating-system-boot-mapped-devices os) @@ -407,8 +411,8 @@ (define (operating-system-boot-mapped-devices os) (let ((devices (operating-system-mapped-devices os)) (file-systems (operating-system-file-systems os))) (filter (lambda (md) - (let ((user (mapped-device-user md file-systems))) - (and user (file-system-needed-for-boot? user)))) + (let ((users (mapped-device-users md file-systems))) + (any file-system-needed-for-boot? users))) devices))) (define (device-mapping-services os) diff --git a/tests/system.scm b/tests/system.scm index 7d55da7174..9416b950e6 100644 --- a/tests/system.scm +++ b/tests/system.scm @@ -19,6 +19,7 @@ (define-module (test-system) #:use-module (gnu) + #:use-module ((gnu services) #:select (service-value)) #:use-module (guix store) #:use-module (srfi srfi-1) #:use-module (srfi srfi-64)) @@ -117,4 +118,26 @@ (define %os-with-mapped-device (type "ext4")) %base-file-systems))))) +(test-equal "non-boot-file-system-service" + '() + + ;; Make sure that mapped devices with at least one needed-for-boot user are + ;; handled exclusively from the initrd. See . + (append-map file-system-dependencies + (service-value + ((@@ (gnu system) non-boot-file-system-service) + (operating-system + (inherit %os-with-mapped-device) + (file-systems + (list (file-system + (mount-point "/foo/bar") + (device "qux:baz") + (type "none") + (dependencies (list %luks-device))) + (file-system + (device (file-system-label "my-root")) + (mount-point "/") + (type "ext4") + (dependencies (list %luks-device)))))))))) + (test-end) -- cgit v1.2.3