From 1bcc87bb685b7985512add221f10e4cb58b5f6f7 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sat, 29 Oct 2016 01:16:24 +0200 Subject: guix download: Add '-o' option. * guix/scripts/download.scm (download-to-file, download-to-store*): New procedures. (%default-options): Add 'download-proc'. (show-help): Adjust description and document '-o'. (%options): Add '-o'. (guix-download): Remove 'store' variable. Add 'fetch' and define 'path' to as its result. * tests/guix-download.sh: Add test. --- guix/scripts/download.scm | 58 +++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 19 deletions(-) (limited to 'guix/scripts') diff --git a/guix/scripts/download.scm b/guix/scripts/download.scm index ec30b05ac0..dffff79729 100644 --- a/guix/scripts/download.scm +++ b/guix/scripts/download.scm @@ -23,12 +23,15 @@ #:use-module (guix hash) #:use-module (guix utils) #:use-module (guix base32) - #:use-module (guix download) - #:use-module ((guix build download) #:select (current-terminal-columns)) - #:use-module ((guix build syscalls) #:select (terminal-columns)) + #:use-module ((guix download) #:hide (url-fetch)) + #:use-module ((guix build download) + #:select (url-fetch current-terminal-columns)) + #:use-module ((guix build syscalls) + #:select (terminal-columns)) #:use-module (web uri) #:use-module (ice-9 match) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-26) #:use-module (srfi srfi-37) #:use-module (rnrs bytevectors) #:use-module (ice-9 binary-ports) @@ -39,15 +42,31 @@ ;;; Command-line options. ;;; +(define (download-to-file url file) + "Download the file at URI to FILE. Return FILE." + (let ((uri (string->uri url))) + (match (uri-scheme uri) + ((or 'file #f) + (copy-file (uri-path uri) file)) + (_ + (url-fetch url file))) + file)) + +(define* (download-to-store* url #:key (verify-certificate? #t)) + (with-store store + (download-to-store store url + #:verify-certificate? verify-certificate?))) + (define %default-options ;; Alist of default option values. `((format . ,bytevector->nix-base32-string) - (verify-certificate? . #t))) + (verify-certificate? . #t) + (download-proc . ,download-to-store*))) (define (show-help) (display (_ "Usage: guix download [OPTION] URL -Download the file at URL, add it to the store, and print its store path -and the hash of its contents. +Download the file at URL to the store or to the given file, and print its +file name and the hash of its contents. Supported formats: 'nix-base32' (default), 'base32', and 'base16' ('hex' and 'hexadecimal' can be used as well).\n")) @@ -56,6 +75,8 @@ Supported formats: 'nix-base32' (default), 'base32', and 'base16' (format #t (_ " --no-check-certificate do not validate the certificate of HTTPS servers ")) + (format #f (_ " + -o, --output=FILE download to FILE")) (newline) (display (_ " -h, --help display this help and exit")) @@ -84,6 +105,12 @@ Supported formats: 'nix-base32' (default), 'base32', and 'base16' (option '("no-check-certificate") #f #f (lambda (opt name arg result) (alist-cons 'verify-certificate? #f result))) + (option '(#\o "output") #t #f + (lambda (opt name arg result) + (alist-cons 'download-proc + (lambda* (url #:key verify-certificate?) + (download-to-file url arg)) + (alist-delete 'download result)))) (option '(#\h "help") #f #f (lambda args @@ -113,24 +140,17 @@ Supported formats: 'nix-base32' (default), 'base32', and 'base16' (with-error-handling (let* ((opts (parse-options)) - (store (open-connection)) (arg (or (assq-ref opts 'argument) (leave (_ "no download URI was specified~%")))) (uri (or (string->uri arg) (leave (_ "~a: failed to parse URI~%") arg))) - (path (case (uri-scheme uri) - ((file) - (add-to-store store (basename (uri-path uri)) - #f "sha256" (uri-path uri))) - (else - (parameterize ((current-terminal-columns - (terminal-columns))) - (download-to-store store (uri->string uri) - (basename (uri-path uri)) - #:verify-certificate? - (assoc-ref opts - 'verify-certificate?)))))) + (fetch (assq-ref opts 'download-proc)) + (path (parameterize ((current-terminal-columns + (terminal-columns))) + (fetch arg + #:verify-certificate? + (assq-ref opts 'verify-certificate?)))) (hash (call-with-input-file (or path (leave (_ "~a: download failed~%") -- cgit v1.2.3 From 94d92c7796a3dd50c27d532315f7d497ac99f08e Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 20 Jul 2015 04:30:16 +0200 Subject: daemon: Add "builtin:download" derivation builder. This ensures that 1) the derivation doesn't change when Guix changes; 2) the derivation closure doesn't contain Guix and its dependencies; 3) we don't have to rely on ugly chroot hacks. Adapted from Nix commit 0a2bee307b20411f5b0dda0c662b1f9bb9e0e131. * nix/libstore/build.cc (DerivationGoal::runChild): Add special case for 'isBuiltin(drv)'. Disable chroot when 'isBuiltin(drv)'. * nix/libstore/builtins.cc, nix/libstore/builtins.hh, nix/scripts/download.in, guix/scripts/perform-download.scm: New files. * guix/ui.scm (show-guix-help)[internal?]: Add 'perform-download'. * nix/local.mk (libstore_a_SOURCES): Add builtins.cc. (libstore_headers): Add builtins.hh. (nodist_pkglibexec_SCRIPTS): Add 'scripts/download'. * config-daemon.ac: Emit 'scripts/download'. * Makefile.am (MODULES): Add 'guix/scripts/perform-download.scm'. * tests/derivations.scm ("unknown built-in builder") ("'download' built-in builder") ("'download' built-in builder, invalid hash") ("'download' built-in builder, not found") ("'download' built-in builder, not fixed-output"): New tests. Co-authored-by: Eelco Dolstra --- .gitignore | 1 + Makefile.am | 1 + config-daemon.ac | 2 + guix/scripts/perform-download.scm | 113 ++++++++++++++++++++++++++++++++++++++ guix/ui.scm | 3 +- nix/libstore/build.cc | 36 +++++++++--- nix/libstore/builtins.cc | 69 +++++++++++++++++++++++ nix/libstore/builtins.hh | 41 ++++++++++++++ nix/local.mk | 5 +- nix/scripts/download.in | 11 ++++ tests/derivations.scm | 70 +++++++++++++++++++++++ 11 files changed, 343 insertions(+), 9 deletions(-) create mode 100644 guix/scripts/perform-download.scm create mode 100644 nix/libstore/builtins.cc create mode 100644 nix/libstore/builtins.hh create mode 100644 nix/scripts/download.in (limited to 'guix/scripts') diff --git a/.gitignore b/.gitignore index 6e892ca687..329d489713 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,4 @@ config.cache stamp-h[0-9] tmp /doc/os-config-lightweight-desktop.texi +/nix/scripts/download diff --git a/Makefile.am b/Makefile.am index 5d3639747f..9d62f48024 100644 --- a/Makefile.am +++ b/Makefile.am @@ -123,6 +123,7 @@ MODULES = \ guix/import/elpa.scm \ guix/scripts.scm \ guix/scripts/download.scm \ + guix/scripts/perform-download.scm \ guix/scripts/build.scm \ guix/scripts/archive.scm \ guix/scripts/import.scm \ diff --git a/config-daemon.ac b/config-daemon.ac index f66f31269d..8a3e6d8b60 100644 --- a/config-daemon.ac +++ b/config-daemon.ac @@ -144,6 +144,8 @@ if test "x$guix_build_daemon" = "xyes"; then AC_CONFIG_FILES([nix/scripts/list-runtime-roots], [chmod +x nix/scripts/list-runtime-roots]) + AC_CONFIG_FILES([nix/scripts/download], + [chmod +x nix/scripts/download]) AC_CONFIG_FILES([nix/scripts/substitute], [chmod +x nix/scripts/substitute]) AC_CONFIG_FILES([nix/scripts/guix-authenticate], diff --git a/guix/scripts/perform-download.scm b/guix/scripts/perform-download.scm new file mode 100644 index 0000000000..0d2e7089aa --- /dev/null +++ b/guix/scripts/perform-download.scm @@ -0,0 +1,113 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2016 Ludovic Courtès +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see . + +(define-module (guix scripts perform-download) + #:use-module (guix ui) + #:use-module (guix derivations) + #:use-module ((guix store) #:select (derivation-path?)) + #:use-module (guix build download) + #:use-module (ice-9 match) + #:export (guix-perform-download)) + +;; This program is a helper for the daemon's 'download' built-in builder. + +(define-syntax derivation-let + (syntax-rules () + ((_ drv ((id name) rest ...) body ...) + (let ((id (assoc-ref (derivation-builder-environment-vars drv) + name))) + (derivation-let drv (rest ...) body ...))) + ((_ drv () body ...) + (begin body ...)))) + +(define %user-module + ;; Module in which content-address mirror procedures are evaluated. + (let ((module (make-fresh-user-module))) + (module-use! module (resolve-interface '(guix base32))) + module)) + +(define (perform-download drv) + "Perform the download described by DRV, a fixed-output derivation." + (derivation-let drv ((url "url") + (output "out") + (executable "executable") + (mirrors "mirrors") + (content-addressed-mirrors "content-addressed-mirrors")) + (unless url + (leave (_ "~a: missing URL~%") (derivation-file-name drv))) + + (let* ((url (call-with-input-string url read)) + (drv-output (assoc-ref (derivation-outputs drv) "out")) + (algo (derivation-output-hash-algo drv-output)) + (hash (derivation-output-hash drv-output))) + (unless (and algo hash) + (leave (_ "~a is not a fixed-output derivation~%") + (derivation-file-name drv))) + + ;; We're invoked by the daemon, which gives us write access to OUTPUT. + (when (url-fetch url output + #:mirrors (if mirrors + (call-with-input-file mirrors read) + '()) + #:content-addressed-mirrors + (if content-addressed-mirrors + (call-with-input-file content-addressed-mirrors + (lambda (port) + (eval (read port) %user-module))) + '()) + #:hashes `((,algo . ,hash)) + + ;; Since DRV's output hash is known, X.509 certificate + ;; validation is pointless. + #:verify-certificate? #f) + (when (and executable (string=? executable "1")) + (chmod output #o755)))))) + +(define (assert-low-privileges) + (when (zero? (getuid)) + (leave (_ "refusing to run with elevated privileges (UID ~a)~%") + (getuid)))) + +(define (guix-perform-download . args) + "Perform the download described by the given fixed-output derivation. + +This is an \"out-of-band\" download in that this code is executed directly by +the daemon and not explicitly described as an input of the derivation. This +allows us to sidestep bootstrapping problems, such downloading the source code +of GnuTLS over HTTPS, before we have built GnuTLS. See +." + (with-error-handling + (match args + (((? derivation-path? drv)) + ;; This program must be invoked by guix-daemon under an unprivileged + ;; UID to prevent things downloading from 'file:///etc/shadow' or + ;; arbitrary code execution via the content-addressed mirror + ;; procedures. (That means we exclude users who did not pass + ;; '--build-users-group'.) + (assert-low-privileges) + (perform-download (call-with-input-file drv read-derivation))) + (("--version") + (show-version-and-exit)) + (x + (leave (_ "fixed-output derivation name expected~%")))))) + +;; Local Variables: +;; eval: (put 'derivation-let 'scheme-indent-function 2) +;; End: + +;; perform-download.scm ends here diff --git a/guix/ui.scm b/guix/ui.scm index 9af8648211..b9fbbfd0e3 100644 --- a/guix/ui.scm +++ b/guix/ui.scm @@ -1184,7 +1184,8 @@ optionally contain a version number and an output name, as in these examples: (define (show-guix-help) (define (internal? command) - (member command '("substitute" "authenticate" "offload"))) + (member command '("substitute" "authenticate" "offload" + "perform-download"))) (format #t (_ "Usage: guix COMMAND ARGS... Run COMMAND with ARGS.\n")) diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc index ae78e65199..889ee3d2bd 100644 --- a/nix/libstore/build.cc +++ b/nix/libstore/build.cc @@ -8,6 +8,7 @@ #include "util.hh" #include "archive.hh" #include "affinity.hh" +#include "builtins.hh" #include #include @@ -2047,7 +2048,12 @@ void DerivationGoal::runChild() commonChildInit(builderOut); #if CHROOT_ENABLED - if (useChroot) { + /* Note: built-in builders are *not* running in a chroot environment + so that we can easily implement them in Guile without having it as + a derivation input (they are running under a separate build user, + though). */ + + if (useChroot && !isBuiltin(drv)) { /* Initialise the loopback interface. */ AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); if (fd == -1) throw SysError("cannot open IP socket"); @@ -2255,6 +2261,28 @@ void DerivationGoal::runChild() throw SysError("setuid failed"); } + restoreSIGPIPE(); + + /* Indicate that we managed to set up the build environment. */ + writeFull(STDERR_FILENO, "\n"); + + /* Execute the program. This should not return. */ + if (isBuiltin(drv)) { + try { + logType = ltFlat; + + auto buildDrv = lookupBuiltinBuilder(drv.builder); + if (buildDrv != NULL) + buildDrv(drv, drvPath); + else + throw Error(format("unsupported builtin function '%1%'") % string(drv.builder, 8)); + _exit(0); + } catch (std::exception & e) { + writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); + _exit(1); + } + } + /* Fill in the arguments. */ Strings args; string builderBasename = baseNameOf(drv.builder); @@ -2262,12 +2290,6 @@ void DerivationGoal::runChild() foreach (Strings::iterator, i, drv.args) args.push_back(rewriteHashes(*i, rewritesToTmp)); - restoreSIGPIPE(); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, "\n"); - - /* Execute the program. This should not return. */ execve(drv.builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); throw SysError(format("executing `%1%'") % drv.builder); diff --git a/nix/libstore/builtins.cc b/nix/libstore/builtins.cc new file mode 100644 index 0000000000..605e44079a --- /dev/null +++ b/nix/libstore/builtins.cc @@ -0,0 +1,69 @@ +/* GNU Guix --- Functional package management for GNU + Copyright (C) 2016 Ludovic Courtès + + This file is part of GNU Guix. + + GNU Guix is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or (at + your option) any later version. + + GNU Guix is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNU Guix. If not, see . */ + +#include +#include +#include + +#include + +namespace nix { + +static void builtinDownload(const Derivation &drv, + const std::string &drvPath) +{ + /* Invoke 'guix perform-download'. */ + Strings args; + args.push_back("perform-download"); + args.push_back(drvPath); + + /* Close all other file descriptors. */ + closeMostFDs(set()); + + const char *const argv[] = { "download", drvPath.c_str(), NULL }; + + /* XXX: Hack our way to use the 'download' script from 'LIBEXECDIR/guix' + or just 'LIBEXECDIR', depending on whether we're running uninstalled or + not. */ + const string subdir = getenv("GUIX_UNINSTALLED") != NULL + ? "" : "/guix"; + + const string program = settings.nixLibexecDir + subdir + "/download"; + execv(program.c_str(), (char *const *) argv); + + throw SysError(format("failed to run download program '%1%'") % program); +} + +static const std::map builtins = +{ + { "download", builtinDownload } +}; + +derivationBuilder lookupBuiltinBuilder(const std::string & name) +{ + if (name.substr(0, 8) == "builtin:") + { + auto realName = name.substr(8); + auto builder = builtins.find(realName); + return builder == builtins.end() ? NULL : builder->second; + } + else + return NULL; +} + +} diff --git a/nix/libstore/builtins.hh b/nix/libstore/builtins.hh new file mode 100644 index 0000000000..0c6db651ab --- /dev/null +++ b/nix/libstore/builtins.hh @@ -0,0 +1,41 @@ +/* GNU Guix --- Functional package management for GNU + Copyright (C) 2016 Ludovic Courtès + + This file is part of GNU Guix. + + GNU Guix is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or (at + your option) any later version. + + GNU Guix is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNU Guix. If not, see . */ + +/* Interface to built-in derivation builders. */ + +#pragma once + +#include +#include +#include + +namespace nix { + + inline bool isBuiltin(const Derivation & drv) + { + return string(drv.builder, 0, 8) == "builtin:"; + } + + /* Build DRV, which lives at DRVPATH. */ + typedef void (*derivationBuilder) (const Derivation &drv, + const std::string &drvPath); + + /* Return the built-in builder called BUILDER, or NULL if none was + found. */ + derivationBuilder lookupBuiltinBuilder(const std::string &builder); +} diff --git a/nix/local.mk b/nix/local.mk index c666edd033..86ef769549 100644 --- a/nix/local.mk +++ b/nix/local.mk @@ -87,6 +87,7 @@ libstore_a_SOURCES = \ %D%/libstore/build.cc \ %D%/libstore/pathlocks.cc \ %D%/libstore/derivations.cc \ + %D%/libstore/builtins.cc \ %D%/libstore/sqlite.cc libstore_headers = \ @@ -98,6 +99,7 @@ libstore_headers = \ %D%/libstore/misc.hh \ %D%/libstore/local-store.hh \ %D%/libstore/sqlite.hh \ + %D%/libstore/builtins.hh \ %D%/libstore/store-api.hh libstore_a_CPPFLAGS = \ @@ -166,7 +168,8 @@ noinst_HEADERS = \ nodist_pkglibexec_SCRIPTS = \ %D%/scripts/list-runtime-roots \ - %D%/scripts/substitute + %D%/scripts/substitute \ + %D%/scripts/download if BUILD_DAEMON_OFFLOAD diff --git a/nix/scripts/download.in b/nix/scripts/download.in new file mode 100644 index 0000000000..4d7088a993 --- /dev/null +++ b/nix/scripts/download.in @@ -0,0 +1,11 @@ +#!@SHELL@ +# A shorthand for "guix perform-download", for use by the daemon. + +if test "x$GUIX_UNINSTALLED" = "x" +then + prefix="@prefix@" + exec_prefix="@exec_prefix@" + exec "@bindir@/guix" perform-download "$@" +else + exec guix perform-download "$@" +fi diff --git a/tests/derivations.scm b/tests/derivations.scm index d8553b223e..449fb47832 100644 --- a/tests/derivations.scm +++ b/tests/derivations.scm @@ -16,6 +16,8 @@ ;;; You should have received a copy of the GNU General Public License ;;; along with GNU Guix. If not, see . +(unsetenv "http_proxy") + (define-module (test-derivations) #:use-module (guix derivations) #:use-module (guix grafts) @@ -24,6 +26,7 @@ #:use-module (guix hash) #:use-module (guix base32) #:use-module (guix tests) + #:use-module (guix tests http) #:use-module ((guix packages) #:select (package-derivation base32)) #:use-module ((guix build utils) #:select (executable-file?)) #:use-module ((gnu packages) #:select (search-bootstrap-binary)) @@ -75,6 +78,9 @@ (lambda (e1 e2) (stringstring (%local-url)))) + #:hash-algo 'sha256 + #:hash (sha256 (string->utf8 text))))) + (and (build-derivations %store (list drv)) + (string=? (call-with-input-file (derivation->output-path drv) + get-string-all) + text)))))) + +(unless (force %http-server-socket) + (test-skip 1)) +(test-assert "'download' built-in builder, invalid hash" + (with-http-server 200 "hello, world!" + (let* ((drv (derivation %store "world" + "builtin:download" '() + #:env-vars `(("url" + . ,(object->string (%local-url)))) + #:hash-algo 'sha256 + #:hash (sha256 (random-bytevector 100))))) ;wrong + (guard (c ((nix-protocol-error? c) + (string-contains (nix-protocol-error-message c) "failed"))) + (build-derivations %store (list drv)) + #f)))) + +(unless (force %http-server-socket) + (test-skip 1)) +(test-assert "'download' built-in builder, not found" + (with-http-server 404 "not found" + (let* ((drv (derivation %store "will-never-be-found" + "builtin:download" '() + #:env-vars `(("url" + . ,(object->string (%local-url)))) + #:hash-algo 'sha256 + #:hash (sha256 (random-bytevector 100))))) + (guard (c ((nix-protocol-error? c) + (string-contains (nix-protocol-error-message (pk c)) "failed"))) + (build-derivations %store (list drv)) + #f)))) + +(test-assert "'download' built-in builder, not fixed-output" + (let* ((source (add-text-to-store %store "hello" "hi!")) + (url (string-append "file://" source)) + (drv (derivation %store "world" + "builtin:download" '() + #:env-vars `(("url" . ,(object->string url)))))) + (guard (c ((nix-protocol-error? c) + (string-contains (nix-protocol-error-message c) "failed"))) + (build-derivations %store (list drv)) + #f))) + (test-equal "derivation-name" "foo-0.0" (let ((drv (derivation %store "foo-0.0" %bash '()))) -- cgit v1.2.3 From e74f64b9e55cbc3052698830001238d2407fed19 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sat, 19 Nov 2016 17:05:07 +0100 Subject: store: Add 'references*'. * guix/store.scm (references*): New procedure. * guix/profiles.scm (manifest-lookup-package)[references*]: Remove. * guix/scripts/system.scm (references*): Remove. * tests/gexp.scm ("gexp->file", "gexp->file + file-append") ("gexp->derivation", "gexp->derivation, cross-compilation") ("gexp->derivation, ungexp + ungexp-native") ("scheme-file", "text-file*", "mixed-text-file"): Remove 'references*' instead of (store-lift references). --- guix/profiles.scm | 4 ---- guix/scripts/system.scm | 3 --- guix/store.scm | 4 ++++ tests/gexp.scm | 59 ++++++++++++++++++++++++------------------------- 4 files changed, 33 insertions(+), 37 deletions(-) (limited to 'guix/scripts') diff --git a/guix/profiles.scm b/guix/profiles.scm index b56b8f4c79..0b317ef51e 100644 --- a/guix/profiles.scm +++ b/guix/profiles.scm @@ -501,10 +501,6 @@ if not found." #t)))) items)) - ;; TODO: Factorize. - (define references* - (store-lift references)) - (with-monad %store-monad (match (manifest-entry-item entry) ((? package? package) diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index 71ddccfa61..bb373a6726 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -77,9 +77,6 @@ ;;; Installation. ;;; -;; TODO: Factorize. -(define references* - (store-lift references)) (define topologically-sorted* (store-lift topologically-sorted)) diff --git a/guix/store.scm b/guix/store.scm index 3047dc39b9..7f54b87db1 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -98,6 +98,7 @@ built-in-builders references references/substitutes + references* requisites referrers optimize-store @@ -1170,6 +1171,9 @@ where FILE is the entry's absolute file name and STAT is the result of (define set-build-options* (store-lift set-build-options)) +(define references* + (store-lift references)) + (define-inlinable (current-system) ;; Consult the %CURRENT-SYSTEM fluid at bind time. This is equivalent to ;; (lift0 %current-system %store-monad), but inlinable, thus avoiding diff --git a/tests/gexp.scm b/tests/gexp.scm index 214e7a5302..354d28f014 100644 --- a/tests/gexp.scm +++ b/tests/gexp.scm @@ -375,7 +375,7 @@ (drv (gexp->file "foo" exp)) (out -> (derivation->output-path drv)) (done (built-derivations (list drv))) - (refs ((store-lift references) out))) + (refs (references* out))) (return (and (equal? sexp (call-with-input-file out read)) (equal? (list guile) refs))))) @@ -386,7 +386,7 @@ (drv (gexp->file "foo" exp)) (out -> (derivation->output-path drv)) (done (built-derivations (list drv))) - (refs ((store-lift references) out))) + (refs (references* out))) (return (and (equal? (string-append guile "/bin/guile") (call-with-input-file out read)) (equal? (list guile) refs))))) @@ -407,8 +407,8 @@ (out -> (derivation->output-path drv)) (out2 -> (derivation->output-path drv "2nd")) (done (built-derivations (list drv))) - (refs ((store-lift references) out)) - (refs2 ((store-lift references) out2)) + (refs (references* out)) + (refs2 (references* out2)) (guile (package-file %bootstrap-guile "bin/guile"))) (return (and (string=? (readlink (string-append out "/foo")) guile) (string=? (readlink out2) file) @@ -481,7 +481,7 @@ (ungexp output)))) (xdrv (gexp->derivation "foo" exp #:target target)) - (refs ((store-lift references) + (refs (references* (derivation-file-name xdrv))) (xcu (package->cross-derivation coreutils target)) @@ -506,7 +506,7 @@ (ungexp output)))) (xdrv (gexp->derivation "foo" exp #:target target)) - (refs ((store-lift references) + (refs (references* (derivation-file-name xdrv))) (xglibc (package->cross-derivation glibc target)) (cu (package->derivation coreutils))) @@ -808,34 +808,33 @@ (out -> (derivation->output-path drv))) (mbegin %store-monad (built-derivations (list drv)) - (mlet %store-monad ((refs ((store-lift references) out))) + (mlet %store-monad ((refs (references* out))) (return (and (equal? refs (list text)) (equal? `(list "foo" ,text) (call-with-input-file out read))))))))) (test-assert "text-file*" - (let ((references (store-lift references))) - (run-with-store %store - (mlet* %store-monad - ((drv (package->derivation %bootstrap-guile)) - (guile -> (derivation->output-path drv)) - (file (text-file "bar" "This is bar.")) - (text (text-file* "foo" - %bootstrap-guile "/bin/guile " - (gexp-input %bootstrap-guile "out") "/bin/guile " - drv "/bin/guile " - file)) - (done (built-derivations (list text))) - (out -> (derivation->output-path text)) - (refs (references out))) - ;; Make sure we get the right references and the right content. - (return (and (lset= string=? refs (list guile file)) - (equal? (call-with-input-file out get-string-all) - (string-append guile "/bin/guile " - guile "/bin/guile " - guile "/bin/guile " - file))))) - #:guile-for-build (package-derivation %store %bootstrap-guile)))) + (run-with-store %store + (mlet* %store-monad + ((drv (package->derivation %bootstrap-guile)) + (guile -> (derivation->output-path drv)) + (file (text-file "bar" "This is bar.")) + (text (text-file* "foo" + %bootstrap-guile "/bin/guile " + (gexp-input %bootstrap-guile "out") "/bin/guile " + drv "/bin/guile " + file)) + (done (built-derivations (list text))) + (out -> (derivation->output-path text)) + (refs (references* out))) + ;; Make sure we get the right references and the right content. + (return (and (lset= string=? refs (list guile file)) + (equal? (call-with-input-file out get-string-all) + (string-append guile "/bin/guile " + guile "/bin/guile " + guile "/bin/guile " + file))))) + #:guile-for-build (package-derivation %store %bootstrap-guile))) (test-assertm "mixed-text-file" (mlet* %store-monad ((file -> (mixed-text-file "mixed" @@ -847,7 +846,7 @@ (guile -> (derivation->output-path guile-drv))) (mbegin %store-monad (built-derivations (list drv)) - (mlet %store-monad ((refs ((store-lift references) out))) + (mlet %store-monad ((refs (references* out))) (return (and (string=? (string-append "export PATH=" guile "/bin") (call-with-input-file out get-string-all)) (equal? refs (list guile)))))))) -- cgit v1.2.3 From fac46e3f5e55f9de6fa2ab8082bc418139590fc0 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sat, 19 Nov 2016 18:06:46 +0100 Subject: lint: Add 'mirror-url' checker. * guix/scripts/lint.scm (origin-uris): New procedure. (check-source): Use it. (check-mirror-url): New procedure. (%checkers): Add 'mirror-url' checker. * tests/lint.scm ("mirror-url") ("mirror-url: one suggestion"): New tests. * doc/guix.texi (Invoking guix lint): Document it. --- doc/guix.texi | 4 +++- guix/scripts/lint.scm | 43 +++++++++++++++++++++++++++++++++++++++---- tests/lint.scm | 19 +++++++++++++++++++ 3 files changed, 61 insertions(+), 5 deletions(-) (limited to 'guix/scripts') diff --git a/doc/guix.texi b/doc/guix.texi index 0e70830d02..7352ea973f 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -5379,9 +5379,11 @@ Identify inputs that should most likely be native inputs. @item source @itemx home-page +@itemx mirror-url @itemx source-file-name Probe @code{home-page} and @code{source} URLs and report those that are -invalid. Check that the source file name is meaningful, e.g. is not +invalid. Suggest a @code{mirror://} URL when applicable. Check that +the source file name is meaningful, e.g. is not just a version number or ``git-checkout'', without a declared @code{file-name} (@pxref{origin Reference}). diff --git a/guix/scripts/lint.scm b/guix/scripts/lint.scm index 6e6f550941..9641d3926a 100644 --- a/guix/scripts/lint.scm +++ b/guix/scripts/lint.scm @@ -65,6 +65,7 @@ check-home-page check-source check-source-file-name + check-mirror-url check-license check-vulnerabilities check-formatting @@ -567,6 +568,14 @@ descriptions maintained upstream." (location->string loc) (package-full-name package) (fill-paragraph (escape-quotes upstream) 77 7))))))) +(define (origin-uris origin) + "Return the list of URIs (strings) for ORIGIN." + (match (origin-uri origin) + ((? string? uri) + (list uri)) + ((uris ...) + uris))) + (define (check-source package) "Emit a warning if PACKAGE has an invalid 'source' field, or if that 'source' is not reachable." @@ -583,10 +592,7 @@ descriptions maintained upstream." (let ((origin (package-source package))) (when (and origin (eqv? (origin-method origin) url-fetch)) - (let* ((strings (origin-uri origin)) - (uris (if (list? strings) - (map string->uri strings) - (list (string->uri strings))))) + (let ((uris (map string->uri (origin-uris origin)))) ;; Just make sure that at least one of the URIs is valid. (call-with-values @@ -626,6 +632,31 @@ descriptions maintained upstream." (_ "the source file name should contain the package name") 'source)))) +(define (check-mirror-url package) + "Check whether PACKAGE uses source URLs that should be 'mirror://'." + (define (check-mirror-uri uri) ;XXX: could be optimized + (let loop ((mirrors %mirrors)) + (match mirrors + (() + #t) + (((mirror-id mirror-urls ...) rest ...) + (match (find (cut string-prefix? <> uri) mirror-urls) + (#f + (loop rest)) + (prefix + (emit-warning package + (format #f (_ "URL should be \ +'mirror://~a/~a'") + mirror-id + (string-drop uri (string-length prefix))) + 'source))))))) + + (let ((origin (package-source package))) + (when (and (origin? origin) + (eqv? (origin-method origin) url-fetch)) + (let ((uris (origin-uris origin))) + (for-each check-mirror-uri uris))))) + (define (check-derivation package) "Emit a warning if we fail to compile PACKAGE to a derivation." (catch #t @@ -863,6 +894,10 @@ or a list thereof") (name 'source) (description "Validate source URLs") (check check-source)) + (lint-checker + (name 'mirror-url) + (description "Suggest 'mirror://' URLs") + (check check-mirror-url)) (lint-checker (name 'source-file-name) (description "Validate file names of sources") diff --git a/tests/lint.scm b/tests/lint.scm index cf1b95ee69..0c534562a4 100644 --- a/tests/lint.scm +++ b/tests/lint.scm @@ -508,6 +508,25 @@ (check-source pkg)))) "not reachable: 404"))) +(test-assert "mirror-url" + (string-null? + (with-warnings + (let ((source (origin + (method url-fetch) + (uri "http://example.org/foo/bar.tar.gz") + (sha256 %null-sha256)))) + (check-mirror-url (dummy-package "x" (source source))))))) + +(test-assert "mirror-url: one suggestion" + (string-contains + (with-warnings + (let ((source (origin + (method url-fetch) + (uri "http://ftp.gnu.org/pub/gnu/foo/foo.tar.gz") + (sha256 %null-sha256)))) + (check-mirror-url (dummy-package "x" (source source))))) + "mirror://gnu/foo/foo.tar.gz")) + (test-assert "cve" (mock ((guix scripts lint) package-vulnerabilities (const '())) (string-null? -- cgit v1.2.3