diff options
author | Marius Bakke <mbakke@fastmail.com> | 2018-09-03 19:20:06 +0200 |
---|---|---|
committer | Marius Bakke <mbakke@fastmail.com> | 2018-09-03 19:20:06 +0200 |
commit | 70dc8db8e7a44e0357c6b0582a710a918bd2e353 (patch) | |
tree | 083102cf532c523068f018e2b113947ca6a3db02 /guix | |
parent | 279ed3efee9c71116d368163f805fe9494518687 (diff) | |
parent | c702749dfd47ea6983768cd5b8cf828898445af0 (diff) | |
download | gnu-guix-70dc8db8e7a44e0357c6b0582a710a918bd2e353.tar gnu-guix-70dc8db8e7a44e0357c6b0582a710a918bd2e353.tar.gz |
Merge branch 'master' into core-updates
Diffstat (limited to 'guix')
-rw-r--r-- | guix/channels.scm | 292 | ||||
-rw-r--r-- | guix/describe.scm | 73 | ||||
-rw-r--r-- | guix/discovery.scm | 13 | ||||
-rw-r--r-- | guix/import/hackage.scm | 1 | ||||
-rw-r--r-- | guix/import/pypi.scm | 173 | ||||
-rw-r--r-- | guix/import/stackage.scm | 45 | ||||
-rw-r--r-- | guix/records.scm | 17 | ||||
-rw-r--r-- | guix/scripts/import/pypi.scm | 28 | ||||
-rw-r--r-- | guix/scripts/import/stackage.scm | 46 | ||||
-rw-r--r-- | guix/scripts/pack.scm | 4 | ||||
-rw-r--r-- | guix/scripts/pull.scm | 218 | ||||
-rw-r--r-- | guix/self.scm | 7 |
12 files changed, 668 insertions, 249 deletions
diff --git a/guix/channels.scm b/guix/channels.scm new file mode 100644 index 0000000000..ec3e05eaf5 --- /dev/null +++ b/guix/channels.scm @@ -0,0 +1,292 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2018 Ludovic Courtès <ludo@gnu.org> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix channels) + #:use-module (guix git) + #:use-module (guix records) + #:use-module (guix gexp) + #:use-module (guix discovery) + #:use-module (guix monads) + #:use-module (guix profiles) + #:use-module (guix derivations) + #:use-module (guix store) + #:use-module (guix i18n) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-9) + #:use-module (srfi srfi-11) + #:autoload (guix self) (whole-package) + #:use-module (ice-9 match) + #:export (channel + channel? + channel-name + channel-url + channel-branch + channel-commit + channel-location + + %default-channels + + channel-instance? + channel-instance-channel + channel-instance-commit + channel-instance-checkout + + latest-channel-instances + channel-instance-derivations + latest-channel-derivations + channel-instances->manifest)) + +;;; Commentary: +;;; +;;; This module implements "channels." A channel is usually a source of +;;; package definitions. There's a special channel, the 'guix' channel, that +;;; provides all of Guix, including its commands and its documentation. +;;; User-defined channels are expected to typically provide a bunch of .scm +;;; files meant to be added to the '%package-search-path'. +;;; +;;; This module provides tools to fetch and update channels from a Git +;;; repository and to build them. +;;; +;;; Code: + +(define-record-type* <channel> channel make-channel + channel? + (name channel-name) + (url channel-url) + (branch channel-branch (default "master")) + (commit channel-commit (default #f)) + (location channel-location + (default (current-source-location)) (innate))) +;; TODO: Add a way to express dependencies among channels. + +(define %default-channels + ;; Default list of channels. + (list (channel + (name 'guix) + (branch "origin/master") + (url "https://git.savannah.gnu.org/git/guix.git")))) + +(define (guix-channel? channel) + "Return true if CHANNEL is the 'guix' channel." + (eq? 'guix (channel-name channel))) + +(define-record-type <channel-instance> + (channel-instance channel commit checkout) + channel-instance? + (channel channel-instance-channel) + (commit channel-instance-commit) + (checkout channel-instance-checkout)) + +(define (channel-reference channel) + "Return the \"reference\" for CHANNEL, an sexp suitable for +'latest-repository-commit'." + (match (channel-commit channel) + (#f `(branch . ,(channel-branch channel))) + (commit `(commit . ,(channel-commit channel))))) + +(define (latest-channel-instances store channels) + "Return a list of channel instances corresponding to the latest checkouts of +CHANNELS." + (map (lambda (channel) + (format (current-error-port) + (G_ "Updating channel '~a' from Git repository at '~a'...~%") + (channel-name channel) + (channel-url channel)) + (let-values (((checkout commit) + (latest-repository-commit store (channel-url channel) + #:ref (channel-reference + channel)))) + (channel-instance channel commit checkout))) + channels)) + +(define %self-build-file + ;; The file containing code to build Guix. This serves the same purpose as + ;; a makefile, and, similarly, is intended to always keep this name. + "build-aux/build-self.scm") + +(define %pull-version + ;; This is the version of the 'guix pull' protocol. It specifies what's + ;; expected from %SELF-BUILD-FILE. The initial version ("0") was when we'd + ;; place a set of compiled Guile modules in ~/.config/guix/latest. + 1) + +(define (standard-module-derivation name source dependencies) + "Return a derivation that builds the Scheme modules in SOURCE and that +depend on DEPENDENCIES, a list of lowerable objects. The assumption is that +SOURCE contains package modules to be added to '%package-module-path'." + (define modules + (scheme-modules* source)) + + ;; FIXME: We should load, say SOURCE/.guix-channel.scm, which would allow + ;; channel publishers to specify things such as the sub-directory where .scm + ;; files live, files to exclude from the channel, preferred substitute URLs, + ;; etc. + (mlet* %store-monad ((compiled + (compiled-modules modules + #:name name + #:module-path (list source) + #:extensions dependencies))) + + (gexp->derivation name + (with-extensions dependencies + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (let ((go (string-append #$output "/lib/guile/" + (effective-version) + "/site-ccache")) + (scm (string-append #$output + "/share/guile/site/" + (effective-version)))) + (mkdir-p (dirname go)) + (symlink #$compiled go) + (mkdir-p (dirname scm)) + (symlink #$source scm)))))))) + +(define* (build-from-source name source + #:key verbose? commit + (dependencies '())) + "Return a derivation to build Guix from SOURCE, using the self-build script +contained therein. Use COMMIT as the version string." + ;; Running the self-build script makes it easier to update the build + ;; procedure: the self-build script of the Guix-to-be-installed contains the + ;; right dependencies, build procedure, etc., which the Guix-in-use may not + ;; be know. + (define script + (string-append source "/" %self-build-file)) + + (if (file-exists? script) + (let ((build (save-module-excursion + (lambda () + (primitive-load script))))) + ;; BUILD must be a monadic procedure of at least one argument: the + ;; source tree. + ;; + ;; Note: BUILD can return #f if it does not support %PULL-VERSION. In + ;; the future we'll fall back to a previous version of the protocol + ;; when that happens. + (build source #:verbose? verbose? #:version commit + #:pull-version %pull-version)) + + ;; Build a set of modules that extend Guix using the standard method. + (standard-module-derivation name source dependencies))) + +(define* (build-channel-instance instance #:optional (dependencies '())) + "Return, as a monadic value, the derivation for INSTANCE, a channel +instance. DEPENDENCIES is a list of extensions providing Guile modules that +INSTANCE depends on." + (build-from-source (symbol->string + (channel-name (channel-instance-channel instance))) + (channel-instance-checkout instance) + #:commit (channel-instance-commit instance) + #:dependencies dependencies)) + +(define (channel-instance-derivations instances) + "Return the list of derivations to build INSTANCES, in the same order as +INSTANCES." + (define core-instance + ;; The 'guix' channel is treated specially: it's an implicit dependency of + ;; all the other channels. + (find (lambda (instance) + (guix-channel? (channel-instance-channel instance))) + instances)) + + (mlet %store-monad ((core (build-channel-instance core-instance))) + (mapm %store-monad + (lambda (instance) + (if (eq? instance core-instance) + (return core) + (build-channel-instance instance + (list core)))) + instances))) + +(define latest-channel-derivations + (let ((latest-channel-instances (store-lift latest-channel-instances))) + (lambda (channels) + "Return, as a monadic value, the list of derivations for the latest +instances of CHANNELS." + (mlet %store-monad ((instances (latest-channel-instances channels))) + (channel-instance-derivations instances))))) + +(define (whole-package-for-legacy name modules) + "Return a full-blown Guix package for MODULES, a derivation that builds Guix +modules in the old ~/.config/guix/latest style." + (define packages + (resolve-interface '(gnu packages guile))) + + (letrec-syntax ((list (syntax-rules (->) + ((_) + '()) + ((_ (module -> variable) rest ...) + (cons (module-ref (resolve-interface + '(gnu packages module)) + 'variable) + (list rest ...))) + ((_ variable rest ...) + (cons (module-ref packages 'variable) + (list rest ...)))))) + (whole-package name modules + + ;; In the "old style", %SELF-BUILD-FILE would simply return a + ;; derivation that builds modules. We have to infer what the + ;; dependencies of these modules were. + (list guile-json guile-git guile-bytestructures + (ssh -> guile-ssh) (tls -> gnutls))))) + +(define (old-style-guix? drv) + "Return true if DRV corresponds to a ~/.config/guix/latest style of +derivation." + ;; Here we rely on a gross historical fact: that derivations produced by the + ;; "old style" (before commit 8a0d9bc8a3f153159d9e239a151c0fa98f1e12d8, + ;; dated May 30, 2018) did not depend on "guix-command.drv". + (not (find (lambda (input) + (string-suffix? "-guix-command.drv" + (derivation-input-path input))) + (derivation-inputs drv)))) + +(define (channel-instances->manifest instances) + "Return a profile manifest with entries for all of INSTANCES, a list of +channel instances." + (define instance->entry + (match-lambda + ((instance drv) + (let ((commit (channel-instance-commit instance)) + (channel (channel-instance-channel instance))) + (with-monad %store-monad + (return (manifest-entry + (name (symbol->string (channel-name channel))) + (version (string-take commit 7)) + (item (if (guix-channel? channel) + (if (old-style-guix? drv) + (whole-package-for-legacy + (string-append name "-" version) + drv) + drv) + drv)) + (properties + `((source (repository + (version 0) + (url ,(channel-url channel)) + (branch ,(channel-branch channel)) + (commit ,commit)))))))))))) + + (mlet* %store-monad ((derivations (channel-instance-derivations instances)) + (entries (mapm %store-monad instance->entry + (zip instances derivations)))) + (return (manifest entries)))) diff --git a/guix/describe.scm b/guix/describe.scm new file mode 100644 index 0000000000..3122a762fe --- /dev/null +++ b/guix/describe.scm @@ -0,0 +1,73 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2018 Ludovic Courtès <ludo@gnu.org> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix describe) + #:use-module (guix memoization) + #:use-module (guix profiles) + #:use-module (srfi srfi-1) + #:use-module (ice-9 match) + #:export (package-path-entries)) + +;;; Commentary: +;;; +;;; This module provides supporting code to allow a Guix instance to find, at +;;; run time, which profile it's in (profiles created by 'guix pull'). That +;;; allows it to read meta-information about itself (e.g., repository URL and +;;; commit ID) and to find other channels available in the same profile. It's +;;; a bit like ELPA's pkg-info.el. +;;; +;;; Code: + +(define current-profile + (mlambda () + "Return the profile (created by 'guix pull') the calling process lives in, +or #f if this is not applicable." + (match (command-line) + ((program . _) + (and (string-suffix? "/bin/guix" program) + ;; Note: We want to do _lexical dot-dot resolution_. Using ".." + ;; for real would instead take us into the /gnu/store directory + ;; that ~/.config/guix/current/bin points to, whereas we want to + ;; obtain ~/.config/guix/current. + (let ((candidate (dirname (dirname program)))) + (and (file-exists? (string-append candidate "/manifest")) + candidate))))))) + +(define current-profile-entries + (mlambda () + "Return the list of entries in the 'guix pull' profile the calling process +lives in, or #f if this is not applicable." + (match (current-profile) + (#f '()) + (profile + (let ((manifest (profile-manifest profile))) + (manifest-entries manifest)))))) + +(define package-path-entries + (mlambda () + "Return a list of package path entries to be added to the package search +path. These entries are taken from the 'guix pull' profile the calling +process lives in, when applicable." + ;; Filter out Guix itself. + (filter-map (lambda (entry) + (and (not (string=? (manifest-entry-name entry) + "guix")) + (string-append (manifest-entry-item entry) + "/share/guile/site/" + (effective-version)))) + (current-profile-entries)))) diff --git a/guix/discovery.scm b/guix/discovery.scm index 2b627d108e..3fc6e2c9e7 100644 --- a/guix/discovery.scm +++ b/guix/discovery.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -27,6 +27,7 @@ #:use-module (ice-9 ftw) #:export (scheme-files scheme-modules + scheme-modules* fold-modules all-modules fold-module-public-variables)) @@ -115,6 +116,16 @@ name and the exception key and arguments." (string-append directory "/" sub-directory) directory)))) +(define* (scheme-modules* directory #:optional sub-directory) + "Return the list of module names found under SUB-DIRECTORY in DIRECTORY. +This is a source-only variant that does not try to load files." + (let ((prefix (string-length directory))) + (map (lambda (file) + (file-name->module-name (string-drop file prefix))) + (scheme-files (if sub-directory + (string-append directory "/" sub-directory) + directory))))) + (define* (fold-modules proc init path #:key (warn (const #f))) "Fold over all the Scheme modules present in PATH, a list of directories. Call (PROC MODULE RESULT) for each module that is found." diff --git a/guix/import/hackage.scm b/guix/import/hackage.scm index 3c00f680bf..54301de2e8 100644 --- a/guix/import/hackage.scm +++ b/guix/import/hackage.scm @@ -44,6 +44,7 @@ %hackage-updater guix-package->hackage-name + hackage-name->package-name hackage-fetch hackage-source-url hackage-cabal-url diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 25560bac46..87b047bdac 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -3,6 +3,7 @@ ;;; Copyright © 2015 Cyril Roelandt <tipecaml@gmail.com> ;;; Copyright © 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2017 Mathieu Othacehe <m.othacehe@gmail.com> +;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -24,6 +25,7 @@ #:use-module (ice-9 match) #:use-module (ice-9 pretty-print) #:use-module (ice-9 regex) + #:use-module (ice-9 receive) #:use-module ((ice-9 rdelim) #:select (read-line)) #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) @@ -36,7 +38,8 @@ #:use-module (guix utils) #:use-module ((guix build utils) #:select ((package-name->name+version - . hyphen-package-name->name+version))) + . hyphen-package-name->name+version) + find-files)) #:use-module (guix import utils) #:use-module ((guix download) #:prefix download:) #:use-module (guix import json) @@ -45,6 +48,7 @@ #:use-module ((guix licenses) #:prefix license:) #:use-module (guix build-system python) #:export (guix-package->pypi-name + pypi-recursive-import pypi->guix-package %pypi-updater)) @@ -114,9 +118,9 @@ package definition." `((propagated-inputs (,'quasiquote ,package-inputs)))))) (define (guess-requirements source-url wheel-url tarball) - "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list of -the required packages specified in the requirements.txt file. TARBALL will be -extracted in the current directory, and will be deleted." + "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list +of the required packages specified in the requirements.txt file. TARBALL will +be extracted in a temporary directory." (define (tarball-directory url) ;; Given the URL of the package's tarball, return the name of the directory @@ -140,8 +144,8 @@ cannot determine package dependencies")) ;; file, remove everything other than the actual name of the required ;; package, and return it. (string-take s - (or (string-index s #\space) - (string-length s)))) + (or (string-index s (lambda (chr) (member chr '(#\space #\> #\= #\<)))) + (string-length s)))) (define (comment? line) ;; Return #t if the given LINE is a comment, #f otherwise. @@ -160,7 +164,7 @@ cannot determine package dependencies")) ((or (string-null? line) (comment? line)) (loop result)) (else - (loop (cons (python->package-name (clean-requirement line)) + (loop (cons (clean-requirement line) result)))))))))) (define (read-wheel-metadata wheel-archive) @@ -180,9 +184,7 @@ cannot determine package dependencies")) (hash-ref (list-ref run_requires 0) "requires") '()))) - (map (lambda (r) - (python->package-name (clean-requirement r))) - requirements))))) + (map clean-requirement requirements))))) (lambda () (delete-file json-file) (rmdir dirname)))))) @@ -197,31 +199,37 @@ cannot determine package dependencies")) (read-wheel-metadata temp)) #f)))) - (define (guess-requirements-from-source) ;; Return the package's requirements by guessing them from the source. (let ((dirname (tarball-directory source-url))) (if (string? dirname) - (let* ((req-file (string-append dirname "/requirements.txt")) - (exit-code (system* "tar" "xf" tarball req-file))) - ;; TODO: support more formats. - (if (zero? exit-code) - (dynamic-wind - (const #t) - (lambda () - (read-requirements req-file)) - (lambda () - (delete-file req-file) - (rmdir dirname))) - (begin - (warning (G_ "'tar xf' failed with exit code ~a\n") - exit-code) - '()))) + (call-with-temporary-directory + (lambda (dir) + (let* ((pypi-name (string-take dirname (string-rindex dirname #\-))) + (req-files (list (string-append dirname "/requirements.txt") + (string-append dirname "/" pypi-name ".egg-info" + "/requires.txt"))) + (exit-codes (map (lambda (file-name) + (parameterize ((current-error-port (%make-void-port "rw+")) + (current-output-port (%make-void-port "rw+"))) + (system* "tar" "xf" tarball "-C" dir file-name))) + req-files))) + ;; Only one of these files needs to exist. + (if (any zero? exit-codes) + (match (find-files dir) + ((file . _) + (read-requirements file)) + (() + (warning (G_ "No requirements file found.\n")))) + (begin + (warning (G_ "Failed to extract requirements files\n")) + '()))))) '()))) ;; First, try to compute the requirements using the wheel, since that is the ;; most reliable option. If a wheel is not provided for this package, try - ;; getting them by reading the "requirements.txt" file from the source. Note + ;; getting them by reading either the "requirements.txt" file or the + ;; "requires.txt" from the egg-info directory from the source tarball. Note ;; that "requirements.txt" is not mandatory, so this is likely to fail. (or (guess-requirements-from-wheel) (guess-requirements-from-source))) @@ -229,16 +237,21 @@ cannot determine package dependencies")) (define (compute-inputs source-url wheel-url tarball) "Given the SOURCE-URL of an already downloaded TARBALL, return a list of -name/variable pairs describing the required inputs of this package." - (sort - (map (lambda (input) - (list input (list 'unquote (string->symbol input)))) - (remove (cut string=? "python-argparse" <>) - (guess-requirements source-url wheel-url tarball))) - (lambda args - (match args - (((a _ ...) (b _ ...)) - (string-ci<? a b)))))) +name/variable pairs describing the required inputs of this package. Also +return the unaltered list of upstream dependency names." + (let ((dependencies + (remove (cut string=? "argparse" <>) + (guess-requirements source-url wheel-url tarball)))) + (values (sort + (map (lambda (input) + (let ((guix-name (python->package-name input))) + (list guix-name (list 'unquote (string->symbol guix-name))))) + dependencies) + (lambda args + (match args + (((a _ ...) (b _ ...)) + (string-ci<? a b))))) + dependencies))) (define (make-pypi-sexp name version source-url wheel-url home-page synopsis description license) @@ -247,46 +260,58 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." (call-with-temporary-output-file (lambda (temp port) (and (url-fetch source-url temp) - `(package - (name ,(python->package-name name)) - (version ,version) - (source (origin - (method url-fetch) + (receive (input-package-names upstream-dependency-names) + (compute-inputs source-url wheel-url temp) + (values + `(package + (name ,(python->package-name name)) + (version ,version) + (source (origin + (method url-fetch) - ;; Sometimes 'pypi-uri' doesn't quite work due to mixed - ;; cases in NAME, for instance, as is the case with - ;; "uwsgi". In that case, fall back to a full URL. - (uri (pypi-uri ,(string-downcase name) version)) - (sha256 - (base32 - ,(guix-hash-url temp))))) - (build-system python-build-system) - ,@(maybe-inputs (compute-inputs source-url wheel-url temp)) - (home-page ,home-page) - (synopsis ,synopsis) - (description ,description) - (license ,(license->symbol license))))))) + ;; Sometimes 'pypi-uri' doesn't quite work due to mixed + ;; cases in NAME, for instance, as is the case with + ;; "uwsgi". In that case, fall back to a full URL. + (uri (pypi-uri ,(string-downcase name) version)) + (sha256 + (base32 + ,(guix-hash-url temp))))) + (build-system python-build-system) + ,@(maybe-inputs input-package-names) + (home-page ,home-page) + (synopsis ,synopsis) + (description ,description) + (license ,(license->symbol license))) + upstream-dependency-names)))))) -(define (pypi->guix-package package-name) - "Fetch the metadata for PACKAGE-NAME from pypi.org, and return the +(define pypi->guix-package + (memoize + (lambda* (package-name) + "Fetch the metadata for PACKAGE-NAME from pypi.org, and return the `package' s-expression corresponding to that package, or #f on failure." - (let ((package (pypi-fetch package-name))) - (and package - (guard (c ((missing-source-error? c) - (let ((package (missing-source-error-package c))) - (leave (G_ "no source release for pypi package ~a ~a~%") - (assoc-ref* package "info" "name") - (assoc-ref* package "info" "version"))))) - (let ((name (assoc-ref* package "info" "name")) - (version (assoc-ref* package "info" "version")) - (release (assoc-ref (latest-source-release package) "url")) - (wheel (assoc-ref (latest-wheel-release package) "url")) - (synopsis (assoc-ref* package "info" "summary")) - (description (assoc-ref* package "info" "summary")) - (home-page (assoc-ref* package "info" "home_page")) - (license (string->license (assoc-ref* package "info" "license")))) - (make-pypi-sexp name version release wheel home-page synopsis - description license)))))) + (let ((package (pypi-fetch package-name))) + (and package + (guard (c ((missing-source-error? c) + (let ((package (missing-source-error-package c))) + (leave (G_ "no source release for pypi package ~a ~a~%") + (assoc-ref* package "info" "name") + (assoc-ref* package "info" "version"))))) + (let ((name (assoc-ref* package "info" "name")) + (version (assoc-ref* package "info" "version")) + (release (assoc-ref (latest-source-release package) "url")) + (wheel (assoc-ref (latest-wheel-release package) "url")) + (synopsis (assoc-ref* package "info" "summary")) + (description (assoc-ref* package "info" "summary")) + (home-page (assoc-ref* package "info" "home_page")) + (license (string->license (assoc-ref* package "info" "license")))) + (make-pypi-sexp name version release wheel home-page synopsis + description license)))))))) + +(define (pypi-recursive-import package-name) + (recursive-import package-name #f + #:repo->guix-package (lambda (name repo) + (pypi->guix-package name)) + #:guix-name python->package-name)) (define (string->license str) "Convert the string STR into a license object." diff --git a/guix/import/stackage.scm b/guix/import/stackage.scm index ec93fbced6..afd5d997ae 100644 --- a/guix/import/stackage.scm +++ b/guix/import/stackage.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2017 Federico Beffa <beffa@fbengineering.ch> +;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -25,10 +26,12 @@ #:use-module (srfi srfi-35) #:use-module (guix import json) #:use-module (guix import hackage) + #:use-module (guix import utils) #:use-module (guix memoization) #:use-module (guix packages) #:use-module (guix upstream) #:export (stackage->guix-package + stackage-recursive-import %stackage-updater)) @@ -45,9 +48,9 @@ (_ #f))) (define (lts-info-packages lts-info) - "Retruns the alist of packages contained in LTS-INFO." + "Returns the alist of packages contained in LTS-INFO." (match lts-info - ((_ ("packages" pkg ...)) pkg) + ((("packages" pkg ...) . _) pkg) (_ '()))) (define (leave-with-message fmt . args) @@ -85,25 +88,33 @@ (define (hackage-name-version name version) (and version (string-append name "@" version))) -(define* (stackage->guix-package package-name ; upstream name - #:key - (include-test-dependencies? #t) - (lts-version "") - (packages-info - (lts-info-packages - (stackage-lts-info-fetch lts-version)))) - "Fetch Cabal file for PACKAGE-NAME from hackage.haskell.org. The retrieved +(define stackage->guix-package + (memoize + (lambda* (package-name ; upstream name + #:key + (include-test-dependencies? #t) + (lts-version "") + (packages-info + (lts-info-packages + (stackage-lts-info-fetch lts-version)))) + "Fetch Cabal file for PACKAGE-NAME from hackage.haskell.org. The retrieved vesion corresponds to the version of PACKAGE-NAME specified in the LTS-VERSION release at stackage.org. Return the `package' S-expression corresponding to that package, or #f on failure. PACKAGES-INFO is the alist with the packages included in the Stackage LTS release." - (let* ((version (lts-package-version packages-info package-name)) - (name-version (hackage-name-version package-name version))) - (if name-version - (hackage->guix-package name-version - #:include-test-dependencies? - include-test-dependencies?) - (leave-with-message "~a: Stackage package not found" package-name)))) + (let* ((version (lts-package-version packages-info package-name)) + (name-version (hackage-name-version package-name version))) + (if name-version + (hackage->guix-package name-version + #:include-test-dependencies? + include-test-dependencies?) + (leave-with-message "~a: Stackage package not found" package-name)))))) + +(define (stackage-recursive-import package-name . args) + (recursive-import package-name #f + #:repo->guix-package (lambda (name repo) + (apply stackage->guix-package (cons name args))) + #:guix-name hackage-name->package-name)) ;;; diff --git a/guix/records.scm b/guix/records.scm index da3ecdaaf8..98f3c8fef0 100644 --- a/guix/records.scm +++ b/guix/records.scm @@ -52,17 +52,6 @@ ((weird _ ...) ;weird! (syntax-violation name "invalid field specifier" #'weird))))) -(define (print-record-abi-mismatch-error port key args - default-printer) - (match args - ((rtd . _) - ;; The source file where this exception is thrown must be recompiled. - (format port "ERROR: ~a: record ABI mismatch; recompilation needed" - rtd)))) - -(set-exception-printer! 'record-abi-mismatch-error - print-record-abi-mismatch-error) - (eval-when (expand load eval) ;; The procedures below are needed both at run time and at expansion time. @@ -81,7 +70,11 @@ interface\" (ABI) for TYPE is equal to COOKIE." (with-syntax ((current-abi (current-abi-identifier type))) #`(unless (eq? current-abi #,cookie) - (throw 'record-abi-mismatch-error #,type))))) + ;; The source file where this exception is thrown must be + ;; recompiled. + (throw 'record-abi-mismatch-error 'abi-check + "~a: record ABI mismatch; recompilation needed" + (list #,type) '()))))) (define-syntax make-syntactic-constructor (syntax-rules () diff --git a/guix/scripts/import/pypi.scm b/guix/scripts/import/pypi.scm index 59a925a3ca..7bd83818ba 100644 --- a/guix/scripts/import/pypi.scm +++ b/guix/scripts/import/pypi.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 David Thompson <davet@gnu.org> +;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -25,6 +26,7 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-37) + #:use-module (srfi srfi-41) #:use-module (ice-9 match) #:use-module (ice-9 format) #:export (guix-import-pypi)) @@ -43,6 +45,8 @@ Import and convert the PyPI package for PACKAGE-NAME.\n")) (display (G_ " -h, --help display this help and exit")) (display (G_ " + -r, --recursive import packages recursively")) + (display (G_ " -V, --version display version information and exit")) (newline) (show-bug-report-information)) @@ -56,6 +60,9 @@ Import and convert the PyPI package for PACKAGE-NAME.\n")) (option '(#\V "version") #f #f (lambda args (show-version-and-exit "guix import pypi"))) + (option '(#\r "recursive") #f #f + (lambda (opt name arg result) + (alist-cons 'recursive #t result))) %standard-import-options)) @@ -81,11 +88,22 @@ Import and convert the PyPI package for PACKAGE-NAME.\n")) (reverse opts)))) (match args ((package-name) - (let ((sexp (pypi->guix-package package-name))) - (unless sexp - (leave (G_ "failed to download meta-data for package '~a'~%") - package-name)) - sexp)) + (if (assoc-ref opts 'recursive) + ;; Recursive import + (map (match-lambda + ((and ('package ('name name) . rest) pkg) + `(define-public ,(string->symbol name) + ,pkg)) + (_ #f)) + (reverse + (stream->list + (pypi-recursive-import package-name)))) + ;; Single import + (let ((sexp (pypi->guix-package package-name))) + (unless sexp + (leave (G_ "failed to download meta-data for package '~a'~%") + package-name)) + sexp))) (() (leave (G_ "too few arguments~%"))) ((many ...) diff --git a/guix/scripts/import/stackage.scm b/guix/scripts/import/stackage.scm index e6676e93e8..b4b12581bf 100644 --- a/guix/scripts/import/stackage.scm +++ b/guix/scripts/import/stackage.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2017 Federico Beffa <beffa@fbengineering.ch> +;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -26,6 +27,7 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-37) + #:use-module (srfi srfi-41) #:use-module (ice-9 match) #:use-module (ice-9 format) #:export (guix-import-stackage)) @@ -43,11 +45,13 @@ (display (G_ "Usage: guix import stackage PACKAGE-NAME Import and convert the LTS Stackage package for PACKAGE-NAME.\n")) (display (G_ " - -r VERSION, --lts-version=VERSION + -l VERSION, --lts-version=VERSION specify the LTS version to use")) (display (G_ " -h, --help display this help and exit")) (display (G_ " + -r, --recursive import packages recursively")) + (display (G_ " -t, --no-test-dependencies don't include test-only dependencies")) (display (G_ " -V, --version display version information and exit")) @@ -68,11 +72,14 @@ Import and convert the LTS Stackage package for PACKAGE-NAME.\n")) (alist-cons 'include-test-dependencies? #f (alist-delete 'include-test-dependencies? result)))) - (option '(#\r "lts-version") #t #f + (option '(#\l "lts-version") #t #f (lambda (opt name arg result) (alist-cons 'lts-version arg (alist-delete 'lts-version result)))) + (option '(#\r "recursive") #f #f + (lambda (opt name arg result) + (alist-cons 'recursive #t result))) %standard-import-options)) @@ -90,6 +97,27 @@ Import and convert the LTS Stackage package for PACKAGE-NAME.\n")) (alist-cons 'argument arg result)) %default-options)) + (define (run-importer package-name opts error-fn) + (let* ((arguments (list + package-name + #:include-test-dependencies? + (assoc-ref opts 'include-test-dependencies?) + #:lts-version (assoc-ref opts 'lts-version))) + (sexp (if (assoc-ref opts 'recursive) + ;; Recursive import + (map (match-lambda + ((and ('package ('name name) . rest) pkg) + `(define-public ,(string->symbol name) + ,pkg)) + (_ #f)) + (reverse + (stream->list + (apply stackage-recursive-import arguments)))) + ;; Single import + (apply stackage->guix-package arguments)))) + (unless sexp (error-fn)) + sexp)) + (let* ((opts (parse-options)) (args (filter-map (match-lambda (('argument . value) @@ -99,15 +127,11 @@ Import and convert the LTS Stackage package for PACKAGE-NAME.\n")) (match args ((package-name) (with-error-handling - (let ((sexp (stackage->guix-package - package-name - #:include-test-dependencies? - (assoc-ref opts 'include-test-dependencies?) - #:lts-version (assoc-ref opts 'lts-version)))) - (unless sexp - (leave (G_ "failed to download cabal file for package '~a'~%") - package-name)) - sexp))) + (run-importer package-name opts + (lambda () + (leave (G_ "failed to download cabal file \ +for package '~a'~%") + package-name))))) (() (leave (G_ "too few arguments~%"))) ((many ...) diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index 729850839b..fb0677de28 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -748,8 +748,8 @@ Create a bundle of PACKAGE.\n")) (build-image (match (assq-ref %formats pack-format) ((? procedure? proc) proc) (#f - (leave (G_ "~a: unknown pack format") - format)))) + (leave (G_ "~a: unknown pack format~%") + pack-format)))) (localstatedir? (assoc-ref opts 'localstatedir?))) (run-with-store store (mlet* %store-monad ((profile (profile-derivation diff --git a/guix/scripts/pull.scm b/guix/scripts/pull.scm index ee68c21a4c..18c04f05dd 100644 --- a/guix/scripts/pull.scm +++ b/guix/scripts/pull.scm @@ -30,26 +30,19 @@ #:use-module (guix grafts) #:use-module (guix memoization) #:use-module (guix monads) + #:use-module (guix channels) #:autoload (guix inferior) (open-inferior) #:use-module (guix scripts build) - #:autoload (guix self) (whole-package) #:use-module (guix git) #:use-module (git) #:use-module (gnu packages) - #:autoload (gnu packages ssh) (guile-ssh) - #:autoload (gnu packages tls) (gnutls) #:use-module ((guix scripts package) #:select (build-and-use-profile)) - #:use-module ((guix build utils) - #:select (with-directory-excursion delete-file-recursively)) - #:use-module ((guix build download) - #:select (%x509-certificate-directory)) #:use-module (gnu packages base) #:use-module (gnu packages guile) #:use-module ((gnu packages bootstrap) #:select (%bootstrap-guile)) #:use-module ((gnu packages certs) #:select (le-certs)) #:use-module (srfi srfi-1) - #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) #:use-module (srfi srfi-35) #:use-module (srfi srfi-37) @@ -57,9 +50,6 @@ #:use-module (ice-9 vlist) #:export (guix-pull)) -(define %repository-url - (or (getenv "GUIX_PULL_URL") "https://git.savannah.gnu.org/git/guix.git")) - ;;; ;;; Command-line options. @@ -67,9 +57,7 @@ (define %default-options ;; Alist of default option values. - `((repository-url . ,%repository-url) - (ref . (branch . "origin/master")) - (system . ,(%current-system)) + `((system . ,(%current-system)) (substitutes? . #t) (build-hook? . #t) (graft? . #t) @@ -81,6 +69,8 @@ Download and deploy the latest version of Guix.\n")) (display (G_ " --verbose produce verbose output")) (display (G_ " + -C, --channels=FILE deploy the channels defined in FILE")) + (display (G_ " --url=URL download from the Git repository at URL")) (display (G_ " --commit=COMMIT download the specified COMMIT")) @@ -105,6 +95,9 @@ Download and deploy the latest version of Guix.\n")) (cons* (option '("verbose") #f #f (lambda (opt name arg result) (alist-cons 'verbose? #t result))) + (option '(#\C "channels") #t #f + (lambda (opt name arg result) + (alist-cons 'channel-file arg result))) (option '(#\l "list-generations") #f #t (lambda (opt name arg result) (cons `(query list-generations ,(or arg "")) @@ -142,70 +135,6 @@ Download and deploy the latest version of Guix.\n")) (define indirect-root-added (store-lift add-indirect-root)) -(define %self-build-file - ;; The file containing code to build Guix. This serves the same purpose as - ;; a makefile, and, similarly, is intended to always keep this name. - "build-aux/build-self.scm") - -(define %pull-version - ;; This is the version of the 'guix pull' protocol. It specifies what's - ;; expected from %SELF-BUILD-FILE. The initial version ("0") was when we'd - ;; place a set of compiled Guile modules in ~/.config/guix/latest. - 1) - -(define* (build-from-source source - #:key verbose? commit) - "Return a derivation to build Guix from SOURCE, using the self-build script -contained therein. Use COMMIT as the version string." - ;; Running the self-build script makes it easier to update the build - ;; procedure: the self-build script of the Guix-to-be-installed contains the - ;; right dependencies, build procedure, etc., which the Guix-in-use may not - ;; be know. - (let* ((script (string-append source "/" %self-build-file)) - (build (primitive-load script))) - ;; BUILD must be a monadic procedure of at least one argument: the source - ;; tree. - ;; - ;; Note: BUILD can return #f if it does not support %PULL-VERSION. In the - ;; future we'll fall back to a previous version of the protocol when that - ;; happens. - (build source #:verbose? verbose? #:version commit - #:pull-version %pull-version))) - -(define (whole-package-for-legacy name modules) - "Return a full-blown Guix package for MODULES, a derivation that builds Guix -modules in the old ~/.config/guix/latest style." - (whole-package name modules - - ;; In the "old style", %SELF-BUILD-FILE would simply return a - ;; derivation that builds modules. We have to infer what the - ;; dependencies of these modules were. - (list guile-json guile-git guile-bytestructures - guile-ssh gnutls))) - -(define* (derivation->manifest-entry drv - #:key url branch commit) - "Return a manifest entry for DRV, which represents Guix at COMMIT. Record -URL, BRANCH, and COMMIT as a property in the manifest entry." - (mbegin %store-monad - (what-to-build (list drv)) - (built-derivations (list drv)) - (let ((out (derivation->output-path drv))) - (return (manifest-entry - (name "guix") - (version (string-take commit 7)) - (item (if (file-exists? (string-append out "/bin/guix")) - drv - (whole-package-for-legacy (string-append name "-" - version) - drv))) - (properties - `((source (repository - (version 0) - (url ,url) - (branch ,branch) - (commit ,commit)))))))))) - (define (display-profile-news profile) "Display what's up in PROFILE--new packages, and all that." (match (memv (generation-number profile) @@ -223,8 +152,8 @@ URL, BRANCH, and COMMIT as a property in the manifest entry." #:heading (G_ "New in this revision:\n")))) (_ #t))) -(define* (build-and-install source config-dir - #:key verbose? url branch commit) +(define* (build-and-install instances config-dir + #:key verbose?) "Build the tool from SOURCE, and install it in CONFIG-DIR." (define update-profile (store-lift build-and-use-profile)) @@ -232,15 +161,9 @@ URL, BRANCH, and COMMIT as a property in the manifest entry." (define profile (string-append config-dir "/current")) - (mlet* %store-monad ((drv (build-from-source source - #:commit commit - #:verbose? verbose?)) - (entry (derivation->manifest-entry drv - #:url url - #:branch branch - #:commit commit))) + (mlet %store-monad ((manifest (channel-instances->manifest instances))) (mbegin %store-monad - (update-profile profile (manifest (list entry))) + (update-profile profile manifest) (return (display-profile-news profile))))) (define (honor-lets-encrypt-certificates! store) @@ -426,45 +349,106 @@ and ALIST2 differ, display HEADING upfront." ((numbers ...) (list-generations profile numbers))))))))) +(define (channel-list opts) + "Return the list of channels to use. If OPTS specify a channel file, +channels are read from there; otherwise, if ~/.config/guix/channels.scm +exists, read it; otherwise %DEFAULT-CHANNELS is used. Apply channel +transformations specified in OPTS (resulting from '--url', '--commit', or +'--branch'), if any." + (define file + (assoc-ref opts 'channel-file)) + + (define default-file + (string-append (config-directory) "/channels.scm")) + + (define (load-channels file) + (let ((result (load* file (make-user-module '((guix channels)))))) + (if (and (list? result) (every channel? result)) + result + (leave (G_ "'~a' did not return a list of channels~%") file)))) + + (define channels + (cond (file + (load-channels file)) + ((file-exists? default-file) + (load-channels default-file)) + (else + %default-channels))) + + (define (environment-variable) + (match (getenv "GUIX_PULL_URL") + (#f #f) + (url + (warning (G_ "The 'GUIX_PULL_URL' environment variable is deprecated. +Use '~/.config/guix/channels.scm' instead.")) + url))) + + (let ((ref (assoc-ref opts 'ref)) + (url (or (assoc-ref opts 'repository-url) + (environment-variable)))) + (if (or ref url) + (match channels + ((one) + ;; When there's only one channel, apply '--url', '--commit', and + ;; '--branch' to this specific channel. + (let ((url (or url (channel-url one)))) + (list (match ref + (('commit . commit) + (channel (inherit one) + (url url) (commit commit) (branch #f))) + (('branch . branch) + (channel (inherit one) + (url url) (commit #f) (branch branch))) + (#f + (channel (inherit one) (url url))))))) + (_ + ;; Otherwise bail out. + (leave + (G_ "'--url', '--commit', and '--branch' are not applicable~%")))) + channels))) + (define (guix-pull . args) - (define (use-le-certs? url) - (string-prefix? "https://git.savannah.gnu.org/" url)) - (with-error-handling (with-git-error-handling - (let* ((opts (parse-command-line args %options - (list %default-options))) - (url (assoc-ref opts 'repository-url)) - (ref (assoc-ref opts 'ref)) - (cache (string-append (cache-directory) "/pull"))) + (let* ((opts (parse-command-line args %options + (list %default-options))) + (cache (string-append (cache-directory) "/pull")) + (channels (channel-list opts))) + (cond ((assoc-ref opts 'query) (process-query opts)) ((assoc-ref opts 'dry-run?) #t) ;XXX: not very useful (else (with-store store - (parameterize ((%graft? (assoc-ref opts 'graft?))) + (parameterize ((%graft? (assoc-ref opts 'graft?)) + (%repository-cache-directory cache)) (set-build-options-from-command-line store opts) - ;; For reproducibility, always refer to the LE certificates - ;; when we know we're talking to Savannah. - (when (use-le-certs? url) - (honor-lets-encrypt-certificates! store)) - - (format (current-error-port) - (G_ "Updating from Git repository at '~a'...~%") - url) - - (let-values (((checkout commit) - (latest-repository-commit store url - #:ref ref - #:cache-directory - cache))) + ;; When certificates are already installed, use them. + ;; Otherwise, use the Let's Encrypt certificates, which we + ;; know Savannah uses. + (let ((certs (or (getenv "SSL_CERT_DIR") "/etc/ssl/certs"))) + (unless (file-exists? certs) + (honor-lets-encrypt-certificates! store))) + (let ((instances (latest-channel-instances store channels))) (format (current-error-port) - (G_ "Building from Git commit ~a...~%") - commit) + (N_ "Building from this channel:~%" + "Building from these channels:~%" + (length instances))) + (for-each (lambda (instance) + (let ((channel + (channel-instance-channel instance))) + (format (current-error-port) + " ~10a~a\t~a~%" + (channel-name channel) + (channel-url channel) + (string-take + (channel-instance-commit instance) + 7)))) + instances) (parameterize ((%guile-for-build (package-derivation store @@ -472,13 +456,7 @@ and ALIST2 differ, display HEADING upfront." %bootstrap-guile (canonical-package guile-2.2))))) (run-with-store store - (build-and-install checkout (config-directory) - #:url url - #:branch (match ref - (('branch . branch) - branch) - (_ #f)) - #:commit commit + (build-and-install instances (config-directory) #:verbose? (assoc-ref opts 'verbose?))))))))))))) diff --git a/guix/self.scm b/guix/self.scm index 5b088c413b..c5da6130bb 100644 --- a/guix/self.scm +++ b/guix/self.scm @@ -206,13 +206,6 @@ list of file-name/file-like objects suitable as inputs to 'imported-files'." (local-file file #:recursive? #t))) (find-files (string-append directory "/" sub-directory) pred))) -(define (scheme-modules* directory sub-directory) - "Return the list of module names found under SUB-DIRECTORY in DIRECTORY." - (let ((prefix (string-length directory))) - (map (lambda (file) - (file-name->module-name (string-drop file prefix))) - (scheme-files (string-append directory "/" sub-directory))))) - (define* (sub-directory item sub-directory) "Return SUB-DIRECTORY within ITEM, which may be a file name or a file-like object." |