summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--guix/import/pypi.scm177
-rw-r--r--tests/pypi.scm79
2 files changed, 166 insertions, 90 deletions
diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index 23a1e69061..537431dd69 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -4,6 +4,7 @@
;;; Copyright © 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org>
;;; Copyright © 2017 Mathieu Othacehe <m.othacehe@gmail.com>
;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net>
+;;; Copyright © 2019 Maxim Cournoyer <maxim.cournoyer@gmail.com>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -26,6 +27,7 @@
#:use-module (ice-9 receive)
#:use-module ((ice-9 rdelim) #:select (read-line))
#:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-11)
#:use-module (srfi srfi-26)
#:use-module (srfi srfi-34)
#:use-module (srfi srfi-35)
@@ -107,14 +109,15 @@ package on PyPI."
((name version _ ...)
(string-append name "-" version ".dist-info"))))
-(define (maybe-inputs package-inputs)
+(define (maybe-inputs package-inputs input-type)
"Given a list of PACKAGE-INPUTS, tries to generate the 'inputs' field of a
-package definition."
+package definition. INPUT-TYPE, a symbol, is used to populate the name of
+the input field."
(match package-inputs
(()
'())
((package-inputs ...)
- `((propagated-inputs (,'quasiquote ,package-inputs))))))
+ `((,input-type (,'quasiquote ,package-inputs))))))
(define %requirement-name-regexp
;; Regexp to match the requirement name in a requirement specification.
@@ -154,9 +157,19 @@ package definition."
(or (regexp-exec %requirement-name-regexp spec)
(error (G_ "Could not extract requirement name in spec:") spec))))
+(define (test-section? name)
+ "Return #t if the section name contains 'test' or 'dev'."
+ (any (cut string-contains-ci name <>)
+ '("test" "dev")))
+
(define (parse-requires.txt requires.txt)
- "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of
-requirement names."
+ "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of lists
+of requirements.
+
+The first list contains the required dependencies while the second the
+optional test dependencies. Note that currently, optional, non-test
+dependencies are omitted since these can be difficult or expensive to
+satisfy."
(define (comment? line)
;; Return #t if the given LINE is a comment, #f otherwise.
@@ -168,26 +181,49 @@ requirement names."
(call-with-input-file requires.txt
(lambda (port)
- (let loop ((result '()))
+ (let loop ((required-deps '())
+ (test-deps '())
+ (inside-test-section? #f)
+ (optional? #f))
(let ((line (read-line port)))
- ;; Stop when a section is encountered, as sections contain optional
- ;; (extra) requirements. Non-optional requirements must appear
- ;; before any section is defined.
(cond
- ((or (eof-object? line) (section-header? line))
+ ((eof-object? line)
;; Duplicates can occur, since the same requirement can be
;; listed multiple times with different conditional markers, e.g.
;; pytest >= 3 ; python_version >= "3.3"
;; pytest < 3 ; python_version < "3.3"
- (reverse (delete-duplicates result)))
+ (map (compose reverse delete-duplicates)
+ (list required-deps test-deps)))
((or (string-null? line) (comment? line))
- (loop result))
- (else
+ (loop required-deps test-deps inside-test-section? optional?))
+ ((section-header? line)
+ ;; Encountering a section means that all the requirements
+ ;; listed below are optional. Since we want to pick only the
+ ;; test dependencies from the optional dependencies, we must
+ ;; track those separately.
+ (loop required-deps test-deps (test-section? line) #t))
+ (inside-test-section?
+ (loop required-deps
+ (cons (specification->requirement-name line)
+ test-deps)
+ inside-test-section? optional?))
+ ((not optional?)
(loop (cons (specification->requirement-name line)
- result)))))))))
+ required-deps)
+ test-deps inside-test-section? optional?))
+ (optional?
+ ;; Skip optional items.
+ (loop required-deps test-deps inside-test-section? optional?))
+ (else
+ (warning (G_ "parse-requires.txt reached an unexpected \
+condition on line ~a~%") line))))))))
(define (parse-wheel-metadata metadata)
- "Given METADATA, a Wheel metadata file, return a list of requirement names."
+ "Given METADATA, a Wheel metadata file, return a list of lists of
+requirements.
+
+Refer to the documentation of PARSE-REQUIRES.TXT for a description of the
+returned value."
;; METADATA is a RFC-2822-like, header based file.
(define (requires-dist-header? line)
@@ -201,21 +237,29 @@ requirement names."
;; Return #t if the given LINE is an "extra" requirement.
(string-match "extra == '(.*)'" line))
+ (define (test-requirement? line)
+ (and=> (match:substring (extra? line) 1) test-section?))
+
(call-with-input-file metadata
(lambda (port)
- (let loop ((requirements '()))
+ (let loop ((required-deps '())
+ (test-deps '()))
(let ((line (read-line port)))
- ;; Stop at the first 'Provides-Extra' section: the non-optional
- ;; requirements appear before the optional ones.
(cond
((eof-object? line)
- (reverse (delete-duplicates requirements)))
+ (map (compose reverse delete-duplicates)
+ (list required-deps test-deps)))
((and (requires-dist-header? line) (not (extra? line)))
(loop (cons (specification->requirement-name
(requires-dist-value line))
- requirements)))
+ required-deps)
+ test-deps))
+ ((and (requires-dist-header? line) (test-requirement? line))
+ (loop required-deps
+ (cons (specification->requirement-name (requires-dist-value line))
+ test-deps)))
(else
- (loop requirements))))))))
+ (loop required-deps test-deps)))))))) ;skip line
(define (guess-requirements source-url wheel-url archive)
"Given SOURCE-URL, WHEEL-URL and an ARCHIVE of the package, return a list
@@ -268,37 +312,46 @@ be extracted in a temporary directory."
(()
(warning (G_ "Cannot guess requirements from source archive:\
no requires.txt file found.~%"))
- '())
+ (list '() '()))
(else (parse-requires.txt (first requires.txt-files)))))))
(begin
(warning (G_ "Unsupported archive format; \
cannot determine package dependencies from source archive: ~a~%")
(basename source-url))
- '())))
+ (list '() '()))))
;; First, try to compute the requirements using the wheel, else, fallback to
;; reading the "requires.txt" from the egg-info directory from the source
- ;; tarball.
+ ;; archive.
(or (guess-requirements-from-wheel)
(guess-requirements-from-source)))
(define (compute-inputs source-url wheel-url archive)
- "Given the SOURCE-URL of an already downloaded ARCHIVE, return a list of
-name/variable pairs describing the required inputs of this package. Also
+ "Given the SOURCE-URL and WHEEL-URL of an already downloaded ARCHIVE, return
+a pair of lists, each consisting of a list of name/variable pairs, for the
+propagated inputs and the native inputs, respectively. Also
return the unaltered list of upstream dependency names."
- (let ((dependencies
- (remove (cut string=? "argparse" <>)
- (guess-requirements source-url wheel-url archive))))
- (values (sort
- (map (lambda (input)
- (let ((guix-name (python->package-name input)))
- (list guix-name (list 'unquote (string->symbol guix-name)))))
- dependencies)
- (lambda args
- (match args
- (((a _ ...) (b _ ...))
- (string-ci<? a b)))))
- dependencies)))
+
+ (define (strip-argparse deps)
+ (remove (cut string=? "argparse" <>) deps))
+
+ (define (requirement->package-name/sort deps)
+ (sort
+ (map (lambda (input)
+ (let ((guix-name (python->package-name input)))
+ (list guix-name (list 'unquote (string->symbol guix-name)))))
+ deps)
+ (lambda args
+ (match args
+ (((a _ ...) (b _ ...))
+ (string-ci<? a b))))))
+
+ (define process-requirements
+ (compose requirement->package-name/sort strip-argparse))
+
+ (let ((dependencies (guess-requirements source-url wheel-url archive)))
+ (values (map process-requirements dependencies)
+ (concatenate dependencies))))
(define (make-pypi-sexp name version source-url wheel-url home-page synopsis
description license)
@@ -307,29 +360,31 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE."
(call-with-temporary-output-file
(lambda (temp port)
(and (url-fetch source-url temp)
- (receive (input-package-names upstream-dependency-names)
+ (receive (guix-dependencies upstream-dependencies)
(compute-inputs source-url wheel-url temp)
- (values
- `(package
- (name ,(python->package-name name))
- (version ,version)
- (source (origin
- (method url-fetch)
-
- ;; Sometimes 'pypi-uri' doesn't quite work due to mixed
- ;; cases in NAME, for instance, as is the case with
- ;; "uwsgi". In that case, fall back to a full URL.
- (uri (pypi-uri ,(string-downcase name) version))
- (sha256
- (base32
- ,(guix-hash-url temp)))))
- (build-system python-build-system)
- ,@(maybe-inputs input-package-names)
- (home-page ,home-page)
- (synopsis ,synopsis)
- (description ,description)
- (license ,(license->symbol license)))
- upstream-dependency-names))))))
+ (match guix-dependencies
+ ((required-inputs test-inputs)
+ (values
+ `(package
+ (name ,(python->package-name name))
+ (version ,version)
+ (source (origin
+ (method url-fetch)
+ ;; Sometimes 'pypi-uri' doesn't quite work due to mixed
+ ;; cases in NAME, for instance, as is the case with
+ ;; "uwsgi". In that case, fall back to a full URL.
+ (uri (pypi-uri ,(string-downcase name) version))
+ (sha256
+ (base32
+ ,(guix-hash-url temp)))))
+ (build-system python-build-system)
+ ,@(maybe-inputs required-inputs 'propagated-inputs)
+ ,@(maybe-inputs test-inputs 'native-inputs)
+ (home-page ,home-page)
+ (synopsis ,synopsis)
+ (description ,description)
+ (license ,(license->symbol license)))
+ upstream-dependencies))))))))
(define pypi->guix-package
(memoize
diff --git a/tests/pypi.scm b/tests/pypi.scm
index 8b42c2f071..43d45f1dd8 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -1,6 +1,7 @@
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2014 David Thompson <davet@gnu.org>
;;; Copyright © 2016 Ricardo Wurmus <rekado@elephly.net>
+;;; Copyright © 2019 Maxim Cournoyer <maxim.cournoyer@gmail.com>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -68,11 +69,6 @@ sha1=da9234ee9982d4bbb3c72346a6de940a148ea686"))
(define test-requires.txt "\
# A comment
# A comment after a space
-bar
-baz > 13.37
-")
-
-(define test-requires-with-sections "\
foo ~= 3
bar != 2
@@ -80,12 +76,25 @@ bar != 2
pytest (>=2.5.0)
")
+;; Beaker contains only optional dependencies.
+(define test-requires.txt-beaker "\
+[crypto]
+pycryptopp>=0.5.12
+
+[cryptography]
+cryptography
+
+[testsuite]
+Mock
+coverage
+")
+
(define test-metadata "\
Classifier: Programming Language :: Python :: 3.7
Requires-Dist: baz ~= 3
Requires-Dist: bar != 2
Provides-Extra: test
-pytest (>=2.5.0)
+Requires-Dist: pytest (>=2.5.0) ; extra == 'test'
")
(define test-metadata-with-extras "
@@ -139,25 +148,31 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
'("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip")
(map specification->requirement-name test-specifications))
-(test-equal "parse-requires.txt, with sections"
- '("foo" "bar")
+(test-equal "parse-requires.txt"
+ (list '("foo" "bar") '("pytest"))
(mock ((ice-9 ports) call-with-input-file
call-with-input-string)
- (parse-requires.txt test-requires-with-sections)))
+ (parse-requires.txt test-requires.txt)))
+
+(test-equal "parse-requires.txt - Beaker"
+ (list '() '("Mock" "coverage"))
+ (mock ((ice-9 ports) call-with-input-file
+ call-with-input-string)
+ (parse-requires.txt test-requires.txt-beaker)))
(test-equal "parse-wheel-metadata, with extras"
- '("wrapt" "bar")
+ (list '("wrapt" "bar") '("tox" "bumpversion"))
(mock ((ice-9 ports) call-with-input-file
call-with-input-string)
(parse-wheel-metadata test-metadata-with-extras)))
(test-equal "parse-wheel-metadata, with extras - Jedi"
- '("parso")
+ (list '("parso") '("pytest"))
(mock ((ice-9 ports) call-with-input-file
call-with-input-string)
(parse-wheel-metadata test-metadata-with-extras-jedi)))
-(test-assert "pypi->guix-package"
+(test-assert "pypi->guix-package, no wheel"
;; Replace network resources with sample data.
(mock ((guix import utils) url-fetch
(lambda (url file-name)
@@ -198,7 +213,10 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
('propagated-inputs
('quasiquote
(("python-bar" ('unquote 'python-bar))
- ("python-baz" ('unquote 'python-baz)))))
+ ("python-foo" ('unquote 'python-foo)))))
+ ('native-inputs
+ ('quasiquote
+ (("python-pytest" ('unquote 'python-pytest)))))
('home-page "http://example.com")
('synopsis "summary")
('description "summary")
@@ -219,25 +237,25 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
(begin
(mkdir-p "foo-1.0.0/foo.egg-info/")
(with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
- (lambda ()
- (display "wrong data to make sure we're testing wheels ")))
+ (lambda ()
+ (display "wrong data to make sure we're testing wheels ")))
(parameterize ((current-output-port (%make-void-port "rw+")))
(system* "tar" "czvf" file-name "foo-1.0.0/"))
- (delete-file-recursively "foo-1.0.0")
- (set! test-source-hash
- (call-with-input-file file-name port-sha256))))
+ (delete-file-recursively "foo-1.0.0")
+ (set! test-source-hash
+ (call-with-input-file file-name port-sha256))))
("https://example.com/foo-1.0.0-py2.py3-none-any.whl"
- (begin
- (mkdir "foo-1.0.0.dist-info")
- (with-output-to-file "foo-1.0.0.dist-info/METADATA"
- (lambda ()
- (display test-metadata)))
- (let ((zip-file (string-append file-name ".zip")))
- ;; zip always adds a "zip" extension to the file it creates,
- ;; so we need to rename it.
- (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA")
- (rename-file zip-file file-name))
- (delete-file-recursively "foo-1.0.0.dist-info")))
+ (begin
+ (mkdir "foo-1.0.0.dist-info")
+ (with-output-to-file "foo-1.0.0.dist-info/METADATA"
+ (lambda ()
+ (display test-metadata)))
+ (let ((zip-file (string-append file-name ".zip")))
+ ;; zip always adds a "zip" extension to the file it creates,
+ ;; so we need to rename it.
+ (system* "zip" "-q" zip-file "foo-1.0.0.dist-info/METADATA")
+ (rename-file zip-file file-name))
+ (delete-file-recursively "foo-1.0.0.dist-info")))
(_ (error "Unexpected URL: " url)))))
(mock ((guix http-client) http-fetch
(lambda (url . rest)
@@ -265,6 +283,9 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
('quasiquote
(("python-bar" ('unquote 'python-bar))
("python-baz" ('unquote 'python-baz)))))
+ ('native-inputs
+ ('quasiquote
+ (("python-pytest" ('unquote 'python-pytest)))))
('home-page "http://example.com")
('synopsis "summary")
('description "summary")
generated by cgit v1.2.3 (git 2.25.4) at 2024-10-30 15:19:47 +0000