aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMaxim Cournoyer <maxim.cournoyer@gmail.com>2019-03-28 00:26:03 -0400
committerMaxim Cournoyer <maxim.cournoyer@gmail.com>2019-07-02 10:08:00 +0900
commitf0190a5dcdc1882e5a058dc7f12bc7ccb166c63d (patch)
treee526adbc703cf8f92a74299703d8553d85795815
parentc799ad7276dcc771912f4fc7edca400ea48b8ec2 (diff)
downloadguix-f0190a5dcdc1882e5a058dc7f12bc7ccb166c63d.tar
guix-f0190a5dcdc1882e5a058dc7f12bc7ccb166c63d.tar.gz
import: pypi: Parse wheel METADATA instead of metadata.json.
With newer Wheel releases, there is no more metadata.json file; the METADATA file should be used instead (see: https://github.com/pypa/wheel/issues/195). This change updates our PyPI importer so that it uses the latter. * guix/import/pypi.scm (define-module): Remove unnecessary modules and export the PARSE-WHEEL-METADATA procedure. (parse-wheel-metadata): Add procedure. (guess-requirements): Use it. * tests/pypi.scm (test-metadata): Test it.
-rw-r--r--guix/import/pypi.scm90
-rw-r--r--tests/pypi.scm60
2 files changed, 101 insertions, 49 deletions
diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm
index a6106ab4ec..7cf1e92101 100644
--- a/guix/import/pypi.scm
+++ b/guix/import/pypi.scm
@@ -21,9 +21,7 @@
;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
(define-module (guix import pypi)
- #:use-module (ice-9 binary-ports)
#:use-module (ice-9 match)
- #:use-module (ice-9 pretty-print)
#:use-module (ice-9 regex)
#:use-module (ice-9 receive)
#:use-module ((ice-9 rdelim) #:select (read-line))
@@ -31,9 +29,6 @@
#:use-module (srfi srfi-26)
#:use-module (srfi srfi-34)
#:use-module (srfi srfi-35)
- #:use-module (rnrs bytevectors)
- #:use-module (json)
- #:use-module (web uri)
#:use-module (guix ui)
#:use-module (guix utils)
#:use-module ((guix build utils)
@@ -49,6 +44,7 @@
#:use-module ((guix licenses) #:prefix license:)
#:use-module (guix build-system python)
#:export (parse-requires.txt
+ parse-wheel-metadata
specification->requirement-name
guix-package->pypi-name
pypi-recursive-import
@@ -177,18 +173,49 @@ requirement names."
;; Stop when a section is encountered, as sections contain optional
;; (extra) requirements. Non-optional requirements must appear
;; before any section is defined.
- (if (or (eof-object? line) (section-header? line))
- ;; Duplicates can occur, since the same requirement can be
- ;; listed multiple times with different conditional markers, e.g.
- ;; pytest >= 3 ; python_version >= "3.3"
- ;; pytest < 3 ; python_version < "3.3"
- (reverse (delete-duplicates result))
- (cond
- ((or (string-null? line) (comment? line))
- (loop result))
- (else
- (loop (cons (specification->requirement-name line)
- result))))))))))
+ (cond
+ ((or (eof-object? line) (section-header? line))
+ ;; Duplicates can occur, since the same requirement can be
+ ;; listed multiple times with different conditional markers, e.g.
+ ;; pytest >= 3 ; python_version >= "3.3"
+ ;; pytest < 3 ; python_version < "3.3"
+ (reverse (delete-duplicates result)))
+ ((or (string-null? line) (comment? line))
+ (loop result))
+ (else
+ (loop (cons (specification->requirement-name line)
+ result)))))))))
+
+(define (parse-wheel-metadata metadata)
+ "Given METADATA, a Wheel metadata file, return a list of requirement names."
+ ;; METADATA is a RFC-2822-like, header based file.
+
+ (define (requires-dist-header? line)
+ ;; Return #t if the given LINE is a Requires-Dist header.
+ (string-match "^Requires-Dist: " line))
+
+ (define (requires-dist-value line)
+ (string-drop line (string-length "Requires-Dist: ")))
+
+ (define (extra? line)
+ ;; Return #t if the given LINE is an "extra" requirement.
+ (string-match "extra == '(.*)'" line))
+
+ (call-with-input-file metadata
+ (lambda (port)
+ (let loop ((requirements '()))
+ (let ((line (read-line port)))
+ ;; Stop at the first 'Provides-Extra' section: the non-optional
+ ;; requirements appear before the optional ones.
+ (cond
+ ((eof-object? line)
+ (reverse (delete-duplicates requirements)))
+ ((and (requires-dist-header? line) (not (extra? line)))
+ (loop (cons (specification->requirement-name
+ (requires-dist-value line))
+ requirements)))
+ (else
+ (loop requirements))))))))
(define (guess-requirements source-url wheel-url archive)
"Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list
@@ -197,25 +224,18 @@ be extracted in a temporary directory."
(define (read-wheel-metadata wheel-archive)
;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
- ;; requirements.
+ ;; requirements, or #f if the metadata file contained therein couldn't be
+ ;; extracted.
(let* ((dirname (wheel-url->extracted-directory wheel-url))
- (json-file (string-append dirname "/metadata.json")))
- (and (zero? (system* "unzip" "-q" wheel-archive json-file))
- (dynamic-wind
- (const #t)
- (lambda ()
- (call-with-input-file json-file
- (lambda (port)
- (let* ((metadata (json->scm port))
- (run_requires (hash-ref metadata "run_requires"))
- (requirements (if run_requires
- (hash-ref (list-ref run_requires 0)
- "requires")
- '())))
- (map specification->requirement-name requirements)))))
- (lambda ()
- (delete-file json-file)
- (rmdir dirname))))))
+ (metadata (string-append dirname "/METADATA")))
+ (call-with-temporary-directory
+ (lambda (dir)
+ (if (zero? (system* "unzip" "-q" wheel-archive "-d" dir metadata))
+ (parse-wheel-metadata (string-append dir "/" metadata))
+ (begin
+ (warning
+ (G_ "Failed to extract file: ~a from wheel.~%") metadata)
+ #f))))))
(define (guess-requirements-from-wheel)
;; Return the package's requirements using the wheel, or #f if an error
diff --git a/tests/pypi.scm b/tests/pypi.scm
index b45d2c9d2f..8b42c2f071 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -22,6 +22,7 @@
#:use-module (guix base32)
#:use-module (guix memoization)
#:use-module (gcrypt hash)
+ #:use-module (guix memoization)
#:use-module (guix tests)
#:use-module (guix build-system python)
#:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p))
@@ -79,17 +80,33 @@ bar != 2
pytest (>=2.5.0)
")
-(define test-metadata
- "{
- \"run_requires\": [
- {
- \"requires\": [
- \"bar\",
- \"baz (>13.37)\"
- ]
- }
- ]
-}")
+(define test-metadata "\
+Classifier: Programming Language :: Python :: 3.7
+Requires-Dist: baz ~= 3
+Requires-Dist: bar != 2
+Provides-Extra: test
+pytest (>=2.5.0)
+")
+
+(define test-metadata-with-extras "
+Classifier: Programming Language :: Python :: 3.7
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Requires-Dist: wrapt (<2,>=1)
+Requires-Dist: bar
+
+Provides-Extra: dev
+Requires-Dist: tox ; extra == 'dev'
+Requires-Dist: bumpversion (<1) ; extra == 'dev'
+")
+
+;;; Provides-Extra can appear before Requires-Dist.
+(define test-metadata-with-extras-jedi "\
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Provides-Extra: testing
+Requires-Dist: parso (>=0.3.0)
+Provides-Extra: testing
+Requires-Dist: pytest (>=3.1.0); extra == 'testing'
+")
(test-begin "pypi")
@@ -128,6 +145,18 @@ pytest (>=2.5.0)
call-with-input-string)
(parse-requires.txt test-requires-with-sections)))
+(test-equal "parse-wheel-metadata, with extras"
+ '("wrapt" "bar")
+ (mock ((ice-9 ports) call-with-input-file
+ call-with-input-string)
+ (parse-wheel-metadata test-metadata-with-extras)))
+
+(test-equal "parse-wheel-metadata, with extras - Jedi"
+ '("parso")
+ (mock ((ice-9 ports) call-with-input-file
+ call-with-input-string)
+ (parse-wheel-metadata test-metadata-with-extras-jedi)))
+
(test-assert "pypi->guix-package"
;; Replace network resources with sample data.
(mock ((guix import utils) url-fetch
@@ -191,7 +220,7 @@ pytest (>=2.5.0)
(mkdir-p "foo-1.0.0/foo.egg-info/")
(with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
(lambda ()
- (display test-requires.txt)))
+ (display "wrong data to make sure we're testing wheels ")))
(parameterize ((current-output-port (%make-void-port "rw+")))
(system* "tar" "czvf" file-name "foo-1.0.0/"))
(delete-file-recursively "foo-1.0.0")
@@ -200,13 +229,13 @@ pytest (>=2.5.0)
("https://example.com/foo-1.0.0-py2.py3-none-any.whl"
(begin
(mkdir "foo-1.0.0.dist-info")
- (with-output-to-file "foo-1.0.0.dist-info/metadata.json"
+ (with-output-to-file "foo-1.0.0.dist-info/METADATA"
(lambda ()
(display test-metadata)))
(let ((zip-file (string-append file-name ".zip")))
;; zip always adds a "zip" extension to the file it creates,
;; so we need to rename it.
- (system* "zip" zip-file "foo-1.0.0.dist-info/metadata.json")
+ (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA")
(rename-file zip-file file-name))
(delete-file-recursively "foo-1.0.0.dist-info")))
(_ (error "Unexpected URL: " url)))))
@@ -218,6 +247,9 @@ pytest (>=2.5.0)
(string-length test-json)))
("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
(_ (error "Unexpected URL: " url)))))
+ ;; Not clearing the memoization cache here would mean returning the value
+ ;; computed in the previous test.
+ (invalidate-memoization! pypi->guix-package)
(match (pypi->guix-package "foo")
(('package
('name "python-foo")