diff options
| -rw-r--r-- | guix/import/pypi.scm | 90 | ||||
| -rw-r--r-- | tests/pypi.scm | 60 |
2 files changed, 101 insertions, 49 deletions
diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index a6106ab4ec..7cf1e92101 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -21,9 +21,7 @@ ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. (define-module (guix import pypi) - #:use-module (ice-9 binary-ports) #:use-module (ice-9 match) - #:use-module (ice-9 pretty-print) #:use-module (ice-9 regex) #:use-module (ice-9 receive) #:use-module ((ice-9 rdelim) #:select (read-line)) @@ -31,9 +29,6 @@ #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) - #:use-module (rnrs bytevectors) - #:use-module (json) - #:use-module (web uri) #:use-module (guix ui) #:use-module (guix utils) #:use-module ((guix build utils) @@ -49,6 +44,7 @@ #:use-module ((guix licenses) #:prefix license:) #:use-module (guix build-system python) #:export (parse-requires.txt + parse-wheel-metadata specification->requirement-name guix-package->pypi-name pypi-recursive-import @@ -177,18 +173,49 @@ requirement names." ;; Stop when a section is encountered, as sections contain optional ;; (extra) requirements. Non-optional requirements must appear ;; before any section is defined. - (if (or (eof-object? line) (section-header? line)) - ;; Duplicates can occur, since the same requirement can be - ;; listed multiple times with different conditional markers, e.g. - ;; pytest >= 3 ; python_version >= "3.3" - ;; pytest < 3 ; python_version < "3.3" - (reverse (delete-duplicates result)) - (cond - ((or (string-null? line) (comment? line)) - (loop result)) - (else - (loop (cons (specification->requirement-name line) - result)))))))))) + (cond + ((or (eof-object? line) (section-header? line)) + ;; Duplicates can occur, since the same requirement can be + ;; listed multiple times with different conditional markers, e.g. + ;; pytest >= 3 ; python_version >= "3.3" + ;; pytest < 3 ; python_version < "3.3" + (reverse (delete-duplicates result))) + ((or (string-null? line) (comment? line)) + (loop result)) + (else + (loop (cons (specification->requirement-name line) + result))))))))) + +(define (parse-wheel-metadata metadata) + "Given METADATA, a Wheel metadata file, return a list of requirement names." + ;; METADATA is a RFC-2822-like, header based file. + + (define (requires-dist-header? line) + ;; Return #t if the given LINE is a Requires-Dist header. + (string-match "^Requires-Dist: " line)) + + (define (requires-dist-value line) + (string-drop line (string-length "Requires-Dist: "))) + + (define (extra? line) + ;; Return #t if the given LINE is an "extra" requirement. + (string-match "extra == '(.*)'" line)) + + (call-with-input-file metadata + (lambda (port) + (let loop ((requirements '())) + (let ((line (read-line port))) + ;; Stop at the first 'Provides-Extra' section: the non-optional + ;; requirements appear before the optional ones. + (cond + ((eof-object? line) + (reverse (delete-duplicates requirements))) + ((and (requires-dist-header? line) (not (extra? line))) + (loop (cons (specification->requirement-name + (requires-dist-value line)) + requirements))) + (else + (loop requirements)))))))) (define (guess-requirements source-url wheel-url archive) "Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list @@ -197,25 +224,18 @@ be extracted in a temporary directory." (define (read-wheel-metadata wheel-archive) ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's - ;; requirements. + ;; requirements, or #f if the metadata file contained therein couldn't be + ;; extracted. (let* ((dirname (wheel-url->extracted-directory wheel-url)) - (json-file (string-append dirname "/metadata.json"))) - (and (zero? (system* "unzip" "-q" wheel-archive json-file)) - (dynamic-wind - (const #t) - (lambda () - (call-with-input-file json-file - (lambda (port) - (let* ((metadata (json->scm port)) - (run_requires (hash-ref metadata "run_requires")) - (requirements (if run_requires - (hash-ref (list-ref run_requires 0) - "requires") - '()))) - (map specification->requirement-name requirements))))) - (lambda () - (delete-file json-file) - (rmdir dirname)))))) + (metadata (string-append dirname "/METADATA"))) + (call-with-temporary-directory + (lambda (dir) + (if (zero? (system* "unzip" "-q" wheel-archive "-d" dir metadata)) + (parse-wheel-metadata (string-append dir "/" metadata)) + (begin + (warning + (G_ "Failed to extract file: ~a from wheel.~%") metadata) + #f)))))) (define (guess-requirements-from-wheel) ;; Return the package's requirements using the wheel, or #f if an error diff --git a/tests/pypi.scm b/tests/pypi.scm index b45d2c9d2f..8b42c2f071 100644 --- a/tests/pypi.scm +++ b/tests/pypi.scm @@ -22,6 +22,7 @@ #:use-module (guix base32) #:use-module (guix memoization) #:use-module (gcrypt hash) + #:use-module (guix memoization) #:use-module (guix tests) #:use-module (guix build-system python) #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p)) @@ -79,17 +80,33 @@ bar != 2 pytest (>=2.5.0) ") -(define test-metadata - "{ - \"run_requires\": [ - { - \"requires\": [ - \"bar\", - \"baz (>13.37)\" - ] - } - ] -}") +(define test-metadata "\ +Classifier: Programming Language :: Python :: 3.7 +Requires-Dist: baz ~= 3 +Requires-Dist: bar != 2 +Provides-Extra: test +pytest (>=2.5.0) +") + +(define test-metadata-with-extras " +Classifier: Programming Language :: Python :: 3.7 +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Requires-Dist: wrapt (<2,>=1) +Requires-Dist: bar + +Provides-Extra: dev +Requires-Dist: tox ; extra == 'dev' +Requires-Dist: bumpversion (<1) ; extra == 'dev' +") + +;;; Provides-Extra can appear before Requires-Dist. +(define test-metadata-with-extras-jedi "\ +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Provides-Extra: testing +Requires-Dist: parso (>=0.3.0) +Provides-Extra: testing +Requires-Dist: pytest (>=3.1.0); extra == 'testing' +") (test-begin "pypi") @@ -128,6 +145,18 @@ pytest (>=2.5.0) call-with-input-string) (parse-requires.txt test-requires-with-sections))) +(test-equal "parse-wheel-metadata, with extras" + '("wrapt" "bar") + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-wheel-metadata test-metadata-with-extras))) + +(test-equal "parse-wheel-metadata, with extras - Jedi" + '("parso") + (mock ((ice-9 ports) call-with-input-file + call-with-input-string) + (parse-wheel-metadata test-metadata-with-extras-jedi))) + (test-assert "pypi->guix-package" ;; Replace network resources with sample data. (mock ((guix import utils) url-fetch @@ -191,7 +220,7 @@ pytest (>=2.5.0) (mkdir-p "foo-1.0.0/foo.egg-info/") (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" (lambda () - (display test-requires.txt))) + (display "wrong data to make sure we're testing wheels "))) (parameterize ((current-output-port (%make-void-port "rw+"))) (system* "tar" "czvf" file-name "foo-1.0.0/")) (delete-file-recursively "foo-1.0.0") @@ -200,13 +229,13 @@ pytest (>=2.5.0) ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" (begin (mkdir "foo-1.0.0.dist-info") - (with-output-to-file "foo-1.0.0.dist-info/metadata.json" + (with-output-to-file "foo-1.0.0.dist-info/METADATA" (lambda () (display test-metadata))) (let ((zip-file (string-append file-name ".zip"))) ;; zip always adds a "zip" extension to the file it creates, ;; so we need to rename it. - (system* "zip" zip-file "foo-1.0.0.dist-info/metadata.json") + (system* "zip" zip-file "foo-1.0.0.dist-info/METADATA") (rename-file zip-file file-name)) (delete-file-recursively "foo-1.0.0.dist-info"))) (_ (error "Unexpected URL: " url))))) @@ -218,6 +247,9 @@ pytest (>=2.5.0) (string-length test-json))) ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) (_ (error "Unexpected URL: " url))))) + ;; Not clearing the memoization cache here would mean returning the value + ;; computed in the previous test. + (invalidate-memoization! pypi->guix-package) (match (pypi->guix-package "foo") (('package ('name "python-foo") |