From 43b846f6b1d2189a7c49c43ccdd8868fe1bd6550 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Sat, 11 Mar 2023 19:43:29 +0000 Subject: guix: Improve download-nar. Previously download-nar worked with gzipped nars and queried berlin.guix.gnu.org (also known as ci.guix.gnu.org). ci.guix.gnu.org no longer serves gzipped nars so this is of limited use. This commit changes download-nar to query both the default substitute servers, and queries for lzipped rather than gzipped nars, since those are available from both. * guix/build/download-nar.scm (urls-for-item): Return urls for lzip rather than gzip compression, and from both default substitute servers. The comment about CDN's is no longer relevant. (restore-gzipped-nar): Rename to restore-lzipped-nar and reimplement accordingly. (download-nar): Add progress reporting and switch to use lzip rather than gzip. * guix/cvs-download.scm (cvs-fetch): Replace guile-zlib with guile-lzlib. * guix/git-download.scm (git-fetch): Replace guile-zlib with guile-lzlib. * guix/hg-download.scm (hg-fetch): Replace guile-zlib with guile-lzlib. * guix/android-repo-download.scm (android-repo-fetch): Add guile-lzlib for download-nar --- guix/android-repo-download.scm | 6 +++- guix/build/download-nar.scm | 73 +++++++++++++++--------------------------- guix/cvs-download.scm | 6 ++-- guix/git-download.scm | 6 ++-- guix/hg-download.scm | 6 ++-- 5 files changed, 39 insertions(+), 58 deletions(-) diff --git a/guix/android-repo-download.scm b/guix/android-repo-download.scm index 1c3502e673..544c5c8be2 100644 --- a/guix/android-repo-download.scm +++ b/guix/android-repo-download.scm @@ -81,6 +81,9 @@ generic name if unset." (define guile-json (module-ref (resolve-interface '(gnu packages guile)) 'guile-json-4)) + (define guile-lzlib + (module-ref (resolve-interface '(gnu packages guile)) 'guile-lzlib)) + (define gnutls (module-ref (resolve-interface '(gnu packages tls)) 'gnutls)) @@ -102,7 +105,8 @@ generic name if unset." (define build (with-imported-modules modules - (with-extensions (list gnutls guile-json) ;for (guix swh) + (with-extensions (list gnutls guile-json ;for (guix swh) + guile-lzlib) #~(begin (use-modules (guix build android-repo) (guix build utils) diff --git a/guix/build/download-nar.scm b/guix/build/download-nar.scm index 867f3c10bb..7de447c61a 100644 --- a/guix/build/download-nar.scm +++ b/guix/build/download-nar.scm @@ -20,7 +20,7 @@ #:use-module (guix build download) #:use-module (guix build utils) #:use-module ((guix serialization) #:hide (dump-port*)) - #:autoload (zlib) (call-with-gzip-input-port) + #:autoload (lzlib) (call-with-lzip-input-port) #:use-module (guix progress) #:use-module (web uri) #:use-module (srfi srfi-11) @@ -42,52 +42,21 @@ "Return the fallback nar URL for ITEM--e.g., \"/gnu/store/cabbag3…-foo-1.2-checkout\"." ;; Here we hard-code nar URLs without checking narinfos. That's probably OK - ;; though. Use berlin.guix.gnu.org instead of its ci.guix.gnu.org front end to - ;; avoid sending these requests to CDN providers without user consent. + ;; though. ;; TODO: Use HTTPS? The downside is the extra dependency. - (let ((bases '("http://berlin.guix.gnu.org")) + (let ((bases '("http://bordeaux.guix.gnu.org" + "http://ci.guix.gnu.org")) (item (basename item))) - (append (map (cut string-append <> "/nar/gzip/" item) bases) + (append (map (cut string-append <> "/nar/lzip/" item) bases) (map (cut string-append <> "/nar/" item) bases)))) -(define (restore-gzipped-nar port item size) - "Restore the gzipped nar read from PORT, of SIZE bytes (compressed), to +(define (restore-lzipped-nar port item size) + "Restore the lzipped nar read from PORT, of SIZE bytes (compressed), to ITEM." - ;; Since PORT is typically a non-file port (for instance because 'http-get' - ;; returns a delimited port), create a child process so we're back to a file - ;; port that can be passed to 'call-with-gzip-input-port'. - (match (pipe) - ((input . output) - (match (primitive-fork) - (0 - (dynamic-wind - (const #t) - (lambda () - (close-port output) - (close-port port) - (catch #t - (lambda () - (call-with-gzip-input-port input - (cut restore-file <> item))) - (lambda (key . args) - (print-exception (current-error-port) - (stack-ref (make-stack #t) 1) - key args) - (primitive-exit 1)))) - (lambda () - (primitive-exit 0)))) - (child - (close-port input) - (dump-port* port output - #:reporter (progress-reporter/file item size - #:abbreviation - store-path-abbreviation)) - (close-port output) - (newline) - (match (waitpid child) - ((_ . status) - (unless (zero? status) - (error "nar decompression failed" status))))))))) + (call-with-lzip-input-port port + (lambda (decompressed-port) + (restore-file decompressed-port + item)))) (define (download-nar item) "Download and extract the normalized archive for ITEM. Return #t on @@ -109,17 +78,25 @@ success, #f otherwise." (values #f #f))))) (if (not port) (loop rest) - (begin + (let* ((reporter (progress-reporter/file + url + size + (current-error-port) + #:abbreviation nar-uri-abbreviation)) + (port-with-progress + (progress-report-port reporter port + #:download-size size))) (if size (format #t "Downloading from ~a (~,2h MiB)...~%" url (/ size (expt 2 20.))) (format #t "Downloading from ~a...~%" url)) - (if (string-contains url "/gzip") - (restore-gzipped-nar port item size) + (if (string-contains url "/lzip") + (restore-lzipped-nar port-with-progress + item + size) (begin - ;; FIXME: Add progress report. - (restore-file port item) - (close-port port))) + (restore-file port-with-progress + item))) #t)))) (() #f)))) diff --git a/guix/cvs-download.scm b/guix/cvs-download.scm index 943d971622..22af2461e9 100644 --- a/guix/cvs-download.scm +++ b/guix/cvs-download.scm @@ -61,8 +61,8 @@ "Return a fixed-output derivation that fetches REF, a object. The output is expected to have recursive hash HASH of type HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." - (define guile-zlib - (module-ref (resolve-interface '(gnu packages guile)) 'guile-zlib)) + (define guile-lzlib + (module-ref (resolve-interface '(gnu packages guile)) 'guile-lzlib)) (define guile-json (module-ref (resolve-interface '(gnu packages guile)) 'guile-json-4)) @@ -77,7 +77,7 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." (define build (with-imported-modules modules (with-extensions (list guile-json gnutls ;for (guix swh) - guile-zlib) + guile-lzlib) #~(begin (use-modules (guix build cvs) (guix build download-nar)) diff --git a/guix/git-download.scm b/guix/git-download.scm index 3dc306e39d..027ef47468 100644 --- a/guix/git-download.scm +++ b/guix/git-download.scm @@ -101,8 +101,8 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." (define guile-json (module-ref (resolve-interface '(gnu packages guile)) 'guile-json-4)) - (define guile-zlib - (module-ref (resolve-interface '(gnu packages guile)) 'guile-zlib)) + (define guile-lzlib + (module-ref (resolve-interface '(gnu packages guile)) 'guile-lzlib)) (define gnutls (module-ref (resolve-interface '(gnu packages tls)) 'gnutls)) @@ -123,7 +123,7 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." (define build (with-imported-modules modules (with-extensions (list guile-json gnutls ;for (guix swh) - guile-zlib) + guile-lzlib) #~(begin (use-modules (guix build git) (guix build utils) diff --git a/guix/hg-download.scm b/guix/hg-download.scm index 382c34922a..8d71553bee 100644 --- a/guix/hg-download.scm +++ b/guix/hg-download.scm @@ -73,8 +73,8 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." ("tar" ,(module-ref (resolve-interface '(gnu packages base)) 'tar)))) - (define guile-zlib - (module-ref (resolve-interface '(gnu packages guile)) 'guile-zlib)) + (define guile-lzlib + (module-ref (resolve-interface '(gnu packages guile)) 'guile-lzlib)) (define guile-json (module-ref (resolve-interface '(gnu packages guile)) 'guile-json-4)) @@ -91,7 +91,7 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." (define build (with-imported-modules modules (with-extensions (list guile-json gnutls ;for (guix swh) - guile-zlib) + guile-lzlib) #~(begin (use-modules (guix build hg) (guix build utils) ;for `set-path-environment-variable' -- cgit v1.2.3