summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJulien Lepiller <julien@lepiller.eu>2020-02-09 19:47:27 +0100
committerGuix Patches Tester <>2020-02-09 19:26:11 +0000
commitff7e068bf718545fa594481e62f4fc5537c583f8 (patch)
tree53ef5889bad0a75f320158efb6d75b6a255e6106
parent27b09f3ab11a30821a5ce0b071aac1bc6156497d (diff)
downloadpatches-ff7e068bf718545fa594481e62f4fc5537c583f8.tar
patches-ff7e068bf718545fa594481e62f4fc5537c583f8.tar.gz
guix: download: Add partial download support.series-2853
* nix/libstore/build.cc (tryToBuild): Do not remove invalid fixed-output derivations. * guix/build/download.scm (http-fetch): Add a range argument. (url-fetch): Performa partial download if a file already exists.
-rw-r--r--guix/build/download.scm36
-rw-r--r--nix/libstore/build.cc1
2 files changed, 28 insertions, 9 deletions
diff --git a/guix/build/download.scm b/guix/build/download.scm
index 0f2d5f402a..c2b710becc 100644
--- a/guix/build/download.scm
+++ b/guix/build/download.scm
@@ -654,11 +654,13 @@ Return the resulting target URI."
#:query (uri-query ref)
#:fragment (uri-fragment ref)))))
-(define* (http-fetch uri #:key timeout (verify-certificate? #t))
+(define* (http-fetch uri #:key timeout (verify-certificate? #t) range)
"Return an input port containing the data at URI, and the expected number of
bytes available or #f. When TIMEOUT is true, bail out if the connection could
not be established in less than TIMEOUT seconds. When VERIFY-CERTIFICATE? is
-true, verify HTTPS certificates; otherwise simply ignore them."
+true, verify HTTPS certificates; otherwise simply ignore them. When RANGE is
+a number, it is the number of bytes we want to skip from the data at URI;
+otherwise the full document is requested."
(define headers
`(;; Some web sites, such as http://dist.schmorp.de, would block you if
@@ -670,6 +672,10 @@ true, verify HTTPS certificates; otherwise simply ignore them."
;; Acceptable" when not explicitly told that everything is accepted.
(Accept . "*/*")
+ ,@(if range
+ `((Range . ,(string-append "bytes=" (number->string range) "-")))
+ '())
+
;; Basic authentication, if needed.
,@(match (uri-userinfo uri)
((? string? str)
@@ -690,7 +696,8 @@ true, verify HTTPS certificates; otherwise simply ignore them."
((code)
(response-code resp)))
(case code
- ((200) ; OK
+ ((200 ; OK
+ 206) ; Partial content
(values port (response-content-length resp)))
((301 ; moved permanently
302 ; found (redirection)
@@ -703,7 +710,8 @@ true, verify HTTPS certificates; otherwise simply ignore them."
(close connection)
(http-fetch uri
#:timeout timeout
- #:verify-certificate? verify-certificate?)))
+ #:verify-certificate? verify-certificate?
+ #:range range)))
(else
(error "download failed" (uri->string uri)
code (response-reason-phrase resp))))))
@@ -775,10 +783,15 @@ otherwise simply ignore them."
((http https)
(false-if-exception*
(let-values (((port size)
- (http-fetch uri
- #:verify-certificate? verify-certificate?
- #:timeout timeout)))
- (call-with-output-file file
+ (if (file-exists? file)
+ (http-fetch uri
+ #:verify-certificate? verify-certificate?
+ #:timeout timeout
+ #:range (stat:size (stat file)))
+ (http-fetch uri
+ #:verify-certificate? verify-certificate?
+ #:timeout timeout))))
+ (call-with-port (open-file file (if (file-exists? file) "a" "w"))
(lambda (output)
(dump-port* port output
#:buffer-size %http-receive-buffer-size
@@ -817,7 +830,12 @@ otherwise simply ignore them."
(let try ((uri (append uri content-addressed-uris)))
(match uri
((uri tail ...)
- (or (fetch uri file)
+ (or (if (file-exists? file)
+ ;; If the file already exists, we do a partial fetch for the
+ ;; remainder. If something went wrong, we remove the file
+ ;; and try to fetch the whole file instead.
+ (or (fetch uri file) (begin (delete-file file) (fetch uri file)))
+ (fetch uri file))
(try tail)))
(()
(format (current-error-port) "failed to download ~s from ~s~%"
diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc
index 17e92c68a7..176ab40226 100644
--- a/nix/libstore/build.cc
+++ b/nix/libstore/build.cc
@@ -1320,6 +1320,7 @@ void DerivationGoal::tryToBuild()
Path path = i->second.path;
if (worker.store.isValidPath(path)) continue;
if (!pathExists(path)) continue;
+ if (fixedOutput) continue;
debug(format("removing invalid path `%1%'") % path);
deletePath(path);
}