From 472a0e82a52a3d5d841e1dfad6b13e26082a5750 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sat, 13 Nov 2021 21:47:15 +0100 Subject: daemon: Do not deduplicate files smaller than 8 KiB. Files smaller than 8 KiB typically represent ~70% of the entries in /gnu/store/.links but only contribute to ~4% of the space savings afforded by deduplication. Not considering these files for deduplication speeds up file insertion in the store and, more importantly, leaves 'removeUnusedLinks' with fewer entries to traverse, thereby speeding it up proportionally. Partly fixes . * config-daemon.ac: Remove symlink hard link check and CAN_LINK_SYMLINK definition. * guix/store/deduplication.scm (%deduplication-minimum-size): New variable. (deduplicate)[loop]: Do not recurse when FILE's size is below %DEDUPLICATION-MINIMUM-SIZE. (dump-port): New procedure. (dump-file/deduplicate)[hash]: Turn into... [dump-and-compute-hash]: ... this thunk. Call 'deduplicate' only when SIZE is greater than %DEDUPLICATION-MINIMUM-SIZE; otherwise call 'dump-port'. * nix/libstore/gc.cc (LocalStore::removeUnusedLinks): Drop files where st.st_size < deduplicationMinSize. * nix/libstore/local-store.hh (deduplicationMinSize): New declaration. * nix/libstore/optimise-store.cc (deduplicationMinSize): New variable. (LocalStore::optimisePath_): Return when PATH is a symlink or smaller than 'deduplicationMinSize'. * tests/derivations.scm ("identical files are deduplicated"): Produce files bigger than %DEDUPLICATION-MINIMUM-SIZE. * tests/nar.scm ("restore-file-set with directories (signed, valid)"): Likewise. * tests/store-deduplication.scm ("deduplicate, below %deduplication-minimum-size"): New test. ("deduplicate", "deduplicate, ENOSPC"): Produce files bigger than %DEDUPLICATION-MINIMUM-SIZE. * tests/store.scm ("substitute, deduplication"): Likewise. --- tests/derivations.scm | 14 +++++++++----- tests/nar.scm | 7 ++++--- tests/store-deduplication.scm | 41 +++++++++++++++++++++++++++++++++++------ tests/store.scm | 4 +++- 4 files changed, 51 insertions(+), 15 deletions(-) (limited to 'tests') diff --git a/tests/derivations.scm b/tests/derivations.scm index cd165d1be6..0775719ea3 100644 --- a/tests/derivations.scm +++ b/tests/derivations.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès +;;; Copyright © 2012-2021 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -170,11 +170,15 @@ (define prefix-len (string-length dir)) #f)))) (test-assert "identical files are deduplicated" - (let* ((build1 (add-text-to-store %store "one.sh" - "echo hello, world > \"$out\"\n" + ;; Note: DATA must be longer than %DEDUPLICATION-MINIMUM-SIZE. + (let* ((data (make-string 9000 #\a)) + (build1 (add-text-to-store %store "one.sh" + (string-append "echo -n " data + " > \"$out\"\n") '())) (build2 (add-text-to-store %store "two.sh" - "# Hey!\necho hello, world > \"$out\"\n" + (string-append "# Hey!\necho -n " + data " > \"$out\"\n") '())) (drv1 (derivation %store "foo" %bash `(,build1) @@ -187,7 +191,7 @@ (define prefix-len (string-length dir)) (file2 (derivation->output-path drv2))) (and (valid-path? %store file1) (valid-path? %store file2) (string=? (call-with-input-file file1 get-string-all) - "hello, world\n") + data) (= (stat:ino (lstat file1)) (stat:ino (lstat file2)))))))) diff --git a/tests/nar.scm b/tests/nar.scm index ba4881caaa..98752f2088 100644 --- a/tests/nar.scm +++ b/tests/nar.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès +;;; Copyright © 2012-2021 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -486,8 +486,9 @@ (define-values (port get-bytevector) ;; their mtime and permissions were not reset. Ensure that this bug is ;; gone. (with-store store - (let* ((text1 (random-text)) - (text2 (random-text)) + ;; Note: TEXT1 and TEXT2 must be longer than %DEDUPLICATION-MINIMUM-SIZE. + (let* ((text1 (string-concatenate (make-list 200 (random-text)))) + (text2 (string-concatenate (make-list 200 (random-text)))) (tree `("tree" directory ("a" regular (data ,text1)) ("b" directory diff --git a/tests/store-deduplication.scm b/tests/store-deduplication.scm index b1c2d93bbd..2950fbc1a3 100644 --- a/tests/store-deduplication.scm +++ b/tests/store-deduplication.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2018, 2020 Ludovic Courtès +;;; Copyright © 2018, 2020-2021 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -30,13 +30,40 @@ (define-module (test-store-deduplication) (test-begin "store-deduplication") +(test-equal "deduplicate, below %deduplication-minimum-size" + (list #t (make-list 5 1)) + + (call-with-temporary-directory + (lambda (store) + ;; Note: DATA must be longer than %DEDUPLICATION-MINIMUM-SIZE. + (let ((data "Hello, world!") + (identical (map (lambda (n) + (string-append store "/" (number->string n) + "/a/b/c")) + (iota 5)))) + (for-each (lambda (file) + (mkdir-p (dirname file)) + (call-with-output-file file + (lambda (port) + (put-bytevector port (string->utf8 data))))) + identical) + + (deduplicate store (nar-sha256 store) #:store store) + + ;; (system (string-append "ls -lRia " store)) + (list (= (length (delete-duplicates + (map (compose stat:ino stat) identical))) + (length identical)) + (map (compose stat:nlink stat) identical)))))) + (test-equal "deduplicate" (cons* #t #f ;inode comparisons 2 (make-list 5 6)) ;'nlink' values (call-with-temporary-directory (lambda (store) - (let ((data (string->utf8 "Hello, world!")) + ;; Note: DATA must be longer than %DEDUPLICATION-MINIMUM-SIZE. + (let ((data (string-concatenate (make-list 1000 "Hello, world!"))) (identical (map (lambda (n) (string-append store "/" (number->string n) "/a/b/c")) @@ -46,7 +73,7 @@ (define-module (test-store-deduplication) (mkdir-p (dirname file)) (call-with-output-file file (lambda (port) - (put-bytevector port data)))) + (put-bytevector port (string->utf8 data))))) identical) ;; Make the parent of IDENTICAL read-only. This should not prevent ;; deduplication from inserting its hard link. @@ -54,7 +81,7 @@ (define-module (test-store-deduplication) (call-with-output-file unique (lambda (port) - (put-bytevector port (string->utf8 "This is unique.")))) + (put-bytevector port (string->utf8 (string-reverse data))))) (deduplicate store (nar-sha256 store) #:store store) @@ -77,8 +104,10 @@ (define-module (test-store-deduplication) (lambda (store) (let ((true-link link) (links 0) - (data1 (string->utf8 "Hello, world!")) - (data2 (string->utf8 "Hi, world!")) + (data1 (string->utf8 + (string-concatenate (make-list 1000 "Hello, world!")))) + (data2 (string->utf8 + (string-concatenate (make-list 1000 "Hi, world!")))) (identical (map (lambda (n) (string-append store "/" (number->string n) "/a/b/c")) diff --git a/tests/store.scm b/tests/store.scm index 2150a0048c..5c9f651d6c 100644 --- a/tests/store.scm +++ b/tests/store.scm @@ -759,7 +759,9 @@ (define lst (test-assert "substitute, deduplication" (with-store s - (let* ((c (random-text)) ; contents of the output + ;; Note: C must be longer than %DEDUPLICATION-MINIMUM-SIZE. + (let* ((c (string-concatenate + (make-list 200 (random-text)))) ; contents of the output (g (package-derivation s %bootstrap-guile)) (d1 (build-expression->derivation s "substitute-me" `(begin ,c (exit 1)) -- cgit v1.2.3