aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLeo Famulari <leo@famulari.name>2019-05-14 18:38:23 -0400
committerLeo Famulari <leo@famulari.name>2019-05-17 07:57:55 -0400
commit08eafef8650922681bad10d9f5a3f92ff91616b6 (patch)
tree17da6003d3a98ce0b42a2b0c6a0149b0bcc35583
parent7211073069d9218074cc90f9ee2c9d2a88543a4e (diff)
downloadguix-08eafef8650922681bad10d9f5a3f92ff91616b6.tar
guix-08eafef8650922681bad10d9f5a3f92ff91616b6.tar.gz
gnu: Borg: Fix a hang in the test suite.
* gnu/packages/patches/borg-fix-hard-link-preloading.patch: New file. * gnu/local.mk (dist_patch_DATA): Add it. * gnu/packages/backup.scm (borg)[source]: Use it.
-rw-r--r--gnu/local.mk1
-rw-r--r--gnu/packages/backup.scm1
-rw-r--r--gnu/packages/patches/borg-fix-hard-link-preloading.patch157
3 files changed, 159 insertions, 0 deletions
diff --git a/gnu/local.mk b/gnu/local.mk
index e76e387e42..ac55e1dd28 100644
--- a/gnu/local.mk
+++ b/gnu/local.mk
@@ -686,6 +686,7 @@ dist_patch_DATA = \
%D%/packages/patches/blender-2.79-newer-ffmpeg.patch \
%D%/packages/patches/blender-2.79-python-3.7-fix.patch \
%D%/packages/patches/boost-fix-icu-build.patch \
+ %D%/packages/patches/borg-fix-hard-link-preloading.patch \
%D%/packages/patches/byobu-writable-status.patch \
%D%/packages/patches/calibre-no-updates-dialog.patch \
%D%/packages/patches/calibre-remove-test-bs4.patch \
diff --git a/gnu/packages/backup.scm b/gnu/packages/backup.scm
index 97c7499c1c..b15c15ad46 100644
--- a/gnu/packages/backup.scm
+++ b/gnu/packages/backup.scm
@@ -519,6 +519,7 @@ detection, and lossless compression.")
(sha256
(base32
"0x95nhv4h34m8cxycbwc4xdz350saaxlgh727b23bgn4ci7gh3vx"))
+ (patches (search-patches "borg-fix-hard-link-preloading.patch"))
(modules '((guix build utils)))
(snippet
'(begin
diff --git a/gnu/packages/patches/borg-fix-hard-link-preloading.patch b/gnu/packages/patches/borg-fix-hard-link-preloading.patch
new file mode 100644
index 0000000000..92a4e22674
--- /dev/null
+++ b/gnu/packages/patches/borg-fix-hard-link-preloading.patch
@@ -0,0 +1,157 @@
+Fix a bug that would cause the test suite to hang:
+
+https://github.com/borgbackup/borg/issues/4350
+
+Patch copied from upstream source repository:
+
+https://github.com/borgbackup/borg/commit/18242ab9e2f26c450b8507aa1d5eceadab8ad027
+
+From 18242ab9e2f26c450b8507aa1d5eceadab8ad027 Mon Sep 17 00:00:00 2001
+From: Thomas Waldmann <tw@waldmann-edv.de>
+Date: Thu, 2 May 2019 21:02:26 +0200
+Subject: [PATCH] preload chunks for hardlink slaves w/o preloaded master,
+ fixes #4350
+
+also split the hardlink extraction test into 2 tests.
+
+(cherry picked from commit f33f318d816505161d1449a02ddfdeb97d6fe80a)
+---
+ src/borg/archive.py | 42 +++++++++++++++++++++++++++++-----
+ src/borg/archiver.py | 5 ++--
+ src/borg/testsuite/archiver.py | 20 +++++++++-------
+ 3 files changed, 51 insertions(+), 16 deletions(-)
+
+diff --git a/src/borg/archive.py b/src/borg/archive.py
+index adc1f42c..0793672a 100644
+--- a/src/borg/archive.py
++++ b/src/borg/archive.py
+@@ -192,7 +192,7 @@ def __init__(self, repository, key):
+ self.repository = repository
+ self.key = key
+
+- def unpack_many(self, ids, filter=None, preload=False):
++ def unpack_many(self, ids, filter=None, partial_extract=False, preload=False, hardlink_masters=None):
+ """
+ Return iterator of items.
+
+@@ -209,12 +209,40 @@ def unpack_many(self, ids, filter=None, preload=False):
+ for item in items:
+ if 'chunks' in item:
+ item.chunks = [ChunkListEntry(*e) for e in item.chunks]
++
++ def preload(chunks):
++ self.repository.preload([c.id for c in chunks])
++
+ if filter:
+ items = [item for item in items if filter(item)]
++
+ if preload:
+- for item in items:
+- if 'chunks' in item:
+- self.repository.preload([c.id for c in item.chunks])
++ if filter and partial_extract:
++ # if we do only a partial extraction, it gets a bit
++ # complicated with computing the preload items: if a hardlink master item is not
++ # selected (== not extracted), we will still need to preload its chunks if a
++ # corresponding hardlink slave is selected (== is extracted).
++ # due to a side effect of the filter() call, we now have hardlink_masters dict populated.
++ masters_preloaded = set()
++ for item in items:
++ if 'chunks' in item: # regular file, maybe a hardlink master
++ preload(item.chunks)
++ # if this is a hardlink master, remember that we already preloaded it:
++ if 'source' not in item and hardlinkable(item.mode) and item.get('hardlink_master', True):
++ masters_preloaded.add(item.path)
++ elif 'source' in item and hardlinkable(item.mode): # hardlink slave
++ source = item.source
++ if source not in masters_preloaded:
++ # we only need to preload *once* (for the 1st selected slave)
++ chunks, _ = hardlink_masters[source]
++ preload(chunks)
++ masters_preloaded.add(source)
++ else:
++ # easy: we do not have a filter, thus all items are selected, thus we need to preload all chunks.
++ for item in items:
++ if 'chunks' in item:
++ preload(item.chunks)
++
+ for item in items:
+ yield item
+
+@@ -433,8 +461,10 @@ def item_filter(self, item, filter=None):
+ return False
+ return filter(item) if filter else True
+
+- def iter_items(self, filter=None, preload=False):
+- for item in self.pipeline.unpack_many(self.metadata.items, preload=preload,
++ def iter_items(self, filter=None, partial_extract=False, preload=False, hardlink_masters=None):
++ assert not (filter and partial_extract and preload) or hardlink_masters is not None
++ for item in self.pipeline.unpack_many(self.metadata.items, partial_extract=partial_extract,
++ preload=preload, hardlink_masters=hardlink_masters,
+ filter=lambda item: self.item_filter(item, filter)):
+ yield item
+
+diff --git a/src/borg/archiver.py b/src/borg/archiver.py
+index 957959d6..dcc20455 100644
+--- a/src/borg/archiver.py
++++ b/src/borg/archiver.py
+@@ -755,7 +755,8 @@ def peek_and_store_hardlink_masters(item, matched):
+ else:
+ pi = None
+
+- for item in archive.iter_items(filter, preload=True):
++ for item in archive.iter_items(filter, partial_extract=partial_extract,
++ preload=True, hardlink_masters=hardlink_masters):
+ orig_path = item.path
+ if strip_components:
+ item.path = os.sep.join(orig_path.split(os.sep)[strip_components:])
+@@ -997,7 +998,7 @@ def item_to_tarinfo(item, original_path):
+ return None, stream
+ return tarinfo, stream
+
+- for item in archive.iter_items(filter, preload=True):
++ for item in archive.iter_items(filter, preload=True, hardlink_masters=hardlink_masters):
+ orig_path = item.path
+ if strip_components:
+ item.path = os.sep.join(orig_path.split(os.sep)[strip_components:])
+diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
+index c35ad800..935b3d79 100644
+--- a/src/borg/testsuite/archiver.py
++++ b/src/borg/testsuite/archiver.py
+@@ -823,7 +823,18 @@ def test_mount_hardlinks(self):
+ assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456'
+
+ @requires_hardlinks
+- def test_extract_hardlinks(self):
++ def test_extract_hardlinks1(self):
++ self._extract_hardlinks_setup()
++ with changedir('output'):
++ self.cmd('extract', self.repository_location + '::test')
++ assert os.stat('input/source').st_nlink == 4
++ assert os.stat('input/abba').st_nlink == 4
++ assert os.stat('input/dir1/hardlink').st_nlink == 4
++ assert os.stat('input/dir1/subdir/hardlink').st_nlink == 4
++ assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456'
++
++ @requires_hardlinks
++ def test_extract_hardlinks2(self):
+ self._extract_hardlinks_setup()
+ with changedir('output'):
+ self.cmd('extract', self.repository_location + '::test', '--strip-components', '2')
+@@ -839,13 +850,6 @@ def test_extract_hardlinks(self):
+ assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456'
+ assert os.stat('input/dir1/aaaa').st_nlink == 2
+ assert os.stat('input/dir1/source2').st_nlink == 2
+- with changedir('output'):
+- self.cmd('extract', self.repository_location + '::test')
+- assert os.stat('input/source').st_nlink == 4
+- assert os.stat('input/abba').st_nlink == 4
+- assert os.stat('input/dir1/hardlink').st_nlink == 4
+- assert os.stat('input/dir1/subdir/hardlink').st_nlink == 4
+- assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456'
+
+ def test_extract_include_exclude(self):
+ self.cmd('init', '--encryption=repokey', self.repository_location)
+--
+2.21.0
+