Merge pull request #3088 from rubdos/reflink

Add reflink option

beets/config_default.yaml

@@ -7,6 +7,7 @@ import:
     move: no
     link: no
     hardlink: no
+    reflink: no
     delete: no
     resume: ask
     incremental: no

beets/importer.py

@@ -222,19 +222,31 @@ class ImportSession(object):
             iconfig['resume'] = False
             iconfig['incremental'] = False
 
-        # Copy, move, link, and hardlink are mutually exclusive.
+        if iconfig['reflink']:
+            iconfig['reflink'] = iconfig['reflink'] \
+                .as_choice(['auto', True, False])
+
+        # Copy, move, reflink, link, and hardlink are mutually exclusive.
         if iconfig['move']:
             iconfig['copy'] = False
             iconfig['link'] = False
             iconfig['hardlink'] = False
+            iconfig['reflink'] = False
         elif iconfig['link']:
             iconfig['copy'] = False
             iconfig['move'] = False
             iconfig['hardlink'] = False
+            iconfig['reflink'] = False
         elif iconfig['hardlink']:
             iconfig['copy'] = False
             iconfig['move'] = False
             iconfig['link'] = False
+            iconfig['reflink'] = False
+        elif iconfig['reflink']:
+            iconfig['copy'] = False
+            iconfig['move'] = False
+            iconfig['link'] = False
+            iconfig['hardlink'] = False
 
         # Only delete when copying.
         if not iconfig['copy']:
@@ -707,7 +719,7 @@ class ImportTask(BaseImportTask):
             item.update(changes)
 
     def manipulate_files(self, operation=None, write=False, session=None):
-        """ Copy, move, link or hardlink (depending on `operation`) the files
+        """ Copy, move, link, hardlink or reflink (depending on `operation`) the files
         as well as write metadata.
 
         `operation` should be an instance of `util.MoveOperation`.
@@ -1536,6 +1548,8 @@ def manipulate_files(session, task):
             operation = MoveOperation.LINK
         elif session.config['hardlink']:
             operation = MoveOperation.HARDLINK
+        elif session.config['reflink']:
+            operation = MoveOperation.REFLINK
         else:
             operation = None
 

beets/library.py

@@ -747,6 +747,16 @@ class Item(LibModel):
             util.hardlink(self.path, dest)
             plugins.send("item_hardlinked", item=self, source=self.path,
                          destination=dest)
+        elif operation == MoveOperation.REFLINK:
+            util.reflink(self.path, dest, fallback=False)
+            plugins.send("item_reflinked", item=self, source=self.path,
+                         destination=dest)
+        elif operation == MoveOperation.REFLINK_AUTO:
+            util.reflink(self.path, dest, fallback=True)
+            plugins.send("item_reflinked", item=self, source=self.path,
+                         destination=dest)
+        else:
+            assert False, 'unknown MoveOperation'
 
         # Either copying or moving succeeded, so update the stored path.
         self.path = dest
@@ -1087,6 +1097,12 @@ class Album(LibModel):
             util.link(old_art, new_art)
         elif operation == MoveOperation.HARDLINK:
             util.hardlink(old_art, new_art)
+        elif operation == MoveOperation.REFLINK:
+            util.reflink(old_art, new_art, fallback=False)
+        elif operation == MoveOperation.REFLINK_AUTO:
+            util.reflink(old_art, new_art, fallback=True)
+        else:
+            assert False, 'unknown MoveOperation'
         self.artpath = new_art
 
     def move(self, operation=MoveOperation.MOVE, basedir=None, store=True):

beets/util/__init__.py

@@ -134,6 +134,8 @@ class MoveOperation(Enum):
     COPY = 1
     LINK = 2
     HARDLINK = 3
+    REFLINK = 4
+    REFLINK_AUTO = 5
 
 
 def normpath(path):
@@ -549,6 +551,35 @@ def hardlink(path, dest, replace=False):
                                   traceback.format_exc())
 
 
+def reflink(path, dest, replace=False, fallback=False):
+    """Create a reflink from `dest` to `path`.
+
+    Raise an `OSError` if `dest` already exists, unless `replace` is
+    True. If `path` == `dest`, then do nothing.
+
+    If reflinking fails and `fallback` is enabled, try copying the file
+    instead. Otherwise, raise an error without trying a plain copy.
+
+    May raise an `ImportError` if the `reflink` module is not available.
+    """
+    import reflink as pyreflink
+
+    if samefile(path, dest):
+        return
+
+    if os.path.exists(syspath(dest)) and not replace:
+        raise FilesystemError(u'file exists', 'rename', (path, dest))
+
+    try:
+        pyreflink.reflink(path, dest)
+    except (NotImplementedError, pyreflink.ReflinkImpossibleError):
+        if fallback:
+            copy(path, dest, replace)
+        else:
+            raise FilesystemError(u'OS/filesystem does not support reflinks.',
+                                  'link', (path, dest), traceback.format_exc())
+
+
 def unique_path(path):
     """Returns a version of ``path`` that does not exist on the
     filesystem. Specifically, if ``path` itself already exists, then

docs/changelog.rst

@@ -13,6 +13,9 @@ New features:
 * :doc:`/plugins/chroma`: Update file metadata after generating fingerprints through the `submit` command.
 * :doc:`/plugins/lastgenre`: Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres to genres.txt and genres-tree.yaml
 * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server.
+* A new :ref:`reflink` config option instructs the importer to create fast,
+  copy-on-write file clones on filesystems that support them. Thanks to
+  :user:`rubdos`.
 * A new :ref:`extra_tags` configuration option allows more tagged metadata
   to be included in MusicBrainz queries.
 * A new :doc:`/plugins/fish` adds `Fish shell`_ tab autocompletion to beets

docs/dev/plugins.rst

@@ -164,6 +164,10 @@ The events currently available are:
   created for a file.
   Parameters: ``item``, ``source`` path, ``destination`` path
 
+* `item_reflinked`: called with an ``Item`` object whenever a reflink is
+  created for a file.
+  Parameters: ``item``, ``source`` path, ``destination`` path
+
 * `item_removed`: called with an ``Item`` object every time an item (singleton
   or album's part) is removed from the library (even when its file is not
   deleted from disk).

docs/reference/config.rst

@@ -475,13 +475,35 @@ hardlink
 ~~~~~~~~
 
 Either ``yes`` or ``no``, indicating whether to use hard links instead of
-moving or copying or symlinking files. (It conflicts with the ``move``,
+moving, copying, or symlinking files. (It conflicts with the ``move``,
 ``copy``, and ``link`` options.) Defaults to ``no``.
 
 As with symbolic links (see :ref:`link`, above), this will not work on Windows
 and you will want to set ``write`` to ``no``.  Otherwise, metadata on the
 original file will be modified.
 
+.. _reflink:
+
+reflink
+~~~~~~~
+
+Either ``yes``, ``no``, or ``auto``, indicating whether to use copy-on-write
+`file clones`_ (a.k.a. "reflinks") instead of copying or moving files.
+The ``auto`` option uses reflinks when possible and falls back to plain
+copying when necessary.
+Defaults to ``no``.
+
+This kind of clone is only available on certain filesystems: for example,
+btrfs and APFS. For more details on filesystem support, see the `pyreflink`_
+documentation.  Note that you need to install ``pyreflink``, either through
+``python -m pip install beets[reflink]`` or ``python -m pip install reflink``.
+
+The option is ignored if ``move`` is enabled (i.e., beets can move or
+copy files but it doesn't make sense to do both).
+
+.. _file clones: https://blogs.oracle.com/otn/save-disk-space-on-linux-by-cloning-files-on-btrfs-and-ocfs2
+.. _pyreflink: https://reflink.readthedocs.io/en/latest/
+
 resume
 ~~~~~~
 

setup.py

@@ -122,6 +122,7 @@ setup(
             'pyxdg',
             'responses>=0.3.0',
             'requests_oauthlib',
+            'reflink',
         ] + (
             # Tests for the thumbnails plugin need pathlib on Python 2 too.
             ['pathlib'] if (sys.version_info < (3, 4, 0)) else []
@@ -163,6 +164,7 @@ setup(
         'scrub': ['mutagen>=1.33'],
         'bpd': ['PyGObject'],
         'replaygain': ['PyGObject'],
+        'reflink': ['reflink'],
     },
     # Non-Python/non-PyPI plugin dependencies:
     #   chroma: chromaprint or fpcalc

test/_common.py

@@ -25,6 +25,8 @@ import six
 import unittest
 from contextlib import contextmanager
 
+import reflink
+
 
 # Mangle the search path to include the beets sources.
 sys.path.insert(0, '..')
@@ -55,6 +57,7 @@ _item_ident = 0
 # OS feature test.
 HAVE_SYMLINK = sys.platform != 'win32'
 HAVE_HARDLINK = sys.platform != 'win32'
+HAVE_REFLINK = reflink.supported_at(tempfile.gettempdir())
 
 
 def item(lib=None):

test/test_files.py

@@ -86,6 +86,24 @@ class MoveTest(_common.TestCase):
         self.i.move(operation=MoveOperation.COPY)
         self.assertExists(self.path)
 
+    def test_reflink_arrives(self):
+        self.i.move(operation=MoveOperation.REFLINK_AUTO)
+        self.assertExists(self.dest)
+
+    def test_reflink_does_not_depart(self):
+        self.i.move(operation=MoveOperation.REFLINK_AUTO)
+        self.assertExists(self.path)
+
+    @unittest.skipUnless(_common.HAVE_REFLINK, "need reflink")
+    def test_force_reflink_arrives(self):
+        self.i.move(operation=MoveOperation.REFLINK)
+        self.assertExists(self.dest)
+
+    @unittest.skipUnless(_common.HAVE_REFLINK, "need reflink")
+    def test_force_reflink_does_not_depart(self):
+        self.i.move(operation=MoveOperation.REFLINK)
+        self.assertExists(self.path)
+
     def test_move_changes_path(self):
         self.i.move()
         self.assertEqual(self.i.path, util.normpath(self.dest))
@@ -268,6 +286,17 @@ class AlbumFileTest(_common.TestCase):
         self.assertTrue(os.path.exists(oldpath))
         self.assertTrue(os.path.exists(self.i.path))
 
+    @unittest.skipUnless(_common.HAVE_REFLINK, "need reflink")
+    def test_albuminfo_move_reflinks_file(self):
+        oldpath = self.i.path
+        self.ai.album = u'newAlbumName'
+        self.ai.move(operation=MoveOperation.REFLINK)
+        self.ai.store()
+        self.i.load()
+
+        self.assertTrue(os.path.exists(oldpath))
+        self.assertTrue(os.path.exists(self.i.path))
+
     def test_albuminfo_move_to_custom_dir(self):
         self.ai.move(basedir=self.otherdir)
         self.i.load()
@@ -549,6 +578,12 @@ class SafeMoveCopyTest(_common.TestCase):
         self.assertExists(self.dest)
         self.assertExists(self.path)
 
+    @unittest.skipUnless(_common.HAVE_REFLINK, "need reflink")
+    def test_successful_reflink(self):
+        util.reflink(self.path, self.dest)
+        self.assertExists(self.dest)
+        self.assertExists(self.path)
+
     def test_unsuccessful_move(self):
         with self.assertRaises(util.FilesystemError):
             util.move(self.path, self.otherpath)
@@ -557,6 +592,11 @@ class SafeMoveCopyTest(_common.TestCase):
         with self.assertRaises(util.FilesystemError):
             util.copy(self.path, self.otherpath)
 
+    @unittest.skipUnless(_common.HAVE_REFLINK, "need reflink")
+    def test_unsuccessful_reflink(self):
+        with self.assertRaises(util.FilesystemError):
+            util.reflink(self.path, self.otherpath)
+
     def test_self_move(self):
         util.move(self.path, self.path)
         self.assertExists(self.path)