diff --git a/beets/config_default.yaml b/beets/config_default.yaml index f3e9acad1..dd140675f 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -7,6 +7,7 @@ import: move: no link: no hardlink: no + reflink: no delete: no resume: ask incremental: no diff --git a/beets/importer.py b/beets/importer.py index 68d5f3d5d..3220b260f 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -222,19 +222,31 @@ class ImportSession(object): iconfig['resume'] = False iconfig['incremental'] = False - # Copy, move, link, and hardlink are mutually exclusive. + if iconfig['reflink']: + iconfig['reflink'] = iconfig['reflink'] \ + .as_choice(['auto', True, False]) + + # Copy, move, reflink, link, and hardlink are mutually exclusive. if iconfig['move']: iconfig['copy'] = False iconfig['link'] = False iconfig['hardlink'] = False + iconfig['reflink'] = False elif iconfig['link']: iconfig['copy'] = False iconfig['move'] = False iconfig['hardlink'] = False + iconfig['reflink'] = False elif iconfig['hardlink']: iconfig['copy'] = False iconfig['move'] = False iconfig['link'] = False + iconfig['reflink'] = False + elif iconfig['reflink']: + iconfig['copy'] = False + iconfig['move'] = False + iconfig['link'] = False + iconfig['hardlink'] = False # Only delete when copying. if not iconfig['copy']: @@ -707,7 +719,7 @@ class ImportTask(BaseImportTask): item.update(changes) def manipulate_files(self, operation=None, write=False, session=None): - """ Copy, move, link or hardlink (depending on `operation`) the files + """ Copy, move, link, hardlink or reflink (depending on `operation`) the files as well as write metadata. `operation` should be an instance of `util.MoveOperation`. @@ -1536,6 +1548,8 @@ def manipulate_files(session, task): operation = MoveOperation.LINK elif session.config['hardlink']: operation = MoveOperation.HARDLINK + elif session.config['reflink']: + operation = MoveOperation.REFLINK else: operation = None diff --git a/beets/library.py b/beets/library.py index e22d4edc0..a060e93d6 100644 --- a/beets/library.py +++ b/beets/library.py @@ -747,6 +747,16 @@ class Item(LibModel): util.hardlink(self.path, dest) plugins.send("item_hardlinked", item=self, source=self.path, destination=dest) + elif operation == MoveOperation.REFLINK: + util.reflink(self.path, dest, fallback=False) + plugins.send("item_reflinked", item=self, source=self.path, + destination=dest) + elif operation == MoveOperation.REFLINK_AUTO: + util.reflink(self.path, dest, fallback=True) + plugins.send("item_reflinked", item=self, source=self.path, + destination=dest) + else: + assert False, 'unknown MoveOperation' # Either copying or moving succeeded, so update the stored path. self.path = dest @@ -1087,6 +1097,12 @@ class Album(LibModel): util.link(old_art, new_art) elif operation == MoveOperation.HARDLINK: util.hardlink(old_art, new_art) + elif operation == MoveOperation.REFLINK: + util.reflink(old_art, new_art, fallback=False) + elif operation == MoveOperation.REFLINK_AUTO: + util.reflink(old_art, new_art, fallback=True) + else: + assert False, 'unknown MoveOperation' self.artpath = new_art def move(self, operation=MoveOperation.MOVE, basedir=None, store=True): diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 384609ee6..248096730 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -134,6 +134,8 @@ class MoveOperation(Enum): COPY = 1 LINK = 2 HARDLINK = 3 + REFLINK = 4 + REFLINK_AUTO = 5 def normpath(path): @@ -549,6 +551,35 @@ def hardlink(path, dest, replace=False): traceback.format_exc()) +def reflink(path, dest, replace=False, fallback=False): + """Create a reflink from `dest` to `path`. + + Raise an `OSError` if `dest` already exists, unless `replace` is + True. If `path` == `dest`, then do nothing. + + If reflinking fails and `fallback` is enabled, try copying the file + instead. Otherwise, raise an error without trying a plain copy. + + May raise an `ImportError` if the `reflink` module is not available. + """ + import reflink as pyreflink + + if samefile(path, dest): + return + + if os.path.exists(syspath(dest)) and not replace: + raise FilesystemError(u'file exists', 'rename', (path, dest)) + + try: + pyreflink.reflink(path, dest) + except (NotImplementedError, pyreflink.ReflinkImpossibleError): + if fallback: + copy(path, dest, replace) + else: + raise FilesystemError(u'OS/filesystem does not support reflinks.', + 'link', (path, dest), traceback.format_exc()) + + def unique_path(path): """Returns a version of ``path`` that does not exist on the filesystem. Specifically, if ``path` itself already exists, then diff --git a/docs/changelog.rst b/docs/changelog.rst index 458e4b8d2..0de3b15a2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,9 @@ New features: * :doc:`/plugins/chroma`: Update file metadata after generating fingerprints through the `submit` command. * :doc:`/plugins/lastgenre`: Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres to genres.txt and genres-tree.yaml * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. +* A new :ref:`reflink` config option instructs the importer to create fast, + copy-on-write file clones on filesystems that support them. Thanks to + :user:`rubdos`. * A new :ref:`extra_tags` configuration option allows more tagged metadata to be included in MusicBrainz queries. * A new :doc:`/plugins/fish` adds `Fish shell`_ tab autocompletion to beets diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 563775fd6..a6aa3d6d7 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -164,6 +164,10 @@ The events currently available are: created for a file. Parameters: ``item``, ``source`` path, ``destination`` path +* `item_reflinked`: called with an ``Item`` object whenever a reflink is + created for a file. + Parameters: ``item``, ``source`` path, ``destination`` path + * `item_removed`: called with an ``Item`` object every time an item (singleton or album's part) is removed from the library (even when its file is not deleted from disk). diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 6aa9f5f53..9dd7447a4 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -475,13 +475,35 @@ hardlink ~~~~~~~~ Either ``yes`` or ``no``, indicating whether to use hard links instead of -moving or copying or symlinking files. (It conflicts with the ``move``, +moving, copying, or symlinking files. (It conflicts with the ``move``, ``copy``, and ``link`` options.) Defaults to ``no``. As with symbolic links (see :ref:`link`, above), this will not work on Windows and you will want to set ``write`` to ``no``. Otherwise, metadata on the original file will be modified. +.. _reflink: + +reflink +~~~~~~~ + +Either ``yes``, ``no``, or ``auto``, indicating whether to use copy-on-write +`file clones`_ (a.k.a. "reflinks") instead of copying or moving files. +The ``auto`` option uses reflinks when possible and falls back to plain +copying when necessary. +Defaults to ``no``. + +This kind of clone is only available on certain filesystems: for example, +btrfs and APFS. For more details on filesystem support, see the `pyreflink`_ +documentation. Note that you need to install ``pyreflink``, either through +``python -m pip install beets[reflink]`` or ``python -m pip install reflink``. + +The option is ignored if ``move`` is enabled (i.e., beets can move or +copy files but it doesn't make sense to do both). + +.. _file clones: https://blogs.oracle.com/otn/save-disk-space-on-linux-by-cloning-files-on-btrfs-and-ocfs2 +.. _pyreflink: https://reflink.readthedocs.io/en/latest/ + resume ~~~~~~ diff --git a/setup.py b/setup.py index 2c3cb2b55..41050307a 100755 --- a/setup.py +++ b/setup.py @@ -122,6 +122,7 @@ setup( 'pyxdg', 'responses>=0.3.0', 'requests_oauthlib', + 'reflink', ] + ( # Tests for the thumbnails plugin need pathlib on Python 2 too. ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] @@ -163,6 +164,7 @@ setup( 'scrub': ['mutagen>=1.33'], 'bpd': ['PyGObject'], 'replaygain': ['PyGObject'], + 'reflink': ['reflink'], }, # Non-Python/non-PyPI plugin dependencies: # chroma: chromaprint or fpcalc diff --git a/test/_common.py b/test/_common.py index 8e3b1dd18..e44fac48b 100644 --- a/test/_common.py +++ b/test/_common.py @@ -25,6 +25,8 @@ import six import unittest from contextlib import contextmanager +import reflink + # Mangle the search path to include the beets sources. sys.path.insert(0, '..') @@ -55,6 +57,7 @@ _item_ident = 0 # OS feature test. HAVE_SYMLINK = sys.platform != 'win32' HAVE_HARDLINK = sys.platform != 'win32' +HAVE_REFLINK = reflink.supported_at(tempfile.gettempdir()) def item(lib=None): diff --git a/test/test_files.py b/test/test_files.py index 13a8b4407..ab82c192e 100644 --- a/test/test_files.py +++ b/test/test_files.py @@ -86,6 +86,24 @@ class MoveTest(_common.TestCase): self.i.move(operation=MoveOperation.COPY) self.assertExists(self.path) + def test_reflink_arrives(self): + self.i.move(operation=MoveOperation.REFLINK_AUTO) + self.assertExists(self.dest) + + def test_reflink_does_not_depart(self): + self.i.move(operation=MoveOperation.REFLINK_AUTO) + self.assertExists(self.path) + + @unittest.skipUnless(_common.HAVE_REFLINK, "need reflink") + def test_force_reflink_arrives(self): + self.i.move(operation=MoveOperation.REFLINK) + self.assertExists(self.dest) + + @unittest.skipUnless(_common.HAVE_REFLINK, "need reflink") + def test_force_reflink_does_not_depart(self): + self.i.move(operation=MoveOperation.REFLINK) + self.assertExists(self.path) + def test_move_changes_path(self): self.i.move() self.assertEqual(self.i.path, util.normpath(self.dest)) @@ -268,6 +286,17 @@ class AlbumFileTest(_common.TestCase): self.assertTrue(os.path.exists(oldpath)) self.assertTrue(os.path.exists(self.i.path)) + @unittest.skipUnless(_common.HAVE_REFLINK, "need reflink") + def test_albuminfo_move_reflinks_file(self): + oldpath = self.i.path + self.ai.album = u'newAlbumName' + self.ai.move(operation=MoveOperation.REFLINK) + self.ai.store() + self.i.load() + + self.assertTrue(os.path.exists(oldpath)) + self.assertTrue(os.path.exists(self.i.path)) + def test_albuminfo_move_to_custom_dir(self): self.ai.move(basedir=self.otherdir) self.i.load() @@ -549,6 +578,12 @@ class SafeMoveCopyTest(_common.TestCase): self.assertExists(self.dest) self.assertExists(self.path) + @unittest.skipUnless(_common.HAVE_REFLINK, "need reflink") + def test_successful_reflink(self): + util.reflink(self.path, self.dest) + self.assertExists(self.dest) + self.assertExists(self.path) + def test_unsuccessful_move(self): with self.assertRaises(util.FilesystemError): util.move(self.path, self.otherpath) @@ -557,6 +592,11 @@ class SafeMoveCopyTest(_common.TestCase): with self.assertRaises(util.FilesystemError): util.copy(self.path, self.otherpath) + @unittest.skipUnless(_common.HAVE_REFLINK, "need reflink") + def test_unsuccessful_reflink(self): + with self.assertRaises(util.FilesystemError): + util.reflink(self.path, self.otherpath) + def test_self_move(self): util.move(self.path, self.path) self.assertExists(self.path)