From b783097329f63db4121112c26a78d9e4402fdd33 Mon Sep 17 00:00:00 2001 From: Thomas Scholtes Date: Tue, 15 Apr 2014 15:40:51 +0200 Subject: [PATCH 1/5] Import zip archives `beet import archive.zip` extracts the archive to a temporary directory and imports the content. The code is very hacky. To make it cleaner the `importer` module needs some refactoring. One thing the code hints at is extending the `ImportTask` class. --- beets/importer.py | 46 ++++++++++++++++++++++++++++++++++++++++++- test/test_importer.py | 30 ++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/beets/importer.py b/beets/importer.py index 97f360d02..9d9c7e962 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -22,6 +22,9 @@ import logging import pickle import itertools from collections import defaultdict +from zipfile import is_zipfile, ZipFile +from tempfile import mkdtemp +import shutil from beets import autotag from beets import library @@ -525,6 +528,11 @@ class ImportTask(object): else: return [self.item] + def cleanup(self): + """Perform clean up during `finalize` stage. + """ + pass + # Utilities. def prune(self, filename): @@ -540,6 +548,16 @@ class ImportTask(object): clutter=config['clutter'].as_str_seq()) +class ArchiveImportTask(ImportTask): + + def __init__(self, toppath): + super(ArchiveImportTask, self).__init__(toppath) + self.sentinel = True + + def cleanup(self): + shutil.rmtree(self.toppath) + + # Full-album pipeline stages. def read_tasks(session): @@ -573,6 +591,26 @@ def read_tasks(session): history_dirs = history_get() for toppath in session.paths: + extracted = None + if is_zipfile(syspath(toppath)): + if not (config['import']['move'] or config['import']['copy']): + log.warn("Cannot import archive. Please set " + "the 'move' or 'copy' option.") + continue + + log.debug('extracting archive {0}' + .format(displayable_path(toppath))) + try: + extracted = mkdtemp() + zip_file = ZipFile(toppath, mode='r') + zip_file.extractall(extracted) + except IOError as exc: + log.error('extraction failed: {0}'.format(exc)) + continue + finally: + zip_file.close() + toppath = extracted + # Check whether the path is to a file. if not os.path.isdir(syspath(toppath)): try: @@ -627,7 +665,11 @@ def read_tasks(session): yield ImportTask(toppath, paths, items) # Indicate the directory is finished. - yield ImportTask.done_sentinel(toppath) + # FIXME hack to delete extraced archives + if extracted is None: + yield ImportTask.done_sentinel(toppath) + else: + yield ArchiveImportTask(extracted) # Show skipped directories. if config['import']['incremental'] and incremental_skipped: @@ -937,6 +979,7 @@ def finalize(session): task.save_progress() if config['import']['incremental']: task.save_history() + task.cleanup() continue items = task.imported_items() @@ -971,6 +1014,7 @@ def finalize(session): task.save_progress() if config['import']['incremental']: task.save_history() + task.cleanup() # Singleton pipeline stages. diff --git a/test/test_importer.py b/test/test_importer.py index 604c4f652..c8c4dde9b 100644 --- a/test/test_importer.py +++ b/test/test_importer.py @@ -17,6 +17,8 @@ import os import shutil import StringIO +from tempfile import mkstemp +from zipfile import ZipFile import _common from _common import unittest @@ -299,6 +301,34 @@ class NonAutotaggedImportTest(_common.TestCase, ImportHelper): self.assertNotExists(os.path.join(self.import_dir, 'the_album')) +class ImportArchiveTest(unittest.TestCase, ImportHelper): + + def setUp(self): + self.setup_beets() + + def tearDown(self): + self.teardown_beets() + + def test_import_zip(self): + zip_path = self.create_zip_archive() + self.assertEqual(len(self.lib.items()), 0) + self.assertEqual(len(self.lib.albums()), 0) + + self._setup_import_session(autotag=False, import_dir=zip_path) + self.importer.run() + self.assertEqual(len(self.lib.items()), 1) + self.assertEqual(len(self.lib.albums()), 1) + + def create_zip_archive(self): + (handle, zip_path) = mkstemp('.zip', dir=self.temp_dir) + os.close(handle) + zip_file = ZipFile(zip_path, mode='w') + zip_file.write(os.path.join(_common.RSRC, 'full.mp3'), + 'full.mp3') + zip_file.close() + return zip_path + + class ImportSingletonTest(_common.TestCase, ImportHelper): """Test ``APPLY`` and ``ASIS`` choices for an import session with singletons config set to True. From e3acdd0cc89156cefc3e27ff220899185d8657e1 Mon Sep 17 00:00:00 2001 From: Thomas Scholtes Date: Tue, 15 Apr 2014 17:23:50 +0200 Subject: [PATCH 2/5] Import tar archives Also refactors the importer code to make better use of ArchiveImportTask. --- beets/importer.py | 72 +++++++++++++++++++++++++++++++++++-------- test/test_importer.py | 31 +++++++++++++------ 2 files changed, 82 insertions(+), 21 deletions(-) diff --git a/beets/importer.py b/beets/importer.py index 9d9c7e962..d31fd1a11 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -22,7 +22,6 @@ import logging import pickle import itertools from collections import defaultdict -from zipfile import is_zipfile, ZipFile from tempfile import mkdtemp import shutil @@ -553,9 +552,60 @@ class ArchiveImportTask(ImportTask): def __init__(self, toppath): super(ArchiveImportTask, self).__init__(toppath) self.sentinel = True + self.extracted = False + + @classmethod + def is_archive(cls, path): + """Returns true if the given path points to an archive that can + be handled. + """ + if not os.path.isfile(path): + return False + + for path_test, _ in cls.handlers(): + if path_test(path): + return True + return False + + @classmethod + def handlers(cls): + """Returns a list of archive handlers. + + Each handler is a `(path_test, ArchiveClass)` tuple. `path_test` + is a function that returns `True` if the given path can be + handled by `ArchiveClass`. `ArchiveClass` is a class that + implements the same interface as `tarfile.TarFile`. + """ + if not hasattr(cls, '_handlers'): + cls._handlers = [] + from zipfile import is_zipfile, ZipFile + cls._handlers.append((is_zipfile, ZipFile)) + from tarfile import is_tarfile, TarFile + cls._handlers.append((is_tarfile, TarFile)) + return cls._handlers def cleanup(self): - shutil.rmtree(self.toppath) + """Removes the temporary directory the archive was extracted to. + """ + if self.extracted: + shutil.rmtree(self.toppath) + + def extract(self): + """Extracts the archive to a temporary directory and sets + `toppath` to that directory. + """ + for path_test, handler_class in self.handlers(): + if path_test(self.toppath): + break + + try: + extract_to = mkdtemp() + archive = handler_class(self.toppath, mode='r') + archive.extractall(extract_to) + finally: + archive.close() + self.extracted = True + self.toppath = extract_to # Full-album pipeline stages. @@ -591,8 +641,9 @@ def read_tasks(session): history_dirs = history_get() for toppath in session.paths: - extracted = None - if is_zipfile(syspath(toppath)): + # Extract archives + archive_task = None + if ArchiveImportTask.is_archive(syspath(toppath)): if not (config['import']['move'] or config['import']['copy']): log.warn("Cannot import archive. Please set " "the 'move' or 'copy' option.") @@ -600,16 +651,13 @@ def read_tasks(session): log.debug('extracting archive {0}' .format(displayable_path(toppath))) + archive_task = ArchiveImportTask(toppath) try: - extracted = mkdtemp() - zip_file = ZipFile(toppath, mode='r') - zip_file.extractall(extracted) + archive_task.extract() except IOError as exc: log.error('extraction failed: {0}'.format(exc)) continue - finally: - zip_file.close() - toppath = extracted + toppath = archive_task.toppath # Check whether the path is to a file. if not os.path.isdir(syspath(toppath)): @@ -666,10 +714,10 @@ def read_tasks(session): # Indicate the directory is finished. # FIXME hack to delete extraced archives - if extracted is None: + if archive_task is None: yield ImportTask.done_sentinel(toppath) else: - yield ArchiveImportTask(extracted) + yield archive_task # Show skipped directories. if config['import']['incremental'] and incremental_skipped: diff --git a/test/test_importer.py b/test/test_importer.py index c8c4dde9b..854b21d4b 100644 --- a/test/test_importer.py +++ b/test/test_importer.py @@ -19,6 +19,7 @@ import shutil import StringIO from tempfile import mkstemp from zipfile import ZipFile +from tarfile import TarFile import _common from _common import unittest @@ -301,7 +302,7 @@ class NonAutotaggedImportTest(_common.TestCase, ImportHelper): self.assertNotExists(os.path.join(self.import_dir, 'the_album')) -class ImportArchiveTest(unittest.TestCase, ImportHelper): +class ImportZipTest(unittest.TestCase, ImportHelper): def setUp(self): self.setup_beets() @@ -310,7 +311,7 @@ class ImportArchiveTest(unittest.TestCase, ImportHelper): self.teardown_beets() def test_import_zip(self): - zip_path = self.create_zip_archive() + zip_path = self.create_archive() self.assertEqual(len(self.lib.items()), 0) self.assertEqual(len(self.lib.albums()), 0) @@ -319,14 +320,26 @@ class ImportArchiveTest(unittest.TestCase, ImportHelper): self.assertEqual(len(self.lib.items()), 1) self.assertEqual(len(self.lib.albums()), 1) - def create_zip_archive(self): - (handle, zip_path) = mkstemp('.zip', dir=self.temp_dir) + def create_archive(self): + (handle, path) = mkstemp(dir=self.temp_dir) os.close(handle) - zip_file = ZipFile(zip_path, mode='w') - zip_file.write(os.path.join(_common.RSRC, 'full.mp3'), - 'full.mp3') - zip_file.close() - return zip_path + archive = ZipFile(path, mode='w') + archive.write(os.path.join(_common.RSRC, 'full.mp3'), + 'full.mp3') + archive.close() + return path + + +class ImportTarTest(ImportZipTest): + + def create_archive(self): + (handle, path) = mkstemp(dir=self.temp_dir) + os.close(handle) + archive = TarFile(path, mode='w') + archive.add(os.path.join(_common.RSRC, 'full.mp3'), + 'full.mp3') + archive.close() + return path class ImportSingletonTest(_common.TestCase, ImportHelper): From 68595ee09d8c82703b0bd1a27ba33d9da1617e2e Mon Sep 17 00:00:00 2001 From: Thomas Scholtes Date: Tue, 15 Apr 2014 17:42:16 +0200 Subject: [PATCH 3/5] Import rar archives --- beets/importer.py | 11 +++++++++++ setup.py | 1 + test/rsrc/archive.rar | Bin 0 -> 2357 bytes test/test_importer.py | 13 +++++++++++++ tox.ini | 1 + 5 files changed, 26 insertions(+) create mode 100644 test/rsrc/archive.rar diff --git a/beets/importer.py b/beets/importer.py index d31fd1a11..61ff1294a 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -548,6 +548,10 @@ class ImportTask(object): class ArchiveImportTask(ImportTask): + """Additional methods for handling archives. + + Use when `toppath` points to a `zip`, `tar`, or `rar` archive. + """ def __init__(self, toppath): super(ArchiveImportTask, self).__init__(toppath) @@ -582,6 +586,13 @@ class ArchiveImportTask(ImportTask): cls._handlers.append((is_zipfile, ZipFile)) from tarfile import is_tarfile, TarFile cls._handlers.append((is_tarfile, TarFile)) + try: + from rarfile import is_rarfile, RarFile + except ImportError: + pass + else: + cls._handlers.append((is_rarfile, RarFile)) + return cls._handlers def cleanup(self): diff --git a/setup.py b/setup.py index e2c917121..c602d275f 100755 --- a/setup.py +++ b/setup.py @@ -99,6 +99,7 @@ setup( 'echonest_tempo': ['pyechonest'], 'lastgenre': ['pylast'], 'web': ['flask'], + 'import': ['rarfile'], }, # Non-Python/non-PyPI plugin dependencies: # replaygain: mp3gain || aacgain diff --git a/test/rsrc/archive.rar b/test/rsrc/archive.rar new file mode 100644 index 0000000000000000000000000000000000000000..e51051155c56a793e6659603e3dcb9126f948f62 GIT binary patch literal 2357 zcmV-53Ci|TVR9iF2LR8Ia{vGh000000002_Y;+)yDgf{Z004yx000B}p?Ia}Sd2s+ zGY9~rfdBwzb!==dZE#}%zz`jkSjde$m;w9jkdi2YChof&1SGRI+UlbO`9jqQYYMqt zs$Ei-H)zWbf&$1Ak^;1r&s46{HdWW*b4|+P$#kmYH)=N#6GaIHuG4jOV4CA~5g=IL zHTWMCQbrTogoK3oci1x*cQ=27>7MlGdCv2l=Xu{e?`QK|_s)EM?>y&w=6?PAp7WF6 zb0_eg#MX$XrUC4)+yigNzFAM;1ic}e=7x#t^bhQAFTl`Yh=q@4L#Z!|)YCboynQT| zn4>gH)XQC*$@A?&hC>Qi@j~|HFwl28x1?2w^oOflLk0`g0U#YC4(Qp5O<@xbK97I_ zNx6GLY5=G%bKmKcN{zW|4CKSDo2y#rAPkF^?qFj>1ChNilgD}?kxtpqMN(9<>OYF zoAIcd`l*%nF=W0eAMHujJTxjM87x9z${uR6~p==0NY^r6i z43~RYc#_cOqA-JZ9VQ!!t&?>|5eWSuyYAiOxTOv)Qv5>iohSkjBp}BgJ+i28ekOzC z0jZGk5Ed*k^UOAY1{5Gxo*ljXI9yBMX2(r4E%?T<7K?^x(Ym_8CuG89hYX0R!?wB3 zD|+9!Xj;OeztRzip#h|=UJ{lq^dYV6=UBGn~#n?W^8HX;mH~O1L-u>zXWYla-Cl;c`>-)n01Vc)~1@Qn4s78 zDp^=;?yPROi0-T|szMfA&R!(#i?fpqDSH9M+0Z}ekbVu93AG~J07mh6fgf~Zt8_zi zd{){9d@5_3w9gS_BJ0$HsIqLN$v2X1|HIgZ^yXMxv2yR~)j7p0SmHi?ikYM2kBcZN zFvC0fOiJ+AwQ+bv&k^#QtO``s(Rpz=o!{+;T^E4p-R_cDJIk}&`;NQZh=*is!?V>F zT1nQ#?=qI_j{MH&VoeyzAHXu)v|!zMADg2r_p~gvo{ouR{LI{z>(^PE-z?{QaT`0^18?OOBP?66~loT=q z!_KiejX%1AfMiv7$9QoA6)Ht4)27&01VLqq>5P4kM4^r0$jj?!TXsPj^Y0!L#Wv;v z^S%Eg3Y@r)Ho_D+r-d|t{ZTM&!rl?IFN>xz7=58iHPJ`8`*Lgbq5{ZjzBsNWXBiCo zbqAKZ2Y%lsM%70*f^gwo88%+|>%<>Z`t%@Ar_wLzev!=)&Dk~`q@fbRofXl&!U#Hs z2WfCXpd`vb!Wv?7Yoa`FkRC?G6a7(q_SIk$v$Vooov(uHi_|kzOTUa!dI@16FT9k> zrPMRF?&@>9zc*cGFz*bRJuRwgP_rcBGC(#8(x7s%ABhig?v(~~xGYhOQ{9SuD9{U~ zS2RH1-?P(EWNDRTe@9jAOsJh6z9>8EnuEarmm$aluo6>7QS`|Q2fZ$lXk~PERnbu} z>hD533Oz8ex1mr|BwUbCq#&Uf^&|1$B>-@ze)llxE+YHneia{T;0HfE{Yv7)_WH9_a{F2wmq+>X{wrU;H|MYZq+x{_Tr2?#ZyL~MrUWlqa&#Ex z&@_Beg5;Ut>DeG8eFdsv7p+f*`vZo-g?UsSa#!9TLTv8om=unM!i;4^Rm_ zepC+~dV~AguSM<-onj`h7PPHlvqTMu%PmqLUf76vhC-wH+dx6 zlT)Fe*}C!sn{L{L<3S%C9i@VzfeLCC-*xnfkvkoYLL^>%LTY5lQ9W7`4q?0q=<|`% z5OXjy@cG2Q^{;auBs}-Hs7|?^nse7FMyO56BO@)N}Dnk;~>NIl@rLF zX9Yi+8=P&rhux!mR%68eVrhKlX(^KVUT6HcAhe>Vrl^0dEfajo@66v()Fphp^Zy-} zzJdh@q@V%W$_ZoAX6k_Tj^ zV_r23j!`Kq9UDC=>&D3;m3hZ^L$Q;OI^+KWh6W8OKe3LXc*+Lko6I*T5YS}0)G@L0 b5>G>ukvl5sq5t646b-lVK*T+J06+%-6)>9n literal 0 HcmV?d00001 diff --git a/test/test_importer.py b/test/test_importer.py index 854b21d4b..d6b211c2d 100644 --- a/test/test_importer.py +++ b/test/test_importer.py @@ -342,6 +342,19 @@ class ImportTarTest(ImportZipTest): return path +class ImportRarTest(ImportZipTest): + + def create_archive(self): + return os.path.join(_common.RSRC, 'archive.rar') + + +@unittest.skip('Implment me!') +class ImportPasswordRarTest(ImportZipTest): + + def create_archive(self): + return os.path.join(_common.RSRC, 'password.rar') + + class ImportSingletonTest(_common.TestCase, ImportHelper): """Test ``APPLY`` and ``ASIS`` choices for an import session with singletons config set to True. diff --git a/tox.ini b/tox.ini index 5e50071c6..e84f536ac 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ deps = flask responses pyechonest + rarfile commands = nosetests {posargs} From 1f742130c45779c49b3fe7f60d6fd599914ab99c Mon Sep 17 00:00:00 2001 From: Thomas Scholtes Date: Tue, 15 Apr 2014 18:26:35 +0200 Subject: [PATCH 4/5] Catch all archive extract errors and skip tests without unrar --- beets/importer.py | 2 +- test/test_importer.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/beets/importer.py b/beets/importer.py index 61ff1294a..951e0ac4e 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -665,7 +665,7 @@ def read_tasks(session): archive_task = ArchiveImportTask(toppath) try: archive_task.extract() - except IOError as exc: + except Exception as exc: log.error('extraction failed: {0}'.format(exc)) continue toppath = archive_task.toppath diff --git a/test/test_importer.py b/test/test_importer.py index d6b211c2d..ca77ac8b0 100644 --- a/test/test_importer.py +++ b/test/test_importer.py @@ -23,7 +23,7 @@ from tarfile import TarFile import _common from _common import unittest -from helper import TestImportSession, TestHelper +from helper import TestImportSession, TestHelper, has_program from beets import library from beets import importer from beets.mediafile import MediaFile @@ -342,6 +342,7 @@ class ImportTarTest(ImportZipTest): return path +@unittest.skipIf(not has_program('unrar'), 'unrar program not found') class ImportRarTest(ImportZipTest): def create_archive(self): From 63097650ffb76ff97e92846513baac9d6a001606 Mon Sep 17 00:00:00 2001 From: Thomas Scholtes Date: Tue, 15 Apr 2014 20:36:46 +0200 Subject: [PATCH 5/5] Add docs and changelog for archive import --- docs/changelog.rst | 1 + docs/reference/cli.rst | 26 ++++++++++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5dcdb2bd1..88eb90789 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,7 @@ New stuff: * There is also a new :doc:`/plugins/keyfinder` that runs a command line program to get the key from audio data and store it in the `initial_key` field. +* Beets can now import `zip`, `tar` and `rar` archives. Fixes: diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 4ee434237..c7d141475 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -43,21 +43,25 @@ import `````` :: - beet import [-CWAPRqst] [-l LOGPATH] DIR... + beet import [-CWAPRqst] [-l LOGPATH] PATH... beet import [options] -L QUERY Add music to your library, attempting to get correct tags for it from MusicBrainz. -Point the command at a directory full of music. The directory can be a single -album or a directory whose leaf subdirectories are albums (the latter case is -true of typical Artist/Album organizations and many people's "downloads" -folders). The music will be copied to a configurable directory structure (see -below) and added to a library database (see below). The command is interactive -and will try to get you to verify MusicBrainz tags that it thinks are suspect. -(This means that importing a large amount of music is therefore very tedious -right now; this is something we need to work on. Read the -:doc:`autotagging guide ` if you need help.) +Point the command at a directory full of music. The directory can be a +single album or a directory whose leaf subdirectories are albums (the +latter case is true of typical Artist/Album organizations and many +people's "downloads" folders). The path can also be a single file or an +archive. Beets supports `zip` and `tar` archives out of the box. To +extract `rar` files you need to install the `rarfile`_ package and the +`unrar` command. The music will be copied to a configurable directory +structure (see below) and added to a library database (see below). The +command is interactive and will try to get you to verify MusicBrainz +tags that it thinks are suspect. (This means that importing a large +amount of music is therefore very tedious right now; this is something +we need to work on. Read the :doc:`autotagging guide ` +if you need help.) * By default, the command copies files your the library directory and updates the ID3 tags on your music. If you'd like to leave your music @@ -116,6 +120,8 @@ right now; this is something we need to work on. Read the ``--group-albums`` option to split the files based on their metadata before matching them as separate albums. +.. _rarfile: https://pypi.python.org/pypi/rarfile/2.2 + .. only:: html Reimporting