diff --git a/beets/importer.py b/beets/importer.py index 558dcc03c..060a188c9 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -179,6 +179,7 @@ class ImportSession(object): self.paths = paths self.query = query self.seen_idents = set() + self._is_resuming = dict() # Normalize the paths. if self.paths: @@ -294,6 +295,50 @@ class ImportSession(object): # User aborted operation. Silently stop. pass + # Incremental and resumed imports + + def already_imported(self, toppath, paths): + """Returns true if the files belonging to this task have already + been imported in a previous session. + """ + if self.is_resuming(toppath) \ + and all(map(lambda p: progress_element(toppath, p), paths)): + return True + if self.config['incremental'] \ + and tuple(paths) in self.history_dirs: + return True + + return False + + @property + def history_dirs(self): + if not hasattr(self, '_history_dirs'): + self._history_dirs = history_get() + return self._history_dirs + + def is_resuming(self, toppath): + """Return `True` if user wants to resume import of this path. + + You have to call `ask_resume` first to determine the return value. + """ + return self._is_resuming.get(toppath, False) + + def ask_resume(self, toppath): + """If import of `toppath` was aborted in an earlier session, ask + user if she wants to resume the import. + + Determines the return value of `is_resuming(toppath)`. + """ + if self.want_resume and has_progress(toppath): + # Either accept immediately or prompt for input to decide. + if self.want_resume is True or \ + self.should_resume(toppath): + log.warn('Resuming interrupted import of %s' % toppath) + self._is_resuming[toppath] = True + else: + # Clear progress; we're starting from the top. + progress_reset(toppath) + # The importer task class. @@ -638,10 +683,6 @@ class SingletonImportTask(ImportTask): def imported_items(self): return [self.item] - def save_history(self): - # TODO we should also save history for singletons - pass - def apply_metadata(self): autotag.apply_item_metadata(self.item, self.match.info) @@ -819,24 +860,11 @@ def read_tasks(session): in the user-specified list of paths. In the case of a singleton import, yields single-item tasks instead. """ - # Look for saved incremental directories. - if session.config['incremental']: - incremental_skipped = 0 - history_dirs = history_get() - + skipped = 0 for toppath in session.paths: # Determine if we want to resume import of the toppath - resuming = False - if session.want_resume and has_progress(toppath): - # Either accept immediately or prompt for input to decide. - if session.want_resume is True or \ - session.should_resume(toppath): - log.warn('Resuming interrupted import of %s' % toppath) - resuming = True - else: - # Clear progress; we're starting from the top. - progress_reset(toppath) + session.ask_resume(toppath) # Extract archives. archive_task = None @@ -860,7 +888,13 @@ def read_tasks(session): # Check whether the path is to a file. if not os.path.isdir(syspath(toppath)): - if resuming and progress_element(toppath, toppath): + # FIXME remove duplicate code. We could put the debug + # statement into `session.alread_imported` but I don't feel + # comfortable triggering an action in a query. + if session.already_imported(toppath, toppath): + log.debug(u'Skipping previously-imported path: {0}' + .format(displayable_path(toppath))) + skipped += 1 continue try: @@ -882,32 +916,33 @@ def read_tasks(session): for _, items in autotag.albums_in_dir(toppath): all_items += items if all_items: + if session.already_imported(toppath, [toppath]): + log.debug(u'Skipping previously-imported path: {0}' + .format(displayable_path(toppath))) + skipped += 1 + continue yield ImportTask(toppath, [toppath], all_items) yield SentinelImportTask(toppath) continue # Produce paths under this directory. for paths, items in autotag.albums_in_dir(toppath): - # Skip according to progress. - if resuming \ - and all(map(lambda p: progress_element(toppath, p), paths)): - continue - - # When incremental, skip paths in the history. - if session.config['incremental'] \ - and tuple(paths) in history_dirs: - log.debug(u'Skipping previously-imported path: %s' % - displayable_path(paths)) - incremental_skipped += 1 - continue - - # Yield all the necessary tasks. if session.config['singletons']: for item in items: - if not (resuming and progress_element(toppath, item.path)): - yield SingletonImportTask(toppath, item) + if session.already_imported(toppath, [item.path]): + log.debug(u'Skipping previously-imported path: {0}' + .format(displayable_path(paths))) + skipped += 1 + continue + yield SingletonImportTask(toppath, item) yield SentinelImportTask(toppath, paths) + else: + if session.already_imported(toppath, paths): + log.debug(u'Skipping previously-imported path: {0}' + .format(displayable_path(paths))) + skipped += 1 + continue yield ImportTask(toppath, paths, items) # Indicate the directory is finished. @@ -918,9 +953,8 @@ def read_tasks(session): yield archive_task # Show skipped directories. - if session.config['incremental'] and incremental_skipped: - log.info(u'Incremental import: skipped %i directories.' % - incremental_skipped) + if skipped: + log.info(u'Skipped {0} directories.'.format(skipped)) def query_tasks(session): diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 8bf048039..a94f4c284 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -25,6 +25,7 @@ import pipes from beets import ui, util, plugins, config from beets.plugins import BeetsPlugin from beetsplug.embedart import embed_item +from beets.util.confit import ConfigTypeError log = logging.getLogger('beets') _fs_lock = threading.Lock() @@ -37,57 +38,53 @@ ALIASES = { } -def _destination(dest_dir, item, keep_new, path_formats): - """Return the path under `dest_dir` where the file should be placed - (possibly after conversion). +def replace_ext(path, ext): + """Return the path with its extension replaced by `ext`. + + The new extension must not contain a leading dot. """ - dest = item.destination(basedir=dest_dir, path_formats=path_formats) - if keep_new: - # When we're keeping the converted file, no extension munging - # occurs. - return dest - else: - # Otherwise, replace the extension. - _, ext = get_format() - return os.path.splitext(dest)[0] + ext + return os.path.splitext(path)[0] + '.' + ext -def get_format(): - """Get the currently configured format command and extension. +def get_format(format=None): + """Return the command tempate and the extension from the config. """ - format = config['convert']['format'].get(unicode).lower() + if not format: + format = config['convert']['format'].get(unicode).lower() format = ALIASES.get(format, format) - format_info = config['convert']['formats'][format].get(dict) - - # Convenience and backwards-compatibility shortcuts. - keys = config['convert'].keys() - if 'command' in keys: - format_info['command'] = config['convert']['command'].get(unicode) - elif 'opts' in keys: - # Undocumented option for backwards compatibility with < 1.3.1. - format_info['command'] = u'ffmpeg -i $source -y {0} $dest'.format( - config['convert']['opts'].get(unicode) - ) - if 'extension' in keys: - format_info['extension'] = config['convert']['extension'].get(unicode) try: - return ( - format_info['command'].encode('utf8'), - (u'.' + format_info['extension']).encode('utf8'), - ) + format_info = config['convert']['formats'][format].get(dict) + command = format_info['command'] + extension = format_info['extension'] except KeyError: raise ui.UserError( u'convert: format {0} needs "command" and "extension" fields' .format(format) ) + except ConfigTypeError: + command = config['convert']['formats'][format].get(str) + extension = format + + # Convenience and backwards-compatibility shortcuts. + keys = config['convert'].keys() + if 'command' in keys: + command = config['convert']['command'].get(unicode) + elif 'opts' in keys: + # Undocumented option for backwards compatibility with < 1.3.1. + command = u'ffmpeg -i $source -y {0} $dest'.format( + config['convert']['opts'].get(unicode) + ) + if 'extension' in keys: + extension = config['convert']['extension'].get(unicode) + + return (command.encode('utf8'), extension.encode('utf8')) -def encode(source, dest, pretend=False): - """Encode ``source`` to ``dest`` using the command from ``get_format()``. +def encode(command, source, dest, pretend=False): + """Encode `source` to `dest` using command template `command`. - Raises an ``ui.UserError`` if the command was not found and a - ``subprocess.CalledProcessError`` if the command exited with a + Raises `subprocess.CalledProcessError` if the command exited with a non-zero status code. """ quiet = config['convert']['quiet'].get() @@ -95,7 +92,6 @@ def encode(source, dest, pretend=False): if not quiet and not pretend: log.info(u'Started encoding {0}'.format(util.displayable_path(source))) - command, _ = get_format() command = Template(command).safe_substitute({ 'source': pipes.quote(source), 'dest': pipes.quote(dest), @@ -117,7 +113,7 @@ def encode(source, dest, pretend=False): raise except OSError as exc: raise ui.UserError( - u'convert: could invoke ffmpeg: {0}'.format(exc) + u"convert: could invoke '{0}': {0}".format(command, exc) ) if not quiet and not pretend: @@ -136,16 +132,22 @@ def should_transcode(item): item.bitrate >= 1000 * maxbr -def convert_item(dest_dir, keep_new, path_formats, pretend=False): +def convert_item(dest_dir, keep_new, path_formats, command, ext, + pretend=False): while True: item = yield - dest = _destination(dest_dir, item, keep_new, path_formats) + dest = item.destination(basedir=dest_dir, path_formats=path_formats) - if os.path.exists(util.syspath(dest)): - log.info(u'Skipping {0} (target file exists)'.format( - util.displayable_path(item.path) - )) - continue + # When keeping the new file in the library, we first move the + # current (pristine) file to the destination. We'll then copy it + # back to its old path or transcode it to a new path. + if keep_new: + original = dest + converted = replace_ext(item.path, ext) + else: + original = item.path + dest = replace_ext(dest, ext) + converted = dest # Ensure that only one thread tries to create directories at a # time. (The existence check is not atomic with the directory @@ -153,19 +155,16 @@ def convert_item(dest_dir, keep_new, path_formats, pretend=False): with _fs_lock: util.mkdirall(dest, pretend) - # When keeping the new file in the library, we first move the - # current (pristine) file to the destination. We'll then copy it - # back to its old path or transcode it to a new path. + if os.path.exists(util.syspath(dest)): + log.info(u'Skipping {0} (target file exists)'.format( + util.displayable_path(item.path) + )) + continue + if keep_new: log.info(u'Moving to {0}'. - format(util.displayable_path(dest))) - util.move(item.path, dest, pretend) - original = dest - _, ext = get_format() - converted = os.path.splitext(item.path)[0] + ext - else: - original = item.path - converted = dest + format(util.displayable_path(original))) + util.move(item.path, original, pretend) if not should_transcode(item): # No transcoding necessary. @@ -173,7 +172,7 @@ def convert_item(dest_dir, keep_new, path_formats, pretend=False): util.copy(original, converted, pretend) else: try: - encode(original, converted, pretend) + encode(command, original, converted, pretend) except subprocess.CalledProcessError: continue @@ -204,12 +203,12 @@ def convert_on_import(lib, item): library. """ if should_transcode(item): - _, ext = get_format() + command, ext = get_format() fd, dest = tempfile.mkstemp(ext) os.close(fd) _temp_files.append(dest) # Delete the transcode later. try: - encode(item.path, dest) + encode(command, item.path, dest) except subprocess.CalledProcessError: return item.path = dest @@ -219,21 +218,24 @@ def convert_on_import(lib, item): def convert_func(lib, opts, args): - dest = opts.dest if opts.dest is not None else \ - config['convert']['dest'].get() - - if not dest: + if not opts.dest: + opts.dest = config['convert']['dest'].get() + if not opts.dest: raise ui.UserError('no convert destination set') + opts.dest = util.bytestring_path(opts.dest) - dest = util.bytestring_path(dest) - threads = opts.threads if opts.threads is not None else \ - config['convert']['threads'].get(int) - keep_new = opts.keep_new + if not opts.threads: + opts.threads = config['convert']['threads'].get(int) - if not config['convert']['paths']: - path_formats = ui.get_path_formats() - else: + if config['convert']['paths']: path_formats = ui.get_path_formats(config['convert']['paths']) + else: + path_formats = ui.get_path_formats() + + if not opts.format: + opts.format = config['convert']['format'].get(unicode).lower() + + command, ext = get_format(opts.format) pretend = opts.pretend if opts.pretend is not None else \ config['convert']['pretend'].get() @@ -248,8 +250,13 @@ def convert_func(lib, opts, args): items = (i for a in lib.albums(ui.decargs(args)) for i in a.items()) else: items = iter(lib.items(ui.decargs(args))) - convert = [convert_item(dest, keep_new, path_formats, pretend) - for _ in range(threads)] + convert = [convert_item(opts.dest, + opts.keep_new, + path_formats, + command, + ext, + pretend) + for _ in range(opts.threads)] pipe = util.pipeline.Pipeline([items, convert]) pipe.run_parallel() @@ -272,29 +279,14 @@ class ConvertPlugin(BeetsPlugin): u'command': u'ffmpeg -i $source -y -vn -acodec alac $dest', u'extension': u'm4a', }, - u'flac': { - u'command': u'ffmpeg -i $source -y -vn -acodec flac $dest', - u'extension': u'flac', - }, - u'mp3': { - u'command': u'ffmpeg -i $source -y -vn -aq 2 $dest', - u'extension': u'mp3', - }, - u'opus': { - u'command': u'ffmpeg -i $source -y -vn -acodec libopus ' - u'-ab 96k $dest', - u'extension': u'opus', - }, - u'ogg': { - u'command': u'ffmpeg -i $source -y -vn -acodec libvorbis ' - u'-aq 2 $dest', - u'extension': u'ogg', - }, - u'windows media': { - u'command': u'ffmpeg -i $source -y -vn -acodec wmav2 ' - u'-vn $dest', - u'extension': u'wma', - }, + u'flac': u'ffmpeg -i $source -y -vn -acodec flac $dest', + u'mp3': u'ffmpeg -i $source -y -vn -aq 2 $dest', + u'opus': + u'ffmpeg -i $source -y -vn -acodec libopus -ab 96k $dest', + u'ogg': + u'ffmpeg -i $source -y -vn -acodec libvorbis -aq 2 $dest', + u'wma': + u'ffmpeg -i $source -y -vn -acodec wmav2 -vn $dest', }, u'max_bitrate': 500, u'auto': False, @@ -318,6 +310,8 @@ class ConvertPlugin(BeetsPlugin): and move the old files') cmd.parser.add_option('-d', '--dest', action='store', help='set the destination directory') + cmd.parser.add_option('-f', '--format', action='store', dest='format', + help='set the destination directory') cmd.func = convert_func return [cmd] diff --git a/docs/changelog.rst b/docs/changelog.rst index 26e095cd0..74ce6195b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -67,6 +67,8 @@ Little improvements and fixes: import. * :doc:`/plugins/chroma`: A new ``auto`` configuration option disables fingerprinting on import. Thanks to ddettrittus. +* :doc:`/plugins/convert`: Add ``--format`` option to select the + transoding command from the command-line. 1.3.6 (May 10, 2014) diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 5eff02f1e..485c80400 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -4,23 +4,17 @@ Convert Plugin The ``convert`` plugin lets you convert parts of your collection to a directory of your choice, transcoding audio and embedding album art along the way. It can transcode to and from any format using a configurable command -line. It will skip files that are already present in the target directory. -Converted files follow the same path formats as your library. - -.. _FFmpeg: http://ffmpeg.org +line. Installation ------------ -First, enable the ``convert`` plugin (see :doc:`/plugins/index`). +Enable the ``convert`` plugin in your configuration (see +:doc:`/plugins/index`). By default, the plugin depends on `FFmpeg`_ to +transcode the audio, so you might want to install it. -To transcode music, this plugin requires the ``ffmpeg`` command-line -tool. If its executable is in your path, it will be found automatically -by the plugin. Otherwise, configure the plugin to locate the executable:: - - convert: - ffmpeg: /usr/bin/ffmpeg +.. _FFmpeg: http://ffmpeg.org Usage @@ -28,11 +22,22 @@ Usage To convert a part of your collection, run ``beet convert QUERY``. This will display all items matching ``QUERY`` and ask you for confirmation before -starting the conversion. The ``-a`` (or ``--album``) option causes the command +starting the conversion. The command will then transcode all the +matching files to the destination directory given by the ``-d`` +(``--dest``) option or the ``dest`` configuration. The path layout +mirrors that of your library, but it may be customized through the +``paths`` configuration. + +The plugin uses a command-line program to transcode the audio. With the +``-f`` (``--format``) option you can choose the transcoding command +and customize the available commands +:ref:`through the configuration `. + +The ``-a`` (or ``--album``) option causes the command to match albums instead of tracks. -The ``-t`` (``--threads``) and ``-d`` (``--dest``) options allow you to specify -or overwrite the respective configuration options. +The ``-t`` (``--threads``) option allows you to specify or overwrite +the respective configuration option. By default, the command places converted files into the destination directory and leaves your library pristine. To instead back up your original files into @@ -48,7 +53,7 @@ The plugin offers several configuration options, all of which live under the * ``dest`` sets the directory the files will be converted (or copied) to. A destination is required---you either have to provide it in the config file - or on the command line using the ``-d`` flag. + or on the command-line using the ``-d`` flag. * ``embed`` indicates whether or not to embed album art in converted items. Default: true. * If you set ``max_bitrate``, all lossy files with a higher bitrate will be @@ -69,39 +74,14 @@ The plugin offers several configuration options, all of which live under the encoding. By default, the plugin will detect the number of processors available and use them all. -These config options control the transcoding process: +.. _convert-format-config: -* ``format`` is the name of the audio file format to transcode to. Files that - are already in the format (and are below the maximum bitrate) will not be - transcoded. The plugin includes default commands for the formats MP3, AAC, - ALAC, FLAC, Opus, Vorbis, and Windows Media; the default is MP3. If you want - to use a different format (or customize the transcoding options), use the - options below. -* ``extension`` is the filename extension to be used for newly transcoded - files. This is implied by the ``format`` option, but you can set it yourself - if you're using a different format. -* ``command`` is the command line to use to transcode audio. A default - command, usually using an FFmpeg invocation, is implied by the ``format`` - option. The tokens ``$source`` and ``$dest`` in the command are replaced - with the paths to the existing and new file. For example, the command - ``ffmpeg -i $source -y -aq 4 $dest`` transcodes to MP3 using FFmpeg at the - V4 quality level. +Configuring the transcoding command +``````````````````````````````````` -Here's an example configuration:: - - convert: - embed: false - format: aac - max_bitrate: 200 - dest: /home/user/MusicForPhone - threads: 4 - paths: - default: $albumartist/$title - -If you have several formats you want to switch between, you can list them -under the ``formats`` key and refer to them using the ``format`` option. Each -key under ``formats`` should contain values for ``command`` and ``extension`` -as described above:: +You can customize the transcoding command through the ``formats`` map +and select a command with the ``--format`` command-line option or the +``format`` configuration.:: convert: format: speex @@ -109,6 +89,28 @@ as described above:: speex: command: ffmpeg -i $source -y -acodec speex $dest extension: spx - wav: - command: ffmpeg -i $source -y -acodec pcm_s16le $dest - extension: wav + wav: ffmpeg -i $source -y -acodec pcm_s16le $dest + +In this example ``beet convert`` will use the *speex* command by +default. To convert the audio to `wav`, run ``beet convert -f wav``. +This will also use the format key (`wav`) as the file extension. + +Each entry in the ``formats`` map consists of a key (the name of the +format) as well as the command and the possibly the file extension. +``extension`` is the filename extension to be used for newly transcoded +files. If only the command is given as a string, the file extension +defaults to the format’s name. ``command`` is the command-line to use +to transcode audio. The tokens ``$source`` and ``$dest`` in the command +are replaced with the paths to the existing and new file. + +The plugin in comes with default commands for the most common audio +formats: `mp3`, `alac`, `flac`, `aac`, `opus`, `ogg`, `wmv`. For +details have a look at the output of ``beet config -d``. + +For a one-command-fits-all solution use the ``convert.command`` and +``convert.extension`` options. If these are set the formats are ignored +and the given command is used for all conversions.:: + + convert: + command: ffmpeg -i $source -y -vn -aq 2 $dest + extension: mp3 diff --git a/test/helper.py b/test/helper.py index aa9c92add..4bcab88ad 100644 --- a/test/helper.py +++ b/test/helper.py @@ -180,24 +180,35 @@ class TestHelper(object): beets.plugins._instances = {} def create_importer(self, item_count=1, album_count=1): - """Returns import session with fixtures. + """Create files to import and return corresponding session. Copies the specified number of files to a subdirectory of - ``self.temp_dir`` and creates a ``TestImportSession`` for this - path. + `self.temp_dir` and creates a `TestImportSession` for this path. """ import_dir = os.path.join(self.temp_dir, 'import') if not os.path.isdir(import_dir): os.mkdir(import_dir) - for i in range(album_count): - album = u'album {0}'.format(i) + album_no = 0 + while album_count: + album = u'album {0}'.format(album_no) album_dir = os.path.join(import_dir, album) + if os.path.exists(album_dir): + album_no += 1 + continue os.mkdir(album_dir) - for j in range(item_count): - title = 'track {0}'.format(j) + album_count -= 1 + + track_no = 0 + album_item_count = item_count + while album_item_count: + title = 'track {0}'.format(track_no) src = os.path.join(_common.RSRC, 'full.mp3') dest = os.path.join(album_dir, '{0}.mp3'.format(title)) + if os.path.exists(dest): + track_no += 1 + continue + album_item_count -= 1 shutil.copy(src, dest) mediafile = MediaFile(dest) mediafile.update({ diff --git a/test/test_convert.py b/test/test_convert.py index 21de94696..24ed44748 100644 --- a/test/test_convert.py +++ b/test/test_convert.py @@ -72,9 +72,18 @@ class ConvertCliTest(unittest.TestCase, TestHelper): self.load_plugins('convert') self.convert_dest = os.path.join(self.temp_dir, 'convert_dest') - self.config['convert']['dest'] = str(self.convert_dest) - self.config['convert']['command'] = u'cp $source $dest' - self.config['convert']['paths']['default'] = u'converted' + self.config['convert'] = { + 'dest': self.convert_dest, + 'paths': {'default': 'converted'}, + 'format': 'mp3', + 'formats': { + 'mp3': 'cp $source $dest', + 'opus': { + 'command': 'cp $source $dest', + 'extension': 'ops', + } + } + } def tearDown(self): self.unload_plugins() @@ -95,6 +104,12 @@ class ConvertCliTest(unittest.TestCase, TestHelper): self.item.load() self.assertEqual(os.path.splitext(self.item.path)[1], '.mp3') + def test_format_option(self): + with control_stdin('y'): + self.run_command('convert', '--format', 'opus', self.item.path) + converted = os.path.join(self.convert_dest, 'converted.ops') + self.assertTrue(os.path.isfile(converted)) + def test_embed_album_art(self): self.config['convert']['embed'] = True image_path = os.path.join(_common.RSRC, 'image-2x3.jpg') diff --git a/test/test_importer.py b/test/test_importer.py index 4f99e5157..80a94833a 100644 --- a/test/test_importer.py +++ b/test/test_importer.py @@ -1159,6 +1159,45 @@ class ResumeImportTest(unittest.TestCase, TestHelper): self.assertIsNotNone(self.lib.items('title:track 1').get()) +class IncrementalImportTest(unittest.TestCase, TestHelper): + + def setUp(self): + self.setup_beets() + self.config['import']['incremental'] = True + + def tearDown(self): + self.teardown_beets() + + def test_incremental_album(self): + importer = self.create_importer(album_count=1) + importer.run() + + # Change album name so the original file would be imported again + # if incremental was off. + album = self.lib.albums().get() + album['album'] = 'edited album' + album.store() + + importer = self.create_importer(album_count=1) + importer.run() + self.assertEqual(len(self.lib.albums()), 2) + + def test_incremental_item(self): + self.config['import']['singletons'] = True + importer = self.create_importer(item_count=1) + importer.run() + + # Change track name so the original file would be imported again + # if incremental was off. + item = self.lib.items().get() + item['artist'] = 'edited artist' + item.store() + + importer = self.create_importer(item_count=1) + importer.run() + self.assertEqual(len(self.lib.items()), 2) + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__)