diff --git a/beets/util/pipeline.py b/beets/util/pipeline.py index 39bc7152e..71ac8ffe6 100644 --- a/beets/util/pipeline.py +++ b/beets/util/pipeline.py @@ -247,9 +247,6 @@ class FirstPipelineThread(PipelineThread): self.out_queue = out_queue self.out_queue.acquire() - self.abort_lock = Lock() - self.abort_flag = False - def run(self): try: while True: diff --git a/beetsplug/chroma.py b/beetsplug/chroma.py index 20d0f5479..91f2fe253 100644 --- a/beetsplug/chroma.py +++ b/beetsplug/chroma.py @@ -329,7 +329,7 @@ def fingerprint_item(log, item, write=False): else: log.info(u'{0}: using existing fingerprint', util.displayable_path(item.path)) - return item.acoustid_fingerprint + return item.acoustid_fingerprint else: log.info(u'{0}: fingerprinting', util.displayable_path(item.path)) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 70363f6eb..275703e97 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -16,6 +16,7 @@ """Converts tracks or albums to external directory """ from __future__ import division, absolute_import, print_function +from beets.util import par_map import os import threading @@ -183,8 +184,8 @@ class ConvertPlugin(BeetsPlugin): def auto_convert(self, config, task): if self.config['auto']: - for item in task.imported_items(): - self.convert_on_import(config.lib, item) + par_map(lambda item: self.convert_on_import(config.lib, item), + task.imported_items()) # Utilities converted from functions to methods on logging overhaul diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 9d6fa23c4..5060c8efe 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -15,26 +15,23 @@ from __future__ import division, absolute_import, print_function -import subprocess -import os import collections +import enum import math +import os +import signal +import six +import subprocess import sys import warnings -import enum -import re -import xml.parsers.expat -from six.moves import zip, queue -import six - from multiprocessing.pool import ThreadPool, RUN +from six.moves import zip, queue from threading import Thread, Event -import signal from beets import ui from beets.plugins import BeetsPlugin -from beets.util import (syspath, command_output, bytestring_path, - displayable_path, py3_path, cpu_count) +from beets.util import (syspath, command_output, displayable_path, + py3_path, cpu_count) # Utilities. @@ -136,252 +133,6 @@ class Backend(object): raise NotImplementedError() -# bsg1770gain backend -class Bs1770gainBackend(Backend): - """bs1770gain is a loudness scanner compliant with ITU-R BS.1770 and - its flavors EBU R128, ATSC A/85 and Replaygain 2.0. - """ - - methods = { - -24: "atsc", - -23: "ebu", - -18: "replaygain", - } - - do_parallel = True - - def __init__(self, config, log): - super(Bs1770gainBackend, self).__init__(config, log) - config.add({ - 'chunk_at': 5000, - 'method': '', - }) - self.chunk_at = config['chunk_at'].as_number() - # backward compatibility to `method` config option - self.__method = config['method'].as_str() - - cmd = 'bs1770gain' - try: - version_out = call([cmd, '--version']) - self.command = cmd - self.version = re.search( - 'bs1770gain ([0-9]+.[0-9]+.[0-9]+), ', - version_out.stdout.decode('utf-8') - ).group(1) - except OSError: - raise FatalReplayGainError( - u'Is bs1770gain installed?' - ) - if not self.command: - raise FatalReplayGainError( - u'no replaygain command found: install bs1770gain' - ) - - def compute_track_gain(self, items, target_level, peak): - """Computes the track gain of the given tracks, returns a list - of TrackGain objects. - """ - - output = self.compute_gain(items, target_level, False) - return output - - def compute_album_gain(self, items, target_level, peak): - """Computes the album gain of the given album, returns an - AlbumGain object. - """ - # TODO: What should be done when not all tracks in the album are - # supported? - - output = self.compute_gain(items, target_level, True) - - if not output: - raise ReplayGainError(u'no output from bs1770gain') - return AlbumGain(output[-1], output[:-1]) - - def isplitter(self, items, chunk_at): - """Break an iterable into chunks of at most size `chunk_at`, - generating lists for each chunk. - """ - iterable = iter(items) - while True: - result = [] - for i in range(chunk_at): - try: - a = next(iterable) - except StopIteration: - break - else: - result.append(a) - if result: - yield result - else: - break - - def compute_gain(self, items, target_level, is_album): - """Computes the track or album gain of a list of items, returns - a list of TrackGain objects. - When computing album gain, the last TrackGain object returned is - the album gain - """ - - if len(items) == 0: - return [] - - albumgaintot = 0.0 - albumpeaktot = 0.0 - returnchunks = [] - - # In the case of very large sets of music, we break the tracks - # into smaller chunks and process them one at a time. This - # avoids running out of memory. - if len(items) > self.chunk_at: - i = 0 - for chunk in self.isplitter(items, self.chunk_at): - i += 1 - returnchunk = self.compute_chunk_gain( - chunk, - is_album, - target_level - ) - albumgaintot += returnchunk[-1].gain - albumpeaktot = max(albumpeaktot, returnchunk[-1].peak) - returnchunks = returnchunks + returnchunk[0:-1] - returnchunks.append(Gain(albumgaintot / i, albumpeaktot)) - return returnchunks - else: - return self.compute_chunk_gain(items, is_album, target_level) - - def compute_chunk_gain(self, items, is_album, target_level): - """Compute ReplayGain values and return a list of results - dictionaries as given by `parse_tool_output`. - """ - # choose method - target_level = db_to_lufs(target_level) - if self.__method != "": - # backward compatibility to `method` option - method = self.__method - gain_adjustment = target_level \ - - [k for k, v in self.methods.items() if v == method][0] - elif target_level in self.methods: - method = self.methods[target_level] - gain_adjustment = 0 - else: - lufs_target = -23 - method = self.methods[lufs_target] - gain_adjustment = target_level - lufs_target - - # Construct shell command. - cmd = [self.command] - cmd += ["--" + method] - cmd += ['--xml', '-p'] - if after_version(self.version, '0.6.0'): - cmd += ['--unit=ebu'] # set units to LU - cmd += ['--suppress-progress'] # don't print % to XML output - - # Workaround for Windows: the underlying tool fails on paths - # with the \\?\ prefix, so we don't use it here. This - # prevents the backend from working with long paths. - args = cmd + [syspath(i.path, prefix=False) for i in items] - path_list = [i.path for i in items] - - # Invoke the command. - self._log.debug( - u'executing {0}', u' '.join(map(displayable_path, args)) - ) - output = call(args).stdout - - self._log.debug(u'analysis finished: {0}', output) - results = self.parse_tool_output(output, path_list, is_album) - - if gain_adjustment: - results = [ - Gain(res.gain + gain_adjustment, res.peak) - for res in results - ] - - self._log.debug(u'{0} items, {1} results', len(items), len(results)) - return results - - def parse_tool_output(self, text, path_list, is_album): - """Given the output from bs1770gain, parse the text and - return a list of dictionaries - containing information about each analyzed file. - """ - per_file_gain = {} - album_gain = {} # mutable variable so it can be set from handlers - parser = xml.parsers.expat.ParserCreate(encoding='utf-8') - state = {'file': None, 'gain': None, 'peak': None} - album_state = {'gain': None, 'peak': None} - - def start_element_handler(name, attrs): - if name == u'track': - state['file'] = bytestring_path(attrs[u'file']) - if state['file'] in per_file_gain: - raise ReplayGainError( - u'duplicate filename in bs1770gain output') - elif name == u'integrated': - if 'lu' in attrs: - state['gain'] = float(attrs[u'lu']) - elif name == u'sample-peak': - if 'factor' in attrs: - state['peak'] = float(attrs[u'factor']) - elif 'amplitude' in attrs: - state['peak'] = float(attrs[u'amplitude']) - - def end_element_handler(name): - if name == u'track': - if state['gain'] is None or state['peak'] is None: - raise ReplayGainError(u'could not parse gain or peak from ' - 'the output of bs1770gain') - per_file_gain[state['file']] = Gain(state['gain'], - state['peak']) - state['gain'] = state['peak'] = None - elif name == u'summary': - if state['gain'] is None or state['peak'] is None: - raise ReplayGainError(u'could not parse gain or peak from ' - 'the output of bs1770gain') - album_gain["album"] = Gain(state['gain'], state['peak']) - state['gain'] = state['peak'] = None - elif len(per_file_gain) == len(path_list): - if state['gain'] is not None: - album_state['gain'] = state['gain'] - if state['peak'] is not None: - album_state['peak'] = state['peak'] - if album_state['gain'] is not None \ - and album_state['peak'] is not None: - album_gain["album"] = Gain( - album_state['gain'], album_state['peak']) - state['gain'] = state['peak'] = None - - parser.StartElementHandler = start_element_handler - parser.EndElementHandler = end_element_handler - - try: - parser.Parse(text, True) - except xml.parsers.expat.ExpatError: - raise ReplayGainError( - u'The bs1770gain tool produced malformed XML. ' - u'Using version >=0.4.10 may solve this problem.') - - if len(per_file_gain) != len(path_list): - raise ReplayGainError( - u'the number of results returned by bs1770gain does not match ' - 'the number of files passed to it') - - # bs1770gain does not return the analysis results in the order that - # files are passed on the command line, because it is sorting the files - # internally. We must recover the order from the filenames themselves. - try: - out = [per_file_gain[os.path.basename(p)] for p in path_list] - except KeyError: - raise ReplayGainError( - u'unrecognized filename in bs1770gain output ' - '(bs1770gain can only deal with utf-8 file names)') - if is_album: - out.append(album_gain["album"]) - return out - - # ffmpeg backend class FfmpegBackend(Backend): """A replaygain backend using ffmpeg's ebur128 filter. @@ -1216,7 +967,6 @@ class ReplayGainPlugin(BeetsPlugin): "command": CommandBackend, "gstreamer": GStreamerBackend, "audiotools": AudioToolsBackend, - "bs1770gain": Bs1770gainBackend, "ffmpeg": FfmpegBackend, } diff --git a/beetsplug/subsonicupdate.py b/beetsplug/subsonicupdate.py index 004439bac..04e903512 100644 --- a/beetsplug/subsonicupdate.py +++ b/beetsplug/subsonicupdate.py @@ -29,26 +29,46 @@ import string import requests +from binascii import hexlify from beets import config from beets.plugins import BeetsPlugin __author__ = 'https://github.com/maffo999' +AUTH_TOKEN_VERSION = (1, 12) class SubsonicUpdate(BeetsPlugin): def __init__(self): super(SubsonicUpdate, self).__init__() - # Set default configuration values config['subsonic'].add({ 'user': 'admin', 'pass': 'admin', 'url': 'http://localhost:4040', }) - config['subsonic']['pass'].redact = True + self._version = None + self._auth = None self.register_listener('import', self.start_scan) + @property + def version(self): + if self._version is None: + self._version = self.__get_version() + return self._version + + @property + def auth(self): + if self._auth is None: + if self.version is not None: + if self.version > AUTH_TOKEN_VERSION: + self._auth = "token" + else: + self._auth = "password" + self._log.info( + u"using '{}' authentication method".format(self._auth)) + return self._auth + @staticmethod def __create_token(): """Create salt and token from given password. @@ -67,10 +87,10 @@ class SubsonicUpdate(BeetsPlugin): return salt, token @staticmethod - def __format_url(): - """Get the Subsonic URL to trigger a scan. Uses either the url - config option or the deprecated host, port, and context_path config - options together. + def __format_url(endpoint): + """Get the Subsonic URL to trigger the given endpoint. + Uses either the url config option or the deprecated host, port, + and context_path config options together. :return: Endpoint for updating Subsonic """ @@ -88,22 +108,55 @@ class SubsonicUpdate(BeetsPlugin): context_path = '' url = "http://{}:{}{}".format(host, port, context_path) - return url + '/rest/startScan' - - def start_scan(self): - user = config['subsonic']['user'].as_str() - url = self.__format_url() - salt, token = self.__create_token() + return url + '/rest/{}'.format(endpoint) + def __get_version(self): + url = self.__format_url("ping.view") payload = { - 'u': user, - 't': token, - 's': salt, - 'v': '1.15.0', # Subsonic 6.1 and newer. 'c': 'beets', 'f': 'json' } + try: + response = requests.get(url, params=payload) + if response.status_code == 200: + json = response.json() + version = json['subsonic-response']['version'] + self._log.info( + u'subsonic version:{0} '.format(version)) + return tuple(int(s) for s in version.split('.')) + else: + self._log.error(u'Error: {0}', json) + return None + except Exception as error: + self._log.error(u'Error: {0}'.format(error)) + return None + def start_scan(self): + user = config['subsonic']['user'].as_str() + url = self.__format_url("startScan.view") + + if self.auth == 'token': + salt, token = self.__create_token() + payload = { + 'u': user, + 't': token, + 's': salt, + 'v': self.version, # Subsonic 6.1 and newer. + 'c': 'beets', + 'f': 'json' + } + elif self.auth == 'password': + password = config['subsonic']['pass'].as_str() + encpass = hexlify(password.encode()).decode() + payload = { + 'u': user, + 'p': 'enc:{}'.format(encpass), + 'v': self.version, + 'c': 'beets', + 'f': 'json' + } + else: + return try: response = requests.get(url, params=payload) json = response.json() diff --git a/docs/changelog.rst b/docs/changelog.rst index 0caf9fd38..18885e58d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,9 @@ Changelog New features: +* Submitting acoustID information on tracks which already have a fingerprint + :bug:`3834` +* conversion uses par_map to parallelize conversion jobs in python3 * Add ``title_case`` config option to lastgenre to make TitleCasing optional. * When config is printed with no available configuration a new message is printed. :bug:`3779` @@ -14,6 +17,8 @@ New features: * :doc:`/plugins/chroma`: Update file metadata after generating fingerprints through the `submit` command. * :doc:`/plugins/lastgenre`: Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres to genres.txt and genres-tree.yaml * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. +* :doc:`/plugins/subsonicupdate`: Automatically choose between token and + password-based authentication based on server version * A new :ref:`reflink` config option instructs the importer to create fast, copy-on-write file clones on filesystems that support them. Thanks to :user:`rubdos`. @@ -59,14 +64,12 @@ New features: Thanks to :user:`samuelnilsson` :bug:`293` * :doc:`/plugins/replaygain`: The new ``ffmpeg`` ReplayGain backend supports - ``R128_`` tags, just like the ``bs1770gain`` backend. + ``R128_`` tags. :bug:`3056` * :doc:`plugins/replaygain`: ``r128_targetlevel`` is a new configuration option for the ReplayGain plugin: It defines the reference volume for files using ``R128_`` tags. ``targetlevel`` only configures the reference volume for ``REPLAYGAIN_`` files. - This also deprecates the ``bs1770gain`` ReplayGain backend's ``method`` - option. Use ``targetlevel`` and ``r128_targetlevel`` instead. :bug:`3065` * A new :doc:`/plugins/parentwork` gets information about the original work, which is useful for classical music. @@ -171,10 +174,12 @@ New features: https://github.com/alastair/python-musicbrainzngs/pull/266 . Thanks to :user:`aereaux`. * :doc:`/plugins/replaygain` now does its analysis in parallel when using - the ``command``, ``ffmpeg`` or ``bs1770gain`` backends. + the ``command`` or ``ffmpeg`` backends. :bug:`3478` -* Add ``extracting_albumdata`` and ``extracting_trackdata`` hooks to allow +* Add ``mb_album_extract`` and ``mb_track_extract`` hooks to allow plugins to add new fields based on MusicBrainz data. Thanks to :user:`dosoe`. +* Removes usage of the bs1770gain replaygain backend. + Thanks to :user:`SamuelCook`. Fixes: @@ -236,8 +241,6 @@ Fixes: :bug:`3437` * :doc:`/plugins/lyrics`: Fix a corner-case with Genius lowercase artist names :bug:`3446` -* :doc:`/plugins/replaygain`: Support ``bs1770gain`` v0.6.0 and up - :bug:`3480` * :doc:`/plugins/parentwork`: Don't save tracks when nothing has changed. :bug:`3492` * Added a warning when configuration files defined in the `include` directive @@ -349,6 +352,7 @@ For packagers: or `repair `_ the test may no longer be necessary. * This version drops support for Python 3.4. +* Removes the optional dependency on bs1770gain. .. _Fish shell: https://fishshell.com/ .. _MediaFile: https://github.com/beetbox/mediafile diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 0100b520e..53daafbb9 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -16,18 +16,14 @@ from __future__ import division, absolute_import, print_function -import unittest import six - -from mock import patch -from test.helper import TestHelper, capture_log, has_program +import unittest +from mediafile import MediaFile from beets import config -from beets.util import CommandOutput -from mediafile import MediaFile from beetsplug.replaygain import (FatalGstreamerPluginReplayGainError, GStreamerBackend) - +from test.helper import TestHelper, has_program try: import gi @@ -41,11 +37,6 @@ if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']): else: GAIN_PROG_AVAILABLE = False -if has_program('bs1770gain'): - LOUDNESS_PROG_AVAILABLE = True -else: - LOUDNESS_PROG_AVAILABLE = False - FFMPEG_AVAILABLE = has_program('ffmpeg', ['-version']) @@ -153,9 +144,7 @@ class ReplayGainCliTestBase(TestHelper): self.assertEqual(max(gains), min(gains)) self.assertNotEqual(max(gains), 0.0) - if not self.backend == "bs1770gain": - # Actually produces peaks == 0.0 ~ self.add_album_fixture - self.assertNotEqual(max(peaks), 0.0) + self.assertNotEqual(max(peaks), 0.0) def test_cli_writes_only_r128_tags(self): if self.backend == "command": @@ -219,62 +208,6 @@ class ReplayGainCmdCliTest(ReplayGainCliTestBase, unittest.TestCase): backend = u'command' -@unittest.skipIf(not LOUDNESS_PROG_AVAILABLE, u'bs1770gain cannot be found') -class ReplayGainLdnsCliTest(ReplayGainCliTestBase, unittest.TestCase): - backend = u'bs1770gain' - - -class ReplayGainLdnsCliMalformedTest(TestHelper, unittest.TestCase): - @patch('beetsplug.replaygain.call') - def setUp(self, call_patch): - self.setup_beets() - self.config['replaygain']['backend'] = 'bs1770gain' - - # Patch call to return nothing, bypassing the bs1770gain installation - # check. - call_patch.return_value = CommandOutput( - stdout=b'bs1770gain 0.0.0, ', stderr=b'' - ) - try: - self.load_plugins('replaygain') - except Exception: - import sys - exc_info = sys.exc_info() - try: - self.tearDown() - except Exception: - pass - six.reraise(exc_info[1], None, exc_info[2]) - - for item in self.add_album_fixture(2).items(): - reset_replaygain(item) - - def tearDown(self): - self.teardown_beets() - self.unload_plugins() - - @patch('beetsplug.replaygain.call') - def test_malformed_output(self, call_patch): - # Return malformed XML (the ampersand should be &) - call_patch.return_value = CommandOutput(stdout=b""" - - - - - - - """, stderr="") - - with capture_log('beets.replaygain') as logs: - self.run_command('replaygain') - - # Count how many lines match the expected error. - matching = [line for line in logs if - 'malformed XML' in line] - - self.assertEqual(len(matching), 2) - - @unittest.skipIf(not FFMPEG_AVAILABLE, u'ffmpeg cannot be found') class ReplayGainFfmpegTest(ReplayGainCliTestBase, unittest.TestCase): backend = u'ffmpeg' diff --git a/test/test_subsonicupdate.py b/test/test_subsonicupdate.py index c47208e65..dd254d593 100644 --- a/test/test_subsonicupdate.py +++ b/test/test_subsonicupdate.py @@ -39,9 +39,21 @@ class SubsonicPluginTest(_common.TestCase, TestHelper): config["subsonic"]["user"] = "admin" config["subsonic"]["pass"] = "admin" config["subsonic"]["url"] = "http://localhost:4040" - + responses.add( + responses.GET, + 'http://localhost:4040/rest/ping.view', + status=200, + body=self.PING_BODY + ) self.subsonicupdate = subsonicupdate.SubsonicUpdate() - + PING_BODY = ''' +{ + "subsonic-response": { + "status": "failed", + "version": "1.15.0" + } +} +''' SUCCESS_BODY = ''' { "subsonic-response": {