From 29d61ca634aeccbe2c8f007c8cfeed4001cdeeff Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 00:11:56 -0800 Subject: [PATCH 01/16] web.exclude_paths_from_items option More exclude_paths_from_items --- beetsplug/web/__init__.py | 9 ++++++++- docs/plugins/web.rst | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index e7b9ec81f..05872f6d2 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -37,7 +37,10 @@ def _rep(obj, expand=False): out = dict(obj) if isinstance(obj, beets.library.Item): - del out['path'] + if app.config['exclude_paths_from_items']: + del out['path'] + else: + out['path'] = out['path'].decode('utf-8') # Get the size (in bytes) of the backing file. This is useful # for the Tomahawk resolver API. @@ -309,6 +312,7 @@ class WebPlugin(BeetsPlugin): 'host': u'127.0.0.1', 'port': 8337, 'cors': '', + 'exclude_paths_from_items': True, }) def commands(self): @@ -327,6 +331,9 @@ class WebPlugin(BeetsPlugin): # Normalizes json output app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False + app.config['exclude_paths_from_items'] = ( + self.config['exclude_paths_from_items']) + # Enable CORS if required. if self.config['cors']: self._log.info(u'Enabling CORS with origin: {0}', diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index f4ae063e0..45000dc26 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -63,6 +63,8 @@ configuration file. The available options are: Default: 8337. - **cors**: The CORS allowed origin (see :ref:`web-cors`, below). Default: CORS is disabled. +- **exclude_paths_from_items**: The 'path' key of items is filtered out of JSON + responses for security reasons. Default: true. Implementation -------------- From 43936cd84c030e2588602c9eb99b47419ca905f8 Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 00:12:07 -0800 Subject: [PATCH 02/16] /item/at_path/ endpoint More at_path /item/by_path docs --- beetsplug/web/__init__.py | 8 ++++++++ docs/plugins/web.rst | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 05872f6d2..88bf299e4 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -221,6 +221,14 @@ def item_query(queries): return g.lib.items(queries) +@app.route('/item/at_path/') +def item_at_path(path): + try: + return flask.jsonify(_rep(beets.library.Item.from_path('/' + path))) + except beets.library.ReadError: + return flask.abort(404) + + @app.route('/item/values/') def item_unique_field_values(key): sort_key = flask.request.args.get('sort_key', key) diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 45000dc26..93a67e7ea 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -162,6 +162,13 @@ response includes all the items requested. If a track is not found it is silentl dropped from the response. +``GET /item/by_path/...`` ++++++++++++++++++++++ + +Look for an item at the given path on the server. If it corresponds to a track, +return the track in the same format as /item/*. + + ``GET /item/query/querystring`` +++++++++++++++++++++++++++++++ From f6cb46d4900b4fe3e8d8453277bf5779e5589eeb Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 09:53:46 -0800 Subject: [PATCH 03/16] Fix broken tests (no new ones yet) --- beetsplug/web/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 88bf299e4..a981ccabc 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -340,7 +340,7 @@ class WebPlugin(BeetsPlugin): app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False app.config['exclude_paths_from_items'] = ( - self.config['exclude_paths_from_items']) + self.config.get('exclude_paths_from_items', True)) # Enable CORS if required. if self.config['cors']: From 50ea74635b365e0d20906572f829c9135c4c81e5 Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 10:08:39 -0800 Subject: [PATCH 04/16] Fix tests I broke --- beetsplug/web/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index a981ccabc..1344b898d 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -37,7 +37,7 @@ def _rep(obj, expand=False): out = dict(obj) if isinstance(obj, beets.library.Item): - if app.config['exclude_paths_from_items']: + if app.config.get('EXCLUDE_PATHS_FROM_ITEMS', True): del out['path'] else: out['path'] = out['path'].decode('utf-8') @@ -339,7 +339,7 @@ class WebPlugin(BeetsPlugin): # Normalizes json output app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False - app.config['exclude_paths_from_items'] = ( + app.config['EXCLUDE_PATHS_FROM_ITEMS'] = ( self.config.get('exclude_paths_from_items', True)) # Enable CORS if required. From cedd93b7786a10308448c33bfd0191729ace03ef Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 10:15:41 -0800 Subject: [PATCH 05/16] Add tests for exclude_paths_from_items --- test/test_web.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/test_web.py b/test/test_web.py index e72ecf33d..d64341f3f 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -21,15 +21,24 @@ class WebPluginTest(_common.LibTestCase): # Add fixtures for track in self.lib.items(): track.remove() - self.lib.add(Item(title=u'title', path='', id=1)) - self.lib.add(Item(title=u'another title', path='', id=2)) + self.lib.add(Item(title=u'title', path='/path_1', id=1)) + self.lib.add(Item(title=u'another title', path='/path_2', id=2)) self.lib.add(Album(album=u'album', id=3)) self.lib.add(Album(album=u'another album', id=4)) web.app.config['TESTING'] = True web.app.config['lib'] = self.lib + web.app.config['EXCLUDE_PATHS_FROM_ITEMS'] = True self.client = web.app.test_client() + def test_config_exclude_paths_from_items(self): + web.app.config['EXCLUDE_PATHS_FROM_ITEMS'] = False + response = self.client.get('/item/1') + response.json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json['path'], u'/path_1') + def test_get_all_items(self): response = self.client.get('/item/') response.json = json.loads(response.data.decode('utf-8')) From c409a71ea1d9eeb5785d75d87d77a6652a0360bf Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 10:31:33 -0800 Subject: [PATCH 06/16] Add missing plus signs to web.rst --- docs/plugins/web.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 93a67e7ea..6bb5b0e4b 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -163,7 +163,7 @@ dropped from the response. ``GET /item/by_path/...`` -+++++++++++++++++++++ ++++++++++++++++++++++++++ Look for an item at the given path on the server. If it corresponds to a track, return the track in the same format as /item/*. From a994df6aa6cce5dff6bee8b08cd8816181b8ffb1 Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 10:34:02 -0800 Subject: [PATCH 07/16] Fix bad doc formatting --- docs/plugins/web.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 6bb5b0e4b..ef5769a2c 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -166,7 +166,7 @@ dropped from the response. +++++++++++++++++++++++++ Look for an item at the given path on the server. If it corresponds to a track, -return the track in the same format as /item/*. +return the track in the same format as ``/item/*``. ``GET /item/query/querystring`` From 05bc4996a832fa9be7f20e2c5c5f8794001d91e0 Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 11:08:24 -0800 Subject: [PATCH 08/16] Rename and invert new config option --- beetsplug/web/__init__.py | 12 ++++++------ docs/plugins/web.rst | 4 ++-- test/test_web.py | 14 +++++++++++--- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 1344b898d..4d5dcba54 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -37,10 +37,10 @@ def _rep(obj, expand=False): out = dict(obj) if isinstance(obj, beets.library.Item): - if app.config.get('EXCLUDE_PATHS_FROM_ITEMS', True): - del out['path'] - else: + if app.config.get('INCLUDE_PATHS', False): out['path'] = out['path'].decode('utf-8') + else: + del out['path'] # Get the size (in bytes) of the backing file. This is useful # for the Tomahawk resolver API. @@ -320,7 +320,7 @@ class WebPlugin(BeetsPlugin): 'host': u'127.0.0.1', 'port': 8337, 'cors': '', - 'exclude_paths_from_items': True, + 'include_paths': False, }) def commands(self): @@ -339,8 +339,8 @@ class WebPlugin(BeetsPlugin): # Normalizes json output app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False - app.config['EXCLUDE_PATHS_FROM_ITEMS'] = ( - self.config.get('exclude_paths_from_items', True)) + app.config['INCLUDE_PATHS'] = ( + self.config.get('include_paths', False)) # Enable CORS if required. if self.config['cors']: diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index ef5769a2c..54c507cf0 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -63,8 +63,8 @@ configuration file. The available options are: Default: 8337. - **cors**: The CORS allowed origin (see :ref:`web-cors`, below). Default: CORS is disabled. -- **exclude_paths_from_items**: The 'path' key of items is filtered out of JSON - responses for security reasons. Default: true. +- **include_paths**: If true, includes paths in item objects. + Default: false. Implementation -------------- diff --git a/test/test_web.py b/test/test_web.py index d64341f3f..871fd1b09 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -28,17 +28,25 @@ class WebPluginTest(_common.LibTestCase): web.app.config['TESTING'] = True web.app.config['lib'] = self.lib - web.app.config['EXCLUDE_PATHS_FROM_ITEMS'] = True + web.app.config['INCLUDE_PATHS'] = False self.client = web.app.test_client() - def test_config_exclude_paths_from_items(self): - web.app.config['EXCLUDE_PATHS_FROM_ITEMS'] = False + def test_config_include_paths_true(self): + web.app.config['INCLUDE_PATHS'] = True response = self.client.get('/item/1') response.json = json.loads(response.data.decode('utf-8')) self.assertEqual(response.status_code, 200) self.assertEqual(response.json['path'], u'/path_1') + def test_config_include_paths_false(self): + web.app.config['INCLUDE_PATHS'] = False + response = self.client.get('/item/1') + response.json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertNotIn('path', response.json) + def test_get_all_items(self): response = self.client.get('/item/') response.json = json.loads(response.data.decode('utf-8')) From 866a650bc0d7d3e48484afca2855dd0828e2683d Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 11:15:18 -0800 Subject: [PATCH 09/16] Rename /item/by_path to /item/path and use PathQuery instead of direct file access --- beetsplug/web/__init__.py | 14 ++++++++------ docs/plugins/web.rst | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 4d5dcba54..d47f1d183 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -221,11 +221,14 @@ def item_query(queries): return g.lib.items(queries) -@app.route('/item/at_path/') +@app.route('/item/path/') def item_at_path(path): - try: - return flask.jsonify(_rep(beets.library.Item.from_path('/' + path))) - except beets.library.ReadError: + g.lib._connection().create_function('bytelower', 1, beets.library._sqlite_bytelower) + query = beets.library.PathQuery('path', u'/' + path) + item = g.lib.items(query).get() + if item: + return flask.jsonify(_rep(item)) + else: return flask.abort(404) @@ -339,8 +342,7 @@ class WebPlugin(BeetsPlugin): # Normalizes json output app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False - app.config['INCLUDE_PATHS'] = ( - self.config.get('include_paths', False)) + app.config['INCLUDE_PATHS'] = self.config['include_paths'] # Enable CORS if required. if self.config['cors']: diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 54c507cf0..f0adacc00 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -162,8 +162,8 @@ response includes all the items requested. If a track is not found it is silentl dropped from the response. -``GET /item/by_path/...`` -+++++++++++++++++++++++++ +``GET /item/path/...`` +++++++++++++++++++++++ Look for an item at the given path on the server. If it corresponds to a track, return the track in the same format as ``/item/*``. From 4434569ddc9b55c6799e8ef56583185ec9d3b4d6 Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 11:28:42 -0800 Subject: [PATCH 10/16] beets.library.Library adds custom bytelower function to all connections, not just one --- beetsplug/web/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index d47f1d183..360e018e6 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -223,7 +223,6 @@ def item_query(queries): @app.route('/item/path/') def item_at_path(path): - g.lib._connection().create_function('bytelower', 1, beets.library._sqlite_bytelower) query = beets.library.PathQuery('path', u'/' + path) item = g.lib.items(query).get() if item: From d8fbdbc16ad4cd1ce3606042c5400001229207fa Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 11:36:57 -0800 Subject: [PATCH 11/16] Update changelog --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index eaf8c7d5a..be7020dc1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,10 @@ New features: non-numeric track index data. For example, some vinyl or tape media will report the side of the record using a letter instead of a number in that field. :bug:`1831` :bug:`2363` +* The :doc:`/plugins/web` has a new endpoint, ``/item/path/foo``, which will + return the item info for the file at the given path, or 404. +* The :doc:`/plugins/web` also has a new config option, ``include_paths``, + which will cause paths to be included in item API responses if set to true. Fixes: From 6b7a6baaf21ad3d017a0d940b7fcaa10c450a039 Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 12:12:20 -0800 Subject: [PATCH 12/16] Add test for /item/path/ endpoint --- test/test_web.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test/test_web.py b/test/test_web.py index 871fd1b09..867e2c3e3 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -4,11 +4,12 @@ from __future__ import division, absolute_import, print_function +import json import unittest +import os.path from six import assertCountEqual from test import _common -import json from beets.library import Item, Album from beetsplug import web @@ -75,6 +76,23 @@ class WebPluginTest(_common.LibTestCase): response = self.client.get('/item/3') self.assertEqual(response.status_code, 404) + def test_get_single_item_by_path(self): + data_path = os.path.join(_common.RSRC, b'full.mp3') + self.lib.add(Item.from_path(data_path)) + response = self.client.get('/item/path' + data_path.decode('utf-8')) + response.json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json['title'], u'full') + + def test_get_single_item_by_path_not_found_if_not_in_library(self): + data_path = os.path.join(_common.RSRC, b'full.mp3') + # data_path points to a valid file, but we have not added the file + # to the library. + response = self.client.get('/item/path' + data_path.decode('utf-8')) + + self.assertEqual(response.status_code, 404) + def test_get_item_empty_query(self): response = self.client.get('/item/query/') response.json = json.loads(response.data.decode('utf-8')) From e2be6ba7814dafc30f9a025dacf8769687ef26c5 Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 12:44:07 -0800 Subject: [PATCH 13/16] Query path with bytestring. Might fix tests. --- beetsplug/web/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 360e018e6..7977eda94 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -223,7 +223,7 @@ def item_query(queries): @app.route('/item/path/') def item_at_path(path): - query = beets.library.PathQuery('path', u'/' + path) + query = beets.library.PathQuery('path', b'/' + path.encode('utf-8')) item = g.lib.items(query).get() if item: return flask.jsonify(_rep(item)) From e3707e45f3308ab7767dd6e690a388004477b43e Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 20:40:30 -0800 Subject: [PATCH 14/16] Maybe fix code and tests for Windows --- beetsplug/web/__init__.py | 9 +++++++-- test/test_web.py | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 7977eda94..f955b6d63 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -176,11 +176,16 @@ class QueryConverter(PathConverter): return ','.join(value) +class EverythingConverter(PathConverter): + regex = '.*?' + + # Flask setup. app = flask.Flask(__name__) app.url_map.converters['idlist'] = IdListConverter app.url_map.converters['query'] = QueryConverter +app.url_map.converters['everything'] = EverythingConverter @app.before_request @@ -221,9 +226,9 @@ def item_query(queries): return g.lib.items(queries) -@app.route('/item/path/') +@app.route('/item/path/') def item_at_path(path): - query = beets.library.PathQuery('path', b'/' + path.encode('utf-8')) + query = beets.library.PathQuery('path', path.encode('utf-8')) item = g.lib.items(query).get() if item: return flask.jsonify(_rep(item)) diff --git a/test/test_web.py b/test/test_web.py index 867e2c3e3..98347d8af 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -79,7 +79,7 @@ class WebPluginTest(_common.LibTestCase): def test_get_single_item_by_path(self): data_path = os.path.join(_common.RSRC, b'full.mp3') self.lib.add(Item.from_path(data_path)) - response = self.client.get('/item/path' + data_path.decode('utf-8')) + response = self.client.get('/item/path/' + data_path.decode('utf-8')) response.json = json.loads(response.data.decode('utf-8')) self.assertEqual(response.status_code, 200) @@ -89,7 +89,7 @@ class WebPluginTest(_common.LibTestCase): data_path = os.path.join(_common.RSRC, b'full.mp3') # data_path points to a valid file, but we have not added the file # to the library. - response = self.client.get('/item/path' + data_path.decode('utf-8')) + response = self.client.get('/item/path/' + data_path.decode('utf-8')) self.assertEqual(response.status_code, 404) From 1426aea4a28d63d2228692a34678f4bf8071c05d Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sat, 14 Jan 2017 21:06:38 -0800 Subject: [PATCH 15/16] Update docs --- docs/plugins/web.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index f0adacc00..9f1bbcd99 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -165,8 +165,11 @@ dropped from the response. ``GET /item/path/...`` ++++++++++++++++++++++ -Look for an item at the given path on the server. If it corresponds to a track, -return the track in the same format as ``/item/*``. +Look for an item at the given absolute path on the server. If it corresponds to +a track, return the track in the same format as ``/item/*``. + +If the server runs UNIX, you'll need to include an extra leading slash: +``http://localhost:8337/item/path//Users/beets/Music/Foo/Bar/Baz.mp3`` ``GET /item/query/querystring`` From 926dce241ca759d407dea4e14ca624f827cb0591 Mon Sep 17 00:00:00 2001 From: Steve Johnson Date: Sun, 15 Jan 2017 11:25:03 -0800 Subject: [PATCH 16/16] Use util.displayable_path instead of naive .decode() --- beetsplug/web/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index f955b6d63..bd4677bd8 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -38,7 +38,7 @@ def _rep(obj, expand=False): if isinstance(obj, beets.library.Item): if app.config.get('INCLUDE_PATHS', False): - out['path'] = out['path'].decode('utf-8') + out['path'] = util.displayable_path(out['path']) else: del out['path']