From 32bd7914e558a3da0dc13ac135bb97cfc984287d Mon Sep 17 00:00:00 2001 From: Malte Ried Date: Tue, 1 Sep 2015 10:57:02 +0200 Subject: [PATCH 1/7] Implemented a much more robust way to check for a case sensitive filesystem --- beets/library.py | 13 +++++++++++-- docs/changelog.rst | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/beets/library.py b/beets/library.py index 85c6e1b40..ca3428683 100644 --- a/beets/library.py +++ b/beets/library.py @@ -61,9 +61,8 @@ class PathQuery(dbcore.FieldQuery): """ super(PathQuery, self).__init__(field, pattern, fast) - # By default, the case sensitivity depends on the platform. if case_sensitive is None: - case_sensitive = platform.system() != 'Windows' + case_sensitive = self.is_filesystem_case_sensitive() self.case_sensitive = case_sensitive # Use a normalized-case pattern for case-insensitive matches. @@ -75,6 +74,16 @@ class PathQuery(dbcore.FieldQuery): # As a directory (prefix). self.dir_path = util.bytestring_path(os.path.join(self.file_path, b'')) + @staticmethod + def is_filesystem_case_sensitive(): + library_path = beets.config['directory'].get() + if os.path.exists(library_path): + # Check if the path to the library exists in lower and upper case + return not (os.path.exists(library_path.lower()) and + os.path.exists(library_path.upper())) + # By default, the case sensitivity depends on the platform. + return platform.system() != 'Windows' + @classmethod def is_path_query(cls, query_part): """Try to guess whether a unicode query part is a path query. diff --git a/docs/changelog.rst b/docs/changelog.rst index 344702e34..a9e92709a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,8 @@ Fixes: option. * The :ref:`list-cmd` command's help output now has a small query and format string example. Thanks to :user:`pkess`. :bug:`1582` +* The check whether the file system is case sensitive or not could lead to + wrong results. It is much more robust now. 1.3.14 (August 2, 2015) From 6ad53e146617d4302ba987b077e364ce1b7df098 Mon Sep 17 00:00:00 2001 From: Malte Ried Date: Wed, 2 Sep 2015 21:17:03 +0200 Subject: [PATCH 2/7] Improved version of case sensitivity checking and move method to beets.util --- beets/library.py | 14 ++------------ beets/util/__init__.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/beets/library.py b/beets/library.py index ca3428683..0f9fddafe 100644 --- a/beets/library.py +++ b/beets/library.py @@ -24,7 +24,6 @@ import unicodedata import time import re from unidecode import unidecode -import platform from beets import logging from beets.mediafile import MediaFile, MutagenError, UnreadableFileError @@ -62,7 +61,8 @@ class PathQuery(dbcore.FieldQuery): super(PathQuery, self).__init__(field, pattern, fast) if case_sensitive is None: - case_sensitive = self.is_filesystem_case_sensitive() + case_sensitive = beets.util.is_filesystem_case_sensitive( + beets.config['directory'].get()) self.case_sensitive = case_sensitive # Use a normalized-case pattern for case-insensitive matches. @@ -74,16 +74,6 @@ class PathQuery(dbcore.FieldQuery): # As a directory (prefix). self.dir_path = util.bytestring_path(os.path.join(self.file_path, b'')) - @staticmethod - def is_filesystem_case_sensitive(): - library_path = beets.config['directory'].get() - if os.path.exists(library_path): - # Check if the path to the library exists in lower and upper case - return not (os.path.exists(library_path.lower()) and - os.path.exists(library_path.upper())) - # By default, the case sensitivity depends on the platform. - return platform.system() != 'Windows' - @classmethod def is_path_query(cls, query_part): """Try to guess whether a unicode query part is a path query. diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 61a95baa2..f0304fc45 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -759,3 +759,41 @@ def interactive_open(target, command=None): command.append(target) return os.execlp(*command) + + +def is_filesystem_case_sensitive(path): + if os.path.exists(path): + # Check if the path to the library exists in lower and upper case + if os.path.exists(path.lower()) and \ + os.path.exists(path.upper()): + # All the paths may exist on the file system. Check if they + # refer to different files + if platform.system() != 'Windows': + # os.path.samefile is only available on Unix systems for + # python < 3.0 + return not os.path.samefile(path.lower(), + path.upper()) + # On windows we create a file with a lower case name + # and try to find the file using an upper case name. + base_file_name = 'beetsCaseSensitivityCheck' + lower = os.path.join(path, base_file_name.lower()) + upper = os.path.join(path, base_file_name.upper()) + # Check if one of the files (upper and lower case) do exist + # already + i = 0 + while os.path.exists(lower) or os.path.exists(upper): + file_name = '{0}{1}'.format(base_file_name, i) + lower = os.path.join(path, file_name.lower()) + upper = os.path.join(path, file_name.upper()) + # Create the file using a lower case name + with open(lower, 'w'): + pass + # Check if the file can be found using the upper cas name + case_sensitive = not os.path.exists(upper) + # Remove the temporary file + os.remove(lower) + return case_sensitive + else: + return True + # By default, the case sensitivity depends on the platform. + return platform.system() != 'Windows' From cd56286e86d93da86a9804513460bfd11b392f72 Mon Sep 17 00:00:00 2001 From: Malte Ried Date: Thu, 3 Sep 2015 22:10:32 +0200 Subject: [PATCH 3/7] Added some documentation --- beets/library.py | 2 ++ beets/util/__init__.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/beets/library.py b/beets/library.py index 0f9fddafe..9d2a1b857 100644 --- a/beets/library.py +++ b/beets/library.py @@ -60,6 +60,8 @@ class PathQuery(dbcore.FieldQuery): """ super(PathQuery, self).__init__(field, pattern, fast) + # By default, the case sensitivity depends on the filesystem + # the library is located on. if case_sensitive is None: case_sensitive = beets.util.is_filesystem_case_sensitive( beets.config['directory'].get()) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index f0304fc45..ed1320807 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -762,6 +762,13 @@ def interactive_open(target, command=None): def is_filesystem_case_sensitive(path): + """Checks if the filesystem at the given path is case sensitive. + If the path does not exist, a case sensitive file system is + assumed if the system is not windows. + + :param path: The path to check for case sensitivity. + :return: True if the file system is case sensitive, False else. + """ if os.path.exists(path): # Check if the path to the library exists in lower and upper case if os.path.exists(path.lower()) and \ From 58ddecf7af017b3ea7774e436a71c7ce9f096e08 Mon Sep 17 00:00:00 2001 From: Malte Ried Date: Fri, 4 Sep 2015 18:34:43 +0200 Subject: [PATCH 4/7] Added a Patch decorator for the os.path.samefile function to repair some previously failed test cases. --- test/test_query.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 04235906d..8507eb312 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -376,12 +376,17 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): self.i.store() self.lib.add_album([self.i]) - self.patcher = patch('beets.library.os.path.exists') - self.patcher.start().return_value = True + self.patcher_exists = patch('beets.library.os.path.exists') + self.patcher_exists.start().return_value = True + + self.patcher_samefile = patch('beets.library.os.path.samefile') + self.patcher_samefile.start().return_value = True def tearDown(self): super(PathQueryTest, self).tearDown() - self.patcher.stop() + + self.patcher_samefile.stop() + self.patcher_exists.stop() def test_path_exact_match(self): q = 'path:/a/b/c.mp3' @@ -504,13 +509,17 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): self.assert_items_matched(results, ['path item', 'caps path']) # test platform-aware default sensitivity - with _common.system_mock('Darwin'): - q = makeq() - self.assertEqual(q.case_sensitive, True) + self.patcher_exists.stop() + try: + with _common.system_mock('Darwin'): + q = makeq() + self.assertEqual(q.case_sensitive, True) - with _common.system_mock('Windows'): - q = makeq() - self.assertEqual(q.case_sensitive, False) + with _common.system_mock('Windows'): + q = makeq() + self.assertEqual(q.case_sensitive, False) + finally: + self.patcher_exists.start() @patch('beets.library.os') def test_path_sep_detection(self, mock_os): @@ -526,7 +535,7 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): def test_path_detection(self): # cover existence test - self.patcher.stop() + self.patcher_exists.stop() is_path = beets.library.PathQuery.is_path_query try: @@ -546,7 +555,7 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): finally: os.chdir(cur_dir) finally: - self.patcher.start() + self.patcher_exists.start() class IntQueryTest(unittest.TestCase, TestHelper): From 42f99999f23d075cb85a9674fd06b42ed181abfb Mon Sep 17 00:00:00 2001 From: Malte Ried Date: Mon, 7 Sep 2015 12:28:19 +0200 Subject: [PATCH 5/7] Improved the case sensitivity detection test. --- test/test_query.py | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 8507eb312..4dcf52a21 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -508,18 +508,36 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): results = self.lib.items(makeq(case_sensitive=False)) self.assert_items_matched(results, ['path item', 'caps path']) - # test platform-aware default sensitivity - self.patcher_exists.stop() - try: - with _common.system_mock('Darwin'): - q = makeq() - self.assertEqual(q.case_sensitive, True) + # Check for correct case sensitivity selection (this check + # only works for non-windows os) + with _common.system_mock('Darwin'): + # exists = True and samefile = True => Case insensitive + q = makeq() + self.assertEqual(q.case_sensitive, False) - with _common.system_mock('Windows'): - q = makeq() - self.assertEqual(q.case_sensitive, False) - finally: - self.patcher_exists.start() + self.patcher_samefile.stop() + self.patcher_samefile.start().return_value = False + + # exists = True and samefile = False => Case sensitive + q = makeq() + self.assertEqual(q.case_sensitive, True) + + self.patcher_samefile.stop() + self.patcher_samefile.start().return_value = True + + # test platform-aware default sensitivity when the library + # path does not exist (exist = False) + self.patcher_exists.stop() + self.patcher_exists.start().return_value = False + with _common.system_mock('Darwin'): + q = makeq() + self.assertEqual(q.case_sensitive, True) + + with _common.system_mock('Windows'): + q = makeq() + self.assertEqual(q.case_sensitive, False) + self.patcher_exists.stop() + self.patcher_exists.start().return_value = True @patch('beets.library.os') def test_path_sep_detection(self, mock_os): From 4c249ac25bf81e8d9f3750b59f0c97253fc2422d Mon Sep 17 00:00:00 2001 From: Malte Ried Date: Fri, 11 Sep 2015 12:35:41 +0200 Subject: [PATCH 6/7] Using GetLongPathNameW to determine file names to check for case sensitivity on windows instead of creating a file. --- beets/util/__init__.py | 45 +++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index a32c44908..1f9279b25 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -16,6 +16,7 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) +import ctypes import os import sys @@ -781,26 +782,30 @@ def is_filesystem_case_sensitive(path): # python < 3.0 return not os.path.samefile(path.lower(), path.upper()) - # On windows we create a file with a lower case name - # and try to find the file using an upper case name. - base_file_name = 'beetsCaseSensitivityCheck' - lower = os.path.join(path, base_file_name.lower()) - upper = os.path.join(path, base_file_name.upper()) - # Check if one of the files (upper and lower case) do exist - # already - i = 0 - while os.path.exists(lower) or os.path.exists(upper): - file_name = '{0}{1}'.format(base_file_name, i) - lower = os.path.join(path, file_name.lower()) - upper = os.path.join(path, file_name.upper()) - # Create the file using a lower case name - with open(lower, 'w'): - pass - # Check if the file can be found using the upper cas name - case_sensitive = not os.path.exists(upper) - # Remove the temporary file - os.remove(lower) - return case_sensitive + + # On windows we use GetLongPathNameW to determine the real path + # using the actual case. + def get_long_path_name(short_path): + if not isinstance(short_path, unicode): + short_path = unicode(short_path) + buf = ctypes.create_unicode_buffer(260) + get_long_path_name_w = ctypes.windll.kernel32.GetLongPathNameW + return_value = get_long_path_name_w(short_path, buf, 260) + if return_value == 0 or return_value > 260: + # An error occurred + return short_path + else: + long_path = buf.value + # GetLongPathNameW does not change the case of the drive + # letter. + if len(long_path) > 1 and long_path[1] == ':': + long_path = long_path[0].upper() + long_path[1:] + return long_path + + lower = get_long_path_name(path.lower()) + upper = get_long_path_name(path.upper()) + + return lower != upper else: return True # By default, the case sensitivity depends on the platform. From 5832e8730ee1b7b104eb1cf40a3aa2638c993886 Mon Sep 17 00:00:00 2001 From: Malte Ried Date: Fri, 11 Sep 2015 20:38:53 +0200 Subject: [PATCH 7/7] Changed the documentation of the PathQueries to reflect the usage of the file system cas-sensitivity. --- docs/reference/query.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/query.rst b/docs/reference/query.rst index e2091bb14..e3ce68004 100644 --- a/docs/reference/query.rst +++ b/docs/reference/query.rst @@ -202,8 +202,8 @@ Note that this only matches items that are *already in your library*, so a path query won't necessarily find *all* the audio files in a directory---just the ones you've already added to your beets library. -Path queries are case-sensitive on most platforms but case-insensitive on -Windows. +Path queries are case-sensitive if the file system the library is located on +is case-sensitive, case-insensitive otherwise. .. _query-sort: