From 412bde5de2018c0770a7859cc3820ccab3404814 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Fri, 6 May 2016 00:15:09 +0100 Subject: [PATCH 1/8] Add library to check if a file is hidden - Add `beets.util.hidden` which adds a `is_hidden` function to check whether or not a file is hidden on the current platform. - Add tests for `beets.util.hidden`. --- beets/util/hidden.py | 143 +++++++++++++++++++++++++++++++++++++++++++ test/test_hidden.py | 74 ++++++++++++++++++++++ 2 files changed, 217 insertions(+) create mode 100644 beets/util/hidden.py create mode 100644 test/test_hidden.py diff --git a/beets/util/hidden.py b/beets/util/hidden.py new file mode 100644 index 000000000..3d7022ae7 --- /dev/null +++ b/beets/util/hidden.py @@ -0,0 +1,143 @@ +"""Simple library to work out if a file is hidden on different platforms.""" + +import ctypes +import ctypes.util +import os.path +import sys + + +# Adjustments for CoreFoundation functions on OS X. +_CF_FUNCTION_MAPPINGS = { + 'CFRelease': { + 'argtypes': [ctypes.c_void_p], + 'restype': None + }, + 'CFURLCreateFromFileSystemRepresentation': { + 'argtypes': [ + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_long, + ctypes.c_int + ], + 'restype': ctypes.c_void_p + }, + 'CFURLCopyResourcePropertyForKey': { + 'argtypes': [ + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_void_p + ], + 'restype': ctypes.c_void_p + }, + 'CFBooleanGetValue': { + 'argtypes': [ctypes.c_void_p], + 'restype': ctypes.c_int + } +} + + +def _dict_attr_copy(source, destination): + """Copy dict values from source on to destination as attributes.""" + for k, v in source.iteritems(): + if isinstance(v, dict): + _dict_attr_copy(v, getattr(destination, k)) + else: + setattr(destination, k, v) + + +def _is_hidden_osx(path): + """Return whether or not a file is hidden on OS X. + + This uses CoreFoundation alongside CFURL to work out if a file has the + "hidden" flag. + """ + # Load CoreFoundation. + cf_path = ctypes.util.find_library('CoreFoundation') + cf = ctypes.cdll.LoadLibrary(cf_path) + + # Copy the adjustments on to the library. + _dict_attr_copy(_CF_FUNCTION_MAPPINGS, cf) + + # Create a URL from the path. + url = cf.CFURLCreateFromFileSystemRepresentation(None, path, len(path), + False) + + # Retrieve the hidden key. + is_hidden_key = ctypes.c_void_p.in_dll(cf, 'kCFURLIsHiddenKey') + + # Create a void pointer and get the address of it. + val = ctypes.c_void_p(0) + val_address = ctypes.addressof(val) + + # Get the value (whether or not the file is hidden) for the hidden key and + # store it in val. + success = cf.CFURLCopyResourcePropertyForKey(url, is_hidden_key, + val_address, None) + + # Check if we were able to get the value for the hidden key. + if success: + + # Retrieve the result as a boolean. + result = cf.CFBooleanGetValue(val) + + # Release the value and URL. + cf.CFRelease(val) + cf.CFRelease(url) + + return bool(result) + else: + return False + + +def _is_hidden_win(path): + """Return whether or not a file is hidden on Windows. + + This uses GetFileAttributes to work out if a file has the "hidden" flag + (FILE_ATTRIBUTE_HIDDEN). + """ + # FILE_ATTRIBUTE_HIDDEN = 2 (0x2) from GetFileAttributes documentation. + hidden_mask = 2 + + # Retrieve the attributes for the file. + attrs = ctypes.windll.kernel32.GetFileAttributesW(path) + + # Ensure we have valid attribues and compare them against the mask. + return attrs >= 0 and bool(attrs & hidden_mask) + + +def _is_hidden_dot(path): + """Return whether or not a file starts with a dot. + + Files starting with a dot are seen as "hidden" files on Unix-based OSes. + """ + return os.path.basename(path).startswith('.') + + +def is_hidden(path): + """Return whether or not a file is hidden. + + This method works differently depending on the platform it is called on. + + On OS X, it uses both the result of `is_hidden_osx` and `is_hidden_dot` to + work out if a file is hidden. + + On Windows, it uses the result of `is_hidden_win` to work out if a file is + hidden. + + On any other operating systems (i.e. Linux), it uses `is_hidden_dot` to + work out if a file is hidden. + """ + # Convert the path to unicode if it is not already. + if not isinstance(path, unicode): + path = path.decode('utf-8') + + # Run platform specific functions depending on the platform + if sys.platform == 'darwin': + return _is_hidden_osx(path) or _is_hidden_dot(path) + elif sys.platform == 'win32': + return _is_hidden_win(path) + else: + return _is_hidden_dot(path) + +__all__ = ['is_hidden'] diff --git a/test/test_hidden.py b/test/test_hidden.py new file mode 100644 index 000000000..a02282ef5 --- /dev/null +++ b/test/test_hidden.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Fabrice Laporte. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Tests for the 'hidden' utility.""" + +from __future__ import division, absolute_import, print_function + +from test._common import unittest +import sys +import tempfile +from beets.util import hidden +import subprocess +import errno +import ctypes + + +class HiddenFileTest(unittest.TestCase): + def setUp(self): + pass + + def test_osx_hidden(self): + if not sys.platform == 'darwin': + self.skipTest('sys.platform is not darwin') + return + + with tempfile.NamedTemporaryFile(delete=False) as f: + try: + command = ["chflags", "hidden", f.name] + subprocess.Popen(command).wait() + except OSError as e: + if e.errno == errno.ENOENT: + self.skipTest("unable to find chflags") + else: + raise e + + self.assertTrue(hidden.is_hidden(f.name)) + + def test_windows_hidden(self): + if not sys.platform == 'windows': + self.skipTest('sys.platform is not windows') + return + + # FILE_ATTRIBUTE_HIDDEN = 2 (0x2) from GetFileAttributes documentation. + hidden_mask = 2 + + with tempfile.NamedTemporaryFile() as f: + # Hide the file using + success = ctypes.windll.kernel32.SetFileAttributesW(f.name, + hidden_mask) + + if not success: + self.skipTest("unable to set file attributes") + + self.assertTrue(hidden.is_hidden(f.name)) + + def test_other_hidden(self): + if sys.platform == 'darwin' or sys.platform == 'windows': + self.skipTest('sys.platform is known') + return + + with temfile.NamedTemporaryFile(prefix='.tmp') as f: + self.assertTrue(hidden.is_hidden(f.name)) From 4b4e788865955b97d4439bde785591fb3a85ccc5 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Fri, 6 May 2016 00:35:43 +0100 Subject: [PATCH 2/8] Add `ignore_hidden` configuration property Add `ignore_hidden` top level configuration property, allowing hidden files to be ignored on import. --- beets/config_default.yaml | 1 + beets/importer.py | 5 ++++- beets/util/__init__.py | 14 ++++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/beets/config_default.yaml b/beets/config_default.yaml index 5bc921167..dfe31ade8 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -26,6 +26,7 @@ import: clutter: ["Thumbs.DB", ".DS_Store"] ignore: [".*", "*~", "System Volume Information"] +ignore_hidden: yes replace: '[\\/]': _ '^\.': _ diff --git a/beets/importer.py b/beets/importer.py index e73a9e08f..f9cd138f4 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -1451,8 +1451,11 @@ def albums_in_dir(path): """ collapse_pat = collapse_paths = collapse_items = None ignore = config['ignore'].as_str_seq() + ignore_hidden = config['ignore_hidden'].get(bool) - for root, dirs, files in sorted_walk(path, ignore=ignore, logger=log): + for root, dirs, files in sorted_walk(path, ignore=ignore, + ignore_hidden=ignore_hidden, + logger=log): items = [os.path.join(root, f) for f in files] # If we're currently collapsing the constituent directories in a # multi-disc album, check whether we should continue collapsing diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 4d46aecd2..327b312fa 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -26,6 +26,7 @@ import traceback import subprocess import platform import shlex +from beets.util import hidden MAX_FILENAME_LENGTH = 200 @@ -151,7 +152,7 @@ def ancestry(path): return out -def sorted_walk(path, ignore=(), logger=None): +def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None): """Like `os.walk`, but yields things in case-insensitive sorted, breadth-first order. Directory and file names matching any glob pattern in `ignore` are skipped. If `logger` is provided, then @@ -185,10 +186,11 @@ def sorted_walk(path, ignore=(), logger=None): # Add to output as either a file or a directory. cur = os.path.join(path, base) - if os.path.isdir(syspath(cur)): - dirs.append(base) - else: - files.append(base) + if (ignore_hidden and not hidden.is_hidden(cur)) or not ignore_hidden: + if os.path.isdir(syspath(cur)): + dirs.append(base) + else: + files.append(base) # Sort lists (case-insensitive) and yield the current level. dirs.sort(key=bytes.lower) @@ -199,7 +201,7 @@ def sorted_walk(path, ignore=(), logger=None): for base in dirs: cur = os.path.join(path, base) # yield from sorted_walk(...) - for res in sorted_walk(cur, ignore, logger): + for res in sorted_walk(cur, ignore, ignore_hidden, logger): yield res From 38a3726b1b459a9e855fba85b9f8c221b0460ac3 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Fri, 6 May 2016 00:45:45 +0100 Subject: [PATCH 3/8] Add documentation --- docs/reference/config.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index b074edf5c..e84dc4a17 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -91,6 +91,15 @@ importing. By default, this consists of ``.*``, ``*~``, and ``System Volume Information`` (i.e., beets ignores Unix-style hidden files, backup files, and a directory that appears at the root of some Windows filesystems). +ignore_hidden +~~~~~~~~~~~~~ + +Either ``yes`` or ``no``; whether to ignore hidden files when importing. On +Windows, the "Hidden" property of files is used to detect whether or not a file +is hidden. On OS X, the file's "IsHidden" flag is used to detect whether or not +a file is hidden. On both OS X and other platforms (excluding Windows), files +(and directories) starting with a dot are detected as hidden files. + .. _replace: replace @@ -799,6 +808,7 @@ Here's an example file:: timid: no log: beetslog.txt ignore: .AppleDouble ._* *~ .DS_Store + ignore_hidden: yes art_filename: albumart plugins: bpd pluginpath: ~/beets/myplugins From d13022e51975c0f1c38e85a0ff1b594cecd93e54 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Fri, 6 May 2016 00:52:22 +0100 Subject: [PATCH 4/8] Use os.lstat instead of CoreFoundation and ctypes magic on OS X --- beets/util/hidden.py | 83 ++++---------------------------------------- 1 file changed, 6 insertions(+), 77 deletions(-) diff --git a/beets/util/hidden.py b/beets/util/hidden.py index 3d7022ae7..6d8fe58ac 100644 --- a/beets/util/hidden.py +++ b/beets/util/hidden.py @@ -1,91 +1,20 @@ """Simple library to work out if a file is hidden on different platforms.""" +import os +import stat import ctypes -import ctypes.util -import os.path import sys -# Adjustments for CoreFoundation functions on OS X. -_CF_FUNCTION_MAPPINGS = { - 'CFRelease': { - 'argtypes': [ctypes.c_void_p], - 'restype': None - }, - 'CFURLCreateFromFileSystemRepresentation': { - 'argtypes': [ - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.c_long, - ctypes.c_int - ], - 'restype': ctypes.c_void_p - }, - 'CFURLCopyResourcePropertyForKey': { - 'argtypes': [ - ctypes.c_void_p, - ctypes.c_void_p, - ctypes.c_void_p, - ctypes.c_void_p - ], - 'restype': ctypes.c_void_p - }, - 'CFBooleanGetValue': { - 'argtypes': [ctypes.c_void_p], - 'restype': ctypes.c_int - } -} - - -def _dict_attr_copy(source, destination): - """Copy dict values from source on to destination as attributes.""" - for k, v in source.iteritems(): - if isinstance(v, dict): - _dict_attr_copy(v, getattr(destination, k)) - else: - setattr(destination, k, v) - - def _is_hidden_osx(path): """Return whether or not a file is hidden on OS X. - This uses CoreFoundation alongside CFURL to work out if a file has the - "hidden" flag. + This uses os.lstat to work out if a file has the "hidden" flag. """ - # Load CoreFoundation. - cf_path = ctypes.util.find_library('CoreFoundation') - cf = ctypes.cdll.LoadLibrary(cf_path) + file_stat = os.lstat(path) - # Copy the adjustments on to the library. - _dict_attr_copy(_CF_FUNCTION_MAPPINGS, cf) - - # Create a URL from the path. - url = cf.CFURLCreateFromFileSystemRepresentation(None, path, len(path), - False) - - # Retrieve the hidden key. - is_hidden_key = ctypes.c_void_p.in_dll(cf, 'kCFURLIsHiddenKey') - - # Create a void pointer and get the address of it. - val = ctypes.c_void_p(0) - val_address = ctypes.addressof(val) - - # Get the value (whether or not the file is hidden) for the hidden key and - # store it in val. - success = cf.CFURLCopyResourcePropertyForKey(url, is_hidden_key, - val_address, None) - - # Check if we were able to get the value for the hidden key. - if success: - - # Retrieve the result as a boolean. - result = cf.CFBooleanGetValue(val) - - # Release the value and URL. - cf.CFRelease(val) - cf.CFRelease(url) - - return bool(result) + if hasattr(file_stat, 'st_flags') and hasattr(stat, 'UF_HIDDEN'): + return (file_stat.st_flags & stat.UF_HIDDEN) == stat.UF_HIDDEN else: return False From 506f1b7351ce70e3e730a277a85fe3b17fbe52c4 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Fri, 6 May 2016 00:55:29 +0100 Subject: [PATCH 5/8] Fix tempfile spelling in hidden test --- test/test_hidden.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_hidden.py b/test/test_hidden.py index a02282ef5..1c2f44848 100644 --- a/test/test_hidden.py +++ b/test/test_hidden.py @@ -70,5 +70,5 @@ class HiddenFileTest(unittest.TestCase): self.skipTest('sys.platform is known') return - with temfile.NamedTemporaryFile(prefix='.tmp') as f: + with tempfile.NamedTemporaryFile(prefix='.tmp') as f: self.assertTrue(hidden.is_hidden(f.name)) From 99326192eb29d9c42c398092ea19b9b88d096e9c Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Fri, 6 May 2016 00:55:55 +0100 Subject: [PATCH 6/8] Add missing copyright and future imports --- beets/util/hidden.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/beets/util/hidden.py b/beets/util/hidden.py index 6d8fe58ac..101e8c2d1 100644 --- a/beets/util/hidden.py +++ b/beets/util/hidden.py @@ -1,4 +1,20 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + """Simple library to work out if a file is hidden on different platforms.""" +from __future__ import division, absolute_import, print_function import os import stat From f8b1a2e565b055d90f15b2a6d7aa16031eb4e1f0 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Fri, 6 May 2016 11:24:28 +0100 Subject: [PATCH 7/8] Remove unnecessary equality check - Remove unnecessary equality check and wrap the value in `bool`. - Remove unnecessary `bool` wrapping. --- beets/util/hidden.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/util/hidden.py b/beets/util/hidden.py index 101e8c2d1..262d371ea 100644 --- a/beets/util/hidden.py +++ b/beets/util/hidden.py @@ -30,7 +30,7 @@ def _is_hidden_osx(path): file_stat = os.lstat(path) if hasattr(file_stat, 'st_flags') and hasattr(stat, 'UF_HIDDEN'): - return (file_stat.st_flags & stat.UF_HIDDEN) == stat.UF_HIDDEN + return bool(file_stat.st_flags & stat.UF_HIDDEN) else: return False @@ -48,7 +48,7 @@ def _is_hidden_win(path): attrs = ctypes.windll.kernel32.GetFileAttributesW(path) # Ensure we have valid attribues and compare them against the mask. - return attrs >= 0 and bool(attrs & hidden_mask) + return attrs >= 0 and attrs & hidden_mask def _is_hidden_dot(path): From 78109d779e8a31f5bb0722e6c7466ba3ddb0698e Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Fri, 6 May 2016 11:27:33 +0100 Subject: [PATCH 8/8] Update changelog to reflect addition of new option --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 99ebc39f5..e3ec794b5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,8 @@ New features: inputs * New :doc:`/plugins/hook` that allows commands to be executed when an event is emitted by beets. :bug:`1561` :bug:`1603` +* :doc:`/reference/config`: New ``ignore_hidden`` configuration option allowing + platform-specific hidden files to be ignored on import. .. _fanart.tv: https://fanart.tv/