Merge pull request #1994 from jackwilsdon/ignore-hidden-files

Add option to ignore hidden files on import

beets/config_default.yaml

@@ -26,6 +26,7 @@ import:
 
 clutter: ["Thumbs.DB", ".DS_Store"]
 ignore: [".*", "*~", "System Volume Information"]
+ignore_hidden: yes
 replace:
     '[\\/]': _
     '^\.': _

beets/importer.py

@@ -1451,8 +1451,11 @@ def albums_in_dir(path):
     """
     collapse_pat = collapse_paths = collapse_items = None
     ignore = config['ignore'].as_str_seq()
+    ignore_hidden = config['ignore_hidden'].get(bool)
 
-    for root, dirs, files in sorted_walk(path, ignore=ignore, logger=log):
+    for root, dirs, files in sorted_walk(path, ignore=ignore,
+                                         ignore_hidden=ignore_hidden,
+                                         logger=log):
         items = [os.path.join(root, f) for f in files]
         # If we're currently collapsing the constituent directories in a
         # multi-disc album, check whether we should continue collapsing

beets/util/__init__.py

@@ -26,6 +26,7 @@ import traceback
 import subprocess
 import platform
 import shlex
+from beets.util import hidden
 
 
 MAX_FILENAME_LENGTH = 200
@@ -151,7 +152,7 @@ def ancestry(path):
     return out
 
 
-def sorted_walk(path, ignore=(), logger=None):
+def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None):
     """Like `os.walk`, but yields things in case-insensitive sorted,
     breadth-first order.  Directory and file names matching any glob
     pattern in `ignore` are skipped. If `logger` is provided, then
@@ -185,10 +186,11 @@ def sorted_walk(path, ignore=(), logger=None):
 
         # Add to output as either a file or a directory.
         cur = os.path.join(path, base)
-        if os.path.isdir(syspath(cur)):
-            dirs.append(base)
-        else:
-            files.append(base)
+        if (ignore_hidden and not hidden.is_hidden(cur)) or not ignore_hidden:
+            if os.path.isdir(syspath(cur)):
+                dirs.append(base)
+            else:
+                files.append(base)
 
     # Sort lists (case-insensitive) and yield the current level.
     dirs.sort(key=bytes.lower)
@@ -199,7 +201,7 @@ def sorted_walk(path, ignore=(), logger=None):
     for base in dirs:
         cur = os.path.join(path, base)
         # yield from sorted_walk(...)
-        for res in sorted_walk(cur, ignore, logger):
+        for res in sorted_walk(cur, ignore, ignore_hidden, logger):
             yield res
 
 

beets/util/hidden.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+# This file is part of beets.
+# Copyright 2016, Adrian Sampson.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+"""Simple library to work out if a file is hidden on different platforms."""
+from __future__ import division, absolute_import, print_function
+
+import os
+import stat
+import ctypes
+import sys
+
+
+def _is_hidden_osx(path):
+    """Return whether or not a file is hidden on OS X.
+
+    This uses os.lstat to work out if a file has the "hidden" flag.
+    """
+    file_stat = os.lstat(path)
+
+    if hasattr(file_stat, 'st_flags') and hasattr(stat, 'UF_HIDDEN'):
+        return bool(file_stat.st_flags & stat.UF_HIDDEN)
+    else:
+        return False
+
+
+def _is_hidden_win(path):
+    """Return whether or not a file is hidden on Windows.
+
+    This uses GetFileAttributes to work out if a file has the "hidden" flag
+    (FILE_ATTRIBUTE_HIDDEN).
+    """
+    # FILE_ATTRIBUTE_HIDDEN = 2 (0x2) from GetFileAttributes documentation.
+    hidden_mask = 2
+
+    # Retrieve the attributes for the file.
+    attrs = ctypes.windll.kernel32.GetFileAttributesW(path)
+
+    # Ensure we have valid attribues and compare them against the mask.
+    return attrs >= 0 and attrs & hidden_mask
+
+
+def _is_hidden_dot(path):
+    """Return whether or not a file starts with a dot.
+
+    Files starting with a dot are seen as "hidden" files on Unix-based OSes.
+    """
+    return os.path.basename(path).startswith('.')
+
+
+def is_hidden(path):
+    """Return whether or not a file is hidden.
+
+    This method works differently depending on the platform it is called on.
+
+    On OS X, it uses both the result of `is_hidden_osx` and `is_hidden_dot` to
+    work out if a file is hidden.
+
+    On Windows, it uses the result of `is_hidden_win` to work out if a file is
+    hidden.
+
+    On any other operating systems (i.e. Linux), it uses `is_hidden_dot` to
+    work out if a file is hidden.
+    """
+    # Convert the path to unicode if it is not already.
+    if not isinstance(path, unicode):
+        path = path.decode('utf-8')
+
+    # Run platform specific functions depending on the platform
+    if sys.platform == 'darwin':
+        return _is_hidden_osx(path) or _is_hidden_dot(path)
+    elif sys.platform == 'win32':
+        return _is_hidden_win(path)
+    else:
+        return _is_hidden_dot(path)
+
+__all__ = ['is_hidden']

docs/changelog.rst

@@ -27,6 +27,8 @@ New features:
   inputs
 * New :doc:`/plugins/hook` that allows commands to be executed when an event is
   emitted by beets. :bug:`1561` :bug:`1603`
+* :doc:`/reference/config`: New ``ignore_hidden`` configuration option allowing
+  platform-specific hidden files to be ignored on import.
 
 .. _fanart.tv: https://fanart.tv/
 

docs/reference/config.rst

@@ -91,6 +91,15 @@ importing. By default, this consists of ``.*``,  ``*~``, and ``System Volume
 Information`` (i.e., beets ignores Unix-style hidden files, backup files, and
 a directory that appears at the root of some Windows filesystems).
 
+ignore_hidden
+~~~~~~~~~~~~~
+
+Either ``yes`` or ``no``; whether to ignore hidden files when importing. On
+Windows, the "Hidden" property of files is used to detect whether or not a file
+is hidden. On OS X, the file's "IsHidden" flag is used to detect whether or not
+a file is hidden. On both OS X and other platforms (excluding Windows), files
+(and directories) starting with a dot are detected as hidden files.
+
 .. _replace:
 
 replace
@@ -799,6 +808,7 @@ Here's an example file::
         timid: no
         log: beetslog.txt
     ignore: .AppleDouble ._* *~ .DS_Store
+    ignore_hidden: yes
     art_filename: albumart
     plugins: bpd
     pluginpath: ~/beets/myplugins

test/test_hidden.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+# This file is part of beets.
+# Copyright 2016, Fabrice Laporte.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+"""Tests for the 'hidden' utility."""
+
+from __future__ import division, absolute_import, print_function
+
+from test._common import unittest
+import sys
+import tempfile
+from beets.util import hidden
+import subprocess
+import errno
+import ctypes
+
+
+class HiddenFileTest(unittest.TestCase):
+    def setUp(self):
+        pass
+
+    def test_osx_hidden(self):
+        if not sys.platform == 'darwin':
+            self.skipTest('sys.platform is not darwin')
+            return
+
+        with tempfile.NamedTemporaryFile(delete=False) as f:
+            try:
+                command = ["chflags", "hidden", f.name]
+                subprocess.Popen(command).wait()
+            except OSError as e:
+                if e.errno == errno.ENOENT:
+                    self.skipTest("unable to find chflags")
+                else:
+                    raise e
+
+            self.assertTrue(hidden.is_hidden(f.name))
+
+    def test_windows_hidden(self):
+        if not sys.platform == 'windows':
+            self.skipTest('sys.platform is not windows')
+            return
+
+        # FILE_ATTRIBUTE_HIDDEN = 2 (0x2) from GetFileAttributes documentation.
+        hidden_mask = 2
+
+        with tempfile.NamedTemporaryFile() as f:
+            # Hide the file using
+            success = ctypes.windll.kernel32.SetFileAttributesW(f.name,
+                                                                hidden_mask)
+
+            if not success:
+                self.skipTest("unable to set file attributes")
+
+            self.assertTrue(hidden.is_hidden(f.name))
+
+    def test_other_hidden(self):
+        if sys.platform == 'darwin' or sys.platform == 'windows':
+            self.skipTest('sys.platform is known')
+            return
+
+        with tempfile.NamedTemporaryFile(prefix='.tmp') as f:
+            self.assertTrue(hidden.is_hidden(f.name))