diff --git a/beets/util/__init__.py b/beets/util/__init__.py
index f3dedcb41..f5ad2da22 100644
--- a/beets/util/__init__.py
+++ b/beets/util/__init__.py
@@ -283,13 +283,13 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
             continue
         clutter = [bytestring_path(c) for c in clutter]
         match_paths = [bytestring_path(d) for d in os.listdir(directory)]
-        if fnmatch_all(match_paths, clutter):
-            # Directory contains only clutter (or nothing).
-            try:
+        try:
+            if fnmatch_all(match_paths, clutter):
+                # Directory contains only clutter (or nothing).
                 shutil.rmtree(directory)
-            except OSError:
+            else:
                 break
-        else:
+        except OSError:
             break
 
 
diff --git a/docs/changelog.rst b/docs/changelog.rst
index b2c8437b6..43b6b20f6 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -195,6 +195,8 @@ Fixes:
   is long.
   Thanks to :user:`ray66`.
   :bug:`3207` :bug:`2752`
+* Fix an unhandled exception when pruning empty directories.
+  :bug:`1996` :bug:`3209`
 
 .. _python-itunes: https://github.com/ocelma/python-itunes