diff --git a/beets/library.py b/beets/library.py
index bf02ef834..9876acf98 100644
--- a/beets/library.py
+++ b/beets/library.py
@@ -1012,8 +1012,6 @@ class Library(BaseLibrary):
         album = self.get_album(item) if with_album else None
 
         self.conn.execute('DELETE FROM items WHERE id=?', (item.id,))
-        if delete:
-            os.unlink(_syspath(item.path))
 
         if album:
             item_iter = album.items()
@@ -1022,7 +1020,18 @@ class Library(BaseLibrary):
             except StopIteration:
                 # Album is empty.
                 album.remove(delete, False)
-    
+
+        if delete:
+            os.unlink(_syspath(item.path))
+
+            # Prune directories.
+            try:
+                os.removedirs(_syspath(os.path.dirname(item.path)))
+            except OSError:
+                # print os.listdir(os.path.dirname(item.path))
+                pass
+
+
     # Browsing.
 
     def artists(self, query=None):
@@ -1220,12 +1229,6 @@ class Album(BaseAlbum):
             artpath = self.artpath
             if artpath:
                 os.unlink(_syspath(artpath))
-
-                # Prune directories.
-                try:
-                    os.removedirs(_syspath(os.path.dirname(artpath)))
-                except OSError:
-                    pass
         
         # Remove album from database.
         self._library.conn.execute(
diff --git a/test/test_files.py b/test/test_files.py
index 6fdcffb15..8148bfec1 100644
--- a/test/test_files.py
+++ b/test/test_files.py
@@ -259,27 +259,33 @@ class RemoveTest(unittest.TestCase):
         if os.path.exists(self.libdir):
             shutil.rmtree(self.libdir)
 
-    def test_removing_last_item_removes_album(self):
-        self.assertEqual(len(self.lib.albums()), 1)
-        self.lib.remove(self.i)
-        self.assertEqual(len(self.lib.albums()), 0)
-
     def test_removing_last_item_removes_empty_dir(self):
         parent = os.path.dirname(self.i.path)
         self.assertTrue(os.path.exists(parent))
-        self.lib.remove(self.i)
+        self.lib.remove(self.i, True)
         self.assertFalse(os.path.exists(parent))
 
     def test_removing_last_item_preserves_nonempty_dir(self):
         parent = os.path.dirname(self.i.path)
         touch(os.path.join(parent, 'dummy.txt'))
-        self.lib.remove(self.i)
+        self.lib.remove(self.i, True)
         self.assertTrue(os.path.exists(parent))
 
-    def test_removing_last_item_preserves_library_dir(self):
+    def test_removing_without_delete_leaves_file(self):
+        path = self.i.path
         self.lib.remove(self.i)
+        self.assertTrue(os.path.exists(path))
+
+    def test_removing_last_item_preserves_library_dir(self):
+        self.lib.remove(self.i, True)
         self.assertTrue(os.path.exists(self.libdir))
 
+    def test_removing_item_outside_of_library_deletes_nothing(self):
+        self.lib.directory = os.path.abspath(os.path.join('rsrc', 'xxx'))
+        parent = os.path.dirname(self.i.path)
+        self.lib.remove(self.i, True)
+        self.assertTrue(os.path.exists(parent))
+
 def suite():
     return unittest.TestLoader().loadTestsFromName(__name__)