diff --git a/test/plugins/test_embedart.py b/test/plugins/test_embedart.py
index cb4d1a421..f2f02137b 100644
--- a/test/plugins/test_embedart.py
+++ b/test/plugins/test_embedart.py
@@ -34,8 +34,35 @@ def require_artresizer_compare(test):
     def wrapper(*args, **kwargs):
         if not ArtResizer.shared.can_compare:
             raise unittest.SkipTest("compare not available")
-        else:
-            return test(*args, **kwargs)
+
+        # PHASH computation in ImageMagick changed at some point in an
+        # undocumented way. Check at a low level that comparisons of our
+        # fixtures give the expected results. Only then, plugin logic tests
+        # below are meaningful.
+        # cf. https://github.com/ImageMagick/ImageMagick/discussions/5191
+        # It would be better to investigate what exactly change in IM and
+        # handle that in ArtResizer.IMBackend.{can_compare,compare}.
+        # Skipping the tests as below is a quick fix to CI, but users may
+        # still see unexpected behaviour.
+        abbey_artpath = os.path.join(_common.RSRC, b"abbey.jpg")
+        abbey_similarpath = os.path.join(_common.RSRC, b"abbey-similar.jpg")
+        abbey_differentpath = os.path.join(_common.RSRC, b"abbey-different.jpg")
+        compare_threshold = 20
+
+        similar_compares_ok = ArtResizer.shared.compare(
+            abbey_artpath,
+            abbey_similarpath,
+            compare_threshold,
+        )
+        different_compares_ok = ArtResizer.shared.compare(
+            abbey_artpath,
+            abbey_differentpath,
+            compare_threshold,
+        )
+        if not similar_compares_ok or different_compares_ok:
+            raise unittest.SkipTest("IM version with broken compare")
+
+        return test(*args, **kwargs)
 
     wrapper.__name__ = test.__name__
     return wrapper