diff --git a/beets/autotag/distance.py b/beets/autotag/distance.py
index e5ec2debb..37c6f84f4 100644
--- a/beets/autotag/distance.py
+++ b/beets/autotag/distance.py
@@ -345,6 +345,12 @@ class Distance:
         dist = string_dist(str1, str2)
         self.add(key, dist)
 
+    def add_data_source(self, before: str | None, after: str | None) -> None:
+        if before != after and (
+            before or len(metadata_plugins.find_metadata_source_plugins()) > 1
+        ):
+            self.add("data_source", metadata_plugins.get_penalty(after))
+
 
 @cache
 def get_track_length_grace() -> float:
@@ -408,9 +414,7 @@ def track_distance(
     if track_info.medium and item.disc:
         dist.add_expr("medium", item.disc != track_info.medium)
 
-    # Plugins.
-    if (actual := track_info.data_source) != item.get("data_source"):
-        dist.add("data_source", metadata_plugins.get_penalty(actual))
+    dist.add_data_source(item.get("data_source"), track_info.data_source)
 
     return dist
 
@@ -526,7 +530,6 @@ def distance(
     for _ in range(len(items) - len(mapping)):
         dist.add("unmatched_tracks", 1.0)
 
-    # Plugins.
-    if (data_source := album_info.data_source) != likelies["data_source"]:
-        dist.add("data_source", metadata_plugins.get_penalty(data_source))
+    dist.add_data_source(likelies["data_source"], album_info.data_source)
+
     return dist
diff --git a/test/autotag/test_distance.py b/test/autotag/test_distance.py
index 91003bbb9..b327bbe44 100644
--- a/test/autotag/test_distance.py
+++ b/test/autotag/test_distance.py
@@ -300,7 +300,7 @@ class TestDataSourceDistance:
     MISMATCH = 0.125
 
     @pytest.fixture(autouse=True)
-    def setup(self, monkeypatch, penalty, weight):
+    def setup(self, monkeypatch, penalty, weight, multiple_data_sources):
         monkeypatch.setitem(Distance._weights, "data_source", weight)
         get_penalty.cache_clear()
 
@@ -320,22 +320,27 @@ class TestDataSourceDistance:
 
         monkeypatch.setattr(
             "beets.metadata_plugins.find_metadata_source_plugins",
-            lambda: [OriginalPlugin(), OtherPlugin()],
+            lambda: (
+                [OriginalPlugin(), OtherPlugin()]
+                if multiple_data_sources
+                else [OtherPlugin()]
+            ),
         )
 
     @pytest.mark.parametrize(
-        "item,info,penalty,weight,expected_distance",
+        "item,info,penalty,weight,multiple_data_sources,expected_distance",
         [
-            _p("Original", "Original", 0.5, 1.0, MATCH, id="match"),
-            _p("Original", "Other", 0.5, 1.0, MISMATCH, id="mismatch"),
-            _p("Original", "unknown", 0.5, 1.0, MISMATCH, id="mismatch-unknown"),  # noqa: E501
-            _p("Original", None, 0.5, 1.0, MISMATCH, id="mismatch-no-info"),
-            _p(None, "Other", 0.5, 1.0, MISMATCH, id="mismatch-no-original"),
-            _p("unknown", "unknown", 0.5, 1.0, MATCH, id="match-unknown"),
-            _p("Original", "Other", 1.0, 1.0, 0.25, id="mismatch-max-penalty"),
-            _p("Original", "Other", 0.5, 5.0, 0.3125, id="mismatch-high-weight"),  # noqa: E501
-            _p("Original", "Other", 0.0, 1.0, MATCH, id="match-no-penalty"),
-            _p("Original", "Other", 0.5, 0.0, MATCH, id="match-no-weight"),
+            _p("Original", "Original", 0.5, 1.0, True, MATCH, id="match"),
+            _p("Original", "Other", 0.5, 1.0, True, MISMATCH, id="mismatch"),
+            _p("Original", "unknown", 0.5, 1.0, True, MISMATCH, id="mismatch-unknown"),  # noqa: E501
+            _p("Original", None, 0.5, 1.0, True, MISMATCH, id="mismatch-no-info"),  # noqa: E501
+            _p(None, "Other", 0.5, 1.0, True, MISMATCH, id="mismatch-no-original-multiple-sources"),  # noqa: E501
+            _p(None, "Other", 0.5, 1.0, False, MATCH, id="match-no-original-but-single-source"),  # noqa: E501
+            _p("unknown", "unknown", 0.5, 1.0, True, MATCH, id="match-unknown"),
+            _p("Original", "Other", 1.0, 1.0, True, 0.25, id="mismatch-max-penalty"),  # noqa: E501
+            _p("Original", "Other", 0.5, 5.0, True, 0.3125, id="mismatch-high-weight"),  # noqa: E501
+            _p("Original", "Other", 0.0, 1.0, True, MATCH, id="match-no-penalty"),  # noqa: E501
+            _p("Original", "Other", 0.5, 0.0, True, MATCH, id="match-no-weight"),  # noqa: E501
         ],
     )  # fmt: skip
     def test_distance(self, item, info, expected_distance):