Defer master file list fetch in source_list until a salt:// URL is encountered

changelog/69823.fixed.md

@@ -0,0 +1,8 @@
+Fix master file list fetching in `source_list`.
+
+Previously, `cp.list_master` and `cp.list_master_dirs` were called eagerly
+for every source entry, even when the source was an HTTP/FTP/local URL that
+does not need the master file list. This caused unnecessary fileserver scans.
+
+Now master file list fetches are deferred until a `salt://` URL is actually
+encountered, improving performance for non-salt sources.

salt/modules/file.py

@@ -4725,18 +4725,16 @@ def source_list(source, source_hash, saltenv):
     if contextkey in __context__:
         return __context__[contextkey]
 
-    # get the master file list
+    # Defer master file list fetches so sources like HTTP/FTP/local
+    # don't pay for a full fileserver scan upfront (#69804).
     if isinstance(source, list):
-        mfiles = [(f, saltenv) for f in __salt__["cp.list_master"](saltenv)]
-        mdirs = [(d, saltenv) for d in __salt__["cp.list_master_dirs"](saltenv)]
-        for single in source:
-            if isinstance(single, dict):
-                single = next(iter(single))
+        _mfiles: dict[str, list] = {}
+        _mdirs: dict[str, list] = {}
 
-            path, senv = salt.utils.url.parse(single)
-            if senv:
-                mfiles += [(f, senv) for f in __salt__["cp.list_master"](senv)]
-                mdirs += [(d, senv) for d in __salt__["cp.list_master_dirs"](senv)]
+        def _ensure_master_files(env: str) -> None:
+            if env not in _mfiles:
+                _mfiles[env] = [(f, env) for f in __salt__["cp.list_master"](env)]
+                _mdirs[env] = [(d, env) for d in __salt__["cp.list_master_dirs"](env)]
 
         ret = None
         for single in source:
@@ -4764,7 +4762,8 @@ def source_list(source, source_hash, saltenv):
                     path, senv = salt.utils.url.parse(single_src)
                     if not senv:
                         senv = saltenv
-                    if (path, saltenv) in mfiles or (path, saltenv) in mdirs:
+                    _ensure_master_files(senv)
+                    if (path, saltenv) in _mfiles[senv] or (path, saltenv) in _mdirs[senv]:
                         ret = (single_src, single_hash)
                         break
                 elif proto.startswith("http") or proto == "ftp":
@@ -4792,9 +4791,11 @@ def source_list(source, source_hash, saltenv):
                 path, senv = salt.utils.url.parse(single)
                 if not senv:
                     senv = saltenv
-                if (path, senv) in mfiles or (path, senv) in mdirs:
-                    ret = (single, source_hash)
-                    break
+                if single.startswith("salt://"):
+                    _ensure_master_files(senv)
+                    if (path, senv) in _mfiles[senv] or (path, senv) in _mdirs[senv]:
+                        ret = (single, source_hash)
+                        break
                 urlparsed_src = urllib.parse.urlparse(single)
                 if salt.utils.platform.is_windows():
                     # urlparse doesn't handle a local Windows path without the

tests/pytests/unit/modules/file/test_file_basics.py

@@ -309,3 +309,45 @@ def test_symlink_lexists_called_follow_symlinks_true():
         filemod.symlink(tfile, a_link, follow_symlinks=True)
         lexists.assert_not_called()
         exists.assert_called()
+
+
+def test_source_list_defers_list_master_for_http():
+    """cp.list_master must not be called when only HTTP sources are provided (#69804)."""
+    list_master = MagicMock(return_value=[])
+    list_master_dirs = MagicMock(return_value=[])
+    with patch.dict(
+        filemod.__salt__,
+        {
+            "cp.list_master": list_master,
+            "cp.list_master_dirs": list_master_dirs,
+            "cp.cache_file": MagicMock(return_value="/tmp/httpd.conf"),
+        },
+    ):
+        with patch("salt.utils.http.query", MagicMock(return_value={})):
+            ret = filemod.source_list(
+                [{"http://t.est.com/httpd.conf": "filehash"}], "", "base"
+            )
+            assert list(ret) == ["http://t.est.com/httpd.conf", "filehash"]
+            list_master.assert_not_called()
+            list_master_dirs.assert_not_called()
+
+
+def test_source_list_calls_list_master_for_salt_url():
+    """cp.list_master must be called when a salt:// source is present (#69804)."""
+    list_master = MagicMock(return_value=["http/httpd.conf"])
+    list_master_dirs = MagicMock(return_value=[])
+    with patch.dict(
+        filemod.__salt__,
+        {
+            "cp.list_master": list_master,
+            "cp.list_master_dirs": list_master_dirs,
+        },
+    ):
+        ret = filemod.source_list(
+            ["salt://http/httpd.conf", "http://fallback.example.com/conf"],
+            "filehash",
+            "base",
+        )
+        assert list(ret) == ["salt://http/httpd.conf", "filehash"]
+        list_master.assert_called_with("base")
+        list_master_dirs.assert_called_with("base")