fix(glob): discard partial dircache entry after prefix-filtered _find

fsspec/asyn.py

@@ -871,9 +871,18 @@ async def _glob(self, path, maxdepth=None, **kwargs):
         # gcsfs, s3fs and adlfs can filter server-side up to the first wildcard.
         if prefix:
             kwargs["prefix"] = prefix
+        root_key = root.rstrip("/")
+        pre_cached = prefix and root and root_key in self.dircache
         allpaths = await self._find(
             root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs
         )
+        # Backends store only prefix-matching files in dircache when prefix= is
+        # used, so a newly created entry is a partial listing that would mislead
+        # subsequent lookups.  Only discard it when _find created it; if the
+        # entry existed before the call the backend served it from cache without
+        # overwriting, so it already holds a full legitimate listing.
+        if prefix and root and not pre_cached:
+            self.dircache.pop(root_key, None)
 
         pattern = glob_translate(path + ("/" if ends_with_sep else ""))
         pattern = re.compile(pattern)

fsspec/tests/test_async.py

@@ -296,11 +296,10 @@ def test_cat_ranges_on_error_raise_propagates():
 
 
 class _PrefixCapturingFS(fsspec.asyn.AsyncFileSystem):
-    """Minimal AsyncFileSystem that records every _find call's kwargs.
-
-    _find ignores the prefix hint and returns all files under *root* so that
-    the client-side glob pattern-matching in _glob still works correctly.
-    This simulates a "naive" backend that silently absorbs unknown kwargs.
+    """AsyncFileSystem that records _find calls and simulates cloud backend
+    dircache behaviour (s3fs, gcsfs, adlfs): prefix-filtered _find stores only
+    matching files in dircache; a second call to the same directory returns the
+    cached partial listing filtered by the new prefix.
     """
 
     protocol = "prefixmock"
@@ -312,15 +311,35 @@ def __init__(self, fs_files=None, **kwargs):
 
     async def _find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs):
         self.find_calls.append({"path": path, "kwargs": dict(kwargs)})
-        root = (path.rstrip("/") + "/") if path else ""
+        prefix = kwargs.get("prefix", "")
+        stripped = path.rstrip("/")
+        # Empty stripped means a root-level glob ("top_*"); no dir prefix.
+        root_prefix = (stripped + "/") if stripped else ""
+
+        if stripped in self.dircache:
+            cached = self.dircache[stripped]
+            results = {
+                e["name"]: e
+                for e in cached
+                if e["name"][len(root_prefix) :].startswith(prefix)
+            }
+            return results if detail else list(results)
+
         results = {
             f: {"name": f, "type": "file", "size": 0}
             for f in self.fs_files
-            if f.startswith(root)
+            if f.startswith(root_prefix) and f[len(root_prefix) :].startswith(prefix)
         }
+        self.dircache[stripped] = list(results.values())
         return results if detail else list(results)
 
     async def _info(self, path, **kwargs):
+        parent = self._parent(path)
+        if parent in self.dircache:
+            for entry in self.dircache[parent]:
+                if entry["name"] == path:
+                    return entry
+            raise FileNotFoundError(path)
         for f in self.fs_files:
             if f == path:
                 return {"name": f, "type": "file", "size": 0}
@@ -404,6 +423,39 @@ def test_glob_prefix_hint(prefix_fs, pattern, expected_results, expected_prefix)
         )
 
 
+def test_glob_prefix_does_not_poison_dircache(prefix_fs):
+    """Two consecutive prefix globs on the same directory must both return correct results."""
+    assert prefix_fs.glob("data/2024/res*") == ["data/2024/results.csv"]
+    assert prefix_fs.glob("data/2024/rep*") == ["data/2024/report.txt"]
+    assert prefix_fs.exists("data/2024/report.txt")
+
+
+def test_glob_prefix_preserves_existing_full_listing(prefix_fs):
+    """A prefix glob must not discard a pre-existing *full* dircache entry.
+
+    When the directory was already fully listed (e.g. by a prior ls/find with
+    no prefix), the backend serves _find from that cache without overwriting it,
+    so _glob must leave the entry intact (pre_cached branch).
+    """
+    # Simulate a prior full listing of "data/2024".
+    full_listing = [
+        {"name": "data/2024/results.csv", "type": "file", "size": 0},
+        {"name": "data/2024/report.txt", "type": "file", "size": 0},
+    ]
+    prefix_fs.dircache["data/2024"] = list(full_listing)
+
+    assert prefix_fs.glob("data/2024/res*") == ["data/2024/results.csv"]
+
+    # The full listing must survive the prefix glob untouched.
+    assert "data/2024" in prefix_fs.dircache
+    assert {e["name"] for e in prefix_fs.dircache["data/2024"]} == {
+        "data/2024/results.csv",
+        "data/2024/report.txt",
+    }
+    # And a subsequent prefix glob for the other file still resolves correctly.
+    assert prefix_fs.glob("data/2024/rep*") == ["data/2024/report.txt"]
+
+
 @pytest.mark.asyncio
 async def test_expand_path_special_characters_async():
     fs_files = [