From 65cbeb87a952d68b838f211de039b61fda946d49 Mon Sep 17 00:00:00 2001 From: Bruno Hays Date: Wed, 24 Jun 2026 13:00:17 +0200 Subject: [PATCH 1/2] fix(glob): discard partial dircache entry after prefix-filtered _find When `_glob` passes `prefix=` to `_find`, backends like s3fs, gcsfs and adlfs perform a server-side filtered listing and store the result in `dircache` under the parent directory key. That entry is a *partial* listing (only files matching the prefix), but it gets treated as a complete directory listing by every subsequent operation on the same path. Consequence: after `glob("dir/train-*")`, a call to `glob("dir/test-*")` or `fs.exists("dir/test-file")` hits the cached train-only listing and returns an empty result / False, even though the test files exist on the remote storage. The regression was introduced in 2026.4.0 by the prefix= optimisation (PR #1996). Fix: after `_find` returns, remove the `dircache` entry for `root` when a prefix was used. The next lookup for the same directory will perform a fresh full listing and cache it correctly. Adds a regression test using a mock backend that faithfully simulates the partial-caching behaviour (cache-hit path returns only the prefix-filtered subset, triggering the exact failure mode). Co-authored-by: Cursor --- fsspec/asyn.py | 9 +++++++++ fsspec/tests/test_async.py | 40 +++++++++++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/fsspec/asyn.py b/fsspec/asyn.py index 32ad3d35d..745ceb788 100644 --- a/fsspec/asyn.py +++ b/fsspec/asyn.py @@ -871,9 +871,18 @@ async def _glob(self, path, maxdepth=None, **kwargs): # gcsfs, s3fs and adlfs can filter server-side up to the first wildcard. if prefix: kwargs["prefix"] = prefix + root_key = root.rstrip("/") + pre_cached = prefix and root and root_key in self.dircache allpaths = await self._find( root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs ) + # Backends store only prefix-matching files in dircache when prefix= is + # used, so a newly created entry is a partial listing that would mislead + # subsequent lookups. Only discard it when _find created it; if the + # entry existed before the call the backend served it from cache without + # overwriting, so it already holds a full legitimate listing. + if prefix and root and not pre_cached: + self.dircache.pop(root_key, None) pattern = glob_translate(path + ("/" if ends_with_sep else "")) pattern = re.compile(pattern) diff --git a/fsspec/tests/test_async.py b/fsspec/tests/test_async.py index bd0593347..6ea4e5477 100644 --- a/fsspec/tests/test_async.py +++ b/fsspec/tests/test_async.py @@ -296,11 +296,10 @@ def test_cat_ranges_on_error_raise_propagates(): class _PrefixCapturingFS(fsspec.asyn.AsyncFileSystem): - """Minimal AsyncFileSystem that records every _find call's kwargs. - - _find ignores the prefix hint and returns all files under *root* so that - the client-side glob pattern-matching in _glob still works correctly. - This simulates a "naive" backend that silently absorbs unknown kwargs. + """AsyncFileSystem that records _find calls and simulates cloud backend + dircache behaviour (s3fs, gcsfs, adlfs): prefix-filtered _find stores only + matching files in dircache; a second call to the same directory returns the + cached partial listing filtered by the new prefix. """ protocol = "prefixmock" @@ -312,15 +311,35 @@ def __init__(self, fs_files=None, **kwargs): async def _find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): self.find_calls.append({"path": path, "kwargs": dict(kwargs)}) - root = (path.rstrip("/") + "/") if path else "" + prefix = kwargs.get("prefix", "") + stripped = path.rstrip("/") + # Empty stripped means a root-level glob ("top_*"); no dir prefix. + root_prefix = (stripped + "/") if stripped else "" + + if stripped in self.dircache: + cached = self.dircache[stripped] + results = { + e["name"]: e + for e in cached + if e["name"][len(root_prefix) :].startswith(prefix) + } + return results if detail else list(results) + results = { f: {"name": f, "type": "file", "size": 0} for f in self.fs_files - if f.startswith(root) + if f.startswith(root_prefix) and f[len(root_prefix) :].startswith(prefix) } + self.dircache[stripped] = list(results.values()) return results if detail else list(results) async def _info(self, path, **kwargs): + parent = self._parent(path) + if parent in self.dircache: + for entry in self.dircache[parent]: + if entry["name"] == path: + return entry + raise FileNotFoundError(path) for f in self.fs_files: if f == path: return {"name": f, "type": "file", "size": 0} @@ -404,6 +423,13 @@ def test_glob_prefix_hint(prefix_fs, pattern, expected_results, expected_prefix) ) +def test_glob_prefix_does_not_poison_dircache(prefix_fs): + """Two consecutive prefix globs on the same directory must both return correct results.""" + assert prefix_fs.glob("data/2024/res*") == ["data/2024/results.csv"] + assert prefix_fs.glob("data/2024/rep*") == ["data/2024/report.txt"] + assert prefix_fs.exists("data/2024/report.txt") + + @pytest.mark.asyncio async def test_expand_path_special_characters_async(): fs_files = [ From 72b5f4b235f5176e447046fa04fe333f78eebb3c Mon Sep 17 00:00:00 2001 From: Bruno Hays Date: Fri, 26 Jun 2026 20:00:05 +0200 Subject: [PATCH 2/2] added test for not discarding previous cache --- fsspec/tests/test_async.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/fsspec/tests/test_async.py b/fsspec/tests/test_async.py index 6ea4e5477..4e8836058 100644 --- a/fsspec/tests/test_async.py +++ b/fsspec/tests/test_async.py @@ -430,6 +430,32 @@ def test_glob_prefix_does_not_poison_dircache(prefix_fs): assert prefix_fs.exists("data/2024/report.txt") +def test_glob_prefix_preserves_existing_full_listing(prefix_fs): + """A prefix glob must not discard a pre-existing *full* dircache entry. + + When the directory was already fully listed (e.g. by a prior ls/find with + no prefix), the backend serves _find from that cache without overwriting it, + so _glob must leave the entry intact (pre_cached branch). + """ + # Simulate a prior full listing of "data/2024". + full_listing = [ + {"name": "data/2024/results.csv", "type": "file", "size": 0}, + {"name": "data/2024/report.txt", "type": "file", "size": 0}, + ] + prefix_fs.dircache["data/2024"] = list(full_listing) + + assert prefix_fs.glob("data/2024/res*") == ["data/2024/results.csv"] + + # The full listing must survive the prefix glob untouched. + assert "data/2024" in prefix_fs.dircache + assert {e["name"] for e in prefix_fs.dircache["data/2024"]} == { + "data/2024/results.csv", + "data/2024/report.txt", + } + # And a subsequent prefix glob for the other file still resolves correctly. + assert prefix_fs.glob("data/2024/rep*") == ["data/2024/report.txt"] + + @pytest.mark.asyncio async def test_expand_path_special_characters_async(): fs_files = [