diff --git a/fsspec/asyn.py b/fsspec/asyn.py index 32ad3d35d..745ceb788 100644 --- a/fsspec/asyn.py +++ b/fsspec/asyn.py @@ -871,9 +871,18 @@ async def _glob(self, path, maxdepth=None, **kwargs): # gcsfs, s3fs and adlfs can filter server-side up to the first wildcard. if prefix: kwargs["prefix"] = prefix + root_key = root.rstrip("/") + pre_cached = prefix and root and root_key in self.dircache allpaths = await self._find( root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs ) + # Backends store only prefix-matching files in dircache when prefix= is + # used, so a newly created entry is a partial listing that would mislead + # subsequent lookups. Only discard it when _find created it; if the + # entry existed before the call the backend served it from cache without + # overwriting, so it already holds a full legitimate listing. + if prefix and root and not pre_cached: + self.dircache.pop(root_key, None) pattern = glob_translate(path + ("/" if ends_with_sep else "")) pattern = re.compile(pattern) diff --git a/fsspec/tests/test_async.py b/fsspec/tests/test_async.py index bd0593347..4e8836058 100644 --- a/fsspec/tests/test_async.py +++ b/fsspec/tests/test_async.py @@ -296,11 +296,10 @@ def test_cat_ranges_on_error_raise_propagates(): class _PrefixCapturingFS(fsspec.asyn.AsyncFileSystem): - """Minimal AsyncFileSystem that records every _find call's kwargs. - - _find ignores the prefix hint and returns all files under *root* so that - the client-side glob pattern-matching in _glob still works correctly. - This simulates a "naive" backend that silently absorbs unknown kwargs. + """AsyncFileSystem that records _find calls and simulates cloud backend + dircache behaviour (s3fs, gcsfs, adlfs): prefix-filtered _find stores only + matching files in dircache; a second call to the same directory returns the + cached partial listing filtered by the new prefix. """ protocol = "prefixmock" @@ -312,15 +311,35 @@ def __init__(self, fs_files=None, **kwargs): async def _find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): self.find_calls.append({"path": path, "kwargs": dict(kwargs)}) - root = (path.rstrip("/") + "/") if path else "" + prefix = kwargs.get("prefix", "") + stripped = path.rstrip("/") + # Empty stripped means a root-level glob ("top_*"); no dir prefix. + root_prefix = (stripped + "/") if stripped else "" + + if stripped in self.dircache: + cached = self.dircache[stripped] + results = { + e["name"]: e + for e in cached + if e["name"][len(root_prefix) :].startswith(prefix) + } + return results if detail else list(results) + results = { f: {"name": f, "type": "file", "size": 0} for f in self.fs_files - if f.startswith(root) + if f.startswith(root_prefix) and f[len(root_prefix) :].startswith(prefix) } + self.dircache[stripped] = list(results.values()) return results if detail else list(results) async def _info(self, path, **kwargs): + parent = self._parent(path) + if parent in self.dircache: + for entry in self.dircache[parent]: + if entry["name"] == path: + return entry + raise FileNotFoundError(path) for f in self.fs_files: if f == path: return {"name": f, "type": "file", "size": 0} @@ -404,6 +423,39 @@ def test_glob_prefix_hint(prefix_fs, pattern, expected_results, expected_prefix) ) +def test_glob_prefix_does_not_poison_dircache(prefix_fs): + """Two consecutive prefix globs on the same directory must both return correct results.""" + assert prefix_fs.glob("data/2024/res*") == ["data/2024/results.csv"] + assert prefix_fs.glob("data/2024/rep*") == ["data/2024/report.txt"] + assert prefix_fs.exists("data/2024/report.txt") + + +def test_glob_prefix_preserves_existing_full_listing(prefix_fs): + """A prefix glob must not discard a pre-existing *full* dircache entry. + + When the directory was already fully listed (e.g. by a prior ls/find with + no prefix), the backend serves _find from that cache without overwriting it, + so _glob must leave the entry intact (pre_cached branch). + """ + # Simulate a prior full listing of "data/2024". + full_listing = [ + {"name": "data/2024/results.csv", "type": "file", "size": 0}, + {"name": "data/2024/report.txt", "type": "file", "size": 0}, + ] + prefix_fs.dircache["data/2024"] = list(full_listing) + + assert prefix_fs.glob("data/2024/res*") == ["data/2024/results.csv"] + + # The full listing must survive the prefix glob untouched. + assert "data/2024" in prefix_fs.dircache + assert {e["name"] for e in prefix_fs.dircache["data/2024"]} == { + "data/2024/results.csv", + "data/2024/report.txt", + } + # And a subsequent prefix glob for the other file still resolves correctly. + assert prefix_fs.glob("data/2024/rep*") == ["data/2024/report.txt"] + + @pytest.mark.asyncio async def test_expand_path_special_characters_async(): fs_files = [