Repository navigation
Expand file tree
/
Copy pathtest_groups.py
More file actions
266 lines (222 loc) · 13.2 KB
/
Copy pathtest_groups.py
File metadata and controls
266 lines (222 loc) · 13.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
"""group_files (docs/COMPOSITE.md, "Grouping files"): which of many files
make one image -- composite parts under their main file, an index file's
members in its group, every other file a plain path that is never read."""
from __future__ import annotations
import os
import shutil
import numpy as np
import pytest
import micro_reader
from micro_reader import FileGroup, _parts
tifffile = pytest.importorskip("tifffile")
import test_companion # noqa: E402
import test_metaxpress # noqa: E402
import test_oir # noqa: E402
from _ndtiff_writer import write_ndtiff # noqa: E402
from _oif_writer import write_oif # noqa: E402
from _vsi_writer import image_volume, write_vsi # noqa: E402
from test_dv import _data as dv_data, write_dv # noqa: E402
ACQUISITION_LOG = "#\n#Tag 0x00FF0408\n\nResolve3D Image File: /data/{name}\n"
def _names(paths, root):
return sorted(os.path.relpath(p, root).replace(os.sep, "/") for p in paths)
def _by_main(grouping, root):
return {os.path.relpath(g.expected, root).replace(os.sep, "/"): g for g in grouping.groups}
@pytest.fixture
def mixed(tmp_path):
"""One folder holding several multi-file images and some plain files."""
root = tmp_path
write_vsi(root, "slide", [image_volume(1, (20, 30))],
{1: {"levels": [np.zeros((20, 30), np.uint16)]}})
write_oif(str(root), "cells", np.zeros((1, 2, 2, 9, 11), np.uint16))
write_dv(root / "a_R3D.dv", dv_data((1, 1, 2, 5, 6)))
(root / "a_R3D.dv.log").write_text(ACQUISITION_LOG.format(name="a_R3D.dv"))
test_oir._write(root, "companion_file") # main + _00001
for name in ("run_MMStack_Pos0.ome.tif", "run_MMStack_Pos0_1.ome.tif"):
tifffile.imwrite(root / name, np.zeros((4, 5), np.uint8))
frames = [({"time": t}, np.zeros((6, 7), np.uint16)) for t in range(4)]
write_ndtiff(str(root / "nd"), frames, per_file=2)
test_metaxpress._write(str(root / "plate"), wells=[(0, 0)], sites=[[True, True]])
tifffile.imwrite(root / "plain.tif", np.zeros((4, 4), np.uint8))
(root / "notes.txt").write_text("not an image")
return root
def test_a_folder_groups_into_images(mixed):
g = micro_reader.group_files(str(mixed))
groups = _by_main(g, mixed)
assert sorted(groups) == ["a_R3D.dv", "cells.oif", "companion_file.oir", "nd/data_NDTiffStack.tif",
"plate/Plate7.HTD", "run_MMStack_Pos0.ome.tif", "slide.vsi"]
vsi = groups["slide.vsi"]
assert (vsi.kind, vsi.format) == ("composite", "vsi")
assert _names(vsi.parts, mixed) == ["_slide_/stack1/frame_t.ets"]
assert os.path.samefile(vsi.main, mixed / "slide.vsi")
assert len(groups["cells.oif"].parts) > 4 and groups["cells.oif"].format == "oif"
assert _names(groups["a_R3D.dv"].parts, mixed) == ["a_R3D.dv.log"]
assert _names(groups["companion_file.oir"].parts, mixed) == ["companion_file_00001"]
assert _names(groups["run_MMStack_Pos0.ome.tif"].parts, mixed) == ["run_MMStack_Pos0_1.ome.tif"]
assert _names(groups["nd/data_NDTiffStack.tif"].parts, mixed) == \
["nd/NDTiff.index", "nd/data_NDTiffStack_1.tif"]
plate = groups["plate/Plate7.HTD"]
assert (plate.kind, plate.format, plate.notes) == ("index", "metaxpress", ())
assert len(plate.members) == 4 and all(os.path.isfile(m) for m in plate.members)
# the plate's thumbnails are not named by the index, but are its parts
assert _names(plate.parts, mixed) == sorted(
f"plate/Plate7_A01_s{s}_w{w}_thumb.TIF" for s in (1, 2) for w in (1, 2))
assert _names(g.singles, mixed) == ["notes.txt", "plain.tif"]
def test_every_file_is_in_one_place(mixed):
g = micro_reader.group_files(str(mixed))
placed = list(g.singles)
for group in g.groups:
placed += list(group.parts) + list(group.members) + [group.main]
keys = [os.path.normcase(os.path.abspath(p)) for p in placed]
assert len(keys) == len(set(keys))
walked = {os.path.normcase(os.path.join(r, n)) for r, _, files in os.walk(mixed) for n in files}
assert set(keys) == walked
def test_a_metaxpress_preview_opens_its_plate(tmp_path):
htd, _ = test_metaxpress._write(str(tmp_path), wells=[(0, 0)], sites=[[True]])
thumb = os.path.join(str(tmp_path), "Plate7_A01_s1_w1_thumb.TIF")
with micro_reader.open(thumb) as f:
assert f.backend == "metaxpress" and os.path.samefile(f.path, htd)
os.remove(htd) # no .HTD: just a TIFF
lone = os.path.join(str(tmp_path), "Plate7_A01_s1_w2_thumb.TIF")
with micro_reader.open(lone) as f:
assert f.backend == "tiff"
def test_harmony_side_folders_open_the_measurement(tmp_path):
import test_harmony
index = test_harmony._write_harmony(str(tmp_path / "Measurement 1"))[0]
images = tmp_path / "Measurement 1" / "Images"
for side, name in (("Assaylayout", "layout.xml"), ("FFC_Profile", "FFC_Profile_Measurement 1.xml")):
os.makedirs(images.parent / side)
(images.parent / side / name).write_text("<Results/>")
for target in (images.parent, images.parent / "Assaylayout", images.parent / "FFC_Profile" /
"FFC_Profile_Measurement 1.xml"):
with micro_reader.open(str(target)) as f:
assert f.backend == "harmony" and os.path.samefile(f.path, index)
(group,) = micro_reader.group_files(str(tmp_path)).groups
assert _names(group.parts, images.parent) == ["Assaylayout/layout.xml",
"FFC_Profile/FFC_Profile_Measurement 1.xml"]
def test_opening_a_plate_folder_reads_only_its_index(tmp_path, monkeypatch):
htd, _ = test_metaxpress._write(str(tmp_path), wells=[(0, 0), (0, 1)], sites=[[True, True]])
reads = []
real = _parts.read_head
monkeypatch.setattr(_parts, "read_head", lambda p: reads.append(os.path.basename(p)) or real(p))
with micro_reader.open(str(tmp_path)) as f:
assert os.path.samefile(f.path, htd)
assert reads == ["Plate7.HTD"]
def test_plain_files_are_never_read(tmp_path, monkeypatch):
for t in range(3):
for z in range(20):
(tmp_path / f"img_t{t}_z{z:03d}.tif").write_bytes(b"")
reads = []
real = _parts.read_head
monkeypatch.setattr(_parts, "read_head", lambda p: reads.append(p) or real(p))
g = micro_reader.group_files(str(tmp_path))
assert (len(g.singles), g.groups, reads) == (60, [], [])
def test_given_files_and_a_missing_main_file(tmp_path):
path = write_oif(str(tmp_path), "cells", np.zeros((1, 2, 1, 9, 11), np.uint16))
planes = sorted(os.path.join(path + ".files", n) for n in os.listdir(path + ".files")
if n.endswith(".tif"))
g = micro_reader.group_files(planes) # parts only: the main is found
(group,) = g.groups
assert os.path.samefile(group.main, path) and list(group.parts) == planes and not g.singles
os.remove(path)
(group,) = micro_reader.group_files(planes).groups # the main file is missing
assert group.main is None and group.expected == path and list(group.parts) == planes
with pytest.raises(FileNotFoundError):
micro_reader.group_files([path])
def test_index_files_of_the_same_images_are_one_group(tmp_path):
first, _ = test_companion._write(str(tmp_path), images=1)
second = os.path.join(str(tmp_path), "copy.companion.ome")
shutil.copyfile(first, second)
g = micro_reader.group_files(str(tmp_path))
(group,) = g.groups
assert group.kind == "index" and group.format == "companion"
assert {os.path.basename(group.main), *map(os.path.basename, group.parts)} == \
{"set.companion.ome", "copy.companion.ome"}
assert group.members and not g.singles
def _ome_set(folder, names):
"""A multi-file OME-TIFF set: one image of len(names) z planes, plane k in
file k; every file's OME-XML names all files by UUID (pointed both ways)."""
uuids = [f"urn:uuid:0000000{k}-0000-0000-0000-000000000000" for k in range(len(names))]
tiffdata = "".join(f'<TiffData FirstZ="{k}" IFD="0" PlaneCount="1">'
f'<UUID FileName="{n}">{u}</UUID></TiffData>'
for k, (n, u) in enumerate(zip(names, uuids)))
for k, name in enumerate(names):
xml = ('<?xml version="1.0" encoding="UTF-8"?>'
'<OME xmlns="http://www.openmicroscopy.org/Schemas/OME/2016-06" '
f'UUID="{uuids[k]}"><Image ID="Image:0" Name="cells"><Pixels ID="Pixels:0" '
f'DimensionOrder="XYZCT" Type="uint8" SizeX="8" SizeY="6" SizeZ="{len(names)}" '
f'SizeC="1" SizeT="1"><Channel ID="Channel:0" SamplesPerPixel="1"/>{tiffdata}'
'</Pixels></Image></OME>')
tifffile.imwrite(os.path.join(folder, name), np.full((6, 8), k, np.uint8),
description=xml, metadata=None)
def test_a_multi_file_ome_tiff_set_is_one_group(tmp_path):
# its files name each other by UUID: one image, as opening any of them gives
# (gap hunt B-09: 86 files of a 4-D set were 86 single images)
_ome_set(str(tmp_path), ["cells_Z0.ome.tif", "cells_Z1.ome.tif", "cells_Z2.ome.tif"])
tifffile.imwrite(str(tmp_path / "alone.ome.tif"), np.zeros((6, 8), np.uint8))
g = micro_reader.group_files(str(tmp_path))
(group,) = g.groups
assert group.kind == "index" and os.path.basename(group.main) == "cells_Z0.ome.tif"
assert {os.path.basename(m) for m in group.members} == {"cells_Z1.ome.tif", "cells_Z2.ome.tif"}
assert _names(g.singles, str(tmp_path)) == ["alone.ome.tif"] # one file: a plain path
with micro_reader.open(group.main) as f:
image = f.images[0]
assert image.shape[image.axes.index("z")] == 3
def test_a_leica_experiment_holds_its_references(tmp_path):
pytest.importorskip("imagecodecs")
from _lif_writer import image, write_xlef, write_xlif
data = np.zeros((2, 3, 20, 30), np.uint16)
write_xlif(str(tmp_path / "a.xlif"), image("a", data, "czyx"), frames="png")
write_xlef(str(tmp_path / "exp.xlef"), ["a.xlif"])
g = micro_reader.group_files(str(tmp_path))
(group,) = g.groups # a.xlif is a member, not a group
assert os.path.basename(group.main) == "exp.xlef" and group.format == "lif"
assert "a.xlif" in {os.path.basename(m) for m in group.members}
assert not g.singles # the frames are members too
def test_an_index_that_cannot_be_read_is_reported(tmp_path):
(tmp_path / "bad.HTD").write_text('"HTSInfoFile", Version 1.0\n"XWells", nonsense\n')
(group,) = micro_reader.group_files(str(tmp_path)).groups
assert group.kind == "index" and not group.members
assert any("cannot be read" in note for note in group.notes)
def test_group_is_a_value():
group = FileGroup("composite", "a.vsi", "a.vsi", "vsi", ("x.ets",))
assert group == FileGroup("composite", "a.vsi", "a.vsi", "vsi", ("x.ets",))
with pytest.raises(AttributeError):
group.main = "b.vsi"
def test_ndtiff_stack_files_open_the_dataset(tmp_path):
frames = [({"time": t}, np.full((6, 7), t, np.uint16)) for t in range(4)]
paths = write_ndtiff(str(tmp_path), frames, per_file=2)
with micro_reader.open(paths[1]) as f: # the second stack file
assert os.path.samefile(f.path, paths[0]) and f.opened == paths[1]
def test_a_cellvoyager_measurement_is_one_group(tmp_path):
"""Its .wpi is the main file; the other entry files, the Harmony-format
export and the settings files are parts; the planes are members."""
import test_cv7000
data = test_cv7000._write(str(tmp_path))
(tmp_path / "Index.idx.xml").write_text("<EvaluationInputData/>") # the instrument's export
g = micro_reader.group_files(str(tmp_path))
(group,) = g.groups
assert (group.kind, group.format, os.path.basename(group.main)) == ("index", "cv7000", "P1.wpi")
assert _names(group.parts, tmp_path) == ["Index.idx.xml", "MeasurementData.mlf",
"MeasurementDetail.mrf", "s.mes", "s.wpp"]
assert len(group.members) == len(data) and not g.singles
for target in ("s.mes", "Index.idx.xml", "MeasurementDetail.mrf", ""): # "" = the folder
with micro_reader.open(os.path.join(str(tmp_path), target) if target else str(tmp_path)) as f:
assert f.backend == "cv7000" and os.path.basename(f.path) == "P1.wpi"
os.remove(tmp_path / "P1.wpi") # no .wpi: the .mrf is the main file
(group,) = micro_reader.group_files(str(tmp_path)).groups
assert os.path.basename(group.main) == "MeasurementDetail.mrf"
assert "MeasurementData.mlf" in _names(group.parts, tmp_path)
def test_cellvoyager_correction_images_are_parts_of_the_plate(tmp_path):
"""The shading / dark-current / OTF files the instrument writes into the
measurement folder belong to the plate's group (not loose images), and
still open alone."""
import test_cv7000
test_cv7000._write(str(tmp_path))
names = ["SC_BP445-45_20x_M01_CH01.tif", "DC_EMCCD _1_CAM1.tif", "OTF_SC_BP525-50_20x_M01_CH02.tif"]
for name in names:
tifffile.imwrite(tmp_path / name, np.zeros((6, 10), "u2"))
g = micro_reader.group_files(str(tmp_path))
(group,) = g.groups
assert set(names) <= set(_names(group.parts, tmp_path)) and not g.singles
with micro_reader.open(tmp_path / names[0]) as f:
assert f.backend == "tiff" and f.images[0].shape == (6, 10)