Repository navigation
Expand file tree
/
Copy pathtest_codecs.py
More file actions
140 lines (116 loc) · 6.58 KB
/
Copy pathtest_codecs.py
File metadata and controls
140 lines (116 loc) · 6.58 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
"""Native decoders against their reference path, bit for bit (PLAN.md,
"Native rule"), plus the error handling both must share."""
from __future__ import annotations
import numpy as np
import pytest
from micro_reader import CorruptFileError, _codecs
zstandard = pytest.importorskip("zstandard") # test-only: makes the frames
needs_native = pytest.mark.skipif(not _codecs.native_available(),
reason="native module not built")
def _payloads():
rng = np.random.default_rng(0)
ramp = (np.add.outer(np.arange(300), np.arange(400)) * 37 % 60000).astype("<u2")
return {
"noise_u8": rng.integers(0, 256, 100_000, dtype=np.uint8),
"image_u16": ramp + rng.integers(0, 200, ramp.shape, dtype="<u2"),
"constant": np.full(1 << 20, 7, np.uint8),
"one_byte": np.array([42], np.uint8),
"large_f32": rng.normal(size=(1024, 1024)).astype("<f4"),
}
@pytest.mark.parametrize("level", [1, 3, 19])
@pytest.mark.parametrize("name", list(_payloads()))
@pytest.mark.parametrize("native", [pytest.param(True, marks=needs_native), False],
ids=["native", "reference"])
def test_zstd_decodes_exactly(name, level, native):
data = _payloads()[name]
frame = zstandard.ZstdCompressor(level=level).compress(data.tobytes())
out = np.empty_like(data)
_codecs.zstd_decompress_into(frame, out, native=native)
np.testing.assert_array_equal(out, data)
@needs_native
def test_native_equals_reference_bit_for_bit():
for data in _payloads().values():
frame = zstandard.ZstdCompressor(level=5).compress(data.tobytes())
a, b = np.empty_like(data), np.empty_like(data)
_codecs.zstd_decompress_into(frame, a, native=True)
_codecs.zstd_decompress_into(frame, b, native=False)
assert a.tobytes() == b.tobytes()
@needs_native
def test_native_accepts_any_byte_buffer():
data = _payloads()["image_u16"]
frame = zstandard.ZstdCompressor().compress(data.tobytes())
for wrapped in (frame, bytearray(frame), memoryview(frame), np.frombuffer(frame, np.uint8)):
out = np.empty_like(data)
_codecs.zstd_decompress_into(wrapped, out, native=True)
np.testing.assert_array_equal(out, data)
@pytest.mark.parametrize("native", [pytest.param(True, marks=needs_native), False],
ids=["native", "reference"])
def test_corrupt_or_mis_sized_frames_raise(native):
"""What zstd can detect: a checksum mismatch (only for frames that carry
one -- a flipped literal byte in a frame without a checksum just decodes
to wrong values, for every decoder) and a truncated frame."""
data = _payloads()["image_u16"]
frame = bytearray(zstandard.ZstdCompressor(write_checksum=True).compress(data.tobytes()))
frame[len(frame) // 2] ^= 0xFF
with pytest.raises(CorruptFileError):
_codecs.zstd_decompress_into(bytes(frame), np.empty_like(data), native=native)
good = zstandard.ZstdCompressor().compress(data.tobytes())
with pytest.raises(CorruptFileError): # truncated
_codecs.zstd_decompress_into(good[:len(good) // 2], np.empty_like(data), native=native)
with pytest.raises(CorruptFileError): # output too small
_codecs.zstd_decompress_into(good, np.empty(data.size - 1, data.dtype), native=native)
with pytest.raises(CorruptFileError): # output too large
_codecs.zstd_decompress_into(good, np.empty(data.size + 1, data.dtype), native=native)
@needs_native
def test_native_refuses_read_only_output():
data = _payloads()["noise_u8"]
frame = zstandard.ZstdCompressor().compress(data.tobytes())
out = np.empty_like(data)
out.setflags(write=False)
with pytest.raises((BufferError, ValueError)):
_codecs.zstd_decompress_into(frame, out, native=True)
def test_hilo_unpack_known_answer():
values = np.array([0x0102, 0xA0B0, 0xFFEE], "<u2")
packed = np.array([0x02, 0xB0, 0xEE, 0x01, 0xA0, 0xFF], np.uint8) # low bytes, high bytes
out = np.empty_like(values)
_codecs.hilo_unpack(packed, out)
np.testing.assert_array_equal(out, values)
@pytest.mark.parametrize("name", list(_payloads()))
@pytest.mark.parametrize("native", [pytest.param(True, marks=needs_native), False],
ids=["native", "reference"])
def test_lzf_decodes_exactly(name, native):
"""LZF as h5py's filter writes it (liblzf, no header): literal runs, short
and long back references, and overlapping ones (the constant payload)."""
imagecodecs = pytest.importorskip("imagecodecs")
data = _payloads()[name]
packed = bytes(imagecodecs.lzf_encode(data.tobytes(), header=False))
out = np.empty_like(data)
_codecs.lzf_decompress_into(packed, out, native=native)
np.testing.assert_array_equal(out, data)
@pytest.mark.parametrize("native", [pytest.param(True, marks=needs_native), False],
ids=["native", "reference"])
def test_corrupt_or_mis_sized_lzf_raises(native):
imagecodecs = pytest.importorskip("imagecodecs")
data = _payloads()["image_u16"]
packed = bytes(imagecodecs.lzf_encode(data.tobytes(), header=False))
with pytest.raises(CorruptFileError): # truncated
_codecs.lzf_decompress_into(packed[:len(packed) // 2], np.empty_like(data), native=native)
with pytest.raises(CorruptFileError): # output too small
_codecs.lzf_decompress_into(packed, np.empty(data.size - 1, data.dtype), native=native)
with pytest.raises(CorruptFileError): # output too large
_codecs.lzf_decompress_into(packed, np.empty(data.size + 1, data.dtype), native=native)
with pytest.raises(CorruptFileError): # a reference before the start
_codecs.lzf_decompress_into(b"\x20\x05", np.empty(3, np.uint8), native=native)
def test_pure_python_packbits_matches_native():
"""The last-resort PackBits decoder (no native module, no imagecodecs)
against the native one, on runs, literals and the -128 no-op."""
from micro_reader._codecs import _packbits_decode
rng = np.random.default_rng(11)
data = np.repeat(rng.integers(0, 256, 400), rng.integers(1, 9, 400)).astype(np.uint8)
imagecodecs = pytest.importorskip("imagecodecs")
packed = bytes(imagecodecs.packbits_encode(data.tobytes())) + b"\x80" # trailing no-op
assert _packbits_decode(packed, data.size) == data.tobytes()
if _codecs.native_available():
out = np.empty_like(data)
assert _codecs.packbits_decode_into(packed, out, native=True) == data.size
np.testing.assert_array_equal(out, data)