diff --git a/dissect/archive/tibx/codec.py b/dissect/archive/tibx/codec.py index 960ca6f..217de96 100644 --- a/dissect/archive/tibx/codec.py +++ b/dissect/archive/tibx/codec.py @@ -22,7 +22,7 @@ from dissect.util.exceptions import CorruptDataError from dissect.archive.tibx.c_tibx import LZ4_BLOCK_HEADER_SIZE -from dissect.archive.tibx.exception import CorruptArchiveError +from dissect.archive.tibx.exception import CorruptArchiveError, UnsupportedFormatError # Generous ceiling for a single decompressed unit; real segments are a few MiB MAX_DECOMPRESSED = 2 << 30 @@ -43,10 +43,17 @@ def decompress_zstd(data: bytes, max_output: int) -> bytes: Python 3.14 it comes from ``backports.zstd``, which only the ``full`` extra installs, and opening an archive and listing what it holds needs no zstd. """ - if sys.version_info >= (3, 14): - from compression import zstd # novermin - else: - from backports import zstd + try: + if sys.version_info >= (3, 14): + from compression import zstd # novermin + else: + from backports import zstd + except ImportError: + # An Error, so that what tolerates an unreadable segment (TIBX.volumes()) does here too. + # Python 3.14 itself may be built without zstd. + raise UnsupportedFormatError( + "reading zstd-compressed TIBX segments needs zstd support: install dissect.archive[full]" + ) try: decompressor = zstd.ZstdDecompressor() diff --git a/dissect/archive/tibx/segment.py b/dissect/archive/tibx/segment.py index b8676db..c6f93c8 100644 --- a/dissect/archive/tibx/segment.py +++ b/dissect/archive/tibx/segment.py @@ -175,16 +175,24 @@ def read_plaintext(store: PageStore, page_index: int, data_key: DataKey | None = plaintext = decrypt_segment(read_compressed(store, segment, pages[1:]), data_key) if segment.compression in COMP_ZSTD: # zstd frame parsing ignores any trailing CBC/PKCS#7 padding - return decompress_zstd(plaintext, segment.length) - # Stored or LZ4: CBC leaves PKCS#7 padding to strip, GCM is already exact -- and - # unpadding it there would eat real bytes whenever the payload happens to end in - # a valid padding pattern. - payload = plaintext if data_key.gcm else _pkcs7_unpad(plaintext) - if segment.compression == COMP_LZ4 and len(payload) != segment.length: - return lz4_block_decompress(payload, segment.length) - if segment.compression in (COMP_NONE, COMP_LZ4) or segment.compression in COMP_STORED_VARIANTS: - return payload[: segment.length] - raise UnsupportedFormatError( - f"encrypted segment at page {page_index} uses compression {segment.compression:#06x}" - ) + out = decompress_zstd(plaintext, segment.length) + else: + # Stored or LZ4: CBC leaves PKCS#7 padding to strip, GCM is already exact -- and + # unpadding it there would eat real bytes whenever the payload happens to end in + # a valid padding pattern. + payload = plaintext if data_key.gcm else _pkcs7_unpad(plaintext) + if segment.compression == COMP_LZ4 and len(payload) != segment.length: + out = lz4_block_decompress(payload, segment.length) + elif segment.compression in (COMP_NONE, COMP_LZ4) or segment.compression in COMP_STORED_VARIANTS: + out = payload[: segment.length] + else: + raise UnsupportedFormatError( + f"encrypted segment at page {page_index} uses compression {segment.compression:#06x}" + ) + # As for a plaintext segment: a short result is damage, not data to zero-fill + if len(out) != segment.length: + raise CorruptArchiveError( + f"encrypted segment at page {page_index} decoded {len(out)} bytes, expected {segment.length}" + ) + return out return decompress(read_compressed(store, segment, pages[1:]), segment) diff --git a/dissect/archive/tibx/tibx.py b/dissect/archive/tibx/tibx.py index f24a064..c52160e 100644 --- a/dissect/archive/tibx/tibx.py +++ b/dissect/archive/tibx/tibx.py @@ -45,7 +45,7 @@ InvalidArchiveError, UnsupportedFormatError, ) -from dissect.archive.tibx.lsm import iter_memtree_cells, read_archive_header +from dissect.archive.tibx.lsm import iter_tree_cells, read_archive_header from dissect.archive.tibx.map import load_extents, load_segment_index from dissect.archive.tibx.page import PageStore from dissect.archive.tibx.segment import read_plaintext @@ -139,15 +139,25 @@ def __init__(self, fh: BinaryIO | list[BinaryIO]): self._reset_snapshot_caches() def _reset_snapshot_caches(self) -> None: - # Everything derived from the currently-selected commit root (not the data key) - self._extents: ExtentTable | None = None + # Everything derived from the live commit root (not the data key) + self._all_extents: ExtentTable | None = None self._segment_index = None - self._segment_cache: OrderedDict[int, bytes] = OrderedDict() - self._segment_cache_bytes = 0 self._segment_layout: dict[tuple[int, int], int] | None = None + self._clear_segment_cache() + self._reset_view_caches() + + def _reset_view_caches(self) -> None: + # Everything that depends on the selected recovery point. A segment's plaintext, + # its location and the layout of the extents it holds do not: a segment is written + # by one backup, and later backups only reference it again. + self._extents: ExtentTable | None = None self._streams: list[TibxVolume] | None = None self._volumes: list[TibxVolume] | None = None + def _clear_segment_cache(self) -> None: + self._segment_cache: OrderedDict[int, bytes] = OrderedDict() + self._segment_cache_bytes = 0 + @classmethod def open(cls, path: str | Path) -> TIBX: """Open an archive from a path, auto-discovering split parts. @@ -252,7 +262,7 @@ def unlock(self, password: str | bytes) -> None: ) self._data_key = unwrap_data_key(self.header, password) - self._segment_cache.clear() + self._clear_segment_cache() def recovery_points(self) -> list[RecoveryPoint]: """The backups in this archive, oldest first. @@ -267,16 +277,26 @@ def recovery_points(self) -> list[RecoveryPoint]: so counting roots reports recovery points that were never taken. """ if self._recovery_points is None: - points: list[RecoveryPoint] = [] + records: dict[int, c_tibx.slice_record] = {} slices = self.header.tree(TLV_SLICES) if slices is not None and slices.has_records: - for cell in iter_memtree_cells(slices): + seen: set[bytes] = set() + # Like every LSM tree, the slices tree moves from its mem-tree into on-disk + # ctrees as it grows; across those runs the newest record of a key wins + for cell in iter_tree_cells(self.store, slices): + if cell.key in seen: + continue + seen.add(cell.key) # The tree opens with an empty placeholder record; a real backup has a value. - if len(cell.value) < len(c_tibx.slice_record) or len(cell.key) < len(c_tibx.slice_key): + if not cell.alive or len(cell.value) < len(c_tibx.slice_record): + continue + if len(cell.key) < len(c_tibx.slice_key): continue key = c_tibx.slice_key(cell.key) - points.append(RecoveryPoint(len(points), key.slice_id, c_tibx.slice_record(cell.value))) - self._recovery_points = points + records[key.slice_id] = c_tibx.slice_record(cell.value) + self._recovery_points = [ + RecoveryPoint(index, slice_id, records[slice_id]) for index, slice_id in enumerate(sorted(records)) + ] return self._recovery_points def use_recovery_point(self, recovery_point: int | str = "latest") -> None: @@ -309,7 +329,7 @@ def use_recovery_point(self, recovery_point: int | str = "latest") -> None: # slice_id as the primary recency signal -- so dropping everything newer than # the chosen slice is exactly the archive as it stood after that backup. self._slice_limit = chosen.slice_id - self._reset_snapshot_caches() + self._reset_view_caches() def disks(self) -> list: """Whole-disk views of this archive. @@ -325,12 +345,18 @@ def disks(self) -> list: def extents(self) -> ExtentTable: """The data_map extents in view: all of them, or those up to the selected slice.""" if self._extents is None: - extents = load_extents(self.store, self.header) + extents = self._unfiltered_extents() if self._slice_limit is not None: extents = extents.select(map(le, extents.slice_ids, repeat(self._slice_limit))) self._extents = extents return self._extents + def _unfiltered_extents(self) -> ExtentTable: + """Every data_map extent, whatever recovery point is selected.""" + if self._all_extents is None: + self._all_extents = load_extents(self.store, self.header) + return self._all_extents + def streams(self) -> list[TibxVolume]: """Every data_map stream, backed-up volumes and Acronis's own metadata alike. @@ -419,15 +445,19 @@ def _extent_base(self, segment_id: int, extent_index: int) -> int: if extent_index == EXTENT_INDEX_WHOLE_SEGMENT: return 0 if self._segment_layout is None: + # From every extent, not just those of the selected recovery point: a volume + # opened before use_recovery_point() still reads through this layout layout: dict[tuple[int, int], int] = {} - shared: dict[int, list[tuple[int, int]]] = defaultdict(list) - extents = self.extents + shared: dict[int, dict[int, int]] = defaultdict(dict) + extents = self._unfiltered_extents() indexes = extents.extent_indexes for row in compress(range(len(indexes)), map(ne, indexes, repeat(EXTENT_INDEX_WHOLE_SEGMENT))): - shared[extents.segment_ids[row]].append((indexes[row], extents.extent_lengths[row])) + # An incremental backup records the unchanged extents again under its own + # slice, so one extent of a segment can appear in several rows: count it once + shared[extents.segment_ids[row]].setdefault(indexes[row], extents.extent_lengths[row]) for shared_segment, entries in shared.items(): offset = 0 - for index, length in sorted(entries, key=itemgetter(0)): + for index, length in sorted(entries.items(), key=itemgetter(0)): layout[(shared_segment, index)] = offset offset += length offset = (offset + EXTENT_ALIGNMENT - 1) & ~(EXTENT_ALIGNMENT - 1) @@ -478,7 +508,9 @@ def __init__(self, tibx: TIBX, volume_id: int, table: ExtentTable, start: int = self._size: int | None = None def __repr__(self) -> str: - return f"" + # The span, not the size: that would read the boot sector, which needs the password + # on an encrypted archive + return f"" @property def extents(self) -> ExtentTable: @@ -631,10 +663,21 @@ def find_split_parts(path: Path) -> list[Path]: """ match = SPLIT_PART_RE.match(path.name) stem = match.group("stem") if match else (path.stem if path.suffix.lower() == ".tibx" else path.name) - parts = sorted(path.parent.glob(f"{stem}-[0-9][0-9][0-9][0-9].tibx"), key=lambda part: part.name) - main = path.parent / f"{stem}.tibx" - if parts and main.exists(): - return [main, *parts] + # Matched by hand rather than globbed: a stem like "Backup [PC1]" is no glob pattern, + # and the extension may be upper case on a case-sensitive filesystem + main = None + parts = [] + try: + candidates = list(path.parent.iterdir()) + except OSError: + return [path] + for candidate in candidates: + if candidate.suffix.lower() == ".tibx" and candidate.name == stem + candidate.suffix: + main = candidate + elif (part := SPLIT_PART_RE.match(candidate.name)) and part.group("stem") == stem: + parts.append((int(part.group("num")), candidate)) + if parts and main is not None: + return [main, *(part for _, part in sorted(parts))] return [path] diff --git a/tests/test_crypto.py b/tests/test_crypto.py index 09d81b3..f103efe 100644 --- a/tests/test_crypto.py +++ b/tests/test_crypto.py @@ -253,3 +253,42 @@ def test_locked_large_stream_is_a_volume_without_the_password() -> None: tibx = TIBX(io.BytesIO(archive)) assert [v.volume_id for v in tibx.volumes()] == [10] + + +def test_locked_volume_repr() -> None: + # repr() must not need the password: it is what logging and the REPL show + archive = build_lsm_archive([ExtentSpec(10, 0, b"x" * 64)], password=PASSWORD) + tibx = TIBX(io.BytesIO(archive)) + + assert "span=64" in repr(tibx.streams()) + + +def test_unlock_resets_segment_cache_budget() -> None: + archive = build_lsm_archive([ExtentSpec(10, 0, b"x" * 0x1000)], password=PASSWORD) + tibx = TIBX(io.BytesIO(archive)) + tibx.unlock(PASSWORD) + tibx.streams()[0].read(0, 16) + assert tibx._segment_cache_bytes + + tibx.unlock(PASSWORD) + assert not tibx._segment_cache + assert tibx._segment_cache_bytes == 0 + + +@pytest.mark.parametrize("alg", [3, 7]) +def test_encrypted_segment_shorter_than_its_length(alg: int) -> None: + # As for a plaintext segment, a segment that decodes short is corrupt: zero-filling the + # rest would pass damage off as volume data + from tests._synth import PAGE, SEGMENT_HEADER_OFFSET, finalize + + archive = build_lsm_archive([ExtentSpec(10, 0, b"x" * 64)], compression=COMP_NONE, password=PASSWORD, alg=alg) + page = bytearray(archive[PAGE : 2 * PAGE]) # the segment follows the ARCH page + header = c_tibx.segment_header(bytes(page[SEGMENT_HEADER_OFFSET:])) + header.length += 16 + page[SEGMENT_HEADER_OFFSET : SEGMENT_HEADER_OFFSET + len(c_tibx.segment_header)] = header.dumps() + archive = archive[:PAGE] + finalize(page) + archive[2 * PAGE :] + + tibx = TIBX(io.BytesIO(archive)) + tibx.unlock(PASSWORD) + with pytest.raises(CorruptArchiveError, match="decoded 64 bytes, expected 80"): + tibx.streams()[0].read(0, 16) diff --git a/tests/test_recovery.py b/tests/test_recovery.py index 7495406..e9fd0fb 100644 --- a/tests/test_recovery.py +++ b/tests/test_recovery.py @@ -5,9 +5,19 @@ import pytest +from dissect.archive.tibx.c_tibx import c_tibx from dissect.archive.tibx.exception import InvalidArchiveError from dissect.archive.tibx.tibx import TIBX -from tests._synth import ExtentSpec, build_lsm_archive +from tests._synth import ( + PAGE, + SLICE_VALUE_LENGTH, + Cell, + ExtentSpec, + build_lsm_archive, + lsb, + lsm_page, + slice_record, +) # A chain lives in one archive: each backup writes its extents under its own slice id, and # the slices tree (TLV[5]) records the backups. Slice ids are not contiguous in the wild -- @@ -102,3 +112,30 @@ def test_archive_without_slices_reports_no_recovery_points() -> None: assert tibx.recovery_points() == [] with pytest.raises(InvalidArchiveError, match="records no backups"): tibx.use_recovery_point(0) + + +def test_recovery_points_from_memtree_and_ctrees() -> None: + # As the slices tree grows its records move from the mem-tree into on-disk ctrees, like + # those of every LSM tree. Across them the newest record of a slice wins. + extents = [ExtentSpec(10, 0, b"x" * 64, slice_id=slice_id) for slice_id in SLICES] + + def cell(slice_id: int, marker: int) -> Cell: + key = c_tibx.slice_key(slice_id=slice_id).dumps() + return Cell(key, slice_record(bytes([marker]) * 16, 1000 * slice_id, 1000 * slice_id + 10)) + + leaf_page = len(build_lsm_archive(extents)) // PAGE + slices = lsb( + 4, + SLICE_VALUE_LENGTH, + memtree_cells=[cell(3, 0x33)], # newer than the ctree's record of slice 3 + ctrees=[(leaf_page * PAGE, 3)], + ) + leaf = lsm_page(c_tibx.PageType.LEAF, b"LEAF", [cell(2, 2), cell(3, 3), cell(5, 5)], key_length=4, compact=True) + tibx = TIBX(io.BytesIO(build_lsm_archive(extents, extra_slots={5: slices}) + leaf)) + + points = tibx.recovery_points() + assert [point.slice_id for point in points] == SLICES + assert [point.guid.bytes_le[0] for point in points] == [2, 0x33, 5] + + tibx.use_recovery_point(2) + assert tibx.extents diff --git a/tests/test_tibx.py b/tests/test_tibx.py index c2f4c9b..cf1e155 100644 --- a/tests/test_tibx.py +++ b/tests/test_tibx.py @@ -15,7 +15,7 @@ import pytest from dissect.archive.tibx.map import Extent, ExtentTable, load_extents, load_segment_index -from dissect.archive.tibx.tibx import MIN_VOLUME_SIZE, TIBX, resolve_extents +from dissect.archive.tibx.tibx import MIN_VOLUME_SIZE, TIBX, find_split_parts, resolve_extents from tests._synth import ( COMP_NONE, PAGE, @@ -35,6 +35,7 @@ segment_map_value, segment_pages, sparse_extents, + write_split_parts, ) if TYPE_CHECKING: @@ -720,5 +721,84 @@ def test_open_without_zstd(monkeypatch: pytest.MonkeyPatch) -> None: tibx_module = importlib.import_module("dissect.archive.tibx") assert tibx_module.TIBX(io.BytesIO(archive)).streams()[0].read(0, 4) == b"AAAA" - with pytest.raises(ImportError): - tibx_module.TIBX(io.BytesIO(zstd_archive)).streams()[0].read(0, 4) + # Reading zstd fails with the parser's own error, which listing volumes tolerates + zstd_tibx = tibx_module.TIBX(io.BytesIO(zstd_archive)) + with pytest.raises(tibx_module.UnsupportedFormatError, match="zstd"): + zstd_tibx.streams()[0].read(0, 4) + zstd_tibx.volumes() + + +def test_shared_segment_extent_recorded_twice() -> None: + # An incremental backup records unchanged extents again under its own slice, pointing + # at the same segment and extent index. Counting such an extent once per row moved + # every later extent of the segment. + tibx = _open( + build_lsm_archive( + [ + ExtentSpec(10, 0, b"A" * 20, segment_group=1), + ExtentSpec(10, 0x1000, b"B" * 20, segment_group=1), + ] + ) + ) + extents = list(tibx.extents) + first = extents[0] + again = first._replace(slice_id=first.slice_id + 1, extent_id=first.extent_id + 10) + tibx._all_extents = ExtentTable.from_extents([first, again, *extents[1:]]) + tibx._reset_view_caches() + + assert tibx.streams()[0].read(0x1000, 4) == b"BBBB" + + +def test_volume_reads_after_selecting_an_older_recovery_point() -> None: + # A volume opened at the latest point keeps reading the extents it was opened with. + # Those of a segment shared by several extents must not lose their place in it. + tibx = _open( + build_lsm_archive( + [ + ExtentSpec(10, 0, b"base" * 16, slice_id=2), + ExtentSpec(10, 0x1000, b"A" * 20, slice_id=3, segment_group=1), + ExtentSpec(10, 0x2000, b"B" * 20, slice_id=3, segment_group=1), + ], + slices=[2, 3], + ) + ) + volume = tibx.streams()[0] + tibx.use_recovery_point(0) + + assert volume.read(0x2000, 4) == b"BBBB" + assert tibx.streams()[0].read(0x2000, 4) == bytes(4) + + +def test_selecting_a_recovery_point_keeps_the_segment_cache() -> None: + # Neither a segment's plaintext nor its location depends on the selected point + tibx = _open(build_lsm_archive([ExtentSpec(10, 0, b"A" * 0x1000)], slices=[2])) + tibx.streams()[0].read(0, 4) + segment_index = tibx._segment_index + tibx.use_recovery_point(0) + + assert tibx._segment_cache + assert tibx._segment_index is segment_index + assert tibx.streams()[0].read(0, 4) == b"AAAA" + + +@pytest.mark.parametrize("name", ["Backup [PC1]", "Backup*"]) +def test_find_split_parts_with_glob_characters(tmp_path: Path, name: str) -> None: + parts = write_split_parts(b"x" * 300, tmp_path, name, (100, 200)) + + assert find_split_parts(parts[0]) == parts + assert find_split_parts(parts[2]) == parts + + +def test_find_split_parts_upper_case_extension(tmp_path: Path) -> None: + parts = write_split_parts(b"x" * 300, tmp_path, "Backup", (100, 200)) + parts = [part.rename(part.with_suffix(".TIBX")) for part in parts] + + assert find_split_parts(parts[0]) == parts + + +def test_find_split_parts_orders_numerically_and_ignores_other_stems(tmp_path: Path) -> None: + parts = write_split_parts(b"x" * 300, tmp_path, "Backup", (100, 200)) + (tmp_path / "Backup2.tibx").write_bytes(b"") + (tmp_path / "Backup2-0001.tibx").write_bytes(b"") + + assert find_split_parts(parts[1]) == parts