Fix review findings in the TIBX parser

dissect/archive/tibx/codec.py

@@ -22,7 +22,7 @@
 from dissect.util.exceptions import CorruptDataError
 
 from dissect.archive.tibx.c_tibx import LZ4_BLOCK_HEADER_SIZE
-from dissect.archive.tibx.exception import CorruptArchiveError
+from dissect.archive.tibx.exception import CorruptArchiveError, UnsupportedFormatError
 
 # Generous ceiling for a single decompressed unit; real segments are a few MiB
 MAX_DECOMPRESSED = 2 << 30
@@ -43,10 +43,17 @@ def decompress_zstd(data: bytes, max_output: int) -> bytes:
     Python 3.14 it comes from ``backports.zstd``, which only the ``full`` extra installs, and
     opening an archive and listing what it holds needs no zstd.
     """
-    if sys.version_info >= (3, 14):
-        from compression import zstd  # novermin
-    else:
-        from backports import zstd
+    try:
+        if sys.version_info >= (3, 14):
+            from compression import zstd  # novermin
+        else:
+            from backports import zstd
+    except ImportError:
+        # An Error, so that what tolerates an unreadable segment (TIBX.volumes()) does here too.
+        # Python 3.14 itself may be built without zstd.
+        raise UnsupportedFormatError(
+            "reading zstd-compressed TIBX segments needs zstd support: install dissect.archive[full]"
+        )
 
     try:
         decompressor = zstd.ZstdDecompressor()

dissect/archive/tibx/segment.py

@@ -175,16 +175,24 @@ def read_plaintext(store: PageStore, page_index: int, data_key: DataKey | None =
         plaintext = decrypt_segment(read_compressed(store, segment, pages[1:]), data_key)
         if segment.compression in COMP_ZSTD:
             # zstd frame parsing ignores any trailing CBC/PKCS#7 padding
-            return decompress_zstd(plaintext, segment.length)
-        # Stored or LZ4: CBC leaves PKCS#7 padding to strip, GCM is already exact -- and
-        # unpadding it there would eat real bytes whenever the payload happens to end in
-        # a valid padding pattern.
-        payload = plaintext if data_key.gcm else _pkcs7_unpad(plaintext)
-        if segment.compression == COMP_LZ4 and len(payload) != segment.length:
-            return lz4_block_decompress(payload, segment.length)
-        if segment.compression in (COMP_NONE, COMP_LZ4) or segment.compression in COMP_STORED_VARIANTS:
-            return payload[: segment.length]
-        raise UnsupportedFormatError(
-            f"encrypted segment at page {page_index} uses compression {segment.compression:#06x}"
-        )
+            out = decompress_zstd(plaintext, segment.length)
+        else:
+            # Stored or LZ4: CBC leaves PKCS#7 padding to strip, GCM is already exact -- and
+            # unpadding it there would eat real bytes whenever the payload happens to end in
+            # a valid padding pattern.
+            payload = plaintext if data_key.gcm else _pkcs7_unpad(plaintext)
+            if segment.compression == COMP_LZ4 and len(payload) != segment.length:
+                out = lz4_block_decompress(payload, segment.length)
+            elif segment.compression in (COMP_NONE, COMP_LZ4) or segment.compression in COMP_STORED_VARIANTS:
+                out = payload[: segment.length]
+            else:
+                raise UnsupportedFormatError(
+                    f"encrypted segment at page {page_index} uses compression {segment.compression:#06x}"
+                )
+        # As for a plaintext segment: a short result is damage, not data to zero-fill
+        if len(out) != segment.length:
+            raise CorruptArchiveError(
+                f"encrypted segment at page {page_index} decoded {len(out)} bytes, expected {segment.length}"
+            )
+        return out
     return decompress(read_compressed(store, segment, pages[1:]), segment)

dissect/archive/tibx/tibx.py

@@ -45,7 +45,7 @@
     InvalidArchiveError,
     UnsupportedFormatError,
 )
-from dissect.archive.tibx.lsm import iter_memtree_cells, read_archive_header
+from dissect.archive.tibx.lsm import iter_tree_cells, read_archive_header
 from dissect.archive.tibx.map import load_extents, load_segment_index
 from dissect.archive.tibx.page import PageStore
 from dissect.archive.tibx.segment import read_plaintext
@@ -139,15 +139,25 @@ def __init__(self, fh: BinaryIO | list[BinaryIO]):
         self._reset_snapshot_caches()
 
     def _reset_snapshot_caches(self) -> None:
-        # Everything derived from the currently-selected commit root (not the data key)
-        self._extents: ExtentTable | None = None
+        # Everything derived from the live commit root (not the data key)
+        self._all_extents: ExtentTable | None = None
         self._segment_index = None
-        self._segment_cache: OrderedDict[int, bytes] = OrderedDict()
-        self._segment_cache_bytes = 0
         self._segment_layout: dict[tuple[int, int], int] | None = None
+        self._clear_segment_cache()
+        self._reset_view_caches()
+
+    def _reset_view_caches(self) -> None:
+        # Everything that depends on the selected recovery point. A segment's plaintext,
+        # its location and the layout of the extents it holds do not: a segment is written
+        # by one backup, and later backups only reference it again.
+        self._extents: ExtentTable | None = None
         self._streams: list[TibxVolume] | None = None
         self._volumes: list[TibxVolume] | None = None
 
+    def _clear_segment_cache(self) -> None:
+        self._segment_cache: OrderedDict[int, bytes] = OrderedDict()
+        self._segment_cache_bytes = 0
+
     @classmethod
     def open(cls, path: str | Path) -> TIBX:
         """Open an archive from a path, auto-discovering split parts.
@@ -252,7 +262,7 @@ def unlock(self, password: str | bytes) -> None:
             )
 
         self._data_key = unwrap_data_key(self.header, password)
-        self._segment_cache.clear()
+        self._clear_segment_cache()
 
     def recovery_points(self) -> list[RecoveryPoint]:
         """The backups in this archive, oldest first.
@@ -267,16 +277,26 @@ def recovery_points(self) -> list[RecoveryPoint]:
         so counting roots reports recovery points that were never taken.
         """
         if self._recovery_points is None:
-            points: list[RecoveryPoint] = []
+            records: dict[int, c_tibx.slice_record] = {}
             slices = self.header.tree(TLV_SLICES)
             if slices is not None and slices.has_records:
-                for cell in iter_memtree_cells(slices):
+                seen: set[bytes] = set()
+                # Like every LSM tree, the slices tree moves from its mem-tree into on-disk
+                # ctrees as it grows; across those runs the newest record of a key wins
+                for cell in iter_tree_cells(self.store, slices):
+                    if cell.key in seen:
+                        continue
+                    seen.add(cell.key)
                     # The tree opens with an empty placeholder record; a real backup has a value.
-                    if len(cell.value) < len(c_tibx.slice_record) or len(cell.key) < len(c_tibx.slice_key):
+                    if not cell.alive or len(cell.value) < len(c_tibx.slice_record):
+                        continue
+                    if len(cell.key) < len(c_tibx.slice_key):
                         continue
                     key = c_tibx.slice_key(cell.key)
-                    points.append(RecoveryPoint(len(points), key.slice_id, c_tibx.slice_record(cell.value)))
-            self._recovery_points = points
+                    records[key.slice_id] = c_tibx.slice_record(cell.value)
+            self._recovery_points = [
+                RecoveryPoint(index, slice_id, records[slice_id]) for index, slice_id in enumerate(sorted(records))
+            ]
         return self._recovery_points
 
     def use_recovery_point(self, recovery_point: int | str = "latest") -> None:
@@ -309,7 +329,7 @@ def use_recovery_point(self, recovery_point: int | str = "latest") -> None:
             # slice_id as the primary recency signal -- so dropping everything newer than
             # the chosen slice is exactly the archive as it stood after that backup.
             self._slice_limit = chosen.slice_id
-        self._reset_snapshot_caches()
+        self._reset_view_caches()
 
     def disks(self) -> list:
         """Whole-disk views of this archive.
@@ -325,12 +345,18 @@ def disks(self) -> list:
     def extents(self) -> ExtentTable:
         """The data_map extents in view: all of them, or those up to the selected slice."""
         if self._extents is None:
-            extents = load_extents(self.store, self.header)
+            extents = self._unfiltered_extents()
             if self._slice_limit is not None:
                 extents = extents.select(map(le, extents.slice_ids, repeat(self._slice_limit)))
             self._extents = extents
         return self._extents
 
+    def _unfiltered_extents(self) -> ExtentTable:
+        """Every data_map extent, whatever recovery point is selected."""
+        if self._all_extents is None:
+            self._all_extents = load_extents(self.store, self.header)
+        return self._all_extents
+
     def streams(self) -> list[TibxVolume]:
         """Every data_map stream, backed-up volumes and Acronis's own metadata alike.
 
@@ -419,15 +445,19 @@ def _extent_base(self, segment_id: int, extent_index: int) -> int:
         if extent_index == EXTENT_INDEX_WHOLE_SEGMENT:
             return 0
         if self._segment_layout is None:
+            # From every extent, not just those of the selected recovery point: a volume
+            # opened before use_recovery_point() still reads through this layout
             layout: dict[tuple[int, int], int] = {}
-            shared: dict[int, list[tuple[int, int]]] = defaultdict(list)
-            extents = self.extents
+            shared: dict[int, dict[int, int]] = defaultdict(dict)
+            extents = self._unfiltered_extents()
             indexes = extents.extent_indexes
             for row in compress(range(len(indexes)), map(ne, indexes, repeat(EXTENT_INDEX_WHOLE_SEGMENT))):
-                shared[extents.segment_ids[row]].append((indexes[row], extents.extent_lengths[row]))
+                # An incremental backup records the unchanged extents again under its own
+                # slice, so one extent of a segment can appear in several rows: count it once
+                shared[extents.segment_ids[row]].setdefault(indexes[row], extents.extent_lengths[row])
             for shared_segment, entries in shared.items():
                 offset = 0
-                for index, length in sorted(entries, key=itemgetter(0)):
+                for index, length in sorted(entries.items(), key=itemgetter(0)):
                     layout[(shared_segment, index)] = offset
                     offset += length
                     offset = (offset + EXTENT_ALIGNMENT - 1) & ~(EXTENT_ALIGNMENT - 1)
@@ -478,7 +508,9 @@ def __init__(self, tibx: TIBX, volume_id: int, table: ExtentTable, start: int =
         self._size: int | None = None
 
     def __repr__(self) -> str:
-        return f"<TibxVolume volume_id={self.volume_id:#x} size={self.size}>"
+        # The span, not the size: that would read the boot sector, which needs the password
+        # on an encrypted archive
+        return f"<TibxVolume volume_id={self.volume_id:#x} span={self.span}>"
 
     @property
     def extents(self) -> ExtentTable:
@@ -631,10 +663,21 @@ def find_split_parts(path: Path) -> list[Path]:
     """
     match = SPLIT_PART_RE.match(path.name)
     stem = match.group("stem") if match else (path.stem if path.suffix.lower() == ".tibx" else path.name)
-    parts = sorted(path.parent.glob(f"{stem}-[0-9][0-9][0-9][0-9].tibx"), key=lambda part: part.name)
-    main = path.parent / f"{stem}.tibx"
-    if parts and main.exists():
-        return [main, *parts]
+    # Matched by hand rather than globbed: a stem like "Backup [PC1]" is no glob pattern,
+    # and the extension may be upper case on a case-sensitive filesystem
+    main = None
+    parts = []
+    try:
+        candidates = list(path.parent.iterdir())
+    except OSError:
+        return [path]
+    for candidate in candidates:
+        if candidate.suffix.lower() == ".tibx" and candidate.name == stem + candidate.suffix:
+            main = candidate
+        elif (part := SPLIT_PART_RE.match(candidate.name)) and part.group("stem") == stem:
+            parts.append((int(part.group("num")), candidate))
+    if parts and main is not None:
+        return [main, *(part for _, part in sorted(parts))]
     return [path]
 
 

tests/test_crypto.py

@@ -253,3 +253,42 @@ def test_locked_large_stream_is_a_volume_without_the_password() -> None:
     tibx = TIBX(io.BytesIO(archive))
 
     assert [v.volume_id for v in tibx.volumes()] == [10]
+
+
+def test_locked_volume_repr() -> None:
+    # repr() must not need the password: it is what logging and the REPL show
+    archive = build_lsm_archive([ExtentSpec(10, 0, b"x" * 64)], password=PASSWORD)
+    tibx = TIBX(io.BytesIO(archive))
+
+    assert "span=64" in repr(tibx.streams())
+
+
+def test_unlock_resets_segment_cache_budget() -> None:
+    archive = build_lsm_archive([ExtentSpec(10, 0, b"x" * 0x1000)], password=PASSWORD)
+    tibx = TIBX(io.BytesIO(archive))
+    tibx.unlock(PASSWORD)
+    tibx.streams()[0].read(0, 16)
+    assert tibx._segment_cache_bytes
+
+    tibx.unlock(PASSWORD)
+    assert not tibx._segment_cache
+    assert tibx._segment_cache_bytes == 0
+
+
+@pytest.mark.parametrize("alg", [3, 7])
+def test_encrypted_segment_shorter_than_its_length(alg: int) -> None:
+    # As for a plaintext segment, a segment that decodes short is corrupt: zero-filling the
+    # rest would pass damage off as volume data
+    from tests._synth import PAGE, SEGMENT_HEADER_OFFSET, finalize
+
+    archive = build_lsm_archive([ExtentSpec(10, 0, b"x" * 64)], compression=COMP_NONE, password=PASSWORD, alg=alg)
+    page = bytearray(archive[PAGE : 2 * PAGE])  # the segment follows the ARCH page
+    header = c_tibx.segment_header(bytes(page[SEGMENT_HEADER_OFFSET:]))
+    header.length += 16
+    page[SEGMENT_HEADER_OFFSET : SEGMENT_HEADER_OFFSET + len(c_tibx.segment_header)] = header.dumps()
+    archive = archive[:PAGE] + finalize(page) + archive[2 * PAGE :]
+
+    tibx = TIBX(io.BytesIO(archive))
+    tibx.unlock(PASSWORD)
+    with pytest.raises(CorruptArchiveError, match="decoded 64 bytes, expected 80"):
+        tibx.streams()[0].read(0, 16)

tests/test_recovery.py

@@ -5,9 +5,19 @@
 
 import pytest
 
+from dissect.archive.tibx.c_tibx import c_tibx
 from dissect.archive.tibx.exception import InvalidArchiveError
 from dissect.archive.tibx.tibx import TIBX
-from tests._synth import ExtentSpec, build_lsm_archive
+from tests._synth import (
+    PAGE,
+    SLICE_VALUE_LENGTH,
+    Cell,
+    ExtentSpec,
+    build_lsm_archive,
+    lsb,
+    lsm_page,
+    slice_record,
+)
 
 # A chain lives in one archive: each backup writes its extents under its own slice id, and
 # the slices tree (TLV[5]) records the backups. Slice ids are not contiguous in the wild --
@@ -102,3 +112,30 @@ def test_archive_without_slices_reports_no_recovery_points() -> None:
     assert tibx.recovery_points() == []
     with pytest.raises(InvalidArchiveError, match="records no backups"):
         tibx.use_recovery_point(0)
+
+
+def test_recovery_points_from_memtree_and_ctrees() -> None:
+    # As the slices tree grows its records move from the mem-tree into on-disk ctrees, like
+    # those of every LSM tree. Across them the newest record of a slice wins.
+    extents = [ExtentSpec(10, 0, b"x" * 64, slice_id=slice_id) for slice_id in SLICES]
+
+    def cell(slice_id: int, marker: int) -> Cell:
+        key = c_tibx.slice_key(slice_id=slice_id).dumps()
+        return Cell(key, slice_record(bytes([marker]) * 16, 1000 * slice_id, 1000 * slice_id + 10))
+
+    leaf_page = len(build_lsm_archive(extents)) // PAGE
+    slices = lsb(
+        4,
+        SLICE_VALUE_LENGTH,
+        memtree_cells=[cell(3, 0x33)],  # newer than the ctree's record of slice 3
+        ctrees=[(leaf_page * PAGE, 3)],
+    )
+    leaf = lsm_page(c_tibx.PageType.LEAF, b"LEAF", [cell(2, 2), cell(3, 3), cell(5, 5)], key_length=4, compact=True)
+    tibx = TIBX(io.BytesIO(build_lsm_archive(extents, extra_slots={5: slices}) + leaf))
+
+    points = tibx.recovery_points()
+    assert [point.slice_id for point in points] == SLICES
+    assert [point.guid.bytes_le[0] for point in points] == [2, 0x33, 5]
+
+    tibx.use_recovery_point(2)
+    assert tibx.extents