legacy: move hardlink helpers, NSIndex1, AES, and PBKDF2 key file methods to borg.legacy, refs #9556

pyproject.toml

@@ -167,6 +167,7 @@ module = [
     "pyfuse3",
     "trio",
     "borg.crypto.low_level",
+    "borg.legacy.crypto.low_level",
     "borg.platform.*",
 ]
 ignore_missing_imports = true

setup.py

@@ -50,6 +50,7 @@
 
 compress_source = "src/borg/compress.pyx"
 crypto_ll_source = "src/borg/crypto/low_level.pyx"
+crypto_legacy_ll_source = "src/borg/legacy/crypto/low_level.pyx"
 buzhash_source = "src/borg/chunkers/buzhash.pyx"
 buzhash64_source = "src/borg/chunkers/buzhash64.pyx"
 reader_source = "src/borg/chunkers/reader.pyx"
@@ -66,6 +67,7 @@
 cython_sources = [
     compress_source,
     crypto_ll_source,
+    crypto_legacy_ll_source,
     buzhash_source,
     buzhash64_source,
     reader_source,
@@ -155,6 +157,10 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s
         dict(sources=[crypto_ll_source]), crypto_ext_lib, dict(extra_compile_args=cflags)
     )
 
+    crypto_legacy_ext_kwargs = members_appended(
+        dict(sources=[crypto_legacy_ll_source]), crypto_ext_lib, dict(extra_compile_args=cflags)
+    )
+
     compress_ext_kwargs = members_appended(
         dict(sources=[compress_source]),
         lib_ext_kwargs(pc, "BORG_LIBLZ4_PREFIX", "lz4", "liblz4", ">= 1.7.0"),
@@ -174,6 +180,7 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s
 
     ext_modules += [
         Extension("borg.crypto.low_level", **crypto_ext_kwargs),
+        Extension("borg.legacy.crypto.low_level", **crypto_legacy_ext_kwargs),
         Extension("borg.compress", **compress_ext_kwargs),
         Extension("borg.hashindex", [hashindex_source], extra_compile_args=cflags),
         Extension("borg.item", [item_source], extra_compile_args=cflags),

src/borg/crypto/key.py

@@ -28,7 +28,7 @@
 from ..repoobj import RepoObj
 
 
-from .low_level import AES, bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256
+from .low_level import bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256
 from .low_level import AES256_OCB, CHACHA20_POLY1305
 from . import low_level
 
@@ -438,9 +438,7 @@ def decrypt_key_file(self, data, passphrase):
             raise UnsupportedKeyFormatError()
         else:
             self._encrypted_key_algorithm = encrypted_key.algorithm
-            if encrypted_key.algorithm == "sha256":
-                return self.decrypt_key_file_pbkdf2(encrypted_key, passphrase)
-            elif encrypted_key.algorithm == "argon2 chacha20-poly1305":
+            if encrypted_key.algorithm == "argon2 chacha20-poly1305":
                 return self.decrypt_key_file_argon2(encrypted_key, passphrase)
             else:
                 raise UnsupportedKeyFormatError()
@@ -478,13 +476,6 @@ def argon2(
         )
         return key
 
-    def decrypt_key_file_pbkdf2(self, encrypted_key, passphrase):
-        key = self.pbkdf2(passphrase, encrypted_key.salt, encrypted_key.iterations, 32)
-        data = AES(key, b"\0" * 16).decrypt(encrypted_key.data)
-        if hmac.compare_digest(hmac_sha256(key, data), encrypted_key.hash):
-            return data
-        return None
-
     def decrypt_key_file_argon2(self, encrypted_key, passphrase):
         key = self.argon2(
             passphrase,
@@ -502,22 +493,11 @@ def decrypt_key_file_argon2(self, encrypted_key, passphrase):
             return None
 
     def encrypt_key_file(self, data, passphrase, algorithm):
-        if algorithm == "sha256":
-            return self.encrypt_key_file_pbkdf2(data, passphrase)
-        elif algorithm == "argon2 chacha20-poly1305":
+        if algorithm == "argon2 chacha20-poly1305":
             return self.encrypt_key_file_argon2(data, passphrase)
         else:
             raise ValueError(f"Unexpected algorithm: {algorithm}")
 
-    def encrypt_key_file_pbkdf2(self, data, passphrase):
-        salt = os.urandom(32)
-        iterations = PBKDF2_ITERATIONS
-        key = self.pbkdf2(passphrase, salt, iterations, 32)
-        hash = hmac_sha256(key, data)
-        cdata = AES(key, b"\0" * 16).encrypt(data)
-        enc_key = EncryptedKey(version=1, salt=salt, iterations=iterations, algorithm="sha256", hash=hash, data=cdata)
-        return msgpack.packb(enc_key.as_dict())
-
     def encrypt_key_file_argon2(self, data, passphrase):
         salt = os.urandom(ARGON2_SALT_BYTES)
         key = self.argon2(passphrase, output_len_in_bytes=32, salt=salt, **ARGON2_ARGS)

src/borg/crypto/low_level.pyx

@@ -657,125 +657,6 @@ cdef class CHACHA20_POLY1305(_AEAD_BASE):
         super().__init__(key, iv=iv, header_len=header_len, aad_offset=aad_offset)
 
 
-cdef class AES:  # legacy
-    """A thin wrapper around the OpenSSL EVP cipher API - for legacy code, like key file encryption"""
-    cdef CIPHER cipher
-    cdef EVP_CIPHER_CTX *ctx
-    cdef unsigned char enc_key[32]
-    cdef int cipher_blk_len
-    cdef int iv_len
-    cdef unsigned char iv[16]
-    cdef long long blocks
-
-    def __init__(self, enc_key, iv=None):
-        assert isinstance(enc_key, bytes) and len(enc_key) == 32
-        self.enc_key = enc_key
-        self.iv_len = 16
-        assert sizeof(self.iv) == self.iv_len
-        self.cipher = EVP_aes_256_ctr
-        self.cipher_blk_len = 16
-        if iv is not None:
-            self.set_iv(iv)
-        else:
-            self.blocks = -1  # make sure set_iv is called before encrypt
-
-    def __cinit__(self, enc_key, iv=None):
-        self.ctx = EVP_CIPHER_CTX_new()
-
-    def __dealloc__(self):
-        EVP_CIPHER_CTX_free(self.ctx)
-
-    def encrypt(self, data, iv=None):
-        if iv is not None:
-            self.set_iv(iv)
-        assert self.blocks == 0, 'iv needs to be set before encrypt is called'
-        cdef Py_buffer idata
-        cdef bint idata_acquired = False
-        cdef unsigned char *odata = NULL
-        cdef int ilen = len(data)
-        cdef int olen = 0
-        cdef int offset
-
-        try:
-            odata = <unsigned char *>PyMem_Malloc(ilen + self.cipher_blk_len)
-            if not odata:
-                raise MemoryError
-
-            idata = ro_buffer(data)
-            idata_acquired = True
-
-            if not EVP_EncryptInit_ex(self.ctx, self.cipher(), NULL, self.enc_key, self.iv):
-                raise Exception('EVP_EncryptInit_ex failed')
-            offset = 0
-            if not EVP_EncryptUpdate(self.ctx, odata, &olen, <const unsigned char*> idata.buf, ilen):
-                raise Exception('EVP_EncryptUpdate failed')
-            offset += olen
-            if not EVP_EncryptFinal_ex(self.ctx, odata+offset, &olen):
-                raise Exception('EVP_EncryptFinal failed')
-            offset += olen
-            self.blocks = self.block_count(offset)
-            return odata[:offset]
-        finally:
-            if odata:
-                PyMem_Free(odata)
-            if idata_acquired:
-                PyBuffer_Release(&idata)
-
-    def decrypt(self, data):
-        cdef Py_buffer idata
-        cdef bint idata_acquired = False
-        cdef unsigned char *odata = NULL
-        cdef int ilen = len(data)
-        cdef int offset
-        cdef int olen = 0
-
-        try:
-            odata = <unsigned char *>PyMem_Malloc(ilen + self.cipher_blk_len)
-            if not odata:
-                raise MemoryError
-
-            idata = ro_buffer(data)
-            idata_acquired = True
-
-            # Set cipher type and mode
-            if not EVP_DecryptInit_ex(self.ctx, self.cipher(), NULL, self.enc_key, self.iv):
-                raise Exception('EVP_DecryptInit_ex failed')
-            offset = 0
-            if not EVP_DecryptUpdate(self.ctx, odata, &olen, <const unsigned char*> idata.buf, ilen):
-                raise Exception('EVP_DecryptUpdate failed')
-            offset += olen
-            if not EVP_DecryptFinal_ex(self.ctx, odata+offset, &olen):
-                # this error check is very important for modes with padding or
-                # authentication. for them, a failure here means corrupted data.
-                # CTR mode does not use padding nor authentication.
-                raise Exception('EVP_DecryptFinal failed')
-            offset += olen
-            self.blocks = self.block_count(ilen)
-            return odata[:offset]
-        finally:
-            if odata:
-                PyMem_Free(odata)
-            if idata_acquired:
-                PyBuffer_Release(&idata)
-
-    def block_count(self, length):
-        return num_cipher_blocks(length, self.cipher_blk_len)
-
-    def set_iv(self, iv):
-        # set_iv needs to be called before each encrypt() call,
-        # because encrypt does a full initialisation of the cipher context.
-        if isinstance(iv, int):
-            iv = iv.to_bytes(self.iv_len, byteorder='big')
-        assert isinstance(iv, bytes) and len(iv) == self.iv_len
-        self.iv = iv
-        self.blocks = 0  # number of cipher blocks encrypted with this IV
-
-    def next_iv(self):
-        # call this after encrypt() to get the next iv (int) for the next encrypt() call
-        iv = int.from_bytes(self.iv[:self.iv_len], byteorder='big')
-        return iv + self.blocks
-
-
 def hmac_sha256(key, data):
     return hmac.digest(key, data, 'sha256')
 

src/borg/hashindex.pyi

@@ -1,7 +1,9 @@
-from typing import NamedTuple, Tuple, Type, IO, Iterator, Any
+from typing import NamedTuple, Tuple, Type, IO, Iterator, Any, MutableMapping
 
 PATH_OR_FILE = str | IO
 
+class HTProxyMixin(MutableMapping): ...
+
 class ChunkIndexEntry(NamedTuple):
     flags: int
     size: int
@@ -22,16 +24,6 @@ class ChunkIndex:
     def __getitem__(self, key: bytes) -> Type[ChunkIndexEntry]: ...
     def __setitem__(self, key: bytes, value: CIE) -> None: ...
 
-class NSIndex1Entry(NamedTuple):
-    segment: int
-    offset: int
-
-class NSIndex1:  # legacy
-    def iteritems(self, *args, **kwargs) -> Iterator: ...
-    def __contains__(self, key: bytes) -> bool: ...
-    def __getitem__(self, key: bytes) -> Any: ...
-    def __setitem__(self, key: bytes, value: Any) -> None: ...
-
 class FuseVersionsIndexEntry(NamedTuple):
     version: int
     hash: bytes

src/borg/hashindex.pyx

@@ -143,103 +143,3 @@ class FuseVersionsIndex(HTProxyMixin, MutableMapping):
     """
     def __init__(self):
         self.ht = HashTableNT(key_size=16, value_type=FuseVersionsIndexEntry, value_format=FuseVersionsIndexEntryFormat)
-
-
-NSIndex1Entry = namedtuple('NSIndex1Entry', 'segment offset')
-NSIndex1EntryFormatT = namedtuple('NSIndex1EntryFormatT', 'segment offset')
-NSIndex1EntryFormat = NSIndex1EntryFormatT(segment="I", offset="I")
-
-
-class NSIndex1(HTProxyMixin, MutableMapping):
-    """
-    Mapping from key256 to (segment32, offset32), as used by the legacy repository index of Borg 1.x.
-    """
-    MAX_VALUE = 2**32 - 1  # borghash has the full uint32_t range
-    MAGIC = b"BORG_IDX"  # borg 1.x
-    HEADER_FMT = "<8sIIBB"  # magic, entries, buckets, ksize, vsize
-    KEY_SIZE = 32
-    VALUE_SIZE = 8
-
-    def __init__(self, capacity=1000, path=None, usable=None):
-        if usable is not None:
-            capacity = usable * 2  # load factor 0.5
-        self.ht = HashTableNT(key_size=self.KEY_SIZE, value_type=NSIndex1Entry, value_format=NSIndex1EntryFormat,
-                              capacity=capacity)
-        if path:
-            self._read(path)
-
-    def iteritems(self, marker=None):
-        do_yield = marker is None
-        for key, value in self.ht.items():
-            if do_yield:
-                yield key, value
-            else:
-                do_yield = key == marker
-
-    @classmethod
-    def read(cls, path):
-        return cls(path=path)
-
-    def size(self):
-        return self.ht.size()  # not quite correct as this is not the on-disk read-only format.
-
-    def write(self, path):
-        if isinstance(path, str):
-            with open(path, 'wb') as fd:
-                self._write_fd(fd)
-        else:
-            self._write_fd(path)
-
-    def _read(self, path):
-        if isinstance(path, str):
-            with open(path, 'rb') as fd:
-                self._read_fd(fd)
-        else:
-            self._read_fd(path)
-
-    def _write_fd(self, fd):
-        used = len(self.ht)
-        header_bytes = struct.pack(self.HEADER_FMT, self.MAGIC, used, used, self.KEY_SIZE, self.VALUE_SIZE)
-        fd.write(header_bytes)
-        # record the header as a separate integrity-hash part if supported
-        hash_part = getattr(fd, "hash_part", None)
-        if hash_part:
-            hash_part("HashHeader")
-        count = 0
-        for key, _ in self.ht.items():
-            value = self.ht._get_raw(key)
-            fd.write(key)
-            fd.write(value)
-            count += 1
-        assert count == used
-
-    def _read_fd(self, fd):
-        header_size = struct.calcsize(self.HEADER_FMT)
-        header_bytes = fd.read(header_size)
-        if len(header_bytes) < header_size:
-            raise ValueError(f"Invalid file: file is too short (header).")
-        # verify the header as a separate integrity-hash part if supported
-        hash_part = getattr(fd, "hash_part", None)
-        if hash_part:
-            hash_part("HashHeader")
-        magic, entries, buckets, ksize, vsize = struct.unpack(self.HEADER_FMT, header_bytes)
-        if magic != self.MAGIC:
-            raise ValueError(f"Invalid file: magic {self.MAGIC.decode()} not found.")
-        assert ksize == self.KEY_SIZE, "invalid key size"
-        assert vsize == self.VALUE_SIZE, "invalid value size"
-        buckets_size = buckets * (ksize + vsize)
-        current_pos = fd.tell()
-        end_of_file = fd.seek(0, os.SEEK_END)
-        if current_pos + buckets_size != end_of_file:
-            raise ValueError(f"Invalid file: file size does not match (buckets).")
-        fd.seek(current_pos)
-        for i in range(buckets):
-            key = fd.read(ksize)
-            value = fd.read(vsize)
-            if value.startswith(b'\xFF\xFF\xFF\xFF'):  # LE for 0xffffffff (empty/unused bucket)
-                continue
-            if value.startswith(b'\xFE\xFF\xFF\xFF'):  # LE for 0xfffffffe (deleted/tombstone bucket)
-                continue
-            self.ht._set_raw(key, value)
-        pos = fd.tell()
-        assert pos == end_of_file

src/borg/helpers/fs.py

@@ -442,15 +442,6 @@ def __init__(self, *, id_type, info_type):
         self.id_type = id_type
         self.info_type = info_type  # can be a single type or a tuple of types
 
-    def borg1_hardlinkable(self, mode):  # legacy
-        return stat.S_ISREG(mode) or stat.S_ISBLK(mode) or stat.S_ISCHR(mode) or stat.S_ISFIFO(mode)
-
-    def borg1_hardlink_master(self, item):  # legacy
-        return item.get("hardlink_master", False) and "source" not in item and self.borg1_hardlinkable(item.mode)
-
-    def borg1_hardlink_slave(self, item):  # legacy
-        return "source" in item and self.borg1_hardlinkable(item.mode)
-
     def hardlink_id_from_path(self, path):
         """compute a hard link id from a path"""
         assert isinstance(path, str)