diff --git a/pyproject.toml b/pyproject.toml index de1f852d51..f330feaeb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -167,6 +167,7 @@ module = [ "pyfuse3", "trio", "borg.crypto.low_level", + "borg.legacy.crypto.low_level", "borg.platform.*", ] ignore_missing_imports = true diff --git a/setup.py b/setup.py index c95c09ed54..7ee2353b5a 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ compress_source = "src/borg/compress.pyx" crypto_ll_source = "src/borg/crypto/low_level.pyx" +crypto_legacy_ll_source = "src/borg/legacy/crypto/low_level.pyx" buzhash_source = "src/borg/chunkers/buzhash.pyx" buzhash64_source = "src/borg/chunkers/buzhash64.pyx" reader_source = "src/borg/chunkers/reader.pyx" @@ -66,6 +67,7 @@ cython_sources = [ compress_source, crypto_ll_source, + crypto_legacy_ll_source, buzhash_source, buzhash64_source, reader_source, @@ -155,6 +157,10 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s dict(sources=[crypto_ll_source]), crypto_ext_lib, dict(extra_compile_args=cflags) ) + crypto_legacy_ext_kwargs = members_appended( + dict(sources=[crypto_legacy_ll_source]), crypto_ext_lib, dict(extra_compile_args=cflags) + ) + compress_ext_kwargs = members_appended( dict(sources=[compress_source]), lib_ext_kwargs(pc, "BORG_LIBLZ4_PREFIX", "lz4", "liblz4", ">= 1.7.0"), @@ -174,6 +180,7 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s ext_modules += [ Extension("borg.crypto.low_level", **crypto_ext_kwargs), + Extension("borg.legacy.crypto.low_level", **crypto_legacy_ext_kwargs), Extension("borg.compress", **compress_ext_kwargs), Extension("borg.hashindex", [hashindex_source], extra_compile_args=cflags), Extension("borg.item", [item_source], extra_compile_args=cflags), diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 30044bf920..555d8b5ef0 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -28,7 +28,7 @@ from ..repoobj import RepoObj -from .low_level import AES, bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256 +from .low_level import bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256 from .low_level import AES256_OCB, CHACHA20_POLY1305 from . import low_level @@ -438,9 +438,7 @@ def decrypt_key_file(self, data, passphrase): raise UnsupportedKeyFormatError() else: self._encrypted_key_algorithm = encrypted_key.algorithm - if encrypted_key.algorithm == "sha256": - return self.decrypt_key_file_pbkdf2(encrypted_key, passphrase) - elif encrypted_key.algorithm == "argon2 chacha20-poly1305": + if encrypted_key.algorithm == "argon2 chacha20-poly1305": return self.decrypt_key_file_argon2(encrypted_key, passphrase) else: raise UnsupportedKeyFormatError() @@ -478,13 +476,6 @@ def argon2( ) return key - def decrypt_key_file_pbkdf2(self, encrypted_key, passphrase): - key = self.pbkdf2(passphrase, encrypted_key.salt, encrypted_key.iterations, 32) - data = AES(key, b"\0" * 16).decrypt(encrypted_key.data) - if hmac.compare_digest(hmac_sha256(key, data), encrypted_key.hash): - return data - return None - def decrypt_key_file_argon2(self, encrypted_key, passphrase): key = self.argon2( passphrase, @@ -502,22 +493,11 @@ def decrypt_key_file_argon2(self, encrypted_key, passphrase): return None def encrypt_key_file(self, data, passphrase, algorithm): - if algorithm == "sha256": - return self.encrypt_key_file_pbkdf2(data, passphrase) - elif algorithm == "argon2 chacha20-poly1305": + if algorithm == "argon2 chacha20-poly1305": return self.encrypt_key_file_argon2(data, passphrase) else: raise ValueError(f"Unexpected algorithm: {algorithm}") - def encrypt_key_file_pbkdf2(self, data, passphrase): - salt = os.urandom(32) - iterations = PBKDF2_ITERATIONS - key = self.pbkdf2(passphrase, salt, iterations, 32) - hash = hmac_sha256(key, data) - cdata = AES(key, b"\0" * 16).encrypt(data) - enc_key = EncryptedKey(version=1, salt=salt, iterations=iterations, algorithm="sha256", hash=hash, data=cdata) - return msgpack.packb(enc_key.as_dict()) - def encrypt_key_file_argon2(self, data, passphrase): salt = os.urandom(ARGON2_SALT_BYTES) key = self.argon2(passphrase, output_len_in_bytes=32, salt=salt, **ARGON2_ARGS) diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index 0d56717fa6..149ea1d738 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -657,125 +657,6 @@ cdef class CHACHA20_POLY1305(_AEAD_BASE): super().__init__(key, iv=iv, header_len=header_len, aad_offset=aad_offset) -cdef class AES: # legacy - """A thin wrapper around the OpenSSL EVP cipher API - for legacy code, like key file encryption""" - cdef CIPHER cipher - cdef EVP_CIPHER_CTX *ctx - cdef unsigned char enc_key[32] - cdef int cipher_blk_len - cdef int iv_len - cdef unsigned char iv[16] - cdef long long blocks - - def __init__(self, enc_key, iv=None): - assert isinstance(enc_key, bytes) and len(enc_key) == 32 - self.enc_key = enc_key - self.iv_len = 16 - assert sizeof(self.iv) == self.iv_len - self.cipher = EVP_aes_256_ctr - self.cipher_blk_len = 16 - if iv is not None: - self.set_iv(iv) - else: - self.blocks = -1 # make sure set_iv is called before encrypt - - def __cinit__(self, enc_key, iv=None): - self.ctx = EVP_CIPHER_CTX_new() - - def __dealloc__(self): - EVP_CIPHER_CTX_free(self.ctx) - - def encrypt(self, data, iv=None): - if iv is not None: - self.set_iv(iv) - assert self.blocks == 0, 'iv needs to be set before encrypt is called' - cdef Py_buffer idata - cdef bint idata_acquired = False - cdef unsigned char *odata = NULL - cdef int ilen = len(data) - cdef int olen = 0 - cdef int offset - - try: - odata = PyMem_Malloc(ilen + self.cipher_blk_len) - if not odata: - raise MemoryError - - idata = ro_buffer(data) - idata_acquired = True - - if not EVP_EncryptInit_ex(self.ctx, self.cipher(), NULL, self.enc_key, self.iv): - raise Exception('EVP_EncryptInit_ex failed') - offset = 0 - if not EVP_EncryptUpdate(self.ctx, odata, &olen, idata.buf, ilen): - raise Exception('EVP_EncryptUpdate failed') - offset += olen - if not EVP_EncryptFinal_ex(self.ctx, odata+offset, &olen): - raise Exception('EVP_EncryptFinal failed') - offset += olen - self.blocks = self.block_count(offset) - return odata[:offset] - finally: - if odata: - PyMem_Free(odata) - if idata_acquired: - PyBuffer_Release(&idata) - - def decrypt(self, data): - cdef Py_buffer idata - cdef bint idata_acquired = False - cdef unsigned char *odata = NULL - cdef int ilen = len(data) - cdef int offset - cdef int olen = 0 - - try: - odata = PyMem_Malloc(ilen + self.cipher_blk_len) - if not odata: - raise MemoryError - - idata = ro_buffer(data) - idata_acquired = True - - # Set cipher type and mode - if not EVP_DecryptInit_ex(self.ctx, self.cipher(), NULL, self.enc_key, self.iv): - raise Exception('EVP_DecryptInit_ex failed') - offset = 0 - if not EVP_DecryptUpdate(self.ctx, odata, &olen, idata.buf, ilen): - raise Exception('EVP_DecryptUpdate failed') - offset += olen - if not EVP_DecryptFinal_ex(self.ctx, odata+offset, &olen): - # this error check is very important for modes with padding or - # authentication. for them, a failure here means corrupted data. - # CTR mode does not use padding nor authentication. - raise Exception('EVP_DecryptFinal failed') - offset += olen - self.blocks = self.block_count(ilen) - return odata[:offset] - finally: - if odata: - PyMem_Free(odata) - if idata_acquired: - PyBuffer_Release(&idata) - - def block_count(self, length): - return num_cipher_blocks(length, self.cipher_blk_len) - - def set_iv(self, iv): - # set_iv needs to be called before each encrypt() call, - # because encrypt does a full initialisation of the cipher context. - if isinstance(iv, int): - iv = iv.to_bytes(self.iv_len, byteorder='big') - assert isinstance(iv, bytes) and len(iv) == self.iv_len - self.iv = iv - self.blocks = 0 # number of cipher blocks encrypted with this IV - - def next_iv(self): - # call this after encrypt() to get the next iv (int) for the next encrypt() call - iv = int.from_bytes(self.iv[:self.iv_len], byteorder='big') - return iv + self.blocks - - def hmac_sha256(key, data): return hmac.digest(key, data, 'sha256') diff --git a/src/borg/hashindex.pyi b/src/borg/hashindex.pyi index 60c4fa8daf..7241de8ebc 100644 --- a/src/borg/hashindex.pyi +++ b/src/borg/hashindex.pyi @@ -1,7 +1,9 @@ -from typing import NamedTuple, Tuple, Type, IO, Iterator, Any +from typing import NamedTuple, Tuple, Type, IO, Iterator, Any, MutableMapping PATH_OR_FILE = str | IO +class HTProxyMixin(MutableMapping): ... + class ChunkIndexEntry(NamedTuple): flags: int size: int @@ -22,16 +24,6 @@ class ChunkIndex: def __getitem__(self, key: bytes) -> Type[ChunkIndexEntry]: ... def __setitem__(self, key: bytes, value: CIE) -> None: ... -class NSIndex1Entry(NamedTuple): - segment: int - offset: int - -class NSIndex1: # legacy - def iteritems(self, *args, **kwargs) -> Iterator: ... - def __contains__(self, key: bytes) -> bool: ... - def __getitem__(self, key: bytes) -> Any: ... - def __setitem__(self, key: bytes, value: Any) -> None: ... - class FuseVersionsIndexEntry(NamedTuple): version: int hash: bytes diff --git a/src/borg/hashindex.pyx b/src/borg/hashindex.pyx index a62a0491a1..e5328b4251 100644 --- a/src/borg/hashindex.pyx +++ b/src/borg/hashindex.pyx @@ -143,103 +143,3 @@ class FuseVersionsIndex(HTProxyMixin, MutableMapping): """ def __init__(self): self.ht = HashTableNT(key_size=16, value_type=FuseVersionsIndexEntry, value_format=FuseVersionsIndexEntryFormat) - - -NSIndex1Entry = namedtuple('NSIndex1Entry', 'segment offset') -NSIndex1EntryFormatT = namedtuple('NSIndex1EntryFormatT', 'segment offset') -NSIndex1EntryFormat = NSIndex1EntryFormatT(segment="I", offset="I") - - -class NSIndex1(HTProxyMixin, MutableMapping): - """ - Mapping from key256 to (segment32, offset32), as used by the legacy repository index of Borg 1.x. - """ - MAX_VALUE = 2**32 - 1 # borghash has the full uint32_t range - MAGIC = b"BORG_IDX" # borg 1.x - HEADER_FMT = "<8sIIBB" # magic, entries, buckets, ksize, vsize - KEY_SIZE = 32 - VALUE_SIZE = 8 - - def __init__(self, capacity=1000, path=None, usable=None): - if usable is not None: - capacity = usable * 2 # load factor 0.5 - self.ht = HashTableNT(key_size=self.KEY_SIZE, value_type=NSIndex1Entry, value_format=NSIndex1EntryFormat, - capacity=capacity) - if path: - self._read(path) - - def iteritems(self, marker=None): - do_yield = marker is None - for key, value in self.ht.items(): - if do_yield: - yield key, value - else: - do_yield = key == marker - - @classmethod - def read(cls, path): - return cls(path=path) - - def size(self): - return self.ht.size() # not quite correct as this is not the on-disk read-only format. - - def write(self, path): - if isinstance(path, str): - with open(path, 'wb') as fd: - self._write_fd(fd) - else: - self._write_fd(path) - - def _read(self, path): - if isinstance(path, str): - with open(path, 'rb') as fd: - self._read_fd(fd) - else: - self._read_fd(path) - - def _write_fd(self, fd): - used = len(self.ht) - header_bytes = struct.pack(self.HEADER_FMT, self.MAGIC, used, used, self.KEY_SIZE, self.VALUE_SIZE) - fd.write(header_bytes) - # record the header as a separate integrity-hash part if supported - hash_part = getattr(fd, "hash_part", None) - if hash_part: - hash_part("HashHeader") - count = 0 - for key, _ in self.ht.items(): - value = self.ht._get_raw(key) - fd.write(key) - fd.write(value) - count += 1 - assert count == used - - def _read_fd(self, fd): - header_size = struct.calcsize(self.HEADER_FMT) - header_bytes = fd.read(header_size) - if len(header_bytes) < header_size: - raise ValueError(f"Invalid file: file is too short (header).") - # verify the header as a separate integrity-hash part if supported - hash_part = getattr(fd, "hash_part", None) - if hash_part: - hash_part("HashHeader") - magic, entries, buckets, ksize, vsize = struct.unpack(self.HEADER_FMT, header_bytes) - if magic != self.MAGIC: - raise ValueError(f"Invalid file: magic {self.MAGIC.decode()} not found.") - assert ksize == self.KEY_SIZE, "invalid key size" - assert vsize == self.VALUE_SIZE, "invalid value size" - buckets_size = buckets * (ksize + vsize) - current_pos = fd.tell() - end_of_file = fd.seek(0, os.SEEK_END) - if current_pos + buckets_size != end_of_file: - raise ValueError(f"Invalid file: file size does not match (buckets).") - fd.seek(current_pos) - for i in range(buckets): - key = fd.read(ksize) - value = fd.read(vsize) - if value.startswith(b'\xFF\xFF\xFF\xFF'): # LE for 0xffffffff (empty/unused bucket) - continue - if value.startswith(b'\xFE\xFF\xFF\xFF'): # LE for 0xfffffffe (deleted/tombstone bucket) - continue - self.ht._set_raw(key, value) - pos = fd.tell() - assert pos == end_of_file diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py index 7fde449633..cf578ed2ac 100644 --- a/src/borg/helpers/fs.py +++ b/src/borg/helpers/fs.py @@ -442,15 +442,6 @@ def __init__(self, *, id_type, info_type): self.id_type = id_type self.info_type = info_type # can be a single type or a tuple of types - def borg1_hardlinkable(self, mode): # legacy - return stat.S_ISREG(mode) or stat.S_ISBLK(mode) or stat.S_ISCHR(mode) or stat.S_ISFIFO(mode) - - def borg1_hardlink_master(self, item): # legacy - return item.get("hardlink_master", False) and "source" not in item and self.borg1_hardlinkable(item.mode) - - def borg1_hardlink_slave(self, item): # legacy - return "source" in item and self.borg1_hardlinkable(item.mode) - def hardlink_id_from_path(self, path): """compute a hard link id from a path""" assert isinstance(path, str) diff --git a/src/borg/legacy/crypto/key.py b/src/borg/legacy/crypto/key.py index 4454015ee6..9ce6d2ef35 100644 --- a/src/borg/legacy/crypto/key.py +++ b/src/borg/legacy/crypto/key.py @@ -1,9 +1,52 @@ +import hmac +import os + from ...constants import * # NOQA -from ...crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b -from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256, AESKeyBase, FlexiKey +from ...crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b, hmac_sha256 +from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256, AESKeyBase, FlexiKey, UnsupportedKeyFormatError +from ...helpers import get_limited_unpacker, msgpack +from ...item import EncryptedKey +from .low_level import AES + + +class Pbkdf2FileMixin: + """Mixin for borg 1.x key files encrypted with PBKDF2 + AES-CTR.""" + + def decrypt_key_file(self, data, passphrase): + unpacker = get_limited_unpacker("key") + unpacker.feed(data) + unpacked = unpacker.unpack() + encrypted_key = EncryptedKey(internal_dict=unpacked) + if encrypted_key.version != 1: + raise UnsupportedKeyFormatError() + self._encrypted_key_algorithm = encrypted_key.algorithm + if encrypted_key.algorithm == "sha256": + return self.decrypt_key_file_pbkdf2(encrypted_key, passphrase) + return super().decrypt_key_file(data, passphrase) + + def encrypt_key_file(self, data, passphrase, algorithm): + if algorithm == "sha256": + return self.encrypt_key_file_pbkdf2(data, passphrase) + return super().encrypt_key_file(data, passphrase, algorithm) + + def decrypt_key_file_pbkdf2(self, encrypted_key, passphrase): + key = self.pbkdf2(passphrase, encrypted_key.salt, encrypted_key.iterations, 32) + data = AES(key, b"\0" * 16).decrypt(encrypted_key.data) + if hmac.compare_digest(hmac_sha256(key, data), encrypted_key.hash): + return data + return None + + def encrypt_key_file_pbkdf2(self, data, passphrase): + salt = os.urandom(32) + iterations = PBKDF2_ITERATIONS + key = self.pbkdf2(passphrase, salt, iterations, 32) + hash = hmac_sha256(key, data) + cdata = AES(key, b"\0" * 16).encrypt(data) + enc_key = EncryptedKey(version=1, salt=salt, iterations=iterations, algorithm="sha256", hash=hash, data=cdata) + return msgpack.packb(enc_key.as_dict()) -class KeyfileKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc] +class KeyfileKey(Pbkdf2FileMixin, ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc] TYPES_ACCEPTABLE = {KeyType.KEYFILE, KeyType.REPO, KeyType.PASSPHRASE} TYPE = KeyType.KEYFILE NAME = "key file" @@ -12,7 +55,7 @@ class KeyfileKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc] CIPHERSUITE = AES256_CTR_HMAC_SHA256 -class RepoKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc] +class RepoKey(Pbkdf2FileMixin, ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc] TYPES_ACCEPTABLE = {KeyType.KEYFILE, KeyType.REPO, KeyType.PASSPHRASE} TYPE = KeyType.REPO NAME = "repokey" @@ -21,7 +64,7 @@ class RepoKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc] CIPHERSUITE = AES256_CTR_HMAC_SHA256 -class Blake2KeyfileKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey): # type: ignore[misc] +class Blake2KeyfileKey(Pbkdf2FileMixin, ID_BLAKE2b_256, AESKeyBase, FlexiKey): # type: ignore[misc] TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO} TYPE = KeyType.BLAKE2KEYFILE NAME = "key file BLAKE2b" @@ -30,7 +73,7 @@ class Blake2KeyfileKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey): # type: ignore[mi CIPHERSUITE = AES256_CTR_BLAKE2b -class Blake2RepoKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey): # type: ignore[misc] +class Blake2RepoKey(Pbkdf2FileMixin, ID_BLAKE2b_256, AESKeyBase, FlexiKey): # type: ignore[misc] TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO} TYPE = KeyType.BLAKE2REPO NAME = "repokey BLAKE2b" diff --git a/src/borg/legacy/crypto/low_level.pyx b/src/borg/legacy/crypto/low_level.pyx new file mode 100644 index 0000000000..ced39f1257 --- /dev/null +++ b/src/borg/legacy/crypto/low_level.pyx @@ -0,0 +1,147 @@ +from cpython cimport PyMem_Malloc, PyMem_Free +from cpython.buffer cimport PyBUF_SIMPLE, PyObject_GetBuffer, PyBuffer_Release + +cdef extern from "openssl/evp.h": + ctypedef struct EVP_CIPHER: + pass + ctypedef struct EVP_CIPHER_CTX: + pass + ctypedef struct ENGINE: + pass + + const EVP_CIPHER *EVP_aes_256_ctr() + + EVP_CIPHER_CTX *EVP_CIPHER_CTX_new() + void EVP_CIPHER_CTX_free(EVP_CIPHER_CTX *a) + + int EVP_EncryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl, + const unsigned char *key, const unsigned char *iv) + int EVP_DecryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl, + const unsigned char *key, const unsigned char *iv) + int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, + const unsigned char *in_, int inl) + int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, + const unsigned char *in_, int inl) + int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl) + int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl) + + +ctypedef const EVP_CIPHER * (* CIPHER)() + + +cdef Py_buffer ro_buffer(object data) except *: + cdef Py_buffer view + PyObject_GetBuffer(data, &view, PyBUF_SIMPLE) + return view + + +cdef class AES: + """A thin wrapper around the OpenSSL EVP cipher API - for legacy key file encryption.""" + cdef CIPHER cipher + cdef EVP_CIPHER_CTX *ctx + cdef unsigned char enc_key[32] + cdef int cipher_blk_len + cdef int iv_len + cdef unsigned char iv[16] + cdef long long blocks + + def __init__(self, enc_key, iv=None): + assert isinstance(enc_key, bytes) and len(enc_key) == 32 + self.enc_key = enc_key + self.iv_len = 16 + assert sizeof(self.iv) == self.iv_len + self.cipher = EVP_aes_256_ctr + self.cipher_blk_len = 16 + if iv is not None: + self.set_iv(iv) + else: + self.blocks = -1 # make sure set_iv is called before encrypt + + def __cinit__(self, enc_key, iv=None): + self.ctx = EVP_CIPHER_CTX_new() + + def __dealloc__(self): + EVP_CIPHER_CTX_free(self.ctx) + + def encrypt(self, data, iv=None): + if iv is not None: + self.set_iv(iv) + assert self.blocks == 0, 'iv needs to be set before encrypt is called' + cdef Py_buffer idata + cdef bint idata_acquired = False + cdef unsigned char *odata = NULL + cdef int ilen = len(data) + cdef int olen = 0 + cdef int offset + + try: + odata = PyMem_Malloc(ilen + self.cipher_blk_len) + if not odata: + raise MemoryError + + idata = ro_buffer(data) + idata_acquired = True + + if not EVP_EncryptInit_ex(self.ctx, self.cipher(), NULL, self.enc_key, self.iv): + raise Exception('EVP_EncryptInit_ex failed') + offset = 0 + if not EVP_EncryptUpdate(self.ctx, odata, &olen, idata.buf, ilen): + raise Exception('EVP_EncryptUpdate failed') + offset += olen + if not EVP_EncryptFinal_ex(self.ctx, odata+offset, &olen): + raise Exception('EVP_EncryptFinal failed') + offset += olen + self.blocks = self.block_count(offset) + return odata[:offset] + finally: + if odata: + PyMem_Free(odata) + if idata_acquired: + PyBuffer_Release(&idata) + + def decrypt(self, data): + cdef Py_buffer idata + cdef bint idata_acquired = False + cdef unsigned char *odata = NULL + cdef int ilen = len(data) + cdef int offset + cdef int olen = 0 + + try: + odata = PyMem_Malloc(ilen + self.cipher_blk_len) + if not odata: + raise MemoryError + + idata = ro_buffer(data) + idata_acquired = True + + if not EVP_DecryptInit_ex(self.ctx, self.cipher(), NULL, self.enc_key, self.iv): + raise Exception('EVP_DecryptInit_ex failed') + offset = 0 + if not EVP_DecryptUpdate(self.ctx, odata, &olen, idata.buf, ilen): + raise Exception('EVP_DecryptUpdate failed') + offset += olen + if not EVP_DecryptFinal_ex(self.ctx, odata+offset, &olen): + raise Exception('EVP_DecryptFinal failed') + offset += olen + self.blocks = self.block_count(ilen) + return odata[:offset] + finally: + if odata: + PyMem_Free(odata) + if idata_acquired: + PyBuffer_Release(&idata) + + def block_count(self, length): + return (length + self.cipher_blk_len - 1) // self.cipher_blk_len + + def set_iv(self, iv): + if isinstance(iv, int): + iv = iv.to_bytes(self.iv_len, byteorder='big') + assert isinstance(iv, bytes) and len(iv) == self.iv_len + self.iv = iv + self.blocks = 0 + + def next_iv(self): + iv = int.from_bytes(self.iv[:self.iv_len], byteorder='big') + return iv + self.blocks diff --git a/src/borg/legacy/hashindex.py b/src/borg/legacy/hashindex.py new file mode 100644 index 0000000000..2dad9db392 --- /dev/null +++ b/src/borg/legacy/hashindex.py @@ -0,0 +1,108 @@ +from collections.abc import MutableMapping +from collections import namedtuple +import os +import struct + +from borghash import HashTableNT + +from ..hashindex import HTProxyMixin + + +NSIndex1Entry = namedtuple("NSIndex1Entry", "segment offset") +NSIndex1EntryFormatT = namedtuple("NSIndex1EntryFormatT", "segment offset") +NSIndex1EntryFormat = NSIndex1EntryFormatT(segment="I", offset="I") + + +class NSIndex1(HTProxyMixin, MutableMapping): + """ + Mapping from key256 to (segment32, offset32), as used by the legacy repository index of Borg 1.x. + """ + + MAX_VALUE = 2**32 - 1 # borghash has the full uint32_t range + MAGIC = b"BORG_IDX" # borg 1.x + HEADER_FMT = "<8sIIBB" # magic, entries, buckets, ksize, vsize + KEY_SIZE = 32 + VALUE_SIZE = 8 + + def __init__(self, capacity=1000, path=None, usable=None): + if usable is not None: + capacity = usable * 2 # load factor 0.5 + self.ht = HashTableNT( + key_size=self.KEY_SIZE, value_type=NSIndex1Entry, value_format=NSIndex1EntryFormat, capacity=capacity + ) + if path: + self._read(path) + + def iteritems(self, marker=None): + do_yield = marker is None + for key, value in self.ht.items(): + if do_yield: + yield key, value + else: + do_yield = key == marker + + @classmethod + def read(cls, path): + return cls(path=path) + + def size(self): + return self.ht.size() # not quite correct as this is not the on-disk read-only format. + + def write(self, path): + if isinstance(path, str): + with open(path, "wb") as fd: + self._write_fd(fd) + else: + self._write_fd(path) + + def _read(self, path): + if isinstance(path, str): + with open(path, "rb") as fd: + self._read_fd(fd) + else: + self._read_fd(path) + + def _write_fd(self, fd): + used = len(self.ht) + header_bytes = struct.pack(self.HEADER_FMT, self.MAGIC, used, used, self.KEY_SIZE, self.VALUE_SIZE) + fd.write(header_bytes) + hash_part = getattr(fd, "hash_part", None) + if hash_part: + hash_part("HashHeader") + count = 0 + for key, _ in self.ht.items(): + value = self.ht._get_raw(key) + fd.write(key) + fd.write(value) + count += 1 + assert count == used + + def _read_fd(self, fd): + header_size = struct.calcsize(self.HEADER_FMT) + header_bytes = fd.read(header_size) + if len(header_bytes) < header_size: + raise ValueError("Invalid file: file is too short (header).") + hash_part = getattr(fd, "hash_part", None) + if hash_part: + hash_part("HashHeader") + magic, entries, buckets, ksize, vsize = struct.unpack(self.HEADER_FMT, header_bytes) + if magic != self.MAGIC: + raise ValueError(f"Invalid file: magic {self.MAGIC.decode()} not found.") + assert ksize == self.KEY_SIZE, "invalid key size" + assert vsize == self.VALUE_SIZE, "invalid value size" + buckets_size = buckets * (ksize + vsize) + current_pos = fd.tell() + end_of_file = fd.seek(0, os.SEEK_END) + if current_pos + buckets_size != end_of_file: + raise ValueError("Invalid file: file size does not match (buckets).") + fd.seek(current_pos) + for i in range(buckets): + key = fd.read(ksize) + value = fd.read(vsize) + if value.startswith(b"\xff\xff\xff\xff"): # LE for 0xffffffff (empty/unused bucket) + continue + if value.startswith(b"\xfe\xff\xff\xff"): # LE for 0xfffffffe (deleted/tombstone bucket) + continue + self.ht._set_raw(key, value) + pos = fd.tell() + assert pos == end_of_file diff --git a/src/borg/legacy/helpers.py b/src/borg/legacy/helpers.py new file mode 100644 index 0000000000..f0cdebf9cf --- /dev/null +++ b/src/borg/legacy/helpers.py @@ -0,0 +1,13 @@ +import stat + + +def borg1_hardlinkable(mode): + return stat.S_ISREG(mode) or stat.S_ISBLK(mode) or stat.S_ISCHR(mode) or stat.S_ISFIFO(mode) + + +def borg1_hardlink_master(item): + return item.get("hardlink_master", False) and "source" not in item and borg1_hardlinkable(item.mode) + + +def borg1_hardlink_slave(item): + return "source" in item and borg1_hardlinkable(item.mode) diff --git a/src/borg/legacy/repository.py b/src/borg/legacy/repository.py index 60e375e3ed..5ae48e7747 100644 --- a/src/borg/legacy/repository.py +++ b/src/borg/legacy/repository.py @@ -16,7 +16,7 @@ import xxhash from ..constants import * # NOQA -from ..hashindex import NSIndex1Entry, NSIndex1 +from .hashindex import NSIndex1Entry, NSIndex1 from ..helpers import Error, ErrorWithTraceback, IntegrityError, format_file_size, parse_file_size from ..helpers import Location from ..helpers import ProgressIndicatorPercent diff --git a/src/borg/legacy/upgrade.py b/src/borg/legacy/upgrade.py index 5ebd91eada..80c3357eea 100644 --- a/src/borg/legacy/upgrade.py +++ b/src/borg/legacy/upgrade.py @@ -6,6 +6,7 @@ from ..helpers import HardLinkManager, join_cmd from ..item import Item from ..logger import create_logger +from .helpers import borg1_hardlink_master, borg1_hardlink_slave logger = create_logger(__name__) @@ -47,10 +48,10 @@ def upgrade_item(self, *, item): "acl_extended", } - if self.hlm.borg1_hardlink_master(item): + if borg1_hardlink_master(item): item.hlid = hlid = self.hlm.hardlink_id_from_path(item.path) self.hlm.remember(id=hlid, info=item.get("chunks")) - elif self.hlm.borg1_hardlink_slave(item): + elif borg1_hardlink_slave(item): item.hlid = hlid = self.hlm.hardlink_id_from_path(item.source) chunks = self.hlm.retrieve(id=hlid) if chunks is not None: diff --git a/src/borg/testsuite/crypto/crypto_test.py b/src/borg/testsuite/crypto/crypto_test.py index 6d72b2dbfa..5d32e39e7b 100644 --- a/src/borg/testsuite/crypto/crypto_test.py +++ b/src/borg/testsuite/crypto/crypto_test.py @@ -6,9 +6,11 @@ from ...crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_OCB, CHACHA20_POLY1305, UNENCRYPTED, IntegrityError from ...crypto.low_level import bytes_to_long, bytes_to_int, long_to_bytes -from ...crypto.low_level import AES, hmac_sha256 +from ...crypto.low_level import hmac_sha256 +from ...legacy.crypto.low_level import AES from hashlib import sha256 from ...crypto.key import CHPOKeyfileKey, AESOCBRepoKey, FlexiKey, KeyBase, PlaintextKey +from ...legacy.crypto.key import KeyfileKey as LegacyKeyfileKey from ...helpers import msgpack, bin_to_hex from .. import BaseTestCase @@ -232,7 +234,7 @@ def test_decrypt_key_file_pbkdf2_sha256_aes256_ctr_hmac_sha256(): encrypted = msgpack.packb( {"version": 1, "algorithm": "sha256", "iterations": 1, "salt": salt, "data": data, "hash": hash} ) - key = CHPOKeyfileKey(None) + key = LegacyKeyfileKey(None) decrypted = key.decrypt_key_file(encrypted, passphrase) diff --git a/src/borg/testsuite/legacyrepository_test.py b/src/borg/testsuite/legacyrepository_test.py index 28e10adcfb..a1da6248a0 100644 --- a/src/borg/testsuite/legacyrepository_test.py +++ b/src/borg/testsuite/legacyrepository_test.py @@ -7,7 +7,7 @@ import pytest from xxhash import xxh64 -from ..hashindex import NSIndex1 +from ..legacy.hashindex import NSIndex1 from ..helpers import Location from ..helpers import IntegrityError from ..helpers import msgpack