From fba64ddf3d26d4a40bc93db7539cefb3ead2dca0 Mon Sep 17 00:00:00 2001 From: Evan Sultanik Date: Fri, 7 Aug 2026 15:54:37 -0400 Subject: [PATCH] Stop indexing nibble-gram lengths that can never be hit Building the substitution alphabet cost roughly 1600 bytes of RAM per certificate byte, which put a hard ceiling on usable key sizes: a 2 MiB key pair needed over 4 GB. Most of that bought nothing. A length-L hit needs the grams from *every* certificate to match at the same offset, so the keyspace is 16**(L*num_secrets) while the certificates supply only 2*min_cert_length offsets. Past a point the index holds one unique key per offset: pure memory, never a hit. Measured with two 256 KiB certificates: gram length distinct keys keys reused blocks used encrypting 4 KiB 1 256 256 3 ( 0.1%) 2 65,511 65,354 4,095 (99.9%) 4 524,260 25 0 ( 0.0%) 8 524,281 0 0 ( 0.0%) 16 524,273 0 0 ( 0.0%) Lengths 4, 8 and 16 were about 75% of the index and were never once used. The default level indexed all of 1, 2, 4 and 8. `select_nibble_gram_lengths` now derives the set from the certificate size and secret count, and the `-1`..`-5` levels cap it rather than choosing it. Two secrets get (1, 2) at any realistic key size; three or more get (1,) alone; a single secret gets (1, 2, 4), which the fixed list could never express. Two further changes to what remains: * Index keys are packed into one `bytes` that interleaves the certificates' nibbles, so a key is a single slice of one interleaved buffer rather than a tuple of per-certificate slices. Measured ~2.2x faster to build than concatenating, at the same memory. `key[j::num_secrets]` recovers certificate j's gram, and the gram length is `len(key) // num_secrets`, which is what the dictionary header needs. * Nibble expansion uses two `bytes.translate` calls instead of a per-byte Python loop, and offsets are stored in `array('I')`. `array('L')` is *native* width -- 8 bytes here, 4 on Windows -- so it was both platform-dependent and twice the size required. The `'L'` in `index_type_map` is deliberately left alone: struct's `<` prefix selects standard sizes, where L is always 4. 2 x 128 KiB 0.57s / 294 MB -> 0.07s / 28 MB 8x faster, 10x less RAM 2 x 512 KiB 2.85s / 1109 MB -> 0.24s / 62 MB 12x faster, 18x less RAM 2 x 2 MiB 12.40s / 4334 MB -> 0.95s / 145 MB 13x faster, 30x less RAM The key count also stops growing with certificate size -- 65,792 at both 512 KiB and 2 MiB, being the complete 1- and 2-gram keyspaces -- so memory now scales only with the offset arrays. Ciphertexts are unchanged in size: measured at 512 B, 4 KiB and 16 KiB of plaintext, the adaptive index and the full fixed list differ by at most one byte, confirming the dropped lengths contributed nothing. Also fixes `stop_when_sufficient`, whose threshold was `(16*L)**N` -- correct only for L=1, by coincidence, which is the only length `-t` passed -- and throttles the status callback, which fired once per nibble offset. `decode` and `encode` now raise `MalformedCiphertextError` and `EncodingError` instead of bare `Exception`; the shorter ciphertexts made `decode`'s case reachable from a truncated file. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01367hFob9sd4xpDmVT4uoFy --- lenticrypt/__init__.py | 8 ++ lenticrypt/__main__.py | 21 ++-- lenticrypt/core.py | 201 ++++++++++++++++++++++++++++---------- tests/test_index.py | 143 +++++++++++++++++++++++++++ tests/test_regressions.py | 7 +- 5 files changed, 318 insertions(+), 62 deletions(-) create mode 100644 tests/test_index.py diff --git a/lenticrypt/__init__.py b/lenticrypt/__init__.py index 6641066..0c72f14 100644 --- a/lenticrypt/__init__.py +++ b/lenticrypt/__init__.py @@ -19,8 +19,12 @@ find_common_nibble_grams, index_type_map, is_power2, + nibbles_of, + pack_grams, read_nibble_grams, read_nibbles, + select_nibble_gram_lengths, + unpack_gram_length, ) __all__ = [ @@ -43,7 +47,11 @@ "find_common_nibble_grams", "index_type_map", "is_power2", + "nibbles_of", + "pack_grams", "read_nibble_grams", "read_nibbles", + "select_nibble_gram_lengths", + "unpack_gram_length", "utils", ] diff --git a/lenticrypt/__main__.py b/lenticrypt/__main__.py index 49ddf9d..3aa6549 100644 --- a/lenticrypt/__main__.py +++ b/lenticrypt/__main__.py @@ -17,6 +17,7 @@ LengthChecksumEncrypter, decrypt, find_common_nibble_grams, + select_nibble_gram_lengths, ) from .exceptions import LenticryptError from .iowrapper import auto_unzip @@ -217,16 +218,15 @@ def missing_combinations(substitution_alphabet: dict, num_secrets: int) -> list[ """ single_grams = substitution_alphabet.get(1, {}) return [ - gram - for gram in ( - tuple(bytes([nibble]) for nibble in combination) - for combination in itertools.product(range(16), repeat=num_secrets) + key + for key in ( + bytes(combination) for combination in itertools.product(range(16), repeat=num_secrets) ) - if gram not in single_grams + if key not in single_grams ] -def _format_missing(missing: Sequence[tuple[bytes, ...]]) -> str: +def _format_missing(missing: Sequence[bytes]) -> str: """Renders missing combinations as hex nibbles. Previously rendered with `chr()` of values 0-15, i.e. control characters, and accumulated by @@ -237,8 +237,8 @@ def _format_missing(missing: Sequence[tuple[bytes, ...]]) -> str: f"{len(missing)} missing nibble combination(s):", ] lines.extend( - " " + " ".join(f"0x{byte[0]:x}" for byte in gram) - for gram in missing[:MAX_REPORTED_COMBINATIONS] + " " + " ".join(f"0x{nibble:x}" for nibble in key) + for key in missing[:MAX_REPORTED_COMBINATIONS] ) if len(missing) > MAX_REPORTED_COMBINATIONS: lines.append(f" ...and {len(missing) - MAX_REPORTED_COMBINATIONS} more") @@ -254,7 +254,10 @@ def do_version(_args: argparse.Namespace, outfile: BinaryIO) -> int: def do_encrypt(args: argparse.Namespace, outfile: BinaryIO) -> int: secrets = tuple(Path(secret).read_bytes() for secret, _plaintext in args.encrypt) - lengths = NIBBLE_GRAM_LENGTHS[: args.level] + # The level caps how far up the ladder we go; the adaptive choice drops lengths that these + # particular secrets are too short to ever hit, which is most of them at realistic key sizes. + lengths = select_nibble_gram_lengths(min(map(len, secrets)), len(secrets))[: args.level] + logger.info(f"Indexing nibble-gram lengths {lengths}") with contextlib.ExitStack() as stack: callback = _progress_callback(args) if callback is not None: diff --git a/lenticrypt/core.py b/lenticrypt/core.py index 5d681b9..a6a0701 100755 --- a/lenticrypt/core.py +++ b/lenticrypt/core.py @@ -51,12 +51,27 @@ "find_common_nibble_grams", "index_type_map", "is_power2", + "nibbles_of", + "pack_grams", + "select_nibble_gram_lengths", + "unpack_gram_length", "read_nibble_grams", "read_nibbles", ] logger = logging.getLogger(name="lenticrypt") +# The block header allocates four bits to `length - 1`, so longer grams cannot be encoded. +MAX_NIBBLE_GRAM_LENGTH = 16 + +# Certificate offsets are stored as unsigned 32-bit ints. `array('L')` is *native* width -- 8 bytes +# here, 4 on Windows -- so it was both platform-dependent and twice the size needed. Note the 'L' in +# `index_type_map` is correct as-is: struct's `<` prefix selects standard sizes, where L is always 4. +OFFSET_TYPECODE = "I" + +# Report progress every 1024 offsets rather than on every one. +PROGRESS_MASK = 0x3FF + # The version of the ciphertext file format, independent of the package version: bumping one does # not imply bumping the other. Prior releases conflated them by encoding this as the semver minor. ENCRYPTION_VERSION: int = 3 @@ -71,66 +86,150 @@ def is_power2(num): return num != 0 and ((num & (num - 1)) == 0) +# Translation tables that map a byte to its high and low nibble, so a whole buffer can be expanded +# with two C-level `bytes.translate` calls instead of a per-byte Python loop. +_HIGH_NIBBLES = bytes(b >> 4 for b in range(256)) +_LOW_NIBBLES = bytes(b & 0x0F for b in range(256)) + + +def nibbles_of(data: bytes) -> bytes: + """Expands `data` into one nibble per byte, high nibble first.""" + expanded = bytearray(len(data) * 2) + expanded[0::2] = data.translate(_HIGH_NIBBLES) + expanded[1::2] = data.translate(_LOW_NIBBLES) + return bytes(expanded) + + def read_nibbles(byte_array: Sequence[int]) -> Generator[int, None, None]: - for b in byte_array: - yield (b & 0b11110000) >> 4 - yield b & 0b00001111 + yield from nibbles_of(bytes(byte_array)) NibbleGramTypeHint = Generator[bytes, None, None] def read_nibble_grams(byte_array: Sequence[int], length: int = 1) -> NibbleGramTypeHint: + """Yields every `length`-nibble window of `byte_array`, one nibble apart.""" if not is_power2(length): - raise ValueError(f"length must be a power of two; received {length}") - - return ( - bytes(ng) - for ng in zip( - *( - itertools.islice(nibbles, i, None) - for i, nibbles in enumerate(itertools.tee(read_nibbles(byte_array), length)) - ) - ) - ) + message = f"length must be a power of two; received {length}" + raise ValueError(message) + nibbles = nibbles_of(bytes(byte_array)) + return (nibbles[i : i + length] for i in range(len(nibbles) - length + 1)) -NibbleGramsTypeHint = Dict[Tuple[bytes, ...], array.array] +# The index maps a *packed* gram key to the certificate offsets where it occurs. The key interleaves +# the certificates' nibbles -- key[j::num_secrets] is certificate j's gram -- so it can be produced +# by a single slice of one interleaved buffer, rather than one slice per certificate plus a concat. +# Measured ~2.2x faster to build than concatenation, at the same memory. +NibbleGramsTypeHint = Dict[bytes, array.array] CommonNibbleGramsTypeHint = Dict[int, NibbleGramsTypeHint] +# A length is worth indexing if its keyspace is small in absolute terms... +ABSOLUTE_CHEAP_KEYSPACE = 1 << 16 +# ...or if the certificates supply enough offsets to populate a useful fraction of it. +MIN_OFFSETS_PER_KEY = 2 + + +def pack_grams(grams: Sequence[bytes]) -> bytes: + """Packs one gram per certificate into a single index key, matching the interleaved layout.""" + # strict: every certificate must contribute a gram of the same length, by construction. + return bytes(itertools.chain.from_iterable(zip(*grams, strict=True))) + + +def unpack_gram_length(key: bytes, num_secrets: int) -> int: + """The nibble-gram length a packed key represents.""" + return len(key) // num_secrets + + +def select_nibble_gram_lengths( + min_cert_length: int, num_secrets: int, max_length: int = MAX_NIBBLE_GRAM_LENGTH +) -> Tuple[int, ...]: + """Chooses which nibble-gram lengths are worth indexing for these certificates. + + A length-L hit needs the grams from *every* certificate to match at the same offset, so the + keyspace is 16**(L*num_secrets) while the certificates supply only 2*min_cert_length offsets. + Past a point the index is one unique key per offset: pure memory, never a hit. Measured with two + 256 KiB certificates, lengths 4, 8 and 16 held 1.57M keys between them, none reused, and were + used for 0% of blocks -- roughly 75% of the index for nothing. + + Length 1 is always included; without it no byte can be encoded at all. + """ + offsets = 2 * min_cert_length + lengths = [] + for length in (1, 2, 4, 8, 16): + if length > max_length: + break + keyspace = 16 ** (length * num_secrets) + cheap = keyspace <= ABSOLUTE_CHEAP_KEYSPACE + useful = offsets >= MIN_OFFSETS_PER_KEY * keyspace + if length == 1 or cheap or useful: + lengths.append(length) + else: + # Longer grams only have larger keyspaces, so nothing after this qualifies either. + break + return tuple(lengths) + def find_common_nibble_grams( certificates: Sequence[Sequence[int]], - nibble_gram_lengths=(1, 2, 4, 8, 16), + nibble_gram_lengths: Optional[Sequence[int]] = None, status_callback: StatusCallbackTypeHint = None, stop_when_sufficient: bool = False, ) -> CommonNibbleGramsTypeHint: - all_nibbles: CommonNibbleGramsTypeHint = {} # maps a nibble value to a common index + """Indexes the offsets at which the certificates share each nibble-gram combination. + + Args: + certificates: The secrets to index. + nibble_gram_lengths: Lengths to index. Defaults to `select_nibble_gram_lengths`, which drops + lengths too long to ever be hit for these certificates. + status_callback: Progress reporter. + stop_when_sufficient: Stop indexing a length once every combination has been seen. + + Returns: + A mapping of nibble-gram length to a mapping of packed gram key to certificate offsets. + """ + num_secrets = len(certificates) min_cert_length = min(len(c) for c in certificates) + if nibble_gram_lengths is None: + nibble_gram_lengths = select_nibble_gram_lengths(min_cert_length, num_secrets) + expanded = [nibbles_of(bytes(c)) for c in certificates] + total_nibbles = min(len(n) for n in expanded) + # One interleaved buffer, so a gram key is a single slice. Transient: 2*num_secrets*cert_size, + # which is megabytes next to an index measured in gigabytes. + interleaved = bytearray(total_nibbles * num_secrets) + for secret, nibbles in enumerate(expanded): + interleaved[secret::num_secrets] = nibbles[:total_nibbles] + interleaved = bytes(interleaved) + del expanded + + all_nibbles: CommonNibbleGramsTypeHint = {} for nibble_gram_length in nibble_gram_lengths: - nibbles: NibbleGramsTypeHint = defaultdict(lambda: array.array("L")) - all_nibbles[nibble_gram_length] = nibbles - range_max = min_cert_length * 2 - nibble_gram_length + 1 - for index, pair in enumerate( - zip(*(read_nibble_grams(c, nibble_gram_length) for c in certificates)) - ): - nibbles[pair].append(index) - if stop_when_sufficient and len(nibbles) >= (16 * nibble_gram_length) ** len( - certificates - ): - return all_nibbles - if status_callback is not None: + nibbles_index: NibbleGramsTypeHint = {} + all_nibbles[nibble_gram_length] = nibbles_index + # 16**(L*N) is the true number of distinct combinations; the previous + # `(16*L)**N` was correct only for L=1, by coincidence. + sufficient = 16 ** (nibble_gram_length * num_secrets) + width = nibble_gram_length * num_secrets + range_max = total_nibbles - nibble_gram_length + 1 + for index in range(range_max): + start = index * num_secrets + key = interleaved[start : start + width] + offsets = nibbles_index.get(key) + if offsets is None: + nibbles_index[key] = offsets = array.array(OFFSET_TYPECODE) + offsets.append(index) + if stop_when_sufficient and len(nibbles_index) >= sufficient: + break + # Throttled: this used to fire once per nibble offset, i.e. millions of Python calls + # plus float arithmetic, all of it invisible work when nothing is watching. + if status_callback is not None and not index & PROGRESS_MASK: status_callback( - index, range_max, "Building Index for %s-nibble-grams" % nibble_gram_length + index, range_max, f"Building Index for {nibble_gram_length}-nibble-grams" ) return all_nibbles READ_BLOCK_BYTES = 4096 -# The block header allocates four bits to `length - 1`, so grams longer than this cannot be encoded. -MAX_NIBBLE_GRAM_LENGTH = 16 - # Distinct unencodable gram tuples to name in the log before falling back to a bare count. MAX_REPORTED_UNENCODABLE = 16 @@ -268,11 +367,11 @@ def _grams_at( def can_encode(self, grams: Tuple[bytes, ...], length: int) -> bool: """Whether this gram tuple can be represented as a ciphertext block.""" - return grams in self.substitution_alphabet[length] + return pack_grams(grams) in self.substitution_alphabet[length] def encode_block(self, grams: Tuple[bytes, ...], length: int) -> bytes: """Encodes one accepted gram tuple as a ciphertext block.""" - index = random.choice(self.substitution_alphabet[length][grams]) + index = random.choice(self.substitution_alphabet[length][pack_grams(grams)]) if index < 256: index_bytes, index_type = 1, "B" # unsigned char elif index < 65536: @@ -414,9 +513,10 @@ def encode(n: int) -> bytearray: return ret n >>= 8 ret = bytearray([n & 0b11111111]) + ret - raise Exception( - f"Integer {orig_n} is too big to encode! The biggest value supported is {MAX_ENCODE_VALUE}." + message = ( + f"Integer {orig_n} is too big to encode; the largest supported value is {MAX_ENCODE_VALUE}" ) + raise EncodingError(message) def decode(byte_array: Union[bytes, bytearray, BinaryIO]) -> Optional[int]: @@ -456,7 +556,8 @@ def decode(byte_array: Union[bytes, bytearray, BinaryIO]) -> Optional[int]: n <<= 8 raw_byte = byte_array.read(1) if len(raw_byte) < 1: - raise Exception("Error: expected another byte in the stream!") + message = "Unexpected end of stream while decoding a variable-width integer" + raise MalformedCiphertextError(message) byte = raw_byte[0] n |= byte return n @@ -469,8 +570,8 @@ def decode(byte_array: Union[bytes, bytearray, BinaryIO]) -> Optional[int]: class DictionaryEncrypter(LengthChecksumEncrypter): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.dictionary: Dict[Tuple[bytes, ...], int] = {} - self.dictionary_items: List[Tuple[bytes, ...]] = [] + self.dictionary: Dict[bytes, int] = {} + self.dictionary_items: List[bytes] = [] # `can_encode` consults the substitution alphabet while building and the dictionary # afterwards; an explicit flag rather than testing the dictionary for emptiness. self._dictionary_built = False @@ -488,9 +589,10 @@ def build_dictionary(self): exactly the `-f/--force-encrypt` case. """ readers = [BufferedNibbleGramReader(stream) for stream in self.to_encrypt] - hits: Dict[Tuple[bytes, ...], int] = {} + hits: Dict[bytes, int] = {} for _length, grams in self.consume_grams(readers, "Building Dictionary"): - hits[grams] = hits.get(grams, 0) + 1 + key = pack_grams(grams) + hits[key] = hits.get(key, 0) + 1 # Every single-nibble gram the secrets can encode needs an entry, so encryption can always # fall back to length 1. Weight 0 keeps observed grams ahead of these fillers. # @@ -500,11 +602,10 @@ def build_dictionary(self): # IndexError on [0]. Iterated lazily rather than materialised as a set: 16**n is 16.7M # tuples for 6 secrets. encodable_single_grams = self.substitution_alphabet.get(1, {}) - for gram in itertools.product( - *([bytes([nibble]) for nibble in range(16)] for _ in range(len(self.to_encrypt))) - ): - if gram not in hits and gram in encodable_single_grams: - hits[gram] = 0 + for combination in itertools.product(range(16), repeat=len(self.to_encrypt)): + key = bytes(combination) + if key not in hits and key in encodable_single_grams: + hits[key] = 0 # Sorted over the mapping rather than a set difference: iteration order of a set of tuples # of `bytes` varies with PYTHONHASHSEED, so dictionary indices differed between runs and # defeated `--seed` reproducibility. The gram itself breaks ties deterministically. @@ -518,18 +619,18 @@ def can_encode(self, grams: Tuple[bytes, ...], length: int) -> bool: # While encrypting, the dictionary is the authority; while building it, the base class's # substitution-alphabet check is used instead. if self._dictionary_built: - return grams in self.dictionary + return pack_grams(grams) in self.dictionary return super().can_encode(grams, length) def encode_block(self, grams: Tuple[bytes, ...], length: int) -> bytes: - return encode(self.dictionary[grams]) + return encode(self.dictionary[pack_grams(grams)]) def get_header(self): yield from super().get_header() # The dictionary itself: one (certificate offset, gram length) pair per entry. yield from iter(encode(len(self.dictionary))) for grams in self.dictionary_items: - gram_length = len(grams[0]) + gram_length = unpack_gram_length(grams, len(self.to_encrypt)) # An explicit check rather than `assert`, which vanishes under -O. build_dictionary only # admits grams the alphabet can encode, so reaching this is a bug, not bad input. offsets = self.substitution_alphabet.get(gram_length, {}).get(grams) diff --git a/tests/test_index.py b/tests/test_index.py new file mode 100644 index 0000000..03e5f90 --- /dev/null +++ b/tests/test_index.py @@ -0,0 +1,143 @@ +"""The nibble-gram index: expansion, packing, and adaptive length selection.""" + +import array +import random + +import pytest + +from lenticrypt import ( + find_common_nibble_grams, + nibbles_of, + pack_grams, + read_nibble_grams, + read_nibbles, + select_nibble_gram_lengths, + unpack_gram_length, +) + + +class TestNibbleExpansion: + def test_matches_the_reference_implementation(self): + """`bytes.translate` twice must equal the per-byte loop it replaced.""" + rng = random.Random(0) + for size in (0, 1, 2, 17, 4096): + data = rng.randbytes(size) + reference = bytearray() + for byte in data: + reference.append((byte & 0b11110000) >> 4) + reference.append(byte & 0b00001111) + assert nibbles_of(data) == bytes(reference) + + def test_every_element_is_a_nibble(self): + assert all(0 <= n <= 15 for n in nibbles_of(bytes(range(256)))) + + def test_read_nibbles_agrees(self): + data = bytes(range(64)) + assert bytes(read_nibbles(data)) == nibbles_of(data) + + +class TestNibbleGrams: + @pytest.mark.parametrize("length", [1, 2, 4, 8, 16]) + def test_windows_are_one_nibble_apart(self, length): + data = bytes(range(32)) + nibbles = nibbles_of(data) + grams = list(read_nibble_grams(data, length)) + assert len(grams) == len(nibbles) - length + 1 + assert grams[0] == nibbles[:length] + assert grams[1] == nibbles[1 : length + 1] + + def test_rejects_non_power_of_two(self): + with pytest.raises(ValueError, match="power of two"): + list(read_nibble_grams(b"abc", 3)) + + +class TestPacking: + @pytest.mark.parametrize("num_secrets", [1, 2, 3, 4]) + @pytest.mark.parametrize("length", [1, 2, 4]) + def test_pack_then_split_round_trips(self, num_secrets, length): + """A packed key must be splittable back into each certificate's gram, since the dictionary + header needs the gram length and the index is keyed on the packed form.""" + rng = random.Random(1) + grams = tuple(bytes(rng.randrange(16) for _ in range(length)) for _ in range(num_secrets)) + key = pack_grams(grams) + assert len(key) == length * num_secrets + assert unpack_gram_length(key, num_secrets) == length + assert tuple(key[i::num_secrets] for i in range(num_secrets)) == grams + + def test_packing_matches_the_index_layout(self): + """`pack_grams` must produce exactly the key the interleaved index builder stores.""" + rng = random.Random(2) + certs = tuple(rng.randbytes(256) for _ in range(2)) + index = find_common_nibble_grams(certs, nibble_gram_lengths=(2,)) + expanded = [nibbles_of(c) for c in certs] + for offset in (0, 1, 5, 100): + grams = tuple(bytes(n[offset : offset + 2]) for n in expanded) + key = pack_grams(grams) + assert key in index[2] + assert offset in index[2][key] + + +class TestAdaptiveLengthSelection: + @pytest.mark.parametrize( + ("num_secrets", "kib", "expected"), + [ + # Two secrets: length 2 needs 65,536 combinations, which a 256 KiB key covers. + (2, 256, (1, 2)), + (2, 32, (1, 2)), + (2, 2048, (1, 2)), + # Three secrets: length 2 needs 16.7M combinations, out of reach at any sane key size. + (3, 32, (1,)), + (3, 512, (1,)), + (4, 512, (1,)), + # One secret: length 4 needs only 65,536, so it is genuinely reachable. + (1, 64, (1, 2, 4)), + ], + ) + def test_selection(self, num_secrets, kib, expected): + assert select_nibble_gram_lengths(kib * 1024, num_secrets) == expected + + def test_length_one_is_always_included(self): + """Without it no byte can be encoded at all, however short the certificates.""" + for num_secrets in (1, 2, 3, 6): + assert select_nibble_gram_lengths(1, num_secrets)[0] == 1 + + def test_respects_an_explicit_cap(self): + assert select_nibble_gram_lengths(64 * 1024, 1, max_length=2) == (1, 2) + + +class TestIndex: + def test_offsets_are_stored_compactly(self): + """`array('L')` is native width -- 8 bytes here, 4 on Windows -- so it was both + platform-dependent and twice the size needed.""" + certs = (bytes(range(256)), bytes(range(255, -1, -1))) + index = find_common_nibble_grams(certs, nibble_gram_lengths=(1,)) + offsets = next(iter(index[1].values())) + assert isinstance(offsets, array.array) + assert offsets.itemsize == 4 + + def test_recorded_offsets_are_correct(self): + rng = random.Random(3) + certs = tuple(rng.randbytes(512) for _ in range(2)) + index = find_common_nibble_grams(certs, nibble_gram_lengths=(1, 2)) + expanded = [nibbles_of(c) for c in certs] + for length, entries in index.items(): + for key, offsets in list(entries.items())[:50]: + for offset in offsets: + rebuilt = pack_grams( + tuple(bytes(n[offset : offset + length]) for n in expanded) + ) + assert rebuilt == key + + def test_defaults_to_adaptive_lengths(self): + rng = random.Random(4) + certs = tuple(rng.randbytes(4096) for _ in range(2)) + assert tuple(find_common_nibble_grams(certs)) == select_nibble_gram_lengths(4096, 2) + + def test_stop_when_sufficient_threshold(self): + """The threshold was `(16*L)**N`, correct only for L=1 by coincidence; it is `16**(L*N)`.""" + # One RNG for both: a fresh `Random(5)` per certificate would make them identical, and + # identical certificates only ever share grams with themselves. + rng = random.Random(5) + certs = tuple(rng.randbytes(1 << 15) for _ in range(2)) + index = find_common_nibble_grams(certs, nibble_gram_lengths=(1,), stop_when_sufficient=True) + assert len(index[1]) == 16**2 diff --git a/tests/test_regressions.py b/tests/test_regressions.py index 9181117..0fe8424 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -20,6 +20,7 @@ decode, decrypt, find_common_nibble_grams, + unpack_gram_length, ) from lenticrypt.exceptions import LenticryptError, UnsupportedVersionError from lenticrypt.iowrapper import IOWrapper @@ -161,8 +162,8 @@ def test_dictionary_header_survives_unencodable_grams(weak_keys): encrypter = DictionaryEncrypter(alphabet, plaintexts) assert bytes(encrypter.get_header()) # Only grams the alphabet can actually encode may enter the dictionary. - for grams in encrypter.dictionary_items: - assert grams in alphabet[len(grams[0])] + for key in encrypter.dictionary_items: + assert key in alphabet[unpack_gram_length(key, 2)] def test_seeded_encryption_is_reproducible(alphabet_2): @@ -216,7 +217,7 @@ def test_missing_combinations_uses_the_alphabet_key_shape(keys_2, weak_keys): assert missing_combinations(find_common_nibble_grams(keys_2, nibble_gram_lengths=(1,)), 2) == [] missing = missing_combinations(find_common_nibble_grams(weak_keys, nibble_gram_lengths=(1,)), 2) assert len(missing) == 255, "weak keys cover only the all-zero gram" - assert all(isinstance(part, bytes) for gram in missing for part in gram) + assert all(isinstance(key, bytes) and len(key) == 2 for key in missing) # --------------------------------------------------------------------------------------------------