From 31e26020ba3aa4ee49f39475ae43a0928afd7d7b Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 2 Jul 2025 19:55:28 +0200 Subject: [PATCH] updates encryption docs --- 10 Notes/Codex Encryption Design.md | 116 +++++++++++++++++- ... => How to create a hash using BearSSL.md} | 26 +++- ...t using symmetric encryption in BearSSL.md | 31 +++++ ... generate a random number using BearSSL.md | 45 +++++++ ...nter to a seq to pass it to a C library.md | 2 +- 5 files changed, 215 insertions(+), 5 deletions(-) rename 10 Notes/{BearSSL hashing.md => How to create a hash using BearSSL.md} (68%) create mode 100644 10 Notes/How to encrypt and decrypt content using symmetric encryption in BearSSL.md create mode 100644 10 Notes/How to generate a random number using BearSSL.md diff --git a/10 Notes/Codex Encryption Design.md b/10 Notes/Codex Encryption Design.md index 38aaaef..1a67c5f 100644 --- a/10 Notes/Codex Encryption Design.md +++ b/10 Notes/Codex Encryption Design.md @@ -18,11 +18,121 @@ where both `MASTER_KEY` and `MASTER_IV` are 256 bit random numbers, and `||` den If storing 512 bits (as opposed to a 256 bit minimum) of key material is a problem, we could derive both by the same key, for example as - key' = SHA256( MASTER_KEY || 0x01 || block_index ), truncated to 192 bits - IV' = SHA256( MASTER_KEY || 0x02 || block_index ), truncated to 128 bits + blockKEY = SHA256( MASTER_KEY || 0x01 || block_index ), truncated to 192 bits + blockIV = SHA256( MASTER_KEY || 0x02 || block_index ), truncated to 128 bits +In our implementation, we will be using the second scheme - starting with a random master key, and then for each block deriving a block level key (`blockKEY`) and block level initialization vector (`blockIV`). -Some context info: +For some introduction and examples on BearSSL, please consult: + +- [[How to generate a random number using BearSSL]] +- [[How to create a hash using BearSSL]] +- [[How to encrypt and decrypt content using symmetric encryption in BearSSL]] + +Before document design considerations for the content encryption in the Codex client, let's first see how to use BearSSL primitives to encrypt and decrypt some content: + +```nim +import std/sequtils +import bearssl/blockx +import stew/byteutils + +import ./rng + +var plaintext = "0123456789abcdef".toBytes +echo "plaintext: ", plaintext.toHex + +let key = newSeqWith(16, Rng.instance.rand(uint8.high).byte) +let ive = newSeqWith(16, Rng.instance.rand(uint8.high).byte) +let ivd = ive + +echo "ive: ", ive.toHex + +var encCtx: AesBigCbcencKeys +aesBigCbcencInit(encCtx, addr key[0], 16.uint) +aesBigCbcencRun(encCtx, addr ive[0], addr plaintext[0], 16.uint) +echo "Encrypted: ", plaintext.toHex + +echo "ivd: ", ivd.toHex + +var decCtx: AesBigCbcdecKeys +aesBigCbcdecInit(decCtx, addr key[0], 16.uint) +aesBigCbcdecRun(decCtx, addr ivd[0], addr plaintext[0], 16.uint) +echo "Decrypted: ", plaintext.toHex +``` + +Important to notice here is that `aesBigCbcencRun` will mutate the provided initialization vector `IV` so that it is ready to use for the subsequent chunk of data - a classical CBC mode for AES. Yet, for Codex, we use slightly modified scheme as already shown above. + +For codex: + +1. we first generate a `MASTER_KEY` - which will be returned to the user +2. from the `MASTER_KEY`, for each block, we derive the corresponding block level encryption key `blockKEY` and block level initialization vector `blockIV` as shown in the proposal above +3. using the derived `blockKEY` and `blockIV`, we then encrypt the block using the BearSSL encryption primitives as demonstrated above. + +As we see above, the block index is used in the process of the key and initialization vector derivation. For this reason we also need to remember to convert the block index to a byte representation - we use big-endian ordering. For this conversion a very simple function can be used: + +```nim +func toBytes[T: SomeInteger](value: T): seq[byte] = + let v = + if system.cpuEndian == bigEndian: value + else: swapBytes(value) + + result = newSeq[byte](sizeof(T)) + copyMem(addr result[0], unsafeAddr v, sizeof(T)) +``` + +or, we can just use the `endians2.toBytes` function from `nim-stew`: `toBytes(blockIndex.uint32, bigEndian)` + +Below we show the code for the derivation of the block level key and initialization vector, followed by the encryption and decryption of a block: + +```nim +import std/sequtils +import bearssl/[blockx, hash] +import stew/[byteutils, endians2] + +import ./rng +import ./hash + +let masterKey = newSeqWith(32, Rng.instance.rand(uint8.high).byte) + +let blockIndex = 1.uint32 +let blockIndexArray = toBytes(blockIndex, bigEndian) + +const KEY_SIZE = 24 # 192 bits for AES-192 +const IV_SIZE = 16 # 128 bits + +let keyForBlock = hash(addr sha256Vtable, masterKey & @[byte(0x01)] & blockIndexArray.toSeq)[0 ..< KEY_SIZE] +let ivForBlock = hash(addr sha256Vtable, masterKey & @[byte(0x02)] & blockIndexArray.toSeq)[0 ..< IV_SIZE] + +var plaintext = newSeqWith(IV_SIZE, Rng.instance.rand(uint8.high).byte) + +let key = keyForBlock +let ive = ivForBlock +let encBuffer = plaintext + +# encryption +var encCtx: AesBigCbcencKeys +aesBigCbcencInit(encCtx, addr key[0], key.len.uint) +aesBigCbcencRun(encCtx, addr ive[0], addr encBuffer[0], ive.len.uint) + +assert encBuffer != plaintext, "Encryption failed, output should differ from input!" + +# decryption +let ivd = ivForBlock + +var decCtx: AesBigCbcdecKeys +aesBigCbcdecInit(decCtx, addr key[0], key.len.uint) +aesBigCbcdecRun(decCtx, addr ivd[0], addr encBuffer[0], ivd.len.uint) + +assert encBuffer == plaintext, "Decryption failed, output should match input!" +``` + +where `rng` and `hash` are defined as shown in [[How to generate a random number using BearSSL]] and [[How to create a hash using BearSSL]]. + +In a similar way we will proceed with other block. + +More data to come. + +### Links - [bearssl](https://bearssl.org/) - [bearssl Nim bindings](https://github.com/status-im/nim-bearssl) diff --git a/10 Notes/BearSSL hashing.md b/10 Notes/How to create a hash using BearSSL.md similarity index 68% rename from 10 Notes/BearSSL hashing.md rename to 10 Notes/How to create a hash using BearSSL.md index c1fb7af..572b412 100644 --- a/10 Notes/BearSSL hashing.md +++ b/10 Notes/How to create a hash using BearSSL.md @@ -24,7 +24,7 @@ BearSSL also simulates an [OOP interface](https://bearssl.org/oop.html), which c ```nim import bearssl/hash -proc hash(hashClass: ptr HashClass, data: openArray[byte]): seq[byte] = +proc hash*(hashClass: ptr HashClass, data: openArray[byte]): seq[byte] = var compatCtx = HashCompatContext() let buffSize = (hashClass[].desc shr HASHDESC_OUT_OFF) and HASHDESC_OUT_MASK result = newSeq[byte](buffSize) @@ -39,8 +39,11 @@ proc hash(hashClass: ptr HashClass, data: openArray[byte]): seq[byte] = Then such a hash function can be conveniently reused for various hashing functions: ```nim +import bearssl/hash import stew/byteutils +import ./hash + let data = "0123456789abcdef".toBytes var sha256HashCtx = Sha256Context() @@ -56,3 +59,24 @@ md5Init(md5Ctx) echo "Hash out[md5]: ", hash(md5Ctx.vtable, data).toHex ``` +This last snippet can be further simplified by using a globally defined *vtable* variables from `nim-bearssl`: + +```nim +import bearssl/hash +import stew/byteutils + +import ./hash + +let data = "0123456789abcdef".toBytes + +var sha256HashCtx = Sha256Context() +let buff = newSeq[byte](sha256SIZE) # 32 bytes for SHA-256 +sha256Init(sha256HashCtx) +sha224Update(sha256HashCtx, addr data[0], data.len.uint) +sha256Out(sha256HashCtx, addr buff[0]) + +echo "Hash out[sha256]: ", buff.toHex +echo "Hash out[sha256]: ", hash(addr sha256Vtable, data).toHex +echo "Hash out[sha1]: ", hash(addr sha1Vtable, data).toHex +echo "Hash out[md5]: ", hash(addr md5Vtable, data).toHex +``` \ No newline at end of file diff --git a/10 Notes/How to encrypt and decrypt content using symmetric encryption in BearSSL.md b/10 Notes/How to encrypt and decrypt content using symmetric encryption in BearSSL.md new file mode 100644 index 0000000..b18edac --- /dev/null +++ b/10 Notes/How to encrypt and decrypt content using symmetric encryption in BearSSL.md @@ -0,0 +1,31 @@ +We show how to use symmetric encryption in BearSSL to encrypt and decrypt the content. +This is a very basic code demonstrating the use of the BearSSL symmetric encryption API. In [[Codex Encryption Design]] we show a more complete example. In the code below we also use [[BearSSL]] to generate a random number. See [[How to generate a random number using BearSSL]] for more details: + +```nim +import std/sequtils +import bearssl/blockx +import stew/byteutils + +import ./rng + +var plaintext = "0123456789abcdef".toBytes +echo "plaintext: ", plaintext.toHex + +let key = newSeqWith(16, Rng.instance.rand(uint8.high).byte) +let ive = newSeqWith(16, Rng.instance.rand(uint8.high).byte) +let ivd = ive + +echo "ive: ", ive.toHex + +var encCtx: AesBigCbcencKeys +aesBigCbcencInit(encCtx, addr key[0], 16.uint) +aesBigCbcencRun(encCtx, addr ive[0], addr plaintext[0], 16.uint) +echo "Encrypted: ", plaintext.toHex + +echo "ivd: ", ivd.toHex + +var decCtx: AesBigCbcdecKeys +aesBigCbcdecInit(decCtx, addr key[0], 16.uint) +aesBigCbcdecRun(decCtx, addr ivd[0], addr plaintext[0], 16.uint) +echo "Decrypted: ", plaintext.toHex +``` \ No newline at end of file diff --git a/10 Notes/How to generate a random number using BearSSL.md b/10 Notes/How to generate a random number using BearSSL.md new file mode 100644 index 0000000..78ec7d0 --- /dev/null +++ b/10 Notes/How to generate a random number using BearSSL.md @@ -0,0 +1,45 @@ +Extracted from [nim-libp2p](https://github.com/vacp2p/nim-libp2p/blob/cfd631457ae865852d17ed77a9d4d3ea37710082/libp2p/crypto/crypto.nim#L161) (`libp2p/crypto/crypto.nim`) and `codex/rng.nim` in [nim-codex](https://github.com/codex-storage/nim-codex) so that it does not depend on libp2p anymore: + +```nim +{.push raises: [].} + +import bearssl/[rand, hash] + +proc newRng*(): ref HmacDrbgContext = + # You should only create one instance of the RNG per application / library + # Ref is used so that it can be shared between components + # TODO consider moving to bearssl + var seeder = prngSeederSystem(nil) + if seeder == nil: + return nil + + var rng = (ref HmacDrbgContext)() + hmacDrbgInit(rng[], addr sha256Vtable, nil, 0) + if seeder(addr rng.vtable) == 0: + return nil + rng + +type + RngSampleError = object of CatchableError + Rng* = ref HmacDrbgContext + +var rng {.threadvar.}: Rng + +proc instance*(t: type Rng): Rng = + if rng.isNil: + rng = newRng() + rng + +# Random helpers: similar as in stdlib, but with HmacDrbgContext rng +# TODO: Move these somewhere else? +const randMax = 18_446_744_073_709_551_615'u64 + +proc rand*(rng: Rng, max: Natural): int = + if max == 0: + return 0 + + while true: + let x = rng[].generate(uint64) + if x < randMax - (randMax mod (uint64(max) + 1'u64)): # against modulo bias + return int(x mod (uint64(max) + 1'u64)) +``` diff --git a/10 Notes/How to get a pointer to a seq to pass it to a C library.md b/10 Notes/How to get a pointer to a seq to pass it to a C library.md index 39fb1d6..c45aa35 100644 --- a/10 Notes/How to get a pointer to a seq to pass it to a C library.md +++ b/10 Notes/How to get a pointer to a seq to pass it to a C library.md @@ -17,5 +17,5 @@ let str = "abcdefghijklmnopqrstuvwxyz" sha256Update(ctx, str.cstring, input[i].len.uint) ``` -Here we use `sha256Update` from [[BearSSL]] library. See [[BearSSL hashing]] for more complete examples. +Here we use `sha256Update` from [[BearSSL]] library. See [[How to create a hash using BearSSL]] for more complete examples. See also [Accessing seq pointer](https://forum.nim-lang.org/t/1489), [Use cstring for C binding](https://forum.nim-lang.org/t/8179), and [Arrays and Sequences in nim](https://forum.nim-lang.org/t/5703) on Nim forum.