From df72a0896bd1e50a6433a7490dd586c2885ea80e Mon Sep 17 00:00:00 2001 From: mratsim Date: Sun, 2 Dec 2018 22:18:17 +0100 Subject: [PATCH 1/3] initial implementation of bigInt dump --- constantine/io.nim | 84 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 79 insertions(+), 5 deletions(-) diff --git a/constantine/io.nim b/constantine/io.nim index f079ea7..10f5cff 100644 --- a/constantine/io.nim +++ b/constantine/io.nim @@ -9,7 +9,9 @@ # - Constant-time validation for parsing secret keys # - Burning memory to ensure secrets are not left after dealloc. -import ./word_types, ./bigints +import + endians, + ./word_types, ./bigints # ############################################################ # @@ -74,7 +76,7 @@ func readDecChar(c: range['0'..'9']): int {.inline.}= # ############################################################ func parseRawUint*( - input: openarray[byte], + src: openarray[byte], bits: static int, endian: static Endianness): BigInt[bits] = ## Parse an unsigned integer from its canonical @@ -90,7 +92,7 @@ func parseRawUint*( acc_len = 0 template body(){.dirty.} = - let src_byte = Word(input[src_idx]) + let src_byte = Word(src[src_idx]) acc = acc and (src_byte shl acc_len) acc_len += 8 # We count bit by bit @@ -102,10 +104,10 @@ func parseRawUint*( acc = src_byte shr (8 - acc_len) when endian == bigEndian: - for src_idx in countdown(input.high, 0): + for src_idx in countdown(src.high, 0): body() else: - for src_idx in 0 ..< input.len: + for src_idx in 0 ..< src.len: body() if acc_len != 0: @@ -116,3 +118,75 @@ func parseRawUint*( # Serialising from internal representation to canonical format # # ############################################################ + +template bigEndian[T: uint16 or uint32 or uint64](outp: pointer, inp: ptr T) = + when T is uint64: + bigEndian64(outp, inp) + elif T is uint32: + bigEndian32(outp, inp) + elif T is uint16: + bigEndian16(outp, inp) + +template littleEndian[T: uint16 or uint32 or uint64](outp: pointer, inp: ptr T) = + when T is uint64: + littleEndian64(outp, inp) + elif T is uint32: + littleEndian32(outp, inp) + elif T is uint16: + littleEndian16(outp, inp) + +func round_step_up(x: Natural, step: static Natural): int {.inline.} = + ## Round the input to the next multiple of "step" + assert (step and (step - 1)) == 0, "Step must be a power of 2" + result = (x + step - 1) and not(step - 1) + +func dumpRawUint*( + dst: var openarray[byte], + src: BigInt, + endian: static Endianness) = + ## Serialize a bigint into its canonical big-endian or little endian + ## representation. + ## A destination buffer of size "BigInt.bits div 8" at minimum is needed. + ## + ## If the buffer is bigger, output will be zero-padded left for big-endian + ## or zero-padded right for little-endian. + ## I.e least significant bit is aligned to buffer boundary + + if dst.len < static(BigInt.bits div 8): + raise newException(ValueError, "BigInt -> Raw int conversion: destination buffer is too small") + + when BigInt.bits == 0: + zeroMem(dst, dst.len) + else: + var + src_idx = 0 + acc = Word(0) + acc_len = 0 + + template body(){.dirty.} = + let w = if src_idx < src.limbs.len: src[src_idx] + else: Word(0) + inc src_idx + + if acc_len == 0: + # Edge case to avoid shifting by 0 + acc = w + acc_len = WordBitSize + else: + let lo = (w shr acc_len) or acc + dec acc_len + acc = w shr (WordBitSize - acc_len) + when endian == bigEndian: + # We're counting down + bigEndian(dst[dst_idx - Word.sizeof], w.unsafeAddr) + else: + littleEndian(dst[dst_idx], w.unsafeAddr) + + when endian == bigEndian: + discard # TODO + else: + let unroll_stop = round_step_up(dst.len, Word.sizeof) + for dst_idx in countup(0, unroll_stop - 1, Word.sizeof): + body() + + # Process the tail - TODO From 2856378427f909af7f72efcbe69af138e84eead2 Mon Sep 17 00:00:00 2001 From: mratsim Date: Mon, 3 Dec 2018 19:56:14 +0100 Subject: [PATCH 2/3] [IO] dumping 2^63 works --- constantine/bigints.nim | 2 + constantine/io.nim | 90 +++++++++++++++++++++++------------------ tests/test_io.nim | 24 ++++++++++- 3 files changed, 75 insertions(+), 41 deletions(-) diff --git a/constantine/bigints.nim b/constantine/bigints.nim index 9bc78c0..ac7a5a6 100644 --- a/constantine/bigints.nim +++ b/constantine/bigints.nim @@ -40,6 +40,8 @@ import ./word_types type Word* = Ct[uint64] +type BaseType* = uint64 # Exported type for conversion in "normal integers" + const WordBitSize* = sizeof(Word) * 8 - 1 ## Limbs are 63-bit by default diff --git a/constantine/io.nim b/constantine/io.nim index 10f5cff..83c5b74 100644 --- a/constantine/io.nim +++ b/constantine/io.nim @@ -119,7 +119,7 @@ func parseRawUint*( # # ############################################################ -template bigEndian[T: uint16 or uint32 or uint64](outp: pointer, inp: ptr T) = +template bigEndianXX[T: uint16 or uint32 or uint64](outp: pointer, inp: ptr T) = when T is uint64: bigEndian64(outp, inp) elif T is uint32: @@ -127,7 +127,7 @@ template bigEndian[T: uint16 or uint32 or uint64](outp: pointer, inp: ptr T) = elif T is uint16: bigEndian16(outp, inp) -template littleEndian[T: uint16 or uint32 or uint64](outp: pointer, inp: ptr T) = +template littleEndianXX[T: uint16 or uint32 or uint64](outp: pointer, inp: ptr T) = when T is uint64: littleEndian64(outp, inp) elif T is uint32: @@ -135,15 +135,55 @@ template littleEndian[T: uint16 or uint32 or uint64](outp: pointer, inp: ptr T) elif T is uint16: littleEndian16(outp, inp) -func round_step_up(x: Natural, step: static Natural): int {.inline.} = - ## Round the input to the next multiple of "step" - assert (step and (step - 1)) == 0, "Step must be a power of 2" - result = (x + step - 1) and not(step - 1) +func dumpRawUintLE( + dst: var openarray[byte], + src: BigInt) {.inline.}= + ## Serialize a bigint into its canonical big-endian representation + ## I.e least significant bit is aligned to buffer boundary + + var + src_idx, dst_idx = 0 + acc: BaseType = 0 + acc_len = 0 + + var tail = dst.len + while tail > 0: + let w = if src_idx < src.limbs.len: src[src_idx].BaseType + else: 0 + inc src_idx + + if acc_len == 0: + # Edge case, we need to refill the buffer to output 64-bit + # as we can only read 63-bit per word + acc = w + acc_len = WordBitSize + else: + let lo = (w shl acc_len) or acc + dec acc_len + acc = w shr (WordBitSize - acc_len) + + if tail >= sizeof(Word): + # Unrolled copy + # debugecho src.repr + littleEndianXX(dst[dst_idx].addr, lo.unsafeAddr) + dst_idx += sizeof(Word) + tail -= sizeof(Word) + else: + # Process the tail + when cpuEndian == littleEndian: + # When requesting little-endian on little-endian platform + # we can just copy each byte + for i in dst_idx ..< tail: + dst[dst_idx] = byte(lo shr (i-dst_idx)) + else: + # We need to copy from the end + for i in 0 ..< tail: + dst[dst_idx] = byte(lo shr (tail-i)) func dumpRawUint*( dst: var openarray[byte], src: BigInt, - endian: static Endianness) = + order: static Endianness) = ## Serialize a bigint into its canonical big-endian or little endian ## representation. ## A destination buffer of size "BigInt.bits div 8" at minimum is needed. @@ -157,36 +197,8 @@ func dumpRawUint*( when BigInt.bits == 0: zeroMem(dst, dst.len) + + when order == littleEndian: + dumpRawUintLE(dst, src) else: - var - src_idx = 0 - acc = Word(0) - acc_len = 0 - - template body(){.dirty.} = - let w = if src_idx < src.limbs.len: src[src_idx] - else: Word(0) - inc src_idx - - if acc_len == 0: - # Edge case to avoid shifting by 0 - acc = w - acc_len = WordBitSize - else: - let lo = (w shr acc_len) or acc - dec acc_len - acc = w shr (WordBitSize - acc_len) - when endian == bigEndian: - # We're counting down - bigEndian(dst[dst_idx - Word.sizeof], w.unsafeAddr) - else: - littleEndian(dst[dst_idx], w.unsafeAddr) - - when endian == bigEndian: - discard # TODO - else: - let unroll_stop = round_step_up(dst.len, Word.sizeof) - for dst_idx in countup(0, unroll_stop - 1, Word.sizeof): - body() - - # Process the tail - TODO + {.error: "Not implemented at the moment".} diff --git a/tests/test_io.nim b/tests/test_io.nim index 43f551b..99c38e9 100644 --- a/tests/test_io.nim +++ b/tests/test_io.nim @@ -5,10 +5,11 @@ # * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -import unittest, +import unittest, random, ../constantine/[io, bigints] -type T = uint64 +randomize(0xDEADBEEF) # Random seed for reproducibility +type T = BaseType suite "IO": test "Parsing raw integers": @@ -29,3 +30,22 @@ suite "IO": check: T(big[0]) == 0 T(big[1]) == 1 + + test "Parsing and dumping round-trip": + block: # "Little-endian" + let x = 1'u64 shl 63 + let x_bytes = cast[array[8, byte]](x) + let big = parseRawUint(x_bytes, 64, littleEndian) # It's fine even on big-endian platform. We only want the byte-pattern + + var r_bytes: array[8, byte] + dumpRawUint(r_bytes, big, littleEndian) + check: x_bytes == r_bytes + + # block: # "Little-endian" + # let x = uint64 rand(0..high(int)) + # let x_bytes = cast[array[8, byte]](x) + # let big = parseRawUint(x_bytes, 64, littleEndian) # It's fine even on big-endian platform. We only want the byte-pattern + + # var r_bytes: array[8, byte] + # dumpRawUint(r_bytes, big, littleEndian) + # check: x_bytes == r_bytes From 059439b2c3830883f50777d853980c08a539d711 Mon Sep 17 00:00:00 2001 From: mratsim Date: Mon, 3 Dec 2018 21:01:29 +0100 Subject: [PATCH 3/3] fix little-endian parsing --- constantine/io.nim | 36 +++++++++++++++++++++++------------- tests/test_io.nim | 29 ++++++++++++++++++++--------- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/constantine/io.nim b/constantine/io.nim index 83c5b74..faf1a9f 100644 --- a/constantine/io.nim +++ b/constantine/io.nim @@ -75,12 +75,11 @@ func readDecChar(c: range['0'..'9']): int {.inline.}= # # ############################################################ -func parseRawUint*( +func parseRawUintLE( src: openarray[byte], - bits: static int, - endian: static Endianness): BigInt[bits] = + bits: static int): BigInt[bits] {.inline.}= ## Parse an unsigned integer from its canonical - ## big-endian or little-endian unsigned representation + ## little-endian unsigned representation ## And store it into a BigInt of size bits ## ## CT: @@ -91,28 +90,39 @@ func parseRawUint*( acc = Word(0) acc_len = 0 - template body(){.dirty.} = + for src_idx in 0 ..< src.len: let src_byte = Word(src[src_idx]) - acc = acc and (src_byte shl acc_len) + # buffer reads + acc = acc or (src_byte shl acc_len) acc_len += 8 # We count bit by bit + # if full, dump if acc_len >= WordBitSize: result[dst_idx] = acc and MaxWord inc dst_idx acc_len -= WordBitSize acc = src_byte shr (8 - acc_len) - when endian == bigEndian: - for src_idx in countdown(src.high, 0): - body() - else: - for src_idx in 0 ..< src.len: - body() - if acc_len != 0: result[dst_idx] = acc +func parseRawUint*( + src: openarray[byte], + bits: static int, + order: static Endianness): BigInt[bits] = + ## Parse an unsigned integer from its canonical + ## big-endian or little-endian unsigned representation + ## And store it into a BigInt of size bits + ## + ## CT: + ## - no leaks + + when order == littleEndian: + parseRawUintLE(src, bits) + else: + {.error: "Not implemented at the moment".} + # ############################################################ # # Serialising from internal representation to canonical format diff --git a/tests/test_io.nim b/tests/test_io.nim index 99c38e9..b3508e0 100644 --- a/tests/test_io.nim +++ b/tests/test_io.nim @@ -31,8 +31,9 @@ suite "IO": T(big[0]) == 0 T(big[1]) == 1 - test "Parsing and dumping round-trip": - block: # "Little-endian" + test "Parsing and dumping round-trip on uint64": + block: + # "Little-endian" - 2^63 let x = 1'u64 shl 63 let x_bytes = cast[array[8, byte]](x) let big = parseRawUint(x_bytes, 64, littleEndian) # It's fine even on big-endian platform. We only want the byte-pattern @@ -41,11 +42,21 @@ suite "IO": dumpRawUint(r_bytes, big, littleEndian) check: x_bytes == r_bytes - # block: # "Little-endian" - # let x = uint64 rand(0..high(int)) - # let x_bytes = cast[array[8, byte]](x) - # let big = parseRawUint(x_bytes, 64, littleEndian) # It's fine even on big-endian platform. We only want the byte-pattern + block: # "Little-endian" - single random + let x = uint64 rand(0..high(int)) + let x_bytes = cast[array[8, byte]](x) + let big = parseRawUint(x_bytes, 64, littleEndian) # It's fine even on big-endian platform. We only want the byte-pattern - # var r_bytes: array[8, byte] - # dumpRawUint(r_bytes, big, littleEndian) - # check: x_bytes == r_bytes + var r_bytes: array[8, byte] + dumpRawUint(r_bytes, big, littleEndian) + check: x_bytes == r_bytes + + block: # "Little-endian" - 10 random cases + for _ in 0 ..< 10: + let x = uint64 rand(0..high(int)) + let x_bytes = cast[array[8, byte]](x) + let big = parseRawUint(x_bytes, 64, littleEndian) # It's fine even on big-endian platform. We only want the byte-pattern + + var r_bytes: array[8, byte] + dumpRawUint(r_bytes, big, littleEndian) + check: x_bytes == r_bytes