From ef5dd8345b2c3484457cc2a29026ef447c46472c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mamy=20Andr=C3=A9-Ratsimbazafy?= Date: Tue, 18 Feb 2020 12:36:42 +0100 Subject: [PATCH] Allow compile-time bigint serialization + terminology: serialize -> export --- constantine/io/endians2.nim | 78 +++++++++++++++++++++++++++++ constantine/io/io_bigints.nim | 39 ++++++--------- constantine/io/io_fields.nim | 6 +-- tests/test_bigints_vs_gmp.nim | 2 +- tests/test_finite_fields.nim | 30 +++++------ tests/test_finite_fields_vs_gmp.nim | 2 +- tests/test_io_bigints.nim | 6 +-- tests/test_io_fields.nim | 14 +++--- 8 files changed, 124 insertions(+), 53 deletions(-) create mode 100644 constantine/io/endians2.nim diff --git a/constantine/io/endians2.nim b/constantine/io/endians2.nim new file mode 100644 index 0000000..081778c --- /dev/null +++ b/constantine/io/endians2.nim @@ -0,0 +1,78 @@ +# Copyright (c) 2018-2019 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +# From https://github.com/status-im/nim-stew/blob/master/stew/endians2.nim +# +# Nim standard library "endians" work with pointers which doesn't work at compile-time +# For auditing purpose and to ensure constant-time safety +# it's better not to introduce a dependency for such a small piece of code + +type + SomeEndianInt* = uint8|uint16|uint32|uint64 + ## types that we support endian conversions for - uint8 is there for + ## for syntactic / generic convenience. Other candidates: + ## * int/uint - uncertain size, thus less suitable for binary interop + ## * intX - over and underflow protection in nim might easily cause issues - + ## need to consider before adding here + +when defined(gcc) or defined(llvm_gcc) or defined(clang): + func swapBytesBuiltin(x: uint8): uint8 = x + func swapBytesBuiltin(x: uint16): uint16 {. + importc: "__builtin_bswap16", nodecl.} + + func swapBytesBuiltin(x: uint32): uint32 {. + importc: "__builtin_bswap32", nodecl.} + + func swapBytesBuiltin(x: uint64): uint64 {. + importc: "__builtin_bswap64", nodecl.} + +elif defined(icc): + func swapBytesBuiltin(x: uint8): uint8 = x + func swapBytesBuiltin(a: uint16): uint16 {.importc: "_bswap16", nodecl.} + func swapBytesBuiltin(a: uint32): uint32 {.importc: "_bswap", nodec.} + func swapBytesBuiltin(a: uint64): uint64 {.importc: "_bswap64", nodecl.} + +elif defined(vcc): + func swapBytesBuiltin(x: uint8): uint8 = x + proc builtin_bswap16(a: uint16): uint16 {. + importc: "_byteswap_ushort", cdecl, header: "".} + + proc builtin_bswap32(a: uint32): uint32 {. + importc: "_byteswap_ulong", cdecl, header: "".} + + proc builtin_bswap64(a: uint64): uint64 {. + importc: "_byteswap_uint64", cdecl, header: "".} + +func swapBytesNim(x: uint8): uint8 = x +func swapBytesNim(x: uint16): uint16 = (x shl 8) or (x shr 8) + +func swapBytesNim(x: uint32): uint32 = + let v = (x shl 16) or (x shr 16) + + ((v shl 8) and 0xff00ff00'u32) or ((v shr 8) and 0x00ff00ff'u32) + +func swapBytesNim(x: uint64): uint64 = + var v = (x shl 32) or (x shr 32) + v = + ((v and 0x0000ffff0000ffff'u64) shl 16) or + ((v and 0xffff0000ffff0000'u64) shr 16) + + ((v and 0x00ff00ff00ff00ff'u64) shl 8) or + ((v and 0xff00ff00ff00ff00'u64) shr 8) + +template swapBytes*[T: SomeEndianInt](x: T): T = + ## Reverse the bytes within an integer, such that the most significant byte + ## changes place with the least significant one, etc + ## + ## Example: + ## doAssert swapBytes(0x01234567'u32) == 0x67452301 + when nimvm: + swapBytesNim(x) + else: + when defined(swapBytesBuiltin): + swapBytesBuiltin(x) + else: + swapBytesNim(x) diff --git a/constantine/io/io_bigints.nim b/constantine/io/io_bigints.nim index 6e75286..564175b 100644 --- a/constantine/io/io_bigints.nim +++ b/constantine/io/io_bigints.nim @@ -11,7 +11,7 @@ # - Burning memory to ensure secrets are not left after dealloc. import - endians, + ./endians2, ../primitives/constant_time, ../math/bigints_checked, ../config/common @@ -151,23 +151,16 @@ func fromUint*( # # ############################################################ -template bigEndianXX[T: uint16 or uint32 or uint64](outp: pointer, inp: ptr T) = - when T is uint64: - bigEndian64(outp, inp) - elif T is uint32: - bigEndian32(outp, inp) - elif T is uint16: - bigEndian16(outp, inp) +template blobFrom*(dst: var openArray[byte], src: SomeEndianInt, startIdx: int, endian: static Endianness) = + ## Write an integer into a raw binary blob + ## Swapping endianness if needed + let s = when endian == cpuEndian: src + else: swapBytes(src) -template littleEndianXX[T: uint16 or uint32 or uint64](outp: pointer, inp: ptr T) = - when T is uint64: - littleEndian64(outp, inp) - elif T is uint32: - littleEndian32(outp, inp) - elif T is uint16: - littleEndian16(outp, inp) + for i in 0 ..< sizeof(src): + dst[startIdx+i] = byte((s shr (i * 8))) -func serializeRawUintLE( +func exportRawUintLE( dst: var openarray[byte], src: BigInt) = ## Serialize a bigint into its canonical little-endian representation @@ -196,7 +189,7 @@ func serializeRawUintLE( if tail >= sizeof(Word): # Unrolled copy - littleEndianXX(dst[dst_idx].addr, lo.unsafeAddr) + dst.blobFrom(src = lo, dst_idx, littleEndian) dst_idx += sizeof(Word) tail -= sizeof(Word) else: @@ -213,7 +206,7 @@ func serializeRawUintLE( dst[dst_idx+i] = byte(lo shr ((tail-i)*8)) return -func serializeRawUintBE( +func exportRawUintBE( dst: var openarray[byte], src: BigInt) = ## Serialize a bigint into its canonical big-endian representation @@ -247,7 +240,7 @@ func serializeRawUintBE( if tail >= sizeof(Word): # Unrolled copy - littleEndianXX(dst[dst_idx].addr, lo.unsafeAddr) + dst.blobFrom(src = lo, dst_idx, littleEndian) dst_idx -= sizeof(Word) tail -= sizeof(Word) else: @@ -264,7 +257,7 @@ func serializeRawUintBE( dst[dst_idx-i] = byte(lo shr ((tail-i)*8)) return -func serializeRawUint*( +func exportRawUint*( dst: var openarray[byte], src: BigInt, dstEndianness: static Endianness) = @@ -283,9 +276,9 @@ func serializeRawUint*( zeroMem(dst, dst.len) when dstEndianness == littleEndian: - serializeRawUintLE(dst, src) + exportRawUintLE(dst, src) else: - serializeRawUintBE(dst, src) + exportRawUintBE(dst, src) # ############################################################ # @@ -438,7 +431,7 @@ func toHex*(big: BigInt, order: static Endianness = bigEndian): string = # 1. Convert Big Int to canonical uint const canonLen = (big.bits + 8 - 1) div 8 var bytes: array[canonLen, byte] - serializeRawUint(bytes, big, cpuEndian) + exportRawUint(bytes, big, cpuEndian) # 2 Convert canonical uint to hex result = bytes.nativeEndianToHex(order) diff --git a/constantine/io/io_fields.nim b/constantine/io/io_fields.nim index 4cf669f..47cb1f7 100644 --- a/constantine/io/io_fields.nim +++ b/constantine/io/io_fields.nim @@ -28,7 +28,7 @@ func fromUint*(dst: var Fq, let raw = (type dst.mres).fromRawUint(cast[array[sizeof(src), byte]](src), cpuEndian) dst.fromBig(raw) -func serializeRawUint*(dst: var openarray[byte], +func exportRawUint*(dst: var openarray[byte], src: Fq, dstEndianness: static Endianness) = ## Serialize a finite field element to its canonical big-endian or little-endian @@ -40,7 +40,7 @@ func serializeRawUint*(dst: var openarray[byte], ## If the buffer is bigger, output will be zero-padded left for big-endian ## or zero-padded right for little-endian. ## I.e least significant bit is aligned to buffer boundary - serializeRawUint(dst, src.toBig(), dstEndianness) + exportRawUint(dst, src.toBig(), dstEndianness) func toHex*(f: Fq, order: static Endianness = bigEndian): string = ## Stringify a finite field element to hex. @@ -51,4 +51,4 @@ func toHex*(f: Fq, order: static Endianness = bigEndian): string = ## ## CT: ## - no leaks - result = f.toBig().toHex() + result = f.toBig().toHex(order) diff --git a/tests/test_bigints_vs_gmp.nim b/tests/test_bigints_vs_gmp.nim index 577e38c..b237728 100644 --- a/tests/test_bigints_vs_gmp.nim +++ b/tests/test_bigints_vs_gmp.nim @@ -138,7 +138,7 @@ proc main() = discard mpz_export(rGMP[0].addr, rW.addr, GMP_LeastSignificantWordFirst, 1, GMP_WordNativeEndian, 0, r) var rConstantine: array[mLen, byte] - serializeRawUint(rConstantine, rTest, littleEndian) + exportRawUint(rConstantine, rTest, littleEndian) # echo "rGMP: ", rGMP.toHex() # echo "rConstantine: ", rConstantine.toHex() diff --git a/tests/test_finite_fields.nim b/tests/test_finite_fields.nim index 283abf0..d9654ac 100644 --- a/tests/test_finite_fields.nim +++ b/tests/test_finite_fields.nim @@ -28,7 +28,7 @@ proc main() = x += y var x_bytes: array[8, byte] - x_bytes.serializeRawUint(x, cpuEndian) + x_bytes.exportRawUint(x, cpuEndian) check: # Check equality in the Montgomery domain @@ -46,7 +46,7 @@ proc main() = x += y var x_bytes: array[8, byte] - x_bytes.serializeRawUint(x, cpuEndian) + x_bytes.exportRawUint(x, cpuEndian) check: # Check equality in the Montgomery domain @@ -64,7 +64,7 @@ proc main() = x += y var x_bytes: array[8, byte] - x_bytes.serializeRawUint(x, cpuEndian) + x_bytes.exportRawUint(x, cpuEndian) check: # Check equality in the Montgomery domain @@ -83,7 +83,7 @@ proc main() = x -= y var x_bytes: array[8, byte] - x_bytes.serializeRawUint(x, cpuEndian) + x_bytes.exportRawUint(x, cpuEndian) check: # Check equality in the Montgomery domain @@ -101,7 +101,7 @@ proc main() = x -= y var x_bytes: array[8, byte] - x_bytes.serializeRawUint(x, cpuEndian) + x_bytes.exportRawUint(x, cpuEndian) check: # Check equality in the Montgomery domain @@ -119,7 +119,7 @@ proc main() = x -= y var x_bytes: array[8, byte] - x_bytes.serializeRawUint(x, cpuEndian) + x_bytes.exportRawUint(x, cpuEndian) check: # Check equality in the Montgomery domain @@ -138,7 +138,7 @@ proc main() = let r = x * y var r_bytes: array[8, byte] - r_bytes.serializeRawUint(r, cpuEndian) + r_bytes.exportRawUint(r, cpuEndian) check: # Check equality in the Montgomery domain @@ -156,7 +156,7 @@ proc main() = let r = x * y var r_bytes: array[8, byte] - r_bytes.serializeRawUint(r, cpuEndian) + r_bytes.exportRawUint(r, cpuEndian) check: # Check equality in the Montgomery domain @@ -175,7 +175,7 @@ proc main() = x += y var x_bytes: array[8, byte] - x_bytes.serializeRawUint(x, cpuEndian) + x_bytes.exportRawUint(x, cpuEndian) let new_x = cast[uint64](x_bytes) check: @@ -194,7 +194,7 @@ proc main() = x += y var x_bytes: array[8, byte] - x_bytes.serializeRawUint(x, cpuEndian) + x_bytes.exportRawUint(x, cpuEndian) let new_x = cast[uint64](x_bytes) check: @@ -213,7 +213,7 @@ proc main() = x += y var x_bytes: array[8, byte] - x_bytes.serializeRawUint(x, cpuEndian) + x_bytes.exportRawUint(x, cpuEndian) let new_x = cast[uint64](x_bytes) check: @@ -233,7 +233,7 @@ proc main() = x -= y var x_bytes: array[8, byte] - x_bytes.serializeRawUint(x, cpuEndian) + x_bytes.exportRawUint(x, cpuEndian) let new_x = cast[uint64](x_bytes) check: @@ -252,7 +252,7 @@ proc main() = x -= y var x_bytes: array[8, byte] - x_bytes.serializeRawUint(x, cpuEndian) + x_bytes.exportRawUint(x, cpuEndian) let new_x = cast[uint64](x_bytes) check: @@ -272,7 +272,7 @@ proc main() = let r = x * y var r_bytes: array[8, byte] - r_bytes.serializeRawUint(r, cpuEndian) + r_bytes.exportRawUint(r, cpuEndian) let new_r = cast[uint64](r_bytes) check: @@ -291,7 +291,7 @@ proc main() = let r = x * y var r_bytes: array[8, byte] - r_bytes.serializeRawUint(r, cpuEndian) + r_bytes.exportRawUint(r, cpuEndian) let new_r = cast[uint64](r_bytes) check: diff --git a/tests/test_finite_fields_vs_gmp.nim b/tests/test_finite_fields_vs_gmp.nim index 839b244..a327b0f 100644 --- a/tests/test_finite_fields_vs_gmp.nim +++ b/tests/test_finite_fields_vs_gmp.nim @@ -111,7 +111,7 @@ proc main() = discard mpz_export(rGMP[0].addr, rW.addr, GMP_LeastSignificantWordFirst, 1, GMP_WordNativeEndian, 0, r) var rConstantine: array[len, byte] - serializeRawUint(rConstantine, rTest, littleEndian) + exportRawUint(rConstantine, rTest, littleEndian) # echo "rGMP: ", rGMP.toHex() # echo "rConstantine: ", rConstantine.toHex() diff --git a/tests/test_io_bigints.nim b/tests/test_io_bigints.nim index 3a840c9..b1de2b2 100644 --- a/tests/test_io_bigints.nim +++ b/tests/test_io_bigints.nim @@ -34,7 +34,7 @@ proc main() = let big = BigInt[64].fromRawUint(x_bytes, littleEndian) # It's fine even on big-endian platform. We only want the byte-pattern var r_bytes: array[8, byte] - serializeRawUint(r_bytes, big, littleEndian) + exportRawUint(r_bytes, big, littleEndian) check: x_bytes == r_bytes block: # "Little-endian" - single random @@ -43,7 +43,7 @@ proc main() = let big = BigInt[64].fromRawUint(x_bytes, littleEndian) # It's fine even on big-endian platform. We only want the byte-pattern var r_bytes: array[8, byte] - serializeRawUint(r_bytes, big, littleEndian) + exportRawUint(r_bytes, big, littleEndian) check: x_bytes == r_bytes block: # "Little-endian" - 10 random cases @@ -53,7 +53,7 @@ proc main() = let big = BigInt[64].fromRawUint(x_bytes, littleEndian) # It's fine even on big-endian platform. We only want the byte-pattern var r_bytes: array[8, byte] - serializeRawUint(r_bytes, big, littleEndian) + exportRawUint(r_bytes, big, littleEndian) check: x_bytes == r_bytes test "Round trip on elliptic curve constants": diff --git a/tests/test_io_fields.nim b/tests/test_io_fields.nim index 3688ed9..5ead349 100644 --- a/tests/test_io_fields.nim +++ b/tests/test_io_fields.nim @@ -26,7 +26,7 @@ proc main() = f.fromUint(x) var r_bytes: array[8, byte] - serializeRawUint(r_bytes, f, littleEndian) + exportRawUint(r_bytes, f, littleEndian) check: x_bytes == r_bytes block: @@ -37,7 +37,7 @@ proc main() = f.fromUint(x) var r_bytes: array[8, byte] - serializeRawUint(r_bytes, f, littleEndian) + exportRawUint(r_bytes, f, littleEndian) check: x_bytes == r_bytes block: @@ -48,7 +48,7 @@ proc main() = f.fromUint(x) var r_bytes: array[8, byte] - serializeRawUint(r_bytes, f, littleEndian) + exportRawUint(r_bytes, f, littleEndian) check: x_bytes == r_bytes block: @@ -59,7 +59,7 @@ proc main() = f.fromUint(x) var r_bytes: array[8, byte] - serializeRawUint(r_bytes, f, littleEndian) + exportRawUint(r_bytes, f, littleEndian) check: x_bytes == r_bytes # Mersenne 127 --------------------------------- @@ -71,7 +71,7 @@ proc main() = f.fromUint(x) var r_bytes: array[16, byte] - serializeRawUint(r_bytes, f, littleEndian) + exportRawUint(r_bytes, f, littleEndian) check: x_bytes == r_bytes[0 ..< 8] block: # "Little-endian" - single random @@ -81,7 +81,7 @@ proc main() = f.fromUint(x) var r_bytes: array[16, byte] - serializeRawUint(r_bytes, f, littleEndian) + exportRawUint(r_bytes, f, littleEndian) check: x_bytes == r_bytes[0 ..< 8] block: # "Little-endian" - 10 random cases @@ -92,7 +92,7 @@ proc main() = f.fromUint(x) var r_bytes: array[16, byte] - serializeRawUint(r_bytes, f, littleEndian) + exportRawUint(r_bytes, f, littleEndian) check: x_bytes == r_bytes[0 ..< 8] test "Round trip on large constant":