From c95cee45bc349fb535376359bec66d7c2cbfd078 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Sat, 6 Jul 2019 20:07:41 +0200 Subject: [PATCH] add byteutils --- README.md | 12 +++- stew/byteutils.nim | 125 +++++++++++++++++++++++++++++++++++++++ tests/all_tests.nim | 3 + tests/test_byteutils.nim | 57 ++++++++++++++++++ 4 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 stew/byteutils.nim create mode 100644 tests/test_byteutils.nim diff --git a/README.md b/README.md index 26d7e49..1b52dc3 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,17 @@ # stew - status e-something w-something +[![Build Status (Travis)](https://img.shields.io/travis/status-im/nim-stew/master.svg?label=Linux%20/%20macOS "Linux/macOS build status (Travis)")](https://travis-ci.org/status-im/nim-stew) +[![Windows build status (Appveyor)](https://img.shields.io/appveyor/ci/nimbus/nim-stew/master.svg?label=Windows "Windows build status (Appveyor)")](https://ci.appveyor.com/project/nimbus/nim-stew) +[![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) +![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg) + `stew` is collection of utilities, std library extensions and budding libraries that are frequently used at Status, but are too small to deserve their own git repository. -We use `stew` as a staging ground for code that has yet to be battle-tested. +We also use `stew` as a staging ground for code that has yet to be +battle-tested. Some of these libraries may eventually be proposed for inclusion in Nim or broken out into separate repositories. @@ -46,7 +53,8 @@ welcome patches. Libraries are documented either in-module or on a separate README in their respective folders -- `bitops2` - an updated version of `bitops.nim`, filling in gaps in original code\ +- `bitops2` - an updated version of `bitops.nim`, filling in gaps in original code +- `byteutils` - utilities that make working with the Nim `byte` type convenient - `shims` - backports of nim `devel` code to the stable version that Status is using ## Using stew in your project diff --git a/stew/byteutils.nim b/stew/byteutils.nim new file mode 100644 index 0000000..e168da9 --- /dev/null +++ b/stew/byteutils.nim @@ -0,0 +1,125 @@ +# byteutils +# Copyright (c) 2018 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + + +######################################################################################################## +#################################### Array utilities ############################################### + +import algorithm + +func initArrayWith*[N: static[int], T](value: T): array[N, T] {.noInit, inline.}= + result.fill(value) + +func `&`*[N1, N2: static[int], T]( + a: array[N1, T], + b: array[N2, T] + ): array[N1 + N2, T] {.inline, noInit.}= + ## Array concatenation + result[0 ..< N1] = a + result[N1 ..< result.len] = b + +######################################################################################################## +##################################### Hex utilities ################################################ + +proc readHexChar*(c: char): byte {.noSideEffect, inline.}= + ## Converts an hex char to a byte + case c + of '0'..'9': result = byte(ord(c) - ord('0')) + of 'a'..'f': result = byte(ord(c) - ord('a') + 10) + of 'A'..'F': result = byte(ord(c) - ord('A') + 10) + else: + raise newException(ValueError, $c & "is not a hexademical character") + +template skip0xPrefix(hexStr: string): int = + ## Returns the index of the first meaningful char in `hexStr` by skipping + ## "0x" prefix + if hexStr[0] == '0' and hexStr[1] in {'x', 'X'}: 2 + else: 0 + +func hexToByteArray*(hexStr: string, output: var openArray[byte], fromIdx, toIdx: int) = + ## Read a hex string and store it in a byte array `output`. No "endianness" reordering is done. + ## Allows specifying the byte range to process into the array + var sIdx = skip0xPrefix(hexStr) + + doAssert(fromIdx >= 0 and toIdx >= fromIdx and fromIdx < output.len and toIdx < output.len) + let sz = toIdx - fromIdx + 1 + + doAssert hexStr.len - sIdx >= 2*sz + + sIdx += fromIdx * 2 + for bIdx in fromIdx ..< sz + fromIdx: + output[bIdx] = hexStr[sIdx].readHexChar shl 4 or hexStr[sIdx + 1].readHexChar + inc(sIdx, 2) + +func hexToByteArray*(hexStr: string, output: var openArray[byte]) {.inline.} = + ## Read a hex string and store it in a byte array `output`. No "endianness" reordering is done. + hexToByteArray(hexStr, output, 0, output.high) + +func hexToByteArray*[N: static[int]](hexStr: string): array[N, byte] {.noInit, inline.}= + ## Read an hex string and store it in a byte array. No "endianness" reordering is done. + hexToByteArray(hexStr, result) + +func hexToPaddedByteArray*[N: static[int]](hexStr: string): array[N, byte] = + ## Read a hex string and store it in a byte array `output`. + ## The string may be shorter than the byte array. + ## No "endianness" reordering is done. + let + p = skip0xPrefix(hexStr) + sz = hexStr.len - p + maxStrSize = result.len * 2 + var + bIdx: int + shift = 4 + + doAssert hexStr.len - p <= maxStrSize + + if sz < maxStrSize: + # include extra byte if odd length + bIdx = result.len - (sz + 1) div 2 + # start with shl of 4 if length is even + shift = 4 - sz mod 2 * 4 + + for sIdx in p ..< hexStr.len: + let nibble = hexStr[sIdx].readHexChar shl shift + result[bIdx] = result[bIdx] or nibble + shift = shift + 4 and 4 + bIdx += shift shr 2 + +func hexToSeqByte*(hexStr: string): seq[byte] = + ## Read an hex string and store it in a sequence of bytes. No "endianness" reordering is done. + doAssert (hexStr.len and 1) == 0 + + let skip = skip0xPrefix(hexStr) + let N = (hexStr.len - skip) div 2 + + result = newSeq[byte](N) + for i in 0 ..< N: + result[i] = hexStr[2*i + skip].readHexChar shl 4 or hexStr[2*i + 1 + skip].readHexChar + +func toHexAux(ba: openarray[byte]): string = + ## Convert a byte-array to its hex representation + ## Output is in lowercase + ## No "endianness" reordering is done. + const hexChars = "0123456789abcdef" + + let sz = ba.len + result = newString(2 * sz) + for i in 0 ..< sz: + result[2*i] = hexChars[int ba[i] shr 4 and 0xF] + result[2*i+1] = hexChars[int ba[i] and 0xF] + +func toHex*(ba: openarray[byte]): string {.inline.} = + ## Convert a byte-array to its hex representation + ## Output is in lowercase + ## No "endianness" reordering is done. + toHexAux(ba) + +func toHex*[N: static[int]](ba: array[N, byte]): string {.inline.} = + ## Convert a big endian byte-array to its hex representation + ## Output is in lowercase + ## No "endianness" reordering is done. + toHexAux(ba) diff --git a/tests/all_tests.nim b/tests/all_tests.nim index 3bfbb6d..189e45c 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -6,3 +6,6 @@ # * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) # # at your option. This file may not be copied, modified, or distributed except according to those terms. + +import + test_byteutils diff --git a/tests/test_byteutils.nim b/tests/test_byteutils.nim new file mode 100644 index 0000000..c21a5d5 --- /dev/null +++ b/tests/test_byteutils.nim @@ -0,0 +1,57 @@ +# byteutils +# Copyright (c) 2018 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +import unittest, + ../stew/byteutils + +suite "Byte utils": + let simpleBArray = [0x12.byte, 0x34, 0x56, 0x78] + + test "hexToByteArray: Inplace partial string": + let s = "0x1234567890" + var a: array[5, byte] + hexToByteArray(s, a, 1, 3) + check a == [0.byte, 0x34, 0x56, 0x78, 0] + + test "hexToByteArray: Inplace full string": + let s = "0xffffffff" + var a: array[4, byte] + hexToByteArray(s, a) + check a == [255.byte, 255, 255, 255] + + test "hexToByteArray: Return array": + let + s = "0x12345678" + a = hexToByteArray[4](s) + check a == simpleBArray + + test "toHex": + check simpleBArray.toHex == "12345678" + + test "Array concatenation": + check simpleBArray & simpleBArray == + [0x12.byte, 0x34, 0x56, 0x78, 0x12, 0x34, 0x56, 0x78] + + test "hexToPaddedByteArray": + block: + let a = hexToPaddedByteArray[4]("0x123") + check a.toHex == "00000123" + block: + let a = hexToPaddedByteArray[4]("0x1234") + check a.toHex == "00001234" + block: + let a = hexToPaddedByteArray[4]("0x1234567") + check a.toHex == "01234567" + block: + let a = hexToPaddedByteArray[4]("0x12345678") + check a.toHex == "12345678" + block: + let a = hexToPaddedByteArray[32]("0x68656c6c6f20776f726c64") + check a.toHex == "00000000000000000000000000000000000000000068656c6c6f20776f726c64" + block: + expect AssertionError: + let a = hexToPaddedByteArray[2]("0x12345")