Fixes for bugs discovered while testing the Hexary Trie

Bugfixes:

* `rlp.rawData` was returning a wider than necessary range when
  called over a `listElem` result.

* Appending `Rlp` objects to a `RlpWriter` was producing incorrect
  results.

New features:

* When the Rlp reading cursor is positioned over a list, you can call
  `enterList` to position the cursor to the first list element.
  Call `skipElem` to move the cursor further.

* `rlpFromHex` will now accept leading `0x` symbols (often used in the
  Ethereum world).

* `rlp.inspect` will now print hex characters by default. Use the
  optional parameter `hexOutput` to control the behavior.

rlp.nim

@@ -41,12 +41,15 @@ proc rlpFromHex*(input: string): Rlp =
   doAssert input.len mod 2 == 0,
           "rlpFromHex expects a string with even number of characters (assuming two characters per byte)"
 
-  let totalBytes = input.len div 2
+  var startByte = if input.len >= 2 and input[0] == '0' and input[1] == 'x': 2
+                  else: 0
+
+  let totalBytes = (input.len - startByte) div 2
   var backingStore = newSeq[byte](totalBytes)
 
   for i in 0 ..< totalBytes:
     var nextByte: int
-    if parseHex(input, nextByte, i*2, 2) == 2:
+    if parseHex(input, nextByte, startByte + i*2, 2) == 2:
       backingStore[i] = byte(nextByte)
     else:
       doAssert false, "rlpFromHex expects a hexademical string, but the input contains non hexademical characters"
@@ -58,8 +61,10 @@ proc rlpFromHex*(input: string): Rlp =
 proc hasData*(self: Rlp): bool =
   position < bytes.len
 
-template rawData*(self: Rlp): BytesRange =
-  self.bytes
+proc currentElemEnd(self: Rlp): int
+
+proc rawData*(self: Rlp): BytesRange =
+  return self.bytes[position ..< self.currentElemEnd]
 
 proc isBlob*(self: Rlp): bool =
   hasData() and bytes[position] < LIST_START_MARKER
@@ -209,7 +214,8 @@ proc toString*(self: Rlp): string =
 
 proc toBytes*(self: Rlp): BytesRange =
   if not isBlob():
-    raise newException(RlpTypeMismatch, "Bytes expected, but the source RLP in not a blob")
+    raise newException(RlpTypeMismatch,
+                       "Bytes expected, but the source RLP in not a blob")
 
   let
     payloadOffset = payloadOffset()
@@ -220,16 +226,18 @@ proc toBytes*(self: Rlp): BytesRange =
   result = bytes.slice(ibegin, iend)
 
 proc currentElemEnd(self: Rlp): int =
+  assert hasData()
   result = position
 
-  if not hasData():
-    return
-
   if isSingleByte():
     result += 1
   elif isBlob() or isList():
     result += payloadOffset() + payloadBytesCount()
 
+proc enterList*(self: var Rlp) =
+  assert isList()
+  position += payloadOffset()
+
 proc skipElem*(rlp: var Rlp) =
   rlp.position = rlp.currentElemEnd
 
@@ -376,7 +384,7 @@ template decode*(bytes: openarray[byte], T: typedesc): T =
   decode(initBytesRange(bytesCopy), T)
 
 proc append*(writer: var RlpWriter; rlp: Rlp) =
-  append(writer, rlp.rawData)
+  appendRawBytes(writer, rlp.rawData)
 
 proc isPrintable(s: string): bool =
   for c in s:
@@ -385,7 +393,7 @@ proc isPrintable(s: string): bool =
 
   return true
 
-proc inspectAux(self: var Rlp, depth: int, output: var string) =
+proc inspectAux(self: var Rlp, depth: int, hexOutput: bool, output: var string) =
   if not hasData():
     return
 
@@ -407,19 +415,26 @@ proc inspectAux(self: var Rlp, depth: int, output: var string) =
     else:
       output.add "blob(" & $str.len & ") ["
       for c in str:
-        output.add $ord(c)
-        output.add ","
-      output[^1] = ']'
+        if hexOutput:
+          output.add toHex(int(c), 2)
+        else:
+          output.add $ord(c)
+          output.add ","
+
+      if hexOutput:
+        output.add ']'
+      else:
+        output[^1] = ']'
   else:
     output.add "{\n"
     for subitem in self:
-      inspectAux(subitem, depth + 1, output)
+      inspectAux(subitem, depth + 1, hexOutput, output)
       output.add "\n"
     indent()
     output.add "}"
 
-proc inspect*(self: Rlp, indent = 0): string =
+proc inspect*(self: Rlp, indent = 0, hexOutput = true): string =
   var rlpCopy = self
   result = newStringOfCap(bytes.len)
-  inspectAux(rlpCopy, indent, result)
+  inspectAux(rlpCopy, indent, hexOutput, result)
 

rlp/writer.nim

@@ -120,6 +120,10 @@ proc appendRawList(self; bytes: BytesRange) =
   output.add(bytes)
   maybeClosePendingLists()
 
+proc appendRawBytes*(self; bytes: BytesRange) =
+  output.add(bytes)
+  maybeClosePendingLists()
+
 proc startList*(self; listSize: int) =
   if listSize == 0:
     appendRawList(BytesRange())

tests/all.nim

@@ -1,3 +1,3 @@
 import
-  test_api_usage, test_json_suite, test_object_serialization
+  test_api_usage, test_object_serialization, test_json_suite
 

tests/test_api_usage.nim

@@ -15,6 +15,9 @@ test "empty bytes are not a proper RLP":
     not rlp.isList
     not rlp.isEmpty
 
+  expect Exception:
+    rlp.skipElem
+
   expect Exception:
     discard rlp.getType
 
@@ -95,6 +98,18 @@ test "encode and decode lists":
     rlp.listElem(1).toString == "Lorem ipsum dolor sit amet"
     rlp.listElem(2).toString == "Donec ligula tortor, egestas eu est vitae"
 
+  # test creating RLPs from other RLPs
+  var list = rlpFromBytes encodeList(rlp.listELem(1), rlp.listELem(0))
+
+  # test that iteration with enterList/skipElem works as expected
+  list.enterList
+  check list.toString == "Lorem ipsum dolor sit amet"
+  list.skipElem
+  check list.toInt(int32) == 6000.int32
+  list.skipElem
+  check(not list.hasData)
+  expect Exception: list.skipElem
+
 test "toBytes":
   let rlp = rlpFromHex("f2cb847f000001827666827666a040ef02798f211da2e8173d37f255be908871ae65060dbb2f77fb29c0421447f4845ab90b50")
   let tok = rlp.listElem(1).toBytes()
@@ -125,8 +140,12 @@ test "encoding length":
   check emptyListRlp.blobLen == 0
 
 test "basic decoding":
-  var rlp = rlpFromHex("856d6f6f7365")
-  check rlp.inspect == q"moose"
+  var rlp1 = rlpFromHex("856d6f6f7365")
+  var rlp2 = rlpFromHex("0x856d6f6f7365")
+
+  check:
+    rlp1.inspect == q"moose"
+    rlp2.inspect == q"moose"
 
 test "malformed/truncated RLP":
   var rlp = rlpFromHex("b8056d6f6f7365")
@@ -144,3 +163,7 @@ test "encode byte arrays":
     rlp.listElem(0).toBytes().toSeq() == @b1
     rlp.listElem(1).toBytes().toSeq() == @b2
     rlp.listElem(2).toBytes().toSeq() == @b3
+
+    # The first byte here is the length of the datum (132 - 128 => 4)
+    $(rlp.listElem(1).rawData) == "R[132, 6, 8, 12, 123]"
+