Pad byte sequences with 0x1

To ensure unique field elements for sequences that
end with 0x0.

poseidon2/io.nim

@@ -2,30 +2,33 @@ import ./types
 import constantine/math/arithmetic
 import constantine/math/io/io_bigints
 
-func fromBytes*(_: type F, bytes: openArray[byte]): F =
+func fromOpenArray(_: type F, bytes: openArray[byte]): F =
-  ## Converts bytes into a field element. The byte array is interpreted as a
+  F.fromBig(B.unmarshal(bytes, littleEndian))
-  ## canonical little-endian big integer. The array should be of length 31 bytes
-  ## or less to ensure that it fits in a field of 254 bits. The remaining
-  ## 6 most-significant bits are set to 0.
-  assert bytes.len <= 31
-  var padded: array[32, byte]
-  copyMem(addr padded[0], unsafeAddr bytes[0], bytes.len)
-  let bigint = B.unmarshal(padded, littleEndian)
-  return F.fromBig(bigint)
 
-iterator elements*(bytes: openArray[byte], _: type F): F =
+func fromBytes*(_: type F, bytes: array[31, byte]): F =
-  ## Converts bytes into field elements. The byte array is converted 31 bytes at
+  ## Converts bytes into a field element. The byte array is interpreted as a
-  ## a time with the `F.fromBytes()` function.
+  ## canonical little-endian big integer.
-  const chunkLen = 31
+  F.fromOpenArray(bytes)
-  var chunkStart = 0
-  while chunkStart < bytes.len:
-    let chunkEnd = min(chunkStart + 31, bytes.len)
-    let element = F.fromBytes(bytes.toOpenArray(chunkStart, chunkEnd - 1))
-    yield element
-    chunkStart += chunkLen
 
 func toBytes*(element: F): array[32, byte] =
   ## Converts a field element into its canonical representation in little-endian
-  ## byte order. Uses at most 254 bits, the remaining 6 most-significant bits
+  ## byte order. Uses at most 254 bits, the remaining most-significant bits are
-  ## are set to 0.
+  ## set to 0.
   assert marshal(result, element.toBig(), littleEndian)
+
+iterator elements*(bytes: openArray[byte], _: type F): F =
+  ## Converts bytes into field elements. The byte array is converted 31 bytes at
+  ## a time with the `F.fromBytes()` function. An end marker (0x1) and
+  ## padding (0x0's) are added to ensure unique field elements for byte
+  ## sequences that end with 0x0's.
+  const chunkLen = 31
+  const endMarker = @[1'u8]
+  var chunkStart = 0
+  while chunkStart + chunkLen <= bytes.len:
+    let chunkEnd = chunkStart + chunkLen - 1
+    let element = F.fromOpenArray(bytes.toOpenArray(chunkStart, chunkEnd))
+    yield element
+    chunkStart += chunkLen
+  let finalChunk = bytes[chunkStart..<bytes.len] & endMarker
+  let finalElement = F.fromOpenArray(finalChunk)
+  yield finalElement

tests/poseidon2/testIo.nim

@@ -7,31 +7,38 @@ import poseidon2/io
 
 suite "conversion to/from bytes":
 
-  test "converts little endian bytes into field elements":
+  func toArray(bytes: openArray[byte]): array[31, byte] =
-    let bytes = toSeq 1'u8..31'u8
+    result[0..<bytes.len] = bytes[0..<bytes.len]
-    let paddedTo32 = bytes & @[0'u8] # most significant byte is not used
-    let expected = F.fromBig(B.unmarshal(paddedTo32, littleEndian))
-    let unmarshalled = F.fromBytes(bytes)
-    check bool(unmarshalled == expected)
 
-  test "pads little endian bytes to the right with 0's":
+  test "converts 31 little endian bytes into a field elements":
-    let bytes = @[0x56'u8, 0x34, 0x12]
+    let bytes = toArray toSeq 1'u8..31'u8
-    let paddedTo32 = bytes & 0'u8.repeat(32 - bytes.len)
+    let paddedTo32 = @bytes & @[0'u8] # most significant byte is not used
     let expected = F.fromBig(B.unmarshal(paddedTo32, littleEndian))
     let unmarshalled = F.fromBytes(bytes)
     check bool(unmarshalled == expected)
 
   test "converts every 31 bytes into a field element":
-    let bytes = toSeq 1'u8..80'u8
+    let bytes = toSeq 1'u8..62'u8
-    let padded = bytes & 0'u8.repeat(93 - bytes.len)
+    let expected1 = F.fromBytes(bytes[0..<31].toArray)
-    let expected1 = F.fromBig(B.unmarshal(padded[ 0..<31] & @[0'u8], littleEndian))
+    let expected2 = F.fromBytes(bytes[31..<62].toArray)
-    let expected2 = F.fromBig(B.unmarshal(padded[31..<62] & @[0'u8], littleEndian))
-    let expected3 = F.fromBig(B.unmarshal(padded[62..<93] & @[0'u8], littleEndian))
     let elements = toSeq bytes.elements(F)
-    check elements.len == 3
     check bool(elements[0] == expected1)
     check bool(elements[1] == expected2)
-    check bool(elements[2] == expected3)
+
+  test "conversion from bytes adds 0x1 as an end marker":
+    let bytes = toSeq 1'u8..62'u8
+    let marker = @[1'u8]
+    let expected = F.fromBytes(marker.toArray)
+    let elements = toSeq bytes.elements(F)
+    check bool(elements[^1] == expected)
+
+  test "conversion from bytes pads the last chunk when it's less than 31 bytes":
+    let bytes = toSeq 1'u8..80'u8
+    let marker = @[1'u8]
+    let padded = bytes[62..<80] & marker & 0'u8.repeat(12)
+    let expected = F.fromBytes(padded.toArray)
+    let elements = toSeq bytes.elements(F)
+    check bool(elements[^1] == expected)
 
   test "converts field element into little-endian bytes":
     var element: F

tests/poseidon2/testPoseidon2.nim

@@ -35,9 +35,9 @@ suite "poseidon2":
   test "merkle root of bytes":
     let bytes = toSeq 1'u8..80'u8
     let root = merkleRoot(bytes)
-    check root.toHex(littleEndian) == "0xa1dffa3f60d166283d60396023d95a1d7996d119e5290fe31131e7c6a7a27817"
+    check root.toHex(littleEndian) == "0x4f317667856aa66125dfa14c09096e08573803c91f87f20e2940238ede66a02d"
 
   test "merkle root of bytes converted to bytes":
     let bytes = toSeq 1'u8..80'u8
     let rootAsBytes = merkleRoot(bytes).toBytes()
-    check rootAsBytes.toHex == "0xa1dffa3f60d166283d60396023d95a1d7996d119e5290fe31131e7c6a7a27817"
+    check rootAsBytes.toHex == "0x4f317667856aa66125dfa14c09096e08573803c91f87f20e2940238ede66a02d"