add `KZGProof` and `Blob` REST JSON serializations (#5526)

* add KZGProof and Blob REST JSON serializations

* error summary updates

* copyright notice

* consistent integer types for Nim 2.0

AllTests-mainnet.md

@@ -430,11 +430,13 @@ OK: 12/12 Fail: 0/12 Skip: 0/12
 OK: 1/1 Fail: 0/1 Skip: 0/1
 ## REST JSON encoding and decoding
 ```diff
-+ DenebSignedBlockContents                                                                   OK
++ Blob                                                                                       OK
++ DenebSignedBlockContents decoding                                                          OK
 + KzgCommitment                                                                              OK
-+ RestPublishedSignedBlockContents                                                           OK
++ KzgProof                                                                                   OK
++ RestPublishedSignedBlockContents decoding                                                  OK
 ```
-OK: 3/3 Fail: 0/3 Skip: 0/3
+OK: 5/5 Fail: 0/5 Skip: 0/5
 ## Remove keystore testing suite
 ```diff
 + Many remotes                                                                               OK
@@ -712,4 +714,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 401/406 Fail: 0/406 Skip: 5/406
+OK: 403/408 Fail: 0/408 Skip: 5/408

beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim

@@ -950,9 +950,26 @@ proc writeValue*(
 ) {.raises: [IOError].} =
   writeValue(writer, hexOriginal(distinctBase(value)))
 
-## KzgCommitment
+## Blob
-# https://github.com/ethereum/beacon-APIs/blob/d934a03187729635bef06ca7f3c067645c3eab15/types/primitive.yaml#L135-L140
+## https://github.com/ethereum/beacon-APIs/blob/v2.4.2/types/primitive.yaml#L129-L133
-proc readValue*(reader: var JsonReader[RestJson], value: var KzgCommitment) {.
+proc readValue*(reader: var JsonReader[RestJson], value: var Blob) {.
+     raises: [IOError, SerializationError].} =
+  try:
+    hexToByteArray(reader.readValue(string), distinctBase(value))
+  except ValueError:
+    raiseUnexpectedValue(reader,
+                         "Blob value should be a valid hex string")
+
+proc writeValue*(
+    writer: var JsonWriter[RestJson], value: Blob
+) {.raises: [IOError].} =
+  writeValue(writer, hexOriginal(distinctBase(value)))
+
+## KzgCommitment and KzgProof; both are the same type, but this makes it
+## explicit.
+## https://github.com/ethereum/beacon-APIs/blob/v2.4.2/types/primitive.yaml#L135-L146
+proc readValue*(reader: var JsonReader[RestJson],
+     value: var (KzgCommitment|KzgProof)) {.
      raises: [IOError, SerializationError].} =
   try:
     hexToByteArray(reader.readValue(string), distinctBase(value))
@@ -961,7 +978,7 @@ proc readValue*(reader: var JsonReader[RestJson], value: var KzgCommitment) {.
                          "KzgCommitment value should be a valid hex string")
 
 proc writeValue*(
-    writer: var JsonWriter[RestJson], value: KzgCommitment
+    writer: var JsonWriter[RestJson], value: KzgCommitment | KzgProof
 ) {.raises: [IOError].} =
   writeValue(writer, hexOriginal(distinctBase(value)))
 

tests/simulation/restapi.sh

@@ -1,4 +1,11 @@
 #!/usr/bin/env bash
+#
+# beacon_chain
+# Copyright (c) 2021-2023 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 set -e
 
@@ -196,6 +203,8 @@ DEPOSIT_CONTRACT_BLOCK="0x000000000000000000000000000000000000000000000000000000
 
 echo Wrote $RUNTIME_CONFIG_FILE:
 
+# DENEB_FORK_EPOCH must be non-FAR_FUTURE_EPOCH to trigger creation of blob
+# sidecar database table.
 tee "$RUNTIME_CONFIG_FILE" <<EOF
 PRESET_BASE: "mainnet"
 MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: ${NUM_VALIDATORS}
@@ -206,6 +215,8 @@ DEPOSIT_CONTRACT_ADDRESS: ${DEPOSIT_CONTRACT_ADDRESS}
 ETH1_FOLLOW_DISTANCE: 1
 ALTAIR_FORK_EPOCH: 0
 BELLATRIX_FORK_EPOCH: 0
+CAPELLA_FORK_EPOCH: 9000
+DENEB_FORK_EPOCH: 10000
 EOF
 
 echo "Creating testnet genesis..."
@@ -218,6 +229,8 @@ ${LOCAL_TESTNET_SIMULATION_BIN} \
   --output-deposit-tree-snapshot="${DEPOSIT_TREE_SNAPSHOT_FILE}" \
   --output-bootstrap-file="${NETWORK_BOOTSTRAP_FILE}" \
   --netkey-file=network_key.json \
+  --capella-fork-epoch=9000 \
+  --deneb-fork-epoch=10000 \
   --insecure-netkey-password=true \
   --genesis-offset=-60 # Chain that has already started allows testing empty slots
 # Make sure we use the newly generated genesis

tests/test_rest_json_serialization.nim

@@ -11,6 +11,8 @@ import
   unittest2,
   ../beacon_chain/spec/eth2_apis/eth2_rest_serialization
 
+from std/strutils import endsWith, startsWith
+
 const denebSignedContents = """
 {
   "signed_block": {
@@ -213,13 +215,13 @@ func fromHex(T: typedesc[KzgCommitment], s: string): T {.
   res
 
 suite "REST JSON encoding and decoding":
-  test "DenebSignedBlockContents":
+  test "DenebSignedBlockContents decoding":
     check: hash_tree_root(RestJson.decode(
       denebSignedContents, DenebSignedBlockContents, requireAllFields = true,
       allowUnknownFields = true)) == Eth2Digest.fromHex(
         "0x6b9fce0e35ee7af9b061f244706c4eda43c16e9dcc5b1cc817ed0671f49d16a8")
 
-  test "RestPublishedSignedBlockContents":
+  test "RestPublishedSignedBlockContents decoding":
     check: hash_tree_root(RestJson.decode(
       denebSignedContents, RestPublishedSignedBlockContents,
       requireAllFields = true, allowUnknownFields = true).denebData) ==
@@ -255,3 +257,66 @@ suite "REST JSON encoding and decoding":
       RestJson.encode(zeroKzgCommitment) != randString
       RestJson.encode(randKzgCommitment) != zeroString
       RestJson.encode(randKzgCommitment) == randString
+
+  test "KzgProof":
+    let
+      zeroString =
+        "\"0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\""
+      randString =
+        "\"0xe2822fdd03685968091c79b1f81d17ed646196c920baecf927a6abbe45cd2d930a692e85ff5d96ebe36d99a57c74d5cb\""
+      zeroKzgProof = KzgProof.fromHex(
+        "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
+      randKzgProof = KzgProof.fromHex(
+        "0xe2822fdd03685968091c79b1f81d17ed646196c920baecf927a6abbe45cd2d930a692e85ff5d96ebe36d99a57c74d5cb")
+
+    check:
+      RestJson.decode(
+        zeroString, KzgProof, requireAllFields = true,
+          allowUnknownFields = true) == zeroKzgProof
+      RestJson.decode(
+        zeroString, KzgProof, requireAllFields = true,
+          allowUnknownFields = true) != randKzgProof
+      RestJson.decode(
+        randString, KzgProof, requireAllFields = true,
+          allowUnknownFields = true) != zeroKzgProof
+      RestJson.decode(
+        randString, KzgProof, requireAllFields = true,
+          allowUnknownFields = true) == randKzgProof
+
+      RestJson.encode(zeroKzgProof) == zeroString
+      RestJson.encode(zeroKzgProof) != randString
+      RestJson.encode(randKzgProof) != zeroString
+      RestJson.encode(randKzgProof) == randString
+
+  test "Blob":
+    let
+      zeroBlob = new Blob
+      nonzeroBlob = new Blob
+      blobLen = distinctBase(nonzeroBlob[]).lenu64
+
+    for i in 0 ..< blobLen:
+      nonzeroBlob[i] = 17.byte
+
+    let
+      zeroString = newClone(RestJson.encode(zeroBlob[]))
+      nonzeroString = newClone(RestJson.encode(nonzeroBlob[]))
+
+    let
+      zeroBlobRoundTrip =
+        newClone(RestJson.decode(
+          zeroString[], Blob, requireAllFields = true, allowUnknownFields = true))
+      nonzeroBlobRoundTrip =
+        newClone(RestJson.decode(
+          nonzeroString[], Blob, requireAllFields = true,
+          allowUnknownFields = true))
+
+    check:
+      zeroString[].startsWith "\"0x0000000000000000000000000000000000000000000000000"
+      nonzeroString[].startsWith "\"0x111111111111111111111111111111111111111111111111"
+      zeroString[].endsWith "0000000000000000000000000000000000000000000000\""
+      nonzeroString[].endsWith "1111111111111111111111111111111111111111111111\""
+      zeroString[].lenu64 == 2*blobLen + 4   # quotation marks and 0x prefix
+      nonzeroString[].lenu64 == 2*blobLen + 4   # quotation marks and 0x prefix
+      zeroBlob[] == zeroBlobRoundTrip[]
+      nonzeroBlob[] == nonzeroBlobRoundTrip[]
+      zeroBlob[] != nonzeroBlob[]