status-im/nimbus-eth2
2
0
Fork 0
mirror of https://github.com/status-im/nimbus-eth2.git synced 2025-02-19 17:58:23 +00:00
Code Issues Projects Releases Wiki Activity

More spec-compliant handling of unknown fields in REST json (#3647)

Browse Source 
* More spec-compliant handling of unknown fields in REST json

* Address review comments
This commit is contained in:
zah 2022-05-20 18:25:26 +03:00 committed by GitHub
parent 50e84156cc
commit 68fb3962c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 92 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim
View File

@ -7,7 +7,8 @@
import std/typetraits import std/typetraits
import stew/[assign2, results, base10, byteutils], presto/common, import stew/[assign2, results, base10, byteutils], presto/common,
libp2p/peerid, serialization, json_serialization, libp2p/peerid, serialization, json_serialization,
json_serialization/std/[options, net, sets] json_serialization/std/[options, net, sets],
chronicles
import ".."/[eth2_ssz_serialization, forks, keystore], import ".."/[eth2_ssz_serialization, forks, keystore],
".."/datatypes/[phase0, altair, bellatrix], ".."/datatypes/[phase0, altair, bellatrix],
".."/mev/bellatrix_mev, ".."/mev/bellatrix_mev,
@ -16,7 +17,7 @@ import ".."/[eth2_ssz_serialization, forks, keystore],
import nimcrypto/utils as ncrutils import nimcrypto/utils as ncrutils
export export
eth2_ssz_serialization, results, peerid, common, serialization, eth2_ssz_serialization, results, peerid, common, serialization, chronicles,
json_serialization, options, net, sets, rest_types, slashing_protection_common json_serialization, options, net, sets, rest_types, slashing_protection_common
from web3/ethtypes import BlockHash from web3/ethtypes import BlockHash
@ -24,6 +25,33 @@ export ethtypes.BlockHash
Json.createFlavor RestJson Json.createFlavor RestJson
## The RestJson format implements JSON serialization in the way specified
## by the Beacon API:
##
## https://ethereum.github.io/beacon-APIs/
##
## In this format, we must always set `allowUnknownFields = true` in the
## decode calls in order to conform the following spec:
##
## All JSON responses return the requested data under a data key in the top
## level of their response. Additional metadata may or may not be present
## in other keys at the top level of the response, dependent on the endpoint.
## The rules that require an increase in version number are as follows:
##
## - no field that is listed in an endpoint shall be removed without an increase
## in the version number
##
## - no field that is listed in an endpoint shall be altered in terms of format
## (e.g. from a string to an array) without an increase in the version number
##
## Note that it is possible for a field to be added to an endpoint's data or
## metadata without an increase in the version number.
##
## TODO nim-json-serializations should allow setting up this policy per format
##
## This also means that when new fields are introduced to the object definitions
## below, one must use the `Option[T]` type.
const const
DecimalSet = {'0' .. '9'} DecimalSet = {'0' .. '9'}
# Base10 (decimal) set of chars # Base10 (decimal) set of chars
@ -93,12 +121,6 @@ type
Web3SignerSignatureResponse | Web3SignerSignatureResponse |
Web3SignerStatusResponse Web3SignerStatusResponse
# These types may be extended with additional fields in the future.
# Locally unknown fields are silently ignored when decoding them.
ExtensibleDecodeTypes* =
GetSpecResponse |
GetSpecVCResponse
SszDecodeTypes* = SszDecodeTypes* =
GetPhase0StateSszResponse | GetPhase0StateSszResponse |
GetPhase0BlockSszResponse GetPhase0BlockSszResponse
@ -338,7 +360,9 @@ proc decodeJsonString*[T](t: typedesc[T],
data: JsonString, data: JsonString,
requireAllFields = true): Result[T, cstring] = requireAllFields = true): Result[T, cstring] =
try: try:
ok(RestJson.decode(string(data), T, requireAllFields = requireAllFields)) ok(RestJson.decode(string(data), T,
requireAllFields = requireAllFields,
allowUnknownFields = true))
except SerializationError: except SerializationError:
err("Unable to deserialize data") err("Unable to deserialize data")
@ -664,6 +688,13 @@ proc readValue*(
raiseUnexpectedValue( raiseUnexpectedValue(
reader, "Expected a valid hex string with " & $value.len() & " bytes") reader, "Expected a valid hex string with " & $value.len() & " bytes")
template unrecognizedFieldWarning =
# TODO: There should be a different notification mechanism for informing the
# caller of a deserialization routine for unexpected fields.
# The chonicles import in this module should be removed.
debug "JSON field not recognized by the current version of Nimbus. Consider upgrading",
fieldName, typeName = typetraits.name(typeof value)
## ForkedBeaconBlock ## ForkedBeaconBlock
proc readValue*[BlockType: Web3SignerForkedBeaconBlock|ForkedBeaconBlock]( proc readValue*[BlockType: Web3SignerForkedBeaconBlock|ForkedBeaconBlock](
reader: var JsonReader[RestJson], reader: var JsonReader[RestJson],
@ -694,7 +725,7 @@ proc readValue*[BlockType: Web3SignerForkedBeaconBlock|ForkedBeaconBlock](
"ForkedBeaconBlock") "ForkedBeaconBlock")
data = some(reader.readValue(JsonString)) data = some(reader.readValue(JsonString))
else: else:
reader.raiseUnexpectedField(fieldName, "ForkedBeaconBlock") unrecognizedFieldWarning()
if version.isNone(): if version.isNone():
reader.raiseUnexpectedValue("Field version is missing") reader.raiseUnexpectedValue("Field version is missing")
@ -705,8 +736,10 @@ proc readValue*[BlockType: Web3SignerForkedBeaconBlock|ForkedBeaconBlock](
of BeaconBlockFork.Phase0: of BeaconBlockFork.Phase0:
let res = let res =
try: try:
some(RestJson.decode(string(data.get()), phase0.BeaconBlock, some(RestJson.decode(string(data.get()),
requireAllFields = true)) phase0.BeaconBlock,
requireAllFields = true,
allowUnknownFields = true))
except SerializationError: except SerializationError:
none[phase0.BeaconBlock]() none[phase0.BeaconBlock]()
if res.isNone(): if res.isNone():
@ -715,8 +748,10 @@ proc readValue*[BlockType: Web3SignerForkedBeaconBlock|ForkedBeaconBlock](
of BeaconBlockFork.Altair: of BeaconBlockFork.Altair:
let res = let res =
try: try:
some(RestJson.decode(string(data.get()), altair.BeaconBlock, some(RestJson.decode(string(data.get()),
requireAllFields = true)) altair.BeaconBlock,
requireAllFields = true,
allowUnknownFields = true))
except SerializationError: except SerializationError:
none[altair.BeaconBlock]() none[altair.BeaconBlock]()
if res.isNone(): if res.isNone():
@ -725,8 +760,10 @@ proc readValue*[BlockType: Web3SignerForkedBeaconBlock|ForkedBeaconBlock](
of BeaconBlockFork.Bellatrix: of BeaconBlockFork.Bellatrix:
let res = let res =
try: try:
some(RestJson.decode(string(data.get()), bellatrix.BeaconBlock, some(RestJson.decode(string(data.get()),
requireAllFields = true)) bellatrix.BeaconBlock,
requireAllFields = true,
allowUnknownFields = true))
except SerializationError: except SerializationError:
none[bellatrix.BeaconBlock]() none[bellatrix.BeaconBlock]()
if res.isNone(): if res.isNone():
@ -835,8 +872,7 @@ proc readValue*(reader: var JsonReader[RestJson],
"RestPublishedBeaconBlockBody") "RestPublishedBeaconBlockBody")
execution_payload = some(reader.readValue(ExecutionPayload)) execution_payload = some(reader.readValue(ExecutionPayload))
else: else:
# Ignore unknown fields unrecognizedFieldWarning()
discard
if randao_reveal.isNone(): if randao_reveal.isNone():
reader.raiseUnexpectedValue("Field `randao_reveal` is missing") reader.raiseUnexpectedValue("Field `randao_reveal` is missing")
@ -949,8 +985,7 @@ proc readValue*(reader: var JsonReader[RestJson],
"RestPublishedBeaconBlock") "RestPublishedBeaconBlock")
blockBody = some(reader.readValue(RestPublishedBeaconBlockBody)) blockBody = some(reader.readValue(RestPublishedBeaconBlockBody))
else: else:
# Ignore unknown fields unrecognizedFieldWarning()
discard
if slot.isNone(): if slot.isNone():
reader.raiseUnexpectedValue("Field `slot` is missing") reader.raiseUnexpectedValue("Field `slot` is missing")
@ -1017,8 +1052,7 @@ proc readValue*(reader: var JsonReader[RestJson],
"RestPublishedSignedBeaconBlock") "RestPublishedSignedBeaconBlock")
signature = some(reader.readValue(ValidatorSig)) signature = some(reader.readValue(ValidatorSig))
else: else:
# Ignore unknown fields unrecognizedFieldWarning()
discard
if signature.isNone(): if signature.isNone():
reader.raiseUnexpectedValue("Field `signature` is missing") reader.raiseUnexpectedValue("Field `signature` is missing")
@ -1081,7 +1115,7 @@ proc readValue*(reader: var JsonReader[RestJson],
"ForkedSignedBeaconBlock") "ForkedSignedBeaconBlock")
data = some(reader.readValue(JsonString)) data = some(reader.readValue(JsonString))
else: else:
reader.raiseUnexpectedField(fieldName, "ForkedSignedBeaconBlock") unrecognizedFieldWarning()
if version.isNone(): if version.isNone():
reader.raiseUnexpectedValue("Field version is missing") reader.raiseUnexpectedValue("Field version is missing")
@ -1092,8 +1126,10 @@ proc readValue*(reader: var JsonReader[RestJson],
of BeaconBlockFork.Phase0: of BeaconBlockFork.Phase0:
let res = let res =
try: try:
some(RestJson.decode(string(data.get()), phase0.SignedBeaconBlock, some(RestJson.decode(string(data.get()),
requireAllFields = true)) phase0.SignedBeaconBlock,
requireAllFields = true,
allowUnknownFields = true))
except SerializationError: except SerializationError:
none[phase0.SignedBeaconBlock]() none[phase0.SignedBeaconBlock]()
if res.isNone(): if res.isNone():
@ -1102,8 +1138,10 @@ proc readValue*(reader: var JsonReader[RestJson],
of BeaconBlockFork.Altair: of BeaconBlockFork.Altair:
let res = let res =
try: try:
some(RestJson.decode(string(data.get()), altair.SignedBeaconBlock, some(RestJson.decode(string(data.get()),
requireAllFields = true)) altair.SignedBeaconBlock,
requireAllFields = true,
allowUnknownFields = true))
except SerializationError: except SerializationError:
none[altair.SignedBeaconBlock]() none[altair.SignedBeaconBlock]()
if res.isNone(): if res.isNone():
@ -1112,8 +1150,10 @@ proc readValue*(reader: var JsonReader[RestJson],
of BeaconBlockFork.Bellatrix: of BeaconBlockFork.Bellatrix:
let res = let res =
try: try:
some(RestJson.decode(string(data.get()), bellatrix.SignedBeaconBlock, some(RestJson.decode(string(data.get()),
requireAllFields = true)) bellatrix.SignedBeaconBlock,
requireAllFields = true,
allowUnknownFields = true))
except SerializationError: except SerializationError:
none[bellatrix.SignedBeaconBlock]() none[bellatrix.SignedBeaconBlock]()
if res.isNone(): if res.isNone():
@ -1165,7 +1205,7 @@ proc readValue*(reader: var JsonReader[RestJson],
"ForkedBeaconState") "ForkedBeaconState")
data = some(reader.readValue(JsonString)) data = some(reader.readValue(JsonString))
else: else:
reader.raiseUnexpectedField(fieldName, "ForkedBeaconState") unrecognizedFieldWarning()
if version.isNone(): if version.isNone():
reader.raiseUnexpectedValue("Field version is missing") reader.raiseUnexpectedValue("Field version is missing")
@ -1188,7 +1228,10 @@ proc readValue*(reader: var JsonReader[RestJson],
of BeaconStateFork.Phase0: of BeaconStateFork.Phase0:
try: try:
tmp[].phase0Data.data = RestJson.decode( tmp[].phase0Data.data = RestJson.decode(
string(data.get()), phase0.BeaconState, requireAllFields = true) string(data.get()),
phase0.BeaconState,
requireAllFields = true,
allowUnknownFields = true)
except SerializationError: except SerializationError:
reader.raiseUnexpectedValue("Incorrect phase0 beacon state format") reader.raiseUnexpectedValue("Incorrect phase0 beacon state format")
@ -1196,7 +1239,10 @@ proc readValue*(reader: var JsonReader[RestJson],
of BeaconStateFork.Altair: of BeaconStateFork.Altair:
try: try:
tmp[].altairData.data = RestJson.decode( tmp[].altairData.data = RestJson.decode(
string(data.get()), altair.BeaconState, requireAllFields = true) string(data.get()),
altair.BeaconState,
requireAllFields = true,
allowUnknownFields = true)
except SerializationError: except SerializationError:
reader.raiseUnexpectedValue("Incorrect altair beacon state format") reader.raiseUnexpectedValue("Incorrect altair beacon state format")
@ -1204,7 +1250,10 @@ proc readValue*(reader: var JsonReader[RestJson],
of BeaconStateFork.Bellatrix: of BeaconStateFork.Bellatrix:
try: try:
tmp[].bellatrixData.data = RestJson.decode( tmp[].bellatrixData.data = RestJson.decode(
string(data.get()), bellatrix.BeaconState, requireAllFields = true) string(data.get()),
bellatrix.BeaconState,
requireAllFields = true,
allowUnknownFields = true)
except SerializationError: except SerializationError:
reader.raiseUnexpectedValue("Incorrect altair beacon state format") reader.raiseUnexpectedValue("Incorrect altair beacon state format")
toValue(bellatrixData) toValue(bellatrixData)
@ -1382,8 +1431,7 @@ proc readValue*(reader: var JsonReader[RestJson],
dataName = fieldName dataName = fieldName
data = some(reader.readValue(JsonString)) data = some(reader.readValue(JsonString))
else: else:
# We ignore all unknown fields. unrecognizedFieldWarning()
discard
if requestKind.isNone(): if requestKind.isNone():
reader.raiseUnexpectedValue("Field `type` is missing") reader.raiseUnexpectedValue("Field `type` is missing")
@ -1620,10 +1668,11 @@ proc readValue*(reader: var JsonReader[RestJson],
reader.raiseUnexpectedValue("Invalid `status` value") reader.raiseUnexpectedValue("Invalid `status` value")
) )
else: else:
# We ignore all unknown fields. unrecognizedFieldWarning()
discard
if status.isNone(): if status.isNone():
reader.raiseUnexpectedValue("Field `status` is missing") reader.raiseUnexpectedValue("Field `status` is missing")
value = RemoteKeystoreStatus(status: status.get(), message: message) value = RemoteKeystoreStatus(status: status.get(), message: message)
## ScryptSalt ## ScryptSalt
@ -1675,8 +1724,7 @@ proc readValue*(reader: var JsonReader[RestJson], value: var Pbkdf2Params) {.
"Pbkdf2Params") "Pbkdf2Params")
salt = some(reader.readValue(Pbkdf2Salt)) salt = some(reader.readValue(Pbkdf2Salt))
else: else:
# Ignore unknown field names. unrecognizedFieldWarning()
discard
if dklen.isNone(): if dklen.isNone():
reader.raiseUnexpectedValue("Field `dklen` is missing") reader.raiseUnexpectedValue("Field `dklen` is missing")
@ -1749,8 +1797,7 @@ proc readValue*(reader: var JsonReader[RestJson], value: var ScryptParams) {.
"ScryptParams") "ScryptParams")
salt = some(reader.readValue(ScryptSalt)) salt = some(reader.readValue(ScryptSalt))
else: else:
# Ignore unknown field names. unrecognizedFieldWarning()
discard
if dklen.isNone(): if dklen.isNone():
reader.raiseUnexpectedValue("Field `dklen` is missing") reader.raiseUnexpectedValue("Field `dklen` is missing")
@ -1833,8 +1880,7 @@ proc readValue*(reader: var JsonReader[RestJson], value: var Keystore) {.
reader.raiseUnexpectedValue("Unexpected negative `version` value") reader.raiseUnexpectedValue("Unexpected negative `version` value")
version = some(res) version = some(res)
else: else:
# Ignore unknown field names. unrecognizedFieldWarning()
discard
if crypto.isNone(): if crypto.isNone():
reader.raiseUnexpectedValue("Field `crypto` is missing") reader.raiseUnexpectedValue("Field `crypto` is missing")
@ -1888,8 +1934,7 @@ proc readValue*(reader: var JsonReader[RestJson],
"KeystoresAndSlashingProtection") "KeystoresAndSlashingProtection")
slashing = some(reader.readValue(SPDIR)) slashing = some(reader.readValue(SPDIR))
else: else:
# Ignore unknown field names. unrecognizedFieldWarning()
discard
if len(keystores) == 0: if len(keystores) == 0:
reader.raiseUnexpectedValue("Missing `keystores` value") reader.raiseUnexpectedValue("Missing `keystores` value")
@ -1912,7 +1957,7 @@ proc decodeBody*[T](t: typedesc[T],
return err("Unsupported content type") return err("Unsupported content type")
let data = let data =
try: try:
RestJson.decode(body.data, T) RestJson.decode(body.data, T, allowUnknownFields = true)
except SerializationError as exc: except SerializationError as exc:
return err("Unable to deserialize data") return err("Unable to deserialize data")
except CatchableError: except CatchableError:
@ -1959,11 +2004,10 @@ proc encodeBytes*[T: EncodeArrays](value: T,
proc decodeBytes*[T: DecodeTypes](t: typedesc[T], value: openArray[byte], proc decodeBytes*[T: DecodeTypes](t: typedesc[T], value: openArray[byte],
contentType: string): RestResult[T] = contentType: string): RestResult[T] =
const isExtensibleType = t is ExtensibleDecodeTypes
case contentType case contentType
of "application/json": of "application/json":
try: try:
ok RestJson.decode(value, T, allowUnknownFields = isExtensibleType) ok RestJson.decode(value, T, allowUnknownFields = true)
except SerializationError: except SerializationError:
err("Serialization error") err("Serialization error")
else: else: