Re-enabled requireAllFields after a fix in nim-json-serialization (#3871)

* Re-enabled requireAllFields after a fix in nim-json-serialization

The problem was that `Option[T]` fields were not treated as optional
when requireAllFields is set to true. This is now fixed in NJS.

* Add makefile targets for recreating the Jenkins simulation runs

* Fix a discrepancy with the REST spec

Jenkinsfile

@@ -49,17 +49,8 @@ def runStages(nodeDir) {
 			stage("Testnet finalization") { timeout(75) {
 				// EXECUTOR_NUMBER will be 0 or 1, since we have 2 executors per Jenkins node
 				sh """#!/bin/bash
-				set -e
-				./scripts/launch_local_testnet.sh --preset minimal --nodes 4 --stop-at-epoch 5 --disable-htop --enable-logtrace \
-					--data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rest-port \
-					\$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) --timeout 600 \
-					--kill-old-processes --light-clients 1 \
-					-- --verify-finalization --discv5:no
-				./scripts/launch_local_testnet.sh --nodes 4 --stop-at-epoch 5 --disable-htop --enable-logtrace \
-					--data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rest-port \
-					\$(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) --timeout 2400 \
-					--kill-old-processes --light-clients 1 \
-					-- --verify-finalization --discv5:no
+				make local-testnet-minimal
+				make local-testnet-mainnet
 				"""
 			} }
 		} catch(e) {

Makefile

@@ -157,6 +157,41 @@ restapi-test:
 		--resttest-delay 30 \
 		--kill-old-processes
 
+local-testnet-minimal:
+	./scripts/launch_local_testnet.sh \
+		--preset minimal \
+		--nodes 4 \
+		--stop-at-epoch 5 \
+		--disable-htop \
+		--enable-logtrace \
+		--data-dir local_testnet0_data \
+		--base-port $$(( 9100 + EXECUTOR_NUMBER * 100 )) \
+		--base-rest-port $$(( 7100 + EXECUTOR_NUMBER * 100 )) \
+		--base-metrics-port $$(( 8108 + EXECUTOR_NUMBER * 100 )) \
+		--timeout 600 \
+		--kill-old-processes \
+		--light-clients 1 \
+		-- \
+		--verify-finalization \
+		--discv5:no
+
+local-testnet-mainnet:
+	./scripts/launch_local_testnet.sh \
+		--nodes 4 \
+		--stop-at-epoch 5 \
+		--disable-htop \
+		--enable-logtrace \
+		--data-dir local_testnet1_data \
+		--base-port $$(( 9100 + EXECUTOR_NUMBER * 100 )) \
+		--base-rest-port $$(( 7100 + EXECUTOR_NUMBER * 100 )) \
+		--base-metrics-port $$(( 8108 + EXECUTOR_NUMBER * 100 )) \
+		--timeout 2400 \
+		--kill-old-processes \
+		--light-clients 1 \
+		-- \
+		--verify-finalization \
+		--discv5:no
+
 # test binaries that can output an XML report
 XML_TEST_BINARIES := \
 	consensus_spec_tests_mainnet \

beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim

@@ -2179,11 +2179,12 @@ proc decodeBody*[T](t: typedesc[T],
   let data =
     try:
       RestJson.decode(body.data, T,
-                      requireAllFields = false,
+                      requireAllFields = true,
                       allowUnknownFields = true)
     except SerializationError as exc:
       debug "Failed to deserialize REST JSON data",
-            err = exc.formatMsg("<data>")
+            err = exc.formatMsg("<data>"),
+            data = string.fromBytes(body.data)
       return err("Unable to deserialize data")
     except CatchableError:
       return err("Unexpected deserialization error")
@@ -2233,11 +2234,12 @@ proc decodeBytes*[T: DecodeTypes](t: typedesc[T], value: openArray[byte],
   of "application/json":
     try:
       ok RestJson.decode(value, T,
-                         requireAllFields = false,
+                         requireAllFields = true,
                          allowUnknownFields = true)
     except SerializationError as exc:
       debug "Failed to deserialize REST JSON data",
-            err = exc.formatMsg("<data>")
+            err = exc.formatMsg("<data>"),
+            data = string.fromBytes(value)
       err("Serialization error")
   else:
     err("Content-Type not supported")

beacon_chain/spec/eth2_apis/rest_types.nim

@@ -560,7 +560,7 @@ type
   GetStateValidatorBalancesResponse* = DataEnclosedObject[seq[RestValidatorBalance]]
   GetStateValidatorResponse* = DataEnclosedObject[RestValidator]
   GetStateValidatorsResponse* = DataEnclosedObject[seq[RestValidator]]
-  GetSyncCommitteeDutiesResponse* = DataRootEnclosedObject[seq[RestSyncCommitteeDuty]]
+  GetSyncCommitteeDutiesResponse* = DataEnclosedObject[seq[RestSyncCommitteeDuty]]
   GetSyncingStatusResponse* = DataEnclosedObject[RestSyncInfo]
   GetVersionResponse* = DataEnclosedObject[RestNodeVersion]
   GetEpochSyncCommitteesResponse* = DataEnclosedObject[RestEpochSyncCommittee]

vendor/nim-json-serialization

@@ -1 +1 @@
-Subproject commit 5a7f9a86cb201606ae669ed4f7f605047c26628c
+Subproject commit 97cf1841190277ed985d1ead88e7712aaf0f3f74

vendor/nim-serialization

@@ -1 +1 @@
-Subproject commit 1d33fa3ced6bc274ed43d99345ceb9cd6bb4dd24
+Subproject commit 493d18b8292fc03aa4f835fd825dea1183f97466

vendor/nim-stew

@@ -1 +1 @@
-Subproject commit 4cab7b08793d25c311efe88d54f948815643bc41
+Subproject commit b55c5a6d7496159a261186807d9bbe3f6f5dac38