Write beacon LC content test vectors to yaml instead of json (#2176)

fluffy/eth_data/yaml_utils.nim

@@ -62,6 +62,7 @@ proc dumpToYaml*[T](value: T, file: string): Result[void, string] =
     except Exception as e:
       raiseAssert(e.msg)
   try:
+    {.gcsafe.}:
       # Dump to yaml, avoiding TAGS and YAML version directives.
       dump(value, s, tagStyle = tsNone, options = options, handles = @[])
   except YamlPresenterJsonError as e:

fluffy/tools/eth_data_exporter/cl_data_exporter.nim

@@ -31,7 +31,7 @@ from beacon_chain/el/el_manager import toBeaconBlockHeader
 export beacon_clock
 
 const
-  largeRequestsTimeout = 60.seconds # Downloading large items such as states.
+  largeRequestsTimeout = 120.seconds # For downloading large items such as states.
   restRequestsTimeout = 30.seconds
 
 proc getBeaconData*(): (RuntimeConfig, ref ForkDigests, BeaconClock) =
@@ -66,15 +66,8 @@ proc exportLCBootstrapUpdate*(
     cfg: RuntimeConfig,
     forkDigests: ref ForkDigests,
 ) {.async.} =
-  let file = "light-client-bootstrap.json"
+  let fileName = "light-client-bootstrap-" & $trustedBlockRoot.data.toHex() & ".yaml"
-  let fh = createAndOpenFile(dataDir, file)
+  existsFile(dataDir, fileName)
-
-  defer:
-    try:
-      fh.close()
-    except IOError as e:
-      fatal "Error occured while closing file", error = e.msg
-      quit 1
 
   let client = RestClientRef.new(restUrl).valueOr:
     error "Cannot connect to server", error = error
@@ -101,14 +94,8 @@ proc exportLCBootstrapUpdate*(
         forkDigest = forkDigestAtEpoch(forkDigests[], epoch(slot), cfg)
         content = encodeBootstrapForked(forkDigest, update)
 
-      let portalContent = JsonPortalContent(
+        file = dataDir / fileName
-        content_key: contentKey.asSeq().to0xHex(), content_value: content.to0xHex()
+      writePortalContentToYaml(file, contentKey.asSeq().to0xHex(), content.to0xHex())
-      )
-
-      var contentTable: JsonPortalContentTable
-      contentTable[$slot] = portalContent
-
-      writePortalContentToJson(fh, contentTable)
 
 proc exportLCUpdates*(
     restUrl: string,
@@ -118,15 +105,8 @@ proc exportLCUpdates*(
     cfg: RuntimeConfig,
     forkDigests: ref ForkDigests,
 ) {.async.} =
-  let file = "light-client-updates.json"
+  let fileName = "light-client-updates-" & $startPeriod & "-" & $count & ".yaml"
-  let fh = createAndOpenFile(dataDir, file)
+  existsFile(dataDir, fileName)
-
-  defer:
-    try:
-      fh.close()
-    except IOError as e:
-      fatal "Error occured while closing file", error = e.msg
-      quit 1
 
   let client = RestClientRef.new(restUrl).valueOr:
     error "Cannot connect to server", error = error
@@ -160,14 +140,8 @@ proc exportLCUpdates*(
 
           content = encodeLightClientUpdatesForked(forkDigest, updates)
 
-        let portalContent = JsonPortalContent(
+          file = dataDir / fileName
-          content_key: contentKey.asSeq().to0xHex(), content_value: content.to0xHex()
+        writePortalContentToYaml(file, contentKey.asSeq().to0xHex(), content.to0xHex())
-        )
-
-        var contentTable: JsonPortalContentTable
-        contentTable[$slot] = portalContent
-
-        writePortalContentToJson(fh, contentTable)
   else:
     error "No updates downloaded"
     quit 1
@@ -175,15 +149,8 @@ proc exportLCUpdates*(
 proc exportLCFinalityUpdate*(
     restUrl: string, dataDir: string, cfg: RuntimeConfig, forkDigests: ref ForkDigests
 ) {.async.} =
-  let file = "light-client-finality-update.json"
+  let fileName = "light-client-finality-update.yaml"
-  let fh = createAndOpenFile(dataDir, file)
+  existsFile(dataDir, fileName)
-
-  defer:
-    try:
-      fh.close()
-    except IOError as e:
-      fatal "Error occured while closing file", error = e.msg
-      quit 1
 
   let client = RestClientRef.new(restUrl).valueOr:
     error "Cannot connect to server", error = error
@@ -211,27 +178,14 @@ proc exportLCFinalityUpdate*(
         )
         content = encodeFinalityUpdateForked(forkDigest, update)
 
-      let portalContent = JsonPortalContent(
+        file = dataDir / fileName
-        content_key: contentKey.asSeq().to0xHex(), content_value: content.to0xHex()
+      writePortalContentToYaml(file, contentKey.asSeq().to0xHex(), content.to0xHex())
-      )
-
-      var contentTable: JsonPortalContentTable
-      contentTable[$finalizedSlot] = portalContent
-
-      writePortalContentToJson(fh, contentTable)
 
 proc exportLCOptimisticUpdate*(
     restUrl: string, dataDir: string, cfg: RuntimeConfig, forkDigests: ref ForkDigests
 ) {.async.} =
-  let file = "light-client-optimistic-update.json"
+  let fileName = "light-client-optimistic-update.yaml"
-  let fh = createAndOpenFile(dataDir, file)
+  existsFile(dataDir, fileName)
-
-  defer:
-    try:
-      fh.close()
-    except IOError as e:
-      fatal "Error occured while closing file", error = e.msg
-      quit 1
 
   let client = RestClientRef.new(restUrl).valueOr:
     error "Cannot connect to server", error = error
@@ -259,14 +213,8 @@ proc exportLCOptimisticUpdate*(
         )
         content = encodeOptimisticUpdateForked(forkDigest, update)
 
-      let portalContent = JsonPortalContent(
+        file = dataDir / fileName
-        content_key: contentKey.asSeq().to0xHex(), content_value: content.to0xHex()
+      writePortalContentToYaml(file, contentKey.asSeq().to0xHex(), content.to0xHex())
-      )
-
-      var contentTable: JsonPortalContentTable
-      contentTable[$slot] = portalContent
-
-      writePortalContentToJson(fh, contentTable)
 
 proc exportHistoricalRoots*(
     restUrl: string, dataDir: string, cfg: RuntimeConfig, forkDigests: ref ForkDigests

fluffy/tools/eth_data_exporter/exporter_common.nim

@@ -14,7 +14,8 @@ import
   faststreams,
   json_serialization,
   json_serialization/std/tables,
-  ../../eth_data/history_data_json_store
+  ../../eth_data/history_data_json_store,
+  ../../eth_data/yaml_utils
 
 export history_data_json_store
 
@@ -28,6 +29,17 @@ proc writePortalContentToJson*(
     fatal "Error occured while writing to file", error = e.msg
     quit 1
 
+proc writePortalContentToYaml*(file: string, contentKey: string, contentValue: string) =
+  let
+    yamlPortalContent =
+      YamlPortalContent(content_key: contentKey, content_value: contentValue)
+    res = yamlPortalContent.dumpToYaml(file)
+  if res.isErr():
+    error "Failed writing content to file", file, error = res.error
+    quit 1
+  else:
+    notice "Successfully wrote content to file", file
+
 proc createAndOpenFile*(dataDir: string, fileName: string): OutputStreamHandle =
   # Creates directory and file, if file already exists
   # program is aborted with info to user, to avoid losing data
@@ -54,3 +66,11 @@ proc createAndOpenFile*(dataDir: string, fileName: string): OutputStreamHandle =
   except IOError as e:
     fatal "Error occurred while opening the file", error = e.msg
     quit 1
+
+proc existsFile*(dataDir: string, fileName: string) =
+  let filePath = dataDir / fileName
+
+  if isFile(filePath):
+    fatal "File under provided path already exists and would be overwritten",
+      path = filePath
+    quit 1