diff --git a/fluffy/eth_data/yaml_utils.nim b/fluffy/eth_data/yaml_utils.nim index 78b2949ab..775e9ff8e 100644 --- a/fluffy/eth_data/yaml_utils.nim +++ b/fluffy/eth_data/yaml_utils.nim @@ -62,8 +62,9 @@ proc dumpToYaml*[T](value: T, file: string): Result[void, string] = except Exception as e: raiseAssert(e.msg) try: - # Dump to yaml, avoiding TAGS and YAML version directives. - dump(value, s, tagStyle = tsNone, options = options, handles = @[]) + {.gcsafe.}: + # Dump to yaml, avoiding TAGS and YAML version directives. + dump(value, s, tagStyle = tsNone, options = options, handles = @[]) except YamlPresenterJsonError as e: return err(e.msg) except YamlSerializationError as e: diff --git a/fluffy/tools/eth_data_exporter/cl_data_exporter.nim b/fluffy/tools/eth_data_exporter/cl_data_exporter.nim index 048ae96bc..d0dd20b8d 100644 --- a/fluffy/tools/eth_data_exporter/cl_data_exporter.nim +++ b/fluffy/tools/eth_data_exporter/cl_data_exporter.nim @@ -31,7 +31,7 @@ from beacon_chain/el/el_manager import toBeaconBlockHeader export beacon_clock const - largeRequestsTimeout = 60.seconds # Downloading large items such as states. + largeRequestsTimeout = 120.seconds # For downloading large items such as states. restRequestsTimeout = 30.seconds proc getBeaconData*(): (RuntimeConfig, ref ForkDigests, BeaconClock) = @@ -66,15 +66,8 @@ proc exportLCBootstrapUpdate*( cfg: RuntimeConfig, forkDigests: ref ForkDigests, ) {.async.} = - let file = "light-client-bootstrap.json" - let fh = createAndOpenFile(dataDir, file) - - defer: - try: - fh.close() - except IOError as e: - fatal "Error occured while closing file", error = e.msg - quit 1 + let fileName = "light-client-bootstrap-" & $trustedBlockRoot.data.toHex() & ".yaml" + existsFile(dataDir, fileName) let client = RestClientRef.new(restUrl).valueOr: error "Cannot connect to server", error = error @@ -101,14 +94,8 @@ proc exportLCBootstrapUpdate*( forkDigest = forkDigestAtEpoch(forkDigests[], epoch(slot), cfg) content = encodeBootstrapForked(forkDigest, update) - let portalContent = JsonPortalContent( - content_key: contentKey.asSeq().to0xHex(), content_value: content.to0xHex() - ) - - var contentTable: JsonPortalContentTable - contentTable[$slot] = portalContent - - writePortalContentToJson(fh, contentTable) + file = dataDir / fileName + writePortalContentToYaml(file, contentKey.asSeq().to0xHex(), content.to0xHex()) proc exportLCUpdates*( restUrl: string, @@ -118,15 +105,8 @@ proc exportLCUpdates*( cfg: RuntimeConfig, forkDigests: ref ForkDigests, ) {.async.} = - let file = "light-client-updates.json" - let fh = createAndOpenFile(dataDir, file) - - defer: - try: - fh.close() - except IOError as e: - fatal "Error occured while closing file", error = e.msg - quit 1 + let fileName = "light-client-updates-" & $startPeriod & "-" & $count & ".yaml" + existsFile(dataDir, fileName) let client = RestClientRef.new(restUrl).valueOr: error "Cannot connect to server", error = error @@ -160,14 +140,8 @@ proc exportLCUpdates*( content = encodeLightClientUpdatesForked(forkDigest, updates) - let portalContent = JsonPortalContent( - content_key: contentKey.asSeq().to0xHex(), content_value: content.to0xHex() - ) - - var contentTable: JsonPortalContentTable - contentTable[$slot] = portalContent - - writePortalContentToJson(fh, contentTable) + file = dataDir / fileName + writePortalContentToYaml(file, contentKey.asSeq().to0xHex(), content.to0xHex()) else: error "No updates downloaded" quit 1 @@ -175,15 +149,8 @@ proc exportLCUpdates*( proc exportLCFinalityUpdate*( restUrl: string, dataDir: string, cfg: RuntimeConfig, forkDigests: ref ForkDigests ) {.async.} = - let file = "light-client-finality-update.json" - let fh = createAndOpenFile(dataDir, file) - - defer: - try: - fh.close() - except IOError as e: - fatal "Error occured while closing file", error = e.msg - quit 1 + let fileName = "light-client-finality-update.yaml" + existsFile(dataDir, fileName) let client = RestClientRef.new(restUrl).valueOr: error "Cannot connect to server", error = error @@ -211,27 +178,14 @@ proc exportLCFinalityUpdate*( ) content = encodeFinalityUpdateForked(forkDigest, update) - let portalContent = JsonPortalContent( - content_key: contentKey.asSeq().to0xHex(), content_value: content.to0xHex() - ) - - var contentTable: JsonPortalContentTable - contentTable[$finalizedSlot] = portalContent - - writePortalContentToJson(fh, contentTable) + file = dataDir / fileName + writePortalContentToYaml(file, contentKey.asSeq().to0xHex(), content.to0xHex()) proc exportLCOptimisticUpdate*( restUrl: string, dataDir: string, cfg: RuntimeConfig, forkDigests: ref ForkDigests ) {.async.} = - let file = "light-client-optimistic-update.json" - let fh = createAndOpenFile(dataDir, file) - - defer: - try: - fh.close() - except IOError as e: - fatal "Error occured while closing file", error = e.msg - quit 1 + let fileName = "light-client-optimistic-update.yaml" + existsFile(dataDir, fileName) let client = RestClientRef.new(restUrl).valueOr: error "Cannot connect to server", error = error @@ -259,14 +213,8 @@ proc exportLCOptimisticUpdate*( ) content = encodeOptimisticUpdateForked(forkDigest, update) - let portalContent = JsonPortalContent( - content_key: contentKey.asSeq().to0xHex(), content_value: content.to0xHex() - ) - - var contentTable: JsonPortalContentTable - contentTable[$slot] = portalContent - - writePortalContentToJson(fh, contentTable) + file = dataDir / fileName + writePortalContentToYaml(file, contentKey.asSeq().to0xHex(), content.to0xHex()) proc exportHistoricalRoots*( restUrl: string, dataDir: string, cfg: RuntimeConfig, forkDigests: ref ForkDigests diff --git a/fluffy/tools/eth_data_exporter/exporter_common.nim b/fluffy/tools/eth_data_exporter/exporter_common.nim index 631dee32f..5a7c73457 100644 --- a/fluffy/tools/eth_data_exporter/exporter_common.nim +++ b/fluffy/tools/eth_data_exporter/exporter_common.nim @@ -14,7 +14,8 @@ import faststreams, json_serialization, json_serialization/std/tables, - ../../eth_data/history_data_json_store + ../../eth_data/history_data_json_store, + ../../eth_data/yaml_utils export history_data_json_store @@ -28,6 +29,17 @@ proc writePortalContentToJson*( fatal "Error occured while writing to file", error = e.msg quit 1 +proc writePortalContentToYaml*(file: string, contentKey: string, contentValue: string) = + let + yamlPortalContent = + YamlPortalContent(content_key: contentKey, content_value: contentValue) + res = yamlPortalContent.dumpToYaml(file) + if res.isErr(): + error "Failed writing content to file", file, error = res.error + quit 1 + else: + notice "Successfully wrote content to file", file + proc createAndOpenFile*(dataDir: string, fileName: string): OutputStreamHandle = # Creates directory and file, if file already exists # program is aborted with info to user, to avoid losing data @@ -54,3 +66,11 @@ proc createAndOpenFile*(dataDir: string, fileName: string): OutputStreamHandle = except IOError as e: fatal "Error occurred while opening the file", error = e.msg quit 1 + +proc existsFile*(dataDir: string, fileName: string) = + let filePath = dataDir / fileName + + if isFile(filePath): + fatal "File under provided path already exists and would be overwritten", + path = filePath + quit 1