Taiko L2 (#483)

* [docs] fix two client scenario: add missing collateral

* [integration] separate step to wait for node to be started

* [cli] add option to specify ethereum private key

* Remove unused imports

* Fix warnings

* [integration] move type definitions to correct place

* [integration] wait a bit longer for a node to start in debug mode

When e.g. running against Taiko test net rpc, the node start
takes longer

* [integration] simplify handling of codex node and client

* [integration] add Taiko integration test

* [contracts] await token approval confirmation before next tx

* [contracts] deployment address of marketplace on Taiko

* [cli] --eth-private-key now takes a file name

Instead of supplying the private key on the command line,
expect the private key to be in a file with the correct
permissions.

* [utils] Fixes undeclared `activeChroniclesStream` on Windows

* [build] update nim-ethers to include PR #52

Co-authored-by: Eric Mastro <eric.mastro@gmail.com>

* [cli] Better error messages when reading eth private key

Co-authored-by: Eric Mastro <eric.mastro@gmail.com>

* [integration] simplify reading of cmd line arguments

Co-authored-by: Eric Mastro <eric.mastro@gmail.com>

* [build] update to latest version of nim-ethers

* [contracts] updated contract address for Taiko L2

* [build] update codex contracts to latest version

---------

Co-authored-by: Eric Mastro <eric.mastro@gmail.com>

Makefile

@@ -92,11 +92,16 @@ testIntegration: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim testIntegration $(NIM_PARAMS) build.nims
 
-# Builds and runs all tests
+# Builds and runs all tests (except for Taiko L2 tests)
 testAll: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim testAll $(NIM_PARAMS) build.nims
 
+# Builds and runs Taiko L2 tests
+testTaiko: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) codex.nims
+
 # nim-libbacktrace
 LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
 libbacktrace:

build.nims

@@ -40,11 +40,15 @@ task build, "build codex binary":
 task test, "Run tests":
   testCodexTask()
 
-task testAll, "Run all tests":
+task testAll, "Run all tests (except for Taiko L2 tests)":
   testCodexTask()
   testContractsTask()
   testIntegrationTask()
 
+task testTaiko, "Run Taiko L2 tests":
+  codexTask()
+  test "testTaiko"
+
 import strutils
 import os
 

codex.nimble

@@ -12,7 +12,7 @@ requires "bearssl >= 0.1.4"
 requires "chronicles >= 0.7.2"
 requires "chronos >= 2.5.2"
 requires "confutils"
-requires "ethers >= 0.5.0 & < 0.6.0"
+requires "ethers >= 0.7.0 & < 0.8.0"
 requires "libbacktrace"
 requires "libp2p"
 requires "metrics"

codex/codex.nim

@@ -8,6 +8,7 @@
 ## those terms.
 
 import std/sequtils
+import std/strutils
 import std/os
 import std/tables
 
@@ -21,6 +22,7 @@ import pkg/nitro
 import pkg/stew/io2
 import pkg/stew/shims/net as stewnet
 import pkg/datastore
+import pkg/ethers except Rng
 
 import ./node
 import ./conf
@@ -50,6 +52,7 @@ type
     maintenance: BlockMaintainer
 
   CodexPrivateKey* = libp2p.PrivateKey # alias
+  EthWallet = ethers.Wallet
 
 proc bootstrapInteractions(
     config: CodexConf,
@@ -60,11 +63,11 @@ proc bootstrapInteractions(
   ##
 
   if not config.persistence and not config.validator:
-    if config.ethAccount.isSome:
+    if config.ethAccount.isSome or config.ethPrivateKey.isSome:
       warn "Ethereum account was set, but neither persistence nor validator is enabled"
     return
 
-  without account =? config.ethAccount:
+  if not config.ethAccount.isSome and not config.ethPrivateKey.isSome:
     if config.persistence:
       error "Persistence enabled, but no Ethereum account was set"
     if config.validator:
@@ -72,7 +75,23 @@ proc bootstrapInteractions(
     quit QuitFailure
 
   let provider = JsonRpcProvider.new(config.ethProvider)
-  let signer = provider.getSigner(account)
+  var signer: Signer
+  if account =? config.ethAccount:
+    signer = provider.getSigner(account)
+  elif keyFile =? config.ethPrivateKey:
+    without isSecure =? checkSecureFile(keyFile):
+      error "Could not check file permissions: does Ethereum private key file exist?"
+      quit QuitFailure
+    if not isSecure:
+      error "Ethereum private key file does not have safe file permissions"
+      quit QuitFailure
+    without key =? keyFile.readAllChars():
+      error "Unable to read Ethereum private key file"
+      quit QuitFailure
+    without wallet =? EthWallet.new(key.strip(), provider):
+      error "Invalid Ethereum private key in file"
+      quit QuitFailure
+    signer = wallet
 
   let deploy = Deployment.new(provider, config)
   without marketplaceAddress =? await deploy.address(Marketplace):

codex/conf.nim

@@ -228,6 +228,12 @@ type
         name: "eth-account"
       .}: Option[EthAddress]
 
+      ethPrivateKey* {.
+        desc: "File containing Ethereum private key for storage contracts"
+        defaultValue: string.none
+        name: "eth-private-key"
+      .}: Option[string]
+
       marketplaceAddress* {.
         desc: "Address of deployed Marketplace contract"
         defaultValue: EthAddress.none

codex/contracts/deployment.nim

@@ -16,6 +16,10 @@ const knownAddresses = {
  # Hardhat localhost network
  "31337": {
   "Marketplace": Address.init("0x59b670e9fA9D0A427751Af201D676719a970857b")
+ }.toTable,
+ # Taiko Alpha-3 Testnet
+ "167005": {
+  "Marketplace": Address.init("0x948CF9291b77Bd7ad84781b9047129Addf1b894F")
  }.toTable
 }.toTable
 

codex/contracts/market.nim

@@ -6,7 +6,6 @@ import pkg/ethers
 import pkg/ethers/testing
 import pkg/upraises
 import pkg/questionable
-import pkg/chronicles
 import ../market
 import ./marketplace
 
@@ -37,7 +36,7 @@ proc approveFunds(market: OnChainMarket, amount: UInt256) {.async.} =
   let tokenAddress = await market.contract.token()
   let token = Erc20Token.new(tokenAddress, market.signer)
 
-  discard await token.approve(market.contract.address(), amount)
+  discard await token.approve(market.contract.address(), amount).confirm(1)
 
 method getSigner*(market: OnChainMarket): Future[Address] {.async.} =
   return await market.signer.getAddress()

codex/utils/fileutils.nim

@@ -18,6 +18,7 @@ import pkg/chronicles
 import stew/io2
 
 export io2
+export chronicles
 
 when defined(windows):
   import stew/[windows/acl]

docs/TWOCLIENTTEST.md

@@ -139,7 +139,8 @@ curl --location 'http://localhost:8081/api/codex/v1/sales/availability' \
 --data '{
     "size": "1000000",
     "duration": "3600",
-    "minPrice": "1000"
+    "minPrice": "1000",
+    "maxCollateral": "1"
 }'
 ```
 
@@ -154,6 +155,7 @@ curl --location 'http://localhost:8080/api/codex/v1/storage/request/<CID>' \
     "reward": "1024",
     "duration": "120",
     "proofProbability": "8"
+    "collateral": "1"
 }'
 ```
 

tests/ethertest.nim

@@ -2,7 +2,6 @@ import std/json
 import pkg/asynctest
 import pkg/ethers
 
-import ./helpers
 import ./checktest
 
 ## Unit testing suite that sets up an Ethereum testing environment.

tests/helpers/multisetup.nim

@@ -13,10 +13,10 @@ template asyncmultisetup* =
     for teardown in teardowns:
       await teardown()
 
-  template setup(setupBody) {.inject.} =
+  template setup(setupBody) {.inject, used.} =
     setups.add(proc {.async.} = setupBody)
 
-  template teardown(teardownBody) {.inject.} =
+  template teardown(teardownBody) {.inject, used.} =
     teardowns.insert(proc {.async.} = teardownBody)
 
 template multisetup* =
@@ -31,8 +31,8 @@ template multisetup* =
     for teardown in teardowns:
       teardown()
 
-  template setup(setupBody) {.inject.} =
+  template setup(setupBody) {.inject, used.} =
     setups.add(proc = setupBody)
 
-  template teardown(teardownBody) {.inject.} =
+  template teardown(teardownBody) {.inject, used.} =
     teardowns.insert(proc = teardownBody)

tests/integration/multinodes.nim

@@ -11,6 +11,49 @@ export ethertest
 export codexclient
 export nodes
 
+type
+  RunningNode* = ref object
+    role*: Role
+    node*: NodeProcess
+    restClient*: CodexClient
+    datadir*: string
+    ethAccount*: Address
+  StartNodes* = object
+    clients*: uint
+    providers*: uint
+    validators*: uint
+  DebugNodes* = object
+    client*: bool
+    provider*: bool
+    validator*: bool
+    topics*: string
+  Role* {.pure.} = enum
+    Client,
+    Provider,
+    Validator
+
+proc new*(_: type RunningNode,
+          role: Role,
+          node: NodeProcess,
+          restClient: CodexClient,
+          datadir: string,
+          ethAccount: Address): RunningNode =
+  RunningNode(role: role,
+              node: node,
+              restClient: restClient,
+              datadir: datadir,
+              ethAccount: ethAccount)
+
+proc init*(_: type StartNodes,
+          clients, providers, validators: uint): StartNodes =
+  StartNodes(clients: clients, providers: providers, validators: validators)
+
+proc init*(_: type DebugNodes,
+          client, provider, validator: bool,
+          topics: string = "validator,proving,market"): DebugNodes =
+  DebugNodes(client: client, provider: provider, validator: validator,
+             topics: topics)
+
 template multinodesuite*(name: string,
   startNodes: StartNodes, debugNodes: DebugNodes, body: untyped) =
 
@@ -49,6 +92,7 @@ template multinodesuite*(name: string,
         .concat(addlOptions)
       if debug: options.add "--log-level=INFO;TRACE: " & debugNodes.topics
       let node = startNode(options, debug = debug)
+      node.waitUntilStarted()
       (node, datadir, accounts[index])
 
     proc newCodexClient(index: int): CodexClient =
@@ -92,13 +136,13 @@ template multinodesuite*(name: string,
         debug "started new validator node and codex client",
           restApiPort = 8080 + index, discPort = 8090 + index, account
 
-    proc clients(): seq[RunningNode] =
+    proc clients(): seq[RunningNode] {.used.} =
       running.filter(proc(r: RunningNode): bool = r.role == Role.Client)
 
-    proc providers(): seq[RunningNode] =
+    proc providers(): seq[RunningNode] {.used.} =
       running.filter(proc(r: RunningNode): bool = r.role == Role.Provider)
 
-    proc validators(): seq[RunningNode] =
+    proc validators(): seq[RunningNode] {.used.} =
       running.filter(proc(r: RunningNode): bool = r.role == Role.Validator)
 
     setup:

tests/integration/nodes.nim

@@ -1,10 +1,16 @@
+import pkg/questionable
+import pkg/confutils
+import pkg/chronicles
+import pkg/libp2p
 import std/osproc
 import std/os
 import std/streams
 import std/strutils
-import pkg/ethers
+import codex/conf
 import ./codexclient
 
+export codexclient
+
 const workingDir = currentSourcePath() / ".." / ".." / ".."
 const executable = "build" / "codex"
 
@@ -13,47 +19,7 @@ type
     process: Process
     arguments: seq[string]
     debug: bool
-  Role* {.pure.} = enum
+    client: ?CodexClient
-    Client,
-    Provider,
-    Validator
-  RunningNode* = ref object
-    role*: Role
-    node*: NodeProcess
-    restClient*: CodexClient
-    datadir*: string
-    ethAccount*: Address
-  StartNodes* = object
-    clients*: uint
-    providers*: uint
-    validators*: uint
-  DebugNodes* = object
-    client*: bool
-    provider*: bool
-    validator*: bool
-    topics*: string
-
-proc new*(_: type RunningNode,
-          role: Role,
-          node: NodeProcess,
-          restClient: CodexClient,
-          datadir: string,
-          ethAccount: Address): RunningNode =
-  RunningNode(role: role,
-              node: node,
-              restClient: restClient,
-              datadir: datadir,
-              ethAccount: ethAccount)
-
-proc init*(_: type StartNodes,
-          clients, providers, validators: uint): StartNodes =
-  StartNodes(clients: clients, providers: providers, validators: validators)
-
-proc init*(_: type DebugNodes,
-          client, provider, validator: bool,
-          topics: string = "validator,proving,market"): DebugNodes =
-  DebugNodes(client: client, provider: provider, validator: validator,
-             topics: topics)
 
 proc start(node: NodeProcess) =
   if node.debug:
@@ -63,16 +29,26 @@ proc start(node: NodeProcess) =
       node.arguments,
       options={poParentStreams}
     )
-    sleep(1000)
   else:
     node.process = osproc.startProcess(
       executable,
       workingDir,
       node.arguments
     )
-    for line in node.process.outputStream.lines:
+
-      if line.contains("Started codex node"):
+proc waitUntilOutput*(node: NodeProcess, output: string) =
-        break
+  if node.debug:
+    raiseAssert "cannot read node output when in debug mode"
+  for line in node.process.outputStream.lines:
+    if line.contains(output):
+      return
+  raiseAssert "node did not output '" & output & "'"
+
+proc waitUntilStarted*(node: NodeProcess) =
+  if node.debug:
+    sleep(5_000)
+  else:
+    node.waitUntilOutput("Started codex node")
 
 proc startNode*(args: openArray[string], debug: string | bool = false): NodeProcess =
   ## Starts a Codex Node with the specified arguments.
@@ -81,13 +57,36 @@ proc startNode*(args: openArray[string], debug: string | bool = false): NodeProc
   node.start()
   node
 
+proc dataDir(node: NodeProcess): string =
+  let config = CodexConf.load(cmdLine = node.arguments)
+  config.dataDir.string
+
+proc apiUrl(node: NodeProcess): string =
+  let config = CodexConf.load(cmdLine = node.arguments)
+  "http://" & config.apiBindAddress & ":" & $config.apiPort & "/api/codex/v1"
+
+proc client*(node: NodeProcess): CodexClient =
+  if client =? node.client:
+    return client
+  let client = CodexClient.new(node.apiUrl)
+  node.client = some client
+  client
+
 proc stop*(node: NodeProcess) =
   if node.process != nil:
     node.process.terminate()
     discard node.process.waitForExit(timeout=5_000)
     node.process.close()
     node.process = nil
+  if client =? node.client:
+    node.client = none CodexClient
+    client.close()
 
 proc restart*(node: NodeProcess) =
   node.stop()
   node.start()
+  node.waitUntilStarted()
+
+proc removeDataDir*(node: NodeProcess) =
+  if dataDir =? node.dataDir:
+    removeDir(dataDir)

tests/integration/testblockexpiration.nim

@@ -32,6 +32,7 @@ ethersuite "Node block expiration tests":
       "--block-mi=1",
       "--block-mn=10"
     ], debug = false)
+    node.waitUntilStarted()
 
   proc uploadTestFile(): string =
     let client = newHttpClient()

tests/integration/testcli.nim

@@ -0,0 +1,40 @@
+import std/unittest
+import std/tempfiles
+import codex/utils/fileutils
+import ./nodes
+
+suite "Command line interface":
+
+  let account = "4242424242424242424242424242424242424242"
+  let key = "4242424242424242424242424242424242424242424242424242424242424242"
+
+  test "complains when persistence is enabled without ethereum account":
+    let node = startNode(@["--persistence"])
+    node.waitUntilOutput("Persistence enabled, but no Ethereum account was set")
+    node.stop()
+
+  test "complains when validator is enabled without ethereum account":
+    let node = startNode(@["--validator"])
+    node.waitUntilOutput("Validator enabled, but no Ethereum account was set")
+    node.stop()
+
+  test "complains when ethereum account is set when not needed":
+    let node = startNode(@["--eth-account=" & account])
+    node.waitUntilOutput("Ethereum account was set, but neither persistence nor validator is enabled")
+    node.stop()
+
+  test "complains when ethereum private key is set when not needed":
+    let keyFile = genTempPath("", "")
+    discard secureWriteFile(keyFile, key)
+    let node = startNode(@["--eth-private-key=" & keyFile])
+    node.waitUntilOutput("Ethereum account was set, but neither persistence nor validator is enabled")
+    node.stop()
+    discard removeFile(keyFile)
+
+  test "complains when ethereum private key file has wrong permissions":
+    let unsafeKeyFile = genTempPath("", "")
+    discard unsafeKeyFile.writeFile(key, 0o666)
+    let node = startNode(@["--persistence", "--eth-private-key=" & unsafeKeyFile])
+    node.waitUntilOutput("Ethereum private key file does not have safe file permissions")
+    node.stop()
+    discard removeFile(unsafeKeyFile)

tests/integration/testproofs.nim

@@ -58,13 +58,17 @@ twonodessuite "Proving integration test", debug1=false, debug2=false:
     await provider.advanceTimeTo(endOfPeriod + 1)
 
   proc startValidator: NodeProcess =
-    startNode([
+    let validator = startNode(
-      "--data-dir=" & validatorDir,
+      [
-      "--api-port=8089",
+        "--data-dir=" & validatorDir,
-      "--disc-port=8099",
+        "--api-port=8089",
-      "--validator",
+        "--disc-port=8099",
-      "--eth-account=" & $accounts[2]
+        "--validator",
-    ], debug = false)
+        "--eth-account=" & $accounts[2]
+      ], debug = false
+    )
+    validator.waitUntilStarted()
+    validator
 
   proc stopValidator(node: NodeProcess) =
     node.stop()

tests/integration/twonodes.nim

@@ -46,6 +46,7 @@ template twonodessuite*(name: string, debug1, debug2: string, body) =
         node1Args.add("--log-level=" & debug1)
 
       node1 = startNode(node1Args, debug = debug1)
+      node1.waitUntilStarted()
 
       let bootstrap = client1.info()["spr"].getStr()
 
@@ -64,6 +65,7 @@ template twonodessuite*(name: string, debug1, debug2: string, body) =
         node2Args.add("--log-level=" & debug2)
 
       node2 = startNode(node2Args, debug = debug2)
+      node2.waitUntilStarted()
 
     teardown:
       client1.close()

tests/testIntegration.nim

@@ -1,3 +1,4 @@
+import ./integration/testcli
 import ./integration/testIntegration
 import ./integration/testblockexpiration
 import ./integration/testproofs

tests/testTaiko.nim

@@ -0,0 +1,74 @@
+import std/times
+import std/os
+import std/json
+import std/tempfiles
+import pkg/asynctest
+import pkg/chronos
+import pkg/stint
+import pkg/questionable
+import pkg/questionable/results
+import ./integration/nodes
+
+suite "Taiko L2 Integration Tests":
+
+  var node1, node2: NodeProcess
+
+  setup:
+    doAssert existsEnv("CODEX_ETH_PRIVATE_KEY"), "Key for Taiko account missing"
+
+    node1 = startNode([
+      "--data-dir=" & createTempDir("", ""),
+      "--api-port=8080",
+      "--nat=127.0.0.1",
+      "--disc-ip=127.0.0.1",
+      "--disc-port=8090",
+      "--persistence",
+      "--eth-provider=https://rpc.test.taiko.xyz"
+    ])
+    node1.waitUntilStarted()
+
+    let bootstrap = node1.client.info()["spr"].getStr()
+
+    node2 = startNode([
+      "--data-dir=" & createTempDir("", ""),
+      "--api-port=8081",
+      "--nat=127.0.0.1",
+      "--disc-ip=127.0.0.1",
+      "--disc-port=8091",
+      "--bootstrap-node=" & bootstrap,
+      "--persistence",
+      "--eth-provider=https://rpc.test.taiko.xyz"
+    ])
+    node2.waitUntilStarted()
+
+  teardown:
+    node1.stop()
+    node2.stop()
+    node1.removeDataDir()
+    node2.removeDataDir()
+
+  test "node 1 buys storage from node 2":
+    discard node2.client.postAvailability(
+      size=0xFFFFF.u256,
+      duration=200.u256,
+      minPrice=300.u256,
+      maxCollateral=300.u256
+    )
+    let cid = !node1.client.upload("some file contents")
+
+    echo "    - requesting storage, expires in 5 minutes"
+    let expiry = getTime().toUnix().uint64 + 5 * 60
+    let purchase = !node1.client.requestStorage(
+      cid,
+      duration=30.u256,
+      reward=400.u256,
+      proofProbability=3.u256,
+      collateral=200.u256,
+      expiry=expiry.u256
+    )
+
+    echo "    - waiting for request to start, timeout 5 minutes"
+    check eventually(node1.client.getPurchase(purchase).?state == success "started", timeout = 5 * 60 * 1000)
+
+    echo "    - waiting for request to finish, timeout 1 minute"
+    check eventually(node1.client.getPurchase(purchase).?state == success "finished", timeout = 1 * 60 * 1000)

vendor/codex-contracts-eth

@@ -1 +1 @@
-Subproject commit 230e7276e271ce53bce36fffdbb25a50621c33b9
+Subproject commit 1854dfba9991a25532de5f6a53cf50e66afb3c8b

vendor/nim-ethers

@@ -1 +1 @@
-Subproject commit 9f4f762e21b433aa31549964d723f47d45da7990
+Subproject commit 8fff63102a3461ddec61714df80840740eaade1f