feat: integrate dagger contracts

Integrate dagger contracts from `nim-dagger-contracts` repo.

Add `dagger-contracts`, `nim-web3`, and all of `nim-web3`’s transitive deps as submodule deps to `nim-dagger`. Note: `nim-web3` and its transitive deps may no longer be needed when we switch to `nim-ethers`.

Add a `testContracts` nimble task to test all of the contracts functionality. Namely, this spins up an ethereum simulator, deploys the contracts (in `dagger-contracts`), runs the contract tests, and finally, regardless of success/error, kills the ethereum sim processes. The nimble task can be run with `./env.sh nimble testContracts`.

We also tested `nim-dagger-contracts` as a submodule dep of `nim-dagger`, and while the tests run as expected, the preference is to merge `nim-dagger-contracts` inside of `nim-dagger` for ease of parallel development. There’s also a high probability that `nim-dagger-contracts` is not being used as a dep by other projects. Are there any strong objections to this?

Co-authored-by: Michael Bradley <michaelsbradleyjr@gmail.com>

.gitignore

@@ -21,3 +21,4 @@ build/
 
 .update.timestamp
 dagger.nims
+deployment-localhost.json

.gitmodules

@@ -54,10 +54,10 @@
 	ignore = untracked
 	branch = master
 [submodule "vendor/upraises"]
-	ignore = untracked
-	branch = master
 	path = vendor/upraises
 	url = https://github.com/markspanbroek/upraises.git
+	ignore = untracked
+	branch = master
 [submodule "vendor/asynctest"]
 	path = vendor/asynctest
 	url = https://github.com/status-im/asynctest.git
@@ -118,9 +118,6 @@
 	url = https://github.com/status-im/stint.git
 	ignore = untracked
 	branch = master
-[submodule "vendor/nim-httputils"]
-	ignore = untracked
-	branch = master
 [submodule "vendor/nim-http-utils"]
 	path = vendor/nim-http-utils
 	url = https://github.com/status-im/nim-http-utils.git
@@ -131,20 +128,11 @@
 	url = https://github.com/status-im/nim-toml-serialization.git
 	ignore = untracked
 	branch = master
-[submodule "vendor/unittest2"]
-	ignore = untracked
-	branch = master
 [submodule "vendor/nim-unittest2"]
 	path = vendor/nim-unittest2
 	url = https://github.com/status-im/nim-unittest2.git
 	ignore = untracked
 	branch = master
-[submodule "vendor/nameresolver"]
-	ignore = untracked
-	branch = master
-[submodule "vendor/nim-nameresolver"]
-	ignore = untracked
-	branch = master
 [submodule "vendor/dnsclient.nim"]
 	path = vendor/dnsclient.nim
 	url = https://github.com/ba0f3/dnsclient.nim.git
@@ -155,3 +143,19 @@
 	url = https://github.com/status-im/nim-websock.git
 	ignore = untracked
 	branch = master
+[submodule "vendor/dagger-contracts"]
+	path = vendor/dagger-contracts
+	url = https://github.com/status-im/dagger-contracts
+	ignore = dirty
+[submodule "vendor/nim-contract-abi"]
+	path = vendor/nim-contract-abi
+	url = https://github.com/status-im/nim-contract-abi
+[submodule "vendor/nim-json-rpc"]
+	path = vendor/nim-json-rpc
+	url = https://github.com/status-im/nim-json-rpc
+[submodule "vendor/nim-zlib"]
+	path = vendor/nim-zlib
+	url = https://github.com/status-im/nim-zlib
+[submodule "vendor/nim-ethers"]
+	path = vendor/nim-ethers
+	url = https://github.com/status-im/nim-ethers

dagger.nimble

@@ -29,9 +29,27 @@ proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
     extra_params &= " " & paramStr(i)
   exec "nim " & lang & " --out:build/" & name & " " & extra_params & " " & srcDir & name & ".nim"
 
-proc test(name: string, params = "-d:chronicles_log_level=DEBUG", lang = "c") =
-  buildBinary name, "tests/", params
+proc test(name: string, srcDir = "tests/", params = "-d:chronicles_log_level=DEBUG", lang = "c") =
+  buildBinary name, srcDir, params
   exec "build/" & name
 
+task testContracts, "Build, deploy and test contracts":
+  exec "cd vendor/dagger-contracts && npm install"
+
+  # start node
+  # Note: combining this command with the previous does not work
+  exec "cd vendor/dagger-contracts && npx hardhat node --no-deploy &"
+
+  # deploy contracts
+  exec "sleep 3 && " &
+       "cd vendor/dagger-contracts && npx hardhat deploy --network localhost --export '../../deployment-localhost.json'"
+
+  # run contract tests using deployed contracts
+  try:
+    test "testContracts", "tests/", "-d:chronicles_log_level=WARN"
+  finally:
+    # kill simulator processes
+    exec "ps -ef | grep hardhat | grep -v grep | awk '{ print $2 }' | xargs kill"
+
 task testAll, "Build & run Dagger tests":
-  test "testAll", "-d:chronicles_log_level=WARN"
+  test "testAll", params = "-d:chronicles_log_level=WARN"

dagger/contracts.nim

@@ -0,0 +1,7 @@
+import contracts/marketplace
+import contracts/storage
+import contracts/deployment
+
+export marketplace
+export storage
+export deployment

dagger/contracts/deployment.nim

@@ -0,0 +1,19 @@
+import std/json
+import pkg/ethers
+import pkg/questionable
+
+type Deployment* = object
+  json: JsonNode
+
+const defaultFile = "./deployment-localhost.json"
+
+## Reads deployment information from a json file. It expects a file that has
+## been exported with Hardhat deploy.
+## See also:
+## https://github.com/wighawag/hardhat-deploy/tree/master#6-hardhat-export
+proc deployment*(file = defaultFile): Deployment =
+  Deployment(json: parseFile(file))
+
+proc address*(deployment: Deployment, Contract: typedesc): ?Address =
+  let address = deployment.json["contracts"][$Contract]["address"].getStr()
+  Address.init(address)

dagger/contracts/marketplace.nim

@@ -0,0 +1,42 @@
+import pkg/stint
+import pkg/contractabi except Address
+import pkg/nimcrypto
+import pkg/chronos
+
+export stint
+
+type
+  StorageRequest* = object
+    duration*: UInt256
+    size*: UInt256
+    contentHash*: Hash
+    proofPeriod*: UInt256
+    proofTimeout*: UInt256
+    nonce*: array[32, byte]
+  StorageBid* = object
+    requestHash*: Hash
+    bidExpiry*: UInt256
+    price*: UInt256
+  Hash = array[32, byte]
+  Signature = array[65, byte]
+
+func hashRequest*(request: StorageRequest): Hash =
+  let encoding = AbiEncoder.encode: (
+    "[dagger.request.v1]",
+    request.duration,
+    request.size,
+    request.contentHash,
+    request.proofPeriod,
+    request.proofTimeout,
+    request.nonce
+  )
+  keccak256.digest(encoding).data
+
+func hashBid*(bid: StorageBid): Hash =
+  let encoding = AbiEncoder.encode: (
+    "[dagger.bid.v1]",
+    bid.requestHash,
+    bid.bidExpiry,
+    bid.price
+  )
+  keccak256.digest(encoding).data

dagger/contracts/storage.nim

@@ -0,0 +1,70 @@
+import pkg/ethers
+import pkg/json_rpc/rpcclient
+import pkg/stint
+import pkg/chronos
+import ./marketplace
+
+export stint
+export contract
+
+type
+  Storage* = ref object of Contract
+  Id = array[32, byte]
+
+proc stakeAmount*(storage: Storage): UInt256 {.contract, view.}
+proc increaseStake*(storage: Storage, amount: UInt256) {.contract.}
+proc withdrawStake*(storage: Storage) {.contract.}
+proc stake*(storage: Storage, account: Address): UInt256 {.contract, view.}
+proc duration*(storage: Storage, id: Id): UInt256 {.contract, view.}
+proc size*(storage: Storage, id: Id): UInt256 {.contract, view.}
+proc contentHash*(storage: Storage, id: Id): array[32, byte] {.contract, view.}
+proc proofPeriod*(storage: Storage, id: Id): UInt256 {.contract, view.}
+proc proofTimeout*(storage: Storage, id: Id): UInt256 {.contract, view.}
+proc price*(storage: Storage, id: Id): UInt256 {.contract, view.}
+proc host*(storage: Storage, id: Id): Address {.contract, view.}
+proc startContract*(storage: Storage, id: Id) {.contract.}
+proc proofEnd*(storage: Storage, id: Id): UInt256 {.contract, view.}
+proc isProofRequired*(storage: Storage,
+                      id: Id,
+                      blocknumber: UInt256): bool {.contract, view.}
+proc submitProof*(storage: Storage,
+                  id: Id,
+                  blocknumber: UInt256,
+                  proof: bool) {.contract.}
+proc markProofAsMissing*(storage: Storage,
+                          id: Id,
+                          blocknumber: UInt256) {.contract.}
+proc finishContract*(storage: Storage, id: Id) {.contract.}
+
+proc newContract(storage: Storage,
+                 duration: UInt256,
+                 size: UInt256,
+                 contentHash: array[32, byte],
+                 proofPeriod: UInt256,
+                 proofTimeout: UInt256,
+                 nonce: array[32, byte],
+                 price: UInt256,
+                 host: Address,
+                 bidExpiry: UInt256,
+                 requestSignature: seq[byte],
+                 bidSignature: seq[byte]) {.contract.}
+
+proc newContract*(storage: Storage,
+                  request: StorageRequest,
+                  bid: StorageBid,
+                  host: Address,
+                  requestSignature: seq[byte],
+                  bidSignature: seq[byte]) {.async.} =
+  await storage.newContract(
+    request.duration,
+    request.size,
+    request.contentHash,
+    request.proofPeriod,
+    request.proofTimeout,
+    request.nonce,
+    bid.price,
+    host,
+    bid.bidExpiry,
+    requestSignature,
+    bidSignature
+  )

dagger/contracts/testtoken.nim

@@ -0,0 +1,10 @@
+import pkg/chronos
+import pkg/stint
+import pkg/ethers
+
+type
+  TestToken* = ref object of Contract
+
+proc mint*(token: TestToken, holder: Address, amount: UInt256) {.contract.}
+proc approve*(token: TestToken, spender: Address, amount: UInt256) {.contract.}
+proc balanceOf*(token: TestToken, account: Address): UInt256 {.contract, view.}

tests/contracts/ethertest.nim

@@ -0,0 +1,49 @@
+import std/json
+import pkg/asynctest
+import pkg/ethers
+
+# Allow multiple setups and teardowns in a test suite
+template multisetup =
+
+  var setups: seq[proc: Future[void] {.gcsafe.}]
+  var teardowns: seq[proc: Future[void] {.gcsafe.}]
+
+  setup:
+    for setup in setups:
+      await setup()
+
+  teardown:
+    for teardown in teardowns:
+      await teardown()
+
+  template setup(setupBody) {.inject.} =
+    setups.add(proc {.async.} = setupBody)
+
+  template teardown(teardownBody) {.inject.} =
+    teardowns.insert(proc {.async.} = teardownBody)
+
+## Unit testing suite that sets up an Ethereum testing environment.
+## Injects a `provider` instance, and a list of `accounts`.
+## Calls the `evm_snapshot` and `evm_revert` methods to ensure that any
+## changes to the blockchain do not persist.
+template ethersuite*(name, body) =
+  suite name:
+
+    var provider {.inject, used.}: JsonRpcProvider
+    var accounts {.inject, used.}: seq[Address]
+    var snapshot: JsonNode
+
+    multisetup()
+
+    setup:
+      provider = JsonRpcProvider.new("ws://localhost:8545")
+      snapshot = await send(provider, "evm_snapshot")
+      accounts = await provider.listAccounts()
+
+    teardown:
+      discard await send(provider, "evm_revert", @[snapshot])
+
+    body
+
+export asynctest
+export ethers

tests/contracts/examples.nim

@@ -0,0 +1,29 @@
+import std/times
+import pkg/stint
+import pkg/nimcrypto
+import dagger/contracts/marketplace
+
+proc randomBytes(amount: static int): array[amount, byte] =
+  doAssert randomBytes(result) == amount
+
+proc example*(_: type StorageRequest): StorageRequest =
+  StorageRequest(
+    duration: 150.u256, # 150 blocks ≈ half an hour
+    size: (1 * 1024 * 1024 * 1024).u256, # 1 Gigabyte
+    contentHash: sha256.digest(0xdeadbeef'u32.toBytes).data,
+    proofPeriod: 8.u256, # 8 blocks ≈ 2 minutes
+    proofTimeout: 4.u256, # 4 blocks ≈ 1 minute
+    nonce: randomBytes(32)
+  )
+
+proc example*(_: type StorageBid): StorageBid =
+  StorageBid(
+    requestHash: hashRequest(StorageRequest.example),
+    bidExpiry: (getTime() + initDuration(hours=1)).toUnix.u256,
+    price: 42.u256
+  )
+
+proc example*(_: type (StorageRequest, StorageBid)): (StorageRequest, StorageBid) =
+  result[0] = StorageRequest.example
+  result[1] = StorageBid.example
+  result[1].requestHash = hashRequest(result[0])

tests/contracts/nim.cfg

@@ -0,0 +1 @@
+--path:"../.."

tests/contracts/testContracts.nim

@@ -0,0 +1,98 @@
+import pkg/chronos
+import pkg/nimcrypto
+import dagger/contracts
+import dagger/contracts/testtoken
+import ./ethertest
+import ./examples
+
+ethersuite "Storage contracts":
+
+  let (request, bid) = (StorageRequest, StorageBid).example
+
+  var client, host: Signer
+  var storage: Storage
+  var token: TestToken
+  var stakeAmount: UInt256
+
+  setup:
+    let deployment = deployment()
+    client = provider.getSigner(accounts[0])
+    host = provider.getSigner(accounts[1])
+    storage = Storage.new(!deployment.address(Storage), provider.getSigner())
+    token = TestToken.new(!deployment.address(TestToken), provider.getSigner())
+    await token.connect(client).mint(await client.getAddress(), 1000.u256)
+    await token.connect(host).mint(await host.getAddress(), 1000.u256)
+    stakeAmount = await storage.stakeAmount()
+
+  proc newContract(): Future[array[32, byte]] {.async.} =
+    await token.connect(host).approve(Address(storage.address), stakeAmount)
+    await storage.connect(host).increaseStake(stakeAmount)
+    await token.connect(client).approve(Address(storage.address), bid.price)
+    let requestHash = hashRequest(request)
+    let bidHash = hashBid(bid)
+    let requestSignature = await client.signMessage(@requestHash)
+    let bidSignature = await host.signMessage(@bidHash)
+    await storage.connect(client).newContract(
+      request,
+      bid,
+      await host.getAddress(),
+      requestSignature,
+      bidSignature
+    )
+    let id = bidHash
+    return id
+
+  proc mineUntilProofRequired(id: array[32, byte]): Future[UInt256] {.async.} =
+    var blocknumber: UInt256
+    var done = false
+    while not done:
+      blocknumber = await provider.getBlockNumber()
+      done = await storage.isProofRequired(id, blocknumber)
+      if not done:
+        discard await provider.send("evm_mine")
+    return blocknumber
+
+  proc mineUntilProofTimeout(id: array[32, byte]) {.async.} =
+    let timeout = await storage.proofTimeout(id)
+    for _ in 0..<timeout.truncate(int):
+      discard await provider.send("evm_mine")
+
+  proc mineUntilEnd(id: array[32, byte]) {.async.} =
+    let proofEnd = await storage.proofEnd(id)
+    while (await provider.getBlockNumber()) < proofEnd:
+      discard await provider.send("evm_mine")
+
+  test "can be created":
+    let id = await newContract()
+    check (await storage.duration(id)) == request.duration
+    check (await storage.size(id)) == request.size
+    check (await storage.contentHash(id)) == request.contentHash
+    check (await storage.proofPeriod(id)) == request.proofPeriod
+    check (await storage.proofTimeout(id)) == request.proofTimeout
+    check (await storage.price(id)) == bid.price
+    check (await storage.host(id)) == (await host.getAddress())
+
+  test "can be started by the host":
+    let id = await newContract()
+    await storage.connect(host).startContract(id)
+    let proofEnd = await storage.proofEnd(id)
+    check proofEnd > 0
+
+  test "accept storage proofs":
+    let id = await newContract()
+    await storage.connect(host).startContract(id)
+    let blocknumber = await mineUntilProofRequired(id)
+    await storage.connect(host).submitProof(id, blocknumber, true)
+
+  test "marks missing proofs":
+    let id = await newContract()
+    await storage.connect(host).startContract(id)
+    let blocknumber = await mineUntilProofRequired(id)
+    await mineUntilProofTimeout(id)
+    await storage.connect(client).markProofAsMissing(id, blocknumber)
+
+  test "can be finished":
+    let id = await newContract()
+    await storage.connect(host).startContract(id)
+    await mineUntilEnd(id)
+    await storage.connect(host).finishContract(id)

tests/contracts/testMarketplace.nim

@@ -0,0 +1,42 @@
+import pkg/asynctest
+import pkg/chronos
+import pkg/nimcrypto
+import pkg/contractabi
+import dagger/contracts
+import ./ethertest
+import ./examples
+
+suite "Marketplace":
+
+  test "hashes requests for storage":
+    let request = StorageRequest.example
+    let encoding = AbiEncoder.encode: (
+      "[dagger.request.v1]",
+      request.duration,
+      request.size,
+      request.contentHash,
+      request.proofPeriod,
+      request.proofTimeout,
+      request.nonce
+    )
+    let expectedHash = keccak256.digest(encoding).data
+    check hashRequest(request) == expectedHash
+
+  test "hashes bids":
+    let bid = StorageBid.example
+    let encoding = AbiEncoder.encode: (
+      "[dagger.bid.v1]",
+      bid.requestHash,
+      bid.bidExpiry,
+      bid.price
+    )
+    let expectedHash = keccak256.digest(encoding).data
+    check hashBid(bid) == expectedHash
+
+ethersuite "Marketplace signatures":
+
+  test "signs request and bid hashes":
+    let hash = hashRequest(StorageRequest.example)
+    let signer = provider.getSigner()
+    let signature = await signer.signMessage(@hash)
+    check signature.len == 65

tests/contracts/testStaking.nim

@@ -0,0 +1,32 @@
+import pkg/chronos
+import pkg/stint
+import dagger/contracts
+import dagger/contracts/testtoken
+import ./ethertest
+
+ethersuite "Staking":
+
+  let stakeAmount = 100.u256
+
+  var storage: Storage
+  var token: TestToken
+
+  setup:
+    let deployment = deployment()
+    storage = Storage.new(!deployment.address(Storage), provider.getSigner())
+    token = TestToken.new(!deployment.address(TestToken), provider.getSigner())
+    await token.mint(accounts[0], 1000.u256)
+
+  test "increases stake":
+    await token.approve(storage.address, stakeAmount)
+    await storage.increaseStake(stakeAmount)
+    let stake = await storage.stake(accounts[0])
+    check stake == stakeAmount
+
+  test "withdraws stake":
+    await token.approve(storage.address, stakeAmount)
+    await storage.increaseStake(stakeAmount)
+    let balanceBefore = await token.balanceOf(accounts[0])
+    await storage.withdrawStake()
+    let balanceAfter = await token.balanceOf(accounts[0])
+    check (balanceAfter - balanceBefore) == stakeAmount

tests/testContracts.nim

@@ -0,0 +1,5 @@
+import ./contracts/testMarketplace
+import ./contracts/testStaking
+import ./contracts/testContracts
+
+{.warning[UnusedImport]:off.}

vendor/dagger-contracts

@@ -0,0 +1 @@
+Subproject commit 41fd33ac7aa0d09075ac0d4d56f82f0888f639be

vendor/nim-contract-abi

@@ -0,0 +1 @@
+Subproject commit a2b8daa320b9ea0d1cfdf7b6442b2316486fda67

vendor/nim-ethers

@@ -0,0 +1 @@
+Subproject commit 27d6e8967268059f6071c56d4449b775ae1f0505

vendor/nim-json-rpc

@@ -0,0 +1 @@
+Subproject commit 5a281760803907f4989cacf109b516381dfbbe11

vendor/nim-zlib

@@ -0,0 +1 @@
+Subproject commit 74cdeb54b21bededb5a515d36f608bc1850555a2