feat: Add validator slot id groups

Related to nim-codex/457, nim-codex/458.

To cover the entire SlotId (uint256) address space, each validator must validate a portion of the SlotId space. When a slot is filled, the SlotId will be put in to a bucket, based on the value of the SlotId and the number of buckets (validators) configured. Similar to `myRequests` and `mySlots`, a function called `validationSlots` can be used to retrieve the `SlotIds` being validated for a particular bucket (validator index). This facilitates loading actively filled slots in need of validation when a validator starts.

contracts/Configuration.sol

@@ -6,6 +6,7 @@ import "@openzeppelin/contracts/token/ERC20/IERC20.sol";
 struct MarketplaceConfig {
   CollateralConfig collateral;
   ProofConfig proofs;
+  ValidationConfig validation;
 }
 
 struct CollateralConfig {
@@ -25,3 +26,12 @@ struct ProofConfig {
   uint8 downtime; // ignore this much recent blocks for proof requirements
   string zkeyHash; // hash of the zkey file which is linked to the verifier
 }
+
+struct ValidationConfig {
+  // Number of validators to cover the entire SlotId space, max 65,535 (2^16-1).
+  // IMPORTANT: This value should be a power of 2 for even distribution,
+  // otherwise, the last validator will have a significantly less number of
+  // SlotIds to validate. The closest power of 2 without overflow is 2^15 =
+  // 32,768, giving each validator a maximum of 3.534e72 slots to validate.
+  uint16 validators;
+}

contracts/FuzzMarketplace.sol

@@ -10,7 +10,8 @@ contract FuzzMarketplace is Marketplace {
     Marketplace(
       MarketplaceConfig(
         CollateralConfig(10, 5, 3, 10),
-        ProofConfig(10, 5, 64, "")
+        ProofConfig(10, 5, 64, ""),
+        ValidationConfig(5)
       ),
       new TestToken(),
       new TestVerifier()

contracts/Marketplace.sol

@@ -7,11 +7,12 @@ import "@openzeppelin/contracts/utils/structs/EnumerableSet.sol";
 import "./Configuration.sol";
 import "./Requests.sol";
 import "./Proofs.sol";
+import "./Validation.sol";
 import "./StateRetrieval.sol";
 import "./Endian.sol";
 import "./Groth16.sol";
 
-contract Marketplace is Proofs, StateRetrieval, Endian {
+contract Marketplace is Proofs, Validation, StateRetrieval, Endian {
   using EnumerableSet for EnumerableSet.Bytes32Set;
   using Requests for Request;
 
@@ -58,7 +59,8 @@ contract Marketplace is Proofs, StateRetrieval, Endian {
     MarketplaceConfig memory configuration,
     IERC20 token_,
     IGroth16Verifier verifier
-  ) Proofs(configuration.proofs, verifier) {
+  ) Proofs(configuration.proofs, verifier)
+    Validation(configuration.validation) {
     _token = token_;
 
     require(
@@ -145,6 +147,8 @@ contract Marketplace is Proofs, StateRetrieval, Endian {
     slot.currentCollateral = collateralAmount;
 
     _addToMySlots(slot.host, slotId);
+    uint16 groupIdx = _getValidatorIndex(slotId);
+    _addToValidationSlots(groupIdx, slotId);
 
     emit SlotFilled(requestId, slotIndex);
     if (context.slotsFilled == request.ask.slots) {
@@ -166,6 +170,8 @@ contract Marketplace is Proofs, StateRetrieval, Endian {
       _payoutCancelledSlot(slot.requestId, slotId);
     } else if (state == SlotState.Failed) {
       _removeFromMySlots(msg.sender, slotId);
+      uint16 groupIdx = _getValidatorIndex(slotId);
+      _removeFromValidationSlots(groupIdx, slotId);
     } else if (state == SlotState.Filled) {
       _forciblyFreeSlot(slotId);
     }
@@ -233,6 +239,8 @@ contract Marketplace is Proofs, StateRetrieval, Endian {
     RequestContext storage context = _requestContexts[requestId];
 
     _removeFromMySlots(slot.host, slotId);
+    uint16 groupIdx = _getValidatorIndex(slotId);
+    _removeFromValidationSlots(groupIdx, slotId);
 
     uint256 slotIndex = slot.slotIndex;
     delete _slots[slotId];
@@ -265,6 +273,8 @@ contract Marketplace is Proofs, StateRetrieval, Endian {
     Slot storage slot = _slots[slotId];
 
     _removeFromMySlots(slot.host, slotId);
+    uint16 groupIdx = _getValidatorIndex(slotId);
+    _removeFromValidationSlots(groupIdx, slotId);
 
     uint256 amount = _requests[requestId].pricePerSlot() +
       slot.currentCollateral;
@@ -279,6 +289,8 @@ contract Marketplace is Proofs, StateRetrieval, Endian {
   ) private requestIsKnown(requestId) {
     Slot storage slot = _slots[slotId];
     _removeFromMySlots(slot.host, slotId);
+    uint16 groupIdx = _getValidatorIndex(slotId);
+    _removeFromValidationSlots(groupIdx, slotId);
 
     uint256 amount = _expiryPayoutAmount(requestId, slot.filledAt) +
       slot.currentCollateral;

contracts/StateRetrieval.sol

@@ -10,6 +10,7 @@ contract StateRetrieval {
 
   mapping(address => EnumerableSet.Bytes32Set) private _requestsPerClient;
   mapping(address => EnumerableSet.Bytes32Set) private _slotsPerHost;
+  mapping(uint16 => EnumerableSet.Bytes32Set) private _slotsPerValidator;
 
   function myRequests() public view returns (RequestId[] memory) {
     return _requestsPerClient[msg.sender].values().toRequestIds();
@@ -19,6 +20,10 @@ contract StateRetrieval {
     return _slotsPerHost[msg.sender].values().toSlotIds();
   }
 
+  function validationSlots(uint16 groupIdx) public view returns (SlotId[] memory) {
+    return _slotsPerValidator[groupIdx].values().toSlotIds();
+  }
+
   function _hasSlots(address host) internal view returns (bool) {
     return _slotsPerHost[host].length() > 0;
   }
@@ -31,6 +36,10 @@ contract StateRetrieval {
     _slotsPerHost[host].add(SlotId.unwrap(slotId));
   }
 
+  function _addToValidationSlots(uint16 groupIdx, SlotId slotId) internal {
+    _slotsPerValidator[groupIdx].add(SlotId.unwrap(slotId));
+  }
+
   function _removeFromMyRequests(address client, RequestId requestId) internal {
     _requestsPerClient[client].remove(RequestId.unwrap(requestId));
   }
@@ -38,4 +47,8 @@ contract StateRetrieval {
   function _removeFromMySlots(address host, SlotId slotId) internal {
     _slotsPerHost[host].remove(SlotId.unwrap(slotId));
   }
+
+  function _removeFromValidationSlots(uint16 groupIdx, SlotId slotId) internal {
+    _slotsPerValidator[groupIdx].remove(SlotId.unwrap(slotId));
+  }
 }

contracts/TestValidation.sol

@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: MIT
+pragma solidity ^0.8.0;
+
+import "./Validation.sol";
+import "./Requests.sol";
+
+contract TestValidation is Validation {
+
+  constructor(
+    ValidationConfig memory config
+  ) Validation(config) {}  // solhint-disable-line no-empty-blocks
+
+  function getValidatorIndex(SlotId slotId) public view returns (uint16) {
+    return _getValidatorIndex(slotId);
+  }
+}

contracts/Validation.sol

@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: MIT
+pragma solidity 0.8.23;
+
+import "./Configuration.sol";
+import "./Requests.sol";
+import "hardhat/console.sol";
+
+/**
+ * @title Validation
+ * @notice Abstract contract that handles distribution of SlotIds to validators
+   based on the number of validators specified in the config.
+ */
+abstract contract Validation {
+  ValidationConfig private _config;
+  uint256 private _idsPerValidator; // number of uint256's in each group of the 2^256 bit space
+
+  /**
+   * Creates a new Validation contract.
+   * @param config network-level validator configuration used to determine
+     number of SlotIds per validator.
+   */
+  constructor(
+    ValidationConfig memory config
+  ) {
+    require(config.validators > 0, "validators must be > 0");
+
+    uint256 high = type(uint256).max;
+
+    // To find the number of SlotIds per validator, we could do
+    // 2^256/validators, except that would overflow. Instead, we use
+    // floor(2^256-1 / validators) + 1. For example, if we used a 4-bit space
+    // (2^4=16) with 2 validators, we'd expect 8 per group: floor(2^4-1 / 2) + 1
+    // = 8
+    if (config.validators == 1) {
+      // max(uint256) + 1 would overflow, so assign 0 and handle as special case
+      // later
+      _idsPerValidator = 0;
+    } else {
+      _idsPerValidator = (high / config.validators) + 1;
+    }
+
+    _config = config;
+  }
+
+  /**
+   * Determines which validator group (0-based index) a SlotId belongs to, based
+     on the number of total validators in the config.
+   * @param slotId SlotID for which to determine the validator group index.
+   */
+  function _getValidatorIndex(SlotId slotId) internal view returns(uint16) {
+    uint256 slotIdInt = uint256(SlotId.unwrap(slotId));
+    return uint16(slotIdInt / _idsPerValidator);
+  }
+}

deploy/marketplace.js

@@ -15,6 +15,9 @@ const CONFIGURATION = {
     // in automine mode, because it can produce a block every second
     downtime: 64,
   },
+  validation: {
+    validators: 3
+  }
 }
 
 async function mine256blocks({ network, ethers }) {

test/Marketplace.test.js

@@ -1114,4 +1114,101 @@ describe("Marketplace", function () {
       expect(await marketplace.mySlots()).to.not.contain(slotId(slot))
     })
   })
+
+  describe("list of validation slots", function () {
+    beforeEach(async function () {
+      switchAccount(client)
+      await token.approve(marketplace.address, price(request))
+      await marketplace.requestStorage(request)
+      switchAccount(host)
+      await token.approve(marketplace.address, request.ask.collateral)
+    })
+
+    it("adds slot to list when filling slot", async function () {
+      await marketplace.fillSlot(slot.request, slot.index, proof)
+      let slot1 = { ...slot, index: slot.index + 1 }
+      await token.approve(marketplace.address, request.ask.collateral)
+      await marketplace.fillSlot(slot.request, slot1.index, proof)
+      const allSlots = (
+        await marketplace.validationSlots(0)).concat(
+        await marketplace.validationSlots(1)).concat(
+        await marketplace.validationSlots(2)
+      )
+      expect(allSlots).to.have.members([
+        slotId(slot),
+        slotId(slot1),
+      ])
+    })
+
+    it("removes slot from list when slot is freed", async function () {
+      await marketplace.fillSlot(slot.request, slot.index, proof)
+      let slot1 = { ...slot, index: slot.index + 1 }
+      await token.approve(marketplace.address, request.ask.collateral)
+      await marketplace.fillSlot(slot.request, slot1.index, proof)
+      await token.approve(marketplace.address, request.ask.collateral)
+      await marketplace.freeSlot(slotId(slot))
+      const allSlots = (
+        await marketplace.validationSlots(0)).concat(
+        await marketplace.validationSlots(1)).concat(
+        await marketplace.validationSlots(2)
+      )
+      expect(allSlots).to.not.have.members([slotId(slot)])
+      expect(allSlots).to.have.members([slotId(slot1)])
+    })
+
+    it("keeps slots when cancelled", async function () {
+      await marketplace.fillSlot(slot.request, slot.index, proof)
+      let slot1 = { ...slot, index: slot.index + 1 }
+
+      await token.approve(marketplace.address, request.ask.collateral)
+      await marketplace.fillSlot(slot.request, slot1.index, proof)
+      await waitUntilCancelled(request)
+      await mine()
+      const allSlots = (
+        await marketplace.validationSlots(0)).concat(
+        await marketplace.validationSlots(1)).concat(
+        await marketplace.validationSlots(2)
+      )
+      expect(allSlots).to.have.members([
+        slotId(slot),
+        slotId(slot1),
+      ])
+    })
+
+    it("removes slot when finished slot is freed", async function () {
+      await waitUntilStarted(marketplace, request, proof, token)
+      await waitUntilFinished(marketplace, requestId(request))
+      await marketplace.freeSlot(slotId(slot))
+      const allSlots = (
+        await marketplace.validationSlots(0)).concat(
+        await marketplace.validationSlots(1)).concat(
+        await marketplace.validationSlots(2)
+      )
+      expect(allSlots).to.not.contain(slotId(slot))
+    })
+
+    it("removes slot when cancelled slot is freed", async function () {
+      await marketplace.fillSlot(slot.request, slot.index, proof)
+      await waitUntilCancelled(request)
+      await marketplace.freeSlot(slotId(slot))
+      const allSlots = (
+        await marketplace.validationSlots(0)).concat(
+        await marketplace.validationSlots(1)).concat(
+        await marketplace.validationSlots(2)
+      )
+      expect(allSlots).to.not.contain(slotId(slot))
+    })
+
+    it("removes slot when failed slot is freed", async function () {
+      await waitUntilStarted(marketplace, request, proof, token)
+      await waitUntilSlotFailed(marketplace, request, slot)
+      await marketplace.freeSlot(slotId(slot))
+      const allSlots = (
+        await marketplace.validationSlots(0)).concat(
+        await marketplace.validationSlots(1)).concat(
+        await marketplace.validationSlots(2)
+      )
+      expect(allSlots).to.not.contain(slotId(slot))
+    })
+  })
 })

test/Validation.test.js

@@ -0,0 +1,107 @@
+const { expect } = require("chai")
+const { ethers } = require("hardhat")
+const {BigNumber, utils} = require("ethers")
+
+describe("Validation", function () {
+  const zero =
+    "0x0000000000000000000000000000000000000000000000000000000000000000"
+  const low =
+    "0x0000000000000000000000000000000000000000000000000000000000000001"
+  const mid =
+    "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
+
+  describe("constructor", function() {
+    // let validation
+    let Validation
+
+    beforeEach(async function () {
+      Validation = await ethers.getContractFactory("TestValidation")
+    })
+
+    it("fails to deploy with > uint16.max validators", async function() {
+      await expect(
+        Validation.deploy({validators: 2**16}) // uint16.max is 2^16-1
+      ).to.be.reverted
+    })
+
+    it("fails to deploy with 0 number of validators", async function() {
+      await expect(
+        Validation.deploy({validators: 0})
+      ).to.be.revertedWith("validators must be > 0")
+    })
+
+    it("successfully deploys with a valid number of validators", async function() {
+      await expect(
+        Validation.deploy({validators: 1})
+      ).to.be.ok
+    })
+  })
+
+  describe("groups of SlotIds per validator", function() {
+
+    let Validation
+
+    const high = ethers.constants.MaxUint256
+
+    function toUInt256Hex(bn) {
+      return utils.hexZeroPad(bn.toHexString(), 32)
+    }
+
+    function random(max) {
+      return Math.floor(Math.random() * max)
+    }
+
+    beforeEach(async function () {
+      Validation = await ethers.getContractFactory("TestValidation")
+    })
+
+    it("tests that the min and max boundary SlotIds into the correct group", async function () {
+      let validators = 2**16-1 // max value of uint16
+      let idsPerGroup = high.div( validators ).add(1) // as in the contract
+      let validation = await Validation.deploy({validators})
+
+      // Returns the minimum SlotId of all allowed SlotIds of the validator
+      // (given its index)
+      function minIdFor(validatorIdx) {
+        return BigNumber.from(validatorIdx).mul(idsPerGroup)
+      }
+      // Returns the maximum SlotId of all allowed SlotIds of the validator
+      // (given its index)
+      function maxIdFor(validatorIdx) {
+        const max = BigNumber.from(validatorIdx + 1).mul(idsPerGroup).sub(1)
+        // Never return more than max value of uint256 because it would
+        // overflow. BigNumber.js lets us do MaxUint256+1 without overflows.
+        if (max.gt(high)) {
+          return high
+        }
+        return max
+      }
+
+      // Generate randomised number of validators. If we fuzzed all possible
+      // number of validators, the test would take far too long to execute. This
+      // should absolutely never fail.
+      let validatorsRandomised = Array.from({ length: 128 }, (_) => random(validators))
+
+      for(let i=0; i<validatorsRandomised.length; i++) {
+        let validatorIdx = validatorsRandomised[i]
+
+        // test the boundary of the SlotIds that are allowed in this particular
+        // validator validatorIdx
+        let min = toUInt256Hex( minIdFor(validatorIdx) )
+        let max = toUInt256Hex( maxIdFor(validatorIdx) )
+
+        try{
+          expect(await validation.getValidatorIndex(min)).to.equal(validatorIdx)
+          expect(await validation.getValidatorIndex(max)).to.equal(validatorIdx)
+        } catch(e) {
+          console.log('FAILING TEST PARAMETERS')
+          console.log('-----------------------------------------------------------------------------------')
+          console.log('validator index:', validatorIdx)
+          console.log('slotId min:     ', min)
+          console.log('slotId max:     ', max)
+          throw e
+        }
+      }
+    })
+  })
+})

test/examples.js

@@ -15,6 +15,9 @@ const exampleConfiguration = () => ({
     downtime: 64,
     zkeyHash: "",
   },
+  validation: {
+    validators: 3
+  }
 })
 
 const exampleRequest = async () => {