wire in circom backend (#698)

* wire in circom backend

* should contain leafs

* adding circom compad and circuits deps

* update windows build

* fix windows build

* improve test names

* move proving defaults to codextypes

* remove unnedded inmports and move defaults to codextypes

* capture error code on backend failure
This commit is contained in:
Dmitriy Ryajov 2024-02-09 15:40:30 -06:00 committed by GitHub
parent 825766eea0
commit e23159b065
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 617 additions and 11 deletions

View File

@ -45,6 +45,7 @@ runs:
if: inputs.os == 'windows' && inputs.cpu == 'amd64' if: inputs.os == 'windows' && inputs.cpu == 'amd64'
uses: msys2/setup-msys2@v2 uses: msys2/setup-msys2@v2
with: with:
path-type: inherit
msystem: UCRT64 msystem: UCRT64
install: > install: >
base-devel base-devel
@ -52,11 +53,13 @@ runs:
mingw-w64-ucrt-x86_64-toolchain mingw-w64-ucrt-x86_64-toolchain
mingw-w64-ucrt-x86_64-cmake mingw-w64-ucrt-x86_64-cmake
mingw-w64-ucrt-x86_64-ntldd-git mingw-w64-ucrt-x86_64-ntldd-git
mingw-w64-ucrt-x86_64-rust
- name: MSYS2 (Windows i386) - name: MSYS2 (Windows i386)
if: inputs.os == 'windows' && inputs.cpu == 'i386' if: inputs.os == 'windows' && inputs.cpu == 'i386'
uses: msys2/setup-msys2@v2 uses: msys2/setup-msys2@v2
with: with:
path-type: inherit
msystem: MINGW32 msystem: MINGW32
install: > install: >
base-devel base-devel
@ -64,6 +67,7 @@ runs:
mingw-w64-i686-toolchain mingw-w64-i686-toolchain
mingw-w64-i686-cmake mingw-w64-i686-cmake
mingw-w64-i686-ntldd-git mingw-w64-i686-ntldd-git
mingw-w64-i686-rust
- name: Derive environment variables - name: Derive environment variables
shell: ${{ inputs.shell }} {0} shell: ${{ inputs.shell }} {0}

View File

@ -27,6 +27,7 @@ jobs:
name: '${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.tests }}' name: '${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.tests }}'
runs-on: ${{ matrix.builder }} runs-on: ${{ matrix.builder }}
timeout-minutes: 80 timeout-minutes: 80
continue-on-error: true
steps: steps:
- name: Checkout sources - name: Checkout sources
uses: actions/checkout@v4 uses: actions/checkout@v4

View File

@ -42,7 +42,6 @@ jobs:
cache_nonce: ${{ needs.matrix.outputs.cache_nonce }} cache_nonce: ${{ needs.matrix.outputs.cache_nonce }}
coverage: coverage:
continue-on-error: true
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout sources - name: Checkout sources

10
.gitmodules vendored
View File

@ -199,3 +199,13 @@
[submodule "vendor/constantine"] [submodule "vendor/constantine"]
path = vendor/constantine path = vendor/constantine
url = https://github.com/mratsim/constantine.git url = https://github.com/mratsim/constantine.git
[submodule "vendor/nim-circom-compat"]
path = vendor/nim-circom-compat
url = https://github.com/codex-storage/nim-circom-compat.git
ignore = untracked
branch = master
[submodule "vendor/codex-storage-proofs-circuits"]
path = vendor/codex-storage-proofs-circuits
url = https://github.com/codex-storage/codex-storage-proofs-circuits.git
ignore = untracked
branch = master

View File

@ -28,6 +28,13 @@ const
DefaultBlockSize* = NBytes 1024*64 DefaultBlockSize* = NBytes 1024*64
DefaultCellSize* = NBytes 2048 DefaultCellSize* = NBytes 2048
# Proving defaults
DefaultMaxSlotDepth* = 32
DefaultMaxDatasetDepth* = 8
DefaultBlockDepth* = 5
DefaultCellElms* = 67
DefaultSamplesNum* = 5
# hashes # hashes
Sha256HashCodec* = multiCodec("sha2-256") Sha256HashCodec* = multiCodec("sha2-256")
Sha512HashCodec* = multiCodec("sha2-512") Sha512HashCodec* = multiCodec("sha2-512")

View File

@ -1,4 +1,6 @@
import ./slots/builder import ./slots/builder
import ./slots/sampler import ./slots/sampler
import ./slots/proofs
import ./slots/types
export builder, sampler export builder, sampler, proofs, types

View File

@ -78,11 +78,11 @@ func toVerifiableProof*(
proof: CodexProof): ?!Poseidon2Proof = proof: CodexProof): ?!Poseidon2Proof =
let let
verifiableProof = Poseidon2Proof( nodes = proof.path.mapIt(
index: proof.index, ? Poseidon2Hash.fromBytes(it.toArray32).toFailure
nleaves: proof.nleaves, )
path: proof.path.mapIt(
? Poseidon2Hash.fromBytes(it.toArray32).toFailure
))
success verifiableProof Poseidon2Proof.init(
index = proof.index,
nleaves = proof.nleaves,
nodes = nodes)

4
codex/slots/proofs.nim Normal file
View File

@ -0,0 +1,4 @@
import ./proofs/backends
import ./proofs/prover
export circomcompat, prover

View File

@ -0,0 +1,3 @@
import ./backends/circomcompat
export circomcompat

View File

@ -0,0 +1,228 @@
## Nim-Codex
## Copyright (c) 2024 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
{.push raises: [].}
import std/sequtils
import pkg/chronos
import pkg/questionable/results
import pkg/circomcompat
import pkg/poseidon2/io
import ../../types
import ../../../stores
import ../../../merkletree
import ../../../codextypes
export circomcompat
type
CircomCompat* = object
slotDepth : int # max depth of the slot tree
datasetDepth : int # max depth of dataset tree
blkDepth : int # depth of the block merkle tree (pow2 for now)
cellElms : int # number of field elements per cell
numSamples : int # number of samples per slot
r1csPath : string # path to the r1cs file
wasmPath : string # path to the wasm file
zKeyPath : string # path to the zkey file
backendCfg : ptr CircomBn254Cfg
CircomG1* = G1
CircomG2* = G2
CircomProof* = Proof
CircomKey* = VerifyingKey
CircomInputs* = Inputs
proc release*(self: CircomCompat) =
## Release the backend
##
self.backendCfg.unsafeAddr.releaseCfg()
proc getVerifyingKey*(
self: CircomCompat): ?!ptr CircomKey =
## Get the verifying key
##
var
cfg: ptr CircomBn254Cfg = self.backendCfg
vkpPtr: ptr VerifyingKey = nil
if cfg.getVerifyingKey(vkpPtr.addr) != ERR_OK or vkpPtr == nil:
return failure("Failed to get verifying key")
success vkpPtr
proc prove*[H](
self: CircomCompat,
input: ProofInput[H]): ?!CircomProof =
## Encode buffers using a backend
##
# NOTE: All inputs are statically sized per circuit
# and adjusted accordingly right before being passed
# to the circom ffi - `setLen` is used to adjust the
# sequence length to the correct size which also 0 pads
# to the correct length
doAssert input.samples.len == self.numSamples,
"Number of samples does not match"
doAssert input.slotProof.len <= self.datasetDepth,
"Number of slot proofs does not match"
doAssert input.samples.allIt(
block:
(it.merklePaths.len <= self.slotDepth + self.blkDepth and
it.cellData.len <= self.cellElms * 32)), "Merkle paths length does not match"
# TODO: All parameters should match circom's static parametter
var
backend: ptr CircomCompatCtx
if initCircomCompat(
self.backendCfg,
addr backend) != ERR_OK or backend == nil:
raiseAssert("failed to initialize CircomCompat backend")
var
entropy = input.entropy.toBytes
dataSetRoot = input.datasetRoot.toBytes
slotRoot = input.slotRoot.toBytes
if backend.pushInputU256Array(
"entropy".cstring, entropy[0].addr, entropy.len.uint32) != ERR_OK:
return failure("Failed to push entropy")
if backend.pushInputU256Array(
"dataSetRoot".cstring, dataSetRoot[0].addr, dataSetRoot.len.uint32) != ERR_OK:
return failure("Failed to push data set root")
if backend.pushInputU256Array(
"slotRoot".cstring, slotRoot[0].addr, slotRoot.len.uint32) != ERR_OK:
return failure("Failed to push data set root")
if backend.pushInputU32(
"nCellsPerSlot".cstring, input.nCellsPerSlot.uint32) != ERR_OK:
return failure("Failed to push nCellsPerSlot")
if backend.pushInputU32(
"nSlotsPerDataSet".cstring, input.nSlotsPerDataSet.uint32) != ERR_OK:
return failure("Failed to push nSlotsPerDataSet")
if backend.pushInputU32(
"slotIndex".cstring, input.slotIndex.uint32) != ERR_OK:
return failure("Failed to push slotIndex")
var
slotProof = input.slotProof.mapIt( it.toBytes ).concat
slotProof.setLen(self.datasetDepth) # zero pad inputs to correct size
# arrays are always flattened
if backend.pushInputU256Array(
"slotProof".cstring,
slotProof[0].addr,
uint (slotProof[0].len * slotProof.len)) != ERR_OK:
return failure("Failed to push slot proof")
for s in input.samples:
var
merklePaths = s.merklePaths.mapIt( it.toBytes )
data = s.cellData
merklePaths.setLen(self.slotDepth) # zero pad inputs to correct size
if backend.pushInputU256Array(
"merklePaths".cstring,
merklePaths[0].addr,
uint (merklePaths[0].len * merklePaths.len)) != ERR_OK:
return failure("Failed to push merkle paths")
data.setLen(self.cellElms * 32) # zero pad inputs to correct size
if backend.pushInputU256Array(
"cellData".cstring,
data[0].addr,
data.len.uint) != ERR_OK:
return failure("Failed to push cell data")
var
proofPtr: ptr Proof = nil
let proof =
try:
if (
let res = self.backendCfg.proveCircuit(backend, proofPtr.addr);
res != ERR_OK) or
proofPtr == nil:
return failure("Failed to prove - err code: " & $res)
proofPtr[]
finally:
if proofPtr != nil:
proofPtr.addr.releaseProof()
if backend != nil:
backend.addr.releaseCircomCompat()
success proof
proc verify*(
self: CircomCompat,
proof: CircomProof,
inputs: CircomInputs,
vkp: CircomKey): ?!bool =
## Verify a proof using a backend
##
var
proofPtr : ptr Proof = unsafeAddr proof
inputsPtr: ptr Inputs = unsafeAddr inputs
vpkPtr: ptr CircomKey = unsafeAddr vkp
let res = verifyCircuit(proofPtr, inputsPtr, vpkPtr)
if res == ERR_OK:
success true
elif res == ERR_FAILED_TO_VERIFY_PROOF:
success false
else:
failure("Failed to verify proof - err code: " & $res)
proc init*(
_: type CircomCompat,
r1csPath : string,
wasmPath : string,
zKeyPath : string = "",
slotDepth = DefaultMaxSlotDepth,
datasetDepth = DefaultMaxDatasetDepth,
blkDepth = DefaultBlockDepth,
cellElms = DefaultCellElms,
numSamples = DefaultSamplesNum): CircomCompat =
## Create a new backend
##
var cfg: ptr CircomBn254Cfg
if initCircomConfig(
r1csPath.cstring,
wasmPath.cstring,
if zKeyPath.len > 0: zKeyPath.cstring else: nil,
addr cfg) != ERR_OK or cfg == nil:
raiseAssert("failed to initialize circom compat config")
CircomCompat(
r1csPath : r1csPath,
wasmPath : wasmPath,
zKeyPath : zKeyPath,
backendCfg : cfg,
slotDepth : slotDepth,
datasetDepth: datasetDepth,
blkDepth : blkDepth,
cellElms : cellElms,
numSamples : numSamples)

View File

@ -0,0 +1,98 @@
## Nim-Codex
## Copyright (c) 2024 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
##
import pkg/chronos
import pkg/chronicles
import pkg/circomcompat
import pkg/poseidon2
import pkg/questionable/results
import pkg/libp2p/cid
import ../../manifest
import ../../merkletree
import ../../stores
import ../../market
import ../../utils/poseidon2digest
import ../builder
import ../sampler
import ./backends
import ../types
export backends
type
AnyProof* = CircomProof
AnyInputs* = CircomInputs
AnyKeys* = CircomKey
AnyHash* = Poseidon2Hash
AnyBackend* = CircomCompat
AnyBuilder* = Poseidon2Builder
AnySampler* = Poseidon2Sampler
Prover* = ref object of RootObj
backend: AnyBackend
store: BlockStore
proc prove*(
self: Prover,
slotIdx: int,
manifest: Manifest,
challenge: ProofChallenge,
nSamples = DefaultSamplesNum): Future[?!AnyProof] {.async.} =
## Prove a statement using backend.
## Returns a future that resolves to a proof.
logScope:
cid = manifest.treeCid
slot = slotIdx
challenge = challenge
trace "Received proof challenge"
without builder =? AnyBuilder.new(self.store, manifest), err:
error "Unable to create slots builder", err = err.msg
return failure(err)
without sampler =? AnySampler.new(slotIdx, self.store, builder), err:
error "Unable to create data sampler", err = err.msg
return failure(err)
without proofInput =? await sampler.getProofInput(challenge, nSamples), err:
error "Unable to get proof input for slot", err = err.msg
return failure(err)
# prove slot
without proof =? self.backend.prove(proofInput), err:
error "Unable to prove slot", err = err.msg
return failure(err)
success proof
proc verify*(
self: Prover,
proof: AnyProof,
inputs: AnyInputs,
vpk: AnyKeys): Future[?!bool] {.async.} =
## Prove a statement using backend.
## Returns a future that resolves to a proof.
self.backend.verify(proof, inputs, vpk)
proc new*(
_: type Prover,
store: BlockStore,
backend: AnyBackend): Prover =
Prover(
backend: backend,
store: store)

View File

@ -0,0 +1,4 @@
pragma circom 2.0.0;
include "sample_cells.circom";
// SampleAndProven( maxDepth, maxLog2NSlots, blockTreeDepth, nFieldElemsPerCell, nSamples )
component main {public [entropy,dataSetRoot,slotIndex]} = SampleAndProve(32, 8, 5, 67, 5);

Binary file not shown.

Binary file not shown.

View File

@ -79,4 +79,5 @@ proc example*(_: type MerkleProof): MerkleProof =
proc example*(_: type Poseidon2Proof): Poseidon2Proof = proc example*(_: type Poseidon2Proof): Poseidon2Proof =
var example = MerkleProof[Poseidon2Hash, PoseidonKeysEnum]() var example = MerkleProof[Poseidon2Hash, PoseidonKeysEnum]()
example.index = 123 example.index = 123
example.path = @[1, 2, 3, 4].mapIt( it.toF )
example example

View File

@ -37,6 +37,31 @@ func toPublicInputs*[H](input: ProofInput[H]): PublicInputs[H] =
entropy: input.entropy entropy: input.entropy
) )
proc toCircomInputs*[H](inputs: PublicInputs[H]): CircomInputs =
var
slotIndex = inputs.slotIndex.toF.toBytes.toArray32
datasetRoot = inputs.datasetRoot.toBytes.toArray32
entropy = inputs.entropy.toBytes.toArray32
elms = [
entropy,
datasetRoot,
slotIndex
]
let inputsPtr = allocShared0(32 * elms.len)
copyMem(inputsPtr, addr elms[0], elms.len * 32)
CircomInputs(
elms: cast[ptr array[32, byte]](inputsPtr),
len: elms.len.uint
)
proc releaseNimInputs*(inputs: var CircomInputs) =
if not inputs.elms.isNil:
deallocShared(inputs.elms)
inputs.elms = nil
func toJsonDecimal*(big: BigInt[254]): string = func toJsonDecimal*(big: BigInt[254]): string =
let s = big.toDecimal.strip( leading = true, trailing = false, chars = {'0'} ) let s = big.toDecimal.strip( leading = true, trailing = false, chars = {'0'} )
if s.len == 0: "0" else: s if s.len == 0: "0" else: s

View File

@ -0,0 +1,137 @@
import std/sequtils
import std/sugar
import std/options
import pkg/chronos
import ../../../asynctest
import pkg/poseidon2
import pkg/datastore
import pkg/codex/slots {.all.}
import pkg/codex/slots/types {.all.}
import pkg/codex/merkletree
import pkg/codex/utils/json
import pkg/codex/codextypes
import pkg/codex/manifest
import pkg/codex/stores
import pkg/constantine/math/arithmetic
import pkg/constantine/math/io/io_fields
import pkg/constantine/math/io/io_bigints
import ./helpers
import ../helpers
suite "Test Circom Compat Backend - control inputs":
let
r1cs = "tests/circuits/fixtures/proof_main.r1cs"
wasm = "tests/circuits/fixtures/proof_main.wasm"
var
circom: CircomCompat
verifyingKeyPtr: ptr CircomKey
proofInput: ProofInput[Poseidon2Hash]
publicInputs: CircomInputs
setup:
let
inputData = readFile("tests/circuits/fixtures/input.json")
inputJson = parseJson(inputData)
proofInput = jsonToProofInput[Poseidon2Hash](inputJson)
publicInputs = toPublicInputs[Poseidon2Hash](proofInput).toCircomInputs
# circom = CircomCompat.init(r1cs, wasm, zkey)
circom = CircomCompat.init(r1cs, wasm)
verifyingKeyPtr = circom.getVerifyingKey().tryGet
teardown:
publicInputs.releaseNimInputs() # this is allocated by nim
verifyingKeyPtr.addr.releaseKey() # this comes from the rust FFI
circom.release() # this comes from the rust FFI
test "Should verify with correct inputs":
let
proof = circom.prove(proofInput).tryGet
check circom.verify(proof, publicInputs, verifyingKeyPtr[]).tryGet
test "Should not verify with incorrect inputs":
proofInput.slotIndex = 1 # change slot index
let
proof = circom.prove(proofInput).tryGet
check circom.verify(proof, publicInputs, verifyingKeyPtr[]).tryGet == false
suite "Test Circom Compat Backend":
let
slotId = 3
samples = 5
ecK = 2
ecM = 2
numDatasetBlocks = 8
blockSize = DefaultBlockSize
cellSize = DefaultCellSize
r1cs = "tests/circuits/fixtures/proof_main.r1cs"
wasm = "tests/circuits/fixtures/proof_main.wasm"
var
store: BlockStore
manifest: Manifest
protected: Manifest
verifiable: Manifest
circom: CircomCompat
verifyingKeyPtr: ptr CircomKey
proofInput: ProofInput[Poseidon2Hash]
publicInputs: CircomInputs
challenge: array[32, byte]
builder: Poseidon2Builder
sampler: Poseidon2Sampler
setup:
let
repoDs = SQLiteDatastore.new(Memory).tryGet()
metaDs = SQLiteDatastore.new(Memory).tryGet()
store = RepoStore.new(repoDs, metaDs)
(manifest, protected, verifiable) =
await createVerifiableManifest(
store,
numDatasetBlocks,
ecK, ecM,
blockSize,
cellSize)
builder = Poseidon2Builder.new(store, verifiable).tryGet
sampler = Poseidon2Sampler.new(slotId, store, builder).tryGet
# circom = CircomCompat.init(r1cs, wasm, zkey)
circom = CircomCompat.init(r1cs, wasm)
verifyingKeyPtr = circom.getVerifyingKey().tryGet
challenge = 1234567.toF.toBytes.toArray32
proofInput = (await sampler.getProofInput(challenge, samples)).tryGet
publicInputs = proofInput.toPublicInputs.toCircomInputs
teardown:
publicInputs.releaseNimInputs() # this is allocated by nim
verifyingKeyPtr.addr.releaseKey() # this comes from the rust FFI
circom.release() # this comes from the rust FFI
test "Should verify with correct input":
var
proof = circom.prove(proofInput).tryGet
check circom.verify(proof, publicInputs, verifyingKeyPtr[]).tryGet
test "Should not verify with incorrect input":
proofInput.slotIndex = 1 # change slot index
let
proof = circom.prove(proofInput).tryGet
check circom.verify(proof, publicInputs, verifyingKeyPtr[]).tryGet == false

View File

@ -26,7 +26,7 @@ import ../backends/helpers
import ../helpers import ../helpers
import ../../helpers import ../../helpers
suite "Test control samples": suite "Test Sampler - control samples":
var var
inputData: string inputData: string
@ -73,7 +73,7 @@ suite "Test control samples":
check datasetProof.verify(proofInput.slotRoot, proofInput.datasetRoot).tryGet check datasetProof.verify(proofInput.slotRoot, proofInput.datasetRoot).tryGet
suite "Test sampler samples": suite "Test Sampler":
let let
slotIndex = 3 slotIndex = 3

View File

@ -0,0 +1,3 @@
import ./backends/testcircomcompat
{.warning[UnusedImport]: off.}

View File

@ -0,0 +1,78 @@
import std/sequtils
import std/sugar
import std/math
import ../../asynctest
import pkg/chronos
import pkg/libp2p/cid
import pkg/datastore
import pkg/codex/merkletree
import pkg/codex/rng
import pkg/codex/manifest
import pkg/codex/chunker
import pkg/codex/blocktype as bt
import pkg/codex/slots
import pkg/codex/stores
import pkg/poseidon2/io
import pkg/codex/utils/poseidon2digest
import pkg/constantine/math/arithmetic
import pkg/constantine/math/io/io_bigints
import pkg/constantine/math/io/io_fields
import ./helpers
import ../helpers
import ./backends/helpers
suite "Test Prover":
let
slotId = 1
samples = 5
blockSize = DefaultBlockSize
cellSize = DefaultCellSize
ecK = 3
ecM = 2
numDatasetBlocks = 8
var
datasetBlocks: seq[bt.Block]
store: BlockStore
manifest: Manifest
protected: Manifest
verifiable: Manifest
sampler: Poseidon2Sampler
setup:
let
repoDs = SQLiteDatastore.new(Memory).tryGet()
metaDs = SQLiteDatastore.new(Memory).tryGet()
store = RepoStore.new(repoDs, metaDs)
(manifest, protected, verifiable) =
await createVerifiableManifest(
store,
numDatasetBlocks,
ecK, ecM,
blockSize,
cellSize)
test "Should sample and prove a slot":
let
r1cs = "tests/circuits/fixtures/proof_main.r1cs"
wasm = "tests/circuits/fixtures/proof_main.wasm"
circomBackend = CircomCompat.init(r1cs, wasm)
prover = Prover.new(store, circomBackend)
challenge = 1234567.toF.toBytes.toArray32
proof = (await prover.prove(1, verifiable, challenge, 5)).tryGet
key = circomBackend.getVerifyingKey().tryGet
builder = Poseidon2Builder.new(store, verifiable).tryGet
sampler = Poseidon2Sampler.new(1, store, builder).tryGet
proofInput = (await sampler.getProofInput(challenge, 5)).tryGet
inputs = proofInput.toPublicInputs.toCircomInputs
check:
(await prover.verify(proof, inputs, key[])).tryGet == true

@ -0,0 +1 @@
Subproject commit c03b43221d68e34bd5015a4e4ee1a0ad3299f8ef

1
vendor/nim-circom-compat vendored Submodule

@ -0,0 +1 @@
Subproject commit fa513b123e081c76ec0bb3237ad886d4830b8071