Refactor Circom compatibility backend and add NimGroth16 backend implementation

This commit is contained in:
Dmitriy Ryajov 2025-05-28 19:03:04 -06:00 committed by Eric
parent 4adba177d0
commit 5101b98521
No known key found for this signature in database
6 changed files with 431 additions and 61 deletions

View File

@ -7,6 +7,8 @@
## This file may not be copied, modified, or distributed except according to
## those terms.
{.deprecated: "use the NimGroth16Backend".}
{.push raises: [].}
import std/sugar
@ -24,7 +26,7 @@ import ./converters
export circomcompat, converters
type
CircomCompat* = object
CircomCompatBackend* = object
slotDepth: int # max depth of the slot tree
datasetDepth: int # max depth of dataset tree
blkDepth: int # depth of the block merkle tree (pow2 for now)
@ -34,13 +36,15 @@ type
wasmPath: string # path to the wasm file
zkeyPath: string # path to the zkey file
backendCfg: ptr CircomBn254Cfg
vkp*: ptr CircomKey
vkp*: ptr CircomCompatKey
NormalizedProofInputs*[H] {.borrow: `.`.} = distinct ProofInputs[H]
CircomCompatBackendRef* = ref CircomCompatBackend
func normalizeInput*[H](
self: CircomCompat, input: ProofInputs[H]
): NormalizedProofInputs[H] =
NormalizedProofInputs*[SomeHash] {.borrow: `.`.} = distinct ProofInputs[SomeHash]
func normalizeInput*[SomeHash](
self: CircomCompatBackendRef, input: ProofInputs[SomeHash]
): NormalizedProofInputs[SomeHash] =
## Parameters in CIRCOM circuits are statically sized and must be properly
## padded before they can be passed onto the circuit. This function takes
## variable length parameters and performs that padding.
@ -53,23 +57,25 @@ func normalizeInput*[H](
for sample in input.samples:
var merklePaths = sample.merklePaths
merklePaths.setLen(self.slotDepth)
Sample[H](cellData: sample.cellData, merklePaths: merklePaths)
Sample[SomeHash](cellData: sample.cellData, merklePaths: merklePaths)
var normSlotProof = input.slotProof
normSlotProof.setLen(self.datasetDepth)
NormalizedProofInputs[H] ProofInputs[H](
entropy: input.entropy,
datasetRoot: input.datasetRoot,
slotIndex: input.slotIndex,
slotRoot: input.slotRoot,
nCellsPerSlot: input.nCellsPerSlot,
nSlotsPerDataSet: input.nSlotsPerDataSet,
slotProof: normSlotProof,
samples: normSamples,
NormalizedProofInputs[SomeHash](
ProofInputs[SomeHash](
entropy: input.entropy,
datasetRoot: input.datasetRoot,
slotIndex: input.slotIndex,
slotRoot: input.slotRoot,
nCellsPerSlot: input.nCellsPerSlot,
nSlotsPerDataSet: input.nSlotsPerDataSet,
slotProof: normSlotProof,
samples: normSamples,
)
)
proc release*(self: CircomCompat) =
proc release*(self: CircomCompatBackendRef) =
## Release the ctx
##
@ -79,7 +85,9 @@ proc release*(self: CircomCompat) =
if not isNil(self.vkp):
self.vkp.unsafeAddr.release_key()
proc prove[H](self: CircomCompat, input: NormalizedProofInputs[H]): ?!CircomProof =
proc prove[SomeHash](
self: CircomCompatBackendRef, input: NormalizedProofInputs[SomeHash]
): Future[?!CircomCompatProof] {.async: (raises: [CancelledError]).} =
doAssert input.samples.len == self.numSamples, "Number of samples does not match"
doAssert input.slotProof.len <= self.datasetDepth,
@ -101,7 +109,7 @@ proc prove[H](self: CircomCompat, input: NormalizedProofInputs[H]): ?!CircomProo
ctx.addr.release_circom_compat()
if init_circom_compat(self.backendCfg, addr ctx) != ERR_OK or ctx == nil:
raiseAssert("failed to initialize CircomCompat ctx")
raiseAssert("failed to initialize CircomCompatBackend ctx")
var
entropy = input.entropy.toBytes
@ -172,12 +180,16 @@ proc prove[H](self: CircomCompat, input: NormalizedProofInputs[H]): ?!CircomProo
success proof
proc prove*[H](self: CircomCompat, input: ProofInputs[H]): ?!CircomProof =
proc prove*[SomeHash](
self: CircomCompatBackendRef, input: ProofInputs[SomeHash]
): Future[?!CircomCompatProof] {.async: (raises: [CancelledError], raw: true).} =
self.prove(self.normalizeInput(input))
proc verify*[H](
self: CircomCompat, proof: CircomProof, inputs: ProofInputs[H]
): ?!bool =
proc verify*[SomeHash](
self: CircomCompatBackendRef,
proof: CircomCompatProof,
inputs: ProofInputs[SomeHash],
): Future[?!bool] {.async: (raises: [CancelledError]).} =
## Verify a proof using a ctx
##
@ -196,8 +208,8 @@ proc verify*[H](
finally:
inputs.releaseCircomInputs()
proc init*(
_: type CircomCompat,
proc new*(
_: type CircomCompatBackendRef,
r1csPath: string,
wasmPath: string,
zkeyPath: string = "",
@ -206,7 +218,7 @@ proc init*(
blkDepth = DefaultBlockDepth,
cellElms = DefaultCellElms,
numSamples = DefaultSamplesNum,
): CircomCompat =
): ?!CircomCompatBackendRef =
## Create a new ctx
##
@ -217,16 +229,16 @@ proc init*(
cfg == nil:
if cfg != nil:
cfg.addr.release_cfg()
raiseAssert("failed to initialize circom compat config")
return failure "failed to initialize circom compat config"
var vkpPtr: ptr VerifyingKey = nil
if cfg.get_verifying_key(vkpPtr.addr) != ERR_OK or vkpPtr == nil:
if vkpPtr != nil:
vkpPtr.addr.release_key()
raiseAssert("Failed to get verifying key")
return failure "Failed to get verifying key"
CircomCompat(
success CircomCompatBackendRef(
r1csPath: r1csPath,
wasmPath: wasmPath,
zkeyPath: zkeyPath,

View File

@ -1,5 +1,5 @@
## Nim-Codex
## Copyright (c) 2024 Status Research & Development GmbH
## Copyright (c) 2025 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
@ -9,21 +9,27 @@
{.push raises: [].}
import pkg/groth16
import pkg/circomcompat
import pkg/constantine/math/io/io_fields
import ../../../contracts
import ../../types
import ../../../merkletree
type
CircomG1* = G1
CircomG2* = G2
CircomCompatG1* = circomcompat.G1
CircomCompatG2* = circomcompat.G2
CircomProof* = Proof
CircomKey* = VerifyingKey
CircomInputs* = Inputs
CircomCompatProof* = circomcompat.Proof
CircomCompatKey* = circomcompat.VerifyingKey
CircomCompatInputs* = circomcompat.Inputs
proc toCircomInputs*(inputs: ProofInputs[Poseidon2Hash]): CircomInputs =
NimGroth16G1* = groth16.G1
NimGroth16G2* = groth16.G2
NimGroth16Proof* = groth16.Proof
proc toCircomInputs*(inputs: ProofInputs[Poseidon2Hash]): CircomCompatInputs =
var
slotIndex = inputs.slotIndex.toF.toBytes.toArray32
datasetRoot = inputs.datasetRoot.toBytes.toArray32
@ -34,21 +40,49 @@ proc toCircomInputs*(inputs: ProofInputs[Poseidon2Hash]): CircomInputs =
let inputsPtr = allocShared0(32 * elms.len)
copyMem(inputsPtr, addr elms[0], elms.len * 32)
CircomInputs(elms: cast[ptr array[32, byte]](inputsPtr), len: elms.len.uint)
CircomCompatInputs(elms: cast[ptr array[32, byte]](inputsPtr), len: elms.len.uint)
proc releaseCircomInputs*(inputs: var CircomInputs) =
proc releaseCircomInputs*(inputs: var CircomCompatInputs) =
if not inputs.elms.isNil:
deallocShared(inputs.elms)
inputs.elms = nil
func toG1*(g: CircomG1): G1Point =
func toG1*(g: CircomCompatG1): G1Point =
G1Point(x: UInt256.fromBytesLE(g.x), y: UInt256.fromBytesLE(g.y))
func toG2*(g: CircomG2): G2Point =
func toG2*(g: CircomCompatG2): G2Point =
G2Point(
x: Fp2Element(real: UInt256.fromBytesLE(g.x[0]), imag: UInt256.fromBytesLE(g.x[1])),
y: Fp2Element(real: UInt256.fromBytesLE(g.y[0]), imag: UInt256.fromBytesLE(g.y[1])),
)
func toGroth16Proof*(proof: CircomProof): Groth16Proof =
func toGroth16Proof*(proof: CircomCompatProof): Groth16Proof =
Groth16Proof(a: proof.a.toG1, b: proof.b.toG2, c: proof.c.toG1)
func toG1*(g: NimGroth16G1): G1Point =
var
x: seq[byte]
y: seq[byte]
assert x.marshal(g.x, Endianness.littleEndian)
assert y.marshal(g.y, Endianness.littleEndian)
G1Point(x: UInt256.fromBytesLE(x), y: UInt256.fromBytesLE(y))
func toG2*(g: NimGroth16G2): G2Point =
var
x: array[2, seq[byte]]
y: array[2, seq[byte]]
assert x[0].marshal(g.x.coords[0], Endianness.littleEndian)
assert x[1].marshal(g.x.coords[1], Endianness.littleEndian)
assert y[0].marshal(g.y.coords[0], Endianness.littleEndian)
assert y[1].marshal(g.y.coords[1], Endianness.littleEndian)
G2Point(
x: Fp2Element(real: UInt256.fromBytesLE(x[0]), imag: UInt256.fromBytesLE(x[1])),
y: Fp2Element(real: UInt256.fromBytesLE(y[0]), imag: UInt256.fromBytesLE(y[1])),
)
func toGroth16Proof*(proof: NimGroth16Proof): Groth16Proof =
Groth16Proof(a: proof.pi_a.toG1, b: proof.pi_b.toG2, c: proof.pi_c.toG1)

View File

@ -0,0 +1,208 @@
## Nim-Codex
## Copyright (c) 2025 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
{.push raises: [].}
import std/sugar
import std/isolation
import std/atomics
import pkg/chronos
import pkg/chronos/threadsync
import pkg/taskpools
import pkg/questionable/results
import pkg/groth16
import pkg/nim/circom_witnessgen
import pkg/nim/circom_witnessgen/load
import pkg/nim/circom_witnessgen/witness
import ../../types
import ../../../stores
import ../../../contracts
import ./converters
export converters
const DefaultCurve* = "bn128"
type
NimGroth16Backend* = object
curve: string # curve name
slotDepth: int # max depth of the slot tree
datasetDepth: int # max depth of dataset tree
blkDepth: int # depth of the block merkle tree (pow2 for now)
cellElms: int # number of field elements per cell
numSamples: int # number of samples per slot
r1cs: R1CS # path to the r1cs file
zkey: ZKey # path to the zkey file
graph*: Graph # path to the graph file generated with circom-witnesscalc
tp: Taskpool # taskpool for async operations
NimGroth16BackendRef* = ref NimGroth16Backend
ProofTask* = object
proof: Isolated[Proof]
self: ptr NimGroth16Backend
inputs: Inputs
signal: ThreadSignalPtr
ok: Atomic[bool]
proc release*(self: NimGroth16BackendRef) =
## Release the ctx
##
discard
proc normalizeInput[SomeHash](
self: NimGroth16BackendRef, input: ProofInputs[SomeHash]
): Inputs =
## Map inputs to witnessgen inputs
##
var normSlotProof = input.slotProof
normSlotProof.setLen(self.datasetDepth)
{
"slotDepth": @[self.slotDepth.toF],
"datasetDepth": @[self.datasetDepth.toF],
"blkDepth": @[self.blkDepth.toF],
"cellElms": @[self.cellElms.toF],
"numSamples": @[self.numSamples.toF],
"entropy": @[input.entropy],
"dataSetRoot": @[input.datasetRoot],
"slotIndex": @[input.slotIndex.toF],
"slotRoot": @[input.slotRoot],
"nCellsPerSlot": @[input.nCellsPerSlot.toF],
"nSlotsPerDataSet": @[input.nSlotsPerDataSet.toF],
"slotProof": normSlotProof,
"cellData": input.samples.mapIt(it.cellData).concat,
"merklePaths": input.samples.mapIt(
block:
var mekrlePaths = it.merklePaths
mekrlePaths.setLen(self.slotDepth)
mekrlePaths
).concat,
}.toTable
proc generateProofTask(task: ptr ProofTask) =
defer:
if task[].signal != nil:
discard task[].signal.fireSync()
try:
trace "Generating witness"
let
witnessValues = generateWitness(task[].self[].graph, task[].inputs)
witness = Witness(
curve: task[].self[].curve,
r: task[].self[].r1cs.r,
nvars: task[].self[].r1cs.cfg.nWires,
values: witnessValues,
)
trace "Generating nim groth16 proof"
var proof = generateProof(task[].self[].zkey, witness, task[].self[].tp)
trace "Proof generated, copying to main thread"
var isolatedProof = isolate(proof)
task[].proof = move isolatedProof
task[].ok.store true
except CatchableError as e:
error "Failed to generate proof", err = e.msg
task[].ok.store false
proc prove*[SomeHash](
self: NimGroth16BackendRef, input: ProofInputs[SomeHash]
): Future[?!NimGroth16Proof] {.async: (raises: [CancelledError]).} =
## Prove a statement using backend.
##
var
signalPtr = ?ThreadSignalPtr.new().mapFailure
task = ProofTask(
self: cast[ptr NimGroth16Backend](self),
signal: signalPtr,
inputs: self.normalizeInput(input),
)
defer:
if signalPtr != nil:
?signalPtr.close().mapFailure
signalPtr = nil
self.tp.spawn generateProofTask(task.addr)
let taskFut = signalPtr.wait()
if err =? catch(await taskFut.join()).errorOption:
# XXX: we need this because there is no way to cancel a task
# and without waiting for it to finish, we'll be writting to free'd
# memory in the task
warn "Error while generating proof, awaiting task to finish", err = err.msg
?catch(await noCancel taskFut)
if err of CancelledError: # reraise cancelled error
trace "Task was cancelled"
raise (ref CancelledError) err
trace "Task failed with error", err = err.msg
return failure err
defer:
task.proof = default(Isolated[Proof])
if not task.ok.load:
trace "Task failed, no proof generated"
return failure("Failed to generate proof")
var proof = task.proof.extract
trace "Task finished successfully, proof generated"
success proof
proc verify*(
self: NimGroth16BackendRef, proof: NimGroth16Proof
): Future[?!bool] {.async: (raises: [CancelledError]).} =
let
vKey = self.zkey.extractVKey
verified = ?verifyProof(vKey, proof).catch
success verified
proc new*(
_: type NimGroth16BackendRef,
graph: string,
r1csPath: string,
zkeyPath: string,
curve = DefaultCurve,
slotDepth = DefaultMaxSlotDepth,
datasetDepth = DefaultMaxDatasetDepth,
blkDepth = DefaultBlockDepth,
cellElms = DefaultCellElms,
numSamples = DefaultSamplesNum,
tp: Taskpool,
): ?!NimGroth16BackendRef =
## Create a new ctx
##
let
graph = ?loadGraph(graph).catch
r1cs = ?parseR1CS(r1csPath).catch
zkey = ?parseZKey(zkeyPath).catch
success NimGroth16BackendRef(
graph: graph,
r1cs: r1cs,
zkey: zkey,
slotDepth: slotDepth,
datasetDepth: datasetDepth,
blkDepth: blkDepth,
cellElms: cellElms,
numSamples: numSamples,
curve: curve,
tp: tp,
)

View File

@ -19,13 +19,13 @@ func toJsonDecimal*(big: BigInt[254]): string =
let s = big.toDecimal.strip(leading = true, trailing = false, chars = {'0'})
if s.len == 0: "0" else: s
func toJson*(g1: CircomG1): JsonNode =
func toJson*(g1: CircomCompatG1): JsonNode =
%*{
"x": Bn254Fr.fromBytes(g1.x).get.toBig.toJsonDecimal,
"y": Bn254Fr.fromBytes(g1.y).get.toBig.toJsonDecimal,
}
func toJson*(g2: CircomG2): JsonNode =
func toJson*(g2: CircomCompatG2): JsonNode =
%*{
"x": [
Bn254Fr.fromBytes(g2.x[0]).get.toBig.toJsonDecimal,
@ -38,8 +38,9 @@ func toJson*(g2: CircomG2): JsonNode =
}
proc toJson*(vpk: VerifyingKey): JsonNode =
let ic =
toSeq(cast[ptr UncheckedArray[CircomG1]](vpk.ic).toOpenArray(0, vpk.icLen.int - 1))
let ic = toSeq(
cast[ptr UncheckedArray[CircomCompatG1]](vpk.ic).toOpenArray(0, vpk.icLen.int - 1)
)
echo ic.len
%*{

View File

@ -24,7 +24,7 @@ suite "Test Circom Compat Backend - control inputs":
zkey = "tests/circuits/fixtures/proof_main.zkey"
var
circom: CircomCompat
circom: CircomCompatBackendRef
proofInputs: ProofInputs[Poseidon2Hash]
setup:
@ -33,22 +33,20 @@ suite "Test Circom Compat Backend - control inputs":
inputJson = !JsonNode.parse(inputData)
proofInputs = Poseidon2Hash.jsonToProofInput(inputJson)
circom = CircomCompat.init(r1cs, wasm, zkey)
circom = CircomCompatBackendRef.new(r1cs, wasm, zkey).tryGet
teardown:
circom.release() # this comes from the rust FFI
test "Should verify with correct inputs":
let proof = circom.prove(proofInputs).tryGet
check circom.verify(proof, proofInputs).tryGet
let proof = (await circom.prove(proofInputs)).tryGet
check (await circom.verify(proof, proofInputs)).tryGet
test "Should not verify with incorrect inputs":
proofInputs.slotIndex = 1 # change slot index
let proof = circom.prove(proofInputs).tryGet
check circom.verify(proof, proofInputs).tryGet == false
let proof = (await circom.prove(proofInputs)).tryGet
check (await circom.verify(proof, proofInputs)).tryGet == false
suite "Test Circom Compat Backend":
let
@ -72,7 +70,7 @@ suite "Test Circom Compat Backend":
manifest: Manifest
protected: Manifest
verifiable: Manifest
circom: CircomCompat
circom: CircomCompatBackendRef
proofInputs: ProofInputs[Poseidon2Hash]
challenge: array[32, byte]
builder: Poseidon2Builder
@ -92,7 +90,7 @@ suite "Test Circom Compat Backend":
builder = Poseidon2Builder.new(store, verifiable).tryGet
sampler = Poseidon2Sampler.new(slotId, store, builder).tryGet
circom = CircomCompat.init(r1cs, wasm, zkey)
circom = CircomCompatBackendRef.new(r1cs, wasm, zkey).tryGet
challenge = 1234567.toF.toBytes.toArray32
proofInputs = (await sampler.getProofInput(challenge, samples)).tryGet
@ -103,13 +101,11 @@ suite "Test Circom Compat Backend":
await metaTmp.destroyDb()
test "Should verify with correct input":
var proof = circom.prove(proofInputs).tryGet
check circom.verify(proof, proofInputs).tryGet
var proof = (await circom.prove(proofInputs)).tryGet
check (await circom.verify(proof, proofInputs)).tryGet
test "Should not verify with incorrect input":
proofInputs.slotIndex = 1 # change slot index
let proof = circom.prove(proofInputs).tryGet
check circom.verify(proof, proofInputs).tryGet == false
let proof = (await circom.prove(proofInputs)).tryGet
check (await circom.verify(proof, proofInputs)).tryGet == false

View File

@ -0,0 +1,119 @@
import std/options
import std/isolation
import ../../../asynctest
import pkg/chronos
import pkg/poseidon2
import pkg/serde/json
import pkg/taskpools
import pkg/codex/slots {.all.}
import pkg/codex/slots/types {.all.}
import pkg/codex/merkletree
import pkg/codex/merkletree/poseidon2
import pkg/codex/codextypes
import pkg/codex/manifest
import pkg/codex/stores
import pkg/groth16
import pkg/nim/circom_witnessgen
import pkg/nim/circom_witnessgen/load
import pkg/nim/circom_witnessgen/witness
import ./helpers
import ../helpers
import ../../helpers
suite "Test NimGoth16 Backend - control inputs":
let
graph = "tests/circuits/fixtures/proof_main.bin"
r1cs = "tests/circuits/fixtures/proof_main.r1cs"
zkey = "tests/circuits/fixtures/proof_main.zkey"
var
nimGroth16: NimGroth16BackendRef
proofInputs: ProofInputs[Poseidon2Hash]
setup:
let
inputData = readFile("tests/circuits/fixtures/input.json")
inputJson = !JsonNode.parse(inputData)
proofInputs = Poseidon2Hash.jsonToProofInput(inputJson)
nimGroth16 = NimGroth16BackendRef.new(graph, r1cs, zkey, tp = Taskpool.new()).tryGet
teardown:
nimGroth16.release()
test "Should verify with correct inputs":
let proof = (await nimGroth16.prove(proofInputs)).tryGet
check (await nimGroth16.verify(proof)).tryGet
# test "Should not verify with incorrect inputs":
# proofInputs.slotIndex = 1 # change slot index
# let proof = (await nimGroth16.prove(proofInputs)).tryGet
# check (await nimGroth16.verify(proof)).tryGet == false
# suite "Test NimGoth16 Backend":
# let
# ecK = 2
# ecM = 2
# slotId = 3
# samples = 5
# numDatasetBlocks = 8
# blockSize = DefaultBlockSize
# cellSize = DefaultCellSize
# graph = "tests/circuits/fixtures/proof_main.bin"
# r1cs = "tests/circuits/fixtures/proof_main.r1cs"
# zkey = "tests/circuits/fixtures/proof_main.zkey"
# repoTmp = TempLevelDb.new()
# metaTmp = TempLevelDb.new()
# var
# store: BlockStore
# manifest: Manifest
# protected: Manifest
# verifiable: Manifest
# nimGroth16: NimGroth16BackendRef
# proofInputs: ProofInputs[Poseidon2Hash]
# challenge: array[32, byte]
# builder: Poseidon2Builder
# sampler: Poseidon2Sampler
# setup:
# let
# repoDs = repoTmp.newDb()
# metaDs = metaTmp.newDb()
# store = RepoStore.new(repoDs, metaDs)
# (manifest, protected, verifiable) = await createVerifiableManifest(
# store, numDatasetBlocks, ecK, ecM, blockSize, cellSize
# )
# builder = Poseidon2Builder.new(store, verifiable).tryGet
# sampler = Poseidon2Sampler.new(slotId, store, builder).tryGet
# nimGroth16 = NimGroth16BackendRef.new(graph, r1cs, zkey, tp = Taskpool.new()).tryGet
# challenge = 1234567.toF.toBytes.toArray32
# proofInputs = (await sampler.getProofInput(challenge, samples)).tryGet
# teardown:
# nimGroth16.release()
# await repoTmp.destroyDb()
# await metaTmp.destroyDb()
# test "Should verify with correct input":
# var proof = (await nimGroth16.prove(proofInputs)).tryGet
# check (await nimGroth16.verify(proof)).tryGet
# test "Should not verify with incorrect input":
# proofInputs.slotIndex = 1 # change slot index
# let proof = (await nimGroth16.prove(proofInputs)).tryGet
# check (await nimGroth16.verify(proof)).tryGet == false