From 948f6c2c7562aa8fcabf044c927ce4a55f3d0fbf Mon Sep 17 00:00:00 2001 From: Balazs Komuves Date: Tue, 15 Oct 2024 11:52:40 +0200 Subject: [PATCH 1/5] add CLI and config for the new hash functions --- reference/nim/proof_input/README.md | 4 ++- reference/nim/proof_input/proof_input.nimble | 1 + reference/nim/proof_input/src/cli.nim | 22 ++++++++++++--- reference/nim/proof_input/src/testmain.nim | 12 ++++----- reference/nim/proof_input/src/types.nim | 28 ++++++++++++++++++++ 5 files changed, 57 insertions(+), 10 deletions(-) diff --git a/reference/nim/proof_input/README.md b/reference/nim/proof_input/README.md index 6993228..cd69add 100644 --- a/reference/nim/proof_input/README.md +++ b/reference/nim/proof_input/README.md @@ -29,6 +29,8 @@ use real data. -K, --ncells = : number of cells inside this slot (eg. 1024; must be a power of two) -o, --output = : the JSON file into which we write the proof input -C, --circom = : the circom main component to create with these parameters + -F, --field = : the underlying field: "bn254" or "goldilocks" + -H, --hash = : the hash function to use: "poseidon2" or "monolith" You can generate only the `.json` or only the `.circom` file, or both at the same time. @@ -39,4 +41,4 @@ NOTE: When using actual files for the slot data (which is untested :), you will need _all_ slots to be able to compute the dataset root. The convention used is that you specify the base name of the file, say `"slotdata"`; and the individual slots file names are derived from this to be for example `"slotdata5.dat"` for -the slot with index 5 (note: indexing starts from 0). \ No newline at end of file +the slot with index 5 (note: indexing starts from 0). diff --git a/reference/nim/proof_input/proof_input.nimble b/reference/nim/proof_input/proof_input.nimble index 114467c..0bb004b 100644 --- a/reference/nim/proof_input/proof_input.nimble +++ b/reference/nim/proof_input/proof_input.nimble @@ -9,3 +9,4 @@ bin = @["cli","testmain"] requires "nim >= 1.6.0" requires "https://github.com/mratsim/constantine#ab6fa6ae1bbbd1b10071a92ec209b381b5d82511" requires "https://github.com/codex-storage/nim-poseidon2#8a54c69032a741160bbc097d009e45a8b5e4d718" +requires "https://github.com/codex-storage/nim-goldilocks-hash" diff --git a/reference/nim/proof_input/src/cli.nim b/reference/nim/proof_input/src/cli.nim index 27eb4e2..c54567d 100644 --- a/reference/nim/proof_input/src/cli.nim +++ b/reference/nim/proof_input/src/cli.nim @@ -26,6 +26,7 @@ import misc #------------------------------------------------------------------------------- type FullConfig = object + hashCfg: HashConfig globCfg: GlobalConfig dsetCfg: DataSetConfig slotIndex: int @@ -34,11 +35,16 @@ type FullConfig = object circomFile: string verbose: bool +const defHashCfg = + HashConfig( field: Goldilocks # BN254 + , hashFun: Poseidon2 + ) + const defGlobCfg = GlobalConfig( maxDepth: 32 , maxLog2NSlots: 8 , cellSize: 2048 - , blockSize: 65536 + , blockSize: 65536 ) const defDSetCfg = @@ -49,7 +55,8 @@ const defDSetCfg = ) const defFullCfg = - FullConfig( globCfg: defGlobCfg + FullConfig( hashCfg: defHashCfg + , globCfg: defGlobCfg , dsetCfg: defDSetCfg , slotIndex: 0 , outFile: "" @@ -81,6 +88,8 @@ proc printHelp() = echo " -K, --ncells = : number of cells inside this slot (eg. 1024; must be a power of two)" echo " -o, --output = : the JSON file into which we write the proof input" echo " -C, --circom = : the circom main component to create with these parameters" + echo " -F, --field = : the underlying field: \"bn254\" or \"goldilocks\"" + echo " -H, --hash = : the hash function to use: \"poseidon2\" or \"monolith\"" echo "" quit() @@ -91,6 +100,7 @@ proc parseCliOptions(): FullConfig = var argCtr: int = 0 + var hashCfg = defHashCfg var globCfg = defGlobCfg var dsetCfg = defDSetCfg var fullCfg = defFullCfg @@ -123,6 +133,8 @@ proc parseCliOptions(): FullConfig = of "K", "ncells" : dsetCfg.ncells = checkPowerOfTwo(parseInt(value),"nCells") of "o", "output" : fullCfg.outFile = value of "C", "circom" : fullCfg.circomFile = value + of "F", "field" : hashCfg.field = parseField(value) + of "H", "hash" : hashCfg.hashFun = parseHashFun(value) else: echo "Unknown option: ", key echo "use --help to get a list of options" @@ -131,6 +143,7 @@ proc parseCliOptions(): FullConfig = of cmdEnd: discard + fullCfg.hashCfg = hashCfg fullCfg.globCfg = globCfg fullCfg.dsetCfg = dsetCfg @@ -140,9 +153,12 @@ proc parseCliOptions(): FullConfig = proc printConfig(fullCfg: FullConfig) = + let hashCfg = fullCfg.hashCfg let globCfg = fullCfg.globCfg let dsetCfg = fullCfg.dsetCfg + echo "field = " & ($hashCfg.field) + echo "hash func. = " & ($hashCfg.hashFun) echo "maxDepth = " & ($globCfg.maxDepth) echo "maxSlots = " & ($pow2(globCfg.maxLog2NSlots)) echo "cellSize = " & ($globCfg.cellSize) @@ -186,7 +202,7 @@ when isMainModule: printConfig(fullCfg) if fullCfg.circomFile == "" and fullCfg.outFile == "": - echo "nothing do!" + echo "nothing to do!" echo "use --help for getting a list of options" quit() diff --git a/reference/nim/proof_input/src/testmain.nim b/reference/nim/proof_input/src/testmain.nim index eeae932..4663d93 100644 --- a/reference/nim/proof_input/src/testmain.nim +++ b/reference/nim/proof_input/src/testmain.nim @@ -1,21 +1,21 @@ import sugar -import std/sequtils +#import std/sequtils -import constantine/math/arithmetic +#import constantine/math/arithmetic import poseidon2/types import poseidon2/merkle import poseidon2/io import types -import blocks -import slot -import dataset -import sample import merkle import gen_input import json +#import blocks +#import slot +#import dataset +#import sample #------------------------------------------------------------------------------- diff --git a/reference/nim/proof_input/src/types.nim b/reference/nim/proof_input/src/types.nim index 3866d42..6ef2a5e 100644 --- a/reference/nim/proof_input/src/types.nim +++ b/reference/nim/proof_input/src/types.nim @@ -106,6 +106,18 @@ type cellSize* : int # size of the cells we prove (2048) blockSize* : int # size of the network block (65536) + HashConfig* = object + field* : FieldSelect + hashFun* : HashSelect + + FieldSelect* = enum + BN254, + Goldilocks + + HashSelect* = enum + Poseidon2, + Monolith + #------------------------------------------------------------------------------- func cellsPerBlock*(glob: GlobalConfig): int = @@ -114,3 +126,19 @@ func cellsPerBlock*(glob: GlobalConfig): int = return k #------------------------------------------------------------------------------- + +func parseField*(str0: string): FieldSelect = + let str = strutils.toLowerAscii(str0) + case str: + of "bn254": return BN254 + of "goldilocks": return Goldilocks + else: raiseAssert("parsefield: unrecognized field `" & str0 & "`") + +func parseHashFun*(str0: string): HashSelect = + let str = strutils.toLowerAscii(str0) + case str: + of "poseidon2": return Poseidon2 + of "monolith": return Monolith + else: raiseAssert("parsefield: unrecognized hash function `" & str0 & "`") + +#------------------------------------------------------------------------------- From 14bab04b197f28f51cb0e8af160b5d0552243da3 Mon Sep 17 00:00:00 2001 From: Balazs Komuves Date: Fri, 18 Oct 2024 17:39:34 +0200 Subject: [PATCH 2/5] it almost compiles (but does not link/builds) after the big hack refactoring --- reference/nim/proof_input/.gitignore | 2 +- reference/nim/proof_input/proof_input.nimble | 6 +- .../src/{blocks.nim => blocks/bn254.nim} | 43 ++++--- .../nim/proof_input/src/blocks/goldilocks.nim | 76 ++++++++++++ reference/nim/proof_input/src/cli.nim | 49 +++++--- reference/nim/proof_input/src/gen_input.nim | 73 ----------- .../nim/proof_input/src/gen_input/bn254.nim | 81 +++++++++++++ .../proof_input/src/gen_input/goldilocks.nim | 87 ++++++++++++++ reference/nim/proof_input/src/json.nim | 113 ++---------------- reference/nim/proof_input/src/json/bn254.nim | 81 +++++++++++++ .../nim/proof_input/src/json/goldilocks.nim | 86 +++++++++++++ reference/nim/proof_input/src/json/shared.nim | 27 +++++ reference/nim/proof_input/src/merkle.nim | 105 +++++----------- .../nim/proof_input/src/merkle/bn254.nim | 65 ++++++++++ .../src/merkle/goldilocks/monolith.nim | 65 ++++++++++ .../src/merkle/goldilocks/poseidon2.nim | 65 ++++++++++ reference/nim/proof_input/src/sample.nim | 43 ------- .../nim/proof_input/src/sample/bn254.nim | 29 +++++ .../nim/proof_input/src/sample/goldilocks.nim | 40 +++++++ reference/nim/proof_input/src/slot.nim | 2 + reference/nim/proof_input/src/types.nim | 82 +++++++------ reference/nim/proof_input/src/types/bn254.nim | 59 +++++++++ .../nim/proof_input/src/types/goldilocks.nim | 44 +++++++ 23 files changed, 955 insertions(+), 368 deletions(-) rename reference/nim/proof_input/src/{blocks.nim => blocks/bn254.nim} (51%) create mode 100644 reference/nim/proof_input/src/blocks/goldilocks.nim delete mode 100644 reference/nim/proof_input/src/gen_input.nim create mode 100644 reference/nim/proof_input/src/gen_input/bn254.nim create mode 100644 reference/nim/proof_input/src/gen_input/goldilocks.nim create mode 100644 reference/nim/proof_input/src/json/bn254.nim create mode 100644 reference/nim/proof_input/src/json/goldilocks.nim create mode 100644 reference/nim/proof_input/src/json/shared.nim create mode 100644 reference/nim/proof_input/src/merkle/bn254.nim create mode 100644 reference/nim/proof_input/src/merkle/goldilocks/monolith.nim create mode 100644 reference/nim/proof_input/src/merkle/goldilocks/poseidon2.nim delete mode 100644 reference/nim/proof_input/src/sample.nim create mode 100644 reference/nim/proof_input/src/sample/bn254.nim create mode 100644 reference/nim/proof_input/src/sample/goldilocks.nim create mode 100644 reference/nim/proof_input/src/types/bn254.nim create mode 100644 reference/nim/proof_input/src/types/goldilocks.nim diff --git a/reference/nim/proof_input/.gitignore b/reference/nim/proof_input/.gitignore index e2cf927..e8d9984 100644 --- a/reference/nim/proof_input/.gitignore +++ b/reference/nim/proof_input/.gitignore @@ -2,4 +2,4 @@ testmain cli *.json -json/ \ No newline at end of file +json_tmp/ \ No newline at end of file diff --git a/reference/nim/proof_input/proof_input.nimble b/reference/nim/proof_input/proof_input.nimble index 0bb004b..cf8ac91 100644 --- a/reference/nim/proof_input/proof_input.nimble +++ b/reference/nim/proof_input/proof_input.nimble @@ -4,9 +4,11 @@ author = "Balazs Komuves" description = "reference implementation for generating the proof inputs" license = "MIT or Apache-2.0" srcDir = "src" -bin = @["cli","testmain"] +bin = @["cli"] +#bin = @["cli","testmain"] requires "nim >= 1.6.0" requires "https://github.com/mratsim/constantine#ab6fa6ae1bbbd1b10071a92ec209b381b5d82511" requires "https://github.com/codex-storage/nim-poseidon2#8a54c69032a741160bbc097d009e45a8b5e4d718" -requires "https://github.com/codex-storage/nim-goldilocks-hash" +requires "https://github.com/codex-storage/nim-goldilocks-hash#0d12b1429345c3f359df47171332e8966dc94ed3" +#requires "../../../../nim-goldilocks-hash/" diff --git a/reference/nim/proof_input/src/blocks.nim b/reference/nim/proof_input/src/blocks/bn254.nim similarity index 51% rename from reference/nim/proof_input/src/blocks.nim rename to reference/nim/proof_input/src/blocks/bn254.nim index 7b695cd..ca1b830 100644 --- a/reference/nim/proof_input/src/blocks.nim +++ b/reference/nim/proof_input/src/blocks/bn254.nim @@ -5,18 +5,28 @@ import std/sequtils #import poseidon2/types import poseidon2/io import poseidon2/sponge -import poseidon2/merkle +#import poseidon2/merkle -import types -import merkle +import ../types +import ../types/bn254 +#import ../merkle +import ../merkle/bn254 #------------------------------------------------------------------------------- -func hashCellOpen( globcfg: GlobalConfig, cellData: openArray[byte] ): Hash = +func merkleTree*( hashcfg: HashConfig, what: openarray[Hash]): MerkleTree[Hash] = + assert( hashcfg.combo == BN254_Poseidon2 ) + return merkleTreeBN254( what ) + +#------------------------------------------------------------------------------- + +func hashCellOpen( hashcfg: HashConfig, globcfg: GlobalConfig, cellData: openArray[byte] ): Hash = + assert( hashcfg.field == BN254 ) + assert( hashcfg.hashFun == Poseidon2 ) assert( cellData.len == globcfg.cellSize , ("cells are expected to be exactly " & $globcfg.cellSize & " bytes") ) return Sponge.digest( cellData, rate=2 ) -func hashCell*( globcfg: GlobalConfig, cellData: Cell): Hash = hashCellOpen(globcfg, cellData) +func hashCell*(hashcfg: HashConfig, globcfg: GlobalConfig, cellData: Cell): Hash = hashCellOpen(hashcfg, globcfg, cellData) #------------------------------------------------------------------------------- @@ -36,23 +46,24 @@ func splitBlockIntoCells( globcfg: GlobalConfig, blockData: openArray[byte] ): s # returns the special hash of a network block (this is a Merkle root built on the # top of the hashes of the 32 cells inside the block) -func hashNetworkBlockOpen( globcfg: GlobalConfig, blockData: openArray[byte] ): Hash = +func hashNetworkBlockOpen( hashcfg: HashConfig, globcfg: GlobalConfig, blockData: openArray[byte] ): Hash = let cells = splitBlockIntoCells(globcfg, blockData) - let leaves = collect( newSeq , (for i in 0..= 0 and index < nleaves ) - var path : seq[Hash] = newSeq[Hash](depth) + var path : seq[H] = newSeq[H](depth) var k = index var m = nleaves for i in 0..0 and k<=64 ) - var r : uint64 = 0 - for i in 0.. Seed -> CellIdx -> CellData genFakeCell cfg (Seed seed) (CellIdx idx) = (mkCellData cfg $ B.pack list) where list = go (fromIntegral $ _cellSize cfg) 1 diff --git a/reference/nim/proof_input/src/types.nim b/reference/nim/proof_input/src/types.nim index 6ef2a5e..6bc68aa 100644 --- a/reference/nim/proof_input/src/types.nim +++ b/reference/nim/proof_input/src/types.nim @@ -2,25 +2,6 @@ import std/strutils import std/sequtils -from constantine/math/io/io_fields import toDecimal - -import poseidon2/types -export types - -#------------------------------------------------------------------------------- - -type Entropy* = F -type Hash* = F -type Root* = Hash - -#------------------------------------------------------------------------------- - -func toDecimalF*(a : F): string = - var s : string = toDecimal(a) - s = s.strip( leading=true, trailing=false, chars={'0'} ) - if s.len == 0: s="0" - return s - #------------------------------------------------------------------------------- type Cell* = seq[byte] @@ -30,27 +11,30 @@ type Block* = seq[byte] type - MerkleProof* = object + MerkleProof*[H] = object leafIndex* : int # linear index of the leaf, starting from 0 - leafValue* : Hash # value of the leaf - merklePath* : seq[Hash] # order: from the bottom to the top + leafValue* : H # value of the leaf + merklePath* : seq[H] # order: from the bottom to the top numberOfLeaves* : int # number of leaves in the tree (=size of input) - MerkleTree* = object - layers*: seq[seq[Hash]] + MerkleTree*[H] = object + layers*: seq[seq[H]] # ^^^ note: the first layer is the bottom layer, and the last layer is the root #------------------------------------------------------------------------------- # the circuit expect merkle path of statically known length, so we need to pad them -func padMerkleProof*( old: MerkleProof, newlen: int ): MerkleProof = +func padMerkleProof*[H]( old: MerkleProof[H], newlen: int ): MerkleProof[H] = let pad = newlen - old.merklePath.len assert( pad >= 0 ) - return MerkleProof( leafIndex: old.leafIndex - , leafValue: old.leafValue - , merklePath: old.merklePath & repeat(zero,pad) - , numberOfLeaves: old.numberOfLeaves - ) + + var zero : H # hackety hack hack, it should be initialized to zero + + return MerkleProof[H]( leafIndex: old.leafIndex + , leafValue: old.leafValue + , merklePath: old.merklePath & repeat(zero,pad) + , numberOfLeaves: old.numberOfLeaves + ) #------------------------------------------------------------------------------- @@ -61,19 +45,19 @@ type BlockIdx* = int SlotIdx* = int - CellProofInput* = object + CellProofInput*[H] = object cellData*: Cell - merkleProof*: MerkleProof + merkleProof*: MerkleProof[H] - SlotProofInput* = object - dataSetRoot*: Root - entropy*: Entropy + SlotProofInput*[H] = object + dataSetRoot*: H # Root + entropy*: H # Entropy nSlots*: int nCells*: int - slotRoot*: Root + slotRoot*: H # Root slotIndex*: SlotIdx - slotProof*: MerkleProof - proofInputs*: seq[CellProofInput] + slotProof*: MerkleProof[H] + proofInputs*: seq[CellProofInput[H]] #------------------------------------------------------------------------------- @@ -109,6 +93,7 @@ type HashConfig* = object field* : FieldSelect hashFun* : HashSelect + combo* : FieldHashCombo FieldSelect* = enum BN254, @@ -118,6 +103,11 @@ type Poseidon2, Monolith + FieldHashCombo* = enum + BN254_Poseidon2, + Goldilocks_Poseidon2, + Goldilocks_Monolith + #------------------------------------------------------------------------------- func cellsPerBlock*(glob: GlobalConfig): int = @@ -142,3 +132,19 @@ func parseHashFun*(str0: string): HashSelect = else: raiseAssert("parsefield: unrecognized hash function `" & str0 & "`") #------------------------------------------------------------------------------- + +{. warning[UnreachableElse]:off .} +func toFieldHashCombo*( field: FieldSelect, hash: HashSelect ): FieldHashCombo = + let msg = "invalid hash function `" & ($hash) & "` choice for field `" & ($field) & "`" + case field: + of BN254: + case hash: + of Poseidon2: return BN254_Poseidon2 + else: raiseAssert(msg) + of Goldilocks: + case hash: + of Poseidon2: return Goldilocks_Poseidon2 + of Monolith: return Goldilocks_Monolith + else: raiseAssert(msg) + +#------------------------------------------------------------------------------- diff --git a/reference/nim/proof_input/src/types/bn254.nim b/reference/nim/proof_input/src/types/bn254.nim new file mode 100644 index 0000000..fac932a --- /dev/null +++ b/reference/nim/proof_input/src/types/bn254.nim @@ -0,0 +1,59 @@ + +import std/strutils +import std/bitops +import std/streams + +import + constantine/math/arithmetic, + constantine/math/io/io_fields, + constantine/math/io/io_bigints, + constantine/math/config/curves + +#from constantine/math/io/io_fields import toDecimal + +import poseidon2/types +import poseidon2/io +export types + +#------------------------------------------------------------------------------- + +type BN254_T* = F +type Entropy* = F +type Hash* = F +type Root* = Hash + +#------------------------------------------------------------------------------- + +func intToBN254*(x: int): F = toF(x) + +func toDecimalF*(a : F): string = + var s : string = toDecimal(a) + s = s.strip( leading=true, trailing=false, chars={'0'} ) + if s.len == 0: s="0" + return s + +func toQuotedDecimalF*(x: F): string = + let s : string = toDecimalF(x) + return ("\"" & s & "\"") + +proc writeF*(h: Stream, prefix: string, x: F) = + h.writeLine(prefix & toQuotedDecimalF(x)) + +#------------------------------------------------------------------------------- + +func extractLowBits[n: static int]( A: BigInt[n], k: int): uint64 = + assert( k>0 and k<=64 ) + var r : uint64 = 0 + for i in 0..0 and k<=56 ) + let val : uint64 = fromF(fld) + let mask : uint64 = (1'u64 shl k) - 1 + return bitand(val, mask) + +#------------------------------------------------------------------------------- + +func digestToJsonString*( d: Digest ): string = + let xs: F4 = fromDigest(d) + return "[ " & toDecimalF(xs[0]) & ", " & + toDecimalF(xs[1]) & ", " & + toDecimalF(xs[2]) & ", " & + toDecimalF(xs[3]) & " ]" \ No newline at end of file From 1c51aca81be560bcd13b77af07afa41b280009c1 Mon Sep 17 00:00:00 2001 From: Balazs Komuves Date: Tue, 22 Oct 2024 12:37:34 +0200 Subject: [PATCH 3/5] more streamlined json output for the goldilocks field (4 field elements per line) --- reference/nim/proof_input/proof_input.nimble | 4 ++-- reference/nim/proof_input/src/json/bn254.nim | 2 +- .../nim/proof_input/src/json/goldilocks.nim | 17 ++++++++++---- reference/nim/proof_input/src/json/shared.nim | 23 +++++++++++++++++++ reference/nim/proof_input/src/types/bn254.nim | 5 +++- .../nim/proof_input/src/types/goldilocks.nim | 15 ++++++++---- 6 files changed, 52 insertions(+), 14 deletions(-) diff --git a/reference/nim/proof_input/proof_input.nimble b/reference/nim/proof_input/proof_input.nimble index cf8ac91..d6a7d6b 100644 --- a/reference/nim/proof_input/proof_input.nimble +++ b/reference/nim/proof_input/proof_input.nimble @@ -10,5 +10,5 @@ bin = @["cli"] requires "nim >= 1.6.0" requires "https://github.com/mratsim/constantine#ab6fa6ae1bbbd1b10071a92ec209b381b5d82511" requires "https://github.com/codex-storage/nim-poseidon2#8a54c69032a741160bbc097d009e45a8b5e4d718" -requires "https://github.com/codex-storage/nim-goldilocks-hash#0d12b1429345c3f359df47171332e8966dc94ed3" -#requires "../../../../nim-goldilocks-hash/" +requires "https://github.com/codex-storage/nim-goldilocks-hash#bd5b805b80b6005a3e5de412dec15783284d205d" +#requires "goldilocks_hash == 0.0.1" diff --git a/reference/nim/proof_input/src/json/bn254.nim b/reference/nim/proof_input/src/json/bn254.nim index 997b7e8..34b7c35 100644 --- a/reference/nim/proof_input/src/json/bn254.nim +++ b/reference/nim/proof_input/src/json/bn254.nim @@ -17,7 +17,7 @@ import shared #------------------------------------------------------------------------------- proc writeFieldElems(h: Stream, prefix: string, xs: seq[F]) = - writeList[F]( h, prefix, xs, writeF ) + writeList[F]( h, prefix, xs, writeLnF ) #------------------------------------------------------------------------------- diff --git a/reference/nim/proof_input/src/json/goldilocks.nim b/reference/nim/proof_input/src/json/goldilocks.nim index b76749f..2784789 100644 --- a/reference/nim/proof_input/src/json/goldilocks.nim +++ b/reference/nim/proof_input/src/json/goldilocks.nim @@ -20,16 +20,23 @@ func bytesToFieldElements( bytes: openArray[byte] ): seq[F] = let digests = padAndDecodeBytesToDigest62(bytes) return digestSeqToFeltSeq(digests) +func bytesToFieldElementsMat( bytes: openArray[byte] ): seq[seq[F]] = + let digests = padAndDecodeBytesToDigest62(bytes) + return digestSeqToFeltSeqSeq(digests) + #------------------------------------------------------------------------------- proc writeFieldElems(h: Stream, prefix: string, xs: seq[F]) = - writeList[F]( h, prefix, xs, writeF ) + writeList[F]( h, prefix, xs, writeLnF ) + +proc writeFieldElemsMat(h: Stream, prefix: string, xs: seq[seq[F]]) = + writeListList[F]( h, prefix, xs, writeF ) #------------------------------------------------------------------------------- proc writeSingleCellData(h: Stream, prefix:string , cell: Cell) = - let flds : seq[F] = bytesToFieldElements(cell) # cell.elements(F).toSeq() - writeFieldElems(h, prefix, flds) + let flds : seq[seq[F]] = bytesToFieldElementsMat(cell) + writeFieldElemsMat(h, prefix, flds) proc writeAllCellData(h: Stream, cells: seq[Cell]) = writeList(h, " ", cells, writeSingleCellData ) @@ -37,8 +44,8 @@ proc writeAllCellData(h: Stream, cells: seq[Cell]) = #------------------------------------------------------------------------------- proc writeSingleMerklePath(h: Stream, prefix: string, path: MerkleProof[Digest]) = - let flds : seq[F] = digestSeqToFeltSeq( path.merklePath ) - writeFieldElems(h, prefix, flds) + let flds : seq[seq[F]] = digestSeqToFeltSeqSeq( path.merklePath ) + writeFieldElemsMat(h, prefix, flds) proc writeAllMerklePaths(h: Stream, paths: seq[MerkleProof[Digest]]) = writeList(h, " ", paths, writeSingleMerklePath ) diff --git a/reference/nim/proof_input/src/json/shared.nim b/reference/nim/proof_input/src/json/shared.nim index 1fb0217..dc72af6 100644 --- a/reference/nim/proof_input/src/json/shared.nim +++ b/reference/nim/proof_input/src/json/shared.nim @@ -24,4 +24,27 @@ proc writeList*[T](h: Stream, prefix: string, xs: seq[T], writeFun: WriteFun[T]) writeFun(h, indent & ", ", xs[i]) h.writeLine( indent & "]" ) +#--------------------------------------- + +proc writeListList*[T](h: Stream, prefix: string, xs: seq[seq[T]], writeFun: WriteFun[T]) = + let n = xs.len + let indent = mkIndent(prefix) + for i in 0.. Date: Fri, 15 Nov 2024 16:30:35 +0100 Subject: [PATCH 4/5] update the workflow scripts and benchmarks --- workflow/.gitignore | 1 + workflow/README.md | 30 +++++++++++++++++++++++------- workflow/cli_args.sh | 4 +++- workflow/prove.sh | 13 ++++++++++++- workflow/setup.sh | 7 ++++++- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/workflow/.gitignore b/workflow/.gitignore index c77d435..659e6b8 100644 --- a/workflow/.gitignore +++ b/workflow/.gitignore @@ -1,3 +1,4 @@ build/ build_big/ +build*/ tmp/ \ No newline at end of file diff --git a/workflow/README.md b/workflow/README.md index 0e21286..f9c1de1 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -16,28 +16,44 @@ NOTE: the examples below assume `bash`. In particular, it won't work with `zsh` To have an overview of what all the different steps and files are, see [PROOFS.md](PROOFS.md). -### Some measurements +### Some benchmarks Approximate time to run this on an M2 (8+4 cores), with 10 samples: - compiling the circuit: 8 seconds - circuit-specific setup (with 1 contributor): 85 seconds - size of the `.zkey` file (only 1 contributor): 110 megabytes +- generating the witness (WASM): 0.3 seconds - proving with `snarkjs` (slow): 7.7 seconds +- proving wiht `zikkurat` (single threaded): 13 seconds +- proving with `nim-groth16` (old version): 2 seconds Same with 50 samples: - compiling: 37 seconds -- circuit-specific setup: ~420 seconds +- circuit-specific setup: ~430 seconds - `.zkey` file: 525 megabytes -- snarkjs prove: 34 seconds +- generating the witness (WASM): 1.2 seconds +- proving with `snarkjs`: 36 seconds +- proving wiht `zikkurat` (single threaded): 52 seconds +- proving with `nim-groth16` (old version): 9.4 seconds And with 100 samples: -- compiling: 76 seconds -- circuit-specific setup: ~1000 seconds -- `.zkey` file -- snarkjs prove: 76 seconds +- compiling: 76 seconds +- circuit-specific setup: ~1050 seconds +- `.zkey` file +- generating the witness (WASM): 2.3 seconds +- proving with `snarkjs`: 76 seconds +- proving wiht `zikkurat` (single threaded): 102 seconds +- proving with `nim-groth16` (old version): 18 seconds + +TODO: + +- add `arkworks` prover +- add `rapidsnarks` prover (doesn't run on ARM) +- update `nim-groth16` to `constantine-0.1` (should be faster because no workarounds) +- add multithreading to `zikkurat` ### Preliminaries diff --git a/workflow/cli_args.sh b/workflow/cli_args.sh index 6ea183b..525791c 100755 --- a/workflow/cli_args.sh +++ b/workflow/cli_args.sh @@ -13,7 +13,9 @@ CLI_ARGS="--depth=$MAXDEPTH \ --seed=$SEED \ --nslots=$NSLOTS \ --ncells=$NCELLS \ - --index=$SLOTINDEX" + --index=$SLOTINDEX \ + --field=bn254 \ + --hash=poseidon2" if [[ "$1" == "--export" ]] then diff --git a/workflow/prove.sh b/workflow/prove.sh index 8fcf7d5..cfd66c8 100755 --- a/workflow/prove.sh +++ b/workflow/prove.sh @@ -16,16 +16,18 @@ ${NIMCLI_DIR}/cli $CLI_ARGS -v --output=input.json # --- generate the witness --- +start=`date +%s` echo "" echo "generating the witness..." cd ${CIRCUIT_MAIN}_js time node generate_witness.js ${CIRCUIT_MAIN}.wasm ../input.json ../witness.wtns cd ${ORIG}/build +end=`date +%s` +echo "Generating the witness took `expr $end - $start` seconds." # --- create the proof --- PROVER="snarkjs" -# PROVER="nim" RS=`which rapidsnark` if [[ ! -z "$RS" ]] @@ -33,9 +35,13 @@ then PROVER="rapidsnark" fi +# PROVER="zikkurat" +PROVER="nim" + echo "" echo "creating the proof... using prover: \`$PROVER\`" +start=`date +%s` case $PROVER in snarkjs) time snarkjs groth16 prove ${CIRCUIT_MAIN}.zkey witness.wtns proof.json public.json @@ -46,11 +52,16 @@ case $PROVER in nim) time nim-groth16 -tpv --zkey=${CIRCUIT_MAIN}.zkey --wtns=witness.wtns -o=proof.json -i=public.json ;; + zikkurat) + time zikkurat-groth16 -tpv --zkey=${CIRCUIT_MAIN}.zkey --wtns=witness.wtns # -o=proof.json -i=public.json + ;; *) echo "unknown prover \`$PROVER\`" exit 99 ;; esac +end=`date +%s` +echo "Creating the proof took `expr $end - $start` seconds." # --- verify the proof --- diff --git a/workflow/setup.sh b/workflow/setup.sh index 076fe65..bc0d9dc 100755 --- a/workflow/setup.sh +++ b/workflow/setup.sh @@ -14,8 +14,13 @@ ${NIMCLI_DIR}/cli $CLI_ARGS -v --circom=${CIRCUIT_MAIN}.circom # --- compile the circuit --- +echo "" +start=`date +%s` CIRCUIT_INCLUDES="-l${CIRCUIT_LIB_DIR} -l${CIRCUIT_POS_DIR} -l${CIRCUIT_PRF_DIR}" -time circom --r1cs --wasm --O2 ${CIRCUIT_INCLUDES} ${CIRCUIT_MAIN}.circom +circom --r1cs --wasm --O2 ${CIRCUIT_INCLUDES} ${CIRCUIT_MAIN}.circom +end=`date +%s` +echo "Compiling the circuit took `expr $end - $start` seconds." +echo "" # --- circuit specific setup --- From d3cfb2035f681bbff76c56d0448f077ee88bb4f6 Mon Sep 17 00:00:00 2001 From: Balazs Komuves Date: Fri, 15 Nov 2024 18:31:13 +0100 Subject: [PATCH 5/5] add arkworks prover measurements --- workflow/README.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index f9c1de1..1560bf3 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -18,14 +18,15 @@ To have an overview of what all the different steps and files are, see [PROOFS.m ### Some benchmarks -Approximate time to run this on an M2 (8+4 cores), with 10 samples: +Approximate time to run this on an M2 macbook pro (8+4 cores), with 10 samples: - compiling the circuit: 8 seconds - circuit-specific setup (with 1 contributor): 85 seconds - size of the `.zkey` file (only 1 contributor): 110 megabytes - generating the witness (WASM): 0.3 seconds - proving with `snarkjs` (slow): 7.7 seconds -- proving wiht `zikkurat` (single threaded): 13 seconds +- proving wiht `zikkurat` (single threaded!): 13 seconds +- proving with `arkworks`: 4.4 seconds (loading the zkey: 6 seconds) - proving with `nim-groth16` (old version): 2 seconds Same with 50 samples: @@ -35,7 +36,8 @@ Same with 50 samples: - `.zkey` file: 525 megabytes - generating the witness (WASM): 1.2 seconds - proving with `snarkjs`: 36 seconds -- proving wiht `zikkurat` (single threaded): 52 seconds +- proving wiht `zikkurat` (single threaded!): 52 seconds +- proving with `arkworks`: 19.8 seconds (loading the zkey: 33 seconds) - proving with `nim-groth16` (old version): 9.4 seconds And with 100 samples: @@ -45,15 +47,16 @@ And with 100 samples: - `.zkey` file - generating the witness (WASM): 2.3 seconds - proving with `snarkjs`: 76 seconds -- proving wiht `zikkurat` (single threaded): 102 seconds +- proving wiht `zikkurat` (single threaded!): 102 seconds +- proving with `arkworks`: 41 seconds (loading the zkey: 66 seconds) - proving with `nim-groth16` (old version): 18 seconds TODO: -- add `arkworks` prover -- add `rapidsnarks` prover (doesn't run on ARM) -- update `nim-groth16` to `constantine-0.1` (should be faster because no workarounds) -- add multithreading to `zikkurat` +- [x] add `arkworks` prover +- [ ] add `rapidsnarks` prover (doesn't run on ARM...) +- [ ] update `nim-groth16` to `constantine-0.1` (should be faster because no workarounds) +- [ ] add multithreading to `zikkurat` ### Preliminaries