diff --git a/reference/nim/proof_input/.gitignore b/reference/nim/proof_input/.gitignore index e2cf927..e8d9984 100644 --- a/reference/nim/proof_input/.gitignore +++ b/reference/nim/proof_input/.gitignore @@ -2,4 +2,4 @@ testmain cli *.json -json/ \ No newline at end of file +json_tmp/ \ No newline at end of file diff --git a/reference/nim/proof_input/README.md b/reference/nim/proof_input/README.md index 6993228..cd69add 100644 --- a/reference/nim/proof_input/README.md +++ b/reference/nim/proof_input/README.md @@ -29,6 +29,8 @@ use real data. -K, --ncells = : number of cells inside this slot (eg. 1024; must be a power of two) -o, --output = : the JSON file into which we write the proof input -C, --circom = : the circom main component to create with these parameters + -F, --field = : the underlying field: "bn254" or "goldilocks" + -H, --hash = : the hash function to use: "poseidon2" or "monolith" You can generate only the `.json` or only the `.circom` file, or both at the same time. @@ -39,4 +41,4 @@ NOTE: When using actual files for the slot data (which is untested :), you will need _all_ slots to be able to compute the dataset root. The convention used is that you specify the base name of the file, say `"slotdata"`; and the individual slots file names are derived from this to be for example `"slotdata5.dat"` for -the slot with index 5 (note: indexing starts from 0). \ No newline at end of file +the slot with index 5 (note: indexing starts from 0). diff --git a/reference/nim/proof_input/proof_input.nimble b/reference/nim/proof_input/proof_input.nimble index 114467c..d6a7d6b 100644 --- a/reference/nim/proof_input/proof_input.nimble +++ b/reference/nim/proof_input/proof_input.nimble @@ -4,8 +4,11 @@ author = "Balazs Komuves" description = "reference implementation for generating the proof inputs" license = "MIT or Apache-2.0" srcDir = "src" -bin = @["cli","testmain"] +bin = @["cli"] +#bin = @["cli","testmain"] requires "nim >= 1.6.0" requires "https://github.com/mratsim/constantine#ab6fa6ae1bbbd1b10071a92ec209b381b5d82511" requires "https://github.com/codex-storage/nim-poseidon2#8a54c69032a741160bbc097d009e45a8b5e4d718" +requires "https://github.com/codex-storage/nim-goldilocks-hash#bd5b805b80b6005a3e5de412dec15783284d205d" +#requires "goldilocks_hash == 0.0.1" diff --git a/reference/nim/proof_input/src/blocks.nim b/reference/nim/proof_input/src/blocks/bn254.nim similarity index 51% rename from reference/nim/proof_input/src/blocks.nim rename to reference/nim/proof_input/src/blocks/bn254.nim index 7b695cd..ca1b830 100644 --- a/reference/nim/proof_input/src/blocks.nim +++ b/reference/nim/proof_input/src/blocks/bn254.nim @@ -5,18 +5,28 @@ import std/sequtils #import poseidon2/types import poseidon2/io import poseidon2/sponge -import poseidon2/merkle +#import poseidon2/merkle -import types -import merkle +import ../types +import ../types/bn254 +#import ../merkle +import ../merkle/bn254 #------------------------------------------------------------------------------- -func hashCellOpen( globcfg: GlobalConfig, cellData: openArray[byte] ): Hash = +func merkleTree*( hashcfg: HashConfig, what: openarray[Hash]): MerkleTree[Hash] = + assert( hashcfg.combo == BN254_Poseidon2 ) + return merkleTreeBN254( what ) + +#------------------------------------------------------------------------------- + +func hashCellOpen( hashcfg: HashConfig, globcfg: GlobalConfig, cellData: openArray[byte] ): Hash = + assert( hashcfg.field == BN254 ) + assert( hashcfg.hashFun == Poseidon2 ) assert( cellData.len == globcfg.cellSize , ("cells are expected to be exactly " & $globcfg.cellSize & " bytes") ) return Sponge.digest( cellData, rate=2 ) -func hashCell*( globcfg: GlobalConfig, cellData: Cell): Hash = hashCellOpen(globcfg, cellData) +func hashCell*(hashcfg: HashConfig, globcfg: GlobalConfig, cellData: Cell): Hash = hashCellOpen(hashcfg, globcfg, cellData) #------------------------------------------------------------------------------- @@ -36,23 +46,24 @@ func splitBlockIntoCells( globcfg: GlobalConfig, blockData: openArray[byte] ): s # returns the special hash of a network block (this is a Merkle root built on the # top of the hashes of the 32 cells inside the block) -func hashNetworkBlockOpen( globcfg: GlobalConfig, blockData: openArray[byte] ): Hash = +func hashNetworkBlockOpen( hashcfg: HashConfig, globcfg: GlobalConfig, blockData: openArray[byte] ): Hash = let cells = splitBlockIntoCells(globcfg, blockData) - let leaves = collect( newSeq , (for i in 0.. : number of cells inside this slot (eg. 1024; must be a power of two)" echo " -o, --output = : the JSON file into which we write the proof input" echo " -C, --circom = : the circom main component to create with these parameters" + echo " -F, --field = : the underlying field: \"bn254\" or \"goldilocks\"" + echo " -H, --hash = : the hash function to use: \"poseidon2\" or \"monolith\"" echo "" quit() @@ -91,6 +110,7 @@ proc parseCliOptions(): FullConfig = var argCtr: int = 0 + var hashCfg = defHashCfg var globCfg = defGlobCfg var dsetCfg = defDSetCfg var fullCfg = defFullCfg @@ -123,6 +143,8 @@ proc parseCliOptions(): FullConfig = of "K", "ncells" : dsetCfg.ncells = checkPowerOfTwo(parseInt(value),"nCells") of "o", "output" : fullCfg.outFile = value of "C", "circom" : fullCfg.circomFile = value + of "F", "field" : hashCfg.field = parseField(value) + of "H", "hash" : hashCfg.hashFun = parseHashFun(value) else: echo "Unknown option: ", key echo "use --help to get a list of options" @@ -131,6 +153,9 @@ proc parseCliOptions(): FullConfig = of cmdEnd: discard + hashCfg.combo = toFieldHashCombo( hashCfg.field , hashCfg.hashFun ) + + fullCfg.hashCfg = hashCfg fullCfg.globCfg = globCfg fullCfg.dsetCfg = dsetCfg @@ -140,9 +165,12 @@ proc parseCliOptions(): FullConfig = proc printConfig(fullCfg: FullConfig) = + let hashCfg = fullCfg.hashCfg let globCfg = fullCfg.globCfg let dsetCfg = fullCfg.dsetCfg + echo "field = " & ($hashCfg.field) + echo "hash func. = " & ($hashCfg.hashFun) echo "maxDepth = " & ($globCfg.maxDepth) echo "maxSlots = " & ($pow2(globCfg.maxLog2NSlots)) echo "cellSize = " & ($globCfg.cellSize) @@ -180,13 +208,13 @@ proc writeCircomMainComponent(fullCfg: FullConfig, fname: string) = when isMainModule: let fullCfg = parseCliOptions() - # echo fullCfg + let hashCfg = fullCfg.hashCfg if fullCfg.verbose: printConfig(fullCfg) if fullCfg.circomFile == "" and fullCfg.outFile == "": - echo "nothing do!" + echo "nothing to do!" echo "use --help for getting a list of options" quit() @@ -196,7 +224,14 @@ when isMainModule: if fullCfg.outFile != "": echo "writing proof input into `" & fullCfg.outFile & "`..." - let prfInput = generateProofInput( fullCfg.globCfg, fullCfg.dsetCfg, fullCfg.slotIndex, toF(fullCfg.entropy) ) - exportProofInput( fullCfg.outFile , prfInput ) + case hashCfg.field + of BN254: + let entropy = intToBN254(fullCfg.entropy) + let prfInput = generateProofInputBN254( hashCfg, fullCfg.globCfg, fullCfg.dsetCfg, fullCfg.slotIndex, entropy ) + exportProofInputBN254( hashCfg, fullCfg.outFile , prfInput ) + of Goldilocks: + let entropy = intToDigest(fullCfg.entropy) + let prfInput = generateProofInputGoldilocks( hashCfg, fullCfg.globCfg, fullCfg.dsetCfg, fullCfg.slotIndex, entropy ) + exportProofInputGoldilocks( hashCfg, fullCfg.outFile , prfInput ) echo "done" diff --git a/reference/nim/proof_input/src/gen_input.nim b/reference/nim/proof_input/src/gen_input.nim deleted file mode 100644 index 15c1f28..0000000 --- a/reference/nim/proof_input/src/gen_input.nim +++ /dev/null @@ -1,73 +0,0 @@ - -# -# generate the input data for the proof -# see `json.nim` to export it in Snarkjs-compatible format -# - -import sugar -import std/sequtils - -import blocks -import slot -import dataset -import sample -import merkle -import types - -#------------------------------------------------------------------------------- - -proc buildSlotTreeFull( globcfg: GlobalConfig, slotCfg: SlotConfig ): (seq[MerkleTree], MerkleTree) = - let ncells = slotCfg.nCells - let nblocks = ncells div cellsPerBlock(globcfg) - assert( nblocks * cellsPerBlock(globcfg) == ncells ) - let blocks : seq[Block] = collect( newSeq, (for i in 0..= 0 and index < nleaves ) - var path : seq[Hash] = newSeq[Hash](depth) + var path : seq[H] = newSeq[H](depth) var k = index var m = nleaves for i in 0..0 and k<=64 ) - var r : uint64 = 0 - for i in 0.. Seed -> CellIdx -> CellData genFakeCell cfg (Seed seed) (CellIdx idx) = (mkCellData cfg $ B.pack list) where list = go (fromIntegral $ _cellSize cfg) 1 diff --git a/reference/nim/proof_input/src/testmain.nim b/reference/nim/proof_input/src/testmain.nim index eeae932..4663d93 100644 --- a/reference/nim/proof_input/src/testmain.nim +++ b/reference/nim/proof_input/src/testmain.nim @@ -1,21 +1,21 @@ import sugar -import std/sequtils +#import std/sequtils -import constantine/math/arithmetic +#import constantine/math/arithmetic import poseidon2/types import poseidon2/merkle import poseidon2/io import types -import blocks -import slot -import dataset -import sample import merkle import gen_input import json +#import blocks +#import slot +#import dataset +#import sample #------------------------------------------------------------------------------- diff --git a/reference/nim/proof_input/src/types.nim b/reference/nim/proof_input/src/types.nim index 3866d42..6bc68aa 100644 --- a/reference/nim/proof_input/src/types.nim +++ b/reference/nim/proof_input/src/types.nim @@ -2,25 +2,6 @@ import std/strutils import std/sequtils -from constantine/math/io/io_fields import toDecimal - -import poseidon2/types -export types - -#------------------------------------------------------------------------------- - -type Entropy* = F -type Hash* = F -type Root* = Hash - -#------------------------------------------------------------------------------- - -func toDecimalF*(a : F): string = - var s : string = toDecimal(a) - s = s.strip( leading=true, trailing=false, chars={'0'} ) - if s.len == 0: s="0" - return s - #------------------------------------------------------------------------------- type Cell* = seq[byte] @@ -30,27 +11,30 @@ type Block* = seq[byte] type - MerkleProof* = object + MerkleProof*[H] = object leafIndex* : int # linear index of the leaf, starting from 0 - leafValue* : Hash # value of the leaf - merklePath* : seq[Hash] # order: from the bottom to the top + leafValue* : H # value of the leaf + merklePath* : seq[H] # order: from the bottom to the top numberOfLeaves* : int # number of leaves in the tree (=size of input) - MerkleTree* = object - layers*: seq[seq[Hash]] + MerkleTree*[H] = object + layers*: seq[seq[H]] # ^^^ note: the first layer is the bottom layer, and the last layer is the root #------------------------------------------------------------------------------- # the circuit expect merkle path of statically known length, so we need to pad them -func padMerkleProof*( old: MerkleProof, newlen: int ): MerkleProof = +func padMerkleProof*[H]( old: MerkleProof[H], newlen: int ): MerkleProof[H] = let pad = newlen - old.merklePath.len assert( pad >= 0 ) - return MerkleProof( leafIndex: old.leafIndex - , leafValue: old.leafValue - , merklePath: old.merklePath & repeat(zero,pad) - , numberOfLeaves: old.numberOfLeaves - ) + + var zero : H # hackety hack hack, it should be initialized to zero + + return MerkleProof[H]( leafIndex: old.leafIndex + , leafValue: old.leafValue + , merklePath: old.merklePath & repeat(zero,pad) + , numberOfLeaves: old.numberOfLeaves + ) #------------------------------------------------------------------------------- @@ -61,19 +45,19 @@ type BlockIdx* = int SlotIdx* = int - CellProofInput* = object + CellProofInput*[H] = object cellData*: Cell - merkleProof*: MerkleProof + merkleProof*: MerkleProof[H] - SlotProofInput* = object - dataSetRoot*: Root - entropy*: Entropy + SlotProofInput*[H] = object + dataSetRoot*: H # Root + entropy*: H # Entropy nSlots*: int nCells*: int - slotRoot*: Root + slotRoot*: H # Root slotIndex*: SlotIdx - slotProof*: MerkleProof - proofInputs*: seq[CellProofInput] + slotProof*: MerkleProof[H] + proofInputs*: seq[CellProofInput[H]] #------------------------------------------------------------------------------- @@ -106,6 +90,24 @@ type cellSize* : int # size of the cells we prove (2048) blockSize* : int # size of the network block (65536) + HashConfig* = object + field* : FieldSelect + hashFun* : HashSelect + combo* : FieldHashCombo + + FieldSelect* = enum + BN254, + Goldilocks + + HashSelect* = enum + Poseidon2, + Monolith + + FieldHashCombo* = enum + BN254_Poseidon2, + Goldilocks_Poseidon2, + Goldilocks_Monolith + #------------------------------------------------------------------------------- func cellsPerBlock*(glob: GlobalConfig): int = @@ -114,3 +116,35 @@ func cellsPerBlock*(glob: GlobalConfig): int = return k #------------------------------------------------------------------------------- + +func parseField*(str0: string): FieldSelect = + let str = strutils.toLowerAscii(str0) + case str: + of "bn254": return BN254 + of "goldilocks": return Goldilocks + else: raiseAssert("parsefield: unrecognized field `" & str0 & "`") + +func parseHashFun*(str0: string): HashSelect = + let str = strutils.toLowerAscii(str0) + case str: + of "poseidon2": return Poseidon2 + of "monolith": return Monolith + else: raiseAssert("parsefield: unrecognized hash function `" & str0 & "`") + +#------------------------------------------------------------------------------- + +{. warning[UnreachableElse]:off .} +func toFieldHashCombo*( field: FieldSelect, hash: HashSelect ): FieldHashCombo = + let msg = "invalid hash function `" & ($hash) & "` choice for field `" & ($field) & "`" + case field: + of BN254: + case hash: + of Poseidon2: return BN254_Poseidon2 + else: raiseAssert(msg) + of Goldilocks: + case hash: + of Poseidon2: return Goldilocks_Poseidon2 + of Monolith: return Goldilocks_Monolith + else: raiseAssert(msg) + +#------------------------------------------------------------------------------- diff --git a/reference/nim/proof_input/src/types/bn254.nim b/reference/nim/proof_input/src/types/bn254.nim new file mode 100644 index 0000000..f7bba1e --- /dev/null +++ b/reference/nim/proof_input/src/types/bn254.nim @@ -0,0 +1,62 @@ + +import std/strutils +import std/bitops +import std/streams + +import + constantine/math/arithmetic, + constantine/math/io/io_fields, + constantine/math/io/io_bigints, + constantine/math/config/curves + +#from constantine/math/io/io_fields import toDecimal + +import poseidon2/types +import poseidon2/io +export types + +#------------------------------------------------------------------------------- + +type BN254_T* = F +type Entropy* = F +type Hash* = F +type Root* = Hash + +#------------------------------------------------------------------------------- + +func intToBN254*(x: int): F = toF(x) + +func toDecimalF*(a : F): string = + var s : string = toDecimal(a) + s = s.strip( leading=true, trailing=false, chars={'0'} ) + if s.len == 0: s="0" + return s + +func toQuotedDecimalF*(x: F): string = + let s : string = toDecimalF(x) + return ("\"" & s & "\"") + +proc writeLnF*(h: Stream, prefix: string, x: F) = + h.writeLine(prefix & toQuotedDecimalF(x)) + +proc writeF*(h: Stream, prefix: string, x: F) = + h.write(prefix & toQuotedDecimalF(x)) + +#------------------------------------------------------------------------------- + +func extractLowBits[n: static int]( A: BigInt[n], k: int): uint64 = + assert( k>0 and k<=64 ) + var r : uint64 = 0 + for i in 0..0 and k<=56 ) + let val : uint64 = fromF(fld) + let mask : uint64 = (1'u64 shl k) - 1 + return bitand(val, mask) + +#------------------------------------------------------------------------------- + +func digestToJsonString*( d: Digest ): string = + let xs: F4 = fromDigest(d) + return "[ " & toQuotedDecimalF(xs[0]) & ", " & + toQuotedDecimalF(xs[1]) & ", " & + toQuotedDecimalF(xs[2]) & ", " & + toQuotedDecimalF(xs[3]) & " ]" + +#------------------------------------------------------------------------------- diff --git a/workflow/.gitignore b/workflow/.gitignore index c77d435..659e6b8 100644 --- a/workflow/.gitignore +++ b/workflow/.gitignore @@ -1,3 +1,4 @@ build/ build_big/ +build*/ tmp/ \ No newline at end of file diff --git a/workflow/README.md b/workflow/README.md index 0e21286..1560bf3 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -16,28 +16,47 @@ NOTE: the examples below assume `bash`. In particular, it won't work with `zsh` To have an overview of what all the different steps and files are, see [PROOFS.md](PROOFS.md). -### Some measurements +### Some benchmarks -Approximate time to run this on an M2 (8+4 cores), with 10 samples: +Approximate time to run this on an M2 macbook pro (8+4 cores), with 10 samples: - compiling the circuit: 8 seconds - circuit-specific setup (with 1 contributor): 85 seconds - size of the `.zkey` file (only 1 contributor): 110 megabytes +- generating the witness (WASM): 0.3 seconds - proving with `snarkjs` (slow): 7.7 seconds +- proving wiht `zikkurat` (single threaded!): 13 seconds +- proving with `arkworks`: 4.4 seconds (loading the zkey: 6 seconds) +- proving with `nim-groth16` (old version): 2 seconds Same with 50 samples: - compiling: 37 seconds -- circuit-specific setup: ~420 seconds +- circuit-specific setup: ~430 seconds - `.zkey` file: 525 megabytes -- snarkjs prove: 34 seconds +- generating the witness (WASM): 1.2 seconds +- proving with `snarkjs`: 36 seconds +- proving wiht `zikkurat` (single threaded!): 52 seconds +- proving with `arkworks`: 19.8 seconds (loading the zkey: 33 seconds) +- proving with `nim-groth16` (old version): 9.4 seconds And with 100 samples: -- compiling: 76 seconds -- circuit-specific setup: ~1000 seconds -- `.zkey` file -- snarkjs prove: 76 seconds +- compiling: 76 seconds +- circuit-specific setup: ~1050 seconds +- `.zkey` file +- generating the witness (WASM): 2.3 seconds +- proving with `snarkjs`: 76 seconds +- proving wiht `zikkurat` (single threaded!): 102 seconds +- proving with `arkworks`: 41 seconds (loading the zkey: 66 seconds) +- proving with `nim-groth16` (old version): 18 seconds + +TODO: + +- [x] add `arkworks` prover +- [ ] add `rapidsnarks` prover (doesn't run on ARM...) +- [ ] update `nim-groth16` to `constantine-0.1` (should be faster because no workarounds) +- [ ] add multithreading to `zikkurat` ### Preliminaries diff --git a/workflow/cli_args.sh b/workflow/cli_args.sh index 6ea183b..525791c 100755 --- a/workflow/cli_args.sh +++ b/workflow/cli_args.sh @@ -13,7 +13,9 @@ CLI_ARGS="--depth=$MAXDEPTH \ --seed=$SEED \ --nslots=$NSLOTS \ --ncells=$NCELLS \ - --index=$SLOTINDEX" + --index=$SLOTINDEX \ + --field=bn254 \ + --hash=poseidon2" if [[ "$1" == "--export" ]] then diff --git a/workflow/prove.sh b/workflow/prove.sh index 8fcf7d5..cfd66c8 100755 --- a/workflow/prove.sh +++ b/workflow/prove.sh @@ -16,16 +16,18 @@ ${NIMCLI_DIR}/cli $CLI_ARGS -v --output=input.json # --- generate the witness --- +start=`date +%s` echo "" echo "generating the witness..." cd ${CIRCUIT_MAIN}_js time node generate_witness.js ${CIRCUIT_MAIN}.wasm ../input.json ../witness.wtns cd ${ORIG}/build +end=`date +%s` +echo "Generating the witness took `expr $end - $start` seconds." # --- create the proof --- PROVER="snarkjs" -# PROVER="nim" RS=`which rapidsnark` if [[ ! -z "$RS" ]] @@ -33,9 +35,13 @@ then PROVER="rapidsnark" fi +# PROVER="zikkurat" +PROVER="nim" + echo "" echo "creating the proof... using prover: \`$PROVER\`" +start=`date +%s` case $PROVER in snarkjs) time snarkjs groth16 prove ${CIRCUIT_MAIN}.zkey witness.wtns proof.json public.json @@ -46,11 +52,16 @@ case $PROVER in nim) time nim-groth16 -tpv --zkey=${CIRCUIT_MAIN}.zkey --wtns=witness.wtns -o=proof.json -i=public.json ;; + zikkurat) + time zikkurat-groth16 -tpv --zkey=${CIRCUIT_MAIN}.zkey --wtns=witness.wtns # -o=proof.json -i=public.json + ;; *) echo "unknown prover \`$PROVER\`" exit 99 ;; esac +end=`date +%s` +echo "Creating the proof took `expr $end - $start` seconds." # --- verify the proof --- diff --git a/workflow/setup.sh b/workflow/setup.sh index 076fe65..bc0d9dc 100755 --- a/workflow/setup.sh +++ b/workflow/setup.sh @@ -14,8 +14,13 @@ ${NIMCLI_DIR}/cli $CLI_ARGS -v --circom=${CIRCUIT_MAIN}.circom # --- compile the circuit --- +echo "" +start=`date +%s` CIRCUIT_INCLUDES="-l${CIRCUIT_LIB_DIR} -l${CIRCUIT_POS_DIR} -l${CIRCUIT_PRF_DIR}" -time circom --r1cs --wasm --O2 ${CIRCUIT_INCLUDES} ${CIRCUIT_MAIN}.circom +circom --r1cs --wasm --O2 ${CIRCUIT_INCLUDES} ${CIRCUIT_MAIN}.circom +end=`date +%s` +echo "Compiling the circuit took `expr $end - $start` seconds." +echo "" # --- circuit specific setup ---