initial two-step prover with CLI

README.md

@@ -11,7 +11,9 @@ Our [`circom`](https://docs.circom.io/) circuit is very similar to the one in [t
 
 ### Benchmarks
 
-TODO after the implementation :)
+TODO: proper benchmarking; but for some preliminary numbers, see below:
+
+#### Circuit parameters
 
 We use the default circuit parameters:
 
@@ -26,6 +28,33 @@ Circuit sizes
 
 So we can see that only less than 10% of the circuit is changing at every proof generation, the rest is changing when a new user registers (and thus the Merkle tree changes).
 
+#### Full proof with `nim-groth16`
+
+Single-threaded (macbook pro M2), excluding witness generation:
+
+    the quotient (FFTs) took 0.0182 seconds
+    pi_A (G1 MSM)       took 0.0263 seconds
+    rho  (G1 MSM)       took 0.0306 seconds
+    pi_B (G2 MSM)       took 0.1572 seconds
+    pi_C (2x G1 MSM)    took 0.0709 seconds
+    --------------------------------------
+    full proof          took 0.3051 seconds
+
+From this we can see that $\pi_B$ dominates, which is a good sign.
+
+Some preliminary numbers for the partial proofs:
+
+    generating full witness    : 0.0013 seconds
+    generating full proof      : 0.2015 seconds
+
+    generating partial witness : 0.0021 seconds
+    generating partial proof   : 0.1362 seconds
+    finishing partial proof    : 0.0630 seconds
+
+So we can already see a nice speedup of about 300%.
+
+Note: This very much just hacked together, and there are further optimization opportunities.
+
 ### Differences from the PSE circuit
 
 Note that we are not generic in the curve/field choice, requiring the BN254 curve. This is only a limitation of the implementation of this PoC, it would work exactly the same with eg. BLS12-381.
@@ -102,3 +131,4 @@ where $\mathcal{F}\subset[1\dots M]$ is the set of witness indices which are unc
 
 The only other significant computation is computing the quotient polynomial; that's usually done with FFT. Some part of that can be partially precomputed, but probably won't give a significant speedup.
 
+

test-input/.gitignore

@@ -1,5 +1,10 @@
 .DS_Store
 cli
+prover_cli
 *.circom
 *.json
-tmp/
+tmp/
+nimble.develop
+nimble.paths
+nimble.lock
+config.nims

test-input/README.md

@@ -2,12 +2,23 @@
 create test input data for `rln-fast`
 -------------------------------------
 
+### Generating test input for the RLN circuit
+
 Quickstart:
 
-    $ nimble build cli
+    $ nimble build -d:release cli
     $ ./cli --help
 
 Examples:
 
     $ ./cli -v --merkle_depth=18 --limit_bits=12 --circom=main.circom --output=input.json --partial=partial.json
     $ ./cli -v -d=16 -b=10 --output=tmp/input.json --partial=tmp/partial.json
+
+### Testing the two-step prover
+
+    $ nimble build -d:release prover_cli
+    $ ./prover_cli --help
+
+Exmaple
+
+    $ DIR=<...> ./prover_cli -i=$DIR/input.json -p=$DIR/partial.json -g=$DIR/rln_main.graph -z=$DIR/rln_main.zkey

test-input/src/cli.nim

@@ -1,4 +1,6 @@
 
+# === test input generator CLI ===
+
 {. warning[UnusedImport]:off .}
 
 import sugar
@@ -9,10 +11,10 @@ import std/parseopt
 
 import std/random
 
-import types
-import json/bn254
-import gen_inputs
-import misc
+import rln/types
+import rln/json/bn254
+import rln/gen_inputs
+import rln/misc
 
 #-------------------------------------------------------------------------------
 

test-input/src/prover_cli.nim

@@ -0,0 +1,198 @@
+
+# === prover CLI ===
+
+{. warning[UnusedImport]:off .}
+
+import sugar
+import std/strutils
+import std/sequtils
+import std/options
+import std/random
+import std/os
+import std/parseopt
+
+import taskpools
+
+import constantine/named/properties_fields
+
+import rln/types
+import rln/json/bn254
+import rln/gen_inputs
+import rln/misc
+
+import groth16/zkey_types
+import groth16/files/zkey
+import groth16/files/witness
+import groth16/prover
+import groth16/prover/shared
+import groth16/prover/types
+import groth16/partial/types
+import groth16/partial/precalc
+import groth16/partial/finish
+import groth16/misc
+
+import circom_witnessgen/types
+import circom_witnessgen/input_json
+import circom_witnessgen/graph
+import circom_witnessgen/load
+import circom_witnessgen/dependencies
+import circom_witnessgen/witness
+import circom_witnessgen/partial
+import circom_witnessgen/export_wtns
+
+#-------------------------------------------------------------------------------
+
+type FullConfig = object
+#  globCfg:     GlobalConfig
+  inputFile:   string
+  partialFile: string
+  graphFile:   string
+  zkeyFile:    string
+  verbose:     bool
+
+const defaultFullCfg =
+  FullConfig( # globCfg:     defaultGlobalConfig
+              inputFile:   ""
+            , partialFile: ""
+            , graphFile:   ""
+            , zkeyFile:    ""
+            , verbose:     false
+            )
+
+#-------------------------------------------------------------------------------
+
+proc printHelp() =
+  echo "usage:"
+  echo "$ ./prover [options] --input=input.json --partial=partial.json"
+  echo ""
+  echo "available options:"
+  echo " -h, --help                          : print this help"
+  echo " -v, --verbose                       : verbose output (print the actual parameters)"
+#  echo " -d, --merkle_depth = <depth>        : Merkle tree depth (default: 20)"
+#  echo " -b, --limit_bits   = <bits>         : log2 of maximum number of messages per epoch (default: 16)"
+  echo " -i, --input        = <input.json>   : the JSON file into which we write the full proof inputs"
+  echo " -p, --partial      = <partial.json> : the JSON file into which we write the partial inputs"
+  echo " -g, --graph        = <rln.graph>    : the witness computation graph file generated by `build-circuit`"
+  echo " -z, --zkey         = <rln.zkey>     : the .zkey (prover and verifier keys) for the Groth16 circuit"
+  echo ""
+
+  quit()
+
+#-------------------------------------------------------------------------------
+
+proc parseCliOptions(): FullConfig =
+
+  var globCfg = defaultGlobalConfig
+  var fullCfg = defaultFullCfg
+
+  var argCtr: int = 0
+  for kind, key, value in getOpt():
+    case kind
+
+    # Positional arguments
+    of cmdArgument:
+      # echo ("arg #" & $argCtr & " = " & key)
+      argCtr += 1
+
+    # Switches
+    of cmdLongOption, cmdShortOption:
+      case key
+
+      of "h", "help"          : printHelp()
+      of "v", "verbose"       : fullCfg.verbose       = true
+#      of "d", "merkle_depth"  : globCfg.merkle_depth  = parseInt(value) 
+#      of "b", "limit_bits"    : globCfg.limit_bits    = parseInt(value)
+      of "i", "input"         : fullCfg.inputFile     = value
+      of "p", "partial"       : fullCfg.partialFile   = value
+      of "g", "graph"         : fullCfg.graphFile     = value
+      of "z", "zkey"          : fullCfg.zkeyFile      = value
+      else:
+        echo "Unknown option: ", key
+        echo "use --help to get a list of options"
+        quit()
+
+    of cmdEnd:
+      discard  
+
+  # fullCfg.globCfg = globCfg
+  
+  return fullCfg
+
+#-------------------------------------------------------------------------------
+
+proc printConfig(fullCfg: FullConfig) =
+
+  # let globCfg = fullCfg.globCfg
+
+  # echo "field           = BN254"
+  # echo "hash function   = Poseidon2"
+  # echo "merkle_depth    = " & ($globCfg.merkle_depth)
+  # echo "limit_bits      = " & ($globCfg.limit_bits)
+
+  echo "full input file        = " & fullCfg.inputFile
+  echo "partial input file     = " & fullCfg.partialFile
+  echo "zkey file              = " & fullCfg.graphFile 
+  echo "computation graph file = " & fullCfg.graphFile 
+
+#-------------------------------------------------------------------------------
+
+type
+  XWitness        = seq[Fr[BN254_Snarks]]                # nim namespacing ......
+  XPartialWitness = seq[Option[Fr[BN254_Snarks]]]        # .... is fucking stupid
+
+when isMainModule:
+
+  let fullCfg = parseCliOptions()
+  # let globCfg = fullCfg.globCfg
+
+  if fullCfg.verbose:
+    printConfig(fullCfg)
+
+  if fullCfg.zkeyFile == "" or fullCfg.inputFile == "" or fullCfg.partialFile == "" or fullCfg.graphFile == "":
+    echo "nothing we can do!"
+    echo "we require .zkey, computation graph, and partial and full input files for the circuit"
+    echo "use --help for getting a list of options"
+    quit()
+
+  let nthreads = 1
+  var pool = Taskpool.new(nthreads)
+
+  let fullInput    : Inputs = loadInputJSON(fullCfg.inputFile  )
+  let partialInput : Inputs = loadInputJSON(fullCfg.partialFile)
+  let xzkey        : ZKey   =     parseZKey(fullCfg.zkeyFile   )    # fucking stupid nim puts the short"module" 
+  let xgraph       : Graph  =     loadGraph(fullCfg.graphFile  )    # names in the same namespace as variables...
+
+  let printFlag  = true
+  let detailFlag = false
+
+  var fullWitness1: XWitness
+  withMeasureTime(printFlag, "generating full witness"):
+    fullWitness1 = generateWitness(xgraph, fullInput)
+  let fullWitness = makeWitnessBN254(fullWitness1)                   # Groth16's witness type is more complicated
+
+  var partialWitness1: XPartialWitness
+  withMeasureTime(printFlag, "generating partial witness"):
+    partialWitness1 = generatePartialWitness(xgraph, partialInput)
+  let partialWitness = makePartialWitness(partialWitness1)           # see above, same again
+ 
+  let mask: Mask = randomMask()
+
+  var fullProof: Proof
+  withMeasureTime(printFlag, "generating full proof"):
+    fullProof = generateProofWithMask(xzkey, fullWitness, mask, pool, detailFlag)
+
+  var partialProof: PartialProof
+  withMeasureTime(printFlag, "generating partial proof"):
+    partialProof = generatePartialProof(xzkey, partialWitness, pool, detailFlag)
+
+  var finishedProof: Proof
+  withMeasureTime(printFlag, "finishing partial proof"):
+    finishedProof = finishPartialProofWithMask(xzkey, fullWitness, partialProof, mask, pool, detailFlag)
+
+  if isEqualProof(fullProof, finishedProof):
+    echo "OK. the two proofs match"
+  else:
+    echo "PROBLEM! the two proofs differ!"
+
+  echo "done"
+

test-input/test_input.nimble

@@ -1,11 +1,13 @@
 
 version       = "0.0.0"
 author        = "Balazs Komuves"
-description   = "create test inputs for the RLN circuit"
+description   = "proof-of-concept for faster RLN Groth16 proving"
 license       = "MIT or Apache-2.0"
 srcDir        = "src"
-bin           = @["cli"]
+bin           = @["cli","prover_cli"]
 
 requires "nim >= 2.0.0"
 requires "https://github.com/mratsim/constantine#bc3845aa492b52f7fef047503b1592e830d1a774"
-requires "https://github.com/logos-storage/nim-poseidon2#7749c368a9302167f94bd0133fb881cb83392caf"
+requires "https://github.com/logos-storage/nim-poseidon2"
+requires "https://github.com/logos-storage/circom-witnessgen#461a7b14d4c2bf76f1f94cc3b91d2beb9d5652fa"
+requires "https://github.com/logos-storage/nim-groth16#73b5ae2734050d64157afbc29d364345ff0ec211"

workflow/README.md

@@ -123,6 +123,10 @@ The output of this step will consist of:
 - `proof.json` containing the proof itself
 - `public.json` containing the public inputs
 
+### Check partial proof timings
+
+    $ DIR=<...> ./prover_cli -i=$DIR/input.json -p=$DIR/partial.json -g=$DIR/rln_main.graph -z=$DIR/rln_main.zkey
+
 ### Verify the proof (on CPU)
 
     $ snarkjs groth16 verify rln_main_verification_key.json public.json proof.json

workflow/prove.sh

@@ -48,7 +48,7 @@ then
 fi
 
 # PROVER="zikkurat"
-# PROVER="nim"
+PROVER="nim"
 
 echo ""
 echo "creating the proof... using prover: \`$PROVER\`"
@@ -62,7 +62,7 @@ case $PROVER in
     time rapidsnark ${CIRCUIT_MAIN}.zkey witness.wtns proof.json public.json
     ;;
   nim)
-    time nim-groth16 -tpv --zkey=${CIRCUIT_MAIN}.zkey --wtns=witness.wtns -o=proof.json -i=public.json
+    time nim-groth16 -tpv --nthreads=1 --zkey=${CIRCUIT_MAIN}.zkey --wtns=witness.wtns -o=proof.json -i=public.json
     ;;
   zikkurat)
     time zikkurat-groth16 -tpv --zkey=${CIRCUIT_MAIN}.zkey --wtns=witness.wtns # -o=proof.json -i=public.json