WIP benchmark runner

ceremony/README.md

@@ -5,4 +5,8 @@ Location for the trusted setup ceremony file
 Copy or symlink the ceremony file here with the name `ceremony.ptau`.
 
 You can find links to the Hermez ceremony files (up to size `2^28`)
-[in the `snarkjs` readme](https://github.com/iden3/snarkjs).
+in the [`snarkjs` readme](https://github.com/iden3/snarkjs).
+
+Alternatively, you can run `setup.sh` to generate a "fake" ceremony
+(totally fine for benchmarking, just don't use it in any real system :)
+But that takes quite a long time.

framework/README.md

@@ -0,0 +1,46 @@
+
+Benchmarking framework
+----------------------
+
+The role of this program is to build, setup, and run benchmarks with various
+parameter settings, collecting the timing results together.
+
+A benchmark is defined by the following shell scripts:
+
+- `build.sh` - build the code
+- `setup.sh` - run some additional setup, for example Groth16 circuit-specific setup
+- `witness.sh` - run witness generation for SNARKs (separate from `setup` because we may want to measure it)
+- `run.sh` - run the benchmark itself
+
+These are run in this order, and results are cached unless explicitely requested.
+All except `run.sh` are optional.
+
+Recommended organization is to put all build artifacts into a `build` subdirectory.
+
+Benchmarks can be parameterized using environment variables. By convention, we
+start the names of these environment variables with the `ZKBENCH_` prefix.
+
+An additional file `benchmark.cfg` specifies the configuration and parameter ranges.
+Example file:
+
+    name: "Poseidon2 Groth16 benchmarks"
+    timeout: 300
+    rerunFrom: build 
+    params:
+      [ PROVER:     [ snarkjs, rapidsnark ]
+      , INPUT_SIZE: [ 256, 512, 1024, 2048 ]
+      , WHICH:      [ hash_sponge, hash_sponge_rate2, hash_merkle ]
+      ]
+
+Note: in case of an arithmetic circuit, every step of the build process must be 
+rerun if the circuit changes, and the circuit depends on the input size...
+The `rerunFrom` parameter allows to set this. Normally you want it te be `run`
+(only rerun the `run.sh` script), but in case of Groth16 you want that to be `build`.
+
+`timeout` (in seconds) sets the maximum target time we should spend on this specific
+benchmark. If the initial run is fast enough, we will rerun it up to 10 times 
+and everage them to get a less noisy result.
+
+`params` corresponds to the possible values of the corresponding environment 
+variables (in this example, `ZKBENCH_PROVER`, etc)
+

framework/src/Runner.hs

@@ -0,0 +1,189 @@
+
+-- | Run a single benchmark
+
+{-# LANGUAGE PackageImports #-}
+module Runner where
+
+--------------------------------------------------------------------------------
+
+import Control.Monad
+
+import Data.Char
+import Data.Maybe
+import Data.Fixed
+
+import Data.Map (Map) 
+import qualified Data.Map as Map
+
+import Text.Printf
+
+import System.IO
+import System.FilePath
+import System.Directory
+import System.Environment
+import System.Process
+
+import "time" Data.Time.Clock
+import "time" Data.Time.Clock.System
+
+--------------------------------------------------------------------------------
+
+quote :: String -> String
+quote str = "`" ++ str ++ "`"
+
+--------------------------------------------------------------------------------
+
+data Phase
+  = Build
+  | Setup
+  | Witness
+  | Run
+  deriving (Eq,Ord,Show)
+
+phaseBaseName :: Phase -> FilePath
+phaseBaseName phase = case phase of
+  Build   -> "build"  
+  Setup   -> "setup"  
+  Witness -> "witness"    
+  Run     -> "run"
+
+parsePhase :: String -> Maybe Phase
+parsePhase str = case map toLower str of
+  "build"   -> Just Build       
+  "setup"   -> Just Setup       
+  "witness" -> Just Witness         
+  "run"     -> Just Run     
+  _         -> Nothing
+
+phaseScript :: Phase -> FilePath
+phaseScript phase = phaseBaseName phase <.> "sh"
+
+phaseLockFile :: Phase -> FilePath
+phaseLockFile phase = phaseBaseName phase <.> "lock"
+
+--------------------------------------------------------------------------------
+
+createLockFile :: FilePath -> IO ()
+createLockFile fpath = do
+  h <- openBinaryFile fpath WriteMode 
+  hClose h
+
+--------------------------------------------------------------------------------
+
+newtype Params 
+  = MkParams (Map String String)
+  deriving (Eq,Show)
+
+mkParams :: [(String,String)] -> Params
+mkParams list = MkParams (Map.fromList list)
+
+extendEnvWithParams :: Params -> [(String,String)] -> [(String,String)]
+extendEnvWithParams (MkParams table) oldEnv = newEnv ++ filteredOld where
+  filteredOld = filter (\pair -> not (fst pair `elem` newKeys)) oldEnv
+  newKeys = map fst newEnv
+  newEnv = [ ("ZKBENCH_" ++ key, value) | (key,value) <- Map.toList table ]
+
+--------------------------------------------------------------------------------
+
+newtype Seconds a
+  = MkSeconds a
+  deriving (Eq,Ord,Show)
+
+fromSeconds :: Seconds a -> a
+fromSeconds (MkSeconds x) = x
+
+--------------------------------------------------------------------------------
+
+data Benchmark = MkBenchmark
+  { _benchDir        :: FilePath
+  , _benchTimeout    :: Seconds Int
+  , _benchRerunFrom  :: Phase 
+  , _benchPhases     :: [Phase]
+  , _benchParams     :: Params
+  }
+  deriving Show
+
+data Result = MkResult
+  { _resParams  :: !Params
+  , _resPhase   :: !Phase  
+  , _resAvgTime :: !(Seconds Double)
+  }
+  deriving Show
+
+--------------------------------------------------------------------------------
+
+runBenchmark :: Bool -> Benchmark -> IO [Result]
+runBenchmark rerunAll bench = do
+  origEnv <- getEnvironment
+  origDir <- getCurrentDirectory 
+
+  path <- canonicalizePath (_benchDir bench)
+  setCurrentDirectory path
+  getCurrentDirectory >>= putStrLn
+
+  mbs <- forM (_benchPhases bench) $ \phase -> do
+    let script = phaseScript phase
+    b <- doesFileExist script
+    if (not b)
+      then do
+        putStrLn ("WARNING: benchmark script " ++ quote script ++ " does not exist!")
+        return Nothing
+      else do
+        let extendedEnv = extendEnvWithParams (_benchParams bench) origEnv
+        let cp = (proc "bash" [script]) { env = Just extendedEnv }
+        mbElapsed <- runSinglePhase 
+          (rerunAll || phase >= _benchRerunFrom bench) 
+          phase
+          (_benchTimeout bench) 
+          cp
+        let f secs = MkResult (_benchParams bench) phase secs
+        return $ fmap f mbElapsed
+
+  return (catMaybes mbs)
+
+--------------------------------------------------------------------------------
+
+-- | runs a process
+runCreateProcess :: CreateProcess -> IO ()
+runCreateProcess cp = withCreateProcess cp $ \stdin stdout stderr handle -> do
+  waitForProcess handle
+  return ()
+
+-- | runs a process and measures the elapsed time (in seconds)
+run1 :: CreateProcess -> IO Double
+run1 cp = do
+  before <- getSystemTime 
+  runCreateProcess cp 
+  after  <- getSystemTime
+  let diff = diffUTCTime (systemToUTCTime after) (systemToUTCTime before) 
+  return (realToFrac diff)
+
+-- | Runs a single phase (eg. @build@ or @run@)
+runSinglePhase :: Bool -> Phase -> Seconds Int -> CreateProcess -> IO (Maybe (Seconds Double))
+runSinglePhase alwaysRerun phase timeout cp = do
+  let lockfile = "build" </> phaseLockFile phase
+  b <- doesFileExist lockfile
+  if (alwaysRerun || not b)
+    then do
+      putStrLn $ "running phase " ++ show phase
+      (n,avg) <- runSinglePhaseAlways phase timeout cp
+      createLockFile lockfile
+      printf "average wall-clock time (from %d runs) for phase `%s` = %.5f seconds\n" n (show phase) (fromSeconds avg)
+      return (Just avg)
+    else do
+      putStrLn $ "skipping phase " ++ show phase
+      return Nothing
+
+-- | Runs a single phase unconditionally; in case of the "Run" phase, possibly
+-- several times to get a more precise measurement
+runSinglePhaseAlways :: Phase -> Seconds Int -> CreateProcess -> IO (Int,Seconds Double)
+runSinglePhaseAlways phase (MkSeconds targetTime) cp = do
+  elapsed1 <- run1 cp
+  let n = if phase == Run 
+            then min 10 (round (fromIntegral targetTime / elapsed1)) :: Integer
+            else 1
+  elapsedRest <- forM [2..n] $ \_ -> run1 cp
+  let avg = sum (elapsed1 : elapsedRest) / fromIntegral n :: Double
+  return (fromInteger n, MkSeconds avg)
+
+--------------------------------------------------------------------------------

framework/src/tmp.hs

@@ -0,0 +1,20 @@
+
+module Main where
+
+import Runner
+
+--------------------------------------------------------------------------------
+
+bench1 = MkBenchmark
+  { _benchDir        = "../../hash/snark/bench/Poseidon2"
+  , _benchTimeout    = MkSeconds 30
+  , _benchRerunFrom  = Build
+  , _benchPhases     = [Build,Setup,Witness,Run]
+  , _benchParams     = mkParams 
+       [ ("INPUT_SIZE" , "64"         )
+       , ("WHICH"      , "hash_sponge")
+       , ("PROVER"     , "snarkjs"    )
+       ]
+  }
+
+main = runBenchmark False bench1

hash/cpu/README.md

@@ -3,5 +3,5 @@ Hash functions CPU benchmarks
 -----------------------------
 
 - `bench` contains the benchmarking scripts
-- `src` contains the 3rd party dependencies as git submodules
+- `external` contains the 3rd party dependencies as git submodules