From 3806b76b423be9c0b3cffbacdb1925a14fa11b0c Mon Sep 17 00:00:00 2001 From: Balazs Komuves Date: Tue, 17 Oct 2023 19:26:53 +0200 Subject: [PATCH] WIP benchmark runner --- ceremony/README.md | 6 +- framework/README.md | 46 ++++++++++ framework/src/Runner.hs | 189 ++++++++++++++++++++++++++++++++++++++++ framework/src/tmp.hs | 20 +++++ hash/cpu/README.md | 2 +- 5 files changed, 261 insertions(+), 2 deletions(-) create mode 100644 framework/README.md create mode 100644 framework/src/Runner.hs create mode 100644 framework/src/tmp.hs diff --git a/ceremony/README.md b/ceremony/README.md index dc273fe..b7e3348 100644 --- a/ceremony/README.md +++ b/ceremony/README.md @@ -5,4 +5,8 @@ Location for the trusted setup ceremony file Copy or symlink the ceremony file here with the name `ceremony.ptau`. You can find links to the Hermez ceremony files (up to size `2^28`) -[in the `snarkjs` readme](https://github.com/iden3/snarkjs). +in the [`snarkjs` readme](https://github.com/iden3/snarkjs). + +Alternatively, you can run `setup.sh` to generate a "fake" ceremony +(totally fine for benchmarking, just don't use it in any real system :) +But that takes quite a long time. \ No newline at end of file diff --git a/framework/README.md b/framework/README.md new file mode 100644 index 0000000..8e1477b --- /dev/null +++ b/framework/README.md @@ -0,0 +1,46 @@ + +Benchmarking framework +---------------------- + +The role of this program is to build, setup, and run benchmarks with various +parameter settings, collecting the timing results together. + +A benchmark is defined by the following shell scripts: + +- `build.sh` - build the code +- `setup.sh` - run some additional setup, for example Groth16 circuit-specific setup +- `witness.sh` - run witness generation for SNARKs (separate from `setup` because we may want to measure it) +- `run.sh` - run the benchmark itself + +These are run in this order, and results are cached unless explicitely requested. +All except `run.sh` are optional. + +Recommended organization is to put all build artifacts into a `build` subdirectory. + +Benchmarks can be parameterized using environment variables. By convention, we +start the names of these environment variables with the `ZKBENCH_` prefix. + +An additional file `benchmark.cfg` specifies the configuration and parameter ranges. +Example file: + + name: "Poseidon2 Groth16 benchmarks" + timeout: 300 + rerunFrom: build + params: + [ PROVER: [ snarkjs, rapidsnark ] + , INPUT_SIZE: [ 256, 512, 1024, 2048 ] + , WHICH: [ hash_sponge, hash_sponge_rate2, hash_merkle ] + ] + +Note: in case of an arithmetic circuit, every step of the build process must be +rerun if the circuit changes, and the circuit depends on the input size... +The `rerunFrom` parameter allows to set this. Normally you want it te be `run` +(only rerun the `run.sh` script), but in case of Groth16 you want that to be `build`. + +`timeout` (in seconds) sets the maximum target time we should spend on this specific +benchmark. If the initial run is fast enough, we will rerun it up to 10 times +and everage them to get a less noisy result. + +`params` corresponds to the possible values of the corresponding environment +variables (in this example, `ZKBENCH_PROVER`, etc) + diff --git a/framework/src/Runner.hs b/framework/src/Runner.hs new file mode 100644 index 0000000..381f429 --- /dev/null +++ b/framework/src/Runner.hs @@ -0,0 +1,189 @@ + +-- | Run a single benchmark + +{-# LANGUAGE PackageImports #-} +module Runner where + +-------------------------------------------------------------------------------- + +import Control.Monad + +import Data.Char +import Data.Maybe +import Data.Fixed + +import Data.Map (Map) +import qualified Data.Map as Map + +import Text.Printf + +import System.IO +import System.FilePath +import System.Directory +import System.Environment +import System.Process + +import "time" Data.Time.Clock +import "time" Data.Time.Clock.System + +-------------------------------------------------------------------------------- + +quote :: String -> String +quote str = "`" ++ str ++ "`" + +-------------------------------------------------------------------------------- + +data Phase + = Build + | Setup + | Witness + | Run + deriving (Eq,Ord,Show) + +phaseBaseName :: Phase -> FilePath +phaseBaseName phase = case phase of + Build -> "build" + Setup -> "setup" + Witness -> "witness" + Run -> "run" + +parsePhase :: String -> Maybe Phase +parsePhase str = case map toLower str of + "build" -> Just Build + "setup" -> Just Setup + "witness" -> Just Witness + "run" -> Just Run + _ -> Nothing + +phaseScript :: Phase -> FilePath +phaseScript phase = phaseBaseName phase <.> "sh" + +phaseLockFile :: Phase -> FilePath +phaseLockFile phase = phaseBaseName phase <.> "lock" + +-------------------------------------------------------------------------------- + +createLockFile :: FilePath -> IO () +createLockFile fpath = do + h <- openBinaryFile fpath WriteMode + hClose h + +-------------------------------------------------------------------------------- + +newtype Params + = MkParams (Map String String) + deriving (Eq,Show) + +mkParams :: [(String,String)] -> Params +mkParams list = MkParams (Map.fromList list) + +extendEnvWithParams :: Params -> [(String,String)] -> [(String,String)] +extendEnvWithParams (MkParams table) oldEnv = newEnv ++ filteredOld where + filteredOld = filter (\pair -> not (fst pair `elem` newKeys)) oldEnv + newKeys = map fst newEnv + newEnv = [ ("ZKBENCH_" ++ key, value) | (key,value) <- Map.toList table ] + +-------------------------------------------------------------------------------- + +newtype Seconds a + = MkSeconds a + deriving (Eq,Ord,Show) + +fromSeconds :: Seconds a -> a +fromSeconds (MkSeconds x) = x + +-------------------------------------------------------------------------------- + +data Benchmark = MkBenchmark + { _benchDir :: FilePath + , _benchTimeout :: Seconds Int + , _benchRerunFrom :: Phase + , _benchPhases :: [Phase] + , _benchParams :: Params + } + deriving Show + +data Result = MkResult + { _resParams :: !Params + , _resPhase :: !Phase + , _resAvgTime :: !(Seconds Double) + } + deriving Show + +-------------------------------------------------------------------------------- + +runBenchmark :: Bool -> Benchmark -> IO [Result] +runBenchmark rerunAll bench = do + origEnv <- getEnvironment + origDir <- getCurrentDirectory + + path <- canonicalizePath (_benchDir bench) + setCurrentDirectory path + getCurrentDirectory >>= putStrLn + + mbs <- forM (_benchPhases bench) $ \phase -> do + let script = phaseScript phase + b <- doesFileExist script + if (not b) + then do + putStrLn ("WARNING: benchmark script " ++ quote script ++ " does not exist!") + return Nothing + else do + let extendedEnv = extendEnvWithParams (_benchParams bench) origEnv + let cp = (proc "bash" [script]) { env = Just extendedEnv } + mbElapsed <- runSinglePhase + (rerunAll || phase >= _benchRerunFrom bench) + phase + (_benchTimeout bench) + cp + let f secs = MkResult (_benchParams bench) phase secs + return $ fmap f mbElapsed + + return (catMaybes mbs) + +-------------------------------------------------------------------------------- + +-- | runs a process +runCreateProcess :: CreateProcess -> IO () +runCreateProcess cp = withCreateProcess cp $ \stdin stdout stderr handle -> do + waitForProcess handle + return () + +-- | runs a process and measures the elapsed time (in seconds) +run1 :: CreateProcess -> IO Double +run1 cp = do + before <- getSystemTime + runCreateProcess cp + after <- getSystemTime + let diff = diffUTCTime (systemToUTCTime after) (systemToUTCTime before) + return (realToFrac diff) + +-- | Runs a single phase (eg. @build@ or @run@) +runSinglePhase :: Bool -> Phase -> Seconds Int -> CreateProcess -> IO (Maybe (Seconds Double)) +runSinglePhase alwaysRerun phase timeout cp = do + let lockfile = "build" phaseLockFile phase + b <- doesFileExist lockfile + if (alwaysRerun || not b) + then do + putStrLn $ "running phase " ++ show phase + (n,avg) <- runSinglePhaseAlways phase timeout cp + createLockFile lockfile + printf "average wall-clock time (from %d runs) for phase `%s` = %.5f seconds\n" n (show phase) (fromSeconds avg) + return (Just avg) + else do + putStrLn $ "skipping phase " ++ show phase + return Nothing + +-- | Runs a single phase unconditionally; in case of the "Run" phase, possibly +-- several times to get a more precise measurement +runSinglePhaseAlways :: Phase -> Seconds Int -> CreateProcess -> IO (Int,Seconds Double) +runSinglePhaseAlways phase (MkSeconds targetTime) cp = do + elapsed1 <- run1 cp + let n = if phase == Run + then min 10 (round (fromIntegral targetTime / elapsed1)) :: Integer + else 1 + elapsedRest <- forM [2..n] $ \_ -> run1 cp + let avg = sum (elapsed1 : elapsedRest) / fromIntegral n :: Double + return (fromInteger n, MkSeconds avg) + +-------------------------------------------------------------------------------- diff --git a/framework/src/tmp.hs b/framework/src/tmp.hs new file mode 100644 index 0000000..9eeea2b --- /dev/null +++ b/framework/src/tmp.hs @@ -0,0 +1,20 @@ + +module Main where + +import Runner + +-------------------------------------------------------------------------------- + +bench1 = MkBenchmark + { _benchDir = "../../hash/snark/bench/Poseidon2" + , _benchTimeout = MkSeconds 30 + , _benchRerunFrom = Build + , _benchPhases = [Build,Setup,Witness,Run] + , _benchParams = mkParams + [ ("INPUT_SIZE" , "64" ) + , ("WHICH" , "hash_sponge") + , ("PROVER" , "snarkjs" ) + ] + } + +main = runBenchmark False bench1 \ No newline at end of file diff --git a/hash/cpu/README.md b/hash/cpu/README.md index c4ad07a..a439fb0 100644 --- a/hash/cpu/README.md +++ b/hash/cpu/README.md @@ -3,5 +3,5 @@ Hash functions CPU benchmarks ----------------------------- - `bench` contains the benchmarking scripts -- `src` contains the 3rd party dependencies as git submodules +- `external` contains the 3rd party dependencies as git submodules