add simple wasm page with state simulator (#334)

* add simple wasm page with state simulator * wip ncli online Co-authored-by: tersec <tersec@users.noreply.github.com>
2020-01-22 16:36:16 +01:00 · 2020-01-22 16:36:16 +01:00 · 23b93adfe6
parent 7e36ba4f4e
commit 23b93adfe6
12 changed files with 604 additions and 15 deletions
--- a/research/state_sim.nim
+++ b/research/state_sim.nim
@ -9,7 +9,7 @@ import
  confutils, stats, times, std/monotimes,
  strformat,
  options, sequtils, random, tables,
-  ../tests/[testutil, testblockutil],
+  ../tests/[testblockutil],
  ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator],
  ../beacon_chain/[attestation_pool, extras, ssz]

@ -20,11 +20,31 @@ type Timers = enum
  tShuffle = "Retrieve committee once using get_beacon_committee"
  tAttest = "Combine committee attestations"

-proc writeJson*(prefix, slot, v: auto) =
+template withTimer(stats: var RunningStat, body: untyped) =
+  let start = cpuTime()
+
+  block:
+    body
+
+  let stop = cpuTime()
+  stats.push stop - start
+
+template withTimerRet(stats: var RunningStat, body: untyped): untyped =
+  let start = cpuTime()
+  let tmp = block:
+    body
+  let stop = cpuTime()
+  stats.push stop - start
+
+  tmp
+
+proc jsonName(prefix, slot: auto): string =
+  fmt"{prefix:04}-{shortLog(slot):08}.json"
+
+proc writeJson*(fn, v: auto) =
  var f: File
  defer: close(f)
-  let fileName = fmt"{prefix:04}-{shortLog(slot):08}.json"
-  Json.saveFile(fileName, v, pretty = true)
+  Json.saveFile(fn, v, pretty = true)

 func verifyConsensus(state: BeaconState, attesterRatio: auto) =
  if attesterRatio < 0.63:
@ -45,16 +65,24 @@ func verifyConsensus(state: BeaconState, attesterRatio: auto) =
 cli do(slots = SLOTS_PER_EPOCH * 6,
       validators = SLOTS_PER_EPOCH * 30, # One per shard is minimum
       json_interval = SLOTS_PER_EPOCH,
+       write_last_json = false,
       prefix = 0,
       attesterRatio {.desc: "ratio of validators that attest in each round"} = 0.73,
       validate = true):
+  echo "Preparing validators..."
  let
    flags = if validate: {} else: {skipValidation}
-    genesisState = initialize_beacon_state_from_eth1(
-      Eth2Digest(), 0,
-      makeInitialDeposits(validators, flags), flags)
+    deposits = makeInitialDeposits(validators, flags)
+
+  echo "Generating Genesis..."
+
+  let
+    genesisState =
+      initialize_beacon_state_from_eth1(Eth2Digest(), 0, deposits, flags)
    genesisBlock = get_initial_beacon_block(genesisState)

+  echo "Starting simulation..."
+
  var
    attestations = initTable[Slot, seq[Attestation]]()
    state = genesisState
@ -65,19 +93,28 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
    blck: SignedBeaconBlock
    cache = get_empty_per_epoch_cache()

-  proc maybeWrite() =
-    if state.slot mod json_interval.uint64 == 0:
-      writeJson(prefix, state.slot, state)
-      write(stdout, ":")
+  proc maybeWrite(last: bool) =
+    if write_last_json:
+      if state.slot mod json_interval.uint64 == 0:
+        write(stdout, ":")
+      else:
+        write(stdout, ".")
+
+      if last:
+        writeJson("state.json", state)
    else:
-      write(stdout, ".")
+      if state.slot mod json_interval.uint64 == 0:
+        writeJson(jsonName(prefix, state.slot), state)
+        write(stdout, ":")
+      else:
+        write(stdout, ".")

  # TODO doAssert against this up-front
  # indexed attestation: validator index beyond max validators per committee
  # len(indices) <= MAX_VALIDATORS_PER_COMMITTEE

  for i in 0..<slots:
-    maybeWrite()
+    maybeWrite(false)
    verifyConsensus(state, attesterRatio)

    let
@ -150,9 +187,10 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
      echo &" slot: {shortLog(state.slot)} ",
        &"epoch: {shortLog(state.slot.compute_epoch_at_slot)}"

-  maybeWrite() # catch that last state as well..

-  echo "done!"
+  maybeWrite(true) # catch that last state as well..
+
+  echo "Done!"

  echo "Validators: ", validators, ", epoch length: ", SLOTS_PER_EPOCH
  echo "Validators per attestation (mean): ", attesters.mean
--- a/research/state_sim.nim.cfg
+++ b/research/state_sim.nim.cfg
@ -0,0 +1 @@
+-u:metrics
--- a/wasm/.gitignore
+++ b/wasm/.gitignore
@ -0,0 +1,2 @@
+state_sim
+
--- a/wasm/README.md
+++ b/wasm/README.md
@ -0,0 +1,14 @@
+# Run nimbus state sim in a browser
+
+Simple runners for in-browser running of WASM versions of applications - based
+on emscripten-generated code.
+
+```
+# Make sure you have built nim-beacon-chain with make first!
+./build.sh
+
+# Run a http server here (wasm + file:/// apparently don't mix)
+python -m SimpleHTTPServer
+
+# Open http://localhost:8000/index.html
+```
--- a/wasm/build.sh
+++ b/wasm/build.sh
@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Simple build script to produce an Emscripten-based wasm version of the state
+# sim.
+# Assumes you have emcc latest-upstream in you PATH, per their install
+# instructions (https://emscripten.org/docs/getting_started/downloads.html)
+#
+# git clone https://github.com/emscripten-core/emsdk.git
+# cd emsdk
+# git pull
+# ./emsdk install latest-upstream
+# ./emsdk activate latest-upstream
+# source ./emsdk_env.sh
+
+# Clean build every time - we use wildcards below so this keeps it simple
+rm -rf state_sim/nimcache
+
+# GC + emcc optimizer leads to crashes - for now, we disable the GC here
+../env.sh nim c \
+  --cpu:i386 --os:linux --gc:none --threads:off \
+  -d:release -d:clang -d:emscripten -d:noSignalHandler -d:usemalloc \
+  --nimcache:state_sim/nimcache \
+  -c ../research/state_sim.nim
+
+../env.sh emcc \
+  -I ../vendor/nimbus-build-system/vendor/Nim/lib \
+  state_sim/nimcache/*.c \
+  ../vendor/nim-blscurve/blscurve/csources/32/{big_384_29.c,ecp2_BLS381.c,rom_curve_BLS381.c,ecp_BLS381.c,fp2_BLS381.c,fp_BLS381.c,rom_field_BLS381.c,pair_BLS381.c,fp12_BLS381.c,fp4_BLS381.c} \
+  -s ERROR_ON_UNDEFINED_SYMBOLS=0 \
+  -s TOTAL_MEMORY=1073741824 \
+  -s EXTRA_EXPORTED_RUNTIME_METHODS=FS \
+  -s WASM=1 \
+  --shell-file state_sim_shell.html \
+  -O3 \
+  -o state_sim/state_sim.html
--- a/wasm/build_ncli.sh
+++ b/wasm/build_ncli.sh
@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Simple build script to produce an Emscripten-based wasm version of the state
+# sim.
+# Assumes you have emcc latest-upstream in you PATH, per their install
+# instructions (https://emscripten.org/docs/getting_started/downloads.html)
+#
+# git clone https://github.com/emscripten-core/emsdk.git
+# cd emsdk
+# git pull
+# ./emsdk install latest-upstream
+# ./emsdk activate latest-upstream
+# source ./emsdk_env.sh
+
+# Clean build every time - we use wildcards below so this keeps it simple
+rm -rf ncli/nimcache
+
+# GC + emcc optimizer leads to crashes - for now, we disable the GC here
+../env.sh nim c \
+  --cpu:i386 --os:linux --gc:none --threads:off \
+  -d:release -d:clang -d:emscripten -d:noSignalHandler -d:usemalloc \
+  --nimcache:ncli/nimcache -d:"network_type=none" \
+  -u:metrics \
+  -c ncli
+
+../env.sh emcc \
+  -I ../vendor/nimbus-build-system/vendor/Nim/lib \
+  ncli/nimcache/*.c \
+  ../vendor/nim-blscurve/blscurve/csources/32/{big_384_29.c,ecp2_BLS381.c,rom_curve_BLS381.c,ecp_BLS381.c,fp2_BLS381.c,fp_BLS381.c,rom_field_BLS381.c,pair_BLS381.c,fp12_BLS381.c,fp4_BLS381.c} \
+  -s ERROR_ON_UNDEFINED_SYMBOLS=0 \
+  -s TOTAL_MEMORY=1073741824 \
+  -s EXTRA_EXPORTED_RUNTIME_METHODS=FS \
+  -s WASM=1 \
+  --shell-file ncli_shell.html \
+  -O3 \
+  -o ncli/ncli.html
--- a/wasm/index.html
+++ b/wasm/index.html
@ -0,0 +1,71 @@
+<!doctype html>
+
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+  <title>Nimbus tooling</title>
+</head>
+
+<style>
+  body,
+  html {
+    font-family: monospace;
+    width: 100%;
+    height: 100%;
+    margin: 0;
+    padding: 0;
+  }
+
+  .row-container {
+    display: flex;
+    width: 100%;
+    height: 100%;
+    flex-direction: column;
+    overflow: hidden;
+  }
+
+  .first-row {
+    background-image: url("https://our.status.im/content/images/2018/12/Artboard-1-copy-15@2x.png");
+    background-position: center; /* Center the image */
+    background-repeat: no-repeat; /* Do not repeat the image */
+    background-size: 100%;
+    border: none;
+    margin: 0;
+    padding: 0;
+  }
+
+  .second-row {
+    flex-grow: 1;
+    border: none;
+    margin: 0;
+    padding: 0;
+  }
+</style>
+<div class="row-container">
+  <div class="first-row">
+    <p><a href="https://github.com/status-im/nim-beacon-chain#state-transition-simulation">Ethereum Beacon Chain state transition simulation</a> (unoptimized work in progress, you might run out of memory)</p>
+    <form action="state_sim/state_sim.html" method="get" target="frame">
+      <table>
+        <tr>
+          <td>Create / Validate BLS signatures</td>
+          <td>Validators</td>
+          <td>Slots</td>
+          <td>Attestation ratio</td>
+        </tr>
+        <tr>
+          <td><input type="radio" name="--validate" value="true">true<input type="radio" name="--validate" value="false"
+              checked="true"> false</td>
+          <td><input type="text" name="--validators" value="100"></input></td>
+          <td><input type="text" name="--slots" value="10"></input></td>
+          <td><input type="text" name="--attesterRatio" value="0.9"></input></td>
+        </tr>
+      </table>
+      <input type="submit" value="Run that state transition!">
+    </form>
+  </div>
+  <iframe name="frame" src="" class="second-row"></iframe>
+</div>
+
+</html>
--- a/wasm/index_ncli.html
+++ b/wasm/index_ncli.html
@ -0,0 +1,71 @@
+<!doctype html>
+
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+  <title>Nimbus ncli/title>
+</head>
+
+<style>
+  body,
+  html {
+    font-family: monospace;
+    width: 100%;
+    height: 100%;
+    margin: 0;
+    padding: 0;
+  }
+
+  .row-container {
+    display: flex;
+    width: 100%;
+    height: 100%;
+    flex-direction: column;
+    overflow: hidden;
+  }
+
+  .first-row {
+    background-image: url("https://our.status.im/content/images/2018/12/Artboard-1-copy-15@2x.png");
+    background-position: center; /* Center the image */
+    background-repeat: no-repeat; /* Do not repeat the image */
+    background-size: 100%;
+    border: none;
+    margin: 0;
+    padding: 0;
+  }
+
+  .second-row {
+    flex-grow: 1;
+    border: none;
+    margin: 0;
+    padding: 0;
+  }
+</style>
+<div class="row-container">
+  <div class="first-row">
+    <p><a href="https://github.com/status-im/nim-beacon-chain#state-transition-simulation">Ethereum Beacon Chain state transition simulation</a> (unoptimized work in progress, you might run out of memory)</p>
+    <form action="ncli/ncli.html" method="get" target="frame">
+      <table>
+        <tr>
+          <td>Create / Validate BLS signatures</td>
+          <td>Validators</td>
+          <td>Slots</td>
+          <td>Attestation ratio</td>
+        </tr>
+        <tr>
+          <td><input type="radio" name="--validate" value="true">true<input type="radio" name="--validate" value="false"
+              checked="true"> false</td>
+          <td><input type="text" name="--validators" value="100"></input></td>
+          <td><input type="text" name="--slots" value="10"></input></td>
+          <td><input type="text" name="--attesterRatio" value="0.9"></input></td>
+        </tr>
+      </table>
+      <input type="submit" value="Run that state transition!">
+    </form>
+  </div>
+  <iframe name="frame" src="" class="second-row"></iframe>
+</div>
+
+</html>
--- a/wasm/index_state_sim.html
+++ b/wasm/index_state_sim.html
@ -0,0 +1,71 @@
+<!doctype html>
+
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+  <title>Nimbus State Sim</title>
+</head>
+
+<style>
+  body,
+  html {
+    font-family: monospace;
+    width: 100%;
+    height: 100%;
+    margin: 0;
+    padding: 0;
+  }
+
+  .row-container {
+    display: flex;
+    width: 100%;
+    height: 100%;
+    flex-direction: column;
+    overflow: hidden;
+  }
+
+  .first-row {
+    background-image: url("https://our.status.im/content/images/2018/12/Artboard-1-copy-15@2x.png");
+    background-position: center; /* Center the image */
+    background-repeat: no-repeat; /* Do not repeat the image */
+    background-size: 100%;
+    border: none;
+    margin: 0;
+    padding: 0;
+  }
+
+  .second-row {
+    flex-grow: 1;
+    border: none;
+    margin: 0;
+    padding: 0;
+  }
+</style>
+<div class="row-container">
+  <div class="first-row">
+    <p><a href="https://github.com/status-im/nim-beacon-chain#state-transition-simulation">Ethereum Beacon Chain state transition simulation</a> (unoptimized work in progress, you might run out of memory)</p>
+    <form action="state_sim/state_sim.html" method="get" target="frame">
+      <table>
+        <tr>
+          <td>Create / Validate BLS signatures</td>
+          <td>Validators</td>
+          <td>Slots</td>
+          <td>Attestation ratio</td>
+        </tr>
+        <tr>
+          <td><input type="radio" name="--validate" value="true">true<input type="radio" name="--validate" value="false"
+              checked="true"> false</td>
+          <td><input type="text" name="--validators" value="100"></input></td>
+          <td><input type="text" name="--slots" value="10"></input></td>
+          <td><input type="text" name="--attesterRatio" value="0.9"></input></td>
+        </tr>
+      </table>
+      <input type="submit" value="Run that state transition!">
+    </form>
+  </div>
+  <iframe name="frame" src="" class="second-row"></iframe>
+</div>
+
+</html>
--- a/wasm/ncli_shell.html
+++ b/wasm/ncli_shell.html
@ -0,0 +1,124 @@
+<!doctype html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+  <title>Nimbus state transition function</title>
+  <style>
+    body,
+    html {
+      font-family: monospace;
+      width: 100%;
+      height: 100%;
+      margin: 0;
+      padding: 0;
+    }
+
+    .row-container {
+      display: flex;
+      width: 100%;
+      height: 100%;
+      align-items: center;
+      flex-direction: column;
+      overflow: hidden;
+    }
+
+    .first-row {
+    }
+
+    .second-row {
+      flex-grow: 1;
+      border: none;
+      margin: 0;
+      padding: 0;
+    }
+
+    textarea.emscripten {
+      font-family: monospace;
+      background-color: beige;
+      width: 95%;
+    }
+
+    div.emscripten_border {
+      border: 1px solid black;
+    }
+  </style>
+</head>
+
+<body height="100%" class="row-container">
+  <div class="first-row">
+    <div class="emscripten" id="status">Running...</div>
+    <hr />
+  </div>
+  <textarea class="emscripten second-row" id="output" rows=50></textarea>
+  <script type='text/javascript'>
+    var statusElement = document.getElementById('status');
+    var progressElement = document.getElementById('progress');
+
+    var Module = {
+      arguments: window.location.search.substr(1).trim().split('&').concat(["--write_last_json:true"]),
+
+      preRun: [],
+      postRun: [() => offerFileAsDownload("state.json", "mime/type")],
+      print: (function () {
+        var element = document.getElementById('output');
+        if (element) element.value = ''; // clear browser cache
+        return function (text) {
+          if (arguments.length > 1) text = Array.prototype.slice.call(arguments).join(' ');
+          // These replacements are necessary if you render to raw HTML
+          //text = text.replace(/&/g, "&amp;");
+          //text = text.replace(/</g, "&lt;");
+          //text = text.replace(/>/g, "&gt;");
+          //text = text.replace('\n', '<br>', 'g');
+          console.log(text);
+          if (element) {
+            element.value += text + "\n";
+            element.scrollTop = element.scrollHeight; // focus on bottom
+          }
+        };
+      })(),
+      printErr: function (text) {
+        if (arguments.length > 1) text = Array.prototype.slice.call(arguments).join(' ');
+        console.error(text);
+      },
+      canvas: (function () { return null; })(),
+      setStatus: function (text) {
+        if (!Module.setStatus.last) Module.setStatus.last = { time: Date.now(), text: '' };
+        if (text === Module.setStatus.last.text) return;
+        statusElement.innerHTML = text;
+      },
+      totalDependencies: 0,
+      monitorRunDependencies: function (left) {
+        this.totalDependencies = Math.max(this.totalDependencies, left);
+        Module.setStatus(left ? 'Preparing... (' + (this.totalDependencies - left) + '/' + this.totalDependencies + ')' : 'All downloads complete.');
+      }
+    };
+    Module.setStatus('Downloading...');
+    window.onerror = function () {
+      Module.setStatus('Exception thrown, see JavaScript console');
+
+      Module.setStatus = function (text) {
+        if (text) Module.printErr('[post-exception status] ' + text);
+      };
+    };
+
+    function offerFileAsDownload(filename, mime) {
+      mime = mime || "application/octet-stream";
+
+      let content = Module.FS.readFile(filename);
+      console.log(`Offering download of "${filename}", with ${content.length} bytes...`);
+
+      var a = document.createElement('a');
+      a.download = filename;
+      a.innerText = "Download state.json";
+      a.href = URL.createObjectURL(new Blob([content], { type: mime }));
+      statusElement.innerHTML = ""
+      statusElement.appendChild(a)
+    }
+
+  </script>
+  {{{ SCRIPT }}}
+</body>
+
+</html>
--- a/wasm/nim.cfg
+++ b/wasm/nim.cfg
@ -0,0 +1,2 @@
+-d:"network_type=none"
+-u:metrics
--- a/wasm/state_sim_shell.html
+++ b/wasm/state_sim_shell.html
@ -0,0 +1,124 @@
+<!doctype html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+  <title>Nimbus state transition function</title>
+  <style>
+    body,
+    html {
+      font-family: monospace;
+      width: 100%;
+      height: 100%;
+      margin: 0;
+      padding: 0;
+    }
+
+    .row-container {
+      display: flex;
+      width: 100%;
+      height: 100%;
+      align-items: center;
+      flex-direction: column;
+      overflow: hidden;
+    }
+
+    .first-row {
+    }
+
+    .second-row {
+      flex-grow: 1;
+      border: none;
+      margin: 0;
+      padding: 0;
+    }
+
+    textarea.emscripten {
+      font-family: monospace;
+      background-color: beige;
+      width: 95%;
+    }
+
+    div.emscripten_border {
+      border: 1px solid black;
+    }
+  </style>
+</head>
+
+<body height="100%" class="row-container">
+  <div class="first-row">
+    <div class="emscripten" id="status">Running...</div>
+    <hr />
+  </div>
+  <textarea class="emscripten second-row" id="output" rows=50></textarea>
+  <script type='text/javascript'>
+    var statusElement = document.getElementById('status');
+    var progressElement = document.getElementById('progress');
+
+    var Module = {
+      arguments: window.location.search.substr(1).trim().split('&').concat(["--write_last_json:true"]),
+
+      preRun: [],
+      postRun: [() => offerFileAsDownload("state.json", "mime/type")],
+      print: (function () {
+        var element = document.getElementById('output');
+        if (element) element.value = ''; // clear browser cache
+        return function (text) {
+          if (arguments.length > 1) text = Array.prototype.slice.call(arguments).join(' ');
+          // These replacements are necessary if you render to raw HTML
+          //text = text.replace(/&/g, "&amp;");
+          //text = text.replace(/</g, "&lt;");
+          //text = text.replace(/>/g, "&gt;");
+          //text = text.replace('\n', '<br>', 'g');
+          console.log(text);
+          if (element) {
+            element.value += text + "\n";
+            element.scrollTop = element.scrollHeight; // focus on bottom
+          }
+        };
+      })(),
+      printErr: function (text) {
+        if (arguments.length > 1) text = Array.prototype.slice.call(arguments).join(' ');
+        console.error(text);
+      },
+      canvas: (function () { return null; })(),
+      setStatus: function (text) {
+        if (!Module.setStatus.last) Module.setStatus.last = { time: Date.now(), text: '' };
+        if (text === Module.setStatus.last.text) return;
+        statusElement.innerHTML = text;
+      },
+      totalDependencies: 0,
+      monitorRunDependencies: function (left) {
+        this.totalDependencies = Math.max(this.totalDependencies, left);
+        Module.setStatus(left ? 'Preparing... (' + (this.totalDependencies - left) + '/' + this.totalDependencies + ')' : 'All downloads complete.');
+      }
+    };
+    Module.setStatus('Downloading...');
+    window.onerror = function () {
+      Module.setStatus('Exception thrown, see JavaScript console');
+
+      Module.setStatus = function (text) {
+        if (text) Module.printErr('[post-exception status] ' + text);
+      };
+    };
+
+    function offerFileAsDownload(filename, mime) {
+      mime = mime || "application/octet-stream";
+
+      let content = Module.FS.readFile(filename);
+      console.log(`Offering download of "${filename}", with ${content.length} bytes...`);
+
+      var a = document.createElement('a');
+      a.download = filename;
+      a.innerText = "Download state.json";
+      a.href = URL.createObjectURL(new Blob([content], { type: mime }));
+      statusElement.innerHTML = ""
+      statusElement.appendChild(a)
+    }
+
+  </script>
+  {{{ SCRIPT }}}
+</body>
+
+</html>