Change raft_api and types

all_test.md

@@ -0,0 +1,2 @@
+* testbasicstatemachine  - 1 millisecond, 288 microseconds, and 233 nanoseconds
+* testbasictimers  - 980 microseconds and 832 nanoseconds

raft/raft_api.nim

@@ -13,12 +13,20 @@ import consensus_module
 
 export types, protocol, consensus_module
 
+proc RaftNodeSmInit[SmCommandType, SmStateType](stateMachine: var RaftNodeStateMachine[SmCommandType, SmStateType])
+
 # Raft Node Public API procedures / functions
-proc RaftNodeCreateNew*[SmCommandType, SmStateType](                     # Create New Raft Node
+proc RaftNodeCreateNew*[SmCommandType, SmStateType](                      # Create New Raft Node
                   id: RaftNodeId, peers: RaftNodePeers,
                   persistentStorage: RaftNodePersistentStorage,
                   msgSendCallback: RaftMessageSendCallback): RaftNode[SmCommandType, SmStateType] =
-  discard
+  var
+    sm = RaftNodeStateMachine[SmCommandType, SmStateType]
+  RaftNodeSmInit[SmCommandType, SmStateType](sm)
+  result = RaftNode[SmCommandType, SmStateType](
+    id: id, state: rnsFollower, currentTerm: 0, votedFor: nil, peers: peers, commitIndex: 0, lastApplied: 0,
+    stateMachine: sm, msgSendCallback: msgSendCallback
+  )
 
 proc RaftNodeLoad*[SmCommandType, SmStateType](
                   persistentStorage: RaftNodePersistentStorage,            # Load Raft Node From Storage
@@ -32,19 +40,19 @@ proc RaftNodeStart*[SmCommandType, SmStateType](node: RaftNode[SmCommandType, Sm
   discard
 
 func RaftNodeIdGet*[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType]): RaftNodeId =                   # Get Raft Node ID
-  discard
+  node.id
 
 func RaftNodeStateGet*[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType]): RaftNodeState =             # Get Raft Node State
-    discard
+    node.state
 
 func RaftNodeTermGet*[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType]): RaftNodeTerm =               # Get Raft Node Term
-  discard
+  node.currentTerm
 
 func RaftNodePeersGet*[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType]): RaftNodePeers =             # Get Raft Node Peers
-  discard
+  node.peers
 
 func RaftNodeIsLeader*[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType]): bool =                      # Check if Raft Node is Leader
-  discard
+  node.state == rnsLeader
 
 # Deliver Raft Message to the Raft Node and dispatch it
 proc RaftNodeMessageDeliver*[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType], raftMessage: RaftMessageBase): RaftMessageResponseBase =
@@ -54,12 +62,6 @@ proc RaftNodeMessageDeliver*[SmCommandType, SmStateType](node: RaftNode[SmComman
 proc RaftNodeClientRequest*[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType], req: RaftNodeClientRequest[SmCommandType]): RaftNodeClientResponse[SmStateType] =
   discard
 
-proc RaftNodeLogIndexGet*[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType]): RaftLogIndex =
-  discard
-
-proc RaftNodeLogEntryGet*[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType], logIndex: RaftLogIndex): Result[RaftNodeLogEntry[SmCommandType], string] =
-  discard
-
 # Abstract State Machine Ops
 func RaftNodeSmStateGet*[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType]): SmStateType =
   node.stateMachine.state
@@ -93,10 +95,13 @@ template RaftTimerStop() =
 proc RaftNodeLogAppend[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType], logEntry: RaftNodeLogEntry[SmCommandType]) =
   discard
 
-proc RaftNodeLastLogIndex[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType]): uint64 =
+proc RaftNodeLogTruncate[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType], truncateIndex: uint64) =
   discard
 
-proc RaftNodeLogTruncate[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType], truncateIndex: uint64) =
+proc RaftNodeLogIndexGet[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType]): RaftLogIndex =
+  discard
+
+proc RaftNodeLogEntryGet[SmCommandType, SmStateType](node: RaftNode[SmCommandType, SmStateType], logIndex: RaftLogIndex): Result[RaftNodeLogEntry[SmCommandType], string] =
   discard
 
 # Private Timers Create Ops

raft/types.nim

@@ -90,9 +90,9 @@ type
     term*: RaftNodeTerm
     index*: RaftLogIndex
     entryType*: LogEntryType                        # Type of entry - data to append, configuration or no op etc.
-    configuration: Option[RaftNodeConfiguration]    # Node configuration
     data*: Option[SmCommandType]                    # Entry data (State Machine Command) - this is mutually exclusive with configuration
                                                     # depending on entryType field
+    configuration*: Option[RaftNodeConfiguration]    # Node configuration
 
   RaftNodeLog*[SmCommandType] = object              # Needs more elaborate definition.
                                                     # Probably this will be a RocksDB/MDBX/SQLite Store Wrapper etc.
@@ -109,7 +109,7 @@ type
   RaftTimerCallback* = proc (timer: RaftTimer) {.nimcall, gcsafe.}   # Pass any function wrapped in a closure
 
   # Raft Node Object type
-  RaftNode*[SmCommandType, SmStateType] = object
+  RaftNode*[SmCommandType, SmStateType] = ref object
     # Timers
     requestVotesTimeout: int
     heartBeatTimeout: int

tests/all_tests.nim

@@ -6,3 +6,11 @@
 # at your option.
 # This file may not be copied, modified, or distributed except according to
 # those terms.
+
+import test_macro
+
+{. warning[UnusedImport]: off .}
+
+cliBuilder:
+  import  ./test_basic_state_machine,
+          ./test_basic_timers

tests/basic_state_machine.nim

@@ -9,7 +9,6 @@
 
 import ../raft/types
 import std/tables
-
 export tables
 
 type

tests/basic_timers.nim

@@ -52,15 +52,13 @@ proc RaftTimerCancelCustomImpl*(timer: RaftTimer): bool {.nimcall, gcsafe, disca
 proc RaftTimerPollThread() {.thread, nimcall, gcsafe.} =
   while running:
     try:
-      withLock(timersChanMtx):
-        debugEcho timersChan.len
       poll()
     except ValueError as e:
-      debugEcho e.msg
+      # debugEcho e.msg
       # Add a 'dummy' timer if no other handles are present to prevent more
       # ValueError exceptions this is a workaround for a asyncdyspatch bug
       # see - https://github.com/nim-lang/Nim/issues/14564
-      addTimer(1, true, proc (fd: AsyncFD): bool {.closure, gcsafe.} = true)
+      addTimer(1, false, proc (fd: AsyncFD): bool {.closure, gcsafe.} = false)
 
 proc RaftTimerJoinPollThread*() {.nimcall, gcsafe.} =
   joinThread(pollThr)

tests/test_basic_state_machine.nim

@@ -11,12 +11,12 @@ import unittest2
 import ../raft/types
 import basic_state_machine
 
-proc smRunner() =
+proc basicStateMachineMain*() =
   var
     sm: RaftBasicSm
     smCommandsLog: seq[SmCommand]
 
-  suite "Test Basic State Machine Implementation":
+  suite "Test Basic State Machine Implementation ":
 
     test "Test Init":
       RaftSmInit(sm)
@@ -47,4 +47,4 @@ proc smRunner() =
       check sm.state[] == {"b": "b", "c": "c", "e": "e", "f": "f", "g": "g", "h": "h"}.toTable
 
 if isMainModule:
-  smRunner()
+  basicStateMachineMain()

tests/test_basic_timers.nim

@@ -22,7 +22,7 @@ const
   WAIT_FOR_SLOW_TIMERS = 200
   FINAL_WAIT = 300
 
-proc timersRunner() =
+proc basicTimersMain*() =
   var
     slowTimers: array[0..MAX_TIMERS, RaftTimer]
     fastTimers: array[0..MAX_TIMERS, RaftTimer]
@@ -33,9 +33,6 @@ proc timersRunner() =
 
   suite "Create and test basic timers":
 
-    test "Start timers":
-      RaftTimerStartCustomImpl(false)
-
     test "Create 'slow' and 'fast' timers":
       for i in 0..MAX_TIMERS:
         slowTimers[i] = RaftTimerCreateCustomImpl(max(SLOW_TIMERS_MIN, rand(SLOW_TIMERS_MAX)), true, RaftDummyTimerCallback)
@@ -44,15 +41,12 @@ proc timersRunner() =
         fastTimers[i] = RaftTimerCreateCustomImpl(max(FAST_TIMERS_MIN, rand(FAST_TIMERS_MAX)), true, RaftDummyTimerCallback)
 
     test "Wait for and cancel 'slow' timers":
-      var fut = sleepAsync(WAIT_FOR_SLOW_TIMERS)
+      waitFor sleepAsync(WAIT_FOR_SLOW_TIMERS)
-      while not fut.finished:
-        discard
       for i in 0..MAX_TIMERS:
         RaftTimerCancelCustomImpl(slowTimers[i])
 
-    test "Wait and stop timers":
+    test "Final wait timers":
       waitFor sleepAsync(FINAL_WAIT)
-      RaftTimerStopCustomImpl()
 
     test "Check timers consistency":
       var
@@ -69,4 +63,4 @@ proc timersRunner() =
       check pass
 
 if isMainModule:
-  timersRunner()
+  basicTimersMain()

tests/test_macro.nim

@@ -0,0 +1,101 @@
+# Nimbus
+# Copyright (c) 2018-2019 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import std/times
+import vm_compile_info
+import macros, strutils, os, unittest2, osproc
+import threadpool
+
+export strutils, os, unittest2, osproc, threadpool
+
+# AppVeyor may go out of memory with the default of 4
+setMinPoolSize(2)
+
+proc executeMyself(numModules: int, names: openArray[string]): int =
+  let appName = getAppFilename()
+  var elpdList = newSeq[Duration](numModules)
+  for i in 0..<numModules:
+    let start = getTime()
+    let execResult = execCmd(appName & " " & $i)
+    let elpd = getTime() - start
+    elpdList[i] = elpd
+    if execResult != 0:
+      stderr.writeLine("subtest no: " & $i & " failed: " & names[i])
+    result = result or execResult
+
+  var f = open("all_test.md", fmWrite)
+  for i in 0..<numModules:
+    f.write("* " & names[i])
+    f.write("  - " & $elpdList[i])
+    f.write("\n")
+  f.close()
+
+proc getImportStmt(stmtList: NimNode): NimNode =
+  result = stmtList[0]
+  result.expectKind nnkImportStmt
+
+proc ofStmt(idx: int, singleModule: NimNode): NimNode =
+  # remove the "test_" prefix
+  let moduleName = normalize(singleModule.toStrLit.strVal).substr(4)
+  let moduleMain = newIdentNode(moduleName & "Main")
+
+  # construct `of` branch
+  # of idx: moduleMain()
+  result = nnkOfBranch.newTree(
+    newLit(idx),
+    newCall(moduleMain)
+  )
+
+proc toModuleNames(importStmt: NimNode): NimNode =
+  result = nnkBracket.newTree
+  for singleModule in importStmt:
+    let x = normalize(singleModule.toStrLit.strVal)
+    result.add newLit(x)
+
+macro cliBuilder*(stmtList: typed): untyped =
+  let importStmt = stmtList.getImportStmt
+  let moduleCount = importStmt.len
+  let moduleNames = importStmt.toModuleNames
+
+  # case paramStr(1).parseInt
+  var caseStmt = nnkCaseStmt.newTree(
+    quote do: paramStr(1).parseInt
+  )
+
+  # of 0: codeStreamMain()
+  # of 1: gasMeterMain()
+  # of 2: memoryMain()
+  # ...
+  for idx, singleModule in importStmt:
+    caseStmt.add ofStmt(idx, singleModule)
+
+  # else:
+  #   echo "invalid argument"
+  caseStmt.add nnkElse.newTree(
+    quote do: echo "invalid argument"
+  )
+
+  result = quote do:
+    if paramCount() == 0:
+      const names = `moduleNames`
+      quit(executeMyself(`moduleCount`, names))
+    else:
+      `caseStmt`
+
+# if you want to add new test module(s)
+# make sure you define an entry poin
+# e.g.
+# proc mytestMain*() =
+#   # put anything you want here
+# and then give it a name `test_mytest.nim`
+# the `mytest` part should match between
+# the proc name and the module name
+
+# if this executable called without any params
+# it will execute each of the test by executing itself
+# repeatedly until all sub-tests are executed.
+# you can execute the sub-test by a number start from zero.

tests/vm_compile_info.nim

@@ -0,0 +1,32 @@
+# Nimbus
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+func vmName(): string =
+  when defined(evmc_enabled):
+    "evmc"
+  else:
+    "nimvm"
+
+const
+  VmName* = vmName()
+  warningMsg = block:
+    var rc = "*** Compiling with " & VmName
+    when defined(legacy_eth66_enabled):
+      rc &= ", legacy-eth/66"
+    when defined(chunked_rlpx_enabled):
+      rc &= ", chunked-rlpx"
+    when defined(boehmgc):
+      rc &= ", boehm/gc"
+    rc &= " enabled"
+    rc
+
+{.warning: warningMsg.}
+
+{.used.}