feat(wakunode2): support configuration via environment variables

apps/wakunode2/config.nim

@@ -12,10 +12,15 @@ import
   libp2p/crypto/crypto,
   libp2p/crypto/secp,
   nimcrypto/utils
+import
+  ../../waku/common/confutils/envvar/defs as confEnvvarDefs,
+  ../../waku/common/confutils/envvar/std/net as confEnvvarNet
 
 export
   confTomlDefs,
-  confTomlNet
+  confTomlNet,
+  confEnvvarDefs,
+  confEnvvarNet
 
 
 type ConfResult*[T] = Result[T, string]
@@ -506,6 +511,12 @@ proc readValue*(r: var TomlReader, value: var crypto.PrivateKey) {.raises: [Seri
   except CatchableError:
     raise newException(SerializationError, getCurrentExceptionMsg())
 
+proc readValue*(r: var EnvvarReader, value: var crypto.PrivateKey) {.raises: [SerializationError].} =
+  try: 
+    value = parseCmdArg(crypto.PrivateKey, r.readValue(string))
+  except CatchableError:
+    raise newException(SerializationError, getCurrentExceptionMsg())
+
 
 {.push warning[ProveInit]: off.}
 
@@ -514,6 +525,8 @@ proc load*(T: type WakuNodeConf, version=""): ConfResult[T] =
     let conf = WakuNodeConf.load(
       version=version,
       secondarySources = proc (conf: WakuNodeConf, sources: auto) =
+        sources.addConfigFile(Envvar, InputFile("wakunode2"))
+
         if conf.configFile.isSome():
           sources.addConfigFile(Toml, conf.configFile.get())
     )

apps/wakunode2/wakunode2.nim

@@ -557,7 +557,7 @@ when isMainModule:
   if conf.logLevel != LogLevel.NONE:
     setLogLevel(conf.logLevel)
   
-
+  
   ##############
   # Node setup #
   ##############

tests/all_tests_v2.nim

@@ -1,3 +1,9 @@
+import
+  # Waku common tests
+  ./v2/test_envvar_serialization,
+  ./v2/test_confutils_envvar,
+  ./v2/test_sqlite_migrations
+
 import
   # Waku v2 tests
   ./v2/test_wakunode,
@@ -36,7 +42,6 @@ import
   ./v2/test_waku_bridge,
   ./v2/test_peer_storage,
   ./v2/test_waku_keepalive,
-  ./v2/test_sqlite_migrations,
   ./v2/test_namespacing_utils,
   ./v2/test_waku_dnsdisc,
   ./v2/test_waku_discv5,

tests/v2/test_confutils_envvar.nim

@@ -0,0 +1,81 @@
+{.used.}
+
+import
+  std/[os, options],
+  stew/results,
+  stew/shims/net as stewNet,
+  testutils/unittests,
+  confutils,
+  confutils/defs,
+  confutils/std/net
+import
+  ../../waku/common/confutils/envvar/defs as confEnvvarDefs,
+  ../../waku/common/confutils/envvar/std/net as confEnvvarNet
+
+
+type ConfResult[T] = Result[T, string]
+
+type TestConf = object
+    configFile* {.
+      desc: "Configuration file path"
+      name: "config-file" }: Option[InputFile]
+    
+    testFile* {.
+      desc: "Configuration test file path"
+      name: "test-file" }: Option[InputFile]
+
+    listenAddress* {.
+      defaultValue: ValidIpAddress.init("127.0.0.1"),
+      desc: "Listening address",
+      name: "listen-address"}: ValidIpAddress
+
+    tcpPort* {.
+      desc: "TCP listening port",
+      defaultValue: 60000,
+      name: "tcp-port" }: Port
+
+
+{.push warning[ProveInit]: off.}
+
+proc load*(T: type TestConf, prefix: string): ConfResult[T] =
+  try:
+    let conf = TestConf.load(
+      secondarySources = proc (conf: TestConf, sources: auto) =
+        sources.addConfigFile(Envvar, InputFile(prefix))
+    )
+    ok(conf)
+  except CatchableError:
+    err(getCurrentExceptionMsg())
+
+{.pop.}
+
+
+suite "nim-confutils - envvar":
+  test "load configuration from environment variables":
+    ## Given
+    let prefix = "test-prefix"
+    
+    let
+      listenAddress = "1.1.1.1"
+      tcpPort = "8080"
+      configFile = "/tmp/test.conf"
+
+    ## When
+    os.putEnv("TEST_PREFIX_CONFIG_FILE", configFile)
+    os.putEnv("TEST_PREFIX_LISTEN_ADDRESS", listenAddress)
+    os.putEnv("TEST_PREFIX_TCP_PORT", tcpPort)
+    
+    let confLoadRes = TestConf.load(prefix)
+
+    ## Then
+    check confLoadRes.isOk()
+
+    let conf = confLoadRes.get()
+    check:
+      conf.listenAddress == ValidIpAddress.init(listenAddress)
+      conf.tcpPort == Port(8080)
+    
+      conf.configFile.isSome()
+      conf.configFile.get().string == configFile
+
+      conf.testFile.isNone()

tests/v2/test_envvar_serialization.nim

@@ -0,0 +1,20 @@
+{.used.}
+
+import
+  testutils/unittests
+import
+  ../../waku/common/envvar_serialization/utils
+
+
+suite "nim-envvar-serialization - utils":
+  test "construct env var key":
+    ## Given
+    let prefix = "some-prefix"
+    let name = @["db-url"]
+    
+    ## When
+    let key = constructKey(prefix, name)
+
+    ## Then
+    check:
+      key == "SOME_PREFIX_DB_URL"

waku/common/confutils/envvar/defs.nim

@@ -0,0 +1,24 @@
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+
+import
+  confutils/defs as confutilsDefs
+import
+  ../../envvar_serialization
+
+export
+  envvar_serialization, confutilsDefs
+
+
+template readConfutilsType(T: type) =
+  template readValue*(r: var EnvvarReader, value: var T) =
+    value = T r.readValue(string)
+
+readConfutilsType InputFile
+readConfutilsType InputDir
+readConfutilsType OutPath
+readConfutilsType OutDir
+readConfutilsType OutFile

waku/common/confutils/envvar/std/net.nim

@@ -0,0 +1,28 @@
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+
+import
+  std/strutils,
+  stew/shims/net
+import
+  ../../../envvar_serialization
+
+export
+  net, 
+  envvar_serialization
+
+
+proc readValue*(r: var EnvvarReader, value: var ValidIpAddress) {.raises: [SerializationError].} =
+  try: 
+    value = ValidIpAddress.init(r.readValue(string))
+  except ValueError:
+    raise newException(EnvvarError, "Invalid IP address")
+
+proc readValue*(r: var EnvvarReader, value: var Port) {.raises: [SerializationError, ValueError].} =
+  try: 
+    value = parseUInt(r.readValue(string)).Port
+  except ValueError:
+    raise newException(EnvvarError, "Invalid Port")

waku/common/envvar_serialization.nim

@@ -0,0 +1,63 @@
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+
+import
+  stew/shims/macros,
+  serialization
+import
+   ./envvar_serialization/reader, 
+   ./envvar_serialization/writer
+
+export
+  serialization, 
+  reader, 
+  writer
+
+
+serializationFormat Envvar
+
+Envvar.setReader EnvvarReader
+Envvar.setWriter EnvvarWriter, PreferredOutput = void
+
+
+template supports*(_: type Envvar, T: type): bool =
+  # The Envvar format should support every type
+  true
+
+template decode*(_: type Envvar,
+                 prefix: string,
+                 RecordType: distinct type,
+                 params: varargs[untyped]): auto =
+  mixin init, ReaderType
+
+  {.noSideEffect.}:
+    var reader = unpackArgs(init, [EnvvarReader, prefix, params])
+    reader.readValue(RecordType)
+
+template encode*(_: type Envvar,
+                 prefix: string,
+                 value: auto,
+                 params: varargs[untyped]) =
+  mixin init, WriterType, writeValue
+
+  {.noSideEffect.}:
+    var writer = unpackArgs(init, [EnvvarWriter, prefix, params])
+    writeValue writer, value
+
+template loadFile*(_: type Envvar,
+                   prefix: string,
+                   RecordType: distinct type,
+                   params: varargs[untyped]): auto =
+  mixin init, ReaderType, readValue
+
+  var reader = unpackArgs(init, [EnvvarReader, prefix, params])
+  reader.readValue(RecordType)
+
+template saveFile*(_: type Envvar, prefix: string, value: auto, params: varargs[untyped]) =
+  mixin init, WriterType, writeValue
+
+  var writer = unpackArgs(init, [EnvvarWriter, prefix, params])
+  writer.writeValue(value)

waku/common/envvar_serialization/reader.nim

@@ -0,0 +1,95 @@
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+
+import
+  std/[tables, typetraits, options, os],
+  serialization/object_serialization,
+  serialization/errors
+import
+  ./utils
+
+
+type
+  EnvvarReader* = object
+    prefix: string
+    key: seq[string]
+
+  EnvvarError* = object of SerializationError
+
+  EnvvarReaderError* = object of EnvvarError
+
+  GenericEnvvarReaderError* = object of EnvvarReaderError
+    deserializedField*: string
+    innerException*: ref CatchableError
+
+
+proc handleReadException*(r: EnvvarReader,
+                          Record: type,
+                          fieldName: string,
+                          field: auto,
+                          err: ref CatchableError) {.raises: [GenericEnvvarReaderError].} =
+  var ex = new GenericEnvvarReaderError
+  ex.deserializedField = fieldName
+  ex.innerException = err
+  raise ex
+
+proc init*(T: type EnvvarReader, prefix: string): T =
+  result.prefix = prefix
+
+proc readValue*[T](r: var EnvvarReader, value: var T) {.raises: [ValueError, SerializationError].} =
+  mixin readValue
+
+  when T is string:
+    let key = constructKey(r.prefix, r.key)
+    value = os.getEnv(key)
+
+  elif T is (SomePrimitives or range):
+    let key = constructKey(r.prefix, r.key)
+    getValue(key, value)
+
+  elif T is Option:
+    template getUnderlyingType[T](_: Option[T]): untyped = T
+    let key = constructKey(r.prefix, r.key)
+    if os.existsEnv(key):
+      type uType = getUnderlyingType(value)
+      when uType is string:
+        value = some(os.getEnv(key))
+      else:
+        value = some(r.readValue(uType))
+
+  elif T is (seq or array):
+    when uTypeIsPrimitives(T):
+      let key = constructKey(r.prefix, r.key)
+      getValue(key, value)
+
+    else:
+      let key = r.key[^1]
+      for i in 0..<value.len:
+        r.key[^1] = key & $i
+        r.readValue(value[i])
+
+  elif T is (object or tuple):
+    type T = type(value)
+    when T.totalSerializedFields > 0:
+      let fields = T.fieldReadersTable(EnvvarReader)
+      var expectedFieldPos = 0
+      r.key.add ""
+      value.enumInstanceSerializedFields(fieldName, field):
+        when T is tuple:
+          r.key[^1] = $expectedFieldPos
+          var reader = fields[][expectedFieldPos].reader
+          expectedFieldPos += 1
+        else:
+          r.key[^1] = fieldName
+          var reader = findFieldReader(fields[], fieldName, expectedFieldPos)
+
+        if reader != nil:
+          reader(value, r)
+      discard r.key.pop()
+
+  else:
+    const typeName = typetraits.name(T)
+    {.fatal: "Failed to convert from Envvar an unsupported type: " & typeName.}

waku/common/envvar_serialization/utils.nim

@@ -0,0 +1,108 @@
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+
+import
+  std/[os, strutils],
+  stew/byteutils, 
+  stew/ranges/ptr_arith
+
+
+type
+  SomePrimitives* = SomeInteger | enum | bool | SomeFloat | char
+
+proc setValue*[T: SomePrimitives](key: string, val: openArray[T]) =
+  os.putEnv(key, byteutils.toHex(makeOpenArray(val[0].unsafeAddr, byte, val.len*sizeof(T))))
+
+proc setValue*(key: string, val: SomePrimitives) =
+  os.putEnv(key, byteutils.toHex(makeOpenArray(val.unsafeAddr, byte, sizeof(val))))
+
+proc decodePaddedHex(hex: string, res: ptr UncheckedArray[byte], outputLen: int) {.raises: [ValueError].} =
+  # make it an even length
+  let
+    inputLen = hex.len and not 0x01
+    numHex = inputLen div 2
+    maxLen = min(outputLen, numHex)
+
+  var
+    offI = hex.len - maxLen * 2
+    offO = outputLen - maxLen
+
+  for i in 0 ..< maxLen:
+    res[i + offO] = hex[2*i + offI].readHexChar shl 4 or hex[2*i + 1 + offI].readHexChar
+
+  # write single nibble from odd length hex
+  if (offO > 0) and (offI > 0):
+    res[offO-1] = hex[offI-1].readHexChar
+
+proc getValue*(key: string, outVal: var string) {.raises: [ValueError].} =
+  let hex = os.getEnv(key)
+  let size = (hex.len div 2) + (hex.len and 0x01)
+  outVal.setLen(size)
+  decodePaddedHex(hex, cast[ptr UncheckedArray[byte]](outVal[0].addr), size)
+
+proc getValue*[T: SomePrimitives](key: string, outVal: var seq[T]) =
+  let hex = os.getEnv(key)
+  let byteSize = (hex.len div 2) + (hex.len and 0x01)
+  let size = (byteSize + sizeof(T) - 1) div sizeof(T)
+  outVal.setLen(size)
+  decodePaddedHex(hex, cast[ptr UncheckedArray[byte]](outVal[0].addr), size * sizeof(T))
+
+proc getValue*[N, T: SomePrimitives](key: string, outVal: var array[N, T]) =
+  let hex = os.getEnv(key)
+  decodePaddedHex(hex, cast[ptr UncheckedArray[byte]](outVal[0].addr), sizeof(outVal))
+
+proc getValue*(key: string, outVal: var SomePrimitives) {.raises: [ValueError].} =
+  let hex = os.getEnv(key)
+  decodePaddedHex(hex, cast[ptr UncheckedArray[byte]](outVal.addr), sizeof(outVal))
+
+template uTypeIsPrimitives*[T](_: type seq[T]): bool =
+  when T is SomePrimitives:
+    true
+  else:
+    false
+
+template uTypeIsPrimitives*[N, T](_: type array[N, T]): bool =
+  when T is SomePrimitives:
+    true
+  else:
+    false
+
+template uTypeIsPrimitives*[T](_: type openArray[T]): bool =
+  when T is SomePrimitives:
+    true
+  else:
+    false
+
+template uTypeIsRecord*(_: typed): bool =
+  false
+
+template uTypeIsRecord*[T](_: type seq[T]): bool =
+  when T is (object or tuple):
+    true
+  else:
+    false
+
+template uTypeIsRecord*[N, T](_: type array[N, T]): bool =
+  when T is (object or tuple):
+    true
+  else:
+    false
+
+
+func constructKey*(prefix: string, keys: openArray[string]): string =
+  var newKey: string
+
+  let envvarPrefix = prefix.strip().toUpper().multiReplace(("-", "_"), (" ", "_"))
+  newKey.add(envvarPrefix)
+
+
+  for k in keys:
+    newKey.add("_")
+
+    let envvarKey = k.toUpper().multiReplace(("-", "_"), (" ", "_"))
+    newKey.add(envvarKey)
+
+  newKey

waku/common/envvar_serialization/writer.nim

@@ -0,0 +1,54 @@
+import
+  typetraits, options, tables, os,
+  serialization, ./utils
+
+type
+  EnvvarWriter* = object
+    prefix: string
+    key: seq[string]
+
+proc init*(T: type EnvvarWriter, prefix: string): T =
+  result.prefix = prefix
+
+proc writeValue*(w: var EnvvarWriter, value: auto) =
+  mixin enumInstanceSerializedFields, writeValue, writeFieldIMPL
+  # TODO: reduce allocation
+
+  when value is string:
+    let key = constructKey(w.prefix, w.key)
+    os.putEnv(key, value)
+
+  elif value is (SomePrimitives or range):
+    let key = constructKey(w.prefix, w.key)
+    setValue(key, value)
+
+  elif value is Option:
+    if value.isSome:
+      w.writeValue value.get
+
+  elif value is (seq or array or openArray):
+    when uTypeIsPrimitives(type value):
+      let key = constructKey(w.prefix, w.key)
+      setValue(key, value)
+
+    elif uTypeIsRecord(type value):
+      let key = w.key[^1]
+      for i in 0..<value.len:
+        w.key[^1] = key & $i
+        w.writeValue(value[i])
+
+    else:
+      const typeName = typetraits.name(value.type)
+      {.fatal: "Failed to convert to Envvar array an unsupported type: " & typeName.}
+
+  elif value is (object or tuple):
+    type RecordType = type value
+    w.key.add ""
+    value.enumInstanceSerializedFields(fieldName, field):
+      w.key[^1] = fieldName
+      w.writeFieldIMPL(FieldTag[RecordType, fieldName], field, value)
+    discard w.key.pop()
+
+  else:
+    const typeName = typetraits.name(value.type)
+    {.fatal: "Failed to convert to Envvar an unsupported type: " & typeName.}