Add 'deposits import' command; Switch to NJS when loading the keystores and improve the data validation

beacon_chain/beacon_node.nim

@@ -1249,6 +1249,12 @@ programMain:
         error "Failed to create launchpad deposit data file", err = err.msg
         quit 1
 
+    of DepositsCmd.`import`:
+      importKeystoresFromDir(
+        rng[],
+        config.importedDepositsDir.string,
+        config.validatorsDir, config.secretsDir)
+
     of DepositsCmd.status:
       echo "The status command is not implemented yet"
       quit 1

beacon_chain/conf.nim

@@ -26,8 +26,9 @@ type
     list    = "Lists details about all wallets"
 
   DepositsCmd* {.pure.} = enum
-    create = "Creates validator keystores and deposits"
-    status = "Displays status information about all deposits"
+    create   = "Creates validator keystores and deposits"
+    `import` = "Imports password-protected keystores interactively"
+    status   = "Displays status information about all deposits"
 
   VCStartUpCmd* = enum
     VCNoCommand
@@ -58,6 +59,10 @@ type
       desc: "A directory containing validator keystores"
       name: "validators-dir" }: Option[InputDir]
 
+    secretsDirFlag* {.
+      desc: "A directory containing validator keystore passwords"
+      name: "secrets-dir" }: Option[InputDir]
+
     walletsDirFlag* {.
       desc: "A directory containing wallet files"
       name: "wallets-dir" }: Option[InputDir]
@@ -125,10 +130,6 @@ type
         abbr: "v"
         name: "validator" }: seq[ValidatorKeyPath]
 
-      secretsDirFlag* {.
-        desc: "A directory containing validator keystore passwords"
-        name: "secrets-dir" }: Option[InputDir]
-
       stateSnapshot* {.
         desc: "SSZ file specifying a recent state snapshot"
         abbr: "s"
@@ -317,12 +318,17 @@ type
 
         newWalletNameFlag* {.
           desc: "An easy-to-remember name for the wallet of your choice"
-          name: "new-wallet-name"}: Option[WalletName]
+          name: "new-wallet-name" }: Option[WalletName]
 
         newWalletFileFlag* {.
           desc: "Output wallet file"
           name: "new-wallet-file" }: Option[OutFile]
 
+      of DepositsCmd.`import`:
+        importedDepositsDir* {.
+          argument
+          desc: "A directory with keystores to import" }: InputDir
+
       of DepositsCmd.status:
         discard
 

beacon_chain/keystore_management.nim

@@ -1,5 +1,5 @@
 import
-  std/[os, json, strutils, terminal, wordwrap],
+  std/[os, strutils, terminal, wordwrap],
   stew/byteutils, chronicles, chronos, web3, stint, json_serialization,
   serialization, blscurve, eth/common/eth_types, eth/keys, confutils, bearssl,
   spec/[datatypes, digest, crypto, keystore],
@@ -18,26 +18,29 @@ type
     mnemonic*: Mnemonic
     nextAccount*: Natural
 
-proc loadKeyStore(conf: BeaconNodeConf|ValidatorClientConf,
+proc loadKeystore(conf: BeaconNodeConf|ValidatorClientConf,
                   validatorsDir, keyName: string): Option[ValidatorPrivKey] =
   let
     keystorePath = validatorsDir / keyName / keystoreFileName
-    keystoreContents = KeyStoreContent:
-      try: readFile(keystorePath)
+    keystore =
+      try: Json.loadFile(keystorePath, Keystore)
       except IOError as err:
         error "Failed to read keystore", err = err.msg, path = keystorePath
         return
+      except SerializationError as err:
+        error "Invalid keystore", err = err.formatMsg(keystorePath)
+        return
 
   let passphrasePath = conf.secretsDir / keyName
   if fileExists(passphrasePath):
     let
-      passphrase = KeyStorePass:
+      passphrase = KeystorePass:
         try: readFile(passphrasePath)
         except IOError as err:
           error "Failed to read passphrase file", err = err.msg, path = passphrasePath
           return
 
-    let res = decryptKeystore(keystoreContents, passphrase)
+    let res = decryptKeystore(keystore, passphrase)
     if res.isOk:
       return res.get.some
     else:
@@ -52,13 +55,13 @@ proc loadKeyStore(conf: BeaconNodeConf|ValidatorClientConf,
   var remainingAttempts = 3
   var prompt = "Please enter passphrase for key \"" & validatorsDir/keyName & "\"\n"
   while remainingAttempts > 0:
-    let passphrase = KeyStorePass:
+    let passphrase = KeystorePass:
       try: readPasswordFromStdin(prompt)
       except IOError:
-        error "STDIN not readable. Cannot obtain KeyStore password"
+        error "STDIN not readable. Cannot obtain Keystore password"
         return
 
-    let decrypted = decryptKeystore(keystoreContents, passphrase)
+    let decrypted = decryptKeystore(keystore, passphrase)
     if decrypted.isOk:
       return decrypted.get.some
     else:
@@ -79,7 +82,7 @@ iterator validatorKeys*(conf: BeaconNodeConf|ValidatorClientConf): ValidatorPriv
     for kind, file in walkDir(validatorsDir):
       if kind == pcDir:
         let keyName = splitFile(file).name
-        let key = loadKeyStore(conf, validatorsDir, keyName)
+        let key = loadKeystore(conf, validatorsDir, keyName)
         if key.isSome:
           yield key.get
         else:
@@ -90,55 +93,66 @@ iterator validatorKeys*(conf: BeaconNodeConf|ValidatorClientConf): ValidatorPriv
     quit 1
 
 type
-  GenerateDepositsError = enum
+  KeystoreGenerationError = enum
     RandomSourceDepleted,
     FailedToCreateValidatoDir
     FailedToCreateSecretFile
     FailedToCreateKeystoreFile
 
-proc generateDeposits*(preset: RuntimePreset,
-                       rng: var BrHmacDrbgContext,
-                       walletData: WalletDataForDeposits,
-                       totalValidators: int,
-                       validatorsDir: string,
-                       secretsDir: string): Result[seq[DepositData], GenerateDepositsError] =
-  var deposits: seq[DepositData]
+proc saveKeystore(rng: var BrHmacDrbgContext,
+                  validatorsDir, secretsDir: string,
+                  signingKey: ValidatorPrivKey, signingPubKey: ValidatorPubKey,
+                  signingKeyPath: KeyPath): Result[void, KeystoreGenerationError] =
+  let
+    keyName = $signingPubKey
+    validatorDir = validatorsDir / keyName
 
-  info "Generating deposits", totalValidators, validatorsDir, secretsDir
+  if not existsDir(validatorDir):
+    var password = KeystorePass getRandomBytes(rng, 32).toHex
+    defer: burnMem(password)
 
-  let withdrawalKeyPath = makeKeyPath(0, withdrawalKeyKind)
-  # TODO: Explain why we are using an empty password
-  var withdrawalKey = keyFromPath(walletData.mnemonic, KeyStorePass"", withdrawalKeyPath)
-  defer: burnMem(withdrawalKey)
-  let withdrawalPubKey = withdrawalKey.toPubKey
-
-  for i in 0 ..< totalValidators:
-    let keyStoreIdx = walletData.nextAccount + i
-    let password = KeyStorePass getRandomBytes(rng, 32).toHex
-
-    let signingKeyPath = withdrawalKeyPath.append keyStoreIdx
-    var signingKey = deriveChildKey(withdrawalKey, keyStoreIdx)
-    defer: burnMem(signingKey)
-    let signingPubKey = signingKey.toPubKey
-
-    let keyStore = encryptKeystore(KdfPbkdf2, rng, signingKey,
-                                   password, signingKeyPath)
     let
-      keyName = $signingPubKey
-      validatorDir = validatorsDir / keyName
+      keyStore = createKeystore(kdfPbkdf2, rng, signingKey,
+                                password, signingKeyPath)
       keystoreFile = validatorDir / keystoreFileName
 
-    if existsDir(validatorDir):
-      continue
-
     try: createDir validatorDir
     except OSError, IOError: return err FailedToCreateValidatoDir
 
     try: writeFile(secretsDir / keyName, password.string)
     except IOError: return err FailedToCreateSecretFile
 
-    try: writeFile(keystoreFile, keyStore.string)
-    except IOError: return err FailedToCreateKeystoreFile
+    try: Json.saveFile(keystoreFile, keyStore)
+    except IOError, SerializationError:
+      return err FailedToCreateKeystoreFile
+
+  ok()
+
+proc generateDeposits*(preset: RuntimePreset,
+                       rng: var BrHmacDrbgContext,
+                       walletData: WalletDataForDeposits,
+                       totalValidators: int,
+                       validatorsDir: string,
+                       secretsDir: string): Result[seq[DepositData], KeystoreGenerationError] =
+  var deposits: seq[DepositData]
+
+  info "Generating deposits", totalValidators, validatorsDir, secretsDir
+
+  let withdrawalKeyPath = makeKeyPath(0, withdrawalKeyKind)
+  # TODO: Explain why we are using an empty password
+  var withdrawalKey = keyFromPath(walletData.mnemonic, KeystorePass"", withdrawalKeyPath)
+  defer: burnMem(withdrawalKey)
+  let withdrawalPubKey = withdrawalKey.toPubKey
+
+  for i in 0 ..< totalValidators:
+    let keyStoreIdx = walletData.nextAccount + i
+    let signingKeyPath = withdrawalKeyPath.append keyStoreIdx
+    var signingKey = deriveChildKey(withdrawalKey, keyStoreIdx)
+    defer: burnMem(signingKey)
+    let signingPubKey = signingKey.toPubKey
+
+    ? saveKeystore(rng, validatorsDir, secretsDir,
+                   signingKey, signingPubKey, signingKeyPath)
 
     deposits.add preset.prepareDeposit(withdrawalPubKey, signingKey, signingPubKey)
 
@@ -153,18 +167,18 @@ const
     minWordLength = minPasswordLen)
 
 proc saveWallet*(wallet: Wallet, outWalletPath: string): Result[void, string] =
-  let
-    uuid = wallet.uuid
-    walletContent = WalletContent Json.encode(wallet, pretty = true)
-
   try: createDir splitFile(outWalletPath).dir
   except OSError, IOError:
     let e = getCurrentException()
     return err("failure to create wallet directory: " & e.msg)
 
-  try: writeFile(outWalletPath, string walletContent)
+  try: Json.saveFile(outWalletPath, wallet, pretty = true)
   except IOError as e:
     return err("failure to write file: " & e.msg)
+  except SerializationError as e:
+    # TODO: Saving a wallet should not produce SerializationErrors.
+    # Investigate the source of this exception.
+    return err("failure to serialize wallet: " & e.formatMsg("wallet"))
 
   ok()
 
@@ -196,6 +210,65 @@ proc resetAttributesNoError() =
     try: stdout.resetAttributes()
     except IOError: discard
 
+proc importKeystoresFromDir*(rng: var BrHmacDrbgContext,
+                             importedDir, validatorsDir, secretsDir: string) =
+  var password: TaintedString
+  defer: burnMem(password)
+
+  try:
+    for file in walkDirRec(importedDir):
+      let ext = splitFile(file).ext
+      if toLowerAscii(ext) != ".json":
+        continue
+
+      let keystore = try:
+        Json.loadFile(file, Keystore)
+      except SerializationError as e:
+        warn "Invalid keystore", err = e.formatMsg(file)
+        continue
+      except IOError as e:
+        warn "Failed to read keystore file", file, err = e.msg
+        continue
+
+      var firstDecryptionAttempt = true
+
+      while true:
+        var secret = decryptCryptoField(keystore.crypto, KeystorePass password)
+
+        if secret.len == 0:
+          if firstDecryptionAttempt:
+            try:
+              echo "Please enter the password for decrypting '$1' " &
+                   "or press ENTER to skip importing this keystore" % [file]
+            except ValueError:
+              raiseAssert "The format string above is correct"
+          else:
+            echo "The entered password was incorrect. Please try again."
+          firstDecryptionAttempt = false
+
+          if not readPasswordInput("Password: ", password):
+            echo "System error while entering password. Please try again."
+
+          if password.len == 0:
+            break
+        else:
+          let privKey = ValidatorPrivKey.fromRaw(secret)
+          if privKey.isOk:
+            let pubKey = privKey.value.toPubKey
+            let status = saveKeystore(rng, validatorsDir, secretsDir,
+                                      privKey.value, pubKey,
+                                      keystore.path)
+            if status.isOk:
+              info "Keystore imported", file
+            else:
+              error "Failed to import keystore", file, err = status.error
+          else:
+            error "Imported keystore holds invalid key", file, err = privKey.error
+          break
+  except OSError:
+    fatal "Failed to access the imported deposits directory"
+    quit 1
+
 proc createWalletInteractively*(
     rng: var BrHmacDrbgContext,
     conf: BeaconNodeConf): Result[WalletDataForDeposits, string] =
@@ -310,9 +383,8 @@ proc createWalletInteractively*(
                      continue
           break
 
-      let wallet = KdfPbkdf2.createWallet(rng, mnemonic,
-                                          name = name,
-                                          password = KeyStorePass password)
+      let wallet = createWallet(kdfPbkdf2, rng, mnemonic,
+                                name = name, password = KeystorePass password)
 
       let outWalletFileFlag = conf.outWalletFile
       let outWalletFile = if outWalletFileFlag.isSome:
@@ -338,27 +410,19 @@ proc loadWallet*(fileName: string): Result[Wallet, string] =
     err e.msg
 
 proc unlockWalletInteractively*(wallet: Wallet): Result[WalletDataForDeposits, string] =
-  var json: JsonNode
-  try:
-    json = parseJson wallet.crypto.string
-  except Exception as e: # TODO: parseJson shouldn't raise general `Exception`
-    if e[] of Defect: raise (ref Defect)(e)
-    else: return err "failure to parse crypto field"
-
-  var password: TaintedString
-
   echo "Please enter the password for unlocking the wallet"
 
   for i in 1..3:
+    var password: TaintedString
     try:
       if not readPasswordInput("Password: ", password):
         return err "failure to read password from stdin"
 
-      var status = decryptoCryptoField(json, KeyStorePass password)
-      if status.isOk:
-        defer: burnMem(status.value)
+      var secret = decryptCryptoField(wallet.crypto, KeystorePass password)
+      if secret.len > 0:
+        defer: burnMem(secret)
         return ok WalletDataForDeposits(
-          mnemonic: Mnemonic string.fromBytes(status.value))
+          mnemonic: Mnemonic string.fromBytes(secret))
       else:
         echo "Unlocking of the wallet failed. Please try again."
     finally:

beacon_chain/spec/keystore.nim

@@ -6,43 +6,69 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
-  json, math, strutils, strformat, typetraits, bearssl,
+  math, strutils, strformat, typetraits, bearssl,
   stew/[results, byteutils, bitseqs, bitops2], stew/shims/macros,
-  eth/keyfile/uuid, blscurve, json_serialization,
+  eth/keyfile/uuid, blscurve, faststreams/textio, json_serialization,
   nimcrypto/[sha2, rijndael, pbkdf2, bcmode, hash, utils],
   ./datatypes, ./crypto, ./digest, ./signatures
 
 export
-  results, burnMem
+  results, burnMem, writeValue, readValue
 
 {.push raises: [Defect].}
 
 type
-  ChecksumParams = object
+  ChecksumFunctionKind* = enum
+    sha256Checksum = "sha256"
 
-  Checksum = object
-    function: string
-    params: ChecksumParams
-    message: string
+  Sha256Params* = object
+  Sha256Digest* = MDigest[256]
 
-  CipherParams = object
-    iv: string
+  ChecksumBytes* = distinct seq[byte]
 
-  Cipher = object
-    function: string
-    params: CipherParams
-    message: string
+  Checksum* = object
+    case function*: ChecksumFunctionKind
+    of sha256Checksum:
+      params*: Sha256Params
+      message*: Sha256Digest
 
-  KdfScrypt* = object
+  Aes128CtrIv* = distinct seq[byte]
+
+  Aes128CtrParams* = object
+    iv*: Aes128CtrIv
+
+  CipherFunctionKind* = enum
+    aes128CtrCipher = "aes-128-ctr"
+
+  CipherBytes* = distinct seq[byte]
+
+  Cipher* = object
+    case function*: CipherFunctionKind
+    of aes128ctrCipher:
+      params*: Aes128CtrParams
+    message*: CipherBytes
+
+  KdfKind* = enum
+    kdfPbkdf2 = "pbkdf2"
+    kdfScrypt = "scrypt"
+
+  ScryptSalt* = distinct seq[byte]
+
+  ScryptParams* = object
     dklen: int
     n, p, r: int
-    salt: string
+    salt: ScryptSalt
 
-  KdfPbkdf2* = object
-    dklen: int
-    c: int
-    prf: string
-    salt: string
+  Pbkdf2Salt* = distinct seq[byte]
+
+  PrfKind* = enum # Pseudo-random-function Kind
+    HmacSha256 = "hmac-sha256"
+
+  Pbkdf2Params* = object
+    dklen*: int
+    c*: int
+    prf*: PrfKind
+    salt*: Pbkdf2Salt
 
   # https://github.com/ethereum/EIPs/blob/4494da0966afa7318ec0157948821b19c4248805/EIPS/eip-2386.md#specification
   Wallet* = object
@@ -52,29 +78,31 @@ type
     walletType* {.serializedFieldName: "type"}: string
     # TODO: The use of `JsonString` can be removed once we
     #       solve the serialization problem for `Crypto[T]`
-    crypto*: JsonString
+    crypto*: Crypto
     nextAccount* {.serializedFieldName: "nextaccount".}: Natural
 
-  KdfParams = KdfPbkdf2 | KdfScrypt
+  Kdf* = object
+    case function*: KdfKind
+    of kdfPbkdf2:
+      pbkdf2Params* {.serializedFieldName: "params".}: Pbkdf2Params
+    of kdfScrypt:
+      scryptParams* {.serializedFieldName: "params".}: ScryptParams
+    message*: string
 
-  Kdf[T: KdfParams] = object
-    function: string
-    params: T
-    message: string
+  Crypto* = object
+    kdf*: Kdf
+    checksum*: Checksum
+    cipher*: Cipher
 
-  Crypto[T: KdfParams] = object
-    kdf: Kdf[T]
-    checksum: Checksum
-    cipher: Cipher
+  Keystore* = object
+    crypto*: Crypto
+    description*: string
+    pubkey*: ValidatorPubKey
+    path*: KeyPath
+    uuid*: string
+    version*: int
 
-  Keystore[T: KdfParams] = object
-    crypto: Crypto[T]
-    pubkey: string
-    path: string
-    uuid: string
-    version: int
-
-  KsResult*[T] = Result[T, cstring]
+  KsResult*[T] = Result[T, string]
 
   Eth2KeyKind* = enum
     signingKeyKind # Also known as voting key
@@ -84,34 +112,32 @@ type
   WalletName* = distinct string
   Mnemonic* = distinct string
   KeyPath* = distinct string
-  KeyStorePass* = distinct string
+  KeystorePass* = distinct string
   KeySeed* = distinct seq[byte]
 
-  KeyStoreContent* = distinct JsonString
-  WalletContent* = distinct JsonString
-
-  SensitiveData = Mnemonic|KeyStorePass|KeySeed
-
   Credentials* = object
     mnemonic*: Mnemonic
-    keyStore*: KeyStoreContent
+    keystore*: Keystore
     signingKey*: ValidatorPrivKey
     withdrawalKey*: ValidatorPrivKey
 
-const
-  saltSize = 32
+  SensitiveData = Mnemonic|KeystorePass|KeySeed
+  SimpleHexEncodedTypes = ScryptSalt|ChecksumBytes|CipherBytes
 
-  scryptParams = KdfScrypt(
-    dklen: saltSize,
+const
+  keyLen = 32
+
+  scryptParams = ScryptParams(
+    dklen: keyLen,
     n: 2^18,
     r: 1,
     p: 8
   )
 
-  pbkdf2Params = KdfPbkdf2(
-    dklen: saltSize,
+  pbkdf2Params = Pbkdf2Params(
+    dklen: keyLen,
     c: 2^18,
-    prf: "hmac-sha256"
+    prf: HmacSha256
   )
 
   # https://eips.ethereum.org/EIPS/eip-2334
@@ -122,8 +148,14 @@ const
   wordListLen = 2048
 
 UUID.serializesAsBaseIn Json
+KeyPath.serializesAsBaseIn Json
 WalletName.serializesAsBaseIn Json
 
+ChecksumFunctionKind.serializesAsTextInJson
+CipherFunctionKind.serializesAsTextInJson
+PrfKind.serializesAsTextInJson
+KdfKind.serializesAsTextInJson
+
 template `$`*(m: Mnemonic): string =
   string(m)
 
@@ -191,7 +223,7 @@ func makeKeyPath*(validatorIdx: Natural,
   except ValueError:
     raiseAssert "All values above can be converted successfully to strings"
 
-func getSeed*(mnemonic: Mnemonic, password: KeyStorePass): KeySeed =
+func getSeed*(mnemonic: Mnemonic, password: KeystorePass): KeySeed =
   # https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki#from-mnemonic-to-seed
   let salt = "mnemonic-" & password.string
   KeySeed sha512.pbkdf2(mnemonic.string, salt, 2048, 64)
@@ -251,7 +283,7 @@ proc deriveMasterKey*(seed: KeySeed): ValidatorPrivKey =
   doAssert success
 
 proc deriveMasterKey*(mnemonic: Mnemonic,
-                      password: KeyStorePass): ValidatorPrivKey =
+                      password: KeystorePass): ValidatorPrivKey =
   deriveMasterKey(getSeed(mnemonic, password))
 
 proc deriveChildKey*(masterKey: ValidatorPrivKey,
@@ -261,106 +293,160 @@ proc deriveChildKey*(masterKey: ValidatorPrivKey,
     result = deriveChildKey(result, idx)
 
 proc keyFromPath*(mnemonic: Mnemonic,
-                  password: KeyStorePass,
+                  password: KeystorePass,
                   path: KeyPath): ValidatorPrivKey =
   deriveChildKey(deriveMasterKey(mnemonic, password), path)
 
-proc shaChecksum(key, cipher: openarray[byte]): array[32, byte] =
+proc shaChecksum(key, cipher: openarray[byte]): Sha256Digest =
   var ctx: sha256
   ctx.init()
   ctx.update(key)
   ctx.update(cipher)
-  result = ctx.finish().data
+  result = ctx.finish()
   ctx.clear()
 
-template tryJsonToCrypto(json: JsonNode; crypto: typedesc): untyped =
-  try:
-    json.to(Crypto[crypto])
-  except Exception:
-    return err "ks: failed to parse crypto"
-
 template hexToBytes(data, name: string): untyped =
   try:
     hexToSeqByte(data)
   except ValueError:
     return err "ks: failed to parse " & name
 
-proc decryptoCryptoField*(json: JsonNode,
-                          password:  KeyStorePass): KsResult[seq[byte]] =
+proc writeJsonHexString(s: OutputStream, data: openarray[byte])
+                       {.raises: [IOError, Defect].} =
+  s.write '"'
+  s.writeHex data
+  s.write '"'
+
+proc readValue*(r: var JsonReader, value: var Pbkdf2Salt)
+               {.raises: [SerializationError, IOError, Defect].} =
+  var s = r.readValue(string)
+
+  if s.len == 0 or s.len mod 16 != 0:
+    r.raiseUnexpectedValue(
+      "The Pbkdf2Salt salf must have a non-zero length divisible by 16")
+
+  try:
+    value = Pbkdf2Salt hexToSeqByte(s)
+  except ValueError:
+    r.raiseUnexpectedValue(
+      "The Pbkdf2Salt must be a valid hex string")
+
+proc readValue*(r: var JsonReader, value: var Aes128CtrIv)
+               {.raises: [SerializationError, IOError, Defect].} =
+  var s = r.readValue(string)
+
+  if s.len != 32:
+    r.raiseUnexpectedValue(
+      "The aes-128-ctr IV must be a string of length 32")
+
+  try:
+    value = Aes128CtrIv hexToSeqByte(s)
+  except ValueError:
+    r.raiseUnexpectedValue(
+      "The aes-128-ctr IV must be a valid hex string")
+
+proc readValue*[T: SimpleHexEncodedTypes](r: var JsonReader, value: var T)
+               {.raises: [SerializationError, IOError, Defect].} =
+  try:
+    value = T hexToSeqByte(r.readValue(string))
+  except ValueError:
+    r.raiseUnexpectedValue("Valid hex string expected")
+
+proc readValue*(r: var JsonReader, value: var Kdf)
+               {.raises: [SerializationError, IOError, Defect].} =
   var
-    decKey: seq[byte]
-    salt: seq[byte]
-    iv: seq[byte]
-    cipherMsg: seq[byte]
-    checksumMsg: seq[byte]
+    functionSpecified = false
+    paramsSpecified = false
 
-  let kdf = json{"kdf", "function"}.getStr
+  for fieldName in readObjectFields(r):
+    case fieldName
+    of "function":
+      value.function = r.readValue(KdfKind)
+      functionSpecified = true
 
-  case kdf
-  of "scrypt":
-    let crypto = tryJsonToCrypto(json, KdfScrypt)
-    return err "ks: scrypt not supported"
-  of "pbkdf2":
-    let
-      crypto = tryJsonToCrypto(json, KdfPbkdf2)
-      kdfParams = crypto.kdf.params
+    of "params":
+      if functionSpecified:
+        case value.function
+        of kdfPbkdf2:
+          r.readValue(value.pbkdf2Params)
+        of kdfScrypt:
+          r.readValue(value.scryptParams)
+      else:
+        r.raiseUnexpectedValue(
+          "The 'params' field must be specified after the 'function' field")
+      paramsSpecified = true
 
-    salt = hexToBytes(kdfParams.salt, "salt")
-    decKey = sha256.pbkdf2(password.string, salt, kdfParams.c, kdfParams.dklen)
-    iv = hexToBytes(crypto.cipher.params.iv, "iv")
-    cipherMsg = hexToBytes(crypto.cipher.message, "cipher")
-    checksumMsg = hexToBytes(crypto.checksum.message, "checksum")
-  else:
-    return err "ks: unknown cipher"
+    of "message":
+      r.readValue(value.message)
 
-  if decKey.len < saltSize:
-    return err "ks: decryption key must be at least 32 bytes"
+    else:
+      r.raiseUnexpectedField(fieldName, "Kdf")
 
-  if iv.len < aes128.sizeBlock:
-    return err "ks: invalid iv"
+  if not (functionSpecified and paramsSpecified):
+    r.raiseUnexpectedValue(
+      "The Kdf value should have sub-fields named 'function' and 'params'")
 
-  let sum = shaChecksum(decKey.toOpenArray(16, 31), cipherMsg)
-  if sum != checksumMsg:
-    return err "ks: invalid checksum"
+template writeValue*(w: var JsonWriter,
+                     value: Pbkdf2Salt|SimpleHexEncodedTypes|Aes128CtrIv) =
+  writeJsonHexString(w.stream, distinctBase value)
+
+template bytes(value: Pbkdf2Salt|SimpleHexEncodedTypes|Aes128CtrIv): seq[byte] =
+  distinctBase value
+
+proc decryptCryptoField*(crypto: Crypto, password: KeystorePass): seq[byte] =
+  ## Returns 0 bytes if the supplied password is incorrect
+
+  let decKey = case crypto.kdf.function
+    of kdfPbkdf2:
+      template params: auto = crypto.kdf.pbkdf2Params
+      sha256.pbkdf2(password.string, params.salt.bytes, params.c, params.dklen)
+    of kdfScrypt:
+      raiseAssert "Scrypt is not supported yet"
+
+  let derivedChecksum = shaChecksum(decKey.toOpenArray(16, 31),
+                                    crypto.cipher.message.bytes)
+  if derivedChecksum != crypto.checksum.message:
+    return
 
   var
     aesCipher: CTR[aes128]
-    secret = newSeq[byte](cipherMsg.len)
+    secret = newSeq[byte](crypto.cipher.message.bytes.len)
 
-  aesCipher.init(decKey.toOpenArray(0, 15), iv)
-  aesCipher.decrypt(cipherMsg, secret)
+  aesCipher.init(decKey.toOpenArray(0, 15), crypto.cipher.params.iv.bytes)
+  aesCipher.decrypt(crypto.cipher.message.bytes, secret)
   aesCipher.clear()
 
-  ok secret
+  return secret
 
-proc decryptKeystore*(data: KeyStoreContent,
-                      password: KeyStorePass): KsResult[ValidatorPrivKey] =
-  # TODO: `parseJson` can raise a general `Exception`
-  let
-    ks = try: parseJson(data.string)
-         except Exception: return err "ks: failed to parse keystore"
-    secret = decryptoCryptoField(ks{"crypto"}, password)
+func cstringToStr(v: cstring): string = $v
 
-  ValidatorPrivKey.fromRaw(? secret)
+proc decryptKeystore*(keystore: Keystore,
+                      password: KeystorePass): KsResult[ValidatorPrivKey] =
+  let decryptedBytes = decryptCryptoField(keystore.crypto, password)
+  if decryptedBytes.len > 0:
+    return ValidatorPrivKey.fromRaw(decryptedBytes).mapErr(cstringToStr)
 
-proc createCryptoField(T: type[KdfParams],
+proc decryptKeystore*(keystore: JsonString,
+                      password: KeystorePass): KsResult[ValidatorPrivKey] =
+  let keystore = try: Json.decode(keystore.string, Keystore)
+                 except SerializationError as e:
+                   return err e.formatMsg("<keystore>")
+  decryptKeystore(keystore, password)
+
+proc createCryptoField(kdfKind: KdfKind,
                        rng: var BrHmacDrbgContext,
                        secret: openarray[byte],
-                       password = KeyStorePass "",
+                       password = KeystorePass "",
                        salt: openarray[byte] = @[],
-                       iv: openarray[byte] = @[]): Crypto[T] =
+                       iv: openarray[byte] = @[]): Crypto =
   type AES = aes128
 
-  var
-    decKey: seq[byte]
-    aesCipher: CTR[AES]
-    cipherMsg = newSeq[byte](secret.len)
-
-  let kdfSalt = if salt.len > 0:
-    doAssert salt.len == saltSize
-    @salt
-  else:
-    getRandomBytes(rng, saltSize)
+  let kdfSalt = Pbkdf2Salt:
+    if salt.len > 0:
+      doAssert salt.len == keyLen
+      @salt
+    else:
+      getRandomBytes(rng, keyLen)
 
   let aesIv = if iv.len > 0:
     doAssert iv.len == AES.sizeBlock
@@ -368,14 +454,21 @@ proc createCryptoField(T: type[KdfParams],
   else:
     getRandomBytes(rng, AES.sizeBlock)
 
-  when T is KdfPbkdf2:
-    decKey = sha256.pbkdf2(password.string, kdfSalt, pbkdf2Params.c,
-                           pbkdf2Params.dklen)
+  let decKey = sha256.pbkdf2(password.string,
+                             kdfSalt.bytes,
+                             pbkdf2Params.c,
+                             pbkdf2Params.dklen)
+  let kdf = case kdfKind
+    of kdfPbkdf2:
+      var params = pbkdf2Params
+      params.salt = kdfSalt
+      Kdf(function: kdfPbkdf2, pbkdf2Params: params, message: "")
+    of kdfScrypt:
+      raiseAssert "Scrypt is not implemented yet"
 
-    var kdf = Kdf[KdfPbkdf2](function: "pbkdf2", params: pbkdf2Params, message: "")
-    kdf.params.salt = byteutils.toHex(kdfSalt)
-  else:
-    {.fatal: "Other KDFs are supported yet".}
+  var
+    aesCipher: CTR[AES]
+    cipherMsg = newSeq[byte](secret.len)
 
   aesCipher.init(decKey.toOpenArray(0, 15), aesIv)
   aesCipher.encrypt(secret, cipherMsg)
@@ -383,46 +476,43 @@ proc createCryptoField(T: type[KdfParams],
 
   let sum = shaChecksum(decKey.toOpenArray(16, 31), cipherMsg)
 
-  Crypto[T](
+  Crypto(
     kdf: kdf,
     checksum: Checksum(
-      function: "sha256",
-      message: byteutils.toHex(sum)),
+      function: sha256Checksum,
+      message: sum),
     cipher: Cipher(
-      function: "aes-128-ctr",
-      params: CipherParams(iv: byteutils.toHex(aesIv)),
-      message: byteutils.toHex(cipherMsg)))
+      function: aes128CtrCipher,
+      params: Aes128CtrParams(iv: Aes128CtrIv aesIv),
+      message: CipherBytes cipherMsg))
 
-proc encryptKeystore*(T: type[KdfParams],
-                      rng: var BrHmacDrbgContext,
-                      privKey: ValidatorPrivkey,
-                      password = KeyStorePass "",
-                      path = KeyPath "",
-                      salt: openarray[byte] = @[],
-                      iv: openarray[byte] = @[],
-                      pretty = true): KeyStoreContent =
+proc createKeystore*(kdfKind: KdfKind,
+                     rng: var BrHmacDrbgContext,
+                     privKey: ValidatorPrivkey,
+                     password = KeystorePass "",
+                     path = KeyPath "",
+                     salt: openarray[byte] = @[],
+                     iv: openarray[byte] = @[]): Keystore =
   let
     secret = privKey.toRaw[^32..^1]
-    cryptoField = createCryptoField(T, rng, secret, password, salt, iv)
+    cryptoField = createCryptoField(kdfKind, rng, secret, password, salt, iv)
     pubkey = privKey.toPubKey()
     uuid = uuidGenerate().expect("Random bytes should be available")
-    keystore = Keystore[T](
-      crypto: cryptoField,
-      pubkey: toHex(pubkey),
-      path: path.string,
-      uuid: $uuid,
-      version: 4)
 
-  KeyStoreContent if pretty: json.pretty(%keystore)
-                  else: $(%keystore)
+  Keystore(
+    crypto: cryptoField,
+    pubkey: pubkey,
+    path: path,
+    uuid: $uuid,
+    version: 4)
 
-proc createWallet*(T: type[KdfParams],
+proc createWallet*(kdfKind: KdfKind,
                    rng: var BrHmacDrbgContext,
                    mnemonic: Mnemonic,
                    name = WalletName "",
                    salt: openarray[byte] = @[],
                    iv: openarray[byte] = @[],
-                   password = KeyStorePass "",
+                   password = KeystorePass "",
                    nextAccount = none(Natural),
                    pretty = true): Wallet =
   let
@@ -430,8 +520,9 @@ proc createWallet*(T: type[KdfParams],
     # Please note that we are passing an empty password here because
     # we want the wallet restoration procedure to depend only on the
     # mnemonic (the user is asked to treat the mnemonic as a password).
-    seed = getSeed(mnemonic, KeyStorePass"")
-    cryptoField = %createCryptoField(T,rng, distinctBase seed, password, salt, iv)
+    seed = getSeed(mnemonic, KeystorePass"")
+    crypto = createCryptoField(kdfKind, rng, distinctBase seed,
+                               password, salt, iv)
 
   Wallet(
     uuid: uuid,
@@ -439,23 +530,9 @@ proc createWallet*(T: type[KdfParams],
           else: WalletName(uuid),
     version: 1,
     walletType: "hierarchical deterministic",
-    crypto: JsonString(if pretty: json.pretty(cryptoField)
-                       else: $cryptoField),
+    crypto: crypto,
     nextAccount: nextAccount.get(0))
 
-proc createWalletContent*(T: type[KdfParams],
-                          rng: var BrHmacDrbgContext,
-                          mnemonic: Mnemonic,
-                          name = WalletName "",
-                          salt: openarray[byte] = @[],
-                          iv: openarray[byte] = @[],
-                          password = KeyStorePass "",
-                          nextAccount = none(Natural),
-                          pretty = true): (UUID, WalletContent) =
-  let wallet = createWallet(
-    T, rng, mnemonic, name, salt, iv, password, nextAccount, pretty)
-  (wallet.uuid, WalletContent Json.encode(wallet, pretty = pretty))
-
 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.2/specs/phase0/deposit-contract.md#withdrawal-credentials
 proc makeWithdrawalCredentials*(k: ValidatorPubKey): Eth2Digest =
   var bytes = eth2digest(k.toRaw())

tests/test_keystore.nim

@@ -8,9 +8,10 @@
 {.used.}
 
 import
-  unittest, ./testutil, json,
-  stew/byteutils, blscurve, eth/keys,
-  ../beacon_chain/spec/[crypto, keystore]
+  json, unittest,
+  stew/byteutils, blscurve, eth/keys, json_serialization,
+  ../beacon_chain/spec/[crypto, keystore],
+  ./testutil
 
 from strutils import replace
 
@@ -94,18 +95,20 @@ suiteReport "Keystore":
     let secret = ValidatorPrivKey.fromRaw(secretBytes).get
 
   timedTest "Pbkdf2 decryption":
-    let decrypt = decryptKeystore(KeyStoreContent pbkdf2Vector,
-                                  KeyStorePass password)
+    let
+      keystore = Json.decode(pbkdf2Vector, Keystore)
+      decrypt = decryptKeystore(keystore, KeystorePass password)
+
     check decrypt.isOk
     check secret == decrypt.get()
 
   timedTest "Pbkdf2 encryption":
-    let encrypt = encryptKeystore(KdfPbkdf2, rng[], secret,
-                                  KeyStorePass password,
+    let keystore = createKeystore(kdfPbkdf2, rng[], secret,
+                                  KeystorePass password,
                                   salt=salt, iv=iv,
                                   path = validateKeyPath "m/12381/60/0/0")
     var
-      encryptJson = parseJson(encrypt.string)
+      encryptJson = parseJson Json.encode(keystore)
       pbkdf2Json = parseJson(pbkdf2Vector)
     encryptJson{"uuid"} = %""
     pbkdf2Json{"uuid"} = %""
@@ -114,26 +117,26 @@ suiteReport "Keystore":
 
   timedTest "Pbkdf2 errors":
     expect Defect:
-      echo encryptKeystore(KdfPbkdf2, rng[], secret, salt = [byte 1]).string
+      echo createKeystore(kdfPbkdf2, rng[], secret, salt = [byte 1])
 
     expect Defect:
-      echo encryptKeystore(KdfPbkdf2, rng[], secret, iv = [byte 1]).string
+      echo createKeystore(kdfPbkdf2, rng[], secret, iv = [byte 1])
 
-    check decryptKeystore(KeyStoreContent pbkdf2Vector,
-                          KeyStorePass "wrong pass").isErr
+    check decryptKeystore(JsonString pbkdf2Vector,
+                          KeystorePass "wrong pass").isErr
 
-    check decryptKeystore(KeyStoreContent pbkdf2Vector,
-                          KeyStorePass "").isErr
+    check decryptKeystore(JsonString pbkdf2Vector,
+                          KeystorePass "").isErr
 
-    check decryptKeystore(KeyStoreContent "{\"a\": 0}",
-                          KeyStorePass "").isErr
+    check decryptKeystore(JsonString "{\"a\": 0}",
+                          KeystorePass "").isErr
 
-    check decryptKeystore(KeyStoreContent "",
-                          KeyStorePass "").isErr
+    check decryptKeystore(JsonString "",
+                          KeystorePass "").isErr
 
     template checkVariant(remove): untyped =
-      check decryptKeystore(KeyStoreContent pbkdf2Vector.replace(remove, ""),
-                            KeyStorePass password).isErr
+      check decryptKeystore(JsonString pbkdf2Vector.replace(remove, ""),
+                            KeystorePass password).isErr
 
     checkVariant "d4e5" # salt
     checkVariant "18b1" # checksum
@@ -143,5 +146,5 @@ suiteReport "Keystore":
     var badKdf = parseJson(pbkdf2Vector)
     badKdf{"crypto", "kdf", "function"} = %"invalid"
 
-    check decryptKeystore(KeyStoreContent $badKdf,
-                          KeyStorePass password).iserr
+    check decryptKeystore(JsonString $badKdf,
+                          KeystorePass password).iserr

vendor/nim-faststreams

@@ -1 +1 @@
-Subproject commit 5df69fc6961e58205189cd92ae2477769fa8c4c0
+Subproject commit 87309f3120d4e627082171a188324d3ee14d8986

vendor/nim-json-serialization

@@ -1 +1 @@
-Subproject commit 9ca88fdcd43f5a4cbc2d86caf057a7bd12575698
+Subproject commit 1dccd4b2ef14c5e3ce30ad3f3a0962e0b98da6a3

vendor/nimcrypto

@@ -1 +1 @@
-Subproject commit f767595f4ddec2b5570b5194feb96954c00a6499
+Subproject commit 029a1f0f1efea17c37b38992c836cf0ac2c803f2