fix(@general): keystore management

src/app/boot/app_controller.nim

@@ -290,6 +290,8 @@ proc start*(self: AppController) =
   self.startupModule.load()
 
 proc load(self: AppController) =
+  self.accountsService.deleteExtraKeyStoreFile()
+  
   self.notificationsManager.init()
 
   self.settingsService.init()

src/app_service/service/accounts/service.nim

@@ -1,5 +1,7 @@
-import json, sequtils, strutils, uuids
+import os, json, sequtils, strutils, uuids
+>>>>>>> 8b5de1363 (fix(@general): keystore management)
 import json_serialization, chronicles
+import times as times
 
 import ./dto/accounts as dto_accounts
 import ./dto/generated_accounts as dto_generated_accounts
@@ -28,6 +30,7 @@ type
     loggedInAccount: AccountDto
     importedAccount: GeneratedAccountDto
     isFirstTimeAccountLogin: bool
+    keyStoreDir: string
 
 proc delete*(self: Service) =
   discard
@@ -36,6 +39,7 @@ proc newService*(fleetConfiguration: FleetConfiguration): Service =
   result = Service()
   result.fleetConfiguration = fleetConfiguration
   result.isFirstTimeAccountLogin = false
+  result.keyStoreDir = main_constants.ROOTKEYSTOREDIR
 
 proc getLoggedInAccount*(self: Service): AccountDto =
   return self.loggedInAccount
@@ -46,6 +50,53 @@ proc getImportedAccount*(self: Service): GeneratedAccountDto =
 proc isFirstTimeAccountLogin*(self: Service): bool =
   return self.isFirstTimeAccountLogin
 
+# Remove extra copied keystore that are not needed (can be remove 1st jan 2023)
+proc deleteExtraKeyStoreFile*(self: Service) =
+  let accounts = status_account.getAccounts().result
+  var deleteCandidates: seq[string] = @[]
+  for path in walkFiles(self.keyStoreDir & "*"):
+    var found = false
+    for account in accounts.getElems():
+      let address = account{"address"}.getStr.replace("0x", "")
+      if path.endsWith(address):
+        found = true
+        break
+
+    if not found:
+      deleteCandidates.add(path)
+
+  if len(deleteCandidates) == 2:
+    return
+
+  let tz = times.utc()
+  let tf = times.initTimeFormat("yyyy-mm-dd'T'HH-mm-ss")
+  proc extractTime(path: string): DateTime =
+    return os.extractFilename(path).split("--")[1].split(".")[0].parse(tf, tz)
+
+  let interval = times.initDuration(seconds = 2)
+  var toDelete: seq[string] = @[]
+  for a in deleteCandidates:
+    let aTime = extractTime(a)
+    var found = false
+    for b in deleteCandidates:
+      let bTime = extractTime(b)
+
+      if aTime - bTime < interval:
+        found = true
+        break
+
+    if found:
+      continue
+    
+    toDelete.add(a)
+
+  for path in toDelete:
+    os.removeFile(path)
+
+proc setKeyStoreDir(self: Service, key: string) = 
+  self.keyStoreDir = joinPath(main_constants.ROOTKEYSTOREDIR, key) & main_constants.sep
+  discard status_general.initKeystore(self.keyStoreDir)
+
 proc compressPk*(publicKey: string): string =
   try:
     let response = status_account.compressPk(publicKey)
@@ -265,11 +316,16 @@ proc getDefaultNodeConfig*(self: Service, installationId: string): JsonNode =
 
   # TODO: commented since it's not necessary (we do the connections thru C bindings). Enable it thru an option once status-nodes are able to be configured in desktop
   # result["ListenAddr"] = if existsEnv("STATUS_PORT"): newJString("0.0.0.0:" & $getEnv("STATUS_PORT")) else: newJString("0.0.0.0:30305")
+  
+  result["KeyStoreDir"] = newJString(self.keyStoreDir.replace(main_constants.STATUSGODIR, ""))
 
 proc setupAccount*(self: Service, accountId, password, displayName: string): string =
   try:
     let installationId = $genUUID()
     let accountDataJson = self.getAccountDataForAccountId(accountId, displayName)
+
+    self.setKeyStoreDir(accountDataJson{"key-uid"}.getStr)
+
     let subaccountDataJson = self.getSubaccountDataForAccountId(accountId, displayName)
     let settingsJson = self.getAccountSettings(accountId, installationId, displayName)
     let nodeConfigJson = self.getDefaultNodeConfig(installationId)
@@ -325,6 +381,14 @@ proc login*(self: Service, account: AccountDto, password: string): string =
       elif(img.imgType == "large"):
         largeImage = img.uri
 
+    # Copy old keystore file to new dir, this code can be remove 1st jan 2023    
+    let keyStoreDir = joinPath(main_constants.ROOTKEYSTOREDIR, account.keyUid) & main_constants.sep
+    if not dirExists(self.keyStoreDir):
+      os.createDir(self.keyStoreDir)
+      for path in walkFiles(main_constants.ROOTKEYSTOREDIR & "*"):
+        os.copyFile(path, self.keyStoreDir & os.extractFilename(path))
+
+    self.setKeyStoreDir(account.keyUid)
     # This is moved from `status-lib` here
     # TODO:
     # If you added a new value in the nodeconfig in status-go, old accounts will not have this value, since the node config
@@ -332,6 +396,7 @@ proc login*(self: Service, account: AccountDto, password: string): string =
     # While this is fixed, you can add here any missing attribute on the node config, and it will be merged with whatever
     # the account has in the db
     var nodeCfg = %* {
+      "KeyStoreDir": self.keyStoreDir.replace(main_constants.STATUSGODIR, ""),
       "ShhextConfig": %* {
         "BandwidthStatsEnabled": true
       },
@@ -372,7 +437,7 @@ proc login*(self: Service, account: AccountDto, password: string): string =
 
 proc verifyAccountPassword*(self: Service, account: string, password: string): bool =
   try:
-    let response = status_account.verifyAccountPassword(account, password, main_constants.KEYSTOREDIR)
+    let response = status_account.verifyAccountPassword(account, password, self.keyStoreDir)
     if(response.result.contains("error")):
       let errMsg = response.result["error"].getStr
       if(errMsg.len == 0):

src/app_service/service/general/service.nim

@@ -1,8 +1,6 @@
-import json, chronicles
+import os, json, chronicles
 
 import ../../../backend/general as status_general
-import ../../../backend/keycard as status_keycard
-import ../../../backend/accounts as status_accounts
 import ../../../constants as app_constants
 
 import ../profile/dto/profile as profile_dto
@@ -20,19 +18,9 @@ proc delete*(self: Service) =
 proc newService*(): Service =
   result = Service()
 
-proc initKeycard(self: Service) =
-  ## This should not be part of the "general service", but part of the "keystore service", but since we don't have
-  ## keycard in place for the refactored part yet but `status-go` part requires keycard to be initialized on the app
-  ## start. This call is added as a part of the "global service".
-  try:
-    discard status_keycard.initKeycard(app_constants.KEYSTOREDIR)
-  except Exception as e:
-    let errDesription = e.msg
-    error "error: ", errDesription
-    return
-
 proc init*(self: Service) =
-  self.initKeycard()
+  if not existsDir(app_constants.ROOTKEYSTOREDIR):
+    createDir(app_constants.ROOTKEYSTOREDIR)
 
 proc startMessenger*(self: Service) =
   try:

src/backend/accounts.nim

@@ -198,21 +198,6 @@ proc saveAccount*(
     }]
   ])
 
-proc loadAccount*(address: string, password: string): RpcResponse[JsonNode] {.raises: [Exception].} =
-  let hashedPassword = hashPassword(password)
-  let payload = %* {
-    "address": address,
-    "password": hashedPassword
-  }
-
-  try:
-    let response = status_go.multiAccountLoadAccount($payload)
-    result.result = Json.decode(response, JsonNode)
-
-  except RpcException as e:
-    error "error doing rpc request", methodName = "storeAccounts", exception=e.msg
-    raise newException(RpcException, e.msg)
-
 proc addPeer*(peer: string): RpcResponse[JsonNode] {.raises: [Exception].} =
   try:
     let response = status_go.addPeer(peer)
@@ -253,18 +238,6 @@ proc login*(name, keyUid, hashedPassword, thumbnail, large: string, nodeCfgObj:
     error "error doing rpc request", methodName = "login", exception=e.msg
     raise newException(RpcException, e.msg)
 
-proc multiAccountImportPrivateKey*(privateKey: string): RpcResponse[JsonNode] =
-  let payload = %* {
-    "privateKey": privateKey
-  }
-  try:
-    let response = status_go.multiAccountImportPrivateKey($payload)
-    result.result = Json.decode(response, JsonNode)
-
-  except RpcException as e:
-    error "error doing rpc request", methodName = "multiAccountImportPrivateKey", exception=e.msg
-    raise newException(RpcException, e.msg)
-
 proc verifyAccountPassword*(address: string, password: string, keystoreDir: string):
   RpcResponse[JsonNode] {.raises: [Exception].} =
   try:

src/backend/core.nim

@@ -68,4 +68,4 @@ proc sendTransaction*(inputJSON: string, password: string): RpcResponse[JsonNode
     result = Json.decode(rpcResponseRaw, RpcResponse[JsonNode])
   except RpcException as e:
     error "error sending tx", inputJSON, exception=e.msg
-    raise newException(RpcException, e.msg)
+    raise newException(RpcException, e.msg)

src/backend/general.nim

@@ -69,4 +69,12 @@ proc generateImages*(imagePath: string, aX, aY, bX, bY: int): RpcResponse[JsonNo
     result.result = Json.decode(response, JsonNode)
   except RpcException as e:
     error "error", methodName = "generateImages", exception=e.msg
-    raise newException(RpcException, e.msg)
+    raise newException(RpcException, e.msg)
+
+proc initKeystore*(keystoreDir: string): RpcResponse[JsonNode] {.raises: [Exception].} =
+  try:
+    let response = status_go.initKeystore(keystoreDir)
+    result.result = Json.decode(response, JsonNode)
+  except RpcException as e:
+    error "error", methodName = "initKeystore", exception=e.msg
+    raise newException(RpcException, e.msg)

src/backend/keycard.nim

@@ -1,13 +0,0 @@
-import json, strutils, chronicles
-import core, utils
-import response_type
-
-import status_go
-
-export response_type
-
-logScope:
-  topics = "rpc-keystore"
-
-proc initKeycard*(keystoreDir: string): RpcResponse[JsonNode] {.raises: [Exception].} =
-  result.result = newJString($status_go.initKeystore(keystoreDir))

src/backend/node_config.nim

@@ -25,7 +25,7 @@ proc switchFleet*(fleet: string, nodeConfig: JsonNode): RpcResponse[JsonNode] {.
     let response = status_go.switchFleet(fleet, $nodeConfig)
     result.result = Json.decode(response, JsonNode)
   except RpcException as e:
-    error "error doing rpc request", methodName = "saveAccountAndLogin", exception=e.msg
+    error "error doing rpc request", methodName = "switchFleet", exception=e.msg
     raise newException(RpcException, e.msg)
 
 proc enableCommunityHistoryArchiveSupport*(): RpcResponse[JsonNode] {.raises: [Exception].} =
@@ -42,4 +42,4 @@ proc disableCommunityHistoryArchiveSupport*(): RpcResponse[JsonNode] {.raises: [
     result = core.callPrivateRPC("disableCommunityHistoryArchiveProtocol".prefix)
   except RpcException as e:
     error "error doing rpc request", methodName = "disableCommunityHistoryArchiveProtocol", exception=e.msg
-    raise newException(RpcException, e.msg)
+    raise newException(RpcException, e.msg)

src/backend/status.nim

@@ -1,106 +0,0 @@
-import statusgo_backend/accounts as statusgo_backend_accounts
-import statusgo_backend/core as statusgo_backend_core
-import statusgo_backend/settings as statusgo_backend_settings
-import chat, accounts, wallet, wallet2, node, network, messages, contacts, profile, stickers, permissions, fleet, settings, mailservers, tokens, provider
-import ../eventemitter
-import bitops, stew/byteutils, chronicles
-import ./types/[setting]
-import ./keycard
-
-import ../backends/backend
-
-export chat, accounts, node, messages, contacts, profile, network, permissions, eventemitter
-
-type Status* = ref object
-  backend*: Backend
-  events*: EventEmitter
-  chat*: ChatModel
-  messages*: MessagesModel
-  accounts*: AccountModel
-  wallet*: WalletModel
-  wallet2*: Wallet2Model
-  node*: NodeModel
-  profile*: ProfileModel
-  contacts*: ContactModel
-  network*: NetworkModel
-  stickers*: StickersModel
-  permissions*: PermissionsModel
-  settings*: SettingsModel
-  tokens*: TokensModel
-  provider*: ProviderModel
-  keycard*: KeycardModel
-
-proc newStatusInstance*(backendName: string): Status =
-  result = Status()
-  result.backend = newBackend(backendName)
-  result.events = createEventEmitter()
-  result.chat = chat.newChatModel(result.events)
-  result.accounts = accounts.newAccountModel(result.events)
-  result.wallet = wallet.newWalletModel(result.events)
-  result.wallet.initEvents()
-  result.wallet2 = wallet2.newWallet2Model(result.events)
-  result.node = node.newNodeModel()
-  result.messages = messages.newMessagesModel(result.events)
-  result.profile = profile.newProfileModel()
-  result.contacts = contacts.newContactModel(result.events)
-  result.network = network.newNetworkModel(result.events)
-  result.stickers = stickers.newStickersModel(result.events)
-  result.permissions = permissions.newPermissionsModel(result.events)
-  result.settings = settings.newSettingsModel(result.events)
-  result.tokens = tokens.newTokensModel(result.events)
-  result.provider = provider.newProviderModel(result.events, result.permissions, result.wallet)
-  result.keycard = newKeycardModel(result.backend)
-
-proc initNode*(self: Status, statusGoDir, keystoreDir: string) =
-  statusgo_backend_accounts.initNode(statusGoDir, keystoreDir)
-
-proc startMessenger*(self: Status) {.exportc, dynlib.} =
-  statusgo_backend_core.startMessenger()
-
-proc reset*(self: Status) {.exportc, dynlib.} =
-  # TODO: remove this once accounts are not tracked in the AccountsModel
-  self.accounts.reset()
-
-  # NOT NEEDED self.chat.reset()
-  # NOT NEEDED self.wallet.reset()
-  # NOT NEEDED self.node.reset()
-  # NOT NEEDED self.profile.reset()
-
-  # TODO: add all resets here
-
-proc getNodeVersion*(self: Status): string  {.exportc, dynlib.} =
-  statusgo_backend_settings.getWeb3ClientVersion()
-
-# TODO: duplicated??
-proc saveSetting*(self: Status, setting: Setting, value: string | bool) =
-  discard statusgo_backend_settings.saveSetting(setting, value)
-
-proc getBloomFilter*(self: Status): string {.exportc, dynlib.} =
-  result = statusgo_backend_core.getBloomFilter()
-
-proc getBloomFilterBitsSet*(self: Status): int {.exportc, dynlib.} =
-  let bloomFilter = statusgo_backend_core.getBloomFilter()
-  var bitCount = 0;
-  for b in hexToSeqByte(bloomFilter):
-    bitCount += countSetBits(b)
-  return bitCount
-
-# C Helpers
-# ==============================================================================
-# This creates extra functions with a simpler API for C interop. This is to avoid
-# having to manually create nim strings, (we can use cstring) instead, and also
-# because functions that accept more than one type for the same parameter are not
-# exported correctly
-
-
-proc newStatusInstance*(): Status {.exportc, dynlib.} =
-  newStatusInstance("statusgo")
-
-proc initNode*(self: Status, statusGoDir, keystoreDir: cstring) {.exportc, dynlib.} =
-  self.initNode($statusGoDir, $keystoreDir)
-
-proc saveStringSetting*(self: Status, setting: Setting, value: cstring) {.exportc, dynlib.} =
-  self.saveSetting(setting, $value)
-
-proc saveBoolSetting*(self: Status, setting: Setting, value: bool) {.exportc, dynlib.} =
-  self.saveSetting(setting, value)

src/constants.nim

@@ -3,7 +3,7 @@ import os, sequtils, strutils
 import # vendor libs
   confutils
 
-const sep = when defined(windows): "\\" else: "/"
+const sep* = when defined(windows): "\\" else: "/"
 
 proc defaultDataDir*(): string =
   let homeDir = getHomeDir()
@@ -56,7 +56,7 @@ let
   OPENURI* = desktopConfig.uri
   DATADIR* = baseDir & sep
   STATUSGODIR* = joinPath(baseDir, "data") & sep
-  KEYSTOREDIR* = joinPath(baseDir, "data", "keystore") & sep
+  ROOTKEYSTOREDIR* = joinPath(baseDir, "data", "keystore") & sep
   TMPDIR* = joinPath(baseDir, "tmp") & sep
   LOGDIR* = joinPath(baseDir, "logs") & sep