Finish faster conversions

src/status_im/data_store/contact_groups.cljs

@@ -5,15 +5,11 @@
             [status-im.data-store.realm.contact-groups :as data-store])
   (:refer-clojure :exclude [exists?]))
 
-(defn- normalize-contacts
-  [item]
-  (update item :contacts vals))
-
 (re-frame/reg-cofx
   :data-store/get-all-contact-groups
   (fn [cofx _]
     (assoc cofx :all-contact-groups (into {}
-                                          (map (comp (juxt :group-id identity) normalize-contacts))
+                                          (map (juxt :group-id identity))
                                           (data-store/get-all-as-list)))))
 
 (re-frame/reg-fx

src/status_im/data_store/realm/accounts.cljs

@@ -2,22 +2,22 @@
   (:require [status-im.data-store.realm.core :as realm]))
 
 (defn get-all-as-list []
-  (->> (realm/get-all realm/base-realm :account)
+  (->> (realm/all-clj (realm/get-all realm/base-realm :account) :account)
-       realm/js-object->clj
+       (mapv #(update % :settings realm/deserialize))))
-       (map #(update % :settings realm/deserialize))
-       (mapv #(realm/fix-map % :networks :id))))
 
 
 (defn get-by-address [address]
-  (-> (realm/get-one-by-field-clj realm/base-realm :account :address address)
+  (-> realm/base-realm
-      (update :settings realm/deserialize)
+      (realm/get-by-field :account :address address)
-      (realm/fix-map :networks :id)))
+      (realm/single-clj :account)
+      (update :settings realm/deserialize)))
 
 (defn- create-account-fn [account update?]
   #(realm/create realm/base-realm :account account update?))
 
 (defn save [account update?]
   (realm/write realm/base-realm
-               (-> (realm/fix-map->vec account :networks)
+               (-> account
                    (update :settings realm/serialize)
-                   (create-account-fn update?))))
+                   (update :networks vals)
+                   (create-account-fn update?))))

src/status_im/data_store/realm/browser.cljs

@@ -7,7 +7,7 @@
   (-> @realm/account-realm
       (realm/get-all :browser)
       (realm/sorted :timestamp :desc)
-      (realm/js-object->clj)))
+      (realm/all-clj :browser)))
 
 (defn save
   [browser update?]
@@ -15,7 +15,7 @@
 
 (defn delete
   [browser-id]
-  (when-let [browser (realm/get-one-by-field @realm/account-realm :browser :browser-id browser-id)]
+  (when-let [browser (realm/single (realm/get-by-field @realm/account-realm :browser :browser-id browser-id))]
     (realm/delete @realm/account-realm browser)))
 
 (defn exists?
@@ -25,4 +25,5 @@
 (defn get-by-id
   [browser-id]
   (-> @realm/account-realm
-      (realm/get-one-by-field-clj :browser :browser-id browser-id)))
+      (realm/get-by-field :browser :browser-id browser-id)
+      (realm/single-clj :browser)))

src/status_im/data_store/realm/chats.cljs

@@ -7,10 +7,8 @@
   (:refer-clojure :exclude [exists?]))
 
 (defn- normalize-chat [{:keys [chat-id] :as chat}]
-  (let [last-clock-value   (messages/get-last-clock-value chat-id)]
+  (let [last-clock-value (messages/get-last-clock-value chat-id)] 
-    (-> chat
+    (assoc chat :last-clock-value  (or last-clock-value 0))))
-        (realm/fix-map->vec :contacts)
-        (assoc :last-clock-value  (or last-clock-value 0)))))
 
 (defn get-all
   []
@@ -18,16 +16,17 @@
        (-> @realm/account-realm
            (realm/get-all :chat)
            (realm/sorted :timestamp :desc)
-           realm/js-object->clj)))
+           (realm/all-clj :chat))))
 
 (defn- get-by-id-obj
   [chat-id]
-  (realm/get-one-by-field @realm/account-realm :chat :chat-id chat-id))
+  (realm/single (realm/get-by-field @realm/account-realm :chat :chat-id chat-id)))
 
 (defn get-by-id
   [chat-id]
   (-> @realm/account-realm
-      (realm/get-one-by-field-clj :chat :chat-id chat-id)
+      (realm/get-by-field :chat :chat-id chat-id)
+      (realm/single-clj :chat)
       normalize-chat))
 
 (defn save
@@ -59,7 +58,7 @@
     (realm/write @realm/account-realm
                  #(aset chat "contacts"
                         (clj->js (into #{} (concat identities
-                                                   (realm/js-object->clj contacts))))))))
+                                                   (realm/list->clj contacts))))))))
 
 (defn remove-contacts
   [chat-id identities]
@@ -68,17 +67,18 @@
     (realm/write @realm/account-realm
                  #(aset chat "contacts"
                         (clj->js (remove (into #{} identities)
-                                         (realm/js-object->clj contacts)))))))
+                                         (realm/list->clj contacts)))))))
 
 (defn save-property
   [chat-id property-name value]
   (realm/write @realm/account-realm
                (fn []
                  (-> @realm/account-realm
-                     (realm/get-one-by-field :chat :chat-id chat-id)
+                     (realm/get-by-field :chat :chat-id chat-id)
+                     realm/single
                      (aset (name property-name) value)))))
 
 (defn get-property
   [chat-id property]
-  (when-let [chat (realm/get-one-by-field @realm/account-realm :chat :chat-id chat-id)]
+  (when-let [chat (realm/single (realm/get-by-field @realm/account-realm :chat :chat-id chat-id))]
     (object/get chat (name property))))

src/status_im/data_store/realm/contact_groups.cljs

@@ -3,14 +3,9 @@
             [status-im.data-store.realm.core :as realm])
   (:refer-clojure :exclude [exists?]))
 
-(defn get-all
-  []
-  (-> @realm/account-realm
-      (realm/get-all :contact-group)))
-
 (defn get-all-as-list
   []
-  (realm/js-object->clj (get-all)))
+  (realm/all-clj (realm/get-all @realm/account-realm :contact-group) :contact-group))
 
 (defn save
   [group update?]
@@ -21,7 +16,8 @@
   (realm/write @realm/account-realm
                (fn []
                  (-> @realm/account-realm
-                     (realm/get-one-by-field :contact-group :group-id group-id)
+                     (realm/get-by-field :contact-group :group-id group-id)
+                     realm/single
                      (aset (name property-name) value)))))
 
 (defn exists?
@@ -30,12 +26,14 @@
 
 (defn delete
   [group-id]
-  (when-let [group (realm/get-one-by-field @realm/account-realm :contact-group :group-id group-id)]
+  (when-let [group (-> @realm/account-realm
+                       (realm/get-by-field :contact-group :group-id group-id)
+                       realm/single)]
     (realm/delete @realm/account-realm group)))
 
 (defn- get-by-id-obj
   [group-id]
-  (realm/get-one-by-field @realm/account-realm :contact-group :group-id group-id))
+  (realm/single (realm/get-by-field @realm/account-realm :contact-group :group-id group-id)))
 
 (defn add-contacts
   [group-id identities]
@@ -44,4 +42,4 @@
     (realm/write @realm/account-realm
                  #(aset group "contacts"
                         (clj->js (into #{} (concat identities
-                                                   (realm/js-object->clj contacts))))))))
+                                                   (realm/list->clj contacts))))))))

src/status_im/data_store/realm/contacts.cljs

@@ -2,23 +2,19 @@
   (:require [status-im.data-store.realm.core :as realm])
   (:refer-clojure :exclude [exists?]))
 
-(defn get-all
-  []
-  (-> @realm/account-realm
-      (realm/get-all :contact)
-      (realm/sorted :name :asc)))
-
 (defn get-all-as-list
   []
-  (realm/js-object->clj (get-all)))
+  (realm/all-clj (realm/get-all @realm/account-realm :contact) :contact))
 
 (defn get-by-id
   [whisper-identity]
-  (realm/get-one-by-field @realm/account-realm :contact :whisper-identity whisper-identity))
+  (realm/single (realm/get-by-field @realm/account-realm :contact :whisper-identity whisper-identity)))
 
 (defn get-by-id-cljs
   [whisper-identity]
-  (realm/get-one-by-field-clj @realm/account-realm :contact :whisper-identity whisper-identity))
+  (-> @realm/account-realm
+      (realm/get-by-field :contact :whisper-identity whisper-identity)
+      (realm/single-clj :contact)))
 
 (defn save
   [contact update?]

src/status_im/data_store/realm/core.cljs

@@ -1,37 +1,36 @@
 (ns status-im.data-store.realm.core
-  (:require [status-im.utils.types :refer [to-string]]
+  (:require [goog.object :as object]
+            [goog.string :as gstr]
+            [clojure.string :as string]
             [status-im.data-store.realm.schemas.account.core :as account]
             [status-im.data-store.realm.schemas.base.core :as base]
             [taoensso.timbre :as log]
             [status-im.utils.fs :as fs]
             [status-im.utils.async :as utils.async]
-            [clojure.string :as str]
-            [goog.string :as gstr]
             [cognitect.transit :as transit]
-            [clojure.walk :as walk]
             [status-im.react-native.js-dependencies :as rn-dependencies]
             [status-im.utils.utils :as utils])
   (:refer-clojure :exclude [exists?]))
 
-(defn realm-version
+(defn- realm-version
   [file-name]
   (.schemaVersion rn-dependencies/realm file-name))
 
-(defn open-realm
+(defn- open-realm
   [options file-name]
   (let [options (merge options {:path file-name})]
     (when (cljs.core/exists? js/window)
       (rn-dependencies/realm. (clj->js options)))))
 
-(defn delete-realm
+(defn- delete-realm
   [file-name]
   (.deleteFile rn-dependencies/realm (clj->js {:path file-name})))
 
-(defn close [realm]
+(defn- close [realm]
   (when realm
     (.close realm)))
 
-(defn migrate-realm [file-name schemas]
+(defn- migrate-realm [file-name schemas]
   (let [current-version (realm-version file-name)]
     (doseq [schema schemas
             :when (> (:schemaVersion schema) current-version)
@@ -39,12 +38,12 @@
       (close migrated-realm)))
   (open-realm (last schemas) file-name))
 
-(defn reset-realm [file-name schemas]
+(defn- reset-realm [file-name schemas]
   (utils/show-popup "Please note" "You must recover or create a new account with this upgrade. Also chatting with accounts in previous releases is incompatible")
   (delete-realm file-name)
   (open-realm (last schemas) file-name))
 
-(defn open-migrated-realm
+(defn- open-migrated-realm
   [file-name schemas]
   ;; TODO: remove for release 0.9.18
   ;; delete the realm file if its schema version is higher
@@ -55,12 +54,18 @@
     (reset-realm file-name schemas)
     (migrate-realm file-name schemas)))
 
+(defn- index-entity-schemas [all-schemas]
+  (into {} (map (juxt :name identity)) (-> all-schemas last :schema)))
+
 (def new-account-filename "new-account")
 
 (def base-realm (open-migrated-realm (.-defaultPath rn-dependencies/realm) base/schemas))
 
 (def account-realm (atom (open-migrated-realm new-account-filename account/schemas)))
 
+(def entity->schemas (merge (index-entity-schemas base/schemas)
+                            (index-entity-schemas account/schemas)))
+
 (def realm-queue (utils.async/task-queue 2000))
 
 (defn close-account-realm []
@@ -86,51 +91,25 @@
     (close-account-realm)
     (log/debug "is new account? " new-account?)
     (if new-account?
-      (let [new-path (str/replace path new-account-filename address)]
+      (let [new-path (string/replace path new-account-filename address)]
         (log/debug "Moving file " path " to " new-path)
         (fs/move-file path new-path #(move-file-handler address % handler)))
       (do
         (reset! account-realm (open-migrated-realm address account/schemas))
         (handler nil)))))
 
+(declare realm-obj->clj)
+
 ;; realm functions
 
-(defn and-query [queries]
-  (str/join " and " queries))
-
-(defn or-query [queries]
-  (str/join " or " queries))
-
 (defn write [realm f]
   (.write realm f))
 
-
-(def transit-special-chars #{"~" "^" "`"})
-(def transit-escape-char "~")
-
-(defn to-be-escaped?
-  "Check if element is a string that begins
-   with a character recognized as special by Transit"
-  [e]
-  (and (string? e)
-       (contains? transit-special-chars (first e))))
-
-(defn prepare-for-transit
-  "Following Transit documentation, escape leading special characters
-  in strings by prepending a ~. This prepares for subsequent
-  fetching from Realm where Transit is used for JSON parsing"
-  [message]
-  (let [walk-fn (fn [e]
-                  (cond->> e
-                    (to-be-escaped? e)
-                    (str transit-escape-char)))]
-    (walk/postwalk walk-fn message)))
-
 (defn create
   ([realm schema-name obj]
    (create realm schema-name obj false))
   ([realm schema-name obj update?]
-   (.create realm (to-string schema-name) (clj->js (prepare-for-transit obj)) update?)))
+   (.create realm (name schema-name) (clj->js obj) update?)))
 
 (defn save
   ([realm schema-name obj]
@@ -149,15 +128,12 @@
   (write realm #(.delete realm obj)))
 
 (defn get-all [realm schema-name]
-  (.objects realm (to-string schema-name)))
+  (.objects realm (name schema-name)))
 
 (defn sorted [results field-name order]
-  (.sorted results (to-string field-name) (if (= order :asc)
+  (.sorted results (name field-name) (if (= order :asc)
-                                            false
+                                       false
-                                            true)))
+                                       true)))
-
-(defn get-count [objs]
-  (.-length objs))
 
 (defn page [results from to]
   (js/Array.prototype.slice.call results from to))
@@ -165,88 +141,83 @@
 (defn filtered [results filter-query]
   (.filtered results filter-query))
 
-(def map->vec
-  (comp vec vals))
-
 (def reader (transit/reader :json))
 (def writer (transit/writer :json))
 
 (defn serialize [o] (transit/write writer o))
 (defn deserialize [o] (try (transit/read reader o) (catch :default e nil)))
 
-(defn- internal-convert [js-object]
+(defn- realm-list->clj-coll [realm-list coll map-fn]
-  (->> js-object
+  (when realm-list
-       (.stringify js/JSON)
+    (into coll (map map-fn) (range 0 (.-length realm-list)))))
-       deserialize
-       walk/keywordize-keys))
 
-(defn js-object->clj
+(defn- list->clj [realm-list]
-  "Converts any js type/object into a map recursively
+  (realm-list->clj-coll realm-list [] #(object/get realm-list %)))
-  Performs 5 times better than iterating over the object keys
-  and that would require special care for collections"
-  [js-object]
-  (let [o (internal-convert js-object)]
-    (if (map? o) (map->vec o) o)))
 
-(defn fix-map->vec
+(defn- object-list->clj [realm-object-list entity-name]
-  "Takes a map m and a keyword k
+  (let [primary-key (-> entity->schemas (get entity-name) :primaryKey name)]
-  Updates the value in k, a map representing a list, into a vector
+    (realm-list->clj-coll realm-object-list
-  example: {:0 0 :1 1} -> [0 1]"
+                          {}
-  [m k]
+                          #(let [realm-obj (object/get realm-object-list %)]
-  (update m k map->vec))
+                             [(object/get realm-obj primary-key) (realm-obj->clj realm-obj entity-name)]))))
 
-(defn fix-map
+(defn- realm-obj->clj [realm-obj entity-name]
-  "Takes a map m, a keyword k and an id id
+  (when realm-obj
-  Updates the value in k, a map representing a list, into a map using
+    (let [{:keys [primaryKey properties]} (get entity->schemas entity-name)]
-  the id extracted from the value as a key
+      (into {}
-  example: {:0 {:id 1 :a 2} :1 {:id 2 :a 2}} -> {1 {:id 1 :a 2} 2 {:id 2 :a 2}}"
+            (map (fn [[prop-name {:keys [type objectType]}]]
-  [m k id]
+                   (let [prop-value (object/get realm-obj (name prop-name))]
-  (update m k #(reduce (fn [acc [_ v]]
+                     [prop-name (case type
-                         (assoc acc (get v id) v))
+                                  "string[]" (list->clj prop-value)
-                       {}
+                                  :list (object-list->clj prop-value objectType)
-                       %)))
+                                  prop-value)])))
+            properties))))
 
-(defn single [result]
+(defn single
-  (aget result 0))
+  "Takes realm results, returns the first one"
+  [result]
+  (object/get result 0))
 
-(defn single-clj [results]
+(defn single-clj
-  (some-> results single internal-convert))
+  "Takes realm results and schema name, returns the first result converted to cljs datastructure"
+  [results schema-name]
+  (-> results single (realm-obj->clj schema-name)))
 
-(defn- get-schema-by-name [opts]
+(defn all-clj
-  (->> opts
+  "Takes realm results and schema name, returns results as converted cljs datastructures in vector"
-       (mapv (fn [{:keys [name] :as schema}]
+  [results schema-name]
-               [(keyword name) schema]))
+  (realm-list->clj-coll results [] #(realm-obj->clj (object/get results %) schema-name)))
-       (into {})))
 
 (defn- field-type [realm schema-name field]
-  (let [schema-by-name (get-schema-by-name (js->clj (.-schema realm) :keywordize-keys true))
+  (let [field-def (get-in entity->schemas [schema-name :properties field])]
-        field-def      (get-in schema-by-name [schema-name :properties field])]
+    (or (:type field-def) field-def)))
-    (if (map? field-def)
-      (:type field-def)
-      field-def)))
 
 (defmulti to-query (fn [_ _ operator _ _] operator))
 
 (defmethod to-query :eq [schema schema-name _ field value]
-  (let [value         (to-string value)
+  (let [field-type    (field-type schema schema-name field)
-        field-type    (field-type schema schema-name field)
         escaped-value (when value (gstr/escapeString (str value)))
         query         (str (name field) "=" (if (= "string" (name field-type))
                                               (str "\"" escaped-value "\"")
                                               value))]
     query))
 
-(defn get-by-field [realm schema-name field value]
+(defn get-by-field
+  "Selects objects from realm identified by schema-name based on value of field"
+  [realm schema-name field value]
   (let [q (to-query realm schema-name :eq field value)]
     (.filtered (.objects realm (name schema-name)) q)))
 
-(defn get-one-by-field [realm schema-name field value]
+(defn- and-query [queries]
-  (single (get-by-field realm schema-name field value)))
+  (string/join " and " queries))
 
-(defn get-one-by-field-clj [realm schema-name field value]
+(defn- or-query [queries]
-  (single-clj (get-by-field realm schema-name field value)))
+  (string/join " or " queries))
 
-(defn get-by-fields [realm schema-name op fields]
+(defn get-by-fields
+  "Selects objects from realm identified by schema name based on field values
+  combined by `:and`/`:or` operator"
+  [realm schema-name op fields]
   (let [queries (map (fn [[k v]]
                        (to-query realm schema-name :eq k v))
                      fields)]
@@ -255,5 +226,8 @@
                  :and (and-query queries)
                  :or (or-query queries)))))
 
-(defn exists? [realm schema-name fields]
+(defn exists?
+  "Returns true if object/s identified by schema-name and field values (`:and`)
+  exists in realm"
+  [realm schema-name fields]
   (pos? (.-length (get-by-fields realm schema-name :and fields))))

src/status_im/data_store/realm/local_storage.cljs

@@ -3,7 +3,7 @@
 
 (defn get-by-chat-id
   [chat-id]
-  (realm/get-one-by-field-clj @realm/account-realm :local-storage :chat-id chat-id))
+  (realm/single-clj (realm/get-by-field @realm/account-realm :local-storage :chat-id chat-id) :local-storage))
 
 (defn save
   [local-storage]

src/status_im/data_store/realm/messages.cljs

@@ -2,14 +2,9 @@
   (:require [status-im.data-store.realm.core :as realm])
   (:refer-clojure :exclude [exists?]))
 
-
-(defn get-all
-  []
-  (realm/get-all @realm/account-realm :message))
-
 (defn get-all-as-list
   []
-  (realm/js-object->clj (get-all)))
+  (realm/all-clj (realm/get-all @realm/account-realm :message) :message))
 
 (defn- transform-message [message]
   (update message :user-statuses
@@ -19,7 +14,9 @@
 
 (defn get-by-id
   [message-id]
-  (some-> (realm/get-one-by-field-clj @realm/account-realm :message :message-id message-id)
+  (some-> @realm/account-realm
+          (realm/get-by-field :message :message-id message-id)
+          (realm/single-clj :message)
           transform-message))
 
 (defn get-by-chat-id
@@ -29,7 +26,7 @@
    (let [messages (-> (realm/get-by-field @realm/account-realm :message :chat-id chat-id)
                       (realm/sorted :timestamp :desc)
                       (realm/page from (+ from number-of-messages))
-                      realm/js-object->clj)]
+                      (realm/all-clj :message))]
      (mapv transform-message messages))))
 
 (defn get-message-ids-by-chat-id
@@ -51,18 +48,11 @@
                                  (aget msg "message-id"))))))
     @chat-id->message-id))
 
-(defn get-by-fields
-  [fields from number-of-messages]
-  (-> (realm/get-by-fields @realm/account-realm :message :and fields)
-      (realm/sorted :timestamp :desc)
-      (realm/page from (+ from number-of-messages))
-      realm/js-object->clj))
-
 (defn get-last-clock-value
   [chat-id]
   (-> (realm/get-by-field @realm/account-realm :message :chat-id chat-id)
       (realm/sorted :clock-value :desc)
-      (realm/single-clj)
+      (realm/single-clj :message)
       :clock-value))
 
 (defn get-unviewed
@@ -70,7 +60,7 @@
   (-> @realm/account-realm
       (realm/get-by-fields :user-status :and {:whisper-identity current-public-key
                                               :status           :received})
-      realm/js-object->clj))
+      (realm/all-clj :user-status)))
 
 (defn exists?
   [message-id]

src/status_im/data_store/realm/requests.cljs

@@ -5,7 +5,7 @@
   []
   (-> @realm/account-realm
       (realm/get-by-field :request :status "open")
-      realm/js-object->clj))
+      (realm/all-clj :request)))
 
 (defn save
   [request]

src/status_im/data_store/realm/schemas/account/v1/local_storage.cljs

@@ -2,6 +2,6 @@
 
 (def schema {:name       :local-storage
              :primaryKey :chat-id
-             :properties {:chat-id "string"
+             :properties {:chat-id :string
-                          :data    {:type    "string"
+                          :data    {:type    :string
                                     :default "{}"}}})

src/status_im/data_store/realm/transport.cljs

@@ -4,8 +4,7 @@
 
 (defn get-all
   []
-  (-> (realm/get-all @realm/account-realm :transport)
+  (realm/all-clj (realm/get-all @realm/account-realm :transport) :transport))
-      realm/js-object->clj))
 
 (defn exists?
   [chat-id]
@@ -17,5 +16,5 @@
 
 (defn delete
   [chat-id]
-  (when-let [chat (realm/get-by-field @realm/account-realm :transport :chat-id chat-id)]
+  (when-let [chat (realm/single (realm/get-by-field @realm/account-realm :transport :chat-id chat-id))]
     (realm/delete @realm/account-realm chat)))

test/cljs/status_im/test/data_store/realm/core.cljs

@@ -7,19 +7,3 @@
   (is (nil? (core/deserialize "giberrish")))
   (is (nil? (core/deserialize nil)))
   (is (nil? (core/deserialize (core/serialize nil)))))
-
-(deftest transit-preparation
-  (testing "Check if leading Transit special characters are properly escaped with tildes"
-    (let [data          {:to-be-escaped1 "~bad string"
-                         :to-be-escaped2 "^another bad string"
-                         :to-be-escaped3 "`and another bad string"
-                         :no-escaping    "no escaping"
-                         :vector-content ["a" "b" "c"]}
-          prepared-data (core/prepare-for-transit data)]
-      (is (= "~~bad string" (:to-be-escaped1 prepared-data)))
-      (is (= "~^another bad string" (:to-be-escaped2 prepared-data)))
-      (is (= "~`and another bad string" (:to-be-escaped3 prepared-data)))
-      (is (= "no escaping" (:no-escaping prepared-data)))
-      (is (= data (-> prepared-data
-                      clj->js
-                      core/internal-convert))))))