diff --git a/src/status_im/data_store/contact_groups.cljs b/src/status_im/data_store/contact_groups.cljs index cc66bc6396..0975ac081d 100644 --- a/src/status_im/data_store/contact_groups.cljs +++ b/src/status_im/data_store/contact_groups.cljs @@ -5,15 +5,11 @@ [status-im.data-store.realm.contact-groups :as data-store]) (:refer-clojure :exclude [exists?])) -(defn- normalize-contacts - [item] - (update item :contacts vals)) - (re-frame/reg-cofx :data-store/get-all-contact-groups (fn [cofx _] (assoc cofx :all-contact-groups (into {} - (map (comp (juxt :group-id identity) normalize-contacts)) + (map (juxt :group-id identity)) (data-store/get-all-as-list))))) (re-frame/reg-fx diff --git a/src/status_im/data_store/realm/accounts.cljs b/src/status_im/data_store/realm/accounts.cljs index cd96790685..576c13cc7b 100644 --- a/src/status_im/data_store/realm/accounts.cljs +++ b/src/status_im/data_store/realm/accounts.cljs @@ -2,22 +2,22 @@ (:require [status-im.data-store.realm.core :as realm])) (defn get-all-as-list [] - (->> (realm/get-all realm/base-realm :account) - realm/js-object->clj - (map #(update % :settings realm/deserialize)) - (mapv #(realm/fix-map % :networks :id)))) + (->> (realm/all-clj (realm/get-all realm/base-realm :account) :account) + (mapv #(update % :settings realm/deserialize)))) (defn get-by-address [address] - (-> (realm/get-one-by-field-clj realm/base-realm :account :address address) - (update :settings realm/deserialize) - (realm/fix-map :networks :id))) + (-> realm/base-realm + (realm/get-by-field :account :address address) + (realm/single-clj :account) + (update :settings realm/deserialize))) (defn- create-account-fn [account update?] #(realm/create realm/base-realm :account account update?)) (defn save [account update?] (realm/write realm/base-realm - (-> (realm/fix-map->vec account :networks) + (-> account (update :settings realm/serialize) - (create-account-fn update?)))) \ No newline at end of file + (update :networks vals) + (create-account-fn update?)))) diff --git a/src/status_im/data_store/realm/browser.cljs b/src/status_im/data_store/realm/browser.cljs index 3a64a6fb93..5f317e24d3 100644 --- a/src/status_im/data_store/realm/browser.cljs +++ b/src/status_im/data_store/realm/browser.cljs @@ -7,7 +7,7 @@ (-> @realm/account-realm (realm/get-all :browser) (realm/sorted :timestamp :desc) - (realm/js-object->clj))) + (realm/all-clj :browser))) (defn save [browser update?] @@ -15,7 +15,7 @@ (defn delete [browser-id] - (when-let [browser (realm/get-one-by-field @realm/account-realm :browser :browser-id browser-id)] + (when-let [browser (realm/single (realm/get-by-field @realm/account-realm :browser :browser-id browser-id))] (realm/delete @realm/account-realm browser))) (defn exists? @@ -25,4 +25,5 @@ (defn get-by-id [browser-id] (-> @realm/account-realm - (realm/get-one-by-field-clj :browser :browser-id browser-id))) \ No newline at end of file + (realm/get-by-field :browser :browser-id browser-id) + (realm/single-clj :browser))) diff --git a/src/status_im/data_store/realm/chats.cljs b/src/status_im/data_store/realm/chats.cljs index a0c001be57..9f1cd5b33b 100644 --- a/src/status_im/data_store/realm/chats.cljs +++ b/src/status_im/data_store/realm/chats.cljs @@ -7,10 +7,8 @@ (:refer-clojure :exclude [exists?])) (defn- normalize-chat [{:keys [chat-id] :as chat}] - (let [last-clock-value (messages/get-last-clock-value chat-id)] - (-> chat - (realm/fix-map->vec :contacts) - (assoc :last-clock-value (or last-clock-value 0))))) + (let [last-clock-value (messages/get-last-clock-value chat-id)] + (assoc chat :last-clock-value (or last-clock-value 0)))) (defn get-all [] @@ -18,16 +16,17 @@ (-> @realm/account-realm (realm/get-all :chat) (realm/sorted :timestamp :desc) - realm/js-object->clj))) + (realm/all-clj :chat)))) (defn- get-by-id-obj [chat-id] - (realm/get-one-by-field @realm/account-realm :chat :chat-id chat-id)) + (realm/single (realm/get-by-field @realm/account-realm :chat :chat-id chat-id))) (defn get-by-id [chat-id] (-> @realm/account-realm - (realm/get-one-by-field-clj :chat :chat-id chat-id) + (realm/get-by-field :chat :chat-id chat-id) + (realm/single-clj :chat) normalize-chat)) (defn save @@ -59,7 +58,7 @@ (realm/write @realm/account-realm #(aset chat "contacts" (clj->js (into #{} (concat identities - (realm/js-object->clj contacts)))))))) + (realm/list->clj contacts)))))))) (defn remove-contacts [chat-id identities] @@ -68,17 +67,18 @@ (realm/write @realm/account-realm #(aset chat "contacts" (clj->js (remove (into #{} identities) - (realm/js-object->clj contacts))))))) + (realm/list->clj contacts))))))) (defn save-property [chat-id property-name value] (realm/write @realm/account-realm (fn [] (-> @realm/account-realm - (realm/get-one-by-field :chat :chat-id chat-id) + (realm/get-by-field :chat :chat-id chat-id) + realm/single (aset (name property-name) value))))) (defn get-property [chat-id property] - (when-let [chat (realm/get-one-by-field @realm/account-realm :chat :chat-id chat-id)] + (when-let [chat (realm/single (realm/get-by-field @realm/account-realm :chat :chat-id chat-id))] (object/get chat (name property)))) diff --git a/src/status_im/data_store/realm/contact_groups.cljs b/src/status_im/data_store/realm/contact_groups.cljs index ecb00001be..02600d2934 100644 --- a/src/status_im/data_store/realm/contact_groups.cljs +++ b/src/status_im/data_store/realm/contact_groups.cljs @@ -3,14 +3,9 @@ [status-im.data-store.realm.core :as realm]) (:refer-clojure :exclude [exists?])) -(defn get-all - [] - (-> @realm/account-realm - (realm/get-all :contact-group))) - (defn get-all-as-list [] - (realm/js-object->clj (get-all))) + (realm/all-clj (realm/get-all @realm/account-realm :contact-group) :contact-group)) (defn save [group update?] @@ -21,7 +16,8 @@ (realm/write @realm/account-realm (fn [] (-> @realm/account-realm - (realm/get-one-by-field :contact-group :group-id group-id) + (realm/get-by-field :contact-group :group-id group-id) + realm/single (aset (name property-name) value))))) (defn exists? @@ -30,12 +26,14 @@ (defn delete [group-id] - (when-let [group (realm/get-one-by-field @realm/account-realm :contact-group :group-id group-id)] + (when-let [group (-> @realm/account-realm + (realm/get-by-field :contact-group :group-id group-id) + realm/single)] (realm/delete @realm/account-realm group))) (defn- get-by-id-obj [group-id] - (realm/get-one-by-field @realm/account-realm :contact-group :group-id group-id)) + (realm/single (realm/get-by-field @realm/account-realm :contact-group :group-id group-id))) (defn add-contacts [group-id identities] @@ -44,4 +42,4 @@ (realm/write @realm/account-realm #(aset group "contacts" (clj->js (into #{} (concat identities - (realm/js-object->clj contacts)))))))) + (realm/list->clj contacts)))))))) diff --git a/src/status_im/data_store/realm/contacts.cljs b/src/status_im/data_store/realm/contacts.cljs index 6f543df2e4..ea2d8781a8 100644 --- a/src/status_im/data_store/realm/contacts.cljs +++ b/src/status_im/data_store/realm/contacts.cljs @@ -2,23 +2,19 @@ (:require [status-im.data-store.realm.core :as realm]) (:refer-clojure :exclude [exists?])) -(defn get-all - [] - (-> @realm/account-realm - (realm/get-all :contact) - (realm/sorted :name :asc))) - (defn get-all-as-list [] - (realm/js-object->clj (get-all))) + (realm/all-clj (realm/get-all @realm/account-realm :contact) :contact)) (defn get-by-id [whisper-identity] - (realm/get-one-by-field @realm/account-realm :contact :whisper-identity whisper-identity)) + (realm/single (realm/get-by-field @realm/account-realm :contact :whisper-identity whisper-identity))) (defn get-by-id-cljs [whisper-identity] - (realm/get-one-by-field-clj @realm/account-realm :contact :whisper-identity whisper-identity)) + (-> @realm/account-realm + (realm/get-by-field :contact :whisper-identity whisper-identity) + (realm/single-clj :contact))) (defn save [contact update?] diff --git a/src/status_im/data_store/realm/core.cljs b/src/status_im/data_store/realm/core.cljs index 2beca6c1f0..6c435d209a 100644 --- a/src/status_im/data_store/realm/core.cljs +++ b/src/status_im/data_store/realm/core.cljs @@ -1,37 +1,36 @@ (ns status-im.data-store.realm.core - (:require [status-im.utils.types :refer [to-string]] + (:require [goog.object :as object] + [goog.string :as gstr] + [clojure.string :as string] [status-im.data-store.realm.schemas.account.core :as account] [status-im.data-store.realm.schemas.base.core :as base] [taoensso.timbre :as log] [status-im.utils.fs :as fs] [status-im.utils.async :as utils.async] - [clojure.string :as str] - [goog.string :as gstr] [cognitect.transit :as transit] - [clojure.walk :as walk] [status-im.react-native.js-dependencies :as rn-dependencies] [status-im.utils.utils :as utils]) (:refer-clojure :exclude [exists?])) -(defn realm-version +(defn- realm-version [file-name] (.schemaVersion rn-dependencies/realm file-name)) -(defn open-realm +(defn- open-realm [options file-name] (let [options (merge options {:path file-name})] (when (cljs.core/exists? js/window) (rn-dependencies/realm. (clj->js options))))) -(defn delete-realm +(defn- delete-realm [file-name] (.deleteFile rn-dependencies/realm (clj->js {:path file-name}))) -(defn close [realm] +(defn- close [realm] (when realm (.close realm))) -(defn migrate-realm [file-name schemas] +(defn- migrate-realm [file-name schemas] (let [current-version (realm-version file-name)] (doseq [schema schemas :when (> (:schemaVersion schema) current-version) @@ -39,12 +38,12 @@ (close migrated-realm))) (open-realm (last schemas) file-name)) -(defn reset-realm [file-name schemas] +(defn- reset-realm [file-name schemas] (utils/show-popup "Please note" "You must recover or create a new account with this upgrade. Also chatting with accounts in previous releases is incompatible") (delete-realm file-name) (open-realm (last schemas) file-name)) -(defn open-migrated-realm +(defn- open-migrated-realm [file-name schemas] ;; TODO: remove for release 0.9.18 ;; delete the realm file if its schema version is higher @@ -55,12 +54,18 @@ (reset-realm file-name schemas) (migrate-realm file-name schemas))) +(defn- index-entity-schemas [all-schemas] + (into {} (map (juxt :name identity)) (-> all-schemas last :schema))) + (def new-account-filename "new-account") (def base-realm (open-migrated-realm (.-defaultPath rn-dependencies/realm) base/schemas)) (def account-realm (atom (open-migrated-realm new-account-filename account/schemas))) +(def entity->schemas (merge (index-entity-schemas base/schemas) + (index-entity-schemas account/schemas))) + (def realm-queue (utils.async/task-queue 2000)) (defn close-account-realm [] @@ -86,51 +91,25 @@ (close-account-realm) (log/debug "is new account? " new-account?) (if new-account? - (let [new-path (str/replace path new-account-filename address)] + (let [new-path (string/replace path new-account-filename address)] (log/debug "Moving file " path " to " new-path) (fs/move-file path new-path #(move-file-handler address % handler))) (do (reset! account-realm (open-migrated-realm address account/schemas)) (handler nil))))) +(declare realm-obj->clj) + ;; realm functions -(defn and-query [queries] - (str/join " and " queries)) - -(defn or-query [queries] - (str/join " or " queries)) - (defn write [realm f] (.write realm f)) - -(def transit-special-chars #{"~" "^" "`"}) -(def transit-escape-char "~") - -(defn to-be-escaped? - "Check if element is a string that begins - with a character recognized as special by Transit" - [e] - (and (string? e) - (contains? transit-special-chars (first e)))) - -(defn prepare-for-transit - "Following Transit documentation, escape leading special characters - in strings by prepending a ~. This prepares for subsequent - fetching from Realm where Transit is used for JSON parsing" - [message] - (let [walk-fn (fn [e] - (cond->> e - (to-be-escaped? e) - (str transit-escape-char)))] - (walk/postwalk walk-fn message))) - (defn create ([realm schema-name obj] (create realm schema-name obj false)) ([realm schema-name obj update?] - (.create realm (to-string schema-name) (clj->js (prepare-for-transit obj)) update?))) + (.create realm (name schema-name) (clj->js obj) update?))) (defn save ([realm schema-name obj] @@ -149,15 +128,12 @@ (write realm #(.delete realm obj))) (defn get-all [realm schema-name] - (.objects realm (to-string schema-name))) + (.objects realm (name schema-name))) (defn sorted [results field-name order] - (.sorted results (to-string field-name) (if (= order :asc) - false - true))) - -(defn get-count [objs] - (.-length objs)) + (.sorted results (name field-name) (if (= order :asc) + false + true))) (defn page [results from to] (js/Array.prototype.slice.call results from to)) @@ -165,88 +141,83 @@ (defn filtered [results filter-query] (.filtered results filter-query)) -(def map->vec - (comp vec vals)) - (def reader (transit/reader :json)) (def writer (transit/writer :json)) (defn serialize [o] (transit/write writer o)) (defn deserialize [o] (try (transit/read reader o) (catch :default e nil))) -(defn- internal-convert [js-object] - (->> js-object - (.stringify js/JSON) - deserialize - walk/keywordize-keys)) +(defn- realm-list->clj-coll [realm-list coll map-fn] + (when realm-list + (into coll (map map-fn) (range 0 (.-length realm-list))))) -(defn js-object->clj - "Converts any js type/object into a map recursively - Performs 5 times better than iterating over the object keys - and that would require special care for collections" - [js-object] - (let [o (internal-convert js-object)] - (if (map? o) (map->vec o) o))) +(defn- list->clj [realm-list] + (realm-list->clj-coll realm-list [] #(object/get realm-list %))) -(defn fix-map->vec - "Takes a map m and a keyword k - Updates the value in k, a map representing a list, into a vector - example: {:0 0 :1 1} -> [0 1]" - [m k] - (update m k map->vec)) +(defn- object-list->clj [realm-object-list entity-name] + (let [primary-key (-> entity->schemas (get entity-name) :primaryKey name)] + (realm-list->clj-coll realm-object-list + {} + #(let [realm-obj (object/get realm-object-list %)] + [(object/get realm-obj primary-key) (realm-obj->clj realm-obj entity-name)])))) -(defn fix-map - "Takes a map m, a keyword k and an id id - Updates the value in k, a map representing a list, into a map using - the id extracted from the value as a key - example: {:0 {:id 1 :a 2} :1 {:id 2 :a 2}} -> {1 {:id 1 :a 2} 2 {:id 2 :a 2}}" - [m k id] - (update m k #(reduce (fn [acc [_ v]] - (assoc acc (get v id) v)) - {} - %))) +(defn- realm-obj->clj [realm-obj entity-name] + (when realm-obj + (let [{:keys [primaryKey properties]} (get entity->schemas entity-name)] + (into {} + (map (fn [[prop-name {:keys [type objectType]}]] + (let [prop-value (object/get realm-obj (name prop-name))] + [prop-name (case type + "string[]" (list->clj prop-value) + :list (object-list->clj prop-value objectType) + prop-value)]))) + properties)))) -(defn single [result] - (aget result 0)) +(defn single + "Takes realm results, returns the first one" + [result] + (object/get result 0)) -(defn single-clj [results] - (some-> results single internal-convert)) +(defn single-clj + "Takes realm results and schema name, returns the first result converted to cljs datastructure" + [results schema-name] + (-> results single (realm-obj->clj schema-name))) -(defn- get-schema-by-name [opts] - (->> opts - (mapv (fn [{:keys [name] :as schema}] - [(keyword name) schema])) - (into {}))) +(defn all-clj + "Takes realm results and schema name, returns results as converted cljs datastructures in vector" + [results schema-name] + (realm-list->clj-coll results [] #(realm-obj->clj (object/get results %) schema-name))) (defn- field-type [realm schema-name field] - (let [schema-by-name (get-schema-by-name (js->clj (.-schema realm) :keywordize-keys true)) - field-def (get-in schema-by-name [schema-name :properties field])] - (if (map? field-def) - (:type field-def) - field-def))) + (let [field-def (get-in entity->schemas [schema-name :properties field])] + (or (:type field-def) field-def))) (defmulti to-query (fn [_ _ operator _ _] operator)) (defmethod to-query :eq [schema schema-name _ field value] - (let [value (to-string value) - field-type (field-type schema schema-name field) + (let [field-type (field-type schema schema-name field) escaped-value (when value (gstr/escapeString (str value))) query (str (name field) "=" (if (= "string" (name field-type)) (str "\"" escaped-value "\"") value))] query)) -(defn get-by-field [realm schema-name field value] +(defn get-by-field + "Selects objects from realm identified by schema-name based on value of field" + [realm schema-name field value] (let [q (to-query realm schema-name :eq field value)] (.filtered (.objects realm (name schema-name)) q))) -(defn get-one-by-field [realm schema-name field value] - (single (get-by-field realm schema-name field value))) +(defn- and-query [queries] + (string/join " and " queries)) -(defn get-one-by-field-clj [realm schema-name field value] - (single-clj (get-by-field realm schema-name field value))) +(defn- or-query [queries] + (string/join " or " queries)) -(defn get-by-fields [realm schema-name op fields] +(defn get-by-fields + "Selects objects from realm identified by schema name based on field values + combined by `:and`/`:or` operator" + [realm schema-name op fields] (let [queries (map (fn [[k v]] (to-query realm schema-name :eq k v)) fields)] @@ -255,5 +226,8 @@ :and (and-query queries) :or (or-query queries))))) -(defn exists? [realm schema-name fields] +(defn exists? + "Returns true if object/s identified by schema-name and field values (`:and`) + exists in realm" + [realm schema-name fields] (pos? (.-length (get-by-fields realm schema-name :and fields)))) diff --git a/src/status_im/data_store/realm/local_storage.cljs b/src/status_im/data_store/realm/local_storage.cljs index aac0f2f9ee..30e94aa128 100644 --- a/src/status_im/data_store/realm/local_storage.cljs +++ b/src/status_im/data_store/realm/local_storage.cljs @@ -3,7 +3,7 @@ (defn get-by-chat-id [chat-id] - (realm/get-one-by-field-clj @realm/account-realm :local-storage :chat-id chat-id)) + (realm/single-clj (realm/get-by-field @realm/account-realm :local-storage :chat-id chat-id) :local-storage)) (defn save [local-storage] diff --git a/src/status_im/data_store/realm/messages.cljs b/src/status_im/data_store/realm/messages.cljs index 1b5dd8ba75..0e2ccf6051 100644 --- a/src/status_im/data_store/realm/messages.cljs +++ b/src/status_im/data_store/realm/messages.cljs @@ -2,14 +2,9 @@ (:require [status-im.data-store.realm.core :as realm]) (:refer-clojure :exclude [exists?])) - -(defn get-all - [] - (realm/get-all @realm/account-realm :message)) - (defn get-all-as-list [] - (realm/js-object->clj (get-all))) + (realm/all-clj (realm/get-all @realm/account-realm :message) :message)) (defn- transform-message [message] (update message :user-statuses @@ -19,7 +14,9 @@ (defn get-by-id [message-id] - (some-> (realm/get-one-by-field-clj @realm/account-realm :message :message-id message-id) + (some-> @realm/account-realm + (realm/get-by-field :message :message-id message-id) + (realm/single-clj :message) transform-message)) (defn get-by-chat-id @@ -29,7 +26,7 @@ (let [messages (-> (realm/get-by-field @realm/account-realm :message :chat-id chat-id) (realm/sorted :timestamp :desc) (realm/page from (+ from number-of-messages)) - realm/js-object->clj)] + (realm/all-clj :message))] (mapv transform-message messages)))) (defn get-message-ids-by-chat-id @@ -51,18 +48,11 @@ (aget msg "message-id")))))) @chat-id->message-id)) -(defn get-by-fields - [fields from number-of-messages] - (-> (realm/get-by-fields @realm/account-realm :message :and fields) - (realm/sorted :timestamp :desc) - (realm/page from (+ from number-of-messages)) - realm/js-object->clj)) - (defn get-last-clock-value [chat-id] (-> (realm/get-by-field @realm/account-realm :message :chat-id chat-id) (realm/sorted :clock-value :desc) - (realm/single-clj) + (realm/single-clj :message) :clock-value)) (defn get-unviewed @@ -70,7 +60,7 @@ (-> @realm/account-realm (realm/get-by-fields :user-status :and {:whisper-identity current-public-key :status :received}) - realm/js-object->clj)) + (realm/all-clj :user-status))) (defn exists? [message-id] diff --git a/src/status_im/data_store/realm/requests.cljs b/src/status_im/data_store/realm/requests.cljs index 4346562c1f..56d749e0a9 100644 --- a/src/status_im/data_store/realm/requests.cljs +++ b/src/status_im/data_store/realm/requests.cljs @@ -5,7 +5,7 @@ [] (-> @realm/account-realm (realm/get-by-field :request :status "open") - realm/js-object->clj)) + (realm/all-clj :request))) (defn save [request] diff --git a/src/status_im/data_store/realm/schemas/account/v1/local_storage.cljs b/src/status_im/data_store/realm/schemas/account/v1/local_storage.cljs index a678e6d5a6..9c6c101e71 100644 --- a/src/status_im/data_store/realm/schemas/account/v1/local_storage.cljs +++ b/src/status_im/data_store/realm/schemas/account/v1/local_storage.cljs @@ -2,6 +2,6 @@ (def schema {:name :local-storage :primaryKey :chat-id - :properties {:chat-id "string" - :data {:type "string" + :properties {:chat-id :string + :data {:type :string :default "{}"}}}) diff --git a/src/status_im/data_store/realm/transport.cljs b/src/status_im/data_store/realm/transport.cljs index 68120d98ce..a9e7657c54 100644 --- a/src/status_im/data_store/realm/transport.cljs +++ b/src/status_im/data_store/realm/transport.cljs @@ -4,8 +4,7 @@ (defn get-all [] - (-> (realm/get-all @realm/account-realm :transport) - realm/js-object->clj)) + (realm/all-clj (realm/get-all @realm/account-realm :transport) :transport)) (defn exists? [chat-id] @@ -17,5 +16,5 @@ (defn delete [chat-id] - (when-let [chat (realm/get-by-field @realm/account-realm :transport :chat-id chat-id)] + (when-let [chat (realm/single (realm/get-by-field @realm/account-realm :transport :chat-id chat-id))] (realm/delete @realm/account-realm chat))) diff --git a/test/cljs/status_im/test/data_store/realm/core.cljs b/test/cljs/status_im/test/data_store/realm/core.cljs index 8c833276ad..755f0edf78 100644 --- a/test/cljs/status_im/test/data_store/realm/core.cljs +++ b/test/cljs/status_im/test/data_store/realm/core.cljs @@ -7,19 +7,3 @@ (is (nil? (core/deserialize "giberrish"))) (is (nil? (core/deserialize nil))) (is (nil? (core/deserialize (core/serialize nil))))) - -(deftest transit-preparation - (testing "Check if leading Transit special characters are properly escaped with tildes" - (let [data {:to-be-escaped1 "~bad string" - :to-be-escaped2 "^another bad string" - :to-be-escaped3 "`and another bad string" - :no-escaping "no escaping" - :vector-content ["a" "b" "c"]} - prepared-data (core/prepare-for-transit data)] - (is (= "~~bad string" (:to-be-escaped1 prepared-data))) - (is (= "~^another bad string" (:to-be-escaped2 prepared-data))) - (is (= "~`and another bad string" (:to-be-escaped3 prepared-data))) - (is (= "no escaping" (:no-escaping prepared-data))) - (is (= data (-> prepared-data - clj->js - core/internal-convert))))))