From a91839bf7e89c559cf26d0440af8016577bcdbe5 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Sep 2015 11:51:19 -0700 Subject: [PATCH 01/35] Store a copy of the encryption key --- shared_realm.cpp | 16 +++++++++++----- shared_realm.hpp | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/shared_realm.cpp b/shared_realm.cpp index a7269591..fc5eaa2e 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -17,9 +17,11 @@ //////////////////////////////////////////////////////////////////////////// #include "shared_realm.hpp" + +#include #include #include -#include + #include using namespace realm; @@ -27,25 +29,29 @@ using namespace realm; RealmCache Realm::s_global_cache; std::mutex Realm::s_init_mutex; -Realm::Config::Config(const Config& c) : path(c.path), read_only(c.read_only), in_memory(c.in_memory), schema_version(c.schema_version), encryption_key(c.encryption_key), migration_function(c.migration_function) +Realm::Config::Config(const Config& c) : path(c.path), read_only(c.read_only), in_memory(c.in_memory), schema_version(c.schema_version), migration_function(c.migration_function) { if (c.schema) { schema = std::make_unique(*c.schema); } + if (c.encryption_key) { + encryption_key = std::make_unique(64); + memcpy(encryption_key.get(), c.encryption_key.get(), 64); + } } Realm::Realm(Config &config) : m_config(config), m_thread_id(std::this_thread::get_id()), m_auto_refresh(true), m_in_transaction(false) { try { if (config.read_only) { - m_read_only_group = std::make_unique(config.path, config.encryption_key.data(), Group::mode_ReadOnly); + m_read_only_group = std::make_unique(config.path, config.encryption_key.get(), Group::mode_ReadOnly); m_group = m_read_only_group.get(); } else { - m_history = realm::make_client_history(config.path, config.encryption_key.data()); + m_history = realm::make_client_history(config.path, config.encryption_key.get()); SharedGroup::DurabilityLevel durability = config.in_memory ? SharedGroup::durability_MemOnly : SharedGroup::durability_Full; - m_shared_group = std::make_unique(*m_history, durability, config.encryption_key.data()); + m_shared_group = std::make_unique(*m_history, durability, config.encryption_key.get()); m_group = nullptr; } } diff --git a/shared_realm.hpp b/shared_realm.hpp index af8b774a..d3c19263 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -45,7 +45,7 @@ namespace realm { std::string path; bool read_only; bool in_memory; - StringData encryption_key; + std::unique_ptr encryption_key; std::unique_ptr schema; uint64_t schema_version; From b4f856bdd0582acdf7afff70d9e8f642fda8b6d7 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 24 Aug 2015 13:12:49 -0700 Subject: [PATCH 02/35] Use NSDMIs for Realm::Config and make it moveable --- shared_realm.hpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/shared_realm.hpp b/shared_realm.hpp index d3c19263..5344f1ef 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -43,16 +43,17 @@ namespace realm { struct Config { std::string path; - bool read_only; - bool in_memory; + bool read_only = false; + bool in_memory = false; std::unique_ptr encryption_key; std::unique_ptr schema; - uint64_t schema_version; + uint64_t schema_version = ObjectStore::NotVersioned; MigrationFunction migration_function; - Config() : read_only(false), in_memory(false), schema_version(ObjectStore::NotVersioned) {}; + Config() = default; + Config(Config&&) = default; Config(const Config& c); }; From 348f4a7b3920add7f1042837f81e5eeb111b591f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 24 Aug 2015 13:14:59 -0700 Subject: [PATCH 03/35] Reduce s_init_mutex's scope --- shared_realm.cpp | 3 ++- shared_realm.hpp | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/shared_realm.cpp b/shared_realm.cpp index fc5eaa2e..fa921fe7 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -23,11 +23,11 @@ #include #include +#include using namespace realm; RealmCache Realm::s_global_cache; -std::mutex Realm::s_init_mutex; Realm::Config::Config(const Config& c) : path(c.path), read_only(c.read_only), in_memory(c.in_memory), schema_version(c.schema_version), migration_function(c.migration_function) { @@ -108,6 +108,7 @@ SharedRealm Realm::get_shared_realm(Config &config) realm = SharedRealm(new Realm(config)); // we want to ensure we are only initializing a single realm at a time + static std::mutex s_init_mutex; std::lock_guard lock(s_init_mutex); uint64_t old_version = ObjectStore::get_schema_version(realm->read_group()); diff --git a/shared_realm.hpp b/shared_realm.hpp index 5344f1ef..31c0a79b 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include @@ -117,8 +116,6 @@ namespace realm { Group *m_group; - static std::mutex s_init_mutex; - public: ExternalNotificationFunction m_external_notifier; From 45890f2772956400ea84f0b075585ce80acebce1 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 24 Aug 2015 13:16:31 -0700 Subject: [PATCH 04/35] Use NSDMIs for Realm --- shared_realm.cpp | 3 +-- shared_realm.hpp | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/shared_realm.cpp b/shared_realm.cpp index fa921fe7..df89301c 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -40,7 +40,7 @@ Realm::Config::Config(const Config& c) : path(c.path), read_only(c.read_only), i } } -Realm::Realm(Config &config) : m_config(config), m_thread_id(std::this_thread::get_id()), m_auto_refresh(true), m_in_transaction(false) +Realm::Realm(Config &config) : m_config(config) { try { if (config.read_only) { @@ -52,7 +52,6 @@ Realm::Realm(Config &config) : m_config(config), m_thread_id(std::this_thread::g SharedGroup::DurabilityLevel durability = config.in_memory ? SharedGroup::durability_MemOnly : SharedGroup::durability_Full; m_shared_group = std::make_unique(*m_history, durability, config.encryption_key.get()); - m_group = nullptr; } } catch (util::File::PermissionDenied const& ex) { diff --git a/shared_realm.hpp b/shared_realm.hpp index 31c0a79b..3ddfb5a5 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -100,9 +100,9 @@ namespace realm { Realm(Config &config); Config m_config; - std::thread::id m_thread_id; - bool m_in_transaction; - bool m_auto_refresh; + std::thread::id m_thread_id = std::this_thread::get_id(); + bool m_in_transaction = false; + bool m_auto_refresh = true; std::set m_notifications; void send_local_notifications(const std::string ¬ification); @@ -114,7 +114,7 @@ namespace realm { std::unique_ptr m_shared_group; std::unique_ptr m_read_only_group; - Group *m_group; + Group *m_group = nullptr; public: ExternalNotificationFunction m_external_notifier; From 25a6734111e4fbf51efdcbeff3f75dd36dcf38c9 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 24 Aug 2015 13:22:42 -0700 Subject: [PATCH 05/35] Eliminate some copies --- shared_realm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared_realm.cpp b/shared_realm.cpp index df89301c..fe97d8b3 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -299,7 +299,7 @@ void Realm::notify() void Realm::send_local_notifications(const std::string &type) { verify_thread(); - for (NotificationFunction notification : m_notifications) { + for (NotificationFunction const& notification : m_notifications) { (*notification)(type); } } From 0ae1bb188e5c68acf7e70f9bed5b41aadd672807 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 26 Aug 2015 11:32:13 -0700 Subject: [PATCH 06/35] Don't cache dynamic realms in the ObjectStore cache either --- shared_realm.cpp | 6 ++++-- shared_realm.hpp | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/shared_realm.cpp b/shared_realm.cpp index fe97d8b3..7a2e7be0 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -29,7 +29,7 @@ using namespace realm; RealmCache Realm::s_global_cache; -Realm::Config::Config(const Config& c) : path(c.path), read_only(c.read_only), in_memory(c.in_memory), schema_version(c.schema_version), migration_function(c.migration_function) +Realm::Config::Config(const Config& c) : path(c.path), read_only(c.read_only), in_memory(c.in_memory), cache(c.cache), schema_version(c.schema_version), migration_function(c.migration_function) { if (c.schema) { schema = std::make_unique(*c.schema); @@ -132,7 +132,9 @@ SharedRealm Realm::get_shared_realm(Config &config) realm->update_schema(*realm->m_config.schema, realm->m_config.schema_version); } - s_global_cache.cache_realm(realm, realm->m_thread_id); + if (config.cache) { + s_global_cache.cache_realm(realm, realm->m_thread_id); + } return realm; } diff --git a/shared_realm.hpp b/shared_realm.hpp index 3ddfb5a5..5c74e13d 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -44,6 +44,7 @@ namespace realm { std::string path; bool read_only = false; bool in_memory = false; + bool cache = true; std::unique_ptr encryption_key; std::unique_ptr schema; From 563a8374d010a84a83b009e186f9ce7de37d5c34 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 26 Aug 2015 13:20:06 -0700 Subject: [PATCH 07/35] Use NSDMIs for realm::Property --- property.hpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/property.hpp b/property.hpp index 22e6f90c..893f772d 100644 --- a/property.hpp +++ b/property.hpp @@ -46,15 +46,12 @@ namespace realm { }; struct Property { - public: - Property() : object_type(""), is_primary(false), is_indexed(false), is_nullable(false) {} - std::string name; PropertyType type; std::string object_type; - bool is_primary; - bool is_indexed; - bool is_nullable; + bool is_primary = false; + bool is_indexed = false; + bool is_nullable = false; size_t table_column; bool requires_index() { return is_primary || is_indexed; } From 55e6cca243b0dc78056851a8d366a0285cf585cb Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 25 Aug 2015 16:39:32 -0700 Subject: [PATCH 08/35] Convert RLMRealmConfiguration to a wrapper around Realm::Config --- shared_realm.cpp | 60 +++++++++++++++++++++++++++++------------------- shared_realm.hpp | 29 +++++++++++++---------- 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/shared_realm.cpp b/shared_realm.cpp index 7a2e7be0..c46b654e 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -22,47 +22,57 @@ #include #include -#include #include using namespace realm; RealmCache Realm::s_global_cache; -Realm::Config::Config(const Config& c) : path(c.path), read_only(c.read_only), in_memory(c.in_memory), cache(c.cache), schema_version(c.schema_version), migration_function(c.migration_function) +Realm::Config::Config(const Config& c) +: path(c.path) +, read_only(c.read_only) +, in_memory(c.in_memory) +, cache(c.cache) +, encryption_key(c.encryption_key) +, schema_version(c.schema_version) +, migration_function(c.migration_function) { if (c.schema) { schema = std::make_unique(*c.schema); } - if (c.encryption_key) { - encryption_key = std::make_unique(64); - memcpy(encryption_key.get(), c.encryption_key.get(), 64); - } } -Realm::Realm(Config &config) : m_config(config) +Realm::Config& Realm::Config::operator=(realm::Realm::Config const& c) +{ + if (&c != this) { + *this = Config(c); + } + return *this; +} + +Realm::Realm(Config config) : m_config(std::move(config)) { try { - if (config.read_only) { - m_read_only_group = std::make_unique(config.path, config.encryption_key.get(), Group::mode_ReadOnly); + if (m_config.read_only) { + m_read_only_group = std::make_unique(m_config.path, m_config.encryption_key.data(), Group::mode_ReadOnly); m_group = m_read_only_group.get(); } else { - m_history = realm::make_client_history(config.path, config.encryption_key.get()); - SharedGroup::DurabilityLevel durability = config.in_memory ? SharedGroup::durability_MemOnly : - SharedGroup::durability_Full; - m_shared_group = std::make_unique(*m_history, durability, config.encryption_key.get()); + m_history = realm::make_client_history(m_config.path, m_config.encryption_key.data()); + SharedGroup::DurabilityLevel durability = m_config.in_memory ? SharedGroup::durability_MemOnly : + SharedGroup::durability_Full; + m_shared_group = std::make_unique(*m_history, durability, m_config.encryption_key.data()); } } catch (util::File::PermissionDenied const& ex) { - throw RealmFileException(RealmFileException::Kind::PermissionDenied, "Unable to open a realm at path '" + config.path + - "'. Please use a path where your app has " + (config.read_only ? "read" : "read-write") + " permissions."); + throw RealmFileException(RealmFileException::Kind::PermissionDenied, "Unable to open a realm at path '" + m_config.path + + "'. Please use a path where your app has " + (m_config.read_only ? "read" : "read-write") + " permissions."); } catch (util::File::Exists const& ex) { - throw RealmFileException(RealmFileException::Kind::Exists, "Unable to open a realm at path '" + config.path + "'"); + throw RealmFileException(RealmFileException::Kind::Exists, "Unable to open a realm at path '" + m_config.path + "'"); } catch (util::File::AccessError const& ex) { - throw RealmFileException(RealmFileException::Kind::AccessError, "Unable to open a realm at path '" + config.path + "'"); + throw RealmFileException(RealmFileException::Kind::AccessError, "Unable to open a realm at path '" + m_config.path + "'"); } catch (IncompatibleLockFile const&) { throw RealmFileException(RealmFileException::Kind::IncompatibleLockFile, "Realm file is currently open in another process " @@ -78,7 +88,7 @@ Group *Realm::read_group() return m_group; } -SharedRealm Realm::get_shared_realm(Config &config) +SharedRealm Realm::get_shared_realm(Config config) { SharedRealm realm = s_global_cache.get_realm(config.path); if (realm) { @@ -104,7 +114,7 @@ SharedRealm Realm::get_shared_realm(Config &config) return realm; } - realm = SharedRealm(new Realm(config)); + realm = SharedRealm(new Realm(std::move(config))); // we want to ensure we are only initializing a single realm at a time static std::mutex s_init_mutex; @@ -153,13 +163,17 @@ bool Realm::update_schema(Schema &schema, uint64_t version) if (!m_config.read_only && ObjectStore::realm_requires_update(read_group(), version, schema)) { // keep old copy to pass to migration function old_config.read_only = true; - SharedRealm old_realm = SharedRealm(new Realm(old_config)), updated_realm = shared_from_this(); + old_config.schema_version = ObjectStore::get_schema_version(read_group()); + old_config.schema = std::make_unique(ObjectStore::schema_from_group(read_group())); + SharedRealm old_realm(new Realm(old_config)); + auto updated_realm = shared_from_this(); // update and migrate begin_transaction(); - changed = ObjectStore::update_realm_with_schema(read_group(), version, *m_config.schema, [=](__unused Group *group, __unused Schema &target_schema) { - m_config.migration_function(old_realm, updated_realm); - }); + changed = ObjectStore::update_realm_with_schema(read_group(), version, *m_config.schema, + [=](__unused Group *group, __unused Schema &target_schema) { + m_config.migration_function(old_realm, updated_realm); + }); commit_transaction(); } else { diff --git a/shared_realm.hpp b/shared_realm.hpp index 5c74e13d..6f868a09 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -19,11 +19,11 @@ #ifndef REALM_REALM_HPP #define REALM_REALM_HPP +#include #include +#include #include #include -#include -#include #include "object_store.hpp" @@ -45,7 +45,7 @@ namespace realm { bool read_only = false; bool in_memory = false; bool cache = true; - std::unique_ptr encryption_key; + std::vector encryption_key; std::unique_ptr schema; uint64_t schema_version = ObjectStore::NotVersioned; @@ -55,18 +55,23 @@ namespace realm { Config() = default; Config(Config&&) = default; Config(const Config& c); + + Config& operator=(Config const&); + Config& operator=(Config&&) = default; }; // Get a cached Realm or create a new one if no cached copies exists - // Caching is done by path - mismatches for inMemory and readOnly Config properties - // will raise an exception - // If schema/schema_version is specified, update_schema is called automatically on the realm - // and a migration is performed. If not specified, the schema version and schema are dynamically - // read from the the existing Realm. - static SharedRealm get_shared_realm(Config &config); + // Caching is done by path - mismatches for in_memory and read_only + // Config properties will raise an exception + // If schema/schema_version is specified, update_schema is called + // automatically on the realm and a migration is performed. If not + // specified, the schema version and schema are dynamically read from + // the the existing Realm. + static SharedRealm get_shared_realm(Config config); - // Updates a Realm to a given target schema/version creating tables and updating indexes as necessary - // Uses the existing migration function on the Config, and the resulting Schema and version with updated + // Updates a Realm to a given target schema/version creating tables and + // updating indexes as necessary. Uses the existing migration function + // on the Config, and the resulting Schema and version with updated // column mappings are set on the realms config upon success. // returns if any changes were made bool update_schema(Schema &schema, uint64_t version); @@ -98,7 +103,7 @@ namespace realm { const std::string DidChangeNotification = "DidChangeNotification"; private: - Realm(Config &config); + Realm(Config config); Config m_config; std::thread::id m_thread_id = std::this_thread::get_id(); From cae4cf2fc0393dca7c1ab1f89b330410581d2260 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 26 Aug 2015 14:53:39 -0700 Subject: [PATCH 09/35] Remove property.hpp include from object_schema.hpp --- object_schema.cpp | 4 ++++ object_schema.hpp | 6 +++--- object_store.hpp | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/object_schema.cpp b/object_schema.cpp index 83edd1f1..41e34685 100644 --- a/object_schema.cpp +++ b/object_schema.cpp @@ -17,13 +17,17 @@ //////////////////////////////////////////////////////////////////////////// #include "object_schema.hpp" + #include "object_store.hpp" +#include "property.hpp" #include #include using namespace realm; +ObjectSchema::~ObjectSchema() = default; + ObjectSchema::ObjectSchema(Group *group, const std::string &name) : name(name) { TableRef tableRef = ObjectStore::table_for_object_type(group, name); Table *table = tableRef.get(); diff --git a/object_schema.hpp b/object_schema.hpp index 56a14d17..51a4c584 100644 --- a/object_schema.hpp +++ b/object_schema.hpp @@ -22,14 +22,14 @@ #include #include -#include "property.hpp" - namespace realm { + class Property; class Group; class ObjectSchema { public: - ObjectSchema() {} + ObjectSchema() = default; + ~ObjectSchema(); // create object schema from existing table // if no table is provided it is looked up in the group diff --git a/object_store.hpp b/object_store.hpp index cacdcd8b..86bbe3ff 100644 --- a/object_store.hpp +++ b/object_store.hpp @@ -26,6 +26,7 @@ #include #include "object_schema.hpp" +#include "property.hpp" namespace realm { class ObjectSchemaValidationException; From e4377bb42a173e68a557f10e6ab18fda906df725 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 26 Aug 2015 15:07:15 -0700 Subject: [PATCH 10/35] Change realm::Schema to a vector rather than a map Much faster to copy and destroy with no loss in lookup performance. --- object_store.cpp | 48 ++++++++++++++++++++++++++++++------------------ object_store.hpp | 4 +--- shared_realm.cpp | 2 +- 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/object_store.cpp b/object_store.cpp index 3d1ec9c3..8dee8d74 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -140,18 +140,24 @@ static inline bool property_has_changed(Property &p1, Property &p2) { return p1.type != p2.type || p1.name != p2.name || p1.object_type != p2.object_type || p1.is_nullable != p2.is_nullable; } +static bool compare_by_name(ObjectSchema const& lft, ObjectSchema const& rgt) { + return lft.name < rgt.name; +} + void ObjectStore::verify_schema(Group *group, Schema &target_schema, bool allow_missing_tables) { + std::sort(begin(target_schema), end(target_schema), compare_by_name); + std::vector errors; for (auto &object_schema : target_schema) { - if (!table_for_object_type(group, object_schema.first)) { + if (!table_for_object_type(group, object_schema.name)) { if (!allow_missing_tables) { - errors.emplace_back(ObjectSchemaValidationException(object_schema.first, - "Missing table for object type '" + object_schema.first + "'.")); + errors.emplace_back(ObjectSchemaValidationException(object_schema.name, + "Missing table for object type '" + object_schema.name + "'.")); } continue; } - auto more_errors = verify_object_schema(group, object_schema.second, target_schema); + auto more_errors = verify_object_schema(group, object_schema, target_schema); errors.insert(errors.end(), more_errors.begin(), more_errors.end()); } if (errors.size()) { @@ -163,6 +169,12 @@ std::vector ObjectStore::verify_object_schema(G std::vector exceptions; ObjectSchema table_schema(group, target_schema.name); + ObjectSchema cmp; + auto schema_contains_table = [&](std::string const& name) { + cmp.name = name; + return std::binary_search(begin(schema), end(schema), cmp, compare_by_name); + }; + // check to see if properties are the same Property *primary = nullptr; for (auto& current_prop : table_schema.properties) { @@ -178,7 +190,7 @@ std::vector ObjectStore::verify_object_schema(G } // check object_type existence - if (current_prop.object_type.length() && schema.find(current_prop.object_type) == schema.end()) { + if (current_prop.object_type.length() && !schema_contains_table(current_prop.object_type)) { exceptions.emplace_back(MissingObjectTypeException(table_schema.name, current_prop)); } @@ -249,11 +261,11 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update std::vector to_update; for (auto& object_schema : target_schema) { bool created = false; - ObjectStore::table_for_object_type_create_if_needed(group, object_schema.first, created); + ObjectStore::table_for_object_type_create_if_needed(group, object_schema.name, created); // we will modify tables for any new objectSchema (table was created) or for all if update_existing is true if (update_existing || created) { - to_update.push_back(&object_schema.second); + to_update.push_back(&object_schema); changed = true; } } @@ -329,7 +341,7 @@ bool ObjectStore::realm_requires_update(Group *group, uint64_t version, Schema & return true; } for (auto& target_schema : schema) { - TableRef table = table_for_object_type(group, target_schema.first); + TableRef table = table_for_object_type(group, target_schema.name); if (!table) { return true; } @@ -377,7 +389,7 @@ Schema ObjectStore::schema_from_group(Group *group) { for (size_t i = 0; i < group->size(); i++) { std::string object_type = object_type_for_table_name(group->get_table_name(i)); if (object_type.length()) { - schema.emplace(object_type, std::move(ObjectSchema(group, object_type))); + schema.emplace_back(group, object_type); } } return schema; @@ -385,13 +397,13 @@ Schema ObjectStore::schema_from_group(Group *group) { bool ObjectStore::indexes_are_up_to_date(Group *group, Schema &schema) { for (auto &object_schema : schema) { - TableRef table = table_for_object_type(group, object_schema.first); + TableRef table = table_for_object_type(group, object_schema.name); if (!table) { continue; } - update_column_mapping(group, object_schema.second); - for (auto& property : object_schema.second.properties) { + update_column_mapping(group, object_schema); + for (auto& property : object_schema.properties) { if (property.requires_index() != table->has_search_index(property.table_column)) { return false; } @@ -403,12 +415,12 @@ bool ObjectStore::indexes_are_up_to_date(Group *group, Schema &schema) { bool ObjectStore::update_indexes(Group *group, Schema &schema) { bool changed = false; for (auto& object_schema : schema) { - TableRef table = table_for_object_type(group, object_schema.first); + TableRef table = table_for_object_type(group, object_schema.name); if (!table) { continue; } - for (auto& property : object_schema.second.properties) { + for (auto& property : object_schema.properties) { if (property.requires_index() == table->has_search_index(property.table_column)) { continue; } @@ -419,7 +431,7 @@ bool ObjectStore::update_indexes(Group *group, Schema &schema) { table->add_search_index(property.table_column); } catch (LogicError const&) { - throw PropertyTypeNotIndexableException(object_schema.first, property); + throw PropertyTypeNotIndexableException(object_schema.name, property); } } else { @@ -432,14 +444,14 @@ bool ObjectStore::update_indexes(Group *group, Schema &schema) { void ObjectStore::validate_primary_column_uniqueness(Group *group, Schema &schema) { for (auto& object_schema : schema) { - auto primary_prop = object_schema.second.primary_key_property(); + auto primary_prop = object_schema.primary_key_property(); if (!primary_prop) { continue; } - TableRef table = table_for_object_type(group, object_schema.first); + TableRef table = table_for_object_type(group, object_schema.name); if (table->get_distinct_view(primary_prop->table_column).size() != table->size()) { - throw DuplicatePrimaryKeyValueException(object_schema.first, *primary_prop); + throw DuplicatePrimaryKeyValueException(object_schema.name, *primary_prop); } } } diff --git a/object_store.hpp b/object_store.hpp index 86bbe3ff..42b8749e 100644 --- a/object_store.hpp +++ b/object_store.hpp @@ -19,7 +19,6 @@ #ifndef REALM_OBJECT_STORE_HPP #define REALM_OBJECT_STORE_HPP -#include #include #include #include @@ -30,8 +29,7 @@ namespace realm { class ObjectSchemaValidationException; - class Schema : public std::map { - }; + using Schema = std::vector; class ObjectStore { public: diff --git a/shared_realm.cpp b/shared_realm.cpp index c46b654e..9d4a4a7f 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -284,7 +284,7 @@ bool Realm::compact() } for (auto &object_schema : *m_config.schema) { - ObjectStore::table_for_object_type(read_group(), object_schema.first)->optimize(); + ObjectStore::table_for_object_type(read_group(), object_schema.name)->optimize(); } m_shared_group->end_read(); From 045c7b20660084f7992a7374a0bf47fa9165a8d1 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 27 Aug 2015 11:38:30 -0700 Subject: [PATCH 11/35] Add Realm::get_schema_version() --- shared_realm.cpp | 12 +++++++++++- shared_realm.hpp | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/shared_realm.cpp b/shared_realm.cpp index 9d4a4a7f..c441df2d 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -133,7 +133,7 @@ SharedRealm Realm::get_shared_realm(Config config) } else if (realm->m_config.read_only) { if (old_version == ObjectStore::NotVersioned) { - throw UnitializedRealmException("Can't open an un-initizliazed Realm without a Schema"); + throw UnitializedRealmException("Can't open an un-initialized Realm without a Schema"); } ObjectStore::verify_schema(realm->read_group(), *realm->m_config.schema, true); } @@ -348,6 +348,16 @@ bool Realm::refresh() return true; } +uint64_t Realm::get_schema_version(const realm::Realm::Config &config) +{ + auto existing_realm = s_global_cache.get_any_realm(config.path); + if (existing_realm) { + return existing_realm->config().schema_version; + } + + return ObjectStore::get_schema_version(Realm(config).read_group()); +} + SharedRealm RealmCache::get_realm(const std::string &path, std::thread::id thread_id) { std::lock_guard lock(m_mutex); diff --git a/shared_realm.hpp b/shared_realm.hpp index 6f868a09..064bfff7 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -76,6 +76,8 @@ namespace realm { // returns if any changes were made bool update_schema(Schema &schema, uint64_t version); + static uint64_t get_schema_version(Config const& config); + const Config &config() const { return m_config; } void begin_transaction(); From 3f226cf342d28666f48c4f91f1610b0c4f7e77e1 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 26 Aug 2015 16:11:45 -0700 Subject: [PATCH 12/35] Rework change notifications Switch to an abstract class rather than std::function in preparation for having more kinds of notifications with different arguments for KVO. --- shared_realm.cpp | 31 ++++++++++++++----------------- shared_realm.hpp | 39 +++++++++++++++++++++------------------ 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/shared_realm.cpp b/shared_realm.cpp index c441df2d..ba71cada 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -223,8 +223,8 @@ void Realm::begin_transaction() LangBindHelper::promote_to_write(*m_shared_group, *m_history); m_in_transaction = true; - if (announce) { - send_local_notifications(DidChangeNotification); + if (announce && m_delegate) { + m_delegate->did_change(); } } @@ -240,8 +240,10 @@ void Realm::commit_transaction() LangBindHelper::commit_and_continue_as_read(*m_shared_group); m_in_transaction = false; - send_external_notifications(); - send_local_notifications(DidChangeNotification); + if (m_delegate) { + m_delegate->transaction_committed(); + m_delegate->did_change(); + } } void Realm::cancel_transaction() @@ -303,24 +305,17 @@ void Realm::notify() if (m_group) { LangBindHelper::advance_read(*m_shared_group, *m_history); } - send_local_notifications(DidChangeNotification); + if (m_delegate) { + m_delegate->did_change(); + } } - else { - send_local_notifications(RefreshRequiredNotification); + else if (m_delegate) { + m_delegate->changes_available(); } } } -void Realm::send_local_notifications(const std::string &type) -{ - verify_thread(); - for (NotificationFunction const& notification : m_notifications) { - (*notification)(type); - } -} - - bool Realm::refresh() { verify_thread(); @@ -344,7 +339,9 @@ bool Realm::refresh() read_group(); } - send_local_notifications(DidChangeNotification); + if (m_delegate) { + m_delegate->did_change(); + } return true; } diff --git a/shared_realm.hpp b/shared_realm.hpp index 064bfff7..0f0598a4 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -21,18 +21,18 @@ #include #include -#include #include #include #include "object_store.hpp" namespace realm { - class RealmCache; + class ClientHistory; class Realm; + class RealmCache; + class RealmDelegate; typedef std::shared_ptr SharedRealm; typedef std::weak_ptr WeakRealm; - class ClientHistory; class Realm : public std::enable_shared_from_this { @@ -90,20 +90,12 @@ namespace realm { bool auto_refresh() { return m_auto_refresh; } void notify(); - typedef std::shared_ptr> NotificationFunction; - void add_notification(NotificationFunction ¬ification) { m_notifications.insert(notification); } - void remove_notification(NotificationFunction notification) { m_notifications.erase(notification); } - void remove_all_notifications() { m_notifications.clear(); } - void invalidate(); bool compact(); std::thread::id thread_id() const { return m_thread_id; } void verify_thread(); - const std::string RefreshRequiredNotification = "RefreshRequiredNotification"; - const std::string DidChangeNotification = "DidChangeNotification"; - private: Realm(Config config); @@ -112,12 +104,6 @@ namespace realm { bool m_in_transaction = false; bool m_auto_refresh = true; - std::set m_notifications; - void send_local_notifications(const std::string ¬ification); - - typedef std::unique_ptr> ExternalNotificationFunction; - void send_external_notifications() { if (m_external_notifier) (*m_external_notifier)(); } - std::unique_ptr m_history; std::unique_ptr m_shared_group; std::unique_ptr m_read_only_group; @@ -125,7 +111,7 @@ namespace realm { Group *m_group = nullptr; public: - ExternalNotificationFunction m_external_notifier; + std::unique_ptr m_delegate; // FIXME private Group *read_group(); @@ -146,6 +132,23 @@ namespace realm { std::mutex m_mutex; }; + class RealmDelegate + { + public: + virtual ~RealmDelegate() = default; + + // The Realm has committed a write transaction, and other Realms at the + // same path should be notified + virtual void transaction_committed() = 0; + + // There are now new versions available for the Realm, but it has not + // had its read version advanced + virtual void changes_available() = 0; + + // The Realm's read version has advanced + virtual void did_change() = 0; + }; + class RealmFileException : public std::runtime_error { public: From 65e1eb5d0817f6ae41055ac046bed8ee99bb6847 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 1 Sep 2015 10:59:28 -0700 Subject: [PATCH 13/35] Add the ability to bypass the Realm cache entirely --- shared_realm.cpp | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/shared_realm.cpp b/shared_realm.cpp index ba71cada..5c2733ed 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -90,31 +90,31 @@ Group *Realm::read_group() SharedRealm Realm::get_shared_realm(Config config) { - SharedRealm realm = s_global_cache.get_realm(config.path); - if (realm) { - if (realm->config().read_only != config.read_only) { - throw MismatchedConfigException("Realm at path already opened with different read permissions."); - } - if (realm->config().in_memory != config.in_memory) { - throw MismatchedConfigException("Realm at path already opened with different inMemory settings."); - } - if (realm->config().encryption_key != config.encryption_key) { - throw MismatchedConfigException("Realm at path already opened with a different encryption key."); - } - if (realm->config().schema_version != config.schema_version && config.schema_version != ObjectStore::NotVersioned) { - throw MismatchedConfigException("Realm at path already opened with different schema version."); - } - // FIXME - enable schma comparison - /*if (realm->config().schema != config.schema) { - throw MismatchedConfigException("Realm at path already opened with different schema"); - }*/ + if (config.cache) { + if (SharedRealm realm = s_global_cache.get_realm(config.path)) { + if (realm->config().read_only != config.read_only) { + throw MismatchedConfigException("Realm at path already opened with different read permissions."); + } + if (realm->config().in_memory != config.in_memory) { + throw MismatchedConfigException("Realm at path already opened with different inMemory settings."); + } + if (realm->config().encryption_key != config.encryption_key) { + throw MismatchedConfigException("Realm at path already opened with a different encryption key."); + } + if (realm->config().schema_version != config.schema_version && config.schema_version != ObjectStore::NotVersioned) { + throw MismatchedConfigException("Realm at path already opened with different schema version."); + } + // FIXME - enable schma comparison + /*if (realm->config().schema != config.schema) { + throw MismatchedConfigException("Realm at path already opened with different schema"); + }*/ + realm->m_config.migration_function = config.migration_function; - realm->m_config.migration_function = config.migration_function; - - return realm; + return realm; + } } - realm = SharedRealm(new Realm(std::move(config))); + SharedRealm realm(new Realm(std::move(config))); // we want to ensure we are only initializing a single realm at a time static std::mutex s_init_mutex; From efdfa08524a4351def6534d869f75aa33a4aad63 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 31 Aug 2015 11:20:20 -0700 Subject: [PATCH 14/35] Port some of the KVO support functionality to the object store --- index_set.cpp | 117 +++++++++++++++++ index_set.hpp | 63 +++++++++ realm_delegate.hpp | 78 +++++++++++ shared_realm.cpp | 313 ++++++++++++++++++++++++++++++++++++++++++--- shared_realm.hpp | 17 --- 5 files changed, 554 insertions(+), 34 deletions(-) create mode 100644 index_set.cpp create mode 100644 index_set.hpp create mode 100644 realm_delegate.hpp diff --git a/index_set.cpp b/index_set.cpp new file mode 100644 index 00000000..d3f70047 --- /dev/null +++ b/index_set.cpp @@ -0,0 +1,117 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "index_set.hpp" + +using namespace realm; + +size_t IndexSet::size() const +{ + size_t size = 0; + for (auto const& range : m_ranges) { + size += range.second - range.first; + } + return size; +} + +std::vector::iterator IndexSet::find(size_t index) +{ + for (auto it = m_ranges.begin(), end = m_ranges.end(); it != end; ++it) { + if (it->second > index) + return it; + } + return m_ranges.end(); +} + +void IndexSet::add(size_t index) +{ + do_add(find(index), index); +} + +void IndexSet::do_add(std::vector::iterator it, size_t index) +{ + bool more_before = it != m_ranges.begin(), valid = it != m_ranges.end(); + if (valid && it->first <= index && it->second > index) { + // index is already in set + } + else if (more_before && (it - 1)->second == index) { + // index is immediate after an existing range + ++(it - 1)->second; + } + else if (more_before && valid && (it - 1)->second == it->first) { + // index joins two existing ranges + (it - 1)->second = it->second; + m_ranges.erase(it); + } + else if (valid && it->first == index + 1) { + // index is immediately before an existing range + --it->first; + } + else { + // index is not next to an existing range + m_ranges.insert(it, {index, index + 1}); + } +} + +void IndexSet::set(size_t len) +{ + m_ranges.clear(); + if (len) { + m_ranges.push_back({0, len}); + } +} + +void IndexSet::insert_at(size_t index) +{ + auto pos = find(index); + if (pos != m_ranges.end()) { + if (pos->first >= index) + ++pos->first; + ++pos->second; + for (auto it = pos + 1; it != m_ranges.end(); ++it) { + ++it->first; + ++it->second; + } + } + do_add(pos, index); +} + +size_t IndexSet::iterator::operator*() const +{ + return m_data->first + m_offset; +} + +IndexSet::iterator& IndexSet::iterator::operator++() +{ + ++m_offset; + if (m_offset + m_data->first == m_data->second) { + ++m_data; + m_offset = 0; + } + return *this; +} + +bool IndexSet::iterator::operator==(iterator other) const +{ + return m_data == other.m_data && m_offset == other.m_offset; +} + +bool IndexSet::iterator::operator!=(iterator other) const +{ + return m_data != other.m_data || m_offset != other.m_offset; +} diff --git a/index_set.hpp b/index_set.hpp new file mode 100644 index 00000000..4e55c40f --- /dev/null +++ b/index_set.hpp @@ -0,0 +1,63 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_INDEX_SET_HPP +#define REALM_INDEX_SET_HPP + +#include + +namespace realm { +class IndexSet { +public: + struct iterator { + size_t operator*() const; + iterator& operator++(); + bool operator==(iterator) const; + bool operator!=(iterator) const; + + iterator(std::pair* data) noexcept : m_data(data) { } + + private: + std::pair* m_data; + size_t m_offset = 0; + }; + + iterator begin() { return iterator(&m_ranges[0]); } + iterator end() { return iterator(&m_ranges[m_ranges.size()]); } + + size_t size() const; + + // Add an index to the set, doing nothing if it's already present + void add(size_t index); + // Set the index set to a single range starting at 0 with length `len` + void set(size_t len); + // Insert an index at the given position, shifting existing indexes back + void insert_at(size_t index); + +private: + using Range = std::pair; + std::vector m_ranges; + + // Find the range which contains the index, or the first one after it if + // none do + std::vector::iterator find(size_t index); + void do_add(std::vector::iterator pos, size_t index); +}; +} // namespace realm + +#endif // REALM_INDEX_SET_HPP diff --git a/realm_delegate.hpp b/realm_delegate.hpp new file mode 100644 index 00000000..2b4f5ba6 --- /dev/null +++ b/realm_delegate.hpp @@ -0,0 +1,78 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_DELEGATE_HPP +#define REALM_DELEGATE_HPP + +#include "index_set.hpp" + +#include +#include + +namespace realm { +class RealmDelegate { +public: + virtual ~RealmDelegate() = default; + + struct ColumnInfo { + bool changed = false; + enum class Kind { + None, + Set, + Insert, + Remove, + SetAll + } kind = Kind::None; + IndexSet indices; + }; + + struct ObserverState { + size_t table_ndx; + size_t row_ndx; + void* info; // opaque user info + std::vector changes; + + // Simple lexographic ordering + friend bool operator<(ObserverState const& lft, ObserverState const& rgt) { + return std::tie(lft.table_ndx, lft.row_ndx) < std::tie(rgt.table_ndx, rgt.row_ndx); + } + }; + + // The Realm has committed a write transaction, and other Realms at the + // same path should be notified + virtual void transaction_committed() = 0; + + // There are now new versions available for the Realm, but it has not + // had its read version advanced + virtual void changes_available() = 0; + + // Called before changing the read transaction. Should return a list of + // ObserverStates for each row for which detailed change information is + // desired. + virtual std::vector get_observed_rows() = 0; + + // The Realm's read version will change + // Only called if get_observed_row() returned a non-empty array. + virtual void will_change(std::vector const&, std::vector const&) = 0; + + // The Realm's read version has changed + virtual void did_change(std::vector const&, std::vector const&) = 0; +}; +} // namespace realm + +#endif /* REALM_DELEGATE_HPP */ diff --git a/shared_realm.cpp b/shared_realm.cpp index 5c2733ed..e2753d85 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -18,14 +18,296 @@ #include "shared_realm.hpp" +#include "realm_delegate.hpp" + #include #include #include #include +#include using namespace realm; +namespace { +class TransactLogHandler { + using ColumnInfo = RealmDelegate::ColumnInfo; + using ObserverState = RealmDelegate::ObserverState; + + size_t currentTable = 0; + std::vector observers; + std::vector invalidated; + ColumnInfo *activeLinkList = nullptr; + RealmDelegate* m_delegate; + + // Get the change info for the given column, creating it if needed + static ColumnInfo& get_change(ObserverState& state, size_t i) + { + if (state.changes.size() <= i) { + state.changes.resize(std::max(state.changes.size() * 2, i + 1)); + } + return state.changes[i]; + } + + // Loop over the columns which were changed in an observer state + template + static void for_each(ObserverState& state, Func&& f) + { + for (size_t i = 0; i < state.changes.size(); ++i) { + auto const& change = state.changes[i]; + if (change.changed) { + f(i, change); + } + } + } + + // Mark the given row/col as needing notifications sent + bool mark_dirty(size_t row_ndx, size_t col_ndx) + { + auto it = lower_bound(begin(observers), end(observers), ObserverState{currentTable, row_ndx, nullptr}); + if (it != end(observers) && it->table_ndx == currentTable && it->row_ndx == row_ndx) { + get_change(*it, col_ndx).changed = true; + } + return true; + } + + // Remove the given observer from the list of observed objects and add it + // to the listed of invalidated objects + void invalidate(ObserverState *o) + { + invalidated.push_back(o->info); + observers.erase(observers.begin() + (o - &observers[0])); + } + +public: + template + TransactLogHandler(RealmDelegate* delegate, SharedGroup& sg, Func&& func) + : m_delegate(delegate) + { + if (!delegate) { + func(); + return; + } + + observers = delegate->get_observed_rows(); + if (observers.empty()) { + auto old_version = sg.get_version_of_current_transaction(); + func(); + if (old_version != sg.get_version_of_current_transaction()) { + delegate->did_change({}, {}); + } + return; + } + + func(*this); + delegate->did_change(observers, invalidated); + } + + // Called at the end of the transaction log immediately before the version + // is advanced + void parse_complete() + { + m_delegate->will_change(observers, invalidated); + } + + // These would require having an observer before schema init + // Maybe do something here to throw an error when multiple processes have different schemas? + bool insert_group_level_table(size_t, size_t, StringData) noexcept { return false; } + bool erase_group_level_table(size_t, size_t) noexcept { return false; } + bool rename_group_level_table(size_t, StringData) noexcept { return false; } + bool insert_column(size_t, DataType, StringData, bool) { return false; } + bool insert_link_column(size_t, DataType, StringData, size_t, size_t) { return false; } + bool erase_column(size_t) { return false; } + bool erase_link_column(size_t, size_t, size_t) { return false; } + bool rename_column(size_t, StringData) { return false; } + bool add_search_index(size_t) { return false; } + bool remove_search_index(size_t) { return false; } + bool add_primary_key(size_t) { return false; } + bool remove_primary_key() { return false; } + bool set_link_type(size_t, LinkType) { return false; } + + bool select_table(size_t group_level_ndx, int, const size_t*) noexcept { + currentTable = group_level_ndx; + return true; + } + + bool insert_empty_rows(size_t, size_t, size_t, bool) { + // rows are only inserted at the end, so no need to do anything + return true; + } + + bool erase_rows(size_t row_ndx, size_t, size_t last_row_ndx, bool unordered) noexcept { + for (size_t i = 0; i < observers.size(); ++i) { + auto& o = observers[i]; + if (o.table_ndx == currentTable) { + if (o.row_ndx == row_ndx) { + invalidate(&o); + --i; + } + else if (unordered && o.row_ndx == last_row_ndx) { + o.row_ndx = row_ndx; + } + else if (!unordered && o.row_ndx > row_ndx) { + o.row_ndx -= 1; + } + } + } + return true; + } + + bool clear_table() noexcept { + for (size_t i = 0; i < observers.size(); ) { + auto& o = observers[i]; + if (o.table_ndx == currentTable) { + invalidate(&o); + } + else { + ++i; + } + } + return true; + } + + bool select_link_list(size_t col, size_t row) { + activeLinkList = nullptr; + for (auto& o : observers) { + if (o.table_ndx == currentTable && o.row_ndx == row) { + activeLinkList = &get_change(o, col); + break; + } + } + return true; + } + + void append_link_list_change(ColumnInfo::Kind kind, size_t index) { + ColumnInfo *o = activeLinkList; + if (!o || o->kind == ColumnInfo::Kind::SetAll) { + // Active LinkList isn't observed or already has multiple kinds of changes + return; + } + + if (o->kind == ColumnInfo::Kind::None) { + o->kind = kind; + o->changed = true; + o->indices.add(index); + } + else if (o->kind == kind) { + if (kind == ColumnInfo::Kind::Remove) { + // Shift the index to compensate for already-removed indices + for (auto i : o->indices) { + if (i <= index) + ++index; + else + break; + } + o->indices.add(index); + } + else if (kind == ColumnInfo::Kind::Insert) { + o->indices.insert_at(index); + } + else { + o->indices.add(index); + } + } + else { + // Array KVO can only send a single kind of change at a time, so + // if there's multiple just give up and send "Set" + o->indices.set(0); + o->kind = ColumnInfo::Kind::SetAll; + } + } + + bool link_list_set(size_t index, size_t) { + append_link_list_change(ColumnInfo::Kind::Set, index); + return true; + } + + bool link_list_insert(size_t index, size_t) { + append_link_list_change(ColumnInfo::Kind::Insert, index); + return true; + } + + bool link_list_erase(size_t index) { + append_link_list_change(ColumnInfo::Kind::Remove, index); + return true; + } + + bool link_list_nullify(size_t index) { + append_link_list_change(ColumnInfo::Kind::Remove, index); + return true; + } + + bool link_list_swap(size_t index1, size_t index2) { + append_link_list_change(ColumnInfo::Kind::Set, index1); + append_link_list_change(ColumnInfo::Kind::Set, index2); + return true; + } + + bool link_list_clear(size_t old_size) { + ColumnInfo *o = activeLinkList; + if (!o || o->kind == ColumnInfo::Kind::SetAll) { + return true; + } + + if (o->kind == ColumnInfo::Kind::Remove) + old_size += o->indices.size(); + else if (o->kind == ColumnInfo::Kind::Insert) + old_size -= o->indices.size(); + + o->indices.set(old_size); + + o->kind = ColumnInfo::Kind::Remove; + o->changed = true; + return true; + } + + bool link_list_move(size_t from, size_t to) { + ColumnInfo *o = activeLinkList; + if (!o || o->kind == ColumnInfo::Kind::SetAll) { + return true; + } + if (from > to) { + std::swap(from, to); + } + + if (o->kind == ColumnInfo::Kind::None) { + o->kind = ColumnInfo::Kind::Set; + o->changed = true; + } + if (o->kind == ColumnInfo::Kind::Set) { + for (size_t i = from; i <= to; ++i) + o->indices.add(i); + } + else { + o->indices.set(0); + o->kind = ColumnInfo::Kind::SetAll; + } + return true; + } + + // Things that just mark the field as modified + bool set_int(size_t col, size_t row, int_fast64_t) { return mark_dirty(row, col); } + bool set_bool(size_t col, size_t row, bool) { return mark_dirty(row, col); } + bool set_float(size_t col, size_t row, float) { return mark_dirty(row, col); } + bool set_double(size_t col, size_t row, double) { return mark_dirty(row, col); } + bool set_string(size_t col, size_t row, StringData) { return mark_dirty(row, col); } + bool set_binary(size_t col, size_t row, BinaryData) { return mark_dirty(row, col); } + bool set_date_time(size_t col, size_t row, DateTime) { return mark_dirty(row, col); } + bool set_table(size_t col, size_t row) { return mark_dirty(row, col); } + bool set_mixed(size_t col, size_t row, const Mixed&) { return mark_dirty(row, col); } + bool set_link(size_t col, size_t row, size_t) { return mark_dirty(row, col); } + bool set_null(size_t col, size_t row) { return mark_dirty(row, col); } + bool nullify_link(size_t col, size_t row) { return mark_dirty(row, col); } + + // Things we don't need to do anything for + bool optimize_table() { return false; } + + // Things that we don't do in the binding + bool select_descriptor(int, const size_t*) { return true; } + bool add_int_to_column(size_t, int_fast64_t) { return false; } +}; +} + RealmCache Realm::s_global_cache; Realm::Config::Config(const Config& c) @@ -214,18 +496,13 @@ void Realm::begin_transaction() throw InvalidTransactionException("The Realm is already in a write transaction"); } - // if the upgrade to write will move the transaction forward, announce the change after promoting - bool announce = m_shared_group->has_changed(); - // make sure we have a read transaction read_group(); - LangBindHelper::promote_to_write(*m_shared_group, *m_history); + TransactLogHandler(m_delegate.get(), *m_shared_group, [&](auto&&... args) { + LangBindHelper::promote_to_write(*m_shared_group, *m_history, std::move(args)...); + }); m_in_transaction = true; - - if (announce && m_delegate) { - m_delegate->did_change(); - } } void Realm::commit_transaction() @@ -242,7 +519,6 @@ void Realm::commit_transaction() if (m_delegate) { m_delegate->transaction_committed(); - m_delegate->did_change(); } } @@ -255,7 +531,9 @@ void Realm::cancel_transaction() throw InvalidTransactionException("Can't cancel a non-existing write transaction"); } - LangBindHelper::rollback_and_continue_as_read(*m_shared_group, *m_history); + TransactLogHandler(m_delegate.get(), *m_shared_group, [&](auto&&... args) { + LangBindHelper::rollback_and_continue_as_read(*m_shared_group, *m_history, std::move(args)...); + }); m_in_transaction = false; } @@ -303,10 +581,12 @@ void Realm::notify() if (m_shared_group->has_changed()) { // Throws if (m_auto_refresh) { if (m_group) { - LangBindHelper::advance_read(*m_shared_group, *m_history); + TransactLogHandler(m_delegate.get(), *m_shared_group, [&](auto&&... args) { + LangBindHelper::advance_read(*m_shared_group, *m_history, std::move(args)...); + }); } - if (m_delegate) { - m_delegate->did_change(); + else if (m_delegate) { + m_delegate->did_change({}, {}); } } else if (m_delegate) { @@ -332,16 +612,15 @@ bool Realm::refresh() } if (m_group) { - LangBindHelper::advance_read(*m_shared_group, *m_history); + TransactLogHandler(m_delegate.get(), *m_shared_group, [&](auto&&... args) { + LangBindHelper::advance_read(*m_shared_group, *m_history, std::move(args)...); + }); } else { // Create the read transaction read_group(); } - if (m_delegate) { - m_delegate->did_change(); - } return true; } diff --git a/shared_realm.hpp b/shared_realm.hpp index 0f0598a4..cdbf4d51 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -132,23 +132,6 @@ namespace realm { std::mutex m_mutex; }; - class RealmDelegate - { - public: - virtual ~RealmDelegate() = default; - - // The Realm has committed a write transaction, and other Realms at the - // same path should be notified - virtual void transaction_committed() = 0; - - // There are now new versions available for the Realm, but it has not - // had its read version advanced - virtual void changes_available() = 0; - - // The Realm's read version has advanced - virtual void did_change() = 0; - }; - class RealmFileException : public std::runtime_error { public: From 0a41c85d0a42e5b8c7cb43ec3fad3fa640ca3049 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 1 Sep 2015 16:21:59 -0700 Subject: [PATCH 15/35] Improve performance of realm_requires_update() and make more things const --- object_schema.cpp | 9 ++++++--- object_schema.hpp | 5 ++++- object_store.cpp | 37 +++++++++++++++---------------------- object_store.hpp | 5 +---- property.hpp | 2 +- shared_realm.cpp | 2 +- shared_realm.hpp | 2 +- 7 files changed, 29 insertions(+), 33 deletions(-) diff --git a/object_schema.cpp b/object_schema.cpp index 41e34685..cfb65f5d 100644 --- a/object_schema.cpp +++ b/object_schema.cpp @@ -60,12 +60,15 @@ ObjectSchema::ObjectSchema(Group *group, const std::string &name) : name(name) { } } -Property *ObjectSchema::property_for_name(const std::string &name) { - for (auto& prop:properties) { - if (prop.name == name) { +Property *ObjectSchema::property_for_name(StringData name) { + for (auto& prop : properties) { + if (StringData(prop.name) == name) { return ∝ } } return nullptr; } +const Property *ObjectSchema::property_for_name(StringData name) const { + return const_cast(this)->property_for_name(name); +} diff --git a/object_schema.hpp b/object_schema.hpp index 51a4c584..5b7bf343 100644 --- a/object_schema.hpp +++ b/object_schema.hpp @@ -19,6 +19,8 @@ #ifndef REALM_OBJECT_SCHEMA_HPP #define REALM_OBJECT_SCHEMA_HPP +#include + #include #include @@ -39,7 +41,8 @@ namespace realm { std::vector properties; std::string primary_key; - Property *property_for_name(const std::string &name); + Property *property_for_name(StringData name); + const Property *property_for_name(StringData name) const; Property *primary_key_property() { return property_for_name(primary_key); } diff --git a/object_store.cpp b/object_store.cpp index 8dee8d74..c751f75f 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -336,19 +336,29 @@ bool ObjectStore::is_schema_at_version(Group *group, uint64_t version) { return old_version == version; } -bool ObjectStore::realm_requires_update(Group *group, uint64_t version, Schema &schema) { +bool ObjectStore::realm_requires_update(Group *group, uint64_t version, Schema const& schema) { if (!is_schema_at_version(group, version)) { return true; } - for (auto& target_schema : schema) { + + for (auto const& target_schema : schema) { TableRef table = table_for_object_type(group, target_schema.name); if (!table) { return true; } + + // Check that all of the property indexes are up to date + size_t count = table->get_column_count(); + for (size_t col = 0; col < count; ++col) { + StringData name = table->get_column_name(col); + bool is_indexed = table->has_search_index(col); + auto prop = target_schema.property_for_name(name); + if (prop && prop->requires_index() != is_indexed) { + return true; + } + } } - if (!indexes_are_up_to_date(group, schema)) { - return true; - } + return false; } @@ -395,23 +405,6 @@ Schema ObjectStore::schema_from_group(Group *group) { return schema; } -bool ObjectStore::indexes_are_up_to_date(Group *group, Schema &schema) { - for (auto &object_schema : schema) { - TableRef table = table_for_object_type(group, object_schema.name); - if (!table) { - continue; - } - - update_column_mapping(group, object_schema); - for (auto& property : object_schema.properties) { - if (property.requires_index() != table->has_search_index(property.table_column)) { - return false; - } - } - } - return true; -} - bool ObjectStore::update_indexes(Group *group, Schema &schema) { bool changed = false; for (auto& object_schema : schema) { diff --git a/object_store.hpp b/object_store.hpp index 42b8749e..5361ec37 100644 --- a/object_store.hpp +++ b/object_store.hpp @@ -53,7 +53,7 @@ namespace realm { // determines if you must call update_realm_with_schema for a given realm. // returns true if there is a schema version mismatch, if there tables which still need to be created, // or if file format or other changes/updates need to be made - static bool realm_requires_update(Group *group, uint64_t version, Schema &schema); + static bool realm_requires_update(Group *group, uint64_t version, Schema const& schema); // updates a Realm to a given target schema/version creating tables and updating indexes as necessary // returns if any changes were made @@ -104,9 +104,6 @@ namespace realm { static std::string table_name_for_object_type(const std::string &class_name); static std::string object_type_for_table_name(const std::string &table_name); - // check if indexes are up to date - if false you need to call update_realm_with_schema - static bool indexes_are_up_to_date(Group *group, Schema &schema); - // returns if any indexes were changed static bool update_indexes(Group *group, Schema &schema); diff --git a/property.hpp b/property.hpp index 893f772d..3883e805 100644 --- a/property.hpp +++ b/property.hpp @@ -54,7 +54,7 @@ namespace realm { bool is_nullable = false; size_t table_column; - bool requires_index() { return is_primary || is_indexed; } + bool requires_index() const { return is_primary || is_indexed; } }; static inline const char *string_for_property_type(PropertyType type) { diff --git a/shared_realm.cpp b/shared_realm.cpp index e2753d85..6acd7593 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -430,7 +430,7 @@ SharedRealm Realm::get_shared_realm(Config config) return realm; } -bool Realm::update_schema(Schema &schema, uint64_t version) +bool Realm::update_schema(Schema const& schema, uint64_t version) { bool changed = false; Config old_config(m_config); diff --git a/shared_realm.hpp b/shared_realm.hpp index cdbf4d51..3ef4bdf1 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -74,7 +74,7 @@ namespace realm { // on the Config, and the resulting Schema and version with updated // column mappings are set on the realms config upon success. // returns if any changes were made - bool update_schema(Schema &schema, uint64_t version); + bool update_schema(Schema const& schema, uint64_t version); static uint64_t get_schema_version(Config const& config); From c3649fbd17b7eec60cd844fe5cefab5c92009410 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 2 Sep 2015 09:31:06 -0700 Subject: [PATCH 16/35] Skip PK uniqueness checking when first creating a Realm file --- object_store.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/object_store.cpp b/object_store.cpp index c751f75f..d8fb943d 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -386,9 +386,9 @@ bool ObjectStore::update_realm_with_schema(Group *group, // apply the migration block if provided and there's any old data if (get_schema_version(group) != ObjectStore::NotVersioned) { migration(group, schema); - } - validate_primary_column_uniqueness(group, schema); + validate_primary_column_uniqueness(group, schema); + } set_schema_version(group, version); return true; From eeb2ddd79422258ce3cedee54ef1d855bb9a80d0 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 2 Sep 2015 11:04:01 -0700 Subject: [PATCH 17/35] Improve array KVO performance a bit --- index_set.cpp | 9 +++++++++ index_set.hpp | 1 + shared_realm.cpp | 9 +-------- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/index_set.cpp b/index_set.cpp index d3f70047..3fe27a46 100644 --- a/index_set.cpp +++ b/index_set.cpp @@ -91,6 +91,15 @@ void IndexSet::insert_at(size_t index) do_add(pos, index); } +void IndexSet::add_shifted(size_t index) +{ + auto it = m_ranges.begin(); + for (auto end = m_ranges.end(); it != end && it->first <= index; ++it) { + index += it->second - it->first; + } + do_add(it, index); +} + size_t IndexSet::iterator::operator*() const { return m_data->first + m_offset; diff --git a/index_set.hpp b/index_set.hpp index 4e55c40f..03d036fe 100644 --- a/index_set.hpp +++ b/index_set.hpp @@ -48,6 +48,7 @@ public: void set(size_t len); // Insert an index at the given position, shifting existing indexes back void insert_at(size_t index); + void add_shifted(size_t index); private: using Range = std::pair; diff --git a/shared_realm.cpp b/shared_realm.cpp index 6acd7593..8f3e416b 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -193,14 +193,7 @@ public: } else if (o->kind == kind) { if (kind == ColumnInfo::Kind::Remove) { - // Shift the index to compensate for already-removed indices - for (auto i : o->indices) { - if (i <= index) - ++index; - else - break; - } - o->indices.add(index); + o->indices.add_shifted(index); } else if (kind == ColumnInfo::Kind::Insert) { o->indices.insert_at(index); From b129ebe8c134871cc2ef05cc4af549f02fce1839 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 2 Sep 2015 13:51:28 -0700 Subject: [PATCH 18/35] Shuffle stuff around and clean some things up --- index_set.cpp | 40 +---- index_set.hpp | 42 +++-- realm_delegate.hpp | 22 ++- shared_realm.cpp | 298 +-------------------------------- transact_log_handler.cpp | 347 +++++++++++++++++++++++++++++++++++++++ transact_log_handler.hpp | 46 ++++++ 6 files changed, 441 insertions(+), 354 deletions(-) create mode 100644 transact_log_handler.cpp create mode 100644 transact_log_handler.hpp diff --git a/index_set.cpp b/index_set.cpp index 3fe27a46..b7e4961c 100644 --- a/index_set.cpp +++ b/index_set.cpp @@ -20,16 +20,7 @@ using namespace realm; -size_t IndexSet::size() const -{ - size_t size = 0; - for (auto const& range : m_ranges) { - size += range.second - range.first; - } - return size; -} - -std::vector::iterator IndexSet::find(size_t index) +IndexSet::iterator IndexSet::find(size_t index) { for (auto it = m_ranges.begin(), end = m_ranges.end(); it != end; ++it) { if (it->second > index) @@ -43,14 +34,14 @@ void IndexSet::add(size_t index) do_add(find(index), index); } -void IndexSet::do_add(std::vector::iterator it, size_t index) +void IndexSet::do_add(iterator it, size_t index) { bool more_before = it != m_ranges.begin(), valid = it != m_ranges.end(); if (valid && it->first <= index && it->second > index) { // index is already in set } else if (more_before && (it - 1)->second == index) { - // index is immediate after an existing range + // index is immediately after an existing range ++(it - 1)->second; } else if (more_before && valid && (it - 1)->second == it->first) { @@ -99,28 +90,3 @@ void IndexSet::add_shifted(size_t index) } do_add(it, index); } - -size_t IndexSet::iterator::operator*() const -{ - return m_data->first + m_offset; -} - -IndexSet::iterator& IndexSet::iterator::operator++() -{ - ++m_offset; - if (m_offset + m_data->first == m_data->second) { - ++m_data; - m_offset = 0; - } - return *this; -} - -bool IndexSet::iterator::operator==(iterator other) const -{ - return m_data == other.m_data && m_offset == other.m_offset; -} - -bool IndexSet::iterator::operator!=(iterator other) const -{ - return m_data != other.m_data || m_offset != other.m_offset; -} diff --git a/index_set.hpp b/index_set.hpp index 03d036fe..7a229177 100644 --- a/index_set.hpp +++ b/index_set.hpp @@ -24,40 +24,38 @@ namespace realm { class IndexSet { public: - struct iterator { - size_t operator*() const; - iterator& operator++(); - bool operator==(iterator) const; - bool operator!=(iterator) const; + using value_type = std::pair; + using iterator = std::vector::iterator; + using const_iterator = std::vector::const_iterator; - iterator(std::pair* data) noexcept : m_data(data) { } - - private: - std::pair* m_data; - size_t m_offset = 0; - }; - - iterator begin() { return iterator(&m_ranges[0]); } - iterator end() { return iterator(&m_ranges[m_ranges.size()]); } - - size_t size() const; + const_iterator begin() const { return m_ranges.begin(); } + const_iterator end() const { return m_ranges.end(); } + bool empty() const { return m_ranges.empty(); } + size_t size() const { return m_ranges.size(); } // Add an index to the set, doing nothing if it's already present void add(size_t index); - // Set the index set to a single range starting at 0 with length `len` + + // Remove all indexes from the set and then add a single range starting from + // zero with the given length void set(size_t len); - // Insert an index at the given position, shifting existing indexes back + + // Insert an index at the given position, shifting existing indexes at or + // after that point back by one void insert_at(size_t index); + + // Add an index which has had all of the ranges in the set before it removed void add_shifted(size_t index); private: - using Range = std::pair; - std::vector m_ranges; + std::vector m_ranges; // Find the range which contains the index, or the first one after it if // none do - std::vector::iterator find(size_t index); - void do_add(std::vector::iterator pos, size_t index); + iterator find(size_t index); + // Insert the index before the given position, combining existing ranges as + // applicable + void do_add(iterator pos, size_t index); }; } // namespace realm diff --git a/realm_delegate.hpp b/realm_delegate.hpp index 2b4f5ba6..c3a41d04 100644 --- a/realm_delegate.hpp +++ b/realm_delegate.hpp @@ -29,6 +29,7 @@ class RealmDelegate { public: virtual ~RealmDelegate() = default; + // Change information for a single field of a row struct ColumnInfo { bool changed = false; enum class Kind { @@ -41,10 +42,19 @@ public: IndexSet indices; }; + // Information about an observed row in a table struct ObserverState { + // Initial table and row which is observed + // May be updated by row insertions and removals size_t table_ndx; size_t row_ndx; - void* info; // opaque user info + + // Opaque userdata for the delegate's use + void* info; + + // Populated with information about which columns were changed + // May be shorter than the actual number of columns if the later columns + // are not modified std::vector changes; // Simple lexographic ordering @@ -68,10 +78,16 @@ public: // The Realm's read version will change // Only called if get_observed_row() returned a non-empty array. - virtual void will_change(std::vector const&, std::vector const&) = 0; + // observers is the vector returned from get_observed_rows() + // invalidated is the `info` pointers for each observed object which was deleted + virtual void will_change(std::vector const& observers, + std::vector const& invalidated) = 0; // The Realm's read version has changed - virtual void did_change(std::vector const&, std::vector const&) = 0; + // observers is the vector returned from get_observed_rows() + // invalidated is the `info` pointers for each observed object which was deleted + virtual void did_change(std::vector const& observers, + std::vector const& invalidated) = 0; }; } // namespace realm diff --git a/shared_realm.cpp b/shared_realm.cpp index 8f3e416b..7d270cc2 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -19,288 +19,15 @@ #include "shared_realm.hpp" #include "realm_delegate.hpp" +#include "transact_log_handler.hpp" #include #include -#include #include -#include using namespace realm; -namespace { -class TransactLogHandler { - using ColumnInfo = RealmDelegate::ColumnInfo; - using ObserverState = RealmDelegate::ObserverState; - - size_t currentTable = 0; - std::vector observers; - std::vector invalidated; - ColumnInfo *activeLinkList = nullptr; - RealmDelegate* m_delegate; - - // Get the change info for the given column, creating it if needed - static ColumnInfo& get_change(ObserverState& state, size_t i) - { - if (state.changes.size() <= i) { - state.changes.resize(std::max(state.changes.size() * 2, i + 1)); - } - return state.changes[i]; - } - - // Loop over the columns which were changed in an observer state - template - static void for_each(ObserverState& state, Func&& f) - { - for (size_t i = 0; i < state.changes.size(); ++i) { - auto const& change = state.changes[i]; - if (change.changed) { - f(i, change); - } - } - } - - // Mark the given row/col as needing notifications sent - bool mark_dirty(size_t row_ndx, size_t col_ndx) - { - auto it = lower_bound(begin(observers), end(observers), ObserverState{currentTable, row_ndx, nullptr}); - if (it != end(observers) && it->table_ndx == currentTable && it->row_ndx == row_ndx) { - get_change(*it, col_ndx).changed = true; - } - return true; - } - - // Remove the given observer from the list of observed objects and add it - // to the listed of invalidated objects - void invalidate(ObserverState *o) - { - invalidated.push_back(o->info); - observers.erase(observers.begin() + (o - &observers[0])); - } - -public: - template - TransactLogHandler(RealmDelegate* delegate, SharedGroup& sg, Func&& func) - : m_delegate(delegate) - { - if (!delegate) { - func(); - return; - } - - observers = delegate->get_observed_rows(); - if (observers.empty()) { - auto old_version = sg.get_version_of_current_transaction(); - func(); - if (old_version != sg.get_version_of_current_transaction()) { - delegate->did_change({}, {}); - } - return; - } - - func(*this); - delegate->did_change(observers, invalidated); - } - - // Called at the end of the transaction log immediately before the version - // is advanced - void parse_complete() - { - m_delegate->will_change(observers, invalidated); - } - - // These would require having an observer before schema init - // Maybe do something here to throw an error when multiple processes have different schemas? - bool insert_group_level_table(size_t, size_t, StringData) noexcept { return false; } - bool erase_group_level_table(size_t, size_t) noexcept { return false; } - bool rename_group_level_table(size_t, StringData) noexcept { return false; } - bool insert_column(size_t, DataType, StringData, bool) { return false; } - bool insert_link_column(size_t, DataType, StringData, size_t, size_t) { return false; } - bool erase_column(size_t) { return false; } - bool erase_link_column(size_t, size_t, size_t) { return false; } - bool rename_column(size_t, StringData) { return false; } - bool add_search_index(size_t) { return false; } - bool remove_search_index(size_t) { return false; } - bool add_primary_key(size_t) { return false; } - bool remove_primary_key() { return false; } - bool set_link_type(size_t, LinkType) { return false; } - - bool select_table(size_t group_level_ndx, int, const size_t*) noexcept { - currentTable = group_level_ndx; - return true; - } - - bool insert_empty_rows(size_t, size_t, size_t, bool) { - // rows are only inserted at the end, so no need to do anything - return true; - } - - bool erase_rows(size_t row_ndx, size_t, size_t last_row_ndx, bool unordered) noexcept { - for (size_t i = 0; i < observers.size(); ++i) { - auto& o = observers[i]; - if (o.table_ndx == currentTable) { - if (o.row_ndx == row_ndx) { - invalidate(&o); - --i; - } - else if (unordered && o.row_ndx == last_row_ndx) { - o.row_ndx = row_ndx; - } - else if (!unordered && o.row_ndx > row_ndx) { - o.row_ndx -= 1; - } - } - } - return true; - } - - bool clear_table() noexcept { - for (size_t i = 0; i < observers.size(); ) { - auto& o = observers[i]; - if (o.table_ndx == currentTable) { - invalidate(&o); - } - else { - ++i; - } - } - return true; - } - - bool select_link_list(size_t col, size_t row) { - activeLinkList = nullptr; - for (auto& o : observers) { - if (o.table_ndx == currentTable && o.row_ndx == row) { - activeLinkList = &get_change(o, col); - break; - } - } - return true; - } - - void append_link_list_change(ColumnInfo::Kind kind, size_t index) { - ColumnInfo *o = activeLinkList; - if (!o || o->kind == ColumnInfo::Kind::SetAll) { - // Active LinkList isn't observed or already has multiple kinds of changes - return; - } - - if (o->kind == ColumnInfo::Kind::None) { - o->kind = kind; - o->changed = true; - o->indices.add(index); - } - else if (o->kind == kind) { - if (kind == ColumnInfo::Kind::Remove) { - o->indices.add_shifted(index); - } - else if (kind == ColumnInfo::Kind::Insert) { - o->indices.insert_at(index); - } - else { - o->indices.add(index); - } - } - else { - // Array KVO can only send a single kind of change at a time, so - // if there's multiple just give up and send "Set" - o->indices.set(0); - o->kind = ColumnInfo::Kind::SetAll; - } - } - - bool link_list_set(size_t index, size_t) { - append_link_list_change(ColumnInfo::Kind::Set, index); - return true; - } - - bool link_list_insert(size_t index, size_t) { - append_link_list_change(ColumnInfo::Kind::Insert, index); - return true; - } - - bool link_list_erase(size_t index) { - append_link_list_change(ColumnInfo::Kind::Remove, index); - return true; - } - - bool link_list_nullify(size_t index) { - append_link_list_change(ColumnInfo::Kind::Remove, index); - return true; - } - - bool link_list_swap(size_t index1, size_t index2) { - append_link_list_change(ColumnInfo::Kind::Set, index1); - append_link_list_change(ColumnInfo::Kind::Set, index2); - return true; - } - - bool link_list_clear(size_t old_size) { - ColumnInfo *o = activeLinkList; - if (!o || o->kind == ColumnInfo::Kind::SetAll) { - return true; - } - - if (o->kind == ColumnInfo::Kind::Remove) - old_size += o->indices.size(); - else if (o->kind == ColumnInfo::Kind::Insert) - old_size -= o->indices.size(); - - o->indices.set(old_size); - - o->kind = ColumnInfo::Kind::Remove; - o->changed = true; - return true; - } - - bool link_list_move(size_t from, size_t to) { - ColumnInfo *o = activeLinkList; - if (!o || o->kind == ColumnInfo::Kind::SetAll) { - return true; - } - if (from > to) { - std::swap(from, to); - } - - if (o->kind == ColumnInfo::Kind::None) { - o->kind = ColumnInfo::Kind::Set; - o->changed = true; - } - if (o->kind == ColumnInfo::Kind::Set) { - for (size_t i = from; i <= to; ++i) - o->indices.add(i); - } - else { - o->indices.set(0); - o->kind = ColumnInfo::Kind::SetAll; - } - return true; - } - - // Things that just mark the field as modified - bool set_int(size_t col, size_t row, int_fast64_t) { return mark_dirty(row, col); } - bool set_bool(size_t col, size_t row, bool) { return mark_dirty(row, col); } - bool set_float(size_t col, size_t row, float) { return mark_dirty(row, col); } - bool set_double(size_t col, size_t row, double) { return mark_dirty(row, col); } - bool set_string(size_t col, size_t row, StringData) { return mark_dirty(row, col); } - bool set_binary(size_t col, size_t row, BinaryData) { return mark_dirty(row, col); } - bool set_date_time(size_t col, size_t row, DateTime) { return mark_dirty(row, col); } - bool set_table(size_t col, size_t row) { return mark_dirty(row, col); } - bool set_mixed(size_t col, size_t row, const Mixed&) { return mark_dirty(row, col); } - bool set_link(size_t col, size_t row, size_t) { return mark_dirty(row, col); } - bool set_null(size_t col, size_t row) { return mark_dirty(row, col); } - bool nullify_link(size_t col, size_t row) { return mark_dirty(row, col); } - - // Things we don't need to do anything for - bool optimize_table() { return false; } - - // Things that we don't do in the binding - bool select_descriptor(int, const size_t*) { return true; } - bool add_int_to_column(size_t, int_fast64_t) { return false; } -}; -} - RealmCache Realm::s_global_cache; Realm::Config::Config(const Config& c) @@ -492,9 +219,7 @@ void Realm::begin_transaction() // make sure we have a read transaction read_group(); - TransactLogHandler(m_delegate.get(), *m_shared_group, [&](auto&&... args) { - LangBindHelper::promote_to_write(*m_shared_group, *m_history, std::move(args)...); - }); + transaction::begin(*m_shared_group, *m_history, m_delegate.get()); m_in_transaction = true; } @@ -507,12 +232,8 @@ void Realm::commit_transaction() throw InvalidTransactionException("Can't commit a non-existing write transaction"); } - LangBindHelper::commit_and_continue_as_read(*m_shared_group); m_in_transaction = false; - - if (m_delegate) { - m_delegate->transaction_committed(); - } + transaction::commit(*m_shared_group, *m_history, m_delegate.get()); } void Realm::cancel_transaction() @@ -524,13 +245,10 @@ void Realm::cancel_transaction() throw InvalidTransactionException("Can't cancel a non-existing write transaction"); } - TransactLogHandler(m_delegate.get(), *m_shared_group, [&](auto&&... args) { - LangBindHelper::rollback_and_continue_as_read(*m_shared_group, *m_history, std::move(args)...); - }); m_in_transaction = false; + transaction::cancel(*m_shared_group, *m_history, m_delegate.get()); } - void Realm::invalidate() { verify_thread(); @@ -574,9 +292,7 @@ void Realm::notify() if (m_shared_group->has_changed()) { // Throws if (m_auto_refresh) { if (m_group) { - TransactLogHandler(m_delegate.get(), *m_shared_group, [&](auto&&... args) { - LangBindHelper::advance_read(*m_shared_group, *m_history, std::move(args)...); - }); + transaction::advance(*m_shared_group, *m_history, m_delegate.get()); } else if (m_delegate) { m_delegate->did_change({}, {}); @@ -605,9 +321,7 @@ bool Realm::refresh() } if (m_group) { - TransactLogHandler(m_delegate.get(), *m_shared_group, [&](auto&&... args) { - LangBindHelper::advance_read(*m_shared_group, *m_history, std::move(args)...); - }); + transaction::advance(*m_shared_group, *m_history, m_delegate.get()); } else { // Create the read transaction diff --git a/transact_log_handler.cpp b/transact_log_handler.cpp new file mode 100644 index 00000000..b02fdcbd --- /dev/null +++ b/transact_log_handler.cpp @@ -0,0 +1,347 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "transact_log_handler.hpp" + +#include "realm_delegate.hpp" + +#include +#include +#include + +using namespace realm; + +namespace { +class TransactLogHandler { + using ColumnInfo = RealmDelegate::ColumnInfo; + using ObserverState = RealmDelegate::ObserverState; + + // Observed table rows which need change information + std::vector m_observers; + // Userdata pointers for rows which have been deleted + std::vector invalidated; + // Delegate to send change information to + RealmDelegate* m_delegate; + + // Index of currently selected table + size_t m_current_table = 0; + // Change information for the currently selected LinkList, if any + ColumnInfo* m_active_linklist = nullptr; + + // Get the change info for the given column, creating it if needed + static ColumnInfo& get_change(ObserverState& state, size_t i) + { + if (state.changes.size() <= i) { + state.changes.resize(std::max(state.changes.size() * 2, i + 1)); + } + return state.changes[i]; + } + + // Loop over the columns which were changed in an observer state + template + static void for_each(ObserverState& state, Func&& f) + { + for (size_t i = 0; i < state.changes.size(); ++i) { + auto const& change = state.changes[i]; + if (change.changed) { + f(i, change); + } + } + } + + // Mark the given row/col as needing notifications sent + bool mark_dirty(size_t row_ndx, size_t col_ndx) + { + auto it = lower_bound(begin(m_observers), end(m_observers), ObserverState{m_current_table, row_ndx, nullptr}); + if (it != end(m_observers) && it->table_ndx == m_current_table && it->row_ndx == row_ndx) { + get_change(*it, col_ndx).changed = true; + } + return true; + } + + // Remove the given observer from the list of observed objects and add it + // to the listed of invalidated objects + void invalidate(ObserverState *o) + { + invalidated.push_back(o->info); + m_observers.erase(m_observers.begin() + (o - &m_observers[0])); + } + +public: + template + TransactLogHandler(RealmDelegate* delegate, SharedGroup& sg, Func&& func) + : m_delegate(delegate) + { + if (!delegate) { + func(); + return; + } + + m_observers = delegate->get_observed_rows(); + if (m_observers.empty()) { + auto old_version = sg.get_version_of_current_transaction(); + func(); + if (old_version != sg.get_version_of_current_transaction()) { + delegate->did_change({}, {}); + } + return; + } + + func(*this); + delegate->did_change(m_observers, invalidated); + } + + // Called at the end of the transaction log immediately before the version + // is advanced + void parse_complete() + { + m_delegate->will_change(m_observers, invalidated); + } + + // These would require having an observer before schema init + // Maybe do something here to throw an error when multiple processes have different schemas? + bool insert_group_level_table(size_t, size_t, StringData) { return false; } + bool erase_group_level_table(size_t, size_t) { return false; } + bool rename_group_level_table(size_t, StringData) { return false; } + bool insert_column(size_t, DataType, StringData, bool) { return false; } + bool insert_link_column(size_t, DataType, StringData, size_t, size_t) { return false; } + bool erase_column(size_t) { return false; } + bool erase_link_column(size_t, size_t, size_t) { return false; } + bool rename_column(size_t, StringData) { return false; } + bool add_search_index(size_t) { return false; } + bool remove_search_index(size_t) { return false; } + bool add_primary_key(size_t) { return false; } + bool remove_primary_key() { return false; } + bool set_link_type(size_t, LinkType) { return false; } + + bool select_table(size_t group_level_ndx, int, const size_t*) noexcept + { + m_current_table = group_level_ndx; + return true; + } + + bool insert_empty_rows(size_t, size_t, size_t, bool) + { + // rows are only inserted at the end, so no need to do anything + return true; + } + + bool erase_rows(size_t row_ndx, size_t, size_t last_row_ndx, bool unordered) + { + for (size_t i = 0; i < m_observers.size(); ++i) { + auto& o = m_observers[i]; + if (o.table_ndx == m_current_table) { + if (o.row_ndx == row_ndx) { + invalidate(&o); + --i; + } + else if (unordered && o.row_ndx == last_row_ndx) { + o.row_ndx = row_ndx; + } + else if (!unordered && o.row_ndx > row_ndx) { + o.row_ndx -= 1; + } + } + } + return true; + } + + bool clear_table() + { + for (size_t i = 0; i < m_observers.size(); ) { + auto& o = m_observers[i]; + if (o.table_ndx == m_current_table) { + invalidate(&o); + } + else { + ++i; + } + } + return true; + } + + bool select_link_list(size_t col, size_t row) + { + m_active_linklist = nullptr; + for (auto& o : m_observers) { + if (o.table_ndx == m_current_table && o.row_ndx == row) { + m_active_linklist = &get_change(o, col); + break; + } + } + return true; + } + + void append_link_list_change(ColumnInfo::Kind kind, size_t index) { + ColumnInfo *o = m_active_linklist; + if (!o || o->kind == ColumnInfo::Kind::SetAll) { + // Active LinkList isn't observed or already has multiple kinds of changes + return; + } + + if (o->kind == ColumnInfo::Kind::None) { + o->kind = kind; + o->changed = true; + o->indices.add(index); + } + else if (o->kind == kind) { + if (kind == ColumnInfo::Kind::Remove) { + o->indices.add_shifted(index); + } + else if (kind == ColumnInfo::Kind::Insert) { + o->indices.insert_at(index); + } + else { + o->indices.add(index); + } + } + else { + // Array KVO can only send a single kind of change at a time, so + // if there's multiple just give up and send "Set" + o->indices.set(0); + o->kind = ColumnInfo::Kind::SetAll; + } + } + + bool link_list_set(size_t index, size_t) + { + append_link_list_change(ColumnInfo::Kind::Set, index); + return true; + } + + bool link_list_insert(size_t index, size_t) + { + append_link_list_change(ColumnInfo::Kind::Insert, index); + return true; + } + + bool link_list_erase(size_t index) + { + append_link_list_change(ColumnInfo::Kind::Remove, index); + return true; + } + + bool link_list_nullify(size_t index) + { + append_link_list_change(ColumnInfo::Kind::Remove, index); + return true; + } + + bool link_list_swap(size_t index1, size_t index2) + { + append_link_list_change(ColumnInfo::Kind::Set, index1); + append_link_list_change(ColumnInfo::Kind::Set, index2); + return true; + } + + bool link_list_clear(size_t old_size) + { + ColumnInfo *o = m_active_linklist; + if (!o || o->kind == ColumnInfo::Kind::SetAll) { + return true; + } + + if (o->kind == ColumnInfo::Kind::Remove) + old_size += o->indices.size(); + else if (o->kind == ColumnInfo::Kind::Insert) + old_size -= o->indices.size(); + + o->indices.set(old_size); + + o->kind = ColumnInfo::Kind::Remove; + o->changed = true; + return true; + } + + bool link_list_move(size_t from, size_t to) + { + ColumnInfo *o = m_active_linklist; + if (!o || o->kind == ColumnInfo::Kind::SetAll) { + return true; + } + if (from > to) { + std::swap(from, to); + } + + if (o->kind == ColumnInfo::Kind::None) { + o->kind = ColumnInfo::Kind::Set; + o->changed = true; + } + if (o->kind == ColumnInfo::Kind::Set) { + for (size_t i = from; i <= to; ++i) + o->indices.add(i); + } + else { + o->indices.set(0); + o->kind = ColumnInfo::Kind::SetAll; + } + return true; + } + + // Things that just mark the field as modified + bool set_int(size_t col, size_t row, int_fast64_t) { return mark_dirty(row, col); } + bool set_bool(size_t col, size_t row, bool) { return mark_dirty(row, col); } + bool set_float(size_t col, size_t row, float) { return mark_dirty(row, col); } + bool set_double(size_t col, size_t row, double) { return mark_dirty(row, col); } + bool set_string(size_t col, size_t row, StringData) { return mark_dirty(row, col); } + bool set_binary(size_t col, size_t row, BinaryData) { return mark_dirty(row, col); } + bool set_date_time(size_t col, size_t row, DateTime) { return mark_dirty(row, col); } + bool set_table(size_t col, size_t row) { return mark_dirty(row, col); } + bool set_mixed(size_t col, size_t row, const Mixed&) { return mark_dirty(row, col); } + bool set_link(size_t col, size_t row, size_t) { return mark_dirty(row, col); } + bool set_null(size_t col, size_t row) { return mark_dirty(row, col); } + bool nullify_link(size_t col, size_t row) { return mark_dirty(row, col); } + + // Doesn't change any data + bool optimize_table() { return true; } + + // Used for subtables, which we currently don't support + bool select_descriptor(int, const size_t*) { return false; } +}; +} // anonymous namespace + +namespace realm { +namespace transaction { +void advance(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate) { + TransactLogHandler(delegate, sg, [&](auto&&... args) { + LangBindHelper::advance_read(sg, history, std::move(args)...); + }); +} + +void begin(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate) { + TransactLogHandler(delegate, sg, [&](auto&&... args) { + LangBindHelper::promote_to_write(sg, history, std::move(args)...); + }); +} + +void commit(SharedGroup& sg, ClientHistory&, RealmDelegate* delegate) { + LangBindHelper::commit_and_continue_as_read(sg); + + if (delegate) { + delegate->transaction_committed(); + } +} + +void cancel(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate) { + TransactLogHandler(delegate, sg, [&](auto&&... args) { + LangBindHelper::rollback_and_continue_as_read(sg, history, std::move(args)...); + }); +} + +} // namespace transaction +} // namespace realm diff --git a/transact_log_handler.hpp b/transact_log_handler.hpp new file mode 100644 index 00000000..3a77848f --- /dev/null +++ b/transact_log_handler.hpp @@ -0,0 +1,46 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_TRANSACT_LOG_HANDLER_HPP +#define REALM_TRANSACT_LOG_HANDLER_HPP + +namespace realm { +class RealmDelegate; +class SharedGroup; +class ClientHistory; + +namespace transaction { +// Advance the read transaction version, with change notifications sent to delegate +// Must not be called from within a write transaction. +void advance(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate); + +// Begin a write transaction +// If the read transaction version is not up to date, will first advance to the +// most recent read transaction and sent notifications to delegate +void begin(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate); + +// Commit a write transaction +void commit(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate); + +// Cancel a write transaction and roll back all changes, with change notifications +// for reverting to the old values sent to delegate +void cancel(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate); +} // namespace transaction +} // namespace realm + +#endif /* REALM_TRANSACT_LOG_HANDLER_HPP */ From e4f29fe2217a7ebdd77a154073e4db156c14a521 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 2 Sep 2015 17:53:21 -0700 Subject: [PATCH 19/35] Move the interprocess notification functionality to the object store --- apple/external_commit_helper.cpp | 226 +++++++++++++++++++++++++++++++ apple/external_commit_helper.hpp | 81 +++++++++++ realm_delegate.hpp | 6 +- shared_realm.cpp | 7 +- shared_realm.hpp | 3 + transact_log_handler.cpp | 2 +- 6 files changed, 318 insertions(+), 7 deletions(-) create mode 100644 apple/external_commit_helper.cpp create mode 100644 apple/external_commit_helper.hpp diff --git a/apple/external_commit_helper.cpp b/apple/external_commit_helper.cpp new file mode 100644 index 00000000..2364e970 --- /dev/null +++ b/apple/external_commit_helper.cpp @@ -0,0 +1,226 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "external_commit_helper.hpp" + +#include "shared_realm.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace realm; + +namespace { +// Write a byte to a pipe to notify anyone waiting for data on the pipe +void notify_fd(int fd) +{ + while (true) { + char c = 0; + ssize_t ret = write(fd, &c, 1); + if (ret == 1) { + break; + } + + // If the pipe's buffer is full, we need to read some of the old data in + // it to make space. We don't just read in the code waiting for + // notifications so that we can notify multiple waiters with a single + // write. + assert(ret == -1 && errno == EAGAIN); + char buff[1024]; + read(fd, buff, sizeof buff); + } +} +} // anonymous namespace + +void ExternalCommitHelper::FdHolder::close() +{ + if (m_fd != -1) { + ::close(m_fd); + } + m_fd = -1; + +} + +// Inter-thread and inter-process notifications of changes are done using a +// named pipe in the filesystem next to the Realm file. Everyone who wants to be +// notified of commits waits for data to become available on the pipe, and anyone +// who commits a write transaction writes data to the pipe after releasing the +// write lock. Note that no one ever actually *reads* from the pipe: the data +// actually written is meaningless, and trying to read from a pipe from multiple +// processes at once is fraught with race conditions. + +// When a RLMRealm instance is created, we add a CFRunLoopSource to the current +// thread's runloop. On each cycle of the run loop, the run loop checks each of +// its sources for work to do, which in the case of CFRunLoopSource is just +// checking if CFRunLoopSourceSignal has been called since the last time it ran, +// and if so invokes the function pointer supplied when the source is created, +// which in our case just invokes `[realm handleExternalChange]`. + +// Listening for external changes is done using kqueue() on a background thread. +// kqueue() lets us efficiently wait until the amount of data which can be read +// from one or more file descriptors has changed, and tells us which of the file +// descriptors it was that changed. We use this to wait on both the shared named +// pipe, and a local anonymous pipe. When data is written to the named pipe, we +// signal the runloop source and wake up the target runloop, and when data is +// written to the anonymous pipe the background thread removes the runloop +// source from the runloop and and shuts down. +ExternalCommitHelper::ExternalCommitHelper(Realm* realm) +: m_realm(realm) +, m_run_loop(CFRunLoopGetCurrent()) +{ + CFRetain(m_run_loop); + + m_kq = kqueue(); + if (m_kq == -1) { + throw std::system_error(errno, std::system_category()); + } + + auto path = realm->config().path + ".note"; + + // Create and open the named pipe + int ret = mkfifo(path.c_str(), 0600); + if (ret == -1) { + int err = errno; + if (err == ENOTSUP) { + // Filesystem doesn't support named pipes, so try putting it in tmp instead + // Hash collisions are okay here because they just result in doing + // extra work, as opposed to correctness problems + std::ostringstream ss; + ss << getenv("TMPDIR"); + ss << "realm_" << std::hash()(path) << ".note"; + path = ss.str(); + ret = mkfifo(path.c_str(), 0600); + err = errno; + } + // the fifo already existing isn't an error + if (ret == -1 && err != EEXIST) { + throw std::system_error(err, std::system_category()); + } + } + + m_notify_fd = open(path.c_str(), O_RDWR); + if (m_notify_fd == -1) { + throw std::system_error(errno, std::system_category()); + } + + // Make writing to the pipe return -1 when the pipe's buffer is full + // rather than blocking until there's space available + ret = fcntl(m_notify_fd, F_SETFL, O_NONBLOCK); + if (ret == -1) { + throw std::system_error(errno, std::system_category()); + } + + // Create the anonymous pipe + int pipeFd[2]; + ret = pipe(pipeFd); + if (ret == -1) { + throw std::system_error(errno, std::system_category()); + } + + m_shutdown_read_fd = pipeFd[0]; + m_shutdown_write_fd = pipeFd[1]; + + // Use the minimum allowed stack size, as we need very little in our listener + // https://developer.apple.com/library/ios/documentation/Cocoa/Conceptual/Multithreading/CreatingThreads/CreatingThreads.html#//apple_ref/doc/uid/10000057i-CH15-SW7 + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 16 * 1024); + + auto fn = [](void *self) -> void * { + static_cast(self)->listen(); + return nullptr; + }; + ret = pthread_create(&m_thread, &attr, fn, this); + pthread_attr_destroy(&attr); + if (ret != 0) { + throw std::system_error(errno, std::system_category()); + } +} + +ExternalCommitHelper::~ExternalCommitHelper() +{ + notify_fd(m_shutdown_write_fd); + pthread_join(m_thread, nullptr); // Wait for the thread to exit +} + +void ExternalCommitHelper::listen() +{ + pthread_setname_np("RLMRealm notification listener"); + + // Create the runloop source + CFRunLoopSourceContext ctx{}; + ctx.info = this; + ctx.perform = [](void *info) { + static_cast(info)->m_realm->notify(); + }; + + CFRunLoopSourceRef signal = CFRunLoopSourceCreate(kCFAllocatorDefault, 0, &ctx); + CFRunLoopAddSource(m_run_loop, signal, kCFRunLoopDefaultMode); + + // Set up the kqueue + // EVFILT_READ indicates that we care about data being available to read + // on the given file descriptor. + // EV_CLEAR makes it wait for the amount of data available to be read to + // change rather than just returning when there is any data to read. + struct kevent ke[2]; + EV_SET(&ke[0], m_notify_fd, EVFILT_READ, EV_ADD | EV_CLEAR, 0, 0, 0); + EV_SET(&ke[1], m_shutdown_read_fd, EVFILT_READ, EV_ADD | EV_CLEAR, 0, 0, 0); + int ret = kevent(m_kq, ke, 2, nullptr, 0, nullptr); + assert(ret == 0); + + while (true) { + struct kevent event; + // Wait for data to become on either fd + // Return code is number of bytes available or -1 on error + ret = kevent(m_kq, nullptr, 0, &event, 1, nullptr); + assert(ret >= 0); + if (ret == 0) { + // Spurious wakeup; just wait again + continue; + } + + // Check which file descriptor had activity: if it's the shutdown + // pipe, then someone called -stop; otherwise it's the named pipe + // and someone committed a write transaction + if (event.ident == (uint32_t)m_shutdown_read_fd) { + CFRunLoopSourceInvalidate(signal); + CFRelease(signal); + CFRelease(m_run_loop); + return; + } + assert(event.ident == (uint32_t)m_notify_fd); + + CFRunLoopSourceSignal(signal); + // Signalling the source makes it run the next time the runloop gets + // to it, but doesn't make the runloop start if it's currently idle + // waiting for events + CFRunLoopWakeUp(m_run_loop); + } +} + +void ExternalCommitHelper::notify_others() +{ + notify_fd(m_notify_fd); +} + diff --git a/apple/external_commit_helper.hpp b/apple/external_commit_helper.hpp new file mode 100644 index 00000000..2793762d --- /dev/null +++ b/apple/external_commit_helper.hpp @@ -0,0 +1,81 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_EXTERNAL_COMMIT_HELPER_HPP +#define REALM_EXTERNAL_COMMIT_HELPER_HPP + +#include + +namespace realm { +class Realm; + +class ExternalCommitHelper { +public: + ExternalCommitHelper(Realm* realm); + ~ExternalCommitHelper(); + + void notify_others(); + +private: + // A RAII holder for a file descriptor which automatically closes the wrapped + // fd when it's deallocated + class FdHolder { + public: + FdHolder() = default; + ~FdHolder() { close(); } + operator int() const { return m_fd; } + + FdHolder& operator=(int newFd) { + close(); + m_fd = newFd; + return *this; + } + + private: + int m_fd = -1; + void close(); + + FdHolder& operator=(FdHolder const&) = delete; + FdHolder(FdHolder const&) = delete; + }; + + void listen(); + + // This is owned by the realm, so it needs to not retain the realm + Realm *const m_realm; + + // Runloop which notifications are delivered on + CFRunLoopRef m_run_loop; + + // The listener thread + pthread_t m_thread; + + // Read-write file descriptor for the named pipe which is waited on for + // changes and written to when a commit is made + FdHolder m_notify_fd; + // File descriptor for the kqueue + FdHolder m_kq; + // The two ends of an anonymous pipe used to notify the kqueue() thread that + // it should be shut down. + FdHolder m_shutdown_read_fd; + FdHolder m_shutdown_write_fd; +}; + +} // namespace realm + +#endif /* REALM_EXTERNAL_COMMIT_HELPER_HPP */ diff --git a/realm_delegate.hpp b/realm_delegate.hpp index c3a41d04..0941b2dc 100644 --- a/realm_delegate.hpp +++ b/realm_delegate.hpp @@ -63,10 +63,6 @@ public: } }; - // The Realm has committed a write transaction, and other Realms at the - // same path should be notified - virtual void transaction_committed() = 0; - // There are now new versions available for the Realm, but it has not // had its read version advanced virtual void changes_available() = 0; @@ -86,6 +82,8 @@ public: // The Realm's read version has changed // observers is the vector returned from get_observed_rows() // invalidated is the `info` pointers for each observed object which was deleted + // Is called with empty change information following a local commit or if + // the Realm isn't in a read transaction yet virtual void did_change(std::vector const& observers, std::vector const& invalidated) = 0; }; diff --git a/shared_realm.cpp b/shared_realm.cpp index 7d270cc2..b201574c 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -18,6 +18,7 @@ #include "shared_realm.hpp" +#include "external_commit_helper.hpp" #include "realm_delegate.hpp" #include "transact_log_handler.hpp" @@ -52,7 +53,9 @@ Realm::Config& Realm::Config::operator=(realm::Realm::Config const& c) return *this; } -Realm::Realm(Config config) : m_config(std::move(config)) +Realm::Realm(Config config) +: m_config(std::move(config)) +, m_notifier(new ExternalCommitHelper(this)) { try { if (m_config.read_only) { @@ -234,6 +237,7 @@ void Realm::commit_transaction() m_in_transaction = false; transaction::commit(*m_shared_group, *m_history, m_delegate.get()); + m_notifier->notify_others(); } void Realm::cancel_transaction() @@ -416,4 +420,3 @@ void RealmCache::clear() m_cache.clear(); } - diff --git a/shared_realm.hpp b/shared_realm.hpp index 3ef4bdf1..eb9ee424 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -28,6 +28,7 @@ namespace realm { class ClientHistory; + class ExternalCommitHelper; class Realm; class RealmCache; class RealmDelegate; @@ -110,6 +111,8 @@ namespace realm { Group *m_group = nullptr; + std::unique_ptr m_notifier; + public: std::unique_ptr m_delegate; diff --git a/transact_log_handler.cpp b/transact_log_handler.cpp index b02fdcbd..3f833b19 100644 --- a/transact_log_handler.cpp +++ b/transact_log_handler.cpp @@ -333,7 +333,7 @@ void commit(SharedGroup& sg, ClientHistory&, RealmDelegate* delegate) { LangBindHelper::commit_and_continue_as_read(sg); if (delegate) { - delegate->transaction_committed(); + delegate->did_change({}, {}); } } From ea5c47510b27bffeb1c234af4ab48969a52eb467 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 3 Sep 2015 10:37:24 -0700 Subject: [PATCH 20/35] Refactor schema initialization a bit Change schema verification to operate on a pair of Schema objects rather than a Schema and a Group to eliminate some redundant work done, defer some of the work done for migrations to within the migration block to avoid doing it unnecessarily, and make passing in a custom schema in the Config when creating a Realm entirely equivalent to calling update_schema() afterwards. --- object_store.cpp | 99 +++++++++++++++++++++++++++--------------------- object_store.hpp | 54 +++++++++++++------------- shared_realm.cpp | 94 ++++++++++++++++++++++++--------------------- shared_realm.hpp | 2 +- 4 files changed, 134 insertions(+), 115 deletions(-) diff --git a/object_store.cpp b/object_store.cpp index d8fb943d..c76d8dc1 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -136,20 +136,24 @@ TableRef ObjectStore::table_for_object_type_create_if_needed(Group *group, const return group->get_or_add_table(table_name_for_object_type(object_type), &created); } -static inline bool property_has_changed(Property &p1, Property &p2) { - return p1.type != p2.type || p1.name != p2.name || p1.object_type != p2.object_type || p1.is_nullable != p2.is_nullable; +static inline bool property_has_changed(Property const& p1, Property const& p2) { + return p1.type != p2.type + || p1.name != p2.name + || p1.object_type != p2.object_type + || p1.is_nullable != p2.is_nullable; } static bool compare_by_name(ObjectSchema const& lft, ObjectSchema const& rgt) { return lft.name < rgt.name; } -void ObjectStore::verify_schema(Group *group, Schema &target_schema, bool allow_missing_tables) { +void ObjectStore::verify_schema(Schema const& actual_schema, Schema &target_schema, bool allow_missing_tables) { std::sort(begin(target_schema), end(target_schema), compare_by_name); std::vector errors; for (auto &object_schema : target_schema) { - if (!table_for_object_type(group, object_schema.name)) { + auto matching_schema = std::lower_bound(begin(actual_schema), end(actual_schema), object_schema, compare_by_name); + if (matching_schema == end(actual_schema) || matching_schema->name != object_schema.name) { if (!allow_missing_tables) { errors.emplace_back(ObjectSchemaValidationException(object_schema.name, "Missing table for object type '" + object_schema.name + "'.")); @@ -157,7 +161,7 @@ void ObjectStore::verify_schema(Group *group, Schema &target_schema, bool allow_ continue; } - auto more_errors = verify_object_schema(group, object_schema, target_schema); + auto more_errors = verify_object_schema(*matching_schema, object_schema, target_schema); errors.insert(errors.end(), more_errors.begin(), more_errors.end()); } if (errors.size()) { @@ -165,9 +169,8 @@ void ObjectStore::verify_schema(Group *group, Schema &target_schema, bool allow_ } } -std::vector ObjectStore::verify_object_schema(Group *group, ObjectSchema &target_schema, Schema &schema) { +std::vector ObjectStore::verify_object_schema(ObjectSchema const& table_schema, ObjectSchema &target_schema, Schema &schema) { std::vector exceptions; - ObjectSchema table_schema(group, target_schema.name); ObjectSchema cmp; auto schema_contains_table = [&](std::string const& name) { @@ -176,7 +179,7 @@ std::vector ObjectStore::verify_object_schema(G }; // check to see if properties are the same - Property *primary = nullptr; + const Property *primary = nullptr; for (auto& current_prop : table_schema.properties) { auto target_prop = target_schema.property_for_name(current_prop.name); @@ -276,12 +279,27 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update ObjectSchema current_schema(group, target_object_schema->name); std::vector &target_props = target_object_schema->properties; + // remove extra columns + size_t deleted = 0; + for (auto& current_prop : current_schema.properties) { + auto target_prop = target_object_schema->property_for_name(current_prop.name); + if (!target_prop || property_has_changed(current_prop, *target_prop)) { + table->remove_column(current_prop.table_column - deleted); + ++deleted; + current_prop.table_column = npos; + changed = true; + } + else { + current_prop.table_column -= deleted; + } + } + // add missing columns for (auto& target_prop : target_props) { auto current_prop = current_schema.property_for_name(target_prop.name); // add any new properties (new name or different type) - if (!current_prop || property_has_changed(*current_prop, target_prop)) { + if (!current_prop || current_prop->table_column == npos) { switch (target_prop.type) { // for objects and arrays, we have to specify target table case PropertyTypeObject: @@ -297,17 +315,8 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update changed = true; } - } - - // remove extra columns - sort(begin(current_schema.properties), end(current_schema.properties), [](Property &i, Property &j) { - return j.table_column < i.table_column; - }); - for (auto& current_prop : current_schema.properties) { - auto target_prop = target_object_schema->property_for_name(current_prop.name); - if (!target_prop || property_has_changed(current_prop, *target_prop)) { - table->remove_column(current_prop.table_column); - changed = true; + else { + target_prop.table_column = current_prop->table_column; } } @@ -336,24 +345,22 @@ bool ObjectStore::is_schema_at_version(Group *group, uint64_t version) { return old_version == version; } -bool ObjectStore::realm_requires_update(Group *group, uint64_t version, Schema const& schema) { - if (!is_schema_at_version(group, version)) { - return true; - } - +bool ObjectStore::needs_update(Schema const& old_schema, Schema& schema) { for (auto const& target_schema : schema) { - TableRef table = table_for_object_type(group, target_schema.name); - if (!table) { + auto matching_schema = std::lower_bound(begin(old_schema), end(old_schema), target_schema, compare_by_name); + if (matching_schema == end(old_schema) || matching_schema->name != target_schema.name) { + // Table doesn't exist return true; } + if (matching_schema->properties.size() != target_schema.properties.size()) { + // If the number of properties don't match then a migration is required + return false; + } + // Check that all of the property indexes are up to date - size_t count = table->get_column_count(); - for (size_t col = 0; col < count; ++col) { - StringData name = table->get_column_name(col); - bool is_indexed = table->has_search_index(col); - auto prop = target_schema.property_for_name(name); - if (prop && prop->requires_index() != is_indexed) { + for (size_t i = 0, count = target_schema.properties.size(); i < count; ++i) { + if (target_schema.properties[i].is_indexed != matching_schema->properties[i].is_indexed) { return true; } } @@ -362,9 +369,8 @@ bool ObjectStore::realm_requires_update(Group *group, uint64_t version, Schema c return false; } -bool ObjectStore::update_realm_with_schema(Group *group, - uint64_t version, - Schema &schema, +bool ObjectStore::update_realm_with_schema(Group *group, Schema const& old_schema, + uint64_t version, Schema &schema, MigrationFunction migration) { // Recheck the schema version after beginning the write transaction as // another process may have done the migration after we opened the read @@ -375,7 +381,11 @@ bool ObjectStore::update_realm_with_schema(Group *group, bool changed = create_metadata_tables(group); changed = create_tables(group, schema, migrating) || changed; - verify_schema(group, schema); + if (!migrating) { + // If we aren't migrating, then verify that all of the tables which + // were already present are valid (newly created ones always are) + verify_schema(old_schema, schema, true); + } changed = update_indexes(group, schema) || changed; @@ -402,6 +412,7 @@ Schema ObjectStore::schema_from_group(Group *group) { schema.emplace_back(group, object_type); } } + std::sort(begin(schema), end(schema), compare_by_name); return schema; } @@ -465,7 +476,7 @@ InvalidSchemaVersionException::InvalidSchemaVersionException(uint64_t old_versio m_what = "Provided schema version " + std::to_string(old_version) + " is less than last set version " + std::to_string(new_version) + "."; } -DuplicatePrimaryKeyValueException::DuplicatePrimaryKeyValueException(std::string object_type, Property &property) : +DuplicatePrimaryKeyValueException::DuplicatePrimaryKeyValueException(std::string object_type, Property const& property) : m_object_type(object_type), m_property(property) { m_what = "Primary key property '" + property.name + "' has duplicate values after migration."; @@ -481,25 +492,25 @@ SchemaValidationException::SchemaValidationException(std::vector MigrationFunction; - static bool update_realm_with_schema(Group *group, uint64_t version, Schema &schema, MigrationFunction migration); + static bool update_realm_with_schema(Group *group, Schema const& old_schema, uint64_t version, + Schema &schema, MigrationFunction migration); // get a table for an object type static realm::TableRef table_for_object_type(Group *group, StringData object_type); @@ -88,10 +88,10 @@ namespace realm { // if update existing is true, updates existing tables, otherwise only adds and initializes new tables static bool create_tables(realm::Group *group, Schema &target_schema, bool update_existing); - // verify a target schema against its table, setting the table_column property on each schema object + // verify a target schema against an expected schema, setting the table_column property on each schema object // updates the column mapping on the target_schema // returns array of validation errors - static std::vector verify_object_schema(Group *group, ObjectSchema &target_schema, Schema &schema); + static std::vector verify_object_schema(ObjectSchema const& expected, ObjectSchema &target_schema, Schema &schema); // get primary key property name for object type static StringData get_primary_key_for_object(Group *group, StringData object_type); @@ -118,7 +118,7 @@ namespace realm { public: ObjectStoreException() = default; ObjectStoreException(const std::string &what) : m_what(what) {} - virtual const char* what() const noexcept { return m_what.c_str(); } + const char* what() const noexcept override { return m_what.c_str(); } protected: std::string m_what; }; @@ -137,9 +137,9 @@ namespace realm { class DuplicatePrimaryKeyValueException : public MigrationException { public: - DuplicatePrimaryKeyValueException(std::string object_type, Property &property); + DuplicatePrimaryKeyValueException(std::string object_type, Property const& property); std::string object_type() { return m_object_type; } - Property &property() { return m_property; } + Property const& property() { return m_property; } private: std::string m_object_type; Property m_property; @@ -166,36 +166,36 @@ namespace realm { class ObjectSchemaPropertyException : public ObjectSchemaValidationException { public: - ObjectSchemaPropertyException(std::string object_type, Property &property) : + ObjectSchemaPropertyException(std::string object_type, Property const& property) : ObjectSchemaValidationException(object_type), m_property(property) {} - Property &property() { return m_property; } + Property const& property() { return m_property; } private: Property m_property; }; class PropertyTypeNotIndexableException : public ObjectSchemaPropertyException { public: - PropertyTypeNotIndexableException(std::string object_type, Property &property); + PropertyTypeNotIndexableException(std::string object_type, Property const& property); }; class ExtraPropertyException : public ObjectSchemaPropertyException { public: - ExtraPropertyException(std::string object_type, Property &property); + ExtraPropertyException(std::string object_type, Property const& property); }; class MissingPropertyException : public ObjectSchemaPropertyException { public: - MissingPropertyException(std::string object_type, Property &property); + MissingPropertyException(std::string object_type, Property const& property); }; class InvalidNullabilityException : public ObjectSchemaPropertyException { public: - InvalidNullabilityException(std::string object_type, Property &property); + InvalidNullabilityException(std::string object_type, Property const& property); }; class MissingObjectTypeException : public ObjectSchemaPropertyException { public: - MissingObjectTypeException(std::string object_type, Property &property); + MissingObjectTypeException(std::string object_type, Property const& property); }; class DuplicatePrimaryKeysException : public ObjectSchemaValidationException { @@ -205,9 +205,9 @@ namespace realm { class MismatchedPropertiesException : public ObjectSchemaValidationException { public: - MismatchedPropertiesException(std::string object_type, Property &old_property, Property &new_property); - Property &old_property() { return m_old_property; } - Property &new_property() { return m_new_property; } + MismatchedPropertiesException(std::string object_type, Property const& old_property, Property const& new_property); + Property const& old_property() { return m_old_property; } + Property const& new_property() { return m_new_property; } private: Property m_old_property, m_new_property; }; diff --git a/shared_realm.cpp b/shared_realm.cpp index b201574c..2143db38 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -121,30 +121,36 @@ SharedRealm Realm::get_shared_realm(Config config) SharedRealm realm(new Realm(std::move(config))); + auto target_schema = std::move(realm->m_config.schema); + auto target_schema_version = realm->m_config.schema_version; + realm->m_config.schema_version = ObjectStore::get_schema_version(realm->read_group()); + // we want to ensure we are only initializing a single realm at a time static std::mutex s_init_mutex; std::lock_guard lock(s_init_mutex); - - uint64_t old_version = ObjectStore::get_schema_version(realm->read_group()); if (auto existing = s_global_cache.get_any_realm(realm->config().path)) { // if there is an existing realm at the current path steal its schema/column mapping // FIXME - need to validate that schemas match realm->m_config.schema = std::make_unique(*existing->m_config.schema); } - else if (!realm->m_config.schema) { - // get schema from group and skip validation - realm->m_config.schema_version = old_version; - realm->m_config.schema = std::make_unique(ObjectStore::schema_from_group(realm->read_group())); - } - else if (realm->m_config.read_only) { - if (old_version == ObjectStore::NotVersioned) { - throw UnitializedRealmException("Can't open an un-initialized Realm without a Schema"); - } - ObjectStore::verify_schema(realm->read_group(), *realm->m_config.schema, true); - } else { - // its a non-cached realm so update/migrate if needed - realm->update_schema(*realm->m_config.schema, realm->m_config.schema_version); + // otherwise get the schema from the group + realm->m_config.schema = std::make_unique(ObjectStore::schema_from_group(realm->read_group())); + + // if a target schema is supplied, verify that it matches or migrate to + // it, as neeeded + if (target_schema) { + if (realm->m_config.read_only) { + if (realm->m_config.schema_version == ObjectStore::NotVersioned) { + throw UnitializedRealmException("Can't open an un-initialized Realm without a Schema"); + } + ObjectStore::verify_schema(*realm->m_config.schema, *target_schema, true); + realm->m_config.schema = std::move(target_schema); + } + else { + realm->update_schema(std::move(target_schema), target_schema_version); + } + } } if (config.cache) { @@ -153,47 +159,49 @@ SharedRealm Realm::get_shared_realm(Config config) return realm; } -bool Realm::update_schema(Schema const& schema, uint64_t version) +bool Realm::update_schema(std::unique_ptr schema, uint64_t version) { - bool changed = false; - Config old_config(m_config); - - // set new version/schema - if (m_config.schema.get() != &schema) { - m_config.schema = std::make_unique(schema); + bool needs_update = !m_config.read_only && (m_config.schema_version != version || ObjectStore::needs_update(*m_config.schema, *schema)); + if (!needs_update) { + ObjectStore::verify_schema(*m_config.schema, *schema, m_config.read_only); + m_config.schema = std::move(schema); + m_config.schema_version = version; + return false; } + + // Store the old config/schema for the migration function, and update + // our schema to the new one + auto old_schema = std::move(m_config.schema); + Config old_config(m_config); + old_config.read_only = true; + old_config.schema = std::move(old_schema); + + m_config.schema = std::move(schema); m_config.schema_version = version; - try { - if (!m_config.read_only && ObjectStore::realm_requires_update(read_group(), version, schema)) { - // keep old copy to pass to migration function - old_config.read_only = true; - old_config.schema_version = ObjectStore::get_schema_version(read_group()); - old_config.schema = std::make_unique(ObjectStore::schema_from_group(read_group())); - SharedRealm old_realm(new Realm(old_config)); - auto updated_realm = shared_from_this(); + auto migration_function = [&](Group*, Schema&) { + SharedRealm old_realm(new Realm(old_config)); + auto updated_realm = shared_from_this(); + m_config.migration_function(old_realm, updated_realm); + }; - // update and migrate - begin_transaction(); - changed = ObjectStore::update_realm_with_schema(read_group(), version, *m_config.schema, - [=](__unused Group *group, __unused Schema &target_schema) { - m_config.migration_function(old_realm, updated_realm); - }); - commit_transaction(); - } - else { - ObjectStore::verify_schema(read_group(), *m_config.schema, m_config.read_only); - } + try { + // update and migrate + begin_transaction(); + bool changed = ObjectStore::update_realm_with_schema(read_group(), *old_config.schema, + version, *m_config.schema, + migration_function); + commit_transaction(); + return changed; } catch (...) { if (is_in_transaction()) { cancel_transaction(); } m_config.schema_version = old_config.schema_version; - m_config.schema = std::move(old_config.schema); + m_config.schema = std::move(m_config.schema); throw; } - return changed; } static void check_read_write(Realm *realm) diff --git a/shared_realm.hpp b/shared_realm.hpp index eb9ee424..a3070c47 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -75,7 +75,7 @@ namespace realm { // on the Config, and the resulting Schema and version with updated // column mappings are set on the realms config upon success. // returns if any changes were made - bool update_schema(Schema const& schema, uint64_t version); + bool update_schema(std::unique_ptr schema, uint64_t version); static uint64_t get_schema_version(Config const& config); From 0eb0bd14d7654ad38cdd936ba46b2ab7596f2875 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 3 Sep 2015 14:48:24 -0700 Subject: [PATCH 21/35] Honor is_nullable when creating columns --- object_store.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/object_store.cpp b/object_store.cpp index c76d8dc1..813b78b9 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -309,7 +309,9 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update break; } default: - target_prop.table_column = table->add_column(DataType(target_prop.type), target_prop.name); + target_prop.table_column = table->add_column(DataType(target_prop.type), + target_prop.name, + target_prop.is_nullable); break; } From f79dec9033aa7dcc87d66cab2039683bdf692261 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 3 Sep 2015 14:54:41 -0700 Subject: [PATCH 22/35] Allow more nullable property types when supported --- object_store.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/object_store.cpp b/object_store.cpp index 813b78b9..1493353f 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -198,6 +198,11 @@ std::vector ObjectStore::verify_object_schema(O } // check nullablity +#if REALM_NULL_STRINGS == 1 + if (current_prop.type == PropertyTypeArray && current_prop.is_nullable) { + exceptions.emplace_back(InvalidNullabilityException(table_schema.name, current_prop)); + } +#else if (current_prop.type == PropertyTypeObject) { if (!current_prop.is_nullable) { exceptions.emplace_back(InvalidNullabilityException(table_schema.name, current_prop)); @@ -208,6 +213,7 @@ std::vector ObjectStore::verify_object_schema(O exceptions.emplace_back(InvalidNullabilityException(table_schema.name, current_prop)); } } +#endif // check primary keys if (current_prop.is_primary) { @@ -515,6 +521,9 @@ MissingPropertyException::MissingPropertyException(std::string object_type, Prop InvalidNullabilityException::InvalidNullabilityException(std::string object_type, Property const& property) : ObjectSchemaPropertyException(object_type, property) { +#if REALM_NULL_STRINGS == 1 + m_what = "'Array' property '" + property.name + "' cannot be nullable"; +#else if (property.type == PropertyTypeObject) { if (!property.is_nullable) { m_what = "'Object' property '" + property.name + "' must be nullable."; @@ -525,6 +534,7 @@ InvalidNullabilityException::InvalidNullabilityException(std::string object_type m_what = "Only 'Object' property types are nullable"; } } +#endif } MissingObjectTypeException::MissingObjectTypeException(std::string object_type, Property const& property) : From 06e0ff837302c6116f98fb63d4dd5d3a1f1b520d Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 3 Sep 2015 17:59:20 -0700 Subject: [PATCH 23/35] Share ExternalCommitHelpers between Realm instances for a single path --- apple/external_commit_helper.cpp | 64 ++++++++++++++++++++++---------- apple/external_commit_helper.hpp | 19 ++++++++-- shared_realm.cpp | 12 +++++- shared_realm.hpp | 4 +- 4 files changed, 73 insertions(+), 26 deletions(-) diff --git a/apple/external_commit_helper.cpp b/apple/external_commit_helper.cpp index 2364e970..1444281a 100644 --- a/apple/external_commit_helper.cpp +++ b/apple/external_commit_helper.cpp @@ -86,10 +86,8 @@ void ExternalCommitHelper::FdHolder::close() // written to the anonymous pipe the background thread removes the runloop // source from the runloop and and shuts down. ExternalCommitHelper::ExternalCommitHelper(Realm* realm) -: m_realm(realm) -, m_run_loop(CFRunLoopGetCurrent()) { - CFRetain(m_run_loop); + add_realm(realm); m_kq = kqueue(); if (m_kq == -1) { @@ -160,23 +158,49 @@ ExternalCommitHelper::ExternalCommitHelper(Realm* realm) ExternalCommitHelper::~ExternalCommitHelper() { + REALM_ASSERT_DEBUG(m_realms.empty()); notify_fd(m_shutdown_write_fd); pthread_join(m_thread, nullptr); // Wait for the thread to exit } +void ExternalCommitHelper::add_realm(realm::Realm* realm) +{ + std::lock_guard lock(m_realms_mutex); + + // Create the runloop source + CFRunLoopSourceContext ctx{}; + ctx.info = realm; + ctx.perform = [](void* info) { + static_cast(info)->notify(); + }; + + CFRunLoopRef runloop = CFRunLoopGetCurrent(); + CFRetain(runloop); + CFRunLoopSourceRef signal = CFRunLoopSourceCreate(kCFAllocatorDefault, 0, &ctx); + CFRunLoopAddSource(runloop, signal, kCFRunLoopDefaultMode); + + m_realms.push_back({realm, runloop, signal}); +} + +void ExternalCommitHelper::remove_realm(realm::Realm* realm) +{ + std::lock_guard lock(m_realms_mutex); + for (auto it = m_realms.begin(); it != m_realms.end(); ++it) { + if (it->realm == realm) { + CFRunLoopSourceInvalidate(it->signal); + CFRelease(it->signal); + CFRelease(it->runloop); + m_realms.erase(it); + return; + } + } + REALM_TERMINATE("Realm not registered"); +} + void ExternalCommitHelper::listen() { pthread_setname_np("RLMRealm notification listener"); - // Create the runloop source - CFRunLoopSourceContext ctx{}; - ctx.info = this; - ctx.perform = [](void *info) { - static_cast(info)->m_realm->notify(); - }; - - CFRunLoopSourceRef signal = CFRunLoopSourceCreate(kCFAllocatorDefault, 0, &ctx); - CFRunLoopAddSource(m_run_loop, signal, kCFRunLoopDefaultMode); // Set up the kqueue // EVFILT_READ indicates that we care about data being available to read @@ -204,18 +228,18 @@ void ExternalCommitHelper::listen() // pipe, then someone called -stop; otherwise it's the named pipe // and someone committed a write transaction if (event.ident == (uint32_t)m_shutdown_read_fd) { - CFRunLoopSourceInvalidate(signal); - CFRelease(signal); - CFRelease(m_run_loop); return; } assert(event.ident == (uint32_t)m_notify_fd); - CFRunLoopSourceSignal(signal); - // Signalling the source makes it run the next time the runloop gets - // to it, but doesn't make the runloop start if it's currently idle - // waiting for events - CFRunLoopWakeUp(m_run_loop); + std::lock_guard lock(m_realms_mutex); + for (auto const& realm : m_realms) { + CFRunLoopSourceSignal(realm.signal); + // Signalling the source makes it run the next time the runloop gets + // to it, but doesn't make the runloop start if it's currently idle + // waiting for events + CFRunLoopWakeUp(realm.runloop); + } } } diff --git a/apple/external_commit_helper.hpp b/apple/external_commit_helper.hpp index 2793762d..acf0e1c4 100644 --- a/apple/external_commit_helper.hpp +++ b/apple/external_commit_helper.hpp @@ -20,6 +20,8 @@ #define REALM_EXTERNAL_COMMIT_HELPER_HPP #include +#include +#include namespace realm { class Realm; @@ -30,6 +32,8 @@ public: ~ExternalCommitHelper(); void notify_others(); + void add_realm(Realm* realm); + void remove_realm(Realm* realm); private: // A RAII holder for a file descriptor which automatically closes the wrapped @@ -54,13 +58,20 @@ private: FdHolder(FdHolder const&) = delete; }; + struct PerRealmInfo { + Realm* realm; + CFRunLoopRef runloop; + CFRunLoopSourceRef signal; + }; + void listen(); - // This is owned by the realm, so it needs to not retain the realm - Realm *const m_realm; + // Currently registered realms and the signal for delivering notifications + // to them + std::vector m_realms; - // Runloop which notifications are delivered on - CFRunLoopRef m_run_loop; + // Mutex which guards m_realms + std::mutex m_realms_mutex; // The listener thread pthread_t m_thread; diff --git a/shared_realm.cpp b/shared_realm.cpp index 2143db38..8b1abd84 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -55,7 +55,6 @@ Realm::Config& Realm::Config::operator=(realm::Realm::Config const& c) Realm::Realm(Config config) : m_config(std::move(config)) -, m_notifier(new ExternalCommitHelper(this)) { try { if (m_config.read_only) { @@ -85,6 +84,12 @@ Realm::Realm(Config config) } } +Realm::~Realm() { + if (m_notifier) { // might not exist yet if an error occurred during init + m_notifier->remove_realm(this); + } +} + Group *Realm::read_group() { if (!m_group) { @@ -132,8 +137,13 @@ SharedRealm Realm::get_shared_realm(Config config) // if there is an existing realm at the current path steal its schema/column mapping // FIXME - need to validate that schemas match realm->m_config.schema = std::make_unique(*existing->m_config.schema); + + realm->m_notifier = existing->m_notifier; + realm->m_notifier->add_realm(realm.get()); } else { + realm->m_notifier = std::make_shared(realm.get()); + // otherwise get the schema from the group realm->m_config.schema = std::make_unique(ObjectStore::schema_from_group(realm->read_group())); diff --git a/shared_realm.hpp b/shared_realm.hpp index a3070c47..3836c5e9 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -97,6 +97,8 @@ namespace realm { std::thread::id thread_id() const { return m_thread_id; } void verify_thread(); + ~Realm(); + private: Realm(Config config); @@ -111,7 +113,7 @@ namespace realm { Group *m_group = nullptr; - std::unique_ptr m_notifier; + std::shared_ptr m_notifier; public: std::unique_ptr m_delegate; From 14004504b1105d2b1f50c96644e1ce3494f4791e Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Sep 2015 09:40:40 -0700 Subject: [PATCH 24/35] Remove an unused function --- object_store.cpp | 11 ----------- object_store.hpp | 3 --- 2 files changed, 14 deletions(-) diff --git a/object_store.cpp b/object_store.cpp index 1493353f..05682adf 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -249,17 +249,6 @@ std::vector ObjectStore::verify_object_schema(O return exceptions; } -void ObjectStore::update_column_mapping(Group *group, ObjectSchema &target_schema) { - ObjectSchema table_schema(group, target_schema.name); - for (auto& target_prop : target_schema.properties) { - auto table_prop = table_schema.property_for_name(target_prop.name); - if (table_prop) { - // Update target property column to match what's in the realm if it exists - target_prop.table_column = table_prop->table_column; - } - } -} - // set references to tables on targetSchema and create/update any missing or out-of-date tables // if update existing is true, updates existing tables, otherwise validates existing tables // NOTE: must be called from within write transaction diff --git a/object_store.hpp b/object_store.hpp index 5778ff64..34169ccd 100644 --- a/object_store.hpp +++ b/object_store.hpp @@ -47,9 +47,6 @@ namespace realm { // throws if the schema is invalid or does not match static void verify_schema(Schema const& actual_schema, Schema &target_schema, bool allow_missing_tables = false); - // updates the target_column member for all properties based on the column indexes in the passed in group - static void update_column_mapping(Group *group, ObjectSchema &target_schema); - // determines if a realm with the given old schema needs non-migration // changes to make it compatible with the given target schema static bool needs_update(Schema const& old_schema, Schema& schema); From dbac77f69b70b780f0521c26dd57a6515d37a8b0 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Sep 2015 09:40:29 -0700 Subject: [PATCH 25/35] Make a bunch of things const --- object_schema.cpp | 7 +++---- object_schema.hpp | 5 ++++- object_store.cpp | 31 ++++++++++++++++++------------- object_store.hpp | 44 +++++++++++++++++++++++--------------------- shared_realm.cpp | 2 +- shared_realm.hpp | 6 +++--- 6 files changed, 52 insertions(+), 43 deletions(-) diff --git a/object_schema.cpp b/object_schema.cpp index cfb65f5d..33cd2497 100644 --- a/object_schema.cpp +++ b/object_schema.cpp @@ -28,9 +28,8 @@ using namespace realm; ObjectSchema::~ObjectSchema() = default; -ObjectSchema::ObjectSchema(Group *group, const std::string &name) : name(name) { - TableRef tableRef = ObjectStore::table_for_object_type(group, name); - Table *table = tableRef.get(); +ObjectSchema::ObjectSchema(const Group *group, const std::string &name) : name(name) { + ConstTableRef table = ObjectStore::table_for_object_type(group, name); size_t count = table->get_column_count(); properties.reserve(count); @@ -44,7 +43,7 @@ ObjectSchema::ObjectSchema(Group *group, const std::string &name) : name(name) { property.table_column = col; if (property.type == PropertyTypeObject || property.type == PropertyTypeArray) { // set link type for objects and arrays - realm::TableRef linkTable = table->get_link_target(col); + ConstTableRef linkTable = table->get_link_target(col); property.object_type = ObjectStore::object_type_for_table_name(linkTable->get_name().data()); } properties.push_back(std::move(property)); diff --git a/object_schema.hpp b/object_schema.hpp index 5b7bf343..aca4af31 100644 --- a/object_schema.hpp +++ b/object_schema.hpp @@ -35,7 +35,7 @@ namespace realm { // create object schema from existing table // if no table is provided it is looked up in the group - ObjectSchema(Group *group, const std::string &name); + ObjectSchema(const Group *group, const std::string &name); std::string name; std::vector properties; @@ -46,6 +46,9 @@ namespace realm { Property *primary_key_property() { return property_for_name(primary_key); } + const Property *primary_key_property() const { + return property_for_name(primary_key); + } }; } diff --git a/object_store.cpp b/object_store.cpp index 05682adf..9e31a1c7 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -45,7 +45,7 @@ const size_t c_object_table_prefix_length = c_object_table_prefix.length(); const uint64_t ObjectStore::NotVersioned = std::numeric_limits::max(); -bool ObjectStore::has_metadata_tables(Group *group) { +bool ObjectStore::has_metadata_tables(const Group *group) { return group->get_table(c_primaryKeyTableName) && group->get_table(c_metadataTableName); } @@ -71,8 +71,8 @@ bool ObjectStore::create_metadata_tables(Group *group) { return changed; } -uint64_t ObjectStore::get_schema_version(Group *group) { - TableRef table = group->get_table(c_metadataTableName); +uint64_t ObjectStore::get_schema_version(const Group *group) { + ConstTableRef table = group->get_table(c_metadataTableName); if (!table || table->get_column_count() == 0) { return ObjectStore::NotVersioned; } @@ -84,8 +84,8 @@ void ObjectStore::set_schema_version(Group *group, uint64_t version) { table->set_int(c_versionColumnIndex, c_zeroRowIndex, version); } -StringData ObjectStore::get_primary_key_for_object(Group *group, StringData object_type) { - TableRef table = group->get_table(c_primaryKeyTableName); +StringData ObjectStore::get_primary_key_for_object(const Group *group, StringData object_type) { + ConstTableRef table = group->get_table(c_primaryKeyTableName); if (!table) { return ""; } @@ -132,6 +132,10 @@ TableRef ObjectStore::table_for_object_type(Group *group, StringData object_type return group->get_table(table_name_for_object_type(object_type)); } +ConstTableRef ObjectStore::table_for_object_type(const Group *group, StringData object_type) { + return group->get_table(table_name_for_object_type(object_type)); +} + TableRef ObjectStore::table_for_object_type_create_if_needed(Group *group, const StringData &object_type, bool &created) { return group->get_or_add_table(table_name_for_object_type(object_type), &created); } @@ -147,7 +151,7 @@ static bool compare_by_name(ObjectSchema const& lft, ObjectSchema const& rgt) { return lft.name < rgt.name; } -void ObjectStore::verify_schema(Schema const& actual_schema, Schema &target_schema, bool allow_missing_tables) { +void ObjectStore::verify_schema(Schema const& actual_schema, Schema& target_schema, bool allow_missing_tables) { std::sort(begin(target_schema), end(target_schema), compare_by_name); std::vector errors; @@ -169,7 +173,9 @@ void ObjectStore::verify_schema(Schema const& actual_schema, Schema &target_sche } } -std::vector ObjectStore::verify_object_schema(ObjectSchema const& table_schema, ObjectSchema &target_schema, Schema &schema) { +std::vector ObjectStore::verify_object_schema(ObjectSchema const& table_schema, + ObjectSchema& target_schema, + Schema const& schema) { std::vector exceptions; ObjectSchema cmp; @@ -334,7 +340,7 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update return changed; } -bool ObjectStore::is_schema_at_version(Group *group, uint64_t version) { +bool ObjectStore::is_schema_at_version(const Group *group, uint64_t version) { uint64_t old_version = get_schema_version(group); if (old_version > version && old_version != NotVersioned) { throw InvalidSchemaVersionException(old_version, version); @@ -342,7 +348,7 @@ bool ObjectStore::is_schema_at_version(Group *group, uint64_t version) { return old_version == version; } -bool ObjectStore::needs_update(Schema const& old_schema, Schema& schema) { +bool ObjectStore::needs_update(Schema const& old_schema, Schema const& schema) { for (auto const& target_schema : schema) { auto matching_schema = std::lower_bound(begin(old_schema), end(old_schema), target_schema, compare_by_name); if (matching_schema == end(old_schema) || matching_schema->name != target_schema.name) { @@ -401,7 +407,7 @@ bool ObjectStore::update_realm_with_schema(Group *group, Schema const& old_schem return true; } -Schema ObjectStore::schema_from_group(Group *group) { +Schema ObjectStore::schema_from_group(const Group *group) { Schema schema; for (size_t i = 0; i < group->size(); i++) { std::string object_type = object_type_for_table_name(group->get_table_name(i)); @@ -443,14 +449,14 @@ bool ObjectStore::update_indexes(Group *group, Schema &schema) { return changed; } -void ObjectStore::validate_primary_column_uniqueness(Group *group, Schema &schema) { +void ObjectStore::validate_primary_column_uniqueness(const Group *group, Schema const& schema) { for (auto& object_schema : schema) { auto primary_prop = object_schema.primary_key_property(); if (!primary_prop) { continue; } - TableRef table = table_for_object_type(group, object_schema.name); + ConstTableRef table = table_for_object_type(group, object_schema.name); if (table->get_distinct_view(primary_prop->table_column).size() != table->size()) { throw DuplicatePrimaryKeyValueException(object_schema.name, *primary_prop); } @@ -567,4 +573,3 @@ DuplicatePrimaryKeysException::DuplicatePrimaryKeysException(std::string object_ { m_what = "Duplicate primary keys for object '" + object_type + "'."; } - diff --git a/object_store.hpp b/object_store.hpp index 34169ccd..89a7108e 100644 --- a/object_store.hpp +++ b/object_store.hpp @@ -37,19 +37,19 @@ namespace realm { static const uint64_t NotVersioned; // get the last set schema version - static uint64_t get_schema_version(Group *group); + static uint64_t get_schema_version(const Group *group); // checks if the schema in the group is at the given version - static bool is_schema_at_version(realm::Group *group, uint64_t version); + static bool is_schema_at_version(const Group *group, uint64_t version); // verify that schema from a group and a target schema are compatible // updates the column mapping on all ObjectSchema properties of the target schema // throws if the schema is invalid or does not match - static void verify_schema(Schema const& actual_schema, Schema &target_schema, bool allow_missing_tables = false); + static void verify_schema(Schema const& actual_schema, Schema& target_schema, bool allow_missing_tables = false); // determines if a realm with the given old schema needs non-migration // changes to make it compatible with the given target schema - static bool needs_update(Schema const& old_schema, Schema& schema); + static bool needs_update(Schema const& old_schema, Schema const& schema); // updates a Realm from old_schema to the given target schema, creating and updating tables as needed // returns if any changes were made @@ -62,9 +62,10 @@ namespace realm { // get a table for an object type static realm::TableRef table_for_object_type(Group *group, StringData object_type); + static realm::ConstTableRef table_for_object_type(const Group *group, StringData object_type); // get existing Schema from a group - static Schema schema_from_group(Group *group); + static Schema schema_from_group(const Group *group); // deletes the table for the given type static void delete_data_for_object(Group *group, const StringData &object_type); @@ -74,7 +75,7 @@ namespace realm { static void set_schema_version(Group *group, uint64_t version); // check if the realm already has all metadata tables - static bool has_metadata_tables(Group *group); + static bool has_metadata_tables(const Group *group); // create any metadata tables that don't already exist // must be in write transaction to set @@ -88,10 +89,12 @@ namespace realm { // verify a target schema against an expected schema, setting the table_column property on each schema object // updates the column mapping on the target_schema // returns array of validation errors - static std::vector verify_object_schema(ObjectSchema const& expected, ObjectSchema &target_schema, Schema &schema); + static std::vector verify_object_schema(ObjectSchema const& expected, + ObjectSchema &target_schema, + Schema const& schema); // get primary key property name for object type - static StringData get_primary_key_for_object(Group *group, StringData object_type); + static StringData get_primary_key_for_object(const Group *group, StringData object_type); // sets primary key property for object type // must be in write transaction to set @@ -105,7 +108,7 @@ namespace realm { static bool update_indexes(Group *group, Schema &schema); // validates that all primary key properties have unique values - static void validate_primary_column_uniqueness(Group *group, Schema &schema); + static void validate_primary_column_uniqueness(const Group *group, Schema const& schema); friend ObjectSchema; }; @@ -126,8 +129,8 @@ namespace realm { class InvalidSchemaVersionException : public MigrationException { public: InvalidSchemaVersionException(uint64_t old_version, uint64_t new_version); - uint64_t old_version() { return m_old_version; } - uint64_t new_version() { return m_new_version; } + uint64_t old_version() const { return m_old_version; } + uint64_t new_version() const { return m_new_version; } private: uint64_t m_old_version, m_new_version; }; @@ -135,8 +138,8 @@ namespace realm { class DuplicatePrimaryKeyValueException : public MigrationException { public: DuplicatePrimaryKeyValueException(std::string object_type, Property const& property); - std::string object_type() { return m_object_type; } - Property const& property() { return m_property; } + std::string object_type() const { return m_object_type; } + Property const& property() const { return m_property; } private: std::string m_object_type; Property m_property; @@ -156,7 +159,7 @@ namespace realm { ObjectSchemaValidationException(std::string object_type) : m_object_type(object_type) {} ObjectSchemaValidationException(std::string object_type, std::string message) : m_object_type(object_type) { m_what = message; } - std::string object_type() { return m_object_type; } + std::string object_type() const { return m_object_type; } protected: std::string m_object_type; }; @@ -165,7 +168,7 @@ namespace realm { public: ObjectSchemaPropertyException(std::string object_type, Property const& property) : ObjectSchemaValidationException(object_type), m_property(property) {} - Property const& property() { return m_property; } + Property const& property() const { return m_property; } private: Property m_property; }; @@ -203,8 +206,8 @@ namespace realm { class MismatchedPropertiesException : public ObjectSchemaValidationException { public: MismatchedPropertiesException(std::string object_type, Property const& old_property, Property const& new_property); - Property const& old_property() { return m_old_property; } - Property const& new_property() { return m_new_property; } + Property const& old_property() const { return m_old_property; } + Property const& new_property() const { return m_new_property; } private: Property m_old_property, m_new_property; }; @@ -212,8 +215,8 @@ namespace realm { class ChangedPrimaryKeyException : public ObjectSchemaValidationException { public: ChangedPrimaryKeyException(std::string object_type, std::string old_primary, std::string new_primary); - std::string old_primary() { return m_old_primary; } - std::string new_primary() { return m_new_primary; } + std::string old_primary() const { return m_old_primary; } + std::string new_primary() const { return m_new_primary; } private: std::string m_old_primary, m_new_primary; }; @@ -221,11 +224,10 @@ namespace realm { class InvalidPrimaryKeyException : public ObjectSchemaValidationException { public: InvalidPrimaryKeyException(std::string object_type, std::string primary_key); - std::string primary_key() { return m_primary_key; } + std::string primary_key() const { return m_primary_key; } private: std::string m_primary_key; }; } #endif /* defined(REALM_OBJECT_STORE_HPP) */ - diff --git a/shared_realm.cpp b/shared_realm.cpp index 8b1abd84..68972e4d 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -221,7 +221,7 @@ static void check_read_write(Realm *realm) } } -void Realm::verify_thread() +void Realm::verify_thread() const { if (m_thread_id != std::this_thread::get_id()) { throw IncorrectThreadException("Realm accessed from incorrect thread."); diff --git a/shared_realm.hpp b/shared_realm.hpp index 3836c5e9..e1c1a0c6 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -84,18 +84,18 @@ namespace realm { void begin_transaction(); void commit_transaction(); void cancel_transaction(); - bool is_in_transaction() { return m_in_transaction; } + bool is_in_transaction() const { return m_in_transaction; } bool refresh(); void set_auto_refresh(bool auto_refresh) { m_auto_refresh = auto_refresh; } - bool auto_refresh() { return m_auto_refresh; } + bool auto_refresh() const { return m_auto_refresh; } void notify(); void invalidate(); bool compact(); std::thread::id thread_id() const { return m_thread_id; } - void verify_thread(); + void verify_thread() const; ~Realm(); From 7de20ea3a6b4d34b52b37372f783a8eddbe0e468 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Sep 2015 09:53:51 -0700 Subject: [PATCH 26/35] USe more const refs to avoid copies --- object_store.cpp | 26 +++++++++++++------------- object_store.hpp | 40 ++++++++++++++++++++-------------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/object_store.cpp b/object_store.cpp index 9e31a1c7..8d2701fd 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -19,8 +19,8 @@ #include "object_store.hpp" #include -#include #include +#include #include #include @@ -479,41 +479,41 @@ InvalidSchemaVersionException::InvalidSchemaVersionException(uint64_t old_versio m_what = "Provided schema version " + std::to_string(old_version) + " is less than last set version " + std::to_string(new_version) + "."; } -DuplicatePrimaryKeyValueException::DuplicatePrimaryKeyValueException(std::string object_type, Property const& property) : +DuplicatePrimaryKeyValueException::DuplicatePrimaryKeyValueException(std::string const& object_type, Property const& property) : m_object_type(object_type), m_property(property) { m_what = "Primary key property '" + property.name + "' has duplicate values after migration."; }; -SchemaValidationException::SchemaValidationException(std::vector errors) : +SchemaValidationException::SchemaValidationException(std::vector const& errors) : m_validation_errors(errors) { m_what ="Migration is required due to the following errors: "; - for (auto error : errors) { + for (auto const& error : errors) { m_what += std::string("\n- ") + error.what(); } } -PropertyTypeNotIndexableException::PropertyTypeNotIndexableException(std::string object_type, Property const& property) : +PropertyTypeNotIndexableException::PropertyTypeNotIndexableException(std::string const& object_type, Property const& property) : ObjectSchemaPropertyException(object_type, property) { m_what = "Can't index property " + object_type + "." + property.name + ": indexing a property of type '" + string_for_property_type(property.type) + "' is currently not supported"; } -ExtraPropertyException::ExtraPropertyException(std::string object_type, Property const& property) : +ExtraPropertyException::ExtraPropertyException(std::string const& object_type, Property const& property) : ObjectSchemaPropertyException(object_type, property) { m_what = "Property '" + property.name + "' has been added to latest object model."; } -MissingPropertyException::MissingPropertyException(std::string object_type, Property const& property) : +MissingPropertyException::MissingPropertyException(std::string const& object_type, Property const& property) : ObjectSchemaPropertyException(object_type, property) { m_what = "Property '" + property.name + "' is missing from latest object model."; } -InvalidNullabilityException::InvalidNullabilityException(std::string object_type, Property const& property) : +InvalidNullabilityException::InvalidNullabilityException(std::string const& object_type, Property const& property) : ObjectSchemaPropertyException(object_type, property) { #if REALM_NULL_STRINGS == 1 @@ -532,13 +532,13 @@ InvalidNullabilityException::InvalidNullabilityException(std::string object_type #endif } -MissingObjectTypeException::MissingObjectTypeException(std::string object_type, Property const& property) : +MissingObjectTypeException::MissingObjectTypeException(std::string const& object_type, Property const& property) : ObjectSchemaPropertyException(object_type, property) { m_what = "Target type '" + property.object_type + "' doesn't exist for property '" + property.name + "'."; } -MismatchedPropertiesException::MismatchedPropertiesException(std::string object_type, Property const& old_property, Property const& new_property) : +MismatchedPropertiesException::MismatchedPropertiesException(std::string const& object_type, Property const& old_property, Property const& new_property) : ObjectSchemaValidationException(object_type), m_old_property(old_property), m_new_property(new_property) { if (new_property.type != old_property.type) { @@ -553,7 +553,7 @@ MismatchedPropertiesException::MismatchedPropertiesException(std::string object_ } } -ChangedPrimaryKeyException::ChangedPrimaryKeyException(std::string object_type, std::string old_primary, std::string new_primary) : ObjectSchemaValidationException(object_type), m_old_primary(old_primary), m_new_primary(new_primary) +ChangedPrimaryKeyException::ChangedPrimaryKeyException(std::string const& object_type, std::string const& old_primary, std::string const& new_primary) : ObjectSchemaValidationException(object_type), m_old_primary(old_primary), m_new_primary(new_primary) { if (old_primary.size()) { m_what = "Property '" + old_primary + "' is no longer a primary key."; @@ -563,13 +563,13 @@ ChangedPrimaryKeyException::ChangedPrimaryKeyException(std::string object_type, } } -InvalidPrimaryKeyException::InvalidPrimaryKeyException(std::string object_type, std::string primary) : +InvalidPrimaryKeyException::InvalidPrimaryKeyException(std::string const& object_type, std::string const& primary) : ObjectSchemaValidationException(object_type), m_primary_key(primary) { m_what = "Specified primary key property '" + primary + "' does not exist."; } -DuplicatePrimaryKeysException::DuplicatePrimaryKeysException(std::string object_type) : ObjectSchemaValidationException(object_type) +DuplicatePrimaryKeysException::DuplicatePrimaryKeysException(std::string const& object_type) : ObjectSchemaValidationException(object_type) { m_what = "Duplicate primary keys for object '" + object_type + "'."; } diff --git a/object_store.hpp b/object_store.hpp index 89a7108e..9b260df8 100644 --- a/object_store.hpp +++ b/object_store.hpp @@ -19,14 +19,14 @@ #ifndef REALM_OBJECT_STORE_HPP #define REALM_OBJECT_STORE_HPP -#include -#include -#include -#include - #include "object_schema.hpp" #include "property.hpp" +#include + +#include +#include + namespace realm { class ObjectSchemaValidationException; using Schema = std::vector; @@ -137,7 +137,7 @@ namespace realm { class DuplicatePrimaryKeyValueException : public MigrationException { public: - DuplicatePrimaryKeyValueException(std::string object_type, Property const& property); + DuplicatePrimaryKeyValueException(std::string const& object_type, Property const& property); std::string object_type() const { return m_object_type; } Property const& property() const { return m_property; } private: @@ -148,16 +148,16 @@ namespace realm { // Schema validation exceptions class SchemaValidationException : public ObjectStoreException { public: - SchemaValidationException(std::vector errors); - std::vector &validation_errors() { return m_validation_errors; } + SchemaValidationException(std::vector const& errors); + std::vector const& validation_errors() const { return m_validation_errors; } private: std::vector m_validation_errors; }; class ObjectSchemaValidationException : public ObjectStoreException { public: - ObjectSchemaValidationException(std::string object_type) : m_object_type(object_type) {} - ObjectSchemaValidationException(std::string object_type, std::string message) : + ObjectSchemaValidationException(std::string const& object_type) : m_object_type(object_type) {} + ObjectSchemaValidationException(std::string const& object_type, std::string const& message) : m_object_type(object_type) { m_what = message; } std::string object_type() const { return m_object_type; } protected: @@ -166,7 +166,7 @@ namespace realm { class ObjectSchemaPropertyException : public ObjectSchemaValidationException { public: - ObjectSchemaPropertyException(std::string object_type, Property const& property) : + ObjectSchemaPropertyException(std::string const& object_type, Property const& property) : ObjectSchemaValidationException(object_type), m_property(property) {} Property const& property() const { return m_property; } private: @@ -175,37 +175,37 @@ namespace realm { class PropertyTypeNotIndexableException : public ObjectSchemaPropertyException { public: - PropertyTypeNotIndexableException(std::string object_type, Property const& property); + PropertyTypeNotIndexableException(std::string const& object_type, Property const& property); }; class ExtraPropertyException : public ObjectSchemaPropertyException { public: - ExtraPropertyException(std::string object_type, Property const& property); + ExtraPropertyException(std::string const& object_type, Property const& property); }; class MissingPropertyException : public ObjectSchemaPropertyException { public: - MissingPropertyException(std::string object_type, Property const& property); + MissingPropertyException(std::string const& object_type, Property const& property); }; class InvalidNullabilityException : public ObjectSchemaPropertyException { public: - InvalidNullabilityException(std::string object_type, Property const& property); + InvalidNullabilityException(std::string const& object_type, Property const& property); }; class MissingObjectTypeException : public ObjectSchemaPropertyException { public: - MissingObjectTypeException(std::string object_type, Property const& property); + MissingObjectTypeException(std::string const& object_type, Property const& property); }; class DuplicatePrimaryKeysException : public ObjectSchemaValidationException { public: - DuplicatePrimaryKeysException(std::string object_type); + DuplicatePrimaryKeysException(std::string const& object_type); }; class MismatchedPropertiesException : public ObjectSchemaValidationException { public: - MismatchedPropertiesException(std::string object_type, Property const& old_property, Property const& new_property); + MismatchedPropertiesException(std::string const& object_type, Property const& old_property, Property const& new_property); Property const& old_property() const { return m_old_property; } Property const& new_property() const { return m_new_property; } private: @@ -214,7 +214,7 @@ namespace realm { class ChangedPrimaryKeyException : public ObjectSchemaValidationException { public: - ChangedPrimaryKeyException(std::string object_type, std::string old_primary, std::string new_primary); + ChangedPrimaryKeyException(std::string const& object_type, std::string const& old_primary, std::string const& new_primary); std::string old_primary() const { return m_old_primary; } std::string new_primary() const { return m_new_primary; } private: @@ -223,7 +223,7 @@ namespace realm { class InvalidPrimaryKeyException : public ObjectSchemaValidationException { public: - InvalidPrimaryKeyException(std::string object_type, std::string primary_key); + InvalidPrimaryKeyException(std::string const& object_type, std::string const& primary_key); std::string primary_key() const { return m_primary_key; } private: std::string m_primary_key; From ba278c5d4f1f6fa7e72fae16948b6b09f588f3db Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Sep 2015 10:02:29 -0700 Subject: [PATCH 27/35] Fix checks for what types of columns can be optional --- object_store.cpp | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/object_store.cpp b/object_store.cpp index 8d2701fd..2745f80e 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -204,22 +204,18 @@ std::vector ObjectStore::verify_object_schema(O } // check nullablity + if (current_prop.is_nullable) { #if REALM_NULL_STRINGS == 1 - if (current_prop.type == PropertyTypeArray && current_prop.is_nullable) { + if (current_prop.type == PropertyTypeArray || current_prop.type == PropertyTypeAny) { +#else + if (current_prop.type != PropertyTypeObject) { +#endif + exceptions.emplace_back(InvalidNullabilityException(table_schema.name, current_prop)); + } + } + else if (current_prop.type == PropertyTypeObject) { exceptions.emplace_back(InvalidNullabilityException(table_schema.name, current_prop)); } -#else - if (current_prop.type == PropertyTypeObject) { - if (!current_prop.is_nullable) { - exceptions.emplace_back(InvalidNullabilityException(table_schema.name, current_prop)); - } - } - else { - if (current_prop.is_nullable) { - exceptions.emplace_back(InvalidNullabilityException(table_schema.name, current_prop)); - } - } -#endif // check primary keys if (current_prop.is_primary) { @@ -516,20 +512,16 @@ MissingPropertyException::MissingPropertyException(std::string const& object_typ InvalidNullabilityException::InvalidNullabilityException(std::string const& object_type, Property const& property) : ObjectSchemaPropertyException(object_type, property) { -#if REALM_NULL_STRINGS == 1 - m_what = "'Array' property '" + property.name + "' cannot be nullable"; -#else if (property.type == PropertyTypeObject) { - if (!property.is_nullable) { - m_what = "'Object' property '" + property.name + "' must be nullable."; - } + m_what = "'Object' property '" + property.name + "' must be nullable."; } else { - if (property.is_nullable) { - m_what = "Only 'Object' property types are nullable"; - } - } +#if REALM_NULL_STRINGS == 1 + m_what = "Array or Mixed property '" + property.name + "' cannot be nullable"; +#else + m_what = "Only 'Object' property types are nullable"; #endif + } } MissingObjectTypeException::MissingObjectTypeException(std::string const& object_type, Property const& property) : From 0c111a2035fccef3ba6b7ba1c2fbefdbeb08974b Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Sep 2015 10:10:18 -0700 Subject: [PATCH 28/35] Fix a comment --- object_store.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/object_store.cpp b/object_store.cpp index 2745f80e..1c7c7071 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -295,7 +295,7 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update for (auto& target_prop : target_props) { auto current_prop = current_schema.property_for_name(target_prop.name); - // add any new properties (new name or different type) + // add any new properties (no old column or old column was removed due to not matching) if (!current_prop || current_prop->table_column == npos) { switch (target_prop.type) { // for objects and arrays, we have to specify target table From 6133eebf8bb13caf0fec80b3c6322d1753b706b9 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Sep 2015 11:30:00 -0700 Subject: [PATCH 29/35] Reduce the scope of a variable --- shared_realm.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shared_realm.cpp b/shared_realm.cpp index 68972e4d..93e1aa13 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -355,8 +355,7 @@ bool Realm::refresh() uint64_t Realm::get_schema_version(const realm::Realm::Config &config) { - auto existing_realm = s_global_cache.get_any_realm(config.path); - if (existing_realm) { + if (auto existing_realm = s_global_cache.get_any_realm(config.path)) { return existing_realm->config().schema_version; } From b381437a454a18a32b5122ba7be2719a902b8b6f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Sep 2015 11:34:02 -0700 Subject: [PATCH 30/35] Make Realm::compact() more robust Throw if it's called on a read-only Realm and ensure the Realm is left in a valid state regardless of the starting state. --- shared_realm.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/shared_realm.cpp b/shared_realm.cpp index 93e1aa13..ab72b638 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -291,20 +291,21 @@ bool Realm::compact() { verify_thread(); - bool success = false; + if (m_config.read_only) { + throw InvalidTransactionException("Can't compact a read-only Realm"); + } if (m_in_transaction) { throw InvalidTransactionException("Can't compact a Realm within a write transaction"); } + Group* group = read_group(); for (auto &object_schema : *m_config.schema) { - ObjectStore::table_for_object_type(read_group(), object_schema.name)->optimize(); + ObjectStore::table_for_object_type(group, object_schema.name)->optimize(); } - m_shared_group->end_read(); - success = m_shared_group->compact(); - m_shared_group->begin_read(); + m_group = nullptr; - return success; + return m_shared_group->compact(); } void Realm::notify() From a0f1dab71b7ce5ddf837245d7e6758b8eff1f40a Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Sep 2015 14:03:58 -0700 Subject: [PATCH 31/35] Add a Schema class, move lookup by name and internal-consistency checks there --- object_store.cpp | 65 ++++------------------------- object_store.hpp | 5 +-- schema.cpp | 106 +++++++++++++++++++++++++++++++++++++++++++++++ schema.hpp | 55 ++++++++++++++++++++++++ shared_realm.cpp | 6 +++ shared_realm.hpp | 1 + 6 files changed, 179 insertions(+), 59 deletions(-) create mode 100644 schema.cpp create mode 100644 schema.hpp diff --git a/object_store.cpp b/object_store.cpp index 1c7c7071..38ed3d4b 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -18,6 +18,8 @@ #include "object_store.hpp" +#include "schema.hpp" + #include #include #include @@ -147,17 +149,11 @@ static inline bool property_has_changed(Property const& p1, Property const& p2) || p1.is_nullable != p2.is_nullable; } -static bool compare_by_name(ObjectSchema const& lft, ObjectSchema const& rgt) { - return lft.name < rgt.name; -} - void ObjectStore::verify_schema(Schema const& actual_schema, Schema& target_schema, bool allow_missing_tables) { - std::sort(begin(target_schema), end(target_schema), compare_by_name); - std::vector errors; for (auto &object_schema : target_schema) { - auto matching_schema = std::lower_bound(begin(actual_schema), end(actual_schema), object_schema, compare_by_name); - if (matching_schema == end(actual_schema) || matching_schema->name != object_schema.name) { + auto matching_schema = actual_schema.find(object_schema); + if (matching_schema == actual_schema.end()) { if (!allow_missing_tables) { errors.emplace_back(ObjectSchemaValidationException(object_schema.name, "Missing table for object type '" + object_schema.name + "'.")); @@ -165,7 +161,7 @@ void ObjectStore::verify_schema(Schema const& actual_schema, Schema& target_sche continue; } - auto more_errors = verify_object_schema(*matching_schema, object_schema, target_schema); + auto more_errors = verify_object_schema(*matching_schema, object_schema); errors.insert(errors.end(), more_errors.begin(), more_errors.end()); } if (errors.size()) { @@ -174,18 +170,10 @@ void ObjectStore::verify_schema(Schema const& actual_schema, Schema& target_sche } std::vector ObjectStore::verify_object_schema(ObjectSchema const& table_schema, - ObjectSchema& target_schema, - Schema const& schema) { + ObjectSchema& target_schema) { std::vector exceptions; - ObjectSchema cmp; - auto schema_contains_table = [&](std::string const& name) { - cmp.name = name; - return std::binary_search(begin(schema), end(schema), cmp, compare_by_name); - }; - // check to see if properties are the same - const Property *primary = nullptr; for (auto& current_prop : table_schema.properties) { auto target_prop = target_schema.property_for_name(current_prop.name); @@ -198,40 +186,6 @@ std::vector ObjectStore::verify_object_schema(O continue; } - // check object_type existence - if (current_prop.object_type.length() && !schema_contains_table(current_prop.object_type)) { - exceptions.emplace_back(MissingObjectTypeException(table_schema.name, current_prop)); - } - - // check nullablity - if (current_prop.is_nullable) { -#if REALM_NULL_STRINGS == 1 - if (current_prop.type == PropertyTypeArray || current_prop.type == PropertyTypeAny) { -#else - if (current_prop.type != PropertyTypeObject) { -#endif - exceptions.emplace_back(InvalidNullabilityException(table_schema.name, current_prop)); - } - } - else if (current_prop.type == PropertyTypeObject) { - exceptions.emplace_back(InvalidNullabilityException(table_schema.name, current_prop)); - } - - // check primary keys - if (current_prop.is_primary) { - if (primary) { - exceptions.emplace_back(DuplicatePrimaryKeysException(table_schema.name)); - } - primary = ¤t_prop; - } - - // check indexable - if (current_prop.is_indexed) { - if (current_prop.type != PropertyTypeString && current_prop.type != PropertyTypeInt) { - exceptions.emplace_back(PropertyTypeNotIndexableException(table_schema.name, current_prop)); - } - } - // create new property with aligned column target_prop->table_column = current_prop.table_column; } @@ -346,8 +300,8 @@ bool ObjectStore::is_schema_at_version(const Group *group, uint64_t version) { bool ObjectStore::needs_update(Schema const& old_schema, Schema const& schema) { for (auto const& target_schema : schema) { - auto matching_schema = std::lower_bound(begin(old_schema), end(old_schema), target_schema, compare_by_name); - if (matching_schema == end(old_schema) || matching_schema->name != target_schema.name) { + auto matching_schema = old_schema.find(target_schema); + if (matching_schema == end(old_schema)) { // Table doesn't exist return true; } @@ -404,14 +358,13 @@ bool ObjectStore::update_realm_with_schema(Group *group, Schema const& old_schem } Schema ObjectStore::schema_from_group(const Group *group) { - Schema schema; + std::vector schema; for (size_t i = 0; i < group->size(); i++) { std::string object_type = object_type_for_table_name(group->get_table_name(i)); if (object_type.length()) { schema.emplace_back(group, object_type); } } - std::sort(begin(schema), end(schema), compare_by_name); return schema; } diff --git a/object_store.hpp b/object_store.hpp index 9b260df8..f38524f8 100644 --- a/object_store.hpp +++ b/object_store.hpp @@ -29,7 +29,7 @@ namespace realm { class ObjectSchemaValidationException; - using Schema = std::vector; + class Schema; class ObjectStore { public: @@ -90,8 +90,7 @@ namespace realm { // updates the column mapping on the target_schema // returns array of validation errors static std::vector verify_object_schema(ObjectSchema const& expected, - ObjectSchema &target_schema, - Schema const& schema); + ObjectSchema &target_schema); // get primary key property name for object type static StringData get_primary_key_for_object(const Group *group, StringData object_type); diff --git a/schema.cpp b/schema.cpp new file mode 100644 index 00000000..bf7a6181 --- /dev/null +++ b/schema.cpp @@ -0,0 +1,106 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "schema.hpp" + +#include "object_schema.hpp" +#include "object_store.hpp" +#include "property.hpp" + +using namespace realm; + +static bool compare_by_name(ObjectSchema const& lft, ObjectSchema const& rgt) { + return lft.name < rgt.name; +} + +Schema::Schema(base types) : base(std::move(types)) { + std::sort(begin(), end(), compare_by_name); +} + +Schema::iterator Schema::find(std::string const& name) +{ + ObjectSchema cmp; + cmp.name = name; + return find(cmp); +} + +Schema::const_iterator Schema::find(std::string const& name) const +{ + return const_cast(this)->find(name); +} + +Schema::iterator Schema::find(ObjectSchema const& object) noexcept +{ + auto it = std::lower_bound(begin(), end(), object, compare_by_name); + if (it != end() && it->name != object.name) { + it = end(); + } + return it; +} + +Schema::const_iterator Schema::find(ObjectSchema const& object) const noexcept +{ + return const_cast(this)->find(object); +} + +void Schema::validate() const +{ + std::vector exceptions; + for (auto const& object : *this) { + const Property *primary = nullptr; + for (auto const& prop : object.properties) { + // check object_type existence + if (!prop.object_type.empty() && find(prop.object_type) == end()) { + exceptions.emplace_back(MissingObjectTypeException(object.name, prop)); + } + + // check nullablity + if (prop.is_nullable) { +#if REALM_NULL_STRINGS == 1 + if (prop.type == PropertyTypeArray || prop.type == PropertyTypeAny) { +#else + if (prop.type != PropertyTypeObject) { +#endif + exceptions.emplace_back(InvalidNullabilityException(object.name, prop)); + } + } + else if (prop.type == PropertyTypeObject) { + exceptions.emplace_back(InvalidNullabilityException(object.name, prop)); + } + + // check primary keys + if (prop.is_primary) { + if (primary) { + exceptions.emplace_back(DuplicatePrimaryKeysException(object.name)); + } + primary = ∝ + } + + // check indexable + if (prop.is_indexed) { + if (prop.type != PropertyTypeString && prop.type != PropertyTypeInt) { + exceptions.emplace_back(PropertyTypeNotIndexableException(object.name, prop)); + } + } + } + } + + if (exceptions.size()) { + throw SchemaValidationException(exceptions); + } +} diff --git a/schema.hpp b/schema.hpp new file mode 100644 index 00000000..0a7fa425 --- /dev/null +++ b/schema.hpp @@ -0,0 +1,55 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_SCHEMA_HPP +#define REALM_SCHEMA_HPP + +#include + +namespace realm { +class ObjectSchema; + +class Schema : private std::vector { +private: + using base = std::vector; +public: + // Create a schema from a vector of ObjectSchema + Schema(base types); + + // find an ObjectSchema by name + iterator find(std::string const& name); + const_iterator find(std::string const& name) const; + + // find an ObjectSchema with the same name as the passed in one + iterator find(ObjectSchema const& object) noexcept; + const_iterator find(ObjectSchema const& object) const noexcept; + + // Verify that this schema is internally consistent (i.e. all properties are + // valid, links link to types that actually exist, etc.) + void validate() const; + + using base::iterator; + using base::const_iterator; + using base::begin; + using base::end; + using base::empty; + using base::size; +}; +} + +#endif /* defined(REALM_SCHEMA_HPP) */ \ No newline at end of file diff --git a/shared_realm.cpp b/shared_realm.cpp index ab72b638..55707918 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -20,6 +20,7 @@ #include "external_commit_helper.hpp" #include "realm_delegate.hpp" +#include "schema.hpp" #include "transact_log_handler.hpp" #include @@ -45,6 +46,8 @@ Realm::Config::Config(const Config& c) } } +Realm::Config::~Config() = default; + Realm::Config& Realm::Config::operator=(realm::Realm::Config const& c) { if (&c != this) { @@ -154,6 +157,7 @@ SharedRealm Realm::get_shared_realm(Config config) if (realm->m_config.schema_version == ObjectStore::NotVersioned) { throw UnitializedRealmException("Can't open an un-initialized Realm without a Schema"); } + target_schema->validate(); ObjectStore::verify_schema(*realm->m_config.schema, *target_schema, true); realm->m_config.schema = std::move(target_schema); } @@ -171,6 +175,8 @@ SharedRealm Realm::get_shared_realm(Config config) bool Realm::update_schema(std::unique_ptr schema, uint64_t version) { + schema->validate(); + bool needs_update = !m_config.read_only && (m_config.schema_version != version || ObjectStore::needs_update(*m_config.schema, *schema)); if (!needs_update) { ObjectStore::verify_schema(*m_config.schema, *schema, m_config.read_only); diff --git a/shared_realm.hpp b/shared_realm.hpp index e1c1a0c6..aa19071e 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -56,6 +56,7 @@ namespace realm { Config() = default; Config(Config&&) = default; Config(const Config& c); + ~Config(); Config& operator=(Config const&); Config& operator=(Config&&) = default; From b7936bb0476a3a993f5477aea5577739a95fa26b Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Sep 2015 14:24:11 -0700 Subject: [PATCH 32/35] Simplify column shifting for removed properties a little --- object_store.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/object_store.cpp b/object_store.cpp index 38ed3d4b..035ce4fc 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -233,16 +233,15 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update // remove extra columns size_t deleted = 0; for (auto& current_prop : current_schema.properties) { + current_prop.table_column -= deleted; + auto target_prop = target_object_schema->property_for_name(current_prop.name); if (!target_prop || property_has_changed(current_prop, *target_prop)) { - table->remove_column(current_prop.table_column - deleted); + table->remove_column(current_prop.table_column); ++deleted; current_prop.table_column = npos; changed = true; } - else { - current_prop.table_column -= deleted; - } } // add missing columns From 95c80c98e63b18d6eae8b0f41c8dba5473e7537f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Sep 2015 14:54:41 -0700 Subject: [PATCH 33/35] Fix error in cleanup after an error during a migration --- shared_realm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared_realm.cpp b/shared_realm.cpp index 55707918..9fe02a0a 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -215,7 +215,7 @@ bool Realm::update_schema(std::unique_ptr schema, uint64_t version) cancel_transaction(); } m_config.schema_version = old_config.schema_version; - m_config.schema = std::move(m_config.schema); + m_config.schema = std::move(old_config.schema); throw; } } From 21d32bf581446e3cfd2dac4ae5c514bb347e66be Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 10 Sep 2015 13:47:06 -0700 Subject: [PATCH 34/35] Add a bit of documentation for RealmDelegate --- realm_delegate.hpp | 122 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 92 insertions(+), 30 deletions(-) diff --git a/realm_delegate.hpp b/realm_delegate.hpp index 0941b2dc..0d9e9dfe 100644 --- a/realm_delegate.hpp +++ b/realm_delegate.hpp @@ -25,24 +25,106 @@ #include namespace realm { +// RealmDelegate is the extension point for adding binding-specific behavior to +// a SharedRealm. It can be used to store additonal data associated with the +// Realm which is needed by the binding, and there are several methods which +// can be overridden to receive notifications of state changes within the Realm. +// +// A simple delegate implementation which lets the user register functions to be +// called on refresh could look like the following: +// +// class DelegateImplementation : public RealmDelegate { +// public: +// // A token returned from add_notification that can be used to remove the +// // notification later +// struct token : private std::list>::iterator { +// token(std::list>::iterator it) : std::list>::iterator(it) { } +// friend class DelegateImplementation; +// }; +// +// token add_notification(std::function func) +// { +// m_registered_notifications.push_back(std::move(func)); +// return token(std::prev(m_registered_notifications.end())); +// } +// +// void remove_notification(token entry) +// { +// m_registered_notifications.erase(entry); +// } +// +// // Override the did_change method to call each registered notification +// void did_change(std::vector const&, std::vector const&) override +// { +// // Loop oddly so that unregistering a notification from within the +// // registered function works +// for (auto it = m_registered_notifications.begin(); it != m_registered_notifications.end(); ) { +// (*it++)(); +// } +// } +// +// private: +// std::list> m_registered_notifications; +// }; class RealmDelegate { public: virtual ~RealmDelegate() = default; + // Called by the Realm when a write transaction is committed to the file by + // a different Realm instance (possibly in a different process) + virtual void changes_available() { } + + struct ObserverState; + + // Override this function if you want to recieve detailed information about + // external changes to a specific set of objects. + // This is called before each operation which may advance the read + // transaction to include + // ObserverStates for each row for which detailed change information is + // desired. + virtual std::vector get_observed_rows() { return {}; } + + // Called immediately before the read transaction is advanced if detailed + // change information was requested (by returning a non-empty array from + // get_observed_rows()). + // The observers vector is the vector returned by get_observed_row(), + // updated with change information. The invalidated vector is a list of the + // `info` fields of observed rows which will be deleted. + virtual void will_change(std::vector const& observers, + std::vector const& invalidated); + + // Called immediately after the read transaction version is advanced. Unlike + // will_change(), this is called even if detailed change information was not + // requested or if the Realm is not actually in a read transactuib, although + // both vectors will be empty in that case. + virtual void did_change(std::vector const& observers, + std::vector const& invalidated); + // Change information for a single field of a row struct ColumnInfo { + // Did this column change? bool changed = false; + // For LinkList columns, what kind of change occurred? + // Always None for other column types enum class Kind { - None, - Set, - Insert, - Remove, - SetAll + None, // No change + Set, // The entries at `indices` were assigned to + Insert, // New values were inserted at each of the indices given + Remove, // Values were removed at each of the indices given + SetAll // The entire LinkList has been replaced with a new set of values } kind = Kind::None; + // The indices where things happened for Set, Insert and Remove + // Not used for None and SetAll IndexSet indices; }; // Information about an observed row in a table + // + // Each object which needs detailed change information should have an + // ObserverState entry in the vector returned from get_observed_rows(), with + // the initial table and row indexes set (and optionally the info field). + // The Realm parses the transaction log, and populates the `changes` vector + // in each ObserverState with information about what changes were made. struct ObserverState { // Initial table and row which is observed // May be updated by row insertions and removals @@ -58,35 +140,15 @@ public: std::vector changes; // Simple lexographic ordering - friend bool operator<(ObserverState const& lft, ObserverState const& rgt) { + friend bool operator<(ObserverState const& lft, ObserverState const& rgt) + { return std::tie(lft.table_ndx, lft.row_ndx) < std::tie(rgt.table_ndx, rgt.row_ndx); } }; - - // There are now new versions available for the Realm, but it has not - // had its read version advanced - virtual void changes_available() = 0; - - // Called before changing the read transaction. Should return a list of - // ObserverStates for each row for which detailed change information is - // desired. - virtual std::vector get_observed_rows() = 0; - - // The Realm's read version will change - // Only called if get_observed_row() returned a non-empty array. - // observers is the vector returned from get_observed_rows() - // invalidated is the `info` pointers for each observed object which was deleted - virtual void will_change(std::vector const& observers, - std::vector const& invalidated) = 0; - - // The Realm's read version has changed - // observers is the vector returned from get_observed_rows() - // invalidated is the `info` pointers for each observed object which was deleted - // Is called with empty change information following a local commit or if - // the Realm isn't in a read transaction yet - virtual void did_change(std::vector const& observers, - std::vector const& invalidated) = 0; }; + +inline void RealmDelegate::will_change(std::vector const&, std::vector const&) { } +inline void RealmDelegate::did_change(std::vector const&, std::vector const&) { } } // namespace realm #endif /* REALM_DELEGATE_HPP */ From f6467778e058b20b2c197036e2368fed5b3dc5b6 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 10 Sep 2015 13:49:50 -0700 Subject: [PATCH 35/35] Send changes_available() even if autorefresh is enabled --- shared_realm.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shared_realm.cpp b/shared_realm.cpp index 9fe02a0a..8b003d30 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -319,6 +319,9 @@ void Realm::notify() verify_thread(); if (m_shared_group->has_changed()) { // Throws + if (m_delegate) { + m_delegate->changes_available(); + } if (m_auto_refresh) { if (m_group) { transaction::advance(*m_shared_group, *m_history, m_delegate.get()); @@ -327,9 +330,6 @@ void Realm::notify() m_delegate->did_change({}, {}); } } - else if (m_delegate) { - m_delegate->changes_available(); - } } }