From b91c69a3b508edbc848e23115c3058f95d7e9403 Mon Sep 17 00:00:00 2001 From: Ari Lazier Date: Fri, 13 Nov 2015 17:34:33 -0800 Subject: [PATCH] Squashed 'src/object-store/' changes from 043f5ff..7701ba1 7701ba1 Merge pull request #16 from realm/tg-close 883ef12 Add Realm::close() and call it in RealmCache::clear() e9ca54e Merge pull request #12 from realm/tg-core-0.94.4 0823a62 Merge pull request #13 from realm/tg-is-empty 62f59d9 Merge pull request #15 from realm/tg-file-error-path b93e5ce Include the path of the file which actually failed to open in exceptions e1e9cd8 Add ObjectStore::is_empty() 52e7e61 Update for core 0.94.4 271432b Merge pull request #11 from realm/kd-rename-delegate-to-bindingcontext db36ca5 Remove Realm suffix 70e1967 Rename realm delegate in transact_log_handler as well 4973827 Rename RealmDelegate to RealmBindingContext 347145b Merge pull request #8 from realm/mar-migrate-required-to-optional 0b45772 Add a test showing our behavior when migrating from an optional column to a required column. 9f1702a Support migrating required columns to optional, preserving their contents. f5b790c Merge pull request #6 from realm/tg-impl 6dfeaf8 Move things which are not part of the API to an impl directory/namespace 513e483 Merge pull request #4 from realm/tg-realm f646777 Send changes_available() even if autorefresh is enabled 21d32bf Add a bit of documentation for RealmDelegate 95c80c9 Fix error in cleanup after an error during a migration b7936bb Simplify column shifting for removed properties a little a0f1dab Add a Schema class, move lookup by name and internal-consistency checks there b381437 Make Realm::compact() more robust 6133eeb Reduce the scope of a variable 0c111a2 Fix a comment ba278c5 Fix checks for what types of columns can be optional 7de20ea USe more const refs to avoid copies dbac77f Make a bunch of things const 1400450 Remove an unused function 06e0ff8 Share ExternalCommitHelpers between Realm instances for a single path f79dec9 Allow more nullable property types when supported 0eb0bd1 Honor is_nullable when creating columns ea5c475 Refactor schema initialization a bit e4f29fe Move the interprocess notification functionality to the object store b129ebe Shuffle stuff around and clean some things up eeb2ddd Improve array KVO performance a bit c3649fb Skip PK uniqueness checking when first creating a Realm file 0a41c85 Improve performance of realm_requires_update() and make more things const efdfa08 Port some of the KVO support functionality to the object store 65e1eb5 Add the ability to bypass the Realm cache entirely 3f226cf Rework change notifications 045c7b2 Add Realm::get_schema_version() e4377bb Change realm::Schema to a vector rather than a map cae4cf2 Remove property.hpp include from object_schema.hpp 55e6cca Convert RLMRealmConfiguration to a wrapper around Realm::Config 563a837 Use NSDMIs for realm::Property 0ae1bb1 Don't cache dynamic realms in the ObjectStore cache either 25a6734 Eliminate some copies 45890f2 Use NSDMIs for Realm 348f4a7 Reduce s_init_mutex's scope b4f856b Use NSDMIs for Realm::Config and make it moveable a91839b Store a copy of the encryption key 0700428 Merge pull request #3 from realm/al-bugfixes b084335 clear Realm cache between tests cb8364c property copy schema from cached realms 8712c8b fixes for latest object store changes 453e4d8 Fix crash when adding a property to a model without updating the schema version. git-subtree-dir: src/object-store git-subtree-split: 7701ba173d6c8d928a4f736a33c2850343b050e3 --- binding_context.hpp | 154 +++++++++++ impl/apple/external_commit_helper.cpp | 251 ++++++++++++++++++ impl/apple/external_commit_helper.hpp | 94 +++++++ impl/transact_log_handler.cpp | 356 ++++++++++++++++++++++++++ impl/transact_log_handler.hpp | 48 ++++ index_set.cpp | 92 +++++++ index_set.hpp | 62 +++++ object_schema.cpp | 20 +- object_schema.hpp | 16 +- object_store.cpp | 321 +++++++++++++---------- object_store.hpp | 123 +++++---- property.hpp | 11 +- schema.cpp | 102 ++++++++ schema.hpp | 55 ++++ shared_realm.cpp | 319 ++++++++++++++--------- shared_realm.hpp | 100 ++++---- 16 files changed, 1742 insertions(+), 382 deletions(-) create mode 100644 binding_context.hpp create mode 100644 impl/apple/external_commit_helper.cpp create mode 100644 impl/apple/external_commit_helper.hpp create mode 100644 impl/transact_log_handler.cpp create mode 100644 impl/transact_log_handler.hpp create mode 100644 index_set.cpp create mode 100644 index_set.hpp create mode 100644 schema.cpp create mode 100644 schema.hpp diff --git a/binding_context.hpp b/binding_context.hpp new file mode 100644 index 00000000..4aa6dd94 --- /dev/null +++ b/binding_context.hpp @@ -0,0 +1,154 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef BINDING_CONTEXT_HPP +#define BINDING_CONTEXT_HPP + +#include "index_set.hpp" + +#include +#include + +namespace realm { +// BindingContext is the extension point for adding binding-specific behavior to +// a SharedRealm. It can be used to store additonal data associated with the +// Realm which is needed by the binding, and there are several methods which +// can be overridden to receive notifications of state changes within the Realm. +// +// A simple implementation which lets the user register functions to be +// called on refresh could look like the following: +// +// class BindingContextImplementation : public BindingContext { +// public: +// // A token returned from add_notification that can be used to remove the +// // notification later +// struct token : private std::list>::iterator { +// token(std::list>::iterator it) : std::list>::iterator(it) { } +// friend class DelegateImplementation; +// }; +// +// token add_notification(std::function func) +// { +// m_registered_notifications.push_back(std::move(func)); +// return token(std::prev(m_registered_notifications.end())); +// } +// +// void remove_notification(token entry) +// { +// m_registered_notifications.erase(entry); +// } +// +// // Override the did_change method to call each registered notification +// void did_change(std::vector const&, std::vector const&) override +// { +// // Loop oddly so that unregistering a notification from within the +// // registered function works +// for (auto it = m_registered_notifications.begin(); it != m_registered_notifications.end(); ) { +// (*it++)(); +// } +// } +// +// private: +// std::list> m_registered_notifications; +// }; +class BindingContext { +public: + virtual ~BindingContext() = default; + + // Called by the Realm when a write transaction is committed to the file by + // a different Realm instance (possibly in a different process) + virtual void changes_available() { } + + struct ObserverState; + + // Override this function if you want to recieve detailed information about + // external changes to a specific set of objects. + // This is called before each operation which may advance the read + // transaction to include + // ObserverStates for each row for which detailed change information is + // desired. + virtual std::vector get_observed_rows() { return {}; } + + // Called immediately before the read transaction is advanced if detailed + // change information was requested (by returning a non-empty array from + // get_observed_rows()). + // The observers vector is the vector returned by get_observed_row(), + // updated with change information. The invalidated vector is a list of the + // `info` fields of observed rows which will be deleted. + virtual void will_change(std::vector const& observers, + std::vector const& invalidated); + + // Called immediately after the read transaction version is advanced. Unlike + // will_change(), this is called even if detailed change information was not + // requested or if the Realm is not actually in a read transactuib, although + // both vectors will be empty in that case. + virtual void did_change(std::vector const& observers, + std::vector const& invalidated); + + // Change information for a single field of a row + struct ColumnInfo { + // Did this column change? + bool changed = false; + // For LinkList columns, what kind of change occurred? + // Always None for other column types + enum class Kind { + None, // No change + Set, // The entries at `indices` were assigned to + Insert, // New values were inserted at each of the indices given + Remove, // Values were removed at each of the indices given + SetAll // The entire LinkList has been replaced with a new set of values + } kind = Kind::None; + // The indices where things happened for Set, Insert and Remove + // Not used for None and SetAll + IndexSet indices; + }; + + // Information about an observed row in a table + // + // Each object which needs detailed change information should have an + // ObserverState entry in the vector returned from get_observed_rows(), with + // the initial table and row indexes set (and optionally the info field). + // The Realm parses the transaction log, and populates the `changes` vector + // in each ObserverState with information about what changes were made. + struct ObserverState { + // Initial table and row which is observed + // May be updated by row insertions and removals + size_t table_ndx; + size_t row_ndx; + + // Opaque userdata for the delegate's use + void* info; + + // Populated with information about which columns were changed + // May be shorter than the actual number of columns if the later columns + // are not modified + std::vector changes; + + // Simple lexographic ordering + friend bool operator<(ObserverState const& lft, ObserverState const& rgt) + { + return std::tie(lft.table_ndx, lft.row_ndx) < std::tie(rgt.table_ndx, rgt.row_ndx); + } + }; +}; + +inline void BindingContext::will_change(std::vector const&, std::vector const&) { } +inline void BindingContext::did_change(std::vector const&, std::vector const&) { } +} // namespace realm + +#endif /* BINDING_CONTEXT_HPP */ diff --git a/impl/apple/external_commit_helper.cpp b/impl/apple/external_commit_helper.cpp new file mode 100644 index 00000000..4cf0b523 --- /dev/null +++ b/impl/apple/external_commit_helper.cpp @@ -0,0 +1,251 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "external_commit_helper.hpp" + +#include "shared_realm.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace realm; +using namespace realm::_impl; + +namespace { +// Write a byte to a pipe to notify anyone waiting for data on the pipe +void notify_fd(int fd) +{ + while (true) { + char c = 0; + ssize_t ret = write(fd, &c, 1); + if (ret == 1) { + break; + } + + // If the pipe's buffer is full, we need to read some of the old data in + // it to make space. We don't just read in the code waiting for + // notifications so that we can notify multiple waiters with a single + // write. + assert(ret == -1 && errno == EAGAIN); + char buff[1024]; + read(fd, buff, sizeof buff); + } +} +} // anonymous namespace + +void ExternalCommitHelper::FdHolder::close() +{ + if (m_fd != -1) { + ::close(m_fd); + } + m_fd = -1; + +} + +// Inter-thread and inter-process notifications of changes are done using a +// named pipe in the filesystem next to the Realm file. Everyone who wants to be +// notified of commits waits for data to become available on the pipe, and anyone +// who commits a write transaction writes data to the pipe after releasing the +// write lock. Note that no one ever actually *reads* from the pipe: the data +// actually written is meaningless, and trying to read from a pipe from multiple +// processes at once is fraught with race conditions. + +// When a RLMRealm instance is created, we add a CFRunLoopSource to the current +// thread's runloop. On each cycle of the run loop, the run loop checks each of +// its sources for work to do, which in the case of CFRunLoopSource is just +// checking if CFRunLoopSourceSignal has been called since the last time it ran, +// and if so invokes the function pointer supplied when the source is created, +// which in our case just invokes `[realm handleExternalChange]`. + +// Listening for external changes is done using kqueue() on a background thread. +// kqueue() lets us efficiently wait until the amount of data which can be read +// from one or more file descriptors has changed, and tells us which of the file +// descriptors it was that changed. We use this to wait on both the shared named +// pipe, and a local anonymous pipe. When data is written to the named pipe, we +// signal the runloop source and wake up the target runloop, and when data is +// written to the anonymous pipe the background thread removes the runloop +// source from the runloop and and shuts down. +ExternalCommitHelper::ExternalCommitHelper(Realm* realm) +{ + add_realm(realm); + + m_kq = kqueue(); + if (m_kq == -1) { + throw std::system_error(errno, std::system_category()); + } + + auto path = realm->config().path + ".note"; + + // Create and open the named pipe + int ret = mkfifo(path.c_str(), 0600); + if (ret == -1) { + int err = errno; + if (err == ENOTSUP) { + // Filesystem doesn't support named pipes, so try putting it in tmp instead + // Hash collisions are okay here because they just result in doing + // extra work, as opposed to correctness problems + std::ostringstream ss; + ss << getenv("TMPDIR"); + ss << "realm_" << std::hash()(path) << ".note"; + path = ss.str(); + ret = mkfifo(path.c_str(), 0600); + err = errno; + } + // the fifo already existing isn't an error + if (ret == -1 && err != EEXIST) { + throw std::system_error(err, std::system_category()); + } + } + + m_notify_fd = open(path.c_str(), O_RDWR); + if (m_notify_fd == -1) { + throw std::system_error(errno, std::system_category()); + } + + // Make writing to the pipe return -1 when the pipe's buffer is full + // rather than blocking until there's space available + ret = fcntl(m_notify_fd, F_SETFL, O_NONBLOCK); + if (ret == -1) { + throw std::system_error(errno, std::system_category()); + } + + // Create the anonymous pipe + int pipeFd[2]; + ret = pipe(pipeFd); + if (ret == -1) { + throw std::system_error(errno, std::system_category()); + } + + m_shutdown_read_fd = pipeFd[0]; + m_shutdown_write_fd = pipeFd[1]; + + // Use the minimum allowed stack size, as we need very little in our listener + // https://developer.apple.com/library/ios/documentation/Cocoa/Conceptual/Multithreading/CreatingThreads/CreatingThreads.html#//apple_ref/doc/uid/10000057i-CH15-SW7 + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 16 * 1024); + + auto fn = [](void *self) -> void * { + static_cast(self)->listen(); + return nullptr; + }; + ret = pthread_create(&m_thread, &attr, fn, this); + pthread_attr_destroy(&attr); + if (ret != 0) { + throw std::system_error(errno, std::system_category()); + } +} + +ExternalCommitHelper::~ExternalCommitHelper() +{ + REALM_ASSERT_DEBUG(m_realms.empty()); + notify_fd(m_shutdown_write_fd); + pthread_join(m_thread, nullptr); // Wait for the thread to exit +} + +void ExternalCommitHelper::add_realm(realm::Realm* realm) +{ + std::lock_guard lock(m_realms_mutex); + + // Create the runloop source + CFRunLoopSourceContext ctx{}; + ctx.info = realm; + ctx.perform = [](void* info) { + static_cast(info)->notify(); + }; + + CFRunLoopRef runloop = CFRunLoopGetCurrent(); + CFRetain(runloop); + CFRunLoopSourceRef signal = CFRunLoopSourceCreate(kCFAllocatorDefault, 0, &ctx); + CFRunLoopAddSource(runloop, signal, kCFRunLoopDefaultMode); + + m_realms.push_back({realm, runloop, signal}); +} + +void ExternalCommitHelper::remove_realm(realm::Realm* realm) +{ + std::lock_guard lock(m_realms_mutex); + for (auto it = m_realms.begin(); it != m_realms.end(); ++it) { + if (it->realm == realm) { + CFRunLoopSourceInvalidate(it->signal); + CFRelease(it->signal); + CFRelease(it->runloop); + m_realms.erase(it); + return; + } + } + REALM_TERMINATE("Realm not registered"); +} + +void ExternalCommitHelper::listen() +{ + pthread_setname_np("RLMRealm notification listener"); + + + // Set up the kqueue + // EVFILT_READ indicates that we care about data being available to read + // on the given file descriptor. + // EV_CLEAR makes it wait for the amount of data available to be read to + // change rather than just returning when there is any data to read. + struct kevent ke[2]; + EV_SET(&ke[0], m_notify_fd, EVFILT_READ, EV_ADD | EV_CLEAR, 0, 0, 0); + EV_SET(&ke[1], m_shutdown_read_fd, EVFILT_READ, EV_ADD | EV_CLEAR, 0, 0, 0); + int ret = kevent(m_kq, ke, 2, nullptr, 0, nullptr); + assert(ret == 0); + + while (true) { + struct kevent event; + // Wait for data to become on either fd + // Return code is number of bytes available or -1 on error + ret = kevent(m_kq, nullptr, 0, &event, 1, nullptr); + assert(ret >= 0); + if (ret == 0) { + // Spurious wakeup; just wait again + continue; + } + + // Check which file descriptor had activity: if it's the shutdown + // pipe, then someone called -stop; otherwise it's the named pipe + // and someone committed a write transaction + if (event.ident == (uint32_t)m_shutdown_read_fd) { + return; + } + assert(event.ident == (uint32_t)m_notify_fd); + + std::lock_guard lock(m_realms_mutex); + for (auto const& realm : m_realms) { + CFRunLoopSourceSignal(realm.signal); + // Signalling the source makes it run the next time the runloop gets + // to it, but doesn't make the runloop start if it's currently idle + // waiting for events + CFRunLoopWakeUp(realm.runloop); + } + } +} + +void ExternalCommitHelper::notify_others() +{ + notify_fd(m_notify_fd); +} + diff --git a/impl/apple/external_commit_helper.hpp b/impl/apple/external_commit_helper.hpp new file mode 100644 index 00000000..d7acb791 --- /dev/null +++ b/impl/apple/external_commit_helper.hpp @@ -0,0 +1,94 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_EXTERNAL_COMMIT_HELPER_HPP +#define REALM_EXTERNAL_COMMIT_HELPER_HPP + +#include +#include +#include + +namespace realm { +class Realm; + +namespace _impl { +class ExternalCommitHelper { +public: + ExternalCommitHelper(Realm* realm); + ~ExternalCommitHelper(); + + void notify_others(); + void add_realm(Realm* realm); + void remove_realm(Realm* realm); + +private: + // A RAII holder for a file descriptor which automatically closes the wrapped + // fd when it's deallocated + class FdHolder { + public: + FdHolder() = default; + ~FdHolder() { close(); } + operator int() const { return m_fd; } + + FdHolder& operator=(int newFd) { + close(); + m_fd = newFd; + return *this; + } + + private: + int m_fd = -1; + void close(); + + FdHolder& operator=(FdHolder const&) = delete; + FdHolder(FdHolder const&) = delete; + }; + + struct PerRealmInfo { + Realm* realm; + CFRunLoopRef runloop; + CFRunLoopSourceRef signal; + }; + + void listen(); + + // Currently registered realms and the signal for delivering notifications + // to them + std::vector m_realms; + + // Mutex which guards m_realms + std::mutex m_realms_mutex; + + // The listener thread + pthread_t m_thread; + + // Read-write file descriptor for the named pipe which is waited on for + // changes and written to when a commit is made + FdHolder m_notify_fd; + // File descriptor for the kqueue + FdHolder m_kq; + // The two ends of an anonymous pipe used to notify the kqueue() thread that + // it should be shut down. + FdHolder m_shutdown_read_fd; + FdHolder m_shutdown_write_fd; +}; + +} // namespace _impl +} // namespace realm + +#endif /* REALM_EXTERNAL_COMMIT_HELPER_HPP */ diff --git a/impl/transact_log_handler.cpp b/impl/transact_log_handler.cpp new file mode 100644 index 00000000..fba97129 --- /dev/null +++ b/impl/transact_log_handler.cpp @@ -0,0 +1,356 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "transact_log_handler.hpp" + +#include "../realm_binding_context.hpp" + +#include +#include +#include + +using namespace realm; + +namespace { +class TransactLogHandler { + using ColumnInfo = RealmBindingContext::ColumnInfo; + using ObserverState = RealmBindingContext::ObserverState; + + // Observed table rows which need change information + std::vector m_observers; + // Userdata pointers for rows which have been deleted + std::vector invalidated; + // Delegate to send change information to + RealmBindingContext* m_binding_context; + + // Index of currently selected table + size_t m_current_table = 0; + // Change information for the currently selected LinkList, if any + ColumnInfo* m_active_linklist = nullptr; + + // Get the change info for the given column, creating it if needed + static ColumnInfo& get_change(ObserverState& state, size_t i) + { + if (state.changes.size() <= i) { + state.changes.resize(std::max(state.changes.size() * 2, i + 1)); + } + return state.changes[i]; + } + + // Loop over the columns which were changed in an observer state + template + static void for_each(ObserverState& state, Func&& f) + { + for (size_t i = 0; i < state.changes.size(); ++i) { + auto const& change = state.changes[i]; + if (change.changed) { + f(i, change); + } + } + } + + // Mark the given row/col as needing notifications sent + bool mark_dirty(size_t row_ndx, size_t col_ndx) + { + auto it = lower_bound(begin(m_observers), end(m_observers), ObserverState{m_current_table, row_ndx, nullptr}); + if (it != end(m_observers) && it->table_ndx == m_current_table && it->row_ndx == row_ndx) { + get_change(*it, col_ndx).changed = true; + } + return true; + } + + // Remove the given observer from the list of observed objects and add it + // to the listed of invalidated objects + void invalidate(ObserverState *o) + { + invalidated.push_back(o->info); + m_observers.erase(m_observers.begin() + (o - &m_observers[0])); + } + +public: + template + TransactLogHandler(RealmBindingContext* binding_context, SharedGroup& sg, Func&& func) + : m_binding_context(binding_context) + { + if (!binding_context) { + func(); + return; + } + + m_observers = binding_context->get_observed_rows(); + if (m_observers.empty()) { + auto old_version = sg.get_version_of_current_transaction(); + func(); + if (old_version != sg.get_version_of_current_transaction()) { + binding_context->did_change({}, {}); + } + return; + } + + func(*this); + binding_context->did_change(m_observers, invalidated); + } + + // Called at the end of the transaction log immediately before the version + // is advanced + void parse_complete() + { + m_binding_context->will_change(m_observers, invalidated); + } + + // These would require having an observer before schema init + // Maybe do something here to throw an error when multiple processes have different schemas? + bool insert_group_level_table(size_t, size_t, StringData) { return false; } + bool erase_group_level_table(size_t, size_t) { return false; } + bool rename_group_level_table(size_t, StringData) { return false; } + bool insert_column(size_t, DataType, StringData, bool) { return false; } + bool insert_link_column(size_t, DataType, StringData, size_t, size_t) { return false; } + bool erase_column(size_t) { return false; } + bool erase_link_column(size_t, size_t, size_t) { return false; } + bool rename_column(size_t, StringData) { return false; } + bool add_search_index(size_t) { return false; } + bool remove_search_index(size_t) { return false; } + bool add_primary_key(size_t) { return false; } + bool remove_primary_key() { return false; } + bool set_link_type(size_t, LinkType) { return false; } + + bool select_table(size_t group_level_ndx, int, const size_t*) noexcept + { + m_current_table = group_level_ndx; + return true; + } + + bool insert_empty_rows(size_t, size_t, size_t, bool) + { + // rows are only inserted at the end, so no need to do anything + return true; + } + + bool erase_rows(size_t row_ndx, size_t, size_t last_row_ndx, bool unordered) + { + for (size_t i = 0; i < m_observers.size(); ++i) { + auto& o = m_observers[i]; + if (o.table_ndx == m_current_table) { + if (o.row_ndx == row_ndx) { + invalidate(&o); + --i; + } + else if (unordered && o.row_ndx == last_row_ndx) { + o.row_ndx = row_ndx; + } + else if (!unordered && o.row_ndx > row_ndx) { + o.row_ndx -= 1; + } + } + } + return true; + } + + bool clear_table() + { + for (size_t i = 0; i < m_observers.size(); ) { + auto& o = m_observers[i]; + if (o.table_ndx == m_current_table) { + invalidate(&o); + } + else { + ++i; + } + } + return true; + } + + bool select_link_list(size_t col, size_t row, size_t) + { + m_active_linklist = nullptr; + for (auto& o : m_observers) { + if (o.table_ndx == m_current_table && o.row_ndx == row) { + m_active_linklist = &get_change(o, col); + break; + } + } + return true; + } + + void append_link_list_change(ColumnInfo::Kind kind, size_t index) { + ColumnInfo *o = m_active_linklist; + if (!o || o->kind == ColumnInfo::Kind::SetAll) { + // Active LinkList isn't observed or already has multiple kinds of changes + return; + } + + if (o->kind == ColumnInfo::Kind::None) { + o->kind = kind; + o->changed = true; + o->indices.add(index); + } + else if (o->kind == kind) { + if (kind == ColumnInfo::Kind::Remove) { + o->indices.add_shifted(index); + } + else if (kind == ColumnInfo::Kind::Insert) { + o->indices.insert_at(index); + } + else { + o->indices.add(index); + } + } + else { + // Array KVO can only send a single kind of change at a time, so + // if there's multiple just give up and send "Set" + o->indices.set(0); + o->kind = ColumnInfo::Kind::SetAll; + } + } + + bool link_list_set(size_t index, size_t) + { + append_link_list_change(ColumnInfo::Kind::Set, index); + return true; + } + + bool link_list_insert(size_t index, size_t) + { + append_link_list_change(ColumnInfo::Kind::Insert, index); + return true; + } + + bool link_list_erase(size_t index) + { + append_link_list_change(ColumnInfo::Kind::Remove, index); + return true; + } + + bool link_list_nullify(size_t index) + { + append_link_list_change(ColumnInfo::Kind::Remove, index); + return true; + } + + bool link_list_swap(size_t index1, size_t index2) + { + append_link_list_change(ColumnInfo::Kind::Set, index1); + append_link_list_change(ColumnInfo::Kind::Set, index2); + return true; + } + + bool link_list_clear(size_t old_size) + { + ColumnInfo *o = m_active_linklist; + if (!o || o->kind == ColumnInfo::Kind::SetAll) { + return true; + } + + if (o->kind == ColumnInfo::Kind::Remove) + old_size += o->indices.size(); + else if (o->kind == ColumnInfo::Kind::Insert) + old_size -= o->indices.size(); + + o->indices.set(old_size); + + o->kind = ColumnInfo::Kind::Remove; + o->changed = true; + return true; + } + + bool link_list_move(size_t from, size_t to) + { + ColumnInfo *o = m_active_linklist; + if (!o || o->kind == ColumnInfo::Kind::SetAll) { + return true; + } + if (from > to) { + std::swap(from, to); + } + + if (o->kind == ColumnInfo::Kind::None) { + o->kind = ColumnInfo::Kind::Set; + o->changed = true; + } + if (o->kind == ColumnInfo::Kind::Set) { + for (size_t i = from; i <= to; ++i) + o->indices.add(i); + } + else { + o->indices.set(0); + o->kind = ColumnInfo::Kind::SetAll; + } + return true; + } + + // Things that just mark the field as modified + bool set_int(size_t col, size_t row, int_fast64_t) { return mark_dirty(row, col); } + bool set_bool(size_t col, size_t row, bool) { return mark_dirty(row, col); } + bool set_float(size_t col, size_t row, float) { return mark_dirty(row, col); } + bool set_double(size_t col, size_t row, double) { return mark_dirty(row, col); } + bool set_string(size_t col, size_t row, StringData) { return mark_dirty(row, col); } + bool set_binary(size_t col, size_t row, BinaryData) { return mark_dirty(row, col); } + bool set_date_time(size_t col, size_t row, DateTime) { return mark_dirty(row, col); } + bool set_table(size_t col, size_t row) { return mark_dirty(row, col); } + bool set_mixed(size_t col, size_t row, const Mixed&) { return mark_dirty(row, col); } + bool set_link(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } + bool set_null(size_t col, size_t row) { return mark_dirty(row, col); } + bool nullify_link(size_t col, size_t row, size_t) { return mark_dirty(row, col); } + + // Doesn't change any data + bool optimize_table() { return true; } + + // Used for subtables, which we currently don't support + bool select_descriptor(int, const size_t*) { return false; } + + // Not implemented + bool insert_substring(size_t, size_t, size_t, StringData) { return false; } + bool erase_substring(size_t, size_t, size_t, size_t) { return false; } + bool swap_rows(size_t, size_t) { return false; } + bool move_column(size_t, size_t) { return false; } + bool move_group_level_table(size_t, size_t) { return false; } +}; +} // anonymous namespace + +namespace realm { +namespace _impl { +namespace transaction { +void advance(SharedGroup& sg, ClientHistory& history, RealmBindingContext* binding_context) { + TransactLogHandler(binding_context, sg, [&](auto&&... args) { + LangBindHelper::advance_read(sg, history, std::move(args)...); + }); +} + +void begin(SharedGroup& sg, ClientHistory& history, RealmBindingContext* binding_context) { + TransactLogHandler(binding_context, sg, [&](auto&&... args) { + LangBindHelper::promote_to_write(sg, history, std::move(args)...); + }); +} + +void commit(SharedGroup& sg, ClientHistory&, RealmBindingContext* binding_context) { + LangBindHelper::commit_and_continue_as_read(sg); + + if (binding_context) { + binding_context->did_change({}, {}); + } +} + +void cancel(SharedGroup& sg, ClientHistory& history, RealmBindingContext* binding_context) { + TransactLogHandler(binding_context, sg, [&](auto&&... args) { + LangBindHelper::rollback_and_continue_as_read(sg, history, std::move(args)...); + }); +} + +} // namespace transaction +} // namespace _impl +} // namespace realm diff --git a/impl/transact_log_handler.hpp b/impl/transact_log_handler.hpp new file mode 100644 index 00000000..321a67c9 --- /dev/null +++ b/impl/transact_log_handler.hpp @@ -0,0 +1,48 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_TRANSACT_LOG_HANDLER_HPP +#define REALM_TRANSACT_LOG_HANDLER_HPP + +namespace realm { +class RealmBindingContext; +class SharedGroup; +class ClientHistory; + +namespace _impl { +namespace transaction { +// Advance the read transaction version, with change notifications sent to delegate +// Must not be called from within a write transaction. +void advance(SharedGroup& sg, ClientHistory& history, RealmBindingContext* binding_context); + +// Begin a write transaction +// If the read transaction version is not up to date, will first advance to the +// most recent read transaction and sent notifications to delegate +void begin(SharedGroup& sg, ClientHistory& history, RealmBindingContext* binding_context); + +// Commit a write transaction +void commit(SharedGroup& sg, ClientHistory& history, RealmBindingContext* binding_context); + +// Cancel a write transaction and roll back all changes, with change notifications +// for reverting to the old values sent to delegate +void cancel(SharedGroup& sg, ClientHistory& history, RealmBindingContext* binding_context); +} // namespace transaction +} // namespace _impl +} // namespace realm + +#endif /* REALM_TRANSACT_LOG_HANDLER_HPP */ diff --git a/index_set.cpp b/index_set.cpp new file mode 100644 index 00000000..b7e4961c --- /dev/null +++ b/index_set.cpp @@ -0,0 +1,92 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "index_set.hpp" + +using namespace realm; + +IndexSet::iterator IndexSet::find(size_t index) +{ + for (auto it = m_ranges.begin(), end = m_ranges.end(); it != end; ++it) { + if (it->second > index) + return it; + } + return m_ranges.end(); +} + +void IndexSet::add(size_t index) +{ + do_add(find(index), index); +} + +void IndexSet::do_add(iterator it, size_t index) +{ + bool more_before = it != m_ranges.begin(), valid = it != m_ranges.end(); + if (valid && it->first <= index && it->second > index) { + // index is already in set + } + else if (more_before && (it - 1)->second == index) { + // index is immediately after an existing range + ++(it - 1)->second; + } + else if (more_before && valid && (it - 1)->second == it->first) { + // index joins two existing ranges + (it - 1)->second = it->second; + m_ranges.erase(it); + } + else if (valid && it->first == index + 1) { + // index is immediately before an existing range + --it->first; + } + else { + // index is not next to an existing range + m_ranges.insert(it, {index, index + 1}); + } +} + +void IndexSet::set(size_t len) +{ + m_ranges.clear(); + if (len) { + m_ranges.push_back({0, len}); + } +} + +void IndexSet::insert_at(size_t index) +{ + auto pos = find(index); + if (pos != m_ranges.end()) { + if (pos->first >= index) + ++pos->first; + ++pos->second; + for (auto it = pos + 1; it != m_ranges.end(); ++it) { + ++it->first; + ++it->second; + } + } + do_add(pos, index); +} + +void IndexSet::add_shifted(size_t index) +{ + auto it = m_ranges.begin(); + for (auto end = m_ranges.end(); it != end && it->first <= index; ++it) { + index += it->second - it->first; + } + do_add(it, index); +} diff --git a/index_set.hpp b/index_set.hpp new file mode 100644 index 00000000..7a229177 --- /dev/null +++ b/index_set.hpp @@ -0,0 +1,62 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_INDEX_SET_HPP +#define REALM_INDEX_SET_HPP + +#include + +namespace realm { +class IndexSet { +public: + using value_type = std::pair; + using iterator = std::vector::iterator; + using const_iterator = std::vector::const_iterator; + + const_iterator begin() const { return m_ranges.begin(); } + const_iterator end() const { return m_ranges.end(); } + bool empty() const { return m_ranges.empty(); } + size_t size() const { return m_ranges.size(); } + + // Add an index to the set, doing nothing if it's already present + void add(size_t index); + + // Remove all indexes from the set and then add a single range starting from + // zero with the given length + void set(size_t len); + + // Insert an index at the given position, shifting existing indexes at or + // after that point back by one + void insert_at(size_t index); + + // Add an index which has had all of the ranges in the set before it removed + void add_shifted(size_t index); + +private: + std::vector m_ranges; + + // Find the range which contains the index, or the first one after it if + // none do + iterator find(size_t index); + // Insert the index before the given position, combining existing ranges as + // applicable + void do_add(iterator pos, size_t index); +}; +} // namespace realm + +#endif // REALM_INDEX_SET_HPP diff --git a/object_schema.cpp b/object_schema.cpp index 83edd1f1..33cd2497 100644 --- a/object_schema.cpp +++ b/object_schema.cpp @@ -17,16 +17,19 @@ //////////////////////////////////////////////////////////////////////////// #include "object_schema.hpp" + #include "object_store.hpp" +#include "property.hpp" #include #include using namespace realm; -ObjectSchema::ObjectSchema(Group *group, const std::string &name) : name(name) { - TableRef tableRef = ObjectStore::table_for_object_type(group, name); - Table *table = tableRef.get(); +ObjectSchema::~ObjectSchema() = default; + +ObjectSchema::ObjectSchema(const Group *group, const std::string &name) : name(name) { + ConstTableRef table = ObjectStore::table_for_object_type(group, name); size_t count = table->get_column_count(); properties.reserve(count); @@ -40,7 +43,7 @@ ObjectSchema::ObjectSchema(Group *group, const std::string &name) : name(name) { property.table_column = col; if (property.type == PropertyTypeObject || property.type == PropertyTypeArray) { // set link type for objects and arrays - realm::TableRef linkTable = table->get_link_target(col); + ConstTableRef linkTable = table->get_link_target(col); property.object_type = ObjectStore::object_type_for_table_name(linkTable->get_name().data()); } properties.push_back(std::move(property)); @@ -56,12 +59,15 @@ ObjectSchema::ObjectSchema(Group *group, const std::string &name) : name(name) { } } -Property *ObjectSchema::property_for_name(const std::string &name) { - for (auto& prop:properties) { - if (prop.name == name) { +Property *ObjectSchema::property_for_name(StringData name) { + for (auto& prop : properties) { + if (StringData(prop.name) == name) { return ∝ } } return nullptr; } +const Property *ObjectSchema::property_for_name(StringData name) const { + return const_cast(this)->property_for_name(name); +} diff --git a/object_schema.hpp b/object_schema.hpp index 56a14d17..4ede6ce5 100644 --- a/object_schema.hpp +++ b/object_schema.hpp @@ -19,30 +19,36 @@ #ifndef REALM_OBJECT_SCHEMA_HPP #define REALM_OBJECT_SCHEMA_HPP +#include + #include #include -#include "property.hpp" - namespace realm { class Group; + struct Property; class ObjectSchema { public: - ObjectSchema() {} + ObjectSchema() = default; + ~ObjectSchema(); // create object schema from existing table // if no table is provided it is looked up in the group - ObjectSchema(Group *group, const std::string &name); + ObjectSchema(const Group *group, const std::string &name); std::string name; std::vector properties; std::string primary_key; - Property *property_for_name(const std::string &name); + Property *property_for_name(StringData name); + const Property *property_for_name(StringData name) const; Property *primary_key_property() { return property_for_name(primary_key); } + const Property *primary_key_property() const { + return property_for_name(primary_key); + } }; } diff --git a/object_store.cpp b/object_store.cpp index cf3dd2ed..99364dc3 100644 --- a/object_store.cpp +++ b/object_store.cpp @@ -18,9 +18,11 @@ #include "object_store.hpp" +#include "schema.hpp" + #include -#include #include +#include #include #include @@ -45,7 +47,7 @@ const size_t c_object_table_prefix_length = c_object_table_prefix.length(); const uint64_t ObjectStore::NotVersioned = std::numeric_limits::max(); -bool ObjectStore::has_metadata_tables(Group *group) { +bool ObjectStore::has_metadata_tables(const Group *group) { return group->get_table(c_primaryKeyTableName) && group->get_table(c_metadataTableName); } @@ -71,8 +73,8 @@ bool ObjectStore::create_metadata_tables(Group *group) { return changed; } -uint64_t ObjectStore::get_schema_version(Group *group) { - TableRef table = group->get_table(c_metadataTableName); +uint64_t ObjectStore::get_schema_version(const Group *group) { + ConstTableRef table = group->get_table(c_metadataTableName); if (!table || table->get_column_count() == 0) { return ObjectStore::NotVersioned; } @@ -84,8 +86,8 @@ void ObjectStore::set_schema_version(Group *group, uint64_t version) { table->set_int(c_versionColumnIndex, c_zeroRowIndex, version); } -StringData ObjectStore::get_primary_key_for_object(Group *group, StringData object_type) { - TableRef table = group->get_table(c_primaryKeyTableName); +StringData ObjectStore::get_primary_key_for_object(const Group *group, StringData object_type) { + ConstTableRef table = group->get_table(c_primaryKeyTableName); if (!table) { return ""; } @@ -132,26 +134,41 @@ TableRef ObjectStore::table_for_object_type(Group *group, StringData object_type return group->get_table(table_name_for_object_type(object_type)); } +ConstTableRef ObjectStore::table_for_object_type(const Group *group, StringData object_type) { + return group->get_table(table_name_for_object_type(object_type)); +} + TableRef ObjectStore::table_for_object_type_create_if_needed(Group *group, const StringData &object_type, bool &created) { return group->get_or_add_table(table_name_for_object_type(object_type), &created); } -static inline bool property_has_changed(Property &p1, Property &p2) { - return p1.type != p2.type || p1.name != p2.name || p1.object_type != p2.object_type || p1.is_nullable != p2.is_nullable; +static inline bool property_has_changed(Property const& p1, Property const& p2) { + return p1.type != p2.type + || p1.name != p2.name + || p1.object_type != p2.object_type + || p1.is_nullable != p2.is_nullable; } -void ObjectStore::verify_schema(Group *group, Schema &target_schema, bool allow_missing_tables) { +static inline bool property_can_be_migrated_to_nullable(const Property& old_property, const Property& new_property) { + return old_property.type == new_property.type + && !old_property.is_nullable + && new_property.is_nullable + && new_property.name == old_property.name; +} + +void ObjectStore::verify_schema(Schema const& actual_schema, Schema& target_schema, bool allow_missing_tables) { std::vector errors; for (auto &object_schema : target_schema) { - if (!table_for_object_type(group, object_schema.first)) { + auto matching_schema = actual_schema.find(object_schema); + if (matching_schema == actual_schema.end()) { if (!allow_missing_tables) { - errors.emplace_back(ObjectSchemaValidationException(object_schema.first, - "Missing table for object type '" + object_schema.first + "'.")); + errors.emplace_back(ObjectSchemaValidationException(object_schema.name, + "Missing table for object type '" + object_schema.name + "'.")); } continue; } - auto more_errors = verify_object_schema(group, object_schema.second, target_schema); + auto more_errors = verify_object_schema(*matching_schema, object_schema); errors.insert(errors.end(), more_errors.begin(), more_errors.end()); } if (errors.size()) { @@ -159,12 +176,11 @@ void ObjectStore::verify_schema(Group *group, Schema &target_schema, bool allow_ } } -std::vector ObjectStore::verify_object_schema(Group *group, ObjectSchema &target_schema, Schema &schema) { +std::vector ObjectStore::verify_object_schema(ObjectSchema const& table_schema, + ObjectSchema& target_schema) { std::vector exceptions; - ObjectSchema table_schema(group, target_schema.name); // check to see if properties are the same - Property *primary = nullptr; for (auto& current_prop : table_schema.properties) { auto target_prop = target_schema.property_for_name(current_prop.name); @@ -177,38 +193,6 @@ std::vector ObjectStore::verify_object_schema(G continue; } - // check object_type existence - if (current_prop.object_type.length() && schema.find(current_prop.object_type) == schema.end()) { - exceptions.emplace_back(MissingObjectTypeException(table_schema.name, current_prop)); - } - - // check nullablity - if (current_prop.type == PropertyTypeObject) { - if (!current_prop.is_nullable) { - exceptions.emplace_back(InvalidNullabilityException(table_schema.name, current_prop)); - } - } - else { - if (current_prop.is_nullable) { - exceptions.emplace_back(InvalidNullabilityException(table_schema.name, current_prop)); - } - } - - // check primary keys - if (current_prop.is_primary) { - if (primary) { - exceptions.emplace_back(DuplicatePrimaryKeysException(table_schema.name)); - } - primary = ¤t_prop; - } - - // check indexable - if (current_prop.is_indexed) { - if (current_prop.type != PropertyTypeString && current_prop.type != PropertyTypeInt) { - exceptions.emplace_back(PropertyTypeNotIndexableException(table_schema.name, current_prop)); - } - } - // create new property with aligned column target_prop->table_column = current_prop.table_column; } @@ -228,13 +212,42 @@ std::vector ObjectStore::verify_object_schema(G return exceptions; } -void ObjectStore::update_column_mapping(Group *group, ObjectSchema &target_schema) { - ObjectSchema table_schema(group, target_schema.name); - for (auto& target_prop : target_schema.properties) { - auto table_prop = table_schema.property_for_name(target_prop.name); - REALM_ASSERT_DEBUG(table_prop); +template +static void copy_property_values(const Property& old_property, const Property& new_property, Table& table, + T (Table::*getter)(std::size_t, std::size_t) const noexcept, + void (Table::*setter)(std::size_t, std::size_t, T)) { + size_t old_column = old_property.table_column, new_column = new_property.table_column; + size_t count = table.size(); + for (size_t i = 0; i < count; i++) { + (table.*setter)(new_column, i, (table.*getter)(old_column, i)); + } +} - target_prop.table_column = table_prop->table_column; +static void copy_property_values(const Property& source, const Property& destination, Table& table) { + switch (destination.type) { + case PropertyTypeInt: + copy_property_values(source, destination, table, &Table::get_int, &Table::set_int); + break; + case PropertyTypeBool: + copy_property_values(source, destination, table, &Table::get_bool, &Table::set_bool); + break; + case PropertyTypeFloat: + copy_property_values(source, destination, table, &Table::get_float, &Table::set_float); + break; + case PropertyTypeDouble: + copy_property_values(source, destination, table, &Table::get_double, &Table::set_double); + break; + case PropertyTypeString: + copy_property_values(source, destination, table, &Table::get_string, &Table::set_string); + break; + case PropertyTypeData: + copy_property_values(source, destination, table, &Table::get_binary, &Table::set_binary); + break; + case PropertyTypeDate: + copy_property_values(source, destination, table, &Table::get_datetime, &Table::set_datetime); + break; + default: + break; } } @@ -248,11 +261,11 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update std::vector to_update; for (auto& object_schema : target_schema) { bool created = false; - ObjectStore::table_for_object_type_create_if_needed(group, object_schema.first, created); + ObjectStore::table_for_object_type_create_if_needed(group, object_schema.name, created); // we will modify tables for any new objectSchema (table was created) or for all if update_existing is true if (update_existing || created) { - to_update.push_back(&object_schema.second); + to_update.push_back(&object_schema); changed = true; } } @@ -263,12 +276,53 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update ObjectSchema current_schema(group, target_object_schema->name); std::vector &target_props = target_object_schema->properties; + // handle columns changing from required to optional + for (auto& current_prop : current_schema.properties) { + auto target_prop = target_object_schema->property_for_name(current_prop.name); + if (!target_prop || !property_can_be_migrated_to_nullable(current_prop, *target_prop)) + continue; + + target_prop->table_column = current_prop.table_column; + current_prop.table_column = current_prop.table_column + 1; + + table->insert_column(target_prop->table_column, DataType(target_prop->type), target_prop->name, target_prop->is_nullable); + copy_property_values(current_prop, *target_prop, *table); + table->remove_column(current_prop.table_column); + + current_prop.table_column = target_prop->table_column; + changed = true; + } + + bool inserted_placeholder_column = false; + + // remove extra columns + size_t deleted = 0; + for (auto& current_prop : current_schema.properties) { + current_prop.table_column -= deleted; + + auto target_prop = target_object_schema->property_for_name(current_prop.name); + if (!target_prop || (property_has_changed(current_prop, *target_prop) + && !property_can_be_migrated_to_nullable(current_prop, *target_prop))) { + if (deleted == current_schema.properties.size() - 1) { + // We're about to remove the last column from the table. Insert a placeholder column to preserve + // the number of rows in the table for the addition of new columns below. + table->add_column(type_Bool, "placeholder"); + inserted_placeholder_column = true; + } + + table->remove_column(current_prop.table_column); + ++deleted; + current_prop.table_column = npos; + changed = true; + } + } + // add missing columns for (auto& target_prop : target_props) { auto current_prop = current_schema.property_for_name(target_prop.name); - // add any new properties (new name or different type) - if (!current_prop || property_has_changed(*current_prop, target_prop)) { + // add any new properties (no old column or old column was removed due to not matching) + if (!current_prop || current_prop->table_column == npos) { switch (target_prop.type) { // for objects and arrays, we have to specify target table case PropertyTypeObject: @@ -278,23 +332,25 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update break; } default: - target_prop.table_column = table->add_column(DataType(target_prop.type), target_prop.name); + target_prop.table_column = table->add_column(DataType(target_prop.type), + target_prop.name, + target_prop.is_nullable); break; } changed = true; } + else { + target_prop.table_column = current_prop->table_column; + } } - // remove extra columns - sort(begin(current_schema.properties), end(current_schema.properties), [](Property &i, Property &j) { - return j.table_column < i.table_column; - }); - for (auto& current_prop : current_schema.properties) { - auto target_prop = target_object_schema->property_for_name(current_prop.name); - if (!target_prop || property_has_changed(current_prop, *target_prop)) { - table->remove_column(current_prop.table_column); - changed = true; + if (inserted_placeholder_column) { + // We inserted a placeholder due to removing all columns from the table. Remove it, and update the indices + // of any columns that we inserted after it. + table->remove_column(0); + for (auto& target_prop : target_props) { + target_prop.table_column--; } } @@ -315,7 +371,7 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update return changed; } -bool ObjectStore::is_schema_at_version(Group *group, uint64_t version) { +bool ObjectStore::is_schema_at_version(const Group *group, uint64_t version) { uint64_t old_version = get_schema_version(group); if (old_version > version && old_version != NotVersioned) { throw InvalidSchemaVersionException(old_version, version); @@ -323,25 +379,32 @@ bool ObjectStore::is_schema_at_version(Group *group, uint64_t version) { return old_version == version; } -bool ObjectStore::realm_requires_update(Group *group, uint64_t version, Schema &schema) { - if (!is_schema_at_version(group, version)) { - return true; - } - for (auto& target_schema : schema) { - TableRef table = table_for_object_type(group, target_schema.first); - if (!table) { +bool ObjectStore::needs_update(Schema const& old_schema, Schema const& schema) { + for (auto const& target_schema : schema) { + auto matching_schema = old_schema.find(target_schema); + if (matching_schema == end(old_schema)) { + // Table doesn't exist return true; } + + if (matching_schema->properties.size() != target_schema.properties.size()) { + // If the number of properties don't match then a migration is required + return false; + } + + // Check that all of the property indexes are up to date + for (size_t i = 0, count = target_schema.properties.size(); i < count; ++i) { + if (target_schema.properties[i].is_indexed != matching_schema->properties[i].is_indexed) { + return true; + } + } } - if (!indexes_are_up_to_date(group, schema)) { - return true; - } + return false; } -bool ObjectStore::update_realm_with_schema(Group *group, - uint64_t version, - Schema &schema, +bool ObjectStore::update_realm_with_schema(Group *group, Schema const& old_schema, + uint64_t version, Schema &schema, MigrationFunction migration) { // Recheck the schema version after beginning the write transaction as // another process may have done the migration after we opened the read @@ -352,7 +415,11 @@ bool ObjectStore::update_realm_with_schema(Group *group, bool changed = create_metadata_tables(group); changed = create_tables(group, schema, migrating) || changed; - verify_schema(group, schema); + if (!migrating) { + // If we aren't migrating, then verify that all of the tables which + // were already present are valid (newly created ones always are) + verify_schema(old_schema, schema, true); + } changed = update_indexes(group, schema) || changed; @@ -363,51 +430,34 @@ bool ObjectStore::update_realm_with_schema(Group *group, // apply the migration block if provided and there's any old data if (get_schema_version(group) != ObjectStore::NotVersioned) { migration(group, schema); - } - validate_primary_column_uniqueness(group, schema); + validate_primary_column_uniqueness(group, schema); + } set_schema_version(group, version); return true; } -Schema ObjectStore::schema_from_group(Group *group) { - Schema schema; +Schema ObjectStore::schema_from_group(const Group *group) { + std::vector schema; for (size_t i = 0; i < group->size(); i++) { std::string object_type = object_type_for_table_name(group->get_table_name(i)); if (object_type.length()) { - schema.emplace(object_type, std::move(ObjectSchema(group, object_type))); + schema.emplace_back(group, object_type); } } return schema; } -bool ObjectStore::indexes_are_up_to_date(Group *group, Schema &schema) { - for (auto &object_schema : schema) { - TableRef table = table_for_object_type(group, object_schema.first); - if (!table) { - continue; - } - - update_column_mapping(group, object_schema.second); - for (auto& property : object_schema.second.properties) { - if (property.requires_index() != table->has_search_index(property.table_column)) { - return false; - } - } - } - return true; -} - bool ObjectStore::update_indexes(Group *group, Schema &schema) { bool changed = false; for (auto& object_schema : schema) { - TableRef table = table_for_object_type(group, object_schema.first); + TableRef table = table_for_object_type(group, object_schema.name); if (!table) { continue; } - for (auto& property : object_schema.second.properties) { + for (auto& property : object_schema.properties) { if (property.requires_index() == table->has_search_index(property.table_column)) { continue; } @@ -418,7 +468,7 @@ bool ObjectStore::update_indexes(Group *group, Schema &schema) { table->add_search_index(property.table_column); } catch (LogicError const&) { - throw PropertyTypeNotIndexableException(object_schema.first, property); + throw PropertyTypeNotIndexableException(object_schema.name, property); } } else { @@ -429,16 +479,16 @@ bool ObjectStore::update_indexes(Group *group, Schema &schema) { return changed; } -void ObjectStore::validate_primary_column_uniqueness(Group *group, Schema &schema) { +void ObjectStore::validate_primary_column_uniqueness(const Group *group, Schema const& schema) { for (auto& object_schema : schema) { - auto primary_prop = object_schema.second.primary_key_property(); + auto primary_prop = object_schema.primary_key_property(); if (!primary_prop) { continue; } - TableRef table = table_for_object_type(group, object_schema.first); + ConstTableRef table = table_for_object_type(group, object_schema.name); if (table->get_distinct_view(primary_prop->table_column).size() != table->size()) { - throw DuplicatePrimaryKeyValueException(object_schema.first, *primary_prop); + throw DuplicatePrimaryKeyValueException(object_schema.name, *primary_prop); } } } @@ -451,7 +501,19 @@ void ObjectStore::delete_data_for_object(Group *group, const StringData &object_ } } - +bool ObjectStore::is_empty(const Group *group) { + for (size_t i = 0; i < group->size(); i++) { + ConstTableRef table = group->get_table(i); + std::string object_type = object_type_for_table_name(table->get_name()); + if (!object_type.length()) { + continue; + } + if (!table->is_empty()) { + return false; + } + } + return true; +} InvalidSchemaVersionException::InvalidSchemaVersionException(uint64_t old_version, uint64_t new_version) : m_old_version(old_version), m_new_version(new_version) @@ -459,62 +521,58 @@ InvalidSchemaVersionException::InvalidSchemaVersionException(uint64_t old_versio m_what = "Provided schema version " + std::to_string(old_version) + " is less than last set version " + std::to_string(new_version) + "."; } -DuplicatePrimaryKeyValueException::DuplicatePrimaryKeyValueException(std::string object_type, Property &property) : +DuplicatePrimaryKeyValueException::DuplicatePrimaryKeyValueException(std::string const& object_type, Property const& property) : m_object_type(object_type), m_property(property) { m_what = "Primary key property '" + property.name + "' has duplicate values after migration."; }; -SchemaValidationException::SchemaValidationException(std::vector errors) : +SchemaValidationException::SchemaValidationException(std::vector const& errors) : m_validation_errors(errors) { m_what ="Migration is required due to the following errors: "; - for (auto error : errors) { + for (auto const& error : errors) { m_what += std::string("\n- ") + error.what(); } } -PropertyTypeNotIndexableException::PropertyTypeNotIndexableException(std::string object_type, Property &property) : +PropertyTypeNotIndexableException::PropertyTypeNotIndexableException(std::string const& object_type, Property const& property) : ObjectSchemaPropertyException(object_type, property) { m_what = "Can't index property " + object_type + "." + property.name + ": indexing a property of type '" + string_for_property_type(property.type) + "' is currently not supported"; } -ExtraPropertyException::ExtraPropertyException(std::string object_type, Property &property) : +ExtraPropertyException::ExtraPropertyException(std::string const& object_type, Property const& property) : ObjectSchemaPropertyException(object_type, property) { m_what = "Property '" + property.name + "' has been added to latest object model."; } -MissingPropertyException::MissingPropertyException(std::string object_type, Property &property) : +MissingPropertyException::MissingPropertyException(std::string const& object_type, Property const& property) : ObjectSchemaPropertyException(object_type, property) { m_what = "Property '" + property.name + "' is missing from latest object model."; } -InvalidNullabilityException::InvalidNullabilityException(std::string object_type, Property &property) : +InvalidNullabilityException::InvalidNullabilityException(std::string const& object_type, Property const& property) : ObjectSchemaPropertyException(object_type, property) { if (property.type == PropertyTypeObject) { - if (!property.is_nullable) { - m_what = "'Object' property '" + property.name + "' must be nullable."; - } + m_what = "'Object' property '" + property.name + "' must be nullable."; } else { - if (property.is_nullable) { - m_what = "Only 'Object' property types are nullable"; - } + m_what = "Array or Mixed property '" + property.name + "' cannot be nullable"; } } -MissingObjectTypeException::MissingObjectTypeException(std::string object_type, Property &property) : +MissingObjectTypeException::MissingObjectTypeException(std::string const& object_type, Property const& property) : ObjectSchemaPropertyException(object_type, property) { m_what = "Target type '" + property.object_type + "' doesn't exist for property '" + property.name + "'."; } -MismatchedPropertiesException::MismatchedPropertiesException(std::string object_type, Property &old_property, Property &new_property) : +MismatchedPropertiesException::MismatchedPropertiesException(std::string const& object_type, Property const& old_property, Property const& new_property) : ObjectSchemaValidationException(object_type), m_old_property(old_property), m_new_property(new_property) { if (new_property.type != old_property.type) { @@ -529,7 +587,7 @@ MismatchedPropertiesException::MismatchedPropertiesException(std::string object_ } } -ChangedPrimaryKeyException::ChangedPrimaryKeyException(std::string object_type, std::string old_primary, std::string new_primary) : ObjectSchemaValidationException(object_type), m_old_primary(old_primary), m_new_primary(new_primary) +ChangedPrimaryKeyException::ChangedPrimaryKeyException(std::string const& object_type, std::string const& old_primary, std::string const& new_primary) : ObjectSchemaValidationException(object_type), m_old_primary(old_primary), m_new_primary(new_primary) { if (old_primary.size()) { m_what = "Property '" + old_primary + "' is no longer a primary key."; @@ -539,14 +597,13 @@ ChangedPrimaryKeyException::ChangedPrimaryKeyException(std::string object_type, } } -InvalidPrimaryKeyException::InvalidPrimaryKeyException(std::string object_type, std::string primary) : +InvalidPrimaryKeyException::InvalidPrimaryKeyException(std::string const& object_type, std::string const& primary) : ObjectSchemaValidationException(object_type), m_primary_key(primary) { m_what = "Specified primary key property '" + primary + "' does not exist."; } -DuplicatePrimaryKeysException::DuplicatePrimaryKeysException(std::string object_type) : ObjectSchemaValidationException(object_type) +DuplicatePrimaryKeysException::DuplicatePrimaryKeysException(std::string const& object_type) : ObjectSchemaValidationException(object_type) { m_what = "Duplicate primary keys for object '" + object_type + "'."; } - diff --git a/object_store.hpp b/object_store.hpp index cacdcd8b..3da680fb 100644 --- a/object_store.hpp +++ b/object_store.hpp @@ -19,18 +19,17 @@ #ifndef REALM_OBJECT_STORE_HPP #define REALM_OBJECT_STORE_HPP -#include -#include -#include -#include -#include - #include "object_schema.hpp" +#include "property.hpp" + +#include + +#include +#include namespace realm { class ObjectSchemaValidationException; - class Schema : public std::map { - }; + class Schema; class ObjectStore { public: @@ -38,47 +37,48 @@ namespace realm { static const uint64_t NotVersioned; // get the last set schema version - static uint64_t get_schema_version(Group *group); + static uint64_t get_schema_version(const Group *group); // checks if the schema in the group is at the given version - static bool is_schema_at_version(realm::Group *group, uint64_t version); + static bool is_schema_at_version(const Group *group, uint64_t version); - // verify a target schema against tables in the given group - // updates the column mapping on all ObjectSchema properties - // throws if the schema is invalid or does not match tables in the given group - static void verify_schema(Group *group, Schema &target_schema, bool allow_missing_tables = false); + // verify that schema from a group and a target schema are compatible + // updates the column mapping on all ObjectSchema properties of the target schema + // throws if the schema is invalid or does not match + static void verify_schema(Schema const& actual_schema, Schema& target_schema, bool allow_missing_tables = false); - // updates the target_column member for all properties based on the column indexes in the passed in group - static void update_column_mapping(Group *group, ObjectSchema &target_schema); - - // determines if you must call update_realm_with_schema for a given realm. - // returns true if there is a schema version mismatch, if there tables which still need to be created, - // or if file format or other changes/updates need to be made - static bool realm_requires_update(Group *group, uint64_t version, Schema &schema); + // determines if a realm with the given old schema needs non-migration + // changes to make it compatible with the given target schema + static bool needs_update(Schema const& old_schema, Schema const& schema); - // updates a Realm to a given target schema/version creating tables and updating indexes as necessary + // updates a Realm from old_schema to the given target schema, creating and updating tables as needed // returns if any changes were made - // passed in schema ar updated with the correct column mapping - // optionally runs migration function/lambda if schema is out of date + // passed in target schema is updated with the correct column mapping + // optionally runs migration function if schema is out of date // NOTE: must be performed within a write transaction typedef std::function MigrationFunction; - static bool update_realm_with_schema(Group *group, uint64_t version, Schema &schema, MigrationFunction migration); + static bool update_realm_with_schema(Group *group, Schema const& old_schema, uint64_t version, + Schema &schema, MigrationFunction migration); // get a table for an object type static realm::TableRef table_for_object_type(Group *group, StringData object_type); + static realm::ConstTableRef table_for_object_type(const Group *group, StringData object_type); // get existing Schema from a group - static Schema schema_from_group(Group *group); + static Schema schema_from_group(const Group *group); // deletes the table for the given type static void delete_data_for_object(Group *group, const StringData &object_type); - private: + // indicates if this group contains any objects + static bool is_empty(const Group *group); + + private: // set a new schema version static void set_schema_version(Group *group, uint64_t version); // check if the realm already has all metadata tables - static bool has_metadata_tables(Group *group); + static bool has_metadata_tables(const Group *group); // create any metadata tables that don't already exist // must be in write transaction to set @@ -89,13 +89,14 @@ namespace realm { // if update existing is true, updates existing tables, otherwise only adds and initializes new tables static bool create_tables(realm::Group *group, Schema &target_schema, bool update_existing); - // verify a target schema against its table, setting the table_column property on each schema object + // verify a target schema against an expected schema, setting the table_column property on each schema object // updates the column mapping on the target_schema // returns array of validation errors - static std::vector verify_object_schema(Group *group, ObjectSchema &target_schema, Schema &schema); + static std::vector verify_object_schema(ObjectSchema const& expected, + ObjectSchema &target_schema); // get primary key property name for object type - static StringData get_primary_key_for_object(Group *group, StringData object_type); + static StringData get_primary_key_for_object(const Group *group, StringData object_type); // sets primary key property for object type // must be in write transaction to set @@ -105,14 +106,11 @@ namespace realm { static std::string table_name_for_object_type(const std::string &class_name); static std::string object_type_for_table_name(const std::string &table_name); - // check if indexes are up to date - if false you need to call update_realm_with_schema - static bool indexes_are_up_to_date(Group *group, Schema &schema); - // returns if any indexes were changed static bool update_indexes(Group *group, Schema &schema); // validates that all primary key properties have unique values - static void validate_primary_column_uniqueness(Group *group, Schema &schema); + static void validate_primary_column_uniqueness(const Group *group, Schema const& schema); friend ObjectSchema; }; @@ -122,7 +120,7 @@ namespace realm { public: ObjectStoreException() = default; ObjectStoreException(const std::string &what) : m_what(what) {} - virtual const char* what() const noexcept { return m_what.c_str(); } + const char* what() const noexcept override { return m_what.c_str(); } protected: std::string m_what; }; @@ -133,17 +131,17 @@ namespace realm { class InvalidSchemaVersionException : public MigrationException { public: InvalidSchemaVersionException(uint64_t old_version, uint64_t new_version); - uint64_t old_version() { return m_old_version; } - uint64_t new_version() { return m_new_version; } + uint64_t old_version() const { return m_old_version; } + uint64_t new_version() const { return m_new_version; } private: uint64_t m_old_version, m_new_version; }; class DuplicatePrimaryKeyValueException : public MigrationException { public: - DuplicatePrimaryKeyValueException(std::string object_type, Property &property); - std::string object_type() { return m_object_type; } - Property &property() { return m_property; } + DuplicatePrimaryKeyValueException(std::string const& object_type, Property const& property); + std::string object_type() const { return m_object_type; } + Property const& property() const { return m_property; } private: std::string m_object_type; Property m_property; @@ -152,87 +150,86 @@ namespace realm { // Schema validation exceptions class SchemaValidationException : public ObjectStoreException { public: - SchemaValidationException(std::vector errors); - std::vector &validation_errors() { return m_validation_errors; } + SchemaValidationException(std::vector const& errors); + std::vector const& validation_errors() const { return m_validation_errors; } private: std::vector m_validation_errors; }; class ObjectSchemaValidationException : public ObjectStoreException { public: - ObjectSchemaValidationException(std::string object_type) : m_object_type(object_type) {} - ObjectSchemaValidationException(std::string object_type, std::string message) : + ObjectSchemaValidationException(std::string const& object_type) : m_object_type(object_type) {} + ObjectSchemaValidationException(std::string const& object_type, std::string const& message) : m_object_type(object_type) { m_what = message; } - std::string object_type() { return m_object_type; } + std::string object_type() const { return m_object_type; } protected: std::string m_object_type; }; class ObjectSchemaPropertyException : public ObjectSchemaValidationException { public: - ObjectSchemaPropertyException(std::string object_type, Property &property) : + ObjectSchemaPropertyException(std::string const& object_type, Property const& property) : ObjectSchemaValidationException(object_type), m_property(property) {} - Property &property() { return m_property; } + Property const& property() const { return m_property; } private: Property m_property; }; class PropertyTypeNotIndexableException : public ObjectSchemaPropertyException { public: - PropertyTypeNotIndexableException(std::string object_type, Property &property); + PropertyTypeNotIndexableException(std::string const& object_type, Property const& property); }; class ExtraPropertyException : public ObjectSchemaPropertyException { public: - ExtraPropertyException(std::string object_type, Property &property); + ExtraPropertyException(std::string const& object_type, Property const& property); }; class MissingPropertyException : public ObjectSchemaPropertyException { public: - MissingPropertyException(std::string object_type, Property &property); + MissingPropertyException(std::string const& object_type, Property const& property); }; class InvalidNullabilityException : public ObjectSchemaPropertyException { public: - InvalidNullabilityException(std::string object_type, Property &property); + InvalidNullabilityException(std::string const& object_type, Property const& property); }; class MissingObjectTypeException : public ObjectSchemaPropertyException { public: - MissingObjectTypeException(std::string object_type, Property &property); + MissingObjectTypeException(std::string const& object_type, Property const& property); }; class DuplicatePrimaryKeysException : public ObjectSchemaValidationException { public: - DuplicatePrimaryKeysException(std::string object_type); + DuplicatePrimaryKeysException(std::string const& object_type); }; class MismatchedPropertiesException : public ObjectSchemaValidationException { public: - MismatchedPropertiesException(std::string object_type, Property &old_property, Property &new_property); - Property &old_property() { return m_old_property; } - Property &new_property() { return m_new_property; } + MismatchedPropertiesException(std::string const& object_type, Property const& old_property, Property const& new_property); + Property const& old_property() const { return m_old_property; } + Property const& new_property() const { return m_new_property; } private: Property m_old_property, m_new_property; }; class ChangedPrimaryKeyException : public ObjectSchemaValidationException { public: - ChangedPrimaryKeyException(std::string object_type, std::string old_primary, std::string new_primary); - std::string old_primary() { return m_old_primary; } - std::string new_primary() { return m_new_primary; } + ChangedPrimaryKeyException(std::string const& object_type, std::string const& old_primary, std::string const& new_primary); + std::string old_primary() const { return m_old_primary; } + std::string new_primary() const { return m_new_primary; } private: std::string m_old_primary, m_new_primary; }; class InvalidPrimaryKeyException : public ObjectSchemaValidationException { public: - InvalidPrimaryKeyException(std::string object_type, std::string primary_key); - std::string primary_key() { return m_primary_key; } + InvalidPrimaryKeyException(std::string const& object_type, std::string const& primary_key); + std::string primary_key() const { return m_primary_key; } private: std::string m_primary_key; }; } #endif /* defined(REALM_OBJECT_STORE_HPP) */ - diff --git a/property.hpp b/property.hpp index 22e6f90c..3883e805 100644 --- a/property.hpp +++ b/property.hpp @@ -46,18 +46,15 @@ namespace realm { }; struct Property { - public: - Property() : object_type(""), is_primary(false), is_indexed(false), is_nullable(false) {} - std::string name; PropertyType type; std::string object_type; - bool is_primary; - bool is_indexed; - bool is_nullable; + bool is_primary = false; + bool is_indexed = false; + bool is_nullable = false; size_t table_column; - bool requires_index() { return is_primary || is_indexed; } + bool requires_index() const { return is_primary || is_indexed; } }; static inline const char *string_for_property_type(PropertyType type) { diff --git a/schema.cpp b/schema.cpp new file mode 100644 index 00000000..911dc4e5 --- /dev/null +++ b/schema.cpp @@ -0,0 +1,102 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "schema.hpp" + +#include "object_schema.hpp" +#include "object_store.hpp" +#include "property.hpp" + +using namespace realm; + +static bool compare_by_name(ObjectSchema const& lft, ObjectSchema const& rgt) { + return lft.name < rgt.name; +} + +Schema::Schema(base types) : base(std::move(types)) { + std::sort(begin(), end(), compare_by_name); +} + +Schema::iterator Schema::find(std::string const& name) +{ + ObjectSchema cmp; + cmp.name = name; + return find(cmp); +} + +Schema::const_iterator Schema::find(std::string const& name) const +{ + return const_cast(this)->find(name); +} + +Schema::iterator Schema::find(ObjectSchema const& object) noexcept +{ + auto it = std::lower_bound(begin(), end(), object, compare_by_name); + if (it != end() && it->name != object.name) { + it = end(); + } + return it; +} + +Schema::const_iterator Schema::find(ObjectSchema const& object) const noexcept +{ + return const_cast(this)->find(object); +} + +void Schema::validate() const +{ + std::vector exceptions; + for (auto const& object : *this) { + const Property *primary = nullptr; + for (auto const& prop : object.properties) { + // check object_type existence + if (!prop.object_type.empty() && find(prop.object_type) == end()) { + exceptions.emplace_back(MissingObjectTypeException(object.name, prop)); + } + + // check nullablity + if (prop.is_nullable) { + if (prop.type == PropertyTypeArray || prop.type == PropertyTypeAny) { + exceptions.emplace_back(InvalidNullabilityException(object.name, prop)); + } + } + else if (prop.type == PropertyTypeObject) { + exceptions.emplace_back(InvalidNullabilityException(object.name, prop)); + } + + // check primary keys + if (prop.is_primary) { + if (primary) { + exceptions.emplace_back(DuplicatePrimaryKeysException(object.name)); + } + primary = ∝ + } + + // check indexable + if (prop.is_indexed) { + if (prop.type != PropertyTypeString && prop.type != PropertyTypeInt) { + exceptions.emplace_back(PropertyTypeNotIndexableException(object.name, prop)); + } + } + } + } + + if (exceptions.size()) { + throw SchemaValidationException(exceptions); + } +} diff --git a/schema.hpp b/schema.hpp new file mode 100644 index 00000000..4d9c7bed --- /dev/null +++ b/schema.hpp @@ -0,0 +1,55 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_SCHEMA_HPP +#define REALM_SCHEMA_HPP + +#include + +namespace realm { +class ObjectSchema; + +class Schema : private std::vector { +private: + using base = std::vector; +public: + // Create a schema from a vector of ObjectSchema + Schema(base types); + + // find an ObjectSchema by name + iterator find(std::string const& name); + const_iterator find(std::string const& name) const; + + // find an ObjectSchema with the same name as the passed in one + iterator find(ObjectSchema const& object) noexcept; + const_iterator find(ObjectSchema const& object) const noexcept; + + // Verify that this schema is internally consistent (i.e. all properties are + // valid, links link to types that actually exist, etc.) + void validate() const; + + using base::iterator; + using base::const_iterator; + using base::begin; + using base::end; + using base::empty; + using base::size; +}; +} + +#endif /* defined(REALM_SCHEMA_HPP) */ diff --git a/shared_realm.cpp b/shared_realm.cpp index 9704b58e..c87dd5d4 100644 --- a/shared_realm.cpp +++ b/shared_realm.cpp @@ -17,55 +17,85 @@ //////////////////////////////////////////////////////////////////////////// #include "shared_realm.hpp" + +#include "external_commit_helper.hpp" +#include "binding_context.hpp" +#include "schema.hpp" +#include "transact_log_handler.hpp" + #include -#include +#include + +#include using namespace realm; +using namespace realm::_impl; RealmCache Realm::s_global_cache; -std::mutex Realm::s_init_mutex; -Realm::Config::Config(const Config& c) : path(c.path), read_only(c.read_only), in_memory(c.in_memory), schema_version(c.schema_version), encryption_key(c.encryption_key), migration_function(c.migration_function) +Realm::Config::Config(const Config& c) +: path(c.path) +, read_only(c.read_only) +, in_memory(c.in_memory) +, cache(c.cache) +, encryption_key(c.encryption_key) +, schema_version(c.schema_version) +, migration_function(c.migration_function) { if (c.schema) { schema = std::make_unique(*c.schema); } } -Realm::Realm(Config &config) : m_config(config), m_thread_id(std::this_thread::get_id()), m_auto_refresh(true), m_in_transaction(false) +Realm::Config::~Config() = default; + +Realm::Config& Realm::Config::operator=(realm::Realm::Config const& c) +{ + if (&c != this) { + *this = Config(c); + } + return *this; +} + +Realm::Realm(Config config) +: m_config(std::move(config)) { try { - if (config.read_only) { - m_read_only_group = std::make_unique(config.path, config.encryption_key.data(), Group::mode_ReadOnly); + if (m_config.read_only) { + m_read_only_group = std::make_unique(m_config.path, m_config.encryption_key.data(), Group::mode_ReadOnly); m_group = m_read_only_group.get(); } else { - m_history = realm::make_client_history(config.path, config.encryption_key.data()); - SharedGroup::DurabilityLevel durability = config.in_memory ? SharedGroup::durability_MemOnly : - SharedGroup::durability_Full; - m_shared_group = std::make_unique(*m_history, durability, config.encryption_key.data()); - m_group = nullptr; + m_history = realm::make_client_history(m_config.path, m_config.encryption_key.data()); + SharedGroup::DurabilityLevel durability = m_config.in_memory ? SharedGroup::durability_MemOnly : + SharedGroup::durability_Full; + m_shared_group = std::make_unique(*m_history, durability, m_config.encryption_key.data()); } } catch (util::File::PermissionDenied const& ex) { - throw RealmFileException(RealmFileException::Kind::PermissionDenied, "Unable to open a realm at path '" + config.path + - "'. Please use a path where your app has " + (config.read_only ? "read" : "read-write") + " permissions."); + throw RealmFileException(RealmFileException::Kind::PermissionDenied, ex.get_path(), + "Unable to open a realm at path '" + ex.get_path() + + "'. Please use a path where your app has " + (m_config.read_only ? "read" : "read-write") + " permissions."); } catch (util::File::Exists const& ex) { - throw RealmFileException(RealmFileException::Kind::Exists, "Unable to open a realm at path '" + config.path + "'"); + throw RealmFileException(RealmFileException::Kind::Exists, ex.get_path(), + "File at path '" + ex.get_path() + "' already exists."); } catch (util::File::AccessError const& ex) { - throw RealmFileException(RealmFileException::Kind::AccessError, "Unable to open a realm at path '" + config.path + "'"); + throw RealmFileException(RealmFileException::Kind::AccessError, ex.get_path(), + "Unable to open a realm at path '" + ex.get_path() + "'"); } - catch (IncompatibleLockFile const&) { - throw RealmFileException(RealmFileException::Kind::IncompatibleLockFile, "Realm file is currently open in another process " - "which cannot share access with this process. All processes sharing a single file must be the same architecture."); + catch (IncompatibleLockFile const& ex) { + throw RealmFileException(RealmFileException::Kind::IncompatibleLockFile, m_config.path, + "Realm file is currently open in another process " + "which cannot share access with this process. All processes sharing a single file must be the same architecture."); } } -Realm::~Realm() -{ - s_global_cache.remove(m_config.path, m_thread_id); +Realm::~Realm() { + if (m_notifier) { // might not exist yet if an error occurred during init + m_notifier->remove_realm(this); + } } Group *Realm::read_group() @@ -76,90 +106,114 @@ Group *Realm::read_group() return m_group; } -SharedRealm Realm::get_shared_realm(Config &config) +SharedRealm Realm::get_shared_realm(Config config) { - SharedRealm realm = s_global_cache.get_realm(config.path); - if (realm) { - if (realm->config().read_only != config.read_only) { - throw MismatchedConfigException("Realm at path already opened with different read permissions."); - } - if (realm->config().in_memory != config.in_memory) { - throw MismatchedConfigException("Realm at path already opened with different inMemory settings."); - } - if (realm->config().encryption_key != config.encryption_key) { - throw MismatchedConfigException("Realm at path already opened with a different encryption key."); - } - if (realm->config().schema_version != config.schema_version && config.schema_version != ObjectStore::NotVersioned) { - throw MismatchedConfigException("Realm at path already opened with different schema version."); - } - // FIXME - enable schma comparison - /*if (realm->config().schema != config.schema) { - throw MismatchedConfigException("Realm at path already opened with different schema"); - }*/ + if (config.cache) { + if (SharedRealm realm = s_global_cache.get_realm(config.path)) { + if (realm->config().read_only != config.read_only) { + throw MismatchedConfigException("Realm at path already opened with different read permissions."); + } + if (realm->config().in_memory != config.in_memory) { + throw MismatchedConfigException("Realm at path already opened with different inMemory settings."); + } + if (realm->config().encryption_key != config.encryption_key) { + throw MismatchedConfigException("Realm at path already opened with a different encryption key."); + } + if (realm->config().schema_version != config.schema_version && config.schema_version != ObjectStore::NotVersioned) { + throw MismatchedConfigException("Realm at path already opened with different schema version."); + } + // FIXME - enable schma comparison + /*if (realm->config().schema != config.schema) { + throw MismatchedConfigException("Realm at path already opened with different schema"); + }*/ + realm->m_config.migration_function = config.migration_function; - realm->m_config.migration_function = config.migration_function; - - return realm; + return realm; + } } - realm = SharedRealm(new Realm(config)); + SharedRealm realm(new Realm(std::move(config))); + + auto target_schema = std::move(realm->m_config.schema); + auto target_schema_version = realm->m_config.schema_version; + realm->m_config.schema_version = ObjectStore::get_schema_version(realm->read_group()); // we want to ensure we are only initializing a single realm at a time + static std::mutex s_init_mutex; std::lock_guard lock(s_init_mutex); - - uint64_t old_version = ObjectStore::get_schema_version(realm->read_group()); - if (!realm->m_config.schema) { - // get schema from group and skip validation - realm->m_config.schema_version = old_version; - realm->m_config.schema = std::make_unique(ObjectStore::schema_from_group(realm->read_group())); - } - else if (realm->m_config.read_only) { - if (old_version == ObjectStore::NotVersioned) { - throw UnitializedRealmException("Can't open an un-initizliazed Realm without a Schema"); - } - ObjectStore::verify_schema(realm->read_group(), *realm->m_config.schema, true); - } - else if(auto existing = s_global_cache.get_any_realm(realm->config().path)) { + if (auto existing = s_global_cache.get_any_realm(realm->config().path)) { // if there is an existing realm at the current path steal its schema/column mapping // FIXME - need to validate that schemas match realm->m_config.schema = std::make_unique(*existing->m_config.schema); + + realm->m_notifier = existing->m_notifier; + realm->m_notifier->add_realm(realm.get()); } else { - // its a non-cached realm so update/migrate if needed - realm->update_schema(*realm->m_config.schema, realm->m_config.schema_version); + realm->m_notifier = std::make_shared(realm.get()); + + // otherwise get the schema from the group + realm->m_config.schema = std::make_unique(ObjectStore::schema_from_group(realm->read_group())); + + // if a target schema is supplied, verify that it matches or migrate to + // it, as neeeded + if (target_schema) { + if (realm->m_config.read_only) { + if (realm->m_config.schema_version == ObjectStore::NotVersioned) { + throw UnitializedRealmException("Can't open an un-initialized Realm without a Schema"); + } + target_schema->validate(); + ObjectStore::verify_schema(*realm->m_config.schema, *target_schema, true); + realm->m_config.schema = std::move(target_schema); + } + else { + realm->update_schema(std::move(target_schema), target_schema_version); + } + } } - s_global_cache.cache_realm(realm, realm->m_thread_id); + if (config.cache) { + s_global_cache.cache_realm(realm, realm->m_thread_id); + } return realm; } -bool Realm::update_schema(Schema &schema, uint64_t version) +bool Realm::update_schema(std::unique_ptr schema, uint64_t version) { - bool changed = false; - Config old_config(m_config); + schema->validate(); - // set new version/schema - if (m_config.schema.get() != &schema) { - m_config.schema = std::make_unique(schema); + bool needs_update = !m_config.read_only && (m_config.schema_version != version || ObjectStore::needs_update(*m_config.schema, *schema)); + if (!needs_update) { + ObjectStore::verify_schema(*m_config.schema, *schema, m_config.read_only); + m_config.schema = std::move(schema); + m_config.schema_version = version; + return false; } + + // Store the old config/schema for the migration function, and update + // our schema to the new one + auto old_schema = std::move(m_config.schema); + Config old_config(m_config); + old_config.read_only = true; + old_config.schema = std::move(old_schema); + + m_config.schema = std::move(schema); m_config.schema_version = version; - try { - if (!m_config.read_only && ObjectStore::realm_requires_update(read_group(), version, schema)) { - // keep old copy to pass to migration function - old_config.read_only = true; - SharedRealm old_realm = SharedRealm(new Realm(old_config)), updated_realm = shared_from_this(); + auto migration_function = [&](Group*, Schema&) { + SharedRealm old_realm(new Realm(old_config)); + auto updated_realm = shared_from_this(); + m_config.migration_function(old_realm, updated_realm); + }; - // update and migrate - begin_transaction(); - changed = ObjectStore::update_realm_with_schema(read_group(), version, *m_config.schema, [=](__unused Group *group, __unused Schema &target_schema) { - m_config.migration_function(old_realm, updated_realm); - }); - commit_transaction(); - } - else { - ObjectStore::verify_schema(read_group(), *m_config.schema, m_config.read_only); - } + try { + // update and migrate + begin_transaction(); + bool changed = ObjectStore::update_realm_with_schema(read_group(), *old_config.schema, + version, *m_config.schema, + migration_function); + commit_transaction(); + return changed; } catch (...) { if (is_in_transaction()) { @@ -169,7 +223,6 @@ bool Realm::update_schema(Schema &schema, uint64_t version) m_config.schema = std::move(old_config.schema); throw; } - return changed; } static void check_read_write(Realm *realm) @@ -179,7 +232,7 @@ static void check_read_write(Realm *realm) } } -void Realm::verify_thread() +void Realm::verify_thread() const { if (m_thread_id != std::this_thread::get_id()) { throw IncorrectThreadException("Realm accessed from incorrect thread."); @@ -195,18 +248,11 @@ void Realm::begin_transaction() throw InvalidTransactionException("The Realm is already in a write transaction"); } - // if the upgrade to write will move the transaction forward, announce the change after promoting - bool announce = m_shared_group->has_changed(); - // make sure we have a read transaction read_group(); - LangBindHelper::promote_to_write(*m_shared_group, *m_history); + transaction::begin(*m_shared_group, *m_history, m_binding_context.get()); m_in_transaction = true; - - if (announce) { - send_local_notifications(DidChangeNotification); - } } void Realm::commit_transaction() @@ -218,11 +264,9 @@ void Realm::commit_transaction() throw InvalidTransactionException("Can't commit a non-existing write transaction"); } - LangBindHelper::commit_and_continue_as_read(*m_shared_group); m_in_transaction = false; - - send_external_notifications(); - send_local_notifications(DidChangeNotification); + transaction::commit(*m_shared_group, *m_history, m_binding_context.get()); + m_notifier->notify_others(); } void Realm::cancel_transaction() @@ -234,11 +278,10 @@ void Realm::cancel_transaction() throw InvalidTransactionException("Can't cancel a non-existing write transaction"); } - LangBindHelper::rollback_and_continue_as_read(*m_shared_group, *m_history); m_in_transaction = false; + transaction::cancel(*m_shared_group, *m_history, m_binding_context.get()); } - void Realm::invalidate() { verify_thread(); @@ -259,20 +302,21 @@ bool Realm::compact() { verify_thread(); - bool success = false; + if (m_config.read_only) { + throw InvalidTransactionException("Can't compact a read-only Realm"); + } if (m_in_transaction) { throw InvalidTransactionException("Can't compact a Realm within a write transaction"); } + Group* group = read_group(); for (auto &object_schema : *m_config.schema) { - ObjectStore::table_for_object_type(read_group(), object_schema.first)->optimize(); + ObjectStore::table_for_object_type(group, object_schema.name)->optimize(); } - m_shared_group->end_read(); - success = m_shared_group->compact(); - m_shared_group->begin_read(); + m_group = nullptr; - return success; + return m_shared_group->compact(); } void Realm::notify() @@ -280,24 +324,17 @@ void Realm::notify() verify_thread(); if (m_shared_group->has_changed()) { // Throws + if (m_binding_context) { + m_binding_context->changes_available(); + } if (m_auto_refresh) { if (m_group) { - LangBindHelper::advance_read(*m_shared_group, *m_history); + transaction::advance(*m_shared_group, *m_history, m_binding_context.get()); + } + else if (m_binding_context) { + m_binding_context->did_change({}, {}); } - send_local_notifications(DidChangeNotification); } - else { - send_local_notifications(RefreshRequiredNotification); - } - } -} - - -void Realm::send_local_notifications(const std::string &type) -{ - verify_thread(); - for (NotificationFunction notification : m_notifications) { - (*notification)(type); } } @@ -318,17 +355,41 @@ bool Realm::refresh() } if (m_group) { - LangBindHelper::advance_read(*m_shared_group, *m_history); + transaction::advance(*m_shared_group, *m_history, m_binding_context.get()); } else { // Create the read transaction read_group(); } - send_local_notifications(DidChangeNotification); return true; } +uint64_t Realm::get_schema_version(const realm::Realm::Config &config) +{ + if (auto existing_realm = s_global_cache.get_any_realm(config.path)) { + return existing_realm->config().schema_version; + } + + return ObjectStore::get_schema_version(Realm(config).read_group()); +} + +void Realm::close() +{ + invalidate(); + + if (m_notifier) { + m_notifier->remove_realm(this); + } + + m_group = nullptr; + m_shared_group = nullptr; + m_history = nullptr; + m_read_only_group = nullptr; + m_notifier = nullptr; + m_binding_context = nullptr; +} + SharedRealm RealmCache::get_realm(const std::string &path, std::thread::id thread_id) { std::lock_guard lock(m_mutex); @@ -355,11 +416,12 @@ SharedRealm RealmCache::get_any_realm(const std::string &path) return SharedRealm(); } - for (auto thread_iter = path_iter->second.begin(); thread_iter != path_iter->second.end(); thread_iter++) { + auto thread_iter = path_iter->second.begin(); + while (thread_iter != path_iter->second.end()) { if (auto realm = thread_iter->second.lock()) { return realm; } - path_iter->second.erase(thread_iter); + path_iter->second.erase(thread_iter++); } return SharedRealm(); @@ -397,3 +459,16 @@ void RealmCache::cache_realm(SharedRealm &realm, std::thread::id thread_id) } } +void RealmCache::clear() +{ + std::lock_guard lock(m_mutex); + for (auto const& path : m_cache) { + for (auto const& thread : path.second) { + if (auto realm = thread.second.lock()) { + realm->close(); + } + } + } + + m_cache.clear(); +} diff --git a/shared_realm.hpp b/shared_realm.hpp index 2807549d..81b14bf7 100644 --- a/shared_realm.hpp +++ b/shared_realm.hpp @@ -19,21 +19,24 @@ #ifndef REALM_REALM_HPP #define REALM_REALM_HPP +#include #include #include #include -#include -#include -#include #include "object_store.hpp" namespace realm { - class RealmCache; + class ClientHistory; class Realm; + class RealmCache; + class BindingContext; typedef std::shared_ptr SharedRealm; typedef std::weak_ptr WeakRealm; - class ClientHistory; + + namespace _impl { + class ExternalCommitHelper; + } class Realm : public std::enable_shared_from_this { @@ -43,88 +46,89 @@ namespace realm { struct Config { std::string path; - bool read_only; - bool in_memory; - StringData encryption_key; + bool read_only = false; + bool in_memory = false; + bool cache = true; + std::vector encryption_key; std::unique_ptr schema; - uint64_t schema_version; + uint64_t schema_version = ObjectStore::NotVersioned; MigrationFunction migration_function; - Config() : read_only(false), in_memory(false), schema_version(ObjectStore::NotVersioned) {}; + Config() = default; + Config(Config&&) = default; Config(const Config& c); + ~Config(); + + Config& operator=(Config const&); + Config& operator=(Config&&) = default; }; // Get a cached Realm or create a new one if no cached copies exists - // Caching is done by path - mismatches for inMemory and readOnly Config properties - // will raise an exception - // If schema/schema_version is specified, update_schema is called automatically on the realm - // and a migration is performed. If not specified, the schema version and schema are dynamically - // read from the the existing Realm. - static SharedRealm get_shared_realm(Config &config); + // Caching is done by path - mismatches for in_memory and read_only + // Config properties will raise an exception + // If schema/schema_version is specified, update_schema is called + // automatically on the realm and a migration is performed. If not + // specified, the schema version and schema are dynamically read from + // the the existing Realm. + static SharedRealm get_shared_realm(Config config); - // Updates a Realm to a given target schema/version creating tables and updating indexes as necessary - // Uses the existing migration function on the Config, and the resulting Schema and version with updated + // Updates a Realm to a given target schema/version creating tables and + // updating indexes as necessary. Uses the existing migration function + // on the Config, and the resulting Schema and version with updated // column mappings are set on the realms config upon success. // returns if any changes were made - bool update_schema(Schema &schema, uint64_t version); + bool update_schema(std::unique_ptr schema, uint64_t version); + + static uint64_t get_schema_version(Config const& config); const Config &config() const { return m_config; } void begin_transaction(); void commit_transaction(); void cancel_transaction(); - bool is_in_transaction() { return m_in_transaction; } + bool is_in_transaction() const { return m_in_transaction; } bool refresh(); void set_auto_refresh(bool auto_refresh) { m_auto_refresh = auto_refresh; } - bool auto_refresh() { return m_auto_refresh; } + bool auto_refresh() const { return m_auto_refresh; } void notify(); - typedef std::shared_ptr> NotificationFunction; - void add_notification(NotificationFunction ¬ification) { m_notifications.insert(notification); } - void remove_notification(NotificationFunction notification) { m_notifications.erase(notification); } - void remove_all_notifications() { m_notifications.clear(); } - void invalidate(); bool compact(); std::thread::id thread_id() const { return m_thread_id; } - void verify_thread(); + void verify_thread() const; - const std::string RefreshRequiredNotification = "RefreshRequiredNotification"; - const std::string DidChangeNotification = "DidChangeNotification"; + // Close this Realm and remove it from the cache. Continuing to use a + // Realm after closing it will produce undefined behavior. + void close(); + + ~Realm(); private: - Realm(Config &config); + Realm(Config config); Config m_config; - std::thread::id m_thread_id; - bool m_in_transaction; - bool m_auto_refresh; - - std::set m_notifications; - void send_local_notifications(const std::string ¬ification); - - typedef std::unique_ptr> ExternalNotificationFunction; - void send_external_notifications() { if (m_external_notifier) (*m_external_notifier)(); } + std::thread::id m_thread_id = std::this_thread::get_id(); + bool m_in_transaction = false; + bool m_auto_refresh = true; std::unique_ptr m_history; std::unique_ptr m_shared_group; std::unique_ptr m_read_only_group; - Group *m_group; + Group *m_group = nullptr; - static std::mutex s_init_mutex; - static RealmCache s_global_cache; + std::shared_ptr<_impl::ExternalCommitHelper> m_notifier; public: - ~Realm(); - ExternalNotificationFunction m_external_notifier; + std::unique_ptr m_binding_context; // FIXME private Group *read_group(); + static RealmCache s_global_cache; }; class RealmCache @@ -134,6 +138,7 @@ namespace realm { SharedRealm get_any_realm(const std::string &path); void remove(const std::string &path, std::thread::id thread_id); void cache_realm(SharedRealm &realm, std::thread::id thread_id = std::this_thread::get_id()); + void clear(); private: std::map> m_cache; @@ -150,7 +155,7 @@ namespace realm { /** Thrown if the user does not have permission to open or create the specified file in the specified access mode when the realm is opened. */ PermissionDenied, - /** Thrown if no_create was specified and the file did already exist when the realm is opened. */ + /** Thrown if create_Always was specified and the file did already exist when the realm is opened. */ Exists, /** Thrown if no_create was specified and the file was not found when the realm is opened. */ NotFound, @@ -159,11 +164,14 @@ namespace realm { architecture mismatch. */ IncompatibleLockFile, }; - RealmFileException(Kind kind, std::string message) : std::runtime_error(message), m_kind(kind) {} + RealmFileException(Kind kind, std::string path, std::string message) : + std::runtime_error(std::move(message)), m_kind(kind), m_path(std::move(path)) {} Kind kind() const { return m_kind; } + const std::string& path() const { return m_path; } private: Kind m_kind; + std::string m_path; }; class MismatchedConfigException : public std::runtime_error