Merge pull request #4 from realm/tg-realm

Merge objectstore changes from obj-c

apple/external_commit_helper.cpp

@@ -0,0 +1,250 @@
+////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2015 Realm Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////
+
+#include "external_commit_helper.hpp"
+
+#include "shared_realm.hpp"
+
+#include <assert.h>
+#include <sys/event.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <system_error>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sstream>
+
+using namespace realm;
+
+namespace {
+// Write a byte to a pipe to notify anyone waiting for data on the pipe
+void notify_fd(int fd)
+{
+    while (true) {
+        char c = 0;
+        ssize_t ret = write(fd, &c, 1);
+        if (ret == 1) {
+            break;
+        }
+
+        // If the pipe's buffer is full, we need to read some of the old data in
+        // it to make space. We don't just read in the code waiting for
+        // notifications so that we can notify multiple waiters with a single
+        // write.
+        assert(ret == -1 && errno == EAGAIN);
+        char buff[1024];
+        read(fd, buff, sizeof buff);
+    }
+}
+} // anonymous namespace
+
+void ExternalCommitHelper::FdHolder::close()
+{
+        if (m_fd != -1) {
+            ::close(m_fd);
+        }
+        m_fd = -1;
+
+}
+
+// Inter-thread and inter-process notifications of changes are done using a
+// named pipe in the filesystem next to the Realm file. Everyone who wants to be
+// notified of commits waits for data to become available on the pipe, and anyone
+// who commits a write transaction writes data to the pipe after releasing the
+// write lock. Note that no one ever actually *reads* from the pipe: the data
+// actually written is meaningless, and trying to read from a pipe from multiple
+// processes at once is fraught with race conditions.
+
+// When a RLMRealm instance is created, we add a CFRunLoopSource to the current
+// thread's runloop. On each cycle of the run loop, the run loop checks each of
+// its sources for work to do, which in the case of CFRunLoopSource is just
+// checking if CFRunLoopSourceSignal has been called since the last time it ran,
+// and if so invokes the function pointer supplied when the source is created,
+// which in our case just invokes `[realm handleExternalChange]`.
+
+// Listening for external changes is done using kqueue() on a background thread.
+// kqueue() lets us efficiently wait until the amount of data which can be read
+// from one or more file descriptors has changed, and tells us which of the file
+// descriptors it was that changed. We use this to wait on both the shared named
+// pipe, and a local anonymous pipe. When data is written to the named pipe, we
+// signal the runloop source and wake up the target runloop, and when data is
+// written to the anonymous pipe the background thread removes the runloop
+// source from the runloop and and shuts down.
+ExternalCommitHelper::ExternalCommitHelper(Realm* realm)
+{
+    add_realm(realm);
+
+    m_kq = kqueue();
+    if (m_kq == -1) {
+        throw std::system_error(errno, std::system_category());
+    }
+
+    auto path = realm->config().path + ".note";
+
+    // Create and open the named pipe
+    int ret = mkfifo(path.c_str(), 0600);
+    if (ret == -1) {
+        int err = errno;
+        if (err == ENOTSUP) {
+            // Filesystem doesn't support named pipes, so try putting it in tmp instead
+            // Hash collisions are okay here because they just result in doing
+            // extra work, as opposed to correctness problems
+            std::ostringstream ss;
+            ss << getenv("TMPDIR");
+            ss << "realm_" << std::hash<std::string>()(path) << ".note";
+            path = ss.str();
+            ret = mkfifo(path.c_str(), 0600);
+            err = errno;
+        }
+        // the fifo already existing isn't an error
+        if (ret == -1 && err != EEXIST) {
+            throw std::system_error(err, std::system_category());
+        }
+    }
+
+    m_notify_fd = open(path.c_str(), O_RDWR);
+    if (m_notify_fd == -1) {
+        throw std::system_error(errno, std::system_category());
+    }
+
+    // Make writing to the pipe return -1 when the pipe's buffer is full
+    // rather than blocking until there's space available
+    ret = fcntl(m_notify_fd, F_SETFL, O_NONBLOCK);
+    if (ret == -1) {
+        throw std::system_error(errno, std::system_category());
+    }
+
+    // Create the anonymous pipe
+    int pipeFd[2];
+    ret = pipe(pipeFd);
+    if (ret == -1) {
+        throw std::system_error(errno, std::system_category());
+    }
+
+    m_shutdown_read_fd = pipeFd[0];
+    m_shutdown_write_fd = pipeFd[1];
+
+    // Use the minimum allowed stack size, as we need very little in our listener
+    // https://developer.apple.com/library/ios/documentation/Cocoa/Conceptual/Multithreading/CreatingThreads/CreatingThreads.html#//apple_ref/doc/uid/10000057i-CH15-SW7
+    pthread_attr_t attr;
+    pthread_attr_init(&attr);
+    pthread_attr_setstacksize(&attr, 16 * 1024);
+
+    auto fn = [](void *self) -> void * {
+        static_cast<ExternalCommitHelper *>(self)->listen();
+        return nullptr;
+    };
+    ret = pthread_create(&m_thread, &attr, fn, this);
+    pthread_attr_destroy(&attr);
+    if (ret != 0) {
+        throw std::system_error(errno, std::system_category());
+    }
+}
+
+ExternalCommitHelper::~ExternalCommitHelper()
+{
+    REALM_ASSERT_DEBUG(m_realms.empty());
+    notify_fd(m_shutdown_write_fd);
+    pthread_join(m_thread, nullptr); // Wait for the thread to exit
+}
+
+void ExternalCommitHelper::add_realm(realm::Realm* realm)
+{
+    std::lock_guard<std::mutex> lock(m_realms_mutex);
+
+    // Create the runloop source
+    CFRunLoopSourceContext ctx{};
+    ctx.info = realm;
+    ctx.perform = [](void* info) {
+        static_cast<Realm*>(info)->notify();
+    };
+
+    CFRunLoopRef runloop = CFRunLoopGetCurrent();
+    CFRetain(runloop);
+    CFRunLoopSourceRef signal = CFRunLoopSourceCreate(kCFAllocatorDefault, 0, &ctx);
+    CFRunLoopAddSource(runloop, signal, kCFRunLoopDefaultMode);
+
+    m_realms.push_back({realm, runloop, signal});
+}
+
+void ExternalCommitHelper::remove_realm(realm::Realm* realm)
+{
+    std::lock_guard<std::mutex> lock(m_realms_mutex);
+    for (auto it = m_realms.begin(); it != m_realms.end(); ++it) {
+        if (it->realm == realm) {
+            CFRunLoopSourceInvalidate(it->signal);
+            CFRelease(it->signal);
+            CFRelease(it->runloop);
+            m_realms.erase(it);
+            return;
+        }
+    }
+    REALM_TERMINATE("Realm not registered");
+}
+
+void ExternalCommitHelper::listen()
+{
+    pthread_setname_np("RLMRealm notification listener");
+
+
+    // Set up the kqueue
+    // EVFILT_READ indicates that we care about data being available to read
+    // on the given file descriptor.
+    // EV_CLEAR makes it wait for the amount of data available to be read to
+    // change rather than just returning when there is any data to read.
+    struct kevent ke[2];
+    EV_SET(&ke[0], m_notify_fd, EVFILT_READ, EV_ADD | EV_CLEAR, 0, 0, 0);
+    EV_SET(&ke[1], m_shutdown_read_fd, EVFILT_READ, EV_ADD | EV_CLEAR, 0, 0, 0);
+    int ret = kevent(m_kq, ke, 2, nullptr, 0, nullptr);
+    assert(ret == 0);
+
+    while (true) {
+        struct kevent event;
+        // Wait for data to become on either fd
+        // Return code is number of bytes available or -1 on error
+        ret = kevent(m_kq, nullptr, 0, &event, 1, nullptr);
+        assert(ret >= 0);
+        if (ret == 0) {
+            // Spurious wakeup; just wait again
+            continue;
+        }
+
+        // Check which file descriptor had activity: if it's the shutdown
+        // pipe, then someone called -stop; otherwise it's the named pipe
+        // and someone committed a write transaction
+        if (event.ident == (uint32_t)m_shutdown_read_fd) {
+            return;
+        }
+        assert(event.ident == (uint32_t)m_notify_fd);
+
+        std::lock_guard<std::mutex> lock(m_realms_mutex);
+        for (auto const& realm : m_realms) {
+            CFRunLoopSourceSignal(realm.signal);
+            // Signalling the source makes it run the next time the runloop gets
+            // to it, but doesn't make the runloop start if it's currently idle
+            // waiting for events
+            CFRunLoopWakeUp(realm.runloop);
+        }
+    }
+}
+
+void ExternalCommitHelper::notify_others()
+{
+    notify_fd(m_notify_fd);
+}
+

apple/external_commit_helper.hpp

@@ -0,0 +1,92 @@
+////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2015 Realm Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////
+
+#ifndef REALM_EXTERNAL_COMMIT_HELPER_HPP
+#define REALM_EXTERNAL_COMMIT_HELPER_HPP
+
+#include <CoreFoundation/CFRunLoop.h>
+#include <mutex>
+#include <vector>
+
+namespace realm {
+class Realm;
+
+class ExternalCommitHelper {
+public:
+    ExternalCommitHelper(Realm* realm);
+    ~ExternalCommitHelper();
+
+    void notify_others();
+    void add_realm(Realm* realm);
+    void remove_realm(Realm* realm);
+
+private:
+    // A RAII holder for a file descriptor which automatically closes the wrapped
+    // fd when it's deallocated
+    class FdHolder {
+    public:
+        FdHolder() = default;
+        ~FdHolder() { close(); }
+        operator int() const { return m_fd; }
+
+        FdHolder& operator=(int newFd) {
+            close();
+            m_fd = newFd;
+            return *this;
+        }
+
+    private:
+        int m_fd = -1;
+        void close();
+
+        FdHolder& operator=(FdHolder const&) = delete;
+        FdHolder(FdHolder const&) = delete;
+    };
+
+    struct PerRealmInfo {
+        Realm* realm;
+        CFRunLoopRef runloop;
+        CFRunLoopSourceRef signal;
+    };
+
+    void listen();
+
+    // Currently registered realms and the signal for delivering notifications
+    // to them
+    std::vector<PerRealmInfo> m_realms;
+
+    // Mutex which guards m_realms
+    std::mutex m_realms_mutex;
+
+    // The listener thread
+    pthread_t m_thread;
+
+    // Read-write file descriptor for the named pipe which is waited on for
+    // changes and written to when a commit is made
+    FdHolder m_notify_fd;
+    // File descriptor for the kqueue
+    FdHolder m_kq;
+    // The two ends of an anonymous pipe used to notify the kqueue() thread that
+    // it should be shut down.
+    FdHolder m_shutdown_read_fd;
+    FdHolder m_shutdown_write_fd;
+};
+
+} // namespace realm
+
+#endif /* REALM_EXTERNAL_COMMIT_HELPER_HPP */

index_set.cpp

@@ -0,0 +1,92 @@
+////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2015 Realm Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////
+
+#include "index_set.hpp"
+
+using namespace realm;
+
+IndexSet::iterator IndexSet::find(size_t index)
+{
+    for (auto it = m_ranges.begin(), end = m_ranges.end(); it != end; ++it) {
+        if (it->second > index)
+            return it;
+    }
+    return m_ranges.end();
+}
+
+void IndexSet::add(size_t index)
+{
+    do_add(find(index), index);
+}
+
+void IndexSet::do_add(iterator it, size_t index)
+{
+    bool more_before = it != m_ranges.begin(), valid = it != m_ranges.end();
+    if (valid && it->first <= index && it->second > index) {
+        // index is already in set
+    }
+    else if (more_before && (it - 1)->second == index) {
+        // index is immediately after an existing range
+        ++(it - 1)->second;
+    }
+    else if (more_before && valid && (it - 1)->second == it->first) {
+        // index joins two existing ranges
+        (it - 1)->second = it->second;
+        m_ranges.erase(it);
+    }
+    else if (valid && it->first == index + 1) {
+        // index is immediately before an existing range
+        --it->first;
+    }
+    else {
+        // index is not next to an existing range
+        m_ranges.insert(it, {index, index + 1});
+    }
+}
+
+void IndexSet::set(size_t len)
+{
+    m_ranges.clear();
+    if (len) {
+        m_ranges.push_back({0, len});
+    }
+}
+
+void IndexSet::insert_at(size_t index)
+{
+    auto pos = find(index);
+    if (pos != m_ranges.end()) {
+        if (pos->first >= index)
+            ++pos->first;
+        ++pos->second;
+        for (auto it = pos + 1; it != m_ranges.end(); ++it) {
+            ++it->first;
+            ++it->second;
+        }
+    }
+    do_add(pos, index);
+}
+
+void IndexSet::add_shifted(size_t index)
+{
+    auto it = m_ranges.begin();
+    for (auto end = m_ranges.end(); it != end && it->first <= index; ++it) {
+        index += it->second - it->first;
+    }
+    do_add(it, index);
+}

index_set.hpp

@@ -0,0 +1,62 @@
+////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2015 Realm Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////
+
+#ifndef REALM_INDEX_SET_HPP
+#define REALM_INDEX_SET_HPP
+
+#include <vector>
+
+namespace realm {
+class IndexSet {
+public:
+    using value_type = std::pair<size_t, size_t>;
+    using iterator = std::vector<value_type>::iterator;
+    using const_iterator = std::vector<value_type>::const_iterator;
+
+    const_iterator begin() const { return m_ranges.begin(); }
+    const_iterator end() const { return m_ranges.end(); }
+    bool empty() const { return m_ranges.empty(); }
+    size_t size() const { return m_ranges.size(); }
+
+    // Add an index to the set, doing nothing if it's already present
+    void add(size_t index);
+
+    // Remove all indexes from the set and then add a single range starting from
+    // zero with the given length
+    void set(size_t len);
+
+    // Insert an index at the given position, shifting existing indexes at or
+    // after that point back by one
+    void insert_at(size_t index);
+
+    // Add an index which has had all of the ranges in the set before it removed
+    void add_shifted(size_t index);
+
+private:
+    std::vector<value_type> m_ranges;
+
+    // Find the range which contains the index, or the first one after it if
+    // none do
+    iterator find(size_t index);
+    // Insert the index before the given position, combining existing ranges as
+    // applicable
+    void do_add(iterator pos, size_t index);
+};
+} // namespace realm
+
+#endif // REALM_INDEX_SET_HPP

object_schema.cpp

@@ -17,16 +17,19 @@
 ////////////////////////////////////////////////////////////////////////////
 
 #include "object_schema.hpp"
+
 #include "object_store.hpp"
+#include "property.hpp"
 
 #include <realm/group_shared.hpp>
 #include <realm/link_view.hpp>
 
 using namespace realm;
 
-ObjectSchema::ObjectSchema(Group *group, const std::string &name) : name(name) {
+ObjectSchema::~ObjectSchema() = default;
-    TableRef tableRef = ObjectStore::table_for_object_type(group, name);
+
-    Table *table = tableRef.get();
+ObjectSchema::ObjectSchema(const Group *group, const std::string &name) : name(name) {
+    ConstTableRef table = ObjectStore::table_for_object_type(group, name);
 
     size_t count = table->get_column_count();
     properties.reserve(count);
@@ -40,7 +43,7 @@ ObjectSchema::ObjectSchema(Group *group, const std::string &name) : name(name) {
         property.table_column = col;
         if (property.type == PropertyTypeObject || property.type == PropertyTypeArray) {
             // set link type for objects and arrays
-            realm::TableRef linkTable = table->get_link_target(col);
+            ConstTableRef linkTable = table->get_link_target(col);
             property.object_type = ObjectStore::object_type_for_table_name(linkTable->get_name().data());
         }
         properties.push_back(std::move(property));
@@ -56,12 +59,15 @@ ObjectSchema::ObjectSchema(Group *group, const std::string &name) : name(name) {
     }
 }
 
-Property *ObjectSchema::property_for_name(const std::string &name) {
+Property *ObjectSchema::property_for_name(StringData name) {
-    for (auto& prop:properties) {
+    for (auto& prop : properties) {
-        if (prop.name == name) {
+        if (StringData(prop.name) == name) {
             return &prop;
         }
     }
     return nullptr;
 }
 
+const Property *ObjectSchema::property_for_name(StringData name) const {
+    return const_cast<ObjectSchema *>(this)->property_for_name(name);
+}

object_schema.hpp

@@ -19,30 +19,36 @@
 #ifndef REALM_OBJECT_SCHEMA_HPP
 #define REALM_OBJECT_SCHEMA_HPP
 
+#include <realm/string_data.hpp>
+
 #include <string>
 #include <vector>
 
-#include "property.hpp"
-
 namespace realm {
+    class Property;
     class Group;
 
     class ObjectSchema {
     public:
-        ObjectSchema() {}
+        ObjectSchema() = default;
+        ~ObjectSchema();
 
         // create object schema from existing table
         // if no table is provided it is looked up in the group
-        ObjectSchema(Group *group, const std::string &name);
+        ObjectSchema(const Group *group, const std::string &name);
 
         std::string name;
         std::vector<Property> properties;
         std::string primary_key;
 
-        Property *property_for_name(const std::string &name);
+        Property *property_for_name(StringData name);
+        const Property *property_for_name(StringData name) const;
         Property *primary_key_property() {
             return property_for_name(primary_key);
         }
+        const Property *primary_key_property() const {
+            return property_for_name(primary_key);
+        }
     };
 }
 

object_store.cpp

@@ -18,9 +18,11 @@
 
 #include "object_store.hpp"
 
+#include "schema.hpp"
+
 #include <realm/group.hpp>
-#include <realm/table.hpp>
 #include <realm/link_view.hpp>
+#include <realm/table.hpp>
 #include <realm/table_view.hpp>
 #include <realm/util/assert.hpp>
 
@@ -45,7 +47,7 @@ const size_t c_object_table_prefix_length = c_object_table_prefix.length();
 
 const uint64_t ObjectStore::NotVersioned = std::numeric_limits<uint64_t>::max();
 
-bool ObjectStore::has_metadata_tables(Group *group) {
+bool ObjectStore::has_metadata_tables(const Group *group) {
     return group->get_table(c_primaryKeyTableName) && group->get_table(c_metadataTableName);
 }
 
@@ -71,8 +73,8 @@ bool ObjectStore::create_metadata_tables(Group *group) {
     return changed;
 }
 
-uint64_t ObjectStore::get_schema_version(Group *group) {
+uint64_t ObjectStore::get_schema_version(const Group *group) {
-    TableRef table = group->get_table(c_metadataTableName);
+    ConstTableRef table = group->get_table(c_metadataTableName);
     if (!table || table->get_column_count() == 0) {
         return ObjectStore::NotVersioned;
     }
@@ -84,8 +86,8 @@ void ObjectStore::set_schema_version(Group *group, uint64_t version) {
     table->set_int(c_versionColumnIndex, c_zeroRowIndex, version);
 }
 
-StringData ObjectStore::get_primary_key_for_object(Group *group, StringData object_type) {
+StringData ObjectStore::get_primary_key_for_object(const Group *group, StringData object_type) {
-    TableRef table = group->get_table(c_primaryKeyTableName);
+    ConstTableRef table = group->get_table(c_primaryKeyTableName);
     if (!table) {
         return "";
     }
@@ -132,26 +134,34 @@ TableRef ObjectStore::table_for_object_type(Group *group, StringData object_type
     return group->get_table(table_name_for_object_type(object_type));
 }
 
+ConstTableRef ObjectStore::table_for_object_type(const Group *group, StringData object_type) {
+    return group->get_table(table_name_for_object_type(object_type));
+}
+
 TableRef ObjectStore::table_for_object_type_create_if_needed(Group *group, const StringData &object_type, bool &created) {
     return group->get_or_add_table(table_name_for_object_type(object_type), &created);
 }
 
-static inline bool property_has_changed(Property &p1, Property &p2) {
+static inline bool property_has_changed(Property const& p1, Property const& p2) {
-    return p1.type != p2.type || p1.name != p2.name || p1.object_type != p2.object_type || p1.is_nullable != p2.is_nullable;
+    return p1.type != p2.type
+        || p1.name != p2.name
+        || p1.object_type != p2.object_type
+        || p1.is_nullable != p2.is_nullable;
 }
 
-void ObjectStore::verify_schema(Group *group, Schema &target_schema, bool allow_missing_tables) {
+void ObjectStore::verify_schema(Schema const& actual_schema, Schema& target_schema, bool allow_missing_tables) {
     std::vector<ObjectSchemaValidationException> errors;
     for (auto &object_schema : target_schema) {
-        if (!table_for_object_type(group, object_schema.first)) {
+        auto matching_schema = actual_schema.find(object_schema);
+        if (matching_schema == actual_schema.end()) {
             if (!allow_missing_tables) {
-                errors.emplace_back(ObjectSchemaValidationException(object_schema.first,
+                errors.emplace_back(ObjectSchemaValidationException(object_schema.name,
-                                    "Missing table for object type '" + object_schema.first + "'."));
+                                    "Missing table for object type '" + object_schema.name + "'."));
             }
             continue;
         }
 
-        auto more_errors = verify_object_schema(group, object_schema.second, target_schema);
+        auto more_errors = verify_object_schema(*matching_schema, object_schema);
         errors.insert(errors.end(), more_errors.begin(), more_errors.end());
     }
     if (errors.size()) {
@@ -159,12 +169,11 @@ void ObjectStore::verify_schema(Group *group, Schema &target_schema, bool allow_
     }
 }
 
-std::vector<ObjectSchemaValidationException> ObjectStore::verify_object_schema(Group *group, ObjectSchema &target_schema, Schema &schema) {
+std::vector<ObjectSchemaValidationException> ObjectStore::verify_object_schema(ObjectSchema const& table_schema,
+                                                                               ObjectSchema& target_schema) {
     std::vector<ObjectSchemaValidationException> exceptions;
-    ObjectSchema table_schema(group, target_schema.name);
 
     // check to see if properties are the same
-    Property *primary = nullptr;
     for (auto& current_prop : table_schema.properties) {
         auto target_prop = target_schema.property_for_name(current_prop.name);
 
@@ -177,38 +186,6 @@ std::vector<ObjectSchemaValidationException> ObjectStore::verify_object_schema(G
             continue;
         }
 
-        // check object_type existence
-        if (current_prop.object_type.length() && schema.find(current_prop.object_type) == schema.end()) {
-            exceptions.emplace_back(MissingObjectTypeException(table_schema.name, current_prop));
-        }
-
-        // check nullablity
-        if (current_prop.type == PropertyTypeObject) {
-            if (!current_prop.is_nullable) {
-                exceptions.emplace_back(InvalidNullabilityException(table_schema.name, current_prop));
-            }
-        }
-        else {
-            if (current_prop.is_nullable) {
-                exceptions.emplace_back(InvalidNullabilityException(table_schema.name, current_prop));
-            }
-        }
-
-        // check primary keys
-        if (current_prop.is_primary) {
-            if (primary) {
-                exceptions.emplace_back(DuplicatePrimaryKeysException(table_schema.name));
-            }
-            primary = &current_prop;
-        }
-
-        // check indexable
-        if (current_prop.is_indexed) {
-            if (current_prop.type != PropertyTypeString && current_prop.type != PropertyTypeInt) {
-                exceptions.emplace_back(PropertyTypeNotIndexableException(table_schema.name, current_prop));
-            }
-        }
-
         // create new property with aligned column
         target_prop->table_column = current_prop.table_column;
     }
@@ -228,17 +205,6 @@ std::vector<ObjectSchemaValidationException> ObjectStore::verify_object_schema(G
     return exceptions;
 }
 
-void ObjectStore::update_column_mapping(Group *group, ObjectSchema &target_schema) {
-    ObjectSchema table_schema(group, target_schema.name);
-    for (auto& target_prop : target_schema.properties) {
-        auto table_prop = table_schema.property_for_name(target_prop.name);
-        if (table_prop) {
-            // Update target property column to match what's in the realm if it exists
-            target_prop.table_column = table_prop->table_column;
-        }
-    }
-}
-
 // set references to tables on targetSchema and create/update any missing or out-of-date tables
 // if update existing is true, updates existing tables, otherwise validates existing tables
 // NOTE: must be called from within write transaction
@@ -249,11 +215,11 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update
     std::vector<ObjectSchema *> to_update;
     for (auto& object_schema : target_schema) {
         bool created = false;
-        ObjectStore::table_for_object_type_create_if_needed(group, object_schema.first, created);
+        ObjectStore::table_for_object_type_create_if_needed(group, object_schema.name, created);
 
         // we will modify tables for any new objectSchema (table was created) or for all if update_existing is true
         if (update_existing || created) {
-            to_update.push_back(&object_schema.second);
+            to_update.push_back(&object_schema);
             changed = true;
         }
     }
@@ -264,12 +230,26 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update
         ObjectSchema current_schema(group, target_object_schema->name);
         std::vector<Property> &target_props = target_object_schema->properties;
 
+        // remove extra columns
+        size_t deleted = 0;
+        for (auto& current_prop : current_schema.properties) {
+            current_prop.table_column -= deleted;
+
+            auto target_prop = target_object_schema->property_for_name(current_prop.name);
+            if (!target_prop || property_has_changed(current_prop, *target_prop)) {
+                table->remove_column(current_prop.table_column);
+                ++deleted;
+                current_prop.table_column = npos;
+                changed = true;
+            }
+        }
+
         // add missing columns
         for (auto& target_prop : target_props) {
             auto current_prop = current_schema.property_for_name(target_prop.name);
 
-            // add any new properties (new name or different type)
+            // add any new properties (no old column or old column was removed due to not matching)
-            if (!current_prop || property_has_changed(*current_prop, target_prop)) {
+            if (!current_prop || current_prop->table_column == npos) {
                 switch (target_prop.type) {
                         // for objects and arrays, we have to specify target table
                     case PropertyTypeObject:
@@ -279,23 +259,16 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update
                         break;
                     }
                     default:
-                        target_prop.table_column = table->add_column(DataType(target_prop.type), target_prop.name);
+                        target_prop.table_column = table->add_column(DataType(target_prop.type),
+                                                                     target_prop.name,
+                                                                     target_prop.is_nullable);
                         break;
                 }
 
                 changed = true;
             }
-        }
+            else {
-
+                target_prop.table_column = current_prop->table_column;
-        // remove extra columns
-        sort(begin(current_schema.properties), end(current_schema.properties), [](Property &i, Property &j) {
-            return j.table_column < i.table_column;
-        });
-        for (auto& current_prop : current_schema.properties) {
-            auto target_prop = target_object_schema->property_for_name(current_prop.name);
-            if (!target_prop || property_has_changed(current_prop, *target_prop)) {
-                table->remove_column(current_prop.table_column);
-                changed = true;
             }
         }
 
@@ -316,7 +289,7 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update
     return changed;
 }
 
-bool ObjectStore::is_schema_at_version(Group *group, uint64_t version) {
+bool ObjectStore::is_schema_at_version(const Group *group, uint64_t version) {
     uint64_t old_version = get_schema_version(group);
     if (old_version > version && old_version != NotVersioned) {
         throw InvalidSchemaVersionException(old_version, version);
@@ -324,25 +297,32 @@ bool ObjectStore::is_schema_at_version(Group *group, uint64_t version) {
     return old_version == version;
 }
 
-bool ObjectStore::realm_requires_update(Group *group, uint64_t version, Schema &schema) {
+bool ObjectStore::needs_update(Schema const& old_schema, Schema const& schema) {
-    if (!is_schema_at_version(group, version)) {
+    for (auto const& target_schema : schema) {
+        auto matching_schema = old_schema.find(target_schema);
+        if (matching_schema == end(old_schema)) {
+            // Table doesn't exist
             return true;
         }
-    for (auto& target_schema : schema) {
+
-        TableRef table = table_for_object_type(group, target_schema.first);
+        if (matching_schema->properties.size() != target_schema.properties.size()) {
-        if (!table) {
+            // If the number of properties don't match then a migration is required
+            return false;
+        }
+
+        // Check that all of the property indexes are up to date
+        for (size_t i = 0, count = target_schema.properties.size(); i < count; ++i) {
+            if (target_schema.properties[i].is_indexed != matching_schema->properties[i].is_indexed) {
                 return true;
             }
         }
-    if (!indexes_are_up_to_date(group, schema)) {
-        return true;
     }
+
     return false;
 }
 
-bool ObjectStore::update_realm_with_schema(Group *group,
+bool ObjectStore::update_realm_with_schema(Group *group, Schema const& old_schema,
-                                           uint64_t version,
+                                           uint64_t version, Schema &schema,
-                                           Schema &schema,
                                            MigrationFunction migration) {
     // Recheck the schema version after beginning the write transaction as
     // another process may have done the migration after we opened the read
@@ -353,7 +333,11 @@ bool ObjectStore::update_realm_with_schema(Group *group,
     bool changed = create_metadata_tables(group);
     changed = create_tables(group, schema, migrating) || changed;
 
-    verify_schema(group, schema);
+    if (!migrating) {
+        // If we aren't migrating, then verify that all of the tables which
+        // were already present are valid (newly created ones always are)
+        verify_schema(old_schema, schema, true);
+    }
 
     changed = update_indexes(group, schema) || changed;
 
@@ -364,51 +348,34 @@ bool ObjectStore::update_realm_with_schema(Group *group,
     // apply the migration block if provided and there's any old data
     if (get_schema_version(group) != ObjectStore::NotVersioned) {
         migration(group, schema);
-    }
 
         validate_primary_column_uniqueness(group, schema);
+    }
 
     set_schema_version(group, version);
     return true;
 }
 
-Schema ObjectStore::schema_from_group(Group *group) {
+Schema ObjectStore::schema_from_group(const Group *group) {
-    Schema schema;
+    std::vector<ObjectSchema> schema;
     for (size_t i = 0; i < group->size(); i++) {
         std::string object_type = object_type_for_table_name(group->get_table_name(i));
         if (object_type.length()) {
-            schema.emplace(object_type, std::move(ObjectSchema(group, object_type)));
+            schema.emplace_back(group, object_type);
         }
     }
     return schema;
 }
 
-bool ObjectStore::indexes_are_up_to_date(Group *group, Schema &schema) {
-    for (auto &object_schema : schema) {
-        TableRef table = table_for_object_type(group, object_schema.first);
-        if (!table) {
-            continue;
-        }
-
-        update_column_mapping(group, object_schema.second);
-        for (auto& property : object_schema.second.properties) {
-            if (property.requires_index() != table->has_search_index(property.table_column)) {
-                return false;
-            }
-        }
-    }
-    return true;
-}
-
 bool ObjectStore::update_indexes(Group *group, Schema &schema) {
     bool changed = false;
     for (auto& object_schema : schema) {
-        TableRef table = table_for_object_type(group, object_schema.first);
+        TableRef table = table_for_object_type(group, object_schema.name);
         if (!table) {
             continue;
         }
 
-        for (auto& property : object_schema.second.properties) {
+        for (auto& property : object_schema.properties) {
             if (property.requires_index() == table->has_search_index(property.table_column)) {
                 continue;
             }
@@ -419,7 +386,7 @@ bool ObjectStore::update_indexes(Group *group, Schema &schema) {
                     table->add_search_index(property.table_column);
                 }
                 catch (LogicError const&) {
-                    throw PropertyTypeNotIndexableException(object_schema.first, property);
+                    throw PropertyTypeNotIndexableException(object_schema.name, property);
                 }
             }
             else {
@@ -430,16 +397,16 @@ bool ObjectStore::update_indexes(Group *group, Schema &schema) {
     return changed;
 }
 
-void ObjectStore::validate_primary_column_uniqueness(Group *group, Schema &schema) {
+void ObjectStore::validate_primary_column_uniqueness(const Group *group, Schema const& schema) {
     for (auto& object_schema : schema) {
-        auto primary_prop = object_schema.second.primary_key_property();
+        auto primary_prop = object_schema.primary_key_property();
         if (!primary_prop) {
             continue;
         }
 
-        TableRef table = table_for_object_type(group, object_schema.first);
+        ConstTableRef table = table_for_object_type(group, object_schema.name);
         if (table->get_distinct_view(primary_prop->table_column).size() != table->size()) {
-            throw DuplicatePrimaryKeyValueException(object_schema.first, *primary_prop);
+            throw DuplicatePrimaryKeyValueException(object_schema.name, *primary_prop);
         }
     }
 }
@@ -460,62 +427,62 @@ InvalidSchemaVersionException::InvalidSchemaVersionException(uint64_t old_versio
     m_what = "Provided schema version " + std::to_string(old_version) + " is less than last set version " + std::to_string(new_version) + ".";
 }
 
-DuplicatePrimaryKeyValueException::DuplicatePrimaryKeyValueException(std::string object_type, Property &property) :
+DuplicatePrimaryKeyValueException::DuplicatePrimaryKeyValueException(std::string const& object_type, Property const& property) :
     m_object_type(object_type), m_property(property)
 {
     m_what = "Primary key property '" + property.name + "' has duplicate values after migration.";
 };
 
 
-SchemaValidationException::SchemaValidationException(std::vector<ObjectSchemaValidationException> errors) :
+SchemaValidationException::SchemaValidationException(std::vector<ObjectSchemaValidationException> const& errors) :
     m_validation_errors(errors)
 {
     m_what ="Migration is required due to the following errors: ";
-    for (auto error : errors) {
+    for (auto const& error : errors) {
         m_what += std::string("\n- ") + error.what();
     }
 }
 
-PropertyTypeNotIndexableException::PropertyTypeNotIndexableException(std::string object_type, Property &property) :
+PropertyTypeNotIndexableException::PropertyTypeNotIndexableException(std::string const& object_type, Property const& property) :
     ObjectSchemaPropertyException(object_type, property)
 {
     m_what = "Can't index property " + object_type + "." + property.name + ": indexing a property of type '" + string_for_property_type(property.type) + "' is currently not supported";
 }
 
-ExtraPropertyException::ExtraPropertyException(std::string object_type, Property &property) :
+ExtraPropertyException::ExtraPropertyException(std::string const& object_type, Property const& property) :
     ObjectSchemaPropertyException(object_type, property)
 {
     m_what = "Property '" + property.name + "' has been added to latest object model.";
 }
 
-MissingPropertyException::MissingPropertyException(std::string object_type, Property &property) :
+MissingPropertyException::MissingPropertyException(std::string const& object_type, Property const& property) :
     ObjectSchemaPropertyException(object_type, property)
 {
     m_what = "Property '" + property.name + "' is missing from latest object model.";
 }
 
-InvalidNullabilityException::InvalidNullabilityException(std::string object_type, Property &property) :
+InvalidNullabilityException::InvalidNullabilityException(std::string const& object_type, Property const& property) :
     ObjectSchemaPropertyException(object_type, property)
 {
     if (property.type == PropertyTypeObject) {
-        if (!property.is_nullable) {
         m_what = "'Object' property '" + property.name + "' must be nullable.";
     }
-    }
     else {
-        if (property.is_nullable) {
+#if REALM_NULL_STRINGS == 1
+        m_what = "Array or Mixed property '" + property.name + "' cannot be nullable";
+#else
         m_what = "Only 'Object' property types are nullable";
-        }
+#endif
     }
 }
 
-MissingObjectTypeException::MissingObjectTypeException(std::string object_type, Property &property) :
+MissingObjectTypeException::MissingObjectTypeException(std::string const& object_type, Property const& property) :
     ObjectSchemaPropertyException(object_type, property)
 {
     m_what = "Target type '" + property.object_type + "' doesn't exist for property '" + property.name + "'.";
 }
 
-MismatchedPropertiesException::MismatchedPropertiesException(std::string object_type, Property &old_property, Property &new_property) :
+MismatchedPropertiesException::MismatchedPropertiesException(std::string const& object_type, Property const& old_property, Property const& new_property) :
     ObjectSchemaValidationException(object_type), m_old_property(old_property), m_new_property(new_property)
 {
     if (new_property.type != old_property.type) {
@@ -530,7 +497,7 @@ MismatchedPropertiesException::MismatchedPropertiesException(std::string object_
     }
 }
 
-ChangedPrimaryKeyException::ChangedPrimaryKeyException(std::string object_type, std::string old_primary, std::string new_primary) : ObjectSchemaValidationException(object_type), m_old_primary(old_primary), m_new_primary(new_primary)
+ChangedPrimaryKeyException::ChangedPrimaryKeyException(std::string const& object_type, std::string const& old_primary, std::string const& new_primary) : ObjectSchemaValidationException(object_type), m_old_primary(old_primary), m_new_primary(new_primary)
 {
     if (old_primary.size()) {
         m_what = "Property '" + old_primary + "' is no longer a primary key.";
@@ -540,14 +507,13 @@ ChangedPrimaryKeyException::ChangedPrimaryKeyException(std::string object_type,
     }
 }
 
-InvalidPrimaryKeyException::InvalidPrimaryKeyException(std::string object_type, std::string primary) :
+InvalidPrimaryKeyException::InvalidPrimaryKeyException(std::string const& object_type, std::string const& primary) :
     ObjectSchemaValidationException(object_type), m_primary_key(primary)
 {
     m_what = "Specified primary key property '" + primary + "' does not exist.";
 }
 
-DuplicatePrimaryKeysException::DuplicatePrimaryKeysException(std::string object_type) : ObjectSchemaValidationException(object_type)
+DuplicatePrimaryKeysException::DuplicatePrimaryKeysException(std::string const& object_type) : ObjectSchemaValidationException(object_type)
 {
     m_what = "Duplicate primary keys for object '" + object_type + "'.";
 }
-

object_store.hpp

@@ -19,18 +19,17 @@
 #ifndef REALM_OBJECT_STORE_HPP
 #define REALM_OBJECT_STORE_HPP
 
-#include <map>
-#include <vector>
-#include <functional>
-#include <realm/link_view.hpp>
-#include <realm/group.hpp>
-
 #include "object_schema.hpp"
+#include "property.hpp"
+
+#include <functional>
+
+#include <realm/group.hpp>
+#include <realm/link_view.hpp>
 
 namespace realm {
     class ObjectSchemaValidationException;
-    class Schema : public std::map<std::string, ObjectSchema> {
+    class Schema;
-    };
 
     class ObjectStore {
       public:
@@ -38,37 +37,35 @@ namespace realm {
         static const uint64_t NotVersioned;
 
         // get the last set schema version
-        static uint64_t get_schema_version(Group *group);
+        static uint64_t get_schema_version(const Group *group);
 
         // checks if the schema in the group is at the given version
-        static bool is_schema_at_version(realm::Group *group, uint64_t version);
+        static bool is_schema_at_version(const Group *group, uint64_t version);
 
-        // verify a target schema against tables in the given group
+        // verify that schema from a group and a target schema are compatible
-        // updates the column mapping on all ObjectSchema properties
+        // updates the column mapping on all ObjectSchema properties of the target schema
-        // throws if the schema is invalid or does not match tables in the given group
+        // throws if the schema is invalid or does not match
-        static void verify_schema(Group *group, Schema &target_schema, bool allow_missing_tables = false);
+        static void verify_schema(Schema const& actual_schema, Schema& target_schema, bool allow_missing_tables = false);
 
-        // updates the target_column member for all properties based on the column indexes in the passed in group
+        // determines if a realm with the given old schema needs non-migration
-        static void update_column_mapping(Group *group, ObjectSchema &target_schema);
+        // changes to make it compatible with the given target schema
+        static bool needs_update(Schema const& old_schema, Schema const& schema);
         
-        // determines if you must call update_realm_with_schema for a given realm.
+        // updates a Realm from old_schema to the given target schema, creating and updating tables as needed
-        // returns true if there is a schema version mismatch, if there tables which still need to be created,
-        // or if file format or other changes/updates need to be made
-        static bool realm_requires_update(Group *group, uint64_t version, Schema &schema);
-        
-        // updates a Realm to a given target schema/version creating tables and updating indexes as necessary
         // returns if any changes were made
-        // passed in schema ar updated with the correct column mapping
+        // passed in target schema is updated with the correct column mapping
-        // optionally runs migration function/lambda if schema is out of date
+        // optionally runs migration function if schema is out of date
         // NOTE: must be performed within a write transaction
         typedef std::function<void(Group *, Schema &)> MigrationFunction;
-        static bool update_realm_with_schema(Group *group, uint64_t version, Schema &schema, MigrationFunction migration);
+        static bool update_realm_with_schema(Group *group, Schema const& old_schema, uint64_t version,
+                                             Schema &schema, MigrationFunction migration);
 
         // get a table for an object type
         static realm::TableRef table_for_object_type(Group *group, StringData object_type);
+        static realm::ConstTableRef table_for_object_type(const Group *group, StringData object_type);
 
         // get existing Schema from a group
-        static Schema schema_from_group(Group *group);
+        static Schema schema_from_group(const Group *group);
 
         // deletes the table for the given type
         static void delete_data_for_object(Group *group, const StringData &object_type);
@@ -78,7 +75,7 @@ namespace realm {
         static void set_schema_version(Group *group, uint64_t version);
 
         // check if the realm already has all metadata tables
-        static bool has_metadata_tables(Group *group);
+        static bool has_metadata_tables(const Group *group);
 
         // create any metadata tables that don't already exist
         // must be in write transaction to set
@@ -89,13 +86,14 @@ namespace realm {
         // if update existing is true, updates existing tables, otherwise only adds and initializes new tables
         static bool create_tables(realm::Group *group, Schema &target_schema, bool update_existing);
 
-        // verify a target schema against its table, setting the table_column property on each schema object
+        // verify a target schema against an expected schema, setting the table_column property on each schema object
         // updates the column mapping on the target_schema
         // returns array of validation errors
-        static std::vector<ObjectSchemaValidationException> verify_object_schema(Group *group, ObjectSchema &target_schema, Schema &schema);
+        static std::vector<ObjectSchemaValidationException> verify_object_schema(ObjectSchema const& expected,
+                                                                                 ObjectSchema &target_schema);
 
         // get primary key property name for object type
-        static StringData get_primary_key_for_object(Group *group, StringData object_type);
+        static StringData get_primary_key_for_object(const Group *group, StringData object_type);
 
         // sets primary key property for object type
         // must be in write transaction to set
@@ -105,14 +103,11 @@ namespace realm {
         static std::string table_name_for_object_type(const std::string &class_name);
         static std::string object_type_for_table_name(const std::string &table_name);
 
-        // check if indexes are up to date - if false you need to call update_realm_with_schema
-        static bool indexes_are_up_to_date(Group *group, Schema &schema);
-
         // returns if any indexes were changed
         static bool update_indexes(Group *group, Schema &schema);
 
         // validates that all primary key properties have unique values
-        static void validate_primary_column_uniqueness(Group *group, Schema &schema);
+        static void validate_primary_column_uniqueness(const Group *group, Schema const& schema);
 
         friend ObjectSchema;
     };
@@ -122,7 +117,7 @@ namespace realm {
       public:
         ObjectStoreException() = default;
         ObjectStoreException(const std::string &what) : m_what(what) {}
-        virtual const char* what() const noexcept { return m_what.c_str(); }
+        const char* what() const noexcept override { return m_what.c_str(); }
       protected:
         std::string m_what;
     };
@@ -133,17 +128,17 @@ namespace realm {
     class InvalidSchemaVersionException : public MigrationException {
       public:
         InvalidSchemaVersionException(uint64_t old_version, uint64_t new_version);
-        uint64_t old_version() { return m_old_version; }
+        uint64_t old_version() const { return m_old_version; }
-        uint64_t new_version() { return m_new_version; }
+        uint64_t new_version() const { return m_new_version; }
       private:
         uint64_t m_old_version, m_new_version;
     };
 
     class DuplicatePrimaryKeyValueException : public MigrationException {
       public:
-        DuplicatePrimaryKeyValueException(std::string object_type, Property &property);
+        DuplicatePrimaryKeyValueException(std::string const& object_type, Property const& property);
-        std::string object_type() { return m_object_type; }
+        std::string object_type() const { return m_object_type; }
-        Property &property() { return m_property; }
+        Property const& property() const { return m_property; }
       private:
         std::string m_object_type;
         Property m_property;
@@ -152,87 +147,86 @@ namespace realm {
     // Schema validation exceptions
     class SchemaValidationException : public ObjectStoreException {
       public:
-        SchemaValidationException(std::vector<ObjectSchemaValidationException> errors);
+        SchemaValidationException(std::vector<ObjectSchemaValidationException> const& errors);
-        std::vector<ObjectSchemaValidationException> &validation_errors() { return m_validation_errors; }
+        std::vector<ObjectSchemaValidationException> const& validation_errors() const { return m_validation_errors; }
       private:
         std::vector<ObjectSchemaValidationException> m_validation_errors;
     };
 
     class ObjectSchemaValidationException : public ObjectStoreException {
       public:
-        ObjectSchemaValidationException(std::string object_type) : m_object_type(object_type) {}
+        ObjectSchemaValidationException(std::string const& object_type) : m_object_type(object_type) {}
-        ObjectSchemaValidationException(std::string object_type, std::string message) :
+        ObjectSchemaValidationException(std::string const& object_type, std::string const& message) :
             m_object_type(object_type) { m_what = message; }
-        std::string object_type() { return m_object_type; }
+        std::string object_type() const { return m_object_type; }
       protected:
         std::string m_object_type;
     };
 
     class ObjectSchemaPropertyException : public ObjectSchemaValidationException {
       public:
-        ObjectSchemaPropertyException(std::string object_type, Property &property) :
+        ObjectSchemaPropertyException(std::string const& object_type, Property const& property) :
             ObjectSchemaValidationException(object_type), m_property(property) {}
-        Property &property() { return m_property; }
+        Property const& property() const { return m_property; }
       private:
         Property m_property;
     };
 
     class PropertyTypeNotIndexableException : public ObjectSchemaPropertyException {
       public:
-        PropertyTypeNotIndexableException(std::string object_type, Property &property);
+        PropertyTypeNotIndexableException(std::string const& object_type, Property const& property);
     };
 
     class ExtraPropertyException : public ObjectSchemaPropertyException {
       public:
-        ExtraPropertyException(std::string object_type, Property &property);
+        ExtraPropertyException(std::string const& object_type, Property const& property);
     };
 
     class MissingPropertyException : public ObjectSchemaPropertyException {
       public:
-        MissingPropertyException(std::string object_type, Property &property);
+        MissingPropertyException(std::string const& object_type, Property const& property);
     };
 
     class InvalidNullabilityException : public ObjectSchemaPropertyException {
       public:
-        InvalidNullabilityException(std::string object_type, Property &property);
+        InvalidNullabilityException(std::string const& object_type, Property const& property);
     };
 
     class MissingObjectTypeException : public ObjectSchemaPropertyException {
     public:
-        MissingObjectTypeException(std::string object_type, Property &property);
+        MissingObjectTypeException(std::string const& object_type, Property const& property);
     };
 
     class DuplicatePrimaryKeysException : public ObjectSchemaValidationException {
     public:
-        DuplicatePrimaryKeysException(std::string object_type);
+        DuplicatePrimaryKeysException(std::string const& object_type);
     };
 
     class MismatchedPropertiesException : public ObjectSchemaValidationException {
       public:
-        MismatchedPropertiesException(std::string object_type, Property &old_property, Property &new_property);
+        MismatchedPropertiesException(std::string const& object_type, Property const& old_property, Property const& new_property);
-        Property &old_property() { return m_old_property; }
+        Property const& old_property() const { return m_old_property; }
-        Property &new_property() { return m_new_property; }
+        Property const& new_property() const { return m_new_property; }
       private:
         Property m_old_property, m_new_property;
     };
 
     class ChangedPrimaryKeyException : public ObjectSchemaValidationException {
       public:
-        ChangedPrimaryKeyException(std::string object_type, std::string old_primary, std::string new_primary);
+        ChangedPrimaryKeyException(std::string const& object_type, std::string const& old_primary, std::string const& new_primary);
-        std::string old_primary() { return m_old_primary; }
+        std::string old_primary() const { return m_old_primary; }
-        std::string new_primary() { return m_new_primary; }
+        std::string new_primary() const { return m_new_primary; }
       private:
         std::string m_old_primary, m_new_primary;
     };
 
     class InvalidPrimaryKeyException : public ObjectSchemaValidationException {
       public:
-        InvalidPrimaryKeyException(std::string object_type, std::string primary_key);
+        InvalidPrimaryKeyException(std::string const& object_type, std::string const& primary_key);
-        std::string primary_key() { return m_primary_key; }
+        std::string primary_key() const { return m_primary_key; }
       private:
         std::string m_primary_key;
     };
 }
 
 #endif /* defined(REALM_OBJECT_STORE_HPP) */
-

property.hpp

@@ -46,18 +46,15 @@ namespace realm {
     };
 
     struct Property {
-    public:
-        Property() : object_type(""), is_primary(false), is_indexed(false), is_nullable(false) {}
-
         std::string name;
         PropertyType type;
         std::string object_type;
-        bool is_primary;
+        bool is_primary = false;
-        bool is_indexed;
+        bool is_indexed = false;
-        bool is_nullable;
+        bool is_nullable = false;
 
         size_t table_column;
-        bool requires_index() { return is_primary || is_indexed; }
+        bool requires_index() const { return is_primary || is_indexed; }
     };
 
     static inline const char *string_for_property_type(PropertyType type) {

realm_delegate.hpp

@@ -0,0 +1,154 @@
+////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2015 Realm Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////
+
+#ifndef REALM_DELEGATE_HPP
+#define REALM_DELEGATE_HPP
+
+#include "index_set.hpp"
+
+#include <tuple>
+#include <vector>
+
+namespace realm {
+// RealmDelegate is the extension point for adding binding-specific behavior to
+// a SharedRealm. It can be used to store additonal data associated with the
+// Realm which is needed by the binding, and there are several methods which
+// can be overridden to receive notifications of state changes within the Realm.
+//
+// A simple delegate implementation which lets the user register functions to be
+// called on refresh could look like the following:
+//
+// class DelegateImplementation : public RealmDelegate {
+// public:
+//     // A token returned from add_notification that can be used to remove the
+//     // notification later
+//     struct token : private std::list<std::function<void ()>>::iterator {
+//         token(std::list<std::function<void ()>>::iterator it) : std::list<std::function<void ()>>::iterator(it) { }
+//         friend class DelegateImplementation;
+//     };
+//
+//     token add_notification(std::function<void ()> func)
+//     {
+//         m_registered_notifications.push_back(std::move(func));
+//         return token(std::prev(m_registered_notifications.end()));
+//     }
+//
+//     void remove_notification(token entry)
+//     {
+//         m_registered_notifications.erase(entry);
+//     }
+//
+//     // Override the did_change method to call each registered notification
+//     void did_change(std::vector<ObserverState> const&, std::vector<void*> const&) override
+//     {
+//         // Loop oddly so that unregistering a notification from within the
+//         // registered function works
+//         for (auto it = m_registered_notifications.begin(); it != m_registered_notifications.end(); ) {
+//             (*it++)();
+//         }
+//     }
+//
+// private:
+//     std::list<std::function<void ()>> m_registered_notifications;
+// };
+class RealmDelegate {
+public:
+    virtual ~RealmDelegate() = default;
+
+    // Called by the Realm when a write transaction is committed to the file by
+    // a different Realm instance (possibly in a different process)
+    virtual void changes_available() { }
+
+    struct ObserverState;
+
+    // Override this function if you want to recieve detailed information about
+    // external changes to a specific set of objects.
+    // This is called before each operation which may advance the read
+    // transaction to include
+    // ObserverStates for each row for which detailed change information is
+    // desired.
+    virtual std::vector<ObserverState> get_observed_rows() { return {}; }
+
+    // Called immediately before the read transaction is advanced if detailed
+    // change information was requested (by returning a non-empty array from
+    // get_observed_rows()).
+    // The observers vector is the vector returned by get_observed_row(),
+    // updated with change information. The invalidated vector is a list of the
+    // `info` fields of observed rows which will be deleted.
+    virtual void will_change(std::vector<ObserverState> const& observers,
+                             std::vector<void*> const& invalidated);
+
+    // Called immediately after the read transaction version is advanced. Unlike
+    // will_change(), this is called even if detailed change information was not
+    // requested or if the Realm is not actually in a read transactuib, although
+    // both vectors will be empty in that case.
+    virtual void did_change(std::vector<ObserverState> const& observers,
+                            std::vector<void*> const& invalidated);
+
+    // Change information for a single field of a row
+    struct ColumnInfo {
+        // Did this column change?
+        bool changed = false;
+        // For LinkList columns, what kind of change occurred?
+        // Always None for other column types
+        enum class Kind {
+            None,   // No change
+            Set,    // The entries at `indices` were assigned to
+            Insert, // New values were inserted at each of the indices given
+            Remove, // Values were removed at each of the indices given
+            SetAll  // The entire LinkList has been replaced with a new set of values
+        } kind = Kind::None;
+        // The indices where things happened for Set, Insert and Remove
+        // Not used for None and SetAll
+        IndexSet indices;
+    };
+
+    // Information about an observed row in a table
+    //
+    // Each object which needs detailed change information should have an
+    // ObserverState entry in the vector returned from get_observed_rows(), with
+    // the initial table and row indexes set (and optionally the info field).
+    // The Realm parses the transaction log, and populates the `changes` vector
+    // in each ObserverState with information about what changes were made.
+    struct ObserverState {
+        // Initial table and row which is observed
+        // May be updated by row insertions and removals
+        size_t table_ndx;
+        size_t row_ndx;
+
+        // Opaque userdata for the delegate's use
+        void* info;
+
+        // Populated with information about which columns were changed
+        // May be shorter than the actual number of columns if the later columns
+        // are not modified
+        std::vector<ColumnInfo> changes;
+
+        // Simple lexographic ordering
+        friend bool operator<(ObserverState const& lft, ObserverState const& rgt)
+        {
+            return std::tie(lft.table_ndx, lft.row_ndx) < std::tie(rgt.table_ndx, rgt.row_ndx);
+        }
+    };
+};
+
+inline void RealmDelegate::will_change(std::vector<ObserverState> const&, std::vector<void*> const&) { }
+inline void RealmDelegate::did_change(std::vector<ObserverState> const&, std::vector<void*> const&) { }
+} // namespace realm
+
+#endif /* REALM_DELEGATE_HPP */

schema.cpp

@@ -0,0 +1,106 @@
+////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2015 Realm Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////
+
+#include "schema.hpp"
+
+#include "object_schema.hpp"
+#include "object_store.hpp"
+#include "property.hpp"
+
+using namespace realm;
+
+static bool compare_by_name(ObjectSchema const& lft, ObjectSchema const& rgt) {
+    return lft.name < rgt.name;
+}
+
+Schema::Schema(base types) : base(std::move(types)) {
+    std::sort(begin(), end(), compare_by_name);
+}
+
+Schema::iterator Schema::find(std::string const& name)
+{
+    ObjectSchema cmp;
+    cmp.name = name;
+    return find(cmp);
+}
+
+Schema::const_iterator Schema::find(std::string const& name) const
+{
+    return const_cast<Schema *>(this)->find(name);
+}
+
+Schema::iterator Schema::find(ObjectSchema const& object) noexcept
+{
+    auto it = std::lower_bound(begin(), end(), object, compare_by_name);
+    if (it != end() && it->name != object.name) {
+        it = end();
+    }
+    return it;
+}
+
+Schema::const_iterator Schema::find(ObjectSchema const& object) const noexcept
+{
+    return const_cast<Schema *>(this)->find(object);
+}
+
+void Schema::validate() const
+{
+    std::vector<ObjectSchemaValidationException> exceptions;
+    for (auto const& object : *this) {
+        const Property *primary = nullptr;
+        for (auto const& prop : object.properties) {
+            // check object_type existence
+            if (!prop.object_type.empty() && find(prop.object_type) == end()) {
+                exceptions.emplace_back(MissingObjectTypeException(object.name, prop));
+            }
+
+            // check nullablity
+            if (prop.is_nullable) {
+#if REALM_NULL_STRINGS == 1
+                if (prop.type == PropertyTypeArray || prop.type == PropertyTypeAny) {
+#else
+                if (prop.type != PropertyTypeObject) {
+#endif
+                    exceptions.emplace_back(InvalidNullabilityException(object.name, prop));
+                }
+            }
+            else if (prop.type == PropertyTypeObject) {
+                exceptions.emplace_back(InvalidNullabilityException(object.name, prop));
+            }
+
+            // check primary keys
+            if (prop.is_primary) {
+                if (primary) {
+                    exceptions.emplace_back(DuplicatePrimaryKeysException(object.name));
+                }
+                primary = &prop;
+            }
+
+            // check indexable
+            if (prop.is_indexed) {
+                if (prop.type != PropertyTypeString && prop.type != PropertyTypeInt) {
+                    exceptions.emplace_back(PropertyTypeNotIndexableException(object.name, prop));
+                }
+            }
+        }
+    }
+
+    if (exceptions.size()) {
+        throw SchemaValidationException(exceptions);
+    }
+}

schema.hpp

@@ -0,0 +1,55 @@
+////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2015 Realm Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////
+
+#ifndef REALM_SCHEMA_HPP
+#define REALM_SCHEMA_HPP
+
+#include <vector>
+
+namespace realm {
+class ObjectSchema;
+
+class Schema : private std::vector<ObjectSchema> {
+private:
+    using base = std::vector<ObjectSchema>;
+public:
+    // Create a schema from a vector of ObjectSchema
+    Schema(base types);
+
+    // find an ObjectSchema by name
+    iterator find(std::string const& name);
+    const_iterator find(std::string const& name) const;
+
+    // find an ObjectSchema with the same name as the passed in one
+    iterator find(ObjectSchema const& object) noexcept;
+    const_iterator find(ObjectSchema const& object) const noexcept;
+
+    // Verify that this schema is internally consistent (i.e. all properties are
+    // valid, links link to types that actually exist, etc.)
+    void validate() const;
+
+    using base::iterator;
+    using base::const_iterator;
+    using base::begin;
+    using base::end;
+    using base::empty;
+    using base::size;
+};
+}
+
+#endif /* defined(REALM_SCHEMA_HPP) */

shared_realm.cpp

@@ -17,47 +17,69 @@
 ////////////////////////////////////////////////////////////////////////////
 
 #include "shared_realm.hpp"
-#include <realm/group_shared.hpp>
+
-#include <realm/lang_bind_helper.hpp>
+#include "external_commit_helper.hpp"
+#include "realm_delegate.hpp"
+#include "schema.hpp"
+#include "transact_log_handler.hpp"
+
 #include <realm/commit_log.hpp>
-#include <memory>
+#include <realm/group_shared.hpp>
+
+#include <mutex>
 
 using namespace realm;
 
 RealmCache Realm::s_global_cache;
-std::mutex Realm::s_init_mutex;
 
-Realm::Config::Config(const Config& c) : path(c.path), read_only(c.read_only), in_memory(c.in_memory), schema_version(c.schema_version), encryption_key(c.encryption_key), migration_function(c.migration_function)
+Realm::Config::Config(const Config& c)
+: path(c.path)
+, read_only(c.read_only)
+, in_memory(c.in_memory)
+, cache(c.cache)
+, encryption_key(c.encryption_key)
+, schema_version(c.schema_version)
+, migration_function(c.migration_function)
 {
     if (c.schema) {
         schema = std::make_unique<Schema>(*c.schema);
     }
 }
 
-Realm::Realm(Config &config) : m_config(config), m_thread_id(std::this_thread::get_id()), m_auto_refresh(true), m_in_transaction(false)
+Realm::Config::~Config() = default;
+
+Realm::Config& Realm::Config::operator=(realm::Realm::Config const& c)
+{
+    if (&c != this) {
+        *this = Config(c);
+    }
+    return *this;
+}
+
+Realm::Realm(Config config)
+: m_config(std::move(config))
 {
     try {
-        if (config.read_only) {
+        if (m_config.read_only) {
-            m_read_only_group = std::make_unique<Group>(config.path, config.encryption_key.data(), Group::mode_ReadOnly);
+            m_read_only_group = std::make_unique<Group>(m_config.path, m_config.encryption_key.data(), Group::mode_ReadOnly);
             m_group = m_read_only_group.get();
         }
         else {
-            m_history = realm::make_client_history(config.path, config.encryption_key.data());
+            m_history = realm::make_client_history(m_config.path, m_config.encryption_key.data());
-            SharedGroup::DurabilityLevel durability = config.in_memory ? SharedGroup::durability_MemOnly :
+            SharedGroup::DurabilityLevel durability = m_config.in_memory ? SharedGroup::durability_MemOnly :
                                                                            SharedGroup::durability_Full;
-            m_shared_group = std::make_unique<SharedGroup>(*m_history, durability, config.encryption_key.data());
+            m_shared_group = std::make_unique<SharedGroup>(*m_history, durability, m_config.encryption_key.data());
-            m_group = nullptr;
         }
     }
     catch (util::File::PermissionDenied const& ex) {
-        throw RealmFileException(RealmFileException::Kind::PermissionDenied, "Unable to open a realm at path '" + config.path +
+        throw RealmFileException(RealmFileException::Kind::PermissionDenied, "Unable to open a realm at path '" + m_config.path +
-                             "'. Please use a path where your app has " + (config.read_only ? "read" : "read-write") + " permissions.");
+                             "'. Please use a path where your app has " + (m_config.read_only ? "read" : "read-write") + " permissions.");
     }
     catch (util::File::Exists const& ex) {
-        throw RealmFileException(RealmFileException::Kind::Exists, "Unable to open a realm at path '" + config.path + "'");
+        throw RealmFileException(RealmFileException::Kind::Exists, "Unable to open a realm at path '" + m_config.path + "'");
     }
     catch (util::File::AccessError const& ex) {
-        throw RealmFileException(RealmFileException::Kind::AccessError, "Unable to open a realm at path '" + config.path + "'");
+        throw RealmFileException(RealmFileException::Kind::AccessError, "Unable to open a realm at path '" + m_config.path + "'");
     }
     catch (IncompatibleLockFile const&) {
         throw RealmFileException(RealmFileException::Kind::IncompatibleLockFile, "Realm file is currently open in another process "
@@ -65,6 +87,12 @@ Realm::Realm(Config &config) : m_config(config), m_thread_id(std::this_thread::g
     }
 }
 
+Realm::~Realm() {
+    if (m_notifier) { // might not exist yet if an error occurred during init
+        m_notifier->remove_realm(this);
+    }
+}
+
 Group *Realm::read_group()
 {
     if (!m_group) {
@@ -73,10 +101,10 @@ Group *Realm::read_group()
     return m_group;
 }
 
-SharedRealm Realm::get_shared_realm(Config &config)
+SharedRealm Realm::get_shared_realm(Config config)
 {
-    SharedRealm realm = s_global_cache.get_realm(config.path);
+    if (config.cache) {
-    if (realm) {
+        if (SharedRealm realm = s_global_cache.get_realm(config.path)) {
             if (realm->config().read_only != config.read_only) {
                 throw MismatchedConfigException("Realm at path already opened with different read permissions.");
             }
@@ -93,70 +121,94 @@ SharedRealm Realm::get_shared_realm(Config &config)
             /*if (realm->config().schema != config.schema) {
                 throw MismatchedConfigException("Realm at path already opened with different schema");
             }*/
-
             realm->m_config.migration_function = config.migration_function;
 
             return realm;
         }
+    }
 
-    realm = SharedRealm(new Realm(config));
+    SharedRealm realm(new Realm(std::move(config)));
+
+    auto target_schema = std::move(realm->m_config.schema);
+    auto target_schema_version = realm->m_config.schema_version;
+    realm->m_config.schema_version = ObjectStore::get_schema_version(realm->read_group());
 
     // we want to ensure we are only initializing a single realm at a time
+    static std::mutex s_init_mutex;
     std::lock_guard<std::mutex> lock(s_init_mutex);
-
-    uint64_t old_version = ObjectStore::get_schema_version(realm->read_group());
     if (auto existing = s_global_cache.get_any_realm(realm->config().path)) {
         // if there is an existing realm at the current path steal its schema/column mapping
         // FIXME - need to validate that schemas match
         realm->m_config.schema = std::make_unique<Schema>(*existing->m_config.schema);
-    }
+
-    else if (!realm->m_config.schema) {
+        realm->m_notifier = existing->m_notifier;
-        // get schema from group and skip validation
+        realm->m_notifier->add_realm(realm.get());
-        realm->m_config.schema_version = old_version;
-        realm->m_config.schema = std::make_unique<Schema>(ObjectStore::schema_from_group(realm->read_group()));
-    }
-    else if (realm->m_config.read_only) {
-        if (old_version == ObjectStore::NotVersioned) {
-            throw UnitializedRealmException("Can't open an un-initizliazed Realm without a Schema");
-        }
-        ObjectStore::verify_schema(realm->read_group(), *realm->m_config.schema, true);
     }
     else {
-        // its a non-cached realm so update/migrate if needed
+        realm->m_notifier = std::make_shared<ExternalCommitHelper>(realm.get());
-        realm->update_schema(*realm->m_config.schema, realm->m_config.schema_version);
+
+        // otherwise get the schema from the group
+        realm->m_config.schema = std::make_unique<Schema>(ObjectStore::schema_from_group(realm->read_group()));
+
+        // if a target schema is supplied, verify that it matches or migrate to
+        // it, as neeeded
+        if (target_schema) {
+            if (realm->m_config.read_only) {
+                if (realm->m_config.schema_version == ObjectStore::NotVersioned) {
+                    throw UnitializedRealmException("Can't open an un-initialized Realm without a Schema");
+                }
+                target_schema->validate();
+                ObjectStore::verify_schema(*realm->m_config.schema, *target_schema, true);
+                realm->m_config.schema = std::move(target_schema);
+            }
+            else {
+                realm->update_schema(std::move(target_schema), target_schema_version);
+            }
+        }
     }
 
+    if (config.cache) {
         s_global_cache.cache_realm(realm, realm->m_thread_id);
+    }
     return realm;
 }
 
-bool Realm::update_schema(Schema &schema, uint64_t version)
+bool Realm::update_schema(std::unique_ptr<Schema> schema, uint64_t version)
 {
-    bool changed = false;
+    schema->validate();
-    Config old_config(m_config);
 
-    // set new version/schema
+    bool needs_update = !m_config.read_only && (m_config.schema_version != version || ObjectStore::needs_update(*m_config.schema, *schema));
-    if (m_config.schema.get() != &schema) {
+    if (!needs_update) {
-        m_config.schema = std::make_unique<Schema>(schema);
+        ObjectStore::verify_schema(*m_config.schema, *schema, m_config.read_only);
+        m_config.schema = std::move(schema);
+        m_config.schema_version = version;
+        return false;
     }
+
+    // Store the old config/schema for the migration function, and update
+    // our schema to the new one
+    auto old_schema = std::move(m_config.schema);
+    Config old_config(m_config);
+    old_config.read_only = true;
+    old_config.schema = std::move(old_schema);
+
+    m_config.schema = std::move(schema);
     m_config.schema_version = version;
 
-    try {
+    auto migration_function = [&](Group*,  Schema&) {
-        if (!m_config.read_only && ObjectStore::realm_requires_update(read_group(), version, schema)) {
+        SharedRealm old_realm(new Realm(old_config));
-            // keep old copy to pass to migration function
+        auto updated_realm = shared_from_this();
-            old_config.read_only = true;
+        m_config.migration_function(old_realm, updated_realm);
-            SharedRealm old_realm = SharedRealm(new Realm(old_config)), updated_realm = shared_from_this();
+    };
 
+    try {
         // update and migrate
         begin_transaction();
-            changed = ObjectStore::update_realm_with_schema(read_group(), version, *m_config.schema, [=](__unused Group *group, __unused Schema &target_schema) {
+        bool changed = ObjectStore::update_realm_with_schema(read_group(), *old_config.schema,
-                m_config.migration_function(old_realm, updated_realm);
+                                                             version, *m_config.schema,
-            });
+                                                             migration_function);
         commit_transaction();
-        }
+        return changed;
-        else {
-            ObjectStore::verify_schema(read_group(), *m_config.schema, m_config.read_only);
-        }
     }
     catch (...) {
         if (is_in_transaction()) {
@@ -166,7 +218,6 @@ bool Realm::update_schema(Schema &schema, uint64_t version)
         m_config.schema = std::move(old_config.schema);
         throw;
     }
-    return changed;
 }
 
 static void check_read_write(Realm *realm)
@@ -176,7 +227,7 @@ static void check_read_write(Realm *realm)
     }
 }
 
-void Realm::verify_thread()
+void Realm::verify_thread() const
 {
     if (m_thread_id != std::this_thread::get_id()) {
         throw IncorrectThreadException("Realm accessed from incorrect thread.");
@@ -192,18 +243,11 @@ void Realm::begin_transaction()
         throw InvalidTransactionException("The Realm is already in a write transaction");
     }
 
-    // if the upgrade to write will move the transaction forward, announce the change after promoting
-    bool announce = m_shared_group->has_changed();
-
     // make sure we have a read transaction
     read_group();
 
-    LangBindHelper::promote_to_write(*m_shared_group, *m_history);
+    transaction::begin(*m_shared_group, *m_history, m_delegate.get());
     m_in_transaction = true;
-
-    if (announce) {
-        send_local_notifications(DidChangeNotification);
-    }
 }
 
 void Realm::commit_transaction()
@@ -215,11 +259,9 @@ void Realm::commit_transaction()
         throw InvalidTransactionException("Can't commit a non-existing write transaction");
     }
 
-    LangBindHelper::commit_and_continue_as_read(*m_shared_group);
     m_in_transaction = false;
-
+    transaction::commit(*m_shared_group, *m_history, m_delegate.get());
-    send_external_notifications();
+    m_notifier->notify_others();
-    send_local_notifications(DidChangeNotification);
 }
 
 void Realm::cancel_transaction()
@@ -231,11 +273,10 @@ void Realm::cancel_transaction()
         throw InvalidTransactionException("Can't cancel a non-existing write transaction");
     }
 
-    LangBindHelper::rollback_and_continue_as_read(*m_shared_group, *m_history);
     m_in_transaction = false;
+    transaction::cancel(*m_shared_group, *m_history, m_delegate.get());
 }
 
-
 void Realm::invalidate()
 {
     verify_thread();
@@ -256,20 +297,21 @@ bool Realm::compact()
 {
     verify_thread();
 
-    bool success = false;
+    if (m_config.read_only) {
+        throw InvalidTransactionException("Can't compact a read-only Realm");
+    }
     if (m_in_transaction) {
         throw InvalidTransactionException("Can't compact a Realm within a write transaction");
     }
 
+    Group* group = read_group();
     for (auto &object_schema : *m_config.schema) {
-        ObjectStore::table_for_object_type(read_group(), object_schema.first)->optimize();
+        ObjectStore::table_for_object_type(group, object_schema.name)->optimize();
     }
-
     m_shared_group->end_read();
-    success = m_shared_group->compact();
+    m_group = nullptr;
-    m_shared_group->begin_read();
 
-    return success;
+    return m_shared_group->compact();
 }
 
 void Realm::notify()
@@ -277,24 +319,17 @@ void Realm::notify()
     verify_thread();
 
     if (m_shared_group->has_changed()) { // Throws
+        if (m_delegate) {
+            m_delegate->changes_available();
+        }
         if (m_auto_refresh) {
             if (m_group) {
-                LangBindHelper::advance_read(*m_shared_group, *m_history);
+                transaction::advance(*m_shared_group, *m_history, m_delegate.get());
             }
-            send_local_notifications(DidChangeNotification);
+            else if (m_delegate) {
-        }
+                m_delegate->did_change({}, {});
-        else {
-            send_local_notifications(RefreshRequiredNotification);
             }
         }
-}
-
-
-void Realm::send_local_notifications(const std::string &type)
-{
-    verify_thread();
-    for (NotificationFunction notification : m_notifications) {
-        (*notification)(type);
     }
 }
 
@@ -315,17 +350,25 @@ bool Realm::refresh()
     }
 
     if (m_group) {
-        LangBindHelper::advance_read(*m_shared_group, *m_history);
+        transaction::advance(*m_shared_group, *m_history, m_delegate.get());
     }
     else {
         // Create the read transaction
         read_group();
     }
 
-    send_local_notifications(DidChangeNotification);
     return true;
 }
 
+uint64_t Realm::get_schema_version(const realm::Realm::Config &config)
+{
+    if (auto existing_realm = s_global_cache.get_any_realm(config.path)) {
+        return existing_realm->config().schema_version;
+    }
+
+    return ObjectStore::get_schema_version(Realm(config).read_group());
+}
+
 SharedRealm RealmCache::get_realm(const std::string &path, std::thread::id thread_id)
 {
     std::lock_guard<std::mutex> lock(m_mutex);
@@ -401,4 +444,3 @@ void RealmCache::clear()
 
     m_cache.clear();
 }
-

shared_realm.hpp

@@ -19,21 +19,21 @@
 #ifndef REALM_REALM_HPP
 #define REALM_REALM_HPP
 
+#include <map>
 #include <memory>
 #include <thread>
 #include <vector>
-#include <mutex>
-#include <set>
-#include <map>
 
 #include "object_store.hpp"
 
 namespace realm {
-    class RealmCache;
+    class ClientHistory;
+    class ExternalCommitHelper;
     class Realm;
+    class RealmCache;
+    class RealmDelegate;
     typedef std::shared_ptr<Realm> SharedRealm;
     typedef std::weak_ptr<Realm> WeakRealm;
-    class ClientHistory;
 
     class Realm : public std::enable_shared_from_this<Realm>
     {
@@ -43,83 +43,81 @@ namespace realm {
         struct Config
         {
             std::string path;
-            bool read_only;
+            bool read_only = false;
-            bool in_memory;
+            bool in_memory = false;
-            StringData encryption_key;
+            bool cache = true;
+            std::vector<char> encryption_key;
 
             std::unique_ptr<Schema> schema;
-            uint64_t schema_version;
+            uint64_t schema_version = ObjectStore::NotVersioned;
 
             MigrationFunction migration_function;
 
-            Config() : read_only(false), in_memory(false), schema_version(ObjectStore::NotVersioned) {};
+            Config() = default;
+            Config(Config&&) = default;
             Config(const Config& c);
+            ~Config();
+
+            Config& operator=(Config const&);
+            Config& operator=(Config&&) = default;
         };
 
         // Get a cached Realm or create a new one if no cached copies exists
-        // Caching is done by path - mismatches for inMemory and readOnly Config properties
+        // Caching is done by path - mismatches for in_memory and read_only
-        // will raise an exception
+        // Config properties will raise an exception
-        // If schema/schema_version is specified, update_schema is called automatically on the realm
+        // If schema/schema_version is specified, update_schema is called
-        // and a migration is performed. If not specified, the schema version and schema are dynamically
+        // automatically on the realm and a migration is performed. If not
-        // read from the the existing Realm.
+        // specified, the schema version and schema are dynamically read from
-        static SharedRealm get_shared_realm(Config &config);
+        // the the existing Realm.
+        static SharedRealm get_shared_realm(Config config);
 
-        // Updates a Realm to a given target schema/version creating tables and updating indexes as necessary
+        // Updates a Realm to a given target schema/version creating tables and
-        // Uses the existing migration function on the Config, and the resulting Schema and version with updated
+        // updating indexes as necessary. Uses the existing migration function
+        // on the Config, and the resulting Schema and version with updated
         // column mappings are set on the realms config upon success.
         // returns if any changes were made
-        bool update_schema(Schema &schema, uint64_t version);
+        bool update_schema(std::unique_ptr<Schema> schema, uint64_t version);
+
+        static uint64_t get_schema_version(Config const& config);
 
         const Config &config() const { return m_config; }
 
         void begin_transaction();
         void commit_transaction();
         void cancel_transaction();
-        bool is_in_transaction() { return m_in_transaction; }
+        bool is_in_transaction() const { return m_in_transaction; }
 
         bool refresh();
         void set_auto_refresh(bool auto_refresh) { m_auto_refresh = auto_refresh; }
-        bool auto_refresh() { return m_auto_refresh; }
+        bool auto_refresh() const { return m_auto_refresh; }
         void notify();
 
-        typedef std::shared_ptr<std::function<void(const std::string)>> NotificationFunction;
-        void add_notification(NotificationFunction &notification) { m_notifications.insert(notification); }
-        void remove_notification(NotificationFunction notification) { m_notifications.erase(notification); }
-        void remove_all_notifications() { m_notifications.clear(); }
-
         void invalidate();
         bool compact();
 
         std::thread::id thread_id() const { return m_thread_id; }
-        void verify_thread();
+        void verify_thread() const;
 
-        const std::string RefreshRequiredNotification = "RefreshRequiredNotification";
+        ~Realm();
-        const std::string DidChangeNotification = "DidChangeNotification";
 
       private:
-        Realm(Config &config);
+        Realm(Config config);
 
         Config m_config;
-        std::thread::id m_thread_id;
+        std::thread::id m_thread_id = std::this_thread::get_id();
-        bool m_in_transaction;
+        bool m_in_transaction = false;
-        bool m_auto_refresh;
+        bool m_auto_refresh = true;
-
-        std::set<NotificationFunction> m_notifications;
-        void send_local_notifications(const std::string &notification);
-
-        typedef std::unique_ptr<std::function<void()>> ExternalNotificationFunction;
-        void send_external_notifications() { if (m_external_notifier) (*m_external_notifier)(); }
 
         std::unique_ptr<ClientHistory> m_history;
         std::unique_ptr<SharedGroup> m_shared_group;
         std::unique_ptr<Group> m_read_only_group;
 
-        Group *m_group;
+        Group *m_group = nullptr;
 
-        static std::mutex s_init_mutex;
+        std::shared_ptr<ExternalCommitHelper> m_notifier;
 
       public:
-        ExternalNotificationFunction m_external_notifier;
+        std::unique_ptr<RealmDelegate> m_delegate;
 
         // FIXME private
         Group *read_group();

transact_log_handler.cpp

@@ -0,0 +1,347 @@
+////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2015 Realm Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////
+
+#include "transact_log_handler.hpp"
+
+#include "realm_delegate.hpp"
+
+#include <realm/commit_log.hpp>
+#include <realm/group_shared.hpp>
+#include <realm/lang_bind_helper.hpp>
+
+using namespace realm;
+
+namespace {
+class TransactLogHandler {
+    using ColumnInfo = RealmDelegate::ColumnInfo;
+    using ObserverState = RealmDelegate::ObserverState;
+
+    // Observed table rows which need change information
+    std::vector<ObserverState> m_observers;
+    // Userdata pointers for rows which have been deleted
+    std::vector<void *> invalidated;
+    // Delegate to send change information to
+    RealmDelegate* m_delegate;
+
+    // Index of currently selected table
+    size_t m_current_table = 0;
+    // Change information for the currently selected LinkList, if any
+    ColumnInfo* m_active_linklist = nullptr;
+
+    // Get the change info for the given column, creating it if needed
+    static ColumnInfo& get_change(ObserverState& state, size_t i)
+    {
+        if (state.changes.size() <= i) {
+            state.changes.resize(std::max(state.changes.size() * 2, i + 1));
+        }
+        return state.changes[i];
+    }
+
+    // Loop over the columns which were changed in an observer state
+    template<typename Func>
+    static void for_each(ObserverState& state, Func&& f)
+    {
+        for (size_t i = 0; i < state.changes.size(); ++i) {
+            auto const& change = state.changes[i];
+            if (change.changed) {
+                f(i, change);
+            }
+        }
+    }
+
+    // Mark the given row/col as needing notifications sent
+    bool mark_dirty(size_t row_ndx, size_t col_ndx)
+    {
+        auto it = lower_bound(begin(m_observers), end(m_observers), ObserverState{m_current_table, row_ndx, nullptr});
+        if (it != end(m_observers) && it->table_ndx == m_current_table && it->row_ndx == row_ndx) {
+            get_change(*it, col_ndx).changed = true;
+        }
+        return true;
+    }
+
+    // Remove the given observer from the list of observed objects and add it
+    // to the listed of invalidated objects
+    void invalidate(ObserverState *o)
+    {
+        invalidated.push_back(o->info);
+        m_observers.erase(m_observers.begin() + (o - &m_observers[0]));
+    }
+
+public:
+    template<typename Func>
+    TransactLogHandler(RealmDelegate* delegate, SharedGroup& sg, Func&& func)
+    : m_delegate(delegate)
+    {
+        if (!delegate) {
+            func();
+            return;
+        }
+
+        m_observers = delegate->get_observed_rows();
+        if (m_observers.empty()) {
+            auto old_version = sg.get_version_of_current_transaction();
+            func();
+            if (old_version != sg.get_version_of_current_transaction()) {
+                delegate->did_change({}, {});
+            }
+            return;
+        }
+
+        func(*this);
+        delegate->did_change(m_observers, invalidated);
+    }
+
+    // Called at the end of the transaction log immediately before the version
+    // is advanced
+    void parse_complete()
+    {
+        m_delegate->will_change(m_observers, invalidated);
+    }
+
+    // These would require having an observer before schema init
+    // Maybe do something here to throw an error when multiple processes have different schemas?
+    bool insert_group_level_table(size_t, size_t, StringData) { return false; }
+    bool erase_group_level_table(size_t, size_t) { return false; }
+    bool rename_group_level_table(size_t, StringData) { return false; }
+    bool insert_column(size_t, DataType, StringData, bool) { return false; }
+    bool insert_link_column(size_t, DataType, StringData, size_t, size_t) { return false; }
+    bool erase_column(size_t) { return false; }
+    bool erase_link_column(size_t, size_t, size_t) { return false; }
+    bool rename_column(size_t, StringData) { return false; }
+    bool add_search_index(size_t) { return false; }
+    bool remove_search_index(size_t) { return false; }
+    bool add_primary_key(size_t) { return false; }
+    bool remove_primary_key() { return false; }
+    bool set_link_type(size_t, LinkType) { return false; }
+
+    bool select_table(size_t group_level_ndx, int, const size_t*) noexcept
+    {
+        m_current_table = group_level_ndx;
+        return true;
+    }
+
+    bool insert_empty_rows(size_t, size_t, size_t, bool)
+    {
+        // rows are only inserted at the end, so no need to do anything
+        return true;
+    }
+
+    bool erase_rows(size_t row_ndx, size_t, size_t last_row_ndx, bool unordered)
+    {
+        for (size_t i = 0; i < m_observers.size(); ++i) {
+            auto& o = m_observers[i];
+            if (o.table_ndx == m_current_table) {
+                if (o.row_ndx == row_ndx) {
+                    invalidate(&o);
+                    --i;
+                }
+                else if (unordered && o.row_ndx == last_row_ndx) {
+                    o.row_ndx = row_ndx;
+                }
+                else if (!unordered && o.row_ndx > row_ndx) {
+                    o.row_ndx -= 1;
+                }
+            }
+        }
+        return true;
+    }
+
+    bool clear_table()
+    {
+        for (size_t i = 0; i < m_observers.size(); ) {
+            auto& o = m_observers[i];
+            if (o.table_ndx == m_current_table) {
+                invalidate(&o);
+            }
+            else {
+                ++i;
+            }
+        }
+        return true;
+    }
+
+    bool select_link_list(size_t col, size_t row)
+    {
+        m_active_linklist = nullptr;
+        for (auto& o : m_observers) {
+            if (o.table_ndx == m_current_table && o.row_ndx == row) {
+                m_active_linklist = &get_change(o, col);
+                break;
+            }
+        }
+        return true;
+    }
+
+    void append_link_list_change(ColumnInfo::Kind kind, size_t index) {
+        ColumnInfo *o = m_active_linklist;
+        if (!o || o->kind == ColumnInfo::Kind::SetAll) {
+            // Active LinkList isn't observed or already has multiple kinds of changes
+            return;
+        }
+
+        if (o->kind == ColumnInfo::Kind::None) {
+            o->kind = kind;
+            o->changed = true;
+            o->indices.add(index);
+        }
+        else if (o->kind == kind) {
+            if (kind == ColumnInfo::Kind::Remove) {
+                o->indices.add_shifted(index);
+            }
+            else if (kind == ColumnInfo::Kind::Insert) {
+                o->indices.insert_at(index);
+            }
+            else {
+                o->indices.add(index);
+            }
+        }
+        else {
+            // Array KVO can only send a single kind of change at a time, so
+            // if there's multiple just give up and send "Set"
+            o->indices.set(0);
+            o->kind = ColumnInfo::Kind::SetAll;
+        }
+    }
+
+    bool link_list_set(size_t index, size_t)
+    {
+        append_link_list_change(ColumnInfo::Kind::Set, index);
+        return true;
+    }
+
+    bool link_list_insert(size_t index, size_t)
+    {
+        append_link_list_change(ColumnInfo::Kind::Insert, index);
+        return true;
+    }
+
+    bool link_list_erase(size_t index)
+    {
+        append_link_list_change(ColumnInfo::Kind::Remove, index);
+        return true;
+    }
+
+    bool link_list_nullify(size_t index)
+    {
+        append_link_list_change(ColumnInfo::Kind::Remove, index);
+        return true;
+    }
+
+    bool link_list_swap(size_t index1, size_t index2)
+    {
+        append_link_list_change(ColumnInfo::Kind::Set, index1);
+        append_link_list_change(ColumnInfo::Kind::Set, index2);
+        return true;
+    }
+
+    bool link_list_clear(size_t old_size)
+    {
+        ColumnInfo *o = m_active_linklist;
+        if (!o || o->kind == ColumnInfo::Kind::SetAll) {
+            return true;
+        }
+
+        if (o->kind == ColumnInfo::Kind::Remove)
+            old_size += o->indices.size();
+        else if (o->kind == ColumnInfo::Kind::Insert)
+            old_size -= o->indices.size();
+
+        o->indices.set(old_size);
+
+        o->kind = ColumnInfo::Kind::Remove;
+        o->changed = true;
+        return true;
+    }
+
+    bool link_list_move(size_t from, size_t to)
+    {
+        ColumnInfo *o = m_active_linklist;
+        if (!o || o->kind == ColumnInfo::Kind::SetAll) {
+            return true;
+        }
+        if (from > to) {
+            std::swap(from, to);
+        }
+
+        if (o->kind == ColumnInfo::Kind::None) {
+            o->kind = ColumnInfo::Kind::Set;
+            o->changed = true;
+        }
+        if (o->kind == ColumnInfo::Kind::Set) {
+            for (size_t i = from; i <= to; ++i)
+                o->indices.add(i);
+        }
+        else {
+            o->indices.set(0);
+            o->kind = ColumnInfo::Kind::SetAll;
+        }
+        return true;
+    }
+
+    // Things that just mark the field as modified
+    bool set_int(size_t col, size_t row, int_fast64_t) { return mark_dirty(row, col); }
+    bool set_bool(size_t col, size_t row, bool) { return mark_dirty(row, col); }
+    bool set_float(size_t col, size_t row, float) { return mark_dirty(row, col); }
+    bool set_double(size_t col, size_t row, double) { return mark_dirty(row, col); }
+    bool set_string(size_t col, size_t row, StringData) { return mark_dirty(row, col); }
+    bool set_binary(size_t col, size_t row, BinaryData) { return mark_dirty(row, col); }
+    bool set_date_time(size_t col, size_t row, DateTime) { return mark_dirty(row, col); }
+    bool set_table(size_t col, size_t row) { return mark_dirty(row, col); }
+    bool set_mixed(size_t col, size_t row, const Mixed&) { return mark_dirty(row, col); }
+    bool set_link(size_t col, size_t row, size_t) { return mark_dirty(row, col); }
+    bool set_null(size_t col, size_t row) { return mark_dirty(row, col); }
+    bool nullify_link(size_t col, size_t row) { return mark_dirty(row, col); }
+
+    // Doesn't change any data
+    bool optimize_table() { return true; }
+
+    // Used for subtables, which we currently don't support
+    bool select_descriptor(int, const size_t*) { return false; }
+};
+} // anonymous namespace
+
+namespace realm {
+namespace transaction {
+void advance(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate) {
+    TransactLogHandler(delegate, sg, [&](auto&&... args) {
+        LangBindHelper::advance_read(sg, history, std::move(args)...);
+    });
+}
+
+void begin(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate) {
+    TransactLogHandler(delegate, sg, [&](auto&&... args) {
+        LangBindHelper::promote_to_write(sg, history, std::move(args)...);
+    });
+}
+
+void commit(SharedGroup& sg, ClientHistory&, RealmDelegate* delegate) {
+    LangBindHelper::commit_and_continue_as_read(sg);
+
+    if (delegate) {
+        delegate->did_change({}, {});
+    }
+}
+
+void cancel(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate) {
+    TransactLogHandler(delegate, sg, [&](auto&&... args) {
+        LangBindHelper::rollback_and_continue_as_read(sg, history, std::move(args)...);
+    });
+}
+
+} // namespace transaction
+} // namespace realm

transact_log_handler.hpp

@@ -0,0 +1,46 @@
+////////////////////////////////////////////////////////////////////////////
+//
+// Copyright 2015 Realm Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////
+
+#ifndef REALM_TRANSACT_LOG_HANDLER_HPP
+#define REALM_TRANSACT_LOG_HANDLER_HPP
+
+namespace realm {
+class RealmDelegate;
+class SharedGroup;
+class ClientHistory;
+
+namespace transaction {
+// Advance the read transaction version, with change notifications sent to delegate
+// Must not be called from within a write transaction.
+void advance(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate);
+
+// Begin a write transaction
+// If the read transaction version is not up to date, will first advance to the
+// most recent read transaction and sent notifications to delegate
+void begin(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate);
+
+// Commit a write transaction
+void commit(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate);
+
+// Cancel a write transaction and roll back all changes, with change notifications
+// for reverting to the old values sent to delegate
+void cancel(SharedGroup& sg, ClientHistory& history, RealmDelegate* delegate);
+} // namespace transaction
+} // namespace realm
+
+#endif /* REALM_TRANSACT_LOG_HANDLER_HPP */