From 9cf26ed2cb1ddc4b33cd432785bec711f6def477 Mon Sep 17 00:00:00 2001 From: Mark Rowe Date: Wed, 6 Jan 2016 14:40:53 -0800 Subject: [PATCH 01/79] Add an initial pass at a CMake-based build system. It currently creates a dynamic library, and builds on OS X only. --- .gitignore | 12 ++++++++++++ .gitmodules | 3 +++ CMake/CompilerFlags.cmake | 10 ++++++++++ CMake/RealmCore.cmake | 28 ++++++++++++++++++++++++++++ CMakeLists.txt | 32 ++++++++++++++++++++++++++++++++ external/pegtl | 1 + 6 files changed, 86 insertions(+) create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 CMake/CompilerFlags.cmake create mode 100644 CMake/RealmCore.cmake create mode 100644 CMakeLists.txt create mode 160000 external/pegtl diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..c1ef0e8a --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +# CMake +.ninja_deps +.ninja_log +CMakeCache.txt +CMakeFiles/ +Makefile +build.ninja +cmake_install.cmake +rules.ninja + +# Build products +librealm-object-store.dylib diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..b7e77b91 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "external/pegtl"] + path = external/pegtl + url = https://github.com/ColinH/PEGTL diff --git a/CMake/CompilerFlags.cmake b/CMake/CompilerFlags.cmake new file mode 100644 index 00000000..a5a6bd36 --- /dev/null +++ b/CMake/CompilerFlags.cmake @@ -0,0 +1,10 @@ +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED on) +set(CMAKE_CXX_EXTENSIONS off) +set(CMAKE_BUILD_TYPE CACHE STRING Debug) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") + +if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang" AND ${CMAKE_GENERATOR} STREQUAL "Ninja") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fcolor-diagnostics") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcolor-diagnostics") +endif() diff --git a/CMake/RealmCore.cmake b/CMake/RealmCore.cmake new file mode 100644 index 00000000..45a5beff --- /dev/null +++ b/CMake/RealmCore.cmake @@ -0,0 +1,28 @@ +include(ExternalProject) + +function(download_realm_core realm_core_version) + set(core_url "https://static.realm.io/downloads/core/realm-core-${realm_core_version}.tar.bz2") + set(core_directory "${CMAKE_CURRENT_SOURCE_DIR}${CMAKE_FILES_DIRECTORY}/core-${realm_core_version}") + + set(core_library_debug ${core_directory}/librealm-dbg.a) + set(core_library_release ${core_directory}/librealm.a) + + ExternalProject_Add(realm-core + URL ${core_url} + PREFIX ${CMAKE_CURRENT_SOURCE_DIR}${CMAKE_FILES_DIRECTORY}/realm-core + DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}${CMAKE_FILES_DIRECTORY} + SOURCE_DIR ${core_directory} + BUILD_BYPRODUCTS ${core_library_debug} ${core_library_release} + USES_TERMINAL_DOWNLOAD 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "") + + add_library(realm STATIC IMPORTED) + add_dependencies(realm realm-core) + set_property(TARGET realm PROPERTY IMPORTED_LOCATION_DEBUG ${core_library_debug}) + set_property(TARGET realm PROPERTY IMPORTED_LOCATION_RELEASE ${core_library_release}) + set_property(TARGET realm PROPERTY IMPORTED_LOCATION ${core_library_release}) + + set(REALM_CORE_INCLUDE_DIR ${core_directory}/include PARENT_SCOPE) +endfunction() diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..539960b7 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,32 @@ +project(realm-object-store) + +cmake_minimum_required(VERSION 3.4.0) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake") + +include(CompilerFlags) + +include(RealmCore) +download_realm_core(0.95.5) + +include_directories(${REALM_CORE_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR} impl external/pegtl) + +set(SOURCES + index_set.cpp + list.cpp + object_schema.cpp + object_store.cpp + results.cpp + schema.cpp + shared_realm.cpp + impl/transact_log_handler.cpp + parser/parser.cpp + parser/query_builder.cpp) + +if(APPLE) + include_directories(impl/apple) + list(APPEND SOURCES impl/apple/external_commit_helper.cpp) + find_library(CF_LIBRARY CoreFoundation) +endif() + +add_library(realm-object-store SHARED ${SOURCES}) +target_link_libraries(realm-object-store realm ${CF_LIBRARY}) diff --git a/external/pegtl b/external/pegtl new file mode 160000 index 00000000..49a5b0a4 --- /dev/null +++ b/external/pegtl @@ -0,0 +1 @@ +Subproject commit 49a5b0a49e154b362ef9cf1e756dd8673ddd4efe From c4191d8af65e325ce8001949c0b04625f001b198 Mon Sep 17 00:00:00 2001 From: Mark Rowe Date: Wed, 6 Jan 2016 15:52:12 -0800 Subject: [PATCH 02/79] Hook the parser tests into the CMake build system. --- .gitignore | 1 + .gitmodules | 3 ++ CMakeLists.txt | 24 ++-------- external/catch | 1 + src/CMakeLists.txt | 22 ++++++++++ .../binding_context.hpp | 0 .../impl}/apple/external_commit_helper.cpp | 0 .../impl}/apple/external_commit_helper.hpp | 0 {impl => src/impl}/transact_log_handler.cpp | 0 {impl => src/impl}/transact_log_handler.hpp | 0 index_set.cpp => src/index_set.cpp | 0 index_set.hpp => src/index_set.hpp | 0 list.cpp => src/list.cpp | 0 list.hpp => src/list.hpp | 0 .../object_accessor.hpp | 0 object_schema.cpp => src/object_schema.cpp | 0 object_schema.hpp => src/object_schema.hpp | 0 object_store.cpp => src/object_store.cpp | 0 object_store.hpp => src/object_store.hpp | 0 {parser => src/parser}/parser.cpp | 0 {parser => src/parser}/parser.hpp | 0 {parser => src/parser}/query_builder.cpp | 0 {parser => src/parser}/query_builder.hpp | 0 {parser => src/parser}/test.sh | 0 property.hpp => src/property.hpp | 0 results.cpp => src/results.cpp | 0 results.hpp => src/results.hpp | 0 schema.cpp => src/schema.cpp | 0 schema.hpp => src/schema.hpp | 0 shared_realm.cpp => src/shared_realm.cpp | 0 shared_realm.hpp => src/shared_realm.hpp | 0 tests/CMakeLists.txt | 5 +++ tests/main.cpp | 2 + parser/test.cpp => tests/parser.cpp | 44 +++++-------------- parser/queryTests.json => tests/query.json | 0 35 files changed, 48 insertions(+), 54 deletions(-) create mode 160000 external/catch create mode 100644 src/CMakeLists.txt rename binding_context.hpp => src/binding_context.hpp (100%) rename {impl => src/impl}/apple/external_commit_helper.cpp (100%) rename {impl => src/impl}/apple/external_commit_helper.hpp (100%) rename {impl => src/impl}/transact_log_handler.cpp (100%) rename {impl => src/impl}/transact_log_handler.hpp (100%) rename index_set.cpp => src/index_set.cpp (100%) rename index_set.hpp => src/index_set.hpp (100%) rename list.cpp => src/list.cpp (100%) rename list.hpp => src/list.hpp (100%) rename object_accessor.hpp => src/object_accessor.hpp (100%) rename object_schema.cpp => src/object_schema.cpp (100%) rename object_schema.hpp => src/object_schema.hpp (100%) rename object_store.cpp => src/object_store.cpp (100%) rename object_store.hpp => src/object_store.hpp (100%) rename {parser => src/parser}/parser.cpp (100%) rename {parser => src/parser}/parser.hpp (100%) rename {parser => src/parser}/query_builder.cpp (100%) rename {parser => src/parser}/query_builder.hpp (100%) rename {parser => src/parser}/test.sh (100%) rename property.hpp => src/property.hpp (100%) rename results.cpp => src/results.cpp (100%) rename results.hpp => src/results.hpp (100%) rename schema.cpp => src/schema.cpp (100%) rename schema.hpp => src/schema.hpp (100%) rename shared_realm.cpp => src/shared_realm.cpp (100%) rename shared_realm.hpp => src/shared_realm.hpp (100%) create mode 100644 tests/CMakeLists.txt create mode 100644 tests/main.cpp rename parser/test.cpp => tests/parser.cpp (72%) rename parser/queryTests.json => tests/query.json (100%) diff --git a/.gitignore b/.gitignore index c1ef0e8a..7d9b2fa3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ rules.ninja # Build products librealm-object-store.dylib +tests/tests diff --git a/.gitmodules b/.gitmodules index b7e77b91..44b8f2ab 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "external/pegtl"] path = external/pegtl url = https://github.com/ColinH/PEGTL +[submodule "external/catch"] + path = external/catch + url = https://github.com/philsquared/Catch diff --git a/CMakeLists.txt b/CMakeLists.txt index 539960b7..ee79aa5a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,25 +8,7 @@ include(CompilerFlags) include(RealmCore) download_realm_core(0.95.5) -include_directories(${REALM_CORE_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR} impl external/pegtl) +include_directories(${REALM_CORE_INCLUDE_DIR} src external/pegtl) -set(SOURCES - index_set.cpp - list.cpp - object_schema.cpp - object_store.cpp - results.cpp - schema.cpp - shared_realm.cpp - impl/transact_log_handler.cpp - parser/parser.cpp - parser/query_builder.cpp) - -if(APPLE) - include_directories(impl/apple) - list(APPEND SOURCES impl/apple/external_commit_helper.cpp) - find_library(CF_LIBRARY CoreFoundation) -endif() - -add_library(realm-object-store SHARED ${SOURCES}) -target_link_libraries(realm-object-store realm ${CF_LIBRARY}) +add_subdirectory(src) +add_subdirectory(tests) diff --git a/external/catch b/external/catch new file mode 160000 index 00000000..f294c984 --- /dev/null +++ b/external/catch @@ -0,0 +1 @@ +Subproject commit f294c9847272b1b92c5119a6f711e57113b5f231 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 00000000..61204003 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,22 @@ +include_directories(impl) + +set(SOURCES + index_set.cpp + list.cpp + object_schema.cpp + object_store.cpp + results.cpp + schema.cpp + shared_realm.cpp + impl/transact_log_handler.cpp + parser/parser.cpp + parser/query_builder.cpp) + +if(APPLE) + include_directories(impl/apple) + list(APPEND SOURCES impl/apple/external_commit_helper.cpp) + find_library(CF_LIBRARY CoreFoundation) +endif() + +add_library(realm-object-store SHARED ${SOURCES}) +target_link_libraries(realm-object-store realm ${CF_LIBRARY}) diff --git a/binding_context.hpp b/src/binding_context.hpp similarity index 100% rename from binding_context.hpp rename to src/binding_context.hpp diff --git a/impl/apple/external_commit_helper.cpp b/src/impl/apple/external_commit_helper.cpp similarity index 100% rename from impl/apple/external_commit_helper.cpp rename to src/impl/apple/external_commit_helper.cpp diff --git a/impl/apple/external_commit_helper.hpp b/src/impl/apple/external_commit_helper.hpp similarity index 100% rename from impl/apple/external_commit_helper.hpp rename to src/impl/apple/external_commit_helper.hpp diff --git a/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp similarity index 100% rename from impl/transact_log_handler.cpp rename to src/impl/transact_log_handler.cpp diff --git a/impl/transact_log_handler.hpp b/src/impl/transact_log_handler.hpp similarity index 100% rename from impl/transact_log_handler.hpp rename to src/impl/transact_log_handler.hpp diff --git a/index_set.cpp b/src/index_set.cpp similarity index 100% rename from index_set.cpp rename to src/index_set.cpp diff --git a/index_set.hpp b/src/index_set.hpp similarity index 100% rename from index_set.hpp rename to src/index_set.hpp diff --git a/list.cpp b/src/list.cpp similarity index 100% rename from list.cpp rename to src/list.cpp diff --git a/list.hpp b/src/list.hpp similarity index 100% rename from list.hpp rename to src/list.hpp diff --git a/object_accessor.hpp b/src/object_accessor.hpp similarity index 100% rename from object_accessor.hpp rename to src/object_accessor.hpp diff --git a/object_schema.cpp b/src/object_schema.cpp similarity index 100% rename from object_schema.cpp rename to src/object_schema.cpp diff --git a/object_schema.hpp b/src/object_schema.hpp similarity index 100% rename from object_schema.hpp rename to src/object_schema.hpp diff --git a/object_store.cpp b/src/object_store.cpp similarity index 100% rename from object_store.cpp rename to src/object_store.cpp diff --git a/object_store.hpp b/src/object_store.hpp similarity index 100% rename from object_store.hpp rename to src/object_store.hpp diff --git a/parser/parser.cpp b/src/parser/parser.cpp similarity index 100% rename from parser/parser.cpp rename to src/parser/parser.cpp diff --git a/parser/parser.hpp b/src/parser/parser.hpp similarity index 100% rename from parser/parser.hpp rename to src/parser/parser.hpp diff --git a/parser/query_builder.cpp b/src/parser/query_builder.cpp similarity index 100% rename from parser/query_builder.cpp rename to src/parser/query_builder.cpp diff --git a/parser/query_builder.hpp b/src/parser/query_builder.hpp similarity index 100% rename from parser/query_builder.hpp rename to src/parser/query_builder.hpp diff --git a/parser/test.sh b/src/parser/test.sh similarity index 100% rename from parser/test.sh rename to src/parser/test.sh diff --git a/property.hpp b/src/property.hpp similarity index 100% rename from property.hpp rename to src/property.hpp diff --git a/results.cpp b/src/results.cpp similarity index 100% rename from results.cpp rename to src/results.cpp diff --git a/results.hpp b/src/results.hpp similarity index 100% rename from results.hpp rename to src/results.hpp diff --git a/schema.cpp b/src/schema.cpp similarity index 100% rename from schema.cpp rename to src/schema.cpp diff --git a/schema.hpp b/src/schema.hpp similarity index 100% rename from schema.hpp rename to src/schema.hpp diff --git a/shared_realm.cpp b/src/shared_realm.cpp similarity index 100% rename from shared_realm.cpp rename to src/shared_realm.cpp diff --git a/shared_realm.hpp b/src/shared_realm.hpp similarity index 100% rename from shared_realm.hpp rename to src/shared_realm.hpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 00000000..a7b20889 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,5 @@ +include_directories(../external/catch/single_include) +add_executable(tests main.cpp parser.cpp) +target_link_libraries(tests realm-object-store) + +add_custom_target(run-tests USES_TERMINAL DEPENDS tests COMMAND ./tests) diff --git a/tests/main.cpp b/tests/main.cpp new file mode 100644 index 00000000..0c7c351f --- /dev/null +++ b/tests/main.cpp @@ -0,0 +1,2 @@ +#define CATCH_CONFIG_MAIN +#include "catch.hpp" diff --git a/parser/test.cpp b/tests/parser.cpp similarity index 72% rename from parser/test.cpp rename to tests/parser.cpp index f834c505..0815d3d6 100644 --- a/parser/test.cpp +++ b/tests/parser.cpp @@ -1,10 +1,8 @@ - -#include "parser.hpp" +#include "catch.hpp" +#include "parser/parser.hpp" #include #include -#include -#include static std::vector valid_queries = { // true/false predicates @@ -131,36 +129,16 @@ static std::vector invalid_queries = { "truepredicate & truepredicate", }; -namespace realm { -namespace parser { - -bool test_grammar() -{ - bool success = true; - for (auto &query : valid_queries) { - std::cout << "valid query: " << query << std::endl; - try { - realm::parser::parse(query); - } catch (std::exception &ex) { - std::cout << "FAILURE - " << ex.what() << std::endl; - success = false; - } +TEST_CASE("valid queries") { + for (auto& query : valid_queries) { + INFO("query: " << query); + CHECK_NOTHROW(realm::parser::parse(query)); } +} - for (auto &query : invalid_queries) { - std::cout << "invalid query: " << query << std::endl; - try { - realm::parser::parse(query); - } catch (std::exception &ex) { - // std::cout << "message: " << ex.what() << std::endl; - continue; - } - std::cout << "FAILURE - query should throw an exception" << std::endl; - success = false; +TEST_CASE("invalid queries") { + for (auto& query : invalid_queries) { + INFO("query: " << query); + CHECK_THROWS(realm::parser::parse(query)); } - - return success; -} - -} } diff --git a/parser/queryTests.json b/tests/query.json similarity index 100% rename from parser/queryTests.json rename to tests/query.json From 9d43f8952e5e88fb0614b16fc274ae6e041a803f Mon Sep 17 00:00:00 2001 From: Mark Rowe Date: Wed, 6 Jan 2016 17:03:27 -0800 Subject: [PATCH 03/79] Rework `download_realm_core` to avoid re-downloading core on clean builds. --- CMake/RealmCore.cmake | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/CMake/RealmCore.cmake b/CMake/RealmCore.cmake index 45a5beff..daf83eba 100644 --- a/CMake/RealmCore.cmake +++ b/CMake/RealmCore.cmake @@ -1,22 +1,37 @@ include(ExternalProject) -function(download_realm_core realm_core_version) - set(core_url "https://static.realm.io/downloads/core/realm-core-${realm_core_version}.tar.bz2") - set(core_directory "${CMAKE_CURRENT_SOURCE_DIR}${CMAKE_FILES_DIRECTORY}/core-${realm_core_version}") +function(download_realm_core core_version) + set(core_url "https://static.realm.io/downloads/core/realm-core-${core_version}.tar.bz2") + set(core_tarball_name "realm-core-${core_version}.tar.bz2") + set(core_temp_tarball "/tmp/${core_tarball_name}") + set(core_directory_parent "${CMAKE_CURRENT_SOURCE_DIR}${CMAKE_FILES_DIRECTORY}") + set(core_directory "${core_directory_parent}/realm-core-${core_version}") + set(core_tarball "${core_directory_parent}/${core_tarball_name}") + + if (NOT EXISTS ${core_tarball}) + if (NOT EXISTS ${core_temp_tarball}) + message("Downloading core ${core_version} from ${core_url}.") + file(DOWNLOAD ${core_url} ${core_temp_tarball}.tmp SHOW_PROGRESS) + file(RENAME ${core_temp_tarball}.tmp ${core_temp_tarball}) + endif() + file(COPY ${core_temp_tarball} DESTINATION ${core_directory_parent}) + endif() set(core_library_debug ${core_directory}/librealm-dbg.a) set(core_library_release ${core_directory}/librealm.a) + set(core_libraries ${core_library_debug} ${core_library_release}) - ExternalProject_Add(realm-core - URL ${core_url} - PREFIX ${CMAKE_CURRENT_SOURCE_DIR}${CMAKE_FILES_DIRECTORY}/realm-core - DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}${CMAKE_FILES_DIRECTORY} - SOURCE_DIR ${core_directory} - BUILD_BYPRODUCTS ${core_library_debug} ${core_library_release} - USES_TERMINAL_DOWNLOAD 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "") + add_custom_command( + COMMENT "Extracting ${core_tarball_name}" + OUTPUT ${core_libraries} + DEPENDS ${core_temp_tarball} + COMMAND ${CMAKE_COMMAND} -E copy ${core_temp_tarball} ${core_directory_parent} + COMMAND ${CMAKE_COMMAND} -E tar xf ${core_tarball} + COMMAND ${CMAKE_COMMAND} -E remove_directory ${core_directory} + COMMAND ${CMAKE_COMMAND} -E rename core ${core_directory} + COMMAND ${CMAKE_COMMAND} -E touch_nocreate ${core_libraries}) + + add_custom_target(realm-core DEPENDS ${core_libraries}) add_library(realm STATIC IMPORTED) add_dependencies(realm realm-core) From 548701c2fd69a977cabb20d6a3688b027ff798f5 Mon Sep 17 00:00:00 2001 From: Mark Rowe Date: Wed, 6 Jan 2016 17:08:20 -0800 Subject: [PATCH 04/79] Remove a bogus forward-declaration. --- src/object_schema.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/object_schema.hpp b/src/object_schema.hpp index 0ede3f5d..4ede6ce5 100644 --- a/src/object_schema.hpp +++ b/src/object_schema.hpp @@ -25,7 +25,6 @@ #include namespace realm { - class Property; class Group; struct Property; From fbb386a7352f293f2eb1541a9e914e32919edbe3 Mon Sep 17 00:00:00 2001 From: Mark Rowe Date: Wed, 6 Jan 2016 17:23:11 -0800 Subject: [PATCH 05/79] Improve things a little for Linux. --- CMake/CompilerFlags.cmake | 11 ++++++++--- CMakeLists.txt | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CMake/CompilerFlags.cmake b/CMake/CompilerFlags.cmake index a5a6bd36..25d78a77 100644 --- a/CMake/CompilerFlags.cmake +++ b/CMake/CompilerFlags.cmake @@ -4,7 +4,12 @@ set(CMAKE_CXX_EXTENSIONS off) set(CMAKE_BUILD_TYPE CACHE STRING Debug) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") -if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang" AND ${CMAKE_GENERATOR} STREQUAL "Ninja") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fcolor-diagnostics") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcolor-diagnostics") +if(${CMAKE_GENERATOR} STREQUAL "Ninja") + if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fcolor-diagnostics") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcolor-diagnostics") + elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always") + endif() endif() diff --git a/CMakeLists.txt b/CMakeLists.txt index ee79aa5a..85138132 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ project(realm-object-store) -cmake_minimum_required(VERSION 3.4.0) +cmake_minimum_required(VERSION 3.2.0) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake") include(CompilerFlags) From 1ff80797ed68866cf5c8ef4be7c2b8dc6357d867 Mon Sep 17 00:00:00 2001 From: Mark Rowe Date: Tue, 19 Jan 2016 10:55:07 -0800 Subject: [PATCH 06/79] Add information on using the build system to the README. --- README.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 30ae7900..948a47e6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Realm Object Store -Cross-platform code used accross bindings. Binding developers can choose to use some or all the included functionality +Cross-platform code used accross bindings. Binding developers can choose to use some or all the included functionality: - `object_store`/`schema`/`object_schema`/`property` - contains the structures and logic used to setup and modify realm files and their schema. - `shared_realm` - wraps the object_store apis to provide transactions, notifications, realm caching, migrations, and other higher level functionality. - `object_accessor`/`results`/`list` - accessor classes, object creation/update pipeline, and helpers for creating platform specific property getters and setters. @@ -8,8 +8,28 @@ Cross-platform code used accross bindings. Binding developers can choose to use ## Building -TBD +The object store's build system currently only suports building for OS X. The object store itself can build for all Apple +platforms when integrated into a binding. + +1. Install CMake. You can download an installer for OS X from the [CMake download page], or install via [Homebrew](http://brew.sh): + ``` + brew install cmake + ``` + +2. Generate build files: + + ``` + cmake . + ``` + +3. Build: + + ``` + make + ``` ## Testing -TBD +``` +make run-tests +``` From 0bae415718a9536ba19a3bcffb4f7bf727834dc3 Mon Sep 17 00:00:00 2001 From: kishikawa katsumi Date: Wed, 25 Nov 2015 00:39:13 +0900 Subject: [PATCH 07/79] Read-only Realm should be opened even in immutable directory Realm files in an app bundle should be opened if marked as read-only --- src/shared_realm.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 2ed7a7c3..91f4b46f 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -158,11 +158,15 @@ SharedRealm Realm::get_shared_realm(Config config) // FIXME - need to validate that schemas match realm->m_config.schema = std::make_unique(*existing->m_config.schema); - realm->m_notifier = existing->m_notifier; - realm->m_notifier->add_realm(realm.get()); + if (!realm->m_config.read_only) { + realm->m_notifier = existing->m_notifier; + realm->m_notifier->add_realm(realm.get()); + } } else { - realm->m_notifier = std::make_shared(realm.get()); + if (!realm->m_config.read_only) { + realm->m_notifier = std::make_shared(realm.get()); + } // otherwise get the schema from the group realm->m_config.schema = std::make_unique(ObjectStore::schema_from_group(realm->read_group())); From 6c25eeb85c0d8e0d6116642d7e8cacf235e789ad Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 16 Nov 2015 11:22:23 -0800 Subject: [PATCH 08/79] Fix race condition in multiprocess schema init If the schema was initialized by a different process between when the old schema was read and the write transaction was began, the schema init code would see the updated schema version but not re-read the schema, resulting in it thinking that a migration was required when the schema actually matched. --- src/shared_realm.cpp | 80 ++++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 2ed7a7c3..17c9e682 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -201,6 +201,7 @@ bool Realm::update_schema(std::unique_ptr schema, uint64_t version) { schema->validate(); + // If the schema version matches, just verify that the schema itself also matches bool needs_update = !m_config.read_only && (m_config.schema_version != version || ObjectStore::needs_update(*m_config.schema, *schema)); if (!needs_update) { ObjectStore::verify_schema(*m_config.schema, *schema, m_config.read_only); @@ -209,41 +210,64 @@ bool Realm::update_schema(std::unique_ptr schema, uint64_t version) return false; } - // Store the old config/schema for the migration function, and update - // our schema to the new one - auto old_schema = std::move(m_config.schema); - Config old_config(m_config); - old_config.read_only = true; - old_config.schema = std::move(old_schema); + begin_transaction(); + struct WriteTransactionGuard { + Realm& realm; + ~WriteTransactionGuard() { + if (realm.is_in_transaction()) { + realm.cancel_transaction(); + } + } + } write_transaction_guard{*this}; - m_config.schema = std::move(schema); - m_config.schema_version = version; + // Recheck the schema version after beginning the write transaction + // If it changed then someone else initialized the schema and we need to + // recheck everything + auto current_schema_version = ObjectStore::get_schema_version(read_group()); + if (current_schema_version != m_config.schema_version) { + cancel_transaction(); + + m_config.schema_version = current_schema_version; + *m_config.schema = ObjectStore::schema_from_group(read_group()); + return update_schema(std::move(schema), version); + } auto migration_function = [&](Group*, Schema&) { - SharedRealm old_realm(new Realm(old_config)); - auto updated_realm = shared_from_this(); - if (m_config.migration_function) { - m_config.migration_function(old_realm, updated_realm); + SharedRealm old_realm(new Realm(m_config)); + old_realm->m_config.read_only = true; + + auto new_realm = shared_from_this(); + m_config.schema = std::move(schema); + m_config.schema_version = version; + + if (!m_config.migration_function) { + return; + } + + try { + m_config.migration_function(old_realm, new_realm); + } + catch (...) { + m_config.schema = std::move(old_realm->m_config.schema); + m_config.schema_version = old_realm->m_config.schema_version; + throw; } }; - try { - // update and migrate - begin_transaction(); - bool changed = ObjectStore::update_realm_with_schema(read_group(), *old_config.schema, - version, *m_config.schema, - migration_function); - commit_transaction(); - return changed; - } - catch (...) { - if (is_in_transaction()) { - cancel_transaction(); - } - m_config.schema_version = old_config.schema_version; - m_config.schema = std::move(old_config.schema); - throw; + bool changed = ObjectStore::update_realm_with_schema(read_group(), *m_config.schema, + version, *schema, + migration_function); + commit_transaction(); + + if (schema) { + // We update the schema after opening the "old" Realm in the migration + // block to reduce the amount of juggling required, but that means that + // the schema hasn't been updated if no migration occurred + m_config.schema = std::move(schema); + m_config.schema_version = version; } + + return changed; } static void check_read_write(Realm *realm) From 9c2d4703ed77a446549fe4d5db78c4e7157eb694 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 17 Nov 2015 14:40:43 -0800 Subject: [PATCH 09/79] Reshuffle the updating of m_config in Realm::update_schema() --- src/shared_realm.cpp | 45 ++++++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 17c9e682..4e07fb3d 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -232,42 +232,33 @@ bool Realm::update_schema(std::unique_ptr schema, uint64_t version) return update_schema(std::move(schema), version); } + Config old_config(m_config); auto migration_function = [&](Group*, Schema&) { - SharedRealm old_realm(new Realm(m_config)); + SharedRealm old_realm(new Realm(old_config)); + // Need to open in read-write mode so that it uses a SharedGroup, but + // users shouldn't actually be able to write via the old realm old_realm->m_config.read_only = true; - auto new_realm = shared_from_this(); - m_config.schema = std::move(schema); - m_config.schema_version = version; - - if (!m_config.migration_function) { - return; - } - - try { - m_config.migration_function(old_realm, new_realm); - } - catch (...) { - m_config.schema = std::move(old_realm->m_config.schema); - m_config.schema_version = old_realm->m_config.schema_version; - throw; + if (m_config.migration_function) { + m_config.migration_function(old_realm, shared_from_this()); } }; - bool changed = ObjectStore::update_realm_with_schema(read_group(), *m_config.schema, - version, *schema, - migration_function); - commit_transaction(); - - if (schema) { - // We update the schema after opening the "old" Realm in the migration - // block to reduce the amount of juggling required, but that means that - // the schema hasn't been updated if no migration occurred + try { m_config.schema = std::move(schema); m_config.schema_version = version; - } - return changed; + bool changed = ObjectStore::update_realm_with_schema(read_group(), *old_config.schema, + version, *m_config.schema, + migration_function); + commit_transaction(); + return changed; + } + catch (...) { + m_config.schema = std::move(old_config.schema); + m_config.schema_version = old_config.schema_version; + throw; + } } static void check_read_write(Realm *realm) From 8d7b5d8d08749476754f53167f8e2b8e88a3ea0f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 17 Nov 2015 14:51:11 -0800 Subject: [PATCH 10/79] Remove the tracking of if any changes were made from update_realm_with_schema() It was no longer actually used for anything since we now check if any changes are actually needed before calling it (to avoid beginning a write transaction when not needed). --- src/object_store.cpp | 30 +++++++----------------------- src/object_store.hpp | 7 +++---- src/shared_realm.cpp | 11 +++++------ src/shared_realm.hpp | 3 +-- 4 files changed, 16 insertions(+), 35 deletions(-) diff --git a/src/object_store.cpp b/src/object_store.cpp index 3aed85c5..29f2828e 100644 --- a/src/object_store.cpp +++ b/src/object_store.cpp @@ -52,13 +52,11 @@ bool ObjectStore::has_metadata_tables(const Group *group) { return group->get_table(c_primaryKeyTableName) && group->get_table(c_metadataTableName); } -bool ObjectStore::create_metadata_tables(Group *group) { - bool changed = false; +void ObjectStore::create_metadata_tables(Group *group) { TableRef table = group->get_or_add_table(c_primaryKeyTableName); if (table->get_column_count() == 0) { table->add_column(type_String, c_primaryKeyObjectClassColumnName); table->add_column(type_String, c_primaryKeyPropertyNameColumnName); - changed = true; } table = group->get_or_add_table(c_metadataTableName); @@ -68,10 +66,7 @@ bool ObjectStore::create_metadata_tables(Group *group) { // set initial version table->add_empty_row(); table->set_int(c_versionColumnIndex, c_zeroRowIndex, ObjectStore::NotVersioned); - changed = true; } - - return changed; } uint64_t ObjectStore::get_schema_version(const Group *group) { @@ -255,9 +250,7 @@ static void copy_property_values(const Property& source, const Property& destina // set references to tables on targetSchema and create/update any missing or out-of-date tables // if update existing is true, updates existing tables, otherwise validates existing tables // NOTE: must be called from within write transaction -bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update_existing) { - bool changed = false; - +void ObjectStore::create_tables(Group *group, Schema &target_schema, bool update_existing) { // first pass to create missing tables std::vector to_update; for (auto& object_schema : target_schema) { @@ -267,7 +260,6 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update // we will modify tables for any new objectSchema (table was created) or for all if update_existing is true if (update_existing || created) { to_update.push_back(&object_schema); - changed = true; } } @@ -291,7 +283,6 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update table->remove_column(current_prop.table_column); current_prop.table_column = target_prop->table_column; - changed = true; } bool inserted_placeholder_column = false; @@ -314,7 +305,6 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update table->remove_column(current_prop.table_column); ++deleted; current_prop.table_column = npos; - changed = true; } } @@ -339,8 +329,6 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update target_prop.is_nullable); break; } - - changed = true; } else { target_prop.table_column = current_prop->table_column; @@ -361,16 +349,13 @@ bool ObjectStore::create_tables(Group *group, Schema &target_schema, bool update // if there is a primary key set, check if it is the same as the old key if (current_schema.primary_key != target_object_schema->primary_key) { set_primary_key_for_object(group, target_object_schema->name, target_object_schema->primary_key); - changed = true; } } else if (current_schema.primary_key.length()) { // there is no primary key, so if there was one nil out set_primary_key_for_object(group, target_object_schema->name, ""); - changed = true; } } - return changed; } bool ObjectStore::is_schema_at_version(const Group *group, uint64_t version) { @@ -405,7 +390,7 @@ bool ObjectStore::needs_update(Schema const& old_schema, Schema const& schema) { return false; } -bool ObjectStore::update_realm_with_schema(Group *group, Schema const& old_schema, +void ObjectStore::update_realm_with_schema(Group *group, Schema const& old_schema, uint64_t version, Schema &schema, MigrationFunction migration) { // Recheck the schema version after beginning the write transaction as @@ -414,8 +399,8 @@ bool ObjectStore::update_realm_with_schema(Group *group, Schema const& old_schem bool migrating = !is_schema_at_version(group, version); // create tables - bool changed = create_metadata_tables(group); - changed = create_tables(group, schema, migrating) || changed; + create_metadata_tables(group); + create_tables(group, schema, migrating); if (!migrating) { // If we aren't migrating, then verify that all of the tables which @@ -423,10 +408,10 @@ bool ObjectStore::update_realm_with_schema(Group *group, Schema const& old_schem verify_schema(old_schema, schema, true); } - changed = update_indexes(group, schema) || changed; + update_indexes(group, schema); if (!migrating) { - return changed; + return; } // apply the migration block if provided and there's any old data @@ -437,7 +422,6 @@ bool ObjectStore::update_realm_with_schema(Group *group, Schema const& old_schem } set_schema_version(group, version); - return true; } Schema ObjectStore::schema_from_group(const Group *group) { diff --git a/src/object_store.hpp b/src/object_store.hpp index d41ec3fd..13b475d0 100644 --- a/src/object_store.hpp +++ b/src/object_store.hpp @@ -52,12 +52,11 @@ namespace realm { static bool needs_update(Schema const& old_schema, Schema const& schema); // updates a Realm from old_schema to the given target schema, creating and updating tables as needed - // returns if any changes were made // passed in target schema is updated with the correct column mapping // optionally runs migration function if schema is out of date // NOTE: must be performed within a write transaction typedef std::function MigrationFunction; - static bool update_realm_with_schema(Group *group, Schema const& old_schema, uint64_t version, + static void update_realm_with_schema(Group *group, Schema const& old_schema, uint64_t version, Schema &schema, MigrationFunction migration); // get a table for an object type @@ -86,11 +85,11 @@ namespace realm { // create any metadata tables that don't already exist // must be in write transaction to set // returns true if it actually did anything - static bool create_metadata_tables(Group *group); + static void create_metadata_tables(Group *group); // set references to tables on targetSchema and create/update any missing or out-of-date tables // if update existing is true, updates existing tables, otherwise only adds and initializes new tables - static bool create_tables(realm::Group *group, Schema &target_schema, bool update_existing); + static void create_tables(realm::Group *group, Schema &target_schema, bool update_existing); // verify a target schema against an expected schema, setting the table_column property on each schema object // updates the column mapping on the target_schema diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 4e07fb3d..8f51a96e 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -197,7 +197,7 @@ SharedRealm Realm::get_shared_realm(Config config) return realm; } -bool Realm::update_schema(std::unique_ptr schema, uint64_t version) +void Realm::update_schema(std::unique_ptr schema, uint64_t version) { schema->validate(); @@ -207,7 +207,7 @@ bool Realm::update_schema(std::unique_ptr schema, uint64_t version) ObjectStore::verify_schema(*m_config.schema, *schema, m_config.read_only); m_config.schema = std::move(schema); m_config.schema_version = version; - return false; + return; } begin_transaction(); @@ -248,11 +248,10 @@ bool Realm::update_schema(std::unique_ptr schema, uint64_t version) m_config.schema = std::move(schema); m_config.schema_version = version; - bool changed = ObjectStore::update_realm_with_schema(read_group(), *old_config.schema, - version, *m_config.schema, - migration_function); + ObjectStore::update_realm_with_schema(read_group(), *old_config.schema, + version, *m_config.schema, + migration_function); commit_transaction(); - return changed; } catch (...) { m_config.schema = std::move(old_config.schema); diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index 85e42224..ddc625dd 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -79,8 +79,7 @@ namespace realm { // updating indexes as necessary. Uses the existing migration function // on the Config, and the resulting Schema and version with updated // column mappings are set on the realms config upon success. - // returns if any changes were made - bool update_schema(std::unique_ptr schema, uint64_t version); + void update_schema(std::unique_ptr schema, uint64_t version); static uint64_t get_schema_version(Config const& config); From 4b001e1842f5043459d98f1429b0ffaf7f905cd4 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 7 Jan 2016 14:27:53 -0800 Subject: [PATCH 11/79] Extract some logic to a helper function to avoid recursion in update_schema() --- src/shared_realm.cpp | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 8f51a96e..4720582c 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -201,12 +201,20 @@ void Realm::update_schema(std::unique_ptr schema, uint64_t version) { schema->validate(); - // If the schema version matches, just verify that the schema itself also matches - bool needs_update = !m_config.read_only && (m_config.schema_version != version || ObjectStore::needs_update(*m_config.schema, *schema)); - if (!needs_update) { + auto needs_update = [&] { + // If the schema version matches, just verify that the schema itself also matches + bool needs_write = !m_config.read_only && (m_config.schema_version != version || ObjectStore::needs_update(*m_config.schema, *schema)); + if (needs_write) { + return true; + } + ObjectStore::verify_schema(*m_config.schema, *schema, m_config.read_only); m_config.schema = std::move(schema); m_config.schema_version = version; + return false; + }; + + if (!needs_update()) { return; } @@ -225,11 +233,13 @@ void Realm::update_schema(std::unique_ptr schema, uint64_t version) // recheck everything auto current_schema_version = ObjectStore::get_schema_version(read_group()); if (current_schema_version != m_config.schema_version) { - cancel_transaction(); - m_config.schema_version = current_schema_version; *m_config.schema = ObjectStore::schema_from_group(read_group()); - return update_schema(std::move(schema), version); + + if (!needs_update()) { + cancel_transaction(); + return; + } } Config old_config(m_config); From 62a729fbfd91ea1ec4a07358bbaae204bb74a831 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 21 Jan 2016 10:06:51 -0800 Subject: [PATCH 12/79] Add headers to CMakeLists --- src/CMakeLists.txt | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 61204003..bef8feb4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,11 +12,26 @@ set(SOURCES parser/parser.cpp parser/query_builder.cpp) +set(HEADERS + index_set.hpp + list.hpp + object_schema.hpp + object_store.hpp + results.hpp + schema.hpp + shared_realm.hpp + impl/transact_log_handler.hpp + parser/parser.hpp + parser/query_builder.hpp) + if(APPLE) include_directories(impl/apple) - list(APPEND SOURCES impl/apple/external_commit_helper.cpp) + list(APPEND SOURCES + impl/apple/external_commit_helper.cpp) + list(APPEND HEADERS + impl/apple/external_commit_helper.hpp) find_library(CF_LIBRARY CoreFoundation) endif() -add_library(realm-object-store SHARED ${SOURCES}) +add_library(realm-object-store SHARED ${SOURCES} ${HEADERS}) target_link_libraries(realm-object-store realm ${CF_LIBRARY}) From 9d1a3da411666da6a1d5017ae7315cac10e7d799 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 21 Jan 2016 11:03:30 -0800 Subject: [PATCH 13/79] Add the required preprocessor flags for using core --- CMake/CompilerFlags.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMake/CompilerFlags.cmake b/CMake/CompilerFlags.cmake index 25d78a77..fffbb8f4 100644 --- a/CMake/CompilerFlags.cmake +++ b/CMake/CompilerFlags.cmake @@ -2,7 +2,7 @@ set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED on) set(CMAKE_CXX_EXTENSIONS off) set(CMAKE_BUILD_TYPE CACHE STRING Debug) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -DREALM_HAVE_CONFIG -DREALM_DEBUG") if(${CMAKE_GENERATOR} STREQUAL "Ninja") if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang") From 0e08e052389fe740c54878cf681cdad01bef0c25 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 21 Jan 2016 13:56:41 -0800 Subject: [PATCH 14/79] Only set REALM_DEBUG for debug builds --- CMake/CompilerFlags.cmake | 4 ++-- CMakeLists.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CMake/CompilerFlags.cmake b/CMake/CompilerFlags.cmake index fffbb8f4..d45392e4 100644 --- a/CMake/CompilerFlags.cmake +++ b/CMake/CompilerFlags.cmake @@ -1,8 +1,8 @@ set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED on) set(CMAKE_CXX_EXTENSIONS off) -set(CMAKE_BUILD_TYPE CACHE STRING Debug) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -DREALM_HAVE_CONFIG -DREALM_DEBUG") +add_compile_options(-Wall -DREALM_HAVE_CONFIG) +add_compile_options("$<$:-DREALM_DEBUG>") if(${CMAKE_GENERATOR} STREQUAL "Ninja") if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang") diff --git a/CMakeLists.txt b/CMakeLists.txt index 85138132..895d4d8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,3 +1,4 @@ +set(CMAKE_BUILD_TYPE Debug CACHE STRING "") project(realm-object-store) cmake_minimum_required(VERSION 3.2.0) From 8d10a65088fa7121894e06c1c9d142e552969185 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 21 Jan 2016 10:45:59 -0800 Subject: [PATCH 15/79] Make Schema constructable from initializer lists This enables the following syntax for defining object schemas, which is useful for writing tests: Schema schema = { {"origin", "", { {"array", PropertyTypeArray, "target"} }}, {"target", "", { {"prop1", PropertyTypeInt}, {"prop2", PropertyTypeFloat}, }}, }; --- src/object_schema.cpp | 27 ++++++++++++++++++++------- src/object_schema.hpp | 4 ++++ src/property.hpp | 2 +- src/schema.hpp | 1 + 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/object_schema.cpp b/src/object_schema.cpp index 33cd2497..3be3953f 100644 --- a/src/object_schema.cpp +++ b/src/object_schema.cpp @@ -28,6 +28,14 @@ using namespace realm; ObjectSchema::~ObjectSchema() = default; +ObjectSchema::ObjectSchema(std::string name, std::string primary_key, std::initializer_list properties) +: name(std::move(name)) +, properties(properties) +, primary_key(std::move(primary_key)) +{ + set_primary_key_property(); +} + ObjectSchema::ObjectSchema(const Group *group, const std::string &name) : name(name) { ConstTableRef table = ObjectStore::table_for_object_type(group, name); @@ -50,13 +58,7 @@ ObjectSchema::ObjectSchema(const Group *group, const std::string &name) : name(n } primary_key = realm::ObjectStore::get_primary_key_for_object(group, name); - if (primary_key.length()) { - auto primary_key_prop = primary_key_property(); - if (!primary_key_prop) { - throw InvalidPrimaryKeyException(name, primary_key); - } - primary_key_prop->is_primary = true; - } + set_primary_key_property(); } Property *ObjectSchema::property_for_name(StringData name) { @@ -71,3 +73,14 @@ Property *ObjectSchema::property_for_name(StringData name) { const Property *ObjectSchema::property_for_name(StringData name) const { return const_cast(this)->property_for_name(name); } + +void ObjectSchema::set_primary_key_property() +{ + if (primary_key.length()) { + auto primary_key_prop = primary_key_property(); + if (!primary_key_prop) { + throw InvalidPrimaryKeyException(name, primary_key); + } + primary_key_prop->is_primary = true; + } +} diff --git a/src/object_schema.hpp b/src/object_schema.hpp index 4ede6ce5..10a2e555 100644 --- a/src/object_schema.hpp +++ b/src/object_schema.hpp @@ -31,6 +31,7 @@ namespace realm { class ObjectSchema { public: ObjectSchema() = default; + ObjectSchema(std::string name, std::string primary_key, std::initializer_list properties); ~ObjectSchema(); // create object schema from existing table @@ -49,6 +50,9 @@ namespace realm { const Property *primary_key_property() const { return property_for_name(primary_key); } + + private: + void set_primary_key_property(); }; } diff --git a/src/property.hpp b/src/property.hpp index 3883e805..05993ae1 100644 --- a/src/property.hpp +++ b/src/property.hpp @@ -53,7 +53,7 @@ namespace realm { bool is_indexed = false; bool is_nullable = false; - size_t table_column; + size_t table_column = -1; bool requires_index() const { return is_primary || is_indexed; } }; diff --git a/src/schema.hpp b/src/schema.hpp index b8d50777..2df1ec9e 100644 --- a/src/schema.hpp +++ b/src/schema.hpp @@ -31,6 +31,7 @@ private: public: // Create a schema from a vector of ObjectSchema Schema(base types); + Schema(std::initializer_list types) : Schema(base(types)) { } // find an ObjectSchema by name iterator find(std::string const& name); From e87a50722386c47497804305cc468e102e91dcde Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 19 Aug 2015 12:27:12 -0700 Subject: [PATCH 16/79] Extract cache management and inter-Realm sharing to RealmCoordinator --- src/CMakeLists.txt | 1 + src/impl/apple/external_commit_helper.cpp | 42 ++-- src/impl/apple/external_commit_helper.hpp | 12 +- src/impl/realm_coordinator.cpp | 182 ++++++++++++++++ src/impl/realm_coordinator.hpp | 77 +++++++ src/results.cpp | 2 + src/shared_realm.cpp | 242 ++++++---------------- src/shared_realm.hpp | 34 ++- 8 files changed, 362 insertions(+), 230 deletions(-) create mode 100644 src/impl/realm_coordinator.cpp create mode 100644 src/impl/realm_coordinator.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bef8feb4..2c9eade2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,6 +8,7 @@ set(SOURCES results.cpp schema.cpp shared_realm.cpp + impl/realm_coordinator.cpp impl/transact_log_handler.cpp parser/parser.cpp parser/query_builder.cpp) diff --git a/src/impl/apple/external_commit_helper.cpp b/src/impl/apple/external_commit_helper.cpp index 4cf0b523..c22fe5df 100644 --- a/src/impl/apple/external_commit_helper.cpp +++ b/src/impl/apple/external_commit_helper.cpp @@ -18,16 +18,16 @@ #include "external_commit_helper.hpp" -#include "shared_realm.hpp" +#include "realm_coordinator.hpp" #include +#include +#include #include #include #include #include -#include #include -#include using namespace realm; using namespace realm::_impl; @@ -56,11 +56,10 @@ void notify_fd(int fd) void ExternalCommitHelper::FdHolder::close() { - if (m_fd != -1) { - ::close(m_fd); - } - m_fd = -1; - + if (m_fd != -1) { + ::close(m_fd); + } + m_fd = -1; } // Inter-thread and inter-process notifications of changes are done using a @@ -86,16 +85,15 @@ void ExternalCommitHelper::FdHolder::close() // signal the runloop source and wake up the target runloop, and when data is // written to the anonymous pipe the background thread removes the runloop // source from the runloop and and shuts down. -ExternalCommitHelper::ExternalCommitHelper(Realm* realm) +ExternalCommitHelper::ExternalCommitHelper(RealmCoordinator& parent) +: m_parent(parent) { - add_realm(realm); - m_kq = kqueue(); if (m_kq == -1) { throw std::system_error(errno, std::system_category()); } - auto path = realm->config().path + ".note"; + auto path = parent.get_path() + ".note"; // Create and open the named pipe int ret = mkfifo(path.c_str(), 0600); @@ -140,28 +138,14 @@ ExternalCommitHelper::ExternalCommitHelper(Realm* realm) m_shutdown_read_fd = pipeFd[0]; m_shutdown_write_fd = pipeFd[1]; - // Use the minimum allowed stack size, as we need very little in our listener - // https://developer.apple.com/library/ios/documentation/Cocoa/Conceptual/Multithreading/CreatingThreads/CreatingThreads.html#//apple_ref/doc/uid/10000057i-CH15-SW7 - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setstacksize(&attr, 16 * 1024); - - auto fn = [](void *self) -> void * { - static_cast(self)->listen(); - return nullptr; - }; - ret = pthread_create(&m_thread, &attr, fn, this); - pthread_attr_destroy(&attr); - if (ret != 0) { - throw std::system_error(errno, std::system_category()); - } + m_thread = std::async(std::launch::async, [=] { listen(); }); } ExternalCommitHelper::~ExternalCommitHelper() { REALM_ASSERT_DEBUG(m_realms.empty()); notify_fd(m_shutdown_write_fd); - pthread_join(m_thread, nullptr); // Wait for the thread to exit + m_thread.wait(); // Wait for the thread to exit } void ExternalCommitHelper::add_realm(realm::Realm* realm) @@ -202,7 +186,6 @@ void ExternalCommitHelper::listen() { pthread_setname_np("RLMRealm notification listener"); - // Set up the kqueue // EVFILT_READ indicates that we care about data being available to read // on the given file descriptor. @@ -248,4 +231,3 @@ void ExternalCommitHelper::notify_others() { notify_fd(m_notify_fd); } - diff --git a/src/impl/apple/external_commit_helper.hpp b/src/impl/apple/external_commit_helper.hpp index d7acb791..9f31d730 100644 --- a/src/impl/apple/external_commit_helper.hpp +++ b/src/impl/apple/external_commit_helper.hpp @@ -20,6 +20,8 @@ #define REALM_EXTERNAL_COMMIT_HELPER_HPP #include +#include +#include #include #include @@ -27,9 +29,13 @@ namespace realm { class Realm; namespace _impl { +class RealmCoordinator; + +// FIXME: split IPC from the local cross-thread stuff +// both are platform-specific but need to be useable separately class ExternalCommitHelper { public: - ExternalCommitHelper(Realm* realm); + ExternalCommitHelper(RealmCoordinator& parent); ~ExternalCommitHelper(); void notify_others(); @@ -67,6 +73,8 @@ private: void listen(); + RealmCoordinator& m_parent; + // Currently registered realms and the signal for delivering notifications // to them std::vector m_realms; @@ -75,7 +83,7 @@ private: std::mutex m_realms_mutex; // The listener thread - pthread_t m_thread; + std::future m_thread; // Read-write file descriptor for the named pipe which is waited on for // changes and written to when a commit is made diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp new file mode 100644 index 00000000..82a6dff1 --- /dev/null +++ b/src/impl/realm_coordinator.cpp @@ -0,0 +1,182 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "realm_coordinator.hpp" + +#include "external_commit_helper.hpp" +#include "object_store.hpp" + +using namespace realm; +using namespace realm::_impl; + +static std::mutex s_coordinator_mutex; +static std::map> s_coordinators_per_path; + +std::shared_ptr RealmCoordinator::get_coordinator(StringData path) +{ + std::lock_guard lock(s_coordinator_mutex); + std::shared_ptr coordinator; + + auto it = s_coordinators_per_path.find(path); + if (it != s_coordinators_per_path.end()) { + coordinator = it->second.lock(); + } + + if (!coordinator) { + s_coordinators_per_path[path] = coordinator = std::make_shared(); + } + + return coordinator; +} + +std::shared_ptr RealmCoordinator::get_existing_coordinator(StringData path) +{ + std::lock_guard lock(s_coordinator_mutex); + auto it = s_coordinators_per_path.find(path); + return it == s_coordinators_per_path.end() ? nullptr : it->second.lock(); +} + +std::shared_ptr RealmCoordinator::get_realm(Realm::Config config) +{ + std::lock_guard lock(m_realm_mutex); + if (!m_notifier && m_cached_realms.empty()) { + m_config = config; + if (!config.read_only) { + m_notifier = std::make_unique(*this); + } + } + else { + if (m_config.read_only != config.read_only) { + throw MismatchedConfigException("Realm at path already opened with different read permissions."); + } + if (m_config.in_memory != config.in_memory) { + throw MismatchedConfigException("Realm at path already opened with different inMemory settings."); + } + if (m_config.encryption_key != config.encryption_key) { + throw MismatchedConfigException("Realm at path already opened with a different encryption key."); + } + if (m_config.schema_version != config.schema_version && config.schema_version != ObjectStore::NotVersioned) { + throw MismatchedConfigException("Realm at path already opened with different schema version."); + } + // FIXME: verify that schema is compatible + // Needs to verify that all tables present in both are identical, and + // then updated m_config with any tables present in config but not in + // it + // Public API currently doesn't make it possible to have non-matching + // schemata so it's not a huge issue + if ((false) && m_config.schema != config.schema) { + throw MismatchedConfigException("Realm at path already opened with different schema"); + } + } + + auto thread_id = std::this_thread::get_id(); + if (config.cache) { + for (auto& weakRealm : m_cached_realms) { + // can be null if we jumped in between ref count hitting zero and + // unregister_realm() getting the lock + if (auto realm = weakRealm.lock()) { + if (realm->thread_id() == thread_id) { + return realm; + } + } + } + } + + auto realm = std::make_shared(config); + realm->init(shared_from_this()); + if (m_notifier) { + m_notifier->add_realm(realm.get()); + } + if (config.cache) { + m_cached_realms.push_back(realm); + } + return realm; +} + +const Schema* RealmCoordinator::get_schema() const noexcept +{ + return m_cached_realms.empty() ? nullptr : m_config.schema.get(); +} + +RealmCoordinator::RealmCoordinator() = default; + +RealmCoordinator::~RealmCoordinator() +{ + std::lock_guard coordinator_lock(s_coordinator_mutex); + for (auto it = s_coordinators_per_path.begin(); it != s_coordinators_per_path.end(); ) { + if (it->second.expired()) { + it = s_coordinators_per_path.erase(it); + } + else { + ++it; + } + } +} + +void RealmCoordinator::unregister_realm(Realm* realm) +{ + std::lock_guard lock(m_realm_mutex); + if (m_notifier) { + m_notifier->remove_realm(realm); + } + for (size_t i = 0; i < m_cached_realms.size(); ++i) { + if (m_cached_realms[i].expired()) { + if (i + 1 < m_cached_realms.size()) { + m_cached_realms[i] = std::move(m_cached_realms.back()); + } + m_cached_realms.pop_back(); + } + } +} + +void RealmCoordinator::clear_cache() +{ + std::vector realms_to_close; + + { + std::lock_guard lock(s_coordinator_mutex); + + // Gather a list of all of the realms which will be removed + for (auto& weak_coordinator : s_coordinators_per_path) { + auto coordinator = weak_coordinator.second.lock(); + if (!coordinator) { + continue; + } + + for (auto& cached_realm : coordinator->m_cached_realms) { + if (auto realm = cached_realm.lock()) { + realms_to_close.push_back(realm); + } + } + } + + s_coordinators_per_path.clear(); + } + + // Close all of the previously cached Realms. This can't be done while + // s_coordinator_mutex is held as it may try to re-lock it. + for (auto& realm : realms_to_close) { + realm->close(); + } +} + +void RealmCoordinator::send_commit_notifications() +{ + REALM_ASSERT(!m_config.read_only); + m_notifier->notify_others(); +} diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp new file mode 100644 index 00000000..f2adc69f --- /dev/null +++ b/src/impl/realm_coordinator.hpp @@ -0,0 +1,77 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_COORDINATOR_HPP +#define REALM_COORDINATOR_HPP + +#include "shared_realm.hpp" + +#include + +namespace realm { +namespace _impl { +class ExternalCommitHelper; + +// RealmCoordinator manages the weak cache of Realm instances and communication +// between per-thread Realm instances for a given file +class RealmCoordinator : public std::enable_shared_from_this { +public: + // Get the coordinator for the given path, creating it if neccesary + static std::shared_ptr get_coordinator(StringData path); + // Get the coordinator for the given path, or null if there is none + static std::shared_ptr get_existing_coordinator(StringData path); + + // Get a thread-local shared Realm with the given configuration + // If the Realm is already open on another thread, validates that the given + // configuration is compatible with the existing one + std::shared_ptr get_realm(Realm::Config config); + + const Schema* get_schema() const noexcept; + uint64_t get_schema_version() const noexcept { return m_config.schema_version; } + const std::string& get_path() const noexcept { return m_config.path; } + + // Asyncronously call notify() on every Realm instance for this coordinator's + // path, including those in other processes + void send_commit_notifications(); + + // Clear the weak Realm cache for all paths + // Should only be called in test code, as continuing to use the previously + // cached instances will have odd results + static void clear_cache(); + + // Explicit constructor/destructor needed for the unique_ptrs to forward-declared types + RealmCoordinator(); + ~RealmCoordinator(); + + // Called by Realm's destructor to ensure the cache is cleaned up promptly + // Do not call directly + void unregister_realm(Realm* realm); + +private: + Realm::Config m_config; + + std::mutex m_realm_mutex; + std::vector> m_cached_realms; + + std::unique_ptr<_impl::ExternalCommitHelper> m_notifier; +}; + +} // namespace _impl +} // namespace realm + +#endif /* REALM_COORDINATOR_HPP */ diff --git a/src/results.cpp b/src/results.cpp index 91a4cc22..a49559ca 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -18,6 +18,8 @@ #include "results.hpp" +#include "object_store.hpp" + #include using namespace realm; diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 30e8f928..85fc8214 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -18,8 +18,10 @@ #include "shared_realm.hpp" -#include "external_commit_helper.hpp" #include "binding_context.hpp" +#include "external_commit_helper.hpp" +#include "object_store.hpp" +#include "realm_coordinator.hpp" #include "schema.hpp" #include "transact_log_handler.hpp" @@ -31,8 +33,6 @@ using namespace realm; using namespace realm::_impl; -RealmCache Realm::s_global_cache; - Realm::Config::Config(const Config& c) : path(c.path) , read_only(c.read_only) @@ -48,7 +48,7 @@ Realm::Config::Config(const Config& c) } } -Realm::Config::Config() = default; +Realm::Config::Config() : schema_version(ObjectStore::NotVersioned) { } Realm::Config::Config(Config&&) = default; Realm::Config::~Config() = default; @@ -104,9 +104,58 @@ Realm::Realm(Config config) } } -Realm::~Realm() { - if (m_notifier) { // might not exist yet if an error occurred during init - m_notifier->remove_realm(this); +void Realm::init(std::shared_ptr coordinator) +{ + m_coordinator = std::move(coordinator); + + // if there is an existing realm at the current path steal its schema/column mapping + if (auto existing = m_coordinator->get_schema()) { + m_config.schema = std::make_unique(*existing); + return; + } + + try { + // otherwise get the schema from the group + auto target_schema = std::move(m_config.schema); + auto target_schema_version = m_config.schema_version; + m_config.schema_version = ObjectStore::get_schema_version(read_group()); + m_config.schema = std::make_unique(ObjectStore::schema_from_group(read_group())); + + // if a target schema is supplied, verify that it matches or migrate to + // it, as neeeded + if (target_schema) { + if (m_config.read_only) { + if (m_config.schema_version == ObjectStore::NotVersioned) { + throw UnitializedRealmException("Can't open an un-initialized Realm without a Schema"); + } + target_schema->validate(); + ObjectStore::verify_schema(*m_config.schema, *target_schema, true); + m_config.schema = std::move(target_schema); + } + else { + update_schema(std::move(target_schema), target_schema_version); + } + + if (!m_config.read_only) { + // End the read transaction created to validation/update the + // schema to avoid pinning the version even if the user never + // actually reads data + invalidate(); + } + } + } + catch (...) { + // Trying to unregister from the coordinator before we finish + // construction will result in a deadlock + m_coordinator = nullptr; + throw; + } +} + +Realm::~Realm() +{ + if (m_coordinator) { + m_coordinator->unregister_realm(this); } } @@ -120,85 +169,7 @@ Group *Realm::read_group() SharedRealm Realm::get_shared_realm(Config config) { - if (config.cache) { - if (SharedRealm realm = s_global_cache.get_realm(config.path)) { - if (realm->config().read_only != config.read_only) { - throw MismatchedConfigException("Realm at path already opened with different read permissions."); - } - if (realm->config().in_memory != config.in_memory) { - throw MismatchedConfigException("Realm at path already opened with different inMemory settings."); - } - if (realm->config().encryption_key != config.encryption_key) { - throw MismatchedConfigException("Realm at path already opened with a different encryption key."); - } - if (realm->config().schema_version != config.schema_version && config.schema_version != ObjectStore::NotVersioned) { - throw MismatchedConfigException("Realm at path already opened with different schema version."); - } - // FIXME - enable schma comparison - /*if (realm->config().schema != config.schema) { - throw MismatchedConfigException("Realm at path already opened with different schema"); - }*/ - realm->m_config.migration_function = config.migration_function; - - return realm; - } - } - - SharedRealm realm(new Realm(std::move(config))); - - auto target_schema = std::move(realm->m_config.schema); - auto target_schema_version = realm->m_config.schema_version; - realm->m_config.schema_version = ObjectStore::get_schema_version(realm->read_group()); - - // we want to ensure we are only initializing a single realm at a time - static std::mutex s_init_mutex; - std::lock_guard lock(s_init_mutex); - if (auto existing = s_global_cache.get_any_realm(realm->config().path)) { - // if there is an existing realm at the current path steal its schema/column mapping - // FIXME - need to validate that schemas match - realm->m_config.schema = std::make_unique(*existing->m_config.schema); - - if (!realm->m_config.read_only) { - realm->m_notifier = existing->m_notifier; - realm->m_notifier->add_realm(realm.get()); - } - } - else { - if (!realm->m_config.read_only) { - realm->m_notifier = std::make_shared(realm.get()); - } - - // otherwise get the schema from the group - realm->m_config.schema = std::make_unique(ObjectStore::schema_from_group(realm->read_group())); - - // if a target schema is supplied, verify that it matches or migrate to - // it, as neeeded - if (target_schema) { - if (realm->m_config.read_only) { - if (realm->m_config.schema_version == ObjectStore::NotVersioned) { - throw UnitializedRealmException("Can't open an un-initialized Realm without a Schema"); - } - target_schema->validate(); - ObjectStore::verify_schema(*realm->m_config.schema, *target_schema, true); - realm->m_config.schema = std::move(target_schema); - } - else { - realm->update_schema(std::move(target_schema), target_schema_version); - } - - if (!realm->m_config.read_only) { - // End the read transaction created to validation/update the - // schema to avoid pinning the version even if the user never - // actually reads data - realm->invalidate(); - } - } - } - - if (config.cache) { - s_global_cache.cache_realm(realm, realm->m_thread_id); - } - return realm; + return RealmCoordinator::get_coordinator(config.path)->get_realm(config); } void Realm::update_schema(std::unique_ptr schema, uint64_t version) @@ -322,7 +293,7 @@ void Realm::commit_transaction() m_in_transaction = false; transaction::commit(*m_shared_group, *m_history, m_binding_context.get()); - m_notifier->notify_others(); + m_coordinator->send_commit_notifications(); } void Realm::cancel_transaction() @@ -392,7 +363,6 @@ void Realm::notify() } } - bool Realm::refresh() { verify_thread(); @@ -421,8 +391,9 @@ bool Realm::refresh() uint64_t Realm::get_schema_version(const realm::Realm::Config &config) { - if (auto existing_realm = s_global_cache.get_any_realm(config.path)) { - return existing_realm->config().schema_version; + auto coordinator = RealmCoordinator::get_existing_coordinator(config.path); + if (coordinator) { + return coordinator->get_schema_version(); } return ObjectStore::get_schema_version(Realm(config).read_group()); @@ -430,97 +401,16 @@ uint64_t Realm::get_schema_version(const realm::Realm::Config &config) void Realm::close() { - if (m_notifier) { - m_notifier->remove_realm(this); + invalidate(); + + if (m_coordinator) { + m_coordinator->unregister_realm(this); } m_group = nullptr; m_shared_group = nullptr; m_history = nullptr; m_read_only_group = nullptr; - m_notifier = nullptr; m_binding_context = nullptr; -} - -SharedRealm RealmCache::get_realm(const std::string &path, std::thread::id thread_id) -{ - std::lock_guard lock(m_mutex); - - auto path_iter = m_cache.find(path); - if (path_iter == m_cache.end()) { - return SharedRealm(); - } - - auto thread_iter = path_iter->second.find(thread_id); - if (thread_iter == path_iter->second.end()) { - return SharedRealm(); - } - - return thread_iter->second.lock(); -} - -SharedRealm RealmCache::get_any_realm(const std::string &path) -{ - std::lock_guard lock(m_mutex); - - auto path_iter = m_cache.find(path); - if (path_iter == m_cache.end()) { - return SharedRealm(); - } - - auto thread_iter = path_iter->second.begin(); - while (thread_iter != path_iter->second.end()) { - if (auto realm = thread_iter->second.lock()) { - return realm; - } - path_iter->second.erase(thread_iter++); - } - - return SharedRealm(); -} - -void RealmCache::remove(const std::string &path, std::thread::id thread_id) -{ - std::lock_guard lock(m_mutex); - - auto path_iter = m_cache.find(path); - if (path_iter == m_cache.end()) { - return; - } - - auto thread_iter = path_iter->second.find(thread_id); - if (thread_iter != path_iter->second.end()) { - path_iter->second.erase(thread_iter); - } - - if (path_iter->second.size() == 0) { - m_cache.erase(path_iter); - } -} - -void RealmCache::cache_realm(SharedRealm &realm, std::thread::id thread_id) -{ - std::lock_guard lock(m_mutex); - - auto path_iter = m_cache.find(realm->config().path); - if (path_iter == m_cache.end()) { - m_cache.emplace(realm->config().path, std::map{{thread_id, realm}}); - } - else { - path_iter->second.emplace(thread_id, realm); - } -} - -void RealmCache::clear() -{ - std::lock_guard lock(m_mutex); - for (auto const& path : m_cache) { - for (auto const& thread : path.second) { - if (auto realm = thread.second.lock()) { - realm->close(); - } - } - } - - m_cache.clear(); + m_coordinator = nullptr; } diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index ddc625dd..79f55496 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -22,20 +22,24 @@ #include "object_store.hpp" #include -#include +#include #include #include namespace realm { - class ClientHistory; - class Realm; - class RealmCache; class BindingContext; + class ClientHistory; + class Group; + class Realm; + class RealmDelegate; + class Schema; + class SharedGroup; typedef std::shared_ptr SharedRealm; typedef std::weak_ptr WeakRealm; namespace _impl { class ExternalCommitHelper; + class RealmCoordinator; } class Realm : public std::enable_shared_from_this @@ -53,7 +57,7 @@ namespace realm { std::vector encryption_key; std::unique_ptr schema; - uint64_t schema_version = ObjectStore::NotVersioned; + uint64_t schema_version; MigrationFunction migration_function; @@ -109,9 +113,10 @@ namespace realm { ~Realm(); - private: + void init(std::shared_ptr<_impl::RealmCoordinator> coordinator); Realm(Config config); + private: Config m_config; std::thread::id m_thread_id = std::this_thread::get_id(); bool m_in_transaction = false; @@ -123,28 +128,13 @@ namespace realm { Group *m_group = nullptr; - std::shared_ptr<_impl::ExternalCommitHelper> m_notifier; + std::shared_ptr<_impl::RealmCoordinator> m_coordinator; public: std::unique_ptr m_binding_context; // FIXME private Group *read_group(); - static RealmCache s_global_cache; - }; - - class RealmCache - { - public: - SharedRealm get_realm(const std::string &path, std::thread::id thread_id = std::this_thread::get_id()); - SharedRealm get_any_realm(const std::string &path); - void remove(const std::string &path, std::thread::id thread_id); - void cache_realm(SharedRealm &realm, std::thread::id thread_id = std::this_thread::get_id()); - void clear(); - - private: - std::map> m_cache; - std::mutex m_mutex; }; class RealmFileException : public std::runtime_error { From c3a9489b0250dabbbe0884553408fb52816be8e5 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 20 Aug 2015 12:27:23 -0700 Subject: [PATCH 17/79] Fix a potential deadlock when opening a realm --- src/impl/realm_coordinator.cpp | 21 +++++++++++++-------- src/impl/realm_coordinator.hpp | 3 ++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 82a6dff1..297a42d3 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -24,6 +24,11 @@ using namespace realm; using namespace realm::_impl; +struct realm::_impl::CachedRealm { + std::weak_ptr realm; + std::thread::id thread_id; +}; + static std::mutex s_coordinator_mutex; static std::map> s_coordinators_per_path; @@ -86,11 +91,11 @@ std::shared_ptr RealmCoordinator::get_realm(Realm::Config config) auto thread_id = std::this_thread::get_id(); if (config.cache) { - for (auto& weakRealm : m_cached_realms) { - // can be null if we jumped in between ref count hitting zero and - // unregister_realm() getting the lock - if (auto realm = weakRealm.lock()) { - if (realm->thread_id() == thread_id) { + for (auto& cachedRealm : m_cached_realms) { + if (cachedRealm.thread_id == thread_id) { + // can be null if we jumped in between ref count hitting zero and + // unregister_realm() getting the lock + if (auto realm = cachedRealm.realm.lock()) { return realm; } } @@ -103,7 +108,7 @@ std::shared_ptr RealmCoordinator::get_realm(Realm::Config config) m_notifier->add_realm(realm.get()); } if (config.cache) { - m_cached_realms.push_back(realm); + m_cached_realms.push_back({realm, thread_id}); } return realm; } @@ -135,7 +140,7 @@ void RealmCoordinator::unregister_realm(Realm* realm) m_notifier->remove_realm(realm); } for (size_t i = 0; i < m_cached_realms.size(); ++i) { - if (m_cached_realms[i].expired()) { + if (m_cached_realms[i].realm.expired()) { if (i + 1 < m_cached_realms.size()) { m_cached_realms[i] = std::move(m_cached_realms.back()); } @@ -159,7 +164,7 @@ void RealmCoordinator::clear_cache() } for (auto& cached_realm : coordinator->m_cached_realms) { - if (auto realm = cached_realm.lock()) { + if (auto realm = cached_realm.realm.lock()) { realms_to_close.push_back(realm); } } diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index f2adc69f..5813d93f 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -26,6 +26,7 @@ namespace realm { namespace _impl { class ExternalCommitHelper; +struct CachedRealm; // RealmCoordinator manages the weak cache of Realm instances and communication // between per-thread Realm instances for a given file @@ -66,7 +67,7 @@ private: Realm::Config m_config; std::mutex m_realm_mutex; - std::vector> m_cached_realms; + std::vector m_cached_realms; std::unique_ptr<_impl::ExternalCommitHelper> m_notifier; }; From f3397d48c03dfbbe50d121b9861743ea5f2dcd0a Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 21 Jan 2016 16:15:25 -0800 Subject: [PATCH 18/79] Add IndexSet tests And fix a bug that resulted in ranges not being merged. --- src/index_set.cpp | 11 ++-- tests/CMakeLists.txt | 8 ++- tests/index_set.cpp | 153 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 6 deletions(-) create mode 100644 tests/index_set.cpp diff --git a/src/index_set.cpp b/src/index_set.cpp index b7e4961c..c244f76a 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -43,11 +43,12 @@ void IndexSet::do_add(iterator it, size_t index) else if (more_before && (it - 1)->second == index) { // index is immediately after an existing range ++(it - 1)->second; - } - else if (more_before && valid && (it - 1)->second == it->first) { - // index joins two existing ranges - (it - 1)->second = it->second; - m_ranges.erase(it); + + if (valid && (it - 1)->second == it->first) { + // index joins two existing ranges + (it - 1)->second = it->second; + m_ranges.erase(it); + } } else if (valid && it->first == index + 1) { // index is immediately before an existing range diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a7b20889..269301c9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,5 +1,11 @@ include_directories(../external/catch/single_include) -add_executable(tests main.cpp parser.cpp) + +set(SOURCES + index_set.cpp + main.cpp + parser.cpp) + +add_executable(tests ${SOURCES}) target_link_libraries(tests realm-object-store) add_custom_target(run-tests USES_TERMINAL DEPENDS tests COMMAND ./tests) diff --git a/tests/index_set.cpp b/tests/index_set.cpp new file mode 100644 index 00000000..50ae889c --- /dev/null +++ b/tests/index_set.cpp @@ -0,0 +1,153 @@ +#include "index_set.hpp" + +#include + +// Catch doesn't have an overload for std::pair, so define one ourselves +// The declaration needs to be before catch.hpp is included for it to be used, +// but the definition needs to be after since it uses Catch's toString() +namespace Catch { +template +std::string toString(std::pair const& value); +} + +#include "catch.hpp" + +namespace Catch { +template + std::string toString(std::pair const& value) { + return "{" + toString(value.first) + ", " + toString(value.second) + "}"; +} +} + +#define REQUIRE_RANGES(index_set, ...) do { \ + std::initializer_list> expected = {__VA_ARGS__}; \ + REQUIRE(index_set.size() == expected.size()); \ + auto begin = index_set.begin(), end = index_set.end(); \ + for (auto range : expected) { \ + REQUIRE(*begin++ == range); \ + } \ +} while (0) + +TEST_CASE("index set") { + realm::IndexSet set; + + SECTION("add() extends existing ranges") { + set.add(1); + REQUIRE_RANGES(set, {1, 2}); + + set.add(2); + REQUIRE_RANGES(set, {1, 3}); + + set.add(0); + REQUIRE_RANGES(set, {0, 3}); + } + + SECTION("add() with gaps") { + set.add(0); + REQUIRE_RANGES(set, {0, 1}); + + set.add(2); + REQUIRE_RANGES(set, {0, 1}, {2, 3}); + } + + SECTION("add() is idempotent") { + set.add(0); + set.add(0); + REQUIRE_RANGES(set, {0, 1}); + } + + SECTION("add() merges existing ranges") { + set.add(0); + set.add(2); + set.add(4); + + set.add(1); + REQUIRE_RANGES(set, {0, 3}, {4, 5}); + } + + SECTION("set() from empty") { + set.set(5); + REQUIRE_RANGES(set, {0, 5}); + } + + SECTION("set() discards existing data") { + set.add(8); + set.add(9); + + set.set(5); + REQUIRE_RANGES(set, {0, 5}); + } + + SECTION("insert_at() extends ranges containing the target index") { + set.add(5); + set.add(6); + + set.insert_at(5); + REQUIRE_RANGES(set, {5, 8}); + + set.insert_at(4); + REQUIRE_RANGES(set, {4, 5}, {6, 9}); + + set.insert_at(9); + REQUIRE_RANGES(set, {4, 5}, {6, 10}); + } + + SECTION("insert_at() does not modify ranges entirely before it") { + set.add(5); + set.add(6); + + set.insert_at(8); + REQUIRE_RANGES(set, {5, 7}, {8, 9}); + } + + SECTION("insert_at() shifts ranges after it") { + set.add(5); + set.add(6); + + set.insert_at(3); + REQUIRE_RANGES(set, {3, 4}, {6, 8}); + } + + SECTION("insert_at() cannot join ranges") { + set.add(5); + set.add(7); + + set.insert_at(6); + REQUIRE_RANGES(set, {5, 7}, {8, 9}); + } + + SECTION("add_shifted() on an empty set is just add()") { + set.add_shifted(5); + REQUIRE_RANGES(set, {5, 6}); + } + + SECTION("add_shifted() before the first range is just add()") { + set.add(10); + set.add_shifted(5); + REQUIRE_RANGES(set, {5, 6}, {10, 11}); + } + + SECTION("add_shifted() on first index of range extends range") { + set.add(5); + set.add_shifted(5); + REQUIRE_RANGES(set, {5, 7}); + + set.add_shifted(5); + REQUIRE_RANGES(set, {5, 8}); + + set.add_shifted(6); + REQUIRE_RANGES(set, {5, 8}, {9, 10}); + } + + SECTION("add_shifted() after ranges shifts by the size of those ranges") { + set.add(5); + set.add_shifted(6); + REQUIRE_RANGES(set, {5, 6}, {7, 8}); + + set.add_shifted(6); // bumped into second range + REQUIRE_RANGES(set, {5, 6}, {7, 9}); + + set.add_shifted(8); + REQUIRE_RANGES(set, {5, 6}, {7, 9}, {11, 12}); + } +} From d5e00c9315f696d10d465d6ec42e21aa24ccd75e Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 13 Nov 2015 10:43:44 -0800 Subject: [PATCH 19/79] Handle allowed schema changes in the transaction log observer --- src/impl/transact_log_handler.cpp | 70 +++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index ed1630ca..031485e4 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -18,7 +18,7 @@ #include "transact_log_handler.hpp" -#include "../binding_context.hpp" +#include "binding_context.hpp" #include #include @@ -41,6 +41,10 @@ class TransactLogHandler { // Change information for the currently selected LinkList, if any ColumnInfo* m_active_linklist = nullptr; + // Tables which were created during the transaction being processed, which + // can have columns inserted without a schema version bump + std::vector m_new_tables; + // Get the change info for the given column, creating it if needed static ColumnInfo& get_change(ObserverState& state, size_t i) { @@ -80,6 +84,21 @@ class TransactLogHandler { m_observers.erase(m_observers.begin() + (o - &m_observers[0])); } + REALM_NORETURN + REALM_NOINLINE + void schema_error() + { + throw std::runtime_error("Schema mismatch detected: another process has modified the Realm file's schema in an incompatible way"); + } + + bool schema_error_unless_new_table() + { + if (std::find(begin(m_new_tables), end(m_new_tables), m_current_table) == end(m_new_tables)) { + schema_error(); + } + return true; + } + public: template TransactLogHandler(BindingContext* binding_context, SharedGroup& sg, Func&& func) @@ -111,21 +130,36 @@ public: m_binding_context->will_change(m_observers, invalidated); } - // These would require having an observer before schema init - // Maybe do something here to throw an error when multiple processes have different schemas? - bool insert_group_level_table(size_t, size_t, StringData) { return false; } - bool erase_group_level_table(size_t, size_t) { return false; } - bool rename_group_level_table(size_t, StringData) { return false; } - bool insert_column(size_t, DataType, StringData, bool) { return false; } - bool insert_link_column(size_t, DataType, StringData, size_t, size_t) { return false; } - bool erase_column(size_t) { return false; } - bool erase_link_column(size_t, size_t, size_t) { return false; } - bool rename_column(size_t, StringData) { return false; } - bool add_search_index(size_t) { return false; } - bool remove_search_index(size_t) { return false; } - bool add_primary_key(size_t) { return false; } - bool remove_primary_key() { return false; } - bool set_link_type(size_t, LinkType) { return false; } + // Schema changes which don't involve a change in the schema version are + // allowed + bool add_search_index(size_t) { return true; } + bool remove_search_index(size_t) { return true; } + + // Creating entirely new tables without a schema version bump is allowed, so + // we need to track if new columns are being added to a new table or an + // existing one + bool insert_group_level_table(size_t table_ndx, size_t, StringData) + { + for (auto& observer : m_observers) { + if (observer.table_ndx >= table_ndx) + ++observer.table_ndx; + } + m_new_tables.push_back(table_ndx); + return true; + } + + bool insert_column(size_t, DataType, StringData, bool) { return schema_error_unless_new_table(); } + bool insert_link_column(size_t, DataType, StringData, size_t, size_t) { return schema_error_unless_new_table(); } + bool add_primary_key(size_t) { return schema_error_unless_new_table(); } + bool set_link_type(size_t, LinkType) { return schema_error_unless_new_table(); } + + // Schema changes which are never allowed while a file is open + bool erase_group_level_table(size_t, size_t) { schema_error(); } + bool rename_group_level_table(size_t, StringData) { schema_error(); } + bool erase_column(size_t) { schema_error(); } + bool erase_link_column(size_t, size_t, size_t) { schema_error(); } + bool rename_column(size_t, StringData) { schema_error(); } + bool remove_primary_key() { schema_error(); } bool select_table(size_t group_level_ndx, int, const size_t*) noexcept { @@ -306,6 +340,8 @@ public: bool nullify_link(size_t col, size_t row, size_t) { return mark_dirty(row, col); } bool set_int_unique(size_t col, size_t row, int_fast64_t) { return mark_dirty(row, col); } bool set_string_unique(size_t col, size_t row, StringData) { return mark_dirty(row, col); } + bool insert_substring(size_t col, size_t row, size_t, StringData) { return mark_dirty(row, col); } + bool erase_substring(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } // Doesn't change any data bool optimize_table() { return true; } @@ -314,8 +350,6 @@ public: bool select_descriptor(int, const size_t*) { return false; } // Not implemented - bool insert_substring(size_t, size_t, size_t, StringData) { return false; } - bool erase_substring(size_t, size_t, size_t, size_t) { return false; } bool swap_rows(size_t, size_t) { return false; } bool move_column(size_t, size_t) { return false; } bool move_group_level_table(size_t, size_t) { return false; } From 2ed90e6d79e2a3315eb4dab108eb2096ba648741 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 13 Nov 2015 11:01:12 -0800 Subject: [PATCH 20/79] Check for invalid schema changes even when KVO is not used --- src/impl/transact_log_handler.cpp | 249 +++++++++++++++++++----------- src/impl/transact_log_handler.hpp | 3 +- src/shared_realm.cpp | 6 +- 3 files changed, 170 insertions(+), 88 deletions(-) diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index 031485e4..299dcaad 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -24,8 +24,122 @@ #include #include -namespace realm { -class TransactLogHandler { +using namespace realm; + +namespace { +// A transaction log handler that just validates that all operations made are +// ones supported by the object store +class TransactLogValidator { + // Index of currently selected table + size_t m_current_table = 0; + + // Tables which were created during the transaction being processed, which + // can have columns inserted without a schema version bump + std::vector m_new_tables; + + REALM_NORETURN + REALM_NOINLINE + void schema_error() + { + throw std::runtime_error("Schema mismatch detected: another process has modified the Realm file's schema in an incompatible way"); + } + + // Throw an exception if the currently modified table already existed before + // the current set of modifications + bool schema_error_unless_new_table() + { + if (std::find(begin(m_new_tables), end(m_new_tables), m_current_table) == end(m_new_tables)) { + schema_error(); + } + return true; + } + +protected: + size_t current_table() const noexcept { return m_current_table; } + +public: + // Schema changes which don't involve a change in the schema version are + // allowed + bool add_search_index(size_t) { return true; } + bool remove_search_index(size_t) { return true; } + + // Creating entirely new tables without a schema version bump is allowed, so + // we need to track if new columns are being added to a new table or an + // existing one + bool insert_group_level_table(size_t table_ndx, size_t, StringData) + { + // Shift any previously added tables after the new one + for (auto& table : m_new_tables) { + if (table >= table_ndx) + ++table; + } + m_new_tables.push_back(table_ndx); + return true; + } + bool insert_column(size_t, DataType, StringData, bool) { return schema_error_unless_new_table(); } + bool insert_link_column(size_t, DataType, StringData, size_t, size_t) { return schema_error_unless_new_table(); } + bool add_primary_key(size_t) { return schema_error_unless_new_table(); } + bool set_link_type(size_t, LinkType) { return schema_error_unless_new_table(); } + + // Removing or renaming things while a Realm is open is never supported + bool erase_group_level_table(size_t, size_t) { schema_error(); } + bool rename_group_level_table(size_t, StringData) { schema_error(); } + bool erase_column(size_t) { schema_error(); } + bool erase_link_column(size_t, size_t, size_t) { schema_error(); } + bool rename_column(size_t, StringData) { schema_error(); } + bool remove_primary_key() { schema_error(); } + bool move_column(size_t, size_t) { schema_error(); } + bool move_group_level_table(size_t, size_t) { schema_error(); } + + bool select_descriptor(int levels, const size_t*) + { + // subtables not supported + return levels == 0; + } + + bool select_table(size_t group_level_ndx, int, const size_t*) noexcept + { + m_current_table = group_level_ndx; + return true; + } + + bool select_link_list(size_t, size_t, size_t) { return true; } + + // Non-schema changes are all allowed + void parse_complete() { } + bool insert_empty_rows(size_t, size_t, size_t, bool) { return true; } + bool erase_rows(size_t, size_t, size_t, bool) { return true; } + bool swap_rows(size_t, size_t) { return true; } + bool clear_table() noexcept { return true; } + bool link_list_set(size_t, size_t) { return true; } + bool link_list_insert(size_t, size_t) { return true; } + bool link_list_erase(size_t) { return true; } + bool link_list_nullify(size_t) { return true; } + bool link_list_clear(size_t) { return true; } + bool link_list_move(size_t, size_t) { return true; } + bool link_list_swap(size_t, size_t) { return true; } + bool set_int(size_t, size_t, int_fast64_t) { return true; } + bool set_bool(size_t, size_t, bool) { return true; } + bool set_float(size_t, size_t, float) { return true; } + bool set_double(size_t, size_t, double) { return true; } + bool set_string(size_t, size_t, StringData) { return true; } + bool set_binary(size_t, size_t, BinaryData) { return true; } + bool set_date_time(size_t, size_t, DateTime) { return true; } + bool set_table(size_t, size_t) { return true; } + bool set_mixed(size_t, size_t, const Mixed&) { return true; } + bool set_link(size_t, size_t, size_t, size_t) { return true; } + bool set_null(size_t, size_t) { return true; } + bool nullify_link(size_t, size_t, size_t) { return true; } + bool insert_substring(size_t, size_t, size_t, StringData) { return true; } + bool erase_substring(size_t, size_t, size_t, size_t) { return true; } + bool optimize_table() { return true; } + bool set_int_unique(size_t, size_t, int_fast64_t) { return true; } + bool set_string_unique(size_t, size_t, StringData) { return true; } +}; + +// Extends TransactLogValidator to also track changes and report it to the +// binding context if any properties are being observed +class TransactLogObserver : public TransactLogValidator { using ColumnInfo = BindingContext::ColumnInfo; using ObserverState = BindingContext::ObserverState; @@ -34,10 +148,8 @@ class TransactLogHandler { // Userdata pointers for rows which have been deleted std::vector invalidated; // Delegate to send change information to - BindingContext* m_binding_context; + BindingContext* m_context; - // Index of currently selected table - size_t m_current_table = 0; // Change information for the currently selected LinkList, if any ColumnInfo* m_active_linklist = nullptr; @@ -69,8 +181,8 @@ class TransactLogHandler { // Mark the given row/col as needing notifications sent bool mark_dirty(size_t row_ndx, size_t col_ndx) { - auto it = lower_bound(begin(m_observers), end(m_observers), ObserverState{m_current_table, row_ndx, nullptr}); - if (it != end(m_observers) && it->table_ndx == m_current_table && it->row_ndx == row_ndx) { + auto it = lower_bound(begin(m_observers), end(m_observers), ObserverState{current_table(), row_ndx, nullptr}); + if (it != end(m_observers) && it->table_ndx == current_table() && it->row_ndx == row_ndx) { get_change(*it, col_ndx).changed = true; } return true; @@ -84,86 +196,57 @@ class TransactLogHandler { m_observers.erase(m_observers.begin() + (o - &m_observers[0])); } - REALM_NORETURN - REALM_NOINLINE - void schema_error() - { - throw std::runtime_error("Schema mismatch detected: another process has modified the Realm file's schema in an incompatible way"); - } - - bool schema_error_unless_new_table() - { - if (std::find(begin(m_new_tables), end(m_new_tables), m_current_table) == end(m_new_tables)) { - schema_error(); - } - return true; - } - public: template - TransactLogHandler(BindingContext* binding_context, SharedGroup& sg, Func&& func) - : m_binding_context(binding_context) + TransactLogObserver(BindingContext* context, SharedGroup& sg, Func&& func, bool validate_schema_changes) + : m_context(context) { - if (!binding_context) { - func(); + if (!context) { + if (validate_schema_changes) { + // The handler functions are non-virtual, so the parent class's + // versions are called if we don't need to track changes to observed + // objects + func(static_cast(*this)); + } + else { + func(); + } return; } - m_observers = binding_context->get_observed_rows(); + m_observers = context->get_observed_rows(); if (m_observers.empty()) { auto old_version = sg.get_version_of_current_transaction(); - func(); + if (validate_schema_changes) { + func(static_cast(*this)); + } + else { + func(); + } if (old_version != sg.get_version_of_current_transaction()) { - binding_context->did_change({}, {}); + context->did_change({}, {}); } return; } func(*this); - binding_context->did_change(m_observers, invalidated); + context->did_change(m_observers, invalidated); } // Called at the end of the transaction log immediately before the version // is advanced void parse_complete() { - m_binding_context->will_change(m_observers, invalidated); + m_context->will_change(m_observers, invalidated); } - // Schema changes which don't involve a change in the schema version are - // allowed - bool add_search_index(size_t) { return true; } - bool remove_search_index(size_t) { return true; } - - // Creating entirely new tables without a schema version bump is allowed, so - // we need to track if new columns are being added to a new table or an - // existing one - bool insert_group_level_table(size_t table_ndx, size_t, StringData) + bool insert_group_level_table(size_t table_ndx, size_t prior_size, StringData name) { for (auto& observer : m_observers) { if (observer.table_ndx >= table_ndx) ++observer.table_ndx; } - m_new_tables.push_back(table_ndx); - return true; - } - - bool insert_column(size_t, DataType, StringData, bool) { return schema_error_unless_new_table(); } - bool insert_link_column(size_t, DataType, StringData, size_t, size_t) { return schema_error_unless_new_table(); } - bool add_primary_key(size_t) { return schema_error_unless_new_table(); } - bool set_link_type(size_t, LinkType) { return schema_error_unless_new_table(); } - - // Schema changes which are never allowed while a file is open - bool erase_group_level_table(size_t, size_t) { schema_error(); } - bool rename_group_level_table(size_t, StringData) { schema_error(); } - bool erase_column(size_t) { schema_error(); } - bool erase_link_column(size_t, size_t, size_t) { schema_error(); } - bool rename_column(size_t, StringData) { schema_error(); } - bool remove_primary_key() { schema_error(); } - - bool select_table(size_t group_level_ndx, int, const size_t*) noexcept - { - m_current_table = group_level_ndx; + TransactLogValidator::insert_group_level_table(table_ndx, prior_size, name); return true; } @@ -177,7 +260,7 @@ public: { for (size_t i = 0; i < m_observers.size(); ++i) { auto& o = m_observers[i]; - if (o.table_ndx == m_current_table) { + if (o.table_ndx == current_table()) { if (o.row_ndx == row_ndx) { invalidate(&o); --i; @@ -197,7 +280,7 @@ public: { for (size_t i = 0; i < m_observers.size(); ) { auto& o = m_observers[i]; - if (o.table_ndx == m_current_table) { + if (o.table_ndx == current_table()) { invalidate(&o); } else { @@ -211,7 +294,7 @@ public: { m_active_linklist = nullptr; for (auto& o : m_observers) { - if (o.table_ndx == m_current_table && o.row_ndx == row) { + if (o.table_ndx == current_table() && o.row_ndx == row) { m_active_linklist = &get_change(o, col); break; } @@ -342,47 +425,41 @@ public: bool set_string_unique(size_t col, size_t row, StringData) { return mark_dirty(row, col); } bool insert_substring(size_t col, size_t row, size_t, StringData) { return mark_dirty(row, col); } bool erase_substring(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } - - // Doesn't change any data - bool optimize_table() { return true; } - - // Used for subtables, which we currently don't support - bool select_descriptor(int, const size_t*) { return false; } - - // Not implemented - bool swap_rows(size_t, size_t) { return false; } - bool move_column(size_t, size_t) { return false; } - bool move_group_level_table(size_t, size_t) { return false; } }; } // anonymous namespace namespace realm { namespace _impl { namespace transaction { -void advance(SharedGroup& sg, ClientHistory& history, BindingContext* binding_context) { - TransactLogHandler(binding_context, sg, [&](auto&&... args) { +void advance(SharedGroup& sg, ClientHistory& history, BindingContext* context) +{ + TransactLogObserver(context, sg, [&](auto&&... args) { LangBindHelper::advance_read(sg, history, std::move(args)...); - }); + }, true); } -void begin(SharedGroup& sg, ClientHistory& history, BindingContext* binding_context) { - TransactLogHandler(binding_context, sg, [&](auto&&... args) { +void begin(SharedGroup& sg, ClientHistory& history, BindingContext* context, + bool validate_schema_changes) +{ + TransactLogObserver(context, sg, [&](auto&&... args) { LangBindHelper::promote_to_write(sg, history, std::move(args)...); - }); + }, validate_schema_changes); } -void commit(SharedGroup& sg, ClientHistory&, BindingContext* binding_context) { +void commit(SharedGroup& sg, ClientHistory&, BindingContext* context) +{ LangBindHelper::commit_and_continue_as_read(sg); - if (binding_context) { - binding_context->did_change({}, {}); + if (context) { + context->did_change({}, {}); } } -void cancel(SharedGroup& sg, ClientHistory& history, BindingContext* binding_context) { - TransactLogHandler(binding_context, sg, [&](auto&&... args) { +void cancel(SharedGroup& sg, ClientHistory& history, BindingContext* context) +{ + TransactLogObserver(context, sg, [&](auto&&... args) { LangBindHelper::rollback_and_continue_as_read(sg, history, std::move(args)...); - }); + }, false); } } // namespace transaction diff --git a/src/impl/transact_log_handler.hpp b/src/impl/transact_log_handler.hpp index 40ddf3f6..f5d3d58a 100644 --- a/src/impl/transact_log_handler.hpp +++ b/src/impl/transact_log_handler.hpp @@ -33,7 +33,8 @@ void advance(SharedGroup& sg, ClientHistory& history, BindingContext* binding_co // Begin a write transaction // If the read transaction version is not up to date, will first advance to the // most recent read transaction and sent notifications to delegate -void begin(SharedGroup& sg, ClientHistory& history, BindingContext* binding_context); +void begin(SharedGroup& sg, ClientHistory& history, BindingContext* binding_context, + bool validate_schema_changes=true); // Commit a write transaction void commit(SharedGroup& sg, ClientHistory& history, BindingContext* binding_context); diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 30e8f928..5c17eb63 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -222,7 +222,11 @@ void Realm::update_schema(std::unique_ptr schema, uint64_t version) return; } - begin_transaction(); + read_group(); + transaction::begin(*m_shared_group, *m_history, m_binding_context.get(), + /* error on schema changes */ false); + m_in_transaction = true; + struct WriteTransactionGuard { Realm& realm; ~WriteTransactionGuard() { From d6daa052e83c4ac5b284169a383710f2768b1651 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 21 Aug 2015 12:27:27 -0700 Subject: [PATCH 21/79] Decouple Realm instance tracking from interprocess notifications --- src/CMakeLists.txt | 2 + src/impl/apple/cached_realm.cpp | 93 +++++++++++++++++++++++ src/impl/apple/cached_realm.hpp | 65 ++++++++++++++++ src/impl/apple/external_commit_helper.cpp | 44 +---------- src/impl/apple/external_commit_helper.hpp | 13 ---- src/impl/realm_coordinator.cpp | 39 +++++----- src/impl/realm_coordinator.hpp | 5 +- 7 files changed, 182 insertions(+), 79 deletions(-) create mode 100644 src/impl/apple/cached_realm.cpp create mode 100644 src/impl/apple/cached_realm.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2c9eade2..ffb715a2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -28,8 +28,10 @@ set(HEADERS if(APPLE) include_directories(impl/apple) list(APPEND SOURCES + impl/apple/cached_realm.cpp impl/apple/external_commit_helper.cpp) list(APPEND HEADERS + impl/apple/cached_realm.hpp impl/apple/external_commit_helper.hpp) find_library(CF_LIBRARY CoreFoundation) endif() diff --git a/src/impl/apple/cached_realm.cpp b/src/impl/apple/cached_realm.cpp new file mode 100644 index 00000000..35317b3c --- /dev/null +++ b/src/impl/apple/cached_realm.cpp @@ -0,0 +1,93 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "cached_realm.hpp" + +#include "shared_realm.hpp" + +using namespace realm; +using namespace realm::_impl; + +CachedRealm::CachedRealm(const std::shared_ptr& realm, bool cache) +: m_realm(realm) +, m_cache(cache) +{ + struct RefCountedWeakPointer { + std::weak_ptr realm; + std::atomic ref_count = {1}; + }; + + CFRunLoopSourceContext ctx{}; + ctx.info = new RefCountedWeakPointer{realm}; + ctx.perform = [](void* info) { + if (auto realm = static_cast(info)->realm.lock()) { + realm->notify(); + } + }; + ctx.retain = [](const void* info) { + static_cast(const_cast(info))->ref_count.fetch_add(1, std::memory_order_relaxed); + return info; + }; + ctx.release = [](const void* info) { + auto ptr = static_cast(const_cast(info)); + if (ptr->ref_count.fetch_add(-1, std::memory_order_acq_rel) == 1) { + delete ptr; + } + }; + + m_runloop = CFRunLoopGetCurrent(); + CFRetain(m_runloop); + m_signal = CFRunLoopSourceCreate(kCFAllocatorDefault, 0, &ctx); + CFRunLoopAddSource(m_runloop, m_signal, kCFRunLoopDefaultMode); +} + +CachedRealm::CachedRealm(CachedRealm&& rgt) +{ + *this = std::move(rgt); +} + +CachedRealm& CachedRealm::operator=(CachedRealm&& rgt) +{ + m_realm = std::move(rgt.m_realm); + m_thread_id = rgt.m_thread_id; + m_cache = rgt.m_cache; + m_runloop = rgt.m_runloop; + m_signal = rgt.m_signal; + rgt.m_runloop = nullptr; + rgt.m_signal = nullptr; + + return *this; +} + +CachedRealm::~CachedRealm() +{ + if (m_signal) { + CFRunLoopSourceInvalidate(m_signal); + CFRelease(m_signal); + CFRelease(m_runloop); + } +} + +void CachedRealm::notify() +{ + CFRunLoopSourceSignal(m_signal); + // Signalling the source makes it run the next time the runloop gets + // to it, but doesn't make the runloop start if it's currently idle + // waiting for events + CFRunLoopWakeUp(m_runloop); +} diff --git a/src/impl/apple/cached_realm.hpp b/src/impl/apple/cached_realm.hpp new file mode 100644 index 00000000..7b1ddd3e --- /dev/null +++ b/src/impl/apple/cached_realm.hpp @@ -0,0 +1,65 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_CACHED_REALM_HPP +#define REALM_CACHED_REALM_HPP + +#include +#include +#include + +namespace realm { +class Realm; + +namespace _impl { + +class CachedRealm { +public: + CachedRealm(const std::shared_ptr& realm, bool cache); + ~CachedRealm(); + + CachedRealm(CachedRealm&&); + CachedRealm& operator=(CachedRealm&&); + + CachedRealm(const CachedRealm&) = delete; + CachedRealm& operator=(const CachedRealm&) = delete; + + // Get a strong reference to the cached realm + std::shared_ptr realm() const { return m_realm.lock(); } + + // Does this CachedRealm store a Realm instance that should be used on the current thread? + bool is_cached_for_current_thread() const { return m_cache && m_thread_id == std::this_thread::get_id(); } + + bool expired() const { return m_realm.expired(); } + + // Asyncronously call notify() on the Realm on the appropriate thread + void notify(); + +private: + std::weak_ptr m_realm; + std::thread::id m_thread_id = std::this_thread::get_id(); + bool m_cache = false; + + CFRunLoopRef m_runloop; + CFRunLoopSourceRef m_signal; +}; + +} // namespace _impl +} // namespace realm + +#endif // REALM_CACHED_REALM_HPP diff --git a/src/impl/apple/external_commit_helper.cpp b/src/impl/apple/external_commit_helper.cpp index c22fe5df..33608ace 100644 --- a/src/impl/apple/external_commit_helper.cpp +++ b/src/impl/apple/external_commit_helper.cpp @@ -143,45 +143,10 @@ ExternalCommitHelper::ExternalCommitHelper(RealmCoordinator& parent) ExternalCommitHelper::~ExternalCommitHelper() { - REALM_ASSERT_DEBUG(m_realms.empty()); notify_fd(m_shutdown_write_fd); m_thread.wait(); // Wait for the thread to exit } -void ExternalCommitHelper::add_realm(realm::Realm* realm) -{ - std::lock_guard lock(m_realms_mutex); - - // Create the runloop source - CFRunLoopSourceContext ctx{}; - ctx.info = realm; - ctx.perform = [](void* info) { - static_cast(info)->notify(); - }; - - CFRunLoopRef runloop = CFRunLoopGetCurrent(); - CFRetain(runloop); - CFRunLoopSourceRef signal = CFRunLoopSourceCreate(kCFAllocatorDefault, 0, &ctx); - CFRunLoopAddSource(runloop, signal, kCFRunLoopDefaultMode); - - m_realms.push_back({realm, runloop, signal}); -} - -void ExternalCommitHelper::remove_realm(realm::Realm* realm) -{ - std::lock_guard lock(m_realms_mutex); - for (auto it = m_realms.begin(); it != m_realms.end(); ++it) { - if (it->realm == realm) { - CFRunLoopSourceInvalidate(it->signal); - CFRelease(it->signal); - CFRelease(it->runloop); - m_realms.erase(it); - return; - } - } - REALM_TERMINATE("Realm not registered"); -} - void ExternalCommitHelper::listen() { pthread_setname_np("RLMRealm notification listener"); @@ -216,14 +181,7 @@ void ExternalCommitHelper::listen() } assert(event.ident == (uint32_t)m_notify_fd); - std::lock_guard lock(m_realms_mutex); - for (auto const& realm : m_realms) { - CFRunLoopSourceSignal(realm.signal); - // Signalling the source makes it run the next time the runloop gets - // to it, but doesn't make the runloop start if it's currently idle - // waiting for events - CFRunLoopWakeUp(realm.runloop); - } + m_parent.on_change(); } } diff --git a/src/impl/apple/external_commit_helper.hpp b/src/impl/apple/external_commit_helper.hpp index 9f31d730..624b8618 100644 --- a/src/impl/apple/external_commit_helper.hpp +++ b/src/impl/apple/external_commit_helper.hpp @@ -20,9 +20,7 @@ #define REALM_EXTERNAL_COMMIT_HELPER_HPP #include -#include #include -#include #include namespace realm { @@ -31,16 +29,12 @@ class Realm; namespace _impl { class RealmCoordinator; -// FIXME: split IPC from the local cross-thread stuff -// both are platform-specific but need to be useable separately class ExternalCommitHelper { public: ExternalCommitHelper(RealmCoordinator& parent); ~ExternalCommitHelper(); void notify_others(); - void add_realm(Realm* realm); - void remove_realm(Realm* realm); private: // A RAII holder for a file descriptor which automatically closes the wrapped @@ -75,13 +69,6 @@ private: RealmCoordinator& m_parent; - // Currently registered realms and the signal for delivering notifications - // to them - std::vector m_realms; - - // Mutex which guards m_realms - std::mutex m_realms_mutex; - // The listener thread std::future m_thread; diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 297a42d3..a25a5036 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -18,17 +18,13 @@ #include "realm_coordinator.hpp" +#include "cached_realm.hpp" #include "external_commit_helper.hpp" #include "object_store.hpp" using namespace realm; using namespace realm::_impl; -struct realm::_impl::CachedRealm { - std::weak_ptr realm; - std::thread::id thread_id; -}; - static std::mutex s_coordinator_mutex; static std::map> s_coordinators_per_path; @@ -59,9 +55,9 @@ std::shared_ptr RealmCoordinator::get_existing_coordinator(Str std::shared_ptr RealmCoordinator::get_realm(Realm::Config config) { std::lock_guard lock(m_realm_mutex); - if (!m_notifier && m_cached_realms.empty()) { + if ((!m_config.read_only && !m_notifier) || (m_config.read_only && m_cached_realms.empty())) { m_config = config; - if (!config.read_only) { + if (!config.read_only && !m_notifier) { m_notifier = std::make_unique(*this); } } @@ -89,13 +85,12 @@ std::shared_ptr RealmCoordinator::get_realm(Realm::Config config) } } - auto thread_id = std::this_thread::get_id(); if (config.cache) { for (auto& cachedRealm : m_cached_realms) { - if (cachedRealm.thread_id == thread_id) { + if (cachedRealm.is_cached_for_current_thread()) { // can be null if we jumped in between ref count hitting zero and // unregister_realm() getting the lock - if (auto realm = cachedRealm.realm.lock()) { + if (auto realm = cachedRealm.realm()) { return realm; } } @@ -104,12 +99,7 @@ std::shared_ptr RealmCoordinator::get_realm(Realm::Config config) auto realm = std::make_shared(config); realm->init(shared_from_this()); - if (m_notifier) { - m_notifier->add_realm(realm.get()); - } - if (config.cache) { - m_cached_realms.push_back({realm, thread_id}); - } + m_cached_realms.emplace_back(realm, m_config.cache); return realm; } @@ -133,14 +123,11 @@ RealmCoordinator::~RealmCoordinator() } } -void RealmCoordinator::unregister_realm(Realm* realm) +void RealmCoordinator::unregister_realm(Realm*) { std::lock_guard lock(m_realm_mutex); - if (m_notifier) { - m_notifier->remove_realm(realm); - } for (size_t i = 0; i < m_cached_realms.size(); ++i) { - if (m_cached_realms[i].realm.expired()) { + if (m_cached_realms[i].expired()) { if (i + 1 < m_cached_realms.size()) { m_cached_realms[i] = std::move(m_cached_realms.back()); } @@ -164,7 +151,7 @@ void RealmCoordinator::clear_cache() } for (auto& cached_realm : coordinator->m_cached_realms) { - if (auto realm = cached_realm.realm.lock()) { + if (auto realm = cached_realm.realm()) { realms_to_close.push_back(realm); } } @@ -185,3 +172,11 @@ void RealmCoordinator::send_commit_notifications() REALM_ASSERT(!m_config.read_only); m_notifier->notify_others(); } + +void RealmCoordinator::on_change() +{ + std::lock_guard lock(m_realm_mutex); + for (auto& realm : m_cached_realms) { + realm.notify(); + } +} diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index 5813d93f..ac889412 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -25,8 +25,8 @@ namespace realm { namespace _impl { +class CachedRealm; class ExternalCommitHelper; -struct CachedRealm; // RealmCoordinator manages the weak cache of Realm instances and communication // between per-thread Realm instances for a given file @@ -63,6 +63,9 @@ public: // Do not call directly void unregister_realm(Realm* realm); + // Called by m_notifier when there's a new commit to send notifications for + void on_change(); + private: Realm::Config m_config; From 7a0c83929f9c19e1619637fcb533eafa3be58eb5 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 23 Nov 2015 14:45:33 -0800 Subject: [PATCH 22/79] Use an unordered map for the Realm coordinator cache --- src/impl/realm_coordinator.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index a25a5036..f3d6c1d8 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -22,11 +22,13 @@ #include "external_commit_helper.hpp" #include "object_store.hpp" +#include + using namespace realm; using namespace realm::_impl; static std::mutex s_coordinator_mutex; -static std::map> s_coordinators_per_path; +static std::unordered_map> s_coordinators_per_path; std::shared_ptr RealmCoordinator::get_coordinator(StringData path) { From e30e2ff278b8eb4122b6a0ccbf8146a25d5ad514 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 23 Nov 2015 14:46:51 -0800 Subject: [PATCH 23/79] Simplify RealmCoordinator::get_coordinator() --- src/impl/realm_coordinator.cpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index f3d6c1d8..cf3289b4 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -33,17 +33,14 @@ static std::unordered_map> s_coordi std::shared_ptr RealmCoordinator::get_coordinator(StringData path) { std::lock_guard lock(s_coordinator_mutex); - std::shared_ptr coordinator; - auto it = s_coordinators_per_path.find(path); - if (it != s_coordinators_per_path.end()) { - coordinator = it->second.lock(); - } - - if (!coordinator) { - s_coordinators_per_path[path] = coordinator = std::make_shared(); + auto& weak_coordinator = s_coordinators_per_path[path]; + if (auto coordinator = weak_coordinator.lock()) { + return coordinator; } + auto coordinator = std::make_shared(); + weak_coordinator = coordinator; return coordinator; } From ebfca16d003a9cd029fa733413652cf7944e59c0 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 30 Nov 2015 10:42:09 -0800 Subject: [PATCH 24/79] Eliminate a config copy when opening Realms --- src/shared_realm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 85fc8214..45960d8b 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -169,7 +169,7 @@ Group *Realm::read_group() SharedRealm Realm::get_shared_realm(Config config) { - return RealmCoordinator::get_coordinator(config.path)->get_realm(config); + return RealmCoordinator::get_coordinator(config.path)->get_realm(std::move(config)); } void Realm::update_schema(std::unique_ptr schema, uint64_t version) From 9b8a0d53460b9dd9f29176cc5e35f3efe1e9134a Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 2 Dec 2015 11:53:06 -0800 Subject: [PATCH 25/79] Log uncaught exceptions in the notifier thread By default the thread just silently goes away. --- src/impl/apple/external_commit_helper.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/impl/apple/external_commit_helper.cpp b/src/impl/apple/external_commit_helper.cpp index 33608ace..f66d4e37 100644 --- a/src/impl/apple/external_commit_helper.cpp +++ b/src/impl/apple/external_commit_helper.cpp @@ -20,6 +20,7 @@ #include "realm_coordinator.hpp" +#include #include #include #include @@ -138,7 +139,21 @@ ExternalCommitHelper::ExternalCommitHelper(RealmCoordinator& parent) m_shutdown_read_fd = pipeFd[0]; m_shutdown_write_fd = pipeFd[1]; - m_thread = std::async(std::launch::async, [=] { listen(); }); + m_thread = std::async(std::launch::async, [=] { + try { + listen(); + } + catch (std::exception const& e) { + fprintf(stderr, "uncaught exception in notifier thread: %s: %s\n", typeid(e).name(), e.what()); + asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "uncaught exception in notifier thread: %s: %s", typeid(e).name(), e.what()); + throw; + } + catch (...) { + fprintf(stderr, "uncaught exception in notifier thread\n"); + asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "uncaught exception in notifier thread"); + throw; + } + }); } ExternalCommitHelper::~ExternalCommitHelper() From 89bd55a5351a7e4f2454dbfa31a5e023f17c8049 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 3 Dec 2015 08:55:29 -0800 Subject: [PATCH 26/79] Actually remove the Realm from the cache when close() is called --- src/impl/apple/cached_realm.cpp | 1 + src/impl/apple/cached_realm.hpp | 8 ++++++++ src/impl/realm_coordinator.cpp | 15 +++++++++------ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/impl/apple/cached_realm.cpp b/src/impl/apple/cached_realm.cpp index 35317b3c..03705d52 100644 --- a/src/impl/apple/cached_realm.cpp +++ b/src/impl/apple/cached_realm.cpp @@ -25,6 +25,7 @@ using namespace realm::_impl; CachedRealm::CachedRealm(const std::shared_ptr& realm, bool cache) : m_realm(realm) +, m_realm_key(realm.get()) , m_cache(cache) { struct RefCountedWeakPointer { diff --git a/src/impl/apple/cached_realm.hpp b/src/impl/apple/cached_realm.hpp index 7b1ddd3e..89806e95 100644 --- a/src/impl/apple/cached_realm.hpp +++ b/src/impl/apple/cached_realm.hpp @@ -45,14 +45,22 @@ public: // Does this CachedRealm store a Realm instance that should be used on the current thread? bool is_cached_for_current_thread() const { return m_cache && m_thread_id == std::this_thread::get_id(); } + // Has the Realm instance been destroyed? bool expired() const { return m_realm.expired(); } // Asyncronously call notify() on the Realm on the appropriate thread void notify(); + // Is this a CachedRealm for the given Realm instance? + // This should be used rather than lock() to avoid deadlocks when the + // reference from lock() is the last remaining one (due to another thread + // releasing them at the same time) + bool is_for_realm(Realm* realm) const { return realm == m_realm_key; } + private: std::weak_ptr m_realm; std::thread::id m_thread_id = std::this_thread::get_id(); + void* m_realm_key; bool m_cache = false; CFRunLoopRef m_runloop; diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index cf3289b4..cfcdd6ad 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -122,16 +122,19 @@ RealmCoordinator::~RealmCoordinator() } } -void RealmCoordinator::unregister_realm(Realm*) +void RealmCoordinator::unregister_realm(Realm* realm) { std::lock_guard lock(m_realm_mutex); for (size_t i = 0; i < m_cached_realms.size(); ++i) { - if (m_cached_realms[i].expired()) { - if (i + 1 < m_cached_realms.size()) { - m_cached_realms[i] = std::move(m_cached_realms.back()); - } - m_cached_realms.pop_back(); + auto& cached_realm = m_cached_realms[i]; + if (!cached_realm.expired() && !cached_realm.is_for_realm(realm)) { + continue; } + + if (i + 1 < m_cached_realms.size()) { + cached_realm = std::move(m_cached_realms.back()); + } + m_cached_realms.pop_back(); } } From 513b3d770c8b1333f411ac8b06504b6adc5d4a3c Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 3 Dec 2015 09:01:32 -0800 Subject: [PATCH 27/79] Add a short explanation of CachedRealm --- src/impl/apple/cached_realm.hpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/impl/apple/cached_realm.hpp b/src/impl/apple/cached_realm.hpp index 89806e95..6b114dc6 100644 --- a/src/impl/apple/cached_realm.hpp +++ b/src/impl/apple/cached_realm.hpp @@ -28,6 +28,11 @@ class Realm; namespace _impl { +// CachedRealm stores a weak reference to a Realm instance, along with all of +// the information about a Realm that needs to be accessed from other threads. +// This is needed to avoid forming strong references to the Realm instances on +// other threads, which can produce deadlocks when the last strong reference to +// a Realm instance is released from within a function holding the cache lock. class CachedRealm { public: CachedRealm(const std::shared_ptr& realm, bool cache); @@ -52,9 +57,6 @@ public: void notify(); // Is this a CachedRealm for the given Realm instance? - // This should be used rather than lock() to avoid deadlocks when the - // reference from lock() is the last remaining one (due to another thread - // releasing them at the same time) bool is_for_realm(Realm* realm) const { return realm == m_realm_key; } private: From 4c195c92e08258357eef3fd2c835759316dba23f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 3 Dec 2015 12:13:19 -0800 Subject: [PATCH 28/79] Remove some unused cruft from ExternalCommitHelper --- src/impl/apple/external_commit_helper.hpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/impl/apple/external_commit_helper.hpp b/src/impl/apple/external_commit_helper.hpp index 624b8618..c6d795a0 100644 --- a/src/impl/apple/external_commit_helper.hpp +++ b/src/impl/apple/external_commit_helper.hpp @@ -19,9 +19,7 @@ #ifndef REALM_EXTERNAL_COMMIT_HELPER_HPP #define REALM_EXTERNAL_COMMIT_HELPER_HPP -#include #include -#include namespace realm { class Realm; @@ -59,12 +57,6 @@ private: FdHolder(FdHolder const&) = delete; }; - struct PerRealmInfo { - Realm* realm; - CFRunLoopRef runloop; - CFRunLoopSourceRef signal; - }; - void listen(); RealmCoordinator& m_parent; From 178c562f2ca675d4f8ed358922089454aabcacf6 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 3 Dec 2015 12:31:56 -0800 Subject: [PATCH 29/79] Add an untested non-Apple ExternalCommitHelper implementation --- src/impl/generic/external_commit_helper.cpp | 46 ++++++++++++++++++ src/impl/generic/external_commit_helper.hpp | 54 +++++++++++++++++++++ src/impl/realm_coordinator.hpp | 2 + 3 files changed, 102 insertions(+) create mode 100644 src/impl/generic/external_commit_helper.cpp create mode 100644 src/impl/generic/external_commit_helper.hpp diff --git a/src/impl/generic/external_commit_helper.cpp b/src/impl/generic/external_commit_helper.cpp new file mode 100644 index 00000000..66417642 --- /dev/null +++ b/src/impl/generic/external_commit_helper.cpp @@ -0,0 +1,46 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "external_commit_helper.hpp" + +#include "realm_coordinator.hpp" + +#include +#include + +using namespace realm; +using namespace realm::_impl; + +ExternalCommitHelper::ExternalCommitHelper(RealmCoordinator& parent) +: m_parent(parent) +, m_history(realm::make_client_history(parent.get_path(), parent.get_encryption_key().data())) +, m_sg(*m_history, parent.is_in_memory() ? SharedGroup::durability_MemOnly : SharedGroup::durability_Full, + parent.get_encryption_key().data()) +, m_thread(std::async(std::launch::async, [=] { + while (m_sg.wait_for_change()) { + m_parent.on_change(); + } +})) +{ +} + +ExternalCommitHelper::~ExternalCommitHelper() +{ + m_sg.wait_for_change_release(); + m_thread.wait(); // Wait for the thread to exit +} diff --git a/src/impl/generic/external_commit_helper.hpp b/src/impl/generic/external_commit_helper.hpp new file mode 100644 index 00000000..3249430e --- /dev/null +++ b/src/impl/generic/external_commit_helper.hpp @@ -0,0 +1,54 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_EXTERNAL_COMMIT_HELPER_HPP +#define REALM_EXTERNAL_COMMIT_HELPER_HPP + +#include + +#include + +namespace realm { +class ClientHistory; + +namespace _impl { +class RealmCoordinator; + +class ExternalCommitHelper { +public: + ExternalCommitHelper(RealmCoordinator& parent); + ~ExternalCommitHelper(); + + // A no-op in this version, but needed for the Apple version + void notify_others() { } + +private: + RealmCoordinator& m_parent; + + // The listener thread + std::future m_thread; + + // A shared group used to listen for changes + std::unique_ptr m_history; + SharedGroup m_sg; +}; + +} // namespace _impl +} // namespace realm + +#endif /* REALM_EXTERNAL_COMMIT_HELPER_HPP */ diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index ac889412..46cfcec3 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -45,6 +45,8 @@ public: const Schema* get_schema() const noexcept; uint64_t get_schema_version() const noexcept { return m_config.schema_version; } const std::string& get_path() const noexcept { return m_config.path; } + const std::vector& get_encryption_key() const noexcept { return m_config.encryption_key; } + bool is_in_memory() const noexcept { return m_config.in_memory; } // Asyncronously call notify() on every Realm instance for this coordinator's // path, including those in other processes From 112c778d8e8af33645e58c57851f028129b392da Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 3 Dec 2015 14:16:48 -0800 Subject: [PATCH 30/79] Extract the non-Apple specific parts of CachedRealm to a base class --- src/CMakeLists.txt | 1 + src/impl/apple/cached_realm.cpp | 14 +++---- src/impl/apple/cached_realm.hpp | 28 ++------------ src/impl/cached_realm_base.hpp | 68 +++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 32 deletions(-) create mode 100644 src/impl/cached_realm_base.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ffb715a2..716b8f7d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,6 +21,7 @@ set(HEADERS results.hpp schema.hpp shared_realm.hpp + impl/cached_realm_base.hpp impl/transact_log_handler.hpp parser/parser.hpp parser/query_builder.hpp) diff --git a/src/impl/apple/cached_realm.cpp b/src/impl/apple/cached_realm.cpp index 03705d52..ca475e46 100644 --- a/src/impl/apple/cached_realm.cpp +++ b/src/impl/apple/cached_realm.cpp @@ -24,9 +24,7 @@ using namespace realm; using namespace realm::_impl; CachedRealm::CachedRealm(const std::shared_ptr& realm, bool cache) -: m_realm(realm) -, m_realm_key(realm.get()) -, m_cache(cache) +: CachedRealmBase(realm, cache) { struct RefCountedWeakPointer { std::weak_ptr realm; @@ -58,15 +56,17 @@ CachedRealm::CachedRealm(const std::shared_ptr& realm, bool cache) } CachedRealm::CachedRealm(CachedRealm&& rgt) +: CachedRealmBase(std::move(rgt)) +, m_runloop(rgt.m_runloop) +, m_signal(rgt.m_signal) { - *this = std::move(rgt); + rgt.m_runloop = nullptr; + rgt.m_signal = nullptr; } CachedRealm& CachedRealm::operator=(CachedRealm&& rgt) { - m_realm = std::move(rgt.m_realm); - m_thread_id = rgt.m_thread_id; - m_cache = rgt.m_cache; + CachedRealmBase::operator=(std::move(rgt)); m_runloop = rgt.m_runloop; m_signal = rgt.m_signal; rgt.m_runloop = nullptr; diff --git a/src/impl/apple/cached_realm.hpp b/src/impl/apple/cached_realm.hpp index 6b114dc6..15be9488 100644 --- a/src/impl/apple/cached_realm.hpp +++ b/src/impl/apple/cached_realm.hpp @@ -19,21 +19,16 @@ #ifndef REALM_CACHED_REALM_HPP #define REALM_CACHED_REALM_HPP +#include "../cached_realm_base.hpp" + #include -#include -#include namespace realm { class Realm; namespace _impl { -// CachedRealm stores a weak reference to a Realm instance, along with all of -// the information about a Realm that needs to be accessed from other threads. -// This is needed to avoid forming strong references to the Realm instances on -// other threads, which can produce deadlocks when the last strong reference to -// a Realm instance is released from within a function holding the cache lock. -class CachedRealm { +class CachedRealm : public CachedRealmBase { public: CachedRealm(const std::shared_ptr& realm, bool cache); ~CachedRealm(); @@ -44,27 +39,10 @@ public: CachedRealm(const CachedRealm&) = delete; CachedRealm& operator=(const CachedRealm&) = delete; - // Get a strong reference to the cached realm - std::shared_ptr realm() const { return m_realm.lock(); } - - // Does this CachedRealm store a Realm instance that should be used on the current thread? - bool is_cached_for_current_thread() const { return m_cache && m_thread_id == std::this_thread::get_id(); } - - // Has the Realm instance been destroyed? - bool expired() const { return m_realm.expired(); } - // Asyncronously call notify() on the Realm on the appropriate thread void notify(); - // Is this a CachedRealm for the given Realm instance? - bool is_for_realm(Realm* realm) const { return realm == m_realm_key; } - private: - std::weak_ptr m_realm; - std::thread::id m_thread_id = std::this_thread::get_id(); - void* m_realm_key; - bool m_cache = false; - CFRunLoopRef m_runloop; CFRunLoopSourceRef m_signal; }; diff --git a/src/impl/cached_realm_base.hpp b/src/impl/cached_realm_base.hpp new file mode 100644 index 00000000..57feceb5 --- /dev/null +++ b/src/impl/cached_realm_base.hpp @@ -0,0 +1,68 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_CACHED_REALM_BASE_HPP +#define REALM_CACHED_REALM_BASE_HPP + +#include +#include + +namespace realm { +class Realm; + +namespace _impl { + +// CachedRealm stores a weak reference to a Realm instance, along with all of +// the information about a Realm that needs to be accessed from other threads. +// This is needed to avoid forming strong references to the Realm instances on +// other threads, which can produce deadlocks when the last strong reference to +// a Realm instance is released from within a function holding the cache lock. +class CachedRealmBase { +public: + CachedRealmBase(const std::shared_ptr& realm, bool cache); + + // Get a strong reference to the cached realm + std::shared_ptr realm() const { return m_realm.lock(); } + + // Does this CachedRealmBase store a Realm instance that should be used on the current thread? + bool is_cached_for_current_thread() const { return m_cache && m_thread_id == std::this_thread::get_id(); } + + // Has the Realm instance been destroyed? + bool expired() const { return m_realm.expired(); } + + // Is this a CachedRealmBase for the given Realm instance? + bool is_for_realm(Realm* realm) const { return realm == m_realm_key; } + +private: + std::weak_ptr m_realm; + std::thread::id m_thread_id = std::this_thread::get_id(); + void* m_realm_key; + bool m_cache = false; +}; + +inline CachedRealmBase::CachedRealmBase(const std::shared_ptr& realm, bool cache) +: m_realm(realm) +, m_realm_key(realm.get()) +, m_cache(cache) +{ +} + +} // namespace _impl +} // namespace realm + +#endif // REALM_CACHED_REALM_BASE_HPP From 4eb49ce6dc2d4a681509ccd8ce43f252a3aa05e8 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 3 Dec 2015 14:19:24 -0800 Subject: [PATCH 31/79] Add a not-very-useful generic CachedRealm implementation --- src/CMakeLists.txt | 7 ++++++ src/impl/generic/cached_realm.hpp | 40 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 src/impl/generic/cached_realm.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 716b8f7d..6775bab7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -35,6 +35,13 @@ if(APPLE) impl/apple/cached_realm.hpp impl/apple/external_commit_helper.hpp) find_library(CF_LIBRARY CoreFoundation) +else() + include_directories(impl/generic) + list(APPEND SOURCES + impl/generic/external_commit_helper.cpp) + list(APPEND HEADERS + impl/generic/cached_realm.hpp + impl/generic/external_commit_helper.hpp) endif() add_library(realm-object-store SHARED ${SOURCES} ${HEADERS}) diff --git a/src/impl/generic/cached_realm.hpp b/src/impl/generic/cached_realm.hpp new file mode 100644 index 00000000..8ceadf0a --- /dev/null +++ b/src/impl/generic/cached_realm.hpp @@ -0,0 +1,40 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_CACHED_REALM_HPP +#define REALM_CACHED_REALM_HPP + +#include "../cached_realm_base.hpp" + +namespace realm { +class Realm; + +namespace _impl { + +class CachedRealm : public CachedRealmBase { +public: + using CachedRealmBase::CachedRealmBase; + + // Do nothing, as this can't be implemented portably + void notify() { } +}; + +} // namespace _impl +} // namespace realm + +#endif // REALM_CACHED_REALM_HPP From e557babaad421e48a6c36acec1e75fb8c93f383f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Dec 2015 12:07:42 -0800 Subject: [PATCH 32/79] Fix the generic implementation of ExternalCommitHelper --- src/impl/generic/external_commit_helper.cpp | 3 +++ src/impl/generic/external_commit_helper.hpp | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/impl/generic/external_commit_helper.cpp b/src/impl/generic/external_commit_helper.cpp index 66417642..e50e884a 100644 --- a/src/impl/generic/external_commit_helper.cpp +++ b/src/impl/generic/external_commit_helper.cpp @@ -32,7 +32,10 @@ ExternalCommitHelper::ExternalCommitHelper(RealmCoordinator& parent) , m_sg(*m_history, parent.is_in_memory() ? SharedGroup::durability_MemOnly : SharedGroup::durability_Full, parent.get_encryption_key().data()) , m_thread(std::async(std::launch::async, [=] { + m_sg.begin_read(); while (m_sg.wait_for_change()) { + m_sg.end_read(); + m_sg.begin_read(); m_parent.on_change(); } })) diff --git a/src/impl/generic/external_commit_helper.hpp b/src/impl/generic/external_commit_helper.hpp index 3249430e..d056566e 100644 --- a/src/impl/generic/external_commit_helper.hpp +++ b/src/impl/generic/external_commit_helper.hpp @@ -40,12 +40,12 @@ public: private: RealmCoordinator& m_parent; - // The listener thread - std::future m_thread; - // A shared group used to listen for changes std::unique_ptr m_history; SharedGroup m_sg; + + // The listener thread + std::future m_thread; }; } // namespace _impl From ad5db727673e4dd8cb6da2741fca0afd81cd51b7 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 11 Dec 2015 14:31:07 -0800 Subject: [PATCH 33/79] Destroy all notifiers before closing realms in clear_cache() --- src/impl/realm_coordinator.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index cfcdd6ad..ca61c8e4 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -140,18 +140,19 @@ void RealmCoordinator::unregister_realm(Realm* realm) void RealmCoordinator::clear_cache() { - std::vector realms_to_close; - + std::vector realms_to_close; { std::lock_guard lock(s_coordinator_mutex); - // Gather a list of all of the realms which will be removed for (auto& weak_coordinator : s_coordinators_per_path) { auto coordinator = weak_coordinator.second.lock(); if (!coordinator) { continue; } + coordinator->m_notifier = nullptr; + + // Gather a list of all of the realms which will be removed for (auto& cached_realm : coordinator->m_cached_realms) { if (auto realm = cached_realm.realm()) { realms_to_close.push_back(realm); @@ -164,8 +165,10 @@ void RealmCoordinator::clear_cache() // Close all of the previously cached Realms. This can't be done while // s_coordinator_mutex is held as it may try to re-lock it. - for (auto& realm : realms_to_close) { - realm->close(); + for (auto& weak_realm : realms_to_close) { + if (auto realm = weak_realm.lock()) { + realm->close(); + } } } From 07c40b45176294072b95eb15ea6920770bfeb792 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 26 Jan 2016 12:00:36 -0800 Subject: [PATCH 34/79] Reformat list.{hpp,cpp} to match core style --- src/list.cpp | 33 ++++++++++++++++++++---------- src/list.hpp | 58 +++++++++++++++++++++++++++++----------------------- 2 files changed, 54 insertions(+), 37 deletions(-) diff --git a/src/list.cpp b/src/list.cpp index 39537a63..747f5eae 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -22,64 +22,75 @@ using namespace realm; -size_t List::size() { +size_t List::size() +{ verify_attached(); return m_link_view->size(); } -Row List::get(std::size_t row_ndx) { +Row List::get(std::size_t row_ndx) +{ verify_attached(); verify_valid_row(row_ndx); return m_link_view->get(row_ndx); } -void List::set(std::size_t row_ndx, std::size_t target_row_ndx) { +void List::set(std::size_t row_ndx, std::size_t target_row_ndx) +{ verify_attached(); verify_in_tranaction(); verify_valid_row(row_ndx); m_link_view->set(row_ndx, target_row_ndx); } -void List::add(std::size_t target_row_ndx) { +void List::add(std::size_t target_row_ndx) +{ verify_attached(); verify_in_tranaction(); m_link_view->add(target_row_ndx); } -void List::insert(std::size_t row_ndx, std::size_t target_row_ndx) { +void List::insert(std::size_t row_ndx, std::size_t target_row_ndx) +{ verify_attached(); verify_in_tranaction(); verify_valid_row(row_ndx, true); m_link_view->insert(row_ndx, target_row_ndx); } -void List::remove(std::size_t row_ndx) { +void List::remove(std::size_t row_ndx) +{ verify_attached(); verify_in_tranaction(); verify_valid_row(row_ndx); m_link_view->remove(row_ndx); } -Query List::get_query() { +Query List::get_query() +{ verify_attached(); return m_link_view->get_target_table().where(m_link_view); } -void List::verify_valid_row(std::size_t row_ndx, bool insertion) { +void List::verify_valid_row(std::size_t row_ndx, bool insertion) +{ size_t size = m_link_view->size(); if (row_ndx > size || (!insertion && row_ndx == size)) { - throw std::out_of_range(std::string("Index ") + std::to_string(row_ndx) + " is outside of range 0..." + std::to_string(size) + "."); + throw std::out_of_range("Index " + std::to_string(row_ndx) + " is outside of range 0..." + + std::to_string(size) + "."); } } -void List::verify_attached() { +void List::verify_attached() +{ if (!m_link_view->is_attached()) { throw std::runtime_error("Tableview is not attached"); } m_link_view->sync_if_needed(); } -void List::verify_in_tranaction() { +void List::verify_in_tranaction() +{ if (!m_realm->is_in_transaction()) { throw std::runtime_error("Can only mutate a list within a transaction."); } diff --git a/src/list.hpp b/src/list.hpp index cf43ece1..99600d1f 100644 --- a/src/list.hpp +++ b/src/list.hpp @@ -23,41 +23,47 @@ #include namespace realm { - class List { - public: - List(SharedRealm &r, const ObjectSchema &s, LinkViewRef l) : m_realm(r), m_object_schema(&s), m_link_view(l) {} +class ObjectSchema; +class List { +public: + List(SharedRealm& r, const ObjectSchema& s, LinkViewRef l) + : m_realm(r) + , m_object_schema(&s) + , m_link_view(l) + { + } - const ObjectSchema &get_object_schema() const { return *m_object_schema; } - SharedRealm realm() { return m_realm; } + const ObjectSchema& get_object_schema() const { return *m_object_schema; } + SharedRealm realm() { return m_realm; } - size_t size(); - Row get(std::size_t row_ndx); - void set(std::size_t row_ndx, std::size_t target_row_ndx); + size_t size(); + Row get(std::size_t row_ndx); + void set(std::size_t row_ndx, std::size_t target_row_ndx); - void add(size_t target_row_ndx); - void remove(size_t list_ndx); - void insert(size_t list_ndx, size_t target_row_ndx); + void add(size_t target_row_ndx); + void remove(size_t list_ndx); + void insert(size_t list_ndx, size_t target_row_ndx); - template - void add(ContextType ctx, ValueType value); + template + void add(ContextType ctx, ValueType value); - template - void insert(ContextType ctx, ValueType value, size_t list_ndx); + template + void insert(ContextType ctx, ValueType value, size_t list_ndx); - template - void set(ContextType ctx, ValueType value, size_t list_ndx); + template + void set(ContextType ctx, ValueType value, size_t list_ndx); - Query get_query(); + Query get_query(); - void verify_valid_row(std::size_t row_ndx, bool insertion = false); - void verify_attached(); - void verify_in_tranaction(); + void verify_valid_row(std::size_t row_ndx, bool insertion = false); + void verify_attached(); + void verify_in_tranaction(); - private: - SharedRealm m_realm; - const ObjectSchema *m_object_schema; - LinkViewRef m_link_view; - }; +private: + SharedRealm m_realm; + const ObjectSchema* m_object_schema; + LinkViewRef m_link_view; +}; } #endif /* REALM_LIST_HPP */ From 356c17ba11cb4ef0f65922dafa83ac3e2bcf3a78 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 26 Jan 2016 12:02:59 -0800 Subject: [PATCH 35/79] Remove list.hpp's dependency on shared_realm.hpp --- src/list.cpp | 11 +++++++++++ src/list.hpp | 17 ++++++++--------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/list.cpp b/src/list.cpp index 747f5eae..6ad8a59e 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -18,10 +18,21 @@ #include "list.hpp" +#include "shared_realm.hpp" + #include using namespace realm; +List::List(std::shared_ptr r, const ObjectSchema& s, LinkViewRef l) +: m_realm(std::move(r)) +, m_object_schema(&s) +, m_link_view(std::move(l)) +{ +} + +List::~List() = default; + size_t List::size() { verify_attached(); diff --git a/src/list.hpp b/src/list.hpp index 99600d1f..d134162e 100644 --- a/src/list.hpp +++ b/src/list.hpp @@ -19,22 +19,21 @@ #ifndef REALM_LIST_HPP #define REALM_LIST_HPP -#include "shared_realm.hpp" #include +#include + namespace realm { class ObjectSchema; +class Realm; + class List { public: - List(SharedRealm& r, const ObjectSchema& s, LinkViewRef l) - : m_realm(r) - , m_object_schema(&s) - , m_link_view(l) - { - } + List(std::shared_ptr r, const ObjectSchema& s, LinkViewRef l); + ~List(); const ObjectSchema& get_object_schema() const { return *m_object_schema; } - SharedRealm realm() { return m_realm; } + const std::shared_ptr& realm() { return m_realm; } size_t size(); Row get(std::size_t row_ndx); @@ -60,7 +59,7 @@ public: void verify_in_tranaction(); private: - SharedRealm m_realm; + std::shared_ptr m_realm; const ObjectSchema* m_object_schema; LinkViewRef m_link_view; }; From 62d573c1d9d74434a15e98c95cc2f2bb0b8d1a12 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 26 Jan 2016 12:05:27 -0800 Subject: [PATCH 36/79] Remove std:: from size_t in list.* --- src/list.cpp | 12 ++++++------ src/list.hpp | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/list.cpp b/src/list.cpp index 6ad8a59e..6b51de9f 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -39,14 +39,14 @@ size_t List::size() return m_link_view->size(); } -Row List::get(std::size_t row_ndx) +Row List::get(size_t row_ndx) { verify_attached(); verify_valid_row(row_ndx); return m_link_view->get(row_ndx); } -void List::set(std::size_t row_ndx, std::size_t target_row_ndx) +void List::set(size_t row_ndx, size_t target_row_ndx) { verify_attached(); verify_in_tranaction(); @@ -54,14 +54,14 @@ void List::set(std::size_t row_ndx, std::size_t target_row_ndx) m_link_view->set(row_ndx, target_row_ndx); } -void List::add(std::size_t target_row_ndx) +void List::add(size_t target_row_ndx) { verify_attached(); verify_in_tranaction(); m_link_view->add(target_row_ndx); } -void List::insert(std::size_t row_ndx, std::size_t target_row_ndx) +void List::insert(size_t row_ndx, size_t target_row_ndx) { verify_attached(); verify_in_tranaction(); @@ -69,7 +69,7 @@ void List::insert(std::size_t row_ndx, std::size_t target_row_ndx) m_link_view->insert(row_ndx, target_row_ndx); } -void List::remove(std::size_t row_ndx) +void List::remove(size_t row_ndx) { verify_attached(); verify_in_tranaction(); @@ -83,7 +83,7 @@ Query List::get_query() return m_link_view->get_target_table().where(m_link_view); } -void List::verify_valid_row(std::size_t row_ndx, bool insertion) +void List::verify_valid_row(size_t row_ndx, bool insertion) { size_t size = m_link_view->size(); if (row_ndx > size || (!insertion && row_ndx == size)) { diff --git a/src/list.hpp b/src/list.hpp index d134162e..2c24482a 100644 --- a/src/list.hpp +++ b/src/list.hpp @@ -36,8 +36,8 @@ public: const std::shared_ptr& realm() { return m_realm; } size_t size(); - Row get(std::size_t row_ndx); - void set(std::size_t row_ndx, std::size_t target_row_ndx); + Row get(size_t row_ndx); + void set(size_t row_ndx, size_t target_row_ndx); void add(size_t target_row_ndx); void remove(size_t list_ndx); @@ -54,7 +54,7 @@ public: Query get_query(); - void verify_valid_row(std::size_t row_ndx, bool insertion = false); + void verify_valid_row(size_t row_ndx, bool insertion = false); void verify_attached(); void verify_in_tranaction(); From 0819f72b1b9d6ce756f415f84dc82768fc4e2d60 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 26 Jan 2016 12:08:40 -0800 Subject: [PATCH 37/79] Remove call to sync_if_needed() on a LinkView --- src/list.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/list.cpp b/src/list.cpp index 6b51de9f..0c78942e 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -97,7 +97,6 @@ void List::verify_attached() if (!m_link_view->is_attached()) { throw std::runtime_error("Tableview is not attached"); } - m_link_view->sync_if_needed(); } void List::verify_in_tranaction() From 243ae3218724e76140de6e5211b252cc34f7868b Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 26 Jan 2016 12:09:07 -0800 Subject: [PATCH 38/79] Fix incorrect error message --- src/list.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/list.cpp b/src/list.cpp index 0c78942e..cbcf0317 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -95,7 +95,7 @@ void List::verify_valid_row(size_t row_ndx, bool insertion) void List::verify_attached() { if (!m_link_view->is_attached()) { - throw std::runtime_error("Tableview is not attached"); + throw std::runtime_error("LinkView is not attached"); } } From 1cbbf1958f10d1c67327f522b1ca39da9b082152 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 26 Jan 2016 12:10:10 -0800 Subject: [PATCH 39/79] Check the thread in all of List's methods --- src/list.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/list.cpp b/src/list.cpp index cbcf0317..08126704 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -97,6 +97,7 @@ void List::verify_attached() if (!m_link_view->is_attached()) { throw std::runtime_error("LinkView is not attached"); } + m_realm->verify_thread(); } void List::verify_in_tranaction() From 6276266d679173f2ad524a22343e275186fbf254 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 26 Jan 2016 14:43:49 -0800 Subject: [PATCH 40/79] Make List const-correct --- src/list.cpp | 14 +++++++------- src/list.hpp | 21 +++++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/list.cpp b/src/list.cpp index 08126704..593aac03 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -24,7 +24,7 @@ using namespace realm; -List::List(std::shared_ptr r, const ObjectSchema& s, LinkViewRef l) +List::List(std::shared_ptr r, const ObjectSchema& s, LinkViewRef l) noexcept : m_realm(std::move(r)) , m_object_schema(&s) , m_link_view(std::move(l)) @@ -33,13 +33,13 @@ List::List(std::shared_ptr r, const ObjectSchema& s, LinkViewRef l) List::~List() = default; -size_t List::size() +size_t List::size() const { verify_attached(); return m_link_view->size(); } -Row List::get(size_t row_ndx) +Row List::get(size_t row_ndx) const { verify_attached(); verify_valid_row(row_ndx); @@ -77,13 +77,13 @@ void List::remove(size_t row_ndx) m_link_view->remove(row_ndx); } -Query List::get_query() +Query List::get_query() const { verify_attached(); return m_link_view->get_target_table().where(m_link_view); } -void List::verify_valid_row(size_t row_ndx, bool insertion) +void List::verify_valid_row(size_t row_ndx, bool insertion) const { size_t size = m_link_view->size(); if (row_ndx > size || (!insertion && row_ndx == size)) { @@ -92,7 +92,7 @@ void List::verify_valid_row(size_t row_ndx, bool insertion) } } -void List::verify_attached() +void List::verify_attached() const { if (!m_link_view->is_attached()) { throw std::runtime_error("LinkView is not attached"); @@ -100,7 +100,7 @@ void List::verify_attached() m_realm->verify_thread(); } -void List::verify_in_tranaction() +void List::verify_in_tranaction() const { if (!m_realm->is_in_transaction()) { throw std::runtime_error("Can only mutate a list within a transaction."); diff --git a/src/list.hpp b/src/list.hpp index 2c24482a..75d478b4 100644 --- a/src/list.hpp +++ b/src/list.hpp @@ -29,20 +29,24 @@ class Realm; class List { public: - List(std::shared_ptr r, const ObjectSchema& s, LinkViewRef l); + List(std::shared_ptr r, const ObjectSchema& s, LinkViewRef l) noexcept; ~List(); const ObjectSchema& get_object_schema() const { return *m_object_schema; } - const std::shared_ptr& realm() { return m_realm; } + const std::shared_ptr& realm() const { return m_realm; } - size_t size(); - Row get(size_t row_ndx); + size_t size() const; + Row get(size_t row_ndx) const; void set(size_t row_ndx, size_t target_row_ndx); void add(size_t target_row_ndx); void remove(size_t list_ndx); void insert(size_t list_ndx, size_t target_row_ndx); + Query get_query() const; + void verify_in_tranaction() const; + + // These are implemented in object_accessor.hpp template void add(ContextType ctx, ValueType value); @@ -52,16 +56,13 @@ public: template void set(ContextType ctx, ValueType value, size_t list_ndx); - Query get_query(); - - void verify_valid_row(size_t row_ndx, bool insertion = false); - void verify_attached(); - void verify_in_tranaction(); - private: std::shared_ptr m_realm; const ObjectSchema* m_object_schema; LinkViewRef m_link_view; + + void verify_valid_row(size_t row_ndx, bool insertion = false) const; + void verify_attached() const; }; } From 91521989629743c2e9a37baec0250706b741f3a0 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 26 Jan 2016 14:44:31 -0800 Subject: [PATCH 41/79] Change List::realm() to List::get_realm() for consistency --- src/list.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/list.hpp b/src/list.hpp index 75d478b4..c4f24df3 100644 --- a/src/list.hpp +++ b/src/list.hpp @@ -33,7 +33,8 @@ public: ~List(); const ObjectSchema& get_object_schema() const { return *m_object_schema; } - const std::shared_ptr& realm() const { return m_realm; } + const std::shared_ptr& get_realm() const { return m_realm; } + Query get_query() const; size_t size() const; Row get(size_t row_ndx) const; @@ -43,7 +44,6 @@ public: void remove(size_t list_ndx); void insert(size_t list_ndx, size_t target_row_ndx); - Query get_query() const; void verify_in_tranaction() const; // These are implemented in object_accessor.hpp From 27acf3f1098e7845b63f1405107576511a367fa3 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 3 Feb 2016 09:24:56 -0800 Subject: [PATCH 42/79] Fix spelling of verify_in_transaction --- src/list.cpp | 10 +++++----- src/list.hpp | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/list.cpp b/src/list.cpp index 593aac03..937cbf02 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -49,7 +49,7 @@ Row List::get(size_t row_ndx) const void List::set(size_t row_ndx, size_t target_row_ndx) { verify_attached(); - verify_in_tranaction(); + verify_in_transaction(); verify_valid_row(row_ndx); m_link_view->set(row_ndx, target_row_ndx); } @@ -57,14 +57,14 @@ void List::set(size_t row_ndx, size_t target_row_ndx) void List::add(size_t target_row_ndx) { verify_attached(); - verify_in_tranaction(); + verify_in_transaction(); m_link_view->add(target_row_ndx); } void List::insert(size_t row_ndx, size_t target_row_ndx) { verify_attached(); - verify_in_tranaction(); + verify_in_transaction(); verify_valid_row(row_ndx, true); m_link_view->insert(row_ndx, target_row_ndx); } @@ -72,7 +72,7 @@ void List::insert(size_t row_ndx, size_t target_row_ndx) void List::remove(size_t row_ndx) { verify_attached(); - verify_in_tranaction(); + verify_in_transaction(); verify_valid_row(row_ndx); m_link_view->remove(row_ndx); } @@ -100,7 +100,7 @@ void List::verify_attached() const m_realm->verify_thread(); } -void List::verify_in_tranaction() const +void List::verify_in_transaction() const { if (!m_realm->is_in_transaction()) { throw std::runtime_error("Can only mutate a list within a transaction."); diff --git a/src/list.hpp b/src/list.hpp index c4f24df3..c2d6f21f 100644 --- a/src/list.hpp +++ b/src/list.hpp @@ -44,7 +44,7 @@ public: void remove(size_t list_ndx); void insert(size_t list_ndx, size_t target_row_ndx); - void verify_in_tranaction() const; + void verify_in_transaction() const; // These are implemented in object_accessor.hpp template From 8e58fc693c90a0d5f66967acbf979af55abcfd4b Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 3 Feb 2016 09:28:11 -0800 Subject: [PATCH 43/79] Call verify_attached() from verify_in_transaction() --- src/list.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/list.cpp b/src/list.cpp index 937cbf02..ac5a6ecf 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -48,7 +48,6 @@ Row List::get(size_t row_ndx) const void List::set(size_t row_ndx, size_t target_row_ndx) { - verify_attached(); verify_in_transaction(); verify_valid_row(row_ndx); m_link_view->set(row_ndx, target_row_ndx); @@ -56,14 +55,12 @@ void List::set(size_t row_ndx, size_t target_row_ndx) void List::add(size_t target_row_ndx) { - verify_attached(); verify_in_transaction(); m_link_view->add(target_row_ndx); } void List::insert(size_t row_ndx, size_t target_row_ndx) { - verify_attached(); verify_in_transaction(); verify_valid_row(row_ndx, true); m_link_view->insert(row_ndx, target_row_ndx); @@ -71,7 +68,6 @@ void List::insert(size_t row_ndx, size_t target_row_ndx) void List::remove(size_t row_ndx) { - verify_attached(); verify_in_transaction(); verify_valid_row(row_ndx); m_link_view->remove(row_ndx); @@ -102,6 +98,7 @@ void List::verify_attached() const void List::verify_in_transaction() const { + verify_attached(); if (!m_realm->is_in_transaction()) { throw std::runtime_error("Can only mutate a list within a transaction."); } From 4c5389dbac642068d90917a3c3575a8c78c1e20e Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 3 Feb 2016 17:57:29 -0800 Subject: [PATCH 44/79] Return RowExpr from List::get() --- src/list.cpp | 2 +- src/list.hpp | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/list.cpp b/src/list.cpp index ac5a6ecf..348c603e 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -39,7 +39,7 @@ size_t List::size() const return m_link_view->size(); } -Row List::get(size_t row_ndx) const +RowExpr List::get(size_t row_ndx) const { verify_attached(); verify_valid_row(row_ndx); diff --git a/src/list.hpp b/src/list.hpp index c2d6f21f..18165e6d 100644 --- a/src/list.hpp +++ b/src/list.hpp @@ -24,6 +24,9 @@ #include namespace realm { +template class BasicRowExpr; +using RowExpr = BasicRowExpr; + class ObjectSchema; class Realm; @@ -37,7 +40,7 @@ public: Query get_query() const; size_t size() const; - Row get(size_t row_ndx) const; + RowExpr get(size_t row_ndx) const; void set(size_t row_ndx, size_t target_row_ndx); void add(size_t target_row_ndx); From fdc67777bb4b74e3ca15d072be284f612837e08f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 3 Feb 2016 17:58:17 -0800 Subject: [PATCH 45/79] Make List default constructable --- src/list.cpp | 5 +++-- src/list.hpp | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/list.cpp b/src/list.cpp index 348c603e..bd1fb43b 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -24,6 +24,9 @@ using namespace realm; +List::List() noexcept = default; +List::~List() = default; + List::List(std::shared_ptr r, const ObjectSchema& s, LinkViewRef l) noexcept : m_realm(std::move(r)) , m_object_schema(&s) @@ -31,8 +34,6 @@ List::List(std::shared_ptr r, const ObjectSchema& s, LinkViewRef l) noexc { } -List::~List() = default; - size_t List::size() const { verify_attached(); diff --git a/src/list.hpp b/src/list.hpp index 18165e6d..c15c6eb9 100644 --- a/src/list.hpp +++ b/src/list.hpp @@ -32,6 +32,7 @@ class Realm; class List { public: + List() noexcept; List(std::shared_ptr r, const ObjectSchema& s, LinkViewRef l) noexcept; ~List(); @@ -61,7 +62,7 @@ public: private: std::shared_ptr m_realm; - const ObjectSchema* m_object_schema; + const ObjectSchema* m_object_schema = nullptr; LinkViewRef m_link_view; void verify_valid_row(size_t row_ndx, bool insertion = false) const; From 74eb195e7f0fbbd0463d66ccc63fe37b45086e1b Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 3 Feb 2016 18:03:02 -0800 Subject: [PATCH 46/79] Add wrappers for a bunch more LinkView methods --- src/list.cpp | 149 ++++++++++++++++++++++++++++++++++++--------------- src/list.hpp | 33 +++++++++--- 2 files changed, 133 insertions(+), 49 deletions(-) diff --git a/src/list.cpp b/src/list.cpp index bd1fb43b..aee57506 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -18,6 +18,7 @@ #include "list.hpp" +#include "results.hpp" #include "shared_realm.hpp" #include @@ -34,46 +35,6 @@ List::List(std::shared_ptr r, const ObjectSchema& s, LinkViewRef l) noexc { } -size_t List::size() const -{ - verify_attached(); - return m_link_view->size(); -} - -RowExpr List::get(size_t row_ndx) const -{ - verify_attached(); - verify_valid_row(row_ndx); - return m_link_view->get(row_ndx); -} - -void List::set(size_t row_ndx, size_t target_row_ndx) -{ - verify_in_transaction(); - verify_valid_row(row_ndx); - m_link_view->set(row_ndx, target_row_ndx); -} - -void List::add(size_t target_row_ndx) -{ - verify_in_transaction(); - m_link_view->add(target_row_ndx); -} - -void List::insert(size_t row_ndx, size_t target_row_ndx) -{ - verify_in_transaction(); - verify_valid_row(row_ndx, true); - m_link_view->insert(row_ndx, target_row_ndx); -} - -void List::remove(size_t row_ndx) -{ - verify_in_transaction(); - verify_valid_row(row_ndx); - m_link_view->remove(row_ndx); -} - Query List::get_query() const { verify_attached(); @@ -89,18 +50,120 @@ void List::verify_valid_row(size_t row_ndx, bool insertion) const } } +bool List::is_valid() const +{ + m_realm->verify_thread(); + return m_link_view && m_link_view->is_attached(); +} + void List::verify_attached() const { - if (!m_link_view->is_attached()) { + if (!is_valid()) { throw std::runtime_error("LinkView is not attached"); } - m_realm->verify_thread(); } void List::verify_in_transaction() const { verify_attached(); if (!m_realm->is_in_transaction()) { - throw std::runtime_error("Can only mutate a list within a transaction."); + throw InvalidTransactionException("Must be in a write transaction"); } } + +size_t List::size() const +{ + verify_attached(); + return m_link_view->size(); +} + +RowExpr List::get(size_t row_ndx) const +{ + verify_attached(); + verify_valid_row(row_ndx); + return m_link_view->get(row_ndx); +} + +size_t List::find(ConstRow const& row) const +{ + verify_attached(); + + if (!row.is_attached() || row.get_table() != &m_link_view->get_target_table()) { + return not_found; + } + + return m_link_view->find(row.get_index()); +} + +void List::add(size_t target_row_ndx) +{ + verify_in_transaction(); + m_link_view->add(target_row_ndx); +} + +void List::insert(size_t row_ndx, size_t target_row_ndx) +{ + verify_in_transaction(); + verify_valid_row(row_ndx, true); + m_link_view->insert(row_ndx, target_row_ndx); +} + +void List::move(size_t source_ndx, size_t dest_ndx) +{ + verify_in_transaction(); + verify_valid_row(source_ndx); + verify_valid_row(dest_ndx); // Can't be one past end due to removing one earlier + m_link_view->move(source_ndx, dest_ndx); +} + +void List::remove(size_t row_ndx) +{ + verify_in_transaction(); + verify_valid_row(row_ndx); + m_link_view->remove(row_ndx); +} + +void List::remove_all() +{ + verify_in_transaction(); + m_link_view->clear(); +} + +void List::set(size_t row_ndx, size_t target_row_ndx) +{ + verify_in_transaction(); + verify_valid_row(row_ndx); + m_link_view->set(row_ndx, target_row_ndx); +} + +void List::swap(size_t ndx1, size_t ndx2) +{ + verify_in_transaction(); + verify_valid_row(ndx1); + verify_valid_row(ndx2); + m_link_view->swap(ndx1, ndx2); +} + +void List::delete_all() +{ + verify_in_transaction(); + m_link_view->remove_all_target_rows(); +} + +Results List::sort(SortOrder order) +{ + return Results(m_realm, *m_object_schema, get_query(), std::move(order)); +} + +// These definitions rely on that LinkViews are interned by core +bool List::operator==(List const& rgt) const noexcept +{ + return m_link_view.get() == rgt.m_link_view.get(); +} + +namespace std { +size_t hash::operator()(realm::List const& list) const +{ + return std::hash()(list.m_link_view.get()); +} +} diff --git a/src/list.hpp b/src/list.hpp index c15c6eb9..464c3bcd 100644 --- a/src/list.hpp +++ b/src/list.hpp @@ -29,6 +29,8 @@ using RowExpr = BasicRowExpr
; class ObjectSchema; class Realm; +class Results; +struct SortOrder; class List { public: @@ -36,19 +38,31 @@ public: List(std::shared_ptr r, const ObjectSchema& s, LinkViewRef l) noexcept; ~List(); - const ObjectSchema& get_object_schema() const { return *m_object_schema; } const std::shared_ptr& get_realm() const { return m_realm; } Query get_query() const; + const ObjectSchema& get_object_schema() const { return *m_object_schema; } + + bool is_valid() const; + void verify_attached() const; + void verify_in_transaction() const; size_t size() const; RowExpr get(size_t row_ndx) const; - void set(size_t row_ndx, size_t target_row_ndx); + size_t find(ConstRow const& row) const; void add(size_t target_row_ndx); - void remove(size_t list_ndx); void insert(size_t list_ndx, size_t target_row_ndx); + void move(size_t source_ndx, size_t dest_ndx); + void remove(size_t list_ndx); + void remove_all(); + void set(size_t row_ndx, size_t target_row_ndx); + void swap(size_t ndx1, size_t ndx2); - void verify_in_transaction() const; + void delete_all(); + + Results sort(SortOrder order); + + bool operator==(List const& rgt) const noexcept; // These are implemented in object_accessor.hpp template @@ -62,11 +76,18 @@ public: private: std::shared_ptr m_realm; - const ObjectSchema* m_object_schema = nullptr; + const ObjectSchema* m_object_schema; LinkViewRef m_link_view; void verify_valid_row(size_t row_ndx, bool insertion = false) const; - void verify_attached() const; + + friend struct std::hash; +}; +} // namespace realm + +namespace std { +template<> struct hash { + size_t operator()(realm::List const&) const; }; } From a3dab7e4b195655aa852a9483aee3e5092467110 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 25 Jan 2016 10:25:28 -0800 Subject: [PATCH 47/79] Add wrappers for platform-specific headers and normalize include paths Building the objectstore code now only requires adding the root `src` directory to the include paths. --- src/CMakeLists.txt | 6 ++--- src/impl/apple/cached_realm.cpp | 2 +- src/impl/apple/cached_realm.hpp | 7 +---- src/impl/apple/external_commit_helper.cpp | 4 +-- src/impl/apple/external_commit_helper.hpp | 5 ---- src/impl/cached_realm.hpp | 30 +++++++++++++++++++++ src/impl/external_commit_helper.hpp | 30 +++++++++++++++++++++ src/impl/generic/cached_realm.hpp | 6 +---- src/impl/generic/external_commit_helper.cpp | 4 +-- src/impl/generic/external_commit_helper.hpp | 4 --- src/impl/realm_coordinator.cpp | 6 ++--- src/impl/transact_log_handler.cpp | 4 +-- src/shared_realm.cpp | 6 ++--- 13 files changed, 77 insertions(+), 37 deletions(-) create mode 100644 src/impl/cached_realm.hpp create mode 100644 src/impl/external_commit_helper.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6775bab7..76233013 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,3 @@ -include_directories(impl) - set(SOURCES index_set.cpp list.cpp @@ -21,13 +19,14 @@ set(HEADERS results.hpp schema.hpp shared_realm.hpp + impl/cached_realm.hpp impl/cached_realm_base.hpp + impl/external_commit_helper.hpp impl/transact_log_handler.hpp parser/parser.hpp parser/query_builder.hpp) if(APPLE) - include_directories(impl/apple) list(APPEND SOURCES impl/apple/cached_realm.cpp impl/apple/external_commit_helper.cpp) @@ -36,7 +35,6 @@ if(APPLE) impl/apple/external_commit_helper.hpp) find_library(CF_LIBRARY CoreFoundation) else() - include_directories(impl/generic) list(APPEND SOURCES impl/generic/external_commit_helper.cpp) list(APPEND HEADERS diff --git a/src/impl/apple/cached_realm.cpp b/src/impl/apple/cached_realm.cpp index ca475e46..1468a8fb 100644 --- a/src/impl/apple/cached_realm.cpp +++ b/src/impl/apple/cached_realm.cpp @@ -16,7 +16,7 @@ // //////////////////////////////////////////////////////////////////////////// -#include "cached_realm.hpp" +#include "impl/cached_realm.hpp" #include "shared_realm.hpp" diff --git a/src/impl/apple/cached_realm.hpp b/src/impl/apple/cached_realm.hpp index 15be9488..5acf874e 100644 --- a/src/impl/apple/cached_realm.hpp +++ b/src/impl/apple/cached_realm.hpp @@ -16,10 +16,7 @@ // //////////////////////////////////////////////////////////////////////////// -#ifndef REALM_CACHED_REALM_HPP -#define REALM_CACHED_REALM_HPP - -#include "../cached_realm_base.hpp" +#include "impl/cached_realm_base.hpp" #include @@ -49,5 +46,3 @@ private: } // namespace _impl } // namespace realm - -#endif // REALM_CACHED_REALM_HPP diff --git a/src/impl/apple/external_commit_helper.cpp b/src/impl/apple/external_commit_helper.cpp index f66d4e37..db04a1de 100644 --- a/src/impl/apple/external_commit_helper.cpp +++ b/src/impl/apple/external_commit_helper.cpp @@ -16,9 +16,9 @@ // //////////////////////////////////////////////////////////////////////////// -#include "external_commit_helper.hpp" +#include "impl/external_commit_helper.hpp" -#include "realm_coordinator.hpp" +#include "impl/realm_coordinator.hpp" #include #include diff --git a/src/impl/apple/external_commit_helper.hpp b/src/impl/apple/external_commit_helper.hpp index c6d795a0..a39876ce 100644 --- a/src/impl/apple/external_commit_helper.hpp +++ b/src/impl/apple/external_commit_helper.hpp @@ -16,9 +16,6 @@ // //////////////////////////////////////////////////////////////////////////// -#ifndef REALM_EXTERNAL_COMMIT_HELPER_HPP -#define REALM_EXTERNAL_COMMIT_HELPER_HPP - #include namespace realm { @@ -77,5 +74,3 @@ private: } // namespace _impl } // namespace realm - -#endif /* REALM_EXTERNAL_COMMIT_HELPER_HPP */ diff --git a/src/impl/cached_realm.hpp b/src/impl/cached_realm.hpp new file mode 100644 index 00000000..3818ca62 --- /dev/null +++ b/src/impl/cached_realm.hpp @@ -0,0 +1,30 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2016 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_CACHED_REALM_HPP +#define REALM_CACHED_REALM_HPP + +#include + +#if REALM_PLATFORM_APPLE +#include "impl/apple/cached_realm.hpp" +#else +#include "impl/generic/cached_realm.hpp" +#endif + +#endif // REALM_CACHED_REALM_HPP diff --git a/src/impl/external_commit_helper.hpp b/src/impl/external_commit_helper.hpp new file mode 100644 index 00000000..c467a4b9 --- /dev/null +++ b/src/impl/external_commit_helper.hpp @@ -0,0 +1,30 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2016 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_EXTERNAL_COMMIT_HELPER_HPP +#define REALM_EXTERNAL_COMMIT_HELPER_HPP + +#include + +#if REALM_PLATFORM_APPLE +#include "impl/apple/external_commit_helper.hpp" +#else +#include "impl/generic/external_commit_helper.hpp" +#endif + +#endif // REALM_EXTERNAL_COMMIT_HELPER_HPP diff --git a/src/impl/generic/cached_realm.hpp b/src/impl/generic/cached_realm.hpp index 8ceadf0a..23a7be11 100644 --- a/src/impl/generic/cached_realm.hpp +++ b/src/impl/generic/cached_realm.hpp @@ -16,10 +16,7 @@ // //////////////////////////////////////////////////////////////////////////// -#ifndef REALM_CACHED_REALM_HPP -#define REALM_CACHED_REALM_HPP - -#include "../cached_realm_base.hpp" +#include "impl/cached_realm_base.hpp" namespace realm { class Realm; @@ -37,4 +34,3 @@ public: } // namespace _impl } // namespace realm -#endif // REALM_CACHED_REALM_HPP diff --git a/src/impl/generic/external_commit_helper.cpp b/src/impl/generic/external_commit_helper.cpp index e50e884a..5071304b 100644 --- a/src/impl/generic/external_commit_helper.cpp +++ b/src/impl/generic/external_commit_helper.cpp @@ -16,9 +16,9 @@ // //////////////////////////////////////////////////////////////////////////// -#include "external_commit_helper.hpp" +#include "impl/external_commit_helper.hpp" -#include "realm_coordinator.hpp" +#include "impl/realm_coordinator.hpp" #include #include diff --git a/src/impl/generic/external_commit_helper.hpp b/src/impl/generic/external_commit_helper.hpp index d056566e..cc7fe5eb 100644 --- a/src/impl/generic/external_commit_helper.hpp +++ b/src/impl/generic/external_commit_helper.hpp @@ -16,9 +16,6 @@ // //////////////////////////////////////////////////////////////////////////// -#ifndef REALM_EXTERNAL_COMMIT_HELPER_HPP -#define REALM_EXTERNAL_COMMIT_HELPER_HPP - #include #include @@ -51,4 +48,3 @@ private: } // namespace _impl } // namespace realm -#endif /* REALM_EXTERNAL_COMMIT_HELPER_HPP */ diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index ca61c8e4..e305b420 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -16,10 +16,10 @@ // //////////////////////////////////////////////////////////////////////////// -#include "realm_coordinator.hpp" +#include "impl/realm_coordinator.hpp" -#include "cached_realm.hpp" -#include "external_commit_helper.hpp" +#include "impl/cached_realm.hpp" +#include "impl/external_commit_helper.hpp" #include "object_store.hpp" #include diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index ed1630ca..72b2a200 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -16,9 +16,9 @@ // //////////////////////////////////////////////////////////////////////////// -#include "transact_log_handler.hpp" +#include "impl/transact_log_handler.hpp" -#include "../binding_context.hpp" +#include "binding_context.hpp" #include #include diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 45960d8b..81c70117 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -19,11 +19,11 @@ #include "shared_realm.hpp" #include "binding_context.hpp" -#include "external_commit_helper.hpp" +#include "impl/external_commit_helper.hpp" +#include "impl/realm_coordinator.hpp" +#include "impl/transact_log_handler.hpp" #include "object_store.hpp" -#include "realm_coordinator.hpp" #include "schema.hpp" -#include "transact_log_handler.hpp" #include #include From 638b4ec35e11137c361dd8ed5f10f50541be69a3 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 1 Feb 2016 11:42:33 -0800 Subject: [PATCH 48/79] Actually update the coordinator's copy of the schema --- src/impl/realm_coordinator.cpp | 10 +++++++++- src/impl/realm_coordinator.hpp | 5 +++++ src/shared_realm.cpp | 3 +++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index e305b420..d59cad08 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -21,6 +21,7 @@ #include "impl/cached_realm.hpp" #include "impl/external_commit_helper.hpp" #include "object_store.hpp" +#include "schema.hpp" #include @@ -96,7 +97,7 @@ std::shared_ptr RealmCoordinator::get_realm(Realm::Config config) } } - auto realm = std::make_shared(config); + auto realm = std::make_shared(std::move(config)); realm->init(shared_from_this()); m_cached_realms.emplace_back(realm, m_config.cache); return realm; @@ -107,6 +108,13 @@ const Schema* RealmCoordinator::get_schema() const noexcept return m_cached_realms.empty() ? nullptr : m_config.schema.get(); } +void RealmCoordinator::update_schema(Schema const& schema) +{ + // FIXME: this should probably be doing some sort of validation and + // notifying all Realm instances of the new schema in some way + m_config.schema = std::make_unique(schema); +} + RealmCoordinator::RealmCoordinator() = default; RealmCoordinator::~RealmCoordinator() diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index 46cfcec3..ee1d748b 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -24,6 +24,8 @@ #include namespace realm { +class Schema; + namespace _impl { class CachedRealm; class ExternalCommitHelper; @@ -68,6 +70,9 @@ public: // Called by m_notifier when there's a new commit to send notifications for void on_change(); + // Update the schema in the cached config + void update_schema(Schema const& new_schema); + private: Realm::Config m_config; diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 81c70117..5c5b818a 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -186,6 +186,7 @@ void Realm::update_schema(std::unique_ptr schema, uint64_t version) ObjectStore::verify_schema(*m_config.schema, *schema, m_config.read_only); m_config.schema = std::move(schema); m_config.schema_version = version; + m_coordinator->update_schema(*m_config.schema); return false; }; @@ -243,6 +244,8 @@ void Realm::update_schema(std::unique_ptr schema, uint64_t version) m_config.schema_version = old_config.schema_version; throw; } + + m_coordinator->update_schema(*m_config.schema); } static void check_read_write(Realm *realm) From 5f5510e170197df4292d0941e568d2e2370e0ff3 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 10 Feb 2016 10:38:32 -0800 Subject: [PATCH 49/79] Remove the Query version of Results::index_of() It didn't work for queries based on a LinkView because for those the arguments to count() are indexes in the LinkView rather than table rows, and there's currently no way to check if a query is based on a LinkView. --- src/results.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/results.cpp b/src/results.cpp index 91a4cc22..ca6ae2a3 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -190,9 +190,6 @@ size_t Results::index_of(size_t row_ndx) case Mode::Table: return row_ndx; case Mode::Query: - if (!m_sort) - return m_query.count(row_ndx, row_ndx + 1) ? m_query.count(0, row_ndx) : not_found; - REALM_FALLTHROUGH; case Mode::TableView: update_tableview(); return m_table_view.find_by_source_ndx(row_ndx); From 48510805d70e7f61e329a36a9b37030d1ee78929 Mon Sep 17 00:00:00 2001 From: Ari Lazier Date: Tue, 16 Feb 2016 11:56:29 -0800 Subject: [PATCH 50/79] test contains can be used as a property name in the parser --- tests/parser.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parser.cpp b/tests/parser.cpp index 0815d3d6..40d81baa 100644 --- a/tests/parser.cpp +++ b/tests/parser.cpp @@ -49,6 +49,7 @@ static std::vector valid_queries = { "0 contains 0", "0 BeGiNsWiTh 0", "0 ENDSWITH 0", + "contains contains 'contains'", // atoms/groups "(0=0)", From c57eb99eb262cd27895b9bbaa19f879eefc8bf5c Mon Sep 17 00:00:00 2001 From: Ari Lazier Date: Tue, 16 Feb 2016 12:10:50 -0800 Subject: [PATCH 51/79] test query with not as a property name --- tests/parser.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parser.cpp b/tests/parser.cpp index 40d81baa..c3788cf1 100644 --- a/tests/parser.cpp +++ b/tests/parser.cpp @@ -50,6 +50,7 @@ static std::vector valid_queries = { "0 BeGiNsWiTh 0", "0 ENDSWITH 0", "contains contains 'contains'", + "NOT NOT != 'NOT'", // atoms/groups "(0=0)", From b1fd7abe74af2de6007d6685ef538051e7ac0323 Mon Sep 17 00:00:00 2001 From: Ari Lazier Date: Tue, 16 Feb 2016 12:19:26 -0800 Subject: [PATCH 52/79] more keyword tests --- tests/parser.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/parser.cpp b/tests/parser.cpp index c3788cf1..93d5f159 100644 --- a/tests/parser.cpp +++ b/tests/parser.cpp @@ -50,7 +50,12 @@ static std::vector valid_queries = { "0 BeGiNsWiTh 0", "0 ENDSWITH 0", "contains contains 'contains'", + "beginswith beginswith 'beginswith'", + "endswith endswith 'endswith'", "NOT NOT != 'NOT'", + "AND == 'AND' AND OR == 'OR'", + // FIXME - bug + // "truepredicate == 'falsepredicate' && truepredicate", // atoms/groups "(0=0)", From 055de6718511e01c0dbf7ace52b2a4b5918009b6 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 18 Feb 2016 15:28:51 -0800 Subject: [PATCH 53/79] Fix dependency information for the core tarball --- CMake/RealmCore.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMake/RealmCore.cmake b/CMake/RealmCore.cmake index daf83eba..fa020932 100644 --- a/CMake/RealmCore.cmake +++ b/CMake/RealmCore.cmake @@ -24,8 +24,7 @@ function(download_realm_core core_version) add_custom_command( COMMENT "Extracting ${core_tarball_name}" OUTPUT ${core_libraries} - DEPENDS ${core_temp_tarball} - COMMAND ${CMAKE_COMMAND} -E copy ${core_temp_tarball} ${core_directory_parent} + DEPENDS ${core_tarball} COMMAND ${CMAKE_COMMAND} -E tar xf ${core_tarball} COMMAND ${CMAKE_COMMAND} -E remove_directory ${core_directory} COMMAND ${CMAKE_COMMAND} -E rename core ${core_directory} From ae9d41f9ce565bae40282012e8caf9c4f8cf4b8b Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 18 Feb 2016 08:39:07 -0800 Subject: [PATCH 54/79] Remove an incorrect std::move() --- src/parser/parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser/parser.cpp b/src/parser/parser.cpp index 2fbcbfbd..c0843c0c 100644 --- a/src/parser/parser.cpp +++ b/src/parser/parser.cpp @@ -331,7 +331,7 @@ Predicate parse(const std::string &query) if (out_predicate.type == Predicate::Type::And && out_predicate.cpnd.sub_predicates.size() == 1) { return std::move(out_predicate.cpnd.sub_predicates.back()); } - return std::move(out_predicate); + return out_predicate; } void analyze_grammar() From bceec93a69bc4c518df9d245c6a654bcef7e6a68 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 23 Feb 2016 14:53:03 -0800 Subject: [PATCH 55/79] Update to core 0.96.2 --- CMakeLists.txt | 2 +- src/impl/transact_log_handler.cpp | 9 ++++---- src/parser/query_builder.cpp | 35 +++++++++++++++++++------------ 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 895d4d8c..e0441f49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake") include(CompilerFlags) include(RealmCore) -download_realm_core(0.95.5) +download_realm_core(0.96.2) include_directories(${REALM_CORE_INCLUDE_DIR} src external/pegtl) diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index b26d5895..cd9a8f9f 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -133,8 +133,9 @@ public: bool insert_substring(size_t, size_t, size_t, StringData) { return true; } bool erase_substring(size_t, size_t, size_t, size_t) { return true; } bool optimize_table() { return true; } - bool set_int_unique(size_t, size_t, int_fast64_t) { return true; } - bool set_string_unique(size_t, size_t, StringData) { return true; } + bool set_int_unique(size_t, size_t, size_t, int_fast64_t) { return true; } + bool set_string_unique(size_t, size_t, size_t, StringData) { return true; } + bool change_link_targets(size_t, size_t) { return true; } }; // Extends TransactLogValidator to also track changes and report it to the @@ -421,8 +422,8 @@ public: bool set_link(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } bool set_null(size_t col, size_t row) { return mark_dirty(row, col); } bool nullify_link(size_t col, size_t row, size_t) { return mark_dirty(row, col); } - bool set_int_unique(size_t col, size_t row, int_fast64_t) { return mark_dirty(row, col); } - bool set_string_unique(size_t col, size_t row, StringData) { return mark_dirty(row, col); } + bool set_int_unique(size_t col, size_t row, size_t, int_fast64_t) { return mark_dirty(row, col); } + bool set_string_unique(size_t col, size_t row, size_t, StringData) { return mark_dirty(row, col); } bool insert_substring(size_t col, size_t row, size_t, StringData) { return mark_dirty(row, col); } bool erase_substring(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } }; diff --git a/src/parser/query_builder.cpp b/src/parser/query_builder.cpp index af0c3d62..384191fb 100644 --- a/src/parser/query_builder.cpp +++ b/src/parser/query_builder.cpp @@ -41,16 +41,24 @@ struct TrueExpression : realm::Expression { if (start != end) return start; - return not_found; + return realm::not_found; + } + void set_base_table(const Table*) override {} + const Table* get_base_table() const override { return nullptr; } + std::unique_ptr clone(QueryNodeHandoverPatches*) const override + { + return std::unique_ptr(new TrueExpression(*this)); } - void set_table() override {} - const Table* get_table() const override { return nullptr; } }; struct FalseExpression : realm::Expression { - size_t find_first(size_t, size_t) const override { return not_found; } - void set_table() override {} - const Table* get_table() const override { return nullptr; } + size_t find_first(size_t, size_t) const override { return realm::not_found; } + void set_base_table(const Table*) override {} + const Table* get_base_table() const override { return nullptr; } + std::unique_ptr clone(QueryNodeHandoverPatches*) const override + { + return std::unique_ptr(new FalseExpression(*this)); + } }; using KeyPath = std::vector; @@ -236,9 +244,11 @@ void add_link_constraint_to_query(realm::Query &query, switch (op) { case Predicate::Operator::NotEqual: query.Not(); - case Predicate::Operator::Equal: - query.links_to(prop_expr.prop->table_column, row_index); + case Predicate::Operator::Equal: { + size_t col = prop_expr.prop->table_column; + query.links_to(col, query.get_table()->get_link_target(col)->get(row_index)); break; + } default: throw std::runtime_error("Only 'equal' and 'not equal' operators supported for object comparison."); } @@ -449,7 +459,7 @@ void update_query_with_predicate(Query &query, const Predicate &pred, Arguments update_query_with_predicate(query, sub, arguments, schema, type); } if (!pred.cpnd.sub_predicates.size()) { - query.and_query(new TrueExpression); + query.and_query(std::unique_ptr(new TrueExpression)); } query.end_group(); break; @@ -461,7 +471,7 @@ void update_query_with_predicate(Query &query, const Predicate &pred, Arguments update_query_with_predicate(query, sub, arguments, schema, type); } if (!pred.cpnd.sub_predicates.size()) { - query.and_query(new FalseExpression); + query.and_query(std::unique_ptr(new FalseExpression)); } query.end_group(); break; @@ -471,16 +481,15 @@ void update_query_with_predicate(Query &query, const Predicate &pred, Arguments break; } case Predicate::Type::True: - query.and_query(new TrueExpression); + query.and_query(std::unique_ptr(new TrueExpression)); break; case Predicate::Type::False: - query.and_query(new FalseExpression); + query.and_query(std::unique_ptr(new FalseExpression)); break; default: throw std::runtime_error("Invalid predicate type"); - break; } } From d165458601c605df209c8138aec42dce2b0d5a9f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 30 Oct 2015 10:39:20 -0700 Subject: [PATCH 56/79] Add support for running queries asynchronously --- src/CMakeLists.txt | 1 + src/impl/apple/cached_realm.hpp | 2 +- src/impl/async_query.cpp | 130 ++++++++++++++ src/impl/async_query.hpp | 75 ++++++++ src/impl/realm_coordinator.cpp | 290 +++++++++++++++++++++++++++++- src/impl/realm_coordinator.hpp | 45 ++++- src/impl/transact_log_handler.cpp | 3 +- src/impl/transact_log_handler.hpp | 5 +- src/object_accessor.hpp | 9 +- src/results.cpp | 39 +++- src/results.hpp | 51 +++++- src/schema.hpp | 2 + src/shared_realm.cpp | 6 +- src/shared_realm.hpp | 8 +- 14 files changed, 650 insertions(+), 16 deletions(-) create mode 100644 src/impl/async_query.cpp create mode 100644 src/impl/async_query.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 76233013..76b8a328 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,6 +6,7 @@ set(SOURCES results.cpp schema.cpp shared_realm.cpp + impl/async_query.cpp impl/realm_coordinator.cpp impl/transact_log_handler.cpp parser/parser.cpp diff --git a/src/impl/apple/cached_realm.hpp b/src/impl/apple/cached_realm.hpp index 5acf874e..f67c160c 100644 --- a/src/impl/apple/cached_realm.hpp +++ b/src/impl/apple/cached_realm.hpp @@ -36,7 +36,7 @@ public: CachedRealm(const CachedRealm&) = delete; CachedRealm& operator=(const CachedRealm&) = delete; - // Asyncronously call notify() on the Realm on the appropriate thread + // Asynchronously call notify() on the Realm on the appropriate thread void notify(); private: diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp new file mode 100644 index 00000000..5f131941 --- /dev/null +++ b/src/impl/async_query.cpp @@ -0,0 +1,130 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "impl/async_query.hpp" + +#include "impl/realm_coordinator.hpp" + +using namespace realm; +using namespace realm::_impl; + +AsyncQuery::AsyncQuery(SortOrder sort, + std::unique_ptr> handover, + std::unique_ptr callback, + RealmCoordinator& parent) +: parent(parent.shared_from_this()) +, m_sort(std::move(sort)) +, m_query_handover(std::move(handover)) +, m_callback(std::move(callback)) +{ +} + +void AsyncQuery::get_results(const SharedRealm& realm, SharedGroup& sg, std::vector>& ret) +{ + if (!m_callback->is_for_current_thread()) { + return; + } + + if (m_error) { + ret.emplace_back([self = shared_from_this()] { + self->m_callback->error(self->m_error); + RealmCoordinator::unregister_query(*self); + }); + return; + } + + if (!m_tv_handover) { + return; + } + if (m_tv_handover->version < sg.get_version_of_current_transaction()) { + // async results are stale; ignore + return; + } +// auto r = Results(realm, +// m_sort, +// std::move(*sg.import_from_handover(std::move(m_tv_handover)))); + Results r; + auto version = sg.get_version_of_current_transaction(); + ret.emplace_back([r = std::move(r), version, &sg, self = shared_from_this()] { + if (sg.get_version_of_current_transaction() == version) { + self->m_callback->deliver(std::move(r)); + } + }); +} + +void AsyncQuery::prepare_update() +{ + // This function must not touch m_tv_handover as it is called without the + // relevant lock held (so that another thread can consume m_tv_handover + // while this is running) + + REALM_ASSERT(m_sg); + + if (m_tv.is_attached()) { + m_did_update = m_tv.sync_if_needed(); + } + else { + m_tv = m_query->find_all(); + if (m_sort) { + m_tv.sort(m_sort.columnIndices, m_sort.ascending); + } + m_did_update = true; + } +} + +void AsyncQuery::prepare_handover() +{ + // Even if the TV didn't change, we need to re-export it if the previous + // export has not been consumed yet, as the old handover object is no longer + // usable due to the version not matching + if (m_did_update || (m_tv_handover && m_tv_handover->version != m_sg->get_version_of_current_transaction())) { + m_tv_handover = m_sg->export_for_handover(m_tv, ConstSourcePayload::Copy); + } +} + +void AsyncQuery::set_error(std::exception_ptr err) +{ + if (!m_error) { + m_error = err; + } +} + +SharedGroup::VersionID AsyncQuery::version() const noexcept +{ + if (m_tv_handover) + return m_tv_handover->version; + if (m_query_handover) + return m_query_handover->version; + return SharedGroup::VersionID{}; +} + +void AsyncQuery::attach_to(realm::SharedGroup& sg) +{ + REALM_ASSERT(!m_sg); + + m_query = sg.import_from_handover(std::move(m_query_handover)); + m_sg = &sg; +} + +void AsyncQuery::detatch() +{ + REALM_ASSERT(m_sg); + + m_query_handover = m_sg->export_for_handover(*m_query, MutableSourcePayload::Move); + m_sg = nullptr; +} diff --git a/src/impl/async_query.hpp b/src/impl/async_query.hpp new file mode 100644 index 00000000..1671e646 --- /dev/null +++ b/src/impl/async_query.hpp @@ -0,0 +1,75 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_ASYNC_QUERY_HPP +#define REALM_ASYNC_QUERY_HPP + +#include "results.hpp" + +#include + +#include + +namespace realm { +namespace _impl { +class AsyncQuery : public std::enable_shared_from_this { +public: + AsyncQuery(SortOrder sort, + std::unique_ptr> handover, + std::unique_ptr callback, + RealmCoordinator& parent); + + void get_results(const SharedRealm& realm, SharedGroup& sg, std::vector>& ret); + + void set_error(std::exception_ptr err); + + // Run/rerun the query if needed + void prepare_update(); + // Update the handover object with the new data produced in prepare_update() + void prepare_handover(); + + // Get the version of the current handover object + SharedGroup::VersionID version() const noexcept; + + void attach_to(SharedGroup& sg); + void detatch(); + + std::shared_ptr parent; + +private: + const SortOrder m_sort; + + std::unique_ptr> m_query_handover; + std::unique_ptr m_query; + + std::unique_ptr> m_tv_handover; + TableView m_tv; + + const std::unique_ptr m_callback; + + SharedGroup* m_sg = nullptr; + + std::exception_ptr m_error; + + bool m_did_update = false; +}; + +} // namespace _impl +} // namespace realm + +#endif /* REALM_ASYNC_QUERY_HPP */ diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index d59cad08..8c0dbc71 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -18,11 +18,20 @@ #include "impl/realm_coordinator.hpp" +#include "impl/async_query.hpp" #include "impl/cached_realm.hpp" #include "impl/external_commit_helper.hpp" +#include "impl/transact_log_handler.hpp" #include "object_store.hpp" #include "schema.hpp" +#include +#include +#include +#include +#include + +#include #include using namespace realm; @@ -58,7 +67,12 @@ std::shared_ptr RealmCoordinator::get_realm(Realm::Config config) if ((!m_config.read_only && !m_notifier) || (m_config.read_only && m_cached_realms.empty())) { m_config = config; if (!config.read_only && !m_notifier) { - m_notifier = std::make_unique(*this); + try { + m_notifier = std::make_unique(*this); + } + catch (std::system_error const& ex) { + throw RealmFileException(RealmFileException::Kind::AccessError, config.path, ex.code().message()); + } } } else { @@ -103,6 +117,11 @@ std::shared_ptr RealmCoordinator::get_realm(Realm::Config config) return realm; } +std::shared_ptr RealmCoordinator::get_realm() +{ + return get_realm(m_config); +} + const Schema* RealmCoordinator::get_schema() const noexcept { return m_cached_realms.empty() ? nullptr : m_config.schema.get(); @@ -186,10 +205,279 @@ void RealmCoordinator::send_commit_notifications() m_notifier->notify_others(); } +void RealmCoordinator::pin_version(uint_fast64_t version, uint_fast32_t index) +{ + if (m_async_error) { + return; + } + + SharedGroup::VersionID versionid(version, index); + if (!m_advancer_sg) { + try { + // Use a temporary Realm instance to open the shared group to reuse + // the error handling there + Realm tmp(m_config); + m_advancer_history = std::move(tmp.m_history); + m_advancer_sg = std::move(tmp.m_shared_group); + m_advancer_sg->begin_read(versionid); + } + catch (...) { + m_async_error = std::current_exception(); + m_advancer_sg = nullptr; + m_advancer_history = nullptr; + } + } + else if (m_new_queries.empty()) { + // If this is the first query then we don't already have a read transaction + m_advancer_sg->begin_read(versionid); + } + else if (versionid < m_advancer_sg->get_version_of_current_transaction()) { + // Ensure we're holding a readlock on the oldest version we have a + // handover object for, as handover objects don't + m_advancer_sg->end_read(); + m_advancer_sg->begin_read(versionid); + } +} + +AsyncQueryCancelationToken RealmCoordinator::register_query(const Results& r, std::unique_ptr target) +{ + return r.get_realm()->m_coordinator->do_register_query(r, std::move(target)); +} + +AsyncQueryCancelationToken RealmCoordinator::do_register_query(const Results& r, std::unique_ptr target) +{ + if (m_config.read_only) { + throw InvalidTransactionException("Cannot create asynchronous query for read-only Realms"); + } + if (r.get_realm()->is_in_transaction()) { + throw InvalidTransactionException("Cannot create asynchronous query while in a write transaction"); + } + + auto handover = r.get_realm()->m_shared_group->export_for_handover(r.get_query(), ConstSourcePayload::Copy); + auto version = handover->version; + auto query = std::make_shared(r.get_sort(), + std::move(handover), + std::move(target), + *this); + + { + std::lock_guard lock(m_query_mutex); + pin_version(version.version, version.index); + m_new_queries.push_back(query); + } + + // Wake up the background worker threads by pretending we made a commit + m_notifier->notify_others(); + return query; +} + +void RealmCoordinator::unregister_query(AsyncQuery& registration) +{ + registration.parent->do_unregister_query(registration); +} + +void RealmCoordinator::do_unregister_query(AsyncQuery& registration) +{ + auto swap_remove = [&](auto& container) { + auto it = std::find_if(container.begin(), container.end(), + [&](auto const& ptr) { return ptr.get() == ®istration; }); + if (it != container.end()) { + std::iter_swap(--container.end(), it); + container.pop_back(); + return true; + } + return false; + }; + + std::lock_guard lock(m_query_mutex); + if (swap_remove(m_queries)) { + // Make sure we aren't holding on to read versions needlessly if there + // are no queries left, but don't close them entirely as opening shared + // groups is expensive + if (!m_running_queries && m_queries.empty() && m_query_sg) { + m_query_sg->end_read(); + } + } + else if (swap_remove(m_new_queries)) { + if (m_new_queries.empty() && m_advancer_sg) { + m_advancer_sg->end_read(); + } + } +} + void RealmCoordinator::on_change() { + run_async_queries(); + std::lock_guard lock(m_realm_mutex); for (auto& realm : m_cached_realms) { realm.notify(); } } + +void RealmCoordinator::run_async_queries() +{ + std::unique_lock lock(m_query_mutex); + + if (m_queries.empty() && m_new_queries.empty()) { + return; + } + + if (!m_async_error) { + open_helper_shared_group(); + } + + if (m_async_error) { + move_new_queries_to_main(); + for (auto& query : m_queries) { + query->set_error(m_async_error); + } + return; + } + + advance_helper_shared_group_to_latest(); + + // Tell other threads not to close the shared group as we need it even + // though we aren't holding the lock + m_running_queries = true; + + // Make a copy of the queries vector so that we can release the lock while + // we run the queries + auto queries_to_run = m_queries; + lock.unlock(); + + for (auto& query : queries_to_run) { + query->prepare_update(); + } + + // Reacquire the lock while updating the fields that are actually read on + // other threads + lock.lock(); + for (auto& query : queries_to_run) { + query->prepare_handover(); + } + + // Check if all queries were removed while we were running them, as if so + // the shared group didn't get closed by do_unregister_query() + m_running_queries = false; + if (m_queries.empty()) { + m_query_sg->end_read(); + } +} + +void RealmCoordinator::open_helper_shared_group() +{ + if (!m_query_sg) { + try { + Realm tmp(m_config); + m_query_history = std::move(tmp.m_history); + m_query_sg = std::move(tmp.m_shared_group); + m_query_sg->begin_read(); + } + catch (...) { + // Store the error to be passed to the async queries + m_async_error = std::current_exception(); + m_query_sg = nullptr; + m_query_history = nullptr; + } + } + else if (m_queries.empty()) { + m_query_sg->begin_read(); + } +} + +void RealmCoordinator::move_new_queries_to_main() +{ + m_queries.reserve(m_queries.size() + m_new_queries.size()); + std::move(m_new_queries.begin(), m_new_queries.end(), std::back_inserter(m_queries)); + m_new_queries.clear(); +} + +void RealmCoordinator::advance_helper_shared_group_to_latest() +{ + if (m_new_queries.empty()) { + LangBindHelper::advance_read(*m_query_sg, *m_query_history); + return; + } + + // Sort newly added queries by their source version so that we can pull them + // all forward to the latest version in a single pass over the transaction log + std::sort(m_new_queries.begin(), m_new_queries.end(), [](auto const& lft, auto const& rgt) { + return lft->version() < rgt->version(); + }); + + // Import all newly added queries to our helper SG + for (auto& query : m_new_queries) { + LangBindHelper::advance_read(*m_advancer_sg, *m_advancer_history, query->version()); + query->attach_to(*m_advancer_sg); + } + + // Advance both SGs to the newest version + LangBindHelper::advance_read(*m_advancer_sg, *m_advancer_history); + LangBindHelper::advance_read(*m_query_sg, *m_query_history, + m_advancer_sg->get_version_of_current_transaction()); + + // Transfer all new queries over to the main SG + for (auto& query : m_new_queries) { + query->detatch(); + query->attach_to(*m_query_sg); + } + + if (!m_new_queries.empty()) { + move_new_queries_to_main(); + m_advancer_sg->end_read(); + } +} + +void RealmCoordinator::advance_to_ready(Realm& realm) +{ + std::vector> async_results; + + { + std::lock_guard lock(m_query_mutex); + + SharedGroup::VersionID version; + for (auto& query : m_queries) { + version = query->version(); + if (version != SharedGroup::VersionID()) { + break; + } + } + + // no untargeted async queries; just advance to latest + if (version.version == 0) { + transaction::advance(*realm.m_shared_group, *realm.m_history, realm.m_binding_context.get()); + return; + } + // async results are out of date; ignore + else if (version < realm.m_shared_group->get_version_of_current_transaction()) { + return; + } + + transaction::advance(*realm.m_shared_group, *realm.m_history, realm.m_binding_context.get(), version); + + for (auto& query : m_queries) { + query->get_results(realm.shared_from_this(), *realm.m_shared_group, async_results); + } + } + + for (auto& results : async_results) { + results(); + } +} + +void RealmCoordinator::process_available_async(Realm& realm) +{ + std::vector> async_results; + + { + std::lock_guard lock(m_query_mutex); + for (auto& query : m_queries) { + query->get_results(realm.shared_from_this(), *realm.m_shared_group, async_results); + } + } + + for (auto& results : async_results) { + results(); + } +} diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index ee1d748b..0c8b120b 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -24,9 +24,15 @@ #include namespace realm { +class AsyncQueryCallback; +class ClientHistory; +class Results; class Schema; +class SharedGroup; +struct AsyncQueryCancelationToken; namespace _impl { +class AsyncQuery; class CachedRealm; class ExternalCommitHelper; @@ -43,6 +49,7 @@ public: // If the Realm is already open on another thread, validates that the given // configuration is compatible with the existing one std::shared_ptr get_realm(Realm::Config config); + std::shared_ptr get_realm(); const Schema* get_schema() const noexcept; uint64_t get_schema_version() const noexcept { return m_config.schema_version; } @@ -50,7 +57,7 @@ public: const std::vector& get_encryption_key() const noexcept { return m_config.encryption_key; } bool is_in_memory() const noexcept { return m_config.in_memory; } - // Asyncronously call notify() on every Realm instance for this coordinator's + // Asynchronously call notify() on every Realm instance for this coordinator's // path, including those in other processes void send_commit_notifications(); @@ -73,13 +80,49 @@ public: // Update the schema in the cached config void update_schema(Schema const& new_schema); + static AsyncQueryCancelationToken register_query(const Results& r, std::unique_ptr); + static void unregister_query(AsyncQuery& registration); + + // Advance the Realm to the most recent transaction version which all async + // work is complete for + void advance_to_ready(Realm& realm); + void process_available_async(Realm& realm); + private: Realm::Config m_config; std::mutex m_realm_mutex; std::vector m_cached_realms; + std::mutex m_query_mutex; + bool m_running_queries = false; + std::vector> m_new_queries; + std::vector> m_queries; + + // SharedGroup used for actually running async queries + // Will have a read transaction iff m_queries is non-empty + std::unique_ptr m_query_history; + std::unique_ptr m_query_sg; + + // SharedGroup used to advance queries in m_new_queries to the main shared + // group's transaction version + // Will have a read transaction iff m_new_queries is non-empty + std::unique_ptr m_advancer_history; + std::unique_ptr m_advancer_sg; + std::exception_ptr m_async_error; + std::unique_ptr<_impl::ExternalCommitHelper> m_notifier; + + AsyncQueryCancelationToken do_register_query(const Results& r, std::unique_ptr); + void do_unregister_query(AsyncQuery& registration); + + // must be called with m_query_mutex locked + void pin_version(uint_fast64_t version, uint_fast32_t index); + + void run_async_queries(); + void open_helper_shared_group(); + void move_new_queries_to_main(); + void advance_helper_shared_group_to_latest(); }; } // namespace _impl diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index cd9a8f9f..44544d17 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -432,7 +432,8 @@ public: namespace realm { namespace _impl { namespace transaction { -void advance(SharedGroup& sg, ClientHistory& history, BindingContext* context) +void advance(SharedGroup& sg, ClientHistory& history, BindingContext* context, + SharedGroup::VersionID version) { TransactLogObserver(context, sg, [&](auto&&... args) { LangBindHelper::advance_read(sg, history, std::move(args)...); diff --git a/src/impl/transact_log_handler.hpp b/src/impl/transact_log_handler.hpp index f5d3d58a..b68845fb 100644 --- a/src/impl/transact_log_handler.hpp +++ b/src/impl/transact_log_handler.hpp @@ -19,6 +19,8 @@ #ifndef REALM_TRANSACT_LOG_HANDLER_HPP #define REALM_TRANSACT_LOG_HANDLER_HPP +#include + namespace realm { class BindingContext; class SharedGroup; @@ -28,7 +30,8 @@ namespace _impl { namespace transaction { // Advance the read transaction version, with change notifications sent to delegate // Must not be called from within a write transaction. -void advance(SharedGroup& sg, ClientHistory& history, BindingContext* binding_context); +void advance(SharedGroup& sg, ClientHistory& history, BindingContext* binding_context, + SharedGroup::VersionID version=SharedGroup::VersionID{}); // Begin a write transaction // If the read transaction version is not up to date, will first advance to the diff --git a/src/object_accessor.hpp b/src/object_accessor.hpp index bad38f9e..1e95638b 100644 --- a/src/object_accessor.hpp +++ b/src/object_accessor.hpp @@ -5,10 +5,13 @@ #ifndef REALM_OBJECT_ACCESSOR_HPP #define REALM_OBJECT_ACCESSOR_HPP -#include -#include "shared_realm.hpp" -#include "schema.hpp" #include "list.hpp" +#include "object_schema.hpp" +#include "object_store.hpp" +#include "schema.hpp" +#include "shared_realm.hpp" + +#include namespace realm { diff --git a/src/results.cpp b/src/results.cpp index dc029157..b3f59945 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -19,6 +19,7 @@ #include "results.hpp" #include "object_store.hpp" +#include "impl/realm_coordinator.hpp" #include @@ -60,6 +61,8 @@ void Results::validate_read() const m_realm->verify_thread(); if (m_table && !m_table->is_attached()) throw InvalidatedException(); + if (m_mode == Mode::TableView && !m_table_view.is_attached()) + throw InvalidatedException(); } void Results::validate_write() const @@ -94,6 +97,11 @@ size_t Results::size() REALM_UNREACHABLE(); } +StringData Results::get_object_type() const noexcept +{ + return get_object_schema().name; +} + RowExpr Results::get(size_t row_ndx) { validate_read(); @@ -301,8 +309,9 @@ Query Results::get_query() const switch (m_mode) { case Mode::Empty: case Mode::Query: - case Mode::TableView: return m_query; + case Mode::TableView: + return m_table_view.get_query(); case Mode::Table: return m_table->where(); } @@ -342,3 +351,31 @@ Results::UnsupportedColumnTypeException::UnsupportedColumnTypeException(size_t c , column_type(table->get_column_type(column)) { } + +AsyncQueryCancelationToken Results::async(std::unique_ptr target) +{ + return _impl::RealmCoordinator::register_query(*this, std::move(target)); +} + +AsyncQueryCancelationToken::~AsyncQueryCancelationToken() +{ + if (m_registration) { + _impl::RealmCoordinator::unregister_query(*m_registration); + } +} + +AsyncQueryCancelationToken::AsyncQueryCancelationToken(AsyncQueryCancelationToken&& rgt) +: m_registration(std::move(rgt.m_registration)) +{ +} + +AsyncQueryCancelationToken& AsyncQueryCancelationToken::operator=(realm::AsyncQueryCancelationToken&& rgt) +{ + if (this != &rgt) { + if (m_registration) { + _impl::RealmCoordinator::unregister_query(*m_registration); + } + m_registration = std::move(rgt.m_registration); + } + return *this; +} diff --git a/src/results.hpp b/src/results.hpp index 27127e8d..01e0368b 100644 --- a/src/results.hpp +++ b/src/results.hpp @@ -29,6 +29,46 @@ namespace realm { template class BasicRowExpr; using RowExpr = BasicRowExpr
; class Mixed; +class Results; +class ObjectSchema; + +namespace _impl { + class AsyncQuery; +} + +// A token which keeps an asynchronous query alive +struct AsyncQueryCancelationToken { + AsyncQueryCancelationToken() = default; + AsyncQueryCancelationToken(std::shared_ptr<_impl::AsyncQuery> registration) : m_registration(std::move(registration)) { } + ~AsyncQueryCancelationToken(); + + AsyncQueryCancelationToken(AsyncQueryCancelationToken&&); + AsyncQueryCancelationToken& operator=(AsyncQueryCancelationToken&&); + + AsyncQueryCancelationToken(AsyncQueryCancelationToken const&) = delete; + AsyncQueryCancelationToken& operator=(AsyncQueryCancelationToken const&) = delete; + +private: + std::shared_ptr<_impl::AsyncQuery> m_registration; +}; + +// Subclass to get notifications about async query things +class AsyncQueryCallback { +public: + virtual ~AsyncQueryCallback() = default; + + // Called with the Results object generated by the query on a thread where + // is_for_current_thread() returned true + virtual void deliver(Results) = 0; + + // If an error occured while running the query on the worker thread, this is + // called with an exception on a thread where is_for_current_thread() + // returned true + virtual void error(std::exception_ptr) = 0; + + // Return whether or not this query is associated with the current thread + virtual bool is_for_current_thread() { return true; } +}; struct SortOrder { std::vector columnIndices; @@ -72,7 +112,7 @@ public: TableView get_tableview(); // Get the object type which will be returned by get() - StringData get_object_type() const noexcept { return get_object_schema().name; } + StringData get_object_type() const noexcept; // Set whether the TableView should sync if needed before accessing results void set_live(bool live); @@ -165,6 +205,13 @@ public: UnsupportedColumnTypeException(size_t column, const Table* table); }; + void update_tableview(); + + // Create an async query from this Results + // The query will be run on a background thread and delivered to the callback, + // and then rerun after each commit (if needed) and redelivered if it changed + AsyncQueryCancelationToken async(std::unique_ptr target); + private: SharedRealm m_realm; const ObjectSchema *m_object_schema; @@ -179,8 +226,6 @@ private: void validate_read() const; void validate_write() const; - void update_tableview(); - template util::Optional aggregate(size_t column, bool return_none_for_empty, Int agg_int, Float agg_float, diff --git a/src/schema.hpp b/src/schema.hpp index 2df1ec9e..3f10129f 100644 --- a/src/schema.hpp +++ b/src/schema.hpp @@ -19,6 +19,8 @@ #ifndef REALM_SCHEMA_HPP #define REALM_SCHEMA_HPP +#include "property.hpp" + #include #include diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 69d48818..71e9538e 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -361,13 +361,16 @@ void Realm::notify() } if (m_auto_refresh) { if (m_group) { - transaction::advance(*m_shared_group, *m_history, m_binding_context.get()); + m_coordinator->advance_to_ready(*this); } else if (m_binding_context) { m_binding_context->did_change({}, {}); } } } + else { + m_coordinator->process_available_async(*this); + } } bool Realm::refresh() @@ -387,6 +390,7 @@ bool Realm::refresh() if (m_group) { transaction::advance(*m_shared_group, *m_history, m_binding_context.get()); + m_coordinator->process_available_async(*this); } else { // Create the read transaction diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index 79f55496..9d7fe302 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -19,9 +19,10 @@ #ifndef REALM_REALM_HPP #define REALM_REALM_HPP -#include "object_store.hpp" +#include #include +#include #include #include #include @@ -42,8 +43,7 @@ namespace realm { class RealmCoordinator; } - class Realm : public std::enable_shared_from_this - { + class Realm : public std::enable_shared_from_this { public: typedef std::function MigrationFunction; @@ -135,6 +135,8 @@ namespace realm { // FIXME private Group *read_group(); + + friend class _impl::RealmCoordinator; }; class RealmFileException : public std::runtime_error { From 8f668fdf09f91da7b3a8b5d05f607439d20783b7 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 8 Dec 2015 14:09:17 -0800 Subject: [PATCH 57/79] Use a single AsyncQuery per Results regardless of number of callbacks added --- src/impl/async_query.cpp | 193 ++++++++++++++++++++++++--------- src/impl/async_query.hpp | 74 ++++++++++--- src/impl/realm_coordinator.cpp | 95 ++++++---------- src/impl/realm_coordinator.hpp | 8 +- src/results.cpp | 56 ++++++++-- src/results.hpp | 37 +++---- src/shared_realm.hpp | 3 +- 7 files changed, 300 insertions(+), 166 deletions(-) diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index 5f131941..d3dd4787 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -19,59 +19,74 @@ #include "impl/async_query.hpp" #include "impl/realm_coordinator.hpp" +#include "results.hpp" using namespace realm; using namespace realm::_impl; -AsyncQuery::AsyncQuery(SortOrder sort, - std::unique_ptr> handover, - std::unique_ptr callback, - RealmCoordinator& parent) -: parent(parent.shared_from_this()) -, m_sort(std::move(sort)) -, m_query_handover(std::move(handover)) -, m_callback(std::move(callback)) +AsyncQuery::AsyncQuery(Results& target) +: m_target_results(&target) +, m_realm(target.get_realm()) +, m_sort(target.get_sort()) +, m_version(m_realm->m_shared_group->get_version_of_current_transaction()) { + Query q = target.get_query(); + m_query_handover = m_realm->m_shared_group->export_for_handover(q, MutableSourcePayload::Move); } -void AsyncQuery::get_results(const SharedRealm& realm, SharedGroup& sg, std::vector>& ret) +size_t AsyncQuery::add_callback(std::function callback) { - if (!m_callback->is_for_current_thread()) { - return; - } + std::lock_guard lock(m_callback_mutex); - if (m_error) { - ret.emplace_back([self = shared_from_this()] { - self->m_callback->error(self->m_error); - RealmCoordinator::unregister_query(*self); - }); - return; - } - - if (!m_tv_handover) { - return; - } - if (m_tv_handover->version < sg.get_version_of_current_transaction()) { - // async results are stale; ignore - return; - } -// auto r = Results(realm, -// m_sort, -// std::move(*sg.import_from_handover(std::move(m_tv_handover)))); - Results r; - auto version = sg.get_version_of_current_transaction(); - ret.emplace_back([r = std::move(r), version, &sg, self = shared_from_this()] { - if (sg.get_version_of_current_transaction() == version) { - self->m_callback->deliver(std::move(r)); + size_t token = 0; + for (auto& callback : m_callbacks) { + if (token <= callback.token) { + token = callback.token + 1; } - }); + } + return token; } -void AsyncQuery::prepare_update() +void AsyncQuery::remove_callback(size_t token) { - // This function must not touch m_tv_handover as it is called without the - // relevant lock held (so that another thread can consume m_tv_handover - // while this is running) + std::lock_guard lock(m_callback_mutex); + if (is_for_current_thread() && m_calling_callbacks) { + // Schedule the removal for after we're done calling callbacks + m_callbacks_to_remove.push_back(token); + return; + } + do_remove_callback(token); +} + +void AsyncQuery::do_remove_callback(size_t token) noexcept +{ + REALM_ASSERT(m_error || m_callbacks.size() > 0); + auto it = find_if(begin(m_callbacks), end(m_callbacks), + [=](const auto& c) { return c.token == token; }); + // We should only fail to find the callback if it was removed due to an error + REALM_ASSERT(m_error || it != end(m_callbacks)); + + if (it != end(m_callbacks)) { + if (it != prev(end(m_callbacks))) { + *it = std::move(m_callbacks.back()); + } + m_callbacks.pop_back(); + } +} + +void AsyncQuery::unregister() noexcept +{ + std::lock_guard lock(m_target_mutex); + RealmCoordinator::unregister_query(*this); + m_target_results = nullptr; + m_realm = nullptr; +} + +void AsyncQuery::run() +{ + // This function must not touch any members touched in deliver(), as they + // may be called concurrently (as it'd be pretty bad for a running query to + // block the main thread trying to pick up the previous results) REALM_ASSERT(m_sg); @@ -80,6 +95,7 @@ void AsyncQuery::prepare_update() } else { m_tv = m_query->find_all(); + m_query = nullptr; if (m_sort) { m_tv.sort(m_sort.columnIndices, m_sort.ascending); } @@ -89,33 +105,103 @@ void AsyncQuery::prepare_update() void AsyncQuery::prepare_handover() { + std::lock_guard lock(m_callback_mutex); + + REALM_ASSERT(m_tv.is_in_sync()); + + m_version = m_sg->get_version_of_current_transaction(); + m_initial_run_complete = true; + // Even if the TV didn't change, we need to re-export it if the previous // export has not been consumed yet, as the old handover object is no longer // usable due to the version not matching - if (m_did_update || (m_tv_handover && m_tv_handover->version != m_sg->get_version_of_current_transaction())) { + if (m_did_update || (m_tv_handover && m_tv_handover->version != m_version)) { m_tv_handover = m_sg->export_for_handover(m_tv, ConstSourcePayload::Copy); } } -void AsyncQuery::set_error(std::exception_ptr err) +void AsyncQuery::deliver(SharedGroup& sg, std::exception_ptr err) { - if (!m_error) { - m_error = err; + if (!is_for_current_thread()) { + return; } -} -SharedGroup::VersionID AsyncQuery::version() const noexcept -{ - if (m_tv_handover) - return m_tv_handover->version; - if (m_query_handover) - return m_query_handover->version; - return SharedGroup::VersionID{}; + std::lock_guard callback_lock(m_callback_mutex); + std::lock_guard target_lock(m_target_mutex); + + // Target results being null here indicates that it was destroyed while we + // were in the process of advancing the Realm version and preparing for + // delivery, i.e. it was destroyed from the "wrong" thread + if (!m_target_results) { + return; + } + + // We can get called before the query has actually had the chance to run if + // we're added immediately before a different set of async results are + // delivered + if (!m_initial_run_complete && !err) { + return; + } + + // Tell remove_callback() to defer actually removing them, so that calling it + // in the callback block works + m_calling_callbacks = true; + + if (err) { + m_error = true; + for (auto& callback : m_callbacks) { + callback.fn(err); + } + + // Remove all the callbacks as we never need to call anything ever again + // after delivering an error + m_callbacks.clear(); + m_callbacks_to_remove.clear(); + m_calling_callbacks = false; + return; + } + + REALM_ASSERT(!m_query_handover); + + auto realm_version = m_realm->m_shared_group->get_version_of_current_transaction(); + if (m_version != realm_version) { + // Realm version can be newer if a commit was made on our thread or the + // user manually called refresh(), or older if a commit was made on a + // different thread and we ran *really* fast in between the check for + // if the shared group has changed and when we pick up async results + return; + } + + // Cannot use m_did_update here as it is used unlocked in run() + bool did_update = false; + if (m_tv_handover) { + Results::AsyncFriend::set_table_view(*m_target_results, + std::move(*sg.import_from_handover(std::move(m_tv_handover)))); + + did_update = true; + } + REALM_ASSERT(!m_tv_handover); + + for (auto& callback : m_callbacks) { + if (did_update || callback.first_run) { + callback.fn(nullptr); + callback.first_run = false; + } + } + + m_calling_callbacks = false; + + // Actually remove any callbacks whose removal was requested during iteration + for (auto token : m_callbacks_to_remove) { + do_remove_callback(token); + } + m_callbacks_to_remove.clear(); } void AsyncQuery::attach_to(realm::SharedGroup& sg) { REALM_ASSERT(!m_sg); + REALM_ASSERT(m_query_handover); m_query = sg.import_from_handover(std::move(m_query_handover)); m_sg = &sg; @@ -124,7 +210,10 @@ void AsyncQuery::attach_to(realm::SharedGroup& sg) void AsyncQuery::detatch() { REALM_ASSERT(m_sg); + REALM_ASSERT(m_query); + REALM_ASSERT(!m_tv.is_attached()); m_query_handover = m_sg->export_for_handover(*m_query, MutableSourcePayload::Move); m_sg = nullptr; + m_query = nullptr; } diff --git a/src/impl/async_query.hpp b/src/impl/async_query.hpp index 1671e646..04f350a9 100644 --- a/src/impl/async_query.hpp +++ b/src/impl/async_query.hpp @@ -23,50 +23,88 @@ #include +#include +#include #include +#include +#include namespace realm { namespace _impl { -class AsyncQuery : public std::enable_shared_from_this { +class AsyncQuery { public: - AsyncQuery(SortOrder sort, - std::unique_ptr> handover, - std::unique_ptr callback, - RealmCoordinator& parent); + AsyncQuery(Results& target); - void get_results(const SharedRealm& realm, SharedGroup& sg, std::vector>& ret); + size_t add_callback(std::function); + void remove_callback(size_t token); - void set_error(std::exception_ptr err); + void unregister() noexcept; // Run/rerun the query if needed - void prepare_update(); - // Update the handover object with the new data produced in prepare_update() + void run(); + // Prepare the handover object if run() did update the TableView void prepare_handover(); + // Update the target results from the handover and call callbacks + void deliver(SharedGroup& sg, std::exception_ptr err); - // Get the version of the current handover object - SharedGroup::VersionID version() const noexcept; - + // Attach the handed-over query to `sg` void attach_to(SharedGroup& sg); + // Create a new query handover object and stop using the previously attached + // SharedGroup void detatch(); - std::shared_ptr parent; + Realm& get_realm() { return *m_target_results->get_realm(); } + // Get the version of the current handover object + SharedGroup::VersionID version() const noexcept { return m_version; } private: - const SortOrder m_sort; + // Target Results to update and a mutex which guards it + std::mutex m_target_mutex; + Results* m_target_results; + std::shared_ptr m_realm; + const SortOrder m_sort; + const std::thread::id m_thread_id = std::this_thread::get_id(); + + // The source Query, in handover from iff m_sg is null + // Only used until the first time the query is actually run, after which + // both will be null std::unique_ptr> m_query_handover; std::unique_ptr m_query; - std::unique_ptr> m_tv_handover; + // The TableView resulting from running the query. Will be detached if the + // Query has not yet been run, in which case m_query or m_query_handover will + // be non-null TableView m_tv; + std::unique_ptr> m_tv_handover; + SharedGroup::VersionID m_version; - const std::unique_ptr m_callback; + struct Callback { + std::function fn; + std::unique_ptr> handover; + size_t token; + bool first_run; + }; + + // Currently registered callbacks and a mutex which must always be held + // while doing anything with them + std::mutex m_callback_mutex; + std::vector m_callbacks; + + // Callbacks which the user has asked to have removed whose removal has been + // deferred until after we're done looping over m_callbacks + std::vector m_callbacks_to_remove; SharedGroup* m_sg = nullptr; - std::exception_ptr m_error; - bool m_did_update = false; + bool m_initial_run_complete = false; + bool m_calling_callbacks = false; + bool m_error = false; + + void do_remove_callback(size_t token) noexcept; + + bool is_for_current_thread() const { return m_thread_id == std::this_thread::get_id(); } }; } // namespace _impl diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 8c0dbc71..1644465f 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -239,48 +239,25 @@ void RealmCoordinator::pin_version(uint_fast64_t version, uint_fast32_t index) } } -AsyncQueryCancelationToken RealmCoordinator::register_query(const Results& r, std::unique_ptr target) +void RealmCoordinator::register_query(std::shared_ptr query) { - return r.get_realm()->m_coordinator->do_register_query(r, std::move(target)); -} - -AsyncQueryCancelationToken RealmCoordinator::do_register_query(const Results& r, std::unique_ptr target) -{ - if (m_config.read_only) { - throw InvalidTransactionException("Cannot create asynchronous query for read-only Realms"); - } - if (r.get_realm()->is_in_transaction()) { - throw InvalidTransactionException("Cannot create asynchronous query while in a write transaction"); - } - - auto handover = r.get_realm()->m_shared_group->export_for_handover(r.get_query(), ConstSourcePayload::Copy); - auto version = handover->version; - auto query = std::make_shared(r.get_sort(), - std::move(handover), - std::move(target), - *this); - + auto version = query->version(); + auto& self = *query->get_realm().m_coordinator; { - std::lock_guard lock(m_query_mutex); - pin_version(version.version, version.index); - m_new_queries.push_back(query); + std::lock_guard lock(self.m_query_mutex); + self.pin_version(version.version, version.index); + self.m_new_queries.push_back(std::move(query)); } // Wake up the background worker threads by pretending we made a commit - m_notifier->notify_others(); - return query; + self.m_notifier->notify_others(); } -void RealmCoordinator::unregister_query(AsyncQuery& registration) -{ - registration.parent->do_unregister_query(registration); -} - -void RealmCoordinator::do_unregister_query(AsyncQuery& registration) +void RealmCoordinator::unregister_query(AsyncQuery& query) { auto swap_remove = [&](auto& container) { auto it = std::find_if(container.begin(), container.end(), - [&](auto const& ptr) { return ptr.get() == ®istration; }); + [&](auto const& ptr) { return ptr.get() == &query; }); if (it != container.end()) { std::iter_swap(--container.end(), it); container.pop_back(); @@ -289,18 +266,19 @@ void RealmCoordinator::do_unregister_query(AsyncQuery& registration) return false; }; - std::lock_guard lock(m_query_mutex); - if (swap_remove(m_queries)) { + auto& self = *query.get_realm().m_coordinator; + std::lock_guard lock(self.m_query_mutex); + if (swap_remove(self.m_queries)) { // Make sure we aren't holding on to read versions needlessly if there // are no queries left, but don't close them entirely as opening shared // groups is expensive - if (!m_running_queries && m_queries.empty() && m_query_sg) { - m_query_sg->end_read(); + if (!self.m_running_queries && self.m_queries.empty() && self.m_query_sg) { + self.m_query_sg->end_read(); } } - else if (swap_remove(m_new_queries)) { - if (m_new_queries.empty() && m_advancer_sg) { - m_advancer_sg->end_read(); + else if (swap_remove(self.m_new_queries)) { + if (self.m_new_queries.empty() && self.m_advancer_sg) { + self.m_advancer_sg->end_read(); } } } @@ -329,9 +307,6 @@ void RealmCoordinator::run_async_queries() if (m_async_error) { move_new_queries_to_main(); - for (auto& query : m_queries) { - query->set_error(m_async_error); - } return; } @@ -347,14 +322,18 @@ void RealmCoordinator::run_async_queries() lock.unlock(); for (auto& query : queries_to_run) { - query->prepare_update(); + query->run(); } // Reacquire the lock while updating the fields that are actually read on // other threads - lock.lock(); - for (auto& query : queries_to_run) { - query->prepare_handover(); + { + // Make sure we don't change the version while another thread is delivering + std::lock_guard version_lock(m_query_version_mutex); + lock.lock(); + for (auto& query : queries_to_run) { + query->prepare_handover(); + } } // Check if all queries were removed while we were running them, as if so @@ -431,8 +410,9 @@ void RealmCoordinator::advance_helper_shared_group_to_latest() void RealmCoordinator::advance_to_ready(Realm& realm) { - std::vector> async_results; + decltype(m_queries) queries; + std::lock_guard lock(m_query_version_mutex); { std::lock_guard lock(m_query_mutex); @@ -455,29 +435,24 @@ void RealmCoordinator::advance_to_ready(Realm& realm) } transaction::advance(*realm.m_shared_group, *realm.m_history, realm.m_binding_context.get(), version); - - for (auto& query : m_queries) { - query->get_results(realm.shared_from_this(), *realm.m_shared_group, async_results); - } + queries = m_queries; } - for (auto& results : async_results) { - results(); + for (auto& query : queries) { + query->deliver(*realm.m_shared_group, m_async_error); } } void RealmCoordinator::process_available_async(Realm& realm) { - std::vector> async_results; - + decltype(m_queries) queries; { std::lock_guard lock(m_query_mutex); - for (auto& query : m_queries) { - query->get_results(realm.shared_from_this(), *realm.m_shared_group, async_results); - } + queries = m_queries; } - for (auto& results : async_results) { - results(); + std::lock_guard lock(m_query_version_mutex); + for (auto& query : queries) { + query->deliver(*realm.m_shared_group, m_async_error); } } diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index 0c8b120b..2a35cd01 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -80,8 +80,8 @@ public: // Update the schema in the cached config void update_schema(Schema const& new_schema); - static AsyncQueryCancelationToken register_query(const Results& r, std::unique_ptr); - static void unregister_query(AsyncQuery& registration); + static void register_query(std::shared_ptr query); + static void unregister_query(AsyncQuery& query); // Advance the Realm to the most recent transaction version which all async // work is complete for @@ -95,6 +95,7 @@ private: std::vector m_cached_realms; std::mutex m_query_mutex; + std::mutex m_query_version_mutex; bool m_running_queries = false; std::vector> m_new_queries; std::vector> m_queries; @@ -113,9 +114,6 @@ private: std::unique_ptr<_impl::ExternalCommitHelper> m_notifier; - AsyncQueryCancelationToken do_register_query(const Results& r, std::unique_ptr); - void do_unregister_query(AsyncQuery& registration); - // must be called with m_query_mutex locked void pin_version(uint_fast64_t version, uint_fast32_t index); diff --git a/src/results.cpp b/src/results.cpp index b3f59945..f19afe1d 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -18,8 +18,9 @@ #include "results.hpp" -#include "object_store.hpp" +#include "impl/async_query.hpp" #include "impl/realm_coordinator.hpp" +#include "object_store.hpp" #include @@ -55,6 +56,13 @@ Results::Results(SharedRealm r, const ObjectSchema &o, Table& table) { } +Results::~Results() +{ + if (m_background_query) { + m_background_query->unregister(); + } +} + void Results::validate_read() const { if (m_realm) @@ -344,6 +352,29 @@ Results Results::filter(Query&& q) const return Results(m_realm, get_object_schema(), get_query().and_query(std::move(q)), get_sort()); } +AsyncQueryCancelationToken Results::async(std::function target) +{ + if (m_realm->config().read_only) { + throw InvalidTransactionException("Cannot create asynchronous query for read-only Realms"); + } + if (m_realm->is_in_transaction()) { + throw InvalidTransactionException("Cannot create asynchronous query while in a write transaction"); + } + + if (!m_background_query) { + m_background_query = std::make_shared<_impl::AsyncQuery>(*this); + _impl::RealmCoordinator::register_query(m_background_query); + } + return {m_background_query, m_background_query->add_callback(std::move(target))}; +} + +void Results::AsyncFriend::set_table_view(Results& results, realm::TableView &&tv) +{ + results.m_table_view = std::move(tv); + results.m_mode = Mode::TableView; + REALM_ASSERT(results.m_table_view.is_in_sync()); +} + Results::UnsupportedColumnTypeException::UnsupportedColumnTypeException(size_t column, const Table* table) : std::runtime_error((std::string)"Operation not supported on '" + table->get_column_name(column).data() + "' columns") , column_index(column) @@ -352,30 +383,37 @@ Results::UnsupportedColumnTypeException::UnsupportedColumnTypeException(size_t c { } -AsyncQueryCancelationToken Results::async(std::unique_ptr target) +AsyncQueryCancelationToken::AsyncQueryCancelationToken(std::shared_ptr<_impl::AsyncQuery> query, size_t token) +: m_query(std::move(query)), m_token(token) { - return _impl::RealmCoordinator::register_query(*this, std::move(target)); } AsyncQueryCancelationToken::~AsyncQueryCancelationToken() { - if (m_registration) { - _impl::RealmCoordinator::unregister_query(*m_registration); + // m_query itself (and not just the pointed-to thing) needs to be accessed + // atomically to ensure that there are no data races when the token is + // destroyed after being modified on a different thread. + // This is needed despite the token not being thread-safe in general as + // users find it very surpringing for obj-c objects to care about what + // thread they are deallocated on. + if (auto query = std::atomic_load(&m_query)) { + query->remove_callback(m_token); } } AsyncQueryCancelationToken::AsyncQueryCancelationToken(AsyncQueryCancelationToken&& rgt) -: m_registration(std::move(rgt.m_registration)) +: m_query(std::atomic_exchange(&rgt.m_query, {})), m_token(rgt.m_token) { } AsyncQueryCancelationToken& AsyncQueryCancelationToken::operator=(realm::AsyncQueryCancelationToken&& rgt) { if (this != &rgt) { - if (m_registration) { - _impl::RealmCoordinator::unregister_query(*m_registration); + if (auto query = std::atomic_load(&m_query)) { + query->remove_callback(m_token); } - m_registration = std::move(rgt.m_registration); + std::atomic_store(&m_query, std::atomic_exchange(&rgt.m_query, {})); + m_token = rgt.m_token; } return *this; } diff --git a/src/results.hpp b/src/results.hpp index 01e0368b..8e0a6ba1 100644 --- a/src/results.hpp +++ b/src/results.hpp @@ -39,7 +39,7 @@ namespace _impl { // A token which keeps an asynchronous query alive struct AsyncQueryCancelationToken { AsyncQueryCancelationToken() = default; - AsyncQueryCancelationToken(std::shared_ptr<_impl::AsyncQuery> registration) : m_registration(std::move(registration)) { } + AsyncQueryCancelationToken(std::shared_ptr<_impl::AsyncQuery> query, size_t token); ~AsyncQueryCancelationToken(); AsyncQueryCancelationToken(AsyncQueryCancelationToken&&); @@ -49,25 +49,8 @@ struct AsyncQueryCancelationToken { AsyncQueryCancelationToken& operator=(AsyncQueryCancelationToken const&) = delete; private: - std::shared_ptr<_impl::AsyncQuery> m_registration; -}; - -// Subclass to get notifications about async query things -class AsyncQueryCallback { -public: - virtual ~AsyncQueryCallback() = default; - - // Called with the Results object generated by the query on a thread where - // is_for_current_thread() returned true - virtual void deliver(Results) = 0; - - // If an error occured while running the query on the worker thread, this is - // called with an exception on a thread where is_for_current_thread() - // returned true - virtual void error(std::exception_ptr) = 0; - - // Return whether or not this query is associated with the current thread - virtual bool is_for_current_thread() { return true; } + std::shared_ptr<_impl::AsyncQuery> m_query; + size_t m_token; }; struct SortOrder { @@ -88,6 +71,7 @@ public: Results() = default; Results(SharedRealm r, const ObjectSchema& o, Table& table); Results(SharedRealm r, const ObjectSchema& o, Query q, SortOrder s = {}); + ~Results(); // Results is copyable and moveable Results(Results const&) = default; @@ -210,7 +194,14 @@ public: // Create an async query from this Results // The query will be run on a background thread and delivered to the callback, // and then rerun after each commit (if needed) and redelivered if it changed - AsyncQueryCancelationToken async(std::unique_ptr target); + AsyncQueryCancelationToken async(std::function target); + + // Helper type to let AsyncQuery update the tableview without giving access + // to any other privates or letting anyone else do so + class AsyncFriend { + friend class _impl::AsyncQuery; + static void set_table_view(Results& results, TableView&& tv); + }; private: SharedRealm m_realm; @@ -221,6 +212,8 @@ private: SortOrder m_sort; bool m_live = true; + std::shared_ptr<_impl::AsyncQuery> m_background_query; + Mode m_mode = Mode::Empty; void validate_read() const; @@ -230,6 +223,8 @@ private: util::Optional aggregate(size_t column, bool return_none_for_empty, Int agg_int, Float agg_float, Double agg_double, DateTime agg_datetime); + + void set_table_view(TableView&& tv); }; } diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index 9d7fe302..fdf064b8 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -39,7 +39,7 @@ namespace realm { typedef std::weak_ptr WeakRealm; namespace _impl { - class ExternalCommitHelper; + class AsyncQuery; class RealmCoordinator; } @@ -136,6 +136,7 @@ namespace realm { // FIXME private Group *read_group(); + friend class _impl::AsyncQuery; friend class _impl::RealmCoordinator; }; From 934263f76a1816caad01f6ae89e72837894e95f7 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 9 Dec 2015 18:05:48 -0800 Subject: [PATCH 58/79] Automatically create the async query when converting to a TableView --- src/results.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/results.cpp b/src/results.cpp index f19afe1d..fe9b40ed 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -177,9 +177,14 @@ void Results::update_tableview() m_mode = Mode::TableView; break; case Mode::TableView: - if (m_live) { - m_table_view.sync_if_needed(); + if (!m_live) { + return; } + if (!m_realm->config().read_only && !m_realm->is_in_transaction() && !m_background_query) { + m_background_query = std::make_shared<_impl::AsyncQuery>(*this); + _impl::RealmCoordinator::register_query(m_background_query); + } + m_table_view.sync_if_needed(); break; } } From 13e10545533ca7071fc4c0c747bf4bdc328c1f3f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 10 Dec 2015 10:50:54 -0800 Subject: [PATCH 59/79] Don't continue to run queries in the background if the result is unused --- src/impl/async_query.cpp | 21 +++++++++++++++++---- src/impl/async_query.hpp | 1 + src/results.cpp | 8 ++++++++ src/results.hpp | 4 ++++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index d3dd4787..097ef13b 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -84,12 +84,21 @@ void AsyncQuery::unregister() noexcept void AsyncQuery::run() { + REALM_ASSERT(m_sg); + + { + std::lock_guard callback_lock(m_callback_mutex); + std::lock_guard target_lock(m_target_mutex); + if (!m_target_results || (m_callbacks.empty() && !m_target_results->wants_background_updates())) { + m_skipped_running = true; + return; + } + } + m_skipped_running = false; + // This function must not touch any members touched in deliver(), as they // may be called concurrently (as it'd be pretty bad for a running query to // block the main thread trying to pick up the previous results) - - REALM_ASSERT(m_sg); - if (m_tv.is_attached()) { m_did_update = m_tv.sync_if_needed(); } @@ -105,10 +114,14 @@ void AsyncQuery::run() void AsyncQuery::prepare_handover() { - std::lock_guard lock(m_callback_mutex); + if (m_skipped_running) { + return; + } + REALM_ASSERT(m_tv.is_attached()); REALM_ASSERT(m_tv.is_in_sync()); + std::lock_guard lock(m_callback_mutex); m_version = m_sg->get_version_of_current_transaction(); m_initial_run_complete = true; diff --git a/src/impl/async_query.hpp b/src/impl/async_query.hpp index 04f350a9..4c3ec0a4 100644 --- a/src/impl/async_query.hpp +++ b/src/impl/async_query.hpp @@ -98,6 +98,7 @@ private: SharedGroup* m_sg = nullptr; bool m_did_update = false; + bool m_skipped_running = false; bool m_initial_run_complete = false; bool m_calling_callbacks = false; bool m_error = false; diff --git a/src/results.cpp b/src/results.cpp index fe9b40ed..a46e83d5 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -184,6 +184,7 @@ void Results::update_tableview() m_background_query = std::make_shared<_impl::AsyncQuery>(*this); _impl::RealmCoordinator::register_query(m_background_query); } + m_has_used_table_view = true; m_table_view.sync_if_needed(); break; } @@ -375,8 +376,15 @@ AsyncQueryCancelationToken Results::async(std::function target); + bool wants_background_updates() const { return m_wants_background_updates; } + // Helper type to let AsyncQuery update the tableview without giving access // to any other privates or letting anyone else do so class AsyncFriend { @@ -215,6 +217,8 @@ private: std::shared_ptr<_impl::AsyncQuery> m_background_query; Mode m_mode = Mode::Empty; + bool m_has_used_table_view = false; + bool m_wants_background_updates = true; void validate_read() const; void validate_write() const; From 8c4f2a4f309639e475403d99ee42a15bc452112d Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 10 Dec 2015 14:29:13 -0800 Subject: [PATCH 60/79] Reduce the scope of class friendships --- src/impl/async_query.cpp | 6 +++--- src/impl/realm_coordinator.cpp | 32 +++++++++++++++++--------------- src/shared_realm.cpp | 29 ++++++++++++++++++++--------- src/shared_realm.hpp | 18 +++++++++++++++--- 4 files changed, 55 insertions(+), 30 deletions(-) diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index 097ef13b..75831434 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -28,10 +28,10 @@ AsyncQuery::AsyncQuery(Results& target) : m_target_results(&target) , m_realm(target.get_realm()) , m_sort(target.get_sort()) -, m_version(m_realm->m_shared_group->get_version_of_current_transaction()) +, m_version(Realm::Internal::get_shared_group(*m_realm).get_version_of_current_transaction()) { Query q = target.get_query(); - m_query_handover = m_realm->m_shared_group->export_for_handover(q, MutableSourcePayload::Move); + m_query_handover = Realm::Internal::get_shared_group(*m_realm).export_for_handover(q, MutableSourcePayload::Move); } size_t AsyncQuery::add_callback(std::function callback) @@ -176,7 +176,7 @@ void AsyncQuery::deliver(SharedGroup& sg, std::exception_ptr err) REALM_ASSERT(!m_query_handover); - auto realm_version = m_realm->m_shared_group->get_version_of_current_transaction(); + auto realm_version = Realm::Internal::get_shared_group(*m_realm).get_version_of_current_transaction(); if (m_version != realm_version) { // Realm version can be newer if a commit was made on our thread or the // user manually called refresh(), or older if a commit was made on a diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 1644465f..29abf8ee 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -214,11 +214,9 @@ void RealmCoordinator::pin_version(uint_fast64_t version, uint_fast32_t index) SharedGroup::VersionID versionid(version, index); if (!m_advancer_sg) { try { - // Use a temporary Realm instance to open the shared group to reuse - // the error handling there - Realm tmp(m_config); - m_advancer_history = std::move(tmp.m_history); - m_advancer_sg = std::move(tmp.m_shared_group); + std::unique_ptr read_only_group; + Realm::open_with_config(m_config, m_advancer_history, m_advancer_sg, read_only_group); + REALM_ASSERT(!read_only_group); m_advancer_sg->begin_read(versionid); } catch (...) { @@ -242,7 +240,7 @@ void RealmCoordinator::pin_version(uint_fast64_t version, uint_fast32_t index) void RealmCoordinator::register_query(std::shared_ptr query) { auto version = query->version(); - auto& self = *query->get_realm().m_coordinator; + auto& self = Realm::Internal::get_coordinator(query->get_realm()); { std::lock_guard lock(self.m_query_mutex); self.pin_version(version.version, version.index); @@ -266,7 +264,7 @@ void RealmCoordinator::unregister_query(AsyncQuery& query) return false; }; - auto& self = *query.get_realm().m_coordinator; + auto& self = Realm::Internal::get_coordinator(query.get_realm()); std::lock_guard lock(self.m_query_mutex); if (swap_remove(self.m_queries)) { // Make sure we aren't holding on to read versions needlessly if there @@ -348,9 +346,9 @@ void RealmCoordinator::open_helper_shared_group() { if (!m_query_sg) { try { - Realm tmp(m_config); - m_query_history = std::move(tmp.m_history); - m_query_sg = std::move(tmp.m_shared_group); + std::unique_ptr read_only_group; + Realm::open_with_config(m_config, m_query_history, m_query_sg, read_only_group); + REALM_ASSERT(!read_only_group); m_query_sg->begin_read(); } catch (...) { @@ -412,6 +410,9 @@ void RealmCoordinator::advance_to_ready(Realm& realm) { decltype(m_queries) queries; + auto& sg = Realm::Internal::get_shared_group(realm); + auto& history = Realm::Internal::get_history(realm); + std::lock_guard lock(m_query_version_mutex); { std::lock_guard lock(m_query_mutex); @@ -426,20 +427,20 @@ void RealmCoordinator::advance_to_ready(Realm& realm) // no untargeted async queries; just advance to latest if (version.version == 0) { - transaction::advance(*realm.m_shared_group, *realm.m_history, realm.m_binding_context.get()); + transaction::advance(sg, history, realm.m_binding_context.get()); return; } // async results are out of date; ignore - else if (version < realm.m_shared_group->get_version_of_current_transaction()) { + else if (version < sg.get_version_of_current_transaction()) { return; } - transaction::advance(*realm.m_shared_group, *realm.m_history, realm.m_binding_context.get(), version); + transaction::advance(sg, history, realm.m_binding_context.get(), version); queries = m_queries; } for (auto& query : queries) { - query->deliver(*realm.m_shared_group, m_async_error); + query->deliver(sg, m_async_error); } } @@ -451,8 +452,9 @@ void RealmCoordinator::process_available_async(Realm& realm) queries = m_queries; } + auto& sg = Realm::Internal::get_shared_group(realm); std::lock_guard lock(m_query_version_mutex); for (auto& query : queries) { - query->deliver(*realm.m_shared_group, m_async_error); + query->deliver(sg, m_async_error); } } diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 71e9538e..ce435ab7 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -62,23 +62,34 @@ Realm::Config& Realm::Config::operator=(realm::Realm::Config const& c) Realm::Realm(Config config) : m_config(std::move(config)) +{ + open_with_config(m_config, m_history, m_shared_group, m_read_only_group); + + if (m_read_only_group) { + m_group = m_read_only_group.get(); + } +} + +void Realm::open_with_config(const Config& config, + std::unique_ptr& history, + std::unique_ptr& shared_group, + std::unique_ptr& read_only_group) { try { - if (m_config.read_only) { - m_read_only_group = std::make_unique(m_config.path, m_config.encryption_key.data(), Group::mode_ReadOnly); - m_group = m_read_only_group.get(); + if (config.read_only) { + read_only_group = std::make_unique(config.path, config.encryption_key.data(), Group::mode_ReadOnly); } else { - m_history = realm::make_client_history(m_config.path, m_config.encryption_key.data()); - SharedGroup::DurabilityLevel durability = m_config.in_memory ? SharedGroup::durability_MemOnly : + history = realm::make_client_history(config.path, config.encryption_key.data()); + SharedGroup::DurabilityLevel durability = config.in_memory ? SharedGroup::durability_MemOnly : SharedGroup::durability_Full; - m_shared_group = std::make_unique(*m_history, durability, m_config.encryption_key.data(), !m_config.disable_format_upgrade); + shared_group = std::make_unique(*history, durability, config.encryption_key.data(), !config.disable_format_upgrade); } } catch (util::File::PermissionDenied const& ex) { throw RealmFileException(RealmFileException::Kind::PermissionDenied, ex.get_path(), "Unable to open a realm at path '" + ex.get_path() + - "'. Please use a path where your app has " + (m_config.read_only ? "read" : "read-write") + " permissions."); + "'. Please use a path where your app has " + (config.read_only ? "read" : "read-write") + " permissions."); } catch (util::File::Exists const& ex) { throw RealmFileException(RealmFileException::Kind::Exists, ex.get_path(), @@ -93,12 +104,12 @@ Realm::Realm(Config config) "Unable to open a realm at path '" + ex.get_path() + "'"); } catch (IncompatibleLockFile const& ex) { - throw RealmFileException(RealmFileException::Kind::IncompatibleLockFile, m_config.path, + throw RealmFileException(RealmFileException::Kind::IncompatibleLockFile, config.path, "Realm file is currently open in another process " "which cannot share access with this process. All processes sharing a single file must be the same architecture."); } catch (FileFormatUpgradeRequired const& ex) { - throw RealmFileException(RealmFileException::Kind::FormatUpgradeRequired, m_config.path, + throw RealmFileException(RealmFileException::Kind::FormatUpgradeRequired, config.path, "The Realm file format must be allowed to be upgraded " "in order to proceed."); } diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index fdf064b8..422af4bf 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -116,6 +116,21 @@ namespace realm { void init(std::shared_ptr<_impl::RealmCoordinator> coordinator); Realm(Config config); + // Give AsyncQuery direct access to the shared group as it needs it to + // call the handover functions + class Internal { + friend class _impl::AsyncQuery; + friend class _impl::RealmCoordinator; + static SharedGroup& get_shared_group(Realm& realm) { return *realm.m_shared_group; } + static ClientHistory& get_history(Realm& realm) { return *realm.m_history; } + static _impl::RealmCoordinator& get_coordinator(Realm& realm) { return *realm.m_coordinator; } + }; + + static void open_with_config(const Config& config, + std::unique_ptr& history, + std::unique_ptr& shared_group, + std::unique_ptr& read_only_group); + private: Config m_config; std::thread::id m_thread_id = std::this_thread::get_id(); @@ -135,9 +150,6 @@ namespace realm { // FIXME private Group *read_group(); - - friend class _impl::AsyncQuery; - friend class _impl::RealmCoordinator; }; class RealmFileException : public std::runtime_error { From 3e90c3057118bb83aefd47494c832f383defc132 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 10 Dec 2015 14:00:02 -0800 Subject: [PATCH 61/79] Add more tests and fix bugs --- src/impl/async_query.cpp | 194 +++++++++++++++++++-------------- src/impl/async_query.hpp | 42 ++++--- src/impl/realm_coordinator.cpp | 75 ++++++------- src/impl/realm_coordinator.hpp | 3 +- 4 files changed, 181 insertions(+), 133 deletions(-) diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index 75831434..7d0e8641 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -28,68 +28,100 @@ AsyncQuery::AsyncQuery(Results& target) : m_target_results(&target) , m_realm(target.get_realm()) , m_sort(target.get_sort()) -, m_version(Realm::Internal::get_shared_group(*m_realm).get_version_of_current_transaction()) +, m_sg_version(Realm::Internal::get_shared_group(*m_realm).get_version_of_current_transaction()) { Query q = target.get_query(); m_query_handover = Realm::Internal::get_shared_group(*m_realm).export_for_handover(q, MutableSourcePayload::Move); } +AsyncQuery::~AsyncQuery() +{ + std::lock_guard lock(m_target_mutex); + m_realm = nullptr; +} + size_t AsyncQuery::add_callback(std::function callback) { - std::lock_guard lock(m_callback_mutex); + m_realm->verify_thread(); - size_t token = 0; - for (auto& callback : m_callbacks) { - if (token <= callback.token) { - token = callback.token + 1; + auto next_token = [=] { + size_t token = 0; + for (auto& callback : m_callbacks) { + if (token <= callback.token) { + token = callback.token + 1; + } } + return token; + }; + + std::lock_guard lock(m_callback_mutex); + auto token = next_token(); + m_callbacks.push_back({std::move(callback), token, -1ULL}); + if (m_callback_index == npos) { // Don't need to wake up if we're already sending notifications + Realm::Internal::get_coordinator(*m_realm).send_commit_notifications(); + m_have_callbacks = true; } return token; } void AsyncQuery::remove_callback(size_t token) { - std::lock_guard lock(m_callback_mutex); - if (is_for_current_thread() && m_calling_callbacks) { - // Schedule the removal for after we're done calling callbacks - m_callbacks_to_remove.push_back(token); - return; - } - do_remove_callback(token); -} + Callback old; + { + std::lock_guard lock(m_callback_mutex); + REALM_ASSERT(m_error || m_callbacks.size() > 0); -void AsyncQuery::do_remove_callback(size_t token) noexcept -{ - REALM_ASSERT(m_error || m_callbacks.size() > 0); - auto it = find_if(begin(m_callbacks), end(m_callbacks), - [=](const auto& c) { return c.token == token; }); - // We should only fail to find the callback if it was removed due to an error - REALM_ASSERT(m_error || it != end(m_callbacks)); - - if (it != end(m_callbacks)) { - if (it != prev(end(m_callbacks))) { - *it = std::move(m_callbacks.back()); + auto it = find_if(begin(m_callbacks), end(m_callbacks), + [=](const auto& c) { return c.token == token; }); + // We should only fail to find the callback if it was removed due to an error + REALM_ASSERT(m_error || it != end(m_callbacks)); + if (it == end(m_callbacks)) { + return; } - m_callbacks.pop_back(); + + size_t idx = distance(begin(m_callbacks), it); + if (m_callback_index != npos && m_callback_index >= idx) { + --m_callback_index; + } + + old = std::move(*it); + m_callbacks.erase(it); + + m_have_callbacks = !m_callbacks.empty(); } } void AsyncQuery::unregister() noexcept { std::lock_guard lock(m_target_mutex); - RealmCoordinator::unregister_query(*this); m_target_results = nullptr; m_realm = nullptr; } +void AsyncQuery::release_query() noexcept +{ + { + std::lock_guard lock(m_target_mutex); + REALM_ASSERT(!m_realm && !m_target_results); + } + + m_query = nullptr; +} + +bool AsyncQuery::is_alive() const noexcept +{ + std::lock_guard lock(m_target_mutex); + return m_target_results != nullptr; +} + void AsyncQuery::run() { REALM_ASSERT(m_sg); { - std::lock_guard callback_lock(m_callback_mutex); std::lock_guard target_lock(m_target_mutex); - if (!m_target_results || (m_callbacks.empty() && !m_target_results->wants_background_updates())) { + // Don't run the query if the results aren't actually going to be used + if (!m_target_results || (!m_have_callbacks && !m_target_results->wants_background_updates())) { m_skipped_running = true; return; } @@ -100,7 +132,7 @@ void AsyncQuery::run() // may be called concurrently (as it'd be pretty bad for a running query to // block the main thread trying to pick up the previous results) if (m_tv.is_attached()) { - m_did_update = m_tv.sync_if_needed(); + m_tv.sync_if_needed(); } else { m_tv = m_query->find_all(); @@ -108,107 +140,111 @@ void AsyncQuery::run() if (m_sort) { m_tv.sort(m_sort.columnIndices, m_sort.ascending); } - m_did_update = true; } } void AsyncQuery::prepare_handover() { if (m_skipped_running) { + m_sg_version = SharedGroup::VersionID{}; return; } REALM_ASSERT(m_tv.is_attached()); REALM_ASSERT(m_tv.is_in_sync()); - std::lock_guard lock(m_callback_mutex); - m_version = m_sg->get_version_of_current_transaction(); + m_sg_version = m_sg->get_version_of_current_transaction(); m_initial_run_complete = true; - // Even if the TV didn't change, we need to re-export it if the previous - // export has not been consumed yet, as the old handover object is no longer - // usable due to the version not matching - if (m_did_update || (m_tv_handover && m_tv_handover->version != m_version)) { - m_tv_handover = m_sg->export_for_handover(m_tv, ConstSourcePayload::Copy); - } -} - -void AsyncQuery::deliver(SharedGroup& sg, std::exception_ptr err) -{ - if (!is_for_current_thread()) { + auto table_version = m_tv.outside_version(); + if (!m_tv_handover && table_version == m_handed_over_table_version) { + // We've already delivered the query results since the last time the + // table changed, so no need to do anything return; } - std::lock_guard callback_lock(m_callback_mutex); + m_tv_handover = m_sg->export_for_handover(m_tv, ConstSourcePayload::Copy); + m_handed_over_table_version = table_version; +} + +bool AsyncQuery::deliver(SharedGroup& sg, std::exception_ptr err) +{ + if (!is_for_current_thread()) { + return false; + } + std::lock_guard target_lock(m_target_mutex); // Target results being null here indicates that it was destroyed while we // were in the process of advancing the Realm version and preparing for // delivery, i.e. it was destroyed from the "wrong" thread if (!m_target_results) { - return; + return false; } // We can get called before the query has actually had the chance to run if // we're added immediately before a different set of async results are // delivered if (!m_initial_run_complete && !err) { - return; + return false; } - // Tell remove_callback() to defer actually removing them, so that calling it - // in the callback block works - m_calling_callbacks = true; - if (err) { - m_error = true; - for (auto& callback : m_callbacks) { - callback.fn(err); - } - - // Remove all the callbacks as we never need to call anything ever again - // after delivering an error - m_callbacks.clear(); - m_callbacks_to_remove.clear(); - m_calling_callbacks = false; - return; + m_error = err; + return m_have_callbacks; } REALM_ASSERT(!m_query_handover); - auto realm_version = Realm::Internal::get_shared_group(*m_realm).get_version_of_current_transaction(); - if (m_version != realm_version) { + auto realm_sg_version = Realm::Internal::get_shared_group(*m_realm).get_version_of_current_transaction(); + if (m_sg_version != realm_sg_version) { // Realm version can be newer if a commit was made on our thread or the // user manually called refresh(), or older if a commit was made on a // different thread and we ran *really* fast in between the check for // if the shared group has changed and when we pick up async results - return; + return false; } - // Cannot use m_did_update here as it is used unlocked in run() - bool did_update = false; if (m_tv_handover) { + m_tv_handover->version = m_sg_version; Results::AsyncFriend::set_table_view(*m_target_results, std::move(*sg.import_from_handover(std::move(m_tv_handover)))); + m_delievered_table_version = m_handed_over_table_version; - did_update = true; } REALM_ASSERT(!m_tv_handover); + return m_have_callbacks; +} - for (auto& callback : m_callbacks) { - if (did_update || callback.first_run) { - callback.fn(nullptr); - callback.first_run = false; +void AsyncQuery::call_callbacks() +{ + REALM_ASSERT(is_for_current_thread()); + + while (auto fn = next_callback()) { + fn(m_error); + } + + if (m_error) { + // Remove all the callbacks as we never need to call anything ever again + // after delivering an error + std::lock_guard callback_lock(m_callback_mutex); + m_callbacks.clear(); + } +} + +std::function AsyncQuery::next_callback() +{ + std::lock_guard callback_lock(m_callback_mutex); + for (++m_callback_index; m_callback_index < m_callbacks.size(); ++m_callback_index) { + auto& callback = m_callbacks[m_callback_index]; + if (m_error || callback.delivered_version != m_delievered_table_version) { + callback.delivered_version = m_delievered_table_version; + return callback.fn; } } - m_calling_callbacks = false; - - // Actually remove any callbacks whose removal was requested during iteration - for (auto token : m_callbacks_to_remove) { - do_remove_callback(token); - } - m_callbacks_to_remove.clear(); + m_callback_index = npos; + return nullptr; } void AsyncQuery::attach_to(realm::SharedGroup& sg) diff --git a/src/impl/async_query.hpp b/src/impl/async_query.hpp index 4c3ec0a4..fef16927 100644 --- a/src/impl/async_query.hpp +++ b/src/impl/async_query.hpp @@ -34,18 +34,22 @@ namespace _impl { class AsyncQuery { public: AsyncQuery(Results& target); + ~AsyncQuery(); size_t add_callback(std::function); void remove_callback(size_t token); void unregister() noexcept; + void release_query() noexcept; // Run/rerun the query if needed void run(); // Prepare the handover object if run() did update the TableView void prepare_handover(); - // Update the target results from the handover and call callbacks - void deliver(SharedGroup& sg, std::exception_ptr err); + // Update the target results from the handover + // Returns if any callbacks need to be invoked + bool deliver(SharedGroup& sg, std::exception_ptr err); + void call_callbacks(); // Attach the handed-over query to `sg` void attach_to(SharedGroup& sg); @@ -55,11 +59,13 @@ public: Realm& get_realm() { return *m_target_results->get_realm(); } // Get the version of the current handover object - SharedGroup::VersionID version() const noexcept { return m_version; } + SharedGroup::VersionID version() const noexcept { return m_sg_version; } + + bool is_alive() const noexcept; private: // Target Results to update and a mutex which guards it - std::mutex m_target_mutex; + mutable std::mutex m_target_mutex; Results* m_target_results; std::shared_ptr m_realm; @@ -77,13 +83,13 @@ private: // be non-null TableView m_tv; std::unique_ptr> m_tv_handover; - SharedGroup::VersionID m_version; + SharedGroup::VersionID m_sg_version; + std::exception_ptr m_error; struct Callback { std::function fn; - std::unique_ptr> handover; size_t token; - bool first_run; + uint_fast64_t delivered_version; }; // Currently registered callbacks and a mutex which must always be held @@ -91,21 +97,27 @@ private: std::mutex m_callback_mutex; std::vector m_callbacks; - // Callbacks which the user has asked to have removed whose removal has been - // deferred until after we're done looping over m_callbacks - std::vector m_callbacks_to_remove; - SharedGroup* m_sg = nullptr; - bool m_did_update = false; + uint_fast64_t m_handed_over_table_version = -1; + uint_fast64_t m_delievered_table_version = -1; + + // Iteration variable for looping over callbacks + // remove_callback() updates this when needed + size_t m_callback_index = npos; + bool m_skipped_running = false; bool m_initial_run_complete = false; - bool m_calling_callbacks = false; - bool m_error = false; - void do_remove_callback(size_t token) noexcept; + // Cached value for if m_callbacks is empty, needed to avoid deadlocks in + // run() due to lock-order inversion between m_callback_mutex and m_target_mutex + // It's okay if this value is stale as at worst it'll result in us doing + // some extra work. + std::atomic m_have_callbacks = {false}; bool is_for_current_thread() const { return m_thread_id == std::this_thread::get_id(); } + + std::function next_callback(); }; } // namespace _impl diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 29abf8ee..91259c6d 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -246,37 +246,40 @@ void RealmCoordinator::register_query(std::shared_ptr query) self.pin_version(version.version, version.index); self.m_new_queries.push_back(std::move(query)); } - - // Wake up the background worker threads by pretending we made a commit - self.m_notifier->notify_others(); } -void RealmCoordinator::unregister_query(AsyncQuery& query) +void RealmCoordinator::clean_up_dead_queries() { auto swap_remove = [&](auto& container) { - auto it = std::find_if(container.begin(), container.end(), - [&](auto const& ptr) { return ptr.get() == &query; }); - if (it != container.end()) { - std::iter_swap(--container.end(), it); + bool did_remove = false; + for (size_t i = 0; i < container.size(); ++i) { + if (container[i]->is_alive()) + continue; + + // Ensure the query is destroyed here even if there's lingering refs + // to the async query elsewhere + container[i]->release_query(); + + if (container.size() > i + 1) + container[i] = std::move(container.back()); container.pop_back(); - return true; + --i; + did_remove = true; } - return false; + return did_remove; }; - auto& self = Realm::Internal::get_coordinator(query.get_realm()); - std::lock_guard lock(self.m_query_mutex); - if (swap_remove(self.m_queries)) { + if (swap_remove(m_queries)) { // Make sure we aren't holding on to read versions needlessly if there // are no queries left, but don't close them entirely as opening shared // groups is expensive - if (!self.m_running_queries && self.m_queries.empty() && self.m_query_sg) { - self.m_query_sg->end_read(); + if (m_queries.empty() && m_query_sg) { + m_query_sg->end_read(); } } - else if (swap_remove(self.m_new_queries)) { - if (self.m_new_queries.empty() && self.m_advancer_sg) { - self.m_advancer_sg->end_read(); + if (swap_remove(m_new_queries)) { + if (m_new_queries.empty() && m_advancer_sg) { + m_advancer_sg->end_read(); } } } @@ -295,6 +298,8 @@ void RealmCoordinator::run_async_queries() { std::unique_lock lock(m_query_mutex); + clean_up_dead_queries(); + if (m_queries.empty() && m_new_queries.empty()) { return; } @@ -310,10 +315,6 @@ void RealmCoordinator::run_async_queries() advance_helper_shared_group_to_latest(); - // Tell other threads not to close the shared group as we need it even - // though we aren't holding the lock - m_running_queries = true; - // Make a copy of the queries vector so that we can release the lock while // we run the queries auto queries_to_run = m_queries; @@ -326,20 +327,13 @@ void RealmCoordinator::run_async_queries() // Reacquire the lock while updating the fields that are actually read on // other threads { - // Make sure we don't change the version while another thread is delivering - std::lock_guard version_lock(m_query_version_mutex); lock.lock(); for (auto& query : queries_to_run) { query->prepare_handover(); } } - // Check if all queries were removed while we were running them, as if so - // the shared group didn't get closed by do_unregister_query() - m_running_queries = false; - if (m_queries.empty()) { - m_query_sg->end_read(); - } + clean_up_dead_queries(); } void RealmCoordinator::open_helper_shared_group() @@ -413,7 +407,6 @@ void RealmCoordinator::advance_to_ready(Realm& realm) auto& sg = Realm::Internal::get_shared_group(realm); auto& history = Realm::Internal::get_history(realm); - std::lock_guard lock(m_query_version_mutex); { std::lock_guard lock(m_query_mutex); @@ -436,25 +429,33 @@ void RealmCoordinator::advance_to_ready(Realm& realm) } transaction::advance(sg, history, realm.m_binding_context.get(), version); - queries = m_queries; + + for (auto& query : m_queries) { + if (query->deliver(sg, m_async_error)) { + queries.push_back(query); + } + } } for (auto& query : queries) { - query->deliver(sg, m_async_error); + query->call_callbacks(); } } void RealmCoordinator::process_available_async(Realm& realm) { + auto& sg = Realm::Internal::get_shared_group(realm); decltype(m_queries) queries; { std::lock_guard lock(m_query_mutex); - queries = m_queries; + for (auto& query : m_queries) { + if (query->deliver(sg, m_async_error)) { + queries.push_back(query); + } + } } - auto& sg = Realm::Internal::get_shared_group(realm); - std::lock_guard lock(m_query_version_mutex); for (auto& query : queries) { - query->deliver(sg, m_async_error); + query->call_callbacks(); } } diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index 2a35cd01..3e153aff 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -81,7 +81,6 @@ public: void update_schema(Schema const& new_schema); static void register_query(std::shared_ptr query); - static void unregister_query(AsyncQuery& query); // Advance the Realm to the most recent transaction version which all async // work is complete for @@ -95,7 +94,6 @@ private: std::vector m_cached_realms; std::mutex m_query_mutex; - std::mutex m_query_version_mutex; bool m_running_queries = false; std::vector> m_new_queries; std::vector> m_queries; @@ -121,6 +119,7 @@ private: void open_helper_shared_group(); void move_new_queries_to_main(); void advance_helper_shared_group_to_latest(); + void clean_up_dead_queries(); }; } // namespace _impl From a95eb509153b1c9a1445717893de43c9cd807f55 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 16 Dec 2015 11:53:04 -0800 Subject: [PATCH 62/79] Don't create background queries for Results on threads without runloops --- src/binding_context.hpp | 4 ++++ src/results.cpp | 2 +- src/shared_realm.cpp | 13 +++++++++++++ src/shared_realm.hpp | 2 ++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/binding_context.hpp b/src/binding_context.hpp index 4aa6dd94..d57ce541 100644 --- a/src/binding_context.hpp +++ b/src/binding_context.hpp @@ -70,6 +70,10 @@ class BindingContext { public: virtual ~BindingContext() = default; + // If the user adds a notification handler to the Realm, will it ever + // actually be called? + virtual bool can_deliver_notifications() const noexcept { return true; } + // Called by the Realm when a write transaction is committed to the file by // a different Realm instance (possibly in a different process) virtual void changes_available() { } diff --git a/src/results.cpp b/src/results.cpp index a46e83d5..cba5e1f7 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -180,7 +180,7 @@ void Results::update_tableview() if (!m_live) { return; } - if (!m_realm->config().read_only && !m_realm->is_in_transaction() && !m_background_query) { + if (!m_background_query && !m_realm->is_in_transaction() && m_realm->can_deliver_notifications()) { m_background_query = std::make_shared<_impl::AsyncQuery>(*this); _impl::RealmCoordinator::register_query(m_background_query); } diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index ce435ab7..272a3367 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -411,6 +411,19 @@ bool Realm::refresh() return true; } +bool Realm::can_deliver_notifications() const noexcept +{ + if (m_config.read_only) { + return false; + } + + if (m_binding_context && !m_binding_context->can_deliver_notifications()) { + return false; + } + + return true; +} + uint64_t Realm::get_schema_version(const realm::Realm::Config &config) { auto coordinator = RealmCoordinator::get_existing_coordinator(config.path); diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index 422af4bf..34aa5b1f 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -107,6 +107,8 @@ namespace realm { void verify_thread() const; void verify_in_write() const; + bool can_deliver_notifications() const noexcept; + // Close this Realm and remove it from the cache. Continuing to use a // Realm after closing it will produce undefined behavior. void close(); From db7d6fa2a82c4106e8358dfd2a2831f4b71f6dc7 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 17 Dec 2015 11:54:58 -0800 Subject: [PATCH 63/79] Don't hold onto the tableview between runs of async queries Updating the table view in advance_read() can be very expensive, and the updated data is never actually used. --- src/impl/async_query.cpp | 49 +++++++++++++++------------------- src/impl/async_query.hpp | 1 - src/impl/realm_coordinator.hpp | 1 - 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index 7d0e8641..0c830119 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -118,53 +118,48 @@ void AsyncQuery::run() { REALM_ASSERT(m_sg); + // This function must not touch any members touched in deliver(), as they + // may be called concurrently (as it'd be pretty bad for a running query to + // block the main thread trying to pick up the previous results) { std::lock_guard target_lock(m_target_mutex); // Don't run the query if the results aren't actually going to be used if (!m_target_results || (!m_have_callbacks && !m_target_results->wants_background_updates())) { - m_skipped_running = true; return; } } - m_skipped_running = false; - // This function must not touch any members touched in deliver(), as they - // may be called concurrently (as it'd be pretty bad for a running query to - // block the main thread trying to pick up the previous results) - if (m_tv.is_attached()) { - m_tv.sync_if_needed(); - } - else { - m_tv = m_query->find_all(); - m_query = nullptr; - if (m_sort) { - m_tv.sort(m_sort.columnIndices, m_sort.ascending); + REALM_ASSERT(!m_tv.is_attached()); + + // If we've run previously, check if we need to rerun + if (m_initial_run_complete) { + // Make an empty tableview from the query to get the table version, since + // Query doesn't expose it + if (m_query->find_all(0, 0, 0).outside_version() == m_handed_over_table_version) { + return; } } + + m_tv = m_query->find_all(); + if (m_sort) { + m_tv.sort(m_sort.columnIndices, m_sort.ascending); + } } void AsyncQuery::prepare_handover() { - if (m_skipped_running) { - m_sg_version = SharedGroup::VersionID{}; + m_sg_version = m_sg->get_version_of_current_transaction(); + + if (!m_tv.is_attached()) { return; } - REALM_ASSERT(m_tv.is_attached()); REALM_ASSERT(m_tv.is_in_sync()); - m_sg_version = m_sg->get_version_of_current_transaction(); m_initial_run_complete = true; - - auto table_version = m_tv.outside_version(); - if (!m_tv_handover && table_version == m_handed_over_table_version) { - // We've already delivered the query results since the last time the - // table changed, so no need to do anything - return; - } - - m_tv_handover = m_sg->export_for_handover(m_tv, ConstSourcePayload::Copy); - m_handed_over_table_version = table_version; + m_handed_over_table_version = m_tv.outside_version(); + m_tv_handover = m_sg->export_for_handover(m_tv, MutableSourcePayload::Move); + m_tv = TableView(); } bool AsyncQuery::deliver(SharedGroup& sg, std::exception_ptr err) diff --git a/src/impl/async_query.hpp b/src/impl/async_query.hpp index fef16927..efc938c5 100644 --- a/src/impl/async_query.hpp +++ b/src/impl/async_query.hpp @@ -106,7 +106,6 @@ private: // remove_callback() updates this when needed size_t m_callback_index = npos; - bool m_skipped_running = false; bool m_initial_run_complete = false; // Cached value for if m_callbacks is empty, needed to avoid deadlocks in diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index 3e153aff..4403bc0b 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -94,7 +94,6 @@ private: std::vector m_cached_realms; std::mutex m_query_mutex; - bool m_running_queries = false; std::vector> m_new_queries; std::vector> m_queries; From c46a2a34de2dcc9c455044bd7c4d689edd65ee8e Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 20 Jan 2016 06:43:46 -0800 Subject: [PATCH 64/79] Fix typos and minor errors in comments --- src/impl/cached_realm_base.hpp | 2 +- src/results.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/impl/cached_realm_base.hpp b/src/impl/cached_realm_base.hpp index 57feceb5..c734d453 100644 --- a/src/impl/cached_realm_base.hpp +++ b/src/impl/cached_realm_base.hpp @@ -27,7 +27,7 @@ class Realm; namespace _impl { -// CachedRealm stores a weak reference to a Realm instance, along with all of +// CachedRealmBase stores a weak reference to a Realm instance, along with all of // the information about a Realm that needs to be accessed from other threads. // This is needed to avoid forming strong references to the Realm instances on // other threads, which can produce deadlocks when the last strong reference to diff --git a/src/results.cpp b/src/results.cpp index cba5e1f7..2916daf7 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -376,8 +376,8 @@ AsyncQueryCancelationToken Results::async(std::function Date: Wed, 27 Jan 2016 06:44:07 -0800 Subject: [PATCH 65/79] Rename AsyncFriend to Interal --- src/impl/async_query.cpp | 4 ++-- src/results.cpp | 2 +- src/results.hpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index 0c830119..3917d23e 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -202,8 +202,8 @@ bool AsyncQuery::deliver(SharedGroup& sg, std::exception_ptr err) if (m_tv_handover) { m_tv_handover->version = m_sg_version; - Results::AsyncFriend::set_table_view(*m_target_results, - std::move(*sg.import_from_handover(std::move(m_tv_handover)))); + Results::Internal::set_table_view(*m_target_results, + std::move(*sg.import_from_handover(std::move(m_tv_handover)))); m_delievered_table_version = m_handed_over_table_version; } diff --git a/src/results.cpp b/src/results.cpp index 2916daf7..c21442f6 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -374,7 +374,7 @@ AsyncQueryCancelationToken Results::async(std::functionadd_callback(std::move(target))}; } -void Results::AsyncFriend::set_table_view(Results& results, realm::TableView &&tv) +void Results::Internal::set_table_view(Results& results, realm::TableView &&tv) { // If the previous TableView was never actually used, then stop generating // new ones until the user actually uses the Results object again diff --git a/src/results.hpp b/src/results.hpp index 5e82bed1..578ebe16 100644 --- a/src/results.hpp +++ b/src/results.hpp @@ -200,7 +200,7 @@ public: // Helper type to let AsyncQuery update the tableview without giving access // to any other privates or letting anyone else do so - class AsyncFriend { + class Internal { friend class _impl::AsyncQuery; static void set_table_view(Results& results, TableView&& tv); }; From ad46e307a277de15311d8671d2accca596186349 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 28 Jan 2016 06:44:10 -0800 Subject: [PATCH 66/79] Update the comments for Realm::Internal --- src/shared_realm.hpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index 34aa5b1f..2360a225 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -118,13 +118,20 @@ namespace realm { void init(std::shared_ptr<_impl::RealmCoordinator> coordinator); Realm(Config config); - // Give AsyncQuery direct access to the shared group as it needs it to - // call the handover functions + // Expose some internal functionality to other parts of the ObjectStore + // without making it public to everyone class Internal { friend class _impl::AsyncQuery; friend class _impl::RealmCoordinator; + + // AsyncQuery needs access to the SharedGroup to be able to call the + // handover functions, which are not very wrappable static SharedGroup& get_shared_group(Realm& realm) { return *realm.m_shared_group; } static ClientHistory& get_history(Realm& realm) { return *realm.m_history; } + + // AsyncQuery needs to be able to access the owning coordinator to + // wake up the worker thread when a callback is added, and + // coordinators need to be able to get themselves from a Realm static _impl::RealmCoordinator& get_coordinator(Realm& realm) { return *realm.m_coordinator; } }; From aa67216574484c215a10082ac74f364d9d3fdf76 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Sun, 31 Jan 2016 06:44:20 -0800 Subject: [PATCH 67/79] Write a much better comment about thread stuff for AsyncQuery --- src/impl/async_query.cpp | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index 3917d23e..4418dcf6 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -36,6 +36,9 @@ AsyncQuery::AsyncQuery(Results& target) AsyncQuery::~AsyncQuery() { + // unregister() may have been called from a different thread than we're being + // destroyed on, so we need to synchronize access to the interesting fields + // modified there std::lock_guard lock(m_target_mutex); m_realm = nullptr; } @@ -114,13 +117,34 @@ bool AsyncQuery::is_alive() const noexcept return m_target_results != nullptr; } +// Most of the inter-thread synchronization for run(), prepare_handover(), +// attach_to(), detach(), release_query() and deliver() is done by +// RealmCoordinator external to this code, which has some potentially +// non-obvious results on which members are and are not safe to use without +// holding a lock. +// +// attach_to(), detach(), run(), prepare_handover(), and release_query() are +// all only ever called on a single thread. call_callbacks() and deliver() are +// called on the same thread. Calls to prepare_handover() and deliver() are +// guarded by a lock. +// +// In total, this means that the safe data flow is as follows: +// - prepare_handover(), attach_to(), detach() and release_query() can read +// members written by each other +// - deliver() can read members written to in prepare_handover(), deliver(), +// and call_callbacks() +// - call_callbacks() and read members written to in deliver() +// +// Separately from this data flow for the query results, all uses of +// m_target_results, m_callbacks, and m_callback_index must be done with the +// appropriate mutex held to avoid race conditions when the Results object is +// destroyed while the background work is running, and to allow removing +// callbacks from any thread. + void AsyncQuery::run() { REALM_ASSERT(m_sg); - // This function must not touch any members touched in deliver(), as they - // may be called concurrently (as it'd be pretty bad for a running query to - // block the main thread trying to pick up the previous results) { std::lock_guard target_lock(m_target_mutex); // Don't run the query if the results aren't actually going to be used From 4e18a99dfdc17f10306818d932b889bb3199b9ea Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 1 Feb 2016 06:44:27 -0800 Subject: [PATCH 68/79] Update some out-of-date comments --- src/impl/async_query.cpp | 3 +++ src/impl/async_query.hpp | 11 ++++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index 4418dcf6..efb03699 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -183,6 +183,9 @@ void AsyncQuery::prepare_handover() m_initial_run_complete = true; m_handed_over_table_version = m_tv.outside_version(); m_tv_handover = m_sg->export_for_handover(m_tv, MutableSourcePayload::Move); + + // detach the TableView as we won't need it again and keeping it around + // makes advance_read() much more expensive m_tv = TableView(); } diff --git a/src/impl/async_query.hpp b/src/impl/async_query.hpp index efc938c5..3f643d39 100644 --- a/src/impl/async_query.hpp +++ b/src/impl/async_query.hpp @@ -72,15 +72,12 @@ private: const SortOrder m_sort; const std::thread::id m_thread_id = std::this_thread::get_id(); - // The source Query, in handover from iff m_sg is null - // Only used until the first time the query is actually run, after which - // both will be null + // The source Query, in handover form iff m_sg is null std::unique_ptr> m_query_handover; std::unique_ptr m_query; - // The TableView resulting from running the query. Will be detached if the - // Query has not yet been run, in which case m_query or m_query_handover will - // be non-null + // The TableView resulting from running the query. Will be detached unless + // the query was (re)run since the last time the handover object was created TableView m_tv; std::unique_ptr> m_tv_handover; SharedGroup::VersionID m_sg_version; @@ -93,7 +90,7 @@ private: }; // Currently registered callbacks and a mutex which must always be held - // while doing anything with them + // while doing anything with them or m_callback_index std::mutex m_callback_mutex; std::vector m_callbacks; From 82843407291b6c1fe63d9dd080c1564ca8e5789a Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 12 Jan 2016 12:55:27 -0800 Subject: [PATCH 69/79] Remove redundant check in RealmCoordinator::advance_helper_shared_group_to_latest() --- src/impl/realm_coordinator.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 91259c6d..eaa2791a 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -394,10 +394,8 @@ void RealmCoordinator::advance_helper_shared_group_to_latest() query->attach_to(*m_query_sg); } - if (!m_new_queries.empty()) { - move_new_queries_to_main(); - m_advancer_sg->end_read(); - } + move_new_queries_to_main(); + m_advancer_sg->end_read(); } void RealmCoordinator::advance_to_ready(Realm& realm) From 1e35324d9765d03d921f7534d4e6a51a6cd2ecaf Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 15 Feb 2016 09:26:56 -0800 Subject: [PATCH 70/79] Avoid holding locks while calling transaction::advance() It can call user code due to change notifications, which leads to deadlocks if that code then tries to add async queries (and advancing is a potentially expensive operation, so doing it while holding a lock inhibits parallelism anyway). --- src/impl/realm_coordinator.cpp | 53 ++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index eaa2791a..7cd7a17a 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -405,34 +405,55 @@ void RealmCoordinator::advance_to_ready(Realm& realm) auto& sg = Realm::Internal::get_shared_group(realm); auto& history = Realm::Internal::get_history(realm); - { - std::lock_guard lock(m_query_mutex); - - SharedGroup::VersionID version; + auto get_query_version = [&] { for (auto& query : m_queries) { - version = query->version(); - if (version != SharedGroup::VersionID()) { - break; + auto version = query->version(); + if (version != SharedGroup::VersionID{}) { + return version; } } + return SharedGroup::VersionID{}; + }; - // no untargeted async queries; just advance to latest - if (version.version == 0) { - transaction::advance(sg, history, realm.m_binding_context.get()); - return; - } - // async results are out of date; ignore - else if (version < sg.get_version_of_current_transaction()) { - return; - } + SharedGroup::VersionID version; + { + std::lock_guard lock(m_query_mutex); + version = get_query_version(); + } + // no async queries; just advance to latest + if (version.version == 0) { + transaction::advance(sg, history, realm.m_binding_context.get()); + return; + } + + // async results are out of date; ignore + if (version < sg.get_version_of_current_transaction()) { + return; + } + + while (true) { + // Advance to the ready version without holding any locks because it + // may end up calling user code (in did_change() notifications) transaction::advance(sg, history, realm.m_binding_context.get(), version); + // Reacquire the lock and recheck the query version, as the queries may + // have advanced to a later version while we didn't hold the lock. If + // so, we need to release the lock and re-advance + std::lock_guard lock(m_query_mutex); + version = get_query_version(); + if (version.version == 0) + return; + if (version != sg.get_version_of_current_transaction()) + continue; + + // Query version now matches the SG version, so we can deliver them for (auto& query : m_queries) { if (query->deliver(sg, m_async_error)) { queries.push_back(query); } } + break; } for (auto& query : queries) { From 99037a7c72a6e467e63db64e0f1a98de5a1570f8 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 22 Feb 2016 18:19:26 -0800 Subject: [PATCH 71/79] Add an AtomicSharedPtr implementation for gcc 4.9 --- src/CMakeLists.txt | 3 +- src/impl/apple/cached_realm.cpp | 2 + src/results.cpp | 11 +-- src/results.hpp | 3 +- src/util/atomic_shared_ptr.hpp | 137 ++++++++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 9 deletions(-) create mode 100644 src/util/atomic_shared_ptr.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 76b8a328..d33a71fb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -25,7 +25,8 @@ set(HEADERS impl/external_commit_helper.hpp impl/transact_log_handler.hpp parser/parser.hpp - parser/query_builder.hpp) + parser/query_builder.hpp + util/atomic_shared_ptr.hpp) if(APPLE) list(APPEND SOURCES diff --git a/src/impl/apple/cached_realm.cpp b/src/impl/apple/cached_realm.cpp index 1468a8fb..33ade9a0 100644 --- a/src/impl/apple/cached_realm.cpp +++ b/src/impl/apple/cached_realm.cpp @@ -20,6 +20,8 @@ #include "shared_realm.hpp" +#include + using namespace realm; using namespace realm::_impl; diff --git a/src/results.cpp b/src/results.cpp index c21442f6..bfdd8e87 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -409,23 +409,20 @@ AsyncQueryCancelationToken::~AsyncQueryCancelationToken() // This is needed despite the token not being thread-safe in general as // users find it very surpringing for obj-c objects to care about what // thread they are deallocated on. - if (auto query = std::atomic_load(&m_query)) { + if (auto query = m_query.exchange({})) { query->remove_callback(m_token); } } -AsyncQueryCancelationToken::AsyncQueryCancelationToken(AsyncQueryCancelationToken&& rgt) -: m_query(std::atomic_exchange(&rgt.m_query, {})), m_token(rgt.m_token) -{ -} +AsyncQueryCancelationToken::AsyncQueryCancelationToken(AsyncQueryCancelationToken&& rgt) = default; AsyncQueryCancelationToken& AsyncQueryCancelationToken::operator=(realm::AsyncQueryCancelationToken&& rgt) { if (this != &rgt) { - if (auto query = std::atomic_load(&m_query)) { + if (auto query = m_query.exchange({})) { query->remove_callback(m_token); } - std::atomic_store(&m_query, std::atomic_exchange(&rgt.m_query, {})); + m_query = std::move(rgt.m_query); m_token = rgt.m_token; } return *this; diff --git a/src/results.hpp b/src/results.hpp index 578ebe16..6310bdd9 100644 --- a/src/results.hpp +++ b/src/results.hpp @@ -20,6 +20,7 @@ #define REALM_RESULTS_HPP #include "shared_realm.hpp" +#include "util/atomic_shared_ptr.hpp" #include #include @@ -49,7 +50,7 @@ struct AsyncQueryCancelationToken { AsyncQueryCancelationToken& operator=(AsyncQueryCancelationToken const&) = delete; private: - std::shared_ptr<_impl::AsyncQuery> m_query; + util::AtomicSharedPtr<_impl::AsyncQuery> m_query; size_t m_token; }; diff --git a/src/util/atomic_shared_ptr.hpp b/src/util/atomic_shared_ptr.hpp new file mode 100644 index 00000000..0ec541e7 --- /dev/null +++ b/src/util/atomic_shared_ptr.hpp @@ -0,0 +1,137 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2016 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_ATOMIC_SHARED_PTR_HPP +#define REALM_ATOMIC_SHARED_PTR_HPP + +#include +#include +#include + +namespace realm { +namespace _impl { + +// Check if std::atomic_load has an overload taking a std::shared_ptr, and set +// HasAtomicPtrOps to either true_type or false_type + +template struct make_void { typedef void type; }; +template using void_t = typename make_void::type; + +template> +struct HasAtomicPtrOps : std::false_type { }; + +template +struct HasAtomicPtrOps()))>> : std::true_type { }; +} // namespace _impl + +namespace util { +// A wrapper for std::shared_ptr that enables sharing a shared_ptr instance +// (and not just a thing *pointed to* by a shared_ptr) between threads. Is +// lock-free iff the underlying shared_ptr implementation supports atomic +// operations. Currently the only implemented operation other than copy/move +// construction/assignment is exchange(). +template>::value> +class AtomicSharedPtr; + +template +class AtomicSharedPtr { +public: + AtomicSharedPtr() = default; + AtomicSharedPtr(std::shared_ptr ptr) : m_ptr(std::move(ptr)) { } + + AtomicSharedPtr(AtomicSharedPtr const& ptr) : m_ptr(std::atomic_load(&ptr.m_ptr)) { } + AtomicSharedPtr(AtomicSharedPtr&& ptr) : m_ptr(std::atomic_exchange(&ptr.m_ptr, {})) { } + + AtomicSharedPtr& operator=(AtomicSharedPtr const& ptr) + { + if (&ptr != this) { + std::atomic_store(&m_ptr, std::atomic_load(&ptr.m_ptr)); + } + return *this; + } + + AtomicSharedPtr& operator=(AtomicSharedPtr&& ptr) + { + std::atomic_store(&m_ptr, std::atomic_exchange(&ptr.m_ptr, {})); + return *this; + } + + std::shared_ptr exchange(std::shared_ptr ptr) + { + return std::atomic_exchange(&m_ptr, std::move(ptr)); + } + +private: + std::shared_ptr m_ptr = nullptr; +}; + +template +class AtomicSharedPtr { +public: + AtomicSharedPtr() = default; + AtomicSharedPtr(std::shared_ptr ptr) : m_ptr(std::move(ptr)) { } + + AtomicSharedPtr(AtomicSharedPtr const& ptr) + { + std::lock_guard lock(ptr.m_mutex); + m_ptr = ptr.m_ptr; + } + AtomicSharedPtr(AtomicSharedPtr&& ptr) + { + std::lock_guard lock(ptr.m_mutex); + m_ptr = std::move(ptr.m_ptr); + } + + AtomicSharedPtr& operator=(AtomicSharedPtr const& ptr) + { + if (&ptr != this) { + // std::lock() ensures that these are locked in a consistent order + // to avoid deadlock + std::lock(m_mutex, ptr.m_mutex); + m_ptr = ptr.m_ptr; + m_mutex.unlock(); + ptr.m_mutex.unlock(); + } + return *this; + } + + AtomicSharedPtr& operator=(AtomicSharedPtr&& ptr) + { + std::lock(m_mutex, ptr.m_mutex); + m_ptr = std::move(ptr.m_ptr); + m_mutex.unlock(); + ptr.m_mutex.unlock(); + return *this; + } + + std::shared_ptr exchange(std::shared_ptr ptr) + { + std::lock_guard lock(m_mutex); + m_ptr.swap(ptr); + return ptr; + } + +private: + std::mutex m_mutex; + std::shared_ptr m_ptr = nullptr; +}; + +} +} + +#endif // REALM_ASYNC_QUERY_HPP From 143cc3b696f73e3ac9455fe2608b6aeff9965bee Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 23 Feb 2016 14:26:31 -0800 Subject: [PATCH 72/79] Fix compilation with GCC 4.9 4.9 does not correctly implement C++14 aggregate initialization. --- src/impl/apple/cached_realm.cpp | 4 ++-- src/parser/parser.hpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/impl/apple/cached_realm.cpp b/src/impl/apple/cached_realm.cpp index 33ade9a0..562d9a72 100644 --- a/src/impl/apple/cached_realm.cpp +++ b/src/impl/apple/cached_realm.cpp @@ -30,11 +30,11 @@ CachedRealm::CachedRealm(const std::shared_ptr& realm, bool cache) { struct RefCountedWeakPointer { std::weak_ptr realm; - std::atomic ref_count = {1}; + std::atomic ref_count; }; CFRunLoopSourceContext ctx{}; - ctx.info = new RefCountedWeakPointer{realm}; + ctx.info = new RefCountedWeakPointer{realm, {1}}; ctx.perform = [](void* info) { if (auto realm = static_cast(info)->realm.lock()) { realm->notify(); diff --git a/src/parser/parser.hpp b/src/parser/parser.hpp index 39834dde..0becab08 100644 --- a/src/parser/parser.hpp +++ b/src/parser/parser.hpp @@ -28,7 +28,7 @@ class Schema; namespace parser { struct Expression { - enum class Type { None, Number, String, KeyPath, Argument, True, False } type = Type::None; + enum class Type { None, Number, String, KeyPath, Argument, True, False } type; std::string s; }; @@ -60,7 +60,7 @@ struct Predicate struct Comparison { Operator op = Operator::None; - Expression expr[2]; + Expression expr[2] = {{Expression::Type::None, ""}, {Expression::Type::None, ""}}; }; struct Compound From cfc88b6fd54c2c70aefcfd9e39c124b5021fe627 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 23 Feb 2016 14:31:59 -0800 Subject: [PATCH 73/79] Rename CachedRealm to WeakRealmNotfier --- src/CMakeLists.txt | 10 +++---- ...ched_realm.cpp => weak_realm_notifier.cpp} | 18 ++++++------ ...ched_realm.hpp => weak_realm_notifier.hpp} | 16 +++++------ ...ched_realm.hpp => weak_realm_notifier.hpp} | 6 ++-- src/impl/realm_coordinator.cpp | 28 +++++++++---------- src/impl/realm_coordinator.hpp | 4 +-- ...ched_realm.hpp => weak_realm_notifier.hpp} | 10 +++---- ..._base.hpp => weak_realm_notifier_base.hpp} | 18 ++++++------ 8 files changed, 55 insertions(+), 55 deletions(-) rename src/impl/apple/{cached_realm.cpp => weak_realm_notifier.cpp} (84%) rename src/impl/apple/{cached_realm.hpp => weak_realm_notifier.hpp} (71%) rename src/impl/generic/{cached_realm.hpp => weak_realm_notifier.hpp} (85%) rename src/impl/{cached_realm.hpp => weak_realm_notifier.hpp} (79%) rename src/impl/{cached_realm_base.hpp => weak_realm_notifier_base.hpp} (75%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d33a71fb..e2e1bd2b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,8 +20,8 @@ set(HEADERS results.hpp schema.hpp shared_realm.hpp - impl/cached_realm.hpp - impl/cached_realm_base.hpp + impl/weak_realm_notifier.hpp + impl/weak_realm_notifier_base.hpp impl/external_commit_helper.hpp impl/transact_log_handler.hpp parser/parser.hpp @@ -30,17 +30,17 @@ set(HEADERS if(APPLE) list(APPEND SOURCES - impl/apple/cached_realm.cpp + impl/apple/weak_realm_notifier.cpp impl/apple/external_commit_helper.cpp) list(APPEND HEADERS - impl/apple/cached_realm.hpp + impl/apple/weak_realm_notifier.hpp impl/apple/external_commit_helper.hpp) find_library(CF_LIBRARY CoreFoundation) else() list(APPEND SOURCES impl/generic/external_commit_helper.cpp) list(APPEND HEADERS - impl/generic/cached_realm.hpp + impl/generic/weak_realm_notifier.hpp impl/generic/external_commit_helper.hpp) endif() diff --git a/src/impl/apple/cached_realm.cpp b/src/impl/apple/weak_realm_notifier.cpp similarity index 84% rename from src/impl/apple/cached_realm.cpp rename to src/impl/apple/weak_realm_notifier.cpp index 562d9a72..da403ad7 100644 --- a/src/impl/apple/cached_realm.cpp +++ b/src/impl/apple/weak_realm_notifier.cpp @@ -16,7 +16,7 @@ // //////////////////////////////////////////////////////////////////////////// -#include "impl/cached_realm.hpp" +#include "impl/weak_realm_notifier.hpp" #include "shared_realm.hpp" @@ -25,8 +25,8 @@ using namespace realm; using namespace realm::_impl; -CachedRealm::CachedRealm(const std::shared_ptr& realm, bool cache) -: CachedRealmBase(realm, cache) +WeakRealmNotifier::WeakRealmNotifier(const std::shared_ptr& realm, bool cache) +: WeakRealmNotifierBase(realm, cache) { struct RefCountedWeakPointer { std::weak_ptr realm; @@ -57,8 +57,8 @@ CachedRealm::CachedRealm(const std::shared_ptr& realm, bool cache) CFRunLoopAddSource(m_runloop, m_signal, kCFRunLoopDefaultMode); } -CachedRealm::CachedRealm(CachedRealm&& rgt) -: CachedRealmBase(std::move(rgt)) +WeakRealmNotifier::WeakRealmNotifier(WeakRealmNotifier&& rgt) +: WeakRealmNotifierBase(std::move(rgt)) , m_runloop(rgt.m_runloop) , m_signal(rgt.m_signal) { @@ -66,9 +66,9 @@ CachedRealm::CachedRealm(CachedRealm&& rgt) rgt.m_signal = nullptr; } -CachedRealm& CachedRealm::operator=(CachedRealm&& rgt) +WeakRealmNotifier& WeakRealmNotifier::operator=(WeakRealmNotifier&& rgt) { - CachedRealmBase::operator=(std::move(rgt)); + WeakRealmNotifierBase::operator=(std::move(rgt)); m_runloop = rgt.m_runloop; m_signal = rgt.m_signal; rgt.m_runloop = nullptr; @@ -77,7 +77,7 @@ CachedRealm& CachedRealm::operator=(CachedRealm&& rgt) return *this; } -CachedRealm::~CachedRealm() +WeakRealmNotifier::~WeakRealmNotifier() { if (m_signal) { CFRunLoopSourceInvalidate(m_signal); @@ -86,7 +86,7 @@ CachedRealm::~CachedRealm() } } -void CachedRealm::notify() +void WeakRealmNotifier::notify() { CFRunLoopSourceSignal(m_signal); // Signalling the source makes it run the next time the runloop gets diff --git a/src/impl/apple/cached_realm.hpp b/src/impl/apple/weak_realm_notifier.hpp similarity index 71% rename from src/impl/apple/cached_realm.hpp rename to src/impl/apple/weak_realm_notifier.hpp index f67c160c..0f94ed7e 100644 --- a/src/impl/apple/cached_realm.hpp +++ b/src/impl/apple/weak_realm_notifier.hpp @@ -16,7 +16,7 @@ // //////////////////////////////////////////////////////////////////////////// -#include "impl/cached_realm_base.hpp" +#include "impl/weak_realm_notifier_base.hpp" #include @@ -25,16 +25,16 @@ class Realm; namespace _impl { -class CachedRealm : public CachedRealmBase { +class WeakRealmNotifier : public WeakRealmNotifierBase { public: - CachedRealm(const std::shared_ptr& realm, bool cache); - ~CachedRealm(); + WeakRealmNotifier(const std::shared_ptr& realm, bool cache); + ~WeakRealmNotifier(); - CachedRealm(CachedRealm&&); - CachedRealm& operator=(CachedRealm&&); + WeakRealmNotifier(WeakRealmNotifier&&); + WeakRealmNotifier& operator=(WeakRealmNotifier&&); - CachedRealm(const CachedRealm&) = delete; - CachedRealm& operator=(const CachedRealm&) = delete; + WeakRealmNotifier(const WeakRealmNotifier&) = delete; + WeakRealmNotifier& operator=(const WeakRealmNotifier&) = delete; // Asynchronously call notify() on the Realm on the appropriate thread void notify(); diff --git a/src/impl/generic/cached_realm.hpp b/src/impl/generic/weak_realm_notifier.hpp similarity index 85% rename from src/impl/generic/cached_realm.hpp rename to src/impl/generic/weak_realm_notifier.hpp index 23a7be11..219551eb 100644 --- a/src/impl/generic/cached_realm.hpp +++ b/src/impl/generic/weak_realm_notifier.hpp @@ -16,16 +16,16 @@ // //////////////////////////////////////////////////////////////////////////// -#include "impl/cached_realm_base.hpp" +#include "impl/weak_realm_notifier_base.hpp" namespace realm { class Realm; namespace _impl { -class CachedRealm : public CachedRealmBase { +class WeakRealmNotifier : public WeakRealmNotifierBase { public: - using CachedRealmBase::CachedRealmBase; + using WeakRealmNotifierBase::WeakRealmNotifierBase; // Do nothing, as this can't be implemented portably void notify() { } diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 7cd7a17a..cc2800b0 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -19,7 +19,7 @@ #include "impl/realm_coordinator.hpp" #include "impl/async_query.hpp" -#include "impl/cached_realm.hpp" +#include "impl/weak_realm_notifier.hpp" #include "impl/external_commit_helper.hpp" #include "impl/transact_log_handler.hpp" #include "object_store.hpp" @@ -64,7 +64,7 @@ std::shared_ptr RealmCoordinator::get_existing_coordinator(Str std::shared_ptr RealmCoordinator::get_realm(Realm::Config config) { std::lock_guard lock(m_realm_mutex); - if ((!m_config.read_only && !m_notifier) || (m_config.read_only && m_cached_realms.empty())) { + if ((!m_config.read_only && !m_notifier) || (m_config.read_only && m_weak_realm_notifiers.empty())) { m_config = config; if (!config.read_only && !m_notifier) { try { @@ -100,7 +100,7 @@ std::shared_ptr RealmCoordinator::get_realm(Realm::Config config) } if (config.cache) { - for (auto& cachedRealm : m_cached_realms) { + for (auto& cachedRealm : m_weak_realm_notifiers) { if (cachedRealm.is_cached_for_current_thread()) { // can be null if we jumped in between ref count hitting zero and // unregister_realm() getting the lock @@ -113,7 +113,7 @@ std::shared_ptr RealmCoordinator::get_realm(Realm::Config config) auto realm = std::make_shared(std::move(config)); realm->init(shared_from_this()); - m_cached_realms.emplace_back(realm, m_config.cache); + m_weak_realm_notifiers.emplace_back(realm, m_config.cache); return realm; } @@ -124,7 +124,7 @@ std::shared_ptr RealmCoordinator::get_realm() const Schema* RealmCoordinator::get_schema() const noexcept { - return m_cached_realms.empty() ? nullptr : m_config.schema.get(); + return m_weak_realm_notifiers.empty() ? nullptr : m_config.schema.get(); } void RealmCoordinator::update_schema(Schema const& schema) @@ -152,16 +152,16 @@ RealmCoordinator::~RealmCoordinator() void RealmCoordinator::unregister_realm(Realm* realm) { std::lock_guard lock(m_realm_mutex); - for (size_t i = 0; i < m_cached_realms.size(); ++i) { - auto& cached_realm = m_cached_realms[i]; - if (!cached_realm.expired() && !cached_realm.is_for_realm(realm)) { + for (size_t i = 0; i < m_weak_realm_notifiers.size(); ++i) { + auto& weak_realm_notifier = m_weak_realm_notifiers[i]; + if (!weak_realm_notifier.expired() && !weak_realm_notifier.is_for_realm(realm)) { continue; } - if (i + 1 < m_cached_realms.size()) { - cached_realm = std::move(m_cached_realms.back()); + if (i + 1 < m_weak_realm_notifiers.size()) { + weak_realm_notifier = std::move(m_weak_realm_notifiers.back()); } - m_cached_realms.pop_back(); + m_weak_realm_notifiers.pop_back(); } } @@ -180,8 +180,8 @@ void RealmCoordinator::clear_cache() coordinator->m_notifier = nullptr; // Gather a list of all of the realms which will be removed - for (auto& cached_realm : coordinator->m_cached_realms) { - if (auto realm = cached_realm.realm()) { + for (auto& weak_realm_notifier : coordinator->m_weak_realm_notifiers) { + if (auto realm = weak_realm_notifier.realm()) { realms_to_close.push_back(realm); } } @@ -289,7 +289,7 @@ void RealmCoordinator::on_change() run_async_queries(); std::lock_guard lock(m_realm_mutex); - for (auto& realm : m_cached_realms) { + for (auto& realm : m_weak_realm_notifiers) { realm.notify(); } } diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index 4403bc0b..3cc60a8b 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -33,7 +33,7 @@ struct AsyncQueryCancelationToken; namespace _impl { class AsyncQuery; -class CachedRealm; +class WeakRealmNotifier; class ExternalCommitHelper; // RealmCoordinator manages the weak cache of Realm instances and communication @@ -91,7 +91,7 @@ private: Realm::Config m_config; std::mutex m_realm_mutex; - std::vector m_cached_realms; + std::vector m_weak_realm_notifiers; std::mutex m_query_mutex; std::vector> m_new_queries; diff --git a/src/impl/cached_realm.hpp b/src/impl/weak_realm_notifier.hpp similarity index 79% rename from src/impl/cached_realm.hpp rename to src/impl/weak_realm_notifier.hpp index 3818ca62..b0e4f595 100644 --- a/src/impl/cached_realm.hpp +++ b/src/impl/weak_realm_notifier.hpp @@ -16,15 +16,15 @@ // //////////////////////////////////////////////////////////////////////////// -#ifndef REALM_CACHED_REALM_HPP -#define REALM_CACHED_REALM_HPP +#ifndef REALM_WEAK_REALM_NOTIFIER_HPP +#define REALM_WEAK_REALM_NOTIFIER_HPP #include #if REALM_PLATFORM_APPLE -#include "impl/apple/cached_realm.hpp" +#include "impl/apple/weak_realm_notifier.hpp" #else -#include "impl/generic/cached_realm.hpp" +#include "impl/generic/weak_realm_notifier.hpp" #endif -#endif // REALM_CACHED_REALM_HPP +#endif // REALM_WEAK_REALM_NOTIFIER_HPP diff --git a/src/impl/cached_realm_base.hpp b/src/impl/weak_realm_notifier_base.hpp similarity index 75% rename from src/impl/cached_realm_base.hpp rename to src/impl/weak_realm_notifier_base.hpp index c734d453..a8c41bab 100644 --- a/src/impl/cached_realm_base.hpp +++ b/src/impl/weak_realm_notifier_base.hpp @@ -16,8 +16,8 @@ // //////////////////////////////////////////////////////////////////////////// -#ifndef REALM_CACHED_REALM_BASE_HPP -#define REALM_CACHED_REALM_BASE_HPP +#ifndef REALM_WEAK_REALM_NOTIFIER_BASE_HPP +#define REALM_WEAK_REALM_NOTIFIER_BASE_HPP #include #include @@ -27,25 +27,25 @@ class Realm; namespace _impl { -// CachedRealmBase stores a weak reference to a Realm instance, along with all of +// WeakRealmNotifierBase stores a weak reference to a Realm instance, along with all of // the information about a Realm that needs to be accessed from other threads. // This is needed to avoid forming strong references to the Realm instances on // other threads, which can produce deadlocks when the last strong reference to // a Realm instance is released from within a function holding the cache lock. -class CachedRealmBase { +class WeakRealmNotifierBase { public: - CachedRealmBase(const std::shared_ptr& realm, bool cache); + WeakRealmNotifierBase(const std::shared_ptr& realm, bool cache); // Get a strong reference to the cached realm std::shared_ptr realm() const { return m_realm.lock(); } - // Does this CachedRealmBase store a Realm instance that should be used on the current thread? + // Does this WeakRealmNotifierBase store a Realm instance that should be used on the current thread? bool is_cached_for_current_thread() const { return m_cache && m_thread_id == std::this_thread::get_id(); } // Has the Realm instance been destroyed? bool expired() const { return m_realm.expired(); } - // Is this a CachedRealmBase for the given Realm instance? + // Is this a WeakRealmNotifierBase for the given Realm instance? bool is_for_realm(Realm* realm) const { return realm == m_realm_key; } private: @@ -55,7 +55,7 @@ private: bool m_cache = false; }; -inline CachedRealmBase::CachedRealmBase(const std::shared_ptr& realm, bool cache) +inline WeakRealmNotifierBase::WeakRealmNotifierBase(const std::shared_ptr& realm, bool cache) : m_realm(realm) , m_realm_key(realm.get()) , m_cache(cache) @@ -65,4 +65,4 @@ inline CachedRealmBase::CachedRealmBase(const std::shared_ptr& realm, boo } // namespace _impl } // namespace realm -#endif // REALM_CACHED_REALM_BASE_HPP +#endif // REALM_WEAK_REALM_NOTIFIER_BASE_HPP From b7b2822082f3bffab5ec8a54647c549d3fef2862 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 25 Feb 2016 11:19:00 -0800 Subject: [PATCH 74/79] Make it possible to disable the commit notifier background worker This makes it much easier to write tests which test the work done on the background thread. --- src/impl/realm_coordinator.cpp | 6 ++++-- src/shared_realm.cpp | 7 ++++--- src/shared_realm.hpp | 32 ++++++++++++++++++++++++++------ 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index cc2800b0..b9ae3e98 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -66,7 +66,7 @@ std::shared_ptr RealmCoordinator::get_realm(Realm::Config config) std::lock_guard lock(m_realm_mutex); if ((!m_config.read_only && !m_notifier) || (m_config.read_only && m_weak_realm_notifiers.empty())) { m_config = config; - if (!config.read_only && !m_notifier) { + if (!config.read_only && !m_notifier && config.automatic_change_notifications) { try { m_notifier = std::make_unique(*this); } @@ -202,7 +202,9 @@ void RealmCoordinator::clear_cache() void RealmCoordinator::send_commit_notifications() { REALM_ASSERT(!m_config.read_only); - m_notifier->notify_others(); + if (m_notifier) { + m_notifier->notify_others(); + } } void RealmCoordinator::pin_version(uint_fast64_t version, uint_fast32_t index) diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 272a3367..16250c23 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -35,13 +35,14 @@ using namespace realm::_impl; Realm::Config::Config(const Config& c) : path(c.path) +, encryption_key(c.encryption_key) +, schema_version(c.schema_version) +, migration_function(c.migration_function) , read_only(c.read_only) , in_memory(c.in_memory) , cache(c.cache) , disable_format_upgrade(c.disable_format_upgrade) -, encryption_key(c.encryption_key) -, schema_version(c.schema_version) -, migration_function(c.migration_function) +, automatic_change_notifications(c.automatic_change_notifications) { if (c.schema) { schema = std::make_unique(*c.schema); diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index 2360a225..5bd90aac 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -47,20 +47,40 @@ namespace realm { public: typedef std::function MigrationFunction; - struct Config - { + struct Config { std::string path; - bool read_only = false; - bool in_memory = false; - bool cache = true; - bool disable_format_upgrade = false; + // User-supplied encryption key. Must be either empty or 64 bytes. std::vector encryption_key; + // Optional schema for the file. If nullptr, the existing schema + // from the file opened will be used. If present, the file will be + // migrated to the schema if needed. std::unique_ptr schema; uint64_t schema_version; MigrationFunction migration_function; + bool read_only = false; + bool in_memory = false; + + // The following are intended for internal/testing purposes and + // should not be publically exposed in binding APIs + + // If false, always return a new Realm instance, and don't return + // that Realm instance for other requests for a cached Realm. Useful + // for dynamic Realms and for tests that need multiple instances on + // one thread + bool cache = true; + // Throw an exception rather than automatically upgrading the file + // format. Used by the browser to warn the user that it'll modify + // the file. + bool disable_format_upgrade = false; + // Disable the background worker thread for producing change + // notifications. Useful for tests for those notifications so that + // everything can be done deterministically on one thread, and + // speeds up tests that don't need notifications. + bool automatic_change_notifications = true; + Config(); Config(Config&&); Config(const Config& c); From 773e7db14d9b9b97ed59e82092cc67ccf0c16d64 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 26 Jan 2016 10:33:53 -0800 Subject: [PATCH 75/79] Add a helper class to generate temp paths for tests --- tests/CMakeLists.txt | 12 +++++++++--- tests/util/test_file.cpp | 31 +++++++++++++++++++++++++++++++ tests/util/test_file.hpp | 15 +++++++++++++++ 3 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 tests/util/test_file.cpp create mode 100644 tests/util/test_file.hpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 269301c9..a0986977 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,11 +1,17 @@ -include_directories(../external/catch/single_include) +include_directories(../external/catch/single_include .) + +set(HEADERS + util/test_file.hpp +) set(SOURCES index_set.cpp main.cpp - parser.cpp) + parser.cpp + util/test_file.cpp +) -add_executable(tests ${SOURCES}) +add_executable(tests ${SOURCES} ${HEADERS}) target_link_libraries(tests realm-object-store) add_custom_target(run-tests USES_TERMINAL DEPENDS tests COMMAND ./tests) diff --git a/tests/util/test_file.cpp b/tests/util/test_file.cpp new file mode 100644 index 00000000..932218c5 --- /dev/null +++ b/tests/util/test_file.cpp @@ -0,0 +1,31 @@ +#include "util/test_file.hpp" + +#include + +#include +#include + +TestFile::TestFile() +{ + static std::string tmpdir = [] { + realm::disable_sync_to_disk(); + + const char* dir = getenv("TMPDIR"); + if (dir && *dir) + return dir; + return "/tmp"; + }(); + path = tmpdir + "/realm.XXXXXX"; + mktemp(&path[0]); + unlink(path.c_str()); +} + +TestFile::~TestFile() +{ + unlink(path.c_str()); +} + +InMemoryTestFile::InMemoryTestFile() +{ + in_memory = true; +} diff --git a/tests/util/test_file.hpp b/tests/util/test_file.hpp new file mode 100644 index 00000000..bd20d671 --- /dev/null +++ b/tests/util/test_file.hpp @@ -0,0 +1,15 @@ +#ifndef REALM_TEST_UTIL_TEST_FILE_HPP +#define REALM_TEST_UTIL_TEST_FILE_HPP + +#include "shared_realm.hpp" + +struct TestFile : realm::Realm::Config { + TestFile(); + ~TestFile(); +}; + +struct InMemoryTestFile : TestFile { + InMemoryTestFile(); +}; + +#endif From 086192f8d376c525ed642d254aba70cc336e1c8a Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 26 Feb 2016 09:49:07 -0800 Subject: [PATCH 76/79] Add minimal Results notification tests --- tests/CMakeLists.txt | 1 + tests/results.cpp | 146 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 tests/results.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a0986977..0d61f0e9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,6 +8,7 @@ set(SOURCES index_set.cpp main.cpp parser.cpp + results.cpp util/test_file.cpp ) diff --git a/tests/results.cpp b/tests/results.cpp new file mode 100644 index 00000000..3dda335e --- /dev/null +++ b/tests/results.cpp @@ -0,0 +1,146 @@ +#include "catch.hpp" + +#include "util/test_file.hpp" + +#include "impl/realm_coordinator.hpp" +#include "object_schema.hpp" +#include "property.hpp" +#include "results.hpp" +#include "schema.hpp" + +#include +#include +#include + +using namespace realm; + +TEST_CASE("Results") { + InMemoryTestFile config; + config.cache = false; + config.automatic_change_notifications = false; + config.schema = std::make_unique(Schema{ + {"object", "", { + {"value", PropertyTypeInt}, + {"link", PropertyTypeObject, "linked to object", false, false, true} + }}, + {"other object", "", { + {"value", PropertyTypeInt} + }}, + {"linking object", "", { + {"link", PropertyTypeObject, "object", false, false, true} + }}, + {"linked to object", "", { + {"value", PropertyTypeInt} + }} + }); + + auto r = Realm::get_shared_realm(config); + auto coordinator = _impl::RealmCoordinator::get_existing_coordinator(config.path); + auto table = r->read_group()->get_table("class_object"); + + r->begin_transaction(); + table->add_empty_row(10); + for (int i = 0; i < 10; ++i) + table->set_int(0, i, i); + r->commit_transaction(); + + Results results(r, *config.schema->find("object"), table->where().greater(0, 0).less(0, 5)); + + SECTION("notifications") { + int notification_calls = 0; + auto token = results.async([&](std::exception_ptr err) { + REQUIRE_FALSE(err); + ++notification_calls; + }); + + coordinator->on_change(); + r->notify(); + + SECTION("initial results are delivered") { + REQUIRE(notification_calls == 1); + } + + SECTION("modifying the table sends a notification asynchronously") { + r->begin_transaction(); + table->set_int(0, 0, 0); + r->commit_transaction(); + + REQUIRE(notification_calls == 1); + coordinator->on_change(); + r->notify(); + REQUIRE(notification_calls == 2); + } + + SECTION("modifying a linked-to table send a notification") { + r->begin_transaction(); + r->read_group()->get_table("class_linked to object")->add_empty_row(); + r->commit_transaction(); + + REQUIRE(notification_calls == 1); + coordinator->on_change(); + r->notify(); + REQUIRE(notification_calls == 2); + } + + SECTION("modifying a a linking table sends a notification") { + r->begin_transaction(); + r->read_group()->get_table("class_linking object")->add_empty_row(); + r->commit_transaction(); + + REQUIRE(notification_calls == 1); + coordinator->on_change(); + r->notify(); + REQUIRE(notification_calls == 2); + } + + SECTION("modifying a an unrelated table does not send a notification") { + r->begin_transaction(); + r->read_group()->get_table("class_other object")->add_empty_row(); + r->commit_transaction(); + + REQUIRE(notification_calls == 1); + coordinator->on_change(); + r->notify(); + REQUIRE(notification_calls == 1); + } + + SECTION("modifications from multiple transactions are collapsed") { + r->begin_transaction(); + table->set_int(0, 0, 0); + r->commit_transaction(); + + r->begin_transaction(); + table->set_int(0, 1, 0); + r->commit_transaction(); + + REQUIRE(notification_calls == 1); + coordinator->on_change(); + r->notify(); + REQUIRE(notification_calls == 2); + } + + SECTION("notifications are not delivered when the token is destroyed before they are calculated") { + r->begin_transaction(); + table->set_int(0, 0, 0); + r->commit_transaction(); + + REQUIRE(notification_calls == 1); + token = {}; + coordinator->on_change(); + r->notify(); + REQUIRE(notification_calls == 1); + } + + SECTION("notifications are not delivered when the token is destroyed before they are delivered") { + r->begin_transaction(); + table->set_int(0, 0, 0); + r->commit_transaction(); + + REQUIRE(notification_calls == 1); + coordinator->on_change(); + token = {}; + r->notify(); + REQUIRE(notification_calls == 1); + } + } +} From 8c56d1338217ef337b970572ed89ffa23d86c567 Mon Sep 17 00:00:00 2001 From: Mark Rowe Date: Wed, 2 Mar 2016 00:21:43 -0800 Subject: [PATCH 77/79] Support building on Linux. By default, Linux uses a locally-built version of core that is prepared in an independent working copy of the realm-core git repository. Support is also added for using an existing local build of core on both OS X and Linux . This can be done by running `cmake -DREALM_CORE_VERSION=/path/to/realm-core`. The generated build system will invoke `sh build.sh build` within the given directory prior to building the object store. --- .gitignore | 2 +- CMake/RealmCore.cmake | 92 ++++++++++++++++++++++++++++++++++++++++++- CMakeLists.txt | 3 +- 3 files changed, 94 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 7d9b2fa3..59c6c914 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,5 @@ cmake_install.cmake rules.ninja # Build products -librealm-object-store.dylib +src/librealm-object-store.* tests/tests diff --git a/CMake/RealmCore.cmake b/CMake/RealmCore.cmake index fa020932..be602e20 100644 --- a/CMake/RealmCore.cmake +++ b/CMake/RealmCore.cmake @@ -1,5 +1,26 @@ include(ExternalProject) +if(${CMAKE_GENERATOR} STREQUAL "Unix Makefiles") + set(MAKE_EQUAL_MAKE "MAKE=$(MAKE)") +endif() + +if (${CMAKE_VERSION} VERSION_GREATER "3.4.0") + set(USES_TERMINAL_BUILD USES_TERMINAL_BUILD 1) +endif() + +function(use_realm_core version_or_path_to_source) + if("${version_or_path_to_source}" MATCHES "^[0-9]+(\\.[0-9])+") + if(APPLE) + download_realm_core(${version_or_path_to_source}) + else() + clone_and_build_realm_core("v${version_or_path_to_source}") + endif() + else() + build_existing_realm_core(${version_or_path_to_source}) + endif() + set(REALM_CORE_INCLUDE_DIR ${REALM_CORE_INCLUDE_DIR} PARENT_SCOPE) +endfunction() + function(download_realm_core core_version) set(core_url "https://static.realm.io/downloads/core/realm-core-${core_version}.tar.bz2") set(core_tarball_name "realm-core-${core_version}.tar.bz2") @@ -10,7 +31,7 @@ function(download_realm_core core_version) if (NOT EXISTS ${core_tarball}) if (NOT EXISTS ${core_temp_tarball}) - message("Downloading core ${core_version} from ${core_url}.") + message("Downloading core ${core_version} from ${core_url}.") file(DOWNLOAD ${core_url} ${core_temp_tarball}.tmp SHOW_PROGRESS) file(RENAME ${core_temp_tarball}.tmp ${core_temp_tarball}) endif() @@ -40,3 +61,72 @@ function(download_realm_core core_version) set(REALM_CORE_INCLUDE_DIR ${core_directory}/include PARENT_SCOPE) endfunction() + +function(clone_and_build_realm_core branch) + set(core_prefix_directory "${CMAKE_CURRENT_SOURCE_DIR}${CMAKE_FILES_DIRECTORY}/realm-core") + ExternalProject_Add(realm-core + GIT_REPOSITORY "git@github.com:realm/realm-core.git" + GIT_TAG ${branch} + PREFIX ${core_prefix_directory} + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND ${MAKE_EQUAL_MAKE} sh build.sh build + INSTALL_COMMAND "" + ${USES_TERMINAL_BUILD} + ) + + ExternalProject_Get_Property(realm-core SOURCE_DIR) + set(core_directory ${SOURCE_DIR}) + set(core_library_debug ${core_directory}/src/realm/librealm-dbg${CMAKE_SHARED_LIBRARY_SUFFIX}) + set(core_library_release ${core_directory}/src/realm/librealm${CMAKE_SHARED_LIBRARY_SUFFIX}) + set(core_libraries ${core_library_debug} ${core_library_release}) + + ExternalProject_Add_Step(realm-core ensure-libraries + COMMAND ${CMAKE_COMMAND} -E touch_nocreate ${core_libraries} + OUTPUT ${core_libraries} + DEPENDEES build + ) + + add_library(realm SHARED IMPORTED) + add_dependencies(realm realm-core) + + set_property(TARGET realm PROPERTY IMPORTED_LOCATION_DEBUG ${core_library_debug}) + set_property(TARGET realm PROPERTY IMPORTED_LOCATION_RELEASE ${core_library_release}) + set_property(TARGET realm PROPERTY IMPORTED_LOCATION ${core_library_release}) + + set(REALM_CORE_INCLUDE_DIR ${core_directory}/src PARENT_SCOPE) +endfunction() + +function(build_existing_realm_core core_directory) + get_filename_component(core_directory ${core_directory} ABSOLUTE) + ExternalProject_Add(realm-core + URL "" + PREFIX ${CMAKE_CURRENT_SOURCE_DIR}${CMAKE_FILES_DIRECTORY}/realm-core + SOURCE_DIR ${core_directory} + BUILD_IN_SOURCE 1 + BUILD_ALWAYS 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND ${MAKE_EQUAL_MAKE} sh build.sh build + INSTALL_COMMAND "" + ${USES_TERMINAL_BUILD} + ) + + set(core_library_debug ${core_directory}/src/realm/librealm-dbg${CMAKE_SHARED_LIBRARY_SUFFIX}) + set(core_library_release ${core_directory}/src/realm/librealm${CMAKE_SHARED_LIBRARY_SUFFIX}) + set(core_libraries ${core_library_debug} ${core_library_release}) + + ExternalProject_Add_Step(realm-core ensure-libraries + COMMAND ${CMAKE_COMMAND} -E touch_nocreate ${core_libraries} + OUTPUT ${core_libraries} + DEPENDEES build + ) + + add_library(realm SHARED IMPORTED) + add_dependencies(realm realm-core) + + set_property(TARGET realm PROPERTY IMPORTED_LOCATION_DEBUG ${core_library_debug}) + set_property(TARGET realm PROPERTY IMPORTED_LOCATION_RELEASE ${core_library_release}) + set_property(TARGET realm PROPERTY IMPORTED_LOCATION ${core_library_release}) + + set(REALM_CORE_INCLUDE_DIR ${core_directory}/src PARENT_SCOPE) +endfunction() diff --git a/CMakeLists.txt b/CMakeLists.txt index e0441f49..39b30790 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,8 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake") include(CompilerFlags) include(RealmCore) -download_realm_core(0.96.2) +set(REALM_CORE_VERSION "0.96.2" CACHE STRING "") +use_realm_core(${REALM_CORE_VERSION}) include_directories(${REALM_CORE_INCLUDE_DIR} src external/pegtl) From ccdc3b07540ece813ed10acc1db0062342b25079 Mon Sep 17 00:00:00 2001 From: Mark Rowe Date: Wed, 2 Mar 2016 00:21:55 -0800 Subject: [PATCH 78/79] Add info to the README about using a local build of core. --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 948a47e6..a641cb36 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,14 @@ platforms when integrated into a binding. make ``` +If you wish to build against a local version of core you can invoke `cmake` like so: + +``` +cmake -DREALM_CORE_VERSION=/path/to/realm-core +``` + +The given core tree will be built as part of the object store build. + ## Testing ``` From 3602cf7588aaf4ea8ef2796ed9a522ec25551440 Mon Sep 17 00:00:00 2001 From: Mark Rowe Date: Wed, 2 Mar 2016 01:16:11 -0800 Subject: [PATCH 79/79] Factor duplicated logic out into a macro. --- CMake/RealmCore.cmake | 52 ++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/CMake/RealmCore.cmake b/CMake/RealmCore.cmake index be602e20..76072544 100644 --- a/CMake/RealmCore.cmake +++ b/CMake/RealmCore.cmake @@ -62,21 +62,7 @@ function(download_realm_core core_version) set(REALM_CORE_INCLUDE_DIR ${core_directory}/include PARENT_SCOPE) endfunction() -function(clone_and_build_realm_core branch) - set(core_prefix_directory "${CMAKE_CURRENT_SOURCE_DIR}${CMAKE_FILES_DIRECTORY}/realm-core") - ExternalProject_Add(realm-core - GIT_REPOSITORY "git@github.com:realm/realm-core.git" - GIT_TAG ${branch} - PREFIX ${core_prefix_directory} - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND ${MAKE_EQUAL_MAKE} sh build.sh build - INSTALL_COMMAND "" - ${USES_TERMINAL_BUILD} - ) - - ExternalProject_Get_Property(realm-core SOURCE_DIR) - set(core_directory ${SOURCE_DIR}) +macro(define_built_realm_core_target core_directory) set(core_library_debug ${core_directory}/src/realm/librealm-dbg${CMAKE_SHARED_LIBRARY_SUFFIX}) set(core_library_release ${core_directory}/src/realm/librealm${CMAKE_SHARED_LIBRARY_SUFFIX}) set(core_libraries ${core_library_debug} ${core_library_release}) @@ -95,6 +81,23 @@ function(clone_and_build_realm_core branch) set_property(TARGET realm PROPERTY IMPORTED_LOCATION ${core_library_release}) set(REALM_CORE_INCLUDE_DIR ${core_directory}/src PARENT_SCOPE) +endmacro() + +function(clone_and_build_realm_core branch) + set(core_prefix_directory "${CMAKE_CURRENT_SOURCE_DIR}${CMAKE_FILES_DIRECTORY}/realm-core") + ExternalProject_Add(realm-core + GIT_REPOSITORY "git@github.com:realm/realm-core.git" + GIT_TAG ${branch} + PREFIX ${core_prefix_directory} + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND ${MAKE_EQUAL_MAKE} sh build.sh build + INSTALL_COMMAND "" + ${USES_TERMINAL_BUILD} + ) + + ExternalProject_Get_Property(realm-core SOURCE_DIR) + define_built_realm_core_target(${SOURCE_DIR}) endfunction() function(build_existing_realm_core core_directory) @@ -111,22 +114,5 @@ function(build_existing_realm_core core_directory) ${USES_TERMINAL_BUILD} ) - set(core_library_debug ${core_directory}/src/realm/librealm-dbg${CMAKE_SHARED_LIBRARY_SUFFIX}) - set(core_library_release ${core_directory}/src/realm/librealm${CMAKE_SHARED_LIBRARY_SUFFIX}) - set(core_libraries ${core_library_debug} ${core_library_release}) - - ExternalProject_Add_Step(realm-core ensure-libraries - COMMAND ${CMAKE_COMMAND} -E touch_nocreate ${core_libraries} - OUTPUT ${core_libraries} - DEPENDEES build - ) - - add_library(realm SHARED IMPORTED) - add_dependencies(realm realm-core) - - set_property(TARGET realm PROPERTY IMPORTED_LOCATION_DEBUG ${core_library_debug}) - set_property(TARGET realm PROPERTY IMPORTED_LOCATION_RELEASE ${core_library_release}) - set_property(TARGET realm PROPERTY IMPORTED_LOCATION ${core_library_release}) - - set(REALM_CORE_INCLUDE_DIR ${core_directory}/src PARENT_SCOPE) + define_built_realm_core_target(${core_directory}) endfunction()