From 273db056053b6f5a8227c722ce7f1c0dde6a7753 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 20 Apr 2016 15:38:49 -0700 Subject: [PATCH 01/93] Fix the initial ref count for WeakRealmNotifier Adding the run loop source to the run loop retains it, so the initial refcount should be 0, not 1. --- src/impl/apple/weak_realm_notifier.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/impl/apple/weak_realm_notifier.cpp b/src/impl/apple/weak_realm_notifier.cpp index da403ad7..0ce6300f 100644 --- a/src/impl/apple/weak_realm_notifier.cpp +++ b/src/impl/apple/weak_realm_notifier.cpp @@ -34,7 +34,7 @@ WeakRealmNotifier::WeakRealmNotifier(const std::shared_ptr& realm, bool c }; CFRunLoopSourceContext ctx{}; - ctx.info = new RefCountedWeakPointer{realm, {1}}; + ctx.info = new RefCountedWeakPointer{realm, {0}}; ctx.perform = [](void* info) { if (auto realm = static_cast(info)->realm.lock()) { realm->notify(); From 7ab91ea75e23b5b2f13648334b912ce8f951f8d1 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 18 Feb 2016 17:21:08 -0800 Subject: [PATCH 02/93] Add cmake target to generate a code coverage report --- CMake/CodeCoverage.cmake | 194 +++++++++++++++++++++++++++++++++++++++ CMakeLists.txt | 1 + tests/CMakeLists.txt | 2 + 3 files changed, 197 insertions(+) create mode 100644 CMake/CodeCoverage.cmake diff --git a/CMake/CodeCoverage.cmake b/CMake/CodeCoverage.cmake new file mode 100644 index 00000000..1ead3e7a --- /dev/null +++ b/CMake/CodeCoverage.cmake @@ -0,0 +1,194 @@ +# Copyright (c) 2012 - 2015, Lars Bilke +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# +# 2012-01-31, Lars Bilke +# - Enable Code Coverage +# +# 2013-09-17, Joakim Söderberg +# - Added support for Clang. +# - Some additional usage instructions. +# +# USAGE: + +# 0. (Mac only) If you use Xcode 5.1 make sure to patch geninfo as described here: +# http://stackoverflow.com/a/22404544/80480 +# +# 1. Copy this file into your cmake modules path. +# +# 2. Add the following line to your CMakeLists.txt: +# INCLUDE(CodeCoverage) +# +# 3. Set compiler flags to turn off optimization and enable coverage: +# SET(CMAKE_CXX_FLAGS "-g -O0 -fprofile-arcs -ftest-coverage") +# SET(CMAKE_C_FLAGS "-g -O0 -fprofile-arcs -ftest-coverage") +# +# 3. Use the function SETUP_TARGET_FOR_COVERAGE to create a custom make target +# which runs your test executable and produces a lcov code coverage report: +# Example: +# SETUP_TARGET_FOR_COVERAGE( +# my_coverage_target # Name for custom target. +# test_driver # Name of the test driver executable that runs the tests. +# # NOTE! This should always have a ZERO as exit code +# # otherwise the coverage generation will not complete. +# coverage # Name of output directory. +# ) +# +# 4. Build a Debug build: +# cmake -DCMAKE_BUILD_TYPE=Debug .. +# make +# make my_coverage_target +# +# + +# Check prereqs +FIND_PROGRAM( GCOV_PATH gcov ) +FIND_PROGRAM( LCOV_PATH lcov ) +FIND_PROGRAM( GENHTML_PATH genhtml ) +FIND_PROGRAM( GCOVR_PATH gcovr PATHS ${CMAKE_SOURCE_DIR}/tests) + +IF(NOT GCOV_PATH) + MESSAGE(FATAL_ERROR "gcov not found! Aborting...") +ENDIF() # NOT GCOV_PATH + +IF(NOT CMAKE_COMPILER_IS_GNUCXX) + # Clang version 3.0.0 and greater now supports gcov as well. + # MESSAGE(WARNING "Compiler is not GNU gcc! Clang Version 3.0.0 and greater supports gcov as well, but older versions don't.") + + IF(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + MESSAGE(FATAL_ERROR "Compiler is not GNU gcc! Aborting...") + ENDIF() +ENDIF() # NOT CMAKE_COMPILER_IS_GNUCXX + +SET(CMAKE_CXX_FLAGS_COVERAGE + "-g -O0 --coverage -fprofile-arcs -ftest-coverage" + CACHE STRING "Flags used by the C++ compiler during coverage builds." + FORCE ) +SET(CMAKE_C_FLAGS_COVERAGE + "-g -O0 --coverage -fprofile-arcs -ftest-coverage" + CACHE STRING "Flags used by the C compiler during coverage builds." + FORCE ) +SET(CMAKE_EXE_LINKER_FLAGS_COVERAGE + "" + CACHE STRING "Flags used for linking binaries during coverage builds." + FORCE ) +SET(CMAKE_SHARED_LINKER_FLAGS_COVERAGE + "" + CACHE STRING "Flags used by the shared libraries linker during coverage builds." + FORCE ) +MARK_AS_ADVANCED( + CMAKE_CXX_FLAGS_COVERAGE + CMAKE_C_FLAGS_COVERAGE + CMAKE_EXE_LINKER_FLAGS_COVERAGE + CMAKE_SHARED_LINKER_FLAGS_COVERAGE ) + +IF ( NOT (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "Coverage")) + MESSAGE( WARNING "Code coverage results with an optimized (non-Debug) build may be misleading" ) +ENDIF() # NOT CMAKE_BUILD_TYPE STREQUAL "Debug" + + +# Param _targetname The name of new the custom make target +# Param _testrunner The name of the target which runs the tests. +# MUST return ZERO always, even on errors. +# If not, no coverage report will be created! +# Param _outputname lcov output is generated as _outputname.info +# HTML report is generated in _outputname/index.html +# Optional fourth parameter is passed as arguments to _testrunner +# Pass them in list form, e.g.: "-j;2" for -j 2 +FUNCTION(SETUP_TARGET_FOR_COVERAGE _targetname _testrunner _outputname) + + IF(NOT LCOV_PATH) + MESSAGE(FATAL_ERROR "lcov not found! Aborting...") + ENDIF() # NOT LCOV_PATH + + IF(NOT GENHTML_PATH) + MESSAGE(FATAL_ERROR "genhtml not found! Aborting...") + ENDIF() # NOT GENHTML_PATH + + # Setup target + ADD_CUSTOM_TARGET(${_targetname} + + # Cleanup lcov + ${LCOV_PATH} --directory . --zerocounters + + # Run tests + COMMAND ${_testrunner} ${ARGV3} + + # Capturing lcov counters and generating report + COMMAND ${LCOV_PATH} --directory . --capture --output-file ${_outputname}.info + COMMAND ${LCOV_PATH} --remove ${_outputname}.info 'tests/*' '/usr/*' --output-file ${_outputname}.info.cleaned + COMMAND ${GENHTML_PATH} -o ${_outputname} ${_outputname}.info.cleaned + COMMAND ${CMAKE_COMMAND} -E remove ${_outputname}.info ${_outputname}.info.cleaned + + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMENT "Resetting code coverage counters to zero.\nProcessing code coverage counters and generating report." + ) + + # Show info where to find the report + ADD_CUSTOM_COMMAND(TARGET ${_targetname} POST_BUILD + COMMAND ; + COMMENT "Open ./${_outputname}/index.html in your browser to view the coverage report." + ) + +ENDFUNCTION() # SETUP_TARGET_FOR_COVERAGE + +# Param _targetname The name of new the custom make target +# Param _testrunner The name of the target which runs the tests +# Param _outputname cobertura output is generated as _outputname.xml +# Optional fourth parameter is passed as arguments to _testrunner +# Pass them in list form, e.g.: "-j;2" for -j 2 +FUNCTION(SETUP_TARGET_FOR_COVERAGE_COBERTURA _targetname _testrunner _outputname) + + IF(NOT PYTHON_EXECUTABLE) + MESSAGE(FATAL_ERROR "Python not found! Aborting...") + ENDIF() # NOT PYTHON_EXECUTABLE + + IF(NOT GCOVR_PATH) + MESSAGE(FATAL_ERROR "gcovr not found! Aborting...") + ENDIF() # NOT GCOVR_PATH + + ADD_CUSTOM_TARGET(${_targetname} + + # Run tests + ${_testrunner} ${ARGV3} + + # Running gcovr + COMMAND ${GCOVR_PATH} -x -r ${CMAKE_SOURCE_DIR} -e '${CMAKE_SOURCE_DIR}/tests/' -o ${_outputname}.xml + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMENT "Running gcovr to produce Cobertura code coverage report." + ) + + # Show info where to find the report + ADD_CUSTOM_COMMAND(TARGET ${_targetname} POST_BUILD + COMMAND ; + COMMENT "Cobertura code coverage report saved in ${_outputname}.xml." + ) + +ENDFUNCTION() # SETUP_TARGET_FOR_COVERAGE_COBERTURA + diff --git a/CMakeLists.txt b/CMakeLists.txt index b4c55f65..286e6ff0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,7 @@ project(realm-object-store) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake") +include(CodeCoverage) include(CompilerFlags) include(Sanitizers) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0d61f0e9..84dd8d1d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -15,4 +15,6 @@ set(SOURCES add_executable(tests ${SOURCES} ${HEADERS}) target_link_libraries(tests realm-object-store) +setup_target_for_coverage(generate-coverage tests coverage) + add_custom_target(run-tests USES_TERMINAL DEPENDS tests COMMAND ./tests) From fafc4232adcbf2d05b2aa271c0d9b190a01fdf16 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 6 May 2016 14:00:44 -0700 Subject: [PATCH 03/93] Rewrite the code coverage generation Make lcov/gcovr an optional dependency that's only needed for Coverage configurations, remove some pointless noisy messages when not generating coverage, and generally simplify the whole thing. --- CMake/CodeCoverage.cmake | 223 +++++++------------------------------- CMake/CompilerFlags.cmake | 1 + CMake/RealmCore.cmake | 2 + tests/CMakeLists.txt | 2 +- 4 files changed, 43 insertions(+), 185 deletions(-) diff --git a/CMake/CodeCoverage.cmake b/CMake/CodeCoverage.cmake index 1ead3e7a..b8308286 100644 --- a/CMake/CodeCoverage.cmake +++ b/CMake/CodeCoverage.cmake @@ -1,194 +1,49 @@ -# Copyright (c) 2012 - 2015, Lars Bilke -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its contributors -# may be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# -# -# 2012-01-31, Lars Bilke -# - Enable Code Coverage -# -# 2013-09-17, Joakim Söderberg -# - Added support for Clang. -# - Some additional usage instructions. -# -# USAGE: +find_program(LCOV_PATH lcov) +find_program(GENHTML_PATH genhtml) +find_program(GCOVR_PATH gcovr PATHS ${CMAKE_SOURCE_DIR}/tests) -# 0. (Mac only) If you use Xcode 5.1 make sure to patch geninfo as described here: -# http://stackoverflow.com/a/22404544/80480 -# -# 1. Copy this file into your cmake modules path. -# -# 2. Add the following line to your CMakeLists.txt: -# INCLUDE(CodeCoverage) -# -# 3. Set compiler flags to turn off optimization and enable coverage: -# SET(CMAKE_CXX_FLAGS "-g -O0 -fprofile-arcs -ftest-coverage") -# SET(CMAKE_C_FLAGS "-g -O0 -fprofile-arcs -ftest-coverage") -# -# 3. Use the function SETUP_TARGET_FOR_COVERAGE to create a custom make target -# which runs your test executable and produces a lcov code coverage report: -# Example: -# SETUP_TARGET_FOR_COVERAGE( -# my_coverage_target # Name for custom target. -# test_driver # Name of the test driver executable that runs the tests. -# # NOTE! This should always have a ZERO as exit code -# # otherwise the coverage generation will not complete. -# coverage # Name of output directory. -# ) -# -# 4. Build a Debug build: -# cmake -DCMAKE_BUILD_TYPE=Debug .. -# make -# make my_coverage_target -# -# +set(CMAKE_CXX_FLAGS_COVERAGE "-g -O0 -fprofile-arcs -ftest-coverage" + CACHE STRING "Flags used by the C++ compiler during coverage builds.") +mark_as_advanced(CMAKE_CXX_FLAGS_COVERAGE) -# Check prereqs -FIND_PROGRAM( GCOV_PATH gcov ) -FIND_PROGRAM( LCOV_PATH lcov ) -FIND_PROGRAM( GENHTML_PATH genhtml ) -FIND_PROGRAM( GCOVR_PATH gcovr PATHS ${CMAKE_SOURCE_DIR}/tests) +if(CMAKE_BUILD_TYPE STREQUAL "Coverage") + if(NOT (LCOV_PATH AND GENHTML_PATH AND GCOVR_PATH)) + message(FATAL_ERROR "Generating a coverage report requires lcov and gcovr") + endif() -IF(NOT GCOV_PATH) - MESSAGE(FATAL_ERROR "gcov not found! Aborting...") -ENDIF() # NOT GCOV_PATH + function(create_coverage_target targetname testrunner) + add_custom_target(${targetname} + # Clear previous coverage information + COMMAND ${LCOV_PATH} --directory . --zerocounters -IF(NOT CMAKE_COMPILER_IS_GNUCXX) - # Clang version 3.0.0 and greater now supports gcov as well. - # MESSAGE(WARNING "Compiler is not GNU gcc! Clang Version 3.0.0 and greater supports gcov as well, but older versions don't.") + # Run the tests + COMMAND ${testrunner} - IF(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") - MESSAGE(FATAL_ERROR "Compiler is not GNU gcc! Aborting...") - ENDIF() -ENDIF() # NOT CMAKE_COMPILER_IS_GNUCXX + # Generate new coverage report + COMMAND ${LCOV_PATH} --directory . --capture --output-file coverage.info + COMMAND ${LCOV_PATH} --extract coverage.info '${CMAKE_SOURCE_DIR}/src/*' --output-file coverage.info.cleaned + COMMAND ${GENHTML_PATH} -o coverage coverage.info.cleaned + COMMAND ${CMAKE_COMMAND} -E remove coverage.info coverage.info.cleaned -SET(CMAKE_CXX_FLAGS_COVERAGE - "-g -O0 --coverage -fprofile-arcs -ftest-coverage" - CACHE STRING "Flags used by the C++ compiler during coverage builds." - FORCE ) -SET(CMAKE_C_FLAGS_COVERAGE - "-g -O0 --coverage -fprofile-arcs -ftest-coverage" - CACHE STRING "Flags used by the C compiler during coverage builds." - FORCE ) -SET(CMAKE_EXE_LINKER_FLAGS_COVERAGE - "" - CACHE STRING "Flags used for linking binaries during coverage builds." - FORCE ) -SET(CMAKE_SHARED_LINKER_FLAGS_COVERAGE - "" - CACHE STRING "Flags used by the shared libraries linker during coverage builds." - FORCE ) -MARK_AS_ADVANCED( - CMAKE_CXX_FLAGS_COVERAGE - CMAKE_C_FLAGS_COVERAGE - CMAKE_EXE_LINKER_FLAGS_COVERAGE - CMAKE_SHARED_LINKER_FLAGS_COVERAGE ) + COMMAND echo Open coverage/index.html in your browser to view the coverage report. -IF ( NOT (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "Coverage")) - MESSAGE( WARNING "Code coverage results with an optimized (non-Debug) build may be misleading" ) -ENDIF() # NOT CMAKE_BUILD_TYPE STREQUAL "Debug" + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + add_custom_target(${targetname}-cobertura + COMMAND ${testrunner} + COMMAND ${GCOVR_PATH} -x -r ${CMAKE_SOURCE_DIR}/src -o coverage.xml + COMMAND echo Code coverage report written to coverage.xml -# Param _targetname The name of new the custom make target -# Param _testrunner The name of the target which runs the tests. -# MUST return ZERO always, even on errors. -# If not, no coverage report will be created! -# Param _outputname lcov output is generated as _outputname.info -# HTML report is generated in _outputname/index.html -# Optional fourth parameter is passed as arguments to _testrunner -# Pass them in list form, e.g.: "-j;2" for -j 2 -FUNCTION(SETUP_TARGET_FOR_COVERAGE _targetname _testrunner _outputname) - - IF(NOT LCOV_PATH) - MESSAGE(FATAL_ERROR "lcov not found! Aborting...") - ENDIF() # NOT LCOV_PATH - - IF(NOT GENHTML_PATH) - MESSAGE(FATAL_ERROR "genhtml not found! Aborting...") - ENDIF() # NOT GENHTML_PATH - - # Setup target - ADD_CUSTOM_TARGET(${_targetname} - - # Cleanup lcov - ${LCOV_PATH} --directory . --zerocounters - - # Run tests - COMMAND ${_testrunner} ${ARGV3} - - # Capturing lcov counters and generating report - COMMAND ${LCOV_PATH} --directory . --capture --output-file ${_outputname}.info - COMMAND ${LCOV_PATH} --remove ${_outputname}.info 'tests/*' '/usr/*' --output-file ${_outputname}.info.cleaned - COMMAND ${GENHTML_PATH} -o ${_outputname} ${_outputname}.info.cleaned - COMMAND ${CMAKE_COMMAND} -E remove ${_outputname}.info ${_outputname}.info.cleaned - - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - COMMENT "Resetting code coverage counters to zero.\nProcessing code coverage counters and generating report." - ) - - # Show info where to find the report - ADD_CUSTOM_COMMAND(TARGET ${_targetname} POST_BUILD - COMMAND ; - COMMENT "Open ./${_outputname}/index.html in your browser to view the coverage report." - ) - -ENDFUNCTION() # SETUP_TARGET_FOR_COVERAGE - -# Param _targetname The name of new the custom make target -# Param _testrunner The name of the target which runs the tests -# Param _outputname cobertura output is generated as _outputname.xml -# Optional fourth parameter is passed as arguments to _testrunner -# Pass them in list form, e.g.: "-j;2" for -j 2 -FUNCTION(SETUP_TARGET_FOR_COVERAGE_COBERTURA _targetname _testrunner _outputname) - - IF(NOT PYTHON_EXECUTABLE) - MESSAGE(FATAL_ERROR "Python not found! Aborting...") - ENDIF() # NOT PYTHON_EXECUTABLE - - IF(NOT GCOVR_PATH) - MESSAGE(FATAL_ERROR "gcovr not found! Aborting...") - ENDIF() # NOT GCOVR_PATH - - ADD_CUSTOM_TARGET(${_targetname} - - # Run tests - ${_testrunner} ${ARGV3} - - # Running gcovr - COMMAND ${GCOVR_PATH} -x -r ${CMAKE_SOURCE_DIR} -e '${CMAKE_SOURCE_DIR}/tests/' -o ${_outputname}.xml - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - COMMENT "Running gcovr to produce Cobertura code coverage report." - ) - - # Show info where to find the report - ADD_CUSTOM_COMMAND(TARGET ${_targetname} POST_BUILD - COMMAND ; - COMMENT "Cobertura code coverage report saved in ${_outputname}.xml." - ) - -ENDFUNCTION() # SETUP_TARGET_FOR_COVERAGE_COBERTURA + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + endfunction() +else() + function(create_coverage_target targetname testrunner) + add_custom_target(${targetname} + COMMAND echo "Configure with -DCMAKE_BUILD_TYPE=Coverage to generate coverage reports") + add_custom_target(${targetname}-cobertura + COMMAND echo "Configure with -DCMAKE_BUILD_TYPE=Coverage to generate coverage reports") + endfunction() +endif() diff --git a/CMake/CompilerFlags.cmake b/CMake/CompilerFlags.cmake index d45392e4..c214b625 100644 --- a/CMake/CompilerFlags.cmake +++ b/CMake/CompilerFlags.cmake @@ -3,6 +3,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED on) set(CMAKE_CXX_EXTENSIONS off) add_compile_options(-Wall -DREALM_HAVE_CONFIG) add_compile_options("$<$:-DREALM_DEBUG>") +add_compile_options("$<$:-DREALM_DEBUG>") if(${CMAKE_GENERATOR} STREQUAL "Ninja") if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang") diff --git a/CMake/RealmCore.cmake b/CMake/RealmCore.cmake index 90256b98..d4d02e9a 100644 --- a/CMake/RealmCore.cmake +++ b/CMake/RealmCore.cmake @@ -60,6 +60,7 @@ function(download_realm_core core_version) add_library(realm STATIC IMPORTED) add_dependencies(realm realm-core) set_property(TARGET realm PROPERTY IMPORTED_LOCATION_DEBUG ${core_library_debug}) + set_property(TARGET realm PROPERTY IMPORTED_LOCATION_COVERAGE ${core_library_debug}) set_property(TARGET realm PROPERTY IMPORTED_LOCATION_RELEASE ${core_library_release}) set_property(TARGET realm PROPERTY IMPORTED_LOCATION ${core_library_release}) @@ -81,6 +82,7 @@ macro(define_built_realm_core_target core_directory) add_dependencies(realm realm-core) set_property(TARGET realm PROPERTY IMPORTED_LOCATION_DEBUG ${core_library_debug}) + set_property(TARGET realm PROPERTY IMPORTED_LOCATION_COVERAGE ${core_library_debug}) set_property(TARGET realm PROPERTY IMPORTED_LOCATION_RELEASE ${core_library_release}) set_property(TARGET realm PROPERTY IMPORTED_LOCATION ${core_library_release}) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 84dd8d1d..b757f8ad 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -15,6 +15,6 @@ set(SOURCES add_executable(tests ${SOURCES} ${HEADERS}) target_link_libraries(tests realm-object-store) -setup_target_for_coverage(generate-coverage tests coverage) +create_coverage_target(generate-coverage tests) add_custom_target(run-tests USES_TERMINAL DEPENDS tests COMMAND ./tests) From 8e24d4331fd480ae4ebec495d353878f9163d5ba Mon Sep 17 00:00:00 2001 From: kishikawa katsumi Date: Mon, 30 Nov 2015 15:52:06 +0900 Subject: [PATCH 04/93] Watch changes for tvOS without named pipes --- src/impl/apple/external_commit_helper.cpp | 38 +++++++++++++++++------ src/impl/apple/external_commit_helper.hpp | 10 ++++-- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/impl/apple/external_commit_helper.cpp b/src/impl/apple/external_commit_helper.cpp index db04a1de..5195a3e3 100644 --- a/src/impl/apple/external_commit_helper.cpp +++ b/src/impl/apple/external_commit_helper.cpp @@ -35,7 +35,7 @@ using namespace realm::_impl; namespace { // Write a byte to a pipe to notify anyone waiting for data on the pipe -void notify_fd(int fd) +void notify_fd(int fd, int read_fd) { while (true) { char c = 0; @@ -50,7 +50,7 @@ void notify_fd(int fd) // write. assert(ret == -1 && errno == EAGAIN); char buff[1024]; - read(fd, buff, sizeof buff); + read(read_fd, buff, sizeof buff); } } } // anonymous namespace @@ -94,6 +94,7 @@ ExternalCommitHelper::ExternalCommitHelper(RealmCoordinator& parent) throw std::system_error(errno, std::system_category()); } +#if !TARGET_OS_TV auto path = parent.get_path() + ".note"; // Create and open the named pipe @@ -129,15 +130,29 @@ ExternalCommitHelper::ExternalCommitHelper(RealmCoordinator& parent) throw std::system_error(errno, std::system_category()); } - // Create the anonymous pipe - int pipeFd[2]; - ret = pipe(pipeFd); +#else // !TARGET_OS_TV + + // tvOS does not support named pipes, so use an anonymous pipe instead + int notification_pipe[2]; + int ret = pipe(notification_pipe); if (ret == -1) { throw std::system_error(errno, std::system_category()); } - m_shutdown_read_fd = pipeFd[0]; - m_shutdown_write_fd = pipeFd[1]; + m_notify_fd = notification_pipe[0]; + m_notify_fd_write = notification_pipe[1]; + +#endif // TARGET_OS_TV + + // Create the anonymous pipe for shutdown notifications + int shutdown_pipe[2]; + ret = pipe(shutdown_pipe); + if (ret == -1) { + throw std::system_error(errno, std::system_category()); + } + + m_shutdown_read_fd = shutdown_pipe[0]; + m_shutdown_write_fd = shutdown_pipe[1]; m_thread = std::async(std::launch::async, [=] { try { @@ -158,7 +173,7 @@ ExternalCommitHelper::ExternalCommitHelper(RealmCoordinator& parent) ExternalCommitHelper::~ExternalCommitHelper() { - notify_fd(m_shutdown_write_fd); + notify_fd(m_shutdown_write_fd, m_shutdown_read_fd); m_thread.wait(); // Wait for the thread to exit } @@ -202,5 +217,10 @@ void ExternalCommitHelper::listen() void ExternalCommitHelper::notify_others() { - notify_fd(m_notify_fd); + if (m_notify_fd_write != -1) { + notify_fd(m_notify_fd_write, m_notify_fd); + } + else { + notify_fd(m_notify_fd, m_notify_fd); + } } diff --git a/src/impl/apple/external_commit_helper.hpp b/src/impl/apple/external_commit_helper.hpp index a39876ce..c87d8b24 100644 --- a/src/impl/apple/external_commit_helper.hpp +++ b/src/impl/apple/external_commit_helper.hpp @@ -61,16 +61,20 @@ private: // The listener thread std::future m_thread; - // Read-write file descriptor for the named pipe which is waited on for - // changes and written to when a commit is made + // Pipe which is waited on for changes and written to when there is a new + // commit to notify others of. When using a named pipe m_notify_fd is + // read-write and m_notify_fd_write is unused; when using an anonymous pipe + // (on tvOS) m_notify_fd is read-only and m_notify_fd_write is write-only. FdHolder m_notify_fd; + FdHolder m_notify_fd_write; + // File descriptor for the kqueue FdHolder m_kq; + // The two ends of an anonymous pipe used to notify the kqueue() thread that // it should be shut down. FdHolder m_shutdown_read_fd; FdHolder m_shutdown_write_fd; }; - } // namespace _impl } // namespace realm From 23d9c1c6e84f8f31cb3f0458e13579b3be4762e8 Mon Sep 17 00:00:00 2001 From: kishikawa katsumi Date: Tue, 16 Feb 2016 22:01:19 +0900 Subject: [PATCH 05/93] Split SchemaValidationException into SchemaValidationException and SchemaMismatchException Because SchemaValidationException is thrown both case that a schema definition is incorrect and case that two schema definitions are mismatched. In the former case, the migration does not solve the problem. But the exception message shows "Migration is required..." Therefore the latter as MismatchException, to distinguish between the two cases. --- src/object_store.cpp | 13 +++++++++++-- src/object_store.hpp | 8 ++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/object_store.cpp b/src/object_store.cpp index 94c309e8..565d5ac0 100644 --- a/src/object_store.cpp +++ b/src/object_store.cpp @@ -171,7 +171,7 @@ void ObjectStore::verify_schema(Schema const& actual_schema, Schema& target_sche errors.insert(errors.end(), more_errors.begin(), more_errors.end()); } if (errors.size()) { - throw SchemaValidationException(errors); + throw SchemaMismatchException(errors); } } @@ -524,7 +524,16 @@ DuplicatePrimaryKeyValueException::DuplicatePrimaryKeyValueException(std::string SchemaValidationException::SchemaValidationException(std::vector const& errors) : m_validation_errors(errors) { - m_what ="Migration is required due to the following errors: "; + m_what = "Schema validation failed due to the following errors: "; + for (auto const& error : errors) { + m_what += std::string("\n- ") + error.what(); + } +} + +SchemaMismatchException::SchemaMismatchException(std::vector const& errors) : +m_validation_errors(errors) +{ + m_what = "Migration is required due to the following errors: "; for (auto const& error : errors) { m_what += std::string("\n- ") + error.what(); } diff --git a/src/object_store.hpp b/src/object_store.hpp index bfbc6acc..28b75c89 100644 --- a/src/object_store.hpp +++ b/src/object_store.hpp @@ -160,6 +160,14 @@ namespace realm { std::vector m_validation_errors; }; + class SchemaMismatchException : public ObjectStoreException { + public: + SchemaMismatchException(std::vector const& errors); + std::vector const& validation_errors() const { return m_validation_errors; } + private: + std::vector m_validation_errors; + }; + class ObjectSchemaValidationException : public ObjectStoreException { public: ObjectSchemaValidationException(std::string const& object_type) : m_object_type(object_type) {} From 6380335fc3ddb5e5e2e67cc55e4c57f51e3ce4db Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 26 Jan 2016 18:08:23 -0800 Subject: [PATCH 06/93] Extract out the parts of AsyncQuery not directly related to query running --- src/CMakeLists.txt | 6 +- src/impl/async_query.cpp | 173 ++++------------------------ src/impl/async_query.hpp | 76 +++--------- src/impl/background_collection.cpp | 179 +++++++++++++++++++++++++++++ src/impl/background_collection.hpp | 114 ++++++++++++++++++ src/impl/realm_coordinator.cpp | 4 +- src/impl/realm_coordinator.hpp | 8 +- src/shared_realm.hpp | 8 +- 8 files changed, 343 insertions(+), 225 deletions(-) create mode 100644 src/impl/background_collection.cpp create mode 100644 src/impl/background_collection.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e2e1bd2b..704de016 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,6 +7,7 @@ set(SOURCES schema.cpp shared_realm.cpp impl/async_query.cpp + impl/background_collection.cpp impl/realm_coordinator.cpp impl/transact_log_handler.cpp parser/parser.cpp @@ -20,10 +21,11 @@ set(HEADERS results.hpp schema.hpp shared_realm.hpp - impl/weak_realm_notifier.hpp - impl/weak_realm_notifier_base.hpp + impl/background_collection.hpp impl/external_commit_helper.hpp impl/transact_log_handler.hpp + impl/weak_realm_notifier.hpp + impl/weak_realm_notifier_base.hpp parser/parser.hpp parser/query_builder.hpp util/atomic_shared_ptr.hpp) diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index ecee53c8..ad17000a 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -25,98 +25,19 @@ using namespace realm; using namespace realm::_impl; AsyncQuery::AsyncQuery(Results& target) -: m_target_results(&target) -, m_realm(target.get_realm()) +: BackgroundCollection(target.get_realm()) +, m_target_results(&target) , m_sort(target.get_sort()) -, m_sg_version(Realm::Internal::get_shared_group(*m_realm).get_version_of_current_transaction()) { Query q = target.get_query(); - m_query_handover = Realm::Internal::get_shared_group(*m_realm).export_for_handover(q, MutableSourcePayload::Move); + m_query_handover = Realm::Internal::get_shared_group(get_realm()).export_for_handover(q, MutableSourcePayload::Move); } -AsyncQuery::~AsyncQuery() +void AsyncQuery::release_data() noexcept { - // unregister() may have been called from a different thread than we're being - // destroyed on, so we need to synchronize access to the interesting fields - // modified there - std::lock_guard lock(m_target_mutex); - m_realm = nullptr; -} - -size_t AsyncQuery::add_callback(std::function callback) -{ - m_realm->verify_thread(); - - auto next_token = [=] { - size_t token = 0; - for (auto& callback : m_callbacks) { - if (token <= callback.token) { - token = callback.token + 1; - } - } - return token; - }; - - std::lock_guard lock(m_callback_mutex); - auto token = next_token(); - m_callbacks.push_back({std::move(callback), token, -1ULL}); - if (m_callback_index == npos) { // Don't need to wake up if we're already sending notifications - Realm::Internal::get_coordinator(*m_realm).send_commit_notifications(); - m_have_callbacks = true; - } - return token; -} - -void AsyncQuery::remove_callback(size_t token) -{ - Callback old; - { - std::lock_guard lock(m_callback_mutex); - REALM_ASSERT(m_error || m_callbacks.size() > 0); - - auto it = find_if(begin(m_callbacks), end(m_callbacks), - [=](const auto& c) { return c.token == token; }); - // We should only fail to find the callback if it was removed due to an error - REALM_ASSERT(m_error || it != end(m_callbacks)); - if (it == end(m_callbacks)) { - return; - } - - size_t idx = distance(begin(m_callbacks), it); - if (m_callback_index != npos && m_callback_index >= idx) { - --m_callback_index; - } - - old = std::move(*it); - m_callbacks.erase(it); - - m_have_callbacks = !m_callbacks.empty(); - } -} - -void AsyncQuery::unregister() noexcept -{ - std::lock_guard lock(m_target_mutex); - m_target_results = nullptr; - m_realm = nullptr; -} - -void AsyncQuery::release_query() noexcept -{ - { - std::lock_guard lock(m_target_mutex); - REALM_ASSERT(!m_realm && !m_target_results); - } - m_query = nullptr; } -bool AsyncQuery::is_alive() const noexcept -{ - std::lock_guard lock(m_target_mutex); - return m_target_results != nullptr; -} - // Most of the inter-thread synchronization for run(), prepare_handover(), // attach_to(), detach(), release_query() and deliver() is done by // RealmCoordinator external to this code, which has some potentially @@ -143,12 +64,12 @@ bool AsyncQuery::is_alive() const noexcept void AsyncQuery::run() { - REALM_ASSERT(m_sg); + m_did_change = false; { std::lock_guard target_lock(m_target_mutex); // Don't run the query if the results aren't actually going to be used - if (!m_target_results || (!m_have_callbacks && !m_target_results->wants_background_updates())) { + if (!m_target_results || (!have_callbacks() && !m_target_results->wants_background_updates())) { return; } } @@ -168,33 +89,31 @@ void AsyncQuery::run() if (m_sort) { m_tv.sort(m_sort.columnIndices, m_sort.ascending); } + + m_did_change = true; } -void AsyncQuery::prepare_handover() +bool AsyncQuery::do_prepare_handover(SharedGroup& sg) { - m_sg_version = m_sg->get_version_of_current_transaction(); - if (!m_tv.is_attached()) { - return; + return false; } REALM_ASSERT(m_tv.is_in_sync()); m_initial_run_complete = true; m_handed_over_table_version = m_tv.sync_if_needed(); - m_tv_handover = m_sg->export_for_handover(m_tv, MutableSourcePayload::Move); + m_tv_handover = sg.export_for_handover(m_tv, MutableSourcePayload::Move); // detach the TableView as we won't need it again and keeping it around // makes advance_read() much more expensive m_tv = TableView(); + + return m_did_change; } -bool AsyncQuery::deliver(SharedGroup& sg, std::exception_ptr err) +bool AsyncQuery::do_deliver(SharedGroup& sg) { - if (!is_for_current_thread()) { - return false; - } - std::lock_guard target_lock(m_target_mutex); // Target results being null here indicates that it was destroyed while we @@ -207,84 +126,32 @@ bool AsyncQuery::deliver(SharedGroup& sg, std::exception_ptr err) // We can get called before the query has actually had the chance to run if // we're added immediately before a different set of async results are // delivered - if (!m_initial_run_complete && !err) { + if (!m_initial_run_complete) { return false; } - if (err) { - m_error = err; - return m_have_callbacks; - } - REALM_ASSERT(!m_query_handover); - auto realm_sg_version = Realm::Internal::get_shared_group(*m_realm).get_version_of_current_transaction(); - if (m_sg_version != realm_sg_version) { - // Realm version can be newer if a commit was made on our thread or the - // user manually called refresh(), or older if a commit was made on a - // different thread and we ran *really* fast in between the check for - // if the shared group has changed and when we pick up async results - return false; - } - if (m_tv_handover) { - m_tv_handover->version = m_sg_version; + m_tv_handover->version = version(); Results::Internal::set_table_view(*m_target_results, std::move(*sg.import_from_handover(std::move(m_tv_handover)))); - m_delivered_table_version = m_handed_over_table_version; - } REALM_ASSERT(!m_tv_handover); - return m_have_callbacks; + return have_callbacks(); } -void AsyncQuery::call_callbacks() +void AsyncQuery::do_attach_to(SharedGroup& sg) { - REALM_ASSERT(is_for_current_thread()); - - while (auto fn = next_callback()) { - fn(m_error); - } - - if (m_error) { - // Remove all the callbacks as we never need to call anything ever again - // after delivering an error - std::lock_guard callback_lock(m_callback_mutex); - m_callbacks.clear(); - } -} - -std::function AsyncQuery::next_callback() -{ - std::lock_guard callback_lock(m_callback_mutex); - for (++m_callback_index; m_callback_index < m_callbacks.size(); ++m_callback_index) { - auto& callback = m_callbacks[m_callback_index]; - if (m_error || callback.delivered_version != m_delivered_table_version) { - callback.delivered_version = m_delivered_table_version; - return callback.fn; - } - } - - m_callback_index = npos; - return nullptr; -} - -void AsyncQuery::attach_to(realm::SharedGroup& sg) -{ - REALM_ASSERT(!m_sg); REALM_ASSERT(m_query_handover); - m_query = sg.import_from_handover(std::move(m_query_handover)); - m_sg = &sg; } -void AsyncQuery::detatch() +void AsyncQuery::do_detach_from(SharedGroup& sg) { - REALM_ASSERT(m_sg); REALM_ASSERT(m_query); REALM_ASSERT(!m_tv.is_attached()); - m_query_handover = m_sg->export_for_handover(*m_query, MutableSourcePayload::Move); - m_sg = nullptr; + m_query_handover = sg.export_for_handover(*m_query, MutableSourcePayload::Move); m_query = nullptr; } diff --git a/src/impl/async_query.hpp b/src/impl/async_query.hpp index cf70b463..82776166 100644 --- a/src/impl/async_query.hpp +++ b/src/impl/async_query.hpp @@ -19,6 +19,7 @@ #ifndef REALM_ASYNC_QUERY_HPP #define REALM_ASYNC_QUERY_HPP +#include "background_collection.hpp" #include "results.hpp" #include @@ -31,46 +32,28 @@ namespace realm { namespace _impl { -class AsyncQuery { +class AsyncQuery : public BackgroundCollection { public: AsyncQuery(Results& target); - ~AsyncQuery(); - - size_t add_callback(std::function); - void remove_callback(size_t token); - - void unregister() noexcept; - void release_query() noexcept; - - // Run/rerun the query if needed - void run(); - // Prepare the handover object if run() did update the TableView - void prepare_handover(); - // Update the target results from the handover - // Returns if any callbacks need to be invoked - bool deliver(SharedGroup& sg, std::exception_ptr err); - void call_callbacks(); - - // Attach the handed-over query to `sg` - void attach_to(SharedGroup& sg); - // Create a new query handover object and stop using the previously attached - // SharedGroup - void detatch(); - - Realm& get_realm() { return *m_target_results->get_realm(); } - // Get the version of the current handover object - SharedGroup::VersionID version() const noexcept { return m_sg_version; } - - bool is_alive() const noexcept; private: + // Run/rerun the query if needed + void run() override; + // Prepare the handover object if run() did update the TableView + bool do_prepare_handover(SharedGroup&) override; + // Update the target results from the handover + // Returns if any callbacks need to be invoked + bool do_deliver(SharedGroup& sg) override; + + void release_data() noexcept override; + void do_attach_to(SharedGroup& sg) override; + void do_detach_from(SharedGroup& sg) override; + // Target Results to update and a mutex which guards it mutable std::mutex m_target_mutex; Results* m_target_results; - std::shared_ptr m_realm; const SortOrder m_sort; - const std::thread::id m_thread_id = std::this_thread::get_id(); // The source Query, in handover form iff m_sg is null std::unique_ptr> m_query_handover; @@ -80,40 +63,11 @@ private: // the query was (re)run since the last time the handover object was created TableView m_tv; std::unique_ptr> m_tv_handover; - SharedGroup::VersionID m_sg_version; - std::exception_ptr m_error; - - struct Callback { - std::function fn; - size_t token; - uint_fast64_t delivered_version; - }; - - // Currently registered callbacks and a mutex which must always be held - // while doing anything with them or m_callback_index - std::mutex m_callback_mutex; - std::vector m_callbacks; - - SharedGroup* m_sg = nullptr; uint_fast64_t m_handed_over_table_version = -1; - uint_fast64_t m_delivered_table_version = -1; - - // Iteration variable for looping over callbacks - // remove_callback() updates this when needed - size_t m_callback_index = npos; + bool m_did_change = false; bool m_initial_run_complete = false; - - // Cached value for if m_callbacks is empty, needed to avoid deadlocks in - // run() due to lock-order inversion between m_callback_mutex and m_target_mutex - // It's okay if this value is stale as at worst it'll result in us doing - // some extra work. - std::atomic m_have_callbacks = {false}; - - bool is_for_current_thread() const { return m_thread_id == std::this_thread::get_id(); } - - std::function next_callback(); }; } // namespace _impl diff --git a/src/impl/background_collection.cpp b/src/impl/background_collection.cpp new file mode 100644 index 00000000..f0d57d48 --- /dev/null +++ b/src/impl/background_collection.cpp @@ -0,0 +1,179 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2016 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "impl/background_collection.hpp" + +#include "impl/realm_coordinator.hpp" +#include "shared_realm.hpp" + +using namespace realm; +using namespace realm::_impl; + +BackgroundCollection::BackgroundCollection(std::shared_ptr realm) +: m_realm(std::move(realm)) +, m_sg_version(Realm::Internal::get_shared_group(*m_realm).get_version_of_current_transaction()) +{ +} + +BackgroundCollection::~BackgroundCollection() +{ + // unregister() may have been called from a different thread than we're being + // destroyed on, so we need to synchronize access to the interesting fields + // modified there + std::lock_guard lock(m_realm_mutex); + m_realm = nullptr; +} + +size_t BackgroundCollection::add_callback(CollectionChangeCallback callback) +{ + m_realm->verify_thread(); + + auto next_token = [=] { + size_t token = 0; + for (auto& callback : m_callbacks) { + if (token <= callback.token) { + token = callback.token + 1; + } + } + return token; + }; + + std::lock_guard lock(m_callback_mutex); + auto token = next_token(); + m_callbacks.push_back({std::move(callback), token, -1ULL}); + if (m_callback_index == npos) { // Don't need to wake up if we're already sending notifications + Realm::Internal::get_coordinator(*m_realm).send_commit_notifications(); + m_have_callbacks = true; + } + return token; +} + +void BackgroundCollection::remove_callback(size_t token) +{ + Callback old; + { + std::lock_guard lock(m_callback_mutex); + REALM_ASSERT(m_error || m_callbacks.size() > 0); + + auto it = find_if(begin(m_callbacks), end(m_callbacks), + [=](const auto& c) { return c.token == token; }); + // We should only fail to find the callback if it was removed due to an error + REALM_ASSERT(m_error || it != end(m_callbacks)); + if (it == end(m_callbacks)) { + return; + } + + size_t idx = distance(begin(m_callbacks), it); + if (m_callback_index != npos && m_callback_index >= idx) { + --m_callback_index; + } + + old = std::move(*it); + m_callbacks.erase(it); + + m_have_callbacks = !m_callbacks.empty(); + } +} + +void BackgroundCollection::unregister() noexcept +{ + std::lock_guard lock(m_realm_mutex); + m_realm = nullptr; +} + +bool BackgroundCollection::is_alive() const noexcept +{ + std::lock_guard lock(m_realm_mutex); + return m_realm != nullptr; +} + +void BackgroundCollection::prepare_handover() +{ + REALM_ASSERT(m_sg); + m_sg_version = m_sg->get_version_of_current_transaction(); + if (do_prepare_handover(*m_sg)) + ++m_results_version; +} + +bool BackgroundCollection::deliver(SharedGroup& sg, + std::exception_ptr err) +{ + if (!is_for_current_thread()) { + return false; + } + + if (err) { + m_error = err; + return have_callbacks(); + } + + auto realm_sg_version = sg.get_version_of_current_transaction(); + if (version() != realm_sg_version) { + // Realm version can be newer if a commit was made on our thread or the + // user manually called refresh(), or older if a commit was made on a + // different thread and we ran *really* fast in between the check for + // if the shared group has changed and when we pick up async results + return false; + } + + return do_deliver(sg); +} + +void BackgroundCollection::call_callbacks() +{ + while (auto fn = next_callback()) { + fn(m_error); + } + + if (m_error) { + // Remove all the callbacks as we never need to call anything ever again + // after delivering an error + std::lock_guard callback_lock(m_callback_mutex); + m_callbacks.clear(); + } +} + +CollectionChangeCallback BackgroundCollection::next_callback() +{ + std::lock_guard callback_lock(m_callback_mutex); + for (++m_callback_index; m_callback_index < m_callbacks.size(); ++m_callback_index) { + auto& callback = m_callbacks[m_callback_index]; + if (m_error || callback.delivered_version != m_results_version) { + callback.delivered_version = m_results_version; + return callback.fn; + } + } + + m_callback_index = npos; + return nullptr; +} + +void BackgroundCollection::attach_to(SharedGroup& sg) +{ + REALM_ASSERT(!m_sg); + + m_sg = &sg; + do_attach_to(sg); +} + +void BackgroundCollection::detach() +{ + REALM_ASSERT(m_sg); + do_detach_from(*m_sg); + m_sg = nullptr; +} diff --git a/src/impl/background_collection.hpp b/src/impl/background_collection.hpp new file mode 100644 index 00000000..9927c33e --- /dev/null +++ b/src/impl/background_collection.hpp @@ -0,0 +1,114 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2016 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_BACKGROUND_COLLECTION_HPP +#define REALM_BACKGROUND_COLLECTION_HPP + +#include + +#include +#include +#include +#include + +namespace realm { +class Realm; + +using CollectionChangeCallback = std::function; + +namespace _impl { +class BackgroundCollection { +public: + BackgroundCollection(std::shared_ptr); + virtual ~BackgroundCollection(); + void unregister() noexcept; + + virtual void release_data() noexcept = 0; + + size_t add_callback(CollectionChangeCallback callback); + void remove_callback(size_t token); + + void call_callbacks(); + + bool is_alive() const noexcept; + + Realm& get_realm() const noexcept { return *m_realm; } + + // Attach the handed-over query to `sg` + void attach_to(SharedGroup& sg); + // Create a new query handover object and stop using the previously attached + // SharedGroup + void detach(); + + virtual void run() { } + void prepare_handover(); + bool deliver(SharedGroup&, std::exception_ptr); + + // Get the version of the current handover object + SharedGroup::VersionID version() const noexcept { return m_sg_version; } + +protected: + bool have_callbacks() const noexcept { return m_have_callbacks; } + +private: + virtual void do_attach_to(SharedGroup&) = 0; + virtual void do_detach_from(SharedGroup&) = 0; + virtual bool do_prepare_handover(SharedGroup&) = 0; + virtual bool do_deliver(SharedGroup&) = 0; + + const std::thread::id m_thread_id = std::this_thread::get_id(); + bool is_for_current_thread() const { return m_thread_id == std::this_thread::get_id(); } + + mutable std::mutex m_realm_mutex; + std::shared_ptr m_realm; + + SharedGroup::VersionID m_sg_version; + SharedGroup* m_sg = nullptr; + + std::exception_ptr m_error; + + uint_fast64_t m_results_version = 0; + + struct Callback { + CollectionChangeCallback fn; + size_t token; + uint_fast64_t delivered_version; + }; + + // Currently registered callbacks and a mutex which must always be held + // while doing anything with them or m_callback_index + std::mutex m_callback_mutex; + std::vector m_callbacks; + + // Cached value for if m_callbacks is empty, needed to avoid deadlocks in + // run() due to lock-order inversion between m_callback_mutex and m_target_mutex + // It's okay if this value is stale as at worst it'll result in us doing + // some extra work. + std::atomic m_have_callbacks = {false}; + + // Iteration variable for looping over callbacks + // remove_callback() updates this when needed + size_t m_callback_index = npos; + + CollectionChangeCallback next_callback(); +}; + +} // namespace _impl +} // namespace realm + +#endif /* REALM_BACKGROUND_COLLECTION_HPP */ diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 3c624c32..f4defbf8 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -268,7 +268,7 @@ void RealmCoordinator::clean_up_dead_queries() // Ensure the query is destroyed here even if there's lingering refs // to the async query elsewhere - container[i]->release_query(); + container[i]->release_data(); if (container.size() > i + 1) container[i] = std::move(container.back()); @@ -399,7 +399,7 @@ void RealmCoordinator::advance_helper_shared_group_to_latest() // Transfer all new queries over to the main SG for (auto& query : m_new_queries) { - query->detatch(); + query->detach(); query->attach_to(*m_query_sg); } diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index 05cf6d6c..c79d5bcf 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -32,9 +32,9 @@ class SharedGroup; struct AsyncQueryCancelationToken; namespace _impl { -class AsyncQuery; -class WeakRealmNotifier; +class BackgroundCollection; class ExternalCommitHelper; +class WeakRealmNotifier; // RealmCoordinator manages the weak cache of Realm instances and communication // between per-thread Realm instances for a given file @@ -97,8 +97,8 @@ private: std::vector m_weak_realm_notifiers; std::mutex m_query_mutex; - std::vector> m_new_queries; - std::vector> m_queries; + std::vector> m_new_queries; + std::vector> m_queries; // SharedGroup used for actually running async queries // Will have a read transaction iff m_queries is non-empty diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index ecd1efae..19908deb 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -42,6 +42,7 @@ namespace realm { namespace _impl { class AsyncQuery; + class BackgroundCollection; class RealmCoordinator; } @@ -144,15 +145,16 @@ namespace realm { // without making it public to everyone class Internal { friend class _impl::AsyncQuery; + friend class _impl::BackgroundCollection; friend class _impl::RealmCoordinator; // AsyncQuery needs access to the SharedGroup to be able to call the // handover functions, which are not very wrappable static SharedGroup& get_shared_group(Realm& realm) { return *realm.m_shared_group; } - // AsyncQuery needs to be able to access the owning coordinator to - // wake up the worker thread when a callback is added, and - // coordinators need to be able to get themselves from a Realm + // BackgroundCollection needs to be able to access the owning + // coordinator to wake up the worker thread when a callback is + // added, and coordinators need to be able to get themselves from a Realm static _impl::RealmCoordinator& get_coordinator(Realm& realm) { return *realm.m_coordinator; } }; From 8f7ec856059bc68389d42c84f8527d45f456ea6d Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 25 Feb 2016 08:45:57 -0800 Subject: [PATCH 07/93] Add minimal transaction log parsing tests --- tests/CMakeLists.txt | 1 + tests/transaction_log_parsing.cpp | 86 +++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 tests/transaction_log_parsing.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b757f8ad..9fbb83b8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -9,6 +9,7 @@ set(SOURCES main.cpp parser.cpp results.cpp + transaction_log_parsing.cpp util/test_file.cpp ) diff --git a/tests/transaction_log_parsing.cpp b/tests/transaction_log_parsing.cpp new file mode 100644 index 00000000..e85c6b8c --- /dev/null +++ b/tests/transaction_log_parsing.cpp @@ -0,0 +1,86 @@ +#include "catch.hpp" + +#include "util/test_file.hpp" + +#include "impl/realm_coordinator.hpp" +#include "impl/transact_log_handler.hpp" +#include "property.hpp" +#include "object_schema.hpp" +#include "schema.hpp" + +#include +#include + +using namespace realm; + +TEST_CASE("Transaction log parsing") { + InMemoryTestFile path; + Realm::Config config = path; + + SECTION("schema change validation") { + config.schema = std::make_unique(Schema{ + {"table", "", { + {"unindexed", PropertyTypeInt}, + {"indexed", PropertyTypeInt, "", false, true} + }}, + }); + auto r = Realm::get_shared_realm(std::move(config)); + r->read_group(); + + auto history = make_client_history(config.path); + SharedGroup sg(*history, SharedGroup::durability_MemOnly); + + SECTION("adding a table is allowed") { + WriteTransaction wt(sg); + TableRef table = wt.add_table("new table"); + table->add_column(type_String, "new col"); + wt.commit(); + + REQUIRE_NOTHROW(r->refresh()); + } + + SECTION("adding an index to an existing column is allowed") { + WriteTransaction wt(sg); + TableRef table = wt.get_table("class_table"); + table->add_search_index(0); + wt.commit(); + + REQUIRE_NOTHROW(r->refresh()); + } + + SECTION("removing an index from an existing column is allowed") { + WriteTransaction wt(sg); + TableRef table = wt.get_table("class_table"); + table->remove_search_index(1); + wt.commit(); + + REQUIRE_NOTHROW(r->refresh()); + } + + SECTION("adding a column to an existing table is not allowed (but eventually should be)") { + WriteTransaction wt(sg); + TableRef table = wt.get_table("class_table"); + table->add_column(type_String, "new col"); + wt.commit(); + + REQUIRE_THROWS(r->refresh()); + } + + SECTION("removing a column is not allowed") { + WriteTransaction wt(sg); + TableRef table = wt.get_table("class_table"); + table->remove_column(1); + wt.commit(); + + REQUIRE_THROWS(r->refresh()); + } + + SECTION("removing a table is not allowed") { + WriteTransaction wt(sg); + wt.get_group().remove_table("class_table"); + wt.commit(); + + REQUIRE_THROWS(r->refresh()); + } + } +} From 6609bcaed70e096d4f56402465bf825120bd71fd Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 21 Jan 2016 10:11:19 -0800 Subject: [PATCH 08/93] Add fine-grained notifications for List --- src/CMakeLists.txt | 5 + src/collection_notifications.cpp | 313 ++++++++++ src/collection_notifications.hpp | 91 +++ src/impl/async_query.cpp | 2 +- src/impl/background_collection.cpp | 9 +- src/impl/background_collection.hpp | 12 +- src/impl/list_notifier.cpp | 141 +++++ src/impl/list_notifier.hpp | 55 ++ src/impl/realm_coordinator.cpp | 180 ++++-- src/impl/realm_coordinator.hpp | 25 +- src/impl/transact_log_handler.cpp | 163 +++++- src/impl/transact_log_handler.hpp | 8 + src/index_set.cpp | 302 +++++++++- src/index_set.hpp | 96 +++- src/list.cpp | 14 + src/list.hpp | 9 + src/object_schema.cpp | 1 + src/object_schema.hpp | 48 +- src/results.cpp | 45 +- src/results.hpp | 30 +- src/schema.cpp | 2 + src/schema.hpp | 2 +- src/shared_realm.hpp | 2 + tests/CMakeLists.txt | 3 + tests/collection_change_indices.cpp | 367 ++++++++++++ tests/index_set.cpp | 334 ++++++++--- tests/list.cpp | 225 ++++++++ tests/transaction_log_parsing.cpp | 862 +++++++++++++++++++++++++++- tests/util/index_helpers.hpp | 22 + 29 files changed, 3124 insertions(+), 244 deletions(-) create mode 100644 src/collection_notifications.cpp create mode 100644 src/collection_notifications.hpp create mode 100644 src/impl/list_notifier.cpp create mode 100644 src/impl/list_notifier.hpp create mode 100644 tests/collection_change_indices.cpp create mode 100644 tests/list.cpp create mode 100644 tests/util/index_helpers.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 704de016..a11820e7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,5 @@ set(SOURCES + collection_notifications.cpp index_set.cpp list.cpp object_schema.cpp @@ -8,12 +9,14 @@ set(SOURCES shared_realm.cpp impl/async_query.cpp impl/background_collection.cpp + impl/list_notifier.cpp impl/realm_coordinator.cpp impl/transact_log_handler.cpp parser/parser.cpp parser/query_builder.cpp) set(HEADERS + collection_notifications.hpp index_set.hpp list.hpp object_schema.hpp @@ -23,6 +26,8 @@ set(HEADERS shared_realm.hpp impl/background_collection.hpp impl/external_commit_helper.hpp + impl/list_notifier.hpp + impl/realm_coordinator.hpp impl/transact_log_handler.hpp impl/weak_realm_notifier.hpp impl/weak_realm_notifier_base.hpp diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp new file mode 100644 index 00000000..57cdd577 --- /dev/null +++ b/src/collection_notifications.cpp @@ -0,0 +1,313 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2016 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "collection_notifications.hpp" + +#include "impl/background_collection.hpp" + +#include +#include +#include + +using namespace realm; + +NotificationToken::NotificationToken(std::shared_ptr<_impl::BackgroundCollection> query, size_t token) +: m_query(std::move(query)), m_token(token) +{ +} + +NotificationToken::~NotificationToken() +{ + // m_query itself (and not just the pointed-to thing) needs to be accessed + // atomically to ensure that there are no data races when the token is + // destroyed after being modified on a different thread. + // This is needed despite the token not being thread-safe in general as + // users find it very surpringing for obj-c objects to care about what + // thread they are deallocated on. + if (auto query = m_query.exchange({})) { + query->remove_callback(m_token); + } +} + +NotificationToken::NotificationToken(NotificationToken&& rgt) = default; + +NotificationToken& NotificationToken::operator=(realm::NotificationToken&& rgt) +{ + if (this != &rgt) { + if (auto query = m_query.exchange({})) { + query->remove_callback(m_token); + } + m_query = std::move(rgt.m_query); + m_token = rgt.m_token; + } + return *this; +} + +CollectionChangeIndices::CollectionChangeIndices(IndexSet deletions, + IndexSet insertions, + IndexSet modifications, + std::vector moves) +: deletions(std::move(deletions)) +, insertions(std::move(insertions)) +, modifications(std::move(modifications)) +, moves(std::move(moves)) +{ + for (auto&& move : this->moves) { + this->deletions.add(move.from); + this->insertions.add(move.to); + } + verify(); +} + +void CollectionChangeIndices::merge(realm::CollectionChangeIndices&& c) +{ + if (c.empty()) + return; + if (empty()) { + *this = std::move(c); + return; + } + + verify(); + c.verify(); + + // First update any old moves + if (!c.moves.empty() || !c.deletions.empty() || !c.insertions.empty()) { + auto it = remove_if(begin(moves), end(moves), [&](auto& old) { + // Check if the moved row was moved again, and if so just update the destination + auto it = find_if(begin(c.moves), end(c.moves), [&](auto const& m) { + return old.to == m.from; + }); + if (it != c.moves.end()) { + old.to = it->to; + *it = c.moves.back(); + c.moves.pop_back(); + ++it; + return false; + } + + // Check if the destination was deleted + if (c.deletions.contains(old.to)) + return true; + + // Update the destination to adjust for any new insertions and deletions + old.to = c.insertions.shift(c.deletions.unshift(old.to)); + return false; + }); + moves.erase(it, end(moves)); + } + + // Ignore new moves of rows which were previously inserted (the implicit + // delete from the move will remove the insert) + if (!insertions.empty()) { + c.moves.erase(remove_if(begin(c.moves), end(c.moves), + [&](auto const& m) { return insertions.contains(m.from); }), + end(c.moves)); + } + + // Update the source position of new moves to compensate for the changes made + // in the old changeset + if (!deletions.empty() || !insertions.empty()) { + for (auto& move : c.moves) + move.from = deletions.shift(insertions.unshift(move.from)); + } + + moves.insert(end(moves), begin(c.moves), end(c.moves)); + + // New deletion indices have been shifted by the insertions, so unshift them + // before adding + deletions.add_shifted_by(insertions, c.deletions); + + // Drop any inserted-then-deleted rows, then merge in new insertions + insertions.erase_at(c.deletions); + insertions.insert_at(c.insertions); + + // Ignore new mmodifications to previously inserted rows + c.modifications.remove(insertions); + + modifications.erase_at(c.deletions); + modifications.shift_for_insert_at(c.insertions); + modifications.add(c.modifications); + + c = {}; + verify(); +} + +void CollectionChangeIndices::modify(size_t ndx) +{ + if (!insertions.contains(ndx)) + modifications.add(ndx); + // FIXME: this breaks mapping old row indices to new + // FIXME: is that a problem? + // If this row was previously moved, unmark it as a move + moves.erase(remove_if(begin(moves), end(moves), + [&](auto move) { return move.to == ndx; }), + end(moves)); +} + +void CollectionChangeIndices::insert(size_t index, size_t count) +{ + modifications.shift_for_insert_at(index, count); + insertions.insert_at(index, count); + + for (auto& move : moves) { + if (move.to >= index) + ++move.to; + } +} + +void CollectionChangeIndices::erase(size_t index) +{ + modifications.erase_at(index); + size_t unshifted = insertions.erase_and_unshift(index); + if (unshifted != npos) + deletions.add_shifted(unshifted); + + for (size_t i = 0; i < moves.size(); ++i) { + auto& move = moves[i]; + if (move.to == index) { + moves.erase(moves.begin() + i); + --i; + } + else if (move.to > index) + --move.to; + } +} + +void CollectionChangeIndices::clear(size_t old_size) +{ + for (auto range : deletions) + old_size += range.second - range.first; + for (auto range : insertions) + old_size -= range.second - range.first; + + modifications.clear(); + insertions.clear(); + moves.clear(); + deletions.set(old_size); +} + +void CollectionChangeIndices::move(size_t from, size_t to) +{ + REALM_ASSERT(from != to); + + bool updated_existing_move = false; + for (auto& move : moves) { + if (move.to != from) { + // Shift other moves if this row is moving from one side of them + // to the other + if (move.to >= to && move.to < from) + ++move.to; + else if (move.to < to && move.to > from) + --move.to; + continue; + } + REALM_ASSERT(!updated_existing_move); + + // Collapse A -> B, B -> C into a single A -> C move + move.to = to; + modifications.erase_at(from); + insertions.erase_at(from); + + modifications.shift_for_insert_at(to); + insertions.insert_at(to); + updated_existing_move = true; + } + if (updated_existing_move) + return; + + if (!insertions.contains(from)) { + auto shifted_from = insertions.unshift(from); + shifted_from = deletions.add_shifted(shifted_from); + + // Don't record it as a move if the source row was newly inserted or + // was previously changed + if (!modifications.contains(from)) + moves.push_back({shifted_from, to}); + } + + modifications.erase_at(from); + insertions.erase_at(from); + + modifications.shift_for_insert_at(to); + insertions.insert_at(to); +} + +void CollectionChangeIndices::move_over(size_t row_ndx, size_t last_row) +{ + REALM_ASSERT(row_ndx <= last_row); + if (row_ndx == last_row) { + erase(row_ndx); + return; + } + + bool updated_existing_move = false; + for (size_t i = 0; i < moves.size(); ++i) { + auto& move = moves[i]; + REALM_ASSERT(move.to <= last_row); + + if (move.to == row_ndx) { + REALM_ASSERT(!updated_existing_move); + moves[i] = moves.back(); + moves.pop_back(); + --i; + updated_existing_move = true; + } + else if (move.to == last_row) { + REALM_ASSERT(!updated_existing_move); + move.to = row_ndx; + updated_existing_move = true; + } + } + if (!updated_existing_move) { + moves.push_back({last_row, row_ndx}); + } + + if (insertions.contains(row_ndx)) { + insertions.remove(row_ndx); + } + else { + if (modifications.contains(row_ndx)) { + modifications.remove(row_ndx); + } + deletions.add(row_ndx); + } + + if (insertions.contains(last_row)) { + insertions.remove(last_row); + insertions.add(row_ndx); + } + else if (modifications.contains(last_row)) { + modifications.remove(last_row); + modifications.add(row_ndx); + } +} + +void CollectionChangeIndices::verify() +{ +#ifdef REALM_DEBUG + for (auto&& move : moves) { + REALM_ASSERT(deletions.contains(move.from)); + REALM_ASSERT(insertions.contains(move.to)); + } + for (auto index : modifications.as_indexes()) + REALM_ASSERT(!insertions.contains(index)); + for (auto index : insertions.as_indexes()) + REALM_ASSERT(!modifications.contains(index)); +#endif +} diff --git a/src/collection_notifications.hpp b/src/collection_notifications.hpp new file mode 100644 index 00000000..b1543221 --- /dev/null +++ b/src/collection_notifications.hpp @@ -0,0 +1,91 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2016 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_COLLECTION_NOTIFICATIONS_HPP +#define REALM_COLLECTION_NOTIFICATIONS_HPP + +#include "index_set.hpp" +#include "util/atomic_shared_ptr.hpp" + +#include +#include + +namespace realm { +namespace _impl { + class BackgroundCollection; +} + +// A token which keeps an asynchronous query alive +struct NotificationToken { + NotificationToken() = default; + NotificationToken(std::shared_ptr<_impl::BackgroundCollection> query, size_t token); + ~NotificationToken(); + + NotificationToken(NotificationToken&&); + NotificationToken& operator=(NotificationToken&&); + + NotificationToken(NotificationToken const&) = delete; + NotificationToken& operator=(NotificationToken const&) = delete; + +private: + util::AtomicSharedPtr<_impl::BackgroundCollection> m_query; + size_t m_token; +}; + +struct CollectionChangeIndices { + struct Move { + size_t from; + size_t to; + + bool operator==(Move m) const { return from == m.from && to == m.to; } + }; + + IndexSet deletions; + IndexSet insertions; + IndexSet modifications; + std::vector moves; + + CollectionChangeIndices(CollectionChangeIndices const&) = default; + CollectionChangeIndices(CollectionChangeIndices&&) = default; + CollectionChangeIndices& operator=(CollectionChangeIndices const&) = default; + CollectionChangeIndices& operator=(CollectionChangeIndices&&) = default; + + CollectionChangeIndices(IndexSet deletions = {}, + IndexSet insertions = {}, + IndexSet modification = {}, + std::vector moves = {}); + + bool empty() const { return deletions.empty() && insertions.empty() && modifications.empty() && moves.empty(); } + + void merge(CollectionChangeIndices&&); + + void insert(size_t ndx, size_t count=1); + void modify(size_t ndx); + void erase(size_t ndx); + void move_over(size_t ndx, size_t last_ndx); + void clear(size_t old_size); + void move(size_t from, size_t to); + +private: + void verify(); +}; + +using CollectionChangeCallback = std::function; +} + +#endif // REALM_COLLECTION_NOTIFICATIONS_HPP diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index ad17000a..9f09f2f5 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -87,7 +87,7 @@ void AsyncQuery::run() m_tv = m_query->find_all(); if (m_sort) { - m_tv.sort(m_sort.columnIndices, m_sort.ascending); + m_tv.sort(m_sort.column_indices, m_sort.ascending); } m_did_change = true; diff --git a/src/impl/background_collection.cpp b/src/impl/background_collection.cpp index f0d57d48..a3e932a3 100644 --- a/src/impl/background_collection.cpp +++ b/src/impl/background_collection.cpp @@ -110,8 +110,7 @@ void BackgroundCollection::prepare_handover() ++m_results_version; } -bool BackgroundCollection::deliver(SharedGroup& sg, - std::exception_ptr err) +bool BackgroundCollection::deliver(SharedGroup& sg, std::exception_ptr err) { if (!is_for_current_thread()) { return false; @@ -131,13 +130,15 @@ bool BackgroundCollection::deliver(SharedGroup& sg, return false; } - return do_deliver(sg); + bool ret = do_deliver(sg); + m_changes_to_deliver = std::move(m_accumulated_changes); + return ret; } void BackgroundCollection::call_callbacks() { while (auto fn = next_callback()) { - fn(m_error); + fn(m_changes_to_deliver, m_error); } if (m_error) { diff --git a/src/impl/background_collection.hpp b/src/impl/background_collection.hpp index 9927c33e..e3508e78 100644 --- a/src/impl/background_collection.hpp +++ b/src/impl/background_collection.hpp @@ -19,6 +19,8 @@ #ifndef REALM_BACKGROUND_COLLECTION_HPP #define REALM_BACKGROUND_COLLECTION_HPP +#include "collection_notifications.hpp" + #include #include @@ -29,9 +31,9 @@ namespace realm { class Realm; -using CollectionChangeCallback = std::function; - namespace _impl { +struct TransactionChangeInfo; + class BackgroundCollection { public: BackgroundCollection(std::shared_ptr); @@ -55,6 +57,8 @@ public: // SharedGroup void detach(); + virtual void add_required_change_info(TransactionChangeInfo&) { } + virtual void run() { } void prepare_handover(); bool deliver(SharedGroup&, std::exception_ptr); @@ -64,6 +68,8 @@ public: protected: bool have_callbacks() const noexcept { return m_have_callbacks; } + bool have_changes() const noexcept { return !m_accumulated_changes.empty(); } + void add_changes(CollectionChangeIndices change) { m_accumulated_changes.merge(std::move(change)); } private: virtual void do_attach_to(SharedGroup&) = 0; @@ -81,6 +87,8 @@ private: SharedGroup* m_sg = nullptr; std::exception_ptr m_error; + CollectionChangeIndices m_accumulated_changes; + CollectionChangeIndices m_changes_to_deliver; uint_fast64_t m_results_version = 0; diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp new file mode 100644 index 00000000..78c0d7ef --- /dev/null +++ b/src/impl/list_notifier.cpp @@ -0,0 +1,141 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "impl/list_notifier.hpp" + +#include "impl/realm_coordinator.hpp" +#include "shared_realm.hpp" + +#include + +using namespace realm; +using namespace realm::_impl; + +// Recursively add `table` and all tables it links to to `out` +static void find_relevant_tables(std::vector& out, Table const& table) +{ + auto table_ndx = table.get_index_in_group(); + if (find(begin(out), end(out), table_ndx) != end(out)) + return; + out.push_back(table_ndx); + + for (size_t i = 0, count = table.get_column_count(); i != count; ++i) { + if (table.get_column_type(i) == type_Link || table.get_column_type(i) == type_LinkList) { + find_relevant_tables(out, *table.get_link_target(i)); + } + } +} + + +ListNotifier::ListNotifier(LinkViewRef lv, std::shared_ptr realm) +: BackgroundCollection(std::move(realm)) +, m_prev_size(lv->size()) +{ + find_relevant_tables(m_relevant_tables, lv->get_target_table()); + + // Find the lv's column, since that isn't tracked directly + size_t row_ndx = lv->get_origin_row_index(); + m_col_ndx = not_found; + auto& table = lv->get_origin_table(); + for (size_t i = 0, count = table.get_column_count(); i != count; ++i) { + if (table.get_column_type(i) == type_LinkList && table.get_linklist(i, row_ndx) == lv) { + m_col_ndx = i; + break; + } + } + REALM_ASSERT(m_col_ndx != not_found); + + auto& sg = Realm::Internal::get_shared_group(get_realm()); + m_lv_handover = sg.export_linkview_for_handover(lv); +} + +void ListNotifier::release_data() noexcept +{ + // FIXME: does this need a lock? + m_lv.reset(); +} + +void ListNotifier::do_attach_to(SharedGroup& sg) +{ + REALM_ASSERT(m_lv_handover); + REALM_ASSERT(!m_lv); + m_lv = sg.import_linkview_from_handover(std::move(m_lv_handover)); +} + +void ListNotifier::do_detach_from(SharedGroup& sg) +{ + REALM_ASSERT(!m_lv_handover); + if (m_lv) { + m_lv_handover = sg.export_linkview_for_handover(m_lv); + m_lv = {}; + } +} + +void ListNotifier::add_required_change_info(TransactionChangeInfo& info) +{ + REALM_ASSERT(!m_lv_handover); + if (!m_lv) { + return; // origin row was deleted after the notification was added + } + + size_t row_ndx = m_lv->get_origin_row_index(); + auto& table = m_lv->get_origin_table(); + info.lists.push_back({table.get_index_in_group(), row_ndx, m_col_ndx, &m_change}); + + auto max = *max_element(begin(m_relevant_tables), end(m_relevant_tables)) + 1; + if (max > info.tables_needed.size()) + info.tables_needed.resize(max, false); + for (auto table_ndx : m_relevant_tables) { + info.tables_needed[table_ndx] = true; + } + + m_info = &info; +} + +void ListNotifier::run() +{ + if (!m_lv) { + // LV was deleted, so report all of the rows being removed if this is + // the first run after that + if (m_prev_size) { + m_change.deletions.set(m_prev_size); + m_prev_size = 0; + } + return; + } + + for (size_t i = 0; i < m_lv->size(); ++i) { + if (m_change.insertions.contains(i) || m_change.modifications.contains(i)) + continue; + if (m_info->row_did_change(m_lv->get_target_table(), m_lv->get(i).get_index())) + m_change.modifications.add(i); + } + + m_prev_size = m_lv->size(); +} + +bool ListNotifier::do_prepare_handover(SharedGroup&) +{ + add_changes(std::move(m_change)); + return true; +} + +bool ListNotifier::do_deliver(SharedGroup&) +{ + return have_callbacks() && have_changes(); +} diff --git a/src/impl/list_notifier.hpp b/src/impl/list_notifier.hpp new file mode 100644 index 00000000..6c666bf7 --- /dev/null +++ b/src/impl/list_notifier.hpp @@ -0,0 +1,55 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_LIST_NOTIFIER_HPP +#define REALM_LIST_NOTIFIER_HPP + +#include "impl/background_collection.hpp" + +#include + +namespace realm { +namespace _impl { +class ListNotifier : public BackgroundCollection { +public: + ListNotifier(LinkViewRef lv, std::shared_ptr realm); + +private: + LinkViewRef m_lv; + std::unique_ptr> m_lv_handover; + CollectionChangeIndices m_change; + size_t m_prev_size; + size_t m_col_ndx; + std::vector m_relevant_tables; + TransactionChangeInfo* m_info; + + void run() override; + + bool do_deliver(SharedGroup& sg) override; + bool do_prepare_handover(SharedGroup&) override; + + void do_attach_to(SharedGroup& sg) override; + void do_detach_from(SharedGroup& sg) override; + + void release_data() noexcept override; + void add_required_change_info(TransactionChangeInfo& info) override; +}; +} +} + +#endif // REALM_LIST_NOTIFIER_HPP diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index f4defbf8..a6f5664b 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -21,6 +21,7 @@ #include "impl/async_query.hpp" #include "impl/weak_realm_notifier.hpp" #include "impl/external_commit_helper.hpp" +#include "impl/list_notifier.hpp" #include "impl/transact_log_handler.hpp" #include "object_store.hpp" #include "schema.hpp" @@ -32,11 +33,45 @@ #include #include +#include #include using namespace realm; using namespace realm::_impl; +bool TransactionChangeInfo::row_did_change(Table const& table, size_t idx, int depth) const +{ + if (depth > 16) // arbitrary limit + return false; + + size_t table_ndx = table.get_index_in_group(); + if (table_ndx < tables.size() && tables[table_ndx].modifications.contains(idx)) + return true; + + for (size_t i = 0, count = table.get_column_count(); i < count; ++i) { + auto type = table.get_column_type(i); + if (type == type_Link) { + auto& target = *table.get_link_target(i); + if (target.is_null_link(i, idx)) + continue; + auto dst = table.get_link(i, idx); + return row_did_change(target, dst, depth + 1); + } + if (type != type_LinkList) + continue; + + auto& target = *table.get_link_target(i); + auto lvr = table.get_linklist(i, idx); + for (size_t j = 0; j < lvr->size(); ++j) { + size_t dst = lvr->get(j).get_index(); + if (row_did_change(target, dst, depth + 1)) + return true; + } + } + + return false; +} + static std::mutex s_coordinator_mutex; static std::unordered_map> s_coordinators_per_path; @@ -247,7 +282,7 @@ void RealmCoordinator::pin_version(uint_fast64_t version, uint_fast32_t index) } } -void RealmCoordinator::register_query(std::shared_ptr query) +void RealmCoordinator::register_query(std::shared_ptr query) { auto version = query->version(); auto& self = Realm::Internal::get_coordinator(query->get_realm()); @@ -319,30 +354,116 @@ void RealmCoordinator::run_async_queries() } if (m_async_error) { - move_new_queries_to_main(); + std::move(m_new_queries.begin(), m_new_queries.end(), std::back_inserter(m_queries)); + m_new_queries.clear(); return; } - advance_helper_shared_group_to_latest(); + std::vector change_info; + SharedGroup::VersionID version; - // Make a copy of the queries vector so that we can release the lock while - // we run the queries - auto queries_to_run = m_queries; + auto new_queries = std::move(m_new_queries); + if (new_queries.empty()) { + change_info.resize(1); + } + else { + change_info.resize(2); + + // Sort newly added queries by their source version so that we can pull them + // all forward to the latest version in a single pass over the transaction log + std::sort(new_queries.begin(), new_queries.end(), + [](auto&& lft, auto&& rgt) { return lft->version() < rgt->version(); }); + version = m_advancer_sg->get_version_of_current_transaction(); + REALM_ASSERT(version == new_queries.front()->version()); + + TransactionChangeInfo* info = &change_info.back(); + + // Advance each of the new queries to the latest version, attaching them + // to the SG at their handover version. This requires a unique + // TransactionChangeInfo for each source version, so that things don't + // see changes from before the version they were handed over from. + // Each Info has all of the changes between that source version and the + // next source version, and they'll be merged together later after + // releasing the lock + for (auto& query : new_queries) { + if (version != query->version()) { + transaction::advance_and_observe_linkviews(*m_advancer_sg, *info, query->version()); + change_info.push_back({{}, std::move(info->lists)}); + info = &change_info.back(); + version = query->version(); + } + query->attach_to(*m_advancer_sg); + query->add_required_change_info(*info); + } + + transaction::advance_and_observe_linkviews(*m_advancer_sg, *info); + + for (auto& query : new_queries) { + query->detach(); + } + version = m_advancer_sg->get_version_of_current_transaction(); + m_advancer_sg->end_read(); + } + + // Make a copy of the queries vector and then release the lock to avoid + // blocking other threads trying to register or unregister queries while we run them + auto queries = m_queries; lock.unlock(); - for (auto& query : queries_to_run) { + for (auto& query : queries) { + query->add_required_change_info(change_info[0]); + } + + transaction::advance_and_observe_linkviews(*m_query_sg, change_info[0], version); + + // Attach the new queries to the main SG and move them to the main list + for (auto& query : new_queries) { + query->attach_to(*m_query_sg); + } + std::move(new_queries.begin(), new_queries.end(), std::back_inserter(queries)); + + for (size_t i = change_info.size() - 1; i > 1; --i) { + auto& cur = change_info[i]; + if (cur.tables.empty()) + continue; + auto& prev = change_info[i - 1]; + if (prev.tables.empty()) { + prev.tables = cur.tables; + continue; + } + + for (size_t j = 0; j < prev.tables.size() && j < cur.tables.size(); ++j) { + prev.tables[j].merge(CollectionChangeIndices{cur.tables[j]}); + } + prev.tables.reserve(cur.tables.size()); + while (prev.tables.size() < cur.tables.size()) { + prev.tables.push_back(cur.tables[prev.tables.size()]); + } + } + + // Copy the list change info if there's multiple LinkViews for the same LinkList + auto id = [](auto const& list) { return std::tie(list.table_ndx, list.col_ndx, list.row_ndx); }; + for (auto& info : change_info) { + for (size_t i = 1; i < info.lists.size(); ++i) { + for (size_t j = i; j > 0; --j) { + if (id(info.lists[i]) == id(info.lists[j - 1])) { + info.lists[j - 1].changes->merge(CollectionChangeIndices{*info.lists[i].changes}); + } + } + } + } + + for (auto& query : queries) { query->run(); } // Reacquire the lock while updating the fields that are actually read on // other threads - { - lock.lock(); - for (auto& query : queries_to_run) { - query->prepare_handover(); - } + lock.lock(); + for (auto& query : queries) { + query->prepare_handover(); } - + m_queries = std::move(queries); clean_up_dead_queries(); } @@ -374,39 +495,6 @@ void RealmCoordinator::move_new_queries_to_main() m_new_queries.clear(); } -void RealmCoordinator::advance_helper_shared_group_to_latest() -{ - if (m_new_queries.empty()) { - LangBindHelper::advance_read(*m_query_sg); - return; - } - - // Sort newly added queries by their source version so that we can pull them - // all forward to the latest version in a single pass over the transaction log - std::sort(m_new_queries.begin(), m_new_queries.end(), [](auto const& lft, auto const& rgt) { - return lft->version() < rgt->version(); - }); - - // Import all newly added queries to our helper SG - for (auto& query : m_new_queries) { - LangBindHelper::advance_read(*m_advancer_sg, query->version()); - query->attach_to(*m_advancer_sg); - } - - // Advance both SGs to the newest version - LangBindHelper::advance_read(*m_advancer_sg); - LangBindHelper::advance_read(*m_query_sg, m_advancer_sg->get_version_of_current_transaction()); - - // Transfer all new queries over to the main SG - for (auto& query : m_new_queries) { - query->detach(); - query->attach_to(*m_query_sg); - } - - move_new_queries_to_main(); - m_advancer_sg->end_read(); -} - void RealmCoordinator::advance_to_ready(Realm& realm) { decltype(m_queries) queries; diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index c79d5bcf..905c17b8 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -19,23 +19,42 @@ #ifndef REALM_COORDINATOR_HPP #define REALM_COORDINATOR_HPP +#include "index_set.hpp" #include "shared_realm.hpp" #include +#include + namespace realm { -class AsyncQueryCallback; class Replication; class Results; class Schema; class SharedGroup; -struct AsyncQueryCancelationToken; +class Table; +struct CollectionChangeIndices; namespace _impl { class BackgroundCollection; class ExternalCommitHelper; +class ListNotifier; class WeakRealmNotifier; +struct ListChangeInfo { + size_t table_ndx; + size_t row_ndx; + size_t col_ndx; + CollectionChangeIndices *changes; +}; + +struct TransactionChangeInfo { + std::vector tables_needed; + std::vector lists; + std::vector tables; + + bool row_did_change(Table const& table, size_t row_ndx, int depth = 0) const; +}; + // RealmCoordinator manages the weak cache of Realm instances and communication // between per-thread Realm instances for a given file class RealmCoordinator : public std::enable_shared_from_this { @@ -83,7 +102,7 @@ public: // Update the schema in the cached config void update_schema(Schema const& new_schema); - static void register_query(std::shared_ptr query); + static void register_query(std::shared_ptr query); // Advance the Realm to the most recent transaction version which all async // work is complete for diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index d95aa32d..1dd75e0f 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -19,6 +19,8 @@ #include "impl/transact_log_handler.hpp" #include "binding_context.hpp" +#include "collection_notifications.hpp" +#include "impl/realm_coordinator.hpp" #include #include @@ -429,6 +431,158 @@ public: bool insert_substring(size_t col, size_t row, size_t, StringData) { return mark_dirty(row, col); } bool erase_substring(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } }; + +// Extends TransactLogValidator to track changes made to LinkViews +class LinkViewObserver : public TransactLogValidator { + _impl::TransactionChangeInfo& m_info; + CollectionChangeIndices* m_active = nullptr; + + CollectionChangeIndices* get_change() + { + auto tbl_ndx = current_table(); + if (tbl_ndx >= m_info.tables_needed.size() || !m_info.tables_needed[tbl_ndx]) + return nullptr; + if (m_info.tables.size() <= tbl_ndx) { + m_info.tables.resize(std::max(m_info.tables.size() * 2, tbl_ndx + 1)); + } + return &m_info.tables[tbl_ndx]; + } + + bool mark_dirty(size_t row, __unused size_t col) + { + if (auto change = get_change()) + change->modify(row); + return true; + } + +public: + LinkViewObserver(_impl::TransactionChangeInfo& info) + : m_info(info) { } + + bool select_link_list(size_t col, size_t row, size_t) + { + mark_dirty(row, col); + + m_active = nullptr; + for (auto& o : m_info.lists) { + if (o.table_ndx == current_table() && o.row_ndx == row && o.col_ndx == col) { + m_active = o.changes; + // need to use last match for multiple source version logic +// break; + } + } + return true; + } + + bool link_list_set(size_t index, size_t) + { + if (m_active) + m_active->modify(index); + return true; + } + + bool link_list_insert(size_t index, size_t) + { + if (m_active) + m_active->insert(index); + return true; + } + + bool link_list_erase(size_t index) + { + if (m_active) + m_active->erase(index); + return true; + } + + bool link_list_nullify(size_t index) + { + return link_list_erase(index); + } + + bool link_list_swap(size_t index1, size_t index2) + { + link_list_set(index1, 0); + link_list_set(index2, 0); + return true; + } + + bool link_list_clear(size_t old_size) + { + if (m_active) + m_active->clear(old_size); + return true; + } + + bool link_list_move(size_t from, size_t to) + { + if (m_active) + m_active->move(from, to); + return true; + } + + bool insert_empty_rows(size_t row_ndx, size_t num_rows_to_insert, size_t, bool unordered) + { + REALM_ASSERT(!unordered); + if (auto change = get_change()) + change->insert(row_ndx, num_rows_to_insert); + + return true; + } + + bool erase_rows(size_t row_ndx, size_t, size_t prior_num_rows, bool unordered) + { + REALM_ASSERT(unordered); + size_t last_row = prior_num_rows - 1; + + for (auto it = begin(m_info.lists); it != end(m_info.lists); ) { + if (it->table_ndx == current_table()) { + if (it->row_ndx == row_ndx) { + *it = std::move(m_info.lists.back()); + m_info.lists.pop_back(); + continue; + } + if (it->row_ndx == last_row - 1) + it->row_ndx = row_ndx; + } + ++it; + } + + if (auto change = get_change()) + change->move_over(row_ndx, last_row); + return true; + } + + bool clear_table() + { + auto tbl_ndx = current_table(); + auto it = remove_if(begin(m_info.lists), end(m_info.lists), + [&](auto const& lv) { return lv.table_ndx == tbl_ndx; }); + m_info.lists.erase(it, end(m_info.lists)); + if (auto change = get_change()) + change->clear(0); // FIXME + return true; + } + + // Things that just mark the field as modified + bool set_int(size_t col, size_t row, int_fast64_t) { return mark_dirty(row, col); } + bool set_bool(size_t col, size_t row, bool) { return mark_dirty(row, col); } + bool set_float(size_t col, size_t row, float) { return mark_dirty(row, col); } + bool set_double(size_t col, size_t row, double) { return mark_dirty(row, col); } + bool set_string(size_t col, size_t row, StringData) { return mark_dirty(row, col); } + bool set_binary(size_t col, size_t row, BinaryData) { return mark_dirty(row, col); } + bool set_olddatetime(size_t col, size_t row, OldDateTime) { return mark_dirty(row, col); } + bool set_timestamp(size_t col, size_t row, Timestamp) { return mark_dirty(row, col); } + bool set_table(size_t col, size_t row) { return mark_dirty(row, col); } + bool set_mixed(size_t col, size_t row, const Mixed&) { return mark_dirty(row, col); } + bool set_link(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } + bool set_null(size_t col, size_t row) { return mark_dirty(row, col); } + bool nullify_link(size_t col, size_t row, size_t) { return mark_dirty(row, col); } + bool insert_substring(size_t col, size_t row, size_t, StringData) { return mark_dirty(row, col); } + bool erase_substring(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } + bool set_int_unique(size_t col, size_t row, size_t, int_fast64_t) { return mark_dirty(row, col); } + bool set_string_unique(size_t col, size_t row, size_t, StringData) { return mark_dirty(row, col); } +}; } // anonymous namespace namespace realm { @@ -437,7 +591,7 @@ namespace transaction { void advance(SharedGroup& sg, BindingContext* context, SharedGroup::VersionID version) { TransactLogObserver(context, sg, [&](auto&&... args) { - LangBindHelper::advance_read(sg, std::move(args)...); + LangBindHelper::advance_read(sg, std::move(args)..., version); }, true); } @@ -464,6 +618,13 @@ void cancel(SharedGroup& sg, BindingContext* context) }, false); } +void advance_and_observe_linkviews(SharedGroup& sg, + TransactionChangeInfo& info, + SharedGroup::VersionID version) +{ + LangBindHelper::advance_read(sg, LinkViewObserver(info), version); +} + } // namespace transaction } // namespace _impl } // namespace realm diff --git a/src/impl/transact_log_handler.hpp b/src/impl/transact_log_handler.hpp index 4249f8f3..278c62db 100644 --- a/src/impl/transact_log_handler.hpp +++ b/src/impl/transact_log_handler.hpp @@ -21,11 +21,15 @@ #include +#include "index_set.hpp" + namespace realm { class BindingContext; class SharedGroup; namespace _impl { +struct TransactionChangeInfo; + namespace transaction { // Advance the read transaction version, with change notifications sent to delegate // Must not be called from within a write transaction. @@ -44,6 +48,10 @@ void commit(SharedGroup& sg, BindingContext* binding_context); // Cancel a write transaction and roll back all changes, with change notifications // for reverting to the old values sent to delegate void cancel(SharedGroup& sg, BindingContext* binding_context); + +void advance_and_observe_linkviews(SharedGroup& sg, + TransactionChangeInfo& info, + SharedGroup::VersionID version=SharedGroup::VersionID{}); } // namespace transaction } // namespace _impl } // namespace realm diff --git a/src/index_set.cpp b/src/index_set.cpp index c244f76a..23e529fe 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -18,11 +18,32 @@ #include "index_set.hpp" +#include + using namespace realm; +const size_t IndexSet::npos; + +IndexSet::IndexSet(std::initializer_list values) +{ + for (size_t v : values) + add(v); +} + +bool IndexSet::contains(size_t index) const +{ + auto it = const_cast(this)->find(index); + return it != m_ranges.end() && it->first <= index; +} + IndexSet::iterator IndexSet::find(size_t index) { - for (auto it = m_ranges.begin(), end = m_ranges.end(); it != end; ++it) { + return find(index, m_ranges.begin()); +} + +IndexSet::iterator IndexSet::find(size_t index, iterator it) +{ + for (auto end = m_ranges.end(); it != end; ++it) { if (it->second > index) return it; } @@ -34,29 +55,39 @@ void IndexSet::add(size_t index) do_add(find(index), index); } -void IndexSet::do_add(iterator it, size_t index) +void IndexSet::add(IndexSet const& other) { - bool more_before = it != m_ranges.begin(), valid = it != m_ranges.end(); - if (valid && it->first <= index && it->second > index) { - // index is already in set + auto it = m_ranges.begin(); + for (size_t index : other.as_indexes()) { + it = do_add(find(index, it), index); } - else if (more_before && (it - 1)->second == index) { - // index is immediately after an existing range - ++(it - 1)->second; +} - if (valid && (it - 1)->second == it->first) { - // index joins two existing ranges - (it - 1)->second = it->second; - m_ranges.erase(it); +size_t IndexSet::add_shifted(size_t index) +{ + auto it = m_ranges.begin(); + for (auto end = m_ranges.end(); it != end && it->first <= index; ++it) { + index += it->second - it->first; + } + do_add(it, index); + return index; +} + +void IndexSet::add_shifted_by(IndexSet const& shifted_by, IndexSet const& values) +{ + auto it = shifted_by.begin(), end = shifted_by.end(); + size_t shift = 0; + size_t skip_until = 0; + for (size_t index : values.as_indexes()) { + for (; it != end && it->first <= index; ++it) { + shift += it->second - it->first; + skip_until = it->second; + } + if (index >= skip_until) { + REALM_ASSERT(index >= shift); + add_shifted(index - shift); + ++shift; } - } - else if (valid && it->first == index + 1) { - // index is immediately before an existing range - --it->first; - } - else { - // index is not next to an existing range - m_ranges.insert(it, {index, index + 1}); } } @@ -68,26 +99,235 @@ void IndexSet::set(size_t len) } } -void IndexSet::insert_at(size_t index) +void IndexSet::insert_at(size_t index, size_t count) { + REALM_ASSERT(count > 0); + auto pos = find(index); + bool in_existing = false; if (pos != m_ranges.end()) { - if (pos->first >= index) - ++pos->first; - ++pos->second; + if (pos->first <= index) + in_existing = true; + else + pos->first += count; + pos->second += count; for (auto it = pos + 1; it != m_ranges.end(); ++it) { - ++it->first; - ++it->second; + it->first += count; + it->second += count; } } - do_add(pos, index); + if (!in_existing) { + for (size_t i = 0; i < count; ++i) + pos = do_add(pos, index + i) + 1; + } } -void IndexSet::add_shifted(size_t index) +void IndexSet::insert_at(IndexSet const& positions) +{ + for (auto range : positions) { + insert_at(range.first, range.second - range.first); + } +} + +void IndexSet::shift_for_insert_at(size_t index, size_t count) +{ + REALM_ASSERT(count > 0); + + auto it = find(index); + if (it == m_ranges.end()) + return; + + if (it->first < index) { + // split the range so that we can exclude `index` + auto old_second = it->second; + it->second = index; + it = m_ranges.insert(it + 1, {index, old_second}); + } + + for (; it != m_ranges.end(); ++it) { + it->first += count; + it->second += count; + } +} + +void IndexSet::shift_for_insert_at(realm::IndexSet const& values) +{ + for (auto range : values) + shift_for_insert_at(range.first, range.second - range.first); +} + +void IndexSet::erase_at(size_t index) +{ + auto it = find(index); + if (it != m_ranges.end()) + do_erase(it, index); +} + +void IndexSet::erase_at(realm::IndexSet const& values) +{ + size_t shift = 0; + for (auto index : values.as_indexes()) + erase_at(index - shift++); +} + +size_t IndexSet::erase_and_unshift(size_t index) +{ + auto shifted = index; + auto it = m_ranges.begin(), end = m_ranges.end(); + for (; it != end && it->second <= index; ++it) { + shifted -= it->second - it->first; + } + if (it == end) + return shifted; + + if (it->first <= index) + shifted = npos; + + do_erase(it, index); + + return shifted; +} + +void IndexSet::do_erase(iterator it, size_t index) +{ + if (it->first <= index) { + --it->second; + if (it->first == it->second) { + it = m_ranges.erase(it); + } + else { + ++it; + } + } + else if (it != m_ranges.begin() && (it - 1)->second + 1 == it->first) { + (it - 1)->second = it->second - 1; + it = m_ranges.erase(it); + } + + for (; it != m_ranges.end(); ++it) { + --it->first; + --it->second; + } +} + +void IndexSet::remove(size_t index) +{ + auto it = find(index); + if (it == m_ranges.end() || it->first > index) + return; + + if (it->first == index) { + ++it->first; + if (it->first == it->second) { + it = m_ranges.erase(it); + } + return; + } + + if (it->second == index + 1) { + --it->second; + return; + } + + auto end = it->second; + it->second = index; + m_ranges.insert(it + 1, {index + 1, end}); +} + +void IndexSet::remove(realm::IndexSet const& values) { auto it = m_ranges.begin(); - for (auto end = m_ranges.end(); it != end && it->first <= index; ++it) { - index += it->second - it->first; + for (auto index : values.as_indexes()) { + it = find(index, it); + if (it == m_ranges.end()) + return; + if (it->first > index) + continue; + + if (it->first == index) { + ++it->first; + if (it->first == it->second) { + it = m_ranges.erase(it); + } + continue; + } + + if (it->second == index + 1) { + --it->second; + continue; + } + + auto end = it->second; + it->second = index; + it = m_ranges.insert(it + 1, {index + 1, end}); } - do_add(it, index); +} + +size_t IndexSet::shift(size_t index) const +{ + for (auto range : m_ranges) { + if (range.first > index) + break; + index += range.second - range.first; + } + return index; +} + +size_t IndexSet::unshift(size_t index) const +{ + REALM_ASSERT_DEBUG(!contains(index)); + auto shifted = index; + for (auto range : m_ranges) { + if (range.first >= index) + break; + shifted -= std::min(range.second, index) - range.first; + } + return shifted; +} + +void IndexSet::clear() +{ + m_ranges.clear(); +} + +IndexSet::iterator IndexSet::do_add(iterator it, size_t index) +{ + verify(); + bool more_before = it != m_ranges.begin(), valid = it != m_ranges.end(); + REALM_ASSERT(!more_before || index >= (it - 1)->second); + if (valid && it->first <= index && it->second > index) { + // index is already in set + return it; + } + if (more_before && (it - 1)->second == index) { + // index is immediately after an existing range + ++(it - 1)->second; + + if (valid && (it - 1)->second == it->first) { + // index joins two existing ranges + (it - 1)->second = it->second; + return m_ranges.erase(it) - 1; + } + return it - 1; + } + if (valid && it->first == index + 1) { + // index is immediately before an existing range + --it->first; + return it; + } + + // index is not next to an existing range + return m_ranges.insert(it, {index, index + 1}); +} + +void IndexSet::verify() const noexcept +{ +#ifdef REALM_DEBUG + size_t prev_end = -1; + for (auto range : m_ranges) { + REALM_ASSERT(range.first < range.second); + REALM_ASSERT(prev_end == size_t(-1) || range.first > prev_end); + prev_end = range.second; + } +#endif } diff --git a/src/index_set.hpp b/src/index_set.hpp index 9988b10f..fef560d5 100644 --- a/src/index_set.hpp +++ b/src/index_set.hpp @@ -20,12 +20,15 @@ #define REALM_INDEX_SET_HPP #include +#include #include #include namespace realm { class IndexSet { public: + static const size_t npos = -1; + using value_type = std::pair; using iterator = std::vector::iterator; using const_iterator = std::vector::const_iterator; @@ -35,8 +38,22 @@ public: bool empty() const { return m_ranges.empty(); } size_t size() const { return m_ranges.size(); } + IndexSet() = default; + IndexSet(std::initializer_list); + + // Check if the index set contains the given index + bool contains(size_t index) const; + // Add an index to the set, doing nothing if it's already present void add(size_t index); + void add(IndexSet const& is); + + // Add an index which has had all of the ranges in the set before it removed + // Returns the unshifted index + size_t add_shifted(size_t index); + // Add indexes which have had the ranges in `shifted_by` added and the ranges + // in the current set removed + void add_shifted_by(IndexSet const& shifted_by, IndexSet const& values); // Remove all indexes from the set and then add a single range starting from // zero with the given length @@ -44,10 +61,78 @@ public: // Insert an index at the given position, shifting existing indexes at or // after that point back by one - void insert_at(size_t index); + void insert_at(size_t index, size_t count=1); + void insert_at(IndexSet const&); - // Add an index which has had all of the ranges in the set before it removed - void add_shifted(size_t index); + // Shift indexes at or after the given point back by one + void shift_for_insert_at(size_t index, size_t count=1); + void shift_for_insert_at(IndexSet const&); + + // Delete an index at the given position, shifting indexes after that point + // forward by one + void erase_at(size_t index); + void erase_at(IndexSet const&); + + size_t erase_and_unshift(size_t index); + + // Remove the indexes at the given index from the set, without shifting + void remove(size_t index); + void remove(IndexSet const&); + + // Shift an index by inserting each of the indexes in this set + size_t shift(size_t index) const; + // Shift an index by deleting each of the indexes in this set + size_t unshift(size_t index) const; + + // Remove all indexes from the set + void clear(); + + void verify() const noexcept; + + // An iterator over the indivual indices in the set rather than the ranges + class IndexInterator : public std::iterator { + public: + IndexInterator(IndexSet::const_iterator it) : m_iterator(it) { } + size_t operator*() const { return m_iterator->first + m_offset; } + bool operator!=(IndexInterator const& it) const { return m_iterator != it.m_iterator; } + + IndexInterator& operator++() + { + ++m_offset; + if (m_iterator->first + m_offset == m_iterator->second) { + ++m_iterator; + m_offset = 0; + } + return *this; + } + + IndexInterator operator++(int) + { + auto value = *this; + ++*this; + return value; + } + + private: + IndexSet::const_iterator m_iterator; + size_t m_offset = 0; + }; + + class IndexIteratableAdaptor { + public: + using value_type = size_t; + using iterator = IndexInterator; + using const_iterator = iterator; + + const_iterator begin() const { return m_index_set.begin(); } + const_iterator end() const { return m_index_set.end(); } + + IndexIteratableAdaptor(IndexSet const& is) : m_index_set(is) { } + private: + IndexSet const& m_index_set; + }; + + IndexIteratableAdaptor as_indexes() const { return *this; } private: std::vector m_ranges; @@ -55,9 +140,12 @@ private: // Find the range which contains the index, or the first one after it if // none do iterator find(size_t index); + iterator find(size_t index, iterator it); // Insert the index before the given position, combining existing ranges as // applicable - void do_add(iterator pos, size_t index); + // returns inserted position + iterator do_add(iterator pos, size_t index); + void do_erase(iterator it, size_t index); }; } // namespace realm diff --git a/src/list.cpp b/src/list.cpp index 78464125..0d3b9f2e 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -17,12 +17,16 @@ //////////////////////////////////////////////////////////////////////////// #include "list.hpp" + +#include "impl/list_notifier.hpp" +#include "impl/realm_coordinator.hpp" #include "results.hpp" #include #include using namespace realm; +using namespace realm::_impl; List::List() noexcept = default; List::~List() = default; @@ -166,3 +170,13 @@ size_t hash::operator()(realm::List const& list) const return std::hash()(list.m_link_view.get()); } } + +NotificationToken List::add_notification_callback(CollectionChangeCallback cb) +{ + verify_attached(); + if (!m_notifier) { + m_notifier = std::make_shared(m_link_view, m_realm); + RealmCoordinator::register_query(m_notifier); + } + return {m_notifier, m_notifier->add_callback(std::move(cb))}; +} diff --git a/src/list.hpp b/src/list.hpp index 464c3bcd..43b20a93 100644 --- a/src/list.hpp +++ b/src/list.hpp @@ -19,6 +19,8 @@ #ifndef REALM_LIST_HPP #define REALM_LIST_HPP +#include "collection_notifications.hpp" + #include #include @@ -32,6 +34,10 @@ class Realm; class Results; struct SortOrder; +namespace _impl { + class BackgroundCollection; +} + class List { public: List() noexcept; @@ -64,6 +70,8 @@ public: bool operator==(List const& rgt) const noexcept; + NotificationToken add_notification_callback(CollectionChangeCallback cb); + // These are implemented in object_accessor.hpp template void add(ContextType ctx, ValueType value); @@ -78,6 +86,7 @@ private: std::shared_ptr m_realm; const ObjectSchema* m_object_schema; LinkViewRef m_link_view; + std::shared_ptr<_impl::BackgroundCollection> m_notifier; void verify_valid_row(size_t row_ndx, bool insertion = false) const; diff --git a/src/object_schema.cpp b/src/object_schema.cpp index f6d7ae2f..dfd4679c 100644 --- a/src/object_schema.cpp +++ b/src/object_schema.cpp @@ -40,6 +40,7 @@ ASSERT_PROPERTY_TYPE_VALUE(Any, Mixed); ASSERT_PROPERTY_TYPE_VALUE(Object, Link); ASSERT_PROPERTY_TYPE_VALUE(Array, LinkList); +ObjectSchema::ObjectSchema() = default; ObjectSchema::~ObjectSchema() = default; ObjectSchema::ObjectSchema(std::string name, std::string primary_key, std::initializer_list properties) diff --git a/src/object_schema.hpp b/src/object_schema.hpp index 10a2e555..f18a5a2d 100644 --- a/src/object_schema.hpp +++ b/src/object_schema.hpp @@ -25,35 +25,35 @@ #include namespace realm { - class Group; - struct Property; +class Group; +struct Property; - class ObjectSchema { - public: - ObjectSchema() = default; - ObjectSchema(std::string name, std::string primary_key, std::initializer_list properties); - ~ObjectSchema(); +class ObjectSchema { +public: + ObjectSchema(); + ObjectSchema(std::string name, std::string primary_key, std::initializer_list properties); + ~ObjectSchema(); - // create object schema from existing table - // if no table is provided it is looked up in the group - ObjectSchema(const Group *group, const std::string &name); + // create object schema from existing table + // if no table is provided it is looked up in the group + ObjectSchema(const Group *group, const std::string &name); - std::string name; - std::vector properties; - std::string primary_key; + std::string name; + std::vector properties; + std::string primary_key; - Property *property_for_name(StringData name); - const Property *property_for_name(StringData name) const; - Property *primary_key_property() { - return property_for_name(primary_key); - } - const Property *primary_key_property() const { - return property_for_name(primary_key); - } + Property *property_for_name(StringData name); + const Property *property_for_name(StringData name) const; + Property *primary_key_property() { + return property_for_name(primary_key); + } + const Property *primary_key_property() const { + return property_for_name(primary_key); + } - private: - void set_primary_key_property(); - }; +private: + void set_primary_key_property(); +}; } #endif /* defined(REALM_OBJECT_SCHEMA_HPP) */ diff --git a/src/results.cpp b/src/results.cpp index fb01bed8..f3f4a277 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -172,7 +172,7 @@ void Results::update_tableview() case Mode::Query: m_table_view = m_query.find_all(); if (m_sort) { - m_table_view.sort(m_sort.columnIndices, m_sort.ascending); + m_table_view.sort(m_sort.column_indices, m_sort.ascending); } m_mode = Mode::TableView; break; @@ -358,7 +358,7 @@ Results Results::filter(Query&& q) const return Results(m_realm, get_object_schema(), get_query().and_query(std::move(q)), get_sort()); } -AsyncQueryCancelationToken Results::async(std::function target) +void Results::prepare_async() { if (m_realm->config().read_only) { throw InvalidTransactionException("Cannot create asynchronous query for read-only Realms"); @@ -371,7 +371,13 @@ AsyncQueryCancelationToken Results::async(std::function(*this); _impl::RealmCoordinator::register_query(m_background_query); } - return {m_background_query, m_background_query->add_callback(std::move(target))}; +} + +NotificationToken Results::async(std::function target) +{ + prepare_async(); + auto wrap = [=](CollectionChangeIndices, std::exception_ptr e) { target(e); }; + return {m_background_query, m_background_query->add_callback(wrap)}; } void Results::Internal::set_table_view(Results& results, realm::TableView &&tv) @@ -386,6 +392,7 @@ void Results::Internal::set_table_view(Results& results, realm::TableView &&tv) results.m_mode = Mode::TableView; results.m_has_used_table_view = false; REALM_ASSERT(results.m_table_view.is_in_sync()); + REALM_ASSERT(results.m_table_view.is_attached()); } Results::UnsupportedColumnTypeException::UnsupportedColumnTypeException(size_t column, const Table* table) @@ -395,35 +402,3 @@ Results::UnsupportedColumnTypeException::UnsupportedColumnTypeException(size_t c , column_type(table->get_column_type(column)) { } - -AsyncQueryCancelationToken::AsyncQueryCancelationToken(std::shared_ptr<_impl::AsyncQuery> query, size_t token) -: m_query(std::move(query)), m_token(token) -{ -} - -AsyncQueryCancelationToken::~AsyncQueryCancelationToken() -{ - // m_query itself (and not just the pointed-to thing) needs to be accessed - // atomically to ensure that there are no data races when the token is - // destroyed after being modified on a different thread. - // This is needed despite the token not being thread-safe in general as - // users find it very surpringing for obj-c objects to care about what - // thread they are deallocated on. - if (auto query = m_query.exchange({})) { - query->remove_callback(m_token); - } -} - -AsyncQueryCancelationToken::AsyncQueryCancelationToken(AsyncQueryCancelationToken&& rgt) = default; - -AsyncQueryCancelationToken& AsyncQueryCancelationToken::operator=(realm::AsyncQueryCancelationToken&& rgt) -{ - if (this != &rgt) { - if (auto query = m_query.exchange({})) { - query->remove_callback(m_token); - } - m_query = std::move(rgt.m_query); - m_token = rgt.m_token; - } - return *this; -} diff --git a/src/results.hpp b/src/results.hpp index c7501179..70000d9e 100644 --- a/src/results.hpp +++ b/src/results.hpp @@ -19,8 +19,8 @@ #ifndef REALM_RESULTS_HPP #define REALM_RESULTS_HPP +#include "collection_notifications.hpp" #include "shared_realm.hpp" -#include "util/atomic_shared_ptr.hpp" #include #include @@ -38,31 +38,11 @@ namespace _impl { class AsyncQuery; } -// A token which keeps an asynchronous query alive -struct AsyncQueryCancelationToken { - AsyncQueryCancelationToken() = default; - AsyncQueryCancelationToken(std::shared_ptr<_impl::AsyncQuery> query, size_t token); - ~AsyncQueryCancelationToken(); - - AsyncQueryCancelationToken(AsyncQueryCancelationToken&&); - AsyncQueryCancelationToken& operator=(AsyncQueryCancelationToken&&); - - AsyncQueryCancelationToken(AsyncQueryCancelationToken const&) = delete; - AsyncQueryCancelationToken& operator=(AsyncQueryCancelationToken const&) = delete; - -private: - util::AtomicSharedPtr<_impl::AsyncQuery> m_query; - size_t m_token; -}; - struct SortOrder { - std::vector columnIndices; + std::vector column_indices; std::vector ascending; - explicit operator bool() const - { - return !columnIndices.empty(); - } + explicit operator bool() const { return !column_indices.empty(); } }; class Results { @@ -196,7 +176,7 @@ public: // Create an async query from this Results // The query will be run on a background thread and delivered to the callback, // and then rerun after each commit (if needed) and redelivered if it changed - AsyncQueryCancelationToken async(std::function target); + NotificationToken async(std::function target); bool wants_background_updates() const { return m_wants_background_updates; } @@ -225,6 +205,8 @@ private: void validate_read() const; void validate_write() const; + void prepare_async(); + template util::Optional aggregate(size_t column, bool return_none_for_empty, Int agg_int, Float agg_float, diff --git a/src/schema.cpp b/src/schema.cpp index 3d3988d7..172ecce9 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -28,6 +28,8 @@ static bool compare_by_name(ObjectSchema const& lft, ObjectSchema const& rgt) { return lft.name < rgt.name; } +Schema::Schema(std::initializer_list types) : Schema(base(types)) { } + Schema::Schema(base types) : base(std::move(types)) { std::sort(begin(), end(), compare_by_name); } diff --git a/src/schema.hpp b/src/schema.hpp index 52c03975..3f11c026 100644 --- a/src/schema.hpp +++ b/src/schema.hpp @@ -34,7 +34,7 @@ private: public: // Create a schema from a vector of ObjectSchema Schema(base types); - Schema(std::initializer_list types) : Schema(base(types)) { } + Schema(std::initializer_list types); // find an ObjectSchema by name iterator find(std::string const& name); diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index 19908deb..3909782d 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -43,6 +43,7 @@ namespace realm { namespace _impl { class AsyncQuery; class BackgroundCollection; + class ListNotifier; class RealmCoordinator; } @@ -145,6 +146,7 @@ namespace realm { // without making it public to everyone class Internal { friend class _impl::AsyncQuery; + friend class _impl::ListNotifier; friend class _impl::BackgroundCollection; friend class _impl::RealmCoordinator; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9fbb83b8..c28b839b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,11 +1,14 @@ include_directories(../external/catch/single_include .) set(HEADERS + util/index_helpers.hpp util/test_file.hpp ) set(SOURCES + collection_change_indices.cpp index_set.cpp + list.cpp main.cpp parser.cpp results.cpp diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp new file mode 100644 index 00000000..d753475b --- /dev/null +++ b/tests/collection_change_indices.cpp @@ -0,0 +1,367 @@ +#include "catch.hpp" + +#include "collection_notifications.hpp" + +#include "util/index_helpers.hpp" + +TEST_CASE("collection change indices") { + using namespace realm; + CollectionChangeIndices c; + + SECTION("stuff") { + SECTION("insert() adds the row to the insertions set") { + c.insert(5); + c.insert(8); + REQUIRE_INDICES(c.insertions, 5, 8); + } + + SECTION("insert() shifts previous insertions and modifications") { + c.insert(5); + c.modify(8); + + c.insert(1); + REQUIRE_INDICES(c.insertions, 1, 6); + REQUIRE_INDICES(c.modifications, 9); + } + + SECTION("insert() does not shift previous deletions") { + c.erase(8); + c.erase(3); + c.insert(5); + + REQUIRE_INDICES(c.insertions, 5); + REQUIRE_INDICES(c.deletions, 3, 8); + } + + SECTION("modify() adds the row to the modifications set") { + c.modify(3); + c.modify(4); + REQUIRE_INDICES(c.modifications, 3, 4); + } + + SECTION("modify() on an inserted row is a no-op") { + c.insert(3); + c.modify(3); + REQUIRE(c.modifications.empty()); + } + + SECTION("modify() doesn't interact with deleted rows") { + c.erase(5); + c.erase(4); + c.erase(3); + + c.modify(4); + REQUIRE_INDICES(c.modifications, 4); + } + + SECTION("erase() adds the row to the deletions set") { + c.erase(5); + REQUIRE_INDICES(c.deletions, 5); + } + + SECTION("erase() is shifted for previous deletions") { + c.erase(5); + c.erase(6); + REQUIRE_INDICES(c.deletions, 5, 7); + } + + SECTION("erase() is shifted for previous insertions") { + c.insert(5); + c.erase(6); + REQUIRE_INDICES(c.deletions, 5); + } + + SECTION("erase() removes previous insertions") { + c.insert(5); + c.erase(5); + REQUIRE(c.insertions.empty()); + REQUIRE(c.deletions.empty()); + } + + SECTION("erase() removes previous modifications") { + c.modify(5); + c.erase(5); + REQUIRE(c.modifications.empty()); + REQUIRE_INDICES(c.deletions, 5); + } + + SECTION("erase() shifts previous modifications") { + c.modify(5); + c.erase(4); + REQUIRE_INDICES(c.modifications, 4); + REQUIRE_INDICES(c.deletions, 4); + } + + SECTION("move() adds the move to the list of moves") { + c.move(5, 6); + REQUIRE_MOVES(c, {5, 6}); + } + + SECTION("move() updates previous moves to the source of this move") { + c.move(5, 6); + c.move(6, 7); + REQUIRE_MOVES(c, {5, 7}); + } + + SECTION("move() shifts previous moves and is shifted by them") { + c.move(5, 10); + c.move(6, 12); + REQUIRE_MOVES(c, {5, 9}, {7, 12}); + + c.move(10, 0); + REQUIRE_MOVES(c, {5, 10}, {7, 12}, {11, 0}); + } + + SECTION("moving a newly inserted row is not reported as a move") { + c.insert(5); + c.move(5, 10); + REQUIRE_INDICES(c.insertions, 10); + REQUIRE(c.moves.empty()); + } + + SECTION("move() shifts previous insertions and modifications") { + c.insert(5); + c.modify(6); + c.move(10, 0); + REQUIRE_INDICES(c.insertions, 0, 6); + REQUIRE_INDICES(c.modifications, 7); + REQUIRE_MOVES(c, {9, 0}); + } + + SECTION("move_over() marks the old last row as moved") { + c.move_over(5, 8); + REQUIRE_MOVES(c, {8, 5}); + } + + SECTION("move_over() removes previous insertions for that row") { + c.insert(5); + c.move_over(5, 8); + REQUIRE(c.insertions.empty()); + } + + SECTION("move_over() removes previous modifications for that row") { + c.modify(5); + c.move_over(5, 8); + REQUIRE(c.modifications.empty()); + } + + SECTION("move_over() updates previous insertions for the old last row") { + c.insert(5); + c.move_over(3, 5); + REQUIRE_INDICES(c.insertions, 3); + } + + SECTION("move_over() updates previous modifications for the old last row") { + c.modify(5); + c.move_over(3, 5); + REQUIRE_INDICES(c.modifications, 3); + } + + SECTION("move_over() removes moves to the target") { + c.move(3, 5); + c.move_over(5, 8); + REQUIRE(c.moves.empty()); + } + + SECTION("move_over() updates moves to the source") { + c.move(3, 8); + c.move_over(5, 8); + REQUIRE_MOVES(c, {3, 5}); + } + + SECTION("move_over() is not shifted by previous calls to move_over()") { + c.move_over(5, 10); + c.move_over(6, 9); + REQUIRE_INDICES(c.deletions, 5, 6); + REQUIRE_MOVES(c, {10, 5}, {9, 6}); + } + } + + SECTION("merge") { + SECTION("deletions are shifted by previous deletions") { + c = {{5}, {}, {}, {}}; + c.merge({{3}, {}, {}, {}}); + REQUIRE_INDICES(c.deletions, 3, 5); + + c = {{5}, {}, {}, {}}; + c.merge({{4}, {}, {}, {}}); + REQUIRE_INDICES(c.deletions, 4, 5); + + c = {{5}, {}, {}, {}}; + c.merge({{5}, {}, {}, {}}); + REQUIRE_INDICES(c.deletions, 5, 6); + + c = {{5}, {}, {}, {}}; + c.merge({{6}, {}, {}, {}}); + REQUIRE_INDICES(c.deletions, 5, 7); + } + + SECTION("deletions are shifted by previous insertions") { + c = {{}, {5}, {}, {}}; + c.merge({{4}, {}, {}, {}}); + REQUIRE_INDICES(c.deletions, 4); + + c = {{}, {5}, {}, {}}; + c.merge({{6}, {}, {}, {}}); + REQUIRE_INDICES(c.deletions, 5); + } + + SECTION("deletions shift previous insertions") { + c = {{}, {2, 3}, {}, {}}; + c.merge({{1}, {}, {}, {}}); + REQUIRE_INDICES(c.insertions, 1, 2); + } + + SECTION("deletions remove previous insertions") { + c = {{}, {1, 2}, {}, {}}; + c.merge({{2}, {}, {}, {}}); + REQUIRE_INDICES(c.insertions, 1); + } + + SECTION("deletions remove previous modifications") { + c = {{}, {}, {2, 3}, {}}; + c.merge({{2}, {}, {}, {}}); + REQUIRE_INDICES(c.modifications, 2); + } + + SECTION("deletions shift previous modifications") { + c = {{}, {}, {2, 3}, {}}; + c.merge({{1}, {}, {}, {}}); + REQUIRE_INDICES(c.modifications, 1, 2); + } + + SECTION("deletions remove previous moves to deleted row") { + c = {{}, {}, {}, {{2, 3}}}; + c.merge({{3}, {}, {}, {}}); + REQUIRE(c.moves.empty()); + } + + SECTION("deletions shift destination of previous moves to after the deleted row") { + c = {{}, {}, {}, {{2, 5}}}; + c.merge({{3}, {}, {}, {}}); + REQUIRE_MOVES(c, {2, 4}); + } + + SECTION("insertions do not interact with previous deletions") { + c = {{1, 3}, {}, {}, {}}; + c.merge({{}, {1, 2, 3}, {}, {}}); + REQUIRE_INDICES(c.deletions, 1, 3); + REQUIRE_INDICES(c.insertions, 1, 2, 3); + } + + SECTION("insertions shift previous insertions") { + c = {{}, {1, 5}, {}, {}}; + c.merge({{}, {1, 4}, {}, {}}); + REQUIRE_INDICES(c.insertions, 1, 2, 4, 7); + } + + SECTION("insertions shift previous modifications") { + c = {{}, {}, {1, 5}, {}}; + c.merge({{}, {1, 4}, {}, {}}); + REQUIRE_INDICES(c.modifications, 2, 7); + REQUIRE_INDICES(c.insertions, 1, 4); + } + + SECTION("insertions shift destination of previous moves") { + c = {{}, {}, {}, {{2, 5}}}; + c.merge({{}, {3}, {}}); + REQUIRE_MOVES(c, {2, 6}); + } + + SECTION("modifications do not interact with previous deletions") { + c = {{1, 2, 3}, {}, {}, {}}; + c.merge({{}, {}, {2}}); + REQUIRE_INDICES(c.deletions, 1, 2, 3); + REQUIRE_INDICES(c.modifications, 2); + } + + SECTION("modifications are discarded for previous insertions") { + c = {{}, {2}, {}, {}}; + c.merge({{}, {}, {1, 2, 3}}); + REQUIRE_INDICES(c.insertions, 2); + REQUIRE_INDICES(c.modifications, 1, 3); + } + + SECTION("modifications are merged with previous modifications") { + c = {{}, {}, {2}, {}}; + c.merge({{}, {}, {1, 2, 3}}); + REQUIRE_INDICES(c.modifications, 1, 2, 3); + } + + SECTION("modifications are discarded for the destination of previous moves") { + c = {{}, {}, {}, {{1, 2}}}; + c.merge({{}, {}, {2, 3}}); + REQUIRE_INDICES(c.modifications, 3); + } + + SECTION("move sources are shifted for previous deletes and insertions") { + c = {{1}, {}, {}, {}}; + c.merge({{}, {}, {}, {{2, 3}}}); + REQUIRE_MOVES(c, {3, 3}); + + c = {{}, {1}, {}, {}}; + c.merge({{}, {}, {}, {{2, 3}}}); + REQUIRE_MOVES(c, {1, 3}); + + c = {{2}, {4}, {}, {}}; + c.merge({{}, {}, {}, {{5, 10}}}); + REQUIRE_MOVES(c, {5, 10}); + } + + SECTION("moves remove previous modifications to source") { + c = {{}, {}, {1}, {}}; + c.merge({{}, {}, {}, {{1, 3}}}); + REQUIRE(c.modifications.empty()); + REQUIRE_MOVES(c, {1, 3}); + } + + SECTION("moves update insertion position for previous inserts of source") { + c = {{}, {1}, {}, {}}; + c.merge({{}, {}, {}, {{1, 3}}}); + REQUIRE(c.moves.empty()); + REQUIRE_INDICES(c.insertions, 3); + } + + SECTION("moves update previous moves to the source") { + c = {{}, {}, {}, {{1, 3}}}; + c.merge({{}, {}, {}, {{3, 5}}}); + REQUIRE_MOVES(c, {1, 5}); + } + + SECTION("moves shift destination of previous moves like an insert/delete pair would") { + c = {{}, {}, {}, {{1, 3}}}; + c.merge({{}, {}, {}, {{2, 5}}}); + REQUIRE_MOVES(c, {1, 2}, {3, 5}); + + c = {{}, {}, {}, {{1, 10}}}; + c.merge({{}, {}, {}, {{2, 5}}}); + REQUIRE_MOVES(c, {1, 10}, {3, 5}); + + c = {{}, {}, {}, {{5, 10}}}; + c.merge({{}, {}, {}, {{12, 2}}}); + REQUIRE_MOVES(c, {5, 11}, {12, 2}); + } + + SECTION("moves shift previous inserts like an insert/delete pair would") { + c = {{}, {5}}; + c.merge({{}, {}, {}, {{2, 6}}}); + REQUIRE_INDICES(c.insertions, 4, 6); + } + + SECTION("moves shift previous modifications like an insert/delete pair would") { + c = {{}, {}, {5}}; + c.merge({{}, {}, {}, {{2, 6}}}); + REQUIRE_INDICES(c.modifications, 4); + } + + SECTION("moves are shifted by previous deletions like an insert/delete pair would") { + c = {{5}}; + c.merge({{}, {}, {}, {{2, 6}}}); + REQUIRE_MOVES(c, {2, 6}); + + c = {{5}}; + c.merge({{}, {}, {}, {{6, 2}}}); + REQUIRE_MOVES(c, {7, 2}); + } + } +} diff --git a/tests/index_set.cpp b/tests/index_set.cpp index 50ae889c..b7f58c6b 100644 --- a/tests/index_set.cpp +++ b/tests/index_set.cpp @@ -1,153 +1,357 @@ -#include "index_set.hpp" - -#include - -// Catch doesn't have an overload for std::pair, so define one ourselves -// The declaration needs to be before catch.hpp is included for it to be used, -// but the definition needs to be after since it uses Catch's toString() -namespace Catch { -template -std::string toString(std::pair const& value); -} - #include "catch.hpp" -namespace Catch { -template - std::string toString(std::pair const& value) { - return "{" + toString(value.first) + ", " + toString(value.second) + "}"; -} -} +#include "index_set.hpp" -#define REQUIRE_RANGES(index_set, ...) do { \ - std::initializer_list> expected = {__VA_ARGS__}; \ - REQUIRE(index_set.size() == expected.size()); \ - auto begin = index_set.begin(), end = index_set.end(); \ - for (auto range : expected) { \ - REQUIRE(*begin++ == range); \ - } \ -} while (0) +#include "util/index_helpers.hpp" TEST_CASE("index set") { realm::IndexSet set; SECTION("add() extends existing ranges") { set.add(1); - REQUIRE_RANGES(set, {1, 2}); + REQUIRE_INDICES(set, 1); set.add(2); - REQUIRE_RANGES(set, {1, 3}); + REQUIRE_INDICES(set, 1, 2); set.add(0); - REQUIRE_RANGES(set, {0, 3}); + REQUIRE_INDICES(set, 0, 1, 2); } SECTION("add() with gaps") { set.add(0); - REQUIRE_RANGES(set, {0, 1}); + REQUIRE_INDICES(set, 0); set.add(2); - REQUIRE_RANGES(set, {0, 1}, {2, 3}); + REQUIRE_INDICES(set, 0, 2); } SECTION("add() is idempotent") { set.add(0); set.add(0); - REQUIRE_RANGES(set, {0, 1}); + REQUIRE_INDICES(set, 0); } SECTION("add() merges existing ranges") { - set.add(0); - set.add(2); - set.add(4); + set = {0, 2, 4}; set.add(1); - REQUIRE_RANGES(set, {0, 3}, {4, 5}); + REQUIRE_INDICES(set, 0, 1, 2, 4); + } + + SECTION("add() combines multiple index sets") { + set = {0, 2, 6}; + + set.add({1, 4, 5}); + REQUIRE_INDICES(set, 0, 1, 2, 4, 5, 6); } SECTION("set() from empty") { set.set(5); - REQUIRE_RANGES(set, {0, 5}); + REQUIRE_INDICES(set, 0, 1, 2, 3, 4); } SECTION("set() discards existing data") { - set.add(8); - set.add(9); + set = {8, 9}; set.set(5); - REQUIRE_RANGES(set, {0, 5}); + REQUIRE_INDICES(set, 0, 1, 2, 3, 4); + } + + SECTION("insert_at() on an empty set is add()") { + set.insert_at(5); + REQUIRE_INDICES(set, 5); } SECTION("insert_at() extends ranges containing the target index") { - set.add(5); - set.add(6); + set = {5, 6}; set.insert_at(5); - REQUIRE_RANGES(set, {5, 8}); + REQUIRE_INDICES(set, 5, 6, 7); set.insert_at(4); - REQUIRE_RANGES(set, {4, 5}, {6, 9}); + REQUIRE_INDICES(set, 4, 6, 7, 8); set.insert_at(9); - REQUIRE_RANGES(set, {4, 5}, {6, 10}); + REQUIRE_INDICES(set, 4, 6, 7, 8, 9); } SECTION("insert_at() does not modify ranges entirely before it") { - set.add(5); - set.add(6); - + set = {5, 6}; set.insert_at(8); - REQUIRE_RANGES(set, {5, 7}, {8, 9}); + REQUIRE_INDICES(set, 5, 6, 8); } SECTION("insert_at() shifts ranges after it") { - set.add(5); - set.add(6); - + set = {5, 6}; set.insert_at(3); - REQUIRE_RANGES(set, {3, 4}, {6, 8}); + REQUIRE_INDICES(set, 3, 6, 7); } SECTION("insert_at() cannot join ranges") { - set.add(5); - set.add(7); - + set = {5, 7}; set.insert_at(6); - REQUIRE_RANGES(set, {5, 7}, {8, 9}); + REQUIRE_INDICES(set, 5, 6, 8); + } + + SECTION("bulk insert_at() on an empty set is add()") { + set.insert_at({5, 6, 8}); + REQUIRE_INDICES(set, 5, 6, 8); + } + + SECTION("bulk insert_at() shifts existing ranges") { + set = {5, 10}; + set.insert_at({3, 8, 14}); + REQUIRE_INDICES(set, 3, 6, 8, 12, 14); + } + + SECTION("bulk insert_at() does not join ranges") { + set = {5, 7}; + set.insert_at({5, 6, 7}); + REQUIRE_INDICES(set, 5, 6, 7, 8, 10); + } + + SECTION("bulk insert_at() extends existing ranges") { + set = {5, 8}; + set.insert_at({5, 9}); + REQUIRE_INDICES(set, 5, 6, 9, 10); + + set = {4, 5}; + set.insert_at({5, 6}); + REQUIRE_INDICES(set, 4, 5, 6, 7); } SECTION("add_shifted() on an empty set is just add()") { set.add_shifted(5); - REQUIRE_RANGES(set, {5, 6}); + REQUIRE_INDICES(set, 5); } SECTION("add_shifted() before the first range is just add()") { set.add(10); set.add_shifted(5); - REQUIRE_RANGES(set, {5, 6}, {10, 11}); + REQUIRE_INDICES(set, 5, 10); } SECTION("add_shifted() on first index of range extends range") { set.add(5); set.add_shifted(5); - REQUIRE_RANGES(set, {5, 7}); + REQUIRE_INDICES(set, 5, 6); set.add_shifted(5); - REQUIRE_RANGES(set, {5, 8}); + REQUIRE_INDICES(set, 5, 6, 7); set.add_shifted(6); - REQUIRE_RANGES(set, {5, 8}, {9, 10}); + REQUIRE_INDICES(set, 5, 6, 7, 9); } SECTION("add_shifted() after ranges shifts by the size of those ranges") { set.add(5); set.add_shifted(6); - REQUIRE_RANGES(set, {5, 6}, {7, 8}); + REQUIRE_INDICES(set, 5, 7); set.add_shifted(6); // bumped into second range - REQUIRE_RANGES(set, {5, 6}, {7, 9}); + REQUIRE_INDICES(set, 5, 7, 8); set.add_shifted(8); - REQUIRE_RANGES(set, {5, 6}, {7, 9}, {11, 12}); + REQUIRE_INDICES(set, 5, 7, 8, 11); + } + + SECTION("add_shifted_by() with an empty shifted by set is just bulka dd_shifted()") { + set = {5}; + set.add_shifted_by({}, {6, 7}); + REQUIRE_INDICES(set, 5, 7, 8); + } + + SECTION("add_shifted_by() shifts backwards for indices in the first set") { + set = {5}; + set.add_shifted_by({0, 2, 3}, {6}); + REQUIRE_INDICES(set, 3, 5); + + set = {5}; + set.add_shifted_by({1, 3}, {4}); + REQUIRE_INDICES(set, 2, 5); + } + + SECTION("add_shifted_by() discards indices in the first set") { + set = {5}; + set.add_shifted_by({3}, {3}); + REQUIRE_INDICES(set, 5); + + set = {5}; + set.add_shifted_by({1, 3}, {3}); + REQUIRE_INDICES(set, 5); + } + + SECTION("shift_for_insert_at() does not modify ranges before it") { + set.add(5); + set.shift_for_insert_at(6); + REQUIRE_INDICES(set, 5); + } + + SECTION("shift_for_insert_at() moves ranges at or after it back") { + set.add(5); + set.shift_for_insert_at(5); + REQUIRE_INDICES(set, 6); + } + + SECTION("shift_for_insert_at() splits ranges containing the index") { + set.add(5); + set.add(6); + set.shift_for_insert_at(6); + REQUIRE_INDICES(set, 5, 7); + } + + SECTION("bulk shift_for_insert_at() updates things") { + set = {5, 6}; + set.shift_for_insert_at({3, 7, 10}); + REQUIRE_INDICES(set, 6, 8); + } + + SECTION("erase_at() shifts ranges after it back") { + set.add(5); + set.erase_at(4); + REQUIRE_INDICES(set, 4); + } + + SECTION("erase_at() shrinks ranges containing the index") { + set = {5, 6, 7}; + + set.erase_at(6); + REQUIRE_INDICES(set, 5, 6); + + set.erase_at(5); + REQUIRE_INDICES(set, 5); + } + + SECTION("erase_at() removes one-element ranges") { + set = {3, 5, 7}; + + set.erase_at(5); + REQUIRE_INDICES(set, 3, 6); + } + + SECTION("erase_at() merges ranges when the gap between them is deleted") { + set.add(3); + set.add(5); + set.erase_at(4); + REQUIRE_INDICES(set, 3, 4); + } + + SECTION("bulk erase_at() does things") { + set = {3, 5, 6, 7, 10, 12}; + set.erase_at({3, 6, 11}); + REQUIRE_INDICES(set, 4, 5, 8, 9); + } + + SECTION("erase_and_unshift() removes the given index") { + set = {1, 2}; + set.erase_and_unshift(2); + REQUIRE_INDICES(set, 1); + } + + SECTION("erase_and_unshift() shifts indexes after the given index") { + set = {1, 5}; + set.erase_and_unshift(2); + REQUIRE_INDICES(set, 1, 4); + } + + SECTION("erase_and_unshift() returns npos for indices in the set") { + set = {1, 3, 5}; + REQUIRE(realm::IndexSet(set).erase_and_unshift(1) == realm::IndexSet::npos); + REQUIRE(realm::IndexSet(set).erase_and_unshift(3) == realm::IndexSet::npos); + REQUIRE(realm::IndexSet(set).erase_and_unshift(5) == realm::IndexSet::npos); + } + + SECTION("erase_and_unshift() returns the same thing as unshift()") { + set = {1, 3, 5, 6}; + REQUIRE(realm::IndexSet(set).erase_and_unshift(0) == 0); + REQUIRE(realm::IndexSet(set).erase_and_unshift(2) == 1); + REQUIRE(realm::IndexSet(set).erase_and_unshift(4) == 2); + REQUIRE(realm::IndexSet(set).erase_and_unshift(7) == 3); + } + + SECTION("shift() adds the number of indexes before the given index in the set to the given index") { + set = {1, 3, 5, 6}; + REQUIRE(set.shift(0) == 0); + REQUIRE(set.shift(1) == 2); + REQUIRE(set.shift(2) == 4); + REQUIRE(set.shift(3) == 7); + REQUIRE(set.shift(4) == 8); + } + + SECTION("unshift() subtracts the number of indexes in the set before the given index from the index") { + set = {1, 3, 5, 6}; + REQUIRE(set.unshift(0) == 0); + REQUIRE(set.unshift(2) == 1); + REQUIRE(set.unshift(4) == 2); + REQUIRE(set.unshift(7) == 3); + REQUIRE(set.unshift(8) == 4); + } + + SECTION("remove() does nothing if the index is not in the set") { + set = {5}; + set.remove(4); + set.remove(6); + REQUIRE_INDICES(set, 5); + } + + SECTION("remove() removes one-element ranges") { + set = {5}; + set.remove(5); + REQUIRE(set.empty()); + } + + SECTION("remove() shrinks ranges beginning with the index") { + set = {5, 6, 7}; + set.remove(5); + REQUIRE_INDICES(set, 6, 7); + } + + SECTION("remove() shrinks ranges ending with the index") { + set = {5, 6, 7}; + set.remove(7); + REQUIRE_INDICES(set, 5, 6); + } + + SECTION("remove() splits ranges containing the index") { + set = {5, 6, 7}; + set.remove(6); + REQUIRE_INDICES(set, 5, 7); + } + + SECTION("bulk remove() does nothing if the indices are not in the set") { + set = {5}; + set.remove({4, 6}); + REQUIRE_INDICES(set, 5); + } + + SECTION("bulk remove() removes one-element ranges") { + set = {5}; + set.remove({5, 6}); + REQUIRE(set.empty()); + } + + SECTION("bulk remove() shrinks ranges beginning with the indices") { + set = {5, 6, 7}; + set.remove({4, 5}); + REQUIRE_INDICES(set, 6, 7); + } + + SECTION("bulk remove() shrinks ranges ending with the indices") { + set = {5, 6, 7}; + set.remove({7, 8}); + REQUIRE_INDICES(set, 5, 6); + } + + SECTION("bulk remove() splits ranges containing the indices") { + set = {5, 6, 7}; + set.remove({3, 6, 8}); + REQUIRE_INDICES(set, 5, 7); + } + + SECTION("bulk remove() correctly removes multiple indices") { + set = {5, 6, 7, 10, 11, 12, 13, 15}; + set.remove({6, 11, 13}); + REQUIRE_INDICES(set, 5, 7, 10, 12, 15); } } diff --git a/tests/list.cpp b/tests/list.cpp new file mode 100644 index 00000000..ab92e304 --- /dev/null +++ b/tests/list.cpp @@ -0,0 +1,225 @@ +#include "catch.hpp" + +#include "util/test_file.hpp" +#include "util/index_helpers.hpp" + +#include "binding_context.hpp" +#include "list.hpp" +#include "property.hpp" +#include "object_schema.hpp" +#include "schema.hpp" + +#include "impl/realm_coordinator.hpp" + +#include +#include +#include + +using namespace realm; + +TEST_CASE("list") { + InMemoryTestFile config; + config.automatic_change_notifications = false; + config.cache = false; + config.schema = std::make_unique(Schema{ + {"origin", "", { + {"array", PropertyTypeArray, "target"} + }}, + {"target", "", { + {"value", PropertyTypeInt} + }}, + }); + + auto r = Realm::get_shared_realm(config); + auto& coordinator = *_impl::RealmCoordinator::get_existing_coordinator(config.path); + + auto origin = r->read_group()->get_table("class_origin"); + auto target = r->read_group()->get_table("class_target"); + + r->begin_transaction(); + + target->add_empty_row(10); + for (int i = 0; i < 10; ++i) + target->set_int(0, i, i); + + origin->add_empty_row(2); + LinkViewRef lv = origin->get_linklist(0, 0); + for (int i = 0; i < 10; ++i) + lv->add(i); + LinkViewRef lv2 = origin->get_linklist(0, 1); + for (int i = 0; i < 10; ++i) + lv2->add(i); + + r->commit_transaction(); + + SECTION("add_notification_block()") { + CollectionChangeIndices change; + List lst(r, *r->config().schema->find("origin"), lv); + + auto write = [&](auto&& f) { + r->begin_transaction(); + f(); + r->commit_transaction(); + + coordinator.on_change(); + r->notify(); + }; + + auto require_change = [&] { + return lst.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { + change = c; + }); + }; + + auto require_no_change = [&] { + return lst.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { + REQUIRE(false); + }); + }; + + SECTION("modifying the list sends a change notifications") { + auto token = require_change(); + write([&] { lst.remove(5); }); + REQUIRE_INDICES(change.deletions, 5); + } + + SECTION("modifying a different list doesn't send a change notification") { + auto token = require_no_change(); + write([&] { lv2->remove(5); }); + } + + SECTION("deleting the list sends a change notification") { + auto token = require_change(); + write([&] { origin->move_last_over(0); }); + REQUIRE_INDICES(change.deletions, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + } + + SECTION("modifying one of the target rows sends a change notification") { + auto token = require_change(); + write([&] { lst.get(5).set_int(0, 6); }); + REQUIRE_INDICES(change.modifications, 5); + } + + SECTION("deleting a target row sends a change notification") { + auto token = require_change(); + write([&] { target->move_last_over(5); }); + REQUIRE_INDICES(change.deletions, 5); + } + + SECTION("adding a row and then modifying the target row does not mark the row as modified") { + auto token = require_change(); + write([&] { + lst.add(5); + target->set_int(0, 5, 10); + }); + REQUIRE_INDICES(change.insertions, 10); + REQUIRE_INDICES(change.modifications, 5); + } + + SECTION("modifying and then moving a row reports move/insert but not modification") { + auto token = require_change(); + write([&] { + target->set_int(0, 5, 10); + lst.move(5, 8); + }); + REQUIRE_INDICES(change.insertions, 8); + REQUIRE_INDICES(change.deletions, 5); + REQUIRE_MOVES(change, {5, 8}); + REQUIRE(change.modifications.empty()); + } + + SECTION("modifying a row which appears multiple times in a list marks them all as modified") { + r->begin_transaction(); + lst.add(5); + r->commit_transaction(); + + auto token = require_change(); + write([&] { target->set_int(0, 5, 10); }); + REQUIRE_INDICES(change.modifications, 5, 10); + } + + SECTION("deleting a row which appears multiple times in a list marks them all as modified") { + r->begin_transaction(); + lst.add(5); + r->commit_transaction(); + + auto token = require_change(); + write([&] { target->move_last_over(5); }); + REQUIRE_INDICES(change.deletions, 5, 10); + } + + SECTION("clearing the target table sends a change notification") { + auto token = require_change(); + write([&] { target->clear(); }); + REQUIRE_INDICES(change.deletions, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + } + + SECTION("moving a target row does not send a change notification") { + // Remove a row from the LV so that we have one to delete that's not in the list + r->begin_transaction(); + lv->remove(2); + r->commit_transaction(); + + auto token = require_no_change(); + write([&] { target->move_last_over(2); }); + } + + SECTION("multiple LinkViws for the same LinkList can get notifications") { + r->begin_transaction(); + target->clear(); + target->add_empty_row(5); + r->commit_transaction(); + + auto get_list = [&] { + auto r = Realm::get_shared_realm(config); + auto lv = r->read_group()->get_table("class_origin")->get_linklist(0, 0); + return List(r, *r->config().schema->find("origin"), lv); + }; + auto change_list = [&] { + r->begin_transaction(); + if (lv->size()) { + target->set_int(0, lv->size() - 1, lv->size()); + } + lv->add(lv->size()); + r->commit_transaction(); + }; + + List lists[3]; + NotificationToken tokens[3]; + CollectionChangeIndices changes[3]; + + for (int i = 0; i < 3; ++i) { + lists[i] = get_list(); + tokens[i] = lists[i].add_notification_callback([i, &changes](CollectionChangeIndices c, std::exception_ptr) { + changes[i] = std::move(c); + }); + change_list(); + } + + // Each of the Lists now has a different source version and state at + // that version, so they should all see different changes despite + // being for the same LinkList + coordinator.on_change(); + r->notify(); + + REQUIRE_INDICES(changes[0].insertions, 0, 1, 2); + REQUIRE(changes[0].modifications.empty()); + + REQUIRE_INDICES(changes[1].insertions, 1, 2); + REQUIRE_INDICES(changes[1].modifications, 0); + + REQUIRE_INDICES(changes[2].insertions, 2); + REQUIRE_INDICES(changes[2].modifications, 1); + + // After making another change, they should all get the same notification + change_list(); + coordinator.on_change(); + r->notify(); + + for (int i = 0; i < 3; ++i) { + REQUIRE_INDICES(changes[i].insertions, 3); + REQUIRE_INDICES(changes[i].modifications, 2); + } + } + } +} diff --git a/tests/transaction_log_parsing.cpp b/tests/transaction_log_parsing.cpp index e85c6b8c..dd5406a4 100644 --- a/tests/transaction_log_parsing.cpp +++ b/tests/transaction_log_parsing.cpp @@ -1,21 +1,110 @@ #include "catch.hpp" +#include "util/index_helpers.hpp" #include "util/test_file.hpp" #include "impl/realm_coordinator.hpp" #include "impl/transact_log_handler.hpp" +#include "collection_notifications.hpp" #include "property.hpp" #include "object_schema.hpp" #include "schema.hpp" #include #include +#include using namespace realm; +class CaptureHelper { +public: + CaptureHelper(std::string const& path, SharedRealm const& r, LinkViewRef lv) + : m_history(make_client_history(path)) + , m_sg(*m_history, SharedGroup::durability_MemOnly) + , m_realm(r) + , m_group(m_sg.begin_read()) + , m_linkview(lv) + { + m_realm->begin_transaction(); + + m_initial.reserve(lv->size()); + for (size_t i = 0; i < lv->size(); ++i) + m_initial.push_back(lv->get(i).get_int(0)); + } + + CollectionChangeIndices finish(size_t ndx) { + m_realm->commit_transaction(); + + CollectionChangeIndices c; + _impl::TransactionChangeInfo info; + info.lists.push_back({ndx, 0, 0, &c}); + info.tables_needed.resize(m_group.size(), true); + _impl::transaction::advance_and_observe_linkviews(m_sg, info); + + if (info.lists.empty()) { + REQUIRE(!m_linkview->is_attached()); + return {}; + } + + validate(c); + return c; + } + + explicit operator bool() const { return m_realm->is_in_transaction(); } + +private: + std::unique_ptr m_history; + SharedGroup m_sg; + SharedRealm m_realm; + Group const& m_group; + + LinkViewRef m_linkview; + std::vector m_initial; + + void validate(CollectionChangeIndices const& info) + { + info.insertions.verify(); + info.deletions.verify(); + info.modifications.verify(); + + std::vector move_sources; + for (auto const& move : info.moves) + move_sources.push_back(m_initial[move.from]); + + // Apply the changes from the transaction log to our copy of the + // initial, using UITableView's batching rules (i.e. delete, then + // insert, then update) + auto it = std::make_reverse_iterator(info.deletions.end()), end = std::make_reverse_iterator(info.deletions.begin()); + for (; it != end; ++it) { + m_initial.erase(m_initial.begin() + it->first, m_initial.begin() + it->second); + } + + for (auto const& range : info.insertions) { + for (auto i = range.first; i < range.second; ++i) + m_initial.insert(m_initial.begin() + i, m_linkview->get(i).get_int(0)); + } + + for (auto const& range : info.modifications) { + for (auto i = range.first; i < range.second; ++i) + m_initial[i] = m_linkview->get(i).get_int(0); + } + + REQUIRE(m_linkview->is_attached()); + + // and make sure we end up with the same end result + REQUIRE(m_initial.size() == m_linkview->size()); + for (auto i = 0; i < m_initial.size(); ++i) + CHECK(m_initial[i] == m_linkview->get(i).get_int(0)); + + // Verify that everything marked as a move actually is one + for (size_t i = 0; i < move_sources.size(); ++i) + CHECK(m_linkview->get(info.moves[i].to).get_int(0) == move_sources[i]); + } +}; + TEST_CASE("Transaction log parsing") { - InMemoryTestFile path; - Realm::Config config = path; + InMemoryTestFile config; + config.automatic_change_notifications = false; SECTION("schema change validation") { config.schema = std::make_unique(Schema{ @@ -24,7 +113,7 @@ TEST_CASE("Transaction log parsing") { {"indexed", PropertyTypeInt, "", false, true} }}, }); - auto r = Realm::get_shared_realm(std::move(config)); + auto r = Realm::get_shared_realm(config); r->read_group(); auto history = make_client_history(config.path); @@ -83,4 +172,771 @@ TEST_CASE("Transaction log parsing") { REQUIRE_THROWS(r->refresh()); } } + + SECTION("row_did_change()") { + config.schema = std::make_unique(Schema{ + {"table", "", { + {"int", PropertyTypeInt}, + {"link", PropertyTypeObject, "table", false, false, true}, + {"array", PropertyTypeArray, "table"} + }}, + }); + + auto r = Realm::get_shared_realm(config); + auto table = r->read_group()->get_table("class_table"); + + r->begin_transaction(); + table->add_empty_row(10); + for (int i = 0; i < 10; ++i) + table->set_int(0, i, i); + r->commit_transaction(); + + auto track_changes = [&](auto&& f) { + auto history = make_client_history(config.path); + SharedGroup sg(*history, SharedGroup::durability_MemOnly); + Group const& g = sg.begin_read(); + + r->begin_transaction(); + f(); + r->commit_transaction(); + + _impl::TransactionChangeInfo info; + info.tables_needed.resize(g.size(), true); + _impl::transaction::advance_and_observe_linkviews(sg, info); + return info; + }; + + SECTION("direct changes are tracked") { + auto info = track_changes([&] { + table->set_int(0, 9, 10); + }); + + REQUIRE_FALSE(info.row_did_change(*table, 8)); + REQUIRE(info.row_did_change(*table, 9)); + } + + SECTION("changes over links are tracked") { + r->begin_transaction(); + for (int i = 0; i < 9; ++i) + table->set_link(1, i, i + 1); + r->commit_transaction(); + + auto info = track_changes([&] { + table->set_int(0, 9, 10); + }); + + REQUIRE(info.row_did_change(*table, 0)); + } + + SECTION("changes over linklists are tracked") { + r->begin_transaction(); + for (int i = 0; i < 9; ++i) + table->get_linklist(2, i)->add(i + 1); + r->commit_transaction(); + + auto info = track_changes([&] { + table->set_int(0, 9, 10); + }); + + REQUIRE(info.row_did_change(*table, 0)); + } + + SECTION("cycles over links do not loop forever") { + r->begin_transaction(); + table->set_link(1, 0, 0); + r->commit_transaction(); + + auto info = track_changes([&] { + table->set_int(0, 9, 10); + }); + REQUIRE_FALSE(info.row_did_change(*table, 0)); + } + + SECTION("cycles over linklists do not loop forever") { + r->begin_transaction(); + table->get_linklist(2, 0)->add(0); + r->commit_transaction(); + + auto info = track_changes([&] { + table->set_int(0, 9, 10); + }); + REQUIRE_FALSE(info.row_did_change(*table, 0)); + } + + SECTION("targets moving is not a change") { + r->begin_transaction(); + table->set_link(1, 0, 9); + table->get_linklist(2, 0)->add(9); + r->commit_transaction(); + + auto info = track_changes([&] { + table->move_last_over(5); + }); + REQUIRE_FALSE(info.row_did_change(*table, 0)); + } + + SECTION("changes made before a row is moved are reported") { + r->begin_transaction(); + table->set_link(1, 0, 9); + r->commit_transaction(); + + auto info = track_changes([&] { + table->set_int(0, 9, 5); + table->move_last_over(5); + }); + REQUIRE(info.row_did_change(*table, 0)); + + r->begin_transaction(); + table->get_linklist(2, 0)->add(8); + r->commit_transaction(); + + info = track_changes([&] { + table->set_int(0, 8, 5); + table->move_last_over(5); + }); + REQUIRE(info.row_did_change(*table, 0)); + } + + SECTION("changes made after a row is moved are reported") { + r->begin_transaction(); + table->set_link(1, 0, 9); + r->commit_transaction(); + + auto info = track_changes([&] { + table->move_last_over(5); + table->set_int(0, 5, 5); + }); + REQUIRE(info.row_did_change(*table, 0)); + + r->begin_transaction(); + table->get_linklist(2, 0)->add(8); + r->commit_transaction(); + + info = track_changes([&] { + table->move_last_over(5); + table->set_int(0, 5, 5); + }); + REQUIRE(info.row_did_change(*table, 0)); + } + } + + SECTION("table change information") { + config.schema = std::make_unique(Schema{ + {"table", "", { + {"value", PropertyTypeInt} + }}, + }); + + auto r = Realm::get_shared_realm(config); + auto& table = *r->read_group()->get_table("class_table"); + + r->begin_transaction(); + table.add_empty_row(10); + for (int i = 0; i < 10; ++i) + table.set_int(0, i, i); + r->commit_transaction(); + + auto track_changes = [&](std::vector tables_needed, auto&& f) { + auto history = make_client_history(config.path); + SharedGroup sg(*history, SharedGroup::durability_MemOnly); + sg.begin_read(); + + r->begin_transaction(); + f(); + r->commit_transaction(); + + _impl::TransactionChangeInfo info; + info.tables_needed = tables_needed; + _impl::transaction::advance_and_observe_linkviews(sg, info); + return info; + }; + + SECTION("modifying a row marks it as modified") { + auto info = track_changes({false, false, true}, [&] { + table.set_int(0, 1, 2); + }); + REQUIRE(info.tables.size() == 3); + REQUIRE_INDICES(info.tables[2].modifications, 1); + } + + SECTION("modifications to untracked tables are ignored") { + auto info = track_changes({false, false, false}, [&] { + table.set_int(0, 1, 2); + }); + REQUIRE(info.tables.size() == 0); + } + + SECTION("new row additions are reported") { + auto info = track_changes({false, false, true}, [&] { + table.add_empty_row(); + table.add_empty_row(); + }); + REQUIRE(info.tables.size() == 3); + REQUIRE_INDICES(info.tables[2].insertions, 10, 11); + } + + SECTION("deleting newly added rows makes them not be reported") { + auto info = track_changes({false, false, true}, [&] { + table.add_empty_row(); + table.add_empty_row(); + table.move_last_over(11); + }); + REQUIRE(info.tables.size() == 3); + REQUIRE_INDICES(info.tables[2].insertions, 10); + REQUIRE(info.tables[2].deletions.empty()); + } + + SECTION("modifying newly added rows is not reported as a modification") { + auto info = track_changes({false, false, true}, [&] { + table.add_empty_row(); + table.set_int(0, 10, 10); + }); + REQUIRE(info.tables.size() == 3); + REQUIRE_INDICES(info.tables[2].insertions, 10); + REQUIRE(info.tables[2].modifications.empty()); + } + + SECTION("move_last_over() does not shift rows other than the last one") { + auto info = track_changes({false, false, true}, [&] { + table.move_last_over(2); + table.move_last_over(3); + }); + REQUIRE(info.tables.size() == 3); + REQUIRE_INDICES(info.tables[2].deletions, 2, 3); + REQUIRE_MOVES(info.tables[2], {9, 2}, {8, 3}); + } + } + + SECTION("LinkView change information") { + config.schema = std::make_unique(Schema{ + {"origin", "", { + {"array", PropertyTypeArray, "target"} + }}, + {"target", "", { + {"value", PropertyTypeInt} + }}, + }); + + auto r = Realm::get_shared_realm(config); + + auto origin = r->read_group()->get_table("class_origin"); + auto target = r->read_group()->get_table("class_target"); + + r->begin_transaction(); + + target->add_empty_row(10); + for (int i = 0; i < 10; ++i) + target->set_int(0, i, i); + + origin->add_empty_row(); + LinkViewRef lv = origin->get_linklist(0, 0); + for (int i = 0; i < 10; ++i) + lv->add(i); + + r->commit_transaction(); + +#define VALIDATE_CHANGES(out) \ + for (CaptureHelper helper(config.path, r, lv); helper; out = helper.finish(origin->get_index_in_group())) + + CollectionChangeIndices changes; + SECTION("single change type") { + SECTION("add single") { + VALIDATE_CHANGES(changes) { + lv->add(0); + } + REQUIRE_INDICES(changes.insertions, 10); + } + SECTION("add multiple") { + VALIDATE_CHANGES(changes) { + lv->add(0); + lv->add(0); + } + REQUIRE_INDICES(changes.insertions, 10, 11); + } + + SECTION("erase single") { + VALIDATE_CHANGES(changes) { + lv->remove(5); + } + REQUIRE_INDICES(changes.deletions, 5); + } + SECTION("erase contiguous forward") { + VALIDATE_CHANGES(changes) { + lv->remove(5); + lv->remove(5); + lv->remove(5); + } + REQUIRE_INDICES(changes.deletions, 5, 6, 7); + } + SECTION("erase contiguous reverse") { + VALIDATE_CHANGES(changes) { + lv->remove(7); + lv->remove(6); + lv->remove(5); + } + REQUIRE_INDICES(changes.deletions, 5, 6, 7); + } + SECTION("erase contiguous mixed") { + VALIDATE_CHANGES(changes) { + lv->remove(5); + lv->remove(6); + lv->remove(5); + } + REQUIRE_INDICES(changes.deletions, 5, 6, 7); + } + SECTION("erase scattered forward") { + VALIDATE_CHANGES(changes) { + lv->remove(3); + lv->remove(4); + lv->remove(5); + } + REQUIRE_INDICES(changes.deletions, 3, 5, 7); + } + SECTION("erase scattered backwards") { + VALIDATE_CHANGES(changes) { + lv->remove(7); + lv->remove(5); + lv->remove(3); + } + REQUIRE_INDICES(changes.deletions, 3, 5, 7); + } + SECTION("erase scattered mixed") { + VALIDATE_CHANGES(changes) { + lv->remove(3); + lv->remove(6); + lv->remove(4); + } + REQUIRE_INDICES(changes.deletions, 3, 5, 7); + } + + SECTION("set single") { + VALIDATE_CHANGES(changes) { + lv->set(5, 0); + } + REQUIRE_INDICES(changes.modifications, 5); + } + SECTION("set contiguous") { + VALIDATE_CHANGES(changes) { + lv->set(5, 0); + lv->set(6, 0); + lv->set(7, 0); + } + REQUIRE_INDICES(changes.modifications, 5, 6, 7); + } + SECTION("set scattered") { + VALIDATE_CHANGES(changes) { + lv->set(5, 0); + lv->set(7, 0); + lv->set(9, 0); + } + REQUIRE_INDICES(changes.modifications, 5, 7, 9); + } + SECTION("set redundant") { + VALIDATE_CHANGES(changes) { + lv->set(5, 0); + lv->set(5, 0); + lv->set(5, 0); + } + REQUIRE_INDICES(changes.modifications, 5); + } + + SECTION("clear") { + VALIDATE_CHANGES(changes) { + lv->clear(); + } + REQUIRE_INDICES(changes.deletions, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + } + + SECTION("move backward") { + VALIDATE_CHANGES(changes) { + lv->move(5, 3); + } + REQUIRE_MOVES(changes, {5, 3}); + } + + SECTION("move forward") { + VALIDATE_CHANGES(changes) { + lv->move(1, 3); + } + REQUIRE_MOVES(changes, {1, 3}); + } + + SECTION("chained moves") { + VALIDATE_CHANGES(changes) { + lv->move(1, 3); + lv->move(3, 5); + } + REQUIRE_MOVES(changes, {1, 5}); + } + + SECTION("backwards chained moves") { + VALIDATE_CHANGES(changes) { + lv->move(5, 3); + lv->move(3, 1); + } + REQUIRE_MOVES(changes, {5, 1}); + } + + SECTION("moves shifting other moves") { + VALIDATE_CHANGES(changes) { + lv->move(1, 5); + lv->move(2, 7); + } + REQUIRE_MOVES(changes, {1, 4}, {3, 7}); + + VALIDATE_CHANGES(changes) { + lv->move(1, 5); + lv->move(7, 0); + } + REQUIRE_MOVES(changes, {1, 6}, {7, 0}); + } + + SECTION("move to current location is a no-op") { + VALIDATE_CHANGES(changes) { + lv->move(5, 5); + } + REQUIRE(changes.insertions.empty()); + REQUIRE(changes.deletions.empty()); + REQUIRE(changes.moves.empty()); + } + + SECTION("delete a target row") { + VALIDATE_CHANGES(changes) { + target->move_last_over(5); + } + REQUIRE_INDICES(changes.deletions, 5); + } + + SECTION("delete all target rows") { + VALIDATE_CHANGES(changes) { + lv->remove_all_target_rows(); + } + REQUIRE_INDICES(changes.deletions, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + } + + SECTION("clear target table") { + VALIDATE_CHANGES(changes) { + target->clear(); + } + REQUIRE_INDICES(changes.deletions, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + } + + SECTION("swap()") { + VALIDATE_CHANGES(changes) { + lv->swap(3, 5); + } + REQUIRE_INDICES(changes.modifications, 3, 5); + } + } + + SECTION("mixed change types") { + SECTION("set -> insert") { + VALIDATE_CHANGES(changes) { + lv->set(5, 0); + lv->insert(5, 0); + } + REQUIRE_INDICES(changes.insertions, 5); + REQUIRE_INDICES(changes.modifications, 6); + + VALIDATE_CHANGES(changes) { + lv->set(4, 0); + lv->insert(5, 0); + } + REQUIRE_INDICES(changes.insertions, 5); + REQUIRE_INDICES(changes.modifications, 4); + } + SECTION("insert -> set") { + VALIDATE_CHANGES(changes) { + lv->insert(5, 0); + lv->set(5, 1); + } + REQUIRE_INDICES(changes.insertions, 5); + REQUIRE(changes.modifications.size() == 0); + + VALIDATE_CHANGES(changes) { + lv->insert(5, 0); + lv->set(6, 1); + } + REQUIRE_INDICES(changes.insertions, 5); + REQUIRE_INDICES(changes.modifications, 6); + + VALIDATE_CHANGES(changes) { + lv->insert(6, 0); + lv->set(5, 1); + } + REQUIRE_INDICES(changes.insertions, 6); + REQUIRE_INDICES(changes.modifications, 5); + } + + SECTION("set -> erase") { + VALIDATE_CHANGES(changes) { + lv->set(5, 0); + lv->remove(5); + } + REQUIRE_INDICES(changes.deletions, 5); + REQUIRE(changes.modifications.size() == 0); + + VALIDATE_CHANGES(changes) { + lv->set(5, 0); + lv->remove(4); + } + REQUIRE_INDICES(changes.deletions, 4); + REQUIRE_INDICES(changes.modifications, 4); + + VALIDATE_CHANGES(changes) { + lv->set(5, 0); + lv->remove(4); + lv->remove(4); + } + REQUIRE_INDICES(changes.deletions, 4, 5); + REQUIRE(changes.modifications.size() == 0); + } + + SECTION("erase -> set") { + VALIDATE_CHANGES(changes) { + lv->remove(5); + lv->set(5, 0); + } + REQUIRE_INDICES(changes.deletions, 5); + REQUIRE_INDICES(changes.modifications, 5); + } + + SECTION("insert -> clear") { + VALIDATE_CHANGES(changes) { + lv->add(0); + lv->clear(); + } + REQUIRE_INDICES(changes.deletions, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + REQUIRE(changes.insertions.size() == 0); + } + + SECTION("set -> clear") { + VALIDATE_CHANGES(changes) { + lv->set(0, 5); + lv->clear(); + } + REQUIRE_INDICES(changes.deletions, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + REQUIRE(changes.modifications.size() == 0); + } + + SECTION("clear -> insert") { + VALIDATE_CHANGES(changes) { + lv->clear(); + lv->add(0); + } + REQUIRE_INDICES(changes.deletions, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + REQUIRE_INDICES(changes.insertions, 0); + } + + SECTION("insert -> delete") { + VALIDATE_CHANGES(changes) { + lv->add(0); + lv->remove(10); + } + REQUIRE(changes.insertions.size() == 0); + REQUIRE(changes.deletions.size() == 0); + + VALIDATE_CHANGES(changes) { + lv->add(0); + lv->remove(9); + } + REQUIRE_INDICES(changes.deletions, 9); + REQUIRE_INDICES(changes.insertions, 9); + + VALIDATE_CHANGES(changes) { + lv->insert(1, 1); + lv->insert(3, 3); + lv->insert(5, 5); + lv->remove(6); + lv->remove(4); + lv->remove(2); + } + REQUIRE_INDICES(changes.deletions, 1, 2, 3); + REQUIRE_INDICES(changes.insertions, 1, 2, 3); + + VALIDATE_CHANGES(changes) { + lv->insert(1, 1); + lv->insert(3, 3); + lv->insert(5, 5); + lv->remove(2); + lv->remove(3); + lv->remove(4); + } + REQUIRE_INDICES(changes.deletions, 1, 2, 3); + REQUIRE_INDICES(changes.insertions, 1, 2, 3); + } + + SECTION("delete -> insert") { + VALIDATE_CHANGES(changes) { + lv->remove(9); + lv->add(0); + } + REQUIRE_INDICES(changes.deletions, 9); + REQUIRE_INDICES(changes.insertions, 9); + } + + SECTION("interleaved delete and insert") { + VALIDATE_CHANGES(changes) { + lv->remove(9); + lv->remove(7); + lv->remove(5); + lv->remove(3); + lv->remove(1); + + lv->insert(4, 9); + lv->insert(3, 7); + lv->insert(2, 5); + lv->insert(1, 3); + lv->insert(0, 1); + + lv->remove(9); + lv->remove(7); + lv->remove(5); + lv->remove(3); + lv->remove(1); + } + + REQUIRE_INDICES(changes.deletions, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + REQUIRE_INDICES(changes.insertions, 0, 1, 2, 3, 4); + } + + SECTION("move after set is just insert+delete") { + VALIDATE_CHANGES(changes) { + lv->set(5, 6); + lv->move(5, 0); + } + + REQUIRE_INDICES(changes.deletions, 5); + REQUIRE_INDICES(changes.insertions, 0); + REQUIRE(changes.moves.empty()); + } + + SECTION("set after move is just insert+delete") { + VALIDATE_CHANGES(changes) { + lv->move(5, 0); + lv->set(0, 6); + } + + REQUIRE_INDICES(changes.deletions, 5); + REQUIRE_INDICES(changes.insertions, 0); + REQUIRE(changes.moves.empty()); + } + + SECTION("delete after move removes original row") { + VALIDATE_CHANGES(changes) { + lv->move(5, 0); + lv->remove(0); + } + + REQUIRE_INDICES(changes.deletions, 5); + REQUIRE(changes.moves.empty()); + } + + SECTION("moving newly inserted row just changes reported index of insert") { + VALIDATE_CHANGES(changes) { + lv->move(5, 0); + lv->remove(0); + } + + REQUIRE_INDICES(changes.deletions, 5); + REQUIRE(changes.moves.empty()); + } + + SECTION("moves shift insertions/changes like any other insertion") { + VALIDATE_CHANGES(changes) { + lv->insert(5, 5); + lv->set(6, 6); + lv->move(7, 4); + } + REQUIRE_INDICES(changes.deletions, 6); + REQUIRE_INDICES(changes.insertions, 4, 6); + REQUIRE_INDICES(changes.modifications, 7); + REQUIRE_MOVES(changes, {6, 4}); + } + + SECTION("clear after delete") { + VALIDATE_CHANGES(changes) { + lv->remove(5); + lv->clear(); + } + REQUIRE_INDICES(changes.deletions, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + } + + SECTION("erase before previous move target") { + VALIDATE_CHANGES(changes) { + lv->move(2, 8); + lv->remove(5); + } + REQUIRE_INDICES(changes.insertions, 7); + REQUIRE_INDICES(changes.deletions, 2, 6); + REQUIRE_MOVES(changes, {2, 7}); + } + + SECTION("insert after move updates move destination") { + VALIDATE_CHANGES(changes) { + lv->move(2, 8); + lv->insert(5, 5); + } + REQUIRE_MOVES(changes, {2, 9}); + } + } + + SECTION("deleting the linkview") { + SECTION("directly") { + VALIDATE_CHANGES(changes) { + origin->move_last_over(0); + } + REQUIRE(!lv->is_attached()); + REQUIRE(changes.insertions.empty()); + REQUIRE(changes.deletions.empty()); + REQUIRE(changes.modifications.empty()); + } + + SECTION("table clear") { + VALIDATE_CHANGES(changes) { + origin->clear(); + } + REQUIRE(!lv->is_attached()); + REQUIRE(changes.insertions.empty()); + REQUIRE(changes.deletions.empty()); + REQUIRE(changes.modifications.empty()); + } + + SECTION("delete a different lv") { + r->begin_transaction(); + origin->add_empty_row(); + r->commit_transaction(); + + VALIDATE_CHANGES(changes) { + origin->move_last_over(1); + } + REQUIRE(changes.insertions.empty()); + REQUIRE(changes.deletions.empty()); + REQUIRE(changes.modifications.empty()); + } + } + + SECTION("modifying a different linkview should not produce notifications") { + r->begin_transaction(); + origin->add_empty_row(); + LinkViewRef lv2 = origin->get_linklist(0, 1); + lv2->add(5); + r->commit_transaction(); + + VALIDATE_CHANGES(changes) { + lv2->add(1); + lv2->add(2); + lv2->remove(0); + lv2->set(0, 6); + lv2->move(1, 0); + lv2->swap(0, 1); + lv2->clear(); + lv2->add(1); + } + + REQUIRE(changes.insertions.empty()); + REQUIRE(changes.deletions.empty()); + REQUIRE(changes.modifications.empty()); + } + } } diff --git a/tests/util/index_helpers.hpp b/tests/util/index_helpers.hpp new file mode 100644 index 00000000..4d847574 --- /dev/null +++ b/tests/util/index_helpers.hpp @@ -0,0 +1,22 @@ +#define REQUIRE_INDICES(index_set, ...) do { \ + index_set.verify(); \ + std::initializer_list expected = {__VA_ARGS__}; \ + auto actual = index_set.as_indexes(); \ + REQUIRE(expected.size() == std::distance(actual.begin(), actual.end())); \ + auto begin = actual.begin(), end = actual.end(); \ + for (auto index : expected) { \ + REQUIRE(*begin++ == index); \ + } \ +} while (0) + +#define REQUIRE_MOVES(c, ...) do { \ + auto actual = (c); \ + std::initializer_list expected = {__VA_ARGS__}; \ + REQUIRE(expected.size() == actual.moves.size()); \ + auto begin = actual.moves.begin(), end = actual.moves.end(); \ + for (auto move : expected) { \ + CHECK(begin->from == move.from); \ + CHECK(begin->to == move.to); \ + ++begin; \ + } \ +} while (0) From f4aaa7c9de2c7dd4305494720ec552da47d5640e Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 8 Jan 2016 10:13:05 -0800 Subject: [PATCH 09/93] Add fine-grained notifications for Results --- src/collection_notifications.cpp | 252 +++++++++++++++++ src/collection_notifications.hpp | 5 + src/impl/async_query.cpp | 59 +++- src/impl/async_query.hpp | 10 + src/impl/background_collection.cpp | 32 +++ src/impl/background_collection.hpp | 7 +- src/impl/list_notifier.cpp | 28 +- src/impl/list_notifier.hpp | 2 +- src/impl/realm_coordinator.cpp | 5 +- src/results.cpp | 8 + src/results.hpp | 1 + tests/collection_change_indices.cpp | 65 +++++ tests/results.cpp | 416 ++++++++++++++++++++++++---- 13 files changed, 803 insertions(+), 87 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 57cdd577..3f82c913 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -311,3 +311,255 @@ void CollectionChangeIndices::verify() REALM_ASSERT(!modifications.contains(index)); #endif } + +namespace { +struct RowInfo { + size_t shifted_row_index; + size_t prev_tv_index; + size_t tv_index; +}; + +void calculate_moves_unsorted(std::vector& new_rows, CollectionChangeIndices& changeset, + std::function row_did_change) +{ + std::sort(begin(new_rows), end(new_rows), [](auto& lft, auto& rgt) { + return lft.tv_index < rgt.tv_index; + }); + + IndexSet::IndexInterator ins = changeset.insertions.begin(), del = changeset.deletions.begin(); + int shift = 0; + for (auto& row : new_rows) { + while (del != changeset.deletions.end() && *del <= row.tv_index) { + ++del; + ++shift; + } + while (ins != changeset.insertions.end() && *ins <= row.tv_index) { + ++ins; + --shift; + } + if (row.prev_tv_index == npos) + continue; + + // For unsorted, non-LV queries a row can only move to an index before + // its original position due to a move_last_over + if (row.tv_index + shift != row.prev_tv_index) { + --shift; + changeset.moves.push_back({row.prev_tv_index, row.tv_index}); + } + // FIXME: currently move implies modification, and so they're mutally exclusive + // this is correct for sorted, but for unsorted a row can move without actually changing + else if (row_did_change(row.shifted_row_index)) { + // FIXME: needlessly quadratic + if (!changeset.insertions.contains(row.tv_index)) + changeset.modifications.add(row.tv_index); + } + } + + // FIXME: this is required for merge(), but it would be nice if it wasn't + for (auto&& move : changeset.moves) { + changeset.insertions.add(move.to); + changeset.deletions.add(move.from); + } +} + +using items = std::vector>; + +struct Match { + size_t i, j, size; +}; + +Match find_longest_match(items const& a, items const& b, + size_t begin1, size_t end1, size_t begin2, size_t end2) +{ + Match best = {begin1, begin2, 0}; + std::vector len_from_j; + len_from_j.resize(end2 - begin2, 0); + std::vector len_from_j_prev = len_from_j; + + for (size_t i = begin1; i < end1; ++i) { + std::fill(begin(len_from_j), end(len_from_j), 0); + + size_t ai = a[i].first; + auto it = lower_bound(begin(b), end(b), std::make_pair(size_t(0), ai), + [](auto a, auto b) { return a.second < b.second; }); + for (; it != end(b) && it->second == ai; ++it) { + size_t j = it->first; + if (j < begin2) + continue; + if (j >= end2) + break; + + size_t off = j - begin2; + size_t size = off == 0 ? 1 : len_from_j_prev[off - 1] + 1; + len_from_j[off] = size; + if (size > best.size) { + best.i = i - size + 1; + best.j = j - size + 1; + best.size = size; + } + } + len_from_j.swap(len_from_j_prev); + } + return best; +} + +void find_longest_matches(items const& a, items const& b_ndx, + size_t begin1, size_t end1, size_t begin2, size_t end2, std::vector& ret) +{ + // FIXME: recursion could get too deep here + Match m = find_longest_match(a, b_ndx, begin1, end1, begin2, end2); + if (!m.size) + return; + if (m.i > begin1 && m.j > begin2) + find_longest_matches(a, b_ndx, begin1, m.i, begin2, m.j, ret); + ret.push_back(m); + if (m.i + m.size < end2 && m.j + m.size < end2) + find_longest_matches(a, b_ndx, m.i + m.size, end1, m.j + m.size, end2, ret); +} + +void calculate_moves_sorted(std::vector& new_rows, CollectionChangeIndices& changeset, + std::function row_did_change) +{ + std::vector> old_candidates; + std::vector> new_candidates; + + std::sort(begin(new_rows), end(new_rows), [](auto& lft, auto& rgt) { + return lft.tv_index < rgt.tv_index; + }); + + IndexSet::IndexInterator ins = changeset.insertions.begin(), del = changeset.deletions.begin(); + int shift = 0; + for (auto& row : new_rows) { + while (del != changeset.deletions.end() && *del <= row.tv_index) { + ++del; + ++shift; + } + while (ins != changeset.insertions.end() && *ins <= row.tv_index) { + ++ins; + --shift; + } + if (row.prev_tv_index == npos) + continue; + + if (row_did_change(row.shifted_row_index)) { + // FIXME: needlessly quadratic + if (!changeset.insertions.contains(row.tv_index)) + changeset.modifications.add(row.tv_index); + } + old_candidates.push_back({row.shifted_row_index, row.prev_tv_index}); + new_candidates.push_back({row.shifted_row_index, row.tv_index}); +// } + } + + std::sort(begin(old_candidates), end(old_candidates), [](auto a, auto b) { + if (a.second != b.second) + return a.second < b.second; + return a.first < b.first; + }); + + // First check if the order of any of the rows actually changed + size_t first_difference = npos; + for (size_t i = 0; i < old_candidates.size(); ++i) { + if (old_candidates[i].first != new_candidates[i].first) { + first_difference = i; + break; + } + } + if (first_difference == npos) + return; + + const auto b_ndx = [&]{ + std::vector> ret; + ret.reserve(new_candidates.size()); + for (size_t i = 0; i < new_candidates.size(); ++i) + ret.push_back(std::make_pair(i, new_candidates[i].first)); + std::sort(begin(ret), end(ret), [](auto a, auto b) { + if (a.second != b.second) + return a.second < b.second; + return a.first < b.first; + }); + return ret; + }(); + + std::vector longest_matches; + find_longest_matches(old_candidates, b_ndx, + first_difference, old_candidates.size(), + first_difference, new_candidates.size(), + longest_matches); + longest_matches.push_back({old_candidates.size(), new_candidates.size(), 0}); + + size_t i = first_difference, j = first_difference; + for (auto match : longest_matches) { + for (; i < match.i; ++i) + changeset.deletions.add(old_candidates[i].second); + for (; j < match.j; ++j) + changeset.insertions.add(new_candidates[j].second); + i += match.size; + j += match.size; + } + + // FIXME: needlessly suboptimal + changeset.modifications.remove(changeset.insertions); +} +} // Anonymous namespace + +CollectionChangeIndices CollectionChangeIndices::calculate(std::vector const& prev_rows, + std::vector const& next_rows, + std::function row_did_change, + bool sort) +{ + CollectionChangeIndices ret; + + std::vector old_rows; + for (size_t i = 0; i < prev_rows.size(); ++i) { + if (prev_rows[i] == npos) + ret.deletions.add(i); + else + old_rows.push_back({prev_rows[i], npos, i}); + } + std::stable_sort(begin(old_rows), end(old_rows), [](auto& lft, auto& rgt) { + return lft.shifted_row_index < rgt.shifted_row_index; + }); + + std::vector new_rows; + for (size_t i = 0; i < next_rows.size(); ++i) { + new_rows.push_back({next_rows[i], npos, i}); + } + std::stable_sort(begin(new_rows), end(new_rows), [](auto& lft, auto& rgt) { + return lft.shifted_row_index < rgt.shifted_row_index; + }); + + size_t i = 0, j = 0; + while (i < old_rows.size() && j < new_rows.size()) { + auto old_index = old_rows[i]; + auto new_index = new_rows[j]; + if (old_index.shifted_row_index == new_index.shifted_row_index) { + new_rows[j].prev_tv_index = old_rows[i].tv_index; + ++i; + ++j; + } + else if (old_index.shifted_row_index < new_index.shifted_row_index) { + ret.deletions.add(old_index.tv_index); + ++i; + } + else { + ret.insertions.add(new_index.tv_index); + ++j; + } + } + + for (; i < old_rows.size(); ++i) + ret.deletions.add(old_rows[i].tv_index); + for (; j < new_rows.size(); ++j) + ret.insertions.add(new_rows[j].tv_index); + + if (sort) { + calculate_moves_sorted(new_rows, ret, row_did_change); + } + else { + calculate_moves_unsorted(new_rows, ret, row_did_change); + } + ret.verify(); + + return ret; +} diff --git a/src/collection_notifications.hpp b/src/collection_notifications.hpp index b1543221..3bf1f9c5 100644 --- a/src/collection_notifications.hpp +++ b/src/collection_notifications.hpp @@ -70,6 +70,11 @@ struct CollectionChangeIndices { IndexSet modification = {}, std::vector moves = {}); + static CollectionChangeIndices calculate(std::vector const& old_rows, + std::vector const& new_rows, + std::function row_did_change, + bool sort); + bool empty() const { return deletions.empty() && insertions.empty() && modifications.empty() && moves.empty(); } void merge(CollectionChangeIndices&&); diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index 9f09f2f5..a91491fa 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -30,6 +30,7 @@ AsyncQuery::AsyncQuery(Results& target) , m_sort(target.get_sort()) { Query q = target.get_query(); + set_table(*q.get_table()); m_query_handover = Realm::Internal::get_shared_group(get_realm()).export_for_handover(q, MutableSourcePayload::Move); } @@ -62,8 +63,26 @@ void AsyncQuery::release_data() noexcept // destroyed while the background work is running, and to allow removing // callbacks from any thread. +static bool map_moves(size_t& idx, CollectionChangeIndices const& changes) +{ + for (auto&& move : changes.moves) { + if (move.from == idx) { + idx = move.to; + return true; + } + } + return false; +} + +void AsyncQuery::do_add_required_change_info(TransactionChangeInfo& info) +{ + REALM_ASSERT(m_query); + m_info = &info; +} + void AsyncQuery::run() { + REALM_ASSERT(m_info); m_did_change = false; { @@ -76,6 +95,8 @@ void AsyncQuery::run() REALM_ASSERT(!m_tv.is_attached()); + size_t table_ndx = m_query->get_table()->get_index_in_group(); + // If we've run previously, check if we need to rerun if (m_initial_run_complete) { // Make an empty tableview from the query to get the table version, since @@ -90,6 +111,39 @@ void AsyncQuery::run() m_tv.sort(m_sort.column_indices, m_sort.ascending); } + if (m_initial_run_complete) { + auto changes = table_ndx < m_info->tables.size() ? &m_info->tables[table_ndx] : nullptr; + + std::vector next_rows; + next_rows.reserve(m_tv.size()); + for (size_t i = 0; i < m_tv.size(); ++i) + next_rows.push_back(m_tv[i].get_index()); + + if (changes) { + for (auto& idx : m_previous_rows) { + if (changes->deletions.contains(idx)) + idx = npos; + else + map_moves(idx, *changes); + REALM_ASSERT_DEBUG(!changes->insertions.contains(idx)); + } + } + + m_changes = CollectionChangeIndices::calculate(m_previous_rows, next_rows, + [&](size_t row) { return m_info->row_did_change(*m_query->get_table(), row); }, + !!m_sort); + m_previous_rows = std::move(next_rows); + if (m_changes.empty()) { + m_tv = {}; + return; + } + } + else { + m_previous_rows.resize(m_tv.size()); + for (size_t i = 0; i < m_tv.size(); ++i) + m_previous_rows[i] = m_tv[i].get_index(); + } + m_did_change = true; } @@ -105,9 +159,12 @@ bool AsyncQuery::do_prepare_handover(SharedGroup& sg) m_handed_over_table_version = m_tv.sync_if_needed(); m_tv_handover = sg.export_for_handover(m_tv, MutableSourcePayload::Move); + add_changes(std::move(m_changes)); + REALM_ASSERT(m_changes.empty()); + // detach the TableView as we won't need it again and keeping it around // makes advance_read() much more expensive - m_tv = TableView(); + m_tv = {}; return m_did_change; } diff --git a/src/impl/async_query.hpp b/src/impl/async_query.hpp index 82776166..324bd84f 100644 --- a/src/impl/async_query.hpp +++ b/src/impl/async_query.hpp @@ -27,11 +27,14 @@ #include #include #include +#include #include #include namespace realm { namespace _impl { +struct TransactionChangeInfo; + class AsyncQuery : public BackgroundCollection { public: AsyncQuery(Results& target); @@ -45,6 +48,8 @@ private: // Returns if any callbacks need to be invoked bool do_deliver(SharedGroup& sg) override; + void do_add_required_change_info(TransactionChangeInfo& info) override; + void release_data() noexcept override; void do_attach_to(SharedGroup& sg) override; void do_detach_from(SharedGroup& sg) override; @@ -64,9 +69,14 @@ private: TableView m_tv; std::unique_ptr> m_tv_handover; + CollectionChangeIndices m_changes; + TransactionChangeInfo* m_info = nullptr; + uint_fast64_t m_handed_over_table_version = -1; bool m_did_change = false; + std::vector m_previous_rows; + bool m_initial_run_complete = false; }; diff --git a/src/impl/background_collection.cpp b/src/impl/background_collection.cpp index a3e932a3..59dba0da 100644 --- a/src/impl/background_collection.cpp +++ b/src/impl/background_collection.cpp @@ -102,6 +102,38 @@ bool BackgroundCollection::is_alive() const noexcept return m_realm != nullptr; } +// Recursively add `table` and all tables it links to to `out` +static void find_relevant_tables(std::vector& out, Table const& table) +{ + auto table_ndx = table.get_index_in_group(); + if (find(begin(out), end(out), table_ndx) != end(out)) + return; + out.push_back(table_ndx); + + for (size_t i = 0, count = table.get_column_count(); i != count; ++i) { + if (table.get_column_type(i) == type_Link || table.get_column_type(i) == type_LinkList) { + find_relevant_tables(out, *table.get_link_target(i)); + } + } +} + +void BackgroundCollection::set_table(Table const& table) +{ + find_relevant_tables(m_relevant_tables, table); +} + +void BackgroundCollection::add_required_change_info(TransactionChangeInfo& info) +{ + auto max = *max_element(begin(m_relevant_tables), end(m_relevant_tables)) + 1; + if (max > info.tables_needed.size()) + info.tables_needed.resize(max, false); + for (auto table_ndx : m_relevant_tables) { + info.tables_needed[table_ndx] = true; + } + + do_add_required_change_info(info); +} + void BackgroundCollection::prepare_handover() { REALM_ASSERT(m_sg); diff --git a/src/impl/background_collection.hpp b/src/impl/background_collection.hpp index e3508e78..26dd28c8 100644 --- a/src/impl/background_collection.hpp +++ b/src/impl/background_collection.hpp @@ -57,7 +57,7 @@ public: // SharedGroup void detach(); - virtual void add_required_change_info(TransactionChangeInfo&) { } + void add_required_change_info(TransactionChangeInfo&); virtual void run() { } void prepare_handover(); @@ -70,12 +70,14 @@ protected: bool have_callbacks() const noexcept { return m_have_callbacks; } bool have_changes() const noexcept { return !m_accumulated_changes.empty(); } void add_changes(CollectionChangeIndices change) { m_accumulated_changes.merge(std::move(change)); } + void set_table(Table const& table); private: virtual void do_attach_to(SharedGroup&) = 0; virtual void do_detach_from(SharedGroup&) = 0; virtual bool do_prepare_handover(SharedGroup&) = 0; virtual bool do_deliver(SharedGroup&) = 0; + virtual void do_add_required_change_info(TransactionChangeInfo&) { } const std::thread::id m_thread_id = std::this_thread::get_id(); bool is_for_current_thread() const { return m_thread_id == std::this_thread::get_id(); } @@ -92,6 +94,9 @@ private: uint_fast64_t m_results_version = 0; + // Tables which this collection needs change information for + std::vector m_relevant_tables; + struct Callback { CollectionChangeCallback fn; size_t token; diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp index 78c0d7ef..f32b4b49 100644 --- a/src/impl/list_notifier.cpp +++ b/src/impl/list_notifier.cpp @@ -26,28 +26,11 @@ using namespace realm; using namespace realm::_impl; -// Recursively add `table` and all tables it links to to `out` -static void find_relevant_tables(std::vector& out, Table const& table) -{ - auto table_ndx = table.get_index_in_group(); - if (find(begin(out), end(out), table_ndx) != end(out)) - return; - out.push_back(table_ndx); - - for (size_t i = 0, count = table.get_column_count(); i != count; ++i) { - if (table.get_column_type(i) == type_Link || table.get_column_type(i) == type_LinkList) { - find_relevant_tables(out, *table.get_link_target(i)); - } - } -} - ListNotifier::ListNotifier(LinkViewRef lv, std::shared_ptr realm) : BackgroundCollection(std::move(realm)) , m_prev_size(lv->size()) { - find_relevant_tables(m_relevant_tables, lv->get_target_table()); - // Find the lv's column, since that isn't tracked directly size_t row_ndx = lv->get_origin_row_index(); m_col_ndx = not_found; @@ -60,6 +43,8 @@ ListNotifier::ListNotifier(LinkViewRef lv, std::shared_ptr realm) } REALM_ASSERT(m_col_ndx != not_found); + set_table(lv->get_target_table()); + auto& sg = Realm::Internal::get_shared_group(get_realm()); m_lv_handover = sg.export_linkview_for_handover(lv); } @@ -86,7 +71,7 @@ void ListNotifier::do_detach_from(SharedGroup& sg) } } -void ListNotifier::add_required_change_info(TransactionChangeInfo& info) +void ListNotifier::do_add_required_change_info(TransactionChangeInfo& info) { REALM_ASSERT(!m_lv_handover); if (!m_lv) { @@ -97,13 +82,6 @@ void ListNotifier::add_required_change_info(TransactionChangeInfo& info) auto& table = m_lv->get_origin_table(); info.lists.push_back({table.get_index_in_group(), row_ndx, m_col_ndx, &m_change}); - auto max = *max_element(begin(m_relevant_tables), end(m_relevant_tables)) + 1; - if (max > info.tables_needed.size()) - info.tables_needed.resize(max, false); - for (auto table_ndx : m_relevant_tables) { - info.tables_needed[table_ndx] = true; - } - m_info = &info; } diff --git a/src/impl/list_notifier.hpp b/src/impl/list_notifier.hpp index 6c666bf7..dcb88ef7 100644 --- a/src/impl/list_notifier.hpp +++ b/src/impl/list_notifier.hpp @@ -47,7 +47,7 @@ private: void do_detach_from(SharedGroup& sg) override; void release_data() noexcept override; - void add_required_change_info(TransactionChangeInfo& info) override; + void do_add_required_change_info(TransactionChangeInfo& info) override; }; } } diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index a6f5664b..9a8d6916 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -51,11 +51,10 @@ bool TransactionChangeInfo::row_did_change(Table const& table, size_t idx, int d for (size_t i = 0, count = table.get_column_count(); i < count; ++i) { auto type = table.get_column_type(i); if (type == type_Link) { - auto& target = *table.get_link_target(i); - if (target.is_null_link(i, idx)) + if (table.is_null_link(i, idx)) continue; auto dst = table.get_link(i, idx); - return row_did_change(target, dst, depth + 1); + return row_did_change(*table.get_link_target(i), dst, depth + 1); } if (type != type_LinkList) continue; diff --git a/src/results.cpp b/src/results.cpp index f3f4a277..bba91cf8 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -46,6 +46,7 @@ Results::Results(SharedRealm r, const ObjectSchema &o, Query q, SortOrder s) , m_sort(std::move(s)) , m_mode(Mode::Query) { + REALM_ASSERT(m_sort.column_indices.size() == m_sort.ascending.size()); } Results::Results(SharedRealm r, const ObjectSchema &o, Table& table) @@ -350,6 +351,7 @@ TableView Results::get_tableview() Results Results::sort(realm::SortOrder&& sort) const { + REALM_ASSERT(sort.column_indices.size() == sort.ascending.size()); return Results(m_realm, get_object_schema(), get_query(), std::move(sort)); } @@ -380,6 +382,12 @@ NotificationToken Results::async(std::function target return {m_background_query, m_background_query->add_callback(wrap)}; } +NotificationToken Results::add_notification_callback(CollectionChangeCallback cb) +{ + prepare_async(); + return {m_background_query, m_background_query->add_callback(std::move(cb))}; +} + void Results::Internal::set_table_view(Results& results, realm::TableView &&tv) { // If the previous TableView was never actually used, then stop generating diff --git a/src/results.hpp b/src/results.hpp index 70000d9e..6529a90a 100644 --- a/src/results.hpp +++ b/src/results.hpp @@ -177,6 +177,7 @@ public: // The query will be run on a background thread and delivered to the callback, // and then rerun after each commit (if needed) and redelivered if it changed NotificationToken async(std::function target); + NotificationToken add_notification_callback(CollectionChangeCallback cb); bool wants_background_updates() const { return m_wants_background_updates; } diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index d753475b..cab25f58 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -177,6 +177,71 @@ TEST_CASE("collection change indices") { } } + SECTION("calculate") { + auto all_modified = [](size_t) { return true; }; + auto none_modified = [](size_t) { return false; }; + + SECTION("no changes") { + c = CollectionChangeIndices::calculate({1, 2, 3}, {1, 2, 3}, none_modified, false); + REQUIRE(c.empty()); + } + + SECTION("inserting from empty") { + c = CollectionChangeIndices::calculate({}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.insertions, 0, 1, 2); + } + + SECTION("deleting all existing") { + c = CollectionChangeIndices::calculate({1, 2, 3}, {}, all_modified, false); + REQUIRE_INDICES(c.deletions, 0, 1, 2); + } + + SECTION("all rows modified without changing order") { + c = CollectionChangeIndices::calculate({1, 2, 3}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.modifications, 0, 1, 2); + } + + SECTION("single insertion in middle") { + c = CollectionChangeIndices::calculate({1, 3}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.insertions, 1); + } + + SECTION("single deletion in middle") { + c = CollectionChangeIndices::calculate({1, 2, 3}, {1, 3}, all_modified, false); + REQUIRE_INDICES(c.deletions, 1); + } + + SECTION("unsorted reordering") { + auto calc = [&](std::vector values) { + return CollectionChangeIndices::calculate({1, 2, 3}, values, none_modified, false); + }; + + // The commented-out permutations are not possible with + // move_last_over() and so are unhandled by unsorted mode + REQUIRE(calc({1, 2, 3}).empty()); + REQUIRE_MOVES(calc({1, 3, 2}), {2, 1}); +// REQUIRE_MOVES(calc({2, 1, 3}), {1, 0}); +// REQUIRE_MOVES(calc({2, 3, 1}), {1, 0}, {2, 1}); + REQUIRE_MOVES(calc({3, 1, 2}), {2, 0}); + REQUIRE_MOVES(calc({3, 2, 1}), {2, 0}, {1, 1}); + } + + SECTION("sorted reordering") { + auto calc = [&](std::vector values) { + return CollectionChangeIndices::calculate({1, 2, 3}, values, all_modified, true); + }; + + REQUIRE(calc({1, 2, 3}).moves.empty()); + return; + // none of these actually work since it just does insert+delete + REQUIRE_MOVES(calc({1, 3, 2}), {2, 1}); + REQUIRE_MOVES(calc({2, 1, 3}), {1, 0}); + REQUIRE_MOVES(calc({2, 3, 1}), {1, 0}, {2, 1}); + REQUIRE_MOVES(calc({3, 1, 2}), {2, 0}); + REQUIRE_MOVES(calc({3, 2, 1}), {2, 0}, {1, 1}); + } + } + SECTION("merge") { SECTION("deletions are shifted by previous deletions") { c = {{5}, {}, {}, {}}; diff --git a/tests/results.cpp b/tests/results.cpp index 3dda335e..62989924 100644 --- a/tests/results.cpp +++ b/tests/results.cpp @@ -1,5 +1,6 @@ #include "catch.hpp" +#include "util/index_helpers.hpp" #include "util/test_file.hpp" #include "impl/realm_coordinator.hpp" @@ -12,6 +13,8 @@ #include #include +#include + using namespace realm; TEST_CASE("Results") { @@ -41,76 +44,38 @@ TEST_CASE("Results") { r->begin_transaction(); table->add_empty_row(10); for (int i = 0; i < 10; ++i) - table->set_int(0, i, i); + table->set_int(0, i, i * 2); r->commit_transaction(); - Results results(r, *config.schema->find("object"), table->where().greater(0, 0).less(0, 5)); + Results results(r, *config.schema->find("object"), table->where().greater(0, 0).less(0, 10)); - SECTION("notifications") { + SECTION("unsorted notifications") { int notification_calls = 0; - auto token = results.async([&](std::exception_ptr err) { + CollectionChangeIndices change; + auto token = results.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { REQUIRE_FALSE(err); + change = c; ++notification_calls; }); coordinator->on_change(); r->notify(); + auto write = [&](auto&& f) { + r->begin_transaction(); + f(); + r->commit_transaction(); + coordinator->on_change(); + r->notify(); + }; + SECTION("initial results are delivered") { REQUIRE(notification_calls == 1); } - SECTION("modifying the table sends a notification asynchronously") { + SECTION("notifications are sent asynchronously") { r->begin_transaction(); - table->set_int(0, 0, 0); - r->commit_transaction(); - - REQUIRE(notification_calls == 1); - coordinator->on_change(); - r->notify(); - REQUIRE(notification_calls == 2); - } - - SECTION("modifying a linked-to table send a notification") { - r->begin_transaction(); - r->read_group()->get_table("class_linked to object")->add_empty_row(); - r->commit_transaction(); - - REQUIRE(notification_calls == 1); - coordinator->on_change(); - r->notify(); - REQUIRE(notification_calls == 2); - } - - SECTION("modifying a a linking table sends a notification") { - r->begin_transaction(); - r->read_group()->get_table("class_linking object")->add_empty_row(); - r->commit_transaction(); - - REQUIRE(notification_calls == 1); - coordinator->on_change(); - r->notify(); - REQUIRE(notification_calls == 2); - } - - SECTION("modifying a an unrelated table does not send a notification") { - r->begin_transaction(); - r->read_group()->get_table("class_other object")->add_empty_row(); - r->commit_transaction(); - - REQUIRE(notification_calls == 1); - coordinator->on_change(); - r->notify(); - REQUIRE(notification_calls == 1); - } - - SECTION("modifications from multiple transactions are collapsed") { - r->begin_transaction(); - table->set_int(0, 0, 0); - r->commit_transaction(); - - r->begin_transaction(); - table->set_int(0, 1, 0); + table->set_int(0, 0, 4); r->commit_transaction(); REQUIRE(notification_calls == 1); @@ -121,7 +86,7 @@ TEST_CASE("Results") { SECTION("notifications are not delivered when the token is destroyed before they are calculated") { r->begin_transaction(); - table->set_int(0, 0, 0); + table->set_int(0, 0, 4); r->commit_transaction(); REQUIRE(notification_calls == 1); @@ -133,7 +98,7 @@ TEST_CASE("Results") { SECTION("notifications are not delivered when the token is destroyed before they are delivered") { r->begin_transaction(); - table->set_int(0, 0, 0); + table->set_int(0, 0, 4); r->commit_transaction(); REQUIRE(notification_calls == 1); @@ -142,5 +107,344 @@ TEST_CASE("Results") { r->notify(); REQUIRE(notification_calls == 1); } + + SECTION("notifications are delivered when a new callback is added from within a callback") { + NotificationToken token2, token3; + bool called = false; + token2 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr) { + token3 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr) { + called = true; + }); + }); + + coordinator->on_change(); + r->notify(); + + REQUIRE(called); + } + + SECTION("notifications are not delivered when a callback is removed from within a callback") { + NotificationToken token2, token3; + token2 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr) { + token3 = {}; + }); + token3 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr) { + REQUIRE(false); + }); + + coordinator->on_change(); + r->notify(); + } + + SECTION("removing the current callback does not stop later ones from being called") { + NotificationToken token2, token3; + bool called = false; + token2 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr) { + token2 = {}; + }); + token3 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr) { + called = true; + }); + + coordinator->on_change(); + r->notify(); + + REQUIRE(called); + } + + SECTION("modifications to unrelated tables do not send notifications") { + write([&] { + r->read_group()->get_table("class_other object")->add_empty_row(); + }); + REQUIRE(notification_calls == 1); + } + + SECTION("irrelevant modifications to linked tables do not send notifications") { + write([&] { + r->read_group()->get_table("class_linked to object")->add_empty_row(); + }); + REQUIRE(notification_calls == 1); + } + + SECTION("irrelevant modifications to linking tables do not send notifications") { + write([&] { + r->read_group()->get_table("class_linking object")->add_empty_row(); + }); + REQUIRE(notification_calls == 1); + } + + SECTION("modifications that leave a non-matching row non-matching do not send notifications") { + write([&] { + table->set_int(0, 6, 13); + }); + REQUIRE(notification_calls == 1); + } + + SECTION("deleting non-matching rows does not send a notification") { + write([&] { + table->move_last_over(0); + table->move_last_over(6); + }); + REQUIRE(notification_calls == 1); + } + + SECTION("modifying a matching row and leaving it matching marks that row as modified") { + write([&] { + table->set_int(0, 1, 3); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.modifications, 0); + } + + SECTION("modifying a matching row to no longer match marks that row as deleted") { + write([&] { + table->set_int(0, 2, 0); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.deletions, 1); + } + + SECTION("modifying a non-matching row to match marks that row as inserted") { + write([&] { + table->set_int(0, 7, 3); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.insertions, 4); + } + + SECTION("deleting a matching row marks that row as deleted") { + write([&] { + table->move_last_over(3); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.deletions, 2); + } + + SECTION("moving a matching row via deletion marks that row as moved") { + write([&] { + table->where().greater_equal(0, 10).find_all().clear(RemoveMode::unordered); + table->move_last_over(0); + }); + REQUIRE(notification_calls == 2); + REQUIRE_MOVES(change, {3, 0}); + } + + SECTION("modifications from multiple transactions are collapsed") { + r->begin_transaction(); + table->set_int(0, 0, 6); + r->commit_transaction(); + + r->begin_transaction(); + table->set_int(0, 1, 0); + r->commit_transaction(); + + REQUIRE(notification_calls == 1); + coordinator->on_change(); + r->notify(); + REQUIRE(notification_calls == 2); + } + } + + // Sort in descending order + results = results.sort({{0}, {false}}); + + SECTION("sorted notifications") { + int notification_calls = 0; + CollectionChangeIndices change; + auto token = results.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { + REQUIRE_FALSE(err); + change = c; + ++notification_calls; + }); + + coordinator->on_change(); + r->notify(); + + auto write = [&](auto&& f) { + r->begin_transaction(); + f(); + r->commit_transaction(); + coordinator->on_change(); + r->notify(); + }; + + SECTION("modifications that leave a non-matching row non-matching do not send notifications") { + write([&] { + table->set_int(0, 6, 13); + }); + REQUIRE(notification_calls == 1); + } + + SECTION("deleting non-matching rows does not send a notification") { + write([&] { + table->move_last_over(0); + table->move_last_over(6); + }); + REQUIRE(notification_calls == 1); + } + + SECTION("modifying a matching row and leaving it matching marks that row as modified") { + write([&] { + table->set_int(0, 1, 3); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.modifications, 3); + } + + SECTION("modifying a matching row to no longer match marks that row as deleted") { + write([&] { + table->set_int(0, 2, 0); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.deletions, 2); + } + + SECTION("modifying a non-matching row to match marks that row as inserted") { + write([&] { + table->set_int(0, 7, 3); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.insertions, 3); + } + + SECTION("deleting a matching row marks that row as deleted") { + write([&] { + table->move_last_over(3); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.deletions, 1); + } + + SECTION("moving a matching row via deletion does not send a notification") { + write([&] { + table->where().greater_equal(0, 10).find_all().clear(RemoveMode::unordered); + table->move_last_over(0); + }); + REQUIRE(notification_calls == 1); + } + + SECTION("modifying a matching row to change its position sends insert+delete") { + write([&] { + table->set_int(0, 2, 9); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.deletions, 2); + REQUIRE_INDICES(change.insertions, 0); + } + + SECTION("modifications from multiple transactions are collapsed") { + r->begin_transaction(); + table->set_int(0, 0, 5); + r->commit_transaction(); + + r->begin_transaction(); + table->set_int(0, 1, 0); + r->commit_transaction(); + + REQUIRE(notification_calls == 1); + coordinator->on_change(); + r->notify(); + REQUIRE(notification_calls == 2); + } + } +} + +TEST_CASE("Async Results error handling") { + InMemoryTestFile config; + config.cache = false; + config.automatic_change_notifications = false; + config.schema = std::make_unique(Schema{ + {"object", "", { + {"value", PropertyTypeInt}, + }}, + }); + + auto r = Realm::get_shared_realm(config); + auto coordinator = _impl::RealmCoordinator::get_existing_coordinator(config.path); + Results results(r, *config.schema->find("object"), *r->read_group()->get_table("class_object")); + + SECTION("error when opening the advancer SG") { + unlink(config.path.c_str()); + + SECTION("error is delivered asynchronously") { + bool called = false; + auto token = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { + REQUIRE(err); + called = true; + }); + + REQUIRE(!called); + coordinator->on_change(); + REQUIRE(!called); + r->notify(); + REQUIRE(called); + } + + SECTION("adding another callback does not send the error again") { + bool called = false; + auto token = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { + REQUIRE(err); + REQUIRE_FALSE(called); + called = true; + }); + + coordinator->on_change(); + r->notify(); + + bool called2 = false; + auto token2 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { + REQUIRE(err); + REQUIRE_FALSE(called2); + called2 = true; + }); + + coordinator->on_change(); + r->notify(); + + REQUIRE(called2); + } + } + + SECTION("error when opening the executor SG") { + SECTION("error is delivered asynchronously") { + bool called = false; + auto token = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { + REQUIRE(err); + called = true; + }); + unlink(config.path.c_str()); + + REQUIRE(!called); + coordinator->on_change(); + REQUIRE(!called); + r->notify(); + REQUIRE(called); + } + + SECTION("adding another callback does not send the error again") { + bool called = false; + auto token = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { + REQUIRE(err); + REQUIRE_FALSE(called); + called = true; + }); + unlink(config.path.c_str()); + + coordinator->on_change(); + r->notify(); + + bool called2 = false; + auto token2 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { + REQUIRE(err); + REQUIRE_FALSE(called2); + called2 = true; + }); + + coordinator->on_change(); + r->notify(); + + REQUIRE(called2); + } + } } From 8c94cd1b2c7b627308f6dc64e832d5173b85ccae Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 2 Feb 2016 15:12:36 -0800 Subject: [PATCH 10/93] Add an afl-based fuzzer for notifications --- CMakeLists.txt | 1 + fuzzer/CMakeLists.txt | 2 + fuzzer/fuzzer.cpp | 364 ++++++++++++++++++++++++++++++++++++++++++ fuzzer/input/0 | 19 +++ fuzzer/input/1 | 33 ++++ 5 files changed, 419 insertions(+) create mode 100644 fuzzer/CMakeLists.txt create mode 100644 fuzzer/fuzzer.cpp create mode 100644 fuzzer/input/0 create mode 100644 fuzzer/input/1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 286e6ff0..ec33ee52 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,4 +16,5 @@ use_realm_core(${REALM_CORE_VERSION}) include_directories(${REALM_CORE_INCLUDE_DIR} src external/pegtl) add_subdirectory(src) +add_subdirectory(fuzzer) add_subdirectory(tests) diff --git a/fuzzer/CMakeLists.txt b/fuzzer/CMakeLists.txt new file mode 100644 index 00000000..65df5418 --- /dev/null +++ b/fuzzer/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(fuzzer fuzzer.cpp) +target_link_libraries(fuzzer realm-object-store) diff --git a/fuzzer/fuzzer.cpp b/fuzzer/fuzzer.cpp new file mode 100644 index 00000000..19a913af --- /dev/null +++ b/fuzzer/fuzzer.cpp @@ -0,0 +1,364 @@ +#include "object_schema.hpp" +#include "property.hpp" +#include "results.hpp" +#include "schema.hpp" +#include "impl/realm_coordinator.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace realm; + +#define FUZZ_SORTED 1 + +#if 0 +#define log(...) fprintf(stderr, __VA_ARGS__) +#else +#define log(...) +#endif + +// Read from a fd until eof into a string +// Needs to use unbuffered i/o to work properly with afl +static void read_all(std::string& buffer, int fd) +{ + buffer.clear(); + size_t offset = 0; + while (true) { + buffer.resize(offset + 4096); + ssize_t bytes_read = read(fd, &buffer[offset], 4096); + if (bytes_read < 4096) { + buffer.resize(offset + bytes_read); + break; + } + offset += 4096; + } +} + +static std::vector read_initial_values(std::istream& input_stream) +{ + std::vector initial_values; + std::string line; + input_stream.seekg(0); + while (std::getline(input_stream, line) && !line.empty()) { + try { + initial_values.push_back(std::stoll(line)); + } + catch (std::invalid_argument) { + // not an error + } + catch (std::out_of_range) { + // not an error + } + } + return initial_values; +} + +struct Change { + enum class Action { + Commit, + Add, + Modify, + Delete + } action; + + size_t index = npos; + int64_t value = 0; +}; + +static std::vector read_changes(std::istream& input_stream) +{ + std::vector ret; + + while (!input_stream.eof()) { + char op = '\0'; + input_stream >> op; + if (!input_stream.good()) + break; + switch (op) { + case 'a': { + int64_t value = 0; + input_stream >> value; + if (input_stream.good()) + ret.push_back({Change::Action::Add, npos, value}); + break; + } + case 'm': { + int64_t value; + size_t ndx; + input_stream >> ndx >> value; + if (input_stream.good()) + ret.push_back({Change::Action::Modify, ndx, value}); + break; + } + case 'd': { + size_t ndx; + input_stream >> ndx; + if (input_stream.good()) + ret.push_back({Change::Action::Delete, ndx, 0}); + break; + } + case 'c': { + ret.push_back({Change::Action::Commit, npos, 0}); + break; + } + default: + return ret; + + } + } + return ret; +} + +static Query query(Table& table) +{ + return table.where().greater(1, 100).less(1, 50000); +} + +static TableView tableview(Table& table) +{ + auto tv = table.where().greater(1, 100).less(1, 50000).find_all(); +#if FUZZ_SORTED + tv.sort({1, 0}, {true, true}); +#endif + return tv; +} + +static int64_t id = 0; + +static void import_initial_values(SharedRealm& r, std::vector& initial_values) +{ + auto& table = *r->read_group()->get_table("class_object"); + + r->begin_transaction(); + table.clear(); + size_t ndx = table.add_empty_row(initial_values.size()); + for (auto value : initial_values) { + table.set_int(0, ndx, id++); + table.set_int(1, ndx++, value); + log("%lld\n", value); + } + r->commit_transaction(); +} + +// Apply the changes from the command file and then return whether a change +// notification should occur +static bool apply_changes(Realm& r, std::istream& input_stream) +{ + auto& table = *r.read_group()->get_table("class_object"); + auto tv = tableview(table); + + std::vector modified; + + log("\n"); + r.begin_transaction(); + for (auto const& change : read_changes(input_stream)) { + switch (change.action) { + case Change::Action::Commit: + log("c\n"); + r.commit_transaction(); + _impl::RealmCoordinator::get_existing_coordinator(r.config().path)->on_change(); + r.begin_transaction(); + break; + + case Change::Action::Add: { + log("a %lld\n", change.value); + size_t ndx = table.add_empty_row(); + table.set_int(0, ndx, id++); + table.set_int(1, ndx, change.value); + break; + } + + case Change::Action::Modify: + if (change.index < table.size()) { + log("m %zu %lld\n", change.index, change.value); + modified.push_back(table.get_int(0, change.index)); + table.set_int(1, change.index, change.value); + } + break; + + case Change::Action::Delete: + if (change.index < table.size()) { + log("d %zu\n", change.index); + table.move_last_over(change.index); + } + break; + } + } + r.commit_transaction(); + log("\n"); + + auto tv2 = tableview(table); + if (tv.size() != tv2.size()) + return true; + + for (size_t i = 0; i < tv.size(); ++i) { + if (!tv.is_row_attached(i)) + return true; + if (tv.get_int(0, i) != tv2.get_int(0, i)) + return true; + if (find(begin(modified), end(modified), tv.get_int(0, i)) != end(modified)) + return true; + } + + return false; +} + +static void verify(CollectionChangeIndices const& changes, std::vector values, Table& table) +{ + auto tv = tableview(table); + + // Apply the changes from the transaction log to our copy of the + // initial, using UITableView's batching rules (i.e. delete, then + // insert, then update) + auto it = std::make_reverse_iterator(changes.deletions.end()), end = std::make_reverse_iterator(changes.deletions.begin()); + for (; it != end; ++it) { + values.erase(values.begin() + it->first, values.begin() + it->second); + } + + for (auto i : changes.insertions.as_indexes()) { + values.insert(values.begin() + i, tv.get_int(1, i)); + } + + if (values.size() != tv.size()) { + abort(); + } + + for (auto i : changes.modifications.as_indexes()) { + if (changes.insertions.contains(i)) + abort(); + values[i] = tv.get_int(1, i); + } + +#if FUZZ_SORTED + if (!std::is_sorted(values.begin(), values.end())) + abort(); +#endif + + for (size_t i = 0; i < values.size(); ++i) { + if (values[i] != tv.get_int(1, i)) { + abort(); + } + } +} + +static void test(Realm::Config const& config, SharedRealm& r, SharedRealm& r2, std::istream& input_stream) +{ + std::vector initial_values = read_initial_values(input_stream); + if (initial_values.empty()) { + return; + } + import_initial_values(r, initial_values); + + auto& table = *r->read_group()->get_table("class_object"); + auto results = Results(r, ObjectSchema(), query(table)) +#if FUZZ_SORTED + .sort({{1, 0}, {true, true}}) +#endif + ; + + initial_values.clear(); + for (size_t i = 0; i < results.size(); ++i) + initial_values.push_back(results.get(i).get_int(1)); + + CollectionChangeIndices changes; + int notification_calls = 0; + auto token = results.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { + if (notification_calls > 0 && c.empty()) + abort(); + changes = c; + ++notification_calls; + }); + + auto& coordinator = *_impl::RealmCoordinator::get_existing_coordinator(config.path); + coordinator.on_change(); r->notify(); + if (notification_calls != 1) { + abort(); + } + + bool expect_notification = apply_changes(*r2, input_stream); + coordinator.on_change(); r->notify(); + + if (notification_calls != 1 + expect_notification) { + abort(); + } + + verify(changes, initial_values, table); +} + +int main(int argc, char** argv) { + std::ios_base::sync_with_stdio(false); + realm::disable_sync_to_disk(); + + Realm::Config config; + config.path = getenv("TMPDIR"); + config.path += "/realm.XXXXXX"; + mktemp(&config.path[0]); + config.cache = false; + config.in_memory = true; + config.automatic_change_notifications = false; + + Schema schema = { + {"object", "", { + {"id", PropertyTypeInt}, + {"value", PropertyTypeInt} + }} + }; + + config.schema = std::make_unique(schema); + unlink(config.path.c_str()); + + auto r = Realm::get_shared_realm(config); + auto r2 = Realm::get_shared_realm(config); + auto& coordinator = *_impl::RealmCoordinator::get_existing_coordinator(config.path); + + auto test_on = [&](auto& buffer) { + id = 0; + + std::istringstream ss(buffer); + test(config, r, r2, ss); + if (r->is_in_transaction()) + r->cancel_transaction(); + r2->invalidate(); + coordinator.on_change(); + }; + + if (argc > 1) { + std::string buffer; + for (int i = 1; i < argc; ++i) { + int fd = open(argv[i], O_RDONLY); + if (fd < 0) + abort(); + read_all(buffer, fd); + close(fd); + + test_on(buffer); + } + unlink(config.path.c_str()); + return 0; + } + +#ifdef __AFL_HAVE_MANUAL_CONTROL + std::string buffer; + while (__AFL_LOOP(1000)) { + read_all(buffer, 0); + test_on(buffer); + } +#else + std::string buffer; + read_all(buffer, 0); + test_on(buffer); +#endif + + unlink(config.path.c_str()); + return 0; +} diff --git a/fuzzer/input/0 b/fuzzer/input/0 new file mode 100644 index 00000000..50132fd0 --- /dev/null +++ b/fuzzer/input/0 @@ -0,0 +1,19 @@ +3 +100 +200 +400 +1000 +2000 +50 +80 +150 +180 +6000 +5000 +60000 + +a 500 +d 12 +c +m 11 10000 +a 800 diff --git a/fuzzer/input/1 b/fuzzer/input/1 new file mode 100644 index 00000000..483456fa --- /dev/null +++ b/fuzzer/input/1 @@ -0,0 +1,33 @@ +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 + +a 114 +a 115 +a 116 +a 117 +a 118 +a 119 +a 120 +a 121 +a 122 +c +m 4 200 +m 3 201 +m 2 202 +m 1 203 +m 5 203 +m 6 204 +m 7 205 +c +d 11 From d46f2c65baaae0ee47a94a91c248f0a46aeae3c4 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Mar 2016 10:54:13 -0800 Subject: [PATCH 11/93] Refactor the transaction log parsers to eliminate some duplication --- src/impl/transact_log_handler.cpp | 136 ++++++++++++------------------ tests/results.cpp | 27 +++++- 2 files changed, 75 insertions(+), 88 deletions(-) diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index 1dd75e0f..5601ce34 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -29,9 +29,30 @@ using namespace realm; namespace { -// A transaction log handler that just validates that all operations made are -// ones supported by the object store -class TransactLogValidator { +template +struct MarkDirtyMixin { + bool mark_dirty(size_t row, size_t col) { static_cast(this)->mark_dirty(row, col); return true; } + + bool set_int(size_t col, size_t row, int_fast64_t) { return mark_dirty(row, col); } + bool set_bool(size_t col, size_t row, bool) { return mark_dirty(row, col); } + bool set_float(size_t col, size_t row, float) { return mark_dirty(row, col); } + bool set_double(size_t col, size_t row, double) { return mark_dirty(row, col); } + bool set_string(size_t col, size_t row, StringData) { return mark_dirty(row, col); } + bool set_binary(size_t col, size_t row, BinaryData) { return mark_dirty(row, col); } + bool set_olddatetime(size_t col, size_t row, OldDateTime) { return mark_dirty(row, col); } + bool set_timestamp(size_t col, size_t row, Timestamp) { return mark_dirty(row, col); } + bool set_table(size_t col, size_t row) { return mark_dirty(row, col); } + bool set_mixed(size_t col, size_t row, const Mixed&) { return mark_dirty(row, col); } + bool set_link(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } + bool set_null(size_t col, size_t row) { return mark_dirty(row, col); } + bool nullify_link(size_t col, size_t row, size_t) { return mark_dirty(row, col); } + bool set_int_unique(size_t col, size_t row, size_t, int_fast64_t) { return mark_dirty(row, col); } + bool set_string_unique(size_t col, size_t row, size_t, StringData) { return mark_dirty(row, col); } + bool insert_substring(size_t col, size_t row, size_t, StringData) { return mark_dirty(row, col); } + bool erase_substring(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } +}; + +class TransactLogValidationMixin { // Index of currently selected table size_t m_current_table = 0; @@ -120,30 +141,20 @@ public: bool link_list_clear(size_t) { return true; } bool link_list_move(size_t, size_t) { return true; } bool link_list_swap(size_t, size_t) { return true; } - bool set_int(size_t, size_t, int_fast64_t) { return true; } - bool set_bool(size_t, size_t, bool) { return true; } - bool set_float(size_t, size_t, float) { return true; } - bool set_double(size_t, size_t, double) { return true; } - bool set_string(size_t, size_t, StringData) { return true; } - bool set_binary(size_t, size_t, BinaryData) { return true; } - bool set_olddatetime(size_t, size_t, OldDateTime) { return true; } - bool set_timestamp(size_t, size_t, Timestamp) { return true; } - bool set_table(size_t, size_t) { return true; } - bool set_mixed(size_t, size_t, const Mixed&) { return true; } - bool set_link(size_t, size_t, size_t, size_t) { return true; } - bool set_null(size_t, size_t) { return true; } - bool nullify_link(size_t, size_t, size_t) { return true; } - bool insert_substring(size_t, size_t, size_t, StringData) { return true; } - bool erase_substring(size_t, size_t, size_t, size_t) { return true; } - bool optimize_table() { return true; } - bool set_int_unique(size_t, size_t, size_t, int_fast64_t) { return true; } - bool set_string_unique(size_t, size_t, size_t, StringData) { return true; } bool change_link_targets(size_t, size_t) { return true; } + bool optimize_table() { return true; } +}; + + +// A transaction log handler that just validates that all operations made are +// ones supported by the object store +struct TransactLogValidator : public TransactLogValidationMixin, public MarkDirtyMixin { + void mark_dirty(size_t, size_t) { } }; // Extends TransactLogValidator to also track changes and report it to the // binding context if any properties are being observed -class TransactLogObserver : public TransactLogValidator { +class TransactLogObserver : public TransactLogValidationMixin, public MarkDirtyMixin { using ColumnInfo = BindingContext::ColumnInfo; using ObserverState = BindingContext::ObserverState; @@ -182,16 +193,6 @@ class TransactLogObserver : public TransactLogValidator { } } - // Mark the given row/col as needing notifications sent - bool mark_dirty(size_t row_ndx, size_t col_ndx) - { - auto it = lower_bound(begin(m_observers), end(m_observers), ObserverState{current_table(), row_ndx, nullptr}); - if (it != end(m_observers) && it->table_ndx == current_table() && it->row_ndx == row_ndx) { - get_change(*it, col_ndx).changed = true; - } - return true; - } - // Remove the given observer from the list of observed objects and add it // to the listed of invalidated objects void invalidate(ObserverState *o) @@ -207,10 +208,7 @@ public: { if (!context) { if (validate_schema_changes) { - // The handler functions are non-virtual, so the parent class's - // versions are called if we don't need to track changes to observed - // objects - func(static_cast(*this)); + func(TransactLogValidator()); } else { func(); @@ -222,7 +220,7 @@ public: if (m_observers.empty()) { auto old_version = sg.get_version_of_current_transaction(); if (validate_schema_changes) { - func(static_cast(*this)); + func(TransactLogValidator()); } else { func(); @@ -237,6 +235,15 @@ public: context->did_change(m_observers, invalidated); } + // Mark the given row/col as needing notifications sent + void mark_dirty(size_t row_ndx, size_t col_ndx) + { + auto it = lower_bound(begin(m_observers), end(m_observers), ObserverState{current_table(), row_ndx, nullptr}); + if (it != end(m_observers) && it->table_ndx == current_table() && it->row_ndx == row_ndx) { + get_change(*it, col_ndx).changed = true; + } + } + // Called at the end of the transaction log immediately before the version // is advanced void parse_complete() @@ -250,7 +257,7 @@ public: if (observer.table_ndx >= table_ndx) ++observer.table_ndx; } - TransactLogValidator::insert_group_level_table(table_ndx, prior_size, name); + TransactLogValidationMixin::insert_group_level_table(table_ndx, prior_size, name); return true; } @@ -411,29 +418,10 @@ public: } return true; } - - // Things that just mark the field as modified - bool set_int(size_t col, size_t row, int_fast64_t) { return mark_dirty(row, col); } - bool set_bool(size_t col, size_t row, bool) { return mark_dirty(row, col); } - bool set_float(size_t col, size_t row, float) { return mark_dirty(row, col); } - bool set_double(size_t col, size_t row, double) { return mark_dirty(row, col); } - bool set_string(size_t col, size_t row, StringData) { return mark_dirty(row, col); } - bool set_binary(size_t col, size_t row, BinaryData) { return mark_dirty(row, col); } - bool set_olddatetime(size_t col, size_t row, OldDateTime) { return mark_dirty(row, col); } - bool set_timestamp(size_t col, size_t row, Timestamp) { return mark_dirty(row, col); } - bool set_table(size_t col, size_t row) { return mark_dirty(row, col); } - bool set_mixed(size_t col, size_t row, const Mixed&) { return mark_dirty(row, col); } - bool set_link(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } - bool set_null(size_t col, size_t row) { return mark_dirty(row, col); } - bool nullify_link(size_t col, size_t row, size_t) { return mark_dirty(row, col); } - bool set_int_unique(size_t col, size_t row, size_t, int_fast64_t) { return mark_dirty(row, col); } - bool set_string_unique(size_t col, size_t row, size_t, StringData) { return mark_dirty(row, col); } - bool insert_substring(size_t col, size_t row, size_t, StringData) { return mark_dirty(row, col); } - bool erase_substring(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } }; // Extends TransactLogValidator to track changes made to LinkViews -class LinkViewObserver : public TransactLogValidator { +class LinkViewObserver : public TransactLogValidationMixin, public MarkDirtyMixin { _impl::TransactionChangeInfo& m_info; CollectionChangeIndices* m_active = nullptr; @@ -448,17 +436,16 @@ class LinkViewObserver : public TransactLogValidator { return &m_info.tables[tbl_ndx]; } - bool mark_dirty(size_t row, __unused size_t col) - { - if (auto change = get_change()) - change->modify(row); - return true; - } - public: LinkViewObserver(_impl::TransactionChangeInfo& info) : m_info(info) { } + void mark_dirty(size_t row, __unused size_t col) + { + if (auto change = get_change()) + change->modify(row); + } + bool select_link_list(size_t col, size_t row, size_t) { mark_dirty(row, col); @@ -563,25 +550,6 @@ public: change->clear(0); // FIXME return true; } - - // Things that just mark the field as modified - bool set_int(size_t col, size_t row, int_fast64_t) { return mark_dirty(row, col); } - bool set_bool(size_t col, size_t row, bool) { return mark_dirty(row, col); } - bool set_float(size_t col, size_t row, float) { return mark_dirty(row, col); } - bool set_double(size_t col, size_t row, double) { return mark_dirty(row, col); } - bool set_string(size_t col, size_t row, StringData) { return mark_dirty(row, col); } - bool set_binary(size_t col, size_t row, BinaryData) { return mark_dirty(row, col); } - bool set_olddatetime(size_t col, size_t row, OldDateTime) { return mark_dirty(row, col); } - bool set_timestamp(size_t col, size_t row, Timestamp) { return mark_dirty(row, col); } - bool set_table(size_t col, size_t row) { return mark_dirty(row, col); } - bool set_mixed(size_t col, size_t row, const Mixed&) { return mark_dirty(row, col); } - bool set_link(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } - bool set_null(size_t col, size_t row) { return mark_dirty(row, col); } - bool nullify_link(size_t col, size_t row, size_t) { return mark_dirty(row, col); } - bool insert_substring(size_t col, size_t row, size_t, StringData) { return mark_dirty(row, col); } - bool erase_substring(size_t col, size_t row, size_t, size_t) { return mark_dirty(row, col); } - bool set_int_unique(size_t col, size_t row, size_t, int_fast64_t) { return mark_dirty(row, col); } - bool set_string_unique(size_t col, size_t row, size_t, StringData) { return mark_dirty(row, col); } }; } // anonymous namespace diff --git a/tests/results.cpp b/tests/results.cpp index 62989924..be8c13d4 100644 --- a/tests/results.cpp +++ b/tests/results.cpp @@ -363,8 +363,28 @@ TEST_CASE("Async Results error handling") { auto coordinator = _impl::RealmCoordinator::get_existing_coordinator(config.path); Results results(r, *config.schema->find("object"), *r->read_group()->get_table("class_object")); + class OpenFileLimiter { + public: + OpenFileLimiter() + { + // Set the max open files to zero so that opening new files will fail + getrlimit(RLIMIT_NOFILE, &m_old); + rlimit rl = m_old; + rl.rlim_cur = 0; + setrlimit(RLIMIT_NOFILE, &rl); + } + + ~OpenFileLimiter() + { + setrlimit(RLIMIT_NOFILE, &m_old); + } + + private: + rlimit m_old; + }; + SECTION("error when opening the advancer SG") { - unlink(config.path.c_str()); + OpenFileLimiter limiter; SECTION("error is delivered asynchronously") { bool called = false; @@ -412,7 +432,7 @@ TEST_CASE("Async Results error handling") { REQUIRE(err); called = true; }); - unlink(config.path.c_str()); + OpenFileLimiter limiter; REQUIRE(!called); coordinator->on_change(); @@ -428,7 +448,7 @@ TEST_CASE("Async Results error handling") { REQUIRE_FALSE(called); called = true; }); - unlink(config.path.c_str()); + OpenFileLimiter limiter; coordinator->on_change(); r->notify(); @@ -445,6 +465,5 @@ TEST_CASE("Async Results error handling") { REQUIRE(called2); } - } } From e25e4c2dcd20a704968d2aee6c89e015a2230e4d Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 4 Mar 2016 11:19:33 -0800 Subject: [PATCH 12/93] Rework handling of mixed move_last_over() and modifications to actually work --- src/collection_notifications.cpp | 74 +++++++++++++---------------- src/impl/async_query.cpp | 12 +++-- src/impl/list_notifier.cpp | 5 ++ tests/collection_change_indices.cpp | 18 +++---- tests/results.cpp | 3 +- tests/transaction_log_parsing.cpp | 20 ++++---- 6 files changed, 68 insertions(+), 64 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 3f82c913..6087af5d 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -150,14 +150,7 @@ void CollectionChangeIndices::merge(realm::CollectionChangeIndices&& c) void CollectionChangeIndices::modify(size_t ndx) { - if (!insertions.contains(ndx)) - modifications.add(ndx); - // FIXME: this breaks mapping old row indices to new - // FIXME: is that a problem? - // If this row was previously moved, unmark it as a move - moves.erase(remove_if(begin(moves), end(moves), - [&](auto move) { return move.to == ndx; }), - end(moves)); + modifications.add(ndx); } void CollectionChangeIndices::insert(size_t index, size_t count) @@ -221,31 +214,30 @@ void CollectionChangeIndices::move(size_t from, size_t to) // Collapse A -> B, B -> C into a single A -> C move move.to = to; - modifications.erase_at(from); - insertions.erase_at(from); - - modifications.shift_for_insert_at(to); - insertions.insert_at(to); updated_existing_move = true; + + insertions.erase_at(from); + insertions.insert_at(to); } - if (updated_existing_move) - return; - if (!insertions.contains(from)) { - auto shifted_from = insertions.unshift(from); - shifted_from = deletions.add_shifted(shifted_from); + if (!updated_existing_move) { + auto shifted_from = insertions.erase_and_unshift(from); + insertions.insert_at(to); - // Don't record it as a move if the source row was newly inserted or - // was previously changed - if (!modifications.contains(from)) + // Don't report deletions/moves for newly inserted rows + if (shifted_from != npos) { + shifted_from = deletions.add_shifted(shifted_from); moves.push_back({shifted_from, to}); + } } + bool modified = modifications.contains(from); modifications.erase_at(from); - insertions.erase_at(from); - modifications.shift_for_insert_at(to); - insertions.insert_at(to); + if (modified) + modifications.insert_at(to); + else + modifications.shift_for_insert_at(to); } void CollectionChangeIndices::move_over(size_t row_ndx, size_t last_row) @@ -255,6 +247,9 @@ void CollectionChangeIndices::move_over(size_t row_ndx, size_t last_row) erase(row_ndx); return; } + move(last_row, row_ndx); + erase(row_ndx + 1); + return; bool updated_existing_move = false; for (size_t i = 0; i < moves.size(); ++i) { @@ -278,24 +273,19 @@ void CollectionChangeIndices::move_over(size_t row_ndx, size_t last_row) moves.push_back({last_row, row_ndx}); } - if (insertions.contains(row_ndx)) { - insertions.remove(row_ndx); - } - else { - if (modifications.contains(row_ndx)) { - modifications.remove(row_ndx); - } - deletions.add(row_ndx); - } + insertions.remove(row_ndx); + modifications.remove(row_ndx); - if (insertions.contains(last_row)) { - insertions.remove(last_row); - insertions.add(row_ndx); - } - else if (modifications.contains(last_row)) { + // not add_shifted() because unordered removal does not shift + // mixed ordered/unordered removal currently not supported + deletions.add(row_ndx); + + if (modifications.contains(last_row)) { modifications.remove(last_row); modifications.add(row_ndx); } + + insertions.add(row_ndx); } void CollectionChangeIndices::verify() @@ -305,10 +295,10 @@ void CollectionChangeIndices::verify() REALM_ASSERT(deletions.contains(move.from)); REALM_ASSERT(insertions.contains(move.to)); } - for (auto index : modifications.as_indexes()) - REALM_ASSERT(!insertions.contains(index)); - for (auto index : insertions.as_indexes()) - REALM_ASSERT(!modifications.contains(index)); +// for (auto index : modifications.as_indexes()) +// REALM_ASSERT(!insertions.contains(index)); +// for (auto index : insertions.as_indexes()) +// REALM_ASSERT(!modifications.contains(index)); #endif } diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index a91491fa..f1c2845b 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -121,11 +121,13 @@ void AsyncQuery::run() if (changes) { for (auto& idx : m_previous_rows) { - if (changes->deletions.contains(idx)) - idx = npos; - else - map_moves(idx, *changes); - REALM_ASSERT_DEBUG(!changes->insertions.contains(idx)); + if (!map_moves(idx, *changes)) { + if (changes->deletions.contains(idx)) + idx = npos; + else { + REALM_ASSERT_DEBUG(!changes->insertions.contains(idx)); + } + } } } diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp index f32b4b49..83d67b7d 100644 --- a/src/impl/list_notifier.cpp +++ b/src/impl/list_notifier.cpp @@ -97,6 +97,11 @@ void ListNotifier::run() return; } + m_change.moves.erase(remove_if(begin(m_change.moves), end(m_change.moves), + [&](auto move) { return m_change.modifications.contains(move.to); }), + end(m_change.moves)); + m_change.modifications.remove(m_change.insertions); + for (size_t i = 0; i < m_lv->size(); ++i) { if (m_change.insertions.contains(i) || m_change.modifications.contains(i)) continue; diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index cab25f58..e30f0593 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -39,10 +39,11 @@ TEST_CASE("collection change indices") { REQUIRE_INDICES(c.modifications, 3, 4); } - SECTION("modify() on an inserted row is a no-op") { + SECTION("modify() on an inserted row marks it as both inserted and modified") { c.insert(3); c.modify(3); - REQUIRE(c.modifications.empty()); + REQUIRE_INDICES(c.insertions, 3); + REQUIRE_INDICES(c.modifications, 3); } SECTION("modify() doesn't interact with deleted rows") { @@ -133,13 +134,13 @@ TEST_CASE("collection change indices") { REQUIRE_MOVES(c, {8, 5}); } - SECTION("move_over() removes previous insertions for that row") { - c.insert(5); + SECTION("move_over() does not mark the old last row as moved if it was newly inserted") { + c.insert(8); c.move_over(5, 8); - REQUIRE(c.insertions.empty()); + REQUIRE(c.moves.empty()); } - SECTION("move_over() removes previous modifications for that row") { + SECTION("move_over() removes previous modifications for the removed row") { c.modify(5); c.move_over(5, 8); REQUIRE(c.modifications.empty()); @@ -160,7 +161,7 @@ TEST_CASE("collection change indices") { SECTION("move_over() removes moves to the target") { c.move(3, 5); c.move_over(5, 8); - REQUIRE(c.moves.empty()); + REQUIRE_MOVES(c, {8, 5}); } SECTION("move_over() updates moves to the source") { @@ -172,7 +173,8 @@ TEST_CASE("collection change indices") { SECTION("move_over() is not shifted by previous calls to move_over()") { c.move_over(5, 10); c.move_over(6, 9); - REQUIRE_INDICES(c.deletions, 5, 6); + REQUIRE_INDICES(c.deletions, 5, 6, 9, 10); + REQUIRE_INDICES(c.insertions, 5, 6); REQUIRE_MOVES(c, {10, 5}, {9, 6}); } } diff --git a/tests/results.cpp b/tests/results.cpp index be8c13d4..ea7751e3 100644 --- a/tests/results.cpp +++ b/tests/results.cpp @@ -204,12 +204,13 @@ TEST_CASE("Results") { REQUIRE_INDICES(change.deletions, 1); } - SECTION("modifying a non-matching row to match marks that row as inserted") { + SECTION("modifying a non-matching row to match marks that row as inserted, but not modified") { write([&] { table->set_int(0, 7, 3); }); REQUIRE(notification_calls == 2); REQUIRE_INDICES(change.insertions, 4); + REQUIRE(change.modifications.empty()); } SECTION("deleting a matching row marks that row as deleted") { diff --git a/tests/transaction_log_parsing.cpp b/tests/transaction_log_parsing.cpp index dd5406a4..dc8b244c 100644 --- a/tests/transaction_log_parsing.cpp +++ b/tests/transaction_log_parsing.cpp @@ -97,8 +97,11 @@ private: CHECK(m_initial[i] == m_linkview->get(i).get_int(0)); // Verify that everything marked as a move actually is one - for (size_t i = 0; i < move_sources.size(); ++i) - CHECK(m_linkview->get(info.moves[i].to).get_int(0) == move_sources[i]); + for (size_t i = 0; i < move_sources.size(); ++i) { + if (!info.modifications.contains(info.moves[i].to)) { + CHECK(m_linkview->get(info.moves[i].to).get_int(0) == move_sources[i]); + } + } } }; @@ -386,14 +389,14 @@ TEST_CASE("Transaction log parsing") { REQUIRE(info.tables[2].deletions.empty()); } - SECTION("modifying newly added rows is not reported as a modification") { + SECTION("modifying newly added rows is reported as a modification") { auto info = track_changes({false, false, true}, [&] { table.add_empty_row(); table.set_int(0, 10, 10); }); REQUIRE(info.tables.size() == 3); REQUIRE_INDICES(info.tables[2].insertions, 10); - REQUIRE(info.tables[2].modifications.empty()); + REQUIRE_INDICES(info.tables[2].modifications, 10); } SECTION("move_last_over() does not shift rows other than the last one") { @@ -402,7 +405,8 @@ TEST_CASE("Transaction log parsing") { table.move_last_over(3); }); REQUIRE(info.tables.size() == 3); - REQUIRE_INDICES(info.tables[2].deletions, 2, 3); + REQUIRE_INDICES(info.tables[2].deletions, 2, 3, 8, 9); + REQUIRE_INDICES(info.tables[2].insertions, 2, 3); REQUIRE_MOVES(info.tables[2], {9, 2}, {8, 3}); } } @@ -651,7 +655,7 @@ TEST_CASE("Transaction log parsing") { lv->set(5, 1); } REQUIRE_INDICES(changes.insertions, 5); - REQUIRE(changes.modifications.size() == 0); + REQUIRE_INDICES(changes.modifications, 5); VALIDATE_CHANGES(changes) { lv->insert(5, 0); @@ -808,7 +812,7 @@ TEST_CASE("Transaction log parsing") { REQUIRE_INDICES(changes.deletions, 5); REQUIRE_INDICES(changes.insertions, 0); - REQUIRE(changes.moves.empty()); + REQUIRE_MOVES(changes, {5, 0}); } SECTION("set after move is just insert+delete") { @@ -819,7 +823,7 @@ TEST_CASE("Transaction log parsing") { REQUIRE_INDICES(changes.deletions, 5); REQUIRE_INDICES(changes.insertions, 0); - REQUIRE(changes.moves.empty()); + REQUIRE_MOVES(changes, {5, 0}); } SECTION("delete after move removes original row") { From 9a0ec0eb28d95e14b2529f91012d7189ebdd6087 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 7 Mar 2016 15:02:17 -0800 Subject: [PATCH 13/93] Actually report deletions for table clears --- src/collection_notifications.cpp | 10 +++--- src/impl/transact_log_handler.cpp | 2 +- src/index_set.cpp | 60 ++++++++++++------------------- src/index_set.hpp | 3 +- 4 files changed, 32 insertions(+), 43 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 6087af5d..fa13697e 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -184,10 +184,12 @@ void CollectionChangeIndices::erase(size_t index) void CollectionChangeIndices::clear(size_t old_size) { - for (auto range : deletions) - old_size += range.second - range.first; - for (auto range : insertions) - old_size -= range.second - range.first; + if (old_size != std::numeric_limits::max()) { + for (auto range : deletions) + old_size += range.second - range.first; + for (auto range : insertions) + old_size -= range.second - range.first; + } modifications.clear(); insertions.clear(); diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index 5601ce34..cabc3046 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -547,7 +547,7 @@ public: [&](auto const& lv) { return lv.table_ndx == tbl_ndx; }); m_info.lists.erase(it, end(m_info.lists)); if (auto change = get_change()) - change->clear(0); // FIXME + change->clear(std::numeric_limits::max()); return true; } }; diff --git a/src/index_set.cpp b/src/index_set.cpp index 23e529fe..ce1ad5b2 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -210,56 +210,42 @@ void IndexSet::do_erase(iterator it, size_t index) } } -void IndexSet::remove(size_t index) +IndexSet::iterator IndexSet::do_remove(iterator it, size_t begin, size_t end) { - auto it = find(index); - if (it == m_ranges.end() || it->first > index) - return; + for (it = find(begin, it); it != m_ranges.end() && it->first < end; it = find(begin, it)) { + // Trim off any part of the range to remove that's before the matching range + begin = std::max(it->first, begin); - if (it->first == index) { - ++it->first; - if (it->first == it->second) { - it = m_ranges.erase(it); + // If the matching range extends to both sides of the range to remove, + // split it on the range to remove + if (it->first < begin && it->second > end) { + it = m_ranges.insert(it + 1, {end, it->second}) - 1; + it->second = begin; } - return; - } - if (it->second == index + 1) { - --it->second; - return; + // Range to delete now coverages (at least) one end of the matching range + if (begin == it->first && end >= it->second) + it = m_ranges.erase(it); + else if (begin == it->first) + it->first = end; + else + it->second = begin; } + return it; +} - auto end = it->second; - it->second = index; - m_ranges.insert(it + 1, {index + 1, end}); +void IndexSet::remove(size_t index, size_t count) +{ + do_remove(find(index), index, index + count); } void IndexSet::remove(realm::IndexSet const& values) { auto it = m_ranges.begin(); - for (auto index : values.as_indexes()) { - it = find(index, it); + for (auto range : values) { + it = do_remove(it, range.first, range.second); if (it == m_ranges.end()) return; - if (it->first > index) - continue; - - if (it->first == index) { - ++it->first; - if (it->first == it->second) { - it = m_ranges.erase(it); - } - continue; - } - - if (it->second == index + 1) { - --it->second; - continue; - } - - auto end = it->second; - it->second = index; - it = m_ranges.insert(it + 1, {index + 1, end}); } } diff --git a/src/index_set.hpp b/src/index_set.hpp index fef560d5..13748e8f 100644 --- a/src/index_set.hpp +++ b/src/index_set.hpp @@ -76,7 +76,7 @@ public: size_t erase_and_unshift(size_t index); // Remove the indexes at the given index from the set, without shifting - void remove(size_t index); + void remove(size_t index, size_t count=1); void remove(IndexSet const&); // Shift an index by inserting each of the indexes in this set @@ -146,6 +146,7 @@ private: // returns inserted position iterator do_add(iterator pos, size_t index); void do_erase(iterator it, size_t index); + iterator do_remove(iterator it, size_t index, size_t count); }; } // namespace realm From a16cd7d42d2c04b6a2e970dc1dbc4aebabfcfee4 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 8 Mar 2016 12:05:19 -0800 Subject: [PATCH 14/93] Add async_query.hpp to the project --- src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a11820e7..0fa896f0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -24,6 +24,7 @@ set(HEADERS results.hpp schema.hpp shared_realm.hpp + impl/async_query.hpp impl/background_collection.hpp impl/external_commit_helper.hpp impl/list_notifier.hpp From a428f813d5ae3679b27a48b70284c6e815e21b4c Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 8 Mar 2016 11:19:13 -0800 Subject: [PATCH 15/93] Skip calling callbacks if two commits cancel each other out when merged --- src/impl/async_query.cpp | 11 +++-------- src/impl/async_query.hpp | 5 +++-- src/impl/background_collection.cpp | 19 +++++++++++-------- src/impl/background_collection.hpp | 10 ++++------ src/impl/list_notifier.cpp | 8 +------- src/impl/list_notifier.hpp | 3 +-- tests/results.cpp | 20 ++++++++++++++++++++ 7 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index f1c2845b..f750a852 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -83,7 +83,6 @@ void AsyncQuery::do_add_required_change_info(TransactionChangeInfo& info) void AsyncQuery::run() { REALM_ASSERT(m_info); - m_did_change = false; { std::lock_guard target_lock(m_target_mutex); @@ -145,14 +144,12 @@ void AsyncQuery::run() for (size_t i = 0; i < m_tv.size(); ++i) m_previous_rows[i] = m_tv[i].get_index(); } - - m_did_change = true; } -bool AsyncQuery::do_prepare_handover(SharedGroup& sg) +void AsyncQuery::do_prepare_handover(SharedGroup& sg) { if (!m_tv.is_attached()) { - return false; + return; } REALM_ASSERT(m_tv.is_in_sync()); @@ -167,8 +164,6 @@ bool AsyncQuery::do_prepare_handover(SharedGroup& sg) // detach the TableView as we won't need it again and keeping it around // makes advance_read() much more expensive m_tv = {}; - - return m_did_change; } bool AsyncQuery::do_deliver(SharedGroup& sg) @@ -197,7 +192,7 @@ bool AsyncQuery::do_deliver(SharedGroup& sg) std::move(*sg.import_from_handover(std::move(m_tv_handover)))); } REALM_ASSERT(!m_tv_handover); - return have_callbacks(); + return true; } void AsyncQuery::do_attach_to(SharedGroup& sg) diff --git a/src/impl/async_query.hpp b/src/impl/async_query.hpp index 324bd84f..457e6fb6 100644 --- a/src/impl/async_query.hpp +++ b/src/impl/async_query.hpp @@ -43,7 +43,7 @@ private: // Run/rerun the query if needed void run() override; // Prepare the handover object if run() did update the TableView - bool do_prepare_handover(SharedGroup&) override; + void do_prepare_handover(SharedGroup&) override; // Update the target results from the handover // Returns if any callbacks need to be invoked bool do_deliver(SharedGroup& sg) override; @@ -54,6 +54,8 @@ private: void do_attach_to(SharedGroup& sg) override; void do_detach_from(SharedGroup& sg) override; + bool should_deliver_initial() const noexcept override { return true; } + // Target Results to update and a mutex which guards it mutable std::mutex m_target_mutex; Results* m_target_results; @@ -73,7 +75,6 @@ private: TransactionChangeInfo* m_info = nullptr; uint_fast64_t m_handed_over_table_version = -1; - bool m_did_change = false; std::vector m_previous_rows; diff --git a/src/impl/background_collection.cpp b/src/impl/background_collection.cpp index 59dba0da..5db90c87 100644 --- a/src/impl/background_collection.cpp +++ b/src/impl/background_collection.cpp @@ -55,7 +55,7 @@ size_t BackgroundCollection::add_callback(CollectionChangeCallback callback) std::lock_guard lock(m_callback_mutex); auto token = next_token(); - m_callbacks.push_back({std::move(callback), token, -1ULL}); + m_callbacks.push_back({std::move(callback), token, false}); if (m_callback_index == npos) { // Don't need to wake up if we're already sending notifications Realm::Internal::get_coordinator(*m_realm).send_commit_notifications(); m_have_callbacks = true; @@ -138,8 +138,7 @@ void BackgroundCollection::prepare_handover() { REALM_ASSERT(m_sg); m_sg_version = m_sg->get_version_of_current_transaction(); - if (do_prepare_handover(*m_sg)) - ++m_results_version; + do_prepare_handover(*m_sg); } bool BackgroundCollection::deliver(SharedGroup& sg, std::exception_ptr err) @@ -162,9 +161,9 @@ bool BackgroundCollection::deliver(SharedGroup& sg, std::exception_ptr err) return false; } - bool ret = do_deliver(sg); + bool should_call_callbacks = do_deliver(sg); m_changes_to_deliver = std::move(m_accumulated_changes); - return ret; + return should_call_callbacks && have_callbacks(); } void BackgroundCollection::call_callbacks() @@ -184,12 +183,16 @@ void BackgroundCollection::call_callbacks() CollectionChangeCallback BackgroundCollection::next_callback() { std::lock_guard callback_lock(m_callback_mutex); + for (++m_callback_index; m_callback_index < m_callbacks.size(); ++m_callback_index) { auto& callback = m_callbacks[m_callback_index]; - if (m_error || callback.delivered_version != m_results_version) { - callback.delivered_version = m_results_version; - return callback.fn; + bool deliver_initial = !callback.initial_delivered && should_deliver_initial(); + if (!m_error && !deliver_initial && m_changes_to_deliver.empty()) { + continue; } + + callback.initial_delivered = true; + return callback.fn; } m_callback_index = npos; diff --git a/src/impl/background_collection.hpp b/src/impl/background_collection.hpp index 26dd28c8..26da3904 100644 --- a/src/impl/background_collection.hpp +++ b/src/impl/background_collection.hpp @@ -68,16 +68,16 @@ public: protected: bool have_callbacks() const noexcept { return m_have_callbacks; } - bool have_changes() const noexcept { return !m_accumulated_changes.empty(); } void add_changes(CollectionChangeIndices change) { m_accumulated_changes.merge(std::move(change)); } void set_table(Table const& table); private: virtual void do_attach_to(SharedGroup&) = 0; virtual void do_detach_from(SharedGroup&) = 0; - virtual bool do_prepare_handover(SharedGroup&) = 0; - virtual bool do_deliver(SharedGroup&) = 0; + virtual void do_prepare_handover(SharedGroup&) = 0; + virtual bool do_deliver(SharedGroup&) { return true; } virtual void do_add_required_change_info(TransactionChangeInfo&) { } + virtual bool should_deliver_initial() const noexcept { return false; } const std::thread::id m_thread_id = std::this_thread::get_id(); bool is_for_current_thread() const { return m_thread_id == std::this_thread::get_id(); } @@ -92,15 +92,13 @@ private: CollectionChangeIndices m_accumulated_changes; CollectionChangeIndices m_changes_to_deliver; - uint_fast64_t m_results_version = 0; - // Tables which this collection needs change information for std::vector m_relevant_tables; struct Callback { CollectionChangeCallback fn; size_t token; - uint_fast64_t delivered_version; + bool initial_delivered; }; // Currently registered callbacks and a mutex which must always be held diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp index 83d67b7d..8bdc4142 100644 --- a/src/impl/list_notifier.cpp +++ b/src/impl/list_notifier.cpp @@ -112,13 +112,7 @@ void ListNotifier::run() m_prev_size = m_lv->size(); } -bool ListNotifier::do_prepare_handover(SharedGroup&) +void ListNotifier::do_prepare_handover(SharedGroup&) { add_changes(std::move(m_change)); - return true; -} - -bool ListNotifier::do_deliver(SharedGroup&) -{ - return have_callbacks() && have_changes(); } diff --git a/src/impl/list_notifier.hpp b/src/impl/list_notifier.hpp index dcb88ef7..5d961306 100644 --- a/src/impl/list_notifier.hpp +++ b/src/impl/list_notifier.hpp @@ -40,8 +40,7 @@ private: void run() override; - bool do_deliver(SharedGroup& sg) override; - bool do_prepare_handover(SharedGroup&) override; + void do_prepare_handover(SharedGroup&) override; void do_attach_to(SharedGroup& sg) override; void do_detach_from(SharedGroup& sg) override; diff --git a/tests/results.cpp b/tests/results.cpp index ea7751e3..15c3ad71 100644 --- a/tests/results.cpp +++ b/tests/results.cpp @@ -235,6 +235,8 @@ TEST_CASE("Results") { table->set_int(0, 0, 6); r->commit_transaction(); + coordinator->on_change(); + r->begin_transaction(); table->set_int(0, 1, 0); r->commit_transaction(); @@ -244,6 +246,24 @@ TEST_CASE("Results") { r->notify(); REQUIRE(notification_calls == 2); } + + SECTION("notifications are not delivered when collapsing transactions results in no net change") { + r->begin_transaction(); + size_t ndx = table->add_empty_row(); + table->set_int(0, ndx, 5); + r->commit_transaction(); + + coordinator->on_change(); + + r->begin_transaction(); + table->move_last_over(ndx); + r->commit_transaction(); + + REQUIRE(notification_calls == 1); + coordinator->on_change(); + r->notify(); + REQUIRE(notification_calls == 1); + } } // Sort in descending order From 424f4e829f43106cf9e4901f21f7b9583d2faa40 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 8 Mar 2016 19:00:31 -0800 Subject: [PATCH 16/93] Prioritize modified rows when calculating changes for sorted results --- src/collection_notifications.cpp | 82 +++++++++++++++++------------ src/impl/async_query.cpp | 1 + src/index_set.cpp | 11 ++++ src/index_set.hpp | 3 ++ tests/collection_change_indices.cpp | 23 ++++++++ tests/index_set.cpp | 30 +++++++++++ 6 files changed, 116 insertions(+), 34 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index fa13697e..262903b1 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -357,10 +357,11 @@ void calculate_moves_unsorted(std::vector& new_rows, CollectionChangeIn using items = std::vector>; struct Match { - size_t i, j, size; + size_t i, j, size, modified; }; Match find_longest_match(items const& a, items const& b, + IndexSet const& modified, size_t begin1, size_t end1, size_t begin2, size_t end2) { Match best = {begin1, begin2, 0}; @@ -384,11 +385,18 @@ Match find_longest_match(items const& a, items const& b, size_t off = j - begin2; size_t size = off == 0 ? 1 : len_from_j_prev[off - 1] + 1; len_from_j[off] = size; - if (size > best.size) { - best.i = i - size + 1; - best.j = j - size + 1; - best.size = size; + if (size > best.size) + best = {i - size + 1, j - size + 1, size, npos}; + // Given two equal-length matches, prefer the one with fewer modified rows + else if (size == best.size) { + if (best.modified == npos) + best.modified = modified.count(best.j - size + 1, best.j + 1); + auto count = modified.count(j - size + 1, j + 1); + if (count < best.modified) + best = {i - size + 1, j - size + 1, size, count}; } + REALM_ASSERT(best.i >= begin1 && best.i + best.size <= end1); + REALM_ASSERT(best.j >= begin2 && best.j + best.size <= end2); } len_from_j.swap(len_from_j_prev); } @@ -396,51 +404,40 @@ Match find_longest_match(items const& a, items const& b, } void find_longest_matches(items const& a, items const& b_ndx, - size_t begin1, size_t end1, size_t begin2, size_t end2, std::vector& ret) + size_t begin1, size_t end1, size_t begin2, size_t end2, + IndexSet const& modified, std::vector& ret) { // FIXME: recursion could get too deep here - Match m = find_longest_match(a, b_ndx, begin1, end1, begin2, end2); + auto m = find_longest_match(a, b_ndx, modified, begin1, end1, begin2, end2); if (!m.size) return; if (m.i > begin1 && m.j > begin2) - find_longest_matches(a, b_ndx, begin1, m.i, begin2, m.j, ret); + find_longest_matches(a, b_ndx, begin1, m.i, begin2, m.j, modified, ret); ret.push_back(m); if (m.i + m.size < end2 && m.j + m.size < end2) - find_longest_matches(a, b_ndx, m.i + m.size, end1, m.j + m.size, end2, ret); + find_longest_matches(a, b_ndx, m.i + m.size, end1, m.j + m.size, end2, modified, ret); } void calculate_moves_sorted(std::vector& new_rows, CollectionChangeIndices& changeset, std::function row_did_change) { + // Remove new insertions, as they're already reported in changeset + new_rows.erase(std::remove_if(begin(new_rows), end(new_rows), + [](auto& row) { return row.prev_tv_index == npos; }), + end(new_rows)); + + std::sort(begin(new_rows), end(new_rows), + [](auto& lft, auto& rgt) { return lft.tv_index < rgt.tv_index; }); + std::vector> old_candidates; std::vector> new_candidates; - - std::sort(begin(new_rows), end(new_rows), [](auto& lft, auto& rgt) { - return lft.tv_index < rgt.tv_index; - }); - - IndexSet::IndexInterator ins = changeset.insertions.begin(), del = changeset.deletions.begin(); - int shift = 0; for (auto& row : new_rows) { - while (del != changeset.deletions.end() && *del <= row.tv_index) { - ++del; - ++shift; - } - while (ins != changeset.insertions.end() && *ins <= row.tv_index) { - ++ins; - --shift; - } - if (row.prev_tv_index == npos) - continue; - if (row_did_change(row.shifted_row_index)) { - // FIXME: needlessly quadratic - if (!changeset.insertions.contains(row.tv_index)) - changeset.modifications.add(row.tv_index); + changeset.modifications.add(row.tv_index); } - old_candidates.push_back({row.shifted_row_index, row.prev_tv_index}); - new_candidates.push_back({row.shifted_row_index, row.tv_index}); -// } + + old_candidates.push_back({row.shifted_row_index, row.prev_tv_index}); + new_candidates.push_back({row.shifted_row_index, row.tv_index}); } std::sort(begin(old_candidates), end(old_candidates), [](auto a, auto b) { @@ -477,7 +474,7 @@ void calculate_moves_sorted(std::vector& new_rows, CollectionChangeIndi find_longest_matches(old_candidates, b_ndx, first_difference, old_candidates.size(), first_difference, new_candidates.size(), - longest_matches); + changeset.modifications, longest_matches); longest_matches.push_back({old_candidates.size(), new_candidates.size(), 0}); size_t i = first_difference, j = first_difference; @@ -553,5 +550,22 @@ CollectionChangeIndices CollectionChangeIndices::calculate(std::vector c } ret.verify(); +#ifdef REALM_DEBUG + { // Verify that applying the calculated change to prev_rows actually produces next_rows + auto rows = prev_rows; + auto it = std::make_reverse_iterator(ret.deletions.end()); + auto end = std::make_reverse_iterator(ret.deletions.begin()); + for (; it != end; ++it) { + rows.erase(rows.begin() + it->first, rows.begin() + it->second); + } + + for (auto i : ret.insertions.as_indexes()) { + rows.insert(rows.begin() + i, next_rows[i]); + } + + REALM_ASSERT(rows == next_rows); + } +#endif + return ret; } diff --git a/src/impl/async_query.cpp b/src/impl/async_query.cpp index f750a852..d77c2524 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/async_query.cpp @@ -133,6 +133,7 @@ void AsyncQuery::run() m_changes = CollectionChangeIndices::calculate(m_previous_rows, next_rows, [&](size_t row) { return m_info->row_did_change(*m_query->get_table(), row); }, !!m_sort); + m_previous_rows = std::move(next_rows); if (m_changes.empty()) { m_tv = {}; diff --git a/src/index_set.cpp b/src/index_set.cpp index ce1ad5b2..0df1b787 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -36,6 +36,17 @@ bool IndexSet::contains(size_t index) const return it != m_ranges.end() && it->first <= index; } +size_t IndexSet::count(size_t start_index, size_t end_index) const +{ + auto it = const_cast(this)->find(start_index); + size_t ret = 0; + for (; end_index > start_index && it != m_ranges.end() && it->first < end_index; ++it) { + ret += std::min(it->second, end_index) - std::max(it->first, start_index); + start_index = it->second; + } + return ret; +} + IndexSet::iterator IndexSet::find(size_t index) { return find(index, m_ranges.begin()); diff --git a/src/index_set.hpp b/src/index_set.hpp index 13748e8f..3186b5b3 100644 --- a/src/index_set.hpp +++ b/src/index_set.hpp @@ -44,6 +44,9 @@ public: // Check if the index set contains the given index bool contains(size_t index) const; + // Counts the number of indices in the set in the given range + size_t count(size_t start_index, size_t end_index) const; + // Add an index to the set, doing nothing if it's already present void add(size_t index); void add(IndexSet const& is); diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index e30f0593..74ef18aa 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -242,6 +242,29 @@ TEST_CASE("collection change indices") { REQUIRE_MOVES(calc({3, 1, 2}), {2, 0}); REQUIRE_MOVES(calc({3, 2, 1}), {2, 0}, {1, 1}); } + + SECTION("merge can collapse insert -> move -> delete to no-op") { + auto four_modified = [](size_t ndx) { return ndx == 4; }; + for (int insert_pos = 0; insert_pos < 4; ++insert_pos) { + for (int move_to_pos = 0; move_to_pos < 4; ++move_to_pos) { + if (insert_pos == move_to_pos) + continue; + CAPTURE(insert_pos); + CAPTURE(move_to_pos); + + std::vector after_insert = {1, 2, 3}; + after_insert.insert(after_insert.begin() + insert_pos, 4); + c = CollectionChangeIndices::calculate({1, 2, 3}, after_insert, four_modified, true); + + std::vector after_move = {1, 2, 3}; + after_move.insert(after_move.begin() + move_to_pos, 4); + c.merge(CollectionChangeIndices::calculate(after_insert, after_move, four_modified, true)); + + c.merge(CollectionChangeIndices::calculate(after_move, {1, 2, 3}, four_modified, true)); + REQUIRE(c.empty()); + } + } + } } SECTION("merge") { diff --git a/tests/index_set.cpp b/tests/index_set.cpp index b7f58c6b..30737fb1 100644 --- a/tests/index_set.cpp +++ b/tests/index_set.cpp @@ -7,6 +7,36 @@ TEST_CASE("index set") { realm::IndexSet set; + SECTION("contains() returns if the index is in the set") { + set = {1, 2, 3, 5}; + REQUIRE_FALSE(set.contains(0)); + REQUIRE(set.contains(1)); + REQUIRE(set.contains(2)); + REQUIRE(set.contains(3)); + REQUIRE_FALSE(set.contains(4)); + REQUIRE(set.contains(5)); + REQUIRE_FALSE(set.contains(6)); + } + + SECTION("count() returns the number of indices int he range in the set") { + set = {1, 2, 3, 5}; + REQUIRE(set.count(0, 6) == 4); + REQUIRE(set.count(0, 5) == 3); + REQUIRE(set.count(0, 4) == 3); + REQUIRE(set.count(0, 3) == 2); + REQUIRE(set.count(0, 2) == 1); + REQUIRE(set.count(0, 1) == 0); + REQUIRE(set.count(0, 0) == 0); + + REQUIRE(set.count(0, 6) == 4); + REQUIRE(set.count(1, 6) == 4); + REQUIRE(set.count(2, 6) == 3); + REQUIRE(set.count(3, 6) == 2); + REQUIRE(set.count(4, 6) == 1); + REQUIRE(set.count(5, 6) == 1); + REQUIRE(set.count(6, 6) == 0); + } + SECTION("add() extends existing ranges") { set.add(1); REQUIRE_INDICES(set, 1); From 4ec1090c05073d685f85d260d0a2b6229576b62e Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 9 Mar 2016 11:53:08 -0800 Subject: [PATCH 17/93] Rename AsyncQuery to ResultsNotifier --- src/CMakeLists.txt | 4 ++-- src/impl/realm_coordinator.cpp | 4 ++-- .../{async_query.cpp => results_notifier.cpp} | 18 +++++++++--------- .../{async_query.hpp => results_notifier.hpp} | 10 +++++----- src/results.cpp | 6 +++--- src/results.hpp | 8 ++++---- src/shared_realm.hpp | 12 ++++++------ 7 files changed, 31 insertions(+), 31 deletions(-) rename src/impl/{async_query.cpp => results_notifier.cpp} (93%) rename src/impl/{async_query.hpp => results_notifier.hpp} (92%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0fa896f0..967ce511 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,10 +7,10 @@ set(SOURCES results.cpp schema.cpp shared_realm.cpp - impl/async_query.cpp impl/background_collection.cpp impl/list_notifier.cpp impl/realm_coordinator.cpp + impl/results_notifier.cpp impl/transact_log_handler.cpp parser/parser.cpp parser/query_builder.cpp) @@ -24,11 +24,11 @@ set(HEADERS results.hpp schema.hpp shared_realm.hpp - impl/async_query.hpp impl/background_collection.hpp impl/external_commit_helper.hpp impl/list_notifier.hpp impl/realm_coordinator.hpp + impl/results_notifier.hpp impl/transact_log_handler.hpp impl/weak_realm_notifier.hpp impl/weak_realm_notifier_base.hpp diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 9a8d6916..472daa5b 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -18,11 +18,11 @@ #include "impl/realm_coordinator.hpp" -#include "impl/async_query.hpp" -#include "impl/weak_realm_notifier.hpp" #include "impl/external_commit_helper.hpp" #include "impl/list_notifier.hpp" +#include "impl/results_notifier.hpp" #include "impl/transact_log_handler.hpp" +#include "impl/weak_realm_notifier.hpp" #include "object_store.hpp" #include "schema.hpp" diff --git a/src/impl/async_query.cpp b/src/impl/results_notifier.cpp similarity index 93% rename from src/impl/async_query.cpp rename to src/impl/results_notifier.cpp index d77c2524..2c79fd28 100644 --- a/src/impl/async_query.cpp +++ b/src/impl/results_notifier.cpp @@ -16,7 +16,7 @@ // //////////////////////////////////////////////////////////////////////////// -#include "impl/async_query.hpp" +#include "impl/results_notifier.hpp" #include "impl/realm_coordinator.hpp" #include "results.hpp" @@ -24,7 +24,7 @@ using namespace realm; using namespace realm::_impl; -AsyncQuery::AsyncQuery(Results& target) +ResultsNotifier::ResultsNotifier(Results& target) : BackgroundCollection(target.get_realm()) , m_target_results(&target) , m_sort(target.get_sort()) @@ -34,7 +34,7 @@ AsyncQuery::AsyncQuery(Results& target) m_query_handover = Realm::Internal::get_shared_group(get_realm()).export_for_handover(q, MutableSourcePayload::Move); } -void AsyncQuery::release_data() noexcept +void ResultsNotifier::release_data() noexcept { m_query = nullptr; } @@ -74,13 +74,13 @@ static bool map_moves(size_t& idx, CollectionChangeIndices const& changes) return false; } -void AsyncQuery::do_add_required_change_info(TransactionChangeInfo& info) +void ResultsNotifier::do_add_required_change_info(TransactionChangeInfo& info) { REALM_ASSERT(m_query); m_info = &info; } -void AsyncQuery::run() +void ResultsNotifier::run() { REALM_ASSERT(m_info); @@ -147,7 +147,7 @@ void AsyncQuery::run() } } -void AsyncQuery::do_prepare_handover(SharedGroup& sg) +void ResultsNotifier::do_prepare_handover(SharedGroup& sg) { if (!m_tv.is_attached()) { return; @@ -167,7 +167,7 @@ void AsyncQuery::do_prepare_handover(SharedGroup& sg) m_tv = {}; } -bool AsyncQuery::do_deliver(SharedGroup& sg) +bool ResultsNotifier::do_deliver(SharedGroup& sg) { std::lock_guard target_lock(m_target_mutex); @@ -196,13 +196,13 @@ bool AsyncQuery::do_deliver(SharedGroup& sg) return true; } -void AsyncQuery::do_attach_to(SharedGroup& sg) +void ResultsNotifier::do_attach_to(SharedGroup& sg) { REALM_ASSERT(m_query_handover); m_query = sg.import_from_handover(std::move(m_query_handover)); } -void AsyncQuery::do_detach_from(SharedGroup& sg) +void ResultsNotifier::do_detach_from(SharedGroup& sg) { REALM_ASSERT(m_query); REALM_ASSERT(!m_tv.is_attached()); diff --git a/src/impl/async_query.hpp b/src/impl/results_notifier.hpp similarity index 92% rename from src/impl/async_query.hpp rename to src/impl/results_notifier.hpp index 457e6fb6..32ab6963 100644 --- a/src/impl/async_query.hpp +++ b/src/impl/results_notifier.hpp @@ -16,8 +16,8 @@ // //////////////////////////////////////////////////////////////////////////// -#ifndef REALM_ASYNC_QUERY_HPP -#define REALM_ASYNC_QUERY_HPP +#ifndef REALM_RESULTS_NOTIFIER_HPP +#define REALM_RESULTS_NOTIFIER_HPP #include "background_collection.hpp" #include "results.hpp" @@ -35,9 +35,9 @@ namespace realm { namespace _impl { struct TransactionChangeInfo; -class AsyncQuery : public BackgroundCollection { +class ResultsNotifier : public BackgroundCollection { public: - AsyncQuery(Results& target); + ResultsNotifier(Results& target); private: // Run/rerun the query if needed @@ -84,4 +84,4 @@ private: } // namespace _impl } // namespace realm -#endif /* REALM_ASYNC_QUERY_HPP */ +#endif /* REALM_RESULTS_NOTIFIER_HPP */ diff --git a/src/results.cpp b/src/results.cpp index bba91cf8..8a1a6129 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -18,8 +18,8 @@ #include "results.hpp" -#include "impl/async_query.hpp" #include "impl/realm_coordinator.hpp" +#include "impl/results_notifier.hpp" #include "object_store.hpp" #include @@ -182,7 +182,7 @@ void Results::update_tableview() return; } if (!m_background_query && !m_realm->is_in_transaction() && m_realm->can_deliver_notifications()) { - m_background_query = std::make_shared<_impl::AsyncQuery>(*this); + m_background_query = std::make_shared<_impl::ResultsNotifier>(*this); _impl::RealmCoordinator::register_query(m_background_query); } m_has_used_table_view = true; @@ -370,7 +370,7 @@ void Results::prepare_async() } if (!m_background_query) { - m_background_query = std::make_shared<_impl::AsyncQuery>(*this); + m_background_query = std::make_shared<_impl::ResultsNotifier>(*this); _impl::RealmCoordinator::register_query(m_background_query); } } diff --git a/src/results.hpp b/src/results.hpp index 6529a90a..0051df47 100644 --- a/src/results.hpp +++ b/src/results.hpp @@ -35,7 +35,7 @@ class Results; class ObjectSchema; namespace _impl { - class AsyncQuery; + class ResultsNotifier; } struct SortOrder { @@ -181,10 +181,10 @@ public: bool wants_background_updates() const { return m_wants_background_updates; } - // Helper type to let AsyncQuery update the tableview without giving access + // Helper type to let ResultsNotifier update the tableview without giving access // to any other privates or letting anyone else do so class Internal { - friend class _impl::AsyncQuery; + friend class _impl::ResultsNotifier; static void set_table_view(Results& results, TableView&& tv); }; @@ -197,7 +197,7 @@ private: SortOrder m_sort; bool m_live = true; - std::shared_ptr<_impl::AsyncQuery> m_background_query; + std::shared_ptr<_impl::ResultsNotifier> m_background_query; Mode m_mode = Mode::Empty; bool m_has_used_table_view = false; diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index 3909782d..187c0211 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -32,19 +32,19 @@ namespace realm { class BindingContext; - class Replication; class Group; class Realm; class RealmDelegate; + class Replication; class SharedGroup; typedef std::shared_ptr SharedRealm; typedef std::weak_ptr WeakRealm; namespace _impl { - class AsyncQuery; class BackgroundCollection; class ListNotifier; class RealmCoordinator; + class ResultsNotifier; } class Realm : public std::enable_shared_from_this { @@ -145,13 +145,13 @@ namespace realm { // Expose some internal functionality to other parts of the ObjectStore // without making it public to everyone class Internal { - friend class _impl::AsyncQuery; - friend class _impl::ListNotifier; friend class _impl::BackgroundCollection; + friend class _impl::ListNotifier; friend class _impl::RealmCoordinator; + friend class _impl::ResultsNotifier; - // AsyncQuery needs access to the SharedGroup to be able to call the - // handover functions, which are not very wrappable + // ResultsNotifier and ListNotifier need access to the SharedGroup + // to be able to call the handover functions, which are not very wrappable static SharedGroup& get_shared_group(Realm& realm) { return *realm.m_shared_group; } // BackgroundCollection needs to be able to access the owning From b920f62ca57f8cbff1adb3fb0523a8d6a66d7bfd Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 9 Mar 2016 12:05:06 -0800 Subject: [PATCH 18/93] Comment and clean up the Notifiers/BackgroundCollection --- src/impl/background_collection.cpp | 13 ++- src/impl/background_collection.hpp | 49 ++++++-- src/impl/list_notifier.cpp | 6 +- src/impl/list_notifier.hpp | 14 ++- src/impl/realm_coordinator.cpp | 174 ++++++++++++++--------------- src/impl/realm_coordinator.hpp | 28 ++--- src/impl/results_notifier.cpp | 63 +++++------ src/impl/results_notifier.hpp | 49 ++++---- src/list.cpp | 2 +- src/results.cpp | 20 ++-- src/results.hpp | 2 +- 11 files changed, 225 insertions(+), 195 deletions(-) diff --git a/src/impl/background_collection.cpp b/src/impl/background_collection.cpp index 5db90c87..09a66133 100644 --- a/src/impl/background_collection.cpp +++ b/src/impl/background_collection.cpp @@ -32,11 +32,9 @@ BackgroundCollection::BackgroundCollection(std::shared_ptr realm) BackgroundCollection::~BackgroundCollection() { - // unregister() may have been called from a different thread than we're being - // destroyed on, so we need to synchronize access to the interesting fields - // modified there - std::lock_guard lock(m_realm_mutex); - m_realm = nullptr; + // Need to do this explicitly to ensure m_realm is destroyed with the mutex + // held to avoid potential double-deletion + unregister(); } size_t BackgroundCollection::add_callback(CollectionChangeCallback callback) @@ -102,6 +100,11 @@ bool BackgroundCollection::is_alive() const noexcept return m_realm != nullptr; } +std::unique_lock BackgroundCollection::lock_target() +{ + return std::unique_lock{m_realm_mutex}; +} + // Recursively add `table` and all tables it links to to `out` static void find_relevant_tables(std::vector& out, Table const& table) { diff --git a/src/impl/background_collection.hpp b/src/impl/background_collection.hpp index 26da3904..2d4ef046 100644 --- a/src/impl/background_collection.hpp +++ b/src/impl/background_collection.hpp @@ -25,7 +25,6 @@ #include #include -#include #include namespace realm { @@ -34,42 +33,72 @@ class Realm; namespace _impl { struct TransactionChangeInfo; +// A base class for a notifier that keeps a collection up to date and/or +// generates detailed change notifications on a background thread. This manages +// most of the lifetime-management issues related to sharing an object between +// the worker thread and the collection on the target thread, along with the +// thread-safe callback collection. class BackgroundCollection { public: BackgroundCollection(std::shared_ptr); virtual ~BackgroundCollection(); + + // ------------------------------------------------------------------------ + // Public API for the collections using this to get notifications: + + // Stop receiving notifications from this background worker + // This must be called in the destructor of the collection void unregister() noexcept; - virtual void release_data() noexcept = 0; - + // Add a callback to be called each time the collection changes + // This can only be called from the target collection's thread + // Returns a token which can be passed to remove_callback() size_t add_callback(CollectionChangeCallback callback); + // Remove a previously added token. The token is no longer valid after + // calling this function and must not be used again. This function can be + // called from any thread. void remove_callback(size_t token); + // ------------------------------------------------------------------------ + // API for RealmCoordinator to manage running things and calling callbacks + + Realm* get_realm() const noexcept { return m_realm.get(); } + + // Get the SharedGroup version which this collection can attach to (if it's + // in handover mode), or can deliver to (if it's been handed over to the BG worker alredad) + SharedGroup::VersionID version() const noexcept { return m_sg_version; } + + // Release references to all core types + // This is called on the worker thread to ensure that non-thread-safe things + // can be destroyed on the correct thread, even if the last reference to the + // BackgroundCollection is released on a different thread + virtual void release_data() noexcept = 0; + + // Call each of the currently registered callbacks, if there have been any + // changes since the last time each of those callbacks was called void call_callbacks(); bool is_alive() const noexcept; - Realm& get_realm() const noexcept { return *m_realm; } - - // Attach the handed-over query to `sg` + // Attach the handed-over query to `sg`. Must not be already attaged to a SharedGroup. void attach_to(SharedGroup& sg); // Create a new query handover object and stop using the previously attached // SharedGroup void detach(); - void add_required_change_info(TransactionChangeInfo&); + // Set `info` as the new ChangeInfo that will be populated by the next + // transaction advance, and register all required information in it + void add_required_change_info(TransactionChangeInfo& info); virtual void run() { } void prepare_handover(); bool deliver(SharedGroup&, std::exception_ptr); - // Get the version of the current handover object - SharedGroup::VersionID version() const noexcept { return m_sg_version; } - protected: bool have_callbacks() const noexcept { return m_have_callbacks; } void add_changes(CollectionChangeIndices change) { m_accumulated_changes.merge(std::move(change)); } void set_table(Table const& table); + std::unique_lock lock_target(); private: virtual void do_attach_to(SharedGroup&) = 0; diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp index 8bdc4142..ef451917 100644 --- a/src/impl/list_notifier.cpp +++ b/src/impl/list_notifier.cpp @@ -1,6 +1,6 @@ //////////////////////////////////////////////////////////////////////////// // -// Copyright 2015 Realm Inc. +// Copyright 2016 Realm Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ using namespace realm; using namespace realm::_impl; - ListNotifier::ListNotifier(LinkViewRef lv, std::shared_ptr realm) : BackgroundCollection(std::move(realm)) , m_prev_size(lv->size()) @@ -45,13 +44,12 @@ ListNotifier::ListNotifier(LinkViewRef lv, std::shared_ptr realm) set_table(lv->get_target_table()); - auto& sg = Realm::Internal::get_shared_group(get_realm()); + auto& sg = Realm::Internal::get_shared_group(*get_realm()); m_lv_handover = sg.export_linkview_for_handover(lv); } void ListNotifier::release_data() noexcept { - // FIXME: does this need a lock? m_lv.reset(); } diff --git a/src/impl/list_notifier.hpp b/src/impl/list_notifier.hpp index 5d961306..4029de85 100644 --- a/src/impl/list_notifier.hpp +++ b/src/impl/list_notifier.hpp @@ -1,6 +1,6 @@ //////////////////////////////////////////////////////////////////////////// // -// Copyright 2015 Realm Inc. +// Copyright 2016 Realm Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -30,12 +30,20 @@ public: ListNotifier(LinkViewRef lv, std::shared_ptr realm); private: + // The linkview, in handover form if this has not been attached to the main + // SharedGroup yet LinkViewRef m_lv; std::unique_ptr> m_lv_handover; - CollectionChangeIndices m_change; + + // The last-seen size of the LinkView so that we can report row deletions + // when the LinkView itself is deleted size_t m_prev_size; + + // The column index of the LinkView size_t m_col_ndx; - std::vector m_relevant_tables; + + // The actual change, calculated in run() and delivered in prepare_handover() + CollectionChangeIndices m_change; TransactionChangeInfo* m_info; void run() override; diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 472daa5b..6d657f1c 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -269,8 +269,8 @@ void RealmCoordinator::pin_version(uint_fast64_t version, uint_fast32_t index) m_advancer_history = nullptr; } } - else if (m_new_queries.empty()) { - // If this is the first query then we don't already have a read transaction + else if (m_new_notifiers.empty()) { + // If this is the first notifier then we don't already have a read transaction m_advancer_sg->begin_read(versionid); } else if (versionid < m_advancer_sg->get_version_of_current_transaction()) { @@ -281,18 +281,18 @@ void RealmCoordinator::pin_version(uint_fast64_t version, uint_fast32_t index) } } -void RealmCoordinator::register_query(std::shared_ptr query) +void RealmCoordinator::register_notifier(std::shared_ptr notifier) { - auto version = query->version(); - auto& self = Realm::Internal::get_coordinator(query->get_realm()); + auto version = notifier->version(); + auto& self = Realm::Internal::get_coordinator(*notifier->get_realm()); { - std::lock_guard lock(self.m_query_mutex); + std::lock_guard lock(self.m_notifier_mutex); self.pin_version(version.version, version.index); - self.m_new_queries.push_back(std::move(query)); + self.m_new_notifiers.push_back(std::move(notifier)); } } -void RealmCoordinator::clean_up_dead_queries() +void RealmCoordinator::clean_up_dead_notifiers() { auto swap_remove = [&](auto& container) { bool did_remove = false; @@ -300,8 +300,8 @@ void RealmCoordinator::clean_up_dead_queries() if (container[i]->is_alive()) continue; - // Ensure the query is destroyed here even if there's lingering refs - // to the async query elsewhere + // Ensure the notifier is destroyed here even if there's lingering refs + // to the async notifier elsewhere container[i]->release_data(); if (container.size() > i + 1) @@ -313,16 +313,16 @@ void RealmCoordinator::clean_up_dead_queries() return did_remove; }; - if (swap_remove(m_queries)) { + if (swap_remove(m_notifiers)) { // Make sure we aren't holding on to read versions needlessly if there - // are no queries left, but don't close them entirely as opening shared + // are no notifiers left, but don't close them entirely as opening shared // groups is expensive - if (m_queries.empty() && m_query_sg) { - m_query_sg->end_read(); + if (m_notifiers.empty() && m_notifier_sg) { + m_notifier_sg->end_read(); } } - if (swap_remove(m_new_queries)) { - if (m_new_queries.empty() && m_advancer_sg) { + if (swap_remove(m_new_notifiers)) { + if (m_new_notifiers.empty() && m_advancer_sg) { m_advancer_sg->end_read(); } } @@ -330,7 +330,7 @@ void RealmCoordinator::clean_up_dead_queries() void RealmCoordinator::on_change() { - run_async_queries(); + run_async_notifiers(); std::lock_guard lock(m_realm_mutex); for (auto& realm : m_weak_realm_notifiers) { @@ -338,13 +338,13 @@ void RealmCoordinator::on_change() } } -void RealmCoordinator::run_async_queries() +void RealmCoordinator::run_async_notifiers() { - std::unique_lock lock(m_query_mutex); + std::unique_lock lock(m_notifier_mutex); - clean_up_dead_queries(); + clean_up_dead_notifiers(); - if (m_queries.empty() && m_new_queries.empty()) { + if (m_notifiers.empty() && m_new_notifiers.empty()) { return; } @@ -353,73 +353,73 @@ void RealmCoordinator::run_async_queries() } if (m_async_error) { - std::move(m_new_queries.begin(), m_new_queries.end(), std::back_inserter(m_queries)); - m_new_queries.clear(); + std::move(m_new_notifiers.begin(), m_new_notifiers.end(), std::back_inserter(m_notifiers)); + m_new_notifiers.clear(); return; } std::vector change_info; SharedGroup::VersionID version; - auto new_queries = std::move(m_new_queries); - if (new_queries.empty()) { + auto new_notifiers = std::move(m_new_notifiers); + if (new_notifiers.empty()) { change_info.resize(1); } else { change_info.resize(2); - // Sort newly added queries by their source version so that we can pull them + // Sort newly added notifiers by their source version so that we can pull them // all forward to the latest version in a single pass over the transaction log - std::sort(new_queries.begin(), new_queries.end(), + std::sort(new_notifiers.begin(), new_notifiers.end(), [](auto&& lft, auto&& rgt) { return lft->version() < rgt->version(); }); version = m_advancer_sg->get_version_of_current_transaction(); - REALM_ASSERT(version == new_queries.front()->version()); + REALM_ASSERT(version == new_notifiers.front()->version()); TransactionChangeInfo* info = &change_info.back(); - // Advance each of the new queries to the latest version, attaching them + // Advance each of the new notifiers to the latest version, attaching them // to the SG at their handover version. This requires a unique // TransactionChangeInfo for each source version, so that things don't // see changes from before the version they were handed over from. // Each Info has all of the changes between that source version and the // next source version, and they'll be merged together later after // releasing the lock - for (auto& query : new_queries) { - if (version != query->version()) { - transaction::advance_and_observe_linkviews(*m_advancer_sg, *info, query->version()); + for (auto& notifier : new_notifiers) { + if (version != notifier->version()) { + transaction::advance_and_observe_linkviews(*m_advancer_sg, *info, notifier->version()); change_info.push_back({{}, std::move(info->lists)}); info = &change_info.back(); - version = query->version(); + version = notifier->version(); } - query->attach_to(*m_advancer_sg); - query->add_required_change_info(*info); + notifier->attach_to(*m_advancer_sg); + notifier->add_required_change_info(*info); } transaction::advance_and_observe_linkviews(*m_advancer_sg, *info); - for (auto& query : new_queries) { - query->detach(); + for (auto& notifier : new_notifiers) { + notifier->detach(); } version = m_advancer_sg->get_version_of_current_transaction(); m_advancer_sg->end_read(); } - // Make a copy of the queries vector and then release the lock to avoid - // blocking other threads trying to register or unregister queries while we run them - auto queries = m_queries; + // Make a copy of the notifiers vector and then release the lock to avoid + // blocking other threads trying to register or unregister notifiers while we run them + auto notifiers = m_notifiers; lock.unlock(); - for (auto& query : queries) { - query->add_required_change_info(change_info[0]); + for (auto& notifier : notifiers) { + notifier->add_required_change_info(change_info[0]); } - transaction::advance_and_observe_linkviews(*m_query_sg, change_info[0], version); + transaction::advance_and_observe_linkviews(*m_notifier_sg, change_info[0], version); - // Attach the new queries to the main SG and move them to the main list - for (auto& query : new_queries) { - query->attach_to(*m_query_sg); + // Attach the new notifiers to the main SG and move them to the main list + for (auto& notifier : new_notifiers) { + notifier->attach_to(*m_notifier_sg); } - std::move(new_queries.begin(), new_queries.end(), std::back_inserter(queries)); + std::move(new_notifiers.begin(), new_notifiers.end(), std::back_inserter(notifiers)); for (size_t i = change_info.size() - 1; i > 1; --i) { auto& cur = change_info[i]; @@ -452,57 +452,57 @@ void RealmCoordinator::run_async_queries() } } - for (auto& query : queries) { - query->run(); + for (auto& notifier : notifiers) { + notifier->run(); } // Reacquire the lock while updating the fields that are actually read on // other threads lock.lock(); - for (auto& query : queries) { - query->prepare_handover(); + for (auto& notifier : notifiers) { + notifier->prepare_handover(); } - m_queries = std::move(queries); - clean_up_dead_queries(); + m_notifiers = std::move(notifiers); + clean_up_dead_notifiers(); } void RealmCoordinator::open_helper_shared_group() { - if (!m_query_sg) { + if (!m_notifier_sg) { try { std::unique_ptr read_only_group; - Realm::open_with_config(m_config, m_query_history, m_query_sg, read_only_group); + Realm::open_with_config(m_config, m_notifier_history, m_notifier_sg, read_only_group); REALM_ASSERT(!read_only_group); - m_query_sg->begin_read(); + m_notifier_sg->begin_read(); } catch (...) { - // Store the error to be passed to the async queries + // Store the error to be passed to the async notifiers m_async_error = std::current_exception(); - m_query_sg = nullptr; - m_query_history = nullptr; + m_notifier_sg = nullptr; + m_notifier_history = nullptr; } } - else if (m_queries.empty()) { - m_query_sg->begin_read(); + else if (m_notifiers.empty()) { + m_notifier_sg->begin_read(); } } -void RealmCoordinator::move_new_queries_to_main() +void RealmCoordinator::move_new_notifiers_to_main() { - m_queries.reserve(m_queries.size() + m_new_queries.size()); - std::move(m_new_queries.begin(), m_new_queries.end(), std::back_inserter(m_queries)); - m_new_queries.clear(); + m_notifiers.reserve(m_notifiers.size() + m_new_notifiers.size()); + std::move(m_new_notifiers.begin(), m_new_notifiers.end(), std::back_inserter(m_notifiers)); + m_new_notifiers.clear(); } void RealmCoordinator::advance_to_ready(Realm& realm) { - decltype(m_queries) queries; + decltype(m_notifiers) notifiers; auto& sg = Realm::Internal::get_shared_group(realm); - auto get_query_version = [&] { - for (auto& query : m_queries) { - auto version = query->version(); + auto get_notifier_version = [&] { + for (auto& notifier : m_notifiers) { + auto version = notifier->version(); if (version != SharedGroup::VersionID{}) { return version; } @@ -512,11 +512,11 @@ void RealmCoordinator::advance_to_ready(Realm& realm) SharedGroup::VersionID version; { - std::lock_guard lock(m_query_mutex); - version = get_query_version(); + std::lock_guard lock(m_notifier_mutex); + version = get_notifier_version(); } - // no async queries; just advance to latest + // no async notifiers; just advance to latest if (version.version == std::numeric_limits::max()) { transaction::advance(sg, realm.m_binding_context.get()); return; @@ -532,44 +532,44 @@ void RealmCoordinator::advance_to_ready(Realm& realm) // may end up calling user code (in did_change() notifications) transaction::advance(sg, realm.m_binding_context.get(), version); - // Reacquire the lock and recheck the query version, as the queries may + // Reacquire the lock and recheck the notifier version, as the notifiers may // have advanced to a later version while we didn't hold the lock. If // so, we need to release the lock and re-advance - std::lock_guard lock(m_query_mutex); - version = get_query_version(); + std::lock_guard lock(m_notifier_mutex); + version = get_notifier_version(); if (version.version == std::numeric_limits::max()) return; if (version != sg.get_version_of_current_transaction()) continue; // Query version now matches the SG version, so we can deliver them - for (auto& query : m_queries) { - if (query->deliver(sg, m_async_error)) { - queries.push_back(query); + for (auto& notifier : m_notifiers) { + if (notifier->deliver(sg, m_async_error)) { + notifiers.push_back(notifier); } } break; } - for (auto& query : queries) { - query->call_callbacks(); + for (auto& notifier : notifiers) { + notifier->call_callbacks(); } } void RealmCoordinator::process_available_async(Realm& realm) { auto& sg = Realm::Internal::get_shared_group(realm); - decltype(m_queries) queries; + decltype(m_notifiers) notifiers; { - std::lock_guard lock(m_query_mutex); - for (auto& query : m_queries) { - if (query->deliver(sg, m_async_error)) { - queries.push_back(query); + std::lock_guard lock(m_notifier_mutex); + for (auto& notifier : m_notifiers) { + if (notifier->deliver(sg, m_async_error)) { + notifiers.push_back(notifier); } } } - for (auto& query : queries) { - query->call_callbacks(); + for (auto& notifier : notifiers) { + notifier->call_callbacks(); } } diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index 905c17b8..bd404e05 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -102,7 +102,7 @@ public: // Update the schema in the cached config void update_schema(Schema const& new_schema); - static void register_query(std::shared_ptr query); + static void register_notifier(std::shared_ptr notifier); // Advance the Realm to the most recent transaction version which all async // work is complete for @@ -115,32 +115,32 @@ private: std::mutex m_realm_mutex; std::vector m_weak_realm_notifiers; - std::mutex m_query_mutex; - std::vector> m_new_queries; - std::vector> m_queries; + std::mutex m_notifier_mutex; + std::vector> m_new_notifiers; + std::vector> m_notifiers; - // SharedGroup used for actually running async queries - // Will have a read transaction iff m_queries is non-empty - std::unique_ptr m_query_history; - std::unique_ptr m_query_sg; + // SharedGroup used for actually running async notifiers + // Will have a read transaction iff m_notifiers is non-empty + std::unique_ptr m_notifier_history; + std::unique_ptr m_notifier_sg; - // SharedGroup used to advance queries in m_new_queries to the main shared + // SharedGroup used to advance notifiers in m_new_notifiers to the main shared // group's transaction version - // Will have a read transaction iff m_new_queries is non-empty + // Will have a read transaction iff m_new_notifiers is non-empty std::unique_ptr m_advancer_history; std::unique_ptr m_advancer_sg; std::exception_ptr m_async_error; std::unique_ptr<_impl::ExternalCommitHelper> m_notifier; - // must be called with m_query_mutex locked + // must be called with m_notifier_mutex locked void pin_version(uint_fast64_t version, uint_fast32_t index); - void run_async_queries(); + void run_async_notifiers(); void open_helper_shared_group(); - void move_new_queries_to_main(); + void move_new_notifiers_to_main(); void advance_helper_shared_group_to_latest(); - void clean_up_dead_queries(); + void clean_up_dead_notifiers(); }; } // namespace _impl diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index 2c79fd28..b40dbefe 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -31,7 +31,7 @@ ResultsNotifier::ResultsNotifier(Results& target) { Query q = target.get_query(); set_table(*q.get_table()); - m_query_handover = Realm::Internal::get_shared_group(get_realm()).export_for_handover(q, MutableSourcePayload::Move); + m_query_handover = Realm::Internal::get_shared_group(*get_realm()).export_for_handover(q, MutableSourcePayload::Move); } void ResultsNotifier::release_data() noexcept @@ -39,30 +39,6 @@ void ResultsNotifier::release_data() noexcept m_query = nullptr; } -// Most of the inter-thread synchronization for run(), prepare_handover(), -// attach_to(), detach(), release_query() and deliver() is done by -// RealmCoordinator external to this code, which has some potentially -// non-obvious results on which members are and are not safe to use without -// holding a lock. -// -// attach_to(), detach(), run(), prepare_handover(), and release_query() are -// all only ever called on a single thread. call_callbacks() and deliver() are -// called on the same thread. Calls to prepare_handover() and deliver() are -// guarded by a lock. -// -// In total, this means that the safe data flow is as follows: -// - prepare_handover(), attach_to(), detach() and release_query() can read -// members written by each other -// - deliver() can read members written to in prepare_handover(), deliver(), -// and call_callbacks() -// - call_callbacks() and read members written to in deliver() -// -// Separately from this data flow for the query results, all uses of -// m_target_results, m_callbacks, and m_callback_index must be done with the -// appropriate mutex held to avoid race conditions when the Results object is -// destroyed while the background work is running, and to allow removing -// callbacks from any thread. - static bool map_moves(size_t& idx, CollectionChangeIndices const& changes) { for (auto&& move : changes.moves) { @@ -74,6 +50,27 @@ static bool map_moves(size_t& idx, CollectionChangeIndices const& changes) return false; } +// Most of the inter-thread synchronization for run(), prepare_handover(), +// attach_to(), detach(), release_data() and deliver() is done by +// RealmCoordinator external to this code, which has some potentially +// non-obvious results on which members are and are not safe to use without +// holding a lock. +// +// add_required_change_info(), attach_to(), detach(), run(), +// prepare_handover(), and release_data() are all only ever called on a single +// background worker thread. call_callbacks() and deliver() are called on the +// target thread. Calls to prepare_handover() and deliver() are guarded by a +// lock. +// +// In total, this means that the safe data flow is as follows: +// - add_Required_change_info(), prepare_handover(), attach_to(), detach() and +// release_data() can read members written by each other +// - deliver() can read members written to in prepare_handover(), deliver(), +// and call_callbacks() +// - call_callbacks() and read members written to in deliver() +// +// Separately from the handover data flow, m_target_results is guarded by the target lock + void ResultsNotifier::do_add_required_change_info(TransactionChangeInfo& info) { REALM_ASSERT(m_query); @@ -83,24 +80,21 @@ void ResultsNotifier::do_add_required_change_info(TransactionChangeInfo& info) void ResultsNotifier::run() { REALM_ASSERT(m_info); + REALM_ASSERT(!m_tv.is_attached()); { - std::lock_guard target_lock(m_target_mutex); + auto lock = lock_target(); // Don't run the query if the results aren't actually going to be used - if (!m_target_results || (!have_callbacks() && !m_target_results->wants_background_updates())) { + if (!get_realm() || (!have_callbacks() && !m_target_results->wants_background_updates())) { return; } } - REALM_ASSERT(!m_tv.is_attached()); - - size_t table_ndx = m_query->get_table()->get_index_in_group(); - // If we've run previously, check if we need to rerun if (m_initial_run_complete) { // Make an empty tableview from the query to get the table version, since // Query doesn't expose it - if (m_query->find_all(0, 0, 0).sync_if_needed() == m_handed_over_table_version) { + if (m_query->find_all(0, 0, 0).sync_if_needed() == m_last_seen_version) { return; } } @@ -109,7 +103,9 @@ void ResultsNotifier::run() if (m_sort) { m_tv.sort(m_sort.column_indices, m_sort.ascending); } + m_last_seen_version = m_tv.sync_if_needed(); + size_t table_ndx = m_query->get_table()->get_index_in_group(); if (m_initial_run_complete) { auto changes = table_ndx < m_info->tables.size() ? &m_info->tables[table_ndx] : nullptr; @@ -156,7 +152,6 @@ void ResultsNotifier::do_prepare_handover(SharedGroup& sg) REALM_ASSERT(m_tv.is_in_sync()); m_initial_run_complete = true; - m_handed_over_table_version = m_tv.sync_if_needed(); m_tv_handover = sg.export_for_handover(m_tv, MutableSourcePayload::Move); add_changes(std::move(m_changes)); @@ -169,7 +164,7 @@ void ResultsNotifier::do_prepare_handover(SharedGroup& sg) bool ResultsNotifier::do_deliver(SharedGroup& sg) { - std::lock_guard target_lock(m_target_mutex); + auto lock = lock_target(); // Target results being null here indicates that it was destroyed while we // were in the process of advancing the Realm version and preparing for diff --git a/src/impl/results_notifier.hpp b/src/impl/results_notifier.hpp index 32ab6963..1e88ed9f 100644 --- a/src/impl/results_notifier.hpp +++ b/src/impl/results_notifier.hpp @@ -24,12 +24,8 @@ #include -#include #include #include -#include -#include -#include namespace realm { namespace _impl { @@ -40,24 +36,8 @@ public: ResultsNotifier(Results& target); private: - // Run/rerun the query if needed - void run() override; - // Prepare the handover object if run() did update the TableView - void do_prepare_handover(SharedGroup&) override; - // Update the target results from the handover - // Returns if any callbacks need to be invoked - bool do_deliver(SharedGroup& sg) override; - - void do_add_required_change_info(TransactionChangeInfo& info) override; - - void release_data() noexcept override; - void do_attach_to(SharedGroup& sg) override; - void do_detach_from(SharedGroup& sg) override; - - bool should_deliver_initial() const noexcept override { return true; } - - // Target Results to update and a mutex which guards it - mutable std::mutex m_target_mutex; + // Target Results to update + // Can only be used with lock_target() held Results* m_target_results; const SortOrder m_sort; @@ -71,14 +51,31 @@ private: TableView m_tv; std::unique_ptr> m_tv_handover; + // The table version from the last time the query was run. Used to avoid + // rerunning the query when there's no chance of it changing. + uint_fast64_t m_last_seen_version = -1; + + // The rows from the previous run of the query, for calculating diffs + std::vector m_previous_rows; + + // The changeset calculated during run() and delivered in do_prepare_handover() CollectionChangeIndices m_changes; TransactionChangeInfo* m_info = nullptr; - uint_fast64_t m_handed_over_table_version = -1; - - std::vector m_previous_rows; - + // Flag for whether or not the query has been run at all, as goofy timing + // can lead to deliver() being called before that bool m_initial_run_complete = false; + + void run() override; + void do_prepare_handover(SharedGroup&) override; + bool do_deliver(SharedGroup& sg) override; + void do_add_required_change_info(TransactionChangeInfo& info) override; + + void release_data() noexcept override; + void do_attach_to(SharedGroup& sg) override; + void do_detach_from(SharedGroup& sg) override; + + bool should_deliver_initial() const noexcept override { return true; } }; } // namespace _impl diff --git a/src/list.cpp b/src/list.cpp index 0d3b9f2e..b4b59aa6 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -176,7 +176,7 @@ NotificationToken List::add_notification_callback(CollectionChangeCallback cb) verify_attached(); if (!m_notifier) { m_notifier = std::make_shared(m_link_view, m_realm); - RealmCoordinator::register_query(m_notifier); + RealmCoordinator::register_notifier(m_notifier); } return {m_notifier, m_notifier->add_callback(std::move(cb))}; } diff --git a/src/results.cpp b/src/results.cpp index 8a1a6129..acf79aad 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -59,8 +59,8 @@ Results::Results(SharedRealm r, const ObjectSchema &o, Table& table) Results::~Results() { - if (m_background_query) { - m_background_query->unregister(); + if (m_notifier) { + m_notifier->unregister(); } } @@ -181,9 +181,9 @@ void Results::update_tableview() if (!m_live) { return; } - if (!m_background_query && !m_realm->is_in_transaction() && m_realm->can_deliver_notifications()) { - m_background_query = std::make_shared<_impl::ResultsNotifier>(*this); - _impl::RealmCoordinator::register_query(m_background_query); + if (!m_notifier && !m_realm->is_in_transaction() && m_realm->can_deliver_notifications()) { + m_notifier = std::make_shared<_impl::ResultsNotifier>(*this); + _impl::RealmCoordinator::register_notifier(m_notifier); } m_has_used_table_view = true; m_table_view.sync_if_needed(); @@ -369,9 +369,9 @@ void Results::prepare_async() throw InvalidTransactionException("Cannot create asynchronous query while in a write transaction"); } - if (!m_background_query) { - m_background_query = std::make_shared<_impl::ResultsNotifier>(*this); - _impl::RealmCoordinator::register_query(m_background_query); + if (!m_notifier) { + m_notifier = std::make_shared<_impl::ResultsNotifier>(*this); + _impl::RealmCoordinator::register_notifier(m_notifier); } } @@ -379,13 +379,13 @@ NotificationToken Results::async(std::function target { prepare_async(); auto wrap = [=](CollectionChangeIndices, std::exception_ptr e) { target(e); }; - return {m_background_query, m_background_query->add_callback(wrap)}; + return {m_notifier, m_notifier->add_callback(wrap)}; } NotificationToken Results::add_notification_callback(CollectionChangeCallback cb) { prepare_async(); - return {m_background_query, m_background_query->add_callback(std::move(cb))}; + return {m_notifier, m_notifier->add_callback(std::move(cb))}; } void Results::Internal::set_table_view(Results& results, realm::TableView &&tv) diff --git a/src/results.hpp b/src/results.hpp index 0051df47..7c6f53cf 100644 --- a/src/results.hpp +++ b/src/results.hpp @@ -197,7 +197,7 @@ private: SortOrder m_sort; bool m_live = true; - std::shared_ptr<_impl::ResultsNotifier> m_background_query; + std::shared_ptr<_impl::ResultsNotifier> m_notifier; Mode m_mode = Mode::Empty; bool m_has_used_table_view = false; From a4298dd92c09e4adbae85d55d222266f5c7f96a5 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 9 Mar 2016 14:10:52 -0800 Subject: [PATCH 19/93] Remove an unused function --- src/impl/realm_coordinator.cpp | 7 ------- src/impl/realm_coordinator.hpp | 1 - 2 files changed, 8 deletions(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 6d657f1c..9ef8a885 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -487,13 +487,6 @@ void RealmCoordinator::open_helper_shared_group() } } -void RealmCoordinator::move_new_notifiers_to_main() -{ - m_notifiers.reserve(m_notifiers.size() + m_new_notifiers.size()); - std::move(m_new_notifiers.begin(), m_new_notifiers.end(), std::back_inserter(m_notifiers)); - m_new_notifiers.clear(); -} - void RealmCoordinator::advance_to_ready(Realm& realm) { decltype(m_notifiers) notifiers; diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index bd404e05..3f20a2cb 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -138,7 +138,6 @@ private: void run_async_notifiers(); void open_helper_shared_group(); - void move_new_notifiers_to_main(); void advance_helper_shared_group_to_latest(); void clean_up_dead_notifiers(); }; From d22c65f28a80aa9618c2be4bfb3265526c53d7e6 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 9 Mar 2016 15:47:48 -0800 Subject: [PATCH 20/93] Partially split out the code for calculating changesets from the struct for delivering them --- src/collection_notifications.cpp | 22 ++++++++-------- src/collection_notifications.hpp | 21 ++++++++++----- src/impl/background_collection.cpp | 34 ++++++++++++++++++++++++ src/impl/background_collection.hpp | 19 ++++++++++--- src/impl/list_notifier.hpp | 2 +- src/impl/realm_coordinator.cpp | 41 +++-------------------------- src/impl/realm_coordinator.hpp | 15 ----------- src/impl/results_notifier.cpp | 2 +- src/impl/results_notifier.hpp | 2 +- src/impl/transact_log_handler.cpp | 7 +++-- tests/collection_change_indices.cpp | 24 ++++++++--------- tests/transaction_log_parsing.cpp | 5 ++-- 12 files changed, 98 insertions(+), 96 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 262903b1..9e8ef54b 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -25,6 +25,7 @@ #include using namespace realm; +using namespace realm::_impl; NotificationToken::NotificationToken(std::shared_ptr<_impl::BackgroundCollection> query, size_t token) : m_query(std::move(query)), m_token(token) @@ -71,10 +72,9 @@ CollectionChangeIndices::CollectionChangeIndices(IndexSet deletions, this->deletions.add(move.from); this->insertions.add(move.to); } - verify(); } -void CollectionChangeIndices::merge(realm::CollectionChangeIndices&& c) +void CollectionChangeBuilder::merge(CollectionChangeBuilder&& c) { if (c.empty()) return; @@ -148,12 +148,12 @@ void CollectionChangeIndices::merge(realm::CollectionChangeIndices&& c) verify(); } -void CollectionChangeIndices::modify(size_t ndx) +void CollectionChangeBuilder::modify(size_t ndx) { modifications.add(ndx); } -void CollectionChangeIndices::insert(size_t index, size_t count) +void CollectionChangeBuilder::insert(size_t index, size_t count) { modifications.shift_for_insert_at(index, count); insertions.insert_at(index, count); @@ -164,7 +164,7 @@ void CollectionChangeIndices::insert(size_t index, size_t count) } } -void CollectionChangeIndices::erase(size_t index) +void CollectionChangeBuilder::erase(size_t index) { modifications.erase_at(index); size_t unshifted = insertions.erase_and_unshift(index); @@ -182,7 +182,7 @@ void CollectionChangeIndices::erase(size_t index) } } -void CollectionChangeIndices::clear(size_t old_size) +void CollectionChangeBuilder::clear(size_t old_size) { if (old_size != std::numeric_limits::max()) { for (auto range : deletions) @@ -197,7 +197,7 @@ void CollectionChangeIndices::clear(size_t old_size) deletions.set(old_size); } -void CollectionChangeIndices::move(size_t from, size_t to) +void CollectionChangeBuilder::move(size_t from, size_t to) { REALM_ASSERT(from != to); @@ -242,7 +242,7 @@ void CollectionChangeIndices::move(size_t from, size_t to) modifications.shift_for_insert_at(to); } -void CollectionChangeIndices::move_over(size_t row_ndx, size_t last_row) +void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) { REALM_ASSERT(row_ndx <= last_row); if (row_ndx == last_row) { @@ -290,7 +290,7 @@ void CollectionChangeIndices::move_over(size_t row_ndx, size_t last_row) insertions.add(row_ndx); } -void CollectionChangeIndices::verify() +void CollectionChangeBuilder::verify() { #ifdef REALM_DEBUG for (auto&& move : moves) { @@ -492,12 +492,12 @@ void calculate_moves_sorted(std::vector& new_rows, CollectionChangeIndi } } // Anonymous namespace -CollectionChangeIndices CollectionChangeIndices::calculate(std::vector const& prev_rows, +CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector const& prev_rows, std::vector const& next_rows, std::function row_did_change, bool sort) { - CollectionChangeIndices ret; + CollectionChangeBuilder ret; std::vector old_rows; for (size_t i = 0; i < prev_rows.size(); ++i) { diff --git a/src/collection_notifications.hpp b/src/collection_notifications.hpp index 3bf1f9c5..95c1fe04 100644 --- a/src/collection_notifications.hpp +++ b/src/collection_notifications.hpp @@ -60,6 +60,8 @@ struct CollectionChangeIndices { IndexSet modifications; std::vector moves; + bool empty() const { return deletions.empty() && insertions.empty() && modifications.empty() && moves.empty(); } + CollectionChangeIndices(CollectionChangeIndices const&) = default; CollectionChangeIndices(CollectionChangeIndices&&) = default; CollectionChangeIndices& operator=(CollectionChangeIndices const&) = default; @@ -69,15 +71,21 @@ struct CollectionChangeIndices { IndexSet insertions = {}, IndexSet modification = {}, std::vector moves = {}); +}; - static CollectionChangeIndices calculate(std::vector const& old_rows, +using CollectionChangeCallback = std::function; + +namespace _impl { +class CollectionChangeBuilder : public CollectionChangeIndices { +public: + using CollectionChangeIndices::CollectionChangeIndices; + + static CollectionChangeBuilder calculate(std::vector const& old_rows, std::vector const& new_rows, std::function row_did_change, bool sort); - bool empty() const { return deletions.empty() && insertions.empty() && modifications.empty() && moves.empty(); } - - void merge(CollectionChangeIndices&&); + void merge(CollectionChangeBuilder&&); void insert(size_t ndx, size_t count=1); void modify(size_t ndx); @@ -89,8 +97,7 @@ struct CollectionChangeIndices { private: void verify(); }; - -using CollectionChangeCallback = std::function; -} +} // namespace _impl +} // namespace realm #endif // REALM_COLLECTION_NOTIFICATIONS_HPP diff --git a/src/impl/background_collection.cpp b/src/impl/background_collection.cpp index 09a66133..e7b1c753 100644 --- a/src/impl/background_collection.cpp +++ b/src/impl/background_collection.cpp @@ -21,9 +21,43 @@ #include "impl/realm_coordinator.hpp" #include "shared_realm.hpp" +#include + using namespace realm; using namespace realm::_impl; +bool TransactionChangeInfo::row_did_change(Table const& table, size_t idx, int depth) const +{ + if (depth > 16) // arbitrary limit + return false; + + size_t table_ndx = table.get_index_in_group(); + if (table_ndx < tables.size() && tables[table_ndx].modifications.contains(idx)) + return true; + + for (size_t i = 0, count = table.get_column_count(); i < count; ++i) { + auto type = table.get_column_type(i); + if (type == type_Link) { + if (table.is_null_link(i, idx)) + continue; + auto dst = table.get_link(i, idx); + return row_did_change(*table.get_link_target(i), dst, depth + 1); + } + if (type != type_LinkList) + continue; + + auto& target = *table.get_link_target(i); + auto lvr = table.get_linklist(i, idx); + for (size_t j = 0; j < lvr->size(); ++j) { + size_t dst = lvr->get(j).get_index(); + if (row_did_change(target, dst, depth + 1)) + return true; + } + } + + return false; +} + BackgroundCollection::BackgroundCollection(std::shared_ptr realm) : m_realm(std::move(realm)) , m_sg_version(Realm::Internal::get_shared_group(*m_realm).get_version_of_current_transaction()) diff --git a/src/impl/background_collection.hpp b/src/impl/background_collection.hpp index 2d4ef046..78da5124 100644 --- a/src/impl/background_collection.hpp +++ b/src/impl/background_collection.hpp @@ -31,7 +31,20 @@ namespace realm { class Realm; namespace _impl { -struct TransactionChangeInfo; +struct ListChangeInfo { + size_t table_ndx; + size_t row_ndx; + size_t col_ndx; + CollectionChangeBuilder* changes; +}; + +struct TransactionChangeInfo { + std::vector tables_needed; + std::vector lists; + std::vector tables; + + bool row_did_change(Table const& table, size_t row_ndx, int depth = 0) const; +}; // A base class for a notifier that keeps a collection up to date and/or // generates detailed change notifications on a background thread. This manages @@ -96,7 +109,7 @@ public: protected: bool have_callbacks() const noexcept { return m_have_callbacks; } - void add_changes(CollectionChangeIndices change) { m_accumulated_changes.merge(std::move(change)); } + void add_changes(CollectionChangeBuilder change) { m_accumulated_changes.merge(std::move(change)); } void set_table(Table const& table); std::unique_lock lock_target(); @@ -118,7 +131,7 @@ private: SharedGroup* m_sg = nullptr; std::exception_ptr m_error; - CollectionChangeIndices m_accumulated_changes; + CollectionChangeBuilder m_accumulated_changes; CollectionChangeIndices m_changes_to_deliver; // Tables which this collection needs change information for diff --git a/src/impl/list_notifier.hpp b/src/impl/list_notifier.hpp index 4029de85..ef3805bf 100644 --- a/src/impl/list_notifier.hpp +++ b/src/impl/list_notifier.hpp @@ -43,7 +43,7 @@ private: size_t m_col_ndx; // The actual change, calculated in run() and delivered in prepare_handover() - CollectionChangeIndices m_change; + CollectionChangeBuilder m_change; TransactionChangeInfo* m_info; void run() override; diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 9ef8a885..61d99259 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -18,9 +18,8 @@ #include "impl/realm_coordinator.hpp" +#include "impl/background_collection.hpp" #include "impl/external_commit_helper.hpp" -#include "impl/list_notifier.hpp" -#include "impl/results_notifier.hpp" #include "impl/transact_log_handler.hpp" #include "impl/weak_realm_notifier.hpp" #include "object_store.hpp" @@ -29,8 +28,6 @@ #include #include #include -#include -#include #include #include @@ -39,38 +36,6 @@ using namespace realm; using namespace realm::_impl; -bool TransactionChangeInfo::row_did_change(Table const& table, size_t idx, int depth) const -{ - if (depth > 16) // arbitrary limit - return false; - - size_t table_ndx = table.get_index_in_group(); - if (table_ndx < tables.size() && tables[table_ndx].modifications.contains(idx)) - return true; - - for (size_t i = 0, count = table.get_column_count(); i < count; ++i) { - auto type = table.get_column_type(i); - if (type == type_Link) { - if (table.is_null_link(i, idx)) - continue; - auto dst = table.get_link(i, idx); - return row_did_change(*table.get_link_target(i), dst, depth + 1); - } - if (type != type_LinkList) - continue; - - auto& target = *table.get_link_target(i); - auto lvr = table.get_linklist(i, idx); - for (size_t j = 0; j < lvr->size(); ++j) { - size_t dst = lvr->get(j).get_index(); - if (row_did_change(target, dst, depth + 1)) - return true; - } - } - - return false; -} - static std::mutex s_coordinator_mutex; static std::unordered_map> s_coordinators_per_path; @@ -432,7 +397,7 @@ void RealmCoordinator::run_async_notifiers() } for (size_t j = 0; j < prev.tables.size() && j < cur.tables.size(); ++j) { - prev.tables[j].merge(CollectionChangeIndices{cur.tables[j]}); + prev.tables[j].merge(CollectionChangeBuilder{cur.tables[j]}); } prev.tables.reserve(cur.tables.size()); while (prev.tables.size() < cur.tables.size()) { @@ -446,7 +411,7 @@ void RealmCoordinator::run_async_notifiers() for (size_t i = 1; i < info.lists.size(); ++i) { for (size_t j = i; j > 0; --j) { if (id(info.lists[i]) == id(info.lists[j - 1])) { - info.lists[j - 1].changes->merge(CollectionChangeIndices{*info.lists[i].changes}); + info.lists[j - 1].changes->merge(CollectionChangeBuilder{*info.lists[i].changes}); } } } diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index 3f20a2cb..ba24d7d0 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -40,21 +40,6 @@ class ExternalCommitHelper; class ListNotifier; class WeakRealmNotifier; -struct ListChangeInfo { - size_t table_ndx; - size_t row_ndx; - size_t col_ndx; - CollectionChangeIndices *changes; -}; - -struct TransactionChangeInfo { - std::vector tables_needed; - std::vector lists; - std::vector tables; - - bool row_did_change(Table const& table, size_t row_ndx, int depth = 0) const; -}; - // RealmCoordinator manages the weak cache of Realm instances and communication // between per-thread Realm instances for a given file class RealmCoordinator : public std::enable_shared_from_this { diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index b40dbefe..282adb16 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -126,7 +126,7 @@ void ResultsNotifier::run() } } - m_changes = CollectionChangeIndices::calculate(m_previous_rows, next_rows, + m_changes = CollectionChangeBuilder::calculate(m_previous_rows, next_rows, [&](size_t row) { return m_info->row_did_change(*m_query->get_table(), row); }, !!m_sort); diff --git a/src/impl/results_notifier.hpp b/src/impl/results_notifier.hpp index 1e88ed9f..96cb0a38 100644 --- a/src/impl/results_notifier.hpp +++ b/src/impl/results_notifier.hpp @@ -59,7 +59,7 @@ private: std::vector m_previous_rows; // The changeset calculated during run() and delivered in do_prepare_handover() - CollectionChangeIndices m_changes; + CollectionChangeBuilder m_changes; TransactionChangeInfo* m_info = nullptr; // Flag for whether or not the query has been run at all, as goofy timing diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index cabc3046..96ae5f41 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -19,8 +19,7 @@ #include "impl/transact_log_handler.hpp" #include "binding_context.hpp" -#include "collection_notifications.hpp" -#include "impl/realm_coordinator.hpp" +#include "impl/background_collection.hpp" #include #include @@ -423,9 +422,9 @@ public: // Extends TransactLogValidator to track changes made to LinkViews class LinkViewObserver : public TransactLogValidationMixin, public MarkDirtyMixin { _impl::TransactionChangeInfo& m_info; - CollectionChangeIndices* m_active = nullptr; + _impl::CollectionChangeBuilder* m_active = nullptr; - CollectionChangeIndices* get_change() + _impl::CollectionChangeBuilder* get_change() { auto tbl_ndx = current_table(); if (tbl_ndx >= m_info.tables_needed.size() || !m_info.tables_needed[tbl_ndx]) diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index 74ef18aa..2be44fe2 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -6,7 +6,7 @@ TEST_CASE("collection change indices") { using namespace realm; - CollectionChangeIndices c; + _impl::CollectionChangeBuilder c; SECTION("stuff") { SECTION("insert() adds the row to the insertions set") { @@ -184,38 +184,38 @@ TEST_CASE("collection change indices") { auto none_modified = [](size_t) { return false; }; SECTION("no changes") { - c = CollectionChangeIndices::calculate({1, 2, 3}, {1, 2, 3}, none_modified, false); + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, none_modified, false); REQUIRE(c.empty()); } SECTION("inserting from empty") { - c = CollectionChangeIndices::calculate({}, {1, 2, 3}, all_modified, false); + c = _impl::CollectionChangeBuilder::calculate({}, {1, 2, 3}, all_modified, false); REQUIRE_INDICES(c.insertions, 0, 1, 2); } SECTION("deleting all existing") { - c = CollectionChangeIndices::calculate({1, 2, 3}, {}, all_modified, false); + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {}, all_modified, false); REQUIRE_INDICES(c.deletions, 0, 1, 2); } SECTION("all rows modified without changing order") { - c = CollectionChangeIndices::calculate({1, 2, 3}, {1, 2, 3}, all_modified, false); + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, all_modified, false); REQUIRE_INDICES(c.modifications, 0, 1, 2); } SECTION("single insertion in middle") { - c = CollectionChangeIndices::calculate({1, 3}, {1, 2, 3}, all_modified, false); + c = _impl::CollectionChangeBuilder::calculate({1, 3}, {1, 2, 3}, all_modified, false); REQUIRE_INDICES(c.insertions, 1); } SECTION("single deletion in middle") { - c = CollectionChangeIndices::calculate({1, 2, 3}, {1, 3}, all_modified, false); + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3}, all_modified, false); REQUIRE_INDICES(c.deletions, 1); } SECTION("unsorted reordering") { auto calc = [&](std::vector values) { - return CollectionChangeIndices::calculate({1, 2, 3}, values, none_modified, false); + return _impl::CollectionChangeBuilder::calculate({1, 2, 3}, values, none_modified, false); }; // The commented-out permutations are not possible with @@ -230,7 +230,7 @@ TEST_CASE("collection change indices") { SECTION("sorted reordering") { auto calc = [&](std::vector values) { - return CollectionChangeIndices::calculate({1, 2, 3}, values, all_modified, true); + return _impl::CollectionChangeBuilder::calculate({1, 2, 3}, values, all_modified, true); }; REQUIRE(calc({1, 2, 3}).moves.empty()); @@ -254,13 +254,13 @@ TEST_CASE("collection change indices") { std::vector after_insert = {1, 2, 3}; after_insert.insert(after_insert.begin() + insert_pos, 4); - c = CollectionChangeIndices::calculate({1, 2, 3}, after_insert, four_modified, true); + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, after_insert, four_modified, true); std::vector after_move = {1, 2, 3}; after_move.insert(after_move.begin() + move_to_pos, 4); - c.merge(CollectionChangeIndices::calculate(after_insert, after_move, four_modified, true)); + c.merge(_impl::CollectionChangeBuilder::calculate(after_insert, after_move, four_modified, true)); - c.merge(CollectionChangeIndices::calculate(after_move, {1, 2, 3}, four_modified, true)); + c.merge(_impl::CollectionChangeBuilder::calculate(after_move, {1, 2, 3}, four_modified, true)); REQUIRE(c.empty()); } } diff --git a/tests/transaction_log_parsing.cpp b/tests/transaction_log_parsing.cpp index dc8b244c..99d2108c 100644 --- a/tests/transaction_log_parsing.cpp +++ b/tests/transaction_log_parsing.cpp @@ -3,9 +3,8 @@ #include "util/index_helpers.hpp" #include "util/test_file.hpp" -#include "impl/realm_coordinator.hpp" +#include "impl/background_collection.hpp" #include "impl/transact_log_handler.hpp" -#include "collection_notifications.hpp" #include "property.hpp" #include "object_schema.hpp" #include "schema.hpp" @@ -35,7 +34,7 @@ public: CollectionChangeIndices finish(size_t ndx) { m_realm->commit_transaction(); - CollectionChangeIndices c; + _impl::CollectionChangeBuilder c; _impl::TransactionChangeInfo info; info.lists.push_back({ndx, 0, 0, &c}); info.tables_needed.resize(m_group.size(), true); From cfb9f0635cb5a3c12781403c407bfc81c576568a Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 10 Mar 2016 16:01:19 -0800 Subject: [PATCH 21/93] Fix calculation of moves for unsorted queries --- src/collection_notifications.cpp | 58 +++++++++-------------------- tests/collection_change_indices.cpp | 7 ++++ 2 files changed, 24 insertions(+), 41 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 9e8ef54b..6eef867e 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -314,44 +314,22 @@ struct RowInfo { void calculate_moves_unsorted(std::vector& new_rows, CollectionChangeIndices& changeset, std::function row_did_change) { - std::sort(begin(new_rows), end(new_rows), [](auto& lft, auto& rgt) { - return lft.tv_index < rgt.tv_index; - }); - - IndexSet::IndexInterator ins = changeset.insertions.begin(), del = changeset.deletions.begin(); - int shift = 0; for (auto& row : new_rows) { - while (del != changeset.deletions.end() && *del <= row.tv_index) { - ++del; - ++shift; - } - while (ins != changeset.insertions.end() && *ins <= row.tv_index) { - ++ins; - --shift; - } - if (row.prev_tv_index == npos) - continue; - - // For unsorted, non-LV queries a row can only move to an index before - // its original position due to a move_last_over - if (row.tv_index + shift != row.prev_tv_index) { - --shift; + // Calculate where this row would be with only previous insertions + // and deletions. We can ignore future insertions/deletions from moves + // because move_last_over() can only move rows to lower indices + size_t expected = row.prev_tv_index + - changeset.deletions.count(0, row.prev_tv_index) + + changeset.insertions.count(0, row.tv_index); + if (row.tv_index != expected) { changeset.moves.push_back({row.prev_tv_index, row.tv_index}); + changeset.insertions.add(row.tv_index); + changeset.deletions.add(row.prev_tv_index); } - // FIXME: currently move implies modification, and so they're mutally exclusive - // this is correct for sorted, but for unsorted a row can move without actually changing - else if (row_did_change(row.shifted_row_index)) { - // FIXME: needlessly quadratic - if (!changeset.insertions.contains(row.tv_index)) - changeset.modifications.add(row.tv_index); + else if (!changeset.insertions.contains(row.tv_index) && row_did_change(row.shifted_row_index)) { + changeset.modifications.add(row.tv_index); } } - - // FIXME: this is required for merge(), but it would be nice if it wasn't - for (auto&& move : changeset.moves) { - changeset.insertions.add(move.to); - changeset.deletions.add(move.from); - } } using items = std::vector>; @@ -421,14 +399,6 @@ void find_longest_matches(items const& a, items const& b_ndx, void calculate_moves_sorted(std::vector& new_rows, CollectionChangeIndices& changeset, std::function row_did_change) { - // Remove new insertions, as they're already reported in changeset - new_rows.erase(std::remove_if(begin(new_rows), end(new_rows), - [](auto& row) { return row.prev_tv_index == npos; }), - end(new_rows)); - - std::sort(begin(new_rows), end(new_rows), - [](auto& lft, auto& rgt) { return lft.tv_index < rgt.tv_index; }); - std::vector> old_candidates; std::vector> new_candidates; for (auto& row : new_rows) { @@ -542,6 +512,12 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c for (; j < new_rows.size(); ++j) ret.insertions.add(new_rows[j].tv_index); + new_rows.erase(std::remove_if(begin(new_rows), end(new_rows), + [](auto& row) { return row.prev_tv_index == npos; }), + end(new_rows)); + std::sort(begin(new_rows), end(new_rows), + [](auto& lft, auto& rgt) { return lft.tv_index < rgt.tv_index; }); + if (sort) { calculate_moves_sorted(new_rows, ret, row_did_change); } diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index 2be44fe2..0b06f3b0 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -213,6 +213,13 @@ TEST_CASE("collection change indices") { REQUIRE_INDICES(c.deletions, 1); } + SECTION("mixed insert and delete") { + c = _impl::CollectionChangeBuilder::calculate({3, 5}, {0, 3}, all_modified, false); + REQUIRE_INDICES(c.deletions, 1); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE(c.moves.empty()); + } + SECTION("unsorted reordering") { auto calc = [&](std::vector values) { return _impl::CollectionChangeBuilder::calculate({1, 2, 3}, values, none_modified, false); From e65ad4d413ddecc30bd0f6b745c28a48f0732343 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 10 Mar 2016 16:58:43 -0800 Subject: [PATCH 22/93] Discard moves which are turned into no-ops when merging --- src/collection_notifications.cpp | 56 ++++++++++++++++----------- src/impl/background_collection.cpp | 1 + src/impl/list_notifier.cpp | 2 +- src/index_set.cpp | 10 +++++ tests/collection_change_indices.cpp | 59 +++++++++++++++++++++++++---- tests/results.cpp | 19 ++++++++++ tests/util/index_helpers.hpp | 1 + 7 files changed, 117 insertions(+), 31 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 6eef867e..ee0506cc 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -102,6 +102,7 @@ void CollectionChangeBuilder::merge(CollectionChangeBuilder&& c) } // Check if the destination was deleted + // Removing the insert for this move will happen later if (c.deletions.contains(old.to)) return true; @@ -114,12 +115,20 @@ void CollectionChangeBuilder::merge(CollectionChangeBuilder&& c) // Ignore new moves of rows which were previously inserted (the implicit // delete from the move will remove the insert) - if (!insertions.empty()) { + if (!insertions.empty() && !c.moves.empty()) { c.moves.erase(remove_if(begin(c.moves), end(c.moves), [&](auto const& m) { return insertions.contains(m.from); }), end(c.moves)); } + // Ensure that any previously modified rows which were moved are still modified + if (!modifications.empty() && !c.moves.empty()) { + for (auto const& move : c.moves) { + if (modifications.contains(move.from)) + c.modifications.add(move.to); + } + } + // Update the source position of new moves to compensate for the changes made // in the old changeset if (!deletions.empty() || !insertions.empty()) { @@ -137,8 +146,17 @@ void CollectionChangeBuilder::merge(CollectionChangeBuilder&& c) insertions.erase_at(c.deletions); insertions.insert_at(c.insertions); - // Ignore new mmodifications to previously inserted rows - c.modifications.remove(insertions); + // Look for moves which are now no-ops, and remove them plus the associated + // insert+delete. Note that this isn't just checking for from == to due to + // that rows can also be shifted by other inserts and deletes + IndexSet to_remove; + moves.erase(remove_if(begin(moves), end(moves), [&](auto const& move) { + if (move.from - deletions.count(0, move.from) != move.to - insertions.count(0, move.to)) + return false; + deletions.remove(move.from); + insertions.remove(move.to); + return true; + }), end(moves)); modifications.erase_at(c.deletions); modifications.shift_for_insert_at(c.insertions); @@ -297,10 +315,6 @@ void CollectionChangeBuilder::verify() REALM_ASSERT(deletions.contains(move.from)); REALM_ASSERT(insertions.contains(move.to)); } -// for (auto index : modifications.as_indexes()) -// REALM_ASSERT(!insertions.contains(index)); -// for (auto index : insertions.as_indexes()) -// REALM_ASSERT(!modifications.contains(index)); #endif } @@ -311,8 +325,7 @@ struct RowInfo { size_t tv_index; }; -void calculate_moves_unsorted(std::vector& new_rows, CollectionChangeIndices& changeset, - std::function row_did_change) +void calculate_moves_unsorted(std::vector& new_rows, CollectionChangeIndices& changeset) { for (auto& row : new_rows) { // Calculate where this row would be with only previous insertions @@ -326,9 +339,6 @@ void calculate_moves_unsorted(std::vector& new_rows, CollectionChangeIn changeset.insertions.add(row.tv_index); changeset.deletions.add(row.prev_tv_index); } - else if (!changeset.insertions.contains(row.tv_index) && row_did_change(row.shifted_row_index)) { - changeset.modifications.add(row.tv_index); - } } } @@ -396,16 +406,11 @@ void find_longest_matches(items const& a, items const& b_ndx, find_longest_matches(a, b_ndx, m.i + m.size, end1, m.j + m.size, end2, modified, ret); } -void calculate_moves_sorted(std::vector& new_rows, CollectionChangeIndices& changeset, - std::function row_did_change) +void calculate_moves_sorted(std::vector& new_rows, CollectionChangeIndices& changeset) { std::vector> old_candidates; std::vector> new_candidates; for (auto& row : new_rows) { - if (row_did_change(row.shifted_row_index)) { - changeset.modifications.add(row.tv_index); - } - old_candidates.push_back({row.shifted_row_index, row.prev_tv_index}); new_candidates.push_back({row.shifted_row_index, row.tv_index}); } @@ -456,9 +461,6 @@ void calculate_moves_sorted(std::vector& new_rows, CollectionChangeIndi i += match.size; j += match.size; } - - // FIXME: needlessly suboptimal - changeset.modifications.remove(changeset.insertions); } } // Anonymous namespace @@ -512,17 +514,25 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c for (; j < new_rows.size(); ++j) ret.insertions.add(new_rows[j].tv_index); + // Filter out the new insertions since we don't need them for any of the + // further calculations new_rows.erase(std::remove_if(begin(new_rows), end(new_rows), [](auto& row) { return row.prev_tv_index == npos; }), end(new_rows)); std::sort(begin(new_rows), end(new_rows), [](auto& lft, auto& rgt) { return lft.tv_index < rgt.tv_index; }); + for (auto& row : new_rows) { + if (row_did_change(row.shifted_row_index)) { + ret.modifications.add(row.tv_index); + } + } + if (sort) { - calculate_moves_sorted(new_rows, ret, row_did_change); + calculate_moves_sorted(new_rows, ret); } else { - calculate_moves_unsorted(new_rows, ret, row_did_change); + calculate_moves_unsorted(new_rows, ret); } ret.verify(); diff --git a/src/impl/background_collection.cpp b/src/impl/background_collection.cpp index e7b1c753..5ce39b45 100644 --- a/src/impl/background_collection.cpp +++ b/src/impl/background_collection.cpp @@ -200,6 +200,7 @@ bool BackgroundCollection::deliver(SharedGroup& sg, std::exception_ptr err) bool should_call_callbacks = do_deliver(sg); m_changes_to_deliver = std::move(m_accumulated_changes); + m_changes_to_deliver.modifications.remove(m_changes_to_deliver.insertions); return should_call_callbacks && have_callbacks(); } diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp index ef451917..e8291fad 100644 --- a/src/impl/list_notifier.cpp +++ b/src/impl/list_notifier.cpp @@ -98,9 +98,9 @@ void ListNotifier::run() m_change.moves.erase(remove_if(begin(m_change.moves), end(m_change.moves), [&](auto move) { return m_change.modifications.contains(move.to); }), end(m_change.moves)); - m_change.modifications.remove(m_change.insertions); for (size_t i = 0; i < m_lv->size(); ++i) { + // FIXME: may need to mark modifications even for inserts (for moves?) if (m_change.insertions.contains(i) || m_change.modifications.contains(i)) continue; if (m_info->row_did_change(m_lv->get_target_table(), m_lv->get(i).get_index())) diff --git a/src/index_set.cpp b/src/index_set.cpp index 0df1b787..3468f439 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -86,6 +86,14 @@ size_t IndexSet::add_shifted(size_t index) void IndexSet::add_shifted_by(IndexSet const& shifted_by, IndexSet const& values) { +#ifdef REALM_DEBUG + size_t expected = std::distance(as_indexes().begin(), as_indexes().end()); + for (auto index : values.as_indexes()) { + if (!shifted_by.contains(index)) + ++expected; + } +#endif + auto it = shifted_by.begin(), end = shifted_by.end(); size_t shift = 0; size_t skip_until = 0; @@ -100,6 +108,8 @@ void IndexSet::add_shifted_by(IndexSet const& shifted_by, IndexSet const& values ++shift; } } + + REALM_ASSERT_DEBUG(std::distance(as_indexes().begin(), as_indexes().end()) == expected); } void IndexSet::set(size_t len) diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index 0b06f3b0..327b5f83 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -8,7 +8,7 @@ TEST_CASE("collection change indices") { using namespace realm; _impl::CollectionChangeBuilder c; - SECTION("stuff") { + SECTION("basic mutation functions") { SECTION("insert() adds the row to the insertions set") { c.insert(5); c.insert(8); @@ -220,6 +220,12 @@ TEST_CASE("collection change indices") { REQUIRE(c.moves.empty()); } + SECTION("modifications are reported for moved rows") { + c = _impl::CollectionChangeBuilder::calculate({3, 5}, {5, 3}, all_modified, false); + REQUIRE_MOVES(c, {1, 0}); + REQUIRE_INDICES(c.modifications, 0, 1); + } + SECTION("unsorted reordering") { auto calc = [&](std::vector values) { return _impl::CollectionChangeBuilder::calculate({1, 2, 3}, values, none_modified, false); @@ -372,11 +378,11 @@ TEST_CASE("collection change indices") { REQUIRE_INDICES(c.modifications, 2); } - SECTION("modifications are discarded for previous insertions") { + SECTION("modifications are not discarded for previous insertions") { c = {{}, {2}, {}, {}}; c.merge({{}, {}, {1, 2, 3}}); REQUIRE_INDICES(c.insertions, 2); - REQUIRE_INDICES(c.modifications, 1, 3); + REQUIRE_INDICES(c.modifications, 1, 2, 3); } SECTION("modifications are merged with previous modifications") { @@ -385,10 +391,10 @@ TEST_CASE("collection change indices") { REQUIRE_INDICES(c.modifications, 1, 2, 3); } - SECTION("modifications are discarded for the destination of previous moves") { + SECTION("modifications are tracked for the destination of previous moves") { c = {{}, {}, {}, {{1, 2}}}; c.merge({{}, {}, {2, 3}}); - REQUIRE_INDICES(c.modifications, 3); + REQUIRE_INDICES(c.modifications, 2, 3); } SECTION("move sources are shifted for previous deletes and insertions") { @@ -405,10 +411,10 @@ TEST_CASE("collection change indices") { REQUIRE_MOVES(c, {5, 10}); } - SECTION("moves remove previous modifications to source") { + SECTION("moves update previous modifications to source") { c = {{}, {}, {1}, {}}; c.merge({{}, {}, {}, {{1, 3}}}); - REQUIRE(c.modifications.empty()); + REQUIRE_INDICES(c.modifications, 3); REQUIRE_MOVES(c, {1, 3}); } @@ -460,5 +466,44 @@ TEST_CASE("collection change indices") { c.merge({{}, {}, {}, {{6, 2}}}); REQUIRE_MOVES(c, {7, 2}); } + + SECTION("leapfrogging rows collapse to an empty changeset") { + c = {{1}, {0}, {}, {{1, 0}}}; + c.merge({{1}, {0}, {}, {{1, 0}}}); + REQUIRE(c.empty()); + } + + SECTION("modify -> move -> unmove leaves row marked as modified") { + c = {{}, {}, {1}}; + c.merge({{1}, {2}, {}, {{1, 2}}}); + c.merge({{1}}); + + REQUIRE_INDICES(c.deletions, 2); + REQUIRE(c.insertions.empty()); + REQUIRE(c.moves.empty()); + REQUIRE_INDICES(c.modifications, 1); + } + + SECTION("modifying a previously moved row which stops being a move due to more deletions") { + // Make it stop being a move in the same transaction as the modify + c = {{1, 2}, {0, 1}, {}, {{1, 0}, {2, 1}}}; + c.merge({{0, 2}, {1}, {0}, {}}); + + REQUIRE_INDICES(c.deletions, 0, 1); + REQUIRE_INDICES(c.insertions, 1); + REQUIRE_INDICES(c.modifications, 0); + REQUIRE(c.moves.empty()); + + // Same net change, but make it no longer a move in the transaction after the modify + c = {{1, 2}, {0, 1}, {}, {{1, 0}, {2, 1}}}; + c.merge({{}, {}, {1}, {}}); + c.merge({{0, 2}, {0}, {}, {{2, 0}}}); + c.merge({{0}, {1}, {}, {}}); + + REQUIRE_INDICES(c.deletions, 0, 1); + REQUIRE_INDICES(c.insertions, 1); + REQUIRE_INDICES(c.modifications, 0); + REQUIRE(c.moves.empty()); + } } } diff --git a/tests/results.cpp b/tests/results.cpp index 15c3ad71..ccf99890 100644 --- a/tests/results.cpp +++ b/tests/results.cpp @@ -247,6 +247,25 @@ TEST_CASE("Results") { REQUIRE(notification_calls == 2); } + SECTION("inserting a row then modifying it in a second transaction does not report it as modified") { + r->begin_transaction(); + size_t ndx = table->add_empty_row(); + table->set_int(0, ndx, 6); + r->commit_transaction(); + + coordinator->on_change(); + + r->begin_transaction(); + table->set_int(0, ndx, 7); + r->commit_transaction(); + + advance_and_notify(*r); + + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.insertions, 4); + REQUIRE(change.modifications.empty()); + } + SECTION("notifications are not delivered when collapsing transactions results in no net change") { r->begin_transaction(); size_t ndx = table->add_empty_row(); diff --git a/tests/util/index_helpers.hpp b/tests/util/index_helpers.hpp index 4d847574..4608932d 100644 --- a/tests/util/index_helpers.hpp +++ b/tests/util/index_helpers.hpp @@ -2,6 +2,7 @@ index_set.verify(); \ std::initializer_list expected = {__VA_ARGS__}; \ auto actual = index_set.as_indexes(); \ + INFO("Checking " #index_set); \ REQUIRE(expected.size() == std::distance(actual.begin(), actual.end())); \ auto begin = actual.begin(), end = actual.end(); \ for (auto index : expected) { \ From b5bd00005c8d4ec1c05d49cdf491454f63f8ebee Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 11 Mar 2016 14:18:05 -0800 Subject: [PATCH 23/93] Skip collecting change information when it isn't needed --- src/impl/background_collection.cpp | 6 ++++-- src/impl/background_collection.hpp | 2 +- src/impl/list_notifier.cpp | 5 +++-- src/impl/list_notifier.hpp | 2 +- src/impl/results_notifier.cpp | 3 ++- src/impl/results_notifier.hpp | 2 +- 6 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/impl/background_collection.cpp b/src/impl/background_collection.cpp index 5ce39b45..b1d4dec9 100644 --- a/src/impl/background_collection.cpp +++ b/src/impl/background_collection.cpp @@ -161,14 +161,16 @@ void BackgroundCollection::set_table(Table const& table) void BackgroundCollection::add_required_change_info(TransactionChangeInfo& info) { + if (!do_add_required_change_info(info)) { + return; + } + auto max = *max_element(begin(m_relevant_tables), end(m_relevant_tables)) + 1; if (max > info.tables_needed.size()) info.tables_needed.resize(max, false); for (auto table_ndx : m_relevant_tables) { info.tables_needed[table_ndx] = true; } - - do_add_required_change_info(info); } void BackgroundCollection::prepare_handover() diff --git a/src/impl/background_collection.hpp b/src/impl/background_collection.hpp index 78da5124..162e0047 100644 --- a/src/impl/background_collection.hpp +++ b/src/impl/background_collection.hpp @@ -118,7 +118,7 @@ private: virtual void do_detach_from(SharedGroup&) = 0; virtual void do_prepare_handover(SharedGroup&) = 0; virtual bool do_deliver(SharedGroup&) { return true; } - virtual void do_add_required_change_info(TransactionChangeInfo&) { } + virtual bool do_add_required_change_info(TransactionChangeInfo&) { return true; } virtual bool should_deliver_initial() const noexcept { return false; } const std::thread::id m_thread_id = std::this_thread::get_id(); diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp index e8291fad..fd6a4ffe 100644 --- a/src/impl/list_notifier.cpp +++ b/src/impl/list_notifier.cpp @@ -69,11 +69,11 @@ void ListNotifier::do_detach_from(SharedGroup& sg) } } -void ListNotifier::do_add_required_change_info(TransactionChangeInfo& info) +bool ListNotifier::do_add_required_change_info(TransactionChangeInfo& info) { REALM_ASSERT(!m_lv_handover); if (!m_lv) { - return; // origin row was deleted after the notification was added + return false; // origin row was deleted after the notification was added } size_t row_ndx = m_lv->get_origin_row_index(); @@ -81,6 +81,7 @@ void ListNotifier::do_add_required_change_info(TransactionChangeInfo& info) info.lists.push_back({table.get_index_in_group(), row_ndx, m_col_ndx, &m_change}); m_info = &info; + return true; } void ListNotifier::run() diff --git a/src/impl/list_notifier.hpp b/src/impl/list_notifier.hpp index ef3805bf..9cc0d775 100644 --- a/src/impl/list_notifier.hpp +++ b/src/impl/list_notifier.hpp @@ -54,7 +54,7 @@ private: void do_detach_from(SharedGroup& sg) override; void release_data() noexcept override; - void do_add_required_change_info(TransactionChangeInfo& info) override; + bool do_add_required_change_info(TransactionChangeInfo& info) override; }; } } diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index 282adb16..de1ce338 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -71,10 +71,11 @@ static bool map_moves(size_t& idx, CollectionChangeIndices const& changes) // // Separately from the handover data flow, m_target_results is guarded by the target lock -void ResultsNotifier::do_add_required_change_info(TransactionChangeInfo& info) +bool ResultsNotifier::do_add_required_change_info(TransactionChangeInfo& info) { REALM_ASSERT(m_query); m_info = &info; + return m_initial_run_complete && have_callbacks(); } void ResultsNotifier::run() diff --git a/src/impl/results_notifier.hpp b/src/impl/results_notifier.hpp index 96cb0a38..2482b770 100644 --- a/src/impl/results_notifier.hpp +++ b/src/impl/results_notifier.hpp @@ -69,7 +69,7 @@ private: void run() override; void do_prepare_handover(SharedGroup&) override; bool do_deliver(SharedGroup& sg) override; - void do_add_required_change_info(TransactionChangeInfo& info) override; + bool do_add_required_change_info(TransactionChangeInfo& info) override; void release_data() noexcept override; void do_attach_to(SharedGroup& sg) override; From 9503a3fc0364b8e11e41f9e0f5ab40f97f4424a3 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 11 Mar 2016 14:43:08 -0800 Subject: [PATCH 24/93] Always send an empty changeset to the first call of a Results notification callback --- src/impl/background_collection.cpp | 17 +++++++++++------ src/impl/background_collection.hpp | 2 +- tests/results.cpp | 10 ++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/impl/background_collection.cpp b/src/impl/background_collection.cpp index b1d4dec9..62e7fa42 100644 --- a/src/impl/background_collection.cpp +++ b/src/impl/background_collection.cpp @@ -208,8 +208,15 @@ bool BackgroundCollection::deliver(SharedGroup& sg, std::exception_ptr err) void BackgroundCollection::call_callbacks() { - while (auto fn = next_callback()) { - fn(m_changes_to_deliver, m_error); + while (auto cb = next_callback()) { + auto fn = cb->fn; + if (!cb->initial_delivered && should_deliver_initial()) { + cb->initial_delivered = true; + fn({}, m_error); // note: may invalidate `cb` + } + else { + fn(m_changes_to_deliver, m_error); + } } if (m_error) { @@ -220,7 +227,7 @@ void BackgroundCollection::call_callbacks() } } -CollectionChangeCallback BackgroundCollection::next_callback() +BackgroundCollection::Callback* BackgroundCollection::next_callback() { std::lock_guard callback_lock(m_callback_mutex); @@ -230,9 +237,7 @@ CollectionChangeCallback BackgroundCollection::next_callback() if (!m_error && !deliver_initial && m_changes_to_deliver.empty()) { continue; } - - callback.initial_delivered = true; - return callback.fn; + return &callback; } m_callback_index = npos; diff --git a/src/impl/background_collection.hpp b/src/impl/background_collection.hpp index 162e0047..d2058a21 100644 --- a/src/impl/background_collection.hpp +++ b/src/impl/background_collection.hpp @@ -158,7 +158,7 @@ private: // remove_callback() updates this when needed size_t m_callback_index = npos; - CollectionChangeCallback next_callback(); + Callback* next_callback(); }; } // namespace _impl diff --git a/tests/results.cpp b/tests/results.cpp index ccf99890..397eb4a8 100644 --- a/tests/results.cpp +++ b/tests/results.cpp @@ -283,6 +283,16 @@ TEST_CASE("Results") { r->notify(); REQUIRE(notification_calls == 1); } + + SECTION("the first call of a notification always passes an empty change even if it previously ran for a different callback") { + auto token2 = results.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr) { + REQUIRE(c.empty()); + }); + + write([&] { + table->set_int(0, table->add_empty_row(), 5); + }); + } } // Sort in descending order From c6def6b81457e84022fe7cab0d7d3f77a11c7b9b Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 11 Mar 2016 14:49:33 -0800 Subject: [PATCH 25/93] Don't parse the transaction logs on the background thread if no change info is needed --- src/impl/realm_coordinator.cpp | 6 +++--- src/impl/transact_log_handler.cpp | 14 ++++++++++---- src/impl/transact_log_handler.hpp | 7 ++++--- tests/transaction_log_parsing.cpp | 6 +++--- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 61d99259..5b60ab4b 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -351,7 +351,7 @@ void RealmCoordinator::run_async_notifiers() // releasing the lock for (auto& notifier : new_notifiers) { if (version != notifier->version()) { - transaction::advance_and_observe_linkviews(*m_advancer_sg, *info, notifier->version()); + transaction::advance(*m_advancer_sg, *info, notifier->version()); change_info.push_back({{}, std::move(info->lists)}); info = &change_info.back(); version = notifier->version(); @@ -360,7 +360,7 @@ void RealmCoordinator::run_async_notifiers() notifier->add_required_change_info(*info); } - transaction::advance_and_observe_linkviews(*m_advancer_sg, *info); + transaction::advance(*m_advancer_sg, *info); for (auto& notifier : new_notifiers) { notifier->detach(); @@ -378,7 +378,7 @@ void RealmCoordinator::run_async_notifiers() notifier->add_required_change_info(change_info[0]); } - transaction::advance_and_observe_linkviews(*m_notifier_sg, change_info[0], version); + transaction::advance(*m_notifier_sg, change_info[0], version); // Attach the new notifiers to the main SG and move them to the main list for (auto& notifier : new_notifiers) { diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index 96ae5f41..19617c14 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -585,11 +585,17 @@ void cancel(SharedGroup& sg, BindingContext* context) }, false); } -void advance_and_observe_linkviews(SharedGroup& sg, - TransactionChangeInfo& info, - SharedGroup::VersionID version) +void advance(SharedGroup& sg, + TransactionChangeInfo& info, + SharedGroup::VersionID version) { - LangBindHelper::advance_read(sg, LinkViewObserver(info), version); + if (info.tables_needed.empty() && info.lists.empty()) { + LangBindHelper::advance_read(sg, version); + } + else { + LangBindHelper::advance_read(sg, LinkViewObserver(info), version); + } + } } // namespace transaction diff --git a/src/impl/transact_log_handler.hpp b/src/impl/transact_log_handler.hpp index 278c62db..c64d09db 100644 --- a/src/impl/transact_log_handler.hpp +++ b/src/impl/transact_log_handler.hpp @@ -49,9 +49,10 @@ void commit(SharedGroup& sg, BindingContext* binding_context); // for reverting to the old values sent to delegate void cancel(SharedGroup& sg, BindingContext* binding_context); -void advance_and_observe_linkviews(SharedGroup& sg, - TransactionChangeInfo& info, - SharedGroup::VersionID version=SharedGroup::VersionID{}); +// Advance the read transaction version, with change information gathered in info +void advance(SharedGroup& sg, + TransactionChangeInfo& info, + SharedGroup::VersionID version=SharedGroup::VersionID{}); } // namespace transaction } // namespace _impl } // namespace realm diff --git a/tests/transaction_log_parsing.cpp b/tests/transaction_log_parsing.cpp index 99d2108c..e8c3ae2a 100644 --- a/tests/transaction_log_parsing.cpp +++ b/tests/transaction_log_parsing.cpp @@ -38,7 +38,7 @@ public: _impl::TransactionChangeInfo info; info.lists.push_back({ndx, 0, 0, &c}); info.tables_needed.resize(m_group.size(), true); - _impl::transaction::advance_and_observe_linkviews(m_sg, info); + _impl::transaction::advance(m_sg, info); if (info.lists.empty()) { REQUIRE(!m_linkview->is_attached()); @@ -204,7 +204,7 @@ TEST_CASE("Transaction log parsing") { _impl::TransactionChangeInfo info; info.tables_needed.resize(g.size(), true); - _impl::transaction::advance_and_observe_linkviews(sg, info); + _impl::transaction::advance(sg, info); return info; }; @@ -349,7 +349,7 @@ TEST_CASE("Transaction log parsing") { _impl::TransactionChangeInfo info; info.tables_needed = tables_needed; - _impl::transaction::advance_and_observe_linkviews(sg, info); + _impl::transaction::advance(sg, info); return info; }; From f1f0327146fc5b374174d57eae9d3c3ddcc4dbc6 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 11 Mar 2016 16:51:35 -0800 Subject: [PATCH 26/93] Speed up unsorted changeset calculations by eliminating most calls to count() --- src/collection_notifications.cpp | 53 ++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index ee0506cc..216d76c8 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -323,22 +323,34 @@ struct RowInfo { size_t shifted_row_index; size_t prev_tv_index; size_t tv_index; + size_t shifted_tv_index; }; -void calculate_moves_unsorted(std::vector& new_rows, CollectionChangeIndices& changeset) +void calculate_moves_unsorted(std::vector& new_rows, IndexSet const& removed, CollectionChangeIndices& changeset) { + size_t expected = 0; for (auto& row : new_rows) { - // Calculate where this row would be with only previous insertions - // and deletions. We can ignore future insertions/deletions from moves - // because move_last_over() can only move rows to lower indices - size_t expected = row.prev_tv_index - - changeset.deletions.count(0, row.prev_tv_index) - + changeset.insertions.count(0, row.tv_index); - if (row.tv_index != expected) { - changeset.moves.push_back({row.prev_tv_index, row.tv_index}); - changeset.insertions.add(row.tv_index); - changeset.deletions.add(row.prev_tv_index); + // With unsorted queries rows only move due to move_last_over(), which + // inherently can only move a row to earlier in the table. + REALM_ASSERT(row.shifted_tv_index >= expected); + if (row.shifted_tv_index == expected) { + ++expected; + continue; } + + // This row isn't just the row after the previous one, but it still may + // not be a move if there were rows deleted between the two, so next + // calcuate what row should be here taking those in to account + size_t calc_expected = row.tv_index - changeset.insertions.count(0, row.tv_index) + removed.count(0, row.prev_tv_index); + if (row.shifted_tv_index == calc_expected) { + expected = calc_expected + 1; + continue; + } + + // The row still isn't the expected one, so it's a move + changeset.moves.push_back({row.prev_tv_index, row.tv_index}); + changeset.insertions.add(row.tv_index); + changeset.deletions.add(row.prev_tv_index); } } @@ -471,12 +483,15 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c { CollectionChangeBuilder ret; + size_t deleted = 0; std::vector old_rows; for (size_t i = 0; i < prev_rows.size(); ++i) { - if (prev_rows[i] == npos) + if (prev_rows[i] == npos) { + ++deleted; ret.deletions.add(i); + } else - old_rows.push_back({prev_rows[i], npos, i}); + old_rows.push_back({prev_rows[i], npos, i, i - deleted}); } std::stable_sort(begin(old_rows), end(old_rows), [](auto& lft, auto& rgt) { return lft.shifted_row_index < rgt.shifted_row_index; @@ -484,23 +499,26 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c std::vector new_rows; for (size_t i = 0; i < next_rows.size(); ++i) { - new_rows.push_back({next_rows[i], npos, i}); + new_rows.push_back({next_rows[i], npos, i, 0}); } std::stable_sort(begin(new_rows), end(new_rows), [](auto& lft, auto& rgt) { return lft.shifted_row_index < rgt.shifted_row_index; }); + IndexSet removed; + size_t i = 0, j = 0; while (i < old_rows.size() && j < new_rows.size()) { auto old_index = old_rows[i]; auto new_index = new_rows[j]; if (old_index.shifted_row_index == new_index.shifted_row_index) { new_rows[j].prev_tv_index = old_rows[i].tv_index; + new_rows[j].shifted_tv_index = old_rows[i].shifted_tv_index; ++i; ++j; } else if (old_index.shifted_row_index < new_index.shifted_row_index) { - ret.deletions.add(old_index.tv_index); + removed.add(old_index.tv_index); ++i; } else { @@ -510,7 +528,7 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c } for (; i < old_rows.size(); ++i) - ret.deletions.add(old_rows[i].tv_index); + removed.add(old_rows[i].tv_index); for (; j < new_rows.size(); ++j) ret.insertions.add(new_rows[j].tv_index); @@ -532,8 +550,9 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c calculate_moves_sorted(new_rows, ret); } else { - calculate_moves_unsorted(new_rows, ret); + calculate_moves_unsorted(new_rows, removed, ret); } + ret.deletions.add(removed); ret.verify(); #ifdef REALM_DEBUG From 1a8a56d10a058dc5d1bc7627cfddf83e976c34bb Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 14 Mar 2016 15:13:19 -0700 Subject: [PATCH 27/93] Use binary search in IndexSet::find() --- src/index_set.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/index_set.cpp b/src/index_set.cpp index 3468f439..e79d7a14 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -54,11 +54,8 @@ IndexSet::iterator IndexSet::find(size_t index) IndexSet::iterator IndexSet::find(size_t index, iterator it) { - for (auto end = m_ranges.end(); it != end; ++it) { - if (it->second > index) - return it; - } - return m_ranges.end(); + return std::lower_bound(it, m_ranges.end(), std::make_pair(size_t(0), index + 1), + [&](auto const& a, auto const& b) { return a.second < b.second; }); } void IndexSet::add(size_t index) From 059f907a4ab535e6cd5a46240d08f511816ec278 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 14 Mar 2016 15:36:01 -0700 Subject: [PATCH 28/93] Make sorted move calculations a bit less gross --- src/collection_notifications.cpp | 253 ++++++++++++++++--------------- 1 file changed, 127 insertions(+), 126 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 216d76c8..a2895a4d 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -320,7 +320,7 @@ void CollectionChangeBuilder::verify() namespace { struct RowInfo { - size_t shifted_row_index; + size_t row_index; size_t prev_tv_index; size_t tv_index; size_t shifted_tv_index; @@ -354,126 +354,125 @@ void calculate_moves_unsorted(std::vector& new_rows, IndexSet const& re } } -using items = std::vector>; +class SortedMoveCalculator { +public: + SortedMoveCalculator(std::vector& new_rows, CollectionChangeIndices& changeset) + : m_modified(changeset.modifications) + { + std::vector old_candidates; + old_candidates.reserve(new_rows.size()); + for (auto& row : new_rows) { + old_candidates.push_back({row.row_index, row.prev_tv_index}); + } + std::sort(begin(old_candidates), end(old_candidates), [](auto a, auto b) { + return std::tie(a.tv_index, a.row_index) < std::tie(b.tv_index, b.row_index); + }); -struct Match { - size_t i, j, size, modified; + // First check if the order of any of the rows actually changed + size_t first_difference = npos; + for (size_t i = 0; i < old_candidates.size(); ++i) { + if (old_candidates[i].row_index != new_rows[i].row_index) { + first_difference = i; + break; + } + } + if (first_difference == npos) + return; + + // A map from row index -> tv index in new results + b.reserve(new_rows.size()); + for (size_t i = 0; i < new_rows.size(); ++i) + b.push_back({new_rows[i].row_index, i}); + std::sort(begin(b), end(b), [](auto a, auto b) { + return std::tie(a.row_index, a.tv_index) < std::tie(b.row_index, b.tv_index); + }); + + a = std::move(old_candidates); + + find_longest_matches(first_difference, a.size(), + first_difference, new_rows.size()); + m_longest_matches.push_back({a.size(), new_rows.size(), 0}); + + size_t i = first_difference, j = first_difference; + for (auto match : m_longest_matches) { + for (; i < match.i; ++i) + changeset.deletions.add(a[i].tv_index); + for (; j < match.j; ++j) + changeset.insertions.add(new_rows[j].tv_index); + i += match.size; + j += match.size; + } + } + +private: + struct Match { + size_t i, j, size, modified; + }; + struct Row { + size_t row_index; + size_t tv_index; + }; + + IndexSet const& m_modified; + std::vector m_longest_matches; + + std::vector a, b; + + Match find_longest_match(size_t begin1, size_t end1, size_t begin2, size_t end2) + { + Match best = {begin1, begin2, 0, 0}; + std::vector len_from_j; + len_from_j.resize(end2 - begin2, 0); + std::vector len_from_j_prev = len_from_j; + + for (size_t i = begin1; i < end1; ++i) { + std::fill(begin(len_from_j), end(len_from_j), 0); + + size_t ai = a[i].row_index; + auto it = lower_bound(begin(b), end(b), Row{ai, 0}, + [](auto a, auto b) { return a.row_index < b.row_index; }); + for (; it != end(b) && it->row_index == ai; ++it) { + size_t j = it->tv_index; + if (j < begin2) + continue; + if (j >= end2) + break; + + size_t off = j - begin2; + size_t size = off == 0 ? 1 : len_from_j_prev[off - 1] + 1; + len_from_j[off] = size; + if (size > best.size) + best = {i - size + 1, j - size + 1, size, npos}; + // Given two equal-length matches, prefer the one with fewer modified rows + else if (size == best.size) { + if (best.modified == npos) + best.modified = m_modified.count(best.j - size + 1, best.j + 1); + auto count = m_modified.count(j - size + 1, j + 1); + if (count < best.modified) + best = {i - size + 1, j - size + 1, size, count}; + } + REALM_ASSERT(best.i >= begin1 && best.i + best.size <= end1); + REALM_ASSERT(best.j >= begin2 && best.j + best.size <= end2); + } + len_from_j.swap(len_from_j_prev); + } + return best; + } + + void find_longest_matches(size_t begin1, size_t end1, size_t begin2, size_t end2) + { + // FIXME: recursion could get too deep here + auto m = find_longest_match(begin1, end1, begin2, end2); + if (!m.size) + return; + if (m.i > begin1 && m.j > begin2) + find_longest_matches(begin1, m.i, begin2, m.j); + m_longest_matches.push_back(m); + if (m.i + m.size < end2 && m.j + m.size < end2) + find_longest_matches(m.i + m.size, end1, m.j + m.size, end2); + } }; -Match find_longest_match(items const& a, items const& b, - IndexSet const& modified, - size_t begin1, size_t end1, size_t begin2, size_t end2) -{ - Match best = {begin1, begin2, 0}; - std::vector len_from_j; - len_from_j.resize(end2 - begin2, 0); - std::vector len_from_j_prev = len_from_j; - - for (size_t i = begin1; i < end1; ++i) { - std::fill(begin(len_from_j), end(len_from_j), 0); - - size_t ai = a[i].first; - auto it = lower_bound(begin(b), end(b), std::make_pair(size_t(0), ai), - [](auto a, auto b) { return a.second < b.second; }); - for (; it != end(b) && it->second == ai; ++it) { - size_t j = it->first; - if (j < begin2) - continue; - if (j >= end2) - break; - - size_t off = j - begin2; - size_t size = off == 0 ? 1 : len_from_j_prev[off - 1] + 1; - len_from_j[off] = size; - if (size > best.size) - best = {i - size + 1, j - size + 1, size, npos}; - // Given two equal-length matches, prefer the one with fewer modified rows - else if (size == best.size) { - if (best.modified == npos) - best.modified = modified.count(best.j - size + 1, best.j + 1); - auto count = modified.count(j - size + 1, j + 1); - if (count < best.modified) - best = {i - size + 1, j - size + 1, size, count}; - } - REALM_ASSERT(best.i >= begin1 && best.i + best.size <= end1); - REALM_ASSERT(best.j >= begin2 && best.j + best.size <= end2); - } - len_from_j.swap(len_from_j_prev); - } - return best; -} - -void find_longest_matches(items const& a, items const& b_ndx, - size_t begin1, size_t end1, size_t begin2, size_t end2, - IndexSet const& modified, std::vector& ret) -{ - // FIXME: recursion could get too deep here - auto m = find_longest_match(a, b_ndx, modified, begin1, end1, begin2, end2); - if (!m.size) - return; - if (m.i > begin1 && m.j > begin2) - find_longest_matches(a, b_ndx, begin1, m.i, begin2, m.j, modified, ret); - ret.push_back(m); - if (m.i + m.size < end2 && m.j + m.size < end2) - find_longest_matches(a, b_ndx, m.i + m.size, end1, m.j + m.size, end2, modified, ret); -} - -void calculate_moves_sorted(std::vector& new_rows, CollectionChangeIndices& changeset) -{ - std::vector> old_candidates; - std::vector> new_candidates; - for (auto& row : new_rows) { - old_candidates.push_back({row.shifted_row_index, row.prev_tv_index}); - new_candidates.push_back({row.shifted_row_index, row.tv_index}); - } - - std::sort(begin(old_candidates), end(old_candidates), [](auto a, auto b) { - if (a.second != b.second) - return a.second < b.second; - return a.first < b.first; - }); - - // First check if the order of any of the rows actually changed - size_t first_difference = npos; - for (size_t i = 0; i < old_candidates.size(); ++i) { - if (old_candidates[i].first != new_candidates[i].first) { - first_difference = i; - break; - } - } - if (first_difference == npos) - return; - - const auto b_ndx = [&]{ - std::vector> ret; - ret.reserve(new_candidates.size()); - for (size_t i = 0; i < new_candidates.size(); ++i) - ret.push_back(std::make_pair(i, new_candidates[i].first)); - std::sort(begin(ret), end(ret), [](auto a, auto b) { - if (a.second != b.second) - return a.second < b.second; - return a.first < b.first; - }); - return ret; - }(); - - std::vector longest_matches; - find_longest_matches(old_candidates, b_ndx, - first_difference, old_candidates.size(), - first_difference, new_candidates.size(), - changeset.modifications, longest_matches); - longest_matches.push_back({old_candidates.size(), new_candidates.size(), 0}); - - size_t i = first_difference, j = first_difference; - for (auto match : longest_matches) { - for (; i < match.i; ++i) - changeset.deletions.add(old_candidates[i].second); - for (; j < match.j; ++j) - changeset.insertions.add(new_candidates[j].second); - i += match.size; - j += match.size; - } -} } // Anonymous namespace CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector const& prev_rows, @@ -485,6 +484,7 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c size_t deleted = 0; std::vector old_rows; + old_rows.reserve(prev_rows.size()); for (size_t i = 0; i < prev_rows.size(); ++i) { if (prev_rows[i] == npos) { ++deleted; @@ -493,16 +493,17 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c else old_rows.push_back({prev_rows[i], npos, i, i - deleted}); } - std::stable_sort(begin(old_rows), end(old_rows), [](auto& lft, auto& rgt) { - return lft.shifted_row_index < rgt.shifted_row_index; + std::sort(begin(old_rows), end(old_rows), [](auto& lft, auto& rgt) { + return lft.row_index < rgt.row_index; }); std::vector new_rows; + new_rows.reserve(next_rows.size()); for (size_t i = 0; i < next_rows.size(); ++i) { new_rows.push_back({next_rows[i], npos, i, 0}); } - std::stable_sort(begin(new_rows), end(new_rows), [](auto& lft, auto& rgt) { - return lft.shifted_row_index < rgt.shifted_row_index; + std::sort(begin(new_rows), end(new_rows), [](auto& lft, auto& rgt) { + return lft.row_index < rgt.row_index; }); IndexSet removed; @@ -511,13 +512,13 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c while (i < old_rows.size() && j < new_rows.size()) { auto old_index = old_rows[i]; auto new_index = new_rows[j]; - if (old_index.shifted_row_index == new_index.shifted_row_index) { + if (old_index.row_index == new_index.row_index) { new_rows[j].prev_tv_index = old_rows[i].tv_index; new_rows[j].shifted_tv_index = old_rows[i].shifted_tv_index; ++i; ++j; } - else if (old_index.shifted_row_index < new_index.shifted_row_index) { + else if (old_index.row_index < new_index.row_index) { removed.add(old_index.tv_index); ++i; } @@ -541,13 +542,13 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c [](auto& lft, auto& rgt) { return lft.tv_index < rgt.tv_index; }); for (auto& row : new_rows) { - if (row_did_change(row.shifted_row_index)) { + if (row_did_change(row.row_index)) { ret.modifications.add(row.tv_index); } } if (sort) { - calculate_moves_sorted(new_rows, ret); + SortedMoveCalculator(new_rows, ret); } else { calculate_moves_unsorted(new_rows, removed, ret); From 7f5277a97b9dfc9adce16433da21e45dd85374db Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 10 Mar 2016 11:02:59 -0800 Subject: [PATCH 29/93] Run RealmCoordinator::on_change() on a different thread when using tsan --- tests/list.cpp | 9 ++---- tests/results.cpp | 44 +++++++++----------------- tests/util/test_file.cpp | 67 ++++++++++++++++++++++++++++++++++++++++ tests/util/test_file.hpp | 2 ++ 4 files changed, 86 insertions(+), 36 deletions(-) diff --git a/tests/list.cpp b/tests/list.cpp index ab92e304..e0d15ccb 100644 --- a/tests/list.cpp +++ b/tests/list.cpp @@ -61,8 +61,7 @@ TEST_CASE("list") { f(); r->commit_transaction(); - coordinator.on_change(); - r->notify(); + advance_and_notify(*r); }; auto require_change = [&] { @@ -199,8 +198,7 @@ TEST_CASE("list") { // Each of the Lists now has a different source version and state at // that version, so they should all see different changes despite // being for the same LinkList - coordinator.on_change(); - r->notify(); + advance_and_notify(*r); REQUIRE_INDICES(changes[0].insertions, 0, 1, 2); REQUIRE(changes[0].modifications.empty()); @@ -213,8 +211,7 @@ TEST_CASE("list") { // After making another change, they should all get the same notification change_list(); - coordinator.on_change(); - r->notify(); + advance_and_notify(*r); for (int i = 0; i < 3; ++i) { REQUIRE_INDICES(changes[i].insertions, 3); diff --git a/tests/results.cpp b/tests/results.cpp index 397eb4a8..db809758 100644 --- a/tests/results.cpp +++ b/tests/results.cpp @@ -58,15 +58,13 @@ TEST_CASE("Results") { ++notification_calls; }); - coordinator->on_change(); - r->notify(); + advance_and_notify(*r); auto write = [&](auto&& f) { r->begin_transaction(); f(); r->commit_transaction(); - coordinator->on_change(); - r->notify(); + advance_and_notify(*r); }; SECTION("initial results are delivered") { @@ -79,8 +77,7 @@ TEST_CASE("Results") { r->commit_transaction(); REQUIRE(notification_calls == 1); - coordinator->on_change(); - r->notify(); + advance_and_notify(*r); REQUIRE(notification_calls == 2); } @@ -91,8 +88,7 @@ TEST_CASE("Results") { REQUIRE(notification_calls == 1); token = {}; - coordinator->on_change(); - r->notify(); + advance_and_notify(*r); REQUIRE(notification_calls == 1); } @@ -117,9 +113,7 @@ TEST_CASE("Results") { }); }); - coordinator->on_change(); - r->notify(); - + advance_and_notify(*r); REQUIRE(called); } @@ -132,8 +126,7 @@ TEST_CASE("Results") { REQUIRE(false); }); - coordinator->on_change(); - r->notify(); + advance_and_notify(*r); } SECTION("removing the current callback does not stop later ones from being called") { @@ -146,8 +139,7 @@ TEST_CASE("Results") { called = true; }); - coordinator->on_change(); - r->notify(); + advance_and_notify(*r); REQUIRE(called); } @@ -307,15 +299,13 @@ TEST_CASE("Results") { ++notification_calls; }); - coordinator->on_change(); - r->notify(); + advance_and_notify(*r); auto write = [&](auto&& f) { r->begin_transaction(); f(); r->commit_transaction(); - coordinator->on_change(); - r->notify(); + advance_and_notify(*r); }; SECTION("modifications that leave a non-matching row non-matching do not send notifications") { @@ -392,8 +382,7 @@ TEST_CASE("Results") { r->commit_transaction(); REQUIRE(notification_calls == 1); - coordinator->on_change(); - r->notify(); + advance_and_notify(*r); REQUIRE(notification_calls == 2); } } @@ -458,8 +447,7 @@ TEST_CASE("Async Results error handling") { called = true; }); - coordinator->on_change(); - r->notify(); + advance_and_notify(*r); bool called2 = false; auto token2 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { @@ -468,9 +456,7 @@ TEST_CASE("Async Results error handling") { called2 = true; }); - coordinator->on_change(); - r->notify(); - + advance_and_notify(*r); REQUIRE(called2); } } @@ -500,8 +486,7 @@ TEST_CASE("Async Results error handling") { }); OpenFileLimiter limiter; - coordinator->on_change(); - r->notify(); + advance_and_notify(*r); bool called2 = false; auto token2 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { @@ -510,8 +495,7 @@ TEST_CASE("Async Results error handling") { called2 = true; }); - coordinator->on_change(); - r->notify(); + advance_and_notify(*r); REQUIRE(called2); } diff --git a/tests/util/test_file.cpp b/tests/util/test_file.cpp index 932218c5..708c0381 100644 --- a/tests/util/test_file.cpp +++ b/tests/util/test_file.cpp @@ -1,10 +1,18 @@ #include "util/test_file.hpp" +#include "impl/realm_coordinator.hpp" + #include #include #include +#if defined(__has_feature) && __has_feature(thread_sanitizer) +#include +#include +#include +#endif + TestFile::TestFile() { static std::string tmpdir = [] { @@ -29,3 +37,62 @@ InMemoryTestFile::InMemoryTestFile() { in_memory = true; } + +#if defined(__has_feature) && __has_feature(thread_sanitizer) +// A helper which synchronously runs on_change() on a fixed background thread +// so that ThreadSanitizer can potentially detect issues +// This deliberately uses an unsafe spinlock for synchronization to ensure that +// the code being tested has to supply all required safety +static class TsanNotifyWorker { +public: + TsanNotifyWorker() + { + m_thread = std::thread([&] { work(); }); + } + + void work() + { + while (true) { + auto value = m_signal.load(std::memory_order_relaxed); + if (value == 0 || value == 1) + continue; + if (value == 2) + return; + + auto c = reinterpret_cast(value); + c->on_change(); + m_signal.store(1, std::memory_order_relaxed); + } + } + + ~TsanNotifyWorker() + { + m_signal = 2; + m_thread.join(); + } + + void on_change(realm::_impl::RealmCoordinator* c) + { + m_signal.store(reinterpret_cast(c), std::memory_order_relaxed); + while (m_signal.load(std::memory_order_relaxed) != 1) ; + } + +private: + std::atomic m_signal{0}; + std::thread m_thread; +} s_worker; + +void advance_and_notify(realm::Realm& realm) +{ + s_worker.on_change(realm::_impl::RealmCoordinator::get_existing_coordinator(realm.config().path).get()); + realm.notify(); +} + +#else // __has_feature(thread_sanitizer) + +void advance_and_notify(realm::Realm& realm) +{ + realm::_impl::RealmCoordinator::get_existing_coordinator(realm.config().path)->on_change(); + realm.notify(); +} +#endif diff --git a/tests/util/test_file.hpp b/tests/util/test_file.hpp index bd20d671..2f6a5135 100644 --- a/tests/util/test_file.hpp +++ b/tests/util/test_file.hpp @@ -12,4 +12,6 @@ struct InMemoryTestFile : TestFile { InMemoryTestFile(); }; +void advance_and_notify(realm::Realm& realm); + #endif From fdeb80f970364d24c8180751937c62d5b830fd8b Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 11 Mar 2016 16:51:35 -0800 Subject: [PATCH 30/93] Speed up IndexSet::count() a bit --- src/index_set.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/index_set.cpp b/src/index_set.cpp index e79d7a14..23930f77 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -39,11 +39,20 @@ bool IndexSet::contains(size_t index) const size_t IndexSet::count(size_t start_index, size_t end_index) const { auto it = const_cast(this)->find(start_index); - size_t ret = 0; - for (; end_index > start_index && it != m_ranges.end() && it->first < end_index; ++it) { - ret += std::min(it->second, end_index) - std::max(it->first, start_index); - start_index = it->second; + const auto end = m_ranges.end(); + if (it == end || it->first >= end_index) { + return 0; } + if (it->second >= end_index) + return std::min(it->second, end_index) - std::max(it->first, start_index); + + // These checks are somewhat redundant, but this loop is hot so pulling instructions out of it helps + size_t ret = it->second - std::max(it->first, start_index); + for (++it; it != end && it->second < end_index; ++it) { + ret += it->second - it->first; + } + if (it != end && it->first < end_index) + ret += end_index - it->first; return ret; } From 88a3b6ed00ed8f75f0050a60f068890c55e41d82 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 15 Mar 2016 14:04:21 -0700 Subject: [PATCH 31/93] Speed up the IndexSet combining operations --- src/index_set.cpp | 140 +++++++++++++++++++++++++++++++++++++++----- src/index_set.hpp | 16 +++-- tests/index_set.cpp | 2 +- 3 files changed, 135 insertions(+), 23 deletions(-) diff --git a/src/index_set.cpp b/src/index_set.cpp index 23930f77..2e553b38 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -92,6 +92,9 @@ size_t IndexSet::add_shifted(size_t index) void IndexSet::add_shifted_by(IndexSet const& shifted_by, IndexSet const& values) { + if (values.empty()) + return; + #ifdef REALM_DEBUG size_t expected = std::distance(as_indexes().begin(), as_indexes().end()); for (auto index : values.as_indexes()) { @@ -100,19 +103,39 @@ void IndexSet::add_shifted_by(IndexSet const& shifted_by, IndexSet const& values } #endif - auto it = shifted_by.begin(), end = shifted_by.end(); - size_t shift = 0; + auto old_ranges = move(m_ranges); + m_ranges.reserve(std::max(old_ranges.size(), values.size())); + + auto old_it = old_ranges.cbegin(), old_end = old_ranges.cend(); + auto shift_it = shifted_by.m_ranges.cbegin(), shift_end = shifted_by.m_ranges.cend(); + size_t skip_until = 0; + size_t old_shift = 0; + size_t new_shift = 0; for (size_t index : values.as_indexes()) { - for (; it != end && it->first <= index; ++it) { - shift += it->second - it->first; - skip_until = it->second; + for (; shift_it != shift_end && shift_it->first <= index; ++shift_it) { + new_shift += shift_it->second - shift_it->first; + skip_until = shift_it->second; } - if (index >= skip_until) { - REALM_ASSERT(index >= shift); - add_shifted(index - shift); - ++shift; + if (index < skip_until) + continue; + + for (; old_it != old_end && old_it->first <= index - new_shift + old_shift; ++old_it) { + for (size_t i = old_it->first; i < old_it->second; ++i) + add_back(i); + old_shift += old_it->second - old_it->first; } + + REALM_ASSERT(index >= new_shift); + add_back(index - new_shift + old_shift); + } + + if (old_it != old_end) { + if (!empty() && old_it->first == m_ranges.back().second) { + m_ranges.back().second = old_it->second; + ++old_it; + } + copy(old_it, old_end, back_inserter(m_ranges)); } REALM_ASSERT_DEBUG(std::distance(as_indexes().begin(), as_indexes().end()) == expected); @@ -151,9 +174,33 @@ void IndexSet::insert_at(size_t index, size_t count) void IndexSet::insert_at(IndexSet const& positions) { - for (auto range : positions) { - insert_at(range.first, range.second - range.first); + if (positions.empty()) + return; + if (empty()) { + m_ranges = positions.m_ranges; + return; } + + auto old_ranges = move(m_ranges); + m_ranges.reserve(std::max(m_ranges.size(), positions.m_ranges.size())); + + IndexIterator begin1 = old_ranges.cbegin(), begin2 = positions.m_ranges.cbegin(); + IndexIterator end1 = old_ranges.cend(), end2 = positions.m_ranges.cend(); + + size_t shift = 0; + while (begin1 != end1 && begin2 != end2) { + if (*begin1 + shift < *begin2) { + add_back(*begin1++ + shift); + } + else { + ++shift; + add_back(*begin2++); + } + } + for (; begin1 != end1; ++begin1) + add_back(*begin1 + shift); + for (; begin2 != end2; ++begin2) + add_back(*begin2); } void IndexSet::shift_for_insert_at(size_t index, size_t count) @@ -179,8 +226,34 @@ void IndexSet::shift_for_insert_at(size_t index, size_t count) void IndexSet::shift_for_insert_at(realm::IndexSet const& values) { - for (auto range : values) - shift_for_insert_at(range.first, range.second - range.first); + if (values.empty()) + return; + + size_t shift = 0; + auto it = find(values.begin()->first); + for (auto range : values) { + for (; it != m_ranges.end() && it->second + shift <= range.first; ++it) { + it->first += shift; + it->second += shift; + } + if (it == m_ranges.end()) + return; + + if (it->first + shift < range.first) { + // split the range so that we can exclude `index` + auto old_second = it->second; + it->first += shift; + it->second = range.first; + it = m_ranges.insert(it + 1, {range.first - shift, old_second}); + } + + shift += range.second - range.first; + } + + for (; it != m_ranges.end(); ++it) { + it->first += shift; + it->second += shift; + } } void IndexSet::erase_at(size_t index) @@ -190,11 +263,34 @@ void IndexSet::erase_at(size_t index) do_erase(it, index); } -void IndexSet::erase_at(realm::IndexSet const& values) +void IndexSet::erase_at(IndexSet const& positions) { + if (empty() || positions.empty()) + return; + + auto old_ranges = move(m_ranges); + m_ranges.reserve(std::max(m_ranges.size(), positions.m_ranges.size())); + + IndexIterator begin1 = old_ranges.cbegin(), begin2 = positions.m_ranges.cbegin(); + IndexIterator end1 = old_ranges.cend(), end2 = positions.m_ranges.cend(); + size_t shift = 0; - for (auto index : values.as_indexes()) - erase_at(index - shift++); + while (begin1 != end1 && begin2 != end2) { + if (*begin1 < *begin2) { + add_back(*begin1++ - shift); + } + else if (*begin1 == *begin2) { + ++shift; + ++begin1; + ++begin2; + } + else { + ++shift; + ++begin2; + } + } + for (; begin1 != end1; ++begin1) + add_back(*begin1 - shift); } size_t IndexSet::erase_and_unshift(size_t index) @@ -303,6 +399,18 @@ void IndexSet::clear() m_ranges.clear(); } +void IndexSet::add_back(size_t index) +{ + if (m_ranges.empty()) + m_ranges.push_back({index, index + 1}); + else if (m_ranges.back().second == index) + ++m_ranges.back().second; + else { + REALM_ASSERT_DEBUG(m_ranges.back().second < index); + m_ranges.push_back({index, index + 1}); + } +} + IndexSet::iterator IndexSet::do_add(iterator it, size_t index) { verify(); diff --git a/src/index_set.hpp b/src/index_set.hpp index 3186b5b3..e58af4b8 100644 --- a/src/index_set.hpp +++ b/src/index_set.hpp @@ -93,13 +93,14 @@ public: void verify() const noexcept; // An iterator over the indivual indices in the set rather than the ranges - class IndexInterator : public std::iterator { + class IndexIterator : public std::iterator { public: - IndexInterator(IndexSet::const_iterator it) : m_iterator(it) { } + IndexIterator(IndexSet::const_iterator it) : m_iterator(it) { } size_t operator*() const { return m_iterator->first + m_offset; } - bool operator!=(IndexInterator const& it) const { return m_iterator != it.m_iterator; } + bool operator==(IndexIterator const& it) const { return m_iterator == it.m_iterator; } + bool operator!=(IndexIterator const& it) const { return m_iterator != it.m_iterator; } - IndexInterator& operator++() + IndexIterator& operator++() { ++m_offset; if (m_iterator->first + m_offset == m_iterator->second) { @@ -109,7 +110,7 @@ public: return *this; } - IndexInterator operator++(int) + IndexIterator operator++(int) { auto value = *this; ++*this; @@ -124,7 +125,7 @@ public: class IndexIteratableAdaptor { public: using value_type = size_t; - using iterator = IndexInterator; + using iterator = IndexIterator; using const_iterator = iterator; const_iterator begin() const { return m_index_set.begin(); } @@ -150,6 +151,9 @@ private: iterator do_add(iterator pos, size_t index); void do_erase(iterator it, size_t index); iterator do_remove(iterator it, size_t index, size_t count); + + // Add an index which must be greater than the largest index in the set + void add_back(size_t index); }; } // namespace realm diff --git a/tests/index_set.cpp b/tests/index_set.cpp index 30737fb1..5e4562e5 100644 --- a/tests/index_set.cpp +++ b/tests/index_set.cpp @@ -186,7 +186,7 @@ TEST_CASE("index set") { REQUIRE_INDICES(set, 5, 7, 8, 11); } - SECTION("add_shifted_by() with an empty shifted by set is just bulka dd_shifted()") { + SECTION("add_shifted_by() with an empty shifted by set is just bulk add_shifted()") { set = {5}; set.add_shifted_by({}, {6, 7}); REQUIRE_INDICES(set, 5, 7, 8); From 5d5f504543463fc05a830f34952b30953cd20db4 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 16 Mar 2016 10:34:16 -0700 Subject: [PATCH 32/93] Greatly speed up sorted changeset calculations --- src/collection_notifications.cpp | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index a2895a4d..a005ba10 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -420,13 +420,24 @@ private: Match find_longest_match(size_t begin1, size_t end1, size_t begin2, size_t end2) { + struct Length { + size_t j, len; + }; + std::vector cur; + std::vector prev; + + auto length = [&](size_t j) -> size_t { + for (auto const& pair : prev) { + if (pair.j + 1 == j) + return pair.len + 1; + } + return 1; + }; + Match best = {begin1, begin2, 0, 0}; - std::vector len_from_j; - len_from_j.resize(end2 - begin2, 0); - std::vector len_from_j_prev = len_from_j; for (size_t i = begin1; i < end1; ++i) { - std::fill(begin(len_from_j), end(len_from_j), 0); + cur.clear(); size_t ai = a[i].row_index; auto it = lower_bound(begin(b), end(b), Row{ai, 0}, @@ -438,9 +449,8 @@ private: if (j >= end2) break; - size_t off = j - begin2; - size_t size = off == 0 ? 1 : len_from_j_prev[off - 1] + 1; - len_from_j[off] = size; + size_t size = length(j); + cur.push_back({j, size}); if (size > best.size) best = {i - size + 1, j - size + 1, size, npos}; // Given two equal-length matches, prefer the one with fewer modified rows @@ -454,7 +464,7 @@ private: REALM_ASSERT(best.i >= begin1 && best.i + best.size <= end1); REALM_ASSERT(best.j >= begin2 && best.j + best.size <= end2); } - len_from_j.swap(len_from_j_prev); + cur.swap(prev); } return best; } From 1b48c7193281abb7b30e8f60715ae02909faf3de Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 16 Mar 2016 11:18:59 -0700 Subject: [PATCH 33/93] Speed up transaction log parsing for queries --- src/collection_notifications.cpp | 60 ++++++++++++++++++-------------- src/index_set.cpp | 8 +---- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index a005ba10..de7cc26e 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -267,45 +267,51 @@ void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) erase(row_ndx); return; } - move(last_row, row_ndx); - erase(row_ndx + 1); - return; + + bool modified = modifications.contains(last_row); + modifications.erase_at(last_row); + if (modified) + modifications.add(row_ndx); + else + modifications.remove(row_ndx); bool updated_existing_move = false; for (size_t i = 0; i < moves.size(); ++i) { auto& move = moves[i]; - REALM_ASSERT(move.to <= last_row); - + // Remove moves to the row being deleted if (move.to == row_ndx) { - REALM_ASSERT(!updated_existing_move); - moves[i] = moves.back(); - moves.pop_back(); + moves.erase(moves.begin() + i); --i; - updated_existing_move = true; + continue; } - else if (move.to == last_row) { - REALM_ASSERT(!updated_existing_move); - move.to = row_ndx; - updated_existing_move = true; - } - } - if (!updated_existing_move) { - moves.push_back({last_row, row_ndx}); + if (move.to != last_row) + continue; + REALM_ASSERT(!updated_existing_move); + + // Collapse A -> B, B -> C into a single A -> C move + move.to = row_ndx; + updated_existing_move = true; + + insertions.erase_at(last_row); + insertions.insert_at(row_ndx); + // Because this is a move, the unshifted source row has already been marked as deleted } - insertions.remove(row_ndx); - modifications.remove(row_ndx); + if (updated_existing_move) + return; - // not add_shifted() because unordered removal does not shift - // mixed ordered/unordered removal currently not supported - deletions.add(row_ndx); - - if (modifications.contains(last_row)) { - modifications.remove(last_row); - modifications.add(row_ndx); + // Don't report deletions/moves if last_row is newly inserted + auto shifted_last_row = insertions.erase_and_unshift(last_row); + if (shifted_last_row != npos) { + shifted_last_row = deletions.add_shifted(shifted_last_row); + moves.push_back({shifted_last_row, row_ndx}); } - insertions.add(row_ndx); + // Don't mark the moved-over row as deleted if it was a new insertion + if (!insertions.contains(row_ndx)) { + deletions.add_shifted(insertions.unshift(row_ndx)); + insertions.add(row_ndx); + } } void CollectionChangeBuilder::verify() diff --git a/src/index_set.cpp b/src/index_set.cpp index 2e553b38..68fa6b2c 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -385,13 +385,7 @@ size_t IndexSet::shift(size_t index) const size_t IndexSet::unshift(size_t index) const { REALM_ASSERT_DEBUG(!contains(index)); - auto shifted = index; - for (auto range : m_ranges) { - if (range.first >= index) - break; - shifted -= std::min(range.second, index) - range.first; - } - return shifted; + return index - count(0, index); } void IndexSet::clear() From 71911ee22112057c83582976467a76a4254cd1ca Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 16 Mar 2016 15:02:24 -0700 Subject: [PATCH 34/93] Add support for wrapping a LinkView in a Results --- src/impl/results_notifier.cpp | 1 + src/impl/results_notifier.hpp | 1 + src/list.cpp | 9 +++- src/list.hpp | 1 + src/results.cpp | 85 +++++++++++++++++++++++++++++++---- src/results.hpp | 11 ++++- tests/list.cpp | 32 ++++++++++++- 7 files changed, 128 insertions(+), 12 deletions(-) diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index de1ce338..c6b0f65c 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -28,6 +28,7 @@ ResultsNotifier::ResultsNotifier(Results& target) : BackgroundCollection(target.get_realm()) , m_target_results(&target) , m_sort(target.get_sort()) +, m_from_linkview(target.get_linkview().get() != nullptr) { Query q = target.get_query(); set_table(*q.get_table()); diff --git a/src/impl/results_notifier.hpp b/src/impl/results_notifier.hpp index 2482b770..bb4bd024 100644 --- a/src/impl/results_notifier.hpp +++ b/src/impl/results_notifier.hpp @@ -41,6 +41,7 @@ private: Results* m_target_results; const SortOrder m_sort; + bool m_from_linkview; // The source Query, in handover form iff m_sg is null std::unique_ptr> m_query_handover; diff --git a/src/list.cpp b/src/list.cpp index b4b59aa6..b6ac8c61 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -155,7 +155,14 @@ void List::delete_all() Results List::sort(SortOrder order) { - return Results(m_realm, *m_object_schema, get_query(), std::move(order)); + verify_attached(); + return Results(m_realm, *m_object_schema, m_link_view, util::none, std::move(order)); +} + +Results List::filter(Query q) +{ + verify_attached(); + return Results(m_realm, *m_object_schema, m_link_view, get_query().and_query(std::move(q))); } // These definitions rely on that LinkViews are interned by core diff --git a/src/list.hpp b/src/list.hpp index 43b20a93..1b4f2a07 100644 --- a/src/list.hpp +++ b/src/list.hpp @@ -67,6 +67,7 @@ public: void delete_all(); Results sort(SortOrder order); + Results filter(Query q); bool operator==(List const& rgt) const noexcept; diff --git a/src/results.cpp b/src/results.cpp index acf79aad..525e4b49 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -57,6 +57,21 @@ Results::Results(SharedRealm r, const ObjectSchema &o, Table& table) { } +Results::Results(SharedRealm r, const ObjectSchema& o, LinkViewRef lv, util::Optional q, SortOrder s) +: m_realm(std::move(r)) +, m_object_schema(&o) +, m_link_view(lv) +, m_table(&lv->get_target_table()) +, m_sort(std::move(s)) +, m_mode(Mode::LinkView) +{ + REALM_ASSERT(m_sort.column_indices.size() == m_sort.ascending.size()); + if (q) { + m_query = std::move(*q); + m_mode = Mode::Query; + } +} + Results::~Results() { if (m_notifier) { @@ -72,6 +87,8 @@ void Results::validate_read() const throw InvalidatedException(); if (m_mode == Mode::TableView && !m_table_view.is_attached()) throw InvalidatedException(); + if (m_mode == Mode::LinkView && !m_link_view->is_attached()) + throw InvalidatedException(); } void Results::validate_write() const @@ -96,9 +113,10 @@ size_t Results::size() { validate_read(); switch (m_mode) { - case Mode::Empty: return 0; - case Mode::Table: return m_table->size(); - case Mode::Query: return m_query.count(); + case Mode::Empty: return 0; + case Mode::Table: return m_table->size(); + case Mode::Query: return m_query.count(); + case Mode::LinkView: return m_link_view->size(); case Mode::TableView: update_tableview(); return m_table_view.size(); @@ -120,12 +138,21 @@ RowExpr Results::get(size_t row_ndx) if (row_ndx < m_table->size()) return m_table->get(row_ndx); break; + case Mode::LinkView: + if (update_linkview()) { + if (row_ndx < m_link_view->size()) + return m_link_view->get(row_ndx); + break; + } + REALM_FALLTHROUGH; case Mode::Query: case Mode::TableView: update_tableview(); - if (row_ndx < m_table_view.size()) - return (!m_live && !m_table_view.is_row_attached(row_ndx)) ? RowExpr() : m_table_view.get(row_ndx); - break; + if (row_ndx >= m_table_view.size()) + break; + if (!m_live && !m_table_view.is_row_attached(row_ndx)) + return {}; + return m_table_view.get(row_ndx); } throw OutOfBoundsIndexException{row_ndx, size()}; @@ -139,6 +166,10 @@ util::Optional Results::first() return none; case Mode::Table: return m_table->size() == 0 ? util::none : util::make_optional(m_table->front()); + case Mode::LinkView: + if (update_linkview()) + return m_link_view->size() == 0 ? util::none : util::make_optional(m_link_view->get(0)); + REALM_FALLTHROUGH; case Mode::Query: case Mode::TableView: update_tableview(); @@ -155,6 +186,10 @@ util::Optional Results::last() return none; case Mode::Table: return m_table->size() == 0 ? util::none : util::make_optional(m_table->back()); + case Mode::LinkView: + if (update_linkview()) + return m_link_view->size() == 0 ? util::none : util::make_optional(m_link_view->get(m_link_view->size() - 1)); + REALM_FALLTHROUGH; case Mode::Query: case Mode::TableView: update_tableview(); @@ -163,12 +198,24 @@ util::Optional Results::last() REALM_UNREACHABLE(); } +bool Results::update_linkview() +{ + if (m_sort) { + m_query = get_query(); + m_mode = Mode::Query; + update_tableview(); + return false; + } + return true; +} + void Results::update_tableview() { validate_read(); switch (m_mode) { case Mode::Empty: case Mode::Table: + case Mode::LinkView: return; case Mode::Query: m_table_view = m_query.find_all(); @@ -214,6 +261,10 @@ size_t Results::index_of(size_t row_ndx) return not_found; case Mode::Table: return row_ndx; + case Mode::LinkView: + if (update_linkview()) + return m_link_view->find(row_ndx); + REALM_FALLTHROUGH; case Mode::Query: case Mode::TableView: update_tableview(); @@ -241,6 +292,10 @@ util::Optional Results::aggregate(size_t column, bool return_none_for_emp if (return_none_for_empty && m_table->size() == 0) return none; return util::Optional(getter(*m_table)); + case Mode::LinkView: + m_query = get_query(); + m_mode = Mode::Query; + REALM_FALLTHROUGH; case Mode::Query: case Mode::TableView: this->update_tableview(); @@ -315,6 +370,10 @@ void Results::clear() update_tableview(); m_table_view.clear(RemoveMode::unordered); break; + case Mode::LinkView: + validate_write(); + m_link_view->remove_all_target_rows(); + break; } } @@ -327,6 +386,8 @@ Query Results::get_query() const return m_query; case Mode::TableView: return m_table_view.get_query(); + case Mode::LinkView: + return m_table->where(m_link_view); case Mode::Table: return m_table->where(); } @@ -339,6 +400,10 @@ TableView Results::get_tableview() switch (m_mode) { case Mode::Empty: return {}; + case Mode::LinkView: + if (update_linkview()) + return m_table->where(m_link_view).find_all(); + REALM_FALLTHROUGH; case Mode::Query: case Mode::TableView: update_tableview(); @@ -352,12 +417,16 @@ TableView Results::get_tableview() Results Results::sort(realm::SortOrder&& sort) const { REALM_ASSERT(sort.column_indices.size() == sort.ascending.size()); - return Results(m_realm, get_object_schema(), get_query(), std::move(sort)); + if (m_link_view) + return Results(m_realm, *m_object_schema, m_link_view, m_query, std::move(sort)); + return Results(m_realm, *m_object_schema, get_query(), std::move(sort)); } Results Results::filter(Query&& q) const { - return Results(m_realm, get_object_schema(), get_query().and_query(std::move(q)), get_sort()); + if (m_link_view) + return Results(m_realm, *m_object_schema, m_link_view, get_query().and_query(std::move(q)), m_sort); + return Results(m_realm, *m_object_schema, get_query().and_query(std::move(q)), m_sort); } void Results::prepare_async() diff --git a/src/results.hpp b/src/results.hpp index 7c6f53cf..65a971df 100644 --- a/src/results.hpp +++ b/src/results.hpp @@ -53,6 +53,7 @@ public: Results() = default; Results(SharedRealm r, const ObjectSchema& o, Table& table); Results(SharedRealm r, const ObjectSchema& o, Query q, SortOrder s = {}); + Results(SharedRealm r, const ObjectSchema& o, LinkViewRef lv, util::Optional q = {}, SortOrder s = {}); ~Results(); // Results is copyable and moveable @@ -80,6 +81,9 @@ public: // Get the object type which will be returned by get() StringData get_object_type() const noexcept; + // Get the LinkView this Results is derived from, if any + LinkViewRef get_linkview() const { return m_link_view; } + // Set whether the TableView should sync if needed before accessing results void set_live(bool live); @@ -125,6 +129,7 @@ public: Empty, // Backed by nothing (for missing tables) Table, // Backed directly by a Table Query, // Backed by a query that has not yet been turned into a TableView + LinkView, // Backed directly by a LinkView TableView // Backed by a TableView created from a Query }; // Get the currrent mode of the Results @@ -171,8 +176,6 @@ public: UnsupportedColumnTypeException(size_t column, const Table* table); }; - void update_tableview(); - // Create an async query from this Results // The query will be run on a background thread and delivered to the callback, // and then rerun after each commit (if needed) and redelivered if it changed @@ -193,6 +196,7 @@ private: const ObjectSchema *m_object_schema; Query m_query; TableView m_table_view; + LinkViewRef m_link_view; Table* m_table = nullptr; SortOrder m_sort; bool m_live = true; @@ -203,6 +207,9 @@ private: bool m_has_used_table_view = false; bool m_wants_background_updates = true; + void update_tableview(); + bool update_linkview(); + void validate_read() const; void validate_write() const; diff --git a/tests/list.cpp b/tests/list.cpp index e0d15ccb..2ccdf65a 100644 --- a/tests/list.cpp +++ b/tests/list.cpp @@ -5,8 +5,9 @@ #include "binding_context.hpp" #include "list.hpp" -#include "property.hpp" #include "object_schema.hpp" +#include "property.hpp" +#include "results.hpp" #include "schema.hpp" #include "impl/realm_coordinator.hpp" @@ -219,4 +220,33 @@ TEST_CASE("list") { } } } + + SECTION("sort()") { + auto objectschema = &*r->config().schema->find("origin"); + List list(r, *objectschema, lv); + auto results = list.sort({{0}, {false}}); + + REQUIRE(&results.get_object_schema() == objectschema); + REQUIRE(results.get_mode() == Results::Mode::LinkView); + REQUIRE(results.size() == 10); + REQUIRE(results.sum(0) == 45); + + for (size_t i = 0; i < 10; ++i) { + REQUIRE(results.get(i).get_index() == 9 - i); + } + } + + SECTION("filter()") { + auto objectschema = &*r->config().schema->find("origin"); + List list(r, *objectschema, lv); + auto results = list.filter(target->where().greater(0, 5)); + + REQUIRE(&results.get_object_schema() == objectschema); + REQUIRE(results.get_mode() == Results::Mode::Query); + REQUIRE(results.size() == 4); + + for (size_t i = 0; i < 4; ++i) { + REQUIRE(results.get(i).get_index() == i + 6); + } + } } From dc7ddfae841cfede33657467c47e1cd4c5013f8d Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 16 Mar 2016 15:44:19 -0700 Subject: [PATCH 35/93] Treat Results from LinkViews as if they were sorted for diff calculations --- src/impl/results_notifier.cpp | 2 +- tests/list.cpp | 121 ++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index c6b0f65c..55300f56 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -130,7 +130,7 @@ void ResultsNotifier::run() m_changes = CollectionChangeBuilder::calculate(m_previous_rows, next_rows, [&](size_t row) { return m_info->row_did_change(*m_query->get_table(), row); }, - !!m_sort); + m_sort || m_from_linkview); m_previous_rows = std::move(next_rows); if (m_changes.empty()) { diff --git a/tests/list.cpp b/tests/list.cpp index 2ccdf65a..7686c4ea 100644 --- a/tests/list.cpp +++ b/tests/list.cpp @@ -221,6 +221,127 @@ TEST_CASE("list") { } } + SECTION("sorted add_notification_block()") { + List lst(r, *r->config().schema->find("origin"), lv); + Results results = lst.sort({{0}, {false}}); + + int notification_calls = 0; + CollectionChangeIndices change; + auto token = results.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { + REQUIRE_FALSE(err); + change = c; + ++notification_calls; + }); + + advance_and_notify(*r); + + auto write = [&](auto&& f) { + r->begin_transaction(); + f(); + r->commit_transaction(); + + advance_and_notify(*r); + }; + + SECTION("add duplicates") { + write([&] { + lst.add(5); + lst.add(5); + lst.add(5); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.insertions, 5, 6, 7); + } + + SECTION("change order by modifying target") { + write([&] { + lst.get(5).set_int(0, 15); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.deletions, 4); + REQUIRE_INDICES(change.insertions, 0); + } + + SECTION("swap") { + write([&] { + lst.swap(1, 2); + }); + REQUIRE(notification_calls == 1); + } + + SECTION("move") { + write([&] { + lst.move(5, 3); + }); + REQUIRE(notification_calls == 1); + } + } + + SECTION("filtered add_notification_block()") { + List lst(r, *r->config().schema->find("origin"), lv); + Results results = lst.filter(target->where().less(0, 9)); + + int notification_calls = 0; + CollectionChangeIndices change; + auto token = results.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { + REQUIRE_FALSE(err); + change = c; + ++notification_calls; + }); + + advance_and_notify(*r); + + auto write = [&](auto&& f) { + r->begin_transaction(); + f(); + r->commit_transaction(); + + advance_and_notify(*r); + }; + + SECTION("add duplicates") { + write([&] { + lst.add(5); + lst.add(5); + lst.add(5); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.insertions, 9, 10, 11); + } + + SECTION("swap") { + write([&] { + lst.swap(1, 2); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.deletions, 2); + REQUIRE_INDICES(change.insertions, 1); + + write([&] { + lst.swap(5, 8); + }); + REQUIRE(notification_calls == 3); + REQUIRE_INDICES(change.deletions, 5, 8); + REQUIRE_INDICES(change.insertions, 5, 8); + } + + SECTION("move") { + write([&] { + lst.move(5, 3); + }); + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.deletions, 5); + REQUIRE_INDICES(change.insertions, 3); + } + + SECTION("move non-matching entry") { + write([&] { + lst.move(9, 3); + }); + REQUIRE(notification_calls == 1); + } + } + SECTION("sort()") { auto objectschema = &*r->config().schema->find("origin"); List list(r, *objectschema, lv); From 0e11a791e9c41fa73006ec880b18039d7853a4e8 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 16 Mar 2016 15:58:50 -0700 Subject: [PATCH 36/93] Improve and expand the IndexSet tests This gets index_set.cpp back up to 100% line coverage from just the targeted unit tests. --- tests/index_set.cpp | 606 ++++++++++++++++++++++++++++---------------- 1 file changed, 384 insertions(+), 222 deletions(-) diff --git a/tests/index_set.cpp b/tests/index_set.cpp index 5e4562e5..db0d9944 100644 --- a/tests/index_set.cpp +++ b/tests/index_set.cpp @@ -4,22 +4,33 @@ #include "util/index_helpers.hpp" -TEST_CASE("index set") { - realm::IndexSet set; - - SECTION("contains() returns if the index is in the set") { - set = {1, 2, 3, 5}; +TEST_CASE("[index_set] contains()") { + SECTION("returns false if the index is before the first entry in the set") { + realm::IndexSet set = {1, 2, 5}; REQUIRE_FALSE(set.contains(0)); - REQUIRE(set.contains(1)); - REQUIRE(set.contains(2)); - REQUIRE(set.contains(3)); - REQUIRE_FALSE(set.contains(4)); - REQUIRE(set.contains(5)); + } + + SECTION("returns false if the index is after the last entry in the set") { + realm::IndexSet set = {1, 2, 5}; REQUIRE_FALSE(set.contains(6)); } - SECTION("count() returns the number of indices int he range in the set") { - set = {1, 2, 3, 5}; + SECTION("returns false if the index is between ranges in the set") { + realm::IndexSet set = {1, 2, 5}; + REQUIRE_FALSE(set.contains(4)); + } + + SECTION("returns true if the index is in the set") { + realm::IndexSet set = {1, 2, 5}; + REQUIRE(set.contains(1)); + REQUIRE(set.contains(2)); + REQUIRE(set.contains(5)); + } +} + +TEST_CASE("[index_set] count()") { + SECTION("returns the number of indices in the set in the given range") { + realm::IndexSet set = {1, 2, 3, 5}; REQUIRE(set.count(0, 6) == 4); REQUIRE(set.count(0, 5) == 3); REQUIRE(set.count(0, 4) == 3); @@ -37,7 +48,21 @@ TEST_CASE("index set") { REQUIRE(set.count(6, 6) == 0); } - SECTION("add() extends existing ranges") { + SECTION("includes full ranges in the middle") { + realm::IndexSet set = {1, 3, 4, 5, 10}; + REQUIRE(set.count(0, 11) == 5); + } + + SECTION("truncates ranges at the beginning and end") { + realm::IndexSet set = {1, 2, 3, 5, 6, 7, 8, 9}; + REQUIRE(set.count(3, 9) == 5); + } +} + +TEST_CASE("[index_set] add()") { + realm::IndexSet set; + + SECTION("extends existing ranges when next to an edge") { set.add(1); REQUIRE_INDICES(set, 1); @@ -48,7 +73,7 @@ TEST_CASE("index set") { REQUIRE_INDICES(set, 0, 1, 2); } - SECTION("add() with gaps") { + SECTION("does not extend ranges over gaps") { set.add(0); REQUIRE_INDICES(set, 0); @@ -56,143 +81,108 @@ TEST_CASE("index set") { REQUIRE_INDICES(set, 0, 2); } - SECTION("add() is idempotent") { + SECTION("does nothing when the index is already in the set") { set.add(0); set.add(0); REQUIRE_INDICES(set, 0); } - SECTION("add() merges existing ranges") { + SECTION("merges existing ranges when adding the index between them") { set = {0, 2, 4}; set.add(1); REQUIRE_INDICES(set, 0, 1, 2, 4); } - SECTION("add() combines multiple index sets") { + SECTION("combines multiple index sets without any shifting") { set = {0, 2, 6}; set.add({1, 4, 5}); REQUIRE_INDICES(set, 0, 1, 2, 4, 5, 6); } +} - SECTION("set() from empty") { - set.set(5); - REQUIRE_INDICES(set, 0, 1, 2, 3, 4); - } +TEST_CASE("[index_set] add_shifted()") { + realm::IndexSet set; - SECTION("set() discards existing data") { - set = {8, 9}; - - set.set(5); - REQUIRE_INDICES(set, 0, 1, 2, 3, 4); - } - - SECTION("insert_at() on an empty set is add()") { - set.insert_at(5); - REQUIRE_INDICES(set, 5); - } - - SECTION("insert_at() extends ranges containing the target index") { - set = {5, 6}; - - set.insert_at(5); - REQUIRE_INDICES(set, 5, 6, 7); - - set.insert_at(4); - REQUIRE_INDICES(set, 4, 6, 7, 8); - - set.insert_at(9); - REQUIRE_INDICES(set, 4, 6, 7, 8, 9); - } - - SECTION("insert_at() does not modify ranges entirely before it") { - set = {5, 6}; - set.insert_at(8); - REQUIRE_INDICES(set, 5, 6, 8); - } - - SECTION("insert_at() shifts ranges after it") { - set = {5, 6}; - set.insert_at(3); - REQUIRE_INDICES(set, 3, 6, 7); - } - - SECTION("insert_at() cannot join ranges") { - set = {5, 7}; - set.insert_at(6); - REQUIRE_INDICES(set, 5, 6, 8); - } - - SECTION("bulk insert_at() on an empty set is add()") { - set.insert_at({5, 6, 8}); - REQUIRE_INDICES(set, 5, 6, 8); - } - - SECTION("bulk insert_at() shifts existing ranges") { - set = {5, 10}; - set.insert_at({3, 8, 14}); - REQUIRE_INDICES(set, 3, 6, 8, 12, 14); - } - - SECTION("bulk insert_at() does not join ranges") { - set = {5, 7}; - set.insert_at({5, 6, 7}); - REQUIRE_INDICES(set, 5, 6, 7, 8, 10); - } - - SECTION("bulk insert_at() extends existing ranges") { - set = {5, 8}; - set.insert_at({5, 9}); - REQUIRE_INDICES(set, 5, 6, 9, 10); - - set = {4, 5}; - set.insert_at({5, 6}); - REQUIRE_INDICES(set, 4, 5, 6, 7); - } - - SECTION("add_shifted() on an empty set is just add()") { + SECTION("on an empty set is just add()") { set.add_shifted(5); REQUIRE_INDICES(set, 5); } - SECTION("add_shifted() before the first range is just add()") { - set.add(10); + SECTION("before the first range is just add()") { + set = {10}; set.add_shifted(5); REQUIRE_INDICES(set, 5, 10); } - SECTION("add_shifted() on first index of range extends range") { - set.add(5); + SECTION("on first index of a range extends the range") { + set = {5}; + set.add_shifted(5); REQUIRE_INDICES(set, 5, 6); set.add_shifted(5); REQUIRE_INDICES(set, 5, 6, 7); + } + SECTION("in the middle of a range is shifted by that range") { + set = {5, 6, 7}; set.add_shifted(6); REQUIRE_INDICES(set, 5, 6, 7, 9); } - SECTION("add_shifted() after ranges shifts by the size of those ranges") { - set.add(5); + SECTION("after the last range adds the total count to the index to be added") { + set = {5}; + set.add_shifted(6); REQUIRE_INDICES(set, 5, 7); - set.add_shifted(6); // bumped into second range - REQUIRE_INDICES(set, 5, 7, 8); - - set.add_shifted(8); - REQUIRE_INDICES(set, 5, 7, 8, 11); + set.add_shifted(10); + REQUIRE_INDICES(set, 5, 7, 12); } - SECTION("add_shifted_by() with an empty shifted by set is just bulk add_shifted()") { - set = {5}; - set.add_shifted_by({}, {6, 7}); + SECTION("in between ranges can be bumped into the next range") { + set = {5, 7}; + set.add_shifted(6); REQUIRE_INDICES(set, 5, 7, 8); } +} - SECTION("add_shifted_by() shifts backwards for indices in the first set") { +TEST_CASE("[index_set] add_shifted_by()") { + realm::IndexSet set; + + SECTION("does nothing given an empty set to add") { + set = {5, 6, 7}; + set.add_shifted_by({5, 6}, {}); + REQUIRE_INDICES(set, 5, 6, 7); + } + + SECTION("does nothing if values is a subset of shifted_by") { + set = {5, 6, 7}; + set.add_shifted_by({3, 4}, {3, 4}); + REQUIRE_INDICES(set, 5, 6, 7); + } + + SECTION("just adds the indices when they are all before the old indices and the shifted-by set is empty") { + set = {5, 6}; + set.add_shifted_by({}, {3, 4}); + REQUIRE_INDICES(set, 3, 4, 5, 6); + } + + SECTION("adds the indices shifted by the old count when they are all after the old indices and the shifted-by set is empty") { + set = {5, 6}; + set.add_shifted_by({}, {7, 8}); + REQUIRE_INDICES(set, 5, 6, 9, 10); + } + + SECTION("acts like bulk add_shifted() when shifted_by is empty") { + set = {5, 10}; + set.add_shifted_by({}, {4, 5, 11}); + REQUIRE_INDICES(set, 4, 5, 6, 10, 13); + } + + SECTION("shifts indices in values back by the number of indices in shifted_by before them") { set = {5}; set.add_shifted_by({0, 2, 3}, {6}); REQUIRE_INDICES(set, 3, 5); @@ -202,97 +192,245 @@ TEST_CASE("index set") { REQUIRE_INDICES(set, 2, 5); } - SECTION("add_shifted_by() discards indices in the first set") { + SECTION("discards indices in both shifted_by and values") { set = {5}; - set.add_shifted_by({3}, {3}); + set.add_shifted_by({2}, {2, 4}); + REQUIRE_INDICES(set, 3, 5); + } +} + +TEST_CASE("[index_set] set()") { + realm::IndexSet set; + + SECTION("clears the existing indices and replaces with the range [0, value)") { + set = {8, 9}; + set.set(5); + REQUIRE_INDICES(set, 0, 1, 2, 3, 4); + } +} + +TEST_CASE("[index_set] insert_at()") { + realm::IndexSet set; + + SECTION("on an empty set is add()") { + set.insert_at(5); REQUIRE_INDICES(set, 5); - set = {5}; - set.add_shifted_by({1, 3}, {3}); - REQUIRE_INDICES(set, 5); + set = {}; + set.insert_at({1, 3, 5}); + REQUIRE_INDICES(set, 1, 3, 5); } - SECTION("shift_for_insert_at() does not modify ranges before it") { - set.add(5); + SECTION("with an empty set is a no-op") { + set = {5, 6}; + set.insert_at(realm::IndexSet{}); + REQUIRE_INDICES(set, 5, 6); + } + + SECTION("extends ranges containing the target range") { + set = {5, 6}; + + set.insert_at(5); + REQUIRE_INDICES(set, 5, 6, 7); + + set.insert_at(6, 2); + REQUIRE_INDICES(set, 5, 6, 7, 8, 9); + + set.insert_at({5, 7, 11}); + REQUIRE_INDICES(set, 5, 6, 7, 8, 9, 10, 11, 12); + } + + SECTION("shifts ranges after the insertion point") { + set = {5, 6}; + + set.insert_at(3); + REQUIRE_INDICES(set, 3, 6, 7); + + set.insert_at(0, 2); + REQUIRE_INDICES(set, 0, 1, 5, 8, 9); + } + + SECTION("does not shift ranges before the insertion point") { + set = {5, 6}; + + set.insert_at(10); + REQUIRE_INDICES(set, 5, 6, 10); + + set.insert_at({15, 16}); + REQUIRE_INDICES(set, 5, 6, 10, 15, 16); + } + + SECTION("can not join ranges") { + set = {5, 7}; + set.insert_at(6); + REQUIRE_INDICES(set, 5, 6, 8); + } + + SECTION("adds later ranges after shifting for previous insertions") { + set = {5, 10}; + set.insert_at({5, 10}); + REQUIRE_INDICES(set, 5, 6, 10, 12); + } +} + +TEST_CASE("[index_set] shift_for_insert_at()") { + realm::IndexSet set; + + SECTION("does nothing given an empty set of insertion points") { + set = {5, 8}; + set.shift_for_insert_at(realm::IndexSet{}); + REQUIRE_INDICES(set, 5, 8); + } + + SECTION("does nothing when called on an empty set") { + set = {}; + set.shift_for_insert_at({5, 8}); + REQUIRE(set.empty()); + } + + SECTION("shifts indices at or after the insertion points") { + set = {5}; + + set.shift_for_insert_at(4); + REQUIRE_INDICES(set, 6); + + set.shift_for_insert_at(6); + REQUIRE_INDICES(set, 7); + + set.shift_for_insert_at({3, 8}); + REQUIRE_INDICES(set, 9); + } + + SECTION("shifts indices by the count specified") { + set = {5}; + set.shift_for_insert_at(3, 10); + REQUIRE_INDICES(set, 15); + } + + SECTION("does not shift indices before the insertion points") { + set = {5}; + set.shift_for_insert_at(6); REQUIRE_INDICES(set, 5); - } - SECTION("shift_for_insert_at() moves ranges at or after it back") { - set.add(5); - set.shift_for_insert_at(5); + set.shift_for_insert_at({3, 8}); REQUIRE_INDICES(set, 6); } - SECTION("shift_for_insert_at() splits ranges containing the index") { - set.add(5); - set.add(6); + SECTION("splits ranges containing the insertion points") { + set = {5, 6, 7, 8}; + set.shift_for_insert_at(6); - REQUIRE_INDICES(set, 5, 7); + REQUIRE_INDICES(set, 5, 7, 8, 9); + + set.shift_for_insert_at({8, 10, 12}); + REQUIRE_INDICES(set, 5, 7, 9, 11); + } +} + +TEST_CASE("[index_set] erase_at()") { + realm::IndexSet set; + + SECTION("is a no-op on an empty set") { + set.erase_at(10); + REQUIRE(set.empty()); + + set.erase_at({1, 5, 8}); + REQUIRE(set.empty()); } - SECTION("bulk shift_for_insert_at() updates things") { - set = {5, 6}; - set.shift_for_insert_at({3, 7, 10}); - REQUIRE_INDICES(set, 6, 8); - } - - SECTION("erase_at() shifts ranges after it back") { - set.add(5); - set.erase_at(4); - REQUIRE_INDICES(set, 4); - } - - SECTION("erase_at() shrinks ranges containing the index") { - set = {5, 6, 7}; - - set.erase_at(6); - REQUIRE_INDICES(set, 5, 6); - - set.erase_at(5); + SECTION("does nothing when given an empty set") { + set = {5}; + set.erase_at(realm::IndexSet{}); REQUIRE_INDICES(set, 5); } - SECTION("erase_at() removes one-element ranges") { - set = {3, 5, 7}; - + SECTION("removes the specified indices") { + set = {5}; set.erase_at(5); - REQUIRE_INDICES(set, 3, 6); + REQUIRE(set.empty()); + + set = {4, 7}; + set.erase_at({4, 7}); + REQUIRE(set.empty()); } - SECTION("erase_at() merges ranges when the gap between them is deleted") { - set.add(3); - set.add(5); + SECTION("does not modify indices before the removed one") { + set = {5, 8}; + set.erase_at(8); + REQUIRE_INDICES(set, 5); + + set = {5, 8, 9}; + set.erase_at({8, 9}); + REQUIRE_INDICES(set, 5); + } + + SECTION("shifts indices after the removed one") { + set = {5, 8}; + set.erase_at(5); + REQUIRE_INDICES(set, 7); + + set = {5, 10, 15, 20}; + set.erase_at({5, 10}); + REQUIRE_INDICES(set, 13, 18); + } + + SECTION("shrinks ranges when used on one of the edges of them") { + set = {5, 6, 7, 8}; + set.erase_at(8); + REQUIRE_INDICES(set, 5, 6, 7); + set.erase_at(5); + REQUIRE_INDICES(set, 5, 6); + + set = {5, 6, 7, 8}; + set.erase_at({5, 8}); + REQUIRE_INDICES(set, 5, 6); + } + + SECTION("shrinks ranges when used in the middle of them") { + set = {5, 6, 7, 8}; + set.erase_at(7); + REQUIRE_INDICES(set, 5, 6, 7); + + set = {5, 6, 7, 8}; + set.erase_at({6, 7}); + REQUIRE_INDICES(set, 5, 6); + } + + SECTION("merges ranges when the gap between them is deleted") { + set = {3, 5}; set.erase_at(4); REQUIRE_INDICES(set, 3, 4); - } - SECTION("bulk erase_at() does things") { - set = {3, 5, 6, 7, 10, 12}; - set.erase_at({3, 6, 11}); - REQUIRE_INDICES(set, 4, 5, 8, 9); + set = {3, 5, 7}; + set.erase_at({4, 6}); + REQUIRE_INDICES(set, 3, 4, 5); } +} - SECTION("erase_and_unshift() removes the given index") { +TEST_CASE("[index_set] erase_and_unshfit()") { + realm::IndexSet set; + + SECTION("removes the given index") { set = {1, 2}; set.erase_and_unshift(2); REQUIRE_INDICES(set, 1); } - SECTION("erase_and_unshift() shifts indexes after the given index") { + SECTION("shifts indexes after the given index") { set = {1, 5}; set.erase_and_unshift(2); REQUIRE_INDICES(set, 1, 4); } - SECTION("erase_and_unshift() returns npos for indices in the set") { + SECTION("returns npos for indices in the set") { set = {1, 3, 5}; REQUIRE(realm::IndexSet(set).erase_and_unshift(1) == realm::IndexSet::npos); REQUIRE(realm::IndexSet(set).erase_and_unshift(3) == realm::IndexSet::npos); REQUIRE(realm::IndexSet(set).erase_and_unshift(5) == realm::IndexSet::npos); } - SECTION("erase_and_unshift() returns the same thing as unshift()") { + SECTION("returns the number of indices in the set before the index for ones not in the set") { set = {1, 3, 5, 6}; REQUIRE(realm::IndexSet(set).erase_and_unshift(0) == 0); REQUIRE(realm::IndexSet(set).erase_and_unshift(2) == 1); @@ -300,7 +438,84 @@ TEST_CASE("index set") { REQUIRE(realm::IndexSet(set).erase_and_unshift(7) == 3); } - SECTION("shift() adds the number of indexes before the given index in the set to the given index") { +} + +TEST_CASE("[index_set] remove()") { + realm::IndexSet set; + + SECTION("is a no-op if the set is empty") { + set.remove(4); + REQUIRE(set.empty()); + + set.remove({1, 2, 3}); + REQUIRE(set.empty()); + } + + SECTION("is a no-op if the set to remove is empty") { + set = {5}; + set.remove(realm::IndexSet{}); + REQUIRE_INDICES(set, 5); + } + + SECTION("is a no-op if the index to remove is not in the set") { + set = {5}; + set.remove(4); + set.remove(6); + set.remove({4, 6}); + REQUIRE_INDICES(set, 5); + } + + SECTION("removes one-element ranges") { + set = {5}; + set.remove(5); + REQUIRE(set.empty()); + + set = {5}; + set.remove({3, 4, 5}); + REQUIRE(set.empty()); + } + + SECTION("shrinks ranges beginning with the index") { + set = {5, 6, 7}; + set.remove(5); + REQUIRE_INDICES(set, 6, 7); + + set = {5, 6, 7}; + set.remove({3, 5}); + REQUIRE_INDICES(set, 6, 7); + } + + SECTION("shrinks ranges ending with the index") { + set = {5, 6, 7}; + set.remove(7); + REQUIRE_INDICES(set, 5, 6); + + set = {5, 6, 7}; + set.remove({3, 7}); + REQUIRE_INDICES(set, 5, 6); + } + + SECTION("splits ranges containing the index") { + set = {5, 6, 7}; + set.remove(6); + REQUIRE_INDICES(set, 5, 7); + + set = {5, 6, 7}; + set.remove({3, 6}); + REQUIRE_INDICES(set, 5, 7); + } + + SECTION("does not shift other indices and uses unshifted positions") { + set = {5, 6, 7, 10, 11, 12, 13, 15}; + set.remove({6, 11, 13}); + REQUIRE_INDICES(set, 5, 7, 10, 12, 15); + } +} + +TEST_CASE("[index_set] shift()") { + realm::IndexSet set; + + SECTION("is ind + count(0, ind), but adds the count-so-far to the stop index") { set = {1, 3, 5, 6}; REQUIRE(set.shift(0) == 0); REQUIRE(set.shift(1) == 2); @@ -308,8 +523,12 @@ TEST_CASE("index set") { REQUIRE(set.shift(3) == 7); REQUIRE(set.shift(4) == 8); } +} - SECTION("unshift() subtracts the number of indexes in the set before the given index from the index") { +TEST_CASE("[index_set] unshift()") { + realm::IndexSet set; + + SECTION("is index - count(0, index)") { set = {1, 3, 5, 6}; REQUIRE(set.unshift(0) == 0); REQUIRE(set.unshift(2) == 1); @@ -317,71 +536,14 @@ TEST_CASE("index set") { REQUIRE(set.unshift(7) == 3); REQUIRE(set.unshift(8) == 4); } +} - SECTION("remove() does nothing if the index is not in the set") { - set = {5}; - set.remove(4); - set.remove(6); - REQUIRE_INDICES(set, 5); - } +TEST_CASE("[index_set] clear()") { + realm::IndexSet set; - SECTION("remove() removes one-element ranges") { - set = {5}; - set.remove(5); + SECTION("removes all indices from the set") { + set = {1, 2, 3}; + set.clear(); REQUIRE(set.empty()); } - - SECTION("remove() shrinks ranges beginning with the index") { - set = {5, 6, 7}; - set.remove(5); - REQUIRE_INDICES(set, 6, 7); - } - - SECTION("remove() shrinks ranges ending with the index") { - set = {5, 6, 7}; - set.remove(7); - REQUIRE_INDICES(set, 5, 6); - } - - SECTION("remove() splits ranges containing the index") { - set = {5, 6, 7}; - set.remove(6); - REQUIRE_INDICES(set, 5, 7); - } - - SECTION("bulk remove() does nothing if the indices are not in the set") { - set = {5}; - set.remove({4, 6}); - REQUIRE_INDICES(set, 5); - } - - SECTION("bulk remove() removes one-element ranges") { - set = {5}; - set.remove({5, 6}); - REQUIRE(set.empty()); - } - - SECTION("bulk remove() shrinks ranges beginning with the indices") { - set = {5, 6, 7}; - set.remove({4, 5}); - REQUIRE_INDICES(set, 6, 7); - } - - SECTION("bulk remove() shrinks ranges ending with the indices") { - set = {5, 6, 7}; - set.remove({7, 8}); - REQUIRE_INDICES(set, 5, 6); - } - - SECTION("bulk remove() splits ranges containing the indices") { - set = {5, 6, 7}; - set.remove({3, 6, 8}); - REQUIRE_INDICES(set, 5, 7); - } - - SECTION("bulk remove() correctly removes multiple indices") { - set = {5, 6, 7, 10, 11, 12, 13, 15}; - set.remove({6, 11, 13}); - REQUIRE_INDICES(set, 5, 7, 10, 12, 15); - } } From edc0d1fc4a838a6089376252de90f4e8665661bf Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 16 Mar 2016 18:11:40 -0700 Subject: [PATCH 37/93] Improve and expand the changeset calculation tests --- src/collection_notifications.cpp | 16 +- tests/collection_change_indices.cpp | 1280 ++++++++++++++++++--------- 2 files changed, 855 insertions(+), 441 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index de7cc26e..110d947f 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -332,7 +332,7 @@ struct RowInfo { size_t shifted_tv_index; }; -void calculate_moves_unsorted(std::vector& new_rows, IndexSet const& removed, CollectionChangeIndices& changeset) +void calculate_moves_unsorted(std::vector& new_rows, IndexSet& removed, CollectionChangeIndices& changeset) { size_t expected = 0; for (auto& row : new_rows) { @@ -356,7 +356,7 @@ void calculate_moves_unsorted(std::vector& new_rows, IndexSet const& re // The row still isn't the expected one, so it's a move changeset.moves.push_back({row.prev_tv_index, row.tv_index}); changeset.insertions.add(row.tv_index); - changeset.deletions.add(row.prev_tv_index); + removed.add(row.prev_tv_index); } } @@ -446,14 +446,18 @@ private: cur.clear(); size_t ai = a[i].row_index; + // Find the TV indicies at which this row appears in the new results + // There should always be at least one (or it would have been filtered out earlier), + // but can be multiple if there are dupes auto it = lower_bound(begin(b), end(b), Row{ai, 0}, [](auto a, auto b) { return a.row_index < b.row_index; }); + REALM_ASSERT(it != end(b) && it->row_index == ai); for (; it != end(b) && it->row_index == ai; ++it) { size_t j = it->tv_index; if (j < begin2) continue; if (j >= end2) - break; + break; // b is sorted by tv_index so this can't transition from false to true size_t size = length(j); cur.push_back({j, size}); @@ -478,6 +482,10 @@ private: void find_longest_matches(size_t begin1, size_t end1, size_t begin2, size_t end2) { // FIXME: recursion could get too deep here + // recursion depth worst case is currently O(N) and each recursion uses 320 bytes of stack + // could reduce worst case to O(sqrt(N)) (and typical case to O(log N)) + // biasing equal selections towards the middle, but that's still + // insufficient for Android's 8 KB stacks auto m = find_longest_match(begin1, end1, begin2, end2); if (!m.size) return; @@ -496,6 +504,8 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c std::function row_did_change, bool sort) { + REALM_ASSERT_DEBUG(sort || std::is_sorted(begin(next_rows), end(next_rows))); + CollectionChangeBuilder ret; size_t deleted = 0; diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index 327b5f83..c4fbafb7 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -4,506 +4,910 @@ #include "util/index_helpers.hpp" -TEST_CASE("collection change indices") { - using namespace realm; +using namespace realm; + +TEST_CASE("[collection_change] insert()") { _impl::CollectionChangeBuilder c; - SECTION("basic mutation functions") { - SECTION("insert() adds the row to the insertions set") { - c.insert(5); - c.insert(8); - REQUIRE_INDICES(c.insertions, 5, 8); - } - - SECTION("insert() shifts previous insertions and modifications") { - c.insert(5); - c.modify(8); - - c.insert(1); - REQUIRE_INDICES(c.insertions, 1, 6); - REQUIRE_INDICES(c.modifications, 9); - } - - SECTION("insert() does not shift previous deletions") { - c.erase(8); - c.erase(3); - c.insert(5); - - REQUIRE_INDICES(c.insertions, 5); - REQUIRE_INDICES(c.deletions, 3, 8); - } - - SECTION("modify() adds the row to the modifications set") { - c.modify(3); - c.modify(4); - REQUIRE_INDICES(c.modifications, 3, 4); - } - - SECTION("modify() on an inserted row marks it as both inserted and modified") { - c.insert(3); - c.modify(3); - REQUIRE_INDICES(c.insertions, 3); - REQUIRE_INDICES(c.modifications, 3); - } - - SECTION("modify() doesn't interact with deleted rows") { - c.erase(5); - c.erase(4); - c.erase(3); - - c.modify(4); - REQUIRE_INDICES(c.modifications, 4); - } - - SECTION("erase() adds the row to the deletions set") { - c.erase(5); - REQUIRE_INDICES(c.deletions, 5); - } - - SECTION("erase() is shifted for previous deletions") { - c.erase(5); - c.erase(6); - REQUIRE_INDICES(c.deletions, 5, 7); - } - - SECTION("erase() is shifted for previous insertions") { - c.insert(5); - c.erase(6); - REQUIRE_INDICES(c.deletions, 5); - } - - SECTION("erase() removes previous insertions") { - c.insert(5); - c.erase(5); - REQUIRE(c.insertions.empty()); - REQUIRE(c.deletions.empty()); - } - - SECTION("erase() removes previous modifications") { - c.modify(5); - c.erase(5); - REQUIRE(c.modifications.empty()); - REQUIRE_INDICES(c.deletions, 5); - } - - SECTION("erase() shifts previous modifications") { - c.modify(5); - c.erase(4); - REQUIRE_INDICES(c.modifications, 4); - REQUIRE_INDICES(c.deletions, 4); - } - - SECTION("move() adds the move to the list of moves") { - c.move(5, 6); - REQUIRE_MOVES(c, {5, 6}); - } - - SECTION("move() updates previous moves to the source of this move") { - c.move(5, 6); - c.move(6, 7); - REQUIRE_MOVES(c, {5, 7}); - } - - SECTION("move() shifts previous moves and is shifted by them") { - c.move(5, 10); - c.move(6, 12); - REQUIRE_MOVES(c, {5, 9}, {7, 12}); - - c.move(10, 0); - REQUIRE_MOVES(c, {5, 10}, {7, 12}, {11, 0}); - } - - SECTION("moving a newly inserted row is not reported as a move") { - c.insert(5); - c.move(5, 10); - REQUIRE_INDICES(c.insertions, 10); - REQUIRE(c.moves.empty()); - } - - SECTION("move() shifts previous insertions and modifications") { - c.insert(5); - c.modify(6); - c.move(10, 0); - REQUIRE_INDICES(c.insertions, 0, 6); - REQUIRE_INDICES(c.modifications, 7); - REQUIRE_MOVES(c, {9, 0}); - } - - SECTION("move_over() marks the old last row as moved") { - c.move_over(5, 8); - REQUIRE_MOVES(c, {8, 5}); - } - - SECTION("move_over() does not mark the old last row as moved if it was newly inserted") { - c.insert(8); - c.move_over(5, 8); - REQUIRE(c.moves.empty()); - } - - SECTION("move_over() removes previous modifications for the removed row") { - c.modify(5); - c.move_over(5, 8); - REQUIRE(c.modifications.empty()); - } - - SECTION("move_over() updates previous insertions for the old last row") { - c.insert(5); - c.move_over(3, 5); - REQUIRE_INDICES(c.insertions, 3); - } - - SECTION("move_over() updates previous modifications for the old last row") { - c.modify(5); - c.move_over(3, 5); - REQUIRE_INDICES(c.modifications, 3); - } - - SECTION("move_over() removes moves to the target") { - c.move(3, 5); - c.move_over(5, 8); - REQUIRE_MOVES(c, {8, 5}); - } - - SECTION("move_over() updates moves to the source") { - c.move(3, 8); - c.move_over(5, 8); - REQUIRE_MOVES(c, {3, 5}); - } - - SECTION("move_over() is not shifted by previous calls to move_over()") { - c.move_over(5, 10); - c.move_over(6, 9); - REQUIRE_INDICES(c.deletions, 5, 6, 9, 10); - REQUIRE_INDICES(c.insertions, 5, 6); - REQUIRE_MOVES(c, {10, 5}, {9, 6}); - } + SECTION("adds the row to the insertions set") { + c.insert(5); + c.insert(8); + REQUIRE_INDICES(c.insertions, 5, 8); } - SECTION("calculate") { - auto all_modified = [](size_t) { return true; }; - auto none_modified = [](size_t) { return false; }; + SECTION("shifts previous insertions and modifications") { + c.insert(5); + c.modify(8); - SECTION("no changes") { - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, none_modified, false); - REQUIRE(c.empty()); - } + c.insert(1); + REQUIRE_INDICES(c.insertions, 1, 6); + REQUIRE_INDICES(c.modifications, 9); + } - SECTION("inserting from empty") { - c = _impl::CollectionChangeBuilder::calculate({}, {1, 2, 3}, all_modified, false); - REQUIRE_INDICES(c.insertions, 0, 1, 2); - } + SECTION("does not shift previous deletions") { + c.erase(8); + c.erase(3); + c.insert(5); - SECTION("deleting all existing") { - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {}, all_modified, false); - REQUIRE_INDICES(c.deletions, 0, 1, 2); - } + REQUIRE_INDICES(c.insertions, 5); + REQUIRE_INDICES(c.deletions, 3, 8); + } - SECTION("all rows modified without changing order") { - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, all_modified, false); - REQUIRE_INDICES(c.modifications, 0, 1, 2); - } + SECTION("shifts destination of previous moves after the insertion point") { + c.moves = {{10, 5}, {10, 2}, {3, 10}}; + c.insert(4); + REQUIRE_MOVES(c, {10, 6}, {10, 2}, {3, 11}); + } +} - SECTION("single insertion in middle") { - c = _impl::CollectionChangeBuilder::calculate({1, 3}, {1, 2, 3}, all_modified, false); - REQUIRE_INDICES(c.insertions, 1); - } +TEST_CASE("[collection_change] modify()") { + _impl::CollectionChangeBuilder c; - SECTION("single deletion in middle") { - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3}, all_modified, false); - REQUIRE_INDICES(c.deletions, 1); - } + SECTION("marks the row as modified") { + c.modify(5); + REQUIRE_INDICES(c.modifications, 5); + } - SECTION("mixed insert and delete") { - c = _impl::CollectionChangeBuilder::calculate({3, 5}, {0, 3}, all_modified, false); - REQUIRE_INDICES(c.deletions, 1); - REQUIRE_INDICES(c.insertions, 0); - REQUIRE(c.moves.empty()); - } + SECTION("also marks newly inserted rows as modified") { + c.insert(5); + c.modify(5); + REQUIRE_INDICES(c.modifications, 5); + } - SECTION("modifications are reported for moved rows") { - c = _impl::CollectionChangeBuilder::calculate({3, 5}, {5, 3}, all_modified, false); - REQUIRE_MOVES(c, {1, 0}); - REQUIRE_INDICES(c.modifications, 0, 1); - } + SECTION("is idempotent") { + c.modify(5); + c.modify(5); + c.modify(5); + c.modify(5); + REQUIRE_INDICES(c.modifications, 5); + } +} - SECTION("unsorted reordering") { - auto calc = [&](std::vector values) { - return _impl::CollectionChangeBuilder::calculate({1, 2, 3}, values, none_modified, false); - }; +TEST_CASE("[collection_change] erase()") { + _impl::CollectionChangeBuilder c; - // The commented-out permutations are not possible with - // move_last_over() and so are unhandled by unsorted mode - REQUIRE(calc({1, 2, 3}).empty()); - REQUIRE_MOVES(calc({1, 3, 2}), {2, 1}); -// REQUIRE_MOVES(calc({2, 1, 3}), {1, 0}); -// REQUIRE_MOVES(calc({2, 3, 1}), {1, 0}, {2, 1}); - REQUIRE_MOVES(calc({3, 1, 2}), {2, 0}); - REQUIRE_MOVES(calc({3, 2, 1}), {2, 0}, {1, 1}); - } + SECTION("adds the row to the deletions set") { + c.erase(5); + REQUIRE_INDICES(c.deletions, 5); + } - SECTION("sorted reordering") { - auto calc = [&](std::vector values) { - return _impl::CollectionChangeBuilder::calculate({1, 2, 3}, values, all_modified, true); - }; + SECTION("is shifted for previous deletions") { + c.erase(5); + c.erase(6); + REQUIRE_INDICES(c.deletions, 5, 7); + } - REQUIRE(calc({1, 2, 3}).moves.empty()); - return; - // none of these actually work since it just does insert+delete - REQUIRE_MOVES(calc({1, 3, 2}), {2, 1}); - REQUIRE_MOVES(calc({2, 1, 3}), {1, 0}); - REQUIRE_MOVES(calc({2, 3, 1}), {1, 0}, {2, 1}); - REQUIRE_MOVES(calc({3, 1, 2}), {2, 0}); - REQUIRE_MOVES(calc({3, 2, 1}), {2, 0}, {1, 1}); - } + SECTION("is shifted for previous insertions") { + c.insert(5); + c.erase(6); + REQUIRE_INDICES(c.deletions, 5); + } - SECTION("merge can collapse insert -> move -> delete to no-op") { - auto four_modified = [](size_t ndx) { return ndx == 4; }; - for (int insert_pos = 0; insert_pos < 4; ++insert_pos) { - for (int move_to_pos = 0; move_to_pos < 4; ++move_to_pos) { - if (insert_pos == move_to_pos) - continue; - CAPTURE(insert_pos); - CAPTURE(move_to_pos); + SECTION("removes previous insertions") { + c.insert(5); + c.erase(5); + REQUIRE(c.insertions.empty()); + REQUIRE(c.deletions.empty()); + } - std::vector after_insert = {1, 2, 3}; - after_insert.insert(after_insert.begin() + insert_pos, 4); - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, after_insert, four_modified, true); + SECTION("removes previous modifications") { + c.modify(5); + c.erase(5); + REQUIRE(c.modifications.empty()); + REQUIRE_INDICES(c.deletions, 5); + } - std::vector after_move = {1, 2, 3}; - after_move.insert(after_move.begin() + move_to_pos, 4); - c.merge(_impl::CollectionChangeBuilder::calculate(after_insert, after_move, four_modified, true)); + SECTION("shifts previous modifications") { + c.modify(5); + c.erase(4); + REQUIRE_INDICES(c.modifications, 4); + REQUIRE_INDICES(c.deletions, 4); + } - c.merge(_impl::CollectionChangeBuilder::calculate(after_move, {1, 2, 3}, four_modified, true)); - REQUIRE(c.empty()); - } + SECTION("removes previous moves to the row being erased") { + c.moves = {{10, 5}}; + c.erase(5); + REQUIRE(c.moves.empty()); + } + + SECTION("shifts the destination of previous moves") { + c.moves = {{10, 5}, {10, 2}, {3, 10}}; + c.erase(4); + REQUIRE_MOVES(c, {10, 4}, {10, 2}, {3, 9}); + } +} + +TEST_CASE("[collection_change] move_over()") { + _impl::CollectionChangeBuilder c; + + SECTION("is just erase when row == last_row") { + c.move_over(10, 10); + REQUIRE_INDICES(c.deletions, 10); + REQUIRE(c.moves.empty()); + } + + SECTION("marks the old last row as moved") { + c.move_over(5, 8); + REQUIRE_MOVES(c, {8, 5}); + } + + SECTION("does not mark the old last row as moved if it was newly inserted") { + c.insert(8); + c.move_over(5, 8); + REQUIRE(c.moves.empty()); + } + + SECTION("removes previous modifications for the removed row") { + c.modify(5); + c.move_over(5, 8); + REQUIRE(c.modifications.empty()); + } + + SECTION("updates previous insertions for the old last row") { + c.insert(5); + c.move_over(3, 5); + REQUIRE_INDICES(c.insertions, 3); + } + + SECTION("updates previous modifications for the old last row") { + c.modify(5); + c.move_over(3, 5); + REQUIRE_INDICES(c.modifications, 3); + } + + SECTION("removes moves to the target") { + c.move(3, 5); + c.move_over(5, 8); + REQUIRE_MOVES(c, {8, 5}); + } + + SECTION("updates moves to the source") { + c.move(3, 8); + c.move_over(5, 8); + REQUIRE_MOVES(c, {3, 5}); + } + + SECTION("is not shifted by previous calls to move_over()") { + c.move_over(5, 10); + c.move_over(6, 9); + REQUIRE_INDICES(c.deletions, 5, 6, 9, 10); + REQUIRE_INDICES(c.insertions, 5, 6); + REQUIRE_MOVES(c, {10, 5}, {9, 6}); + } +} + +TEST_CASE("[collection_change] clear()") { + _impl::CollectionChangeBuilder c; + + SECTION("removes all insertions") { + c.insertions = {1, 2, 3}; + c.clear(0); + REQUIRE(c.insertions.empty()); + } + + SECTION("removes all modifications") { + c.modifications = {1, 2, 3}; + c.clear(0); + REQUIRE(c.modifications.empty()); + } + + SECTION("removes all moves") { + c.moves = {{1, 3}}; + c.clear(0); + REQUIRE(c.moves.empty()); + } + + SECTION("sets deletions to the number of rows before any changes") { + c.insertions = {1, 2, 3}; + c.clear(5); + REQUIRE_INDICES(c.deletions, 0, 1); + + c.deletions = {1, 2, 3}; + c.clear(5); + REQUIRE_INDICES(c.deletions, 0, 1, 2, 3, 4, 5, 6, 7); + } + + SECTION("sets deletions SIZE_T_MAX if that if the given previous size") { + c.insertions = {1, 2, 3}; + c.clear(SIZE_T_MAX); + REQUIRE(c.deletions.size() == 1); + REQUIRE(c.deletions.begin()->first == 0); + REQUIRE(c.deletions.begin()->second == SIZE_T_MAX); + } +} + +TEST_CASE("[collection_change] move()") { + _impl::CollectionChangeBuilder c; + + SECTION("adds the move to the list of moves") { + c.move(5, 6); + REQUIRE_MOVES(c, {5, 6}); + } + + SECTION("updates previous moves to the source of this move") { + c.move(5, 6); + c.move(6, 7); + REQUIRE_MOVES(c, {5, 7}); + } + + SECTION("shifts previous moves and is shifted by them") { + c.move(5, 10); + c.move(6, 12); + REQUIRE_MOVES(c, {5, 9}, {7, 12}); + + c.move(10, 0); + REQUIRE_MOVES(c, {5, 10}, {7, 12}, {11, 0}); + } + + SECTION("does not report a move if the source is newly inserted") { + c.insert(5); + c.move(5, 10); + REQUIRE_INDICES(c.insertions, 10); + REQUIRE(c.moves.empty()); + } + + SECTION("shifts previous insertions and modifications") { + c.insert(5); + c.modify(6); + c.move(10, 0); + REQUIRE_INDICES(c.insertions, 0, 6); + REQUIRE_INDICES(c.modifications, 7); + REQUIRE_MOVES(c, {9, 0}); + } + + SECTION("marks the target row as modified if the source row was") { + c.modify(5); + + c.move(5, 10); + REQUIRE_INDICES(c.modifications, 10); + + c.move(6, 12); + REQUIRE_INDICES(c.modifications, 9); + } +} + +TEST_CASE("[collection_change] calculate() unsorted") { + _impl::CollectionChangeBuilder c; + + auto all_modified = [](size_t) { return true; }; + auto none_modified = [](size_t) { return false; }; + const auto npos = size_t(-1); + + SECTION("returns an empty set when input and output are identical") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, none_modified, false); + REQUIRE(c.empty()); + } + + SECTION("marks all as inserted when prev is empty") { + c = _impl::CollectionChangeBuilder::calculate({}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.insertions, 0, 1, 2); + } + + SECTION("marks all as deleted when new is empty") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {}, all_modified, false); + REQUIRE_INDICES(c.deletions, 0, 1, 2); + } + + SECTION("marks npos rows in prev as deleted") { + c = _impl::CollectionChangeBuilder::calculate({npos, 1, 2, 3, npos}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.deletions, 0, 4); + } + + SECTION("marks modified rows which do not move as modified") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.modifications, 0, 1, 2); + } + + SECTION("does not mark unmodified rows as modified") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, none_modified, false); + REQUIRE(c.modifications.empty()); + } + + SECTION("marks newly added rows as insertions") { + c = _impl::CollectionChangeBuilder::calculate({2, 3}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.insertions, 0); + + c = _impl::CollectionChangeBuilder::calculate({1, 3}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.insertions, 1); + + c = _impl::CollectionChangeBuilder::calculate({1, 2}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.insertions, 2); + } + + SECTION("marks removed rows as deleted") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2}, all_modified, false); + REQUIRE_INDICES(c.deletions, 2); + + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3}, all_modified, false); + REQUIRE_INDICES(c.deletions, 1); + + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {2, 3}, all_modified, false); + REQUIRE_INDICES(c.deletions, 0); + } + + SECTION("marks rows as both inserted and deleted") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3, 4}, all_modified, false); + REQUIRE_INDICES(c.deletions, 1); + REQUIRE_INDICES(c.insertions, 2); + REQUIRE(c.moves.empty()); + } + + SECTION("marks rows as modified even if they moved") { + c = _impl::CollectionChangeBuilder::calculate({5, 3}, {3, 5}, all_modified, false); + REQUIRE_MOVES(c, {1, 0}); + REQUIRE_INDICES(c.modifications, 0, 1); + } + + SECTION("does not mark rows as modified if they are new") { + c = _impl::CollectionChangeBuilder::calculate({3}, {3, 5}, all_modified, false); + REQUIRE_INDICES(c.modifications, 0); + } + + SECTION("reports moves which can be produced by move_last_over()") { + auto calc = [&](std::vector values) { + return _impl::CollectionChangeBuilder::calculate(values, {1, 2, 3}, none_modified, false); + }; + + REQUIRE(calc({1, 2, 3}).empty()); + REQUIRE_MOVES(calc({1, 3, 2}), {2, 1}); + REQUIRE_MOVES(calc({2, 1, 3}), {1, 0}); + REQUIRE_MOVES(calc({2, 3, 1}), {2, 0}); + REQUIRE_MOVES(calc({3, 1, 2}), {1, 0}, {2, 1}); + REQUIRE_MOVES(calc({3, 2, 1}), {2, 0}, {1, 1}); + } +} + +TEST_CASE("[collection_change] calculate() sorted") { + _impl::CollectionChangeBuilder c; + + auto all_modified = [](size_t) { return true; }; + auto none_modified = [](size_t) { return false; }; + const auto npos = size_t(-1); + + SECTION("returns an empty set when input and output are identical") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, none_modified, true); + REQUIRE(c.empty()); + } + + SECTION("marks all as inserted when prev is empty") { + c = _impl::CollectionChangeBuilder::calculate({}, {1, 2, 3}, all_modified, true); + REQUIRE_INDICES(c.insertions, 0, 1, 2); + } + + SECTION("marks all as deleted when new is empty") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {}, all_modified, true); + REQUIRE_INDICES(c.deletions, 0, 1, 2); + } + + SECTION("marks npos rows in prev as deleted") { + c = _impl::CollectionChangeBuilder::calculate({npos, 1, 2, 3, npos}, {1, 2, 3}, all_modified, true); + REQUIRE_INDICES(c.deletions, 0, 4); + } + + SECTION("marks modified rows which do not move as modified") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, all_modified, true); + REQUIRE_INDICES(c.modifications, 0, 1, 2); + } + + SECTION("does not mark unmodified rows as modified") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, none_modified, true); + REQUIRE(c.modifications.empty()); + } + + SECTION("marks newly added rows as insertions") { + c = _impl::CollectionChangeBuilder::calculate({2, 3}, {1, 2, 3}, all_modified, true); + REQUIRE_INDICES(c.insertions, 0); + + c = _impl::CollectionChangeBuilder::calculate({1, 3}, {1, 2, 3}, all_modified, true); + REQUIRE_INDICES(c.insertions, 1); + + c = _impl::CollectionChangeBuilder::calculate({1, 2}, {1, 2, 3}, all_modified, true); + REQUIRE_INDICES(c.insertions, 2); + } + + SECTION("marks removed rows as deleted") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2}, all_modified, true); + REQUIRE_INDICES(c.deletions, 2); + + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3}, all_modified, true); + REQUIRE_INDICES(c.deletions, 1); + + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {2, 3}, all_modified, true); + REQUIRE_INDICES(c.deletions, 0); + } + + SECTION("marks rows as both inserted and deleted") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3, 4}, all_modified, true); + REQUIRE_INDICES(c.deletions, 1); + REQUIRE_INDICES(c.insertions, 2); + REQUIRE(c.moves.empty()); + } + + SECTION("marks rows as modified even if they moved") { + c = _impl::CollectionChangeBuilder::calculate({3, 5}, {5, 3}, all_modified, true); + REQUIRE_INDICES(c.deletions, 1); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.modifications, 0, 1); + } + + SECTION("does not mark rows as modified if they are new") { + c = _impl::CollectionChangeBuilder::calculate({3}, {3, 5}, all_modified, true); + REQUIRE_INDICES(c.modifications, 0); + } + + SECTION("reports inserts/deletes for simple reorderings") { + auto calc = [&](std::vector old_rows, std::vector new_rows) { + return _impl::CollectionChangeBuilder::calculate(old_rows, new_rows, none_modified, true); + }; + + c = calc({1, 2, 3}, {1, 2, 3}); + REQUIRE(c.insertions.empty()); + REQUIRE(c.deletions.empty()); + + c = calc({1, 2, 3}, {1, 3, 2}); + REQUIRE_INDICES(c.insertions, 1); + REQUIRE_INDICES(c.deletions, 2); + + c = calc({1, 2, 3}, {2, 1, 3}); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.deletions, 1); + + c = calc({1, 2, 3}, {2, 3, 1}); + REQUIRE_INDICES(c.insertions, 2); + REQUIRE_INDICES(c.deletions, 0); + + c = calc({1, 2, 3}, {3, 1, 2}); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.deletions, 2); + + c = calc({1, 2, 3}, {3, 2, 1}); + REQUIRE_INDICES(c.insertions, 0, 1); + REQUIRE_INDICES(c.deletions, 1, 2); + + c = calc({1, 3, 2}, {1, 2, 3}); + REQUIRE_INDICES(c.insertions, 1); + REQUIRE_INDICES(c.deletions, 2); + + c = calc({1, 3, 2}, {1, 3, 2}); + REQUIRE(c.insertions.empty()); + REQUIRE(c.deletions.empty()); + + c = calc({1, 3, 2}, {2, 1, 3}); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.deletions, 2); + + c = calc({1, 3, 2}, {2, 3, 1}); + REQUIRE_INDICES(c.insertions, 0, 1); + REQUIRE_INDICES(c.deletions, 1, 2); + + c = calc({1, 3, 2}, {3, 1, 2}); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.deletions, 1); + + c = calc({1, 3, 2}, {3, 2, 1}); + REQUIRE_INDICES(c.insertions, 2); + REQUIRE_INDICES(c.deletions, 0); + + c = calc({2, 1, 3}, {1, 2, 3}); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.deletions, 1); + + c = calc({2, 1, 3}, {1, 3, 2}); + REQUIRE_INDICES(c.insertions, 2); + REQUIRE_INDICES(c.deletions, 0); + + c = calc({2, 1, 3}, {2, 1, 3}); + REQUIRE(c.insertions.empty()); + REQUIRE(c.deletions.empty()); + + c = calc({2, 1, 3}, {2, 3, 1}); + REQUIRE_INDICES(c.insertions, 1); + REQUIRE_INDICES(c.deletions, 2); + + c = calc({2, 1, 3}, {3, 1, 2}); + REQUIRE_INDICES(c.insertions, 0, 1); + REQUIRE_INDICES(c.deletions, 1, 2); + + c = calc({2, 1, 3}, {3, 2, 1}); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.deletions, 2); + + c = calc({2, 3, 1}, {1, 2, 3}); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.deletions, 2); + + c = calc({2, 3, 1}, {1, 3, 2}); + REQUIRE_INDICES(c.insertions, 0, 1); + REQUIRE_INDICES(c.deletions, 1, 2); + + c = calc({2, 3, 1}, {2, 1, 3}); + REQUIRE_INDICES(c.insertions, 1); + REQUIRE_INDICES(c.deletions, 2); + + c = calc({2, 3, 1}, {2, 3, 1}); + REQUIRE(c.insertions.empty()); + REQUIRE(c.deletions.empty()); + + c = calc({2, 3, 1}, {3, 1, 2}); + REQUIRE_INDICES(c.insertions, 2); + REQUIRE_INDICES(c.deletions, 0); + + c = calc({2, 3, 1}, {3, 2, 1}); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.deletions, 1); + + c = calc({3, 1, 2}, {1, 2, 3}); + REQUIRE_INDICES(c.insertions, 2); + REQUIRE_INDICES(c.deletions, 0); + + c = calc({3, 1, 2}, {1, 3, 2}); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.deletions, 1); + + c = calc({3, 1, 2}, {2, 1, 3}); + REQUIRE_INDICES(c.insertions, 0, 1); + REQUIRE_INDICES(c.deletions, 1, 2); + + c = calc({3, 1, 2}, {2, 3, 1}); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.deletions, 2); + + c = calc({3, 1, 2}, {3, 1, 2}); + REQUIRE(c.insertions.empty()); + REQUIRE(c.deletions.empty()); + + c = calc({3, 1, 2}, {3, 2, 1}); + REQUIRE_INDICES(c.insertions, 1); + REQUIRE_INDICES(c.deletions, 2); + + c = calc({3, 2, 1}, {1, 2, 3}); + REQUIRE_INDICES(c.insertions, 0, 1); + REQUIRE_INDICES(c.deletions, 1, 2); + + c = calc({3, 2, 1}, {1, 3, 2}); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.deletions, 2); + + c = calc({3, 2, 1}, {2, 1, 3}); + REQUIRE_INDICES(c.insertions, 2); + REQUIRE_INDICES(c.deletions, 0); + + c = calc({3, 2, 1}, {2, 3, 1}); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.deletions, 1); + + c = calc({3, 2, 1}, {3, 1, 2}); + REQUIRE_INDICES(c.insertions, 1); + REQUIRE_INDICES(c.deletions, 2); + + c = calc({3, 2, 1}, {3, 2, 1}); + REQUIRE(c.insertions.empty()); + REQUIRE(c.deletions.empty()); + } + + SECTION("prefers to produce diffs where modified rows are the ones to move when it is ambiguous") { + auto two_modified = [](size_t ndx) { return ndx == 2; }; + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3, 2}, two_modified, true); + REQUIRE_INDICES(c.deletions, 1); + REQUIRE_INDICES(c.insertions, 2); + + auto three_modified = [](size_t ndx) { return ndx == 3; }; + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3, 2}, three_modified, true); + REQUIRE_INDICES(c.deletions, 2); + REQUIRE_INDICES(c.insertions, 1); + } + + SECTION("prefers smaller diffs over larger diffs moving only modified rows") { + auto two_modified = [](size_t ndx) { return ndx == 2; }; + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {2, 3, 1}, two_modified, true); + REQUIRE_INDICES(c.deletions, 0); + REQUIRE_INDICES(c.insertions, 2); + } + + SECTION("supports duplicate indices") { + c = _impl::CollectionChangeBuilder::calculate({1, 1, 2, 2, 3, 3}, + {1, 2, 3, 1, 2, 3}, + all_modified, true); + REQUIRE_INDICES(c.deletions, 3, 5); + REQUIRE_INDICES(c.insertions, 1, 2); + } + + SECTION("deletes and inserts the last option when any in a range could be deleted") { + c = _impl::CollectionChangeBuilder::calculate({3, 2, 1, 1, 2, 3}, + {1, 1, 2, 2, 3, 3}, + all_modified, true); + REQUIRE_INDICES(c.deletions, 0, 1); + REQUIRE_INDICES(c.insertions, 3, 5); + } + + SECTION("reports insertions/deletions when the number of duplicate entries changes") { + c = _impl::CollectionChangeBuilder::calculate({1, 1, 1, 1, 2, 3}, + {1, 2, 3, 1}, + all_modified, true); + REQUIRE_INDICES(c.deletions, 1, 2, 3); + REQUIRE_INDICES(c.insertions, 3); + + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3, 1}, + {1, 1, 1, 1, 2, 3}, + all_modified, true); + REQUIRE_INDICES(c.deletions, 3); + REQUIRE_INDICES(c.insertions, 1, 2, 3); + } + + SECTION("properly recurses into smaller subblocks") { + std::vector prev = {10, 1, 2, 11, 3, 4, 5, 12, 6, 7, 13}; + std::vector next = {13, 1, 2, 12, 3, 4, 5, 11, 6, 7, 10}; + c = _impl::CollectionChangeBuilder::calculate(prev, next, all_modified, true); + REQUIRE_INDICES(c.deletions, 0, 3, 7, 10); + REQUIRE_INDICES(c.insertions, 0, 3, 7, 10); + } + + SECTION("produces diffs which let merge collapse insert -> move -> delete to no-op") { + auto four_modified = [](size_t ndx) { return ndx == 4; }; + for (int insert_pos = 0; insert_pos < 4; ++insert_pos) { + for (int move_to_pos = 0; move_to_pos < 4; ++move_to_pos) { + if (insert_pos == move_to_pos) + continue; + CAPTURE(insert_pos); + CAPTURE(move_to_pos); + + std::vector after_insert = {1, 2, 3}; + after_insert.insert(after_insert.begin() + insert_pos, 4); + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, after_insert, four_modified, true); + + std::vector after_move = {1, 2, 3}; + after_move.insert(after_move.begin() + move_to_pos, 4); + c.merge(_impl::CollectionChangeBuilder::calculate(after_insert, after_move, four_modified, true)); + + c.merge(_impl::CollectionChangeBuilder::calculate(after_move, {1, 2, 3}, four_modified, true)); + REQUIRE(c.empty()); } } } +} - SECTION("merge") { - SECTION("deletions are shifted by previous deletions") { - c = {{5}, {}, {}, {}}; - c.merge({{3}, {}, {}, {}}); - REQUIRE_INDICES(c.deletions, 3, 5); +TEST_CASE("[collection_change] merge()") { + _impl::CollectionChangeBuilder c; - c = {{5}, {}, {}, {}}; - c.merge({{4}, {}, {}, {}}); - REQUIRE_INDICES(c.deletions, 4, 5); + SECTION("is a no-op if the new set is empty") { + c = {{1, 2, 3}, {4, 5}, {6, 7}, {{8, 9}}}; + c.merge({}); + REQUIRE_INDICES(c.deletions, 1, 2, 3, 8); + REQUIRE_INDICES(c.insertions, 4, 5, 9); + REQUIRE_INDICES(c.modifications, 6, 7); + REQUIRE_MOVES(c, {8, 9}); + } - c = {{5}, {}, {}, {}}; - c.merge({{5}, {}, {}, {}}); - REQUIRE_INDICES(c.deletions, 5, 6); + SECTION("replaces the set with the new set if the old set is empty") { + c.merge({{1, 2, 3}, {4, 5}, {6, 7}, {{8, 9}}}); + REQUIRE_INDICES(c.deletions, 1, 2, 3, 8); + REQUIRE_INDICES(c.insertions, 4, 5, 9); + REQUIRE_INDICES(c.modifications, 6, 7); + REQUIRE_MOVES(c, {8, 9}); + } - c = {{5}, {}, {}, {}}; - c.merge({{6}, {}, {}, {}}); - REQUIRE_INDICES(c.deletions, 5, 7); - } + SECTION("shifts deletions by previous deletions") { + c = {{5}, {}, {}, {}}; + c.merge({{3}, {}, {}, {}}); + REQUIRE_INDICES(c.deletions, 3, 5); - SECTION("deletions are shifted by previous insertions") { - c = {{}, {5}, {}, {}}; - c.merge({{4}, {}, {}, {}}); - REQUIRE_INDICES(c.deletions, 4); + c = {{5}, {}, {}, {}}; + c.merge({{4}, {}, {}, {}}); + REQUIRE_INDICES(c.deletions, 4, 5); - c = {{}, {5}, {}, {}}; - c.merge({{6}, {}, {}, {}}); - REQUIRE_INDICES(c.deletions, 5); - } + c = {{5}, {}, {}, {}}; + c.merge({{5}, {}, {}, {}}); + REQUIRE_INDICES(c.deletions, 5, 6); - SECTION("deletions shift previous insertions") { - c = {{}, {2, 3}, {}, {}}; - c.merge({{1}, {}, {}, {}}); - REQUIRE_INDICES(c.insertions, 1, 2); - } + c = {{5}, {}, {}, {}}; + c.merge({{6}, {}, {}, {}}); + REQUIRE_INDICES(c.deletions, 5, 7); + } - SECTION("deletions remove previous insertions") { - c = {{}, {1, 2}, {}, {}}; - c.merge({{2}, {}, {}, {}}); - REQUIRE_INDICES(c.insertions, 1); - } + SECTION("shifts deletions by previous insertions") { + c = {{}, {5}, {}, {}}; + c.merge({{4}, {}, {}, {}}); + REQUIRE_INDICES(c.deletions, 4); - SECTION("deletions remove previous modifications") { - c = {{}, {}, {2, 3}, {}}; - c.merge({{2}, {}, {}, {}}); - REQUIRE_INDICES(c.modifications, 2); - } + c = {{}, {5}, {}, {}}; + c.merge({{6}, {}, {}, {}}); + REQUIRE_INDICES(c.deletions, 5); + } - SECTION("deletions shift previous modifications") { - c = {{}, {}, {2, 3}, {}}; - c.merge({{1}, {}, {}, {}}); - REQUIRE_INDICES(c.modifications, 1, 2); - } + SECTION("shifts previous insertions by deletions") { + c = {{}, {2, 3}, {}, {}}; + c.merge({{1}, {}, {}, {}}); + REQUIRE_INDICES(c.insertions, 1, 2); + } - SECTION("deletions remove previous moves to deleted row") { - c = {{}, {}, {}, {{2, 3}}}; - c.merge({{3}, {}, {}, {}}); - REQUIRE(c.moves.empty()); - } + SECTION("removes previous insertions for newly deleted rows") { + c = {{}, {1, 2}, {}, {}}; + c.merge({{2}, {}, {}, {}}); + REQUIRE_INDICES(c.insertions, 1); + } - SECTION("deletions shift destination of previous moves to after the deleted row") { - c = {{}, {}, {}, {{2, 5}}}; - c.merge({{3}, {}, {}, {}}); - REQUIRE_MOVES(c, {2, 4}); - } + SECTION("removes previous modifications for newly deleted rows") { + c = {{}, {}, {2, 3}, {}}; + c.merge({{2}, {}, {}, {}}); + REQUIRE_INDICES(c.modifications, 2); + } - SECTION("insertions do not interact with previous deletions") { - c = {{1, 3}, {}, {}, {}}; - c.merge({{}, {1, 2, 3}, {}, {}}); - REQUIRE_INDICES(c.deletions, 1, 3); - REQUIRE_INDICES(c.insertions, 1, 2, 3); - } + SECTION("shifts previous modifications for deletions of other rows") { + c = {{}, {}, {2, 3}, {}}; + c.merge({{1}, {}, {}, {}}); + REQUIRE_INDICES(c.modifications, 1, 2); + } - SECTION("insertions shift previous insertions") { - c = {{}, {1, 5}, {}, {}}; - c.merge({{}, {1, 4}, {}, {}}); - REQUIRE_INDICES(c.insertions, 1, 2, 4, 7); - } + SECTION("removes moves to rows which have been deleted") { + c = {{}, {}, {}, {{2, 3}}}; + c.merge({{3}, {}, {}, {}}); + REQUIRE(c.moves.empty()); + } - SECTION("insertions shift previous modifications") { - c = {{}, {}, {1, 5}, {}}; - c.merge({{}, {1, 4}, {}, {}}); - REQUIRE_INDICES(c.modifications, 2, 7); - REQUIRE_INDICES(c.insertions, 1, 4); - } + SECTION("shifts destinations of previous moves to reflect new deletions") { + c = {{}, {}, {}, {{2, 5}}}; + c.merge({{3}, {}, {}, {}}); + REQUIRE_MOVES(c, {2, 4}); + } - SECTION("insertions shift destination of previous moves") { - c = {{}, {}, {}, {{2, 5}}}; - c.merge({{}, {3}, {}}); - REQUIRE_MOVES(c, {2, 6}); - } + SECTION("does not modify old deletions based on new insertions") { + c = {{1, 3}, {}, {}, {}}; + c.merge({{}, {1, 2, 3}, {}, {}}); + REQUIRE_INDICES(c.deletions, 1, 3); + REQUIRE_INDICES(c.insertions, 1, 2, 3); + } - SECTION("modifications do not interact with previous deletions") { - c = {{1, 2, 3}, {}, {}, {}}; - c.merge({{}, {}, {2}}); - REQUIRE_INDICES(c.deletions, 1, 2, 3); - REQUIRE_INDICES(c.modifications, 2); - } + SECTION("shifts previous insertions to reflect new insertions") { + c = {{}, {1, 5}, {}, {}}; + c.merge({{}, {1, 4}, {}, {}}); + REQUIRE_INDICES(c.insertions, 1, 2, 4, 7); + } - SECTION("modifications are not discarded for previous insertions") { - c = {{}, {2}, {}, {}}; - c.merge({{}, {}, {1, 2, 3}}); - REQUIRE_INDICES(c.insertions, 2); - REQUIRE_INDICES(c.modifications, 1, 2, 3); - } + SECTION("shifts previous modifications to reflect new insertions") { + c = {{}, {}, {1, 5}, {}}; + c.merge({{}, {1, 4}, {}, {}}); + REQUIRE_INDICES(c.modifications, 2, 7); + REQUIRE_INDICES(c.insertions, 1, 4); + } - SECTION("modifications are merged with previous modifications") { - c = {{}, {}, {2}, {}}; - c.merge({{}, {}, {1, 2, 3}}); - REQUIRE_INDICES(c.modifications, 1, 2, 3); - } + SECTION("shifts previous move destinations to reflect new insertions") { + c = {{}, {}, {}, {{2, 5}}}; + c.merge({{}, {3}, {}}); + REQUIRE_MOVES(c, {2, 6}); + } - SECTION("modifications are tracked for the destination of previous moves") { - c = {{}, {}, {}, {{1, 2}}}; - c.merge({{}, {}, {2, 3}}); - REQUIRE_INDICES(c.modifications, 2, 3); - } + SECTION("does not modify old deletions based on new modifications") { + c = {{1, 2, 3}, {}, {}, {}}; + c.merge({{}, {}, {2}}); + REQUIRE_INDICES(c.deletions, 1, 2, 3); + REQUIRE_INDICES(c.modifications, 2); + } - SECTION("move sources are shifted for previous deletes and insertions") { - c = {{1}, {}, {}, {}}; - c.merge({{}, {}, {}, {{2, 3}}}); - REQUIRE_MOVES(c, {3, 3}); + SECTION("tracks modifications made to previously inserted rows") { + c = {{}, {2}, {}, {}}; + c.merge({{}, {}, {1, 2, 3}}); + REQUIRE_INDICES(c.insertions, 2); + REQUIRE_INDICES(c.modifications, 1, 2, 3); + } - c = {{}, {1}, {}, {}}; - c.merge({{}, {}, {}, {{2, 3}}}); - REQUIRE_MOVES(c, {1, 3}); + SECTION("unions modifications with old modifications") { + c = {{}, {}, {2}, {}}; + c.merge({{}, {}, {1, 2, 3}}); + REQUIRE_INDICES(c.modifications, 1, 2, 3); + } - c = {{2}, {4}, {}, {}}; - c.merge({{}, {}, {}, {{5, 10}}}); - REQUIRE_MOVES(c, {5, 10}); - } + SECTION("tracks modifications for previous moves") { + c = {{}, {}, {}, {{1, 2}}}; + c.merge({{}, {}, {2, 3}}); + REQUIRE_INDICES(c.modifications, 2, 3); + } - SECTION("moves update previous modifications to source") { - c = {{}, {}, {1}, {}}; - c.merge({{}, {}, {}, {{1, 3}}}); - REQUIRE_INDICES(c.modifications, 3); - REQUIRE_MOVES(c, {1, 3}); - } + SECTION("updates new move sources to reflect previous inserts and deletes") { + c = {{1}, {}, {}, {}}; + c.merge({{}, {}, {}, {{2, 3}}}); + REQUIRE_MOVES(c, {3, 3}); - SECTION("moves update insertion position for previous inserts of source") { - c = {{}, {1}, {}, {}}; - c.merge({{}, {}, {}, {{1, 3}}}); - REQUIRE(c.moves.empty()); - REQUIRE_INDICES(c.insertions, 3); - } + c = {{}, {1}, {}, {}}; + c.merge({{}, {}, {}, {{2, 3}}}); + REQUIRE_MOVES(c, {1, 3}); - SECTION("moves update previous moves to the source") { - c = {{}, {}, {}, {{1, 3}}}; - c.merge({{}, {}, {}, {{3, 5}}}); - REQUIRE_MOVES(c, {1, 5}); - } + c = {{2}, {4}, {}, {}}; + c.merge({{}, {}, {}, {{5, 10}}}); + REQUIRE_MOVES(c, {5, 10}); + } - SECTION("moves shift destination of previous moves like an insert/delete pair would") { - c = {{}, {}, {}, {{1, 3}}}; - c.merge({{}, {}, {}, {{2, 5}}}); - REQUIRE_MOVES(c, {1, 2}, {3, 5}); + SECTION("updates the row modified for rows moved after a modification") { + c = {{}, {}, {1}, {}}; + c.merge({{}, {}, {}, {{1, 3}}}); + REQUIRE_INDICES(c.modifications, 3); + REQUIRE_MOVES(c, {1, 3}); + } - c = {{}, {}, {}, {{1, 10}}}; - c.merge({{}, {}, {}, {{2, 5}}}); - REQUIRE_MOVES(c, {1, 10}, {3, 5}); + SECTION("updates the row inserted for moves of previously new rows") { + c = {{}, {1}, {}, {}}; + c.merge({{}, {}, {}, {{1, 3}}}); + REQUIRE(c.moves.empty()); + REQUIRE_INDICES(c.insertions, 3); + } - c = {{}, {}, {}, {{5, 10}}}; - c.merge({{}, {}, {}, {{12, 2}}}); - REQUIRE_MOVES(c, {5, 11}, {12, 2}); - } + SECTION("updates old moves when the destination is moved again") { + c = {{}, {}, {}, {{1, 3}}}; + c.merge({{}, {}, {}, {{3, 5}}}); + REQUIRE_MOVES(c, {1, 5}); + } - SECTION("moves shift previous inserts like an insert/delete pair would") { - c = {{}, {5}}; - c.merge({{}, {}, {}, {{2, 6}}}); - REQUIRE_INDICES(c.insertions, 4, 6); - } + SECTION("shifts destination of previous moves to reflect new moves like an insert/delete pair would") { + c = {{}, {}, {}, {{1, 3}}}; + c.merge({{}, {}, {}, {{2, 5}}}); + REQUIRE_MOVES(c, {1, 2}, {3, 5}); - SECTION("moves shift previous modifications like an insert/delete pair would") { - c = {{}, {}, {5}}; - c.merge({{}, {}, {}, {{2, 6}}}); - REQUIRE_INDICES(c.modifications, 4); - } + c = {{}, {}, {}, {{1, 10}}}; + c.merge({{}, {}, {}, {{2, 5}}}); + REQUIRE_MOVES(c, {1, 10}, {3, 5}); - SECTION("moves are shifted by previous deletions like an insert/delete pair would") { - c = {{5}}; - c.merge({{}, {}, {}, {{2, 6}}}); - REQUIRE_MOVES(c, {2, 6}); + c = {{}, {}, {}, {{5, 10}}}; + c.merge({{}, {}, {}, {{12, 2}}}); + REQUIRE_MOVES(c, {5, 11}, {12, 2}); + } - c = {{5}}; - c.merge({{}, {}, {}, {{6, 2}}}); - REQUIRE_MOVES(c, {7, 2}); - } + SECTION("moves shift previous inserts like an insert/delete pair would") { + c = {{}, {5}}; + c.merge({{}, {}, {}, {{2, 6}}}); + REQUIRE_INDICES(c.insertions, 4, 6); + } - SECTION("leapfrogging rows collapse to an empty changeset") { - c = {{1}, {0}, {}, {{1, 0}}}; - c.merge({{1}, {0}, {}, {{1, 0}}}); - REQUIRE(c.empty()); - } + SECTION("moves shift previous modifications like an insert/delete pair would") { + c = {{}, {}, {5}}; + c.merge({{}, {}, {}, {{2, 6}}}); + REQUIRE_INDICES(c.modifications, 4); + } - SECTION("modify -> move -> unmove leaves row marked as modified") { - c = {{}, {}, {1}}; - c.merge({{1}, {2}, {}, {{1, 2}}}); - c.merge({{1}}); + SECTION("moves are shifted by previous deletions like an insert/delete pair would") { + c = {{5}}; + c.merge({{}, {}, {}, {{2, 6}}}); + REQUIRE_MOVES(c, {2, 6}); - REQUIRE_INDICES(c.deletions, 2); - REQUIRE(c.insertions.empty()); - REQUIRE(c.moves.empty()); - REQUIRE_INDICES(c.modifications, 1); - } + c = {{5}}; + c.merge({{}, {}, {}, {{6, 2}}}); + REQUIRE_MOVES(c, {7, 2}); + } - SECTION("modifying a previously moved row which stops being a move due to more deletions") { - // Make it stop being a move in the same transaction as the modify - c = {{1, 2}, {0, 1}, {}, {{1, 0}, {2, 1}}}; - c.merge({{0, 2}, {1}, {0}, {}}); + SECTION("leapfrogging rows collapse to an empty changeset") { + c = {{1}, {0}, {}, {{1, 0}}}; + c.merge({{1}, {0}, {}, {{1, 0}}}); + REQUIRE(c.empty()); + } - REQUIRE_INDICES(c.deletions, 0, 1); - REQUIRE_INDICES(c.insertions, 1); - REQUIRE_INDICES(c.modifications, 0); - REQUIRE(c.moves.empty()); + SECTION("modify -> move -> unmove leaves row marked as modified") { + c = {{}, {}, {1}}; + c.merge({{1}, {2}, {}, {{1, 2}}}); + c.merge({{1}}); - // Same net change, but make it no longer a move in the transaction after the modify - c = {{1, 2}, {0, 1}, {}, {{1, 0}, {2, 1}}}; - c.merge({{}, {}, {1}, {}}); - c.merge({{0, 2}, {0}, {}, {{2, 0}}}); - c.merge({{0}, {1}, {}, {}}); + REQUIRE_INDICES(c.deletions, 2); + REQUIRE(c.insertions.empty()); + REQUIRE(c.moves.empty()); + REQUIRE_INDICES(c.modifications, 1); + } - REQUIRE_INDICES(c.deletions, 0, 1); - REQUIRE_INDICES(c.insertions, 1); - REQUIRE_INDICES(c.modifications, 0); - REQUIRE(c.moves.empty()); - } + SECTION("modifying a previously moved row which stops being a move due to more deletions") { + // Make it stop being a move in the same transaction as the modify + c = {{1, 2}, {0, 1}, {}, {{1, 0}, {2, 1}}}; + c.merge({{0, 2}, {1}, {0}, {}}); + + REQUIRE_INDICES(c.deletions, 0, 1); + REQUIRE_INDICES(c.insertions, 1); + REQUIRE_INDICES(c.modifications, 0); + REQUIRE(c.moves.empty()); + + // Same net change, but make it no longer a move in the transaction after the modify + c = {{1, 2}, {0, 1}, {}, {{1, 0}, {2, 1}}}; + c.merge({{}, {}, {1}, {}}); + c.merge({{0, 2}, {0}, {}, {{2, 0}}}); + c.merge({{0}, {1}, {}, {}}); + + REQUIRE_INDICES(c.deletions, 0, 1); + REQUIRE_INDICES(c.insertions, 1); + REQUIRE_INDICES(c.modifications, 0); + REQUIRE(c.moves.empty()); } } From 47cee90d32f2ccf7be24784c73d27cd1501e212d Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 18 Mar 2016 10:23:26 -0700 Subject: [PATCH 38/93] Refactor the fuzzer a bit --- fuzzer/CMakeLists.txt | 5 +- fuzzer/command_file.cpp | 222 +++++++++++++++++++++++++++++++++++++++ fuzzer/command_file.hpp | 38 +++++++ fuzzer/fuzzer.cpp | 223 +++++++++++----------------------------- fuzzer/input-lv/0 | 38 +++++++ fuzzer/input/0 | 1 + fuzzer/input/1 | 1 + 7 files changed, 362 insertions(+), 166 deletions(-) create mode 100644 fuzzer/command_file.cpp create mode 100644 fuzzer/command_file.hpp create mode 100644 fuzzer/input-lv/0 diff --git a/fuzzer/CMakeLists.txt b/fuzzer/CMakeLists.txt index 65df5418..b646497a 100644 --- a/fuzzer/CMakeLists.txt +++ b/fuzzer/CMakeLists.txt @@ -1,2 +1,5 @@ -add_executable(fuzzer fuzzer.cpp) +add_executable(fuzzer + command_file.hpp + command_file.cpp + fuzzer.cpp) target_link_libraries(fuzzer realm-object-store) diff --git a/fuzzer/command_file.cpp b/fuzzer/command_file.cpp new file mode 100644 index 00000000..28123abe --- /dev/null +++ b/fuzzer/command_file.cpp @@ -0,0 +1,222 @@ +#include "command_file.hpp" + +#include "impl/realm_coordinator.hpp" +#include "shared_realm.hpp" + +#include +#include + +#include + +using namespace fuzzer; +using namespace realm; + +#if 0 +#define log(...) fprintf(stderr, __VA_ARGS__) +#else +#define log(...) +#endif + +template +static T read_value(std::istream& input) +{ + T ret; + input >> ret; + return ret; +} + +template +static auto make_reader(void (*fn)(RealmState&, Args...)) { + return [=](std::istream& input) { + return std::bind(fn, std::placeholders::_1, read_value(input)...); + }; +} + +static void run_add(RealmState& state, int64_t value) +{ + log("add %lld\n", value); + size_t ndx = state.table.add_empty_row(); + state.table.set_int(0, ndx, state.uid++); + state.table.set_int(1, ndx, value); +} + +static void run_modify(RealmState& state, size_t index, int64_t value) +{ + if (index < state.table.size()) { + log("modify %zu %lld\n", index, value); + state.table.set_int(1, index, value); + state.modified.push_back(state.table.get_int(0, index)); + } +} + +static void run_delete(RealmState& state, size_t index) +{ + if (index < state.table.size()) { + log("delete %zu\n", index); + state.table.move_last_over(index); + } +} + +static void run_commit(RealmState& state) +{ + log("commit\n"); + state.realm.commit_transaction(); + state.coordinator.on_change(); + state.realm.begin_transaction(); +} + +static void run_lv_insert(RealmState& state, size_t pos, size_t target) +{ + if (target < state.table.size() && pos <= state.lv->size()) { + log("lv insert %zu %zu\n", pos, target); + state.lv->insert(pos, target); + } +} + +static void run_lv_set(RealmState& state, size_t pos, size_t target) +{ + if (target < state.table.size() && pos < state.lv->size()) { + log("lv set %zu %zu\n", pos, target); + // We can't reliably detect self-assignment for verification, so don't do it + if (state.lv->get(pos).get_index() != target) + state.lv->set(pos, target); + } +} + +static void run_lv_move(RealmState& state, size_t from, size_t to) +{ + if (from < state.lv->size() && to < state.lv->size()) { + log("lv move %zu %zu\n", from, to); + // FIXME: only do the move if it has an effect to avoid getting a + // notification which we weren't expecting. This is really urgh. + for (size_t i = std::min(from, to); i < std::max(from, to); ++i) { + if (state.lv->get(i).get_index() != state.lv->get(i + 1).get_index()) { + state.lv->move(from, to); + break; + } + } + } +} + +static void run_lv_swap(RealmState& state, size_t ndx1, size_t ndx2) +{ + if (ndx1 < state.lv->size() && ndx2 < state.lv->size()) { + log("lv swap %zu %zu\n", ndx1, ndx2); + if (state.lv->get(ndx1).get_index() != state.lv->get(ndx2).get_index()) { + state.lv->swap(ndx1, ndx2); + // FIXME: swap() needs to produce moves so that a pair of swaps can + // be collapsed away. Currently it just marks the rows as modified. + state.modified.push_back(state.lv->get(ndx1).get_int(0)); + state.modified.push_back(state.lv->get(ndx2).get_int(0)); + } + } +} + +static void run_lv_remove(RealmState& state, size_t pos) +{ + if (pos < state.lv->size()) { + log("lv remove %zu\n", pos); + state.lv->remove(pos); + } +} + +static void run_lv_remove_target(RealmState& state, size_t pos) +{ + if (pos < state.lv->size()) { + log("lv target remove %zu\n", pos); + state.lv->remove_target_row(pos); + } +} + +static std::map(std::istream&)>> readers = { + // Row functions + {'a', make_reader(run_add)}, + {'c', make_reader(run_commit)}, + {'d', make_reader(run_delete)}, + {'m', make_reader(run_modify)}, + + // LinkView functions + {'i', make_reader(run_lv_insert)}, + {'s', make_reader(run_lv_set)}, + {'o', make_reader(run_lv_move)}, + {'w', make_reader(run_lv_swap)}, + {'r', make_reader(run_lv_remove)}, + {'t', make_reader(run_lv_remove_target)}, +}; + +template +static std::vector read_int_list(std::istream& input_stream) +{ + std::vector ret; + std::string line; + while (std::getline(input_stream, line) && !line.empty()) { + try { + ret.push_back(std::stoll(line)); + log("%lld\n", (long long)ret.back()); + } + catch (std::invalid_argument) { + // not an error + } + catch (std::out_of_range) { + // not an error + } + } + log("\n"); + return ret; +} + +CommandFile::CommandFile(std::istream& input) +: initial_values(read_int_list(input)) +, initial_list_indices(read_int_list(input)) +{ + if (!input.good()) + return; + + while (input.good()) { + char op = '\0'; + input >> op; + if (!input.good()) + break; + + auto it = readers.find(op); + if (it == readers.end()) + continue; + + auto fn = it->second(input); + if (!input.good()) + return; + commands.push_back(std::move(fn)); + } +} + +void CommandFile::import(RealmState& state) +{ + auto& table = state.table; + + state.realm.begin_transaction(); + + table.clear(); + size_t ndx = table.add_empty_row(initial_values.size()); + for (auto value : initial_values) { + table.set_int(0, ndx, state.uid++); + table.set_int(1, ndx++, value); + } + + state.lv->clear(); + for (auto value : initial_list_indices) { + if (value < table.size()) + state.lv->add(value); + } + + state.realm.commit_transaction(); + +} + +void CommandFile::run(RealmState& state) +{ + state.realm.begin_transaction(); + for (auto& command : commands) { + command(state); + } + state.realm.commit_transaction(); +} diff --git a/fuzzer/command_file.hpp b/fuzzer/command_file.hpp new file mode 100644 index 00000000..f06f5433 --- /dev/null +++ b/fuzzer/command_file.hpp @@ -0,0 +1,38 @@ +#include + +#include +#include +#include +#include + +namespace realm { + class Table; + class LinkView; + class Realm; + namespace _impl { + class RealmCoordinator; + } +} + +namespace fuzzer { +struct RealmState { + realm::Realm& realm; + realm::_impl::RealmCoordinator& coordinator; + + realm::Table& table; + realm::LinkViewRef lv; + int64_t uid = 0; + std::vector modified; +}; + +struct CommandFile { + std::vector initial_values; + std::vector initial_list_indices; + std::vector> commands; + + CommandFile(std::istream& input); + + void import(RealmState& state); + void run(RealmState& state); +}; +} \ No newline at end of file diff --git a/fuzzer/fuzzer.cpp b/fuzzer/fuzzer.cpp index 19a913af..3a6d5648 100644 --- a/fuzzer/fuzzer.cpp +++ b/fuzzer/fuzzer.cpp @@ -1,3 +1,6 @@ +#include "command_file.hpp" + +#include "list.hpp" #include "object_schema.hpp" #include "property.hpp" #include "results.hpp" @@ -10,22 +13,15 @@ #include #include -#include #include -#include #include #include #include using namespace realm; -#define FUZZ_SORTED 1 - -#if 0 -#define log(...) fprintf(stderr, __VA_ARGS__) -#else -#define log(...) -#endif +#define FUZZ_SORTED 0 +#define FUZZ_LINKVIEW 1 // Read from a fd until eof into a string // Needs to use unbuffered i/o to work properly with afl @@ -44,160 +40,32 @@ static void read_all(std::string& buffer, int fd) } } -static std::vector read_initial_values(std::istream& input_stream) +static Query query(fuzzer::RealmState& state) { - std::vector initial_values; - std::string line; - input_stream.seekg(0); - while (std::getline(input_stream, line) && !line.empty()) { - try { - initial_values.push_back(std::stoll(line)); - } - catch (std::invalid_argument) { - // not an error - } - catch (std::out_of_range) { - // not an error - } - } - return initial_values; +#if FUZZ_LINKVIEW + return state.table.where(state.lv); +#else + return state.table.where().greater(1, 100).less(1, 50000); +#endif } -struct Change { - enum class Action { - Commit, - Add, - Modify, - Delete - } action; - - size_t index = npos; - int64_t value = 0; -}; - -static std::vector read_changes(std::istream& input_stream) +static TableView tableview(fuzzer::RealmState& state) { - std::vector ret; - - while (!input_stream.eof()) { - char op = '\0'; - input_stream >> op; - if (!input_stream.good()) - break; - switch (op) { - case 'a': { - int64_t value = 0; - input_stream >> value; - if (input_stream.good()) - ret.push_back({Change::Action::Add, npos, value}); - break; - } - case 'm': { - int64_t value; - size_t ndx; - input_stream >> ndx >> value; - if (input_stream.good()) - ret.push_back({Change::Action::Modify, ndx, value}); - break; - } - case 'd': { - size_t ndx; - input_stream >> ndx; - if (input_stream.good()) - ret.push_back({Change::Action::Delete, ndx, 0}); - break; - } - case 'c': { - ret.push_back({Change::Action::Commit, npos, 0}); - break; - } - default: - return ret; - - } - } - return ret; -} - -static Query query(Table& table) -{ - return table.where().greater(1, 100).less(1, 50000); -} - -static TableView tableview(Table& table) -{ - auto tv = table.where().greater(1, 100).less(1, 50000).find_all(); + auto tv = query(state).find_all(); #if FUZZ_SORTED tv.sort({1, 0}, {true, true}); #endif return tv; } -static int64_t id = 0; - -static void import_initial_values(SharedRealm& r, std::vector& initial_values) -{ - auto& table = *r->read_group()->get_table("class_object"); - - r->begin_transaction(); - table.clear(); - size_t ndx = table.add_empty_row(initial_values.size()); - for (auto value : initial_values) { - table.set_int(0, ndx, id++); - table.set_int(1, ndx++, value); - log("%lld\n", value); - } - r->commit_transaction(); -} - // Apply the changes from the command file and then return whether a change // notification should occur -static bool apply_changes(Realm& r, std::istream& input_stream) +static bool apply_changes(fuzzer::CommandFile& commands, fuzzer::RealmState& state) { - auto& table = *r.read_group()->get_table("class_object"); - auto tv = tableview(table); + auto tv = tableview(state); + commands.run(state); - std::vector modified; - - log("\n"); - r.begin_transaction(); - for (auto const& change : read_changes(input_stream)) { - switch (change.action) { - case Change::Action::Commit: - log("c\n"); - r.commit_transaction(); - _impl::RealmCoordinator::get_existing_coordinator(r.config().path)->on_change(); - r.begin_transaction(); - break; - - case Change::Action::Add: { - log("a %lld\n", change.value); - size_t ndx = table.add_empty_row(); - table.set_int(0, ndx, id++); - table.set_int(1, ndx, change.value); - break; - } - - case Change::Action::Modify: - if (change.index < table.size()) { - log("m %zu %lld\n", change.index, change.value); - modified.push_back(table.get_int(0, change.index)); - table.set_int(1, change.index, change.value); - } - break; - - case Change::Action::Delete: - if (change.index < table.size()) { - log("d %zu\n", change.index); - table.move_last_over(change.index); - } - break; - } - } - r.commit_transaction(); - log("\n"); - - auto tv2 = tableview(table); + auto tv2 = tableview(state); if (tv.size() != tv2.size()) return true; @@ -206,16 +74,16 @@ static bool apply_changes(Realm& r, std::istream& input_stream) return true; if (tv.get_int(0, i) != tv2.get_int(0, i)) return true; - if (find(begin(modified), end(modified), tv.get_int(0, i)) != end(modified)) + if (find(begin(state.modified), end(state.modified), tv.get_int(0, i)) != end(state.modified)) return true; } return false; } -static void verify(CollectionChangeIndices const& changes, std::vector values, Table& table) +static void verify(CollectionChangeIndices const& changes, std::vector values, fuzzer::RealmState& state) { - auto tv = tableview(table); + auto tv = tableview(state); // Apply the changes from the transaction log to our copy of the // initial, using UITableView's batching rules (i.e. delete, then @@ -253,20 +121,41 @@ static void verify(CollectionChangeIndices const& changes, std::vector static void test(Realm::Config const& config, SharedRealm& r, SharedRealm& r2, std::istream& input_stream) { - std::vector initial_values = read_initial_values(input_stream); - if (initial_values.empty()) { + fuzzer::RealmState state = { + *r, + *_impl::RealmCoordinator::get_existing_coordinator(r->config().path), + *r->read_group()->get_table("class_object"), + r->read_group()->get_table("class_linklist")->get_linklist(0, 0), + 0, + {} + }; + + fuzzer::CommandFile command(input_stream); + if (command.initial_values.empty()) { return; } - import_initial_values(r, initial_values); + command.import(state); - auto& table = *r->read_group()->get_table("class_object"); - auto results = Results(r, ObjectSchema(), query(table)) + fuzzer::RealmState state2 = { + *r2, + state.coordinator, + *r2->read_group()->get_table("class_object"), + r2->read_group()->get_table("class_linklist")->get_linklist(0, 0), + state.uid, + {} + }; + +#if FUZZ_LINKVIEW && !FUZZ_SORTED + auto results = List(r, ObjectSchema(), state.lv); +#else + auto results = Results(r, ObjectSchema(), query(state)) #if FUZZ_SORTED .sort({{1, 0}, {true, true}}) #endif ; +#endif // FUZZ_LINKVIEW - initial_values.clear(); + std::vector initial_values; for (size_t i = 0; i < results.size(); ++i) initial_values.push_back(results.get(i).get_int(1)); @@ -279,20 +168,19 @@ static void test(Realm::Config const& config, SharedRealm& r, SharedRealm& r2, s ++notification_calls; }); - auto& coordinator = *_impl::RealmCoordinator::get_existing_coordinator(config.path); - coordinator.on_change(); r->notify(); + state.coordinator.on_change(); r->notify(); if (notification_calls != 1) { abort(); } - bool expect_notification = apply_changes(*r2, input_stream); - coordinator.on_change(); r->notify(); + bool expect_notification = apply_changes(command, state2); + state.coordinator.on_change(); r->notify(); if (notification_calls != 1 + expect_notification) { abort(); } - verify(changes, initial_values, table); + verify(changes, initial_values, state); } int main(int argc, char** argv) { @@ -311,6 +199,9 @@ int main(int argc, char** argv) { {"object", "", { {"id", PropertyTypeInt}, {"value", PropertyTypeInt} + }}, + {"linklist", "", { + {"list", PropertyTypeArray, "object"} }} }; @@ -321,9 +212,11 @@ int main(int argc, char** argv) { auto r2 = Realm::get_shared_realm(config); auto& coordinator = *_impl::RealmCoordinator::get_existing_coordinator(config.path); - auto test_on = [&](auto& buffer) { - id = 0; + r->begin_transaction(); + r->read_group()->get_table("class_linklist")->add_empty_row(); + r->commit_transaction(); + auto test_on = [&](auto& buffer) { std::istringstream ss(buffer); test(config, r, r2, ss); if (r->is_in_transaction()) diff --git a/fuzzer/input-lv/0 b/fuzzer/input-lv/0 new file mode 100644 index 00000000..cb6a3bd9 --- /dev/null +++ b/fuzzer/input-lv/0 @@ -0,0 +1,38 @@ +3 +100 +200 +400 +1000 +2000 +50 +80 +150 +180 +6000 +5000 +60000 + +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 + +a 500 +d 12 +c +m 11 10000 +a 800 +i 5 13 +s 3 8 +o 2 10 +w 1 6 +r 7 +t 11 diff --git a/fuzzer/input/0 b/fuzzer/input/0 index 50132fd0..675ab157 100644 --- a/fuzzer/input/0 +++ b/fuzzer/input/0 @@ -12,6 +12,7 @@ 5000 60000 + a 500 d 12 c diff --git a/fuzzer/input/1 b/fuzzer/input/1 index 483456fa..1cbf4855 100644 --- a/fuzzer/input/1 +++ b/fuzzer/input/1 @@ -12,6 +12,7 @@ 112 113 + a 114 a 115 a 116 From 1289c4806c4c76072c8367c3de6332b3f0c37021 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 21 Mar 2016 10:39:23 -0700 Subject: [PATCH 39/93] Fix handling of move_last_over() on the second-to-last row --- src/collection_notifications.cpp | 2 +- tests/collection_change_indices.cpp | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 110d947f..3fd03df7 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -263,7 +263,7 @@ void CollectionChangeBuilder::move(size_t from, size_t to) void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) { REALM_ASSERT(row_ndx <= last_row); - if (row_ndx == last_row) { + if (row_ndx == last_row || row_ndx + 1 == last_row) { erase(row_ndx); return; } diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index c4fbafb7..8526fd07 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -126,6 +126,17 @@ TEST_CASE("[collection_change] move_over()") { REQUIRE(c.moves.empty()); } + SECTION("is just erase when row + 1 == last_row") { + c.move_over(0, 6); + c.move_over(4, 5); + c.move_over(0, 4); + c.move_over(2, 3); + + REQUIRE_INDICES(c.deletions, 0, 2, 4, 5, 6); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_MOVES(c, {5, 0}); + } + SECTION("marks the old last row as moved") { c.move_over(5, 8); REQUIRE_MOVES(c, {8, 5}); From 51edc6909ce99b1fc5922df1d491ec65305943f3 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 21 Mar 2016 10:46:43 -0700 Subject: [PATCH 40/93] Log more in the fuzzer --- fuzzer/command_file.cpp | 2 +- fuzzer/fuzzer.cpp | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/fuzzer/command_file.cpp b/fuzzer/command_file.cpp index 28123abe..e3f836ae 100644 --- a/fuzzer/command_file.cpp +++ b/fuzzer/command_file.cpp @@ -52,7 +52,7 @@ static void run_modify(RealmState& state, size_t index, int64_t value) static void run_delete(RealmState& state, size_t index) { if (index < state.table.size()) { - log("delete %zu\n", index); + log("delete %zu (%lld)\n", index, state.table.get_int(1, index)); state.table.move_last_over(index); } } diff --git a/fuzzer/fuzzer.cpp b/fuzzer/fuzzer.cpp index 3a6d5648..b1c9aebb 100644 --- a/fuzzer/fuzzer.cpp +++ b/fuzzer/fuzzer.cpp @@ -22,6 +22,7 @@ using namespace realm; #define FUZZ_SORTED 0 #define FUZZ_LINKVIEW 1 +#define FUZZ_LOG 0 // Read from a fd until eof into a string // Needs to use unbuffered i/o to work properly with afl @@ -63,6 +64,11 @@ static TableView tableview(fuzzer::RealmState& state) static bool apply_changes(fuzzer::CommandFile& commands, fuzzer::RealmState& state) { auto tv = tableview(state); +#if FUZZ_LOG + for (size_t i = 0; i < tv.size(); ++i) + fprintf(stderr, "pre: %lld\n", tv.get_int(0, i)); +#endif + commands.run(state); auto tv2 = tableview(state); @@ -70,6 +76,9 @@ static bool apply_changes(fuzzer::CommandFile& commands, fuzzer::RealmState& sta return true; for (size_t i = 0; i < tv.size(); ++i) { +#if FUZZ_LOG + fprintf(stderr, "%lld %lld\n", tv.get_int(0, i), tv2.get_int(0, i)); +#endif if (!tv.is_row_attached(i)) return true; if (tv.get_int(0, i) != tv2.get_int(0, i)) @@ -114,6 +123,9 @@ static void verify(CollectionChangeIndices const& changes, std::vector for (size_t i = 0; i < values.size(); ++i) { if (values[i] != tv.get_int(1, i)) { +#if FUZZ_LOG + fprintf(stderr, "%lld %lld\n", values[i], tv.get_int(1, i)); +#endif abort(); } } From 0a3158ce741989c1e182529fe46d17194e2d82c6 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 21 Mar 2016 11:15:29 -0700 Subject: [PATCH 41/93] Fix marking deletions when chaining move_last_over() --- src/collection_notifications.cpp | 14 ++++++++++++-- tests/collection_change_indices.cpp | 9 +++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 3fd03df7..5e95e6be 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -292,8 +292,17 @@ void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) move.to = row_ndx; updated_existing_move = true; - insertions.erase_at(last_row); - insertions.insert_at(row_ndx); + if (!insertions.empty()) { + REALM_ASSERT(std::prev(insertions.end())->second - 1 <= last_row); + insertions.remove(last_row); + } + + // Don't mark the moved-over row as deleted if it was a new insertion + if (!insertions.contains(row_ndx)) { + deletions.add_shifted(insertions.unshift(row_ndx)); + insertions.add(row_ndx); + } + // Because this is a move, the unshifted source row has already been marked as deleted } @@ -312,6 +321,7 @@ void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) deletions.add_shifted(insertions.unshift(row_ndx)); insertions.add(row_ndx); } + verify(); } void CollectionChangeBuilder::verify() diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index 8526fd07..fcf15e75 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -185,6 +185,15 @@ TEST_CASE("[collection_change] move_over()") { REQUIRE_INDICES(c.insertions, 5, 6); REQUIRE_MOVES(c, {10, 5}, {9, 6}); } + + SECTION("marks the moved-over row as deleted when chaining moves") { + c.move_over(5, 10); + c.move_over(0, 5); + + REQUIRE_INDICES(c.deletions, 0, 5, 10); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_MOVES(c, {10, 0}); + } } TEST_CASE("[collection_change] clear()") { From feed7c3479e05fef1c34d7de2587747c2e287174 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 21 Mar 2016 12:47:21 -0700 Subject: [PATCH 42/93] Update moves when there is another move to exactly the previous move target --- src/collection_notifications.cpp | 2 +- tests/collection_change_indices.cpp | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 5e95e6be..cec46d73 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -226,7 +226,7 @@ void CollectionChangeBuilder::move(size_t from, size_t to) // to the other if (move.to >= to && move.to < from) ++move.to; - else if (move.to < to && move.to > from) + else if (move.to <= to && move.to > from) --move.to; continue; } diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index fcf15e75..dacaefad 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -284,6 +284,17 @@ TEST_CASE("[collection_change] move()") { c.move(6, 12); REQUIRE_INDICES(c.modifications, 9); } + + SECTION("bumps previous moves to the same location") { + c.move(5, 10); + c.move(7, 10); + REQUIRE_MOVES(c, {5, 9}, {8, 10}); + + c = {}; + c.move(5, 10); + c.move(15, 10); + REQUIRE_MOVES(c, {5, 11}, {15, 10}); + } } TEST_CASE("[collection_change] calculate() unsorted") { From ef632804ef282e8dc7d0ac5ea7ad7143e6613229 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 21 Mar 2016 13:03:10 -0700 Subject: [PATCH 43/93] Clean up stale moves for linkviews even without a merge --- src/collection_notifications.cpp | 26 +++++++++++++++----------- src/collection_notifications.hpp | 1 + src/impl/list_notifier.cpp | 1 - src/impl/transact_log_handler.cpp | 8 ++++++++ tests/collection_change_indices.cpp | 7 +++++++ 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index cec46d73..59c835ad 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -146,17 +146,7 @@ void CollectionChangeBuilder::merge(CollectionChangeBuilder&& c) insertions.erase_at(c.deletions); insertions.insert_at(c.insertions); - // Look for moves which are now no-ops, and remove them plus the associated - // insert+delete. Note that this isn't just checking for from == to due to - // that rows can also be shifted by other inserts and deletes - IndexSet to_remove; - moves.erase(remove_if(begin(moves), end(moves), [&](auto const& move) { - if (move.from - deletions.count(0, move.from) != move.to - insertions.count(0, move.to)) - return false; - deletions.remove(move.from); - insertions.remove(move.to); - return true; - }), end(moves)); + clean_up_stale_moves(); modifications.erase_at(c.deletions); modifications.shift_for_insert_at(c.insertions); @@ -166,6 +156,20 @@ void CollectionChangeBuilder::merge(CollectionChangeBuilder&& c) verify(); } +void CollectionChangeBuilder::clean_up_stale_moves() +{ + // Look for moves which are now no-ops, and remove them plus the associated + // insert+delete. Note that this isn't just checking for from == to due to + // that rows can also be shifted by other inserts and deletes + moves.erase(remove_if(begin(moves), end(moves), [&](auto const& move) { + if (move.from - deletions.count(0, move.from) != move.to - insertions.count(0, move.to)) + return false; + deletions.remove(move.from); + insertions.remove(move.to); + return true; + }), end(moves)); +} + void CollectionChangeBuilder::modify(size_t ndx) { modifications.add(ndx); diff --git a/src/collection_notifications.hpp b/src/collection_notifications.hpp index 95c1fe04..019ee907 100644 --- a/src/collection_notifications.hpp +++ b/src/collection_notifications.hpp @@ -86,6 +86,7 @@ public: bool sort); void merge(CollectionChangeBuilder&&); + void clean_up_stale_moves(); void insert(size_t ndx, size_t count=1); void modify(size_t ndx); diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp index fd6a4ffe..5d69f4e3 100644 --- a/src/impl/list_notifier.cpp +++ b/src/impl/list_notifier.cpp @@ -101,7 +101,6 @@ void ListNotifier::run() end(m_change.moves)); for (size_t i = 0; i < m_lv->size(); ++i) { - // FIXME: may need to mark modifications even for inserts (for moves?) if (m_change.insertions.contains(i) || m_change.modifications.contains(i)) continue; if (m_info->row_did_change(m_lv->get_target_table(), m_lv->get(i).get_index())) diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index 19617c14..87c18c25 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -445,6 +445,14 @@ public: change->modify(row); } + void parse_complete() + { + for (auto& list : m_info.lists) + { + list.changes->clean_up_stale_moves(); + } + } + bool select_link_list(size_t col, size_t row, size_t) { mark_dirty(row, col); diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index dacaefad..2fac5fa9 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -295,6 +295,13 @@ TEST_CASE("[collection_change] move()") { c.move(15, 10); REQUIRE_MOVES(c, {5, 11}, {15, 10}); } + + SECTION("collapses redundant swaps of adjacent rows to a no-op") { + c.move(7, 8); + c.move(7, 8); + c.clean_up_stale_moves(); + REQUIRE(c.empty()); + } } TEST_CASE("[collection_change] calculate() unsorted") { From a0b16305c9413ff35b588f542cc7b4269ed5e4bc Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 21 Mar 2016 14:20:20 -0700 Subject: [PATCH 44/93] Fix dangling pointers to a reallocated vector when there are multiple source versions for handover --- src/impl/realm_coordinator.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 5b60ab4b..4425d2d3 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -331,8 +331,6 @@ void RealmCoordinator::run_async_notifiers() change_info.resize(1); } else { - change_info.resize(2); - // Sort newly added notifiers by their source version so that we can pull them // all forward to the latest version in a single pass over the transaction log std::sort(new_notifiers.begin(), new_notifiers.end(), @@ -340,6 +338,18 @@ void RealmCoordinator::run_async_notifiers() version = m_advancer_sg->get_version_of_current_transaction(); REALM_ASSERT(version == new_notifiers.front()->version()); + // Preallocate the required amount of space in the vector so that we can + // safely give out pointers to within the vector + { + size_t count = 2; + for (auto it = new_notifiers.begin(), next = it + 1; next != new_notifiers.end(); ++it, ++next) { + if ((*it)->version() != (*next)->version()) + ++count; + } + change_info.reserve(count); + change_info.resize(2); + } + TransactionChangeInfo* info = &change_info.back(); // Advance each of the new notifiers to the latest version, attaching them From 7a74a2255817be15bdcecccdcf300996e7d92374 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 21 Mar 2016 15:02:53 -0700 Subject: [PATCH 45/93] Fix tracking of modifications after linkview moves --- src/collection_notifications.cpp | 2 ++ src/impl/list_notifier.cpp | 13 ++++++++----- tests/collection_change_indices.cpp | 8 ++++++++ tests/list.cpp | 22 ++++++++++++++++++++++ 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 59c835ad..0457211d 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -94,6 +94,8 @@ void CollectionChangeBuilder::merge(CollectionChangeBuilder&& c) return old.to == m.from; }); if (it != c.moves.end()) { + if (modifications.contains(it->from)) + c.modifications.add(it->to); old.to = it->to; *it = c.moves.back(); c.moves.pop_back(); diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp index 5d69f4e3..aba0244a 100644 --- a/src/impl/list_notifier.cpp +++ b/src/impl/list_notifier.cpp @@ -96,17 +96,20 @@ void ListNotifier::run() return; } - m_change.moves.erase(remove_if(begin(m_change.moves), end(m_change.moves), - [&](auto move) { return m_change.modifications.contains(move.to); }), - end(m_change.moves)); - for (size_t i = 0; i < m_lv->size(); ++i) { - if (m_change.insertions.contains(i) || m_change.modifications.contains(i)) + if (m_change.modifications.contains(i)) continue; if (m_info->row_did_change(m_lv->get_target_table(), m_lv->get(i).get_index())) m_change.modifications.add(i); } + for (auto const& move : m_change.moves) { + if (m_change.modifications.contains(move.to)) + continue; + if (m_info->row_did_change(m_lv->get_target_table(), m_lv->get(move.to).get_index())) + m_change.modifications.add(move.to); + } + m_prev_size = m_lv->size(); } diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index 2fac5fa9..4c8dd2e6 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -861,6 +861,14 @@ TEST_CASE("[collection_change] merge()") { REQUIRE_MOVES(c, {1, 3}); } + SECTION("updates the row modified for chained moves") { + c = {{}, {}, {1}, {}}; + c.merge({{}, {}, {}, {{1, 3}}}); + c.merge({{}, {}, {}, {{3, 5}}}); + REQUIRE_INDICES(c.modifications, 5); + REQUIRE_MOVES(c, {1, 5}); + } + SECTION("updates the row inserted for moves of previously new rows") { c = {{}, {1}, {}, {}}; c.merge({{}, {}, {}, {{1, 3}}}); diff --git a/tests/list.cpp b/tests/list.cpp index 7686c4ea..5b38a4d4 100644 --- a/tests/list.cpp +++ b/tests/list.cpp @@ -219,6 +219,28 @@ TEST_CASE("list") { REQUIRE_INDICES(changes[i].modifications, 2); } } + + SECTION("modifications are reported for rows that are moved and then moved back in a second transaction") { + auto token = require_change(); + + r->begin_transaction(); + lv->get(5).set_int(0, 10); + lv->get(1).set_int(0, 10); + lv->move(5, 8); + lv->move(1, 2); + r->commit_transaction(); + + coordinator.on_change(); + + write([&]{ + lv->move(8, 5); + }); + + REQUIRE_INDICES(change.deletions, 1); + REQUIRE_INDICES(change.insertions, 2); + REQUIRE_INDICES(change.modifications, 5); + REQUIRE_MOVES(change, {1, 2}); + } } SECTION("sorted add_notification_block()") { From 20d9da973b541c7d87e81ba4cbf01718ab6ea8d7 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 22 Mar 2016 07:57:50 -0700 Subject: [PATCH 46/93] Make List and Results notifications more consistent Deliver the initial results for both of them and include the changeset in the initial delivery for both, rather than having them behave weirdly differently. --- src/impl/background_collection.cpp | 19 ++++++------------- src/impl/background_collection.hpp | 3 +-- src/impl/list_notifier.cpp | 2 +- src/impl/results_notifier.hpp | 2 -- tests/list.cpp | 12 +++++++++--- tests/results.cpp | 4 ++-- 6 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/impl/background_collection.cpp b/src/impl/background_collection.cpp index 62e7fa42..7e623ab6 100644 --- a/src/impl/background_collection.cpp +++ b/src/impl/background_collection.cpp @@ -208,15 +208,8 @@ bool BackgroundCollection::deliver(SharedGroup& sg, std::exception_ptr err) void BackgroundCollection::call_callbacks() { - while (auto cb = next_callback()) { - auto fn = cb->fn; - if (!cb->initial_delivered && should_deliver_initial()) { - cb->initial_delivered = true; - fn({}, m_error); // note: may invalidate `cb` - } - else { - fn(m_changes_to_deliver, m_error); - } + while (auto fn = next_callback()) { + fn(m_changes_to_deliver, m_error); } if (m_error) { @@ -227,17 +220,17 @@ void BackgroundCollection::call_callbacks() } } -BackgroundCollection::Callback* BackgroundCollection::next_callback() +CollectionChangeCallback BackgroundCollection::next_callback() { std::lock_guard callback_lock(m_callback_mutex); for (++m_callback_index; m_callback_index < m_callbacks.size(); ++m_callback_index) { auto& callback = m_callbacks[m_callback_index]; - bool deliver_initial = !callback.initial_delivered && should_deliver_initial(); - if (!m_error && !deliver_initial && m_changes_to_deliver.empty()) { + if (!m_error && callback.initial_delivered && m_changes_to_deliver.empty()) { continue; } - return &callback; + callback.initial_delivered = true; + return callback.fn; } m_callback_index = npos; diff --git a/src/impl/background_collection.hpp b/src/impl/background_collection.hpp index d2058a21..425c7b2e 100644 --- a/src/impl/background_collection.hpp +++ b/src/impl/background_collection.hpp @@ -119,7 +119,6 @@ private: virtual void do_prepare_handover(SharedGroup&) = 0; virtual bool do_deliver(SharedGroup&) { return true; } virtual bool do_add_required_change_info(TransactionChangeInfo&) { return true; } - virtual bool should_deliver_initial() const noexcept { return false; } const std::thread::id m_thread_id = std::this_thread::get_id(); bool is_for_current_thread() const { return m_thread_id == std::this_thread::get_id(); } @@ -158,7 +157,7 @@ private: // remove_callback() updates this when needed size_t m_callback_index = npos; - Callback* next_callback(); + CollectionChangeCallback next_callback(); }; } // namespace _impl diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp index aba0244a..a0143552 100644 --- a/src/impl/list_notifier.cpp +++ b/src/impl/list_notifier.cpp @@ -86,7 +86,7 @@ bool ListNotifier::do_add_required_change_info(TransactionChangeInfo& info) void ListNotifier::run() { - if (!m_lv) { + if (!m_lv || !m_lv->is_attached()) { // LV was deleted, so report all of the rows being removed if this is // the first run after that if (m_prev_size) { diff --git a/src/impl/results_notifier.hpp b/src/impl/results_notifier.hpp index bb4bd024..81f659c4 100644 --- a/src/impl/results_notifier.hpp +++ b/src/impl/results_notifier.hpp @@ -75,8 +75,6 @@ private: void release_data() noexcept override; void do_attach_to(SharedGroup& sg) override; void do_detach_from(SharedGroup& sg) override; - - bool should_deliver_initial() const noexcept override { return true; } }; } // namespace _impl diff --git a/tests/list.cpp b/tests/list.cpp index 5b38a4d4..7db9a953 100644 --- a/tests/list.cpp +++ b/tests/list.cpp @@ -66,15 +66,21 @@ TEST_CASE("list") { }; auto require_change = [&] { - return lst.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { + auto token = lst.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { change = c; }); + advance_and_notify(*r); + return token; }; auto require_no_change = [&] { - return lst.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { - REQUIRE(false); + bool first = true; + auto token = lst.add_notification_callback([&, first](CollectionChangeIndices c, std::exception_ptr err) mutable { + REQUIRE(first); + first = false; }); + advance_and_notify(*r); + return token; }; SECTION("modifying the list sends a change notifications") { diff --git a/tests/results.cpp b/tests/results.cpp index db809758..e0216c59 100644 --- a/tests/results.cpp +++ b/tests/results.cpp @@ -276,9 +276,9 @@ TEST_CASE("Results") { REQUIRE(notification_calls == 1); } - SECTION("the first call of a notification always passes an empty change even if it previously ran for a different callback") { + SECTION("the first call of a notification can include changes if it previously ran for a different callback") { auto token2 = results.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr) { - REQUIRE(c.empty()); + REQUIRE(!c.empty()); }); write([&] { From bab5540cf64ef0c440a14654f1ca45cc34556b55 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 22 Mar 2016 11:37:05 -0700 Subject: [PATCH 47/93] Fix incorrect results when the second-to-last row is deleted --- src/collection_notifications.cpp | 2 +- tests/collection_change_indices.cpp | 1 + tests/results.cpp | 20 ++++++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 0457211d..99c40714 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -269,7 +269,7 @@ void CollectionChangeBuilder::move(size_t from, size_t to) void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) { REALM_ASSERT(row_ndx <= last_row); - if (row_ndx == last_row || row_ndx + 1 == last_row) { + if (row_ndx == last_row) { erase(row_ndx); return; } diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index 4c8dd2e6..d02ed52b 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -131,6 +131,7 @@ TEST_CASE("[collection_change] move_over()") { c.move_over(4, 5); c.move_over(0, 4); c.move_over(2, 3); + c.clean_up_stale_moves(); REQUIRE_INDICES(c.deletions, 0, 2, 4, 5, 6); REQUIRE_INDICES(c.insertions, 0); diff --git a/tests/results.cpp b/tests/results.cpp index e0216c59..fb039d44 100644 --- a/tests/results.cpp +++ b/tests/results.cpp @@ -385,6 +385,26 @@ TEST_CASE("Results") { advance_and_notify(*r); REQUIRE(notification_calls == 2); } + + SECTION("moving a matching row by deleting all other rows") { + r->begin_transaction(); + table->clear(); + table->add_empty_row(2); + table->set_int(0, 0, 15); + table->set_int(0, 1, 5); + r->commit_transaction(); + advance_and_notify(*r); + + write([&] { + table->move_last_over(0); + table->add_empty_row(); + table->set_int(0, 1, 3); + }); + + REQUIRE(notification_calls == 3); + REQUIRE(change.deletions.empty()); + REQUIRE_INDICES(change.insertions, 1); + } } } From 784c34e052ca7a76860c6cfe715c65305c42f7a4 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 22 Mar 2016 16:54:04 -0700 Subject: [PATCH 48/93] Add property.hpp to HEADERS --- src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 967ce511..29db5138 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,6 +21,7 @@ set(HEADERS list.hpp object_schema.hpp object_store.hpp + property.hpp results.hpp schema.hpp shared_realm.hpp From dd336120b2ed0644fe253134383828b13b00a3c1 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 22 Mar 2016 16:40:55 -0700 Subject: [PATCH 49/93] Fix compilation with GCC 4.9 --- fuzzer/command_file.hpp | 2 +- fuzzer/fuzzer.cpp | 5 +++-- src/collection_notifications.cpp | 4 ++-- src/index_set.cpp | 4 +++- src/index_set.hpp | 10 ++++++++++ src/property.hpp | 18 ++++++++++++++++++ src/results.cpp | 2 +- tests/collection_change_indices.cpp | 6 ++++-- tests/transaction_log_parsing.cpp | 5 +++-- tests/util/index_helpers.hpp | 4 ++-- 10 files changed, 47 insertions(+), 13 deletions(-) diff --git a/fuzzer/command_file.hpp b/fuzzer/command_file.hpp index f06f5433..7de23d43 100644 --- a/fuzzer/command_file.hpp +++ b/fuzzer/command_file.hpp @@ -21,7 +21,7 @@ struct RealmState { realm::Table& table; realm::LinkViewRef lv; - int64_t uid = 0; + int64_t uid; std::vector modified; }; diff --git a/fuzzer/fuzzer.cpp b/fuzzer/fuzzer.cpp index b1c9aebb..e3572769 100644 --- a/fuzzer/fuzzer.cpp +++ b/fuzzer/fuzzer.cpp @@ -97,7 +97,8 @@ static void verify(CollectionChangeIndices const& changes, std::vector // Apply the changes from the transaction log to our copy of the // initial, using UITableView's batching rules (i.e. delete, then // insert, then update) - auto it = std::make_reverse_iterator(changes.deletions.end()), end = std::make_reverse_iterator(changes.deletions.begin()); + auto it = util::make_reverse_iterator(changes.deletions.end()); + auto end = util::make_reverse_iterator(changes.deletions.begin()); for (; it != end; ++it) { values.erase(values.begin() + it->first, values.begin() + it->second); } @@ -207,7 +208,7 @@ int main(int argc, char** argv) { config.in_memory = true; config.automatic_change_notifications = false; - Schema schema = { + Schema schema{ {"object", "", { {"id", PropertyTypeInt}, {"value", PropertyTypeInt} diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 99c40714..11abf418 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -601,8 +601,8 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c #ifdef REALM_DEBUG { // Verify that applying the calculated change to prev_rows actually produces next_rows auto rows = prev_rows; - auto it = std::make_reverse_iterator(ret.deletions.end()); - auto end = std::make_reverse_iterator(ret.deletions.begin()); + auto it = util::make_reverse_iterator(ret.deletions.end()); + auto end = util::make_reverse_iterator(ret.deletions.begin()); for (; it != end; ++it) { rows.erase(rows.begin() + it->first, rows.begin() + it->second); } diff --git a/src/index_set.cpp b/src/index_set.cpp index 68fa6b2c..a9c2d5a0 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -20,6 +20,8 @@ #include +#include + using namespace realm; const size_t IndexSet::npos; @@ -138,7 +140,7 @@ void IndexSet::add_shifted_by(IndexSet const& shifted_by, IndexSet const& values copy(old_it, old_end, back_inserter(m_ranges)); } - REALM_ASSERT_DEBUG(std::distance(as_indexes().begin(), as_indexes().end()) == expected); + REALM_ASSERT_DEBUG((size_t)std::distance(as_indexes().begin(), as_indexes().end()) == expected); } void IndexSet::set(size_t len) diff --git a/src/index_set.hpp b/src/index_set.hpp index e58af4b8..944149bc 100644 --- a/src/index_set.hpp +++ b/src/index_set.hpp @@ -155,6 +155,16 @@ private: // Add an index which must be greater than the largest index in the set void add_back(size_t index); }; + +namespace util { +// This was added in C++14 but is missing from libstdc++ 4.9 +template +std::reverse_iterator make_reverse_iterator(Iterator it) +{ + return std::reverse_iterator(it); +} +} // namespace util + } // namespace realm #endif // REALM_INDEX_SET_HPP diff --git a/src/property.hpp b/src/property.hpp index 25a3bda7..7f968bd2 100644 --- a/src/property.hpp +++ b/src/property.hpp @@ -52,6 +52,20 @@ namespace realm { || type == PropertyTypeDate || type == PropertyTypeString; } + +#if __GNUC__ < 5 + // GCC 4.9 does not support C++14 braced-init with NSDMIs + Property(std::string name="", PropertyType type=PropertyTypeInt, std::string object_type="", + bool is_primary=false, bool is_indexed=false, bool is_nullable=false) + : name(std::move(name)) + , type(type) + , object_type(std::move(object_type)) + , is_primary(is_primary) + , is_indexed(is_indexed) + , is_nullable(is_nullable) + { + } +#endif }; static inline const char *string_for_property_type(PropertyType type) { @@ -76,6 +90,10 @@ namespace realm { return "object"; case PropertyTypeArray: return "array"; +#if __GNUC__ + default: + __builtin_unreachable(); +#endif } } } diff --git a/src/results.cpp b/src/results.cpp index 525e4b49..b5048003 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -293,7 +293,7 @@ util::Optional Results::aggregate(size_t column, bool return_none_for_emp return none; return util::Optional(getter(*m_table)); case Mode::LinkView: - m_query = get_query(); + m_query = this->get_query(); m_mode = Mode::Query; REALM_FALLTHROUGH; case Mode::Query: diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index d02ed52b..5d67a5e2 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -4,6 +4,8 @@ #include "util/index_helpers.hpp" +#include + using namespace realm; TEST_CASE("[collection_change] insert()") { @@ -230,10 +232,10 @@ TEST_CASE("[collection_change] clear()") { SECTION("sets deletions SIZE_T_MAX if that if the given previous size") { c.insertions = {1, 2, 3}; - c.clear(SIZE_T_MAX); + c.clear(std::numeric_limits::max()); REQUIRE(c.deletions.size() == 1); REQUIRE(c.deletions.begin()->first == 0); - REQUIRE(c.deletions.begin()->second == SIZE_T_MAX); + REQUIRE(c.deletions.begin()->second == std::numeric_limits::max()); } } diff --git a/tests/transaction_log_parsing.cpp b/tests/transaction_log_parsing.cpp index e8c3ae2a..76a20f46 100644 --- a/tests/transaction_log_parsing.cpp +++ b/tests/transaction_log_parsing.cpp @@ -73,7 +73,8 @@ private: // Apply the changes from the transaction log to our copy of the // initial, using UITableView's batching rules (i.e. delete, then // insert, then update) - auto it = std::make_reverse_iterator(info.deletions.end()), end = std::make_reverse_iterator(info.deletions.begin()); + auto it = util::make_reverse_iterator(info.deletions.end()); + auto end = util::make_reverse_iterator(info.deletions.begin()); for (; it != end; ++it) { m_initial.erase(m_initial.begin() + it->first, m_initial.begin() + it->second); } @@ -92,7 +93,7 @@ private: // and make sure we end up with the same end result REQUIRE(m_initial.size() == m_linkview->size()); - for (auto i = 0; i < m_initial.size(); ++i) + for (size_t i = 0; i < m_initial.size(); ++i) CHECK(m_initial[i] == m_linkview->get(i).get_int(0)); // Verify that everything marked as a move actually is one diff --git a/tests/util/index_helpers.hpp b/tests/util/index_helpers.hpp index 4608932d..5ca6803c 100644 --- a/tests/util/index_helpers.hpp +++ b/tests/util/index_helpers.hpp @@ -4,7 +4,7 @@ auto actual = index_set.as_indexes(); \ INFO("Checking " #index_set); \ REQUIRE(expected.size() == std::distance(actual.begin(), actual.end())); \ - auto begin = actual.begin(), end = actual.end(); \ + auto begin = actual.begin(); \ for (auto index : expected) { \ REQUIRE(*begin++ == index); \ } \ @@ -14,7 +14,7 @@ auto actual = (c); \ std::initializer_list expected = {__VA_ARGS__}; \ REQUIRE(expected.size() == actual.moves.size()); \ - auto begin = actual.moves.begin(), end = actual.moves.end(); \ + auto begin = actual.moves.begin(); \ for (auto move : expected) { \ CHECK(begin->from == move.from); \ CHECK(begin->to == move.to); \ From b81c950056802acad7f57c612d069f0ed7c6d842 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 23 Mar 2016 07:42:28 -0700 Subject: [PATCH 50/93] Put the fuzzer realm file in the working directory mkstemp() eventually becomes a bottleneck due to thousands of stale files from crashes. --- fuzzer/fuzzer.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fuzzer/fuzzer.cpp b/fuzzer/fuzzer.cpp index e3572769..72be0dca 100644 --- a/fuzzer/fuzzer.cpp +++ b/fuzzer/fuzzer.cpp @@ -201,9 +201,7 @@ int main(int argc, char** argv) { realm::disable_sync_to_disk(); Realm::Config config; - config.path = getenv("TMPDIR"); - config.path += "/realm.XXXXXX"; - mktemp(&config.path[0]); + config.path = "fuzzer.realm"; config.cache = false; config.in_memory = true; config.automatic_change_notifications = false; From f051337cd341c7824f04ff947cb6181884549608 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 23 Mar 2016 08:48:07 -0700 Subject: [PATCH 51/93] Rename erase_and_unshift() to erase_or_unshift() --- src/collection_notifications.cpp | 6 +++--- src/index_set.cpp | 2 +- src/index_set.hpp | 3 ++- tests/index_set.cpp | 20 ++++++++++---------- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 11abf418..d060dfdb 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -191,7 +191,7 @@ void CollectionChangeBuilder::insert(size_t index, size_t count) void CollectionChangeBuilder::erase(size_t index) { modifications.erase_at(index); - size_t unshifted = insertions.erase_and_unshift(index); + size_t unshifted = insertions.erase_or_unshift(index); if (unshifted != npos) deletions.add_shifted(unshifted); @@ -247,7 +247,7 @@ void CollectionChangeBuilder::move(size_t from, size_t to) } if (!updated_existing_move) { - auto shifted_from = insertions.erase_and_unshift(from); + auto shifted_from = insertions.erase_or_unshift(from); insertions.insert_at(to); // Don't report deletions/moves for newly inserted rows @@ -316,7 +316,7 @@ void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) return; // Don't report deletions/moves if last_row is newly inserted - auto shifted_last_row = insertions.erase_and_unshift(last_row); + auto shifted_last_row = insertions.erase_or_unshift(last_row); if (shifted_last_row != npos) { shifted_last_row = deletions.add_shifted(shifted_last_row); moves.push_back({shifted_last_row, row_ndx}); diff --git a/src/index_set.cpp b/src/index_set.cpp index a9c2d5a0..4edfe084 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -295,7 +295,7 @@ void IndexSet::erase_at(IndexSet const& positions) add_back(*begin1 - shift); } -size_t IndexSet::erase_and_unshift(size_t index) +size_t IndexSet::erase_or_unshift(size_t index) { auto shifted = index; auto it = m_ranges.begin(), end = m_ranges.end(); diff --git a/src/index_set.hpp b/src/index_set.hpp index 944149bc..924f35e6 100644 --- a/src/index_set.hpp +++ b/src/index_set.hpp @@ -76,7 +76,8 @@ public: void erase_at(size_t index); void erase_at(IndexSet const&); - size_t erase_and_unshift(size_t index); + // If the given index is in the set remove it and return npos; otherwise unshift() it + size_t erase_or_unshift(size_t index); // Remove the indexes at the given index from the set, without shifting void remove(size_t index, size_t count=1); diff --git a/tests/index_set.cpp b/tests/index_set.cpp index db0d9944..05c4bbf2 100644 --- a/tests/index_set.cpp +++ b/tests/index_set.cpp @@ -408,34 +408,34 @@ TEST_CASE("[index_set] erase_at()") { } } -TEST_CASE("[index_set] erase_and_unshfit()") { +TEST_CASE("[index_set] erase_or_unshift()") { realm::IndexSet set; SECTION("removes the given index") { set = {1, 2}; - set.erase_and_unshift(2); + set.erase_or_unshift(2); REQUIRE_INDICES(set, 1); } SECTION("shifts indexes after the given index") { set = {1, 5}; - set.erase_and_unshift(2); + set.erase_or_unshift(2); REQUIRE_INDICES(set, 1, 4); } SECTION("returns npos for indices in the set") { set = {1, 3, 5}; - REQUIRE(realm::IndexSet(set).erase_and_unshift(1) == realm::IndexSet::npos); - REQUIRE(realm::IndexSet(set).erase_and_unshift(3) == realm::IndexSet::npos); - REQUIRE(realm::IndexSet(set).erase_and_unshift(5) == realm::IndexSet::npos); + REQUIRE(realm::IndexSet(set).erase_or_unshift(1) == realm::IndexSet::npos); + REQUIRE(realm::IndexSet(set).erase_or_unshift(3) == realm::IndexSet::npos); + REQUIRE(realm::IndexSet(set).erase_or_unshift(5) == realm::IndexSet::npos); } SECTION("returns the number of indices in the set before the index for ones not in the set") { set = {1, 3, 5, 6}; - REQUIRE(realm::IndexSet(set).erase_and_unshift(0) == 0); - REQUIRE(realm::IndexSet(set).erase_and_unshift(2) == 1); - REQUIRE(realm::IndexSet(set).erase_and_unshift(4) == 2); - REQUIRE(realm::IndexSet(set).erase_and_unshift(7) == 3); + REQUIRE(realm::IndexSet(set).erase_or_unshift(0) == 0); + REQUIRE(realm::IndexSet(set).erase_or_unshift(2) == 1); + REQUIRE(realm::IndexSet(set).erase_or_unshift(4) == 2); + REQUIRE(realm::IndexSet(set).erase_or_unshift(7) == 3); } } From f6004877692ad30c47a903c8f185ca40853a81d9 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 23 Mar 2016 08:57:34 -0700 Subject: [PATCH 52/93] Speed up CollectionChangeBuilder::move_over() --- src/collection_notifications.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index d060dfdb..d3129363 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -269,15 +269,18 @@ void CollectionChangeBuilder::move(size_t from, size_t to) void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) { REALM_ASSERT(row_ndx <= last_row); + REALM_ASSERT(insertions.empty() || prev(insertions.end())->second - 1 <= last_row); + REALM_ASSERT(modifications.empty() || prev(modifications.end())->second - 1 <= last_row); if (row_ndx == last_row) { erase(row_ndx); return; } bool modified = modifications.contains(last_row); - modifications.erase_at(last_row); - if (modified) + if (modified) { + modifications.remove(last_row); modifications.add(row_ndx); + } else modifications.remove(row_ndx); @@ -316,8 +319,11 @@ void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) return; // Don't report deletions/moves if last_row is newly inserted - auto shifted_last_row = insertions.erase_or_unshift(last_row); - if (shifted_last_row != npos) { + if (!insertions.empty() && prev(insertions.end())->second == last_row + 1) { + insertions.remove(last_row); + } + else { + auto shifted_last_row = insertions.unshift(last_row); shifted_last_row = deletions.add_shifted(shifted_last_row); moves.push_back({shifted_last_row, row_ndx}); } From 7a75e2bae297ab025bd06643dd33c7fef806bd0f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 23 Mar 2016 10:01:14 -0700 Subject: [PATCH 53/93] Use a better data structure for IndexSet Switch to a chunked vector-of-vectors which makes mid-insertions on large sets much faster, and cache the begin/end/count for each chunk to make lookups much more cache-friendly. --- src/impl/transact_log_handler.cpp | 4 +- src/index_set.cpp | 545 ++++++++++++++++++++-------- src/index_set.hpp | 186 +++++++++- tests/collection_change_indices.cpp | 3 +- tests/index_set.cpp | 40 +- tests/transaction_log_parsing.cpp | 14 +- 6 files changed, 617 insertions(+), 175 deletions(-) diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index 87c18c25..19af17eb 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -382,9 +382,9 @@ public: } if (o->kind == ColumnInfo::Kind::Remove) - old_size += o->indices.size(); + old_size += o->indices.count(); else if (o->kind == ColumnInfo::Kind::Insert) - old_size -= o->indices.size(); + old_size -= o->indices.count(); o->indices.set(old_size); diff --git a/src/index_set.cpp b/src/index_set.cpp index 4edfe084..b9b9525c 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -23,9 +23,247 @@ #include using namespace realm; +using namespace realm::_impl; const size_t IndexSet::npos; +template +void MutableChunkedRangeVectorIterator::set(size_t front, size_t back) +{ + this->m_outer->count -= this->m_inner->second - this->m_inner->first; + if (this->offset() == 0) { + this->m_outer->begin = front; + } + if (this->m_inner == &this->m_outer->data.back()) { + this->m_outer->end = back; + } + this->m_outer->count += back - front; + this->m_inner->first = front; + this->m_inner->second = back; +} + +template +void MutableChunkedRangeVectorIterator::adjust(ptrdiff_t front, ptrdiff_t back) +{ + if (this->offset() == 0) { + this->m_outer->begin += front; + } + if (this->m_inner == &this->m_outer->data.back()) { + this->m_outer->end += back; + } + this->m_outer->count += -front + back; + this->m_inner->first += front; + this->m_inner->second += back; +} + +template +void MutableChunkedRangeVectorIterator::shift(ptrdiff_t distance) +{ + if (this->offset() == 0) { + this->m_outer->begin += distance; + } + if (this->m_inner == &this->m_outer->data.back()) { + this->m_outer->end += distance; + } + this->m_inner->first += distance; + this->m_inner->second += distance; +} + +void ChunkedRangeVector::push_back(value_type value) +{ + if (!empty() && m_data.back().data.size() < max_size) { + auto& range = m_data.back(); + REALM_ASSERT(range.end <= value.first); + + range.data.push_back(value); + range.count += value.second - value.first; + range.end = value.second; + } + else { + m_data.push_back({{std::move(value)}, value.first, value.second, value.second - value.first}); + } + verify(); +} + +ChunkedRangeVector::iterator ChunkedRangeVector::insert(iterator pos, value_type value) +{ + if (pos.m_outer == m_data.end()) { + push_back(std::move(value)); + return std::prev(end()); + } + + pos = ensure_space(pos); + auto& chunk = *pos.m_outer; + pos.m_inner = &*chunk.data.insert(pos.m_outer->data.begin() + pos.offset(), value); + chunk.count += value.second - value.first; + chunk.begin = std::min(chunk.begin, value.first); + chunk.end = std::max(chunk.end, value.second); + + verify(); + return pos; +} + +ChunkedRangeVector::iterator ChunkedRangeVector::ensure_space(iterator pos) +{ + if (pos.m_outer->data.size() + 1 <= max_size) + return pos; + + auto offset = pos.offset(); + + // Split the chunk in half to make space for the new insertion + auto new_pos = m_data.insert(pos.m_outer + 1, Chunk{}); + auto prev = new_pos - 1; + auto to_move = max_size / 2; + new_pos->data.reserve(to_move); + new_pos->data.assign(prev->data.end() - to_move, prev->data.end()); + prev->data.resize(prev->data.size() - to_move); + + size_t moved_count = 0; + for (auto range : new_pos->data) + moved_count += range.second - range.first; + + prev->end = prev->data.back().second; + prev->count -= moved_count; + new_pos->begin = new_pos->data.front().first; + new_pos->end = new_pos->data.back().second; + new_pos->count = moved_count; + + if (offset >= to_move) { + pos.m_outer = new_pos; + offset -= to_move; + } + else { + pos.m_outer = prev; + } + pos.m_end = m_data.end(); + pos.m_inner = &pos.m_outer->data[offset]; + verify(); + return pos; +} + +ChunkedRangeVector::iterator ChunkedRangeVector::erase(iterator pos) +{ + auto offset = pos.offset(); + auto& chunk = *pos.m_outer; + chunk.count -= pos->second - pos->first; + chunk.data.erase(chunk.data.begin() + offset); + + if (chunk.data.size() == 0) { + pos.m_outer = m_data.erase(pos.m_outer); + pos.m_end = m_data.end(); + pos.m_inner = pos.m_outer == m_data.end() ? nullptr : &pos.m_outer->data.back(); + verify(); + return pos; + } + + chunk.begin = chunk.data.front().first; + chunk.end = chunk.data.back().second; + if (offset < chunk.data.size()) + pos.m_inner = &chunk.data[offset]; + else { + ++pos.m_outer; + pos.m_inner = pos.m_outer == pos.m_end ? nullptr : &pos.m_outer->data.front(); + } + + verify(); + return pos; +} + +void ChunkedRangeVector::verify() const noexcept +{ +#ifdef REALM_DEBUG + size_t prev_end = -1; + for (auto range : *this) { + REALM_ASSERT(range.first < range.second); + REALM_ASSERT(prev_end == size_t(-1) || range.first > prev_end); + prev_end = range.second; + } + + for (auto& chunk : m_data) { + REALM_ASSERT(!chunk.data.empty()); + REALM_ASSERT(chunk.data.front().first == chunk.begin); + REALM_ASSERT(chunk.data.back().second == chunk.end); + REALM_ASSERT(chunk.count <= chunk.end - chunk.begin); + size_t count = 0; + for (auto range : chunk.data) + count += range.second - range.first; + REALM_ASSERT(count == chunk.count); + } +#endif +} + +namespace { +class ChunkedRangeVectorBuilder { +public: + using value_type = std::pair; + + ChunkedRangeVectorBuilder(ChunkedRangeVector const& expected); + void push_back(size_t index); + void push_back(std::pair range); + std::vector finalize(); +private: + std::vector m_data; + size_t m_outer_pos = 0; +}; + +ChunkedRangeVectorBuilder::ChunkedRangeVectorBuilder(ChunkedRangeVector const& expected) +{ + size_t size = 0; + for (auto const& chunk : expected.m_data) + size += chunk.data.size(); + m_data.resize(size / ChunkedRangeVector::max_size + 1); + for (size_t i = 0; i < m_data.size() - 1; ++i) + m_data[i].data.reserve(ChunkedRangeVector::max_size); +} + +void ChunkedRangeVectorBuilder::push_back(size_t index) +{ + push_back({index, index + 1}); +} + +void ChunkedRangeVectorBuilder::push_back(std::pair range) +{ + auto& chunk = m_data[m_outer_pos]; + if (chunk.data.empty()) { + chunk.data.push_back(range); + chunk.count = range.second - range.first; + chunk.begin = range.first; + } + else if (range.first == chunk.data.back().second) { + chunk.data.back().second = range.second; + chunk.count += range.second - range.first; + } + else if (chunk.data.size() < ChunkedRangeVector::max_size) { + chunk.data.push_back(range); + chunk.count += range.second - range.first; + } + else { + chunk.end = chunk.data.back().second; + ++m_outer_pos; + if (m_outer_pos >= m_data.size()) + m_data.push_back({{range}, range.first, 0, 1}); + else { + auto& chunk = m_data[m_outer_pos]; + chunk.data.push_back(range); + chunk.begin = range.first; + chunk.count = range.second - range.first; + } + } +} + +std::vector ChunkedRangeVectorBuilder::finalize() +{ + if (!m_data.empty()) { + m_data.resize(m_outer_pos + 1); + if (m_data.back().data.empty()) + m_data.pop_back(); + else + m_data.back().end = m_data.back().data.back().second; + } + return std::move(m_data); +} +} + IndexSet::IndexSet(std::initializer_list values) { for (size_t v : values) @@ -35,24 +273,48 @@ IndexSet::IndexSet(std::initializer_list values) bool IndexSet::contains(size_t index) const { auto it = const_cast(this)->find(index); - return it != m_ranges.end() && it->first <= index; + return it != end() && it->first <= index; } size_t IndexSet::count(size_t start_index, size_t end_index) const { auto it = const_cast(this)->find(start_index); - const auto end = m_ranges.end(); + const auto end = this->end(); if (it == end || it->first >= end_index) { return 0; } if (it->second >= end_index) return std::min(it->second, end_index) - std::max(it->first, start_index); - // These checks are somewhat redundant, but this loop is hot so pulling instructions out of it helps - size_t ret = it->second - std::max(it->first, start_index); - for (++it; it != end && it->second < end_index; ++it) { - ret += it->second - it->first; + size_t ret = 0; + + if (start_index > it->first || it.offset() != 0) { + // Start index is in the middle of a chunk, so start by counting the + // rest of that chunk + ret = it->second - std::max(it->first, start_index); + for (++it; it != end && it->second < end_index && it.offset() != 0; ++it) { + ret += it->second - it->first; + } + if (it != end && it->first < end_index && it.offset() != 0) + ret += end_index - it->first; + if (it == end || it->second >= end_index) + return ret; } + + // Now count all complete chunks that fall within the range + while (it != end && it.outer()->end <= end_index) { + REALM_ASSERT_DEBUG(it.offset() == 0); + ret += it.outer()->count; + it.next_chunk(); + } + + // Cound all complete ranges within the last chunk + while (it != end && it->second <= end_index) { + ret += it->second - it->first; + ++it; + } + + // And finally add in the partial last range if (it != end && it->first < end_index) ret += end_index - it->first; return ret; @@ -60,13 +322,25 @@ size_t IndexSet::count(size_t start_index, size_t end_index) const IndexSet::iterator IndexSet::find(size_t index) { - return find(index, m_ranges.begin()); + return find(index, begin()); } -IndexSet::iterator IndexSet::find(size_t index, iterator it) +IndexSet::iterator IndexSet::find(size_t index, iterator begin) { - return std::lower_bound(it, m_ranges.end(), std::make_pair(size_t(0), index + 1), - [&](auto const& a, auto const& b) { return a.second < b.second; }); + auto it = std::find_if(begin.outer(), m_data.end(), + [&](auto const& lft) { return lft.end > index; }); + if (it == m_data.end()) + return end(); + if (index < it->begin) + return iterator(it, m_data.end(), &it->data[0]); + auto inner_begin = it->data.begin(); + if (it == begin.outer()) + inner_begin += begin.offset(); + auto inner = std::lower_bound(inner_begin, it->data.end(), index, + [&](auto const& lft, auto) { return lft.second <= index; }); + REALM_ASSERT_DEBUG(inner != it->data.end()); + + return iterator(it, m_data.end(), &*inner); } void IndexSet::add(size_t index) @@ -76,7 +350,7 @@ void IndexSet::add(size_t index) void IndexSet::add(IndexSet const& other) { - auto it = m_ranges.begin(); + auto it = begin(); for (size_t index : other.as_indexes()) { it = do_add(find(index, it), index); } @@ -84,10 +358,16 @@ void IndexSet::add(IndexSet const& other) size_t IndexSet::add_shifted(size_t index) { - auto it = m_ranges.begin(); - for (auto end = m_ranges.end(); it != end && it->first <= index; ++it) { + iterator it = begin(), end = this->end(); + + // Shift for any complete chunks before the target + for (; it != end && it.outer()->end <= index; it.next_chunk()) + index += it.outer()->count; + + // And any ranges within the last partial chunk + for (; it != end && it->first <= index; ++it) index += it->second - it->first; - } + do_add(it, index); return index; } @@ -105,11 +385,10 @@ void IndexSet::add_shifted_by(IndexSet const& shifted_by, IndexSet const& values } #endif - auto old_ranges = move(m_ranges); - m_ranges.reserve(std::max(old_ranges.size(), values.size())); + ChunkedRangeVectorBuilder builder(*this); - auto old_it = old_ranges.cbegin(), old_end = old_ranges.cend(); - auto shift_it = shifted_by.m_ranges.cbegin(), shift_end = shifted_by.m_ranges.cend(); + auto old_it = cbegin(), old_end = cend(); + auto shift_it = shifted_by.cbegin(), shift_end = shifted_by.cend(); size_t skip_until = 0; size_t old_shift = 0; @@ -124,30 +403,25 @@ void IndexSet::add_shifted_by(IndexSet const& shifted_by, IndexSet const& values for (; old_it != old_end && old_it->first <= index - new_shift + old_shift; ++old_it) { for (size_t i = old_it->first; i < old_it->second; ++i) - add_back(i); + builder.push_back(i); old_shift += old_it->second - old_it->first; } REALM_ASSERT(index >= new_shift); - add_back(index - new_shift + old_shift); + builder.push_back(index - new_shift + old_shift); } - if (old_it != old_end) { - if (!empty() && old_it->first == m_ranges.back().second) { - m_ranges.back().second = old_it->second; - ++old_it; - } - copy(old_it, old_end, back_inserter(m_ranges)); - } + copy(old_it, old_end, std::back_inserter(builder)); + m_data = builder.finalize(); REALM_ASSERT_DEBUG((size_t)std::distance(as_indexes().begin(), as_indexes().end()) == expected); } void IndexSet::set(size_t len) { - m_ranges.clear(); + clear(); if (len) { - m_ranges.push_back({0, len}); + push_back({0, len}); } } @@ -156,22 +430,25 @@ void IndexSet::insert_at(size_t index, size_t count) REALM_ASSERT(count > 0); auto pos = find(index); + auto end = this->end(); bool in_existing = false; - if (pos != m_ranges.end()) { - if (pos->first <= index) + if (pos != end) { + if (pos->first <= index) { in_existing = true; - else - pos->first += count; - pos->second += count; - for (auto it = pos + 1; it != m_ranges.end(); ++it) { - it->first += count; - it->second += count; + pos.adjust(0, count); } + else { + pos.shift(count); + } + for (auto it = std::next(pos); it != end; ++it) + it.shift(count); } if (!in_existing) { for (size_t i = 0; i < count; ++i) - pos = do_add(pos, index + i) + 1; + pos = std::next(do_add(pos, index + i)); } + + verify(); } void IndexSet::insert_at(IndexSet const& positions) @@ -179,30 +456,30 @@ void IndexSet::insert_at(IndexSet const& positions) if (positions.empty()) return; if (empty()) { - m_ranges = positions.m_ranges; + *this = positions; return; } - auto old_ranges = move(m_ranges); - m_ranges.reserve(std::max(m_ranges.size(), positions.m_ranges.size())); - - IndexIterator begin1 = old_ranges.cbegin(), begin2 = positions.m_ranges.cbegin(); - IndexIterator end1 = old_ranges.cend(), end2 = positions.m_ranges.cend(); + IndexIterator begin1 = cbegin(), begin2 = positions.cbegin(); + IndexIterator end1 = cend(), end2 = positions.cend(); + ChunkedRangeVectorBuilder builder(*this); size_t shift = 0; while (begin1 != end1 && begin2 != end2) { if (*begin1 + shift < *begin2) { - add_back(*begin1++ + shift); + builder.push_back(*begin1++ + shift); } else { ++shift; - add_back(*begin2++); + builder.push_back(*begin2++); } } for (; begin1 != end1; ++begin1) - add_back(*begin1 + shift); + builder.push_back(*begin1 + shift); for (; begin2 != end2; ++begin2) - add_back(*begin2); + builder.push_back(*begin2); + + m_data = builder.finalize(); } void IndexSet::shift_for_insert_at(size_t index, size_t count) @@ -210,58 +487,53 @@ void IndexSet::shift_for_insert_at(size_t index, size_t count) REALM_ASSERT(count > 0); auto it = find(index); - if (it == m_ranges.end()) + if (it == end()) return; - if (it->first < index) { - // split the range so that we can exclude `index` - auto old_second = it->second; - it->second = index; - it = m_ranges.insert(it + 1, {index, old_second}); - } + for (auto pos = it, end = this->end(); pos != end; ++pos) + pos.shift(count); - for (; it != m_ranges.end(); ++it) { - it->first += count; - it->second += count; + // If the range contained the insertion point, split the range and move + // the part of it before the insertion point back + if (it->first < index + count) { + auto old_second = it->second; + it.set(it->first - count, index); + insert(std::next(it), {index + count, old_second}); } + verify(); } void IndexSet::shift_for_insert_at(realm::IndexSet const& values) { - if (values.empty()) + if (empty() || values.empty()) + return; + if (values.m_data.front().begin >= m_data.back().end) return; + IndexIterator begin1 = cbegin(), begin2 = values.cbegin(); + IndexIterator end1 = cend(), end2 = values.cend(); + + ChunkedRangeVectorBuilder builder(*this); size_t shift = 0; - auto it = find(values.begin()->first); - for (auto range : values) { - for (; it != m_ranges.end() && it->second + shift <= range.first; ++it) { - it->first += shift; - it->second += shift; + while (begin1 != end1 && begin2 != end2) { + if (*begin1 + shift < *begin2) { + builder.push_back(*begin1++ + shift); } - if (it == m_ranges.end()) - return; - - if (it->first + shift < range.first) { - // split the range so that we can exclude `index` - auto old_second = it->second; - it->first += shift; - it->second = range.first; - it = m_ranges.insert(it + 1, {range.first - shift, old_second}); + else { + ++shift; + begin2++; } - - shift += range.second - range.first; } + for (; begin1 != end1; ++begin1) + builder.push_back(*begin1 + shift); - for (; it != m_ranges.end(); ++it) { - it->first += shift; - it->second += shift; - } + m_data = builder.finalize(); } void IndexSet::erase_at(size_t index) { auto it = find(index); - if (it != m_ranges.end()) + if (it != end()) do_erase(it, index); } @@ -270,16 +542,15 @@ void IndexSet::erase_at(IndexSet const& positions) if (empty() || positions.empty()) return; - auto old_ranges = move(m_ranges); - m_ranges.reserve(std::max(m_ranges.size(), positions.m_ranges.size())); + ChunkedRangeVectorBuilder builder(*this); - IndexIterator begin1 = old_ranges.cbegin(), begin2 = positions.m_ranges.cbegin(); - IndexIterator end1 = old_ranges.cend(), end2 = positions.m_ranges.cend(); + IndexIterator begin1 = cbegin(), begin2 = positions.cbegin(); + IndexIterator end1 = cend(), end2 = positions.cend(); size_t shift = 0; while (begin1 != end1 && begin2 != end2) { if (*begin1 < *begin2) { - add_back(*begin1++ - shift); + builder.push_back(*begin1++ - shift); } else if (*begin1 == *begin2) { ++shift; @@ -292,16 +563,24 @@ void IndexSet::erase_at(IndexSet const& positions) } } for (; begin1 != end1; ++begin1) - add_back(*begin1 - shift); + builder.push_back(*begin1 - shift); + + m_data = builder.finalize(); } size_t IndexSet::erase_or_unshift(size_t index) { auto shifted = index; - auto it = m_ranges.begin(), end = m_ranges.end(); - for (; it != end && it->second <= index; ++it) { + iterator it = begin(), end = this->end(); + + // Shift for any complete chunks before the target + for (; it != end && it.outer()->end <= index; it.next_chunk()) + shifted -= it.outer()->count; + + // And any ranges within the last partial chunk + for (; it != end && it->second <= index; ++it) shifted -= it->second - it->first; - } + if (it == end) return shifted; @@ -316,45 +595,43 @@ size_t IndexSet::erase_or_unshift(size_t index) void IndexSet::do_erase(iterator it, size_t index) { if (it->first <= index) { - --it->second; - if (it->first == it->second) { - it = m_ranges.erase(it); + if (it->first + 1 == it->second) { + it = erase(it); } else { + it.adjust(0, -1); ++it; } } - else if (it != m_ranges.begin() && (it - 1)->second + 1 == it->first) { - (it - 1)->second = it->second - 1; - it = m_ranges.erase(it); + else if (it != begin() && std::prev(it)->second + 1 == it->first) { + std::prev(it).adjust(0, it->second - it->first); + it = erase(it); } - for (; it != m_ranges.end(); ++it) { - --it->first; - --it->second; - } + for (; it != end(); ++it) + it.shift(-1); } IndexSet::iterator IndexSet::do_remove(iterator it, size_t begin, size_t end) { - for (it = find(begin, it); it != m_ranges.end() && it->first < end; it = find(begin, it)) { + for (it = find(begin, it); it != this->end() && it->first < end; it = find(begin, it)) { // Trim off any part of the range to remove that's before the matching range begin = std::max(it->first, begin); // If the matching range extends to both sides of the range to remove, // split it on the range to remove if (it->first < begin && it->second > end) { - it = m_ranges.insert(it + 1, {end, it->second}) - 1; - it->second = begin; + auto old_second = it->second; + it.set(it->first, begin); + it = std::prev(insert(std::next(it), {end, old_second})); } - // Range to delete now coverages (at least) one end of the matching range - if (begin == it->first && end >= it->second) - it = m_ranges.erase(it); + else if (begin == it->first && end >= it->second) + it = erase(it); else if (begin == it->first) - it->first = end; + it.set(end, it->second); else - it->second = begin; + it.set(it->first, begin); } return it; } @@ -366,17 +643,18 @@ void IndexSet::remove(size_t index, size_t count) void IndexSet::remove(realm::IndexSet const& values) { - auto it = m_ranges.begin(); + auto it = begin(); for (auto range : values) { it = do_remove(it, range.first, range.second); - if (it == m_ranges.end()) + if (it == end()) return; } } size_t IndexSet::shift(size_t index) const { - for (auto range : m_ranges) { + // FIXME: optimize + for (auto range : *this) { if (range.first > index) break; index += range.second - range.first; @@ -392,59 +670,36 @@ size_t IndexSet::unshift(size_t index) const void IndexSet::clear() { - m_ranges.clear(); -} - -void IndexSet::add_back(size_t index) -{ - if (m_ranges.empty()) - m_ranges.push_back({index, index + 1}); - else if (m_ranges.back().second == index) - ++m_ranges.back().second; - else { - REALM_ASSERT_DEBUG(m_ranges.back().second < index); - m_ranges.push_back({index, index + 1}); - } + m_data.clear(); } IndexSet::iterator IndexSet::do_add(iterator it, size_t index) { verify(); - bool more_before = it != m_ranges.begin(), valid = it != m_ranges.end(); - REALM_ASSERT(!more_before || index >= (it - 1)->second); + bool more_before = it != begin(), valid = it != end(); + REALM_ASSERT(!more_before || index >= std::prev(it)->second); if (valid && it->first <= index && it->second > index) { // index is already in set return it; } - if (more_before && (it - 1)->second == index) { + if (more_before && std::prev(it)->second == index) { + auto prev = std::prev(it); // index is immediately after an existing range - ++(it - 1)->second; + prev.adjust(0, 1); - if (valid && (it - 1)->second == it->first) { + if (valid && prev->second == it->first) { // index joins two existing ranges - (it - 1)->second = it->second; - return m_ranges.erase(it) - 1; + prev.adjust(0, it->second - it->first); + return std::prev(erase(it)); } - return it - 1; + return prev; } if (valid && it->first == index + 1) { // index is immediately before an existing range - --it->first; + it.adjust(-1, 0); return it; } // index is not next to an existing range - return m_ranges.insert(it, {index, index + 1}); -} - -void IndexSet::verify() const noexcept -{ -#ifdef REALM_DEBUG - size_t prev_end = -1; - for (auto range : m_ranges) { - REALM_ASSERT(range.first < range.second); - REALM_ASSERT(prev_end == size_t(-1) || range.first > prev_end); - prev_end = range.second; - } -#endif + return insert(it, {index, index + 1}); } diff --git a/src/index_set.hpp b/src/index_set.hpp index 924f35e6..3a4ee1ea 100644 --- a/src/index_set.hpp +++ b/src/index_set.hpp @@ -25,18 +25,115 @@ #include namespace realm { -class IndexSet { +class IndexSet; + +namespace _impl { +template +class MutableChunkedRangeVectorIterator; + +// An iterator for ChunkedRangeVector, templated on the vector iterator/const_iterator +template +class ChunkedRangeVectorIterator { +public: + using iterator_category = std::bidirectional_iterator_tag; + using value_type = typename std::remove_referencedata.begin())>::type; + using difference_type = ptrdiff_t; + using pointer = const value_type*; + using reference = const value_type&; + + ChunkedRangeVectorIterator(OuterIterator outer, OuterIterator end, value_type* inner) + : m_outer(outer), m_end(end), m_inner(inner) { } + + reference operator*() const { return *m_inner; } + pointer operator->() const { return m_inner; } + + template bool operator==(Other const& it) const; + template bool operator!=(Other const& it) const; + + ChunkedRangeVectorIterator& operator++(); + ChunkedRangeVectorIterator operator++(int); + + ChunkedRangeVectorIterator& operator--(); + ChunkedRangeVectorIterator operator--(int); + + // Advance directly to the next outer block + void next_chunk(); + + OuterIterator outer() const { return m_outer; } + size_t offset() const { return m_inner - &m_outer->data[0]; } + +private: + OuterIterator m_outer; + OuterIterator m_end; + value_type* m_inner; + friend struct ChunkedRangeVector; + friend class MutableChunkedRangeVectorIterator; +}; + +// A mutable iterator that adds some invariant-preserving mutation methods +template +class MutableChunkedRangeVectorIterator : public ChunkedRangeVectorIterator { +public: + using ChunkedRangeVectorIterator::ChunkedRangeVectorIterator; + + // Set this iterator to the given range and update the parent if needed + void set(size_t begin, size_t end); + // Adjust the begin and end of this iterator by the given amounts and + // update the parent if needed + void adjust(ptrdiff_t front, ptrdiff_t back); + // Shift this iterator by the given amount and update the parent if needed + void shift(ptrdiff_t distance); +}; + +// A vector which stores ranges in chunks with a maximum size +struct ChunkedRangeVector { + struct Chunk { + std::vector> data; + size_t begin; + size_t end; + size_t count; + }; + std::vector m_data; + + using value_type = std::pair; + using iterator = MutableChunkedRangeVectorIterator; + using const_iterator = ChunkedRangeVectorIterator; + +#ifdef REALM_DEBUG + static const size_t max_size = 4; +#else + static const size_t max_size = 4096 / sizeof(std::pair); +#endif + + iterator begin() { return empty() ? end() : iterator(m_data.begin(), m_data.end(), &m_data[0].data[0]); } + iterator end() { return iterator(m_data.end(), m_data.end(), nullptr); } + const_iterator begin() const { return cbegin(); } + const_iterator end() const { return cend(); } + const_iterator cbegin() const { return empty() ? cend() : const_iterator(m_data.cbegin(), m_data.end(), &m_data[0].data[0]); } + const_iterator cend() const { return const_iterator(m_data.end(), m_data.end(), nullptr); } + + bool empty() const noexcept { return m_data.empty(); } + + iterator insert(iterator pos, value_type value); + iterator erase(iterator pos); + void push_back(value_type value); + iterator ensure_space(iterator pos); + + void verify() const noexcept; +}; +} // namespace _impl + +class IndexSet : private _impl::ChunkedRangeVector { public: static const size_t npos = -1; - using value_type = std::pair; - using iterator = std::vector::iterator; - using const_iterator = std::vector::const_iterator; - - const_iterator begin() const { return m_ranges.begin(); } - const_iterator end() const { return m_ranges.end(); } - bool empty() const { return m_ranges.empty(); } - size_t size() const { return m_ranges.size(); } + using ChunkedRangeVector::value_type; + using ChunkedRangeVector::iterator; + using ChunkedRangeVector::const_iterator; + using ChunkedRangeVector::begin; + using ChunkedRangeVector::end; + using ChunkedRangeVector::empty; + using ChunkedRangeVector::verify; IndexSet() = default; IndexSet(std::initializer_list); @@ -45,7 +142,7 @@ public: bool contains(size_t index) const; // Counts the number of indices in the set in the given range - size_t count(size_t start_index, size_t end_index) const; + size_t count(size_t start_index=0, size_t end_index=-1) const; // Add an index to the set, doing nothing if it's already present void add(size_t index); @@ -91,8 +188,6 @@ public: // Remove all indexes from the set void clear(); - void verify() const noexcept; - // An iterator over the indivual indices in the set rather than the ranges class IndexIterator : public std::iterator { public: @@ -140,8 +235,6 @@ public: IndexIteratableAdaptor as_indexes() const { return *this; } private: - std::vector m_ranges; - // Find the range which contains the index, or the first one after it if // none do iterator find(size_t index); @@ -153,8 +246,7 @@ private: void do_erase(iterator it, size_t index); iterator do_remove(iterator it, size_t index, size_t count); - // Add an index which must be greater than the largest index in the set - void add_back(size_t index); + void shift_until_end_by(iterator begin, ptrdiff_t shift); }; namespace util { @@ -166,6 +258,68 @@ std::reverse_iterator make_reverse_iterator(Iterator it) } } // namespace util + +namespace _impl { +template +template +inline bool ChunkedRangeVectorIterator::operator==(OtherIterator const& it) const +{ + return m_outer == it.outer() && m_inner == it.operator->(); +} + +template +template +inline bool ChunkedRangeVectorIterator::operator!=(OtherIterator const& it) const +{ + return !(*this == it); +} + +template +inline ChunkedRangeVectorIterator& ChunkedRangeVectorIterator::operator++() +{ + ++m_inner; + if (offset() == m_outer->data.size()) + next_chunk(); + return *this; +} + +template +inline ChunkedRangeVectorIterator ChunkedRangeVectorIterator::operator++(int) +{ + auto value = *this; + ++*this; + return value; +} + +template +inline ChunkedRangeVectorIterator& ChunkedRangeVectorIterator::operator--() +{ + if (!m_inner || m_inner == &m_outer->data.front()) { + --m_outer; + m_inner = &m_outer->data.back(); + } + else { + --m_inner; + } + return *this; +} + +template +inline ChunkedRangeVectorIterator ChunkedRangeVectorIterator::operator--(int) +{ + auto value = *this; + --*this; + return value; +} + +template +inline void ChunkedRangeVectorIterator::next_chunk() +{ + ++m_outer; + m_inner = m_outer != m_end ? &m_outer->data[0] : nullptr; +} +} // namespace _impl + } // namespace realm #endif // REALM_INDEX_SET_HPP diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index 5d67a5e2..a15b7b18 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -233,7 +233,8 @@ TEST_CASE("[collection_change] clear()") { SECTION("sets deletions SIZE_T_MAX if that if the given previous size") { c.insertions = {1, 2, 3}; c.clear(std::numeric_limits::max()); - REQUIRE(c.deletions.size() == 1); + REQUIRE(!c.deletions.empty()); + REQUIRE(++c.deletions.begin() == c.deletions.end()); REQUIRE(c.deletions.begin()->first == 0); REQUIRE(c.deletions.begin()->second == std::numeric_limits::max()); } diff --git a/tests/index_set.cpp b/tests/index_set.cpp index 05c4bbf2..df5af2f1 100644 --- a/tests/index_set.cpp +++ b/tests/index_set.cpp @@ -57,6 +57,20 @@ TEST_CASE("[index_set] count()") { realm::IndexSet set = {1, 2, 3, 5, 6, 7, 8, 9}; REQUIRE(set.count(3, 9) == 5); } + + SECTION("handles full chunks well") { + size_t count = realm::_impl::ChunkedRangeVector::max_size * 4; + realm::IndexSet set; + for (size_t i = 0; i < count; ++i) { + set.add(i * 3); + set.add(i * 3 + 1); + } + + for (size_t i = 0; i < count * 3; ++i) { + REQUIRE(set.count(i) == 2 * count - (i + 1) * 2 / 3); + REQUIRE(set.count(0, i) == (i + 1) / 3 + (i + 2) / 3); + } + } } TEST_CASE("[index_set] add()") { @@ -100,6 +114,12 @@ TEST_CASE("[index_set] add()") { set.add({1, 4, 5}); REQUIRE_INDICES(set, 0, 1, 2, 4, 5, 6); } + + SECTION("handles front additions of ranges") { + for (size_t i = 20; i > 0; i -= 2) + set.add(i); + REQUIRE_INDICES(set, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20); + } } TEST_CASE("[index_set] add_shifted()") { @@ -172,14 +192,14 @@ TEST_CASE("[index_set] add_shifted_by()") { SECTION("adds the indices shifted by the old count when they are all after the old indices and the shifted-by set is empty") { set = {5, 6}; - set.add_shifted_by({}, {7, 8}); - REQUIRE_INDICES(set, 5, 6, 9, 10); + set.add_shifted_by({}, {7, 9, 11, 13}); + REQUIRE_INDICES(set, 5, 6, 9, 11, 13, 15); } SECTION("acts like bulk add_shifted() when shifted_by is empty") { - set = {5, 10}; + set = {5, 10, 15, 20, 25}; set.add_shifted_by({}, {4, 5, 11}); - REQUIRE_INDICES(set, 4, 5, 6, 10, 13); + REQUIRE_INDICES(set, 4, 5, 6, 10, 13, 15, 20, 25); } SECTION("shifts indices in values back by the number of indices in shifted_by before them") { @@ -288,6 +308,18 @@ TEST_CASE("[index_set] shift_for_insert_at()") { REQUIRE(set.empty()); } + SECTION("does nothing when the insertion points are all after the current indices") { + set = {10, 20}; + set.shift_for_insert_at({30, 40}); + REQUIRE_INDICES(set, 10, 20); + } + + SECTION("does shift when the insertion points are all before the current indices") { + set = {10, 20}; + set.shift_for_insert_at({2, 4}); + REQUIRE_INDICES(set, 12, 22); + } + SECTION("shifts indices at or after the insertion points") { set = {5}; diff --git a/tests/transaction_log_parsing.cpp b/tests/transaction_log_parsing.cpp index 76a20f46..7363c6c2 100644 --- a/tests/transaction_log_parsing.cpp +++ b/tests/transaction_log_parsing.cpp @@ -366,7 +366,7 @@ TEST_CASE("Transaction log parsing") { auto info = track_changes({false, false, false}, [&] { table.set_int(0, 1, 2); }); - REQUIRE(info.tables.size() == 0); + REQUIRE(info.tables.empty()); } SECTION("new row additions are reported") { @@ -678,7 +678,7 @@ TEST_CASE("Transaction log parsing") { lv->remove(5); } REQUIRE_INDICES(changes.deletions, 5); - REQUIRE(changes.modifications.size() == 0); + REQUIRE(changes.modifications.empty()); VALIDATE_CHANGES(changes) { lv->set(5, 0); @@ -693,7 +693,7 @@ TEST_CASE("Transaction log parsing") { lv->remove(4); } REQUIRE_INDICES(changes.deletions, 4, 5); - REQUIRE(changes.modifications.size() == 0); + REQUIRE(changes.modifications.empty()); } SECTION("erase -> set") { @@ -711,7 +711,7 @@ TEST_CASE("Transaction log parsing") { lv->clear(); } REQUIRE_INDICES(changes.deletions, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); - REQUIRE(changes.insertions.size() == 0); + REQUIRE(changes.insertions.empty()); } SECTION("set -> clear") { @@ -720,7 +720,7 @@ TEST_CASE("Transaction log parsing") { lv->clear(); } REQUIRE_INDICES(changes.deletions, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); - REQUIRE(changes.modifications.size() == 0); + REQUIRE(changes.modifications.empty()); } SECTION("clear -> insert") { @@ -737,8 +737,8 @@ TEST_CASE("Transaction log parsing") { lv->add(0); lv->remove(10); } - REQUIRE(changes.insertions.size() == 0); - REQUIRE(changes.deletions.size() == 0); + REQUIRE(changes.insertions.empty()); + REQUIRE(changes.deletions.empty()); VALIDATE_CHANGES(changes) { lv->add(0); From 5eb18ed487d4136b30942c09061175e60eb83e3b Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 28 Mar 2016 12:49:25 -0700 Subject: [PATCH 54/93] Allow non-empty no-op changesets when there should be no notification in the fuzzer --- fuzzer/fuzzer.cpp | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/fuzzer/fuzzer.cpp b/fuzzer/fuzzer.cpp index 72be0dca..a1cc8fd8 100644 --- a/fuzzer/fuzzer.cpp +++ b/fuzzer/fuzzer.cpp @@ -90,7 +90,7 @@ static bool apply_changes(fuzzer::CommandFile& commands, fuzzer::RealmState& sta return false; } -static void verify(CollectionChangeIndices const& changes, std::vector values, fuzzer::RealmState& state) +static auto verify(CollectionChangeIndices const& changes, std::vector values, fuzzer::RealmState& state) { auto tv = tableview(state); @@ -130,6 +130,15 @@ static void verify(CollectionChangeIndices const& changes, std::vector abort(); } } + + return values; +} + +static void verify_no_op(CollectionChangeIndices const& changes, std::vector values, fuzzer::RealmState& state) +{ + auto new_values = verify(changes, values, state); + if (!std::equal(begin(values), end(values), begin(new_values), end(new_values))) + abort(); } static void test(Realm::Config const& config, SharedRealm& r, SharedRealm& r2, std::istream& input_stream) @@ -189,11 +198,15 @@ static void test(Realm::Config const& config, SharedRealm& r, SharedRealm& r2, s bool expect_notification = apply_changes(command, state2); state.coordinator.on_change(); r->notify(); - if (notification_calls != 1 + expect_notification) { - abort(); + if (expect_notification) { + if (notification_calls != 2) + abort(); + verify(changes, initial_values, state); + } + else { + if (notification_calls == 2) + verify_no_op(changes, initial_values, state); } - - verify(changes, initial_values, state); } int main(int argc, char** argv) { From 510c049855932bd83f60eb38c59257d0af191b0f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 28 Mar 2016 13:48:38 -0700 Subject: [PATCH 55/93] Don't run the LV commands at all if fuzzing non-linkview --- fuzzer/command_file.cpp | 6 ++++++ fuzzer/fuzzer.cpp | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/fuzzer/command_file.cpp b/fuzzer/command_file.cpp index e3f836ae..7fbde719 100644 --- a/fuzzer/command_file.cpp +++ b/fuzzer/command_file.cpp @@ -67,6 +67,7 @@ static void run_commit(RealmState& state) static void run_lv_insert(RealmState& state, size_t pos, size_t target) { + if (!state.lv) return; if (target < state.table.size() && pos <= state.lv->size()) { log("lv insert %zu %zu\n", pos, target); state.lv->insert(pos, target); @@ -75,6 +76,7 @@ static void run_lv_insert(RealmState& state, size_t pos, size_t target) static void run_lv_set(RealmState& state, size_t pos, size_t target) { + if (!state.lv) return; if (target < state.table.size() && pos < state.lv->size()) { log("lv set %zu %zu\n", pos, target); // We can't reliably detect self-assignment for verification, so don't do it @@ -85,6 +87,7 @@ static void run_lv_set(RealmState& state, size_t pos, size_t target) static void run_lv_move(RealmState& state, size_t from, size_t to) { + if (!state.lv) return; if (from < state.lv->size() && to < state.lv->size()) { log("lv move %zu %zu\n", from, to); // FIXME: only do the move if it has an effect to avoid getting a @@ -100,6 +103,7 @@ static void run_lv_move(RealmState& state, size_t from, size_t to) static void run_lv_swap(RealmState& state, size_t ndx1, size_t ndx2) { + if (!state.lv) return; if (ndx1 < state.lv->size() && ndx2 < state.lv->size()) { log("lv swap %zu %zu\n", ndx1, ndx2); if (state.lv->get(ndx1).get_index() != state.lv->get(ndx2).get_index()) { @@ -114,6 +118,7 @@ static void run_lv_swap(RealmState& state, size_t ndx1, size_t ndx2) static void run_lv_remove(RealmState& state, size_t pos) { + if (!state.lv) return; if (pos < state.lv->size()) { log("lv remove %zu\n", pos); state.lv->remove(pos); @@ -122,6 +127,7 @@ static void run_lv_remove(RealmState& state, size_t pos) static void run_lv_remove_target(RealmState& state, size_t pos) { + if (!state.lv) return; if (pos < state.lv->size()) { log("lv target remove %zu\n", pos); state.lv->remove_target_row(pos); diff --git a/fuzzer/fuzzer.cpp b/fuzzer/fuzzer.cpp index a1cc8fd8..6cd5a7d0 100644 --- a/fuzzer/fuzzer.cpp +++ b/fuzzer/fuzzer.cpp @@ -162,7 +162,11 @@ static void test(Realm::Config const& config, SharedRealm& r, SharedRealm& r2, s *r2, state.coordinator, *r2->read_group()->get_table("class_object"), +#if FUZZ_LINKVIEW r2->read_group()->get_table("class_linklist")->get_linklist(0, 0), +#else + {}, +#endif state.uid, {} }; From 45705b18d2c71ff537173d18f37bece80ac2dc40 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 28 Mar 2016 13:15:00 -0700 Subject: [PATCH 56/93] Build all of the fuzzer variants --- fuzzer/CMakeLists.txt | 18 +++++++++++++----- fuzzer/fuzz-sorted-linkview.cpp | 3 +++ fuzzer/fuzz-sorted-query.cpp | 3 +++ fuzzer/fuzz-unsorted-linkview.cpp | 3 +++ fuzzer/fuzz-unsorted-query.cpp | 3 +++ fuzzer/fuzzer.cpp | 8 +++++++- 6 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 fuzzer/fuzz-sorted-linkview.cpp create mode 100644 fuzzer/fuzz-sorted-query.cpp create mode 100644 fuzzer/fuzz-unsorted-linkview.cpp create mode 100644 fuzzer/fuzz-unsorted-query.cpp diff --git a/fuzzer/CMakeLists.txt b/fuzzer/CMakeLists.txt index b646497a..a23d7c16 100644 --- a/fuzzer/CMakeLists.txt +++ b/fuzzer/CMakeLists.txt @@ -1,5 +1,13 @@ -add_executable(fuzzer - command_file.hpp - command_file.cpp - fuzzer.cpp) -target_link_libraries(fuzzer realm-object-store) +macro(build_fuzzer_variant variant) + add_executable(${variant} command_file.hpp command_file.cpp ${variant}.cpp) + target_link_libraries(${variant} realm-object-store) + set_target_properties(${variant} PROPERTIES + EXCLUDE_FROM_ALL 1 + EXCLUDE_FROM_DEFAULT_BUILD 1) +endmacro() + +build_fuzzer_variant(fuzzer) +build_fuzzer_variant(fuzz-sorted-query) +build_fuzzer_variant(fuzz-unsorted-query) +build_fuzzer_variant(fuzz-sorted-linkview) +build_fuzzer_variant(fuzz-unsorted-linkview) diff --git a/fuzzer/fuzz-sorted-linkview.cpp b/fuzzer/fuzz-sorted-linkview.cpp new file mode 100644 index 00000000..13d9bec0 --- /dev/null +++ b/fuzzer/fuzz-sorted-linkview.cpp @@ -0,0 +1,3 @@ +#define FUZZ_SORTED 1 +#define FUZZ_LINKVIEW 1 +#include "fuzzer.cpp" diff --git a/fuzzer/fuzz-sorted-query.cpp b/fuzzer/fuzz-sorted-query.cpp new file mode 100644 index 00000000..b32e9dc3 --- /dev/null +++ b/fuzzer/fuzz-sorted-query.cpp @@ -0,0 +1,3 @@ +#define FUZZ_SORTED 1 +#define FUZZ_LINKVIEW 0 +#include "fuzzer.cpp" diff --git a/fuzzer/fuzz-unsorted-linkview.cpp b/fuzzer/fuzz-unsorted-linkview.cpp new file mode 100644 index 00000000..24d25184 --- /dev/null +++ b/fuzzer/fuzz-unsorted-linkview.cpp @@ -0,0 +1,3 @@ +#define FUZZ_SORTED 0 +#define FUZZ_LINKVIEW 1 +#include "fuzzer.cpp" diff --git a/fuzzer/fuzz-unsorted-query.cpp b/fuzzer/fuzz-unsorted-query.cpp new file mode 100644 index 00000000..6dec4c74 --- /dev/null +++ b/fuzzer/fuzz-unsorted-query.cpp @@ -0,0 +1,3 @@ +#define FUZZ_SORTED 0 +#define FUZZ_LINKVIEW 0 +#include "fuzzer.cpp" diff --git a/fuzzer/fuzzer.cpp b/fuzzer/fuzzer.cpp index 6cd5a7d0..6b088e85 100644 --- a/fuzzer/fuzzer.cpp +++ b/fuzzer/fuzzer.cpp @@ -20,8 +20,14 @@ using namespace realm; +#ifndef FUZZ_SORTED #define FUZZ_SORTED 0 -#define FUZZ_LINKVIEW 1 +#endif + +#ifndef FUZZ_LINKVIEW +#define FUZZ_LINKVIEW 0 +#endif + #define FUZZ_LOG 0 // Read from a fd until eof into a string From 8623aa6c6b15e3b08fe2de124e8b84b2fb9f2994 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 28 Mar 2016 15:46:43 -0700 Subject: [PATCH 57/93] Actually unregister List notifiers when the List is destroyed --- src/list.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/list.cpp b/src/list.cpp index b6ac8c61..38adfac8 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -29,7 +29,12 @@ using namespace realm; using namespace realm::_impl; List::List() noexcept = default; -List::~List() = default; +List::~List() +{ + if (m_notifier) { + m_notifier->unregister(); + } +} List::List(std::shared_ptr r, const ObjectSchema& s, LinkViewRef l) noexcept : m_realm(std::move(r)) From ae9516dbb7392fe2e8133bd7f94850ebdd8a5bbc Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 29 Mar 2016 16:41:50 -0700 Subject: [PATCH 58/93] Improve performance for large numbers of deletions --- src/collection_notifications.cpp | 57 ++++++++++++++------------------ 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index d3129363..ac995d37 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -284,52 +284,43 @@ void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) else modifications.remove(row_ndx); + bool row_is_insertion = insertions.contains(row_ndx); + bool last_is_insertion = !insertions.empty() && prev(insertions.end())->second == last_row + 1; + REALM_ASSERT_DEBUG(insertions.empty() || prev(insertions.end())->second <= last_row + 1); + bool updated_existing_move = false; - for (size_t i = 0; i < moves.size(); ++i) { - auto& move = moves[i]; - // Remove moves to the row being deleted - if (move.to == row_ndx) { - moves.erase(moves.begin() + i); - --i; - continue; + if (row_is_insertion || last_is_insertion) { + for (size_t i = 0; i < moves.size(); ++i) { + auto& move = moves[i]; + // Remove moves to the row being deleted + if (move.to == row_ndx) { + moves.erase(moves.begin() + i); + --i; + continue; + } + if (move.to != last_row) + continue; + REALM_ASSERT(!updated_existing_move); + + // Collapse A -> B, B -> C into a single A -> C move + move.to = row_ndx; + updated_existing_move = true; } - if (move.to != last_row) - continue; - REALM_ASSERT(!updated_existing_move); - - // Collapse A -> B, B -> C into a single A -> C move - move.to = row_ndx; - updated_existing_move = true; - - if (!insertions.empty()) { - REALM_ASSERT(std::prev(insertions.end())->second - 1 <= last_row); - insertions.remove(last_row); - } - - // Don't mark the moved-over row as deleted if it was a new insertion - if (!insertions.contains(row_ndx)) { - deletions.add_shifted(insertions.unshift(row_ndx)); - insertions.add(row_ndx); - } - - // Because this is a move, the unshifted source row has already been marked as deleted } - if (updated_existing_move) - return; - // Don't report deletions/moves if last_row is newly inserted - if (!insertions.empty() && prev(insertions.end())->second == last_row + 1) { + if (last_is_insertion) { insertions.remove(last_row); } - else { + // If it was previously moved, the unshifted source row has already been marked as deleted + else if (!updated_existing_move) { auto shifted_last_row = insertions.unshift(last_row); shifted_last_row = deletions.add_shifted(shifted_last_row); moves.push_back({shifted_last_row, row_ndx}); } // Don't mark the moved-over row as deleted if it was a new insertion - if (!insertions.contains(row_ndx)) { + if (!row_is_insertion) { deletions.add_shifted(insertions.unshift(row_ndx)); insertions.add(row_ndx); } From 2e807166298ef60039e316828e35e82afad69992 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 29 Mar 2016 16:56:38 -0700 Subject: [PATCH 59/93] Improve performance of move mapping when there are a lot of deletions --- src/impl/results_notifier.cpp | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index 55300f56..b097cec0 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -40,17 +40,6 @@ void ResultsNotifier::release_data() noexcept m_query = nullptr; } -static bool map_moves(size_t& idx, CollectionChangeIndices const& changes) -{ - for (auto&& move : changes.moves) { - if (move.from == idx) { - idx = move.to; - return true; - } - } - return false; -} - // Most of the inter-thread synchronization for run(), prepare_handover(), // attach_to(), detach(), release_data() and deliver() is done by // RealmCoordinator external to this code, which has some potentially @@ -117,14 +106,18 @@ void ResultsNotifier::run() next_rows.push_back(m_tv[i].get_index()); if (changes) { + auto move_map = changes->moves; + std::sort(begin(move_map), end(move_map), + [](auto const& a, auto const& b) { return a.from < b.from; }); for (auto& idx : m_previous_rows) { - if (!map_moves(idx, *changes)) { - if (changes->deletions.contains(idx)) - idx = npos; - else { - REALM_ASSERT_DEBUG(!changes->insertions.contains(idx)); - } - } + auto it = lower_bound(begin(move_map), end(move_map), idx, + [](auto const& a, auto b) { return a.from < b; }); + if (it != move_map.end() && it->from == idx) + idx = it->to; + else if (changes->deletions.contains(idx)) + idx = npos; + else + REALM_ASSERT_DEBUG(!changes->insertions.contains(idx)); } } From bc78c02e9d9f9226bb786cd7810b68f63e6980b7 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 30 Mar 2016 09:38:48 -0700 Subject: [PATCH 60/93] Fix quadratic runtime of move_last_over() parsing --- src/collection_notifications.cpp | 52 ++++++++++++++++++----------- src/collection_notifications.hpp | 6 ++++ src/impl/results_notifier.cpp | 8 ++--- src/impl/transact_log_handler.cpp | 6 ++-- tests/collection_change_indices.cpp | 18 +++++++--- tests/transaction_log_parsing.cpp | 2 +- 6 files changed, 61 insertions(+), 31 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index ac995d37..5955d908 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -172,6 +172,19 @@ void CollectionChangeBuilder::clean_up_stale_moves() }), end(moves)); } +void CollectionChangeBuilder::parse_complete() +{ + moves.reserve(m_move_mapping.size()); + for (auto move : m_move_mapping) { + REALM_ASSERT_DEBUG(deletions.contains(move.second)); + REALM_ASSERT_DEBUG(insertions.contains(move.first)); + moves.push_back({move.second, move.first}); + } + m_move_mapping.clear(); + std::sort(begin(moves), end(moves), + [](auto const& a, auto const& b) { return a.from < b.from; }); +} + void CollectionChangeBuilder::modify(size_t ndx) { modifications.add(ndx); @@ -218,12 +231,14 @@ void CollectionChangeBuilder::clear(size_t old_size) modifications.clear(); insertions.clear(); moves.clear(); + m_move_mapping.clear(); deletions.set(old_size); } void CollectionChangeBuilder::move(size_t from, size_t to) { REALM_ASSERT(from != to); + REALM_ASSERT(m_move_mapping.empty()); bool updated_existing_move = false; for (auto& move : moves) { @@ -268,6 +283,7 @@ void CollectionChangeBuilder::move(size_t from, size_t to) void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) { + REALM_ASSERT(moves.empty()); REALM_ASSERT(row_ndx <= last_row); REALM_ASSERT(insertions.empty() || prev(insertions.end())->second - 1 <= last_row); REALM_ASSERT(modifications.empty() || prev(modifications.end())->second - 1 <= last_row); @@ -288,35 +304,33 @@ void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) bool last_is_insertion = !insertions.empty() && prev(insertions.end())->second == last_row + 1; REALM_ASSERT_DEBUG(insertions.empty() || prev(insertions.end())->second <= last_row + 1); - bool updated_existing_move = false; - if (row_is_insertion || last_is_insertion) { - for (size_t i = 0; i < moves.size(); ++i) { - auto& move = moves[i]; - // Remove moves to the row being deleted - if (move.to == row_ndx) { - moves.erase(moves.begin() + i); - --i; - continue; - } - if (move.to != last_row) - continue; - REALM_ASSERT(!updated_existing_move); - - // Collapse A -> B, B -> C into a single A -> C move - move.to = row_ndx; - updated_existing_move = true; + // Collapse A -> B, B -> C into a single A -> C move + bool last_was_already_moved = false; + if (last_is_insertion) { + auto it = m_move_mapping.find(last_row); + if (it != m_move_mapping.end() && it->first == last_row) { + m_move_mapping[row_ndx] = it->second; + m_move_mapping.erase(it); + last_was_already_moved = true; } } + // Remove moves to the row being deleted + if (row_is_insertion && !last_was_already_moved) { + auto it = m_move_mapping.find(row_ndx); + if (it != m_move_mapping.end() && it->first == row_ndx) + m_move_mapping.erase(it); + } + // Don't report deletions/moves if last_row is newly inserted if (last_is_insertion) { insertions.remove(last_row); } // If it was previously moved, the unshifted source row has already been marked as deleted - else if (!updated_existing_move) { + else if (!last_was_already_moved) { auto shifted_last_row = insertions.unshift(last_row); shifted_last_row = deletions.add_shifted(shifted_last_row); - moves.push_back({shifted_last_row, row_ndx}); + m_move_mapping[row_ndx] = shifted_last_row; } // Don't mark the moved-over row as deleted if it was a new insertion diff --git a/src/collection_notifications.hpp b/src/collection_notifications.hpp index 019ee907..d6779abc 100644 --- a/src/collection_notifications.hpp +++ b/src/collection_notifications.hpp @@ -24,6 +24,7 @@ #include #include +#include namespace realm { namespace _impl { @@ -95,8 +96,13 @@ public: void clear(size_t old_size); void move(size_t from, size_t to); + void parse_complete(); + private: + std::unordered_map m_move_mapping; + void verify(); + }; } // namespace _impl } // namespace realm diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index b097cec0..44782305 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -106,13 +106,11 @@ void ResultsNotifier::run() next_rows.push_back(m_tv[i].get_index()); if (changes) { - auto move_map = changes->moves; - std::sort(begin(move_map), end(move_map), - [](auto const& a, auto const& b) { return a.from < b.from; }); + auto const& moves = changes->moves; for (auto& idx : m_previous_rows) { - auto it = lower_bound(begin(move_map), end(move_map), idx, + auto it = lower_bound(begin(moves), end(moves), idx, [](auto const& a, auto b) { return a.from < b; }); - if (it != move_map.end() && it->from == idx) + if (it != moves.end() && it->from == idx) idx = it->to; else if (changes->deletions.contains(idx)) idx = npos; diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index 19af17eb..c0ea4531 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -447,8 +447,10 @@ public: void parse_complete() { - for (auto& list : m_info.lists) - { + for (auto& table : m_info.tables) { + table.parse_complete(); + } + for (auto& list : m_info.lists) { list.changes->clean_up_stale_moves(); } } diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index a15b7b18..585f56b8 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -133,6 +133,7 @@ TEST_CASE("[collection_change] move_over()") { c.move_over(4, 5); c.move_over(0, 4); c.move_over(2, 3); + c.parse_complete(); c.clean_up_stale_moves(); REQUIRE_INDICES(c.deletions, 0, 2, 4, 5, 6); @@ -142,56 +143,65 @@ TEST_CASE("[collection_change] move_over()") { SECTION("marks the old last row as moved") { c.move_over(5, 8); + c.parse_complete(); REQUIRE_MOVES(c, {8, 5}); } SECTION("does not mark the old last row as moved if it was newly inserted") { c.insert(8); c.move_over(5, 8); + c.parse_complete(); REQUIRE(c.moves.empty()); } SECTION("removes previous modifications for the removed row") { c.modify(5); c.move_over(5, 8); + c.parse_complete(); REQUIRE(c.modifications.empty()); } SECTION("updates previous insertions for the old last row") { c.insert(5); c.move_over(3, 5); + c.parse_complete(); REQUIRE_INDICES(c.insertions, 3); } SECTION("updates previous modifications for the old last row") { c.modify(5); c.move_over(3, 5); + c.parse_complete(); REQUIRE_INDICES(c.modifications, 3); } SECTION("removes moves to the target") { - c.move(3, 5); + c.move_over(5, 10); c.move_over(5, 8); + c.parse_complete(); REQUIRE_MOVES(c, {8, 5}); } SECTION("updates moves to the source") { - c.move(3, 8); + c.move_over(8, 10); c.move_over(5, 8); - REQUIRE_MOVES(c, {3, 5}); + c.parse_complete(); + REQUIRE_MOVES(c, {10, 5}); } SECTION("is not shifted by previous calls to move_over()") { c.move_over(5, 10); c.move_over(6, 9); + c.parse_complete(); REQUIRE_INDICES(c.deletions, 5, 6, 9, 10); REQUIRE_INDICES(c.insertions, 5, 6); - REQUIRE_MOVES(c, {10, 5}, {9, 6}); + REQUIRE_MOVES(c, {9, 6}, {10, 5}); } SECTION("marks the moved-over row as deleted when chaining moves") { c.move_over(5, 10); c.move_over(0, 5); + c.parse_complete(); REQUIRE_INDICES(c.deletions, 0, 5, 10); REQUIRE_INDICES(c.insertions, 0); diff --git a/tests/transaction_log_parsing.cpp b/tests/transaction_log_parsing.cpp index 7363c6c2..aed20b8a 100644 --- a/tests/transaction_log_parsing.cpp +++ b/tests/transaction_log_parsing.cpp @@ -407,7 +407,7 @@ TEST_CASE("Transaction log parsing") { REQUIRE(info.tables.size() == 3); REQUIRE_INDICES(info.tables[2].deletions, 2, 3, 8, 9); REQUIRE_INDICES(info.tables[2].insertions, 2, 3); - REQUIRE_MOVES(info.tables[2], {9, 2}, {8, 3}); + REQUIRE_MOVES(info.tables[2], {8, 3}, {9, 2}); } } From a86265f4dc57627299a2fbee0ccdf3670b9acded Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 31 Mar 2016 11:06:27 -0700 Subject: [PATCH 61/93] Move CollectionChangeBuilder to background_collection.hpp --- src/collection_notifications.cpp | 574 ---------------------------- src/collection_notifications.hpp | 42 -- src/impl/background_collection.cpp | 569 +++++++++++++++++++++++++++ src/impl/background_collection.hpp | 36 ++ tests/collection_change_indices.cpp | 2 +- 5 files changed, 606 insertions(+), 617 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 5955d908..cca23764 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -20,10 +20,6 @@ #include "impl/background_collection.hpp" -#include -#include -#include - using namespace realm; using namespace realm::_impl; @@ -58,573 +54,3 @@ NotificationToken& NotificationToken::operator=(realm::NotificationToken&& rgt) } return *this; } - -CollectionChangeIndices::CollectionChangeIndices(IndexSet deletions, - IndexSet insertions, - IndexSet modifications, - std::vector moves) -: deletions(std::move(deletions)) -, insertions(std::move(insertions)) -, modifications(std::move(modifications)) -, moves(std::move(moves)) -{ - for (auto&& move : this->moves) { - this->deletions.add(move.from); - this->insertions.add(move.to); - } -} - -void CollectionChangeBuilder::merge(CollectionChangeBuilder&& c) -{ - if (c.empty()) - return; - if (empty()) { - *this = std::move(c); - return; - } - - verify(); - c.verify(); - - // First update any old moves - if (!c.moves.empty() || !c.deletions.empty() || !c.insertions.empty()) { - auto it = remove_if(begin(moves), end(moves), [&](auto& old) { - // Check if the moved row was moved again, and if so just update the destination - auto it = find_if(begin(c.moves), end(c.moves), [&](auto const& m) { - return old.to == m.from; - }); - if (it != c.moves.end()) { - if (modifications.contains(it->from)) - c.modifications.add(it->to); - old.to = it->to; - *it = c.moves.back(); - c.moves.pop_back(); - ++it; - return false; - } - - // Check if the destination was deleted - // Removing the insert for this move will happen later - if (c.deletions.contains(old.to)) - return true; - - // Update the destination to adjust for any new insertions and deletions - old.to = c.insertions.shift(c.deletions.unshift(old.to)); - return false; - }); - moves.erase(it, end(moves)); - } - - // Ignore new moves of rows which were previously inserted (the implicit - // delete from the move will remove the insert) - if (!insertions.empty() && !c.moves.empty()) { - c.moves.erase(remove_if(begin(c.moves), end(c.moves), - [&](auto const& m) { return insertions.contains(m.from); }), - end(c.moves)); - } - - // Ensure that any previously modified rows which were moved are still modified - if (!modifications.empty() && !c.moves.empty()) { - for (auto const& move : c.moves) { - if (modifications.contains(move.from)) - c.modifications.add(move.to); - } - } - - // Update the source position of new moves to compensate for the changes made - // in the old changeset - if (!deletions.empty() || !insertions.empty()) { - for (auto& move : c.moves) - move.from = deletions.shift(insertions.unshift(move.from)); - } - - moves.insert(end(moves), begin(c.moves), end(c.moves)); - - // New deletion indices have been shifted by the insertions, so unshift them - // before adding - deletions.add_shifted_by(insertions, c.deletions); - - // Drop any inserted-then-deleted rows, then merge in new insertions - insertions.erase_at(c.deletions); - insertions.insert_at(c.insertions); - - clean_up_stale_moves(); - - modifications.erase_at(c.deletions); - modifications.shift_for_insert_at(c.insertions); - modifications.add(c.modifications); - - c = {}; - verify(); -} - -void CollectionChangeBuilder::clean_up_stale_moves() -{ - // Look for moves which are now no-ops, and remove them plus the associated - // insert+delete. Note that this isn't just checking for from == to due to - // that rows can also be shifted by other inserts and deletes - moves.erase(remove_if(begin(moves), end(moves), [&](auto const& move) { - if (move.from - deletions.count(0, move.from) != move.to - insertions.count(0, move.to)) - return false; - deletions.remove(move.from); - insertions.remove(move.to); - return true; - }), end(moves)); -} - -void CollectionChangeBuilder::parse_complete() -{ - moves.reserve(m_move_mapping.size()); - for (auto move : m_move_mapping) { - REALM_ASSERT_DEBUG(deletions.contains(move.second)); - REALM_ASSERT_DEBUG(insertions.contains(move.first)); - moves.push_back({move.second, move.first}); - } - m_move_mapping.clear(); - std::sort(begin(moves), end(moves), - [](auto const& a, auto const& b) { return a.from < b.from; }); -} - -void CollectionChangeBuilder::modify(size_t ndx) -{ - modifications.add(ndx); -} - -void CollectionChangeBuilder::insert(size_t index, size_t count) -{ - modifications.shift_for_insert_at(index, count); - insertions.insert_at(index, count); - - for (auto& move : moves) { - if (move.to >= index) - ++move.to; - } -} - -void CollectionChangeBuilder::erase(size_t index) -{ - modifications.erase_at(index); - size_t unshifted = insertions.erase_or_unshift(index); - if (unshifted != npos) - deletions.add_shifted(unshifted); - - for (size_t i = 0; i < moves.size(); ++i) { - auto& move = moves[i]; - if (move.to == index) { - moves.erase(moves.begin() + i); - --i; - } - else if (move.to > index) - --move.to; - } -} - -void CollectionChangeBuilder::clear(size_t old_size) -{ - if (old_size != std::numeric_limits::max()) { - for (auto range : deletions) - old_size += range.second - range.first; - for (auto range : insertions) - old_size -= range.second - range.first; - } - - modifications.clear(); - insertions.clear(); - moves.clear(); - m_move_mapping.clear(); - deletions.set(old_size); -} - -void CollectionChangeBuilder::move(size_t from, size_t to) -{ - REALM_ASSERT(from != to); - REALM_ASSERT(m_move_mapping.empty()); - - bool updated_existing_move = false; - for (auto& move : moves) { - if (move.to != from) { - // Shift other moves if this row is moving from one side of them - // to the other - if (move.to >= to && move.to < from) - ++move.to; - else if (move.to <= to && move.to > from) - --move.to; - continue; - } - REALM_ASSERT(!updated_existing_move); - - // Collapse A -> B, B -> C into a single A -> C move - move.to = to; - updated_existing_move = true; - - insertions.erase_at(from); - insertions.insert_at(to); - } - - if (!updated_existing_move) { - auto shifted_from = insertions.erase_or_unshift(from); - insertions.insert_at(to); - - // Don't report deletions/moves for newly inserted rows - if (shifted_from != npos) { - shifted_from = deletions.add_shifted(shifted_from); - moves.push_back({shifted_from, to}); - } - } - - bool modified = modifications.contains(from); - modifications.erase_at(from); - - if (modified) - modifications.insert_at(to); - else - modifications.shift_for_insert_at(to); -} - -void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) -{ - REALM_ASSERT(moves.empty()); - REALM_ASSERT(row_ndx <= last_row); - REALM_ASSERT(insertions.empty() || prev(insertions.end())->second - 1 <= last_row); - REALM_ASSERT(modifications.empty() || prev(modifications.end())->second - 1 <= last_row); - if (row_ndx == last_row) { - erase(row_ndx); - return; - } - - bool modified = modifications.contains(last_row); - if (modified) { - modifications.remove(last_row); - modifications.add(row_ndx); - } - else - modifications.remove(row_ndx); - - bool row_is_insertion = insertions.contains(row_ndx); - bool last_is_insertion = !insertions.empty() && prev(insertions.end())->second == last_row + 1; - REALM_ASSERT_DEBUG(insertions.empty() || prev(insertions.end())->second <= last_row + 1); - - // Collapse A -> B, B -> C into a single A -> C move - bool last_was_already_moved = false; - if (last_is_insertion) { - auto it = m_move_mapping.find(last_row); - if (it != m_move_mapping.end() && it->first == last_row) { - m_move_mapping[row_ndx] = it->second; - m_move_mapping.erase(it); - last_was_already_moved = true; - } - } - - // Remove moves to the row being deleted - if (row_is_insertion && !last_was_already_moved) { - auto it = m_move_mapping.find(row_ndx); - if (it != m_move_mapping.end() && it->first == row_ndx) - m_move_mapping.erase(it); - } - - // Don't report deletions/moves if last_row is newly inserted - if (last_is_insertion) { - insertions.remove(last_row); - } - // If it was previously moved, the unshifted source row has already been marked as deleted - else if (!last_was_already_moved) { - auto shifted_last_row = insertions.unshift(last_row); - shifted_last_row = deletions.add_shifted(shifted_last_row); - m_move_mapping[row_ndx] = shifted_last_row; - } - - // Don't mark the moved-over row as deleted if it was a new insertion - if (!row_is_insertion) { - deletions.add_shifted(insertions.unshift(row_ndx)); - insertions.add(row_ndx); - } - verify(); -} - -void CollectionChangeBuilder::verify() -{ -#ifdef REALM_DEBUG - for (auto&& move : moves) { - REALM_ASSERT(deletions.contains(move.from)); - REALM_ASSERT(insertions.contains(move.to)); - } -#endif -} - -namespace { -struct RowInfo { - size_t row_index; - size_t prev_tv_index; - size_t tv_index; - size_t shifted_tv_index; -}; - -void calculate_moves_unsorted(std::vector& new_rows, IndexSet& removed, CollectionChangeIndices& changeset) -{ - size_t expected = 0; - for (auto& row : new_rows) { - // With unsorted queries rows only move due to move_last_over(), which - // inherently can only move a row to earlier in the table. - REALM_ASSERT(row.shifted_tv_index >= expected); - if (row.shifted_tv_index == expected) { - ++expected; - continue; - } - - // This row isn't just the row after the previous one, but it still may - // not be a move if there were rows deleted between the two, so next - // calcuate what row should be here taking those in to account - size_t calc_expected = row.tv_index - changeset.insertions.count(0, row.tv_index) + removed.count(0, row.prev_tv_index); - if (row.shifted_tv_index == calc_expected) { - expected = calc_expected + 1; - continue; - } - - // The row still isn't the expected one, so it's a move - changeset.moves.push_back({row.prev_tv_index, row.tv_index}); - changeset.insertions.add(row.tv_index); - removed.add(row.prev_tv_index); - } -} - -class SortedMoveCalculator { -public: - SortedMoveCalculator(std::vector& new_rows, CollectionChangeIndices& changeset) - : m_modified(changeset.modifications) - { - std::vector old_candidates; - old_candidates.reserve(new_rows.size()); - for (auto& row : new_rows) { - old_candidates.push_back({row.row_index, row.prev_tv_index}); - } - std::sort(begin(old_candidates), end(old_candidates), [](auto a, auto b) { - return std::tie(a.tv_index, a.row_index) < std::tie(b.tv_index, b.row_index); - }); - - // First check if the order of any of the rows actually changed - size_t first_difference = npos; - for (size_t i = 0; i < old_candidates.size(); ++i) { - if (old_candidates[i].row_index != new_rows[i].row_index) { - first_difference = i; - break; - } - } - if (first_difference == npos) - return; - - // A map from row index -> tv index in new results - b.reserve(new_rows.size()); - for (size_t i = 0; i < new_rows.size(); ++i) - b.push_back({new_rows[i].row_index, i}); - std::sort(begin(b), end(b), [](auto a, auto b) { - return std::tie(a.row_index, a.tv_index) < std::tie(b.row_index, b.tv_index); - }); - - a = std::move(old_candidates); - - find_longest_matches(first_difference, a.size(), - first_difference, new_rows.size()); - m_longest_matches.push_back({a.size(), new_rows.size(), 0}); - - size_t i = first_difference, j = first_difference; - for (auto match : m_longest_matches) { - for (; i < match.i; ++i) - changeset.deletions.add(a[i].tv_index); - for (; j < match.j; ++j) - changeset.insertions.add(new_rows[j].tv_index); - i += match.size; - j += match.size; - } - } - -private: - struct Match { - size_t i, j, size, modified; - }; - struct Row { - size_t row_index; - size_t tv_index; - }; - - IndexSet const& m_modified; - std::vector m_longest_matches; - - std::vector a, b; - - Match find_longest_match(size_t begin1, size_t end1, size_t begin2, size_t end2) - { - struct Length { - size_t j, len; - }; - std::vector cur; - std::vector prev; - - auto length = [&](size_t j) -> size_t { - for (auto const& pair : prev) { - if (pair.j + 1 == j) - return pair.len + 1; - } - return 1; - }; - - Match best = {begin1, begin2, 0, 0}; - - for (size_t i = begin1; i < end1; ++i) { - cur.clear(); - - size_t ai = a[i].row_index; - // Find the TV indicies at which this row appears in the new results - // There should always be at least one (or it would have been filtered out earlier), - // but can be multiple if there are dupes - auto it = lower_bound(begin(b), end(b), Row{ai, 0}, - [](auto a, auto b) { return a.row_index < b.row_index; }); - REALM_ASSERT(it != end(b) && it->row_index == ai); - for (; it != end(b) && it->row_index == ai; ++it) { - size_t j = it->tv_index; - if (j < begin2) - continue; - if (j >= end2) - break; // b is sorted by tv_index so this can't transition from false to true - - size_t size = length(j); - cur.push_back({j, size}); - if (size > best.size) - best = {i - size + 1, j - size + 1, size, npos}; - // Given two equal-length matches, prefer the one with fewer modified rows - else if (size == best.size) { - if (best.modified == npos) - best.modified = m_modified.count(best.j - size + 1, best.j + 1); - auto count = m_modified.count(j - size + 1, j + 1); - if (count < best.modified) - best = {i - size + 1, j - size + 1, size, count}; - } - REALM_ASSERT(best.i >= begin1 && best.i + best.size <= end1); - REALM_ASSERT(best.j >= begin2 && best.j + best.size <= end2); - } - cur.swap(prev); - } - return best; - } - - void find_longest_matches(size_t begin1, size_t end1, size_t begin2, size_t end2) - { - // FIXME: recursion could get too deep here - // recursion depth worst case is currently O(N) and each recursion uses 320 bytes of stack - // could reduce worst case to O(sqrt(N)) (and typical case to O(log N)) - // biasing equal selections towards the middle, but that's still - // insufficient for Android's 8 KB stacks - auto m = find_longest_match(begin1, end1, begin2, end2); - if (!m.size) - return; - if (m.i > begin1 && m.j > begin2) - find_longest_matches(begin1, m.i, begin2, m.j); - m_longest_matches.push_back(m); - if (m.i + m.size < end2 && m.j + m.size < end2) - find_longest_matches(m.i + m.size, end1, m.j + m.size, end2); - } -}; - -} // Anonymous namespace - -CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector const& prev_rows, - std::vector const& next_rows, - std::function row_did_change, - bool sort) -{ - REALM_ASSERT_DEBUG(sort || std::is_sorted(begin(next_rows), end(next_rows))); - - CollectionChangeBuilder ret; - - size_t deleted = 0; - std::vector old_rows; - old_rows.reserve(prev_rows.size()); - for (size_t i = 0; i < prev_rows.size(); ++i) { - if (prev_rows[i] == npos) { - ++deleted; - ret.deletions.add(i); - } - else - old_rows.push_back({prev_rows[i], npos, i, i - deleted}); - } - std::sort(begin(old_rows), end(old_rows), [](auto& lft, auto& rgt) { - return lft.row_index < rgt.row_index; - }); - - std::vector new_rows; - new_rows.reserve(next_rows.size()); - for (size_t i = 0; i < next_rows.size(); ++i) { - new_rows.push_back({next_rows[i], npos, i, 0}); - } - std::sort(begin(new_rows), end(new_rows), [](auto& lft, auto& rgt) { - return lft.row_index < rgt.row_index; - }); - - IndexSet removed; - - size_t i = 0, j = 0; - while (i < old_rows.size() && j < new_rows.size()) { - auto old_index = old_rows[i]; - auto new_index = new_rows[j]; - if (old_index.row_index == new_index.row_index) { - new_rows[j].prev_tv_index = old_rows[i].tv_index; - new_rows[j].shifted_tv_index = old_rows[i].shifted_tv_index; - ++i; - ++j; - } - else if (old_index.row_index < new_index.row_index) { - removed.add(old_index.tv_index); - ++i; - } - else { - ret.insertions.add(new_index.tv_index); - ++j; - } - } - - for (; i < old_rows.size(); ++i) - removed.add(old_rows[i].tv_index); - for (; j < new_rows.size(); ++j) - ret.insertions.add(new_rows[j].tv_index); - - // Filter out the new insertions since we don't need them for any of the - // further calculations - new_rows.erase(std::remove_if(begin(new_rows), end(new_rows), - [](auto& row) { return row.prev_tv_index == npos; }), - end(new_rows)); - std::sort(begin(new_rows), end(new_rows), - [](auto& lft, auto& rgt) { return lft.tv_index < rgt.tv_index; }); - - for (auto& row : new_rows) { - if (row_did_change(row.row_index)) { - ret.modifications.add(row.tv_index); - } - } - - if (sort) { - SortedMoveCalculator(new_rows, ret); - } - else { - calculate_moves_unsorted(new_rows, removed, ret); - } - ret.deletions.add(removed); - ret.verify(); - -#ifdef REALM_DEBUG - { // Verify that applying the calculated change to prev_rows actually produces next_rows - auto rows = prev_rows; - auto it = util::make_reverse_iterator(ret.deletions.end()); - auto end = util::make_reverse_iterator(ret.deletions.begin()); - for (; it != end; ++it) { - rows.erase(rows.begin() + it->first, rows.begin() + it->second); - } - - for (auto i : ret.insertions.as_indexes()) { - rows.insert(rows.begin() + i, next_rows[i]); - } - - REALM_ASSERT(rows == next_rows); - } -#endif - - return ret; -} diff --git a/src/collection_notifications.hpp b/src/collection_notifications.hpp index d6779abc..a3023b39 100644 --- a/src/collection_notifications.hpp +++ b/src/collection_notifications.hpp @@ -23,8 +23,6 @@ #include "util/atomic_shared_ptr.hpp" #include -#include -#include namespace realm { namespace _impl { @@ -62,49 +60,9 @@ struct CollectionChangeIndices { std::vector moves; bool empty() const { return deletions.empty() && insertions.empty() && modifications.empty() && moves.empty(); } - - CollectionChangeIndices(CollectionChangeIndices const&) = default; - CollectionChangeIndices(CollectionChangeIndices&&) = default; - CollectionChangeIndices& operator=(CollectionChangeIndices const&) = default; - CollectionChangeIndices& operator=(CollectionChangeIndices&&) = default; - - CollectionChangeIndices(IndexSet deletions = {}, - IndexSet insertions = {}, - IndexSet modification = {}, - std::vector moves = {}); }; using CollectionChangeCallback = std::function; - -namespace _impl { -class CollectionChangeBuilder : public CollectionChangeIndices { -public: - using CollectionChangeIndices::CollectionChangeIndices; - - static CollectionChangeBuilder calculate(std::vector const& old_rows, - std::vector const& new_rows, - std::function row_did_change, - bool sort); - - void merge(CollectionChangeBuilder&&); - void clean_up_stale_moves(); - - void insert(size_t ndx, size_t count=1); - void modify(size_t ndx); - void erase(size_t ndx); - void move_over(size_t ndx, size_t last_ndx); - void clear(size_t old_size); - void move(size_t from, size_t to); - - void parse_complete(); - -private: - std::unordered_map m_move_mapping; - - void verify(); - -}; -} // namespace _impl } // namespace realm #endif // REALM_COLLECTION_NOTIFICATIONS_HPP diff --git a/src/impl/background_collection.cpp b/src/impl/background_collection.cpp index 7e623ab6..d0166cf1 100644 --- a/src/impl/background_collection.cpp +++ b/src/impl/background_collection.cpp @@ -251,3 +251,572 @@ void BackgroundCollection::detach() do_detach_from(*m_sg); m_sg = nullptr; } + +CollectionChangeBuilder::CollectionChangeBuilder(IndexSet deletions, + IndexSet insertions, + IndexSet modifications, + std::vector moves) +{ + this->deletions = std::move(deletions); + this->insertions = std::move(insertions); + this->modifications = std::move(modifications); + this->moves = std::move(moves); + + for (auto&& move : this->moves) { + this->deletions.add(move.from); + this->insertions.add(move.to); + } +} + +void CollectionChangeBuilder::merge(CollectionChangeBuilder&& c) +{ + if (c.empty()) + return; + if (empty()) { + *this = std::move(c); + return; + } + + verify(); + c.verify(); + + // First update any old moves + if (!c.moves.empty() || !c.deletions.empty() || !c.insertions.empty()) { + auto it = remove_if(begin(moves), end(moves), [&](auto& old) { + // Check if the moved row was moved again, and if so just update the destination + auto it = find_if(begin(c.moves), end(c.moves), [&](auto const& m) { + return old.to == m.from; + }); + if (it != c.moves.end()) { + if (modifications.contains(it->from)) + c.modifications.add(it->to); + old.to = it->to; + *it = c.moves.back(); + c.moves.pop_back(); + ++it; + return false; + } + + // Check if the destination was deleted + // Removing the insert for this move will happen later + if (c.deletions.contains(old.to)) + return true; + + // Update the destination to adjust for any new insertions and deletions + old.to = c.insertions.shift(c.deletions.unshift(old.to)); + return false; + }); + moves.erase(it, end(moves)); + } + + // Ignore new moves of rows which were previously inserted (the implicit + // delete from the move will remove the insert) + if (!insertions.empty() && !c.moves.empty()) { + c.moves.erase(remove_if(begin(c.moves), end(c.moves), + [&](auto const& m) { return insertions.contains(m.from); }), + end(c.moves)); + } + + // Ensure that any previously modified rows which were moved are still modified + if (!modifications.empty() && !c.moves.empty()) { + for (auto const& move : c.moves) { + if (modifications.contains(move.from)) + c.modifications.add(move.to); + } + } + + // Update the source position of new moves to compensate for the changes made + // in the old changeset + if (!deletions.empty() || !insertions.empty()) { + for (auto& move : c.moves) + move.from = deletions.shift(insertions.unshift(move.from)); + } + + moves.insert(end(moves), begin(c.moves), end(c.moves)); + + // New deletion indices have been shifted by the insertions, so unshift them + // before adding + deletions.add_shifted_by(insertions, c.deletions); + + // Drop any inserted-then-deleted rows, then merge in new insertions + insertions.erase_at(c.deletions); + insertions.insert_at(c.insertions); + + clean_up_stale_moves(); + + modifications.erase_at(c.deletions); + modifications.shift_for_insert_at(c.insertions); + modifications.add(c.modifications); + + c = {}; + verify(); +} + +void CollectionChangeBuilder::clean_up_stale_moves() +{ + // Look for moves which are now no-ops, and remove them plus the associated + // insert+delete. Note that this isn't just checking for from == to due to + // that rows can also be shifted by other inserts and deletes + moves.erase(remove_if(begin(moves), end(moves), [&](auto const& move) { + if (move.from - deletions.count(0, move.from) != move.to - insertions.count(0, move.to)) + return false; + deletions.remove(move.from); + insertions.remove(move.to); + return true; + }), end(moves)); +} + +void CollectionChangeBuilder::parse_complete() +{ + moves.reserve(m_move_mapping.size()); + for (auto move : m_move_mapping) { + REALM_ASSERT_DEBUG(deletions.contains(move.second)); + REALM_ASSERT_DEBUG(insertions.contains(move.first)); + moves.push_back({move.second, move.first}); + } + m_move_mapping.clear(); + std::sort(begin(moves), end(moves), + [](auto const& a, auto const& b) { return a.from < b.from; }); +} + +void CollectionChangeBuilder::modify(size_t ndx) +{ + modifications.add(ndx); +} + +void CollectionChangeBuilder::insert(size_t index, size_t count) +{ + modifications.shift_for_insert_at(index, count); + insertions.insert_at(index, count); + + for (auto& move : moves) { + if (move.to >= index) + ++move.to; + } +} + +void CollectionChangeBuilder::erase(size_t index) +{ + modifications.erase_at(index); + size_t unshifted = insertions.erase_or_unshift(index); + if (unshifted != npos) + deletions.add_shifted(unshifted); + + for (size_t i = 0; i < moves.size(); ++i) { + auto& move = moves[i]; + if (move.to == index) { + moves.erase(moves.begin() + i); + --i; + } + else if (move.to > index) + --move.to; + } +} + +void CollectionChangeBuilder::clear(size_t old_size) +{ + if (old_size != std::numeric_limits::max()) { + for (auto range : deletions) + old_size += range.second - range.first; + for (auto range : insertions) + old_size -= range.second - range.first; + } + + modifications.clear(); + insertions.clear(); + moves.clear(); + m_move_mapping.clear(); + deletions.set(old_size); +} + +void CollectionChangeBuilder::move(size_t from, size_t to) +{ + REALM_ASSERT(from != to); + + bool updated_existing_move = false; + for (auto& move : moves) { + if (move.to != from) { + // Shift other moves if this row is moving from one side of them + // to the other + if (move.to >= to && move.to < from) + ++move.to; + else if (move.to <= to && move.to > from) + --move.to; + continue; + } + REALM_ASSERT(!updated_existing_move); + + // Collapse A -> B, B -> C into a single A -> C move + move.to = to; + updated_existing_move = true; + + insertions.erase_at(from); + insertions.insert_at(to); + } + + if (!updated_existing_move) { + auto shifted_from = insertions.erase_or_unshift(from); + insertions.insert_at(to); + + // Don't report deletions/moves for newly inserted rows + if (shifted_from != npos) { + shifted_from = deletions.add_shifted(shifted_from); + moves.push_back({shifted_from, to}); + } + } + + bool modified = modifications.contains(from); + modifications.erase_at(from); + + if (modified) + modifications.insert_at(to); + else + modifications.shift_for_insert_at(to); +} + +void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) +{ + REALM_ASSERT(row_ndx <= last_row); + REALM_ASSERT(insertions.empty() || prev(insertions.end())->second - 1 <= last_row); + REALM_ASSERT(modifications.empty() || prev(modifications.end())->second - 1 <= last_row); + if (row_ndx == last_row) { + erase(row_ndx); + return; + } + + bool modified = modifications.contains(last_row); + if (modified) { + modifications.remove(last_row); + modifications.add(row_ndx); + } + else + modifications.remove(row_ndx); + + bool row_is_insertion = insertions.contains(row_ndx); + bool last_is_insertion = !insertions.empty() && prev(insertions.end())->second == last_row + 1; + REALM_ASSERT_DEBUG(insertions.empty() || prev(insertions.end())->second <= last_row + 1); + + // Collapse A -> B, B -> C into a single A -> C move + bool last_was_already_moved = false; + if (last_is_insertion) { + auto it = m_move_mapping.find(last_row); + if (it != m_move_mapping.end() && it->first == last_row) { + m_move_mapping[row_ndx] = it->second; + m_move_mapping.erase(it); + last_was_already_moved = true; + } + } + + // Remove moves to the row being deleted + if (row_is_insertion && !last_was_already_moved) { + auto it = m_move_mapping.find(row_ndx); + if (it != m_move_mapping.end() && it->first == row_ndx) + m_move_mapping.erase(it); + } + + // Don't report deletions/moves if last_row is newly inserted + if (last_is_insertion) { + insertions.remove(last_row); + } + // If it was previously moved, the unshifted source row has already been marked as deleted + else if (!last_was_already_moved) { + auto shifted_last_row = insertions.unshift(last_row); + shifted_last_row = deletions.add_shifted(shifted_last_row); + m_move_mapping[row_ndx] = shifted_last_row; + } + + // Don't mark the moved-over row as deleted if it was a new insertion + if (!row_is_insertion) { + deletions.add_shifted(insertions.unshift(row_ndx)); + insertions.add(row_ndx); + } + verify(); +} + +void CollectionChangeBuilder::verify() +{ +#ifdef REALM_DEBUG + for (auto&& move : moves) { + REALM_ASSERT(deletions.contains(move.from)); + REALM_ASSERT(insertions.contains(move.to)); + } +#endif +} + +namespace { +struct RowInfo { + size_t row_index; + size_t prev_tv_index; + size_t tv_index; + size_t shifted_tv_index; +}; + +void calculate_moves_unsorted(std::vector& new_rows, IndexSet& removed, CollectionChangeIndices& changeset) +{ + size_t expected = 0; + for (auto& row : new_rows) { + // With unsorted queries rows only move due to move_last_over(), which + // inherently can only move a row to earlier in the table. + REALM_ASSERT(row.shifted_tv_index >= expected); + if (row.shifted_tv_index == expected) { + ++expected; + continue; + } + + // This row isn't just the row after the previous one, but it still may + // not be a move if there were rows deleted between the two, so next + // calcuate what row should be here taking those in to account + size_t calc_expected = row.tv_index - changeset.insertions.count(0, row.tv_index) + removed.count(0, row.prev_tv_index); + if (row.shifted_tv_index == calc_expected) { + expected = calc_expected + 1; + continue; + } + + // The row still isn't the expected one, so it's a move + changeset.moves.push_back({row.prev_tv_index, row.tv_index}); + changeset.insertions.add(row.tv_index); + removed.add(row.prev_tv_index); + } +} + +class SortedMoveCalculator { +public: + SortedMoveCalculator(std::vector& new_rows, CollectionChangeIndices& changeset) + : m_modified(changeset.modifications) + { + std::vector old_candidates; + old_candidates.reserve(new_rows.size()); + for (auto& row : new_rows) { + old_candidates.push_back({row.row_index, row.prev_tv_index}); + } + std::sort(begin(old_candidates), end(old_candidates), [](auto a, auto b) { + return std::tie(a.tv_index, a.row_index) < std::tie(b.tv_index, b.row_index); + }); + + // First check if the order of any of the rows actually changed + size_t first_difference = npos; + for (size_t i = 0; i < old_candidates.size(); ++i) { + if (old_candidates[i].row_index != new_rows[i].row_index) { + first_difference = i; + break; + } + } + if (first_difference == npos) + return; + + // A map from row index -> tv index in new results + b.reserve(new_rows.size()); + for (size_t i = 0; i < new_rows.size(); ++i) + b.push_back({new_rows[i].row_index, i}); + std::sort(begin(b), end(b), [](auto a, auto b) { + return std::tie(a.row_index, a.tv_index) < std::tie(b.row_index, b.tv_index); + }); + + a = std::move(old_candidates); + + find_longest_matches(first_difference, a.size(), + first_difference, new_rows.size()); + m_longest_matches.push_back({a.size(), new_rows.size(), 0}); + + size_t i = first_difference, j = first_difference; + for (auto match : m_longest_matches) { + for (; i < match.i; ++i) + changeset.deletions.add(a[i].tv_index); + for (; j < match.j; ++j) + changeset.insertions.add(new_rows[j].tv_index); + i += match.size; + j += match.size; + } + } + +private: + struct Match { + size_t i, j, size, modified; + }; + struct Row { + size_t row_index; + size_t tv_index; + }; + + IndexSet const& m_modified; + std::vector m_longest_matches; + + std::vector a, b; + + Match find_longest_match(size_t begin1, size_t end1, size_t begin2, size_t end2) + { + struct Length { + size_t j, len; + }; + std::vector cur; + std::vector prev; + + auto length = [&](size_t j) -> size_t { + for (auto const& pair : prev) { + if (pair.j + 1 == j) + return pair.len + 1; + } + return 1; + }; + + Match best = {begin1, begin2, 0, 0}; + + for (size_t i = begin1; i < end1; ++i) { + cur.clear(); + + size_t ai = a[i].row_index; + // Find the TV indicies at which this row appears in the new results + // There should always be at least one (or it would have been filtered out earlier), + // but can be multiple if there are dupes + auto it = lower_bound(begin(b), end(b), Row{ai, 0}, + [](auto a, auto b) { return a.row_index < b.row_index; }); + REALM_ASSERT(it != end(b) && it->row_index == ai); + for (; it != end(b) && it->row_index == ai; ++it) { + size_t j = it->tv_index; + if (j < begin2) + continue; + if (j >= end2) + break; // b is sorted by tv_index so this can't transition from false to true + + size_t size = length(j); + cur.push_back({j, size}); + if (size > best.size) + best = {i - size + 1, j - size + 1, size, npos}; + // Given two equal-length matches, prefer the one with fewer modified rows + else if (size == best.size) { + if (best.modified == npos) + best.modified = m_modified.count(best.j - size + 1, best.j + 1); + auto count = m_modified.count(j - size + 1, j + 1); + if (count < best.modified) + best = {i - size + 1, j - size + 1, size, count}; + } + REALM_ASSERT(best.i >= begin1 && best.i + best.size <= end1); + REALM_ASSERT(best.j >= begin2 && best.j + best.size <= end2); + } + cur.swap(prev); + } + return best; + } + + void find_longest_matches(size_t begin1, size_t end1, size_t begin2, size_t end2) + { + // FIXME: recursion could get too deep here + // recursion depth worst case is currently O(N) and each recursion uses 320 bytes of stack + // could reduce worst case to O(sqrt(N)) (and typical case to O(log N)) + // biasing equal selections towards the middle, but that's still + // insufficient for Android's 8 KB stacks + auto m = find_longest_match(begin1, end1, begin2, end2); + if (!m.size) + return; + if (m.i > begin1 && m.j > begin2) + find_longest_matches(begin1, m.i, begin2, m.j); + m_longest_matches.push_back(m); + if (m.i + m.size < end2 && m.j + m.size < end2) + find_longest_matches(m.i + m.size, end1, m.j + m.size, end2); + } +}; + +} // Anonymous namespace + +CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector const& prev_rows, + std::vector const& next_rows, + std::function row_did_change, + bool sort) +{ + REALM_ASSERT_DEBUG(sort || std::is_sorted(begin(next_rows), end(next_rows))); + + CollectionChangeBuilder ret; + + size_t deleted = 0; + std::vector old_rows; + old_rows.reserve(prev_rows.size()); + for (size_t i = 0; i < prev_rows.size(); ++i) { + if (prev_rows[i] == npos) { + ++deleted; + ret.deletions.add(i); + } + else + old_rows.push_back({prev_rows[i], npos, i, i - deleted}); + } + std::sort(begin(old_rows), end(old_rows), [](auto& lft, auto& rgt) { + return lft.row_index < rgt.row_index; + }); + + std::vector new_rows; + new_rows.reserve(next_rows.size()); + for (size_t i = 0; i < next_rows.size(); ++i) { + new_rows.push_back({next_rows[i], npos, i, 0}); + } + std::sort(begin(new_rows), end(new_rows), [](auto& lft, auto& rgt) { + return lft.row_index < rgt.row_index; + }); + + IndexSet removed; + + size_t i = 0, j = 0; + while (i < old_rows.size() && j < new_rows.size()) { + auto old_index = old_rows[i]; + auto new_index = new_rows[j]; + if (old_index.row_index == new_index.row_index) { + new_rows[j].prev_tv_index = old_rows[i].tv_index; + new_rows[j].shifted_tv_index = old_rows[i].shifted_tv_index; + ++i; + ++j; + } + else if (old_index.row_index < new_index.row_index) { + removed.add(old_index.tv_index); + ++i; + } + else { + ret.insertions.add(new_index.tv_index); + ++j; + } + } + + for (; i < old_rows.size(); ++i) + removed.add(old_rows[i].tv_index); + for (; j < new_rows.size(); ++j) + ret.insertions.add(new_rows[j].tv_index); + + // Filter out the new insertions since we don't need them for any of the + // further calculations + new_rows.erase(std::remove_if(begin(new_rows), end(new_rows), + [](auto& row) { return row.prev_tv_index == npos; }), + end(new_rows)); + std::sort(begin(new_rows), end(new_rows), + [](auto& lft, auto& rgt) { return lft.tv_index < rgt.tv_index; }); + + for (auto& row : new_rows) { + if (row_did_change(row.row_index)) { + ret.modifications.add(row.tv_index); + } + } + + if (sort) { + SortedMoveCalculator(new_rows, ret); + } + else { + calculate_moves_unsorted(new_rows, removed, ret); + } + ret.deletions.add(removed); + ret.verify(); + +#ifdef REALM_DEBUG + { // Verify that applying the calculated change to prev_rows actually produces next_rows + auto rows = prev_rows; + auto it = util::make_reverse_iterator(ret.deletions.end()); + auto end = util::make_reverse_iterator(ret.deletions.begin()); + for (; it != end; ++it) { + rows.erase(rows.begin() + it->first, rows.begin() + it->second); + } + + for (auto i : ret.insertions.as_indexes()) { + rows.insert(rows.begin() + i, next_rows[i]); + } + + REALM_ASSERT(rows == next_rows); + } +#endif + + return ret; +} diff --git a/src/impl/background_collection.hpp b/src/impl/background_collection.hpp index 425c7b2e..84d0df09 100644 --- a/src/impl/background_collection.hpp +++ b/src/impl/background_collection.hpp @@ -26,11 +26,47 @@ #include #include #include +#include namespace realm { class Realm; namespace _impl { +class CollectionChangeBuilder : public CollectionChangeIndices { +public: + CollectionChangeBuilder(CollectionChangeBuilder const&) = default; + CollectionChangeBuilder(CollectionChangeBuilder&&) = default; + CollectionChangeBuilder& operator=(CollectionChangeBuilder const&) = default; + CollectionChangeBuilder& operator=(CollectionChangeBuilder&&) = default; + + CollectionChangeBuilder(IndexSet deletions = {}, + IndexSet insertions = {}, + IndexSet modification = {}, + std::vector moves = {}); + + static CollectionChangeBuilder calculate(std::vector const& old_rows, + std::vector const& new_rows, + std::function row_did_change, + bool sort); + + void merge(CollectionChangeBuilder&&); + void clean_up_stale_moves(); + + void insert(size_t ndx, size_t count=1); + void modify(size_t ndx); + void erase(size_t ndx); + void move_over(size_t ndx, size_t last_ndx); + void clear(size_t old_size); + void move(size_t from, size_t to); + + void parse_complete(); + +private: + std::unordered_map m_move_mapping; + + void verify(); + +}; struct ListChangeInfo { size_t table_ndx; size_t row_ndx; diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index 585f56b8..7eddc254 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -1,6 +1,6 @@ #include "catch.hpp" -#include "collection_notifications.hpp" +#include "impl/background_collection.hpp" #include "util/index_helpers.hpp" From 155d9497938d046e9cc35e4f98694de17f7a1b0f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 30 Mar 2016 12:12:05 -0700 Subject: [PATCH 62/93] Only track inserts and deletes for tables being queried directly # Conflicts: # src/collection_notifications.cpp # src/collection_notifications.hpp --- src/impl/background_collection.cpp | 18 ++++++++++++------ src/impl/background_collection.hpp | 7 ++++--- src/impl/realm_coordinator.cpp | 2 +- src/impl/results_notifier.cpp | 6 ++++++ src/impl/transact_log_handler.cpp | 14 ++++++++++---- tests/transaction_log_parsing.cpp | 9 ++++++--- 6 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/impl/background_collection.cpp b/src/impl/background_collection.cpp index d0166cf1..552f457b 100644 --- a/src/impl/background_collection.cpp +++ b/src/impl/background_collection.cpp @@ -166,10 +166,10 @@ void BackgroundCollection::add_required_change_info(TransactionChangeInfo& info) } auto max = *max_element(begin(m_relevant_tables), end(m_relevant_tables)) + 1; - if (max > info.tables_needed.size()) - info.tables_needed.resize(max, false); + if (max > info.table_modifications_needed.size()) + info.table_modifications_needed.resize(max, false); for (auto table_ndx : m_relevant_tables) { - info.tables_needed[table_ndx] = true; + info.table_modifications_needed[table_ndx] = true; } } @@ -384,9 +384,12 @@ void CollectionChangeBuilder::modify(size_t ndx) modifications.add(ndx); } -void CollectionChangeBuilder::insert(size_t index, size_t count) +void CollectionChangeBuilder::insert(size_t index, size_t count, bool track_moves) { modifications.shift_for_insert_at(index, count); + if (!track_moves) + return; + insertions.insert_at(index, count); for (auto& move : moves) { @@ -474,12 +477,12 @@ void CollectionChangeBuilder::move(size_t from, size_t to) modifications.shift_for_insert_at(to); } -void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) +void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row, bool track_moves) { REALM_ASSERT(row_ndx <= last_row); REALM_ASSERT(insertions.empty() || prev(insertions.end())->second - 1 <= last_row); REALM_ASSERT(modifications.empty() || prev(modifications.end())->second - 1 <= last_row); - if (row_ndx == last_row) { + if (track_moves && row_ndx == last_row) { erase(row_ndx); return; } @@ -492,6 +495,9 @@ void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row) else modifications.remove(row_ndx); + if (!track_moves) + return; + bool row_is_insertion = insertions.contains(row_ndx); bool last_is_insertion = !insertions.empty() && prev(insertions.end())->second == last_row + 1; REALM_ASSERT_DEBUG(insertions.empty() || prev(insertions.end())->second <= last_row + 1); diff --git a/src/impl/background_collection.hpp b/src/impl/background_collection.hpp index 84d0df09..2772f591 100644 --- a/src/impl/background_collection.hpp +++ b/src/impl/background_collection.hpp @@ -52,10 +52,10 @@ public: void merge(CollectionChangeBuilder&&); void clean_up_stale_moves(); - void insert(size_t ndx, size_t count=1); + void insert(size_t ndx, size_t count=1, bool track_moves=true); void modify(size_t ndx); void erase(size_t ndx); - void move_over(size_t ndx, size_t last_ndx); + void move_over(size_t ndx, size_t last_ndx, bool track_moves=true); void clear(size_t old_size); void move(size_t from, size_t to); @@ -75,7 +75,8 @@ struct ListChangeInfo { }; struct TransactionChangeInfo { - std::vector tables_needed; + std::vector table_modifications_needed; + std::vector table_moves_needed; std::vector lists; std::vector tables; diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 4425d2d3..55ee1f9e 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -362,7 +362,7 @@ void RealmCoordinator::run_async_notifiers() for (auto& notifier : new_notifiers) { if (version != notifier->version()) { transaction::advance(*m_advancer_sg, *info, notifier->version()); - change_info.push_back({{}, std::move(info->lists)}); + change_info.push_back({{}, {}, std::move(info->lists)}); info = &change_info.back(); version = notifier->version(); } diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index 44782305..8d04f771 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -65,6 +65,12 @@ bool ResultsNotifier::do_add_required_change_info(TransactionChangeInfo& info) { REALM_ASSERT(m_query); m_info = &info; + + auto table_ndx = m_query->get_table()->get_index_in_group(); + if (info.table_moves_needed.size() <= table_ndx) + info.table_moves_needed.resize(table_ndx + 1); + info.table_moves_needed[table_ndx] = true; + return m_initial_run_complete && have_callbacks(); } diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index c0ea4531..41c0701d 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -427,7 +427,7 @@ class LinkViewObserver : public TransactLogValidationMixin, public MarkDirtyMixi _impl::CollectionChangeBuilder* get_change() { auto tbl_ndx = current_table(); - if (tbl_ndx >= m_info.tables_needed.size() || !m_info.tables_needed[tbl_ndx]) + if (tbl_ndx >= m_info.table_modifications_needed.size() || !m_info.table_modifications_needed[tbl_ndx]) return nullptr; if (m_info.tables.size() <= tbl_ndx) { m_info.tables.resize(std::max(m_info.tables.size() * 2, tbl_ndx + 1)); @@ -435,6 +435,12 @@ class LinkViewObserver : public TransactLogValidationMixin, public MarkDirtyMixi return &m_info.tables[tbl_ndx]; } + bool need_move_info() const + { + auto tbl_ndx = current_table(); + return tbl_ndx < m_info.table_moves_needed.size() && m_info.table_moves_needed[tbl_ndx]; + } + public: LinkViewObserver(_impl::TransactionChangeInfo& info) : m_info(info) { } @@ -521,7 +527,7 @@ public: { REALM_ASSERT(!unordered); if (auto change = get_change()) - change->insert(row_ndx, num_rows_to_insert); + change->insert(row_ndx, num_rows_to_insert, need_move_info()); return true; } @@ -545,7 +551,7 @@ public: } if (auto change = get_change()) - change->move_over(row_ndx, last_row); + change->move_over(row_ndx, last_row, need_move_info()); return true; } @@ -599,7 +605,7 @@ void advance(SharedGroup& sg, TransactionChangeInfo& info, SharedGroup::VersionID version) { - if (info.tables_needed.empty() && info.lists.empty()) { + if (info.table_modifications_needed.empty() && info.lists.empty()) { LangBindHelper::advance_read(sg, version); } else { diff --git a/tests/transaction_log_parsing.cpp b/tests/transaction_log_parsing.cpp index aed20b8a..da394289 100644 --- a/tests/transaction_log_parsing.cpp +++ b/tests/transaction_log_parsing.cpp @@ -37,7 +37,8 @@ public: _impl::CollectionChangeBuilder c; _impl::TransactionChangeInfo info; info.lists.push_back({ndx, 0, 0, &c}); - info.tables_needed.resize(m_group.size(), true); + info.table_modifications_needed.resize(m_group.size(), true); + info.table_moves_needed.resize(m_group.size(), true); _impl::transaction::advance(m_sg, info); if (info.lists.empty()) { @@ -204,7 +205,8 @@ TEST_CASE("Transaction log parsing") { r->commit_transaction(); _impl::TransactionChangeInfo info; - info.tables_needed.resize(g.size(), true); + info.table_modifications_needed.resize(g.size(), true); + info.table_moves_needed.resize(g.size(), true); _impl::transaction::advance(sg, info); return info; }; @@ -349,7 +351,8 @@ TEST_CASE("Transaction log parsing") { r->commit_transaction(); _impl::TransactionChangeInfo info; - info.tables_needed = tables_needed; + info.table_modifications_needed = tables_needed; + info.table_moves_needed = tables_needed; _impl::transaction::advance(sg, info); return info; }; From c12d87427a7f1cfc3a7086d7514c54b9f2214f2f Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 11 Apr 2016 13:24:04 -0700 Subject: [PATCH 63/93] Clean up includes and forward declarations a bit --- src/collection_notifications.hpp | 3 +++ src/impl/background_collection.hpp | 4 +++- src/impl/list_notifier.cpp | 1 - src/impl/realm_coordinator.cpp | 3 +-- src/impl/realm_coordinator.hpp | 10 ++-------- src/impl/results_notifier.cpp | 1 - src/impl/results_notifier.hpp | 5 ----- src/impl/transact_log_handler.cpp | 2 +- src/impl/transact_log_handler.hpp | 3 --- src/index_set.hpp | 7 ++++--- src/list.cpp | 1 + src/list.hpp | 6 ++++-- src/object_accessor.hpp | 2 +- src/object_store.hpp | 8 +++----- src/results.hpp | 2 -- src/schema.cpp | 3 ++- src/schema.hpp | 3 --- src/shared_realm.cpp | 3 --- src/shared_realm.hpp | 5 ----- 19 files changed, 25 insertions(+), 47 deletions(-) diff --git a/src/collection_notifications.hpp b/src/collection_notifications.hpp index a3023b39..f6366f48 100644 --- a/src/collection_notifications.hpp +++ b/src/collection_notifications.hpp @@ -23,6 +23,9 @@ #include "util/atomic_shared_ptr.hpp" #include +#include +#include +#include namespace realm { namespace _impl { diff --git a/src/impl/background_collection.hpp b/src/impl/background_collection.hpp index 2772f591..2c8047e6 100644 --- a/src/impl/background_collection.hpp +++ b/src/impl/background_collection.hpp @@ -23,8 +23,10 @@ #include -#include +#include +#include #include +#include #include #include diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp index a0143552..11a74bb4 100644 --- a/src/impl/list_notifier.cpp +++ b/src/impl/list_notifier.cpp @@ -18,7 +18,6 @@ #include "impl/list_notifier.hpp" -#include "impl/realm_coordinator.hpp" #include "shared_realm.hpp" #include diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 55ee1f9e..1fd0cb2e 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -28,9 +28,8 @@ #include #include #include +#include -#include -#include #include using namespace realm; diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index ba24d7d0..899f2c03 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -19,25 +19,19 @@ #ifndef REALM_COORDINATOR_HPP #define REALM_COORDINATOR_HPP -#include "index_set.hpp" #include "shared_realm.hpp" -#include - -#include +#include namespace realm { class Replication; -class Results; class Schema; class SharedGroup; -class Table; -struct CollectionChangeIndices; +class StringData; namespace _impl { class BackgroundCollection; class ExternalCommitHelper; -class ListNotifier; class WeakRealmNotifier; // RealmCoordinator manages the weak cache of Realm instances and communication diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index 8d04f771..dcd51b8e 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -18,7 +18,6 @@ #include "impl/results_notifier.hpp" -#include "impl/realm_coordinator.hpp" #include "results.hpp" using namespace realm; diff --git a/src/impl/results_notifier.hpp b/src/impl/results_notifier.hpp index 81f659c4..3dfc81f6 100644 --- a/src/impl/results_notifier.hpp +++ b/src/impl/results_notifier.hpp @@ -24,13 +24,8 @@ #include -#include -#include - namespace realm { namespace _impl { -struct TransactionChangeInfo; - class ResultsNotifier : public BackgroundCollection { public: ResultsNotifier(Results& target); diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index 41c0701d..d33a64fc 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -20,8 +20,8 @@ #include "binding_context.hpp" #include "impl/background_collection.hpp" +#include "index_set.hpp" -#include #include #include diff --git a/src/impl/transact_log_handler.hpp b/src/impl/transact_log_handler.hpp index c64d09db..96dbbfda 100644 --- a/src/impl/transact_log_handler.hpp +++ b/src/impl/transact_log_handler.hpp @@ -21,11 +21,8 @@ #include -#include "index_set.hpp" - namespace realm { class BindingContext; -class SharedGroup; namespace _impl { struct TransactionChangeInfo; diff --git a/src/index_set.hpp b/src/index_set.hpp index 3a4ee1ea..f0f7599e 100644 --- a/src/index_set.hpp +++ b/src/index_set.hpp @@ -19,14 +19,15 @@ #ifndef REALM_INDEX_SET_HPP #define REALM_INDEX_SET_HPP +#include #include +#include #include +#include +#include #include -#include namespace realm { -class IndexSet; - namespace _impl { template class MutableChunkedRangeVectorIterator; diff --git a/src/list.cpp b/src/list.cpp index 38adfac8..fec20772 100644 --- a/src/list.cpp +++ b/src/list.cpp @@ -22,6 +22,7 @@ #include "impl/realm_coordinator.hpp" #include "results.hpp" +#include #include #include diff --git a/src/list.hpp b/src/list.hpp index 1b4f2a07..2be94748 100644 --- a/src/list.hpp +++ b/src/list.hpp @@ -21,15 +21,17 @@ #include "collection_notifications.hpp" -#include +#include +#include +#include #include namespace realm { -template class BasicRowExpr; using RowExpr = BasicRowExpr; class ObjectSchema; +class Query; class Realm; class Results; struct SortOrder; diff --git a/src/object_accessor.hpp b/src/object_accessor.hpp index 85821a10..e2e1d086 100644 --- a/src/object_accessor.hpp +++ b/src/object_accessor.hpp @@ -25,7 +25,7 @@ #include "schema.hpp" #include "shared_realm.hpp" -#include +#include namespace realm { diff --git a/src/object_store.hpp b/src/object_store.hpp index 28b75c89..670cbf53 100644 --- a/src/object_store.hpp +++ b/src/object_store.hpp @@ -22,14 +22,12 @@ #include "schema.hpp" #include "property.hpp" +#include + #include -#include -#include - -#include - namespace realm { + class Group; class ObjectSchemaValidationException; class Schema; diff --git a/src/results.hpp b/src/results.hpp index 65a971df..388b85fe 100644 --- a/src/results.hpp +++ b/src/results.hpp @@ -23,7 +23,6 @@ #include "shared_realm.hpp" #include -#include #include #include @@ -31,7 +30,6 @@ namespace realm { template class BasicRowExpr; using RowExpr = BasicRowExpr
; class Mixed; -class Results; class ObjectSchema; namespace _impl { diff --git a/src/schema.cpp b/src/schema.cpp index 172ecce9..50cfc670 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -18,10 +18,11 @@ #include "schema.hpp" -#include "object_schema.hpp" #include "object_store.hpp" #include "property.hpp" +#include + using namespace realm; static bool compare_by_name(ObjectSchema const& lft, ObjectSchema const& rgt) { diff --git a/src/schema.hpp b/src/schema.hpp index 3f11c026..3a4e026f 100644 --- a/src/schema.hpp +++ b/src/schema.hpp @@ -20,14 +20,11 @@ #define REALM_SCHEMA_HPP #include "object_schema.hpp" -#include "property.hpp" #include #include namespace realm { -class ObjectSchema; - class Schema : private std::vector { private: using base = std::vector; diff --git a/src/shared_realm.cpp b/src/shared_realm.cpp index 865cbe6b..dc0ba898 100644 --- a/src/shared_realm.cpp +++ b/src/shared_realm.cpp @@ -19,7 +19,6 @@ #include "shared_realm.hpp" #include "binding_context.hpp" -#include "impl/external_commit_helper.hpp" #include "impl/realm_coordinator.hpp" #include "impl/transact_log_handler.hpp" #include "object_store.hpp" @@ -28,8 +27,6 @@ #include #include -#include - using namespace realm; using namespace realm::_impl; diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index 187c0211..f1ea3b2a 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -21,20 +21,15 @@ #include "schema.hpp" -#include - #include -#include #include #include #include -#include namespace realm { class BindingContext; class Group; class Realm; - class RealmDelegate; class Replication; class SharedGroup; typedef std::shared_ptr SharedRealm; From 2ce0e2e37f53d5b67bd469e17e42c741b1074dd8 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 11 Apr 2016 14:36:35 -0700 Subject: [PATCH 64/93] Remove transaction log handlers for old pk functions --- src/impl/transact_log_handler.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index d33a64fc..129384a9 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -100,7 +100,6 @@ public: } bool insert_column(size_t, DataType, StringData, bool) { return schema_error_unless_new_table(); } bool insert_link_column(size_t, DataType, StringData, size_t, size_t) { return schema_error_unless_new_table(); } - bool add_primary_key(size_t) { return schema_error_unless_new_table(); } bool set_link_type(size_t, LinkType) { return schema_error_unless_new_table(); } // Removing or renaming things while a Realm is open is never supported @@ -109,7 +108,6 @@ public: bool erase_column(size_t) { schema_error(); } bool erase_link_column(size_t, size_t, size_t) { schema_error(); } bool rename_column(size_t, StringData) { schema_error(); } - bool remove_primary_key() { schema_error(); } bool move_column(size_t, size_t) { schema_error(); } bool move_group_level_table(size_t, size_t) { schema_error(); } From 32b05314f564bc0f9b726c422819f400a7cc6df9 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 11 Apr 2016 14:37:46 -0700 Subject: [PATCH 65/93] Remove extraneous semicolons --- src/object_accessor.hpp | 4 ++-- src/parser/query_builder.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/object_accessor.hpp b/src/object_accessor.hpp index e2e1d086..3fea1741 100644 --- a/src/object_accessor.hpp +++ b/src/object_accessor.hpp @@ -143,7 +143,7 @@ namespace realm { "Setting invalid property '" + prop_name + "' on object '" + m_object_schema->name + "'."); } set_property_value_impl(ctx, *prop, value, try_update); - }; + } template inline ValueType Object::get_property_value(ContextType ctx, std::string prop_name) @@ -154,7 +154,7 @@ namespace realm { "Getting invalid property '" + prop_name + "' on object '" + m_object_schema->name + "'."); } return get_property_value_impl(ctx, *prop); - }; + } template inline void Object::set_property_value_impl(ContextType ctx, const Property &property, ValueType value, bool try_update) diff --git a/src/parser/query_builder.hpp b/src/parser/query_builder.hpp index bccc2e13..f9dba1bd 100644 --- a/src/parser/query_builder.hpp +++ b/src/parser/query_builder.hpp @@ -50,7 +50,7 @@ class Arguments { template class ArgumentConverter : public Arguments { public: - ArgumentConverter(ContextType context, std::vector arguments) : m_arguments(arguments), m_ctx(context) {}; + ArgumentConverter(ContextType context, std::vector arguments) : m_arguments(arguments), m_ctx(context) {} using Accessor = realm::NativeAccessor; virtual bool bool_for_argument(size_t argument_index) { return Accessor::to_bool(m_ctx, argument_at(argument_index)); } From 83b4d8ded24ac44c7e9b1c45344dc0d32e8d1a74 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 12 Apr 2016 15:32:52 -0700 Subject: [PATCH 66/93] Make RealmCoordinator::run_async_notifiers a bit less gross --- src/impl/realm_coordinator.cpp | 173 ++++++++++++++++++--------------- 1 file changed, 96 insertions(+), 77 deletions(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 1fd0cb2e..a9866899 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -302,6 +302,85 @@ void RealmCoordinator::on_change() } } +static SharedGroup::VersionID advance_mixed_version_notifiers(std::vector>& notifiers, + SharedGroup& sg, + std::vector& change_info) +{ + if (notifiers.empty()) + return SharedGroup::VersionID{}; + + // Sort the notifiers by their source version so that we can pull them + // all forward to the latest version in a single pass over the transaction log + std::sort(notifiers.begin(), notifiers.end(), + [](auto&& lft, auto&& rgt) { return lft->version() < rgt->version(); }); + auto version = sg.get_version_of_current_transaction(); + REALM_ASSERT(version == notifiers.front()->version()); + + // Preallocate the required amount of space in the vector so that we can + // safely give out pointers to within the vector + { + size_t count = 1; + for (auto it = notifiers.begin(), next = it + 1; next != notifiers.end(); ++it, ++next) { + if ((*it)->version() != (*next)->version()) + ++count; + } + change_info.reserve(count); + change_info.resize(1); + } + + TransactionChangeInfo* info = &change_info.back(); + + // Advance each of the new notifiers to the latest version, attaching them + // to the SG at their handover version. This requires a unique + // TransactionChangeInfo for each source version, so that things don't + // see changes from before the version they were handed over from. + // Each Info has all of the changes between that source version and the + // next source version, and they'll be merged together later after + // releasing the lock + for (auto& notifier : notifiers) { + if (version != notifier->version()) { + transaction::advance(sg, *info, notifier->version()); + change_info.push_back({{}, {}, std::move(info->lists)}); + info = &change_info.back(); + version = notifier->version(); + } + notifier->attach_to(sg); + notifier->add_required_change_info(*info); + } + + transaction::advance(sg, *info); + + // We now need to combine the transaction change info objects so that all of + // the notifiers see the complete set of changes from their first version to + // the most recent one + for (size_t i = change_info.size() - 1; i > 0; --i) { + auto& cur = change_info[i]; + if (cur.tables.empty()) + continue; + auto& prev = change_info[i - 1]; + if (prev.tables.empty()) { + prev.tables = cur.tables; + continue; + } + + for (size_t j = 0; j < prev.tables.size() && j < cur.tables.size(); ++j) { + prev.tables[j].merge(CollectionChangeBuilder{cur.tables[j]}); + } + prev.tables.reserve(cur.tables.size()); + while (prev.tables.size() < cur.tables.size()) { + prev.tables.push_back(cur.tables[prev.tables.size()]); + } + } + + for (auto& notifier : notifiers) { + notifier->detach(); + } + version = sg.get_version_of_current_transaction(); + sg.end_read(); + + return version; +} + void RealmCoordinator::run_async_notifiers() { std::unique_lock lock(m_notifier_mutex); @@ -322,72 +401,24 @@ void RealmCoordinator::run_async_notifiers() return; } - std::vector change_info; - SharedGroup::VersionID version; - + // Advance all of the new notifiers to the most recent version, if any auto new_notifiers = std::move(m_new_notifiers); - if (new_notifiers.empty()) { - change_info.resize(1); - } - else { - // Sort newly added notifiers by their source version so that we can pull them - // all forward to the latest version in a single pass over the transaction log - std::sort(new_notifiers.begin(), new_notifiers.end(), - [](auto&& lft, auto&& rgt) { return lft->version() < rgt->version(); }); - version = m_advancer_sg->get_version_of_current_transaction(); - REALM_ASSERT(version == new_notifiers.front()->version()); - - // Preallocate the required amount of space in the vector so that we can - // safely give out pointers to within the vector - { - size_t count = 2; - for (auto it = new_notifiers.begin(), next = it + 1; next != new_notifiers.end(); ++it, ++next) { - if ((*it)->version() != (*next)->version()) - ++count; - } - change_info.reserve(count); - change_info.resize(2); - } - - TransactionChangeInfo* info = &change_info.back(); - - // Advance each of the new notifiers to the latest version, attaching them - // to the SG at their handover version. This requires a unique - // TransactionChangeInfo for each source version, so that things don't - // see changes from before the version they were handed over from. - // Each Info has all of the changes between that source version and the - // next source version, and they'll be merged together later after - // releasing the lock - for (auto& notifier : new_notifiers) { - if (version != notifier->version()) { - transaction::advance(*m_advancer_sg, *info, notifier->version()); - change_info.push_back({{}, {}, std::move(info->lists)}); - info = &change_info.back(); - version = notifier->version(); - } - notifier->attach_to(*m_advancer_sg); - notifier->add_required_change_info(*info); - } - - transaction::advance(*m_advancer_sg, *info); - - for (auto& notifier : new_notifiers) { - notifier->detach(); - } - version = m_advancer_sg->get_version_of_current_transaction(); - m_advancer_sg->end_read(); - } + std::vector incremental_change_info; + auto version = advance_mixed_version_notifiers(new_notifiers, *m_advancer_sg, + incremental_change_info); // Make a copy of the notifiers vector and then release the lock to avoid // blocking other threads trying to register or unregister notifiers while we run them auto notifiers = m_notifiers; lock.unlock(); + // Advance the non-new notifiers to the same version as we advanced the new + // ones to (or the latest if there were no new ones) + TransactionChangeInfo change_info; for (auto& notifier : notifiers) { - notifier->add_required_change_info(change_info[0]); + notifier->add_required_change_info(change_info); } - - transaction::advance(*m_notifier_sg, change_info[0], version); + transaction::advance(*m_notifier_sg, change_info, version); // Attach the new notifiers to the main SG and move them to the main list for (auto& notifier : new_notifiers) { @@ -395,28 +426,9 @@ void RealmCoordinator::run_async_notifiers() } std::move(new_notifiers.begin(), new_notifiers.end(), std::back_inserter(notifiers)); - for (size_t i = change_info.size() - 1; i > 1; --i) { - auto& cur = change_info[i]; - if (cur.tables.empty()) - continue; - auto& prev = change_info[i - 1]; - if (prev.tables.empty()) { - prev.tables = cur.tables; - continue; - } - - for (size_t j = 0; j < prev.tables.size() && j < cur.tables.size(); ++j) { - prev.tables[j].merge(CollectionChangeBuilder{cur.tables[j]}); - } - prev.tables.reserve(cur.tables.size()); - while (prev.tables.size() < cur.tables.size()) { - prev.tables.push_back(cur.tables[prev.tables.size()]); - } - } - // Copy the list change info if there's multiple LinkViews for the same LinkList auto id = [](auto const& list) { return std::tie(list.table_ndx, list.col_ndx, list.row_ndx); }; - for (auto& info : change_info) { + auto merge_linkview_info = [&](TransactionChangeInfo& info) { for (size_t i = 1; i < info.lists.size(); ++i) { for (size_t j = i; j > 0; --j) { if (id(info.lists[i]) == id(info.lists[j - 1])) { @@ -424,8 +436,15 @@ void RealmCoordinator::run_async_notifiers() } } } + }; + + merge_linkview_info(change_info); + for (auto& info : incremental_change_info) { + merge_linkview_info(info); } + // Change info is now all ready, so the notifiers can now perform their + // background work for (auto& notifier : notifiers) { notifier->run(); } From 69cefd052eab1f1f663f3c1f2b49ceec7e281e8c Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 13 Apr 2016 12:48:15 -0700 Subject: [PATCH 67/93] Deliver the new TableView even if it did not change Even if the new TV has the same rows as the old one, we need to hand it over to the destination thread to bump the outside version of the destination thread's TV to avoid rerunning the query there. --- src/impl/results_notifier.cpp | 33 +++++++++++++++++++++------------ src/impl/results_notifier.hpp | 3 +++ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index dcd51b8e..f53f060d 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -73,7 +73,7 @@ bool ResultsNotifier::do_add_required_change_info(TransactionChangeInfo& info) return m_initial_run_complete && have_callbacks(); } -void ResultsNotifier::run() +bool ResultsNotifier::need_to_run() { REALM_ASSERT(m_info); REALM_ASSERT(!m_tv.is_attached()); @@ -82,7 +82,7 @@ void ResultsNotifier::run() auto lock = lock_target(); // Don't run the query if the results aren't actually going to be used if (!get_realm() || (!have_callbacks() && !m_target_results->wants_background_updates())) { - return; + return false; } } @@ -91,16 +91,15 @@ void ResultsNotifier::run() // Make an empty tableview from the query to get the table version, since // Query doesn't expose it if (m_query->find_all(0, 0, 0).sync_if_needed() == m_last_seen_version) { - return; + return false; } } - m_tv = m_query->find_all(); - if (m_sort) { - m_tv.sort(m_sort.column_indices, m_sort.ascending); - } - m_last_seen_version = m_tv.sync_if_needed(); + return true; +} +void ResultsNotifier::calculate_changes() +{ size_t table_ndx = m_query->get_table()->get_index_in_group(); if (m_initial_run_complete) { auto changes = table_ndx < m_info->tables.size() ? &m_info->tables[table_ndx] : nullptr; @@ -129,10 +128,6 @@ void ResultsNotifier::run() m_sort || m_from_linkview); m_previous_rows = std::move(next_rows); - if (m_changes.empty()) { - m_tv = {}; - return; - } } else { m_previous_rows.resize(m_tv.size()); @@ -141,6 +136,20 @@ void ResultsNotifier::run() } } +void ResultsNotifier::run() +{ + if (!need_to_run()) + return; + + m_tv = m_query->find_all(); + if (m_sort) { + m_tv.sort(m_sort.column_indices, m_sort.ascending); + } + m_last_seen_version = m_tv.sync_if_needed(); + + calculate_changes(); +} + void ResultsNotifier::do_prepare_handover(SharedGroup& sg) { if (!m_tv.is_attached()) { diff --git a/src/impl/results_notifier.hpp b/src/impl/results_notifier.hpp index 3dfc81f6..03b4be60 100644 --- a/src/impl/results_notifier.hpp +++ b/src/impl/results_notifier.hpp @@ -62,6 +62,9 @@ private: // can lead to deliver() being called before that bool m_initial_run_complete = false; + bool need_to_run(); + void calculate_changes(); + void run() override; void do_prepare_handover(SharedGroup&) override; bool do_deliver(SharedGroup& sg) override; From ea3a2f471114a366c74dbbf753733c7c441bbd32 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 14 Apr 2016 11:28:19 -0700 Subject: [PATCH 68/93] Refactor the incremental change tracking for mixed source versions --- src/impl/realm_coordinator.cpp | 189 ++++++++++++++++++--------------- 1 file changed, 105 insertions(+), 84 deletions(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index a9866899..5e7314d3 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -302,84 +302,98 @@ void RealmCoordinator::on_change() } } -static SharedGroup::VersionID advance_mixed_version_notifiers(std::vector>& notifiers, - SharedGroup& sg, - std::vector& change_info) -{ - if (notifiers.empty()) - return SharedGroup::VersionID{}; - - // Sort the notifiers by their source version so that we can pull them - // all forward to the latest version in a single pass over the transaction log - std::sort(notifiers.begin(), notifiers.end(), - [](auto&& lft, auto&& rgt) { return lft->version() < rgt->version(); }); - auto version = sg.get_version_of_current_transaction(); - REALM_ASSERT(version == notifiers.front()->version()); - - // Preallocate the required amount of space in the vector so that we can - // safely give out pointers to within the vector +namespace { +class IncrementalChangeInfo { +public: + IncrementalChangeInfo(SharedGroup& sg, + std::vector>& notifiers) + : m_sg(sg) { + if (notifiers.empty()) + return; + + auto cmp = [&](auto&& lft, auto&& rgt) { + return lft->version() < rgt->version(); + }; + + // Sort the notifiers by their source version so that we can pull them + // all forward to the latest version in a single pass over the transaction log + std::sort(notifiers.begin(), notifiers.end(), cmp); + + // Preallocate the required amount of space in the vector so that we can + // safely give out pointers to within the vector size_t count = 1; for (auto it = notifiers.begin(), next = it + 1; next != notifiers.end(); ++it, ++next) { - if ((*it)->version() != (*next)->version()) + if (cmp(*it, *next)) ++count; } - change_info.reserve(count); - change_info.resize(1); + m_info.reserve(count); + m_info.resize(1); + m_current = &m_info[0]; } - TransactionChangeInfo* info = &change_info.back(); + TransactionChangeInfo& current() const { return *m_current; } - // Advance each of the new notifiers to the latest version, attaching them - // to the SG at their handover version. This requires a unique - // TransactionChangeInfo for each source version, so that things don't - // see changes from before the version they were handed over from. - // Each Info has all of the changes between that source version and the - // next source version, and they'll be merged together later after - // releasing the lock - for (auto& notifier : notifiers) { - if (version != notifier->version()) { - transaction::advance(sg, *info, notifier->version()); - change_info.push_back({{}, {}, std::move(info->lists)}); - info = &change_info.back(); - version = notifier->version(); + bool advance_incremental(SharedGroup::VersionID version) + { + if (version != m_sg.get_version_of_current_transaction()) { + transaction::advance(m_sg, *m_current, version); + // FIXME: needs to copy tables to watch? + m_info.push_back({{}, {}, std::move(m_current->lists)}); + m_current = &m_info.back(); + return true; } - notifier->attach_to(sg); - notifier->add_required_change_info(*info); + return false; } - transaction::advance(sg, *info); - - // We now need to combine the transaction change info objects so that all of - // the notifiers see the complete set of changes from their first version to - // the most recent one - for (size_t i = change_info.size() - 1; i > 0; --i) { - auto& cur = change_info[i]; - if (cur.tables.empty()) - continue; - auto& prev = change_info[i - 1]; - if (prev.tables.empty()) { - prev.tables = cur.tables; - continue; + void advance_to_final(SharedGroup::VersionID version) + { + if (!m_current) { + transaction::advance(m_sg, nullptr, version); + return; } - for (size_t j = 0; j < prev.tables.size() && j < cur.tables.size(); ++j) { - prev.tables[j].merge(CollectionChangeBuilder{cur.tables[j]}); + transaction::advance(m_sg, *m_current, version); + + // We now need to combine the transaction change info objects so that all of + // the notifiers see the complete set of changes from their first version to + // the most recent one + for (size_t i = m_info.size() - 1; i > 0; --i) { + auto& cur = m_info[i]; + if (cur.tables.empty()) + continue; + auto& prev = m_info[i - 1]; + if (prev.tables.empty()) { + prev.tables = cur.tables; + continue; + } + + for (size_t j = 0; j < prev.tables.size() && j < cur.tables.size(); ++j) { + prev.tables[j].merge(CollectionChangeBuilder{cur.tables[j]}); + } + prev.tables.reserve(cur.tables.size()); + while (prev.tables.size() < cur.tables.size()) { + prev.tables.push_back(cur.tables[prev.tables.size()]); + } } - prev.tables.reserve(cur.tables.size()); - while (prev.tables.size() < cur.tables.size()) { - prev.tables.push_back(cur.tables[prev.tables.size()]); + + // Copy the list change info if there's multiple LinkViews for the same LinkList + auto id = [](auto const& list) { return std::tie(list.table_ndx, list.col_ndx, list.row_ndx); }; + for (size_t i = 1; i < m_current->lists.size(); ++i) { + for (size_t j = i; j > 0; --j) { + if (id(m_current->lists[i]) == id(m_current->lists[j - 1])) { + m_current->lists[j - 1].changes->merge(CollectionChangeBuilder{*m_current->lists[i].changes}); + } + } } } - for (auto& notifier : notifiers) { - notifier->detach(); - } - version = sg.get_version_of_current_transaction(); - sg.end_read(); - - return version; -} +private: + std::vector m_info; + TransactionChangeInfo* m_current = nullptr; + SharedGroup& m_sg; +}; +} // anonymous namespace void RealmCoordinator::run_async_notifiers() { @@ -401,11 +415,35 @@ void RealmCoordinator::run_async_notifiers() return; } + SharedGroup::VersionID version; + // Advance all of the new notifiers to the most recent version, if any auto new_notifiers = std::move(m_new_notifiers); - std::vector incremental_change_info; - auto version = advance_mixed_version_notifiers(new_notifiers, *m_advancer_sg, - incremental_change_info); + IncrementalChangeInfo new_notifier_change_info(*m_advancer_sg, new_notifiers); + + if (!new_notifiers.empty()) { + REALM_ASSERT(m_advancer_sg->get_version_of_current_transaction() == new_notifiers.front()->version()); + + // Advance each of the new notifiers to the latest version, attaching them + // to the SG at their handover version. This requires a unique + // TransactionChangeInfo for each source version, so that things don't + // see changes from before the version they were handed over from. + // Each Info has all of the changes between that source version and the + // next source version, and they'll be merged together later after + // releasing the lock + for (auto& notifier : new_notifiers) { + new_notifier_change_info.advance_incremental(notifier->version()); + notifier->attach_to(*m_advancer_sg); + notifier->add_required_change_info(new_notifier_change_info.current()); + } + new_notifier_change_info.advance_to_final(SharedGroup::VersionID{}); + + for (auto& notifier : new_notifiers) { + notifier->detach(); + } + version = m_advancer_sg->get_version_of_current_transaction(); + m_advancer_sg->end_read(); + } // Make a copy of the notifiers vector and then release the lock to avoid // blocking other threads trying to register or unregister notifiers while we run them @@ -414,11 +452,11 @@ void RealmCoordinator::run_async_notifiers() // Advance the non-new notifiers to the same version as we advanced the new // ones to (or the latest if there were no new ones) - TransactionChangeInfo change_info; + IncrementalChangeInfo change_info(*m_notifier_sg, notifiers); for (auto& notifier : notifiers) { - notifier->add_required_change_info(change_info); + notifier->add_required_change_info(change_info.current()); } - transaction::advance(*m_notifier_sg, change_info, version); + change_info.advance_to_final(version); // Attach the new notifiers to the main SG and move them to the main list for (auto& notifier : new_notifiers) { @@ -426,23 +464,6 @@ void RealmCoordinator::run_async_notifiers() } std::move(new_notifiers.begin(), new_notifiers.end(), std::back_inserter(notifiers)); - // Copy the list change info if there's multiple LinkViews for the same LinkList - auto id = [](auto const& list) { return std::tie(list.table_ndx, list.col_ndx, list.row_ndx); }; - auto merge_linkview_info = [&](TransactionChangeInfo& info) { - for (size_t i = 1; i < info.lists.size(); ++i) { - for (size_t j = i; j > 0; --j) { - if (id(info.lists[i]) == id(info.lists[j - 1])) { - info.lists[j - 1].changes->merge(CollectionChangeBuilder{*info.lists[i].changes}); - } - } - } - }; - - merge_linkview_info(change_info); - for (auto& info : incremental_change_info) { - merge_linkview_info(info); - } - // Change info is now all ready, so the notifiers can now perform their // background work for (auto& notifier : notifiers) { From 238e9e3b6b25904a54d35f0838e84e0faaf671b5 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 14 Apr 2016 12:21:30 -0700 Subject: [PATCH 69/93] Fix tracking of which tables need change info with multiple source notifier versions --- src/impl/realm_coordinator.cpp | 6 +++-- tests/list.cpp | 42 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 5e7314d3..7f0f6ff2 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -338,8 +338,10 @@ public: { if (version != m_sg.get_version_of_current_transaction()) { transaction::advance(m_sg, *m_current, version); - // FIXME: needs to copy tables to watch? - m_info.push_back({{}, {}, std::move(m_current->lists)}); + m_info.push_back({ + m_current->table_modifications_needed, + m_current->table_moves_needed, + std::move(m_current->lists)}); m_current = &m_info.back(); return true; } diff --git a/tests/list.cpp b/tests/list.cpp index 7db9a953..37f2381a 100644 --- a/tests/list.cpp +++ b/tests/list.cpp @@ -29,6 +29,12 @@ TEST_CASE("list") { {"target", "", { {"value", PropertyTypeInt} }}, + {"other_origin", "", { + {"array", PropertyTypeArray, "other_target"} + }}, + {"other_target", "", { + {"value", PropertyTypeInt} + }}, }); auto r = Realm::get_shared_realm(config); @@ -226,6 +232,42 @@ TEST_CASE("list") { } } + SECTION("tables-of-interest are tracked properly for multiple source versions") { + auto other_origin = r->read_group()->get_table("class_other_origin"); + auto other_target = r->read_group()->get_table("class_other_target"); + + r->begin_transaction(); + other_target->add_empty_row(); + other_origin->add_empty_row(); + LinkViewRef lv2 = other_origin->get_linklist(0, 0); + lv2->add(0); + r->commit_transaction(); + + List lst2(r, *r->config().schema->find("other_origin"), lv2); + + // Add a callback for list1, advance the version, then add a + // callback for list2, so that the notifiers added at each source + // version have different tables watched for modifications + CollectionChangeIndices changes1, changes2; + auto token1 = lst.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr) { + changes1 = std::move(c); + }); + + r->begin_transaction(); r->commit_transaction(); + + auto token2 = lst2.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr) { + changes2 = std::move(c); + }); + + r->begin_transaction(); + target->set_int(0, 0, 10); + r->commit_transaction(); + advance_and_notify(*r); + + REQUIRE_INDICES(changes1.modifications, 0); + REQUIRE(changes2.empty()); + } + SECTION("modifications are reported for rows that are moved and then moved back in a second transaction") { auto token = require_change(); From e75ff494217dc7b211c6594a2ae3ab14e32d1f11 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 13 Apr 2016 15:38:13 -0700 Subject: [PATCH 70/93] Fix check for the target results being invalid in ResultsNotifier::deliver() m_target_results is no longer actually set to null when the notifier is unregistered, so check the thing which is (i.e. m_realm). --- src/impl/results_notifier.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index f53f060d..7f3c7320 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -173,10 +173,10 @@ bool ResultsNotifier::do_deliver(SharedGroup& sg) { auto lock = lock_target(); - // Target results being null here indicates that it was destroyed while we + // Target realm being null here indicates that we were unregistered while we // were in the process of advancing the Realm version and preparing for - // delivery, i.e. it was destroyed from the "wrong" thread - if (!m_target_results) { + // delivery, i.e. the results was destroyed from the "wrong" thread + if (!get_realm()) { return false; } From 31c3982bff1652da96362ac161c30ecc52cf3b87 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 15 Apr 2016 13:57:12 -0700 Subject: [PATCH 71/93] Fix some typos --- src/impl/background_collection.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/impl/background_collection.hpp b/src/impl/background_collection.hpp index 2c8047e6..830cd8bf 100644 --- a/src/impl/background_collection.hpp +++ b/src/impl/background_collection.hpp @@ -132,7 +132,7 @@ public: bool is_alive() const noexcept; - // Attach the handed-over query to `sg`. Must not be already attaged to a SharedGroup. + // Attach the handed-over query to `sg`. Must not be already attached to a SharedGroup. void attach_to(SharedGroup& sg); // Create a new query handover object and stop using the previously attached // SharedGroup From fe5564f40e3f945d46867b4c6e2b467d61008d31 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 19 Apr 2016 13:59:15 -0700 Subject: [PATCH 72/93] Change all of the mentions of 'query' in NotificationToken to 'notifier' --- src/collection_notifications.cpp | 16 ++++++++-------- src/collection_notifications.hpp | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index cca23764..7eae2cb8 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -23,21 +23,21 @@ using namespace realm; using namespace realm::_impl; -NotificationToken::NotificationToken(std::shared_ptr<_impl::BackgroundCollection> query, size_t token) -: m_query(std::move(query)), m_token(token) +NotificationToken::NotificationToken(std::shared_ptr<_impl::BackgroundCollection> notifier, size_t token) +: m_notifier(std::move(notifier)), m_token(token) { } NotificationToken::~NotificationToken() { - // m_query itself (and not just the pointed-to thing) needs to be accessed + // m_notifier itself (and not just the pointed-to thing) needs to be accessed // atomically to ensure that there are no data races when the token is // destroyed after being modified on a different thread. // This is needed despite the token not being thread-safe in general as // users find it very surpringing for obj-c objects to care about what // thread they are deallocated on. - if (auto query = m_query.exchange({})) { - query->remove_callback(m_token); + if (auto notifier = m_notifier.exchange({})) { + notifier->remove_callback(m_token); } } @@ -46,10 +46,10 @@ NotificationToken::NotificationToken(NotificationToken&& rgt) = default; NotificationToken& NotificationToken::operator=(realm::NotificationToken&& rgt) { if (this != &rgt) { - if (auto query = m_query.exchange({})) { - query->remove_callback(m_token); + if (auto notifier = m_notifier.exchange({})) { + notifier->remove_callback(m_token); } - m_query = std::move(rgt.m_query); + m_notifier = std::move(rgt.m_notifier); m_token = rgt.m_token; } return *this; diff --git a/src/collection_notifications.hpp b/src/collection_notifications.hpp index f6366f48..c2dc9d03 100644 --- a/src/collection_notifications.hpp +++ b/src/collection_notifications.hpp @@ -35,7 +35,7 @@ namespace _impl { // A token which keeps an asynchronous query alive struct NotificationToken { NotificationToken() = default; - NotificationToken(std::shared_ptr<_impl::BackgroundCollection> query, size_t token); + NotificationToken(std::shared_ptr<_impl::BackgroundCollection> notifier, size_t token); ~NotificationToken(); NotificationToken(NotificationToken&&); @@ -45,7 +45,7 @@ struct NotificationToken { NotificationToken& operator=(NotificationToken const&) = delete; private: - util::AtomicSharedPtr<_impl::BackgroundCollection> m_query; + util::AtomicSharedPtr<_impl::BackgroundCollection> m_notifier; size_t m_token; }; From 953e1b15a84c912a892ca4e3e7de71f848acb6dd Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 19 Apr 2016 14:01:09 -0700 Subject: [PATCH 73/93] Rename BackgroundCollection to CollectionNotifier --- src/CMakeLists.txt | 4 +-- src/collection_notifications.cpp | 4 +-- src/collection_notifications.hpp | 6 ++-- ...collection.cpp => collection_notifier.cpp} | 32 +++++++++---------- ...collection.hpp => collection_notifier.hpp} | 8 ++--- src/impl/list_notifier.cpp | 2 +- src/impl/list_notifier.hpp | 4 +-- src/impl/realm_coordinator.cpp | 6 ++-- src/impl/realm_coordinator.hpp | 8 ++--- src/impl/results_notifier.cpp | 2 +- src/impl/results_notifier.hpp | 4 +-- src/impl/transact_log_handler.cpp | 2 +- src/list.hpp | 2 +- src/shared_realm.hpp | 6 ++-- tests/collection_change_indices.cpp | 2 +- tests/transaction_log_parsing.cpp | 2 +- 16 files changed, 47 insertions(+), 47 deletions(-) rename src/impl/{background_collection.cpp => collection_notifier.cpp} (96%) rename src/impl/{background_collection.hpp => collection_notifier.hpp} (97%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 29db5138..b244ed81 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,7 +7,7 @@ set(SOURCES results.cpp schema.cpp shared_realm.cpp - impl/background_collection.cpp + impl/collection_notifier.cpp impl/list_notifier.cpp impl/realm_coordinator.cpp impl/results_notifier.cpp @@ -25,7 +25,7 @@ set(HEADERS results.hpp schema.hpp shared_realm.hpp - impl/background_collection.hpp + impl/collection_notifier.hpp impl/external_commit_helper.hpp impl/list_notifier.hpp impl/realm_coordinator.hpp diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 7eae2cb8..9a11941d 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -18,12 +18,12 @@ #include "collection_notifications.hpp" -#include "impl/background_collection.hpp" +#include "impl/collection_notifier.hpp" using namespace realm; using namespace realm::_impl; -NotificationToken::NotificationToken(std::shared_ptr<_impl::BackgroundCollection> notifier, size_t token) +NotificationToken::NotificationToken(std::shared_ptr<_impl::CollectionNotifier> notifier, size_t token) : m_notifier(std::move(notifier)), m_token(token) { } diff --git a/src/collection_notifications.hpp b/src/collection_notifications.hpp index c2dc9d03..8fc2c0c8 100644 --- a/src/collection_notifications.hpp +++ b/src/collection_notifications.hpp @@ -29,13 +29,13 @@ namespace realm { namespace _impl { - class BackgroundCollection; + class CollectionNotifier; } // A token which keeps an asynchronous query alive struct NotificationToken { NotificationToken() = default; - NotificationToken(std::shared_ptr<_impl::BackgroundCollection> notifier, size_t token); + NotificationToken(std::shared_ptr<_impl::CollectionNotifier> notifier, size_t token); ~NotificationToken(); NotificationToken(NotificationToken&&); @@ -45,7 +45,7 @@ struct NotificationToken { NotificationToken& operator=(NotificationToken const&) = delete; private: - util::AtomicSharedPtr<_impl::BackgroundCollection> m_notifier; + util::AtomicSharedPtr<_impl::CollectionNotifier> m_notifier; size_t m_token; }; diff --git a/src/impl/background_collection.cpp b/src/impl/collection_notifier.cpp similarity index 96% rename from src/impl/background_collection.cpp rename to src/impl/collection_notifier.cpp index 552f457b..acf95d71 100644 --- a/src/impl/background_collection.cpp +++ b/src/impl/collection_notifier.cpp @@ -16,7 +16,7 @@ // //////////////////////////////////////////////////////////////////////////// -#include "impl/background_collection.hpp" +#include "impl/collection_notifier.hpp" #include "impl/realm_coordinator.hpp" #include "shared_realm.hpp" @@ -58,20 +58,20 @@ bool TransactionChangeInfo::row_did_change(Table const& table, size_t idx, int d return false; } -BackgroundCollection::BackgroundCollection(std::shared_ptr realm) +CollectionNotifier::CollectionNotifier(std::shared_ptr realm) : m_realm(std::move(realm)) , m_sg_version(Realm::Internal::get_shared_group(*m_realm).get_version_of_current_transaction()) { } -BackgroundCollection::~BackgroundCollection() +CollectionNotifier::~CollectionNotifier() { // Need to do this explicitly to ensure m_realm is destroyed with the mutex // held to avoid potential double-deletion unregister(); } -size_t BackgroundCollection::add_callback(CollectionChangeCallback callback) +size_t CollectionNotifier::add_callback(CollectionChangeCallback callback) { m_realm->verify_thread(); @@ -95,7 +95,7 @@ size_t BackgroundCollection::add_callback(CollectionChangeCallback callback) return token; } -void BackgroundCollection::remove_callback(size_t token) +void CollectionNotifier::remove_callback(size_t token) { Callback old; { @@ -122,19 +122,19 @@ void BackgroundCollection::remove_callback(size_t token) } } -void BackgroundCollection::unregister() noexcept +void CollectionNotifier::unregister() noexcept { std::lock_guard lock(m_realm_mutex); m_realm = nullptr; } -bool BackgroundCollection::is_alive() const noexcept +bool CollectionNotifier::is_alive() const noexcept { std::lock_guard lock(m_realm_mutex); return m_realm != nullptr; } -std::unique_lock BackgroundCollection::lock_target() +std::unique_lock CollectionNotifier::lock_target() { return std::unique_lock{m_realm_mutex}; } @@ -154,12 +154,12 @@ static void find_relevant_tables(std::vector& out, Table const& table) } } -void BackgroundCollection::set_table(Table const& table) +void CollectionNotifier::set_table(Table const& table) { find_relevant_tables(m_relevant_tables, table); } -void BackgroundCollection::add_required_change_info(TransactionChangeInfo& info) +void CollectionNotifier::add_required_change_info(TransactionChangeInfo& info) { if (!do_add_required_change_info(info)) { return; @@ -173,14 +173,14 @@ void BackgroundCollection::add_required_change_info(TransactionChangeInfo& info) } } -void BackgroundCollection::prepare_handover() +void CollectionNotifier::prepare_handover() { REALM_ASSERT(m_sg); m_sg_version = m_sg->get_version_of_current_transaction(); do_prepare_handover(*m_sg); } -bool BackgroundCollection::deliver(SharedGroup& sg, std::exception_ptr err) +bool CollectionNotifier::deliver(SharedGroup& sg, std::exception_ptr err) { if (!is_for_current_thread()) { return false; @@ -206,7 +206,7 @@ bool BackgroundCollection::deliver(SharedGroup& sg, std::exception_ptr err) return should_call_callbacks && have_callbacks(); } -void BackgroundCollection::call_callbacks() +void CollectionNotifier::call_callbacks() { while (auto fn = next_callback()) { fn(m_changes_to_deliver, m_error); @@ -220,7 +220,7 @@ void BackgroundCollection::call_callbacks() } } -CollectionChangeCallback BackgroundCollection::next_callback() +CollectionChangeCallback CollectionNotifier::next_callback() { std::lock_guard callback_lock(m_callback_mutex); @@ -237,7 +237,7 @@ CollectionChangeCallback BackgroundCollection::next_callback() return nullptr; } -void BackgroundCollection::attach_to(SharedGroup& sg) +void CollectionNotifier::attach_to(SharedGroup& sg) { REALM_ASSERT(!m_sg); @@ -245,7 +245,7 @@ void BackgroundCollection::attach_to(SharedGroup& sg) do_attach_to(sg); } -void BackgroundCollection::detach() +void CollectionNotifier::detach() { REALM_ASSERT(m_sg); do_detach_from(*m_sg); diff --git a/src/impl/background_collection.hpp b/src/impl/collection_notifier.hpp similarity index 97% rename from src/impl/background_collection.hpp rename to src/impl/collection_notifier.hpp index 830cd8bf..a21ff042 100644 --- a/src/impl/background_collection.hpp +++ b/src/impl/collection_notifier.hpp @@ -90,10 +90,10 @@ struct TransactionChangeInfo { // most of the lifetime-management issues related to sharing an object between // the worker thread and the collection on the target thread, along with the // thread-safe callback collection. -class BackgroundCollection { +class CollectionNotifier { public: - BackgroundCollection(std::shared_ptr); - virtual ~BackgroundCollection(); + CollectionNotifier(std::shared_ptr); + virtual ~CollectionNotifier(); // ------------------------------------------------------------------------ // Public API for the collections using this to get notifications: @@ -123,7 +123,7 @@ public: // Release references to all core types // This is called on the worker thread to ensure that non-thread-safe things // can be destroyed on the correct thread, even if the last reference to the - // BackgroundCollection is released on a different thread + // CollectionNotifier is released on a different thread virtual void release_data() noexcept = 0; // Call each of the currently registered callbacks, if there have been any diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp index 11a74bb4..2d14602b 100644 --- a/src/impl/list_notifier.cpp +++ b/src/impl/list_notifier.cpp @@ -26,7 +26,7 @@ using namespace realm; using namespace realm::_impl; ListNotifier::ListNotifier(LinkViewRef lv, std::shared_ptr realm) -: BackgroundCollection(std::move(realm)) +: CollectionNotifier(std::move(realm)) , m_prev_size(lv->size()) { // Find the lv's column, since that isn't tracked directly diff --git a/src/impl/list_notifier.hpp b/src/impl/list_notifier.hpp index 9cc0d775..82b4e414 100644 --- a/src/impl/list_notifier.hpp +++ b/src/impl/list_notifier.hpp @@ -19,13 +19,13 @@ #ifndef REALM_LIST_NOTIFIER_HPP #define REALM_LIST_NOTIFIER_HPP -#include "impl/background_collection.hpp" +#include "impl/collection_notifier.hpp" #include namespace realm { namespace _impl { -class ListNotifier : public BackgroundCollection { +class ListNotifier : public CollectionNotifier { public: ListNotifier(LinkViewRef lv, std::shared_ptr realm); diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 7f0f6ff2..78a2f7ae 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -18,7 +18,7 @@ #include "impl/realm_coordinator.hpp" -#include "impl/background_collection.hpp" +#include "impl/collection_notifier.hpp" #include "impl/external_commit_helper.hpp" #include "impl/transact_log_handler.hpp" #include "impl/weak_realm_notifier.hpp" @@ -245,7 +245,7 @@ void RealmCoordinator::pin_version(uint_fast64_t version, uint_fast32_t index) } } -void RealmCoordinator::register_notifier(std::shared_ptr notifier) +void RealmCoordinator::register_notifier(std::shared_ptr notifier) { auto version = notifier->version(); auto& self = Realm::Internal::get_coordinator(*notifier->get_realm()); @@ -306,7 +306,7 @@ namespace { class IncrementalChangeInfo { public: IncrementalChangeInfo(SharedGroup& sg, - std::vector>& notifiers) + std::vector>& notifiers) : m_sg(sg) { if (notifiers.empty()) diff --git a/src/impl/realm_coordinator.hpp b/src/impl/realm_coordinator.hpp index 899f2c03..2a6f74b2 100644 --- a/src/impl/realm_coordinator.hpp +++ b/src/impl/realm_coordinator.hpp @@ -30,7 +30,7 @@ class SharedGroup; class StringData; namespace _impl { -class BackgroundCollection; +class CollectionNotifier; class ExternalCommitHelper; class WeakRealmNotifier; @@ -81,7 +81,7 @@ public: // Update the schema in the cached config void update_schema(Schema const& new_schema); - static void register_notifier(std::shared_ptr notifier); + static void register_notifier(std::shared_ptr notifier); // Advance the Realm to the most recent transaction version which all async // work is complete for @@ -95,8 +95,8 @@ private: std::vector m_weak_realm_notifiers; std::mutex m_notifier_mutex; - std::vector> m_new_notifiers; - std::vector> m_notifiers; + std::vector> m_new_notifiers; + std::vector> m_notifiers; // SharedGroup used for actually running async notifiers // Will have a read transaction iff m_notifiers is non-empty diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index 7f3c7320..c6ae69c9 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -24,7 +24,7 @@ using namespace realm; using namespace realm::_impl; ResultsNotifier::ResultsNotifier(Results& target) -: BackgroundCollection(target.get_realm()) +: CollectionNotifier(target.get_realm()) , m_target_results(&target) , m_sort(target.get_sort()) , m_from_linkview(target.get_linkview().get() != nullptr) diff --git a/src/impl/results_notifier.hpp b/src/impl/results_notifier.hpp index 03b4be60..de0b2d65 100644 --- a/src/impl/results_notifier.hpp +++ b/src/impl/results_notifier.hpp @@ -19,14 +19,14 @@ #ifndef REALM_RESULTS_NOTIFIER_HPP #define REALM_RESULTS_NOTIFIER_HPP -#include "background_collection.hpp" +#include "collection_notifier.hpp" #include "results.hpp" #include namespace realm { namespace _impl { -class ResultsNotifier : public BackgroundCollection { +class ResultsNotifier : public CollectionNotifier { public: ResultsNotifier(Results& target); diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index 129384a9..8aedd052 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -19,7 +19,7 @@ #include "impl/transact_log_handler.hpp" #include "binding_context.hpp" -#include "impl/background_collection.hpp" +#include "impl/collection_notifier.hpp" #include "index_set.hpp" #include diff --git a/src/list.hpp b/src/list.hpp index 2be94748..8abd601d 100644 --- a/src/list.hpp +++ b/src/list.hpp @@ -89,7 +89,7 @@ private: std::shared_ptr m_realm; const ObjectSchema* m_object_schema; LinkViewRef m_link_view; - std::shared_ptr<_impl::BackgroundCollection> m_notifier; + std::shared_ptr<_impl::CollectionNotifier> m_notifier; void verify_valid_row(size_t row_ndx, bool insertion = false) const; diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index f1ea3b2a..25e5acfd 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -36,7 +36,7 @@ namespace realm { typedef std::weak_ptr WeakRealm; namespace _impl { - class BackgroundCollection; + class CollectionNotifier; class ListNotifier; class RealmCoordinator; class ResultsNotifier; @@ -140,7 +140,7 @@ namespace realm { // Expose some internal functionality to other parts of the ObjectStore // without making it public to everyone class Internal { - friend class _impl::BackgroundCollection; + friend class _impl::CollectionNotifier; friend class _impl::ListNotifier; friend class _impl::RealmCoordinator; friend class _impl::ResultsNotifier; @@ -149,7 +149,7 @@ namespace realm { // to be able to call the handover functions, which are not very wrappable static SharedGroup& get_shared_group(Realm& realm) { return *realm.m_shared_group; } - // BackgroundCollection needs to be able to access the owning + // CollectionNotifier needs to be able to access the owning // coordinator to wake up the worker thread when a callback is // added, and coordinators need to be able to get themselves from a Realm static _impl::RealmCoordinator& get_coordinator(Realm& realm) { return *realm.m_coordinator; } diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index 7eddc254..1ca24f16 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -1,6 +1,6 @@ #include "catch.hpp" -#include "impl/background_collection.hpp" +#include "impl/collection_notifier.hpp" #include "util/index_helpers.hpp" diff --git a/tests/transaction_log_parsing.cpp b/tests/transaction_log_parsing.cpp index da394289..319a08ca 100644 --- a/tests/transaction_log_parsing.cpp +++ b/tests/transaction_log_parsing.cpp @@ -3,7 +3,7 @@ #include "util/index_helpers.hpp" #include "util/test_file.hpp" -#include "impl/background_collection.hpp" +#include "impl/collection_notifier.hpp" #include "impl/transact_log_handler.hpp" #include "property.hpp" #include "object_schema.hpp" From c0350b900136ffbe2df72ec3f255704ff469fa0e Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 19 Apr 2016 14:05:15 -0700 Subject: [PATCH 74/93] Rename CollectionChangeIndices to CollectionChangeSet --- src/collection_notifications.hpp | 4 ++-- src/impl/collection_notifier.cpp | 4 ++-- src/impl/collection_notifier.hpp | 4 ++-- src/results.cpp | 2 +- tests/list.cpp | 24 +++++++++++----------- tests/results.cpp | 34 +++++++++++++++---------------- tests/transaction_log_parsing.cpp | 6 +++--- tests/util/index_helpers.hpp | 2 +- 8 files changed, 40 insertions(+), 40 deletions(-) diff --git a/src/collection_notifications.hpp b/src/collection_notifications.hpp index 8fc2c0c8..ed75c315 100644 --- a/src/collection_notifications.hpp +++ b/src/collection_notifications.hpp @@ -49,7 +49,7 @@ private: size_t m_token; }; -struct CollectionChangeIndices { +struct CollectionChangeSet { struct Move { size_t from; size_t to; @@ -65,7 +65,7 @@ struct CollectionChangeIndices { bool empty() const { return deletions.empty() && insertions.empty() && modifications.empty() && moves.empty(); } }; -using CollectionChangeCallback = std::function; +using CollectionChangeCallback = std::function; } // namespace realm #endif // REALM_COLLECTION_NOTIFICATIONS_HPP diff --git a/src/impl/collection_notifier.cpp b/src/impl/collection_notifier.cpp index acf95d71..aa96ea8b 100644 --- a/src/impl/collection_notifier.cpp +++ b/src/impl/collection_notifier.cpp @@ -557,7 +557,7 @@ struct RowInfo { size_t shifted_tv_index; }; -void calculate_moves_unsorted(std::vector& new_rows, IndexSet& removed, CollectionChangeIndices& changeset) +void calculate_moves_unsorted(std::vector& new_rows, IndexSet& removed, CollectionChangeSet& changeset) { size_t expected = 0; for (auto& row : new_rows) { @@ -587,7 +587,7 @@ void calculate_moves_unsorted(std::vector& new_rows, IndexSet& removed, class SortedMoveCalculator { public: - SortedMoveCalculator(std::vector& new_rows, CollectionChangeIndices& changeset) + SortedMoveCalculator(std::vector& new_rows, CollectionChangeSet& changeset) : m_modified(changeset.modifications) { std::vector old_candidates; diff --git a/src/impl/collection_notifier.hpp b/src/impl/collection_notifier.hpp index a21ff042..eb906d8d 100644 --- a/src/impl/collection_notifier.hpp +++ b/src/impl/collection_notifier.hpp @@ -34,7 +34,7 @@ namespace realm { class Realm; namespace _impl { -class CollectionChangeBuilder : public CollectionChangeIndices { +class CollectionChangeBuilder : public CollectionChangeSet { public: CollectionChangeBuilder(CollectionChangeBuilder const&) = default; CollectionChangeBuilder(CollectionChangeBuilder&&) = default; @@ -170,7 +170,7 @@ private: std::exception_ptr m_error; CollectionChangeBuilder m_accumulated_changes; - CollectionChangeIndices m_changes_to_deliver; + CollectionChangeSet m_changes_to_deliver; // Tables which this collection needs change information for std::vector m_relevant_tables; diff --git a/src/results.cpp b/src/results.cpp index b5048003..1095731a 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -447,7 +447,7 @@ void Results::prepare_async() NotificationToken Results::async(std::function target) { prepare_async(); - auto wrap = [=](CollectionChangeIndices, std::exception_ptr e) { target(e); }; + auto wrap = [=](CollectionChangeSet, std::exception_ptr e) { target(e); }; return {m_notifier, m_notifier->add_callback(wrap)}; } diff --git a/tests/list.cpp b/tests/list.cpp index 37f2381a..79153e7a 100644 --- a/tests/list.cpp +++ b/tests/list.cpp @@ -60,7 +60,7 @@ TEST_CASE("list") { r->commit_transaction(); SECTION("add_notification_block()") { - CollectionChangeIndices change; + CollectionChangeSet change; List lst(r, *r->config().schema->find("origin"), lv); auto write = [&](auto&& f) { @@ -72,7 +72,7 @@ TEST_CASE("list") { }; auto require_change = [&] { - auto token = lst.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { + auto token = lst.add_notification_callback([&](CollectionChangeSet c, std::exception_ptr err) { change = c; }); advance_and_notify(*r); @@ -81,7 +81,7 @@ TEST_CASE("list") { auto require_no_change = [&] { bool first = true; - auto token = lst.add_notification_callback([&, first](CollectionChangeIndices c, std::exception_ptr err) mutable { + auto token = lst.add_notification_callback([&, first](CollectionChangeSet c, std::exception_ptr err) mutable { REQUIRE(first); first = false; }); @@ -198,11 +198,11 @@ TEST_CASE("list") { List lists[3]; NotificationToken tokens[3]; - CollectionChangeIndices changes[3]; + CollectionChangeSet changes[3]; for (int i = 0; i < 3; ++i) { lists[i] = get_list(); - tokens[i] = lists[i].add_notification_callback([i, &changes](CollectionChangeIndices c, std::exception_ptr) { + tokens[i] = lists[i].add_notification_callback([i, &changes](CollectionChangeSet c, std::exception_ptr) { changes[i] = std::move(c); }); change_list(); @@ -248,14 +248,14 @@ TEST_CASE("list") { // Add a callback for list1, advance the version, then add a // callback for list2, so that the notifiers added at each source // version have different tables watched for modifications - CollectionChangeIndices changes1, changes2; - auto token1 = lst.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr) { + CollectionChangeSet changes1, changes2; + auto token1 = lst.add_notification_callback([&](CollectionChangeSet c, std::exception_ptr) { changes1 = std::move(c); }); r->begin_transaction(); r->commit_transaction(); - auto token2 = lst2.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr) { + auto token2 = lst2.add_notification_callback([&](CollectionChangeSet c, std::exception_ptr) { changes2 = std::move(c); }); @@ -296,8 +296,8 @@ TEST_CASE("list") { Results results = lst.sort({{0}, {false}}); int notification_calls = 0; - CollectionChangeIndices change; - auto token = results.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { + CollectionChangeSet change; + auto token = results.add_notification_callback([&](CollectionChangeSet c, std::exception_ptr err) { REQUIRE_FALSE(err); change = c; ++notification_calls; @@ -352,8 +352,8 @@ TEST_CASE("list") { Results results = lst.filter(target->where().less(0, 9)); int notification_calls = 0; - CollectionChangeIndices change; - auto token = results.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { + CollectionChangeSet change; + auto token = results.add_notification_callback([&](CollectionChangeSet c, std::exception_ptr err) { REQUIRE_FALSE(err); change = c; ++notification_calls; diff --git a/tests/results.cpp b/tests/results.cpp index fb039d44..fffff6c8 100644 --- a/tests/results.cpp +++ b/tests/results.cpp @@ -51,8 +51,8 @@ TEST_CASE("Results") { SECTION("unsorted notifications") { int notification_calls = 0; - CollectionChangeIndices change; - auto token = results.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { + CollectionChangeSet change; + auto token = results.add_notification_callback([&](CollectionChangeSet c, std::exception_ptr err) { REQUIRE_FALSE(err); change = c; ++notification_calls; @@ -107,8 +107,8 @@ TEST_CASE("Results") { SECTION("notifications are delivered when a new callback is added from within a callback") { NotificationToken token2, token3; bool called = false; - token2 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr) { - token3 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr) { + token2 = results.add_notification_callback([&](CollectionChangeSet, std::exception_ptr) { + token3 = results.add_notification_callback([&](CollectionChangeSet, std::exception_ptr) { called = true; }); }); @@ -119,10 +119,10 @@ TEST_CASE("Results") { SECTION("notifications are not delivered when a callback is removed from within a callback") { NotificationToken token2, token3; - token2 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr) { + token2 = results.add_notification_callback([&](CollectionChangeSet, std::exception_ptr) { token3 = {}; }); - token3 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr) { + token3 = results.add_notification_callback([&](CollectionChangeSet, std::exception_ptr) { REQUIRE(false); }); @@ -132,10 +132,10 @@ TEST_CASE("Results") { SECTION("removing the current callback does not stop later ones from being called") { NotificationToken token2, token3; bool called = false; - token2 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr) { + token2 = results.add_notification_callback([&](CollectionChangeSet, std::exception_ptr) { token2 = {}; }); - token3 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr) { + token3 = results.add_notification_callback([&](CollectionChangeSet, std::exception_ptr) { called = true; }); @@ -277,7 +277,7 @@ TEST_CASE("Results") { } SECTION("the first call of a notification can include changes if it previously ran for a different callback") { - auto token2 = results.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr) { + auto token2 = results.add_notification_callback([&](CollectionChangeSet c, std::exception_ptr) { REQUIRE(!c.empty()); }); @@ -292,8 +292,8 @@ TEST_CASE("Results") { SECTION("sorted notifications") { int notification_calls = 0; - CollectionChangeIndices change; - auto token = results.add_notification_callback([&](CollectionChangeIndices c, std::exception_ptr err) { + CollectionChangeSet change; + auto token = results.add_notification_callback([&](CollectionChangeSet c, std::exception_ptr err) { REQUIRE_FALSE(err); change = c; ++notification_calls; @@ -447,7 +447,7 @@ TEST_CASE("Async Results error handling") { SECTION("error is delivered asynchronously") { bool called = false; - auto token = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { + auto token = results.add_notification_callback([&](CollectionChangeSet, std::exception_ptr err) { REQUIRE(err); called = true; }); @@ -461,7 +461,7 @@ TEST_CASE("Async Results error handling") { SECTION("adding another callback does not send the error again") { bool called = false; - auto token = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { + auto token = results.add_notification_callback([&](CollectionChangeSet, std::exception_ptr err) { REQUIRE(err); REQUIRE_FALSE(called); called = true; @@ -470,7 +470,7 @@ TEST_CASE("Async Results error handling") { advance_and_notify(*r); bool called2 = false; - auto token2 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { + auto token2 = results.add_notification_callback([&](CollectionChangeSet, std::exception_ptr err) { REQUIRE(err); REQUIRE_FALSE(called2); called2 = true; @@ -484,7 +484,7 @@ TEST_CASE("Async Results error handling") { SECTION("error when opening the executor SG") { SECTION("error is delivered asynchronously") { bool called = false; - auto token = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { + auto token = results.add_notification_callback([&](CollectionChangeSet, std::exception_ptr err) { REQUIRE(err); called = true; }); @@ -499,7 +499,7 @@ TEST_CASE("Async Results error handling") { SECTION("adding another callback does not send the error again") { bool called = false; - auto token = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { + auto token = results.add_notification_callback([&](CollectionChangeSet, std::exception_ptr err) { REQUIRE(err); REQUIRE_FALSE(called); called = true; @@ -509,7 +509,7 @@ TEST_CASE("Async Results error handling") { advance_and_notify(*r); bool called2 = false; - auto token2 = results.add_notification_callback([&](CollectionChangeIndices, std::exception_ptr err) { + auto token2 = results.add_notification_callback([&](CollectionChangeSet, std::exception_ptr err) { REQUIRE(err); REQUIRE_FALSE(called2); called2 = true; diff --git a/tests/transaction_log_parsing.cpp b/tests/transaction_log_parsing.cpp index 319a08ca..92748568 100644 --- a/tests/transaction_log_parsing.cpp +++ b/tests/transaction_log_parsing.cpp @@ -31,7 +31,7 @@ public: m_initial.push_back(lv->get(i).get_int(0)); } - CollectionChangeIndices finish(size_t ndx) { + CollectionChangeSet finish(size_t ndx) { m_realm->commit_transaction(); _impl::CollectionChangeBuilder c; @@ -61,7 +61,7 @@ private: LinkViewRef m_linkview; std::vector m_initial; - void validate(CollectionChangeIndices const& info) + void validate(CollectionChangeSet const& info) { info.insertions.verify(); info.deletions.verify(); @@ -445,7 +445,7 @@ TEST_CASE("Transaction log parsing") { #define VALIDATE_CHANGES(out) \ for (CaptureHelper helper(config.path, r, lv); helper; out = helper.finish(origin->get_index_in_group())) - CollectionChangeIndices changes; + CollectionChangeSet changes; SECTION("single change type") { SECTION("add single") { VALIDATE_CHANGES(changes) { diff --git a/tests/util/index_helpers.hpp b/tests/util/index_helpers.hpp index 5ca6803c..bb5145b5 100644 --- a/tests/util/index_helpers.hpp +++ b/tests/util/index_helpers.hpp @@ -12,7 +12,7 @@ #define REQUIRE_MOVES(c, ...) do { \ auto actual = (c); \ - std::initializer_list expected = {__VA_ARGS__}; \ + std::initializer_list expected = {__VA_ARGS__}; \ REQUIRE(expected.size() == actual.moves.size()); \ auto begin = actual.moves.begin(); \ for (auto move : expected) { \ From 23c16f8e67dc75eb4be6f07834c6ff59dd5671ed Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 19 Apr 2016 15:13:28 -0700 Subject: [PATCH 75/93] Use aggregate initialization for the base class in CollectionChangeBuilder --- src/impl/collection_notifier.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/impl/collection_notifier.cpp b/src/impl/collection_notifier.cpp index aa96ea8b..2a039f81 100644 --- a/src/impl/collection_notifier.cpp +++ b/src/impl/collection_notifier.cpp @@ -256,12 +256,8 @@ CollectionChangeBuilder::CollectionChangeBuilder(IndexSet deletions, IndexSet insertions, IndexSet modifications, std::vector moves) +: CollectionChangeSet({std::move(deletions), std::move(insertions), std::move(modifications), std::move(moves)}) { - this->deletions = std::move(deletions); - this->insertions = std::move(insertions); - this->modifications = std::move(modifications); - this->moves = std::move(moves); - for (auto&& move : this->moves) { this->deletions.add(move.from); this->insertions.add(move.to); From 66666a75b0415f457b35dee8d589d8ee43b0181a Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 19 Apr 2016 15:16:15 -0700 Subject: [PATCH 76/93] Iterate backwards to pick the last valid list in select_link_list --- src/impl/transact_log_handler.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index 8aedd052..86db5544 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -464,11 +464,13 @@ public: mark_dirty(row, col); m_active = nullptr; - for (auto& o : m_info.lists) { - if (o.table_ndx == current_table() && o.row_ndx == row && o.col_ndx == col) { - m_active = o.changes; - // need to use last match for multiple source version logic -// break; + // When there are multiple source versions there could be multiple + // change objects for a single LinkView, in which case we need to use + // the last one + for (auto it = m_info.lists.rbegin(), end = m_info.lists.rend(); it != end; ++it) { + if (it->table_ndx == current_table() && it->row_ndx == row && it->col_ndx == col) { + m_active = it->changes; + break; } } return true; From 9883944df4e6c8c8d9e881d10f4682c3175e3517 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 19 Apr 2016 15:47:23 -0700 Subject: [PATCH 77/93] Pull CollectionChangeBuilder into its own file --- src/CMakeLists.txt | 2 + src/impl/collection_change_builder.cpp | 595 +++++++++++++++++++++++++ src/impl/collection_change_builder.hpp | 65 +++ src/impl/collection_notifier.cpp | 571 ------------------------ src/impl/collection_notifier.hpp | 37 +- 5 files changed, 663 insertions(+), 607 deletions(-) create mode 100644 src/impl/collection_change_builder.cpp create mode 100644 src/impl/collection_change_builder.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b244ed81..8f849049 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,6 +7,7 @@ set(SOURCES results.cpp schema.cpp shared_realm.cpp + impl/collection_change_builder.cpp impl/collection_notifier.cpp impl/list_notifier.cpp impl/realm_coordinator.cpp @@ -25,6 +26,7 @@ set(HEADERS results.hpp schema.hpp shared_realm.hpp + impl/collection_change_builder.hpp impl/collection_notifier.hpp impl/external_commit_helper.hpp impl/list_notifier.hpp diff --git a/src/impl/collection_change_builder.cpp b/src/impl/collection_change_builder.cpp new file mode 100644 index 00000000..3617bdef --- /dev/null +++ b/src/impl/collection_change_builder.cpp @@ -0,0 +1,595 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2016 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "impl/collection_change_builder.hpp" + +#include + +using namespace realm; +using namespace realm::_impl; + +CollectionChangeBuilder::CollectionChangeBuilder(IndexSet deletions, + IndexSet insertions, + IndexSet modifications, + std::vector moves) +: CollectionChangeSet({std::move(deletions), std::move(insertions), std::move(modifications), std::move(moves)}) +{ + for (auto&& move : this->moves) { + this->deletions.add(move.from); + this->insertions.add(move.to); + } +} + +void CollectionChangeBuilder::merge(CollectionChangeBuilder&& c) +{ + if (c.empty()) + return; + if (empty()) { + *this = std::move(c); + return; + } + + verify(); + c.verify(); + + // First update any old moves + if (!c.moves.empty() || !c.deletions.empty() || !c.insertions.empty()) { + auto it = remove_if(begin(moves), end(moves), [&](auto& old) { + // Check if the moved row was moved again, and if so just update the destination + auto it = find_if(begin(c.moves), end(c.moves), [&](auto const& m) { + return old.to == m.from; + }); + if (it != c.moves.end()) { + if (modifications.contains(it->from)) + c.modifications.add(it->to); + old.to = it->to; + *it = c.moves.back(); + c.moves.pop_back(); + ++it; + return false; + } + + // Check if the destination was deleted + // Removing the insert for this move will happen later + if (c.deletions.contains(old.to)) + return true; + + // Update the destination to adjust for any new insertions and deletions + old.to = c.insertions.shift(c.deletions.unshift(old.to)); + return false; + }); + moves.erase(it, end(moves)); + } + + // Ignore new moves of rows which were previously inserted (the implicit + // delete from the move will remove the insert) + if (!insertions.empty() && !c.moves.empty()) { + c.moves.erase(remove_if(begin(c.moves), end(c.moves), + [&](auto const& m) { return insertions.contains(m.from); }), + end(c.moves)); + } + + // Ensure that any previously modified rows which were moved are still modified + if (!modifications.empty() && !c.moves.empty()) { + for (auto const& move : c.moves) { + if (modifications.contains(move.from)) + c.modifications.add(move.to); + } + } + + // Update the source position of new moves to compensate for the changes made + // in the old changeset + if (!deletions.empty() || !insertions.empty()) { + for (auto& move : c.moves) + move.from = deletions.shift(insertions.unshift(move.from)); + } + + moves.insert(end(moves), begin(c.moves), end(c.moves)); + + // New deletion indices have been shifted by the insertions, so unshift them + // before adding + deletions.add_shifted_by(insertions, c.deletions); + + // Drop any inserted-then-deleted rows, then merge in new insertions + insertions.erase_at(c.deletions); + insertions.insert_at(c.insertions); + + clean_up_stale_moves(); + + modifications.erase_at(c.deletions); + modifications.shift_for_insert_at(c.insertions); + modifications.add(c.modifications); + + c = {}; + verify(); +} + +void CollectionChangeBuilder::clean_up_stale_moves() +{ + // Look for moves which are now no-ops, and remove them plus the associated + // insert+delete. Note that this isn't just checking for from == to due to + // that rows can also be shifted by other inserts and deletes + moves.erase(remove_if(begin(moves), end(moves), [&](auto const& move) { + if (move.from - deletions.count(0, move.from) != move.to - insertions.count(0, move.to)) + return false; + deletions.remove(move.from); + insertions.remove(move.to); + return true; + }), end(moves)); +} + +void CollectionChangeBuilder::parse_complete() +{ + moves.reserve(m_move_mapping.size()); + for (auto move : m_move_mapping) { + REALM_ASSERT_DEBUG(deletions.contains(move.second)); + REALM_ASSERT_DEBUG(insertions.contains(move.first)); + moves.push_back({move.second, move.first}); + } + m_move_mapping.clear(); + std::sort(begin(moves), end(moves), + [](auto const& a, auto const& b) { return a.from < b.from; }); +} + +void CollectionChangeBuilder::modify(size_t ndx) +{ + modifications.add(ndx); +} + +void CollectionChangeBuilder::insert(size_t index, size_t count, bool track_moves) +{ + modifications.shift_for_insert_at(index, count); + if (!track_moves) + return; + + insertions.insert_at(index, count); + + for (auto& move : moves) { + if (move.to >= index) + ++move.to; + } +} + +void CollectionChangeBuilder::erase(size_t index) +{ + modifications.erase_at(index); + size_t unshifted = insertions.erase_or_unshift(index); + if (unshifted != IndexSet::npos) + deletions.add_shifted(unshifted); + + for (size_t i = 0; i < moves.size(); ++i) { + auto& move = moves[i]; + if (move.to == index) { + moves.erase(moves.begin() + i); + --i; + } + else if (move.to > index) + --move.to; + } +} + +void CollectionChangeBuilder::clear(size_t old_size) +{ + if (old_size != std::numeric_limits::max()) { + for (auto range : deletions) + old_size += range.second - range.first; + for (auto range : insertions) + old_size -= range.second - range.first; + } + + modifications.clear(); + insertions.clear(); + moves.clear(); + m_move_mapping.clear(); + deletions.set(old_size); +} + +void CollectionChangeBuilder::move(size_t from, size_t to) +{ + REALM_ASSERT(from != to); + + bool updated_existing_move = false; + for (auto& move : moves) { + if (move.to != from) { + // Shift other moves if this row is moving from one side of them + // to the other + if (move.to >= to && move.to < from) + ++move.to; + else if (move.to <= to && move.to > from) + --move.to; + continue; + } + REALM_ASSERT(!updated_existing_move); + + // Collapse A -> B, B -> C into a single A -> C move + move.to = to; + updated_existing_move = true; + + insertions.erase_at(from); + insertions.insert_at(to); + } + + if (!updated_existing_move) { + auto shifted_from = insertions.erase_or_unshift(from); + insertions.insert_at(to); + + // Don't report deletions/moves for newly inserted rows + if (shifted_from != IndexSet::npos) { + shifted_from = deletions.add_shifted(shifted_from); + moves.push_back({shifted_from, to}); + } + } + + bool modified = modifications.contains(from); + modifications.erase_at(from); + + if (modified) + modifications.insert_at(to); + else + modifications.shift_for_insert_at(to); +} + +void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row, bool track_moves) +{ + REALM_ASSERT(row_ndx <= last_row); + REALM_ASSERT(insertions.empty() || prev(insertions.end())->second - 1 <= last_row); + REALM_ASSERT(modifications.empty() || prev(modifications.end())->second - 1 <= last_row); + if (track_moves && row_ndx == last_row) { + erase(row_ndx); + return; + } + + bool modified = modifications.contains(last_row); + if (modified) { + modifications.remove(last_row); + modifications.add(row_ndx); + } + else + modifications.remove(row_ndx); + + if (!track_moves) + return; + + bool row_is_insertion = insertions.contains(row_ndx); + bool last_is_insertion = !insertions.empty() && prev(insertions.end())->second == last_row + 1; + REALM_ASSERT_DEBUG(insertions.empty() || prev(insertions.end())->second <= last_row + 1); + + // Collapse A -> B, B -> C into a single A -> C move + bool last_was_already_moved = false; + if (last_is_insertion) { + auto it = m_move_mapping.find(last_row); + if (it != m_move_mapping.end() && it->first == last_row) { + m_move_mapping[row_ndx] = it->second; + m_move_mapping.erase(it); + last_was_already_moved = true; + } + } + + // Remove moves to the row being deleted + if (row_is_insertion && !last_was_already_moved) { + auto it = m_move_mapping.find(row_ndx); + if (it != m_move_mapping.end() && it->first == row_ndx) + m_move_mapping.erase(it); + } + + // Don't report deletions/moves if last_row is newly inserted + if (last_is_insertion) { + insertions.remove(last_row); + } + // If it was previously moved, the unshifted source row has already been marked as deleted + else if (!last_was_already_moved) { + auto shifted_last_row = insertions.unshift(last_row); + shifted_last_row = deletions.add_shifted(shifted_last_row); + m_move_mapping[row_ndx] = shifted_last_row; + } + + // Don't mark the moved-over row as deleted if it was a new insertion + if (!row_is_insertion) { + deletions.add_shifted(insertions.unshift(row_ndx)); + insertions.add(row_ndx); + } + verify(); +} + +void CollectionChangeBuilder::verify() +{ +#ifdef REALM_DEBUG + for (auto&& move : moves) { + REALM_ASSERT(deletions.contains(move.from)); + REALM_ASSERT(insertions.contains(move.to)); + } +#endif +} + +namespace { +struct RowInfo { + size_t row_index; + size_t prev_tv_index; + size_t tv_index; + size_t shifted_tv_index; +}; + +void calculate_moves_unsorted(std::vector& new_rows, IndexSet& removed, CollectionChangeSet& changeset) +{ + size_t expected = 0; + for (auto& row : new_rows) { + // With unsorted queries rows only move due to move_last_over(), which + // inherently can only move a row to earlier in the table. + REALM_ASSERT(row.shifted_tv_index >= expected); + if (row.shifted_tv_index == expected) { + ++expected; + continue; + } + + // This row isn't just the row after the previous one, but it still may + // not be a move if there were rows deleted between the two, so next + // calcuate what row should be here taking those in to account + size_t calc_expected = row.tv_index - changeset.insertions.count(0, row.tv_index) + removed.count(0, row.prev_tv_index); + if (row.shifted_tv_index == calc_expected) { + expected = calc_expected + 1; + continue; + } + + // The row still isn't the expected one, so it's a move + changeset.moves.push_back({row.prev_tv_index, row.tv_index}); + changeset.insertions.add(row.tv_index); + removed.add(row.prev_tv_index); + } +} + +class SortedMoveCalculator { +public: + SortedMoveCalculator(std::vector& new_rows, CollectionChangeSet& changeset) + : m_modified(changeset.modifications) + { + std::vector old_candidates; + old_candidates.reserve(new_rows.size()); + for (auto& row : new_rows) { + old_candidates.push_back({row.row_index, row.prev_tv_index}); + } + std::sort(begin(old_candidates), end(old_candidates), [](auto a, auto b) { + return std::tie(a.tv_index, a.row_index) < std::tie(b.tv_index, b.row_index); + }); + + // First check if the order of any of the rows actually changed + size_t first_difference = IndexSet::npos; + for (size_t i = 0; i < old_candidates.size(); ++i) { + if (old_candidates[i].row_index != new_rows[i].row_index) { + first_difference = i; + break; + } + } + if (first_difference == IndexSet::npos) + return; + + // A map from row index -> tv index in new results + b.reserve(new_rows.size()); + for (size_t i = 0; i < new_rows.size(); ++i) + b.push_back({new_rows[i].row_index, i}); + std::sort(begin(b), end(b), [](auto a, auto b) { + return std::tie(a.row_index, a.tv_index) < std::tie(b.row_index, b.tv_index); + }); + + a = std::move(old_candidates); + + find_longest_matches(first_difference, a.size(), + first_difference, new_rows.size()); + m_longest_matches.push_back({a.size(), new_rows.size(), 0}); + + size_t i = first_difference, j = first_difference; + for (auto match : m_longest_matches) { + for (; i < match.i; ++i) + changeset.deletions.add(a[i].tv_index); + for (; j < match.j; ++j) + changeset.insertions.add(new_rows[j].tv_index); + i += match.size; + j += match.size; + } + } + +private: + struct Match { + size_t i, j, size, modified; + }; + struct Row { + size_t row_index; + size_t tv_index; + }; + + IndexSet const& m_modified; + std::vector m_longest_matches; + + std::vector a, b; + + Match find_longest_match(size_t begin1, size_t end1, size_t begin2, size_t end2) + { + struct Length { + size_t j, len; + }; + std::vector cur; + std::vector prev; + + auto length = [&](size_t j) -> size_t { + for (auto const& pair : prev) { + if (pair.j + 1 == j) + return pair.len + 1; + } + return 1; + }; + + Match best = {begin1, begin2, 0, 0}; + + for (size_t i = begin1; i < end1; ++i) { + cur.clear(); + + size_t ai = a[i].row_index; + // Find the TV indicies at which this row appears in the new results + // There should always be at least one (or it would have been filtered out earlier), + // but can be multiple if there are dupes + auto it = lower_bound(begin(b), end(b), Row{ai, 0}, + [](auto a, auto b) { return a.row_index < b.row_index; }); + REALM_ASSERT(it != end(b) && it->row_index == ai); + for (; it != end(b) && it->row_index == ai; ++it) { + size_t j = it->tv_index; + if (j < begin2) + continue; + if (j >= end2) + break; // b is sorted by tv_index so this can't transition from false to true + + size_t size = length(j); + cur.push_back({j, size}); + if (size > best.size) + best = {i - size + 1, j - size + 1, size, IndexSet::npos}; + // Given two equal-length matches, prefer the one with fewer modified rows + else if (size == best.size) { + if (best.modified == IndexSet::npos) + best.modified = m_modified.count(best.j - size + 1, best.j + 1); + auto count = m_modified.count(j - size + 1, j + 1); + if (count < best.modified) + best = {i - size + 1, j - size + 1, size, count}; + } + REALM_ASSERT(best.i >= begin1 && best.i + best.size <= end1); + REALM_ASSERT(best.j >= begin2 && best.j + best.size <= end2); + } + cur.swap(prev); + } + return best; + } + + void find_longest_matches(size_t begin1, size_t end1, size_t begin2, size_t end2) + { + // FIXME: recursion could get too deep here + // recursion depth worst case is currently O(N) and each recursion uses 320 bytes of stack + // could reduce worst case to O(sqrt(N)) (and typical case to O(log N)) + // biasing equal selections towards the middle, but that's still + // insufficient for Android's 8 KB stacks + auto m = find_longest_match(begin1, end1, begin2, end2); + if (!m.size) + return; + if (m.i > begin1 && m.j > begin2) + find_longest_matches(begin1, m.i, begin2, m.j); + m_longest_matches.push_back(m); + if (m.i + m.size < end2 && m.j + m.size < end2) + find_longest_matches(m.i + m.size, end1, m.j + m.size, end2); + } +}; + +} // Anonymous namespace + +CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector const& prev_rows, + std::vector const& next_rows, + std::function row_did_change, + bool sort) +{ + REALM_ASSERT_DEBUG(sort || std::is_sorted(begin(next_rows), end(next_rows))); + + CollectionChangeBuilder ret; + + size_t deleted = 0; + std::vector old_rows; + old_rows.reserve(prev_rows.size()); + for (size_t i = 0; i < prev_rows.size(); ++i) { + if (prev_rows[i] == IndexSet::npos) { + ++deleted; + ret.deletions.add(i); + } + else + old_rows.push_back({prev_rows[i], IndexSet::npos, i, i - deleted}); + } + std::sort(begin(old_rows), end(old_rows), [](auto& lft, auto& rgt) { + return lft.row_index < rgt.row_index; + }); + + std::vector new_rows; + new_rows.reserve(next_rows.size()); + for (size_t i = 0; i < next_rows.size(); ++i) { + new_rows.push_back({next_rows[i], IndexSet::npos, i, 0}); + } + std::sort(begin(new_rows), end(new_rows), [](auto& lft, auto& rgt) { + return lft.row_index < rgt.row_index; + }); + + IndexSet removed; + + size_t i = 0, j = 0; + while (i < old_rows.size() && j < new_rows.size()) { + auto old_index = old_rows[i]; + auto new_index = new_rows[j]; + if (old_index.row_index == new_index.row_index) { + new_rows[j].prev_tv_index = old_rows[i].tv_index; + new_rows[j].shifted_tv_index = old_rows[i].shifted_tv_index; + ++i; + ++j; + } + else if (old_index.row_index < new_index.row_index) { + removed.add(old_index.tv_index); + ++i; + } + else { + ret.insertions.add(new_index.tv_index); + ++j; + } + } + + for (; i < old_rows.size(); ++i) + removed.add(old_rows[i].tv_index); + for (; j < new_rows.size(); ++j) + ret.insertions.add(new_rows[j].tv_index); + + // Filter out the new insertions since we don't need them for any of the + // further calculations + new_rows.erase(std::remove_if(begin(new_rows), end(new_rows), + [](auto& row) { return row.prev_tv_index == IndexSet::npos; }), + end(new_rows)); + std::sort(begin(new_rows), end(new_rows), + [](auto& lft, auto& rgt) { return lft.tv_index < rgt.tv_index; }); + + for (auto& row : new_rows) { + if (row_did_change(row.row_index)) { + ret.modifications.add(row.tv_index); + } + } + + if (sort) { + SortedMoveCalculator(new_rows, ret); + } + else { + calculate_moves_unsorted(new_rows, removed, ret); + } + ret.deletions.add(removed); + ret.verify(); + +#ifdef REALM_DEBUG + { // Verify that applying the calculated change to prev_rows actually produces next_rows + auto rows = prev_rows; + auto it = util::make_reverse_iterator(ret.deletions.end()); + auto end = util::make_reverse_iterator(ret.deletions.begin()); + for (; it != end; ++it) { + rows.erase(rows.begin() + it->first, rows.begin() + it->second); + } + + for (auto i : ret.insertions.as_indexes()) { + rows.insert(rows.begin() + i, next_rows[i]); + } + + REALM_ASSERT(rows == next_rows); + } +#endif + + return ret; +} diff --git a/src/impl/collection_change_builder.hpp b/src/impl/collection_change_builder.hpp new file mode 100644 index 00000000..a0d6ba48 --- /dev/null +++ b/src/impl/collection_change_builder.hpp @@ -0,0 +1,65 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2016 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_COLLECTION_CHANGE_BUILDER_HPP +#define REALM_COLLECTION_CHANGE_BUILDER_HPP + +#include "collection_notifications.hpp" + +#include + +namespace realm { +namespace _impl { +class CollectionChangeBuilder : public CollectionChangeSet { +public: + CollectionChangeBuilder(CollectionChangeBuilder const&) = default; + CollectionChangeBuilder(CollectionChangeBuilder&&) = default; + CollectionChangeBuilder& operator=(CollectionChangeBuilder const&) = default; + CollectionChangeBuilder& operator=(CollectionChangeBuilder&&) = default; + + CollectionChangeBuilder(IndexSet deletions = {}, + IndexSet insertions = {}, + IndexSet modification = {}, + std::vector moves = {}); + + static CollectionChangeBuilder calculate(std::vector const& old_rows, + std::vector const& new_rows, + std::function row_did_change, + bool sort); + + void merge(CollectionChangeBuilder&&); + void clean_up_stale_moves(); + + void insert(size_t ndx, size_t count=1, bool track_moves=true); + void modify(size_t ndx); + void erase(size_t ndx); + void move_over(size_t ndx, size_t last_ndx, bool track_moves=true); + void clear(size_t old_size); + void move(size_t from, size_t to); + + void parse_complete(); + +private: + std::unordered_map m_move_mapping; + + void verify(); +}; +} // namespace _impl +} // namespace realm + +#endif // REALM_COLLECTION_CHANGE_BUILDER_HPP diff --git a/src/impl/collection_notifier.cpp b/src/impl/collection_notifier.cpp index 2a039f81..eb92e6fa 100644 --- a/src/impl/collection_notifier.cpp +++ b/src/impl/collection_notifier.cpp @@ -251,574 +251,3 @@ void CollectionNotifier::detach() do_detach_from(*m_sg); m_sg = nullptr; } - -CollectionChangeBuilder::CollectionChangeBuilder(IndexSet deletions, - IndexSet insertions, - IndexSet modifications, - std::vector moves) -: CollectionChangeSet({std::move(deletions), std::move(insertions), std::move(modifications), std::move(moves)}) -{ - for (auto&& move : this->moves) { - this->deletions.add(move.from); - this->insertions.add(move.to); - } -} - -void CollectionChangeBuilder::merge(CollectionChangeBuilder&& c) -{ - if (c.empty()) - return; - if (empty()) { - *this = std::move(c); - return; - } - - verify(); - c.verify(); - - // First update any old moves - if (!c.moves.empty() || !c.deletions.empty() || !c.insertions.empty()) { - auto it = remove_if(begin(moves), end(moves), [&](auto& old) { - // Check if the moved row was moved again, and if so just update the destination - auto it = find_if(begin(c.moves), end(c.moves), [&](auto const& m) { - return old.to == m.from; - }); - if (it != c.moves.end()) { - if (modifications.contains(it->from)) - c.modifications.add(it->to); - old.to = it->to; - *it = c.moves.back(); - c.moves.pop_back(); - ++it; - return false; - } - - // Check if the destination was deleted - // Removing the insert for this move will happen later - if (c.deletions.contains(old.to)) - return true; - - // Update the destination to adjust for any new insertions and deletions - old.to = c.insertions.shift(c.deletions.unshift(old.to)); - return false; - }); - moves.erase(it, end(moves)); - } - - // Ignore new moves of rows which were previously inserted (the implicit - // delete from the move will remove the insert) - if (!insertions.empty() && !c.moves.empty()) { - c.moves.erase(remove_if(begin(c.moves), end(c.moves), - [&](auto const& m) { return insertions.contains(m.from); }), - end(c.moves)); - } - - // Ensure that any previously modified rows which were moved are still modified - if (!modifications.empty() && !c.moves.empty()) { - for (auto const& move : c.moves) { - if (modifications.contains(move.from)) - c.modifications.add(move.to); - } - } - - // Update the source position of new moves to compensate for the changes made - // in the old changeset - if (!deletions.empty() || !insertions.empty()) { - for (auto& move : c.moves) - move.from = deletions.shift(insertions.unshift(move.from)); - } - - moves.insert(end(moves), begin(c.moves), end(c.moves)); - - // New deletion indices have been shifted by the insertions, so unshift them - // before adding - deletions.add_shifted_by(insertions, c.deletions); - - // Drop any inserted-then-deleted rows, then merge in new insertions - insertions.erase_at(c.deletions); - insertions.insert_at(c.insertions); - - clean_up_stale_moves(); - - modifications.erase_at(c.deletions); - modifications.shift_for_insert_at(c.insertions); - modifications.add(c.modifications); - - c = {}; - verify(); -} - -void CollectionChangeBuilder::clean_up_stale_moves() -{ - // Look for moves which are now no-ops, and remove them plus the associated - // insert+delete. Note that this isn't just checking for from == to due to - // that rows can also be shifted by other inserts and deletes - moves.erase(remove_if(begin(moves), end(moves), [&](auto const& move) { - if (move.from - deletions.count(0, move.from) != move.to - insertions.count(0, move.to)) - return false; - deletions.remove(move.from); - insertions.remove(move.to); - return true; - }), end(moves)); -} - -void CollectionChangeBuilder::parse_complete() -{ - moves.reserve(m_move_mapping.size()); - for (auto move : m_move_mapping) { - REALM_ASSERT_DEBUG(deletions.contains(move.second)); - REALM_ASSERT_DEBUG(insertions.contains(move.first)); - moves.push_back({move.second, move.first}); - } - m_move_mapping.clear(); - std::sort(begin(moves), end(moves), - [](auto const& a, auto const& b) { return a.from < b.from; }); -} - -void CollectionChangeBuilder::modify(size_t ndx) -{ - modifications.add(ndx); -} - -void CollectionChangeBuilder::insert(size_t index, size_t count, bool track_moves) -{ - modifications.shift_for_insert_at(index, count); - if (!track_moves) - return; - - insertions.insert_at(index, count); - - for (auto& move : moves) { - if (move.to >= index) - ++move.to; - } -} - -void CollectionChangeBuilder::erase(size_t index) -{ - modifications.erase_at(index); - size_t unshifted = insertions.erase_or_unshift(index); - if (unshifted != npos) - deletions.add_shifted(unshifted); - - for (size_t i = 0; i < moves.size(); ++i) { - auto& move = moves[i]; - if (move.to == index) { - moves.erase(moves.begin() + i); - --i; - } - else if (move.to > index) - --move.to; - } -} - -void CollectionChangeBuilder::clear(size_t old_size) -{ - if (old_size != std::numeric_limits::max()) { - for (auto range : deletions) - old_size += range.second - range.first; - for (auto range : insertions) - old_size -= range.second - range.first; - } - - modifications.clear(); - insertions.clear(); - moves.clear(); - m_move_mapping.clear(); - deletions.set(old_size); -} - -void CollectionChangeBuilder::move(size_t from, size_t to) -{ - REALM_ASSERT(from != to); - - bool updated_existing_move = false; - for (auto& move : moves) { - if (move.to != from) { - // Shift other moves if this row is moving from one side of them - // to the other - if (move.to >= to && move.to < from) - ++move.to; - else if (move.to <= to && move.to > from) - --move.to; - continue; - } - REALM_ASSERT(!updated_existing_move); - - // Collapse A -> B, B -> C into a single A -> C move - move.to = to; - updated_existing_move = true; - - insertions.erase_at(from); - insertions.insert_at(to); - } - - if (!updated_existing_move) { - auto shifted_from = insertions.erase_or_unshift(from); - insertions.insert_at(to); - - // Don't report deletions/moves for newly inserted rows - if (shifted_from != npos) { - shifted_from = deletions.add_shifted(shifted_from); - moves.push_back({shifted_from, to}); - } - } - - bool modified = modifications.contains(from); - modifications.erase_at(from); - - if (modified) - modifications.insert_at(to); - else - modifications.shift_for_insert_at(to); -} - -void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row, bool track_moves) -{ - REALM_ASSERT(row_ndx <= last_row); - REALM_ASSERT(insertions.empty() || prev(insertions.end())->second - 1 <= last_row); - REALM_ASSERT(modifications.empty() || prev(modifications.end())->second - 1 <= last_row); - if (track_moves && row_ndx == last_row) { - erase(row_ndx); - return; - } - - bool modified = modifications.contains(last_row); - if (modified) { - modifications.remove(last_row); - modifications.add(row_ndx); - } - else - modifications.remove(row_ndx); - - if (!track_moves) - return; - - bool row_is_insertion = insertions.contains(row_ndx); - bool last_is_insertion = !insertions.empty() && prev(insertions.end())->second == last_row + 1; - REALM_ASSERT_DEBUG(insertions.empty() || prev(insertions.end())->second <= last_row + 1); - - // Collapse A -> B, B -> C into a single A -> C move - bool last_was_already_moved = false; - if (last_is_insertion) { - auto it = m_move_mapping.find(last_row); - if (it != m_move_mapping.end() && it->first == last_row) { - m_move_mapping[row_ndx] = it->second; - m_move_mapping.erase(it); - last_was_already_moved = true; - } - } - - // Remove moves to the row being deleted - if (row_is_insertion && !last_was_already_moved) { - auto it = m_move_mapping.find(row_ndx); - if (it != m_move_mapping.end() && it->first == row_ndx) - m_move_mapping.erase(it); - } - - // Don't report deletions/moves if last_row is newly inserted - if (last_is_insertion) { - insertions.remove(last_row); - } - // If it was previously moved, the unshifted source row has already been marked as deleted - else if (!last_was_already_moved) { - auto shifted_last_row = insertions.unshift(last_row); - shifted_last_row = deletions.add_shifted(shifted_last_row); - m_move_mapping[row_ndx] = shifted_last_row; - } - - // Don't mark the moved-over row as deleted if it was a new insertion - if (!row_is_insertion) { - deletions.add_shifted(insertions.unshift(row_ndx)); - insertions.add(row_ndx); - } - verify(); -} - -void CollectionChangeBuilder::verify() -{ -#ifdef REALM_DEBUG - for (auto&& move : moves) { - REALM_ASSERT(deletions.contains(move.from)); - REALM_ASSERT(insertions.contains(move.to)); - } -#endif -} - -namespace { -struct RowInfo { - size_t row_index; - size_t prev_tv_index; - size_t tv_index; - size_t shifted_tv_index; -}; - -void calculate_moves_unsorted(std::vector& new_rows, IndexSet& removed, CollectionChangeSet& changeset) -{ - size_t expected = 0; - for (auto& row : new_rows) { - // With unsorted queries rows only move due to move_last_over(), which - // inherently can only move a row to earlier in the table. - REALM_ASSERT(row.shifted_tv_index >= expected); - if (row.shifted_tv_index == expected) { - ++expected; - continue; - } - - // This row isn't just the row after the previous one, but it still may - // not be a move if there were rows deleted between the two, so next - // calcuate what row should be here taking those in to account - size_t calc_expected = row.tv_index - changeset.insertions.count(0, row.tv_index) + removed.count(0, row.prev_tv_index); - if (row.shifted_tv_index == calc_expected) { - expected = calc_expected + 1; - continue; - } - - // The row still isn't the expected one, so it's a move - changeset.moves.push_back({row.prev_tv_index, row.tv_index}); - changeset.insertions.add(row.tv_index); - removed.add(row.prev_tv_index); - } -} - -class SortedMoveCalculator { -public: - SortedMoveCalculator(std::vector& new_rows, CollectionChangeSet& changeset) - : m_modified(changeset.modifications) - { - std::vector old_candidates; - old_candidates.reserve(new_rows.size()); - for (auto& row : new_rows) { - old_candidates.push_back({row.row_index, row.prev_tv_index}); - } - std::sort(begin(old_candidates), end(old_candidates), [](auto a, auto b) { - return std::tie(a.tv_index, a.row_index) < std::tie(b.tv_index, b.row_index); - }); - - // First check if the order of any of the rows actually changed - size_t first_difference = npos; - for (size_t i = 0; i < old_candidates.size(); ++i) { - if (old_candidates[i].row_index != new_rows[i].row_index) { - first_difference = i; - break; - } - } - if (first_difference == npos) - return; - - // A map from row index -> tv index in new results - b.reserve(new_rows.size()); - for (size_t i = 0; i < new_rows.size(); ++i) - b.push_back({new_rows[i].row_index, i}); - std::sort(begin(b), end(b), [](auto a, auto b) { - return std::tie(a.row_index, a.tv_index) < std::tie(b.row_index, b.tv_index); - }); - - a = std::move(old_candidates); - - find_longest_matches(first_difference, a.size(), - first_difference, new_rows.size()); - m_longest_matches.push_back({a.size(), new_rows.size(), 0}); - - size_t i = first_difference, j = first_difference; - for (auto match : m_longest_matches) { - for (; i < match.i; ++i) - changeset.deletions.add(a[i].tv_index); - for (; j < match.j; ++j) - changeset.insertions.add(new_rows[j].tv_index); - i += match.size; - j += match.size; - } - } - -private: - struct Match { - size_t i, j, size, modified; - }; - struct Row { - size_t row_index; - size_t tv_index; - }; - - IndexSet const& m_modified; - std::vector m_longest_matches; - - std::vector a, b; - - Match find_longest_match(size_t begin1, size_t end1, size_t begin2, size_t end2) - { - struct Length { - size_t j, len; - }; - std::vector cur; - std::vector prev; - - auto length = [&](size_t j) -> size_t { - for (auto const& pair : prev) { - if (pair.j + 1 == j) - return pair.len + 1; - } - return 1; - }; - - Match best = {begin1, begin2, 0, 0}; - - for (size_t i = begin1; i < end1; ++i) { - cur.clear(); - - size_t ai = a[i].row_index; - // Find the TV indicies at which this row appears in the new results - // There should always be at least one (or it would have been filtered out earlier), - // but can be multiple if there are dupes - auto it = lower_bound(begin(b), end(b), Row{ai, 0}, - [](auto a, auto b) { return a.row_index < b.row_index; }); - REALM_ASSERT(it != end(b) && it->row_index == ai); - for (; it != end(b) && it->row_index == ai; ++it) { - size_t j = it->tv_index; - if (j < begin2) - continue; - if (j >= end2) - break; // b is sorted by tv_index so this can't transition from false to true - - size_t size = length(j); - cur.push_back({j, size}); - if (size > best.size) - best = {i - size + 1, j - size + 1, size, npos}; - // Given two equal-length matches, prefer the one with fewer modified rows - else if (size == best.size) { - if (best.modified == npos) - best.modified = m_modified.count(best.j - size + 1, best.j + 1); - auto count = m_modified.count(j - size + 1, j + 1); - if (count < best.modified) - best = {i - size + 1, j - size + 1, size, count}; - } - REALM_ASSERT(best.i >= begin1 && best.i + best.size <= end1); - REALM_ASSERT(best.j >= begin2 && best.j + best.size <= end2); - } - cur.swap(prev); - } - return best; - } - - void find_longest_matches(size_t begin1, size_t end1, size_t begin2, size_t end2) - { - // FIXME: recursion could get too deep here - // recursion depth worst case is currently O(N) and each recursion uses 320 bytes of stack - // could reduce worst case to O(sqrt(N)) (and typical case to O(log N)) - // biasing equal selections towards the middle, but that's still - // insufficient for Android's 8 KB stacks - auto m = find_longest_match(begin1, end1, begin2, end2); - if (!m.size) - return; - if (m.i > begin1 && m.j > begin2) - find_longest_matches(begin1, m.i, begin2, m.j); - m_longest_matches.push_back(m); - if (m.i + m.size < end2 && m.j + m.size < end2) - find_longest_matches(m.i + m.size, end1, m.j + m.size, end2); - } -}; - -} // Anonymous namespace - -CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector const& prev_rows, - std::vector const& next_rows, - std::function row_did_change, - bool sort) -{ - REALM_ASSERT_DEBUG(sort || std::is_sorted(begin(next_rows), end(next_rows))); - - CollectionChangeBuilder ret; - - size_t deleted = 0; - std::vector old_rows; - old_rows.reserve(prev_rows.size()); - for (size_t i = 0; i < prev_rows.size(); ++i) { - if (prev_rows[i] == npos) { - ++deleted; - ret.deletions.add(i); - } - else - old_rows.push_back({prev_rows[i], npos, i, i - deleted}); - } - std::sort(begin(old_rows), end(old_rows), [](auto& lft, auto& rgt) { - return lft.row_index < rgt.row_index; - }); - - std::vector new_rows; - new_rows.reserve(next_rows.size()); - for (size_t i = 0; i < next_rows.size(); ++i) { - new_rows.push_back({next_rows[i], npos, i, 0}); - } - std::sort(begin(new_rows), end(new_rows), [](auto& lft, auto& rgt) { - return lft.row_index < rgt.row_index; - }); - - IndexSet removed; - - size_t i = 0, j = 0; - while (i < old_rows.size() && j < new_rows.size()) { - auto old_index = old_rows[i]; - auto new_index = new_rows[j]; - if (old_index.row_index == new_index.row_index) { - new_rows[j].prev_tv_index = old_rows[i].tv_index; - new_rows[j].shifted_tv_index = old_rows[i].shifted_tv_index; - ++i; - ++j; - } - else if (old_index.row_index < new_index.row_index) { - removed.add(old_index.tv_index); - ++i; - } - else { - ret.insertions.add(new_index.tv_index); - ++j; - } - } - - for (; i < old_rows.size(); ++i) - removed.add(old_rows[i].tv_index); - for (; j < new_rows.size(); ++j) - ret.insertions.add(new_rows[j].tv_index); - - // Filter out the new insertions since we don't need them for any of the - // further calculations - new_rows.erase(std::remove_if(begin(new_rows), end(new_rows), - [](auto& row) { return row.prev_tv_index == npos; }), - end(new_rows)); - std::sort(begin(new_rows), end(new_rows), - [](auto& lft, auto& rgt) { return lft.tv_index < rgt.tv_index; }); - - for (auto& row : new_rows) { - if (row_did_change(row.row_index)) { - ret.modifications.add(row.tv_index); - } - } - - if (sort) { - SortedMoveCalculator(new_rows, ret); - } - else { - calculate_moves_unsorted(new_rows, removed, ret); - } - ret.deletions.add(removed); - ret.verify(); - -#ifdef REALM_DEBUG - { // Verify that applying the calculated change to prev_rows actually produces next_rows - auto rows = prev_rows; - auto it = util::make_reverse_iterator(ret.deletions.end()); - auto end = util::make_reverse_iterator(ret.deletions.begin()); - for (; it != end; ++it) { - rows.erase(rows.begin() + it->first, rows.begin() + it->second); - } - - for (auto i : ret.insertions.as_indexes()) { - rows.insert(rows.begin() + i, next_rows[i]); - } - - REALM_ASSERT(rows == next_rows); - } -#endif - - return ret; -} diff --git a/src/impl/collection_notifier.hpp b/src/impl/collection_notifier.hpp index eb906d8d..2159b45c 100644 --- a/src/impl/collection_notifier.hpp +++ b/src/impl/collection_notifier.hpp @@ -19,7 +19,7 @@ #ifndef REALM_BACKGROUND_COLLECTION_HPP #define REALM_BACKGROUND_COLLECTION_HPP -#include "collection_notifications.hpp" +#include "impl/collection_change_builder.hpp" #include @@ -34,41 +34,6 @@ namespace realm { class Realm; namespace _impl { -class CollectionChangeBuilder : public CollectionChangeSet { -public: - CollectionChangeBuilder(CollectionChangeBuilder const&) = default; - CollectionChangeBuilder(CollectionChangeBuilder&&) = default; - CollectionChangeBuilder& operator=(CollectionChangeBuilder const&) = default; - CollectionChangeBuilder& operator=(CollectionChangeBuilder&&) = default; - - CollectionChangeBuilder(IndexSet deletions = {}, - IndexSet insertions = {}, - IndexSet modification = {}, - std::vector moves = {}); - - static CollectionChangeBuilder calculate(std::vector const& old_rows, - std::vector const& new_rows, - std::function row_did_change, - bool sort); - - void merge(CollectionChangeBuilder&&); - void clean_up_stale_moves(); - - void insert(size_t ndx, size_t count=1, bool track_moves=true); - void modify(size_t ndx); - void erase(size_t ndx); - void move_over(size_t ndx, size_t last_ndx, bool track_moves=true); - void clear(size_t old_size); - void move(size_t from, size_t to); - - void parse_complete(); - -private: - std::unordered_map m_move_mapping; - - void verify(); - -}; struct ListChangeInfo { size_t table_ndx; size_t row_ndx; From 95900f5e17bdb3565d7ca378fec7adf84b49ce15 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 19 Apr 2016 16:07:40 -0700 Subject: [PATCH 78/93] Make more member functions in CollectionNotifier pure virtual --- src/impl/collection_notifier.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/impl/collection_notifier.hpp b/src/impl/collection_notifier.hpp index 2159b45c..1d98efc8 100644 --- a/src/impl/collection_notifier.hpp +++ b/src/impl/collection_notifier.hpp @@ -107,7 +107,7 @@ public: // transaction advance, and register all required information in it void add_required_change_info(TransactionChangeInfo& info); - virtual void run() { } + virtual void run() = 0; void prepare_handover(); bool deliver(SharedGroup&, std::exception_ptr); @@ -122,7 +122,7 @@ private: virtual void do_detach_from(SharedGroup&) = 0; virtual void do_prepare_handover(SharedGroup&) = 0; virtual bool do_deliver(SharedGroup&) { return true; } - virtual bool do_add_required_change_info(TransactionChangeInfo&) { return true; } + virtual bool do_add_required_change_info(TransactionChangeInfo&) = 0; const std::thread::id m_thread_id = std::this_thread::get_id(); bool is_for_current_thread() const { return m_thread_id == std::this_thread::get_id(); } From eee6e55eb0181e247eeb2ba6f3582fcbcdc462bf Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 20 Apr 2016 13:45:29 -0700 Subject: [PATCH 79/93] Refactor SortedMoveCalculator and add more of an explanation --- src/impl/collection_change_builder.cpp | 193 ++++++++++++++++--------- src/impl/collection_change_builder.hpp | 2 + 2 files changed, 127 insertions(+), 68 deletions(-) diff --git a/src/impl/collection_change_builder.cpp b/src/impl/collection_change_builder.cpp index 3617bdef..aae22217 100644 --- a/src/impl/collection_change_builder.cpp +++ b/src/impl/collection_change_builder.cpp @@ -352,78 +352,65 @@ void calculate_moves_unsorted(std::vector& new_rows, IndexSet& removed, } } -class SortedMoveCalculator { +class LongestCommonSubsequenceCalculator { public: - SortedMoveCalculator(std::vector& new_rows, CollectionChangeSet& changeset) - : m_modified(changeset.modifications) - { - std::vector old_candidates; - old_candidates.reserve(new_rows.size()); - for (auto& row : new_rows) { - old_candidates.push_back({row.row_index, row.prev_tv_index}); - } - std::sort(begin(old_candidates), end(old_candidates), [](auto a, auto b) { - return std::tie(a.tv_index, a.row_index) < std::tie(b.tv_index, b.row_index); - }); - - // First check if the order of any of the rows actually changed - size_t first_difference = IndexSet::npos; - for (size_t i = 0; i < old_candidates.size(); ++i) { - if (old_candidates[i].row_index != new_rows[i].row_index) { - first_difference = i; - break; - } - } - if (first_difference == IndexSet::npos) - return; - - // A map from row index -> tv index in new results - b.reserve(new_rows.size()); - for (size_t i = 0; i < new_rows.size(); ++i) - b.push_back({new_rows[i].row_index, i}); - std::sort(begin(b), end(b), [](auto a, auto b) { - return std::tie(a.row_index, a.tv_index) < std::tie(b.row_index, b.tv_index); - }); - - a = std::move(old_candidates); - - find_longest_matches(first_difference, a.size(), - first_difference, new_rows.size()); - m_longest_matches.push_back({a.size(), new_rows.size(), 0}); - - size_t i = first_difference, j = first_difference; - for (auto match : m_longest_matches) { - for (; i < match.i; ++i) - changeset.deletions.add(a[i].tv_index); - for (; j < match.j; ++j) - changeset.insertions.add(new_rows[j].tv_index); - i += match.size; - j += match.size; - } - } - -private: - struct Match { - size_t i, j, size, modified; - }; + // A pair of an index in the table and an index in the table view struct Row { size_t row_index; size_t tv_index; }; - IndexSet const& m_modified; + struct Match { + // The index in `a` at which this match begins + size_t i; + // The index in `b` at which this match begins + size_t j; + // The length of this match + size_t size; + // The number of rows in this block which were modified + size_t modified; + }; std::vector m_longest_matches; - std::vector a, b; + LongestCommonSubsequenceCalculator(std::vector& a, std::vector& b, + size_t start_index, + IndexSet const& modifications) + : m_modified(modifications) + , a(a), b(b) + { + find_longest_matches(start_index, a.size(), + start_index, b.size()); + m_longest_matches.push_back({a.size(), b.size(), 0}); + } +private: + IndexSet const& m_modified; + + // The two arrays of rows being diffed + // a is sorted by tv_index, b is sorted by row_index + std::vector &a, &b; + + // Find the longest matching range in (a + begin1, a + end1) and (b + begin2, b + end2) + // "Matching" is defined as "has the same row index"; the TV index is just + // there to let us turn an index in a/b into an index which can be reported + // in the output changeset. + // + // This is done with the O(N) space variant of the dynamic programming + // algorithm for longest common subsequence, where N is the maximum number + // of the most common row index (which for everything but linkview-derived + // TVs will be 1). Match find_longest_match(size_t begin1, size_t end1, size_t begin2, size_t end2) { struct Length { size_t j, len; }; - std::vector cur; + // The length of the matching block for each `j` for the previously checked row std::vector prev; + // The length of the matching block for each `j` for the row currently being checked + std::vector cur; + // Calculate the length of the matching block *ending* at b[j], which + // is 1 if b[j - 1] did not match, and b[j - 1] + 1 otherwise. auto length = [&](size_t j) -> size_t { for (auto const& pair : prev) { if (pair.j + 1 == j) @@ -432,17 +419,15 @@ private: return 1; }; - Match best = {begin1, begin2, 0, 0}; - - for (size_t i = begin1; i < end1; ++i) { - cur.clear(); - + // Iterate over each `j` which has the same row index as a[i] and falls + // within the range begin2 <= j < end2 + auto for_each_b_match = [&](size_t i, auto&& f) { size_t ai = a[i].row_index; // Find the TV indicies at which this row appears in the new results - // There should always be at least one (or it would have been filtered out earlier), - // but can be multiple if there are dupes - auto it = lower_bound(begin(b), end(b), Row{ai, 0}, - [](auto a, auto b) { return a.row_index < b.row_index; }); + // There should always be at least one (or it would have been + // filtered out earlier), but there can be multiple if there are dupes + auto it = lower_bound(begin(b), end(b), ai, + [](auto lft, auto rgt) { return lft.row_index < rgt; }); REALM_ASSERT(it != end(b) && it->row_index == ai); for (; it != end(b) && it->row_index == ai; ++it) { size_t j = it->tv_index; @@ -450,9 +435,23 @@ private: continue; if (j >= end2) break; // b is sorted by tv_index so this can't transition from false to true + f(j); + } + }; + Match best = {begin1, begin2, 0, 0}; + for (size_t i = begin1; i < end1; ++i) { + // prev = std::move(cur), but avoids discarding prev's heap allocation + cur.swap(prev); + cur.clear(); + + for_each_b_match(i, [&](size_t j) { size_t size = length(j); + cur.push_back({j, size}); + + // If the matching block ending at a[i] and b[j] is longer than + // the previous one, select it as the best if (size > best.size) best = {i - size + 1, j - size + 1, size, IndexSet::npos}; // Given two equal-length matches, prefer the one with fewer modified rows @@ -463,10 +462,11 @@ private: if (count < best.modified) best = {i - size + 1, j - size + 1, size, count}; } + + // The best block should always fall within the range being searched REALM_ASSERT(best.i >= begin1 && best.i + best.size <= end1); REALM_ASSERT(best.j >= begin2 && best.j + best.size <= end2); - } - cur.swap(prev); + }); } return best; } @@ -489,6 +489,57 @@ private: } }; +void calculate_moves_sorted(std::vector& rows, CollectionChangeSet& changeset) +{ + // The RowInfo array contains information about the old and new TV indices of + // each row, which we need to turn into two sequences of rows, which we'll + // then find matches in + std::vector a, b; + + a.reserve(rows.size()); + for (auto& row : rows) { + a.push_back({row.row_index, row.prev_tv_index}); + } + std::sort(begin(a), end(a), [](auto lft, auto rgt) { + return std::tie(lft.tv_index, lft.row_index) < std::tie(rgt.tv_index, rgt.row_index); + }); + + // Before constructing `b`, first find the first index in `a` which will + // actually differ in `b`, and skip everything else if there aren't any + size_t first_difference = IndexSet::npos; + for (size_t i = 0; i < a.size(); ++i) { + if (a[i].row_index != rows[i].row_index) { + first_difference = i; + break; + } + } + if (first_difference == IndexSet::npos) + return; + + // Note that `b` is sorted by row_index, while `a` is sorted by tv_index + b.reserve(rows.size()); + for (size_t i = 0; i < rows.size(); ++i) + b.push_back({rows[i].row_index, i}); + std::sort(begin(b), end(b), [](auto lft, auto rgt) { + return std::tie(lft.row_index, lft.tv_index) < std::tie(rgt.row_index, rgt.tv_index); + }); + + // Calculate the LCS of the two sequences + auto matches = LongestCommonSubsequenceCalculator(a, b, first_difference, + changeset.modifications).m_longest_matches; + + // And then insert and delete rows as needed to align them + size_t i = first_difference, j = first_difference; + for (auto match : matches) { + for (; i < match.i; ++i) + changeset.deletions.add(a[i].tv_index); + for (; j < match.j; ++j) + changeset.insertions.add(rows[j].tv_index); + i += match.size; + j += match.size; + } +} + } // Anonymous namespace CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector const& prev_rows, @@ -524,8 +575,14 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c return lft.row_index < rgt.row_index; }); + // Don't add rows which were modified to not match the query to `deletions` + // immediately because the unsorted move logic needs to be able to distinuish + // them from rows which were outright deleted IndexSet removed; + // Now that our old and new sets of rows are sorted by row index, we can + // iterate over them and either record old+new TV indices for rows present + // in both, or mark them as inserted/deleted if they appear only in one size_t i = 0, j = 0; while (i < old_rows.size() && j < new_rows.size()) { auto old_index = old_rows[i]; @@ -566,7 +623,7 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c } if (sort) { - SortedMoveCalculator(new_rows, ret); + calculate_moves_sorted(new_rows, ret); } else { calculate_moves_unsorted(new_rows, removed, ret); diff --git a/src/impl/collection_change_builder.hpp b/src/impl/collection_change_builder.hpp index a0d6ba48..6e9f78c1 100644 --- a/src/impl/collection_change_builder.hpp +++ b/src/impl/collection_change_builder.hpp @@ -37,6 +37,8 @@ public: IndexSet modification = {}, std::vector moves = {}); + // Calculate where rows need to be inserted or deleted from old_rows to turn + // it into new_rows, and check all matching rows for modifications static CollectionChangeBuilder calculate(std::vector const& old_rows, std::vector const& new_rows, std::function row_did_change, From 8879212111e5a655c39d1f9bb589a407b9160ed3 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 25 Apr 2016 10:15:56 -0700 Subject: [PATCH 80/93] Fix assertion failure when a notifier is removed before it runs --- src/impl/realm_coordinator.cpp | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 78a2f7ae..edb0df09 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -235,13 +235,17 @@ void RealmCoordinator::pin_version(uint_fast64_t version, uint_fast32_t index) } else if (m_new_notifiers.empty()) { // If this is the first notifier then we don't already have a read transaction + REALM_ASSERT_3(m_advancer_sg->get_transact_stage(), ==, SharedGroup::transact_Ready); m_advancer_sg->begin_read(versionid); } - else if (versionid < m_advancer_sg->get_version_of_current_transaction()) { - // Ensure we're holding a readlock on the oldest version we have a - // handover object for, as handover objects don't - m_advancer_sg->end_read(); - m_advancer_sg->begin_read(versionid); + else { + REALM_ASSERT_3(m_advancer_sg->get_transact_stage(), ==, SharedGroup::transact_Reading); + if (versionid < m_advancer_sg->get_version_of_current_transaction()) { + // Ensure we're holding a readlock on the oldest version we have a + // handover object for, as handover objects don't + m_advancer_sg->end_read(); + m_advancer_sg->begin_read(versionid); + } } } @@ -282,10 +286,12 @@ void RealmCoordinator::clean_up_dead_notifiers() // are no notifiers left, but don't close them entirely as opening shared // groups is expensive if (m_notifiers.empty() && m_notifier_sg) { + REALM_ASSERT_3(m_notifier_sg->get_transact_stage(), ==, SharedGroup::transact_Reading); m_notifier_sg->end_read(); } } if (swap_remove(m_new_notifiers)) { + REALM_ASSERT_3(m_advancer_sg->get_transact_stage(), ==, SharedGroup::transact_Reading); if (m_new_notifiers.empty() && m_advancer_sg) { m_advancer_sg->end_read(); } @@ -424,7 +430,14 @@ void RealmCoordinator::run_async_notifiers() IncrementalChangeInfo new_notifier_change_info(*m_advancer_sg, new_notifiers); if (!new_notifiers.empty()) { - REALM_ASSERT(m_advancer_sg->get_version_of_current_transaction() == new_notifiers.front()->version()); + REALM_ASSERT_3(m_advancer_sg->get_transact_stage(), ==, SharedGroup::transact_Reading); + REALM_ASSERT_3(m_advancer_sg->get_version_of_current_transaction().version, + <=, new_notifiers.front()->version().version); + + // The advancer SG can be at an older version than the oldest new notifier + // if a notifier was added and then removed before it ever got the chance + // to run, as we don't move the pin forward when removing dead notifiers + transaction::advance(*m_advancer_sg, nullptr, new_notifiers.front()->version()); // Advance each of the new notifiers to the latest version, attaching them // to the SG at their handover version. This requires a unique @@ -446,6 +459,7 @@ void RealmCoordinator::run_async_notifiers() version = m_advancer_sg->get_version_of_current_transaction(); m_advancer_sg->end_read(); } + REALM_ASSERT_3(m_advancer_sg->get_transact_stage(), ==, SharedGroup::transact_Ready); // Make a copy of the notifiers vector and then release the lock to avoid // blocking other threads trying to register or unregister notifiers while we run them From 632d757014c785f4f791a04c566f4b9fa0d2e5aa Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 25 Apr 2016 12:02:59 -0700 Subject: [PATCH 81/93] Always deliver results to the correct SharedGroup If there are multiple Realm instances for a single file on a single thread due to disabling caching we need to actually deliver the results to the appropriate SharedGroup for each notifier rather than delivering them all to the first one. --- src/impl/collection_notifier.cpp | 9 ++++++--- src/impl/collection_notifier.hpp | 6 +----- src/impl/realm_coordinator.cpp | 4 ++-- tests/list.cpp | 6 ++++-- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/impl/collection_notifier.cpp b/src/impl/collection_notifier.cpp index eb92e6fa..249775da 100644 --- a/src/impl/collection_notifier.cpp +++ b/src/impl/collection_notifier.cpp @@ -180,10 +180,13 @@ void CollectionNotifier::prepare_handover() do_prepare_handover(*m_sg); } -bool CollectionNotifier::deliver(SharedGroup& sg, std::exception_ptr err) +bool CollectionNotifier::deliver(Realm& realm, SharedGroup& sg, std::exception_ptr err) { - if (!is_for_current_thread()) { - return false; + { + std::lock_guard lock(m_realm_mutex); + if (m_realm.get() != &realm) { + return false; + } } if (err) { diff --git a/src/impl/collection_notifier.hpp b/src/impl/collection_notifier.hpp index 1d98efc8..383984d4 100644 --- a/src/impl/collection_notifier.hpp +++ b/src/impl/collection_notifier.hpp @@ -27,7 +27,6 @@ #include #include #include -#include #include namespace realm { @@ -109,7 +108,7 @@ public: virtual void run() = 0; void prepare_handover(); - bool deliver(SharedGroup&, std::exception_ptr); + bool deliver(Realm&, SharedGroup&, std::exception_ptr); protected: bool have_callbacks() const noexcept { return m_have_callbacks; } @@ -124,9 +123,6 @@ private: virtual bool do_deliver(SharedGroup&) { return true; } virtual bool do_add_required_change_info(TransactionChangeInfo&) = 0; - const std::thread::id m_thread_id = std::this_thread::get_id(); - bool is_for_current_thread() const { return m_thread_id == std::this_thread::get_id(); } - mutable std::mutex m_realm_mutex; std::shared_ptr m_realm; diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index edb0df09..09693088 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -567,7 +567,7 @@ void RealmCoordinator::advance_to_ready(Realm& realm) // Query version now matches the SG version, so we can deliver them for (auto& notifier : m_notifiers) { - if (notifier->deliver(sg, m_async_error)) { + if (notifier->deliver(realm, sg, m_async_error)) { notifiers.push_back(notifier); } } @@ -586,7 +586,7 @@ void RealmCoordinator::process_available_async(Realm& realm) { std::lock_guard lock(m_notifier_mutex); for (auto& notifier : m_notifiers) { - if (notifier->deliver(sg, m_async_error)) { + if (notifier->deliver(realm, sg, m_async_error)) { notifiers.push_back(notifier); } } diff --git a/tests/list.cpp b/tests/list.cpp index 79153e7a..4d079b03 100644 --- a/tests/list.cpp +++ b/tests/list.cpp @@ -211,7 +211,8 @@ TEST_CASE("list") { // Each of the Lists now has a different source version and state at // that version, so they should all see different changes despite // being for the same LinkList - advance_and_notify(*r); + for (auto& list : lists) + advance_and_notify(*list.get_realm()); REQUIRE_INDICES(changes[0].insertions, 0, 1, 2); REQUIRE(changes[0].modifications.empty()); @@ -224,7 +225,8 @@ TEST_CASE("list") { // After making another change, they should all get the same notification change_list(); - advance_and_notify(*r); + for (auto& list : lists) + advance_and_notify(*list.get_realm()); for (int i = 0; i < 3; ++i) { REQUIRE_INDICES(changes[i].insertions, 3); From 3218740fd9a5714efce3543f5700073b28c44b39 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 26 Apr 2016 10:58:30 -0700 Subject: [PATCH 82/93] Fix the check for a deleted LV in ListNotifier::add_required_change_info() --- src/impl/list_notifier.cpp | 5 ++++- tests/list.cpp | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp index 2d14602b..aec9afda 100644 --- a/src/impl/list_notifier.cpp +++ b/src/impl/list_notifier.cpp @@ -71,7 +71,7 @@ void ListNotifier::do_detach_from(SharedGroup& sg) bool ListNotifier::do_add_required_change_info(TransactionChangeInfo& info) { REALM_ASSERT(!m_lv_handover); - if (!m_lv) { + if (!m_lv || !m_lv->is_attached()) { return false; // origin row was deleted after the notification was added } @@ -92,6 +92,9 @@ void ListNotifier::run() m_change.deletions.set(m_prev_size); m_prev_size = 0; } + else { + m_change = {}; + } return; } diff --git a/tests/list.cpp b/tests/list.cpp index 4d079b03..c355a6ed 100644 --- a/tests/list.cpp +++ b/tests/list.cpp @@ -104,6 +104,11 @@ TEST_CASE("list") { auto token = require_change(); write([&] { origin->move_last_over(0); }); REQUIRE_INDICES(change.deletions, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + + // Should not resend delete all notification after another commit + change = {}; + write([&] { target->add_empty_row(); }); + REQUIRE(change.empty()); } SECTION("modifying one of the target rows sends a change notification") { From 4df552ba2d20b541cf9fc3ec6f62ec1644f66558 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 26 Apr 2016 14:45:56 -0700 Subject: [PATCH 83/93] Clean up old move info even when the row being deleted is the last one --- src/impl/collection_change_builder.cpp | 9 +++++++-- tests/collection_change_indices.cpp | 13 +++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/impl/collection_change_builder.cpp b/src/impl/collection_change_builder.cpp index aae22217..2fa652f9 100644 --- a/src/impl/collection_change_builder.cpp +++ b/src/impl/collection_change_builder.cpp @@ -249,8 +249,13 @@ void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row, bool tr REALM_ASSERT(row_ndx <= last_row); REALM_ASSERT(insertions.empty() || prev(insertions.end())->second - 1 <= last_row); REALM_ASSERT(modifications.empty() || prev(modifications.end())->second - 1 <= last_row); - if (track_moves && row_ndx == last_row) { - erase(row_ndx); + + if (row_ndx == last_row) { + auto shifted_from = insertions.erase_or_unshift(row_ndx); + if (shifted_from != IndexSet::npos) + deletions.add_shifted(shifted_from); + modifications.remove(row_ndx); + m_move_mapping.erase(row_ndx); return; } diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index 1ca24f16..42625bfb 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -124,7 +124,10 @@ TEST_CASE("[collection_change] move_over()") { SECTION("is just erase when row == last_row") { c.move_over(10, 10); + c.parse_complete(); + REQUIRE_INDICES(c.deletions, 10); + REQUIRE(c.insertions.empty()); REQUIRE(c.moves.empty()); } @@ -189,6 +192,16 @@ TEST_CASE("[collection_change] move_over()") { REQUIRE_MOVES(c, {10, 5}); } + SECTION("removes moves to the row when row == last_row") { + c.move_over(0, 1); + c.move_over(0, 0); + c.parse_complete(); + + REQUIRE_INDICES(c.deletions, 0, 1); + REQUIRE(c.insertions.empty()); + REQUIRE(c.moves.empty()); + } + SECTION("is not shifted by previous calls to move_over()") { c.move_over(5, 10); c.move_over(6, 9); From f9364b50a4385b631521315d9a25ef4da845a02d Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 2 May 2016 16:24:54 -0700 Subject: [PATCH 84/93] Return the correct iterator from ChunkedVector::erase() When a chunk is removed entirely it should return an iterator to the first element in the next chunk, not the last. --- src/index_set.cpp | 2 +- tests/index_set.cpp | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/index_set.cpp b/src/index_set.cpp index b9b9525c..2c566e13 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -151,7 +151,7 @@ ChunkedRangeVector::iterator ChunkedRangeVector::erase(iterator pos) if (chunk.data.size() == 0) { pos.m_outer = m_data.erase(pos.m_outer); pos.m_end = m_data.end(); - pos.m_inner = pos.m_outer == m_data.end() ? nullptr : &pos.m_outer->data.back(); + pos.m_inner = pos.m_outer == m_data.end() ? nullptr : &pos.m_outer->data.front(); verify(); return pos; } diff --git a/tests/index_set.cpp b/tests/index_set.cpp index df5af2f1..ff62a05d 100644 --- a/tests/index_set.cpp +++ b/tests/index_set.cpp @@ -120,6 +120,17 @@ TEST_CASE("[index_set] add()") { set.add(i); REQUIRE_INDICES(set, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20); } + + SECTION("merges ranges even when they are in different chunks") { + realm::IndexSet set2; + for (int i = 0; i < 20; ++i) { + set.add(i * 2); + set2.add(i); + set2.add(i * 2); + } + set.add(set2); + REQUIRE(set.count() == 30); + } } TEST_CASE("[index_set] add_shifted()") { From d8a69b87dcce18a8e950f4ffa5566cbbd1e4fc69 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 29 Apr 2016 10:47:54 -0700 Subject: [PATCH 85/93] Improve change calculation performance for nontrivial object graphs Skip doing any checking at all if none of the tables reachable from the root table have been modified (which can happen if the table version was bumped due to insertions, unrelated backlinks, or unlinked-to rows being deleted in linked tables). Add cycle checking rather than relying on the max depth to handle it, as the worst case was O(N^16) if the cycle involved a LinkList of size N. Track which rows have been confirmed to have not been modified. Cache the information about the links for each of the relevant tables as checking the table schema can get somewhat expensive. --- src/impl/collection_notifier.cpp | 157 ++++++++++---- src/impl/collection_notifier.hpp | 47 ++++- src/impl/list_notifier.cpp | 5 +- src/impl/results_notifier.cpp | 2 +- tests/transaction_log_parsing.cpp | 338 +++++++++++++++++------------- 5 files changed, 355 insertions(+), 194 deletions(-) diff --git a/src/impl/collection_notifier.cpp b/src/impl/collection_notifier.cpp index 249775da..f9e2157c 100644 --- a/src/impl/collection_notifier.cpp +++ b/src/impl/collection_notifier.cpp @@ -26,31 +26,87 @@ using namespace realm; using namespace realm::_impl; -bool TransactionChangeInfo::row_did_change(Table const& table, size_t idx, int depth) const +std::function +CollectionNotifier::get_modification_checker(TransactionChangeInfo const& info, + Table const& root_table) { - if (depth > 16) // arbitrary limit + // First check if any of the tables accessible from the root table were + // actually modified. This can be false if there were only insertions, or + // deletions which were not linked to by any row in the linking table + auto table_modified = [&](auto& tbl) { + return tbl.table_ndx < info.tables.size() + && !info.tables[tbl.table_ndx].modifications.empty(); + }; + if (!any_of(begin(m_related_tables), end(m_related_tables), table_modified)) { + return [](size_t) { return false; }; + } + + return DeepChangeChecker(info, root_table, m_related_tables); +} + +void DeepChangeChecker::find_related_tables(std::vector& out, Table const& table) +{ + auto table_ndx = table.get_index_in_group(); + if (any_of(begin(out), end(out), [=](auto& tbl) { return tbl.table_ndx == table_ndx; })) + return; + + size_t info = out.size(); + out.push_back({table_ndx, {}}); + for (size_t i = 0, count = table.get_column_count(); i != count; ++i) { + auto type = table.get_column_type(i); + if (type == type_Link || type == type_LinkList) { + out[info].links.push_back({i, type == type_LinkList}); + find_related_tables(out, *table.get_link_target(i)); + } + } +} + +DeepChangeChecker::DeepChangeChecker(TransactionChangeInfo const& info, + Table const& root_table, + std::vector const& related_tables) +: m_info(info) +, m_root_table(root_table) +, m_root_table_ndx(root_table.get_index_in_group()) +, m_root_modifications(m_root_table_ndx < info.tables.size() ? &info.tables[m_root_table_ndx].modifications : nullptr) +, m_related_tables(related_tables) +{ +} + +bool DeepChangeChecker::check_outgoing_links(size_t table_ndx, + Table const& table, + size_t row_ndx, size_t depth) +{ + auto it = find_if(begin(m_related_tables), end(m_related_tables), + [&](auto&& tbl) { return tbl.table_ndx == table_ndx; }); + if (it == m_related_tables.end()) return false; - size_t table_ndx = table.get_index_in_group(); - if (table_ndx < tables.size() && tables[table_ndx].modifications.contains(idx)) - return true; - - for (size_t i = 0, count = table.get_column_count(); i < count; ++i) { - auto type = table.get_column_type(i); - if (type == type_Link) { - if (table.is_null_link(i, idx)) - continue; - auto dst = table.get_link(i, idx); - return row_did_change(*table.get_link_target(i), dst, depth + 1); + // Check if we're already checking if the destination of the link is + // modified, and if not add it to the stack + auto already_checking = [&](size_t col) { + for (auto p = m_current_path.begin(); p < m_current_path.begin() + depth; ++p) { + if (p->table == table_ndx && p->row == row_ndx && p->col == col) + return true; } - if (type != type_LinkList) - continue; + m_current_path[depth] = {table_ndx, row_ndx, col, false}; + return false; + }; - auto& target = *table.get_link_target(i); - auto lvr = table.get_linklist(i, idx); - for (size_t j = 0; j < lvr->size(); ++j) { + for (auto const& link : it->links) { + if (already_checking(link.col_ndx)) + continue; + if (!link.is_list) { + if (table.is_null_link(link.col_ndx, row_ndx)) + continue; + auto dst = table.get_link(link.col_ndx, row_ndx); + return check_row(*table.get_link_target(link.col_ndx), dst, depth + 1); + } + + auto& target = *table.get_link_target(link.col_ndx); + auto lvr = table.get_linklist(link.col_ndx, row_ndx); + for (size_t j = 0, size = lvr->size(); j < size; ++j) { size_t dst = lvr->get(j).get_index(); - if (row_did_change(target, dst, depth + 1)) + if (check_row(target, dst, depth + 1)) return true; } } @@ -58,6 +114,39 @@ bool TransactionChangeInfo::row_did_change(Table const& table, size_t idx, int d return false; } +bool DeepChangeChecker::check_row(Table const& table, size_t idx, size_t depth) +{ + // Arbitrary upper limit on the maximum depth to search + if (depth >= m_current_path.size()) { + // Don't mark any of the intermediate rows checked along the path as + // not modified, as a search starting from them might hit a modification + for (size_t i = 1; i < m_current_path.size(); ++i) + m_current_path[i].depth_exceeded = true; + return false; + } + + size_t table_ndx = table.get_index_in_group(); + if (depth > 0 && table_ndx < m_info.tables.size() && m_info.tables[table_ndx].modifications.contains(idx)) + return true; + + if (m_not_modified.size() <= table_ndx) + m_not_modified.resize(table_ndx + 1); + if (m_not_modified[table_ndx].contains(idx)) + return false; + + bool ret = check_outgoing_links(table_ndx, table, idx, depth); + if (!ret && !m_current_path[depth].depth_exceeded) + m_not_modified[table_ndx].add(idx); + return ret; +} + +bool DeepChangeChecker::operator()(size_t ndx) +{ + if (m_root_modifications && m_root_modifications->contains(ndx)) + return true; + return check_row(m_root_table, ndx, 0); +} + CollectionNotifier::CollectionNotifier(std::shared_ptr realm) : m_realm(std::move(realm)) , m_sg_version(Realm::Internal::get_shared_group(*m_realm).get_version_of_current_transaction()) @@ -139,24 +228,10 @@ std::unique_lock CollectionNotifier::lock_target() return std::unique_lock{m_realm_mutex}; } -// Recursively add `table` and all tables it links to to `out` -static void find_relevant_tables(std::vector& out, Table const& table) -{ - auto table_ndx = table.get_index_in_group(); - if (find(begin(out), end(out), table_ndx) != end(out)) - return; - out.push_back(table_ndx); - - for (size_t i = 0, count = table.get_column_count(); i != count; ++i) { - if (table.get_column_type(i) == type_Link || table.get_column_type(i) == type_LinkList) { - find_relevant_tables(out, *table.get_link_target(i)); - } - } -} - void CollectionNotifier::set_table(Table const& table) { - find_relevant_tables(m_relevant_tables, table); + m_related_tables.clear(); + DeepChangeChecker::find_related_tables(m_related_tables, table); } void CollectionNotifier::add_required_change_info(TransactionChangeInfo& info) @@ -165,11 +240,13 @@ void CollectionNotifier::add_required_change_info(TransactionChangeInfo& info) return; } - auto max = *max_element(begin(m_relevant_tables), end(m_relevant_tables)) + 1; - if (max > info.table_modifications_needed.size()) - info.table_modifications_needed.resize(max, false); - for (auto table_ndx : m_relevant_tables) { - info.table_modifications_needed[table_ndx] = true; + auto max = max_element(begin(m_related_tables), end(m_related_tables), + [](auto&& a, auto&& b) { return a.table_ndx < b.table_ndx; }); + + if (max->table_ndx >= info.table_modifications_needed.size()) + info.table_modifications_needed.resize(max->table_ndx + 1, false); + for (auto& tbl : m_related_tables) { + info.table_modifications_needed[tbl.table_ndx] = true; } } diff --git a/src/impl/collection_notifier.hpp b/src/impl/collection_notifier.hpp index 383984d4..211d5ced 100644 --- a/src/impl/collection_notifier.hpp +++ b/src/impl/collection_notifier.hpp @@ -23,6 +23,7 @@ #include +#include #include #include #include @@ -45,8 +46,47 @@ struct TransactionChangeInfo { std::vector table_moves_needed; std::vector lists; std::vector tables; +}; - bool row_did_change(Table const& table, size_t row_ndx, int depth = 0) const; +class DeepChangeChecker { +public: + struct OutgoingLink { + size_t col_ndx; + bool is_list; + }; + struct RelatedTable { + size_t table_ndx; + std::vector links; + }; + + DeepChangeChecker(TransactionChangeInfo const& info, Table const& root_table, + std::vector const& related_tables); + + bool operator()(size_t row_ndx); + + // Recursively add `table` and all tables it links to to `out`, along with + // information about the links from them + static void find_related_tables(std::vector& out, Table const& table); + +private: + TransactionChangeInfo const& m_info; + Table const& m_root_table; + const size_t m_root_table_ndx; + IndexSet const* const m_root_modifications; + std::vector m_not_modified; + std::vector const& m_related_tables; + + struct Path { + size_t table; + size_t row; + size_t col; + bool depth_exceeded; + }; + std::array m_current_path; + + bool check_row(Table const& table, size_t row_ndx, size_t depth = 0); + bool check_outgoing_links(size_t table_ndx, Table const& table, + size_t row_ndx, size_t depth = 0); }; // A base class for a notifier that keeps a collection up to date and/or @@ -116,6 +156,8 @@ protected: void set_table(Table const& table); std::unique_lock lock_target(); + std::function get_modification_checker(TransactionChangeInfo const&, Table const&); + private: virtual void do_attach_to(SharedGroup&) = 0; virtual void do_detach_from(SharedGroup&) = 0; @@ -133,8 +175,7 @@ private: CollectionChangeBuilder m_accumulated_changes; CollectionChangeSet m_changes_to_deliver; - // Tables which this collection needs change information for - std::vector m_relevant_tables; + std::vector m_related_tables; struct Callback { CollectionChangeCallback fn; diff --git a/src/impl/list_notifier.cpp b/src/impl/list_notifier.cpp index aec9afda..64bf6daf 100644 --- a/src/impl/list_notifier.cpp +++ b/src/impl/list_notifier.cpp @@ -98,17 +98,18 @@ void ListNotifier::run() return; } + auto row_did_change = get_modification_checker(*m_info, m_lv->get_target_table()); for (size_t i = 0; i < m_lv->size(); ++i) { if (m_change.modifications.contains(i)) continue; - if (m_info->row_did_change(m_lv->get_target_table(), m_lv->get(i).get_index())) + if (row_did_change(m_lv->get(i).get_index())) m_change.modifications.add(i); } for (auto const& move : m_change.moves) { if (m_change.modifications.contains(move.to)) continue; - if (m_info->row_did_change(m_lv->get_target_table(), m_lv->get(move.to).get_index())) + if (row_did_change(m_lv->get(move.to).get_index())) m_change.modifications.add(move.to); } diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index c6ae69c9..c72fdd0a 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -124,7 +124,7 @@ void ResultsNotifier::calculate_changes() } m_changes = CollectionChangeBuilder::calculate(m_previous_rows, next_rows, - [&](size_t row) { return m_info->row_did_change(*m_query->get_table(), row); }, + get_modification_checker(*m_info, *m_query->get_table()), m_sort || m_from_linkview); m_previous_rows = std::move(next_rows); diff --git a/tests/transaction_log_parsing.cpp b/tests/transaction_log_parsing.cpp index 92748568..b4654d24 100644 --- a/tests/transaction_log_parsing.cpp +++ b/tests/transaction_log_parsing.cpp @@ -177,154 +177,6 @@ TEST_CASE("Transaction log parsing") { } } - SECTION("row_did_change()") { - config.schema = std::make_unique(Schema{ - {"table", "", { - {"int", PropertyTypeInt}, - {"link", PropertyTypeObject, "table", false, false, true}, - {"array", PropertyTypeArray, "table"} - }}, - }); - - auto r = Realm::get_shared_realm(config); - auto table = r->read_group()->get_table("class_table"); - - r->begin_transaction(); - table->add_empty_row(10); - for (int i = 0; i < 10; ++i) - table->set_int(0, i, i); - r->commit_transaction(); - - auto track_changes = [&](auto&& f) { - auto history = make_client_history(config.path); - SharedGroup sg(*history, SharedGroup::durability_MemOnly); - Group const& g = sg.begin_read(); - - r->begin_transaction(); - f(); - r->commit_transaction(); - - _impl::TransactionChangeInfo info; - info.table_modifications_needed.resize(g.size(), true); - info.table_moves_needed.resize(g.size(), true); - _impl::transaction::advance(sg, info); - return info; - }; - - SECTION("direct changes are tracked") { - auto info = track_changes([&] { - table->set_int(0, 9, 10); - }); - - REQUIRE_FALSE(info.row_did_change(*table, 8)); - REQUIRE(info.row_did_change(*table, 9)); - } - - SECTION("changes over links are tracked") { - r->begin_transaction(); - for (int i = 0; i < 9; ++i) - table->set_link(1, i, i + 1); - r->commit_transaction(); - - auto info = track_changes([&] { - table->set_int(0, 9, 10); - }); - - REQUIRE(info.row_did_change(*table, 0)); - } - - SECTION("changes over linklists are tracked") { - r->begin_transaction(); - for (int i = 0; i < 9; ++i) - table->get_linklist(2, i)->add(i + 1); - r->commit_transaction(); - - auto info = track_changes([&] { - table->set_int(0, 9, 10); - }); - - REQUIRE(info.row_did_change(*table, 0)); - } - - SECTION("cycles over links do not loop forever") { - r->begin_transaction(); - table->set_link(1, 0, 0); - r->commit_transaction(); - - auto info = track_changes([&] { - table->set_int(0, 9, 10); - }); - REQUIRE_FALSE(info.row_did_change(*table, 0)); - } - - SECTION("cycles over linklists do not loop forever") { - r->begin_transaction(); - table->get_linklist(2, 0)->add(0); - r->commit_transaction(); - - auto info = track_changes([&] { - table->set_int(0, 9, 10); - }); - REQUIRE_FALSE(info.row_did_change(*table, 0)); - } - - SECTION("targets moving is not a change") { - r->begin_transaction(); - table->set_link(1, 0, 9); - table->get_linklist(2, 0)->add(9); - r->commit_transaction(); - - auto info = track_changes([&] { - table->move_last_over(5); - }); - REQUIRE_FALSE(info.row_did_change(*table, 0)); - } - - SECTION("changes made before a row is moved are reported") { - r->begin_transaction(); - table->set_link(1, 0, 9); - r->commit_transaction(); - - auto info = track_changes([&] { - table->set_int(0, 9, 5); - table->move_last_over(5); - }); - REQUIRE(info.row_did_change(*table, 0)); - - r->begin_transaction(); - table->get_linklist(2, 0)->add(8); - r->commit_transaction(); - - info = track_changes([&] { - table->set_int(0, 8, 5); - table->move_last_over(5); - }); - REQUIRE(info.row_did_change(*table, 0)); - } - - SECTION("changes made after a row is moved are reported") { - r->begin_transaction(); - table->set_link(1, 0, 9); - r->commit_transaction(); - - auto info = track_changes([&] { - table->move_last_over(5); - table->set_int(0, 5, 5); - }); - REQUIRE(info.row_did_change(*table, 0)); - - r->begin_transaction(); - table->get_linklist(2, 0)->add(8); - r->commit_transaction(); - - info = track_changes([&] { - table->move_last_over(5); - table->set_int(0, 5, 5); - }); - REQUIRE(info.row_did_change(*table, 0)); - } - } - SECTION("table change information") { config.schema = std::make_unique(Schema{ {"table", "", { @@ -947,3 +799,193 @@ TEST_CASE("Transaction log parsing") { } } } + +TEST_CASE("DeepChangeChecker") { + InMemoryTestFile config; + config.automatic_change_notifications = false; + + config.schema = std::make_unique(Schema{ + {"table", "", { + {"int", PropertyTypeInt}, + {"link", PropertyTypeObject, "table", false, false, true}, + {"array", PropertyTypeArray, "table"} + }}, + }); + + auto r = Realm::get_shared_realm(config); + auto table = r->read_group()->get_table("class_table"); + + r->begin_transaction(); + table->add_empty_row(10); + for (int i = 0; i < 10; ++i) + table->set_int(0, i, i); + r->commit_transaction(); + + auto track_changes = [&](auto&& f) { + auto history = make_client_history(config.path); + SharedGroup sg(*history, SharedGroup::durability_MemOnly); + Group const& g = sg.begin_read(); + + r->begin_transaction(); + f(); + r->commit_transaction(); + + _impl::TransactionChangeInfo info; + info.table_modifications_needed.resize(g.size(), true); + info.table_moves_needed.resize(g.size(), true); + _impl::transaction::advance(sg, info); + return info; + }; + + std::vector<_impl::DeepChangeChecker::RelatedTable> tables; + _impl::DeepChangeChecker::find_related_tables(tables, *table); + + SECTION("direct changes are tracked") { + auto info = track_changes([&] { + table->set_int(0, 9, 10); + }); + + _impl::DeepChangeChecker checker(info, *table, tables); + REQUIRE_FALSE(checker(8)); + REQUIRE(checker(9)); + } + + SECTION("changes over links are tracked") { + r->begin_transaction(); + for (int i = 0; i < 9; ++i) + table->set_link(1, i, i + 1); + r->commit_transaction(); + + auto info = track_changes([&] { + table->set_int(0, 9, 10); + }); + + REQUIRE(_impl::DeepChangeChecker(info, *table, tables)(0)); + } + + SECTION("changes over linklists are tracked") { + r->begin_transaction(); + for (int i = 0; i < 9; ++i) + table->get_linklist(2, i)->add(i + 1); + r->commit_transaction(); + + auto info = track_changes([&] { + table->set_int(0, 9, 10); + }); + + REQUIRE(_impl::DeepChangeChecker(info, *table, tables)(0)); + } + + SECTION("cycles over links do not loop forever") { + r->begin_transaction(); + table->set_link(1, 0, 0); + r->commit_transaction(); + + auto info = track_changes([&] { + table->set_int(0, 9, 10); + }); + REQUIRE_FALSE(_impl::DeepChangeChecker(info, *table, tables)(0)); + } + + SECTION("cycles over linklists do not loop forever") { + r->begin_transaction(); + table->get_linklist(2, 0)->add(0); + r->commit_transaction(); + + auto info = track_changes([&] { + table->set_int(0, 9, 10); + }); + REQUIRE_FALSE(_impl::DeepChangeChecker(info, *table, tables)(0)); + } + + SECTION("link chains are tracked up to 16 levels deep") { + r->begin_transaction(); + table->add_empty_row(10); + for (int i = 0; i < 19; ++i) + table->set_link(1, i, i + 1); + r->commit_transaction(); + + auto info = track_changes([&] { + table->set_int(0, 19, -1); + }); + + _impl::DeepChangeChecker checker(info, *table, tables); + CHECK(checker(19)); + CHECK(checker(18)); + CHECK(checker(4)); + CHECK_FALSE(checker(3)); + CHECK_FALSE(checker(2)); + + // Check in other orders to make sure that the caching doesn't effect + // the results + _impl::DeepChangeChecker checker2(info, *table, tables); + CHECK_FALSE(checker2(2)); + CHECK_FALSE(checker2(3)); + CHECK(checker2(4)); + CHECK(checker2(18)); + CHECK(checker2(19)); + + _impl::DeepChangeChecker checker3(info, *table, tables); + CHECK(checker2(4)); + CHECK_FALSE(checker2(3)); + CHECK_FALSE(checker2(2)); + CHECK(checker2(18)); + CHECK(checker2(19)); + } + + SECTION("targets moving is not a change") { + r->begin_transaction(); + table->set_link(1, 0, 9); + table->get_linklist(2, 0)->add(9); + r->commit_transaction(); + + auto info = track_changes([&] { + table->move_last_over(5); + }); + REQUIRE_FALSE(_impl::DeepChangeChecker(info, *table, tables)(0)); + } + + SECTION("changes made before a row is moved are reported") { + r->begin_transaction(); + table->set_link(1, 0, 9); + r->commit_transaction(); + + auto info = track_changes([&] { + table->set_int(0, 9, 5); + table->move_last_over(5); + }); + REQUIRE(_impl::DeepChangeChecker(info, *table, tables)(0)); + + r->begin_transaction(); + table->get_linklist(2, 0)->add(8); + r->commit_transaction(); + + info = track_changes([&] { + table->set_int(0, 8, 5); + table->move_last_over(5); + }); + REQUIRE(_impl::DeepChangeChecker(info, *table, tables)(0)); + } + + SECTION("changes made after a row is moved are reported") { + r->begin_transaction(); + table->set_link(1, 0, 9); + r->commit_transaction(); + + auto info = track_changes([&] { + table->move_last_over(5); + table->set_int(0, 5, 5); + }); + REQUIRE(_impl::DeepChangeChecker(info, *table, tables)(0)); + + r->begin_transaction(); + table->get_linklist(2, 0)->add(8); + r->commit_transaction(); + + info = track_changes([&] { + table->move_last_over(5); + table->set_int(0, 5, 5); + }); + REQUIRE(_impl::DeepChangeChecker(info, *table, tables)(0)); + } +} From 4cf5d5db4c92f13fcfddd18685693e0eed35de74 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 3 May 2016 13:30:38 -0700 Subject: [PATCH 86/93] Report modification paths as pre-delete/inserts as required for UITableView --- src/impl/collection_notifier.cpp | 7 ++++++- tests/results.cpp | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/impl/collection_notifier.cpp b/src/impl/collection_notifier.cpp index f9e2157c..5b18e8b1 100644 --- a/src/impl/collection_notifier.cpp +++ b/src/impl/collection_notifier.cpp @@ -282,7 +282,12 @@ bool CollectionNotifier::deliver(Realm& realm, SharedGroup& sg, std::exception_p bool should_call_callbacks = do_deliver(sg); m_changes_to_deliver = std::move(m_accumulated_changes); - m_changes_to_deliver.modifications.remove(m_changes_to_deliver.insertions); + + // FIXME: ugh + // fixup modifications to be source rows rather than dest rows + m_changes_to_deliver.modifications.erase_at(m_changes_to_deliver.insertions); + m_changes_to_deliver.modifications.shift_for_insert_at(m_changes_to_deliver.deletions); + return should_call_callbacks && have_callbacks(); } diff --git a/tests/results.cpp b/tests/results.cpp index fffff6c8..85b58692 100644 --- a/tests/results.cpp +++ b/tests/results.cpp @@ -258,6 +258,18 @@ TEST_CASE("Results") { REQUIRE(change.modifications.empty()); } + SECTION("modification indices are pre-insert/delete") { + r->begin_transaction(); + table->set_int(0, 2, 0); + table->set_int(0, 3, 6); + r->commit_transaction(); + advance_and_notify(*r); + + REQUIRE(notification_calls == 2); + REQUIRE_INDICES(change.deletions, 1); + REQUIRE_INDICES(change.modifications, 2); + } + SECTION("notifications are not delivered when collapsing transactions results in no net change") { r->begin_transaction(); size_t ndx = table->add_empty_row(); From 03108713ee93044c0fba59356ec4728f03d4c54d Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Thu, 21 Jan 2016 10:45:41 -0800 Subject: [PATCH 87/93] Make PropertyType an enum class --- src/object_accessor.hpp | 44 +++++++++++++------------- src/object_schema.cpp | 6 ++-- src/object_store.cpp | 20 ++++++------ src/parser/query_builder.cpp | 20 ++++++------ src/property.hpp | 52 +++++++++++++++---------------- src/schema.cpp | 4 +-- tests/list.cpp | 8 ++--- tests/results.cpp | 12 +++---- tests/transaction_log_parsing.cpp | 16 +++++----- 9 files changed, 91 insertions(+), 91 deletions(-) diff --git a/src/object_accessor.hpp b/src/object_accessor.hpp index 3fea1741..e8924269 100644 --- a/src/object_accessor.hpp +++ b/src/object_accessor.hpp @@ -172,31 +172,31 @@ namespace realm { } switch (property.type) { - case PropertyTypeBool: + case PropertyType::Bool: m_row.set_bool(column, Accessor::to_bool(ctx, value)); break; - case PropertyTypeInt: + case PropertyType::Int: m_row.set_int(column, Accessor::to_long(ctx, value)); break; - case PropertyTypeFloat: + case PropertyType::Float: m_row.set_float(column, Accessor::to_float(ctx, value)); break; - case PropertyTypeDouble: + case PropertyType::Double: m_row.set_double(column, Accessor::to_double(ctx, value)); break; - case PropertyTypeString: + case PropertyType::String: m_row.set_string(column, Accessor::to_string(ctx, value)); break; - case PropertyTypeData: + case PropertyType::Data: m_row.set_binary(column, BinaryData(Accessor::to_binary(ctx, value))); break; - case PropertyTypeAny: + case PropertyType::Any: m_row.set_mixed(column, Accessor::to_mixed(ctx, value)); break; - case PropertyTypeDate: + case PropertyType::Date: m_row.set_timestamp(column, Accessor::to_timestamp(ctx, value)); break; - case PropertyTypeObject: { + case PropertyType::Object: { if (Accessor::is_null(ctx, value)) { m_row.nullify_link(column); } @@ -205,7 +205,7 @@ namespace realm { } break; } - case PropertyTypeArray: { + case PropertyType::Array: { realm::LinkViewRef link_view = m_row.get_linklist(column); link_view->clear(); if (!Accessor::is_null(ctx, value)) { @@ -231,23 +231,23 @@ namespace realm { } switch (property.type) { - case PropertyTypeBool: + case PropertyType::Bool: return Accessor::from_bool(ctx, m_row.get_bool(column)); - case PropertyTypeInt: + case PropertyType::Int: return Accessor::from_long(ctx, m_row.get_int(column)); - case PropertyTypeFloat: + case PropertyType::Float: return Accessor::from_float(ctx, m_row.get_float(column)); - case PropertyTypeDouble: + case PropertyType::Double: return Accessor::from_double(ctx, m_row.get_double(column)); - case PropertyTypeString: + case PropertyType::String: return Accessor::from_string(ctx, m_row.get_string(column)); - case PropertyTypeData: + case PropertyType::Data: return Accessor::from_binary(ctx, m_row.get_binary(column)); - case PropertyTypeAny: + case PropertyType::Any: throw "Any not supported"; - case PropertyTypeDate: + case PropertyType::Date: return Accessor::from_timestamp(ctx, m_row.get_timestamp(column)); - case PropertyTypeObject: { + case PropertyType::Object: { auto linkObjectSchema = m_realm->config().schema->find(property.object_type); TableRef table = ObjectStore::table_for_object_type(m_realm->read_group(), linkObjectSchema->name); if (m_row.is_null_link(property.table_column)) { @@ -255,7 +255,7 @@ namespace realm { } return Accessor::from_object(ctx, std::move(Object(m_realm, *linkObjectSchema, table->get(m_row.get_link(column))))); } - case PropertyTypeArray: { + case PropertyType::Array: { auto arrayObjectSchema = m_realm->config().schema->find(property.object_type); return Accessor::from_list(ctx, std::move(List(m_realm, *arrayObjectSchema, static_cast(m_row.get_linklist(column))))); } @@ -281,7 +281,7 @@ namespace realm { if (primary_prop) { // search for existing object based on primary key type ValueType primary_value = Accessor::dict_value_for_key(ctx, value, object_schema.primary_key); - if (primary_prop->type == PropertyTypeString) { + if (primary_prop->type == PropertyType::String) { row_index = table->find_first_string(primary_prop->table_column, Accessor::to_string(ctx, primary_value)); } else { @@ -312,7 +312,7 @@ namespace realm { if (Accessor::has_default_value_for_property(ctx, realm.get(), object_schema, prop.name)) { object.set_property_value_impl(ctx, prop, Accessor::default_value_for_property(ctx, realm.get(), object_schema, prop.name), try_update); } - else if (prop.is_nullable || prop.type == PropertyTypeArray) { + else if (prop.is_nullable || prop.type == PropertyType::Array) { object.set_property_value_impl(ctx, prop, Accessor::null_value(ctx), try_update); } else { diff --git a/src/object_schema.cpp b/src/object_schema.cpp index dfd4679c..f794c34a 100644 --- a/src/object_schema.cpp +++ b/src/object_schema.cpp @@ -27,7 +27,7 @@ using namespace realm; #define ASSERT_PROPERTY_TYPE_VALUE(property, type) \ - static_assert(static_cast(PropertyType##property) == type_##type, \ + static_assert(static_cast(PropertyType::property) == type_##type, \ "PropertyType and DataType must have the same values") ASSERT_PROPERTY_TYPE_VALUE(Int, Int); @@ -62,9 +62,9 @@ ObjectSchema::ObjectSchema(const Group *group, const std::string &name) : name(n property.type = (PropertyType)table->get_column_type(col); property.is_indexed = table->has_search_index(col); property.is_primary = false; - property.is_nullable = table->is_nullable(col) || property.type == PropertyTypeObject; + property.is_nullable = table->is_nullable(col) || property.type == PropertyType::Object; property.table_column = col; - if (property.type == PropertyTypeObject || property.type == PropertyTypeArray) { + if (property.type == PropertyType::Object || property.type == PropertyType::Array) { // set link type for objects and arrays ConstTableRef linkTable = table->get_link_target(col); property.object_type = ObjectStore::object_type_for_table_name(linkTable->get_name().data()); diff --git a/src/object_store.cpp b/src/object_store.cpp index 565d5ac0..83742be5 100644 --- a/src/object_store.cpp +++ b/src/object_store.cpp @@ -224,25 +224,25 @@ static void copy_property_values(const Property& old_property, const Property& n static void copy_property_values(const Property& source, const Property& destination, Table& table) { switch (destination.type) { - case PropertyTypeInt: + case PropertyType::Int: copy_property_values(source, destination, table, &Table::get_int, &Table::set_int); break; - case PropertyTypeBool: + case PropertyType::Bool: copy_property_values(source, destination, table, &Table::get_bool, &Table::set_bool); break; - case PropertyTypeFloat: + case PropertyType::Float: copy_property_values(source, destination, table, &Table::get_float, &Table::set_float); break; - case PropertyTypeDouble: + case PropertyType::Double: copy_property_values(source, destination, table, &Table::get_double, &Table::set_double); break; - case PropertyTypeString: + case PropertyType::String: copy_property_values(source, destination, table, &Table::get_string, &Table::set_string); break; - case PropertyTypeData: + case PropertyType::Data: copy_property_values(source, destination, table, &Table::get_binary, &Table::set_binary); break; - case PropertyTypeDate: + case PropertyType::Date: copy_property_values(source, destination, table, &Table::get_timestamp, &Table::set_timestamp); break; default: @@ -319,8 +319,8 @@ void ObjectStore::create_tables(Group *group, Schema &target_schema, bool update if (!current_prop || current_prop->table_column == npos) { switch (target_prop.type) { // for objects and arrays, we have to specify target table - case PropertyTypeObject: - case PropertyTypeArray: { + case PropertyType::Object: + case PropertyType::Array: { TableRef link_table = ObjectStore::table_for_object_type(group, target_prop.object_type); REALM_ASSERT(link_table); target_prop.table_column = table->add_column_link(DataType(target_prop.type), target_prop.name, *link_table); @@ -560,7 +560,7 @@ MissingPropertyException::MissingPropertyException(std::string const& object_typ InvalidNullabilityException::InvalidNullabilityException(std::string const& object_type, Property const& property) : ObjectSchemaPropertyException(object_type, property) { - if (property.type == PropertyTypeObject) { + if (property.type == PropertyType::Object) { m_what = "'Object' property '" + property.name + "' must be nullable."; } else { diff --git a/src/parser/query_builder.cpp b/src/parser/query_builder.cpp index f55a3b71..28334a88 100644 --- a/src/parser/query_builder.cpp +++ b/src/parser/query_builder.cpp @@ -94,7 +94,7 @@ struct PropertyExpression KeyPath key_path = key_path_from_string(key_path_string); for (size_t index = 0; index < key_path.size(); index++) { if (prop) { - precondition(prop->type == PropertyTypeObject || prop->type == PropertyTypeArray, + precondition(prop->type == PropertyType::Object || prop->type == PropertyType::Array, (std::string)"Property '" + key_path[index] + "' is not a link in object of type '" + desc->name + "'"); indexes.push_back(prop->table_column); @@ -400,36 +400,36 @@ void do_add_comparison_to_query(Query &query, const Schema &schema, const Object { auto type = expr.prop->type; switch (type) { - case PropertyTypeBool: + case PropertyType::Bool: add_bool_constraint_to_query(query, cmp.op, value_of_type_for_query(expr.table_getter, lhs, args), value_of_type_for_query(expr.table_getter, rhs, args)); break; - case PropertyTypeDate: + case PropertyType::Date: add_numeric_constraint_to_query(query, cmp.op, value_of_type_for_query(expr.table_getter, lhs, args), value_of_type_for_query(expr.table_getter, rhs, args)); break; - case PropertyTypeDouble: + case PropertyType::Double: add_numeric_constraint_to_query(query, cmp.op, value_of_type_for_query(expr.table_getter, lhs, args), value_of_type_for_query(expr.table_getter, rhs, args)); break; - case PropertyTypeFloat: + case PropertyType::Float: add_numeric_constraint_to_query(query, cmp.op, value_of_type_for_query(expr.table_getter, lhs, args), value_of_type_for_query(expr.table_getter, rhs, args)); break; - case PropertyTypeInt: + case PropertyType::Int: add_numeric_constraint_to_query(query, cmp.op, value_of_type_for_query(expr.table_getter, lhs, args), value_of_type_for_query(expr.table_getter, rhs, args)); break; - case PropertyTypeString: + case PropertyType::String: add_string_constraint_to_query(query, cmp, value_of_type_for_query(expr.table_getter, lhs, args), value_of_type_for_query(expr.table_getter, rhs, args)); break; - case PropertyTypeData: + case PropertyType::Data: add_binary_constraint_to_query(query, cmp.op, value_of_type_for_query(expr.table_getter, lhs, args), value_of_type_for_query(expr.table_getter, rhs, args)); break; - case PropertyTypeObject: - case PropertyTypeArray: + case PropertyType::Object: + case PropertyType::Array: add_link_constraint_to_query(query, cmp.op, expr, link_argument(lhs, rhs, args)); break; default: { diff --git a/src/property.hpp b/src/property.hpp index 7f968bd2..edd22a75 100644 --- a/src/property.hpp +++ b/src/property.hpp @@ -22,17 +22,17 @@ #include namespace realm { - enum PropertyType { - PropertyTypeInt = 0, - PropertyTypeBool = 1, - PropertyTypeFloat = 9, - PropertyTypeDouble = 10, - PropertyTypeString = 2, - PropertyTypeData = 4, - PropertyTypeAny = 6, // deprecated and will be removed in the future - PropertyTypeDate = 8, - PropertyTypeObject = 12, - PropertyTypeArray = 13, + enum class PropertyType { + Int = 0, + Bool = 1, + Float = 9, + Double = 10, + String = 2, + Data = 4, + Any = 6, // deprecated and will be removed in the future + Date = 8, + Object = 12, + Array = 13, }; struct Property { @@ -47,15 +47,15 @@ namespace realm { bool requires_index() const { return is_primary || is_indexed; } bool is_indexable() const { - return type == PropertyTypeInt - || type == PropertyTypeBool - || type == PropertyTypeDate - || type == PropertyTypeString; + return type == PropertyType::Int + || type == PropertyType::Bool + || type == PropertyType::Date + || type == PropertyType::String; } #if __GNUC__ < 5 // GCC 4.9 does not support C++14 braced-init with NSDMIs - Property(std::string name="", PropertyType type=PropertyTypeInt, std::string object_type="", + Property(std::string name="", PropertyType type=PropertyType::Int, std::string object_type="", bool is_primary=false, bool is_indexed=false, bool is_nullable=false) : name(std::move(name)) , type(type) @@ -70,25 +70,25 @@ namespace realm { static inline const char *string_for_property_type(PropertyType type) { switch (type) { - case PropertyTypeString: + case PropertyType::String: return "string"; - case PropertyTypeInt: + case PropertyType::Int: return "int"; - case PropertyTypeBool: + case PropertyType::Bool: return "bool"; - case PropertyTypeDate: + case PropertyType::Date: return "date"; - case PropertyTypeData: + case PropertyType::Data: return "data"; - case PropertyTypeDouble: + case PropertyType::Double: return "double"; - case PropertyTypeFloat: + case PropertyType::Float: return "float"; - case PropertyTypeAny: + case PropertyType::Any: return "any"; - case PropertyTypeObject: + case PropertyType::Object: return "object"; - case PropertyTypeArray: + case PropertyType::Array: return "array"; #if __GNUC__ default: diff --git a/src/schema.cpp b/src/schema.cpp index 50cfc670..2d036041 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -74,11 +74,11 @@ void Schema::validate() const // check nullablity if (prop.is_nullable) { - if (prop.type == PropertyTypeArray || prop.type == PropertyTypeAny) { + if (prop.type == PropertyType::Array || prop.type == PropertyType::Any) { exceptions.emplace_back(InvalidNullabilityException(object.name, prop)); } } - else if (prop.type == PropertyTypeObject) { + else if (prop.type == PropertyType::Object) { exceptions.emplace_back(InvalidNullabilityException(object.name, prop)); } diff --git a/tests/list.cpp b/tests/list.cpp index c355a6ed..4495762a 100644 --- a/tests/list.cpp +++ b/tests/list.cpp @@ -24,16 +24,16 @@ TEST_CASE("list") { config.cache = false; config.schema = std::make_unique(Schema{ {"origin", "", { - {"array", PropertyTypeArray, "target"} + {"array", PropertyType::Array, "target"} }}, {"target", "", { - {"value", PropertyTypeInt} + {"value", PropertyType::Int} }}, {"other_origin", "", { - {"array", PropertyTypeArray, "other_target"} + {"array", PropertyType::Array, "other_target"} }}, {"other_target", "", { - {"value", PropertyTypeInt} + {"value", PropertyType::Int} }}, }); diff --git a/tests/results.cpp b/tests/results.cpp index 85b58692..9113d76e 100644 --- a/tests/results.cpp +++ b/tests/results.cpp @@ -23,17 +23,17 @@ TEST_CASE("Results") { config.automatic_change_notifications = false; config.schema = std::make_unique(Schema{ {"object", "", { - {"value", PropertyTypeInt}, - {"link", PropertyTypeObject, "linked to object", false, false, true} + {"value", PropertyType::Int}, + {"link", PropertyType::Object, "linked to object", false, false, true} }}, {"other object", "", { - {"value", PropertyTypeInt} + {"value", PropertyType::Int} }}, {"linking object", "", { - {"link", PropertyTypeObject, "object", false, false, true} + {"link", PropertyType::Object, "object", false, false, true} }}, {"linked to object", "", { - {"value", PropertyTypeInt} + {"value", PropertyType::Int} }} }); @@ -426,7 +426,7 @@ TEST_CASE("Async Results error handling") { config.automatic_change_notifications = false; config.schema = std::make_unique(Schema{ {"object", "", { - {"value", PropertyTypeInt}, + {"value", PropertyType::Int}, }}, }); diff --git a/tests/transaction_log_parsing.cpp b/tests/transaction_log_parsing.cpp index b4654d24..fd35503b 100644 --- a/tests/transaction_log_parsing.cpp +++ b/tests/transaction_log_parsing.cpp @@ -113,8 +113,8 @@ TEST_CASE("Transaction log parsing") { SECTION("schema change validation") { config.schema = std::make_unique(Schema{ {"table", "", { - {"unindexed", PropertyTypeInt}, - {"indexed", PropertyTypeInt, "", false, true} + {"unindexed", PropertyType::Int}, + {"indexed", PropertyType::Int, "", false, true} }}, }); auto r = Realm::get_shared_realm(config); @@ -180,7 +180,7 @@ TEST_CASE("Transaction log parsing") { SECTION("table change information") { config.schema = std::make_unique(Schema{ {"table", "", { - {"value", PropertyTypeInt} + {"value", PropertyType::Int} }}, }); @@ -269,10 +269,10 @@ TEST_CASE("Transaction log parsing") { SECTION("LinkView change information") { config.schema = std::make_unique(Schema{ {"origin", "", { - {"array", PropertyTypeArray, "target"} + {"array", PropertyType::Array, "target"} }}, {"target", "", { - {"value", PropertyTypeInt} + {"value", PropertyType::Int} }}, }); @@ -806,9 +806,9 @@ TEST_CASE("DeepChangeChecker") { config.schema = std::make_unique(Schema{ {"table", "", { - {"int", PropertyTypeInt}, - {"link", PropertyTypeObject, "table", false, false, true}, - {"array", PropertyTypeArray, "table"} + {"int", PropertyType::Int}, + {"link", PropertyType::Object, "table", false, false, true}, + {"array", PropertyType::Array, "table"} }}, }); From ef1c6ddc63d71a21c87434c056c6097ba8644abb Mon Sep 17 00:00:00 2001 From: Mark Rowe Date: Thu, 14 Apr 2016 18:09:29 -0700 Subject: [PATCH 88/93] Add support for fine-grained notifications from backlink TVs --- src/impl/collection_change_builder.cpp | 6 +- src/impl/results_notifier.cpp | 13 +- src/impl/results_notifier.hpp | 2 +- src/results.cpp | 52 +++++- src/results.hpp | 4 + tests/collection_change_indices.cpp | 212 ++++++++++++------------- 6 files changed, 163 insertions(+), 126 deletions(-) diff --git a/src/impl/collection_change_builder.cpp b/src/impl/collection_change_builder.cpp index 2fa652f9..c81cc502 100644 --- a/src/impl/collection_change_builder.cpp +++ b/src/impl/collection_change_builder.cpp @@ -550,9 +550,9 @@ void calculate_moves_sorted(std::vector& rows, CollectionChangeSet& cha CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector const& prev_rows, std::vector const& next_rows, std::function row_did_change, - bool sort) + bool rows_are_in_table_order) { - REALM_ASSERT_DEBUG(sort || std::is_sorted(begin(next_rows), end(next_rows))); + REALM_ASSERT_DEBUG(!rows_are_in_table_order || std::is_sorted(begin(next_rows), end(next_rows))); CollectionChangeBuilder ret; @@ -627,7 +627,7 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c } } - if (sort) { + if (!rows_are_in_table_order) { calculate_moves_sorted(new_rows, ret); } else { diff --git a/src/impl/results_notifier.cpp b/src/impl/results_notifier.cpp index c72fdd0a..6e91f3d0 100644 --- a/src/impl/results_notifier.cpp +++ b/src/impl/results_notifier.cpp @@ -27,7 +27,7 @@ ResultsNotifier::ResultsNotifier(Results& target) : CollectionNotifier(target.get_realm()) , m_target_results(&target) , m_sort(target.get_sort()) -, m_from_linkview(target.get_linkview().get() != nullptr) +, m_target_is_in_table_order(target.is_in_table_order()) { Query q = target.get_query(); set_table(*q.get_table()); @@ -87,12 +87,8 @@ bool ResultsNotifier::need_to_run() } // If we've run previously, check if we need to rerun - if (m_initial_run_complete) { - // Make an empty tableview from the query to get the table version, since - // Query doesn't expose it - if (m_query->find_all(0, 0, 0).sync_if_needed() == m_last_seen_version) { - return false; - } + if (m_initial_run_complete && m_query->sync_view_if_needed() == m_last_seen_version) { + return false; } return true; @@ -125,7 +121,7 @@ void ResultsNotifier::calculate_changes() m_changes = CollectionChangeBuilder::calculate(m_previous_rows, next_rows, get_modification_checker(*m_info, *m_query->get_table()), - m_sort || m_from_linkview); + m_target_is_in_table_order && !m_sort); m_previous_rows = std::move(next_rows); } @@ -141,6 +137,7 @@ void ResultsNotifier::run() if (!need_to_run()) return; + m_query->sync_view_if_needed(); m_tv = m_query->find_all(); if (m_sort) { m_tv.sort(m_sort.column_indices, m_sort.ascending); diff --git a/src/impl/results_notifier.hpp b/src/impl/results_notifier.hpp index de0b2d65..8028fbd0 100644 --- a/src/impl/results_notifier.hpp +++ b/src/impl/results_notifier.hpp @@ -36,7 +36,7 @@ private: Results* m_target_results; const SortOrder m_sort; - bool m_from_linkview; + bool m_target_is_in_table_order; // The source Query, in handover form iff m_sg is null std::unique_ptr> m_query_handover; diff --git a/src/results.cpp b/src/results.cpp index 1095731a..5e9064f9 100644 --- a/src/results.cpp +++ b/src/results.cpp @@ -72,6 +72,17 @@ Results::Results(SharedRealm r, const ObjectSchema& o, LinkViewRef lv, util::Opt } } +Results::Results(SharedRealm r, const ObjectSchema& o, TableView tv, SortOrder s) +: m_realm(std::move(r)) +, m_object_schema(&o) +, m_table_view(std::move(tv)) +, m_table(&m_table_view.get_parent()) +, m_sort(std::move(s)) +, m_mode(Mode::TableView) +{ + REALM_ASSERT(m_sort.column_indices.size() == m_sort.ascending.size()); +} + Results::~Results() { if (m_notifier) { @@ -85,7 +96,7 @@ void Results::validate_read() const m_realm->verify_thread(); if (m_table && !m_table->is_attached()) throw InvalidatedException(); - if (m_mode == Mode::TableView && !m_table_view.is_attached()) + if (m_mode == Mode::TableView && (!m_table_view.is_attached() || m_table_view.depends_on_deleted_object())) throw InvalidatedException(); if (m_mode == Mode::LinkView && !m_link_view->is_attached()) throw InvalidatedException(); @@ -115,8 +126,10 @@ size_t Results::size() switch (m_mode) { case Mode::Empty: return 0; case Mode::Table: return m_table->size(); - case Mode::Query: return m_query.count(); case Mode::LinkView: return m_link_view->size(); + case Mode::Query: + m_query.sync_view_if_needed(); + return m_query.count(); case Mode::TableView: update_tableview(); return m_table_view.size(); @@ -218,6 +231,7 @@ void Results::update_tableview() case Mode::LinkView: return; case Mode::Query: + m_query.sync_view_if_needed(); m_table_view = m_query.find_all(); if (m_sort) { m_table_view.sort(m_sort.column_indices, m_sort.ascending); @@ -384,8 +398,19 @@ Query Results::get_query() const case Mode::Empty: case Mode::Query: return m_query; - case Mode::TableView: - return m_table_view.get_query(); + case Mode::TableView: { + // A TableView has an associated Query if it was produced by Query::find_all. This is indicated + // by TableView::get_query returning a Query with a non-null table. + Query query = m_table_view.get_query(); + if (query.get_table()) { + return query; + } + + // The TableView has no associated query so create one with no conditions that is restricted + // to the rows in the TableView. + m_table_view.sync_if_needed(); + return Query(*m_table, std::make_unique(m_table_view)); + } case Mode::LinkView: return m_table->where(m_link_view); case Mode::Table: @@ -417,15 +442,11 @@ TableView Results::get_tableview() Results Results::sort(realm::SortOrder&& sort) const { REALM_ASSERT(sort.column_indices.size() == sort.ascending.size()); - if (m_link_view) - return Results(m_realm, *m_object_schema, m_link_view, m_query, std::move(sort)); return Results(m_realm, *m_object_schema, get_query(), std::move(sort)); } Results Results::filter(Query&& q) const { - if (m_link_view) - return Results(m_realm, *m_object_schema, m_link_view, get_query().and_query(std::move(q)), m_sort); return Results(m_realm, *m_object_schema, get_query().and_query(std::move(q)), m_sort); } @@ -457,6 +478,21 @@ NotificationToken Results::add_notification_callback(CollectionChangeCallback cb return {m_notifier, m_notifier->add_callback(std::move(cb))}; } +bool Results::is_in_table_order() const +{ + switch (m_mode) { + case Mode::Empty: + case Mode::Table: + return true; + case Mode::LinkView: + return false; + case Mode::Query: + return m_query.produces_results_in_table_order() && !m_sort; + case Mode::TableView: + return m_table_view.is_in_table_order(); + } +} + void Results::Internal::set_table_view(Results& results, realm::TableView &&tv) { // If the previous TableView was never actually used, then stop generating diff --git a/src/results.hpp b/src/results.hpp index 388b85fe..6b25982a 100644 --- a/src/results.hpp +++ b/src/results.hpp @@ -51,6 +51,7 @@ public: Results() = default; Results(SharedRealm r, const ObjectSchema& o, Table& table); Results(SharedRealm r, const ObjectSchema& o, Query q, SortOrder s = {}); + Results(SharedRealm r, const ObjectSchema& o, TableView tv, SortOrder s); Results(SharedRealm r, const ObjectSchema& o, LinkViewRef lv, util::Optional q = {}, SortOrder s = {}); ~Results(); @@ -182,6 +183,9 @@ public: bool wants_background_updates() const { return m_wants_background_updates; } + // Returns whether the rows are guaranteed to be in table order. + bool is_in_table_order() const; + // Helper type to let ResultsNotifier update the tableview without giving access // to any other privates or letting anyone else do so class Internal { diff --git a/tests/collection_change_indices.cpp b/tests/collection_change_indices.cpp index 42625bfb..d0f9ec92 100644 --- a/tests/collection_change_indices.cpp +++ b/tests/collection_change_indices.cpp @@ -338,97 +338,6 @@ TEST_CASE("[collection_change] calculate() unsorted") { auto none_modified = [](size_t) { return false; }; const auto npos = size_t(-1); - SECTION("returns an empty set when input and output are identical") { - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, none_modified, false); - REQUIRE(c.empty()); - } - - SECTION("marks all as inserted when prev is empty") { - c = _impl::CollectionChangeBuilder::calculate({}, {1, 2, 3}, all_modified, false); - REQUIRE_INDICES(c.insertions, 0, 1, 2); - } - - SECTION("marks all as deleted when new is empty") { - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {}, all_modified, false); - REQUIRE_INDICES(c.deletions, 0, 1, 2); - } - - SECTION("marks npos rows in prev as deleted") { - c = _impl::CollectionChangeBuilder::calculate({npos, 1, 2, 3, npos}, {1, 2, 3}, all_modified, false); - REQUIRE_INDICES(c.deletions, 0, 4); - } - - SECTION("marks modified rows which do not move as modified") { - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, all_modified, false); - REQUIRE_INDICES(c.modifications, 0, 1, 2); - } - - SECTION("does not mark unmodified rows as modified") { - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, none_modified, false); - REQUIRE(c.modifications.empty()); - } - - SECTION("marks newly added rows as insertions") { - c = _impl::CollectionChangeBuilder::calculate({2, 3}, {1, 2, 3}, all_modified, false); - REQUIRE_INDICES(c.insertions, 0); - - c = _impl::CollectionChangeBuilder::calculate({1, 3}, {1, 2, 3}, all_modified, false); - REQUIRE_INDICES(c.insertions, 1); - - c = _impl::CollectionChangeBuilder::calculate({1, 2}, {1, 2, 3}, all_modified, false); - REQUIRE_INDICES(c.insertions, 2); - } - - SECTION("marks removed rows as deleted") { - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2}, all_modified, false); - REQUIRE_INDICES(c.deletions, 2); - - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3}, all_modified, false); - REQUIRE_INDICES(c.deletions, 1); - - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {2, 3}, all_modified, false); - REQUIRE_INDICES(c.deletions, 0); - } - - SECTION("marks rows as both inserted and deleted") { - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3, 4}, all_modified, false); - REQUIRE_INDICES(c.deletions, 1); - REQUIRE_INDICES(c.insertions, 2); - REQUIRE(c.moves.empty()); - } - - SECTION("marks rows as modified even if they moved") { - c = _impl::CollectionChangeBuilder::calculate({5, 3}, {3, 5}, all_modified, false); - REQUIRE_MOVES(c, {1, 0}); - REQUIRE_INDICES(c.modifications, 0, 1); - } - - SECTION("does not mark rows as modified if they are new") { - c = _impl::CollectionChangeBuilder::calculate({3}, {3, 5}, all_modified, false); - REQUIRE_INDICES(c.modifications, 0); - } - - SECTION("reports moves which can be produced by move_last_over()") { - auto calc = [&](std::vector values) { - return _impl::CollectionChangeBuilder::calculate(values, {1, 2, 3}, none_modified, false); - }; - - REQUIRE(calc({1, 2, 3}).empty()); - REQUIRE_MOVES(calc({1, 3, 2}), {2, 1}); - REQUIRE_MOVES(calc({2, 1, 3}), {1, 0}); - REQUIRE_MOVES(calc({2, 3, 1}), {2, 0}); - REQUIRE_MOVES(calc({3, 1, 2}), {1, 0}, {2, 1}); - REQUIRE_MOVES(calc({3, 2, 1}), {2, 0}, {1, 1}); - } -} - -TEST_CASE("[collection_change] calculate() sorted") { - _impl::CollectionChangeBuilder c; - - auto all_modified = [](size_t) { return true; }; - auto none_modified = [](size_t) { return false; }; - const auto npos = size_t(-1); - SECTION("returns an empty set when input and output are identical") { c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, none_modified, true); REQUIRE(c.empty()); @@ -489,9 +398,8 @@ TEST_CASE("[collection_change] calculate() sorted") { } SECTION("marks rows as modified even if they moved") { - c = _impl::CollectionChangeBuilder::calculate({3, 5}, {5, 3}, all_modified, true); - REQUIRE_INDICES(c.deletions, 1); - REQUIRE_INDICES(c.insertions, 0); + c = _impl::CollectionChangeBuilder::calculate({5, 3}, {3, 5}, all_modified, true); + REQUIRE_MOVES(c, {1, 0}); REQUIRE_INDICES(c.modifications, 0, 1); } @@ -500,9 +408,101 @@ TEST_CASE("[collection_change] calculate() sorted") { REQUIRE_INDICES(c.modifications, 0); } + SECTION("reports moves which can be produced by move_last_over()") { + auto calc = [&](std::vector values) { + return _impl::CollectionChangeBuilder::calculate(values, {1, 2, 3}, none_modified, true); + }; + + REQUIRE(calc({1, 2, 3}).empty()); + REQUIRE_MOVES(calc({1, 3, 2}), {2, 1}); + REQUIRE_MOVES(calc({2, 1, 3}), {1, 0}); + REQUIRE_MOVES(calc({2, 3, 1}), {2, 0}); + REQUIRE_MOVES(calc({3, 1, 2}), {1, 0}, {2, 1}); + REQUIRE_MOVES(calc({3, 2, 1}), {2, 0}, {1, 1}); + } +} + +TEST_CASE("[collection_change] calculate() sorted") { + _impl::CollectionChangeBuilder c; + + auto all_modified = [](size_t) { return true; }; + auto none_modified = [](size_t) { return false; }; + const auto npos = size_t(-1); + + SECTION("returns an empty set when input and output are identical") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, none_modified, false); + REQUIRE(c.empty()); + } + + SECTION("marks all as inserted when prev is empty") { + c = _impl::CollectionChangeBuilder::calculate({}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.insertions, 0, 1, 2); + } + + SECTION("marks all as deleted when new is empty") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {}, all_modified, false); + REQUIRE_INDICES(c.deletions, 0, 1, 2); + } + + SECTION("marks npos rows in prev as deleted") { + c = _impl::CollectionChangeBuilder::calculate({npos, 1, 2, 3, npos}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.deletions, 0, 4); + } + + SECTION("marks modified rows which do not move as modified") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.modifications, 0, 1, 2); + } + + SECTION("does not mark unmodified rows as modified") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2, 3}, none_modified, false); + REQUIRE(c.modifications.empty()); + } + + SECTION("marks newly added rows as insertions") { + c = _impl::CollectionChangeBuilder::calculate({2, 3}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.insertions, 0); + + c = _impl::CollectionChangeBuilder::calculate({1, 3}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.insertions, 1); + + c = _impl::CollectionChangeBuilder::calculate({1, 2}, {1, 2, 3}, all_modified, false); + REQUIRE_INDICES(c.insertions, 2); + } + + SECTION("marks removed rows as deleted") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 2}, all_modified, false); + REQUIRE_INDICES(c.deletions, 2); + + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3}, all_modified, false); + REQUIRE_INDICES(c.deletions, 1); + + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {2, 3}, all_modified, false); + REQUIRE_INDICES(c.deletions, 0); + } + + SECTION("marks rows as both inserted and deleted") { + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3, 4}, all_modified, false); + REQUIRE_INDICES(c.deletions, 1); + REQUIRE_INDICES(c.insertions, 2); + REQUIRE(c.moves.empty()); + } + + SECTION("marks rows as modified even if they moved") { + c = _impl::CollectionChangeBuilder::calculate({3, 5}, {5, 3}, all_modified, false); + REQUIRE_INDICES(c.deletions, 1); + REQUIRE_INDICES(c.insertions, 0); + REQUIRE_INDICES(c.modifications, 0, 1); + } + + SECTION("does not mark rows as modified if they are new") { + c = _impl::CollectionChangeBuilder::calculate({3}, {3, 5}, all_modified, false); + REQUIRE_INDICES(c.modifications, 0); + } + SECTION("reports inserts/deletes for simple reorderings") { auto calc = [&](std::vector old_rows, std::vector new_rows) { - return _impl::CollectionChangeBuilder::calculate(old_rows, new_rows, none_modified, true); + return _impl::CollectionChangeBuilder::calculate(old_rows, new_rows, none_modified, false); }; c = calc({1, 2, 3}, {1, 2, 3}); @@ -652,19 +652,19 @@ TEST_CASE("[collection_change] calculate() sorted") { SECTION("prefers to produce diffs where modified rows are the ones to move when it is ambiguous") { auto two_modified = [](size_t ndx) { return ndx == 2; }; - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3, 2}, two_modified, true); + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3, 2}, two_modified, false); REQUIRE_INDICES(c.deletions, 1); REQUIRE_INDICES(c.insertions, 2); auto three_modified = [](size_t ndx) { return ndx == 3; }; - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3, 2}, three_modified, true); + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {1, 3, 2}, three_modified, false); REQUIRE_INDICES(c.deletions, 2); REQUIRE_INDICES(c.insertions, 1); } SECTION("prefers smaller diffs over larger diffs moving only modified rows") { auto two_modified = [](size_t ndx) { return ndx == 2; }; - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {2, 3, 1}, two_modified, true); + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, {2, 3, 1}, two_modified, false); REQUIRE_INDICES(c.deletions, 0); REQUIRE_INDICES(c.insertions, 2); } @@ -672,7 +672,7 @@ TEST_CASE("[collection_change] calculate() sorted") { SECTION("supports duplicate indices") { c = _impl::CollectionChangeBuilder::calculate({1, 1, 2, 2, 3, 3}, {1, 2, 3, 1, 2, 3}, - all_modified, true); + all_modified, false); REQUIRE_INDICES(c.deletions, 3, 5); REQUIRE_INDICES(c.insertions, 1, 2); } @@ -680,7 +680,7 @@ TEST_CASE("[collection_change] calculate() sorted") { SECTION("deletes and inserts the last option when any in a range could be deleted") { c = _impl::CollectionChangeBuilder::calculate({3, 2, 1, 1, 2, 3}, {1, 1, 2, 2, 3, 3}, - all_modified, true); + all_modified, false); REQUIRE_INDICES(c.deletions, 0, 1); REQUIRE_INDICES(c.insertions, 3, 5); } @@ -688,13 +688,13 @@ TEST_CASE("[collection_change] calculate() sorted") { SECTION("reports insertions/deletions when the number of duplicate entries changes") { c = _impl::CollectionChangeBuilder::calculate({1, 1, 1, 1, 2, 3}, {1, 2, 3, 1}, - all_modified, true); + all_modified, false); REQUIRE_INDICES(c.deletions, 1, 2, 3); REQUIRE_INDICES(c.insertions, 3); c = _impl::CollectionChangeBuilder::calculate({1, 2, 3, 1}, {1, 1, 1, 1, 2, 3}, - all_modified, true); + all_modified, false); REQUIRE_INDICES(c.deletions, 3); REQUIRE_INDICES(c.insertions, 1, 2, 3); } @@ -702,7 +702,7 @@ TEST_CASE("[collection_change] calculate() sorted") { SECTION("properly recurses into smaller subblocks") { std::vector prev = {10, 1, 2, 11, 3, 4, 5, 12, 6, 7, 13}; std::vector next = {13, 1, 2, 12, 3, 4, 5, 11, 6, 7, 10}; - c = _impl::CollectionChangeBuilder::calculate(prev, next, all_modified, true); + c = _impl::CollectionChangeBuilder::calculate(prev, next, all_modified, false); REQUIRE_INDICES(c.deletions, 0, 3, 7, 10); REQUIRE_INDICES(c.insertions, 0, 3, 7, 10); } @@ -718,13 +718,13 @@ TEST_CASE("[collection_change] calculate() sorted") { std::vector after_insert = {1, 2, 3}; after_insert.insert(after_insert.begin() + insert_pos, 4); - c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, after_insert, four_modified, true); + c = _impl::CollectionChangeBuilder::calculate({1, 2, 3}, after_insert, four_modified, false); std::vector after_move = {1, 2, 3}; after_move.insert(after_move.begin() + move_to_pos, 4); - c.merge(_impl::CollectionChangeBuilder::calculate(after_insert, after_move, four_modified, true)); + c.merge(_impl::CollectionChangeBuilder::calculate(after_insert, after_move, four_modified, false)); - c.merge(_impl::CollectionChangeBuilder::calculate(after_move, {1, 2, 3}, four_modified, true)); + c.merge(_impl::CollectionChangeBuilder::calculate(after_move, {1, 2, 3}, four_modified, false)); REQUIRE(c.empty()); } } From 8d9e5db09248133c1f3402dd2eea5050f7445dcc Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 3 May 2016 17:04:49 -0700 Subject: [PATCH 89/93] Give a variable a less terrible name and add a comment --- src/impl/collection_notifier.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/impl/collection_notifier.cpp b/src/impl/collection_notifier.cpp index 5b18e8b1..d9ec92c6 100644 --- a/src/impl/collection_notifier.cpp +++ b/src/impl/collection_notifier.cpp @@ -50,12 +50,17 @@ void DeepChangeChecker::find_related_tables(std::vector& out, Tabl if (any_of(begin(out), end(out), [=](auto& tbl) { return tbl.table_ndx == table_ndx; })) return; - size_t info = out.size(); + // We need to add this table to `out` before recurring so that the check + // above works, but we can't store a pointer to the thing being populated + // because the recursive calls may resize `out`, so instead look it up by + // index every time + size_t out_index = out.size(); out.push_back({table_ndx, {}}); + for (size_t i = 0, count = table.get_column_count(); i != count; ++i) { auto type = table.get_column_type(i); if (type == type_Link || type == type_LinkList) { - out[info].links.push_back({i, type == type_LinkList}); + out[out_index].links.push_back({i, type == type_LinkList}); find_related_tables(out, *table.get_link_target(i)); } } From b42bf25c9945cb8119e9bcd5dde67189a7d917ce Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 10 May 2016 11:33:00 -0700 Subject: [PATCH 90/93] Fix compilation with latest core --- src/index_set.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/index_set.cpp b/src/index_set.cpp index 2c566e13..a5c30c2b 100644 --- a/src/index_set.cpp +++ b/src/index_set.cpp @@ -414,7 +414,9 @@ void IndexSet::add_shifted_by(IndexSet const& shifted_by, IndexSet const& values copy(old_it, old_end, std::back_inserter(builder)); m_data = builder.finalize(); - REALM_ASSERT_DEBUG((size_t)std::distance(as_indexes().begin(), as_indexes().end()) == expected); +#ifdef REALM_DEBUG + REALM_ASSERT((size_t)std::distance(as_indexes().begin(), as_indexes().end()) == expected); +#endif } void IndexSet::set(size_t len) From 2bcf42904a9764fc57bf977e90d2732d47323b62 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 10 May 2016 11:41:47 -0700 Subject: [PATCH 91/93] Fix some typos in comments --- src/collection_notifications.cpp | 2 +- src/impl/collection_change_builder.cpp | 4 ++-- src/impl/collection_notifier.cpp | 3 ++- src/impl/realm_coordinator.cpp | 2 +- src/impl/transact_log_handler.cpp | 2 +- src/index_set.hpp | 2 +- src/shared_realm.hpp | 2 +- 7 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/collection_notifications.cpp b/src/collection_notifications.cpp index 9a11941d..83dc89e1 100644 --- a/src/collection_notifications.cpp +++ b/src/collection_notifications.cpp @@ -34,7 +34,7 @@ NotificationToken::~NotificationToken() // atomically to ensure that there are no data races when the token is // destroyed after being modified on a different thread. // This is needed despite the token not being thread-safe in general as - // users find it very surpringing for obj-c objects to care about what + // users find it very surprising for obj-c objects to care about what // thread they are deallocated on. if (auto notifier = m_notifier.exchange({})) { notifier->remove_callback(m_token); diff --git a/src/impl/collection_change_builder.cpp b/src/impl/collection_change_builder.cpp index c81cc502..2aa75b20 100644 --- a/src/impl/collection_change_builder.cpp +++ b/src/impl/collection_change_builder.cpp @@ -581,8 +581,8 @@ CollectionChangeBuilder CollectionChangeBuilder::calculate(std::vector c }); // Don't add rows which were modified to not match the query to `deletions` - // immediately because the unsorted move logic needs to be able to distinuish - // them from rows which were outright deleted + // immediately because the unsorted move logic needs to be able to + // distinguish them from rows which were outright deleted IndexSet removed; // Now that our old and new sets of rows are sorted by row index, we can diff --git a/src/impl/collection_notifier.cpp b/src/impl/collection_notifier.cpp index d9ec92c6..7751c3ac 100644 --- a/src/impl/collection_notifier.cpp +++ b/src/impl/collection_notifier.cpp @@ -288,8 +288,9 @@ bool CollectionNotifier::deliver(Realm& realm, SharedGroup& sg, std::exception_p bool should_call_callbacks = do_deliver(sg); m_changes_to_deliver = std::move(m_accumulated_changes); - // FIXME: ugh // fixup modifications to be source rows rather than dest rows + // FIXME: the actual change calculations should be updated to just calculate + // the correct thing instead m_changes_to_deliver.modifications.erase_at(m_changes_to_deliver.insertions); m_changes_to_deliver.modifications.shift_for_insert_at(m_changes_to_deliver.deletions); diff --git a/src/impl/realm_coordinator.cpp b/src/impl/realm_coordinator.cpp index 09693088..51922f24 100644 --- a/src/impl/realm_coordinator.cpp +++ b/src/impl/realm_coordinator.cpp @@ -385,7 +385,7 @@ public: } } - // Copy the list change info if there's multiple LinkViews for the same LinkList + // Copy the list change info if there are multiple LinkViews for the same LinkList auto id = [](auto const& list) { return std::tie(list.table_ndx, list.col_ndx, list.row_ndx); }; for (size_t i = 1; i < m_current->lists.size(); ++i) { for (size_t j = i; j > 0; --j) { diff --git a/src/impl/transact_log_handler.cpp b/src/impl/transact_log_handler.cpp index 86db5544..e0880bee 100644 --- a/src/impl/transact_log_handler.cpp +++ b/src/impl/transact_log_handler.cpp @@ -335,7 +335,7 @@ public: } else { // Array KVO can only send a single kind of change at a time, so - // if there's multiple just give up and send "Set" + // if there are multiple just give up and send "Set" o->indices.set(0); o->kind = ColumnInfo::Kind::SetAll; } diff --git a/src/index_set.hpp b/src/index_set.hpp index f0f7599e..0cf00fd9 100644 --- a/src/index_set.hpp +++ b/src/index_set.hpp @@ -189,7 +189,7 @@ public: // Remove all indexes from the set void clear(); - // An iterator over the indivual indices in the set rather than the ranges + // An iterator over the individual indices in the set rather than the ranges class IndexIterator : public std::iterator { public: IndexIterator(IndexSet::const_iterator it) : m_iterator(it) { } diff --git a/src/shared_realm.hpp b/src/shared_realm.hpp index 25e5acfd..90ff1665 100644 --- a/src/shared_realm.hpp +++ b/src/shared_realm.hpp @@ -63,7 +63,7 @@ namespace realm { bool in_memory = false; // The following are intended for internal/testing purposes and - // should not be publically exposed in binding APIs + // should not be publicly exposed in binding APIs // If false, always return a new Realm instance, and don't return // that Realm instance for other requests for a cached Realm. Useful From e6d09b513eb1d61652ec0e0eef0eea7b33b065e8 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 10 May 2016 13:44:51 -0700 Subject: [PATCH 92/93] Fix an assertion failure in IndexSet::do_add() following a table clear We don't track insertions and deletions for tables that are merely linked to by tables actually being observed (for performance reasons, since we don't need that information), but the check for that was missing in one place. This would be merely a slowdown rather than a crash, but deletions.add_shifted() can overflow size_t if the passed-in index represents a newly inserted row and the check for that didn't work due to not tracking insertions for the table. The only remotely realistic way to actually have size_t overflow is to have previously cleared the table (the table clear instruction does not include the old size of the table, so it just marks {0, SIZE_T_MAX} as deleted). Fixes #3537. --- src/impl/collection_change_builder.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/impl/collection_change_builder.cpp b/src/impl/collection_change_builder.cpp index 2aa75b20..da0ff5ab 100644 --- a/src/impl/collection_change_builder.cpp +++ b/src/impl/collection_change_builder.cpp @@ -251,11 +251,13 @@ void CollectionChangeBuilder::move_over(size_t row_ndx, size_t last_row, bool tr REALM_ASSERT(modifications.empty() || prev(modifications.end())->second - 1 <= last_row); if (row_ndx == last_row) { - auto shifted_from = insertions.erase_or_unshift(row_ndx); - if (shifted_from != IndexSet::npos) - deletions.add_shifted(shifted_from); + if (track_moves) { + auto shifted_from = insertions.erase_or_unshift(row_ndx); + if (shifted_from != IndexSet::npos) + deletions.add_shifted(shifted_from); + m_move_mapping.erase(row_ndx); + } modifications.remove(row_ndx); - m_move_mapping.erase(row_ndx); return; } From db55770bfae8a8a5dbeb15ddf8ffa1f8ba5b1b4a Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Tue, 10 May 2016 14:38:26 -0700 Subject: [PATCH 93/93] Move the notifications fuzzer to the tests directory --- CMakeLists.txt | 1 - tests/CMakeLists.txt | 2 ++ {fuzzer => tests/notifications-fuzzer}/CMakeLists.txt | 0 {fuzzer => tests/notifications-fuzzer}/command_file.cpp | 0 {fuzzer => tests/notifications-fuzzer}/command_file.hpp | 0 {fuzzer => tests/notifications-fuzzer}/fuzz-sorted-linkview.cpp | 0 {fuzzer => tests/notifications-fuzzer}/fuzz-sorted-query.cpp | 0 .../notifications-fuzzer}/fuzz-unsorted-linkview.cpp | 0 {fuzzer => tests/notifications-fuzzer}/fuzz-unsorted-query.cpp | 0 {fuzzer => tests/notifications-fuzzer}/fuzzer.cpp | 0 {fuzzer => tests/notifications-fuzzer}/input-lv/0 | 0 {fuzzer => tests/notifications-fuzzer}/input/0 | 0 {fuzzer => tests/notifications-fuzzer}/input/1 | 0 13 files changed, 2 insertions(+), 1 deletion(-) rename {fuzzer => tests/notifications-fuzzer}/CMakeLists.txt (100%) rename {fuzzer => tests/notifications-fuzzer}/command_file.cpp (100%) rename {fuzzer => tests/notifications-fuzzer}/command_file.hpp (100%) rename {fuzzer => tests/notifications-fuzzer}/fuzz-sorted-linkview.cpp (100%) rename {fuzzer => tests/notifications-fuzzer}/fuzz-sorted-query.cpp (100%) rename {fuzzer => tests/notifications-fuzzer}/fuzz-unsorted-linkview.cpp (100%) rename {fuzzer => tests/notifications-fuzzer}/fuzz-unsorted-query.cpp (100%) rename {fuzzer => tests/notifications-fuzzer}/fuzzer.cpp (100%) rename {fuzzer => tests/notifications-fuzzer}/input-lv/0 (100%) rename {fuzzer => tests/notifications-fuzzer}/input/0 (100%) rename {fuzzer => tests/notifications-fuzzer}/input/1 (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index ec33ee52..286e6ff0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,5 +16,4 @@ use_realm_core(${REALM_CORE_VERSION}) include_directories(${REALM_CORE_INCLUDE_DIR} src external/pegtl) add_subdirectory(src) -add_subdirectory(fuzzer) add_subdirectory(tests) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c28b839b..6e055b4d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -22,3 +22,5 @@ target_link_libraries(tests realm-object-store) create_coverage_target(generate-coverage tests) add_custom_target(run-tests USES_TERMINAL DEPENDS tests COMMAND ./tests) + +add_subdirectory(notifications-fuzzer) diff --git a/fuzzer/CMakeLists.txt b/tests/notifications-fuzzer/CMakeLists.txt similarity index 100% rename from fuzzer/CMakeLists.txt rename to tests/notifications-fuzzer/CMakeLists.txt diff --git a/fuzzer/command_file.cpp b/tests/notifications-fuzzer/command_file.cpp similarity index 100% rename from fuzzer/command_file.cpp rename to tests/notifications-fuzzer/command_file.cpp diff --git a/fuzzer/command_file.hpp b/tests/notifications-fuzzer/command_file.hpp similarity index 100% rename from fuzzer/command_file.hpp rename to tests/notifications-fuzzer/command_file.hpp diff --git a/fuzzer/fuzz-sorted-linkview.cpp b/tests/notifications-fuzzer/fuzz-sorted-linkview.cpp similarity index 100% rename from fuzzer/fuzz-sorted-linkview.cpp rename to tests/notifications-fuzzer/fuzz-sorted-linkview.cpp diff --git a/fuzzer/fuzz-sorted-query.cpp b/tests/notifications-fuzzer/fuzz-sorted-query.cpp similarity index 100% rename from fuzzer/fuzz-sorted-query.cpp rename to tests/notifications-fuzzer/fuzz-sorted-query.cpp diff --git a/fuzzer/fuzz-unsorted-linkview.cpp b/tests/notifications-fuzzer/fuzz-unsorted-linkview.cpp similarity index 100% rename from fuzzer/fuzz-unsorted-linkview.cpp rename to tests/notifications-fuzzer/fuzz-unsorted-linkview.cpp diff --git a/fuzzer/fuzz-unsorted-query.cpp b/tests/notifications-fuzzer/fuzz-unsorted-query.cpp similarity index 100% rename from fuzzer/fuzz-unsorted-query.cpp rename to tests/notifications-fuzzer/fuzz-unsorted-query.cpp diff --git a/fuzzer/fuzzer.cpp b/tests/notifications-fuzzer/fuzzer.cpp similarity index 100% rename from fuzzer/fuzzer.cpp rename to tests/notifications-fuzzer/fuzzer.cpp diff --git a/fuzzer/input-lv/0 b/tests/notifications-fuzzer/input-lv/0 similarity index 100% rename from fuzzer/input-lv/0 rename to tests/notifications-fuzzer/input-lv/0 diff --git a/fuzzer/input/0 b/tests/notifications-fuzzer/input/0 similarity index 100% rename from fuzzer/input/0 rename to tests/notifications-fuzzer/input/0 diff --git a/fuzzer/input/1 b/tests/notifications-fuzzer/input/1 similarity index 100% rename from fuzzer/input/1 rename to tests/notifications-fuzzer/input/1