From 1e0d26211fafe749f302165dfd7339914c5c0e68 Mon Sep 17 00:00:00 2001 From: bhartnett <51288821+bhartnett@users.noreply.github.com> Date: Tue, 1 Oct 2024 11:22:08 +0800 Subject: [PATCH] Update RocksDb to v9.6.1 (#70) * Update RocksDb to v9.6.1 --- rocksdb/lib/rocksdb.h | 90 ++++++++++++++++++++ rocksdb/lib/rocksdb_gen.nim | 165 ++++++++++++++++++++++++++++++++++++ vendor/rocksdb | 2 +- vendor/vcpkg | 2 +- 4 files changed, 257 insertions(+), 2 deletions(-) diff --git a/rocksdb/lib/rocksdb.h b/rocksdb/lib/rocksdb.h index 887d186..b1ff902 100644 --- a/rocksdb/lib/rocksdb.h +++ b/rocksdb/lib/rocksdb.h @@ -456,6 +456,9 @@ rocksdb_create_column_family_with_ttl( extern ROCKSDB_LIBRARY_API void rocksdb_drop_column_family( rocksdb_t* db, rocksdb_column_family_handle_t* handle, char** errptr); +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* + rocksdb_get_default_column_family_handle(rocksdb_t* db); + extern ROCKSDB_LIBRARY_API void rocksdb_column_family_handle_destroy( rocksdb_column_family_handle_t*); @@ -524,6 +527,13 @@ extern ROCKSDB_LIBRARY_API char* rocksdb_get_cf_with_ts( rocksdb_column_family_handle_t* column_family, const char* key, size_t keylen, size_t* vallen, char** ts, size_t* tslen, char** errptr); +/** + * Returns a malloc() buffer with the DB identity, assigning the length to + * *id_len. Returns NULL if an error occurred. + */ +extern ROCKSDB_LIBRARY_API char* rocksdb_get_db_identity(rocksdb_t* db, + size_t* id_len); + // if values_list[i] == NULL and errs[i] == NULL, // then we got status.IsNotFound(), which we will not return. // all errors except status status.ok() and status.IsNotFound() are returned. @@ -662,6 +672,19 @@ extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes_cf( const size_t* range_start_key_len, const char* const* range_limit_key, const size_t* range_limit_key_len, uint64_t* sizes, char** errptr); +enum { + rocksdb_size_approximation_flags_none = 0, + rocksdb_size_approximation_flags_include_memtable = 1 << 0, + rocksdb_size_approximation_flags_include_files = 1 << 1, +}; + +extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes_cf_with_flags( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + int num_ranges, const char* const* range_start_key, + const size_t* range_start_key_len, const char* const* range_limit_key, + const size_t* range_limit_key_len, uint8_t include_flags, uint64_t* sizes, + char** errptr); + extern ROCKSDB_LIBRARY_API void rocksdb_compact_range(rocksdb_t* db, const char* start_key, size_t start_key_len, @@ -749,6 +772,8 @@ extern ROCKSDB_LIBRARY_API const char* rocksdb_iter_timestamp( const rocksdb_iterator_t*, size_t* tslen); extern ROCKSDB_LIBRARY_API void rocksdb_iter_get_error( const rocksdb_iterator_t*, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_refresh( + const rocksdb_iterator_t* iter, char** errptr); extern ROCKSDB_LIBRARY_API void rocksdb_wal_iter_next( rocksdb_wal_iterator_t* iter); @@ -769,6 +794,10 @@ extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_writebatch_create( void); extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_writebatch_create_from( const char* rep, size_t size); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* +rocksdb_writebatch_create_with_params(size_t reserved_bytes, size_t max_bytes, + size_t protection_bytes_per_key, + size_t default_cf_ts_sz); extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_destroy( rocksdb_writebatch_t*); extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_clear(rocksdb_writebatch_t*); @@ -856,6 +885,13 @@ extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_iterate( rocksdb_writebatch_t*, void* state, void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), void (*deleted)(void*, const char* k, size_t klen)); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_iterate_cf( + rocksdb_writebatch_t*, void* state, + void (*put_cf)(void*, uint32_t cfid, const char* k, size_t klen, + const char* v, size_t vlen), + void (*deleted_cf)(void*, uint32_t cfid, const char* k, size_t klen), + void (*merge_cf)(void*, uint32_t cfid, const char* k, size_t klen, + const char* v, size_t vlen)); extern ROCKSDB_LIBRARY_API const char* rocksdb_writebatch_data( rocksdb_writebatch_t*, size_t* size); extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_set_save_point( @@ -864,6 +900,9 @@ extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_rollback_to_save_point( rocksdb_writebatch_t*, char** errptr); extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_pop_save_point( rocksdb_writebatch_t*, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_update_timestamps( + rocksdb_writebatch_t* wb, const char* ts, size_t tslen, void* state, + size_t (*get_ts_size)(void*, uint32_t), char** errptr); /* Write batch with index */ @@ -872,6 +911,11 @@ rocksdb_writebatch_wi_create(size_t reserved_bytes, unsigned char overwrite_keys); extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* rocksdb_writebatch_wi_create_from(const char* rep, size_t size); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* +rocksdb_writebatch_wi_create_with_params( + rocksdb_comparator_t* backup_index_comparator, size_t reserved_bytes, + unsigned char overwrite_key, size_t max_bytes, + size_t protection_bytes_per_key); extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_destroy( rocksdb_writebatch_wi_t*); extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_clear( @@ -982,6 +1026,9 @@ extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* rocksdb_writebatch_wi_create_iterator_with_base_cf( rocksdb_writebatch_wi_t* wbwi, rocksdb_iterator_t* base_iterator, rocksdb_column_family_handle_t* cf); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_update_timestamps( + rocksdb_writebatch_wi_t* wbwi, const char* ts, size_t tslen, void* state, + size_t (*get_ts_size)(void*, uint32_t), char** errptr); /* Options utils */ @@ -1086,6 +1133,21 @@ rocksdb_block_based_options_set_pin_top_level_index_and_filter( rocksdb_block_based_table_options_t*, unsigned char); extern ROCKSDB_LIBRARY_API void rocksdb_options_set_block_based_table_factory( rocksdb_options_t* opt, rocksdb_block_based_table_options_t* table_options); +enum { + rocksdb_block_based_k_fallback_pinning_tier = 0, + rocksdb_block_based_k_none_pinning_tier = 1, + rocksdb_block_based_k_flush_and_similar_pinning_tier = 2, + rocksdb_block_based_k_all_pinning_tier = 3, +}; +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_top_level_index_pinning_tier( + rocksdb_block_based_table_options_t*, int); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_partition_pinning_tier( + rocksdb_block_based_table_options_t*, int); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_unpartitioned_pinning_tier( + rocksdb_block_based_table_options_t*, int); extern ROCKSDB_LIBRARY_API void rocksdb_options_set_write_buffer_manager( rocksdb_options_t* opt, rocksdb_write_buffer_manager_t* wbm); @@ -1185,6 +1247,11 @@ extern ROCKSDB_LIBRARY_API int rocksdb_options_get_info_log_level( rocksdb_options_t*); extern ROCKSDB_LIBRARY_API rocksdb_logger_t* rocksdb_logger_create_stderr_logger(int log_level, const char* prefix); +extern ROCKSDB_LIBRARY_API rocksdb_logger_t* +rocksdb_logger_create_callback_logger(int log_level, + void (*)(void* priv, unsigned lev, + char* msg, size_t len), + void* priv); extern ROCKSDB_LIBRARY_API void rocksdb_logger_destroy( rocksdb_logger_t* logger); extern ROCKSDB_LIBRARY_API void rocksdb_options_set_write_buffer_size( @@ -1595,6 +1662,17 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_plain_table_factory( rocksdb_options_t*, uint32_t, int, double, size_t, size_t, char, unsigned char, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_write_dbid_to_manifest(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_write_dbid_to_manifest( + rocksdb_options_t*, unsigned char); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_track_and_verify_wals_in_manifest(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_track_and_verify_wals_in_manifest(rocksdb_options_t*, + unsigned char); + extern ROCKSDB_LIBRARY_API void rocksdb_options_set_min_level_to_compress( rocksdb_options_t* opt, int level); @@ -1704,6 +1782,18 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_compression( extern ROCKSDB_LIBRARY_API int rocksdb_options_get_wal_compression( rocksdb_options_t* opt); +enum { + rocksdb_k_by_compensated_size_compaction_pri = 0, + rocksdb_k_oldest_largest_seq_first_compaction_pri = 1, + rocksdb_k_oldest_smallest_seq_first_compaction_pri = 2, + rocksdb_k_min_overlapping_ratio_compaction_pri = 3, + rocksdb_k_round_robin_compaction_pri = 4 +}; +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compaction_pri( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_compaction_pri( + rocksdb_options_t*); + /* RateLimiter */ extern ROCKSDB_LIBRARY_API rocksdb_ratelimiter_t* rocksdb_ratelimiter_create( int64_t rate_bytes_per_sec, int64_t refill_period_us, int32_t fairness); diff --git a/rocksdb/lib/rocksdb_gen.nim b/rocksdb/lib/rocksdb_gen.nim index 65a77df..6a77a53 100644 --- a/rocksdb/lib/rocksdb_gen.nim +++ b/rocksdb/lib/rocksdb_gen.nim @@ -579,6 +579,12 @@ proc rocksdb_drop_column_family*( db: ptr rocksdb_t, handle: ptr rocksdb_column_family_handle_t, errptr: cstringArray ) {.cdecl, importc: "rocksdb_drop_column_family", dynlib: librocksdb.} +proc rocksdb_get_default_column_family_handle*( + db: ptr rocksdb_t +): ptr rocksdb_column_family_handle_t {. + cdecl, importc: "rocksdb_get_default_column_family_handle", dynlib: librocksdb +.} + proc rocksdb_column_family_handle_destroy*( a1: ptr rocksdb_column_family_handle_t ) {.cdecl, importc: "rocksdb_column_family_handle_destroy", dynlib: librocksdb.} @@ -719,6 +725,15 @@ proc rocksdb_get_cf_with_ts*( errptr: cstringArray, ): cstring {.cdecl, importc: "rocksdb_get_cf_with_ts", dynlib: librocksdb.} +## +## Returns a malloc() buffer with the DB identity, assigning the length to +## *id_len. Returns NULL if an error occurred. +## + +proc rocksdb_get_db_identity*( + db: ptr rocksdb_t, id_len: ptr csize_t +): cstring {.cdecl, importc: "rocksdb_get_db_identity", dynlib: librocksdb.} + ## if values_list[i] == NULL and errs[i] == NULL, ## then we got status.IsNotFound(), which we will not return. ## all errors except status status.ok() and status.IsNotFound() are returned. @@ -948,6 +963,24 @@ proc rocksdb_approximate_sizes_cf*( errptr: cstringArray, ) {.cdecl, importc: "rocksdb_approximate_sizes_cf", dynlib: librocksdb.} +const + rocksdb_size_approximation_flags_none* = 0 + rocksdb_size_approximation_flags_include_memtable* = 1 shl 0 + rocksdb_size_approximation_flags_include_files* = 1 shl 1 + +proc rocksdb_approximate_sizes_cf_with_flags*( + db: ptr rocksdb_t, + column_family: ptr rocksdb_column_family_handle_t, + num_ranges: cint, + range_start_key: cstringArray, + range_start_key_len: ptr csize_t, + range_limit_key: cstringArray, + range_limit_key_len: ptr csize_t, + include_flags: uint8, + sizes: ptr uint64, + errptr: cstringArray, +) {.cdecl, importc: "rocksdb_approximate_sizes_cf_with_flags", dynlib: librocksdb.} + proc rocksdb_compact_range*( db: ptr rocksdb_t, start_key: cstring, @@ -1102,6 +1135,10 @@ proc rocksdb_iter_get_error*( a1: ptr rocksdb_iterator_t, errptr: cstringArray ) {.cdecl, importc: "rocksdb_iter_get_error", dynlib: librocksdb.} +proc rocksdb_iter_refresh*( + iter: ptr rocksdb_iterator_t, errptr: cstringArray +) {.cdecl, importc: "rocksdb_iter_refresh", dynlib: librocksdb.} + proc rocksdb_wal_iter_next*( iter: ptr rocksdb_wal_iterator_t ) {.cdecl, importc: "rocksdb_wal_iter_next", dynlib: librocksdb.} @@ -1140,6 +1177,15 @@ proc rocksdb_writebatch_create_from*( cdecl, importc: "rocksdb_writebatch_create_from", dynlib: librocksdb .} +proc rocksdb_writebatch_create_with_params*( + reserved_bytes: csize_t, + max_bytes: csize_t, + protection_bytes_per_key: csize_t, + default_cf_ts_sz: csize_t, +): ptr rocksdb_writebatch_t {. + cdecl, importc: "rocksdb_writebatch_create_with_params", dynlib: librocksdb +.} + proc rocksdb_writebatch_destroy*( a1: ptr rocksdb_writebatch_t ) {.cdecl, importc: "rocksdb_writebatch_destroy", dynlib: librocksdb.} @@ -1333,6 +1379,18 @@ proc rocksdb_writebatch_iterate*( deleted: proc(a1: pointer, k: cstring, klen: csize_t) {.cdecl.}, ) {.cdecl, importc: "rocksdb_writebatch_iterate", dynlib: librocksdb.} +proc rocksdb_writebatch_iterate_cf*( + a1: ptr rocksdb_writebatch_t, + state: pointer, + put_cf: proc( + a1: pointer, cfid: uint32, k: cstring, klen: csize_t, v: cstring, vlen: csize_t + ) {.cdecl.}, + deleted_cf: proc(a1: pointer, cfid: uint32, k: cstring, klen: csize_t) {.cdecl.}, + merge_cf: proc( + a1: pointer, cfid: uint32, k: cstring, klen: csize_t, v: cstring, vlen: csize_t + ) {.cdecl.}, +) {.cdecl, importc: "rocksdb_writebatch_iterate_cf", dynlib: librocksdb.} + proc rocksdb_writebatch_data*( a1: ptr rocksdb_writebatch_t, size: ptr csize_t ): cstring {.cdecl, importc: "rocksdb_writebatch_data", dynlib: librocksdb.} @@ -1349,6 +1407,15 @@ proc rocksdb_writebatch_pop_save_point*( a1: ptr rocksdb_writebatch_t, errptr: cstringArray ) {.cdecl, importc: "rocksdb_writebatch_pop_save_point", dynlib: librocksdb.} +proc rocksdb_writebatch_update_timestamps*( + wb: ptr rocksdb_writebatch_t, + ts: cstring, + tslen: csize_t, + state: pointer, + get_ts_size: proc(a1: pointer, a2: uint32): csize_t {.cdecl.}, + errptr: cstringArray, +) {.cdecl, importc: "rocksdb_writebatch_update_timestamps", dynlib: librocksdb.} + ## Write batch with index proc rocksdb_writebatch_wi_create*( @@ -1363,6 +1430,16 @@ proc rocksdb_writebatch_wi_create_from*( cdecl, importc: "rocksdb_writebatch_wi_create_from", dynlib: librocksdb .} +proc rocksdb_writebatch_wi_create_with_params*( + backup_index_comparator: ptr rocksdb_comparator_t, + reserved_bytes: csize_t, + overwrite_key: uint8, + max_bytes: csize_t, + protection_bytes_per_key: csize_t, +): ptr rocksdb_writebatch_wi_t {. + cdecl, importc: "rocksdb_writebatch_wi_create_with_params", dynlib: librocksdb +.} + proc rocksdb_writebatch_wi_destroy*( a1: ptr rocksdb_writebatch_wi_t ) {.cdecl, importc: "rocksdb_writebatch_wi_destroy", dynlib: librocksdb.} @@ -1626,6 +1703,15 @@ proc rocksdb_writebatch_wi_create_iterator_with_base_cf*( dynlib: librocksdb .} +proc rocksdb_writebatch_wi_update_timestamps*( + wbwi: ptr rocksdb_writebatch_wi_t, + ts: cstring, + tslen: csize_t, + state: pointer, + get_ts_size: proc(a1: pointer, a2: uint32): csize_t {.cdecl.}, + errptr: cstringArray, +) {.cdecl, importc: "rocksdb_writebatch_wi_update_timestamps", dynlib: librocksdb.} + ## Options utils ## Load the latest rocksdb options from the specified db_path. ## @@ -1838,6 +1924,36 @@ proc rocksdb_options_set_block_based_table_factory*( cdecl, importc: "rocksdb_options_set_block_based_table_factory", dynlib: librocksdb .} +const + rocksdb_block_based_k_fallback_pinning_tier* = 0 + rocksdb_block_based_k_none_pinning_tier* = 1 + rocksdb_block_based_k_flush_and_similar_pinning_tier* = 2 + rocksdb_block_based_k_all_pinning_tier* = 3 + +proc rocksdb_block_based_options_set_top_level_index_pinning_tier*( + a1: ptr rocksdb_block_based_table_options_t, a2: cint +) {. + cdecl, + importc: "rocksdb_block_based_options_set_top_level_index_pinning_tier", + dynlib: librocksdb +.} + +proc rocksdb_block_based_options_set_partition_pinning_tier*( + a1: ptr rocksdb_block_based_table_options_t, a2: cint +) {. + cdecl, + importc: "rocksdb_block_based_options_set_partition_pinning_tier", + dynlib: librocksdb +.} + +proc rocksdb_block_based_options_set_unpartitioned_pinning_tier*( + a1: ptr rocksdb_block_based_table_options_t, a2: cint +) {. + cdecl, + importc: "rocksdb_block_based_options_set_unpartitioned_pinning_tier", + dynlib: librocksdb +.} + proc rocksdb_options_set_write_buffer_manager*( opt: ptr rocksdb_options_t, wbm: ptr rocksdb_write_buffer_manager_t ) {.cdecl, importc: "rocksdb_options_set_write_buffer_manager", dynlib: librocksdb.} @@ -2057,6 +2173,14 @@ proc rocksdb_logger_create_stderr_logger*( cdecl, importc: "rocksdb_logger_create_stderr_logger", dynlib: librocksdb .} +proc rocksdb_logger_create_callback_logger*( + log_level: cint, + a2: proc(priv: pointer, lev: cuint, msg: cstring, len: csize_t) {.cdecl.}, + priv: pointer, +): ptr rocksdb_logger_t {. + cdecl, importc: "rocksdb_logger_create_callback_logger", dynlib: librocksdb +.} + proc rocksdb_logger_destroy*( logger: ptr rocksdb_logger_t ) {.cdecl, importc: "rocksdb_logger_destroy", dynlib: librocksdb.} @@ -3074,6 +3198,32 @@ proc rocksdb_options_set_plain_table_factory*( a9: uint8, ) {.cdecl, importc: "rocksdb_options_set_plain_table_factory", dynlib: librocksdb.} +proc rocksdb_options_get_write_dbid_to_manifest*( + a1: ptr rocksdb_options_t +): uint8 {. + cdecl, importc: "rocksdb_options_get_write_dbid_to_manifest", dynlib: librocksdb +.} + +proc rocksdb_options_set_write_dbid_to_manifest*( + a1: ptr rocksdb_options_t, a2: uint8 +) {.cdecl, importc: "rocksdb_options_set_write_dbid_to_manifest", dynlib: librocksdb.} + +proc rocksdb_options_get_track_and_verify_wals_in_manifest*( + a1: ptr rocksdb_options_t +): uint8 {. + cdecl, + importc: "rocksdb_options_get_track_and_verify_wals_in_manifest", + dynlib: librocksdb +.} + +proc rocksdb_options_set_track_and_verify_wals_in_manifest*( + a1: ptr rocksdb_options_t, a2: uint8 +) {. + cdecl, + importc: "rocksdb_options_set_track_and_verify_wals_in_manifest", + dynlib: librocksdb +.} + proc rocksdb_options_set_min_level_to_compress*( opt: ptr rocksdb_options_t, level: cint ) {.cdecl, importc: "rocksdb_options_set_min_level_to_compress", dynlib: librocksdb.} @@ -3284,6 +3434,21 @@ proc rocksdb_options_get_wal_compression*( opt: ptr rocksdb_options_t ): cint {.cdecl, importc: "rocksdb_options_get_wal_compression", dynlib: librocksdb.} +const + rocksdb_k_by_compensated_size_compaction_pri* = 0 + rocksdb_k_oldest_largest_seq_first_compaction_pri* = 1 + rocksdb_k_oldest_smallest_seq_first_compaction_pri* = 2 + rocksdb_k_min_overlapping_ratio_compaction_pri* = 3 + rocksdb_k_round_robin_compaction_pri* = 4 + +proc rocksdb_options_set_compaction_pri*( + a1: ptr rocksdb_options_t, a2: cint +) {.cdecl, importc: "rocksdb_options_set_compaction_pri", dynlib: librocksdb.} + +proc rocksdb_options_get_compaction_pri*( + a1: ptr rocksdb_options_t +): cint {.cdecl, importc: "rocksdb_options_get_compaction_pri", dynlib: librocksdb.} + ## RateLimiter proc rocksdb_ratelimiter_create*( diff --git a/vendor/rocksdb b/vendor/rocksdb index 08f9322..13d5230 160000 --- a/vendor/rocksdb +++ b/vendor/rocksdb @@ -1 +1 @@ -Subproject commit 08f93221f50700f19f11555fb46abfe708a716d1 +Subproject commit 13d5230e5da650cf93e6dccb389c82d316d355c6 diff --git a/vendor/vcpkg b/vendor/vcpkg index 1de2026..c82f746 160000 --- a/vendor/vcpkg +++ b/vendor/vcpkg @@ -1 +1 @@ -Subproject commit 1de2026f28ead93ff1773e6e680387643e914ea1 +Subproject commit c82f74667287d3dc386bce81e44964370c91a289