mirror of
https://github.com/facebook/rocksdb.git
synced 2026-11-06 02:52:35 -05:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b026bfdf4a | ||
|
|
ce7b6bced6 | ||
|
|
2f3fac5a1d | ||
|
|
b2ef1cb1ae | ||
|
|
05ab235fe7 | ||
|
|
36a76d543e | ||
|
|
21b393c7f3 | ||
|
|
8d7926a766 | ||
|
|
9ed80413e0 | ||
|
|
15ec7d4f95 | ||
|
|
0612f472b2 | ||
|
|
187cb23caa | ||
|
|
848cf937d9 | ||
|
|
c5cfed2e4a |
No files matched your search
@@ -207,7 +207,6 @@ set(SOURCES
|
||||
util/filter_policy.cc
|
||||
util/hash.cc
|
||||
util/histogram.cc
|
||||
util/histogram_windowing.cc
|
||||
util/instrumented_mutex.cc
|
||||
util/iostats_context.cc
|
||||
tools/ldb_cmd.cc
|
||||
@@ -239,7 +238,6 @@ set(SOURCES
|
||||
util/thread_status_updater.cc
|
||||
util/thread_status_util.cc
|
||||
util/thread_status_util_debug.cc
|
||||
util/transaction_test_util.cc
|
||||
util/xfunc.cc
|
||||
util/xxhash.cc
|
||||
utilities/backupable/backupable_db.cc
|
||||
@@ -360,7 +358,6 @@ set(TESTS
|
||||
db/perf_context_test.cc
|
||||
db/plain_table_db_test.cc
|
||||
db/prefix_test.cc
|
||||
db/repair_test.cc
|
||||
db/skiplist_test.cc
|
||||
db/table_properties_collector_test.cc
|
||||
db/version_builder_test.cc
|
||||
@@ -399,7 +396,6 @@ set(TESTS
|
||||
util/iostats_context_test.cc
|
||||
util/memenv_test.cc
|
||||
util/mock_env_test.cc
|
||||
util/options_settable_test.cc
|
||||
util/options_test.cc
|
||||
util/rate_limiter_test.cc
|
||||
util/slice_transform_test.cc
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
# RocksDB default options change log
|
||||
## 4.7.0 (4/8/2016)
|
||||
* options.write_buffer_size changes from 4MB to 64MB
|
||||
* options.target_file_size_base changes from 2MB to 64MB
|
||||
* options.max_bytes_for_level_base changes from 10MB to 256MB
|
||||
* options.soft_pending_compaction_bytes_limit changes from 0 (disabled) to 64GB
|
||||
* options.hard_pending_compaction_bytes_limit changes from 0 (disabled) to 256GB
|
||||
* table_cache_numshardbits changes from 4 to 6
|
||||
* max_file_opening_threads changes from 1 to 16
|
||||
@@ -1,9 +1,4 @@
|
||||
# Rocksdb Change Log
|
||||
## 4.7.0 (4/8/2016)
|
||||
### Public API Change
|
||||
* rename options compaction_measure_io_stats to report_bg_io_stats and include flush too.
|
||||
* Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options.
|
||||
|
||||
## 4.6.0 (3/10/2016)
|
||||
### Public API Changes
|
||||
* Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier.
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
This is the list of all known third-party language bindings for RocksDB. If something is missing, please open a pull request to add it.
|
||||
|
||||
* Java - https://github.com/facebook/rocksdb/tree/master/java
|
||||
* Python - http://pyrocksdb.readthedocs.org/en/latest/
|
||||
* Perl - https://metacpan.org/pod/RocksDB
|
||||
* Node.js - https://npmjs.org/package/rocksdb
|
||||
* Go - https://github.com/tecbot/gorocksdb
|
||||
* Ruby - http://rubygems.org/gems/rocksdb-ruby
|
||||
* Haskell - https://hackage.haskell.org/package/rocksdb-haskell
|
||||
* PHP - https://github.com/Photonios/rocksdb-php
|
||||
* C# - https://github.com/warrenfalk/rocksdb-sharp
|
||||
@@ -174,15 +174,6 @@ else
|
||||
pg = -pg
|
||||
endif
|
||||
|
||||
# USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc.
|
||||
ifdef COMPILE_WITH_UBSAN
|
||||
DISABLE_JEMALLOC=1
|
||||
EXEC_LDFLAGS += -fsanitize=undefined
|
||||
PLATFORM_CCFLAGS += -fsanitize=undefined
|
||||
PLATFORM_CXXFLAGS += -fsanitize=undefined
|
||||
endif
|
||||
|
||||
|
||||
ifndef DISABLE_JEMALLOC
|
||||
EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS)
|
||||
PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
|
||||
@@ -326,7 +317,6 @@ TESTS = \
|
||||
rate_limiter_test \
|
||||
delete_scheduler_test \
|
||||
options_test \
|
||||
options_settable_test \
|
||||
options_util_test \
|
||||
event_logger_test \
|
||||
cuckoo_table_builder_test \
|
||||
@@ -606,7 +596,7 @@ ldb_tests: ldb
|
||||
crash_test: whitebox_crash_test blackbox_crash_test
|
||||
|
||||
blackbox_crash_test: db_stress
|
||||
python -u tools/db_crashtest.py --simple blackbox
|
||||
python -u tools/db_crashtest.py --simple blackbox
|
||||
python -u tools/db_crashtest.py blackbox
|
||||
|
||||
whitebox_crash_test: db_stress
|
||||
@@ -623,18 +613,8 @@ asan_crash_test:
|
||||
COMPILE_WITH_ASAN=1 $(MAKE) crash_test
|
||||
$(MAKE) clean
|
||||
|
||||
ubsan_check:
|
||||
$(MAKE) clean
|
||||
COMPILE_WITH_UBSAN=1 $(MAKE) check -j32
|
||||
$(MAKE) clean
|
||||
|
||||
ubsan_crash_test:
|
||||
$(MAKE) clean
|
||||
COMPILE_WITH_UBSAN=1 $(MAKE) crash_test
|
||||
$(MAKE) clean
|
||||
|
||||
valgrind_check: $(TESTS)
|
||||
for t in $(filter-out %skiplist_test options_settable_test,$(TESTS)); do \
|
||||
for t in $(filter-out %skiplist_test,$(TESTS)); do \
|
||||
$(VALGRIND_VER) $(VALGRIND_OPTS) ./$$t; \
|
||||
ret_code=$$?; \
|
||||
if [ $$ret_code -ne 0 ]; then \
|
||||
@@ -1021,9 +1001,6 @@ compact_files_test: db/compact_files_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
options_test: util/options_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
options_settable_test: util/options_settable_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
options_util_test: utilities/options/options_util_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
@@ -1066,9 +1043,6 @@ transaction_test: utilities/transactions/transaction_test.o $(LIBOBJECTS) $(TEST
|
||||
sst_dump: tools/sst_dump.o $(LIBOBJECTS)
|
||||
$(AM_LINK)
|
||||
|
||||
repair_test: db/repair_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
ldb_cmd_test: tools/ldb_cmd_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
@@ -1240,7 +1214,7 @@ jdb_bench:
|
||||
|
||||
commit_prereq: build_tools/rocksdb-lego-determinator \
|
||||
build_tools/precommit_checker.py
|
||||
J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit tsan asan ubsan lite
|
||||
J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit tsan asan lite
|
||||
$(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava;
|
||||
|
||||
xfunc:
|
||||
|
||||
@@ -52,7 +52,3 @@ Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtub
|
||||
|
||||
## VWO, Wingify
|
||||
[VWO's](https://vwo.com/) Smart Code checker and URL helper uses RocksDB to store all the URLs where VWO's Smart Code is installed.
|
||||
|
||||
## quasardb
|
||||
[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark.
|
||||
quasardb uses a heavily tuned RocksDB as its persistence layer.
|
||||
@@ -1,6 +1,6 @@
|
||||
GCC_BASE=/mnt/gvfs/third-party2/gcc/ebc96bc2fb751b5a0300b8d91a95bdf24ac1d88b/4.8.1/centos6-native/cc6c9dc
|
||||
CLANG_BASE=/mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/9d9ecb9/
|
||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/f97108c12512b3b0789ac4515d836bdb1eae1142/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
|
||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/ea2fd1278810d3af2ea52218d2767e09d786dbd0/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
|
||||
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f5484f168c0e4d19823d41df052c5870c6e575a4/2.17/gcc-4.8.1-glibc-2.17/99df8fc
|
||||
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/cbf6f1f209e5bd160bdc5d971744e039f36b1566/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/6d39cb54708049f527e713ad19f2aadb9d3667e8/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
@@ -8,9 +8,9 @@ BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/2ddd45f0853bfc8bb1c27f0f447236a1a26c338a
|
||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/cb6c4880fcb4fee471574ba6af63a3882155a16a/0.5.1/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/c7275a4ceae0aca0929e56964a31dafc53c1ee96/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/d9acac8a9a2c9378eb696e22ffa8bd0e97d9206b/master/gcc-4.8.1-glibc-2.17/8d31e51
|
||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/40791a3fef9206a77f2c4bc51f8169e5bf10d68e/master/gcc-4.8.1-glibc-2.17/8d31e51
|
||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/41bfa3759ce52c071f5fd547ec9ecd2522929f0a/trunk/gcc-4.8.1-glibc-2.17/675d945
|
||||
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/303048f72efc92ae079e62dfc84823401aecfd94/trunk/gcc-4.8.1-glibc-2.17/675d945
|
||||
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/1a48835975c66d30e47770ec419758ed3b9ba010/3.10.62-62_fbk17_03959_ge29cc63/gcc-4.8.1-glibc-2.17/da39a3e
|
||||
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/71454c53fffcb716a0beb9a90047aff7fb5c984a/2.26/centos6-native/da39a3e
|
||||
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/a5b8152b2a15ce8a98808cf954fbccec825a97bc/2.25/centos6-native/da39a3e
|
||||
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/af85c56f424cd5edfc2c97588299b44ecdec96bb/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
@@ -184,7 +184,7 @@ parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.')
|
||||
|
||||
# <test ....>
|
||||
parser.add_argument('test', nargs='+',
|
||||
help='CI test(s) to run. e.g: unit punit asan tsan ubsan')
|
||||
help='CI test(s) to run. e.g: unit punit asan tsan')
|
||||
|
||||
print("Please follow log %s" % Log.LOG_FILE)
|
||||
|
||||
|
||||
@@ -67,7 +67,6 @@ ASAN="COMPILE_WITH_ASAN=1"
|
||||
CLANG="USE_CLANG=1"
|
||||
LITE="OPT=-DROCKSDB_LITE"
|
||||
TSAN="COMPILE_WITH_TSAN=1"
|
||||
UBSAN="COMPILE_WITH_UBSAN=1"
|
||||
DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
|
||||
PARSER="'parser':'egrep \'Failure|^#|Abort|Expected|Actual|GoogleTestFailure|^==\''"
|
||||
|
||||
@@ -377,48 +376,6 @@ ASAN_CRASH_TEST_COMMANDS="[
|
||||
}
|
||||
]"
|
||||
|
||||
#
|
||||
# RocksDB test under undefined behavior sanitizer
|
||||
#
|
||||
UBSAN_TEST_COMMANDS="[
|
||||
{
|
||||
'name':'Rocksdb Unit Test under UBSAN',
|
||||
'oncall':'$ONCALL',
|
||||
'steps': [
|
||||
$CLEANUP_ENV,
|
||||
{
|
||||
'name':'Test RocksDB debug under UBSAN',
|
||||
'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make J=1 ubsan_check',
|
||||
'user':'root',
|
||||
$PARSER
|
||||
}
|
||||
],
|
||||
$REPORT
|
||||
}
|
||||
]"
|
||||
|
||||
#
|
||||
# RocksDB crash testing under udnefined behavior sanitizer
|
||||
#
|
||||
UBSAN_CRASH_TEST_COMMANDS="[
|
||||
{
|
||||
'name':'Rocksdb crash test under UBSAN',
|
||||
'oncall':'$ONCALL',
|
||||
'timeout': 86400,
|
||||
'steps': [
|
||||
$CLEANUP_ENV,
|
||||
{
|
||||
'name':'Build and run RocksDB debug ubsan_crash_test',
|
||||
'timeout': 86400,
|
||||
'shell':'$SHM $DEBUG make J=1 ubsan_crash_test',
|
||||
'user':'root',
|
||||
$PARSER
|
||||
},
|
||||
],
|
||||
$REPORT
|
||||
}
|
||||
]"
|
||||
|
||||
#
|
||||
# RocksDB unit test under valgrind
|
||||
#
|
||||
@@ -505,9 +462,9 @@ run_format_compatible()
|
||||
if [ -e "build_detect_platform" ]
|
||||
then
|
||||
sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file
|
||||
rm -rf build_detect_platform
|
||||
rm -rf build_detect_platform
|
||||
cp $TEST_TMPDIR/temp_build_file build_detect_platform
|
||||
chmod +x build_detect_platform
|
||||
chmod +x build_detect_platform
|
||||
fi
|
||||
|
||||
make ldb -j32
|
||||
@@ -675,12 +632,6 @@ case $1 in
|
||||
asan_crash)
|
||||
echo $ASAN_CRASH_TEST_COMMANDS
|
||||
;;
|
||||
ubsan)
|
||||
echo $UBSAN_TEST_COMMANDS
|
||||
;;
|
||||
ubsan_crash)
|
||||
echo $UBSAN_CRASH_TEST_COMMANDS
|
||||
;;
|
||||
valgrind)
|
||||
echo $VALGRIND_TEST_COMMANDS
|
||||
;;
|
||||
|
||||
@@ -9,7 +9,7 @@ Param(
|
||||
[string]$WorkFolder = "", # Direct tests to use that folder
|
||||
[int]$Limit = -1, # -1 means run all otherwise limit for testing purposes
|
||||
[string]$Exclude = "", # Expect a comma separated list, no spaces
|
||||
[string]$Run = "db_test", # Run db_test|db_test2|tests|testname1,testname2...
|
||||
[string]$Run = "db_test", # Run db_test|tests|testname1,testname2...
|
||||
# Number of async tasks that would run concurrently. Recommend a number below 64.
|
||||
# However, CPU utlization really depends on the storage media. Recommend ram based disk.
|
||||
# a value of 1 will run everything serially
|
||||
@@ -46,25 +46,15 @@ if($WorkFolder -eq "") {
|
||||
$Env:TEST_TMPDIR = $WorkFolder
|
||||
}
|
||||
|
||||
Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
|
||||
|
||||
# Use JEMALLOC executables
|
||||
if($Run -ceq "db_test" -or
|
||||
$Run -ceq "db_test2" ) {
|
||||
|
||||
$file_name = $Run
|
||||
|
||||
if($EnableJE) {
|
||||
$file_name += "_je"
|
||||
}
|
||||
|
||||
$file_name += ".exe"
|
||||
|
||||
$db_test = -Join ($BinariesFolder, $file_name)
|
||||
|
||||
Write-Output "Binaries: $BinariesFolder db_test: $db_test"
|
||||
if($EnableJE) {
|
||||
$db_test = -Join ($BinariesFolder, "db_test_je.exe")
|
||||
} else {
|
||||
$db_test = -Join ($BinariesFolder, "db_test.exe")
|
||||
}
|
||||
|
||||
Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
|
||||
Write-Output "Binaries: $BinariesFolder db_test: $db_test"
|
||||
|
||||
#Exclusions that we do not want to run
|
||||
$ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
|
||||
@@ -163,8 +153,7 @@ function Discover-TestBinaries([string]$Pattern, $HashTable) {
|
||||
|
||||
$TestsToRun = [ordered]@{}
|
||||
|
||||
if($Run -ceq "db_test" -or
|
||||
$Run -ceq "db_test2") {
|
||||
if($Run -ceq "db_test") {
|
||||
Normalize-DbTests -HashTable $TestsToRun
|
||||
} elseif($Run -ceq "tests") {
|
||||
if($EnableJE) {
|
||||
@@ -236,8 +225,7 @@ function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
|
||||
Write-Host "Starting $k"
|
||||
$log_path = ($TestToLog.$k)
|
||||
|
||||
if($Run -ceq "db_test" -or
|
||||
$Run -ceq "db_test2") {
|
||||
if($Run -ceq "db_test") {
|
||||
$job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
|
||||
} else {
|
||||
[string]$Exe = -Join ($BinariesFolder, $k)
|
||||
|
||||
+7
-17
@@ -41,17 +41,13 @@ TableBuilder* NewTableBuilder(
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
||||
int_tbl_prop_collector_factories,
|
||||
uint32_t column_family_id, const std::string& column_family_name,
|
||||
WritableFileWriter* file, const CompressionType compression_type,
|
||||
uint32_t column_family_id, WritableFileWriter* file,
|
||||
const CompressionType compression_type,
|
||||
const CompressionOptions& compression_opts, const bool skip_filters) {
|
||||
assert((column_family_id ==
|
||||
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
|
||||
column_family_name.empty());
|
||||
return ioptions.table_factory->NewTableBuilder(
|
||||
TableBuilderOptions(ioptions, internal_comparator,
|
||||
int_tbl_prop_collector_factories, compression_type,
|
||||
compression_opts, skip_filters,
|
||||
column_family_name),
|
||||
compression_opts, skip_filters),
|
||||
column_family_id, file);
|
||||
}
|
||||
|
||||
@@ -62,16 +58,12 @@ Status BuildTable(
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
||||
int_tbl_prop_collector_factories,
|
||||
uint32_t column_family_id, const std::string& column_family_name,
|
||||
std::vector<SequenceNumber> snapshots,
|
||||
uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
|
||||
SequenceNumber earliest_write_conflict_snapshot,
|
||||
const CompressionType compression,
|
||||
const CompressionOptions& compression_opts, bool paranoid_file_checks,
|
||||
InternalStats* internal_stats, const Env::IOPriority io_priority,
|
||||
TableProperties* table_properties, int level) {
|
||||
assert((column_family_id ==
|
||||
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
|
||||
column_family_name.empty());
|
||||
TableProperties* table_properties) {
|
||||
// Reports the IOStats for flush for every following bytes.
|
||||
const size_t kReportFlushIOStatsEvery = 1048576;
|
||||
Status s;
|
||||
@@ -95,8 +87,7 @@ Status BuildTable(
|
||||
|
||||
builder = NewTableBuilder(
|
||||
ioptions, internal_comparator, int_tbl_prop_collector_factories,
|
||||
column_family_id, column_family_name, file_writer.get(), compression,
|
||||
compression_opts);
|
||||
column_family_id, file_writer.get(), compression, compression_opts);
|
||||
}
|
||||
|
||||
MergeHelper merge(env, internal_comparator.user_comparator(),
|
||||
@@ -158,8 +149,7 @@ Status BuildTable(
|
||||
ReadOptions(), env_options, internal_comparator, meta->fd, nullptr,
|
||||
(internal_stats == nullptr) ? nullptr
|
||||
: internal_stats->GetFileReadHist(0),
|
||||
false /* for_compaction */, nullptr /* arena */,
|
||||
false /* skip_filter */, level));
|
||||
false));
|
||||
s = it->status();
|
||||
if (s.ok() && paranoid_file_checks) {
|
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
|
||||
+4
-11
@@ -33,16 +33,13 @@ class WritableFileWriter;
|
||||
class InternalStats;
|
||||
class InternalIterator;
|
||||
|
||||
// @param column_family_name Name of the column family that is also identified
|
||||
// by column_family_id, or empty string if unknown. It must outlive the
|
||||
// TableBuilder returned by this function.
|
||||
TableBuilder* NewTableBuilder(
|
||||
const ImmutableCFOptions& options,
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
||||
int_tbl_prop_collector_factories,
|
||||
uint32_t column_family_id, const std::string& column_family_name,
|
||||
WritableFileWriter* file, const CompressionType compression_type,
|
||||
uint32_t column_family_id, WritableFileWriter* file,
|
||||
const CompressionType compression_type,
|
||||
const CompressionOptions& compression_opts,
|
||||
const bool skip_filters = false);
|
||||
|
||||
@@ -51,9 +48,6 @@ TableBuilder* NewTableBuilder(
|
||||
// *meta will be filled with metadata about the generated table.
|
||||
// If no data is present in *iter, meta->file_size will be set to
|
||||
// zero, and no Table file will be produced.
|
||||
//
|
||||
// @param column_family_name Name of the column family that is also identified
|
||||
// by column_family_id, or empty string if unknown.
|
||||
extern Status BuildTable(
|
||||
const std::string& dbname, Env* env, const ImmutableCFOptions& options,
|
||||
const EnvOptions& env_options, TableCache* table_cache,
|
||||
@@ -61,13 +55,12 @@ extern Status BuildTable(
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
||||
int_tbl_prop_collector_factories,
|
||||
uint32_t column_family_id, const std::string& column_family_name,
|
||||
std::vector<SequenceNumber> snapshots,
|
||||
uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
|
||||
SequenceNumber earliest_write_conflict_snapshot,
|
||||
const CompressionType compression,
|
||||
const CompressionOptions& compression_opts, bool paranoid_file_checks,
|
||||
InternalStats* internal_stats,
|
||||
const Env::IOPriority io_priority = Env::IO_HIGH,
|
||||
TableProperties* table_properties = nullptr, int level = -1);
|
||||
TableProperties* table_properties = nullptr);
|
||||
|
||||
} // namespace rocksdb
|
||||
@@ -1288,11 +1288,6 @@ void rocksdb_block_based_options_set_cache_index_and_filter_blocks(
|
||||
options->rep.cache_index_and_filter_blocks = v;
|
||||
}
|
||||
|
||||
void rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(
|
||||
rocksdb_block_based_table_options_t* options, unsigned char v) {
|
||||
options->rep.pin_l0_filter_and_index_blocks_in_cache = v;
|
||||
}
|
||||
|
||||
void rocksdb_block_based_options_set_skip_table_builder_flush(
|
||||
rocksdb_block_based_table_options_t* options, unsigned char v) {
|
||||
options->rep.skip_table_builder_flush = v;
|
||||
|
||||
+7
-4
@@ -147,10 +147,13 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
|
||||
const ColumnFamilyOptions& src) {
|
||||
ColumnFamilyOptions result = src;
|
||||
result.comparator = icmp;
|
||||
size_t clamp_max = std::conditional<
|
||||
sizeof(size_t) == 4, std::integral_constant<size_t, 0xffffffff>,
|
||||
std::integral_constant<size_t, 64ull << 30>>::type::value;
|
||||
ClipToRange(&result.write_buffer_size, ((size_t)64) << 10, clamp_max);
|
||||
#ifdef OS_MACOSX
|
||||
// TODO(icanadi) make write_buffer_size uint64_t instead of size_t
|
||||
ClipToRange(&result.write_buffer_size, ((size_t)64) << 10, ((size_t)1) << 30);
|
||||
#else
|
||||
ClipToRange(&result.write_buffer_size,
|
||||
((size_t)64) << 10, ((size_t)64) << 30);
|
||||
#endif
|
||||
// if user sets arena_block_size, we trust user to use this value. Otherwise,
|
||||
// calculate a proper value from writer_buffer_size;
|
||||
if (result.arena_block_size <= 0) {
|
||||
|
||||
@@ -465,8 +465,6 @@ class ColumnFamilySet {
|
||||
// Don't call while iterating over ColumnFamilySet
|
||||
void FreeDeadColumnFamilies();
|
||||
|
||||
Cache* get_table_cache() { return table_cache_; }
|
||||
|
||||
private:
|
||||
friend class ColumnFamilyData;
|
||||
// helper function that gets called from cfd destructor
|
||||
|
||||
+30
-1
@@ -162,6 +162,9 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
|
||||
deletion_compaction_(_deletion_compaction),
|
||||
inputs_(std::move(_inputs)),
|
||||
grandparents_(std::move(_grandparents)),
|
||||
grandparent_index_(0),
|
||||
seen_key_(false),
|
||||
overlapped_bytes_(0),
|
||||
score_(_score),
|
||||
bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
|
||||
is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
|
||||
@@ -287,6 +290,32 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Compaction::ShouldStopBefore(const Slice& internal_key) {
|
||||
// Scan to find earliest grandparent file that contains key.
|
||||
const InternalKeyComparator* icmp = &cfd_->internal_comparator();
|
||||
while (grandparent_index_ < grandparents_.size() &&
|
||||
icmp->Compare(internal_key,
|
||||
grandparents_[grandparent_index_]->largest.Encode()) > 0) {
|
||||
if (seen_key_) {
|
||||
overlapped_bytes_ += grandparents_[grandparent_index_]->fd.GetFileSize();
|
||||
}
|
||||
assert(grandparent_index_ + 1 >= grandparents_.size() ||
|
||||
icmp->Compare(grandparents_[grandparent_index_]->largest.Encode(),
|
||||
grandparents_[grandparent_index_+1]->smallest.Encode())
|
||||
< 0);
|
||||
grandparent_index_++;
|
||||
}
|
||||
seen_key_ = true;
|
||||
|
||||
if (overlapped_bytes_ > max_grandparent_overlap_bytes_) {
|
||||
// Too much overlap for current output; start new output
|
||||
overlapped_bytes_ = 0;
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Mark (or clear) each file that is being compacted
|
||||
void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) {
|
||||
for (size_t i = 0; i < num_input_levels(); i++) {
|
||||
@@ -392,7 +421,7 @@ void Compaction::Summary(char* output, int len) {
|
||||
snprintf(output + write, len - write, "]");
|
||||
}
|
||||
|
||||
uint64_t Compaction::OutputFilePreallocationSize() const {
|
||||
uint64_t Compaction::OutputFilePreallocationSize() {
|
||||
uint64_t preallocation_size = 0;
|
||||
|
||||
if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel ||
|
||||
|
||||
+15
-17
@@ -104,7 +104,7 @@ class Compaction {
|
||||
}
|
||||
|
||||
// Returns the LevelFilesBrief of the specified compaction input level.
|
||||
const LevelFilesBrief* input_levels(size_t compaction_input_level) const {
|
||||
LevelFilesBrief* input_levels(size_t compaction_input_level) {
|
||||
return &input_levels_[compaction_input_level];
|
||||
}
|
||||
|
||||
@@ -132,6 +132,10 @@ class Compaction {
|
||||
bool KeyNotExistsBeyondOutputLevel(const Slice& user_key,
|
||||
std::vector<size_t>* level_ptrs) const;
|
||||
|
||||
// Returns true iff we should stop building the current output
|
||||
// before processing "internal_key".
|
||||
bool ShouldStopBefore(const Slice& internal_key);
|
||||
|
||||
// Clear all files to indicate that they are not being compacted
|
||||
// Delete this compaction from the list of running compactions.
|
||||
//
|
||||
@@ -147,13 +151,13 @@ class Compaction {
|
||||
double score() const { return score_; }
|
||||
|
||||
// Is this compaction creating a file in the bottom most level?
|
||||
bool bottommost_level() const { return bottommost_level_; }
|
||||
bool bottommost_level() { return bottommost_level_; }
|
||||
|
||||
// Does this compaction include all sst files?
|
||||
bool is_full_compaction() const { return is_full_compaction_; }
|
||||
bool is_full_compaction() { return is_full_compaction_; }
|
||||
|
||||
// Was this compaction triggered manually by the client?
|
||||
bool is_manual_compaction() const { return is_manual_compaction_; }
|
||||
bool is_manual_compaction() { return is_manual_compaction_; }
|
||||
|
||||
// Used when allow_trivial_move option is set in
|
||||
// Universal compaction. If all the input files are
|
||||
@@ -166,21 +170,19 @@ class Compaction {
|
||||
// Used when allow_trivial_move option is set in
|
||||
// Universal compaction. Returns true, if the input files
|
||||
// are non-overlapping and can be trivially moved.
|
||||
bool is_trivial_move() const { return is_trivial_move_; }
|
||||
bool is_trivial_move() { return is_trivial_move_; }
|
||||
|
||||
// How many total levels are there?
|
||||
int number_levels() const { return number_levels_; }
|
||||
|
||||
// Return the MutableCFOptions that should be used throughout the compaction
|
||||
// procedure
|
||||
const MutableCFOptions* mutable_cf_options() const {
|
||||
return &mutable_cf_options_;
|
||||
}
|
||||
const MutableCFOptions* mutable_cf_options() { return &mutable_cf_options_; }
|
||||
|
||||
// Returns the size in bytes that the output file should be preallocated to.
|
||||
// In level compaction, that is max_file_size_. In universal compaction, that
|
||||
// is the sum of all input file sizes.
|
||||
uint64_t OutputFilePreallocationSize() const;
|
||||
uint64_t OutputFilePreallocationSize();
|
||||
|
||||
void SetInputVersion(Version* input_version);
|
||||
|
||||
@@ -223,14 +225,6 @@ class Compaction {
|
||||
|
||||
CompactionReason compaction_reason() { return compaction_reason_; }
|
||||
|
||||
const std::vector<FileMetaData*>& grandparents() const {
|
||||
return grandparents_;
|
||||
}
|
||||
|
||||
uint64_t max_grandparent_overlap_bytes() const {
|
||||
return max_grandparent_overlap_bytes_;
|
||||
}
|
||||
|
||||
private:
|
||||
// mark (or clear) all files that are being compacted
|
||||
void MarkFilesBeingCompacted(bool mark_as_compacted);
|
||||
@@ -274,6 +268,10 @@ class Compaction {
|
||||
// State used to check for number of of overlapping grandparent files
|
||||
// (grandparent == "output_level_ + 1")
|
||||
std::vector<FileMetaData*> grandparents_;
|
||||
size_t grandparent_index_; // Index in grandparent_starts_
|
||||
bool seen_key_; // Some output key has been seen
|
||||
uint64_t overlapped_bytes_; // Bytes of overlap between current output
|
||||
// and grandparent files
|
||||
const double score_; // score that was used to pick this compaction.
|
||||
|
||||
// Is this compaction creating a file in the bottom most level?
|
||||
|
||||
@@ -14,7 +14,7 @@ CompactionIterator::CompactionIterator(
|
||||
InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
|
||||
SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
|
||||
SequenceNumber earliest_write_conflict_snapshot, Env* env,
|
||||
bool expect_valid_internal_key, const Compaction* compaction,
|
||||
bool expect_valid_internal_key, Compaction* compaction,
|
||||
const CompactionFilter* compaction_filter, LogBuffer* log_buffer)
|
||||
: input_(input),
|
||||
cmp_(cmp),
|
||||
|
||||
@@ -42,7 +42,7 @@ class CompactionIterator {
|
||||
std::vector<SequenceNumber>* snapshots,
|
||||
SequenceNumber earliest_write_conflict_snapshot, Env* env,
|
||||
bool expect_valid_internal_key,
|
||||
const Compaction* compaction = nullptr,
|
||||
Compaction* compaction = nullptr,
|
||||
const CompactionFilter* compaction_filter = nullptr,
|
||||
LogBuffer* log_buffer = nullptr);
|
||||
|
||||
@@ -92,7 +92,7 @@ class CompactionIterator {
|
||||
const SequenceNumber earliest_write_conflict_snapshot_;
|
||||
Env* env_;
|
||||
bool expect_valid_internal_key_;
|
||||
const Compaction* compaction_;
|
||||
Compaction* compaction_;
|
||||
const CompactionFilter* compaction_filter_;
|
||||
LogBuffer* log_buffer_;
|
||||
bool bottommost_level_;
|
||||
|
||||
+4
-50
@@ -63,7 +63,7 @@ namespace rocksdb {
|
||||
|
||||
// Maintains state for each sub-compaction
|
||||
struct CompactionJob::SubcompactionState {
|
||||
const Compaction* compaction;
|
||||
Compaction* compaction;
|
||||
std::unique_ptr<CompactionIterator> c_iter;
|
||||
|
||||
// The boundaries of the key-range this compaction is interested in. No two
|
||||
@@ -104,14 +104,6 @@ struct CompactionJob::SubcompactionState {
|
||||
uint64_t num_output_records;
|
||||
CompactionJobStats compaction_job_stats;
|
||||
uint64_t approx_size;
|
||||
// An index that used to speed up Compaction::ShouldStopBefore().
|
||||
size_t grandparent_index = 0;
|
||||
// The number of bytes overlapping between the current output and
|
||||
// grandparent files used in Compaction::ShouldStopBefore().
|
||||
uint64_t overlapped_bytes = 0;
|
||||
// A flag determine whether the key has been seen in
|
||||
// Compaction::ShouldStopBefore()
|
||||
bool seen_key = false;
|
||||
|
||||
SubcompactionState(Compaction* c, Slice* _start, Slice* _end,
|
||||
uint64_t size = 0)
|
||||
@@ -123,10 +115,7 @@ struct CompactionJob::SubcompactionState {
|
||||
total_bytes(0),
|
||||
num_input_records(0),
|
||||
num_output_records(0),
|
||||
approx_size(size),
|
||||
grandparent_index(0),
|
||||
overlapped_bytes(0),
|
||||
seen_key(false) {
|
||||
approx_size(size) {
|
||||
assert(compaction != nullptr);
|
||||
}
|
||||
|
||||
@@ -145,9 +134,6 @@ struct CompactionJob::SubcompactionState {
|
||||
num_output_records = std::move(o.num_output_records);
|
||||
compaction_job_stats = std::move(o.compaction_job_stats);
|
||||
approx_size = std::move(o.approx_size);
|
||||
grandparent_index = std::move(o.grandparent_index);
|
||||
overlapped_bytes = std::move(o.overlapped_bytes);
|
||||
seen_key = std::move(o.seen_key);
|
||||
return *this;
|
||||
}
|
||||
|
||||
@@ -155,38 +141,6 @@ struct CompactionJob::SubcompactionState {
|
||||
SubcompactionState(const SubcompactionState&) = delete;
|
||||
|
||||
SubcompactionState& operator=(const SubcompactionState&) = delete;
|
||||
|
||||
// Returns true iff we should stop building the current output
|
||||
// before processing "internal_key".
|
||||
bool ShouldStopBefore(const Slice& internal_key) {
|
||||
const InternalKeyComparator* icmp =
|
||||
&compaction->column_family_data()->internal_comparator();
|
||||
const std::vector<FileMetaData*>& grandparents = compaction->grandparents();
|
||||
|
||||
// Scan to find earliest grandparent file that contains key.
|
||||
while (grandparent_index < grandparents.size() &&
|
||||
icmp->Compare(internal_key,
|
||||
grandparents[grandparent_index]->largest.Encode()) >
|
||||
0) {
|
||||
if (seen_key) {
|
||||
overlapped_bytes += grandparents[grandparent_index]->fd.GetFileSize();
|
||||
}
|
||||
assert(grandparent_index + 1 >= grandparents.size() ||
|
||||
icmp->Compare(
|
||||
grandparents[grandparent_index]->largest.Encode(),
|
||||
grandparents[grandparent_index + 1]->smallest.Encode()) < 0);
|
||||
grandparent_index++;
|
||||
}
|
||||
seen_key = true;
|
||||
|
||||
if (overlapped_bytes > compaction->max_grandparent_overlap_bytes()) {
|
||||
// Too much overlap for current output; start new output
|
||||
overlapped_bytes = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// Maintains state for the entire compaction
|
||||
@@ -715,7 +669,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
|
||||
if (end != nullptr &&
|
||||
cfd->user_comparator()->Compare(c_iter->user_key(), *end) >= 0) {
|
||||
break;
|
||||
} else if (sub_compact->ShouldStopBefore(key) &&
|
||||
} else if (sub_compact->compaction->ShouldStopBefore(key) &&
|
||||
sub_compact->builder != nullptr) {
|
||||
status = FinishCompactionOutputFile(input->status(), sub_compact);
|
||||
if (!status.ok()) {
|
||||
@@ -1019,7 +973,7 @@ Status CompactionJob::OpenCompactionOutputFile(
|
||||
cfd->ioptions()->optimize_filters_for_hits && bottommost_level_;
|
||||
sub_compact->builder.reset(NewTableBuilder(
|
||||
*cfd->ioptions(), cfd->internal_comparator(),
|
||||
cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
|
||||
cfd->int_tbl_prop_collector_factories(), cfd->GetID(),
|
||||
sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
|
||||
cfd->ioptions()->compression_opts, skip_filters));
|
||||
LogFlush(db_options_.info_log);
|
||||
|
||||
@@ -660,7 +660,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
|
||||
options.max_subcompactions = max_subcompactions_;
|
||||
options.bytes_per_sync = 512 * 1024;
|
||||
|
||||
options.report_bg_io_stats = true;
|
||||
options.compaction_measure_io_stats = true;
|
||||
for (int test = 0; test < 2; ++test) {
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
@@ -195,7 +195,6 @@ TEST_F(CompactionPickerTest, LevelMaxScore) {
|
||||
NewVersionStorage(6, kCompactionStyleLevel);
|
||||
mutable_cf_options_.target_file_size_base = 10000000;
|
||||
mutable_cf_options_.target_file_size_multiplier = 10;
|
||||
mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
|
||||
Add(0, 1U, "150", "200", 1000000000U);
|
||||
// Level 1 score 1.2
|
||||
Add(1, 66U, "150", "200", 6000000U);
|
||||
@@ -492,7 +491,6 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
|
||||
NewVersionStorage(6, kCompactionStyleLevel);
|
||||
mutable_cf_options_.target_file_size_base = 10000000;
|
||||
mutable_cf_options_.target_file_size_multiplier = 10;
|
||||
mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
|
||||
mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
|
||||
|
||||
Add(2, 6U, "150", "179", 50000000U);
|
||||
@@ -519,7 +517,6 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
|
||||
NewVersionStorage(6, kCompactionStyleLevel);
|
||||
mutable_cf_options_.target_file_size_base = 10000000;
|
||||
mutable_cf_options_.target_file_size_multiplier = 10;
|
||||
mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
|
||||
mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
|
||||
|
||||
Add(2, 6U, "150", "175",
|
||||
@@ -547,21 +544,20 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
|
||||
|
||||
TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) {
|
||||
NewVersionStorage(6, kCompactionStyleLevel);
|
||||
mutable_cf_options_.max_bytes_for_level_base = 10000000;
|
||||
mutable_cf_options_.max_bytes_for_level_multiplier = 10;
|
||||
mutable_cf_options_.target_file_size_base = 10000000;
|
||||
mutable_cf_options_.target_file_size_multiplier = 10;
|
||||
mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
|
||||
|
||||
// file 7 and 8 over lap with the same file, but file 8 is smaller so
|
||||
// it will be picked.
|
||||
Add(2, 6U, "150", "167", 60000000U); // Overlaps with file 26, 27
|
||||
Add(2, 7U, "168", "169", 60000000U); // Overlaps with file 27
|
||||
Add(2, 8U, "201", "300", 61000000U); // Overlaps with file 28, but the file
|
||||
// itself is larger. Should be picked.
|
||||
Add(2, 6U, "150", "175", 60000000U); // Overlaps with file 26, 27
|
||||
Add(2, 7U, "176", "200", 60000000U); // Overlaps with file 27
|
||||
Add(2, 8U, "201", "300", 61000000U); // Overlaps with file 27
|
||||
|
||||
Add(3, 26U, "160", "165", 260000000U);
|
||||
Add(3, 27U, "166", "170", 260000000U);
|
||||
Add(3, 28U, "180", "400", 260000000U);
|
||||
Add(3, 29U, "401", "500", 260000000U);
|
||||
Add(3, 26U, "166", "170", 260000000U);
|
||||
Add(3, 27U, "180", "400", 260000000U);
|
||||
Add(3, 28U, "401", "500", 260000000U);
|
||||
UpdateVersionStorageInfo();
|
||||
|
||||
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
||||
|
||||
@@ -166,12 +166,10 @@ TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef SNAPPY
|
||||
TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
|
||||
ReadOptions read_options;
|
||||
auto table_options = GetTableOptions();
|
||||
auto options = GetOptions(table_options);
|
||||
options.compression = CompressionType::kSnappyCompression;
|
||||
InitTable(options);
|
||||
|
||||
std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
|
||||
@@ -229,7 +227,6 @@ TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
|
||||
delete iter;
|
||||
iter = nullptr;
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
|
||||
@@ -368,10 +368,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
|
||||
// entries are deleted. The compaction should create bunch of 'DeleteFile'
|
||||
// entries in VersionEdit, but none of the 'AddFile's.
|
||||
TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
|
||||
options.disable_auto_compactions = true;
|
||||
options.create_if_missing = true;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
// put some data
|
||||
@@ -399,10 +400,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
|
||||
|
||||
TEST_F(DBTestCompactionFilter, CompactionFilterWithValueChange) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.num_levels = 3;
|
||||
options.compaction_filter_factory =
|
||||
std::make_shared<ChangeFilterFactory>();
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// Write 100K+1 keys, these are written to a few files
|
||||
@@ -465,7 +467,8 @@ TEST_F(DBTestCompactionFilter, CompactionFilterWithMergeOperator) {
|
||||
PutFixed64(&three, 3);
|
||||
PutFixed64(&four, 4);
|
||||
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options = CurrentOptions(options);
|
||||
options.create_if_missing = true;
|
||||
options.merge_operator = MergeOperators::CreateUInt64AddOperator();
|
||||
options.num_levels = 3;
|
||||
@@ -618,10 +621,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextCfId) {
|
||||
// Compaction filters should only be applied to records that are newer than the
|
||||
// latest snapshot. This test inserts records and applies a delete filter.
|
||||
TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
|
||||
options.disable_auto_compactions = true;
|
||||
options.create_if_missing = true;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
// Put some data.
|
||||
@@ -655,10 +659,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
|
||||
// records newer than the snapshot will also be processed
|
||||
TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
|
||||
std::string five = ToString(5);
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_filter_factory = std::make_shared<DeleteISFilterFactory>();
|
||||
options.disable_auto_compactions = true;
|
||||
options.create_if_missing = true;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
// Put some data.
|
||||
|
||||
+25
-13
@@ -288,7 +288,8 @@ TEST_F(DBCompactionTest, SkipStatsUpdateTest) {
|
||||
}
|
||||
|
||||
TEST_F(DBCompactionTest, TestTableReaderForCompaction) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options = CurrentOptions(options);
|
||||
options.env = env_;
|
||||
options.new_table_reader_for_compaction_inputs = true;
|
||||
options.max_open_files = 100;
|
||||
@@ -492,13 +493,14 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) {
|
||||
TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
|
||||
const int kNumKeysPerFile = 100;
|
||||
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.write_buffer_size = 110 << 10; // 110KB
|
||||
options.arena_block_size = 4 << 10;
|
||||
options.num_levels = 3;
|
||||
options.level0_file_num_compaction_trigger = 3;
|
||||
options.max_subcompactions = max_subcompactions_;
|
||||
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
Random rnd(301);
|
||||
@@ -536,7 +538,7 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
|
||||
// and see number of compactions scheduled to be less than allowed.
|
||||
const int kNumKeysPerFile = 100;
|
||||
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.write_buffer_size = 110 << 10; // 110KB
|
||||
options.arena_block_size = 4 << 10;
|
||||
options.num_levels = 3;
|
||||
@@ -547,6 +549,7 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
|
||||
options.base_background_compactions = 1;
|
||||
options.max_background_compactions = 3;
|
||||
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
|
||||
options = CurrentOptions(options);
|
||||
|
||||
// Block all threads in thread pool.
|
||||
const size_t kTotalTasks = 4;
|
||||
@@ -629,9 +632,10 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
|
||||
}
|
||||
|
||||
TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.write_buffer_size = 100000000; // Large write buffer
|
||||
options.max_subcompactions = max_subcompactions_;
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
Random rnd(301);
|
||||
@@ -658,8 +662,9 @@ TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
|
||||
|
||||
TEST_F(DBCompactionTest, MinorCompactionsHappen) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.write_buffer_size = 10000;
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
const int N = 500;
|
||||
@@ -684,7 +689,7 @@ TEST_F(DBCompactionTest, MinorCompactionsHappen) {
|
||||
}
|
||||
|
||||
TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_style = kCompactionStyleLevel;
|
||||
options.level0_file_num_compaction_trigger = 3;
|
||||
|
||||
@@ -698,6 +703,7 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
|
||||
const size_t key_len =
|
||||
static_cast<size_t>(compact_opt.output_file_size_limit) / 5;
|
||||
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
std::vector<const Snapshot*> snaps;
|
||||
@@ -741,8 +747,10 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
|
||||
// if the database is shutdown during the memtable compaction.
|
||||
TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.env = env_;
|
||||
options.write_buffer_size = 1000000;
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// Trigger a long memtable compaction and reopen the database during it
|
||||
@@ -766,9 +774,10 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveOneFile) {
|
||||
[&](void* arg) { trivial_move++; });
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.write_buffer_size = 100000000;
|
||||
options.max_subcompactions = max_subcompactions_;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
int32_t num_keys = 80;
|
||||
@@ -1360,9 +1369,10 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveToLastLevelWithFiles) {
|
||||
[&](void* arg) { non_trivial_move++; });
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.write_buffer_size = 100000000;
|
||||
options.max_subcompactions = max_subcompactions_;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
int32_t value_size = 10 * 1024; // 10 KB
|
||||
@@ -1651,7 +1661,7 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
|
||||
int max_key_universal_insert = 600;
|
||||
|
||||
// Stage 1: generate a db with level compaction
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.write_buffer_size = 110 << 10; // 110KB
|
||||
options.arena_block_size = 4 << 10;
|
||||
options.num_levels = 4;
|
||||
@@ -1661,6 +1671,7 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
|
||||
options.target_file_size_base = 200 << 10; // 200KB
|
||||
options.target_file_size_multiplier = 1;
|
||||
options.max_subcompactions = max_subcompactions_;
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
for (int i = 0; i <= max_key_level_insert; i++) {
|
||||
@@ -2375,9 +2386,10 @@ TEST_P(DBCompactionTestWithParam, ForceBottommostLevelCompaction) {
|
||||
[&](void* arg) { non_trivial_move++; });
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.write_buffer_size = 100000000;
|
||||
options.max_subcompactions = max_subcompactions_;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
int32_t value_size = 10 * 1024; // 10 KB
|
||||
@@ -2467,14 +2479,14 @@ class CompactionPriTest : public DBTestBase,
|
||||
};
|
||||
|
||||
TEST_P(CompactionPriTest, Test) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.write_buffer_size = 16 * 1024;
|
||||
options.compaction_pri = static_cast<CompactionPri>(compaction_pri_);
|
||||
options.hard_pending_compaction_bytes_limit = 256 * 1024;
|
||||
options.max_bytes_for_level_base = 64 * 1024;
|
||||
options.max_bytes_for_level_multiplier = 4;
|
||||
options.compression = kNoCompression;
|
||||
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
Random rnd(301);
|
||||
|
||||
+14
-48
@@ -405,21 +405,6 @@ DBImpl::~DBImpl() {
|
||||
}
|
||||
logs_.clear();
|
||||
|
||||
// Table cache may have table handles holding blocks from the block cache.
|
||||
// We need to release them before the block cache is destroyed. The block
|
||||
// cache may be destroyed inside versions_.reset(), when column family data
|
||||
// list is destroyed, so leaving handles in table cache after
|
||||
// versions_.reset() may cause issues.
|
||||
// Here we clean all unreferenced handles in table cache.
|
||||
// Now we assume all user queries have finished, so only version set itself
|
||||
// can possibly hold the blocks from block cache. After releasing unreferenced
|
||||
// handles here, only handles held by version set left and inside
|
||||
// versions_.reset(), we will release them. There, we need to make sure every
|
||||
// time a handle is released, we erase it from the cache too. By doing that,
|
||||
// we can guarantee that after versions_.reset(), table cache is empty
|
||||
// so the cache can be safely destroyed.
|
||||
table_cache_->EraseUnRefEntries();
|
||||
|
||||
// versions need to be destroyed before table_cache since it can hold
|
||||
// references to table_cache.
|
||||
versions_.reset();
|
||||
@@ -844,8 +829,7 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
|
||||
|
||||
// Delete old info log files.
|
||||
size_t old_info_log_file_count = old_info_log_files.size();
|
||||
if (old_info_log_file_count != 0 &&
|
||||
old_info_log_file_count >= db_options_.keep_log_file_num) {
|
||||
if (old_info_log_file_count >= db_options_.keep_log_file_num) {
|
||||
std::sort(old_info_log_files.begin(), old_info_log_files.end());
|
||||
size_t end = old_info_log_file_count - db_options_.keep_log_file_num;
|
||||
for (unsigned int i = 0; i <= end; i++) {
|
||||
@@ -1118,23 +1102,6 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
stream.EndArray();
|
||||
}
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
if (db_options_.wal_filter != nullptr) {
|
||||
std::map<std::string, uint32_t> cf_name_id_map;
|
||||
std::map<uint32_t, uint64_t> cf_lognumber_map;
|
||||
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
||||
cf_name_id_map.insert(
|
||||
std::make_pair(cfd->GetName(), cfd->GetID()));
|
||||
cf_lognumber_map.insert(
|
||||
std::make_pair(cfd->GetID(), cfd->GetLogNumber()));
|
||||
}
|
||||
|
||||
db_options_.wal_filter->ColumnFamilyLogNumberMap(
|
||||
cf_lognumber_map,
|
||||
cf_name_id_map);
|
||||
}
|
||||
#endif
|
||||
|
||||
bool continue_replay_log = true;
|
||||
for (auto log_number : log_numbers) {
|
||||
// The previous incarnation may not have written any MANIFEST
|
||||
@@ -1201,7 +1168,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
continue_replay_log &&
|
||||
reader.ReadRecord(&record, &scratch, db_options_.wal_recovery_mode) &&
|
||||
status.ok()) {
|
||||
if (record.size() < WriteBatchInternal::kHeader) {
|
||||
if (record.size() < 12) {
|
||||
reporter.Corruption(record.size(),
|
||||
Status::Corruption("log record too small"));
|
||||
continue;
|
||||
@@ -1214,8 +1181,8 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
bool batch_changed = false;
|
||||
|
||||
WalFilter::WalProcessingOption wal_processing_option =
|
||||
db_options_.wal_filter->LogRecordFound(log_number, fname, batch,
|
||||
&new_batch, &batch_changed);
|
||||
db_options_.wal_filter->LogRecord(batch, &new_batch,
|
||||
&batch_changed);
|
||||
|
||||
switch (wal_processing_option) {
|
||||
case WalFilter::WalProcessingOption::kContinueProcessing:
|
||||
@@ -1452,8 +1419,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
|
||||
s = BuildTable(
|
||||
dbname_, env_, *cfd->ioptions(), env_options_, cfd->table_cache(),
|
||||
iter.get(), &meta, cfd->internal_comparator(),
|
||||
cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
|
||||
snapshot_seqs, earliest_write_conflict_snapshot,
|
||||
cfd->int_tbl_prop_collector_factories(), cfd->GetID(), snapshot_seqs,
|
||||
earliest_write_conflict_snapshot,
|
||||
GetCompressionFlush(*cfd->ioptions()),
|
||||
cfd->ioptions()->compression_opts, paranoid_file_checks,
|
||||
cfd->internal_stats(), Env::IO_HIGH, &info.table_properties);
|
||||
@@ -1513,13 +1480,12 @@ Status DBImpl::FlushMemTableToOutputFile(
|
||||
std::vector<SequenceNumber> snapshot_seqs =
|
||||
snapshots_.GetAll(&earliest_write_conflict_snapshot);
|
||||
|
||||
FlushJob flush_job(dbname_, cfd, db_options_, mutable_cf_options,
|
||||
env_options_, versions_.get(), &mutex_, &shutting_down_,
|
||||
snapshot_seqs, earliest_write_conflict_snapshot,
|
||||
job_context, log_buffer, directories_.GetDbDir(),
|
||||
directories_.GetDataDir(0U),
|
||||
GetCompressionFlush(*cfd->ioptions()), stats_,
|
||||
&event_logger_, mutable_cf_options.report_bg_io_stats);
|
||||
FlushJob flush_job(
|
||||
dbname_, cfd, db_options_, mutable_cf_options, env_options_,
|
||||
versions_.get(), &mutex_, &shutting_down_, snapshot_seqs,
|
||||
earliest_write_conflict_snapshot, job_context, log_buffer,
|
||||
directories_.GetDbDir(), directories_.GetDataDir(0U),
|
||||
GetCompressionFlush(*cfd->ioptions()), stats_, &event_logger_);
|
||||
|
||||
FileMetaData file_meta;
|
||||
|
||||
@@ -1868,7 +1834,7 @@ Status DBImpl::CompactFilesImpl(
|
||||
directories_.GetDataDir(c->output_path_id()), stats_, &mutex_, &bg_error_,
|
||||
snapshot_seqs, earliest_write_conflict_snapshot, table_cache_,
|
||||
&event_logger_, c->mutable_cf_options()->paranoid_file_checks,
|
||||
c->mutable_cf_options()->report_bg_io_stats, dbname_,
|
||||
c->mutable_cf_options()->compaction_measure_io_stats, dbname_,
|
||||
nullptr); // Here we pass a nullptr for CompactionJobStats because
|
||||
// CompactFiles does not trigger OnCompactionCompleted(),
|
||||
// which is the only place where CompactionJobStats is
|
||||
@@ -3074,7 +3040,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
|
||||
&bg_error_, snapshot_seqs, earliest_write_conflict_snapshot,
|
||||
table_cache_, &event_logger_,
|
||||
c->mutable_cf_options()->paranoid_file_checks,
|
||||
c->mutable_cf_options()->report_bg_io_stats, dbname_,
|
||||
c->mutable_cf_options()->compaction_measure_io_stats, dbname_,
|
||||
&compaction_job_stats);
|
||||
compaction_job.Prepare();
|
||||
|
||||
|
||||
+1
-1
@@ -523,7 +523,7 @@ class DBImpl : public DB {
|
||||
SequenceNumber* max_sequence, bool read_only);
|
||||
|
||||
// The following two methods are used to flush a memtable to
|
||||
// storage. The first one is used at database RecoveryTime (when the
|
||||
// storage. The first one is used atdatabase RecoveryTime (when the
|
||||
// database is opened) and is heavyweight because it holds the mutex
|
||||
// for the entire period. The second method WriteLevel0Table supports
|
||||
// concurrent flush memtables to storage.
|
||||
|
||||
@@ -18,11 +18,12 @@ class DBTestInPlaceUpdate : public DBTestBase {
|
||||
|
||||
TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.inplace_update_support = true;
|
||||
options.env = env_;
|
||||
options.write_buffer_size = 100000;
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// Update key with values of smaller size
|
||||
@@ -40,11 +41,12 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) {
|
||||
|
||||
TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.inplace_update_support = true;
|
||||
options.env = env_;
|
||||
options.write_buffer_size = 100000;
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// Update key with values of larger size
|
||||
@@ -62,7 +64,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) {
|
||||
|
||||
TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.inplace_update_support = true;
|
||||
|
||||
@@ -70,6 +72,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
|
||||
options.write_buffer_size = 100000;
|
||||
options.inplace_callback =
|
||||
rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerSize;
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// Update key with values of smaller size
|
||||
@@ -89,7 +92,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
|
||||
|
||||
TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.inplace_update_support = true;
|
||||
|
||||
@@ -97,6 +100,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
|
||||
options.write_buffer_size = 100000;
|
||||
options.inplace_callback =
|
||||
rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerVarintSize;
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// Update key with values of smaller varint size
|
||||
@@ -116,7 +120,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
|
||||
|
||||
TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.inplace_update_support = true;
|
||||
|
||||
@@ -124,6 +128,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
|
||||
options.write_buffer_size = 100000;
|
||||
options.inplace_callback =
|
||||
rocksdb::DBTestInPlaceUpdate::updateInPlaceLargerSize;
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// Update key with values of larger size
|
||||
@@ -141,7 +146,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
|
||||
|
||||
TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.inplace_update_support = true;
|
||||
|
||||
@@ -149,6 +154,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) {
|
||||
options.write_buffer_size = 100000;
|
||||
options.inplace_callback =
|
||||
rocksdb::DBTestInPlaceUpdate::updateInPlaceNoAction;
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// Callback function requests no actions from db
|
||||
|
||||
+20
-25
@@ -15,7 +15,6 @@
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/merge_context.h"
|
||||
#include "port/port.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/iterator.h"
|
||||
@@ -252,7 +251,7 @@ class DBIter: public Iterator {
|
||||
bool prefix_same_as_start_;
|
||||
bool iter_pinned_;
|
||||
// List of operands for merge operator.
|
||||
MergeContext merge_context_;
|
||||
std::deque<std::string> merge_operands_;
|
||||
LocalStatistics local_stats_;
|
||||
|
||||
// No copying allowed
|
||||
@@ -412,9 +411,9 @@ void DBIter::MergeValuesNewToOld() {
|
||||
return;
|
||||
}
|
||||
|
||||
merge_context_.Clear();
|
||||
// Start the merge process by pushing the first operand
|
||||
merge_context_.PushOperand(iter_->value());
|
||||
std::deque<std::string> operands;
|
||||
operands.push_front(iter_->value().ToString());
|
||||
|
||||
ParsedInternalKey ikey;
|
||||
for (iter_->Next(); iter_->Valid(); iter_->Next()) {
|
||||
@@ -439,8 +438,7 @@ void DBIter::MergeValuesNewToOld() {
|
||||
{
|
||||
StopWatchNano timer(env_, statistics_ != nullptr);
|
||||
PERF_TIMER_GUARD(merge_operator_time_nanos);
|
||||
user_merge_operator_->FullMerge(ikey.user_key, &val,
|
||||
merge_context_.GetOperands(),
|
||||
user_merge_operator_->FullMerge(ikey.user_key, &val, operands,
|
||||
&saved_value_, logger_);
|
||||
RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
|
||||
timer.ElapsedNanos());
|
||||
@@ -452,7 +450,7 @@ void DBIter::MergeValuesNewToOld() {
|
||||
// hit a merge, add the value as an operand and run associative merge.
|
||||
// when complete, add result to operands and continue.
|
||||
const Slice& val = iter_->value();
|
||||
merge_context_.PushOperand(val);
|
||||
operands.push_front(val.ToString());
|
||||
} else {
|
||||
assert(false);
|
||||
}
|
||||
@@ -465,9 +463,8 @@ void DBIter::MergeValuesNewToOld() {
|
||||
// a deletion marker.
|
||||
// feed null as the existing value to the merge operator, such that
|
||||
// client can differentiate this scenario and do things accordingly.
|
||||
user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
|
||||
merge_context_.GetOperands(), &saved_value_,
|
||||
logger_);
|
||||
user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands,
|
||||
&saved_value_, logger_);
|
||||
RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
|
||||
}
|
||||
}
|
||||
@@ -559,7 +556,7 @@ void DBIter::PrevInternal() {
|
||||
// saved_value_
|
||||
bool DBIter::FindValueForCurrentKey() {
|
||||
assert(iter_->Valid());
|
||||
merge_context_.Clear();
|
||||
merge_operands_.clear();
|
||||
// last entry before merge (could be kTypeDeletion, kTypeSingleDeletion or
|
||||
// kTypeValue)
|
||||
ValueType last_not_merge_type = kTypeDeletion;
|
||||
@@ -579,19 +576,19 @@ bool DBIter::FindValueForCurrentKey() {
|
||||
last_key_entry_type = ikey.type;
|
||||
switch (last_key_entry_type) {
|
||||
case kTypeValue:
|
||||
merge_context_.Clear();
|
||||
merge_operands_.clear();
|
||||
saved_value_ = iter_->value().ToString();
|
||||
last_not_merge_type = kTypeValue;
|
||||
break;
|
||||
case kTypeDeletion:
|
||||
case kTypeSingleDeletion:
|
||||
merge_context_.Clear();
|
||||
merge_operands_.clear();
|
||||
last_not_merge_type = last_key_entry_type;
|
||||
PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
|
||||
break;
|
||||
case kTypeMerge:
|
||||
assert(user_merge_operator_ != nullptr);
|
||||
merge_context_.PushOperandBack(iter_->value());
|
||||
merge_operands_.push_back(iter_->value().ToString());
|
||||
break;
|
||||
default:
|
||||
assert(false);
|
||||
@@ -614,8 +611,8 @@ bool DBIter::FindValueForCurrentKey() {
|
||||
StopWatchNano timer(env_, statistics_ != nullptr);
|
||||
PERF_TIMER_GUARD(merge_operator_time_nanos);
|
||||
user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
|
||||
merge_context_.GetOperands(),
|
||||
&saved_value_, logger_);
|
||||
merge_operands_, &saved_value_,
|
||||
logger_);
|
||||
RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
|
||||
timer.ElapsedNanos());
|
||||
} else {
|
||||
@@ -626,8 +623,8 @@ bool DBIter::FindValueForCurrentKey() {
|
||||
StopWatchNano timer(env_, statistics_ != nullptr);
|
||||
PERF_TIMER_GUARD(merge_operator_time_nanos);
|
||||
user_merge_operator_->FullMerge(saved_key_.GetKey(), &temp_slice,
|
||||
merge_context_.GetOperands(),
|
||||
&saved_value_, logger_);
|
||||
merge_operands_, &saved_value_,
|
||||
logger_);
|
||||
RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
|
||||
timer.ElapsedNanos());
|
||||
}
|
||||
@@ -670,11 +667,11 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
|
||||
|
||||
// kTypeMerge. We need to collect all kTypeMerge values and save them
|
||||
// in operands
|
||||
merge_context_.Clear();
|
||||
std::deque<std::string> operands;
|
||||
while (iter_->Valid() &&
|
||||
user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()) &&
|
||||
ikey.type == kTypeMerge) {
|
||||
merge_context_.PushOperand(iter_->value());
|
||||
operands.push_front(iter_->value().ToString());
|
||||
iter_->Next();
|
||||
FindParseableKey(&ikey, kForward);
|
||||
}
|
||||
@@ -685,8 +682,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
|
||||
{
|
||||
StopWatchNano timer(env_, statistics_ != nullptr);
|
||||
PERF_TIMER_GUARD(merge_operator_time_nanos);
|
||||
user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
|
||||
merge_context_.GetOperands(),
|
||||
user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands,
|
||||
&saved_value_, logger_);
|
||||
RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
|
||||
}
|
||||
@@ -704,9 +700,8 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
|
||||
{
|
||||
StopWatchNano timer(env_, statistics_ != nullptr);
|
||||
PERF_TIMER_GUARD(merge_operator_time_nanos);
|
||||
user_merge_operator_->FullMerge(saved_key_.GetKey(), &val,
|
||||
merge_context_.GetOperands(), &saved_value_,
|
||||
logger_);
|
||||
user_merge_operator_->FullMerge(saved_key_.GetKey(), &val, operands,
|
||||
&saved_value_, logger_);
|
||||
RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
|
||||
}
|
||||
valid_ = true;
|
||||
|
||||
@@ -253,35 +253,6 @@ TEST_F(DBPropertiesTest, ValidatePropertyInfo) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DBPropertiesTest, ValidateSampleNumber) {
|
||||
// When "max_open_files" is -1, we read all the files for
|
||||
// "rocksdb.estimate-num-keys" computation, which is the ground truth.
|
||||
// Otherwise, we sample 20 newest files to make an estimation.
|
||||
// Formula: lastest_20_files_active_key_ratio * total_files
|
||||
Options options = CurrentOptions();
|
||||
options.disable_auto_compactions = true;
|
||||
options.level0_stop_writes_trigger = 1000;
|
||||
DestroyAndReopen(options);
|
||||
int key = 0;
|
||||
for (int files = 20; files >= 10; files -= 10) {
|
||||
for (int i = 0; i < files; i++) {
|
||||
int rows = files / 10;
|
||||
for (int j = 0; j < rows; j++) {
|
||||
db_->Put(WriteOptions(), std::to_string(++key), "foo");
|
||||
}
|
||||
db_->Flush(FlushOptions());
|
||||
}
|
||||
}
|
||||
std::string num;
|
||||
Reopen(options);
|
||||
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
|
||||
ASSERT_EQ("45", num);
|
||||
options.max_open_files = -1;
|
||||
Reopen(options);
|
||||
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
|
||||
ASSERT_EQ("50", num);
|
||||
}
|
||||
|
||||
TEST_F(DBPropertiesTest, AggregatedTableProperties) {
|
||||
for (int kTableCount = 40; kTableCount <= 100; kTableCount += 30) {
|
||||
const int kKeysPerTable = 100;
|
||||
|
||||
@@ -215,34 +215,6 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfTablesInRange) {
|
||||
TestGetPropertiesOfTablesInRange(std::move(ranges));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DBTablePropertiesTest, GetColumnFamilyNameProperty) {
|
||||
std::string kExtraCfName = "pikachu";
|
||||
CreateAndReopenWithCF({kExtraCfName}, CurrentOptions());
|
||||
|
||||
// Create one table per CF, then verify it was created with the column family
|
||||
// name property.
|
||||
for (int cf = 0; cf < 2; ++cf) {
|
||||
Put(cf, "key", "val");
|
||||
Flush(cf);
|
||||
|
||||
TablePropertiesCollection fname_to_props;
|
||||
ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props));
|
||||
ASSERT_EQ(1U, fname_to_props.size());
|
||||
|
||||
std::string expected_cf_name;
|
||||
if (cf > 0) {
|
||||
expected_cf_name = kExtraCfName;
|
||||
} else {
|
||||
expected_cf_name = kDefaultColumnFamilyName;
|
||||
}
|
||||
ASSERT_EQ(expected_cf_name,
|
||||
fname_to_props.begin()->second->column_family_name);
|
||||
ASSERT_EQ(cf, static_cast<uint32_t>(
|
||||
fname_to_props.begin()->second->column_family_id));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
+437
-19
@@ -228,9 +228,10 @@ TEST_F(DBTest, MemEnvTest) {
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
TEST_F(DBTest, WriteEmptyBatch) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.env = env_;
|
||||
options.write_buffer_size = 100000;
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
ASSERT_OK(Put(1, "foo", "bar"));
|
||||
@@ -280,12 +281,13 @@ TEST_F(DBTest, ReadOnlyDB) {
|
||||
|
||||
TEST_F(DBTest, CompactedDB) {
|
||||
const uint64_t kFileSize = 1 << 20;
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.disable_auto_compactions = true;
|
||||
options.write_buffer_size = kFileSize;
|
||||
options.target_file_size_base = kFileSize;
|
||||
options.max_bytes_for_level_base = 1 << 30;
|
||||
options.compression = kNoCompression;
|
||||
options = CurrentOptions(options);
|
||||
Reopen(options);
|
||||
// 1 L0 file, use CompactedDB if max_open_files = -1
|
||||
ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, '1')));
|
||||
@@ -762,10 +764,11 @@ TEST_F(DBTest, DISABLED_VeryLargeValue) {
|
||||
std::string key1(kKeySize, 'c');
|
||||
std::string key2(kKeySize, 'd');
|
||||
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.env = env_;
|
||||
options.write_buffer_size = 100000; // Small write buffer
|
||||
options.paranoid_checks = true;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
ASSERT_OK(Put("boo", "v1"));
|
||||
@@ -808,8 +811,10 @@ TEST_F(DBTest, DISABLED_VeryLargeValue) {
|
||||
|
||||
TEST_F(DBTest, GetFromImmutableLayer) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.env = env_;
|
||||
options.write_buffer_size = 100000; // Small write buffer
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
ASSERT_OK(Put(1, "foo", "v1"));
|
||||
@@ -1911,9 +1916,11 @@ TEST_F(DBTest, Recover) {
|
||||
|
||||
TEST_F(DBTest, RecoverWithTableHandle) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.write_buffer_size = 100;
|
||||
options.disable_auto_compactions = true;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
@@ -2361,9 +2368,10 @@ TEST_F(DBTest, CompressedCache) {
|
||||
// Iteration 4: both block cache and compressed cache, but DB is not
|
||||
// compressed
|
||||
for (int iter = 0; iter < 4; iter++) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.write_buffer_size = 64*1024; // small write buffer
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
options = CurrentOptions(options);
|
||||
|
||||
BlockBasedTableOptions table_options;
|
||||
switch (iter) {
|
||||
@@ -2628,9 +2636,10 @@ TEST_F(DBTest, MinLevelToCompress2) {
|
||||
|
||||
TEST_F(DBTest, RepeatedWritesToSameKey) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.env = env_;
|
||||
options.write_buffer_size = 100000; // Small write buffer
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// We must have at most one file per level except for level-0,
|
||||
@@ -2707,10 +2716,11 @@ static bool Between(uint64_t val, uint64_t low, uint64_t high) {
|
||||
}
|
||||
|
||||
TEST_F(DBTest, ApproximateSizesMemTable) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.write_buffer_size = 100000000; // Large write buffer
|
||||
options.compression = kNoCompression;
|
||||
options.create_if_missing = true;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
const int N = 128;
|
||||
@@ -2813,10 +2823,11 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
|
||||
|
||||
TEST_F(DBTest, ApproximateSizes) {
|
||||
do {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.write_buffer_size = 100000000; // Large write buffer
|
||||
options.compression = kNoCompression;
|
||||
options.create_if_missing = true;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
@@ -7832,7 +7843,8 @@ TEST_F(DBTest, MergeTestTime) {
|
||||
this->env_->addon_time_.store(0);
|
||||
this->env_->time_elapse_only_sleep_ = true;
|
||||
this->env_->no_sleep_ = true;
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options = CurrentOptions(options);
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
options.merge_operator.reset(new DelayedMergeOperator(this));
|
||||
DestroyAndReopen(options);
|
||||
@@ -7872,7 +7884,8 @@ TEST_F(DBTest, MergeTestTime) {
|
||||
#ifndef ROCKSDB_LITE
|
||||
TEST_P(DBTestWithParam, MergeCompactionTimeTest) {
|
||||
SetPerfLevel(kEnableTime);
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options = CurrentOptions(options);
|
||||
options.compaction_filter_factory = std::make_shared<KeepFilterFactory>();
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
options.merge_operator.reset(new DelayedMergeOperator(this));
|
||||
@@ -7891,13 +7904,14 @@ TEST_P(DBTestWithParam, MergeCompactionTimeTest) {
|
||||
}
|
||||
|
||||
TEST_P(DBTestWithParam, FilterCompactionTimeTest) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_filter_factory =
|
||||
std::make_shared<DelayFilterFactory>(this);
|
||||
options.disable_auto_compactions = true;
|
||||
options.create_if_missing = true;
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
options.max_subcompactions = max_subcompactions_;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
// put some data
|
||||
@@ -7939,8 +7953,9 @@ TEST_F(DBTest, TestLogCleanup) {
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
TEST_F(DBTest, EmptyCompactedDB) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.max_open_files = -1;
|
||||
options = CurrentOptions(options);
|
||||
Close();
|
||||
ASSERT_OK(ReadOnlyReopen(options));
|
||||
Status s = Put("new", "value");
|
||||
@@ -8176,8 +8191,9 @@ TEST_F(DBTest, AutomaticConflictsWithManualCompaction) {
|
||||
// Github issue #595
|
||||
// Large write batch with column families
|
||||
TEST_F(DBTest, LargeBatchWithColumnFamilies) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.env = env_;
|
||||
options = CurrentOptions(options);
|
||||
options.write_buffer_size = 100000; // Small write buffer
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
int64_t j = 0;
|
||||
@@ -8271,10 +8287,11 @@ TEST_F(DBTest, DelayedWriteRate) {
|
||||
const int kEntriesPerMemTable = 100;
|
||||
const int kTotalFlushes = 20;
|
||||
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
env_->SetBackgroundThreads(1, Env::LOW);
|
||||
options.env = env_;
|
||||
env_->no_sleep_ = true;
|
||||
options = CurrentOptions(options);
|
||||
options.write_buffer_size = 100000000;
|
||||
options.max_write_buffer_number = 256;
|
||||
options.max_background_compactions = 1;
|
||||
@@ -8335,9 +8352,10 @@ TEST_F(DBTest, DelayedWriteRate) {
|
||||
}
|
||||
|
||||
TEST_F(DBTest, HardLimit) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.env = env_;
|
||||
env_->SetBackgroundThreads(1, Env::LOW);
|
||||
options = CurrentOptions(options);
|
||||
options.max_write_buffer_number = 256;
|
||||
options.write_buffer_size = 110 << 10; // 110KB
|
||||
options.arena_block_size = 4 * 1024;
|
||||
@@ -8385,8 +8403,9 @@ TEST_F(DBTest, HardLimit) {
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
TEST_F(DBTest, SoftLimit) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.env = env_;
|
||||
options = CurrentOptions(options);
|
||||
options.write_buffer_size = 100000; // Small write buffer
|
||||
options.max_write_buffer_number = 256;
|
||||
options.level0_file_num_compaction_trigger = 1;
|
||||
@@ -8502,8 +8521,9 @@ TEST_F(DBTest, SoftLimit) {
|
||||
}
|
||||
|
||||
TEST_F(DBTest, LastWriteBufferDelay) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.env = env_;
|
||||
options = CurrentOptions(options);
|
||||
options.write_buffer_size = 100000;
|
||||
options.max_write_buffer_number = 4;
|
||||
options.delayed_write_rate = 20000;
|
||||
@@ -10088,8 +10108,9 @@ INSTANTIATE_TEST_CASE_P(DBTestWithParam, DBTestWithParam,
|
||||
::testing::Bool()));
|
||||
|
||||
TEST_F(DBTest, PauseBackgroundWorkTest) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.write_buffer_size = 100000; // Small write buffer
|
||||
options = CurrentOptions(options);
|
||||
Reopen(options);
|
||||
|
||||
std::vector<std::thread> threads;
|
||||
@@ -10113,6 +10134,403 @@ TEST_F(DBTest, PauseBackgroundWorkTest) {
|
||||
ASSERT_EQ(true, done.load());
|
||||
}
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
namespace {
|
||||
void ValidateKeyExistence(DB* db, const std::vector<Slice>& keys_must_exist,
|
||||
const std::vector<Slice>& keys_must_not_exist) {
|
||||
// Ensure that expected keys exist
|
||||
std::vector<std::string> values;
|
||||
if (keys_must_exist.size() > 0) {
|
||||
std::vector<Status> status_list =
|
||||
db->MultiGet(ReadOptions(), keys_must_exist, &values);
|
||||
for (size_t i = 0; i < keys_must_exist.size(); i++) {
|
||||
ASSERT_OK(status_list[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure that given keys don't exist
|
||||
if (keys_must_not_exist.size() > 0) {
|
||||
std::vector<Status> status_list =
|
||||
db->MultiGet(ReadOptions(), keys_must_not_exist, &values);
|
||||
for (size_t i = 0; i < keys_must_not_exist.size(); i++) {
|
||||
ASSERT_TRUE(status_list[i].IsNotFound());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_F(DBTest, WalFilterTest) {
|
||||
class TestWalFilter : public WalFilter {
|
||||
private:
|
||||
// Processing option that is requested to be applied at the given index
|
||||
WalFilter::WalProcessingOption wal_processing_option_;
|
||||
// Index at which to apply wal_processing_option_
|
||||
// At other indexes default wal_processing_option::kContinueProcessing is
|
||||
// returned.
|
||||
size_t apply_option_at_record_index_;
|
||||
// Current record index, incremented with each record encountered.
|
||||
size_t current_record_index_;
|
||||
|
||||
public:
|
||||
TestWalFilter(WalFilter::WalProcessingOption wal_processing_option,
|
||||
size_t apply_option_for_record_index)
|
||||
: wal_processing_option_(wal_processing_option),
|
||||
apply_option_at_record_index_(apply_option_for_record_index),
|
||||
current_record_index_(0) {}
|
||||
|
||||
virtual WalProcessingOption LogRecord(const WriteBatch& batch,
|
||||
WriteBatch* new_batch,
|
||||
bool* batch_changed) const override {
|
||||
WalFilter::WalProcessingOption option_to_return;
|
||||
|
||||
if (current_record_index_ == apply_option_at_record_index_) {
|
||||
option_to_return = wal_processing_option_;
|
||||
} else {
|
||||
option_to_return = WalProcessingOption::kContinueProcessing;
|
||||
}
|
||||
|
||||
// Filter is passed as a const object for RocksDB to not modify the
|
||||
// object, however we modify it for our own purpose here and hence
|
||||
// cast the constness away.
|
||||
(const_cast<TestWalFilter*>(this)->current_record_index_)++;
|
||||
|
||||
return option_to_return;
|
||||
}
|
||||
|
||||
virtual const char* Name() const override { return "TestWalFilter"; }
|
||||
};
|
||||
|
||||
// Create 3 batches with two keys each
|
||||
std::vector<std::vector<std::string>> batch_keys(3);
|
||||
|
||||
batch_keys[0].push_back("key1");
|
||||
batch_keys[0].push_back("key2");
|
||||
batch_keys[1].push_back("key3");
|
||||
batch_keys[1].push_back("key4");
|
||||
batch_keys[2].push_back("key5");
|
||||
batch_keys[2].push_back("key6");
|
||||
|
||||
// Test with all WAL processing options
|
||||
for (int option = 0;
|
||||
option < static_cast<int>(
|
||||
WalFilter::WalProcessingOption::kWalProcessingOptionMax);
|
||||
option++) {
|
||||
Options options = OptionsForLogIterTest();
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// Write given keys in given batches
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
WriteBatch batch;
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
|
||||
}
|
||||
dbfull()->Write(WriteOptions(), &batch);
|
||||
}
|
||||
|
||||
WalFilter::WalProcessingOption wal_processing_option =
|
||||
static_cast<WalFilter::WalProcessingOption>(option);
|
||||
|
||||
// Create a test filter that would apply wal_processing_option at the first
|
||||
// record
|
||||
size_t apply_option_for_record_index = 1;
|
||||
TestWalFilter test_wal_filter(wal_processing_option,
|
||||
apply_option_for_record_index);
|
||||
|
||||
// Reopen database with option to use WAL filter
|
||||
options = OptionsForLogIterTest();
|
||||
options.wal_filter = &test_wal_filter;
|
||||
Status status =
|
||||
TryReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||
if (wal_processing_option ==
|
||||
WalFilter::WalProcessingOption::kCorruptedRecord) {
|
||||
assert(!status.ok());
|
||||
// In case of corruption we can turn off paranoid_checks to reopen
|
||||
// databse
|
||||
options.paranoid_checks = false;
|
||||
ReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||
} else {
|
||||
assert(status.ok());
|
||||
}
|
||||
|
||||
// Compute which keys we expect to be found
|
||||
// and which we expect not to be found after recovery.
|
||||
std::vector<Slice> keys_must_exist;
|
||||
std::vector<Slice> keys_must_not_exist;
|
||||
switch (wal_processing_option) {
|
||||
case WalFilter::WalProcessingOption::kCorruptedRecord:
|
||||
case WalFilter::WalProcessingOption::kContinueProcessing: {
|
||||
fprintf(stderr, "Testing with complete WAL processing\n");
|
||||
// we expect all records to be processed
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: {
|
||||
fprintf(stderr,
|
||||
"Testing with ignoring record %" ROCKSDB_PRIszt " only\n",
|
||||
apply_option_for_record_index);
|
||||
// We expect the record with apply_option_for_record_index to be not
|
||||
// found.
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
if (i == apply_option_for_record_index) {
|
||||
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
||||
} else {
|
||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case WalFilter::WalProcessingOption::kStopReplay: {
|
||||
fprintf(stderr,
|
||||
"Testing with stopping replay from record %" ROCKSDB_PRIszt
|
||||
"\n",
|
||||
apply_option_for_record_index);
|
||||
// We expect records beyond apply_option_for_record_index to be not
|
||||
// found.
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
if (i >= apply_option_for_record_index) {
|
||||
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
||||
} else {
|
||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
assert(false); // unhandled case
|
||||
}
|
||||
|
||||
bool checked_after_reopen = false;
|
||||
|
||||
while (true) {
|
||||
// Ensure that expected keys exists
|
||||
// and not expected keys don't exist after recovery
|
||||
ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
|
||||
|
||||
if (checked_after_reopen) {
|
||||
break;
|
||||
}
|
||||
|
||||
// reopen database again to make sure previous log(s) are not used
|
||||
//(even if they were skipped)
|
||||
// reopn database with option to use WAL filter
|
||||
options = OptionsForLogIterTest();
|
||||
ReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||
|
||||
checked_after_reopen = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DBTest, WalFilterTestWithChangeBatch) {
|
||||
class ChangeBatchHandler : public WriteBatch::Handler {
|
||||
private:
|
||||
// Batch to insert keys in
|
||||
WriteBatch* new_write_batch_;
|
||||
// Number of keys to add in the new batch
|
||||
size_t num_keys_to_add_in_new_batch_;
|
||||
// Number of keys added to new batch
|
||||
size_t num_keys_added_;
|
||||
|
||||
public:
|
||||
ChangeBatchHandler(WriteBatch* new_write_batch,
|
||||
size_t num_keys_to_add_in_new_batch)
|
||||
: new_write_batch_(new_write_batch),
|
||||
num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
|
||||
num_keys_added_(0) {}
|
||||
virtual void Put(const Slice& key, const Slice& value) override {
|
||||
if (num_keys_added_ < num_keys_to_add_in_new_batch_) {
|
||||
new_write_batch_->Put(key, value);
|
||||
++num_keys_added_;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class TestWalFilterWithChangeBatch : public WalFilter {
|
||||
private:
|
||||
// Index at which to start changing records
|
||||
size_t change_records_from_index_;
|
||||
// Number of keys to add in the new batch
|
||||
size_t num_keys_to_add_in_new_batch_;
|
||||
// Current record index, incremented with each record encountered.
|
||||
size_t current_record_index_;
|
||||
|
||||
public:
|
||||
TestWalFilterWithChangeBatch(size_t change_records_from_index,
|
||||
size_t num_keys_to_add_in_new_batch)
|
||||
: change_records_from_index_(change_records_from_index),
|
||||
num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
|
||||
current_record_index_(0) {}
|
||||
|
||||
virtual WalProcessingOption LogRecord(const WriteBatch& batch,
|
||||
WriteBatch* new_batch,
|
||||
bool* batch_changed) const override {
|
||||
if (current_record_index_ >= change_records_from_index_) {
|
||||
ChangeBatchHandler handler(new_batch, num_keys_to_add_in_new_batch_);
|
||||
batch.Iterate(&handler);
|
||||
*batch_changed = true;
|
||||
}
|
||||
|
||||
// Filter is passed as a const object for RocksDB to not modify the
|
||||
// object, however we modify it for our own purpose here and hence
|
||||
// cast the constness away.
|
||||
(const_cast<TestWalFilterWithChangeBatch*>(this)
|
||||
->current_record_index_)++;
|
||||
|
||||
return WalProcessingOption::kContinueProcessing;
|
||||
}
|
||||
|
||||
virtual const char* Name() const override {
|
||||
return "TestWalFilterWithChangeBatch";
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<std::vector<std::string>> batch_keys(3);
|
||||
|
||||
batch_keys[0].push_back("key1");
|
||||
batch_keys[0].push_back("key2");
|
||||
batch_keys[1].push_back("key3");
|
||||
batch_keys[1].push_back("key4");
|
||||
batch_keys[2].push_back("key5");
|
||||
batch_keys[2].push_back("key6");
|
||||
|
||||
Options options = OptionsForLogIterTest();
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// Write given keys in given batches
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
WriteBatch batch;
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
|
||||
}
|
||||
dbfull()->Write(WriteOptions(), &batch);
|
||||
}
|
||||
|
||||
// Create a test filter that would apply wal_processing_option at the first
|
||||
// record
|
||||
size_t change_records_from_index = 1;
|
||||
size_t num_keys_to_add_in_new_batch = 1;
|
||||
TestWalFilterWithChangeBatch test_wal_filter_with_change_batch(
|
||||
change_records_from_index, num_keys_to_add_in_new_batch);
|
||||
|
||||
// Reopen database with option to use WAL filter
|
||||
options = OptionsForLogIterTest();
|
||||
options.wal_filter = &test_wal_filter_with_change_batch;
|
||||
ReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||
|
||||
// Ensure that all keys exist before change_records_from_index_
|
||||
// And after that index only single key exists
|
||||
// as our filter adds only single key for each batch
|
||||
std::vector<Slice> keys_must_exist;
|
||||
std::vector<Slice> keys_must_not_exist;
|
||||
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
if (i >= change_records_from_index && j >= num_keys_to_add_in_new_batch) {
|
||||
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
||||
} else {
|
||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool checked_after_reopen = false;
|
||||
|
||||
while (true) {
|
||||
// Ensure that expected keys exists
|
||||
// and not expected keys don't exist after recovery
|
||||
ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
|
||||
|
||||
if (checked_after_reopen) {
|
||||
break;
|
||||
}
|
||||
|
||||
// reopen database again to make sure previous log(s) are not used
|
||||
//(even if they were skipped)
|
||||
// reopn database with option to use WAL filter
|
||||
options = OptionsForLogIterTest();
|
||||
ReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||
|
||||
checked_after_reopen = true;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DBTest, WalFilterTestWithChangeBatchExtraKeys) {
|
||||
class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
|
||||
public:
|
||||
virtual WalProcessingOption LogRecord(const WriteBatch& batch,
|
||||
WriteBatch* new_batch,
|
||||
bool* batch_changed) const override {
|
||||
*new_batch = batch;
|
||||
new_batch->Put("key_extra", "value_extra");
|
||||
*batch_changed = true;
|
||||
return WalProcessingOption::kContinueProcessing;
|
||||
}
|
||||
|
||||
virtual const char* Name() const override {
|
||||
return "WalFilterTestWithChangeBatchExtraKeys";
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<std::vector<std::string>> batch_keys(3);
|
||||
|
||||
batch_keys[0].push_back("key1");
|
||||
batch_keys[0].push_back("key2");
|
||||
batch_keys[1].push_back("key3");
|
||||
batch_keys[1].push_back("key4");
|
||||
batch_keys[2].push_back("key5");
|
||||
batch_keys[2].push_back("key6");
|
||||
|
||||
Options options = OptionsForLogIterTest();
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// Write given keys in given batches
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
WriteBatch batch;
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
|
||||
}
|
||||
dbfull()->Write(WriteOptions(), &batch);
|
||||
}
|
||||
|
||||
// Create a test filter that would add extra keys
|
||||
TestWalFilterWithChangeBatchAddExtraKeys test_wal_filter_extra_keys;
|
||||
|
||||
// Reopen database with option to use WAL filter
|
||||
options = OptionsForLogIterTest();
|
||||
options.wal_filter = &test_wal_filter_extra_keys;
|
||||
Status status = TryReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||
ASSERT_TRUE(status.IsNotSupported());
|
||||
|
||||
// Reopen without filter, now reopen should succeed - previous
|
||||
// attempt to open must not have altered the db.
|
||||
options = OptionsForLogIterTest();
|
||||
ReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||
|
||||
std::vector<Slice> keys_must_exist;
|
||||
std::vector<Slice> keys_must_not_exist; // empty vector
|
||||
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||
}
|
||||
}
|
||||
|
||||
ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
|
||||
}
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
class SliceTransformLimitedDomain : public SliceTransform {
|
||||
const char* Name() const override { return "SliceTransformLimitedDomain"; }
|
||||
|
||||
|
||||
-797
@@ -9,82 +9,14 @@
|
||||
#include <cstdlib>
|
||||
#include "db/db_test_util.h"
|
||||
#include "port/stack_trace.h"
|
||||
#include "rocksdb/wal_filter.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
static uint64_t TestGetTickerCount(const Options& options,
|
||||
Tickers ticker_type) {
|
||||
return options.statistics->getTickerCount(ticker_type);
|
||||
}
|
||||
|
||||
class DBTest2 : public DBTestBase {
|
||||
public:
|
||||
DBTest2() : DBTestBase("/db_test2") {}
|
||||
};
|
||||
|
||||
class PrefixFullBloomWithReverseComparator
|
||||
: public DBTestBase,
|
||||
public ::testing::WithParamInterface<bool> {
|
||||
public:
|
||||
PrefixFullBloomWithReverseComparator()
|
||||
: DBTestBase("/prefix_bloom_reverse") {}
|
||||
virtual void SetUp() override { if_cache_filter_ = GetParam(); }
|
||||
bool if_cache_filter_;
|
||||
};
|
||||
|
||||
TEST_P(PrefixFullBloomWithReverseComparator,
|
||||
PrefixFullBloomWithReverseComparator) {
|
||||
Options options = last_options_;
|
||||
options.comparator = ReverseBytewiseComparator();
|
||||
options.prefix_extractor.reset(NewCappedPrefixTransform(3));
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
BlockBasedTableOptions bbto;
|
||||
if (if_cache_filter_) {
|
||||
bbto.no_block_cache = false;
|
||||
bbto.cache_index_and_filter_blocks = true;
|
||||
bbto.block_cache = NewLRUCache(1);
|
||||
}
|
||||
bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
|
||||
bbto.whole_key_filtering = false;
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
||||
DestroyAndReopen(options);
|
||||
|
||||
ASSERT_OK(dbfull()->Put(WriteOptions(), "bar123", "foo"));
|
||||
ASSERT_OK(dbfull()->Put(WriteOptions(), "bar234", "foo2"));
|
||||
ASSERT_OK(dbfull()->Put(WriteOptions(), "foo123", "foo3"));
|
||||
|
||||
dbfull()->Flush(FlushOptions());
|
||||
|
||||
if (bbto.block_cache) {
|
||||
bbto.block_cache->EraseUnRefEntries();
|
||||
}
|
||||
|
||||
unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
|
||||
iter->Seek("bar345");
|
||||
ASSERT_OK(iter->status());
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("bar234", iter->key().ToString());
|
||||
ASSERT_EQ("foo2", iter->value().ToString());
|
||||
iter->Next();
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("bar123", iter->key().ToString());
|
||||
ASSERT_EQ("foo", iter->value().ToString());
|
||||
|
||||
iter->Seek("foo234");
|
||||
ASSERT_OK(iter->status());
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("foo123", iter->key().ToString());
|
||||
ASSERT_EQ("foo3", iter->value().ToString());
|
||||
|
||||
iter->Seek("bar");
|
||||
ASSERT_OK(iter->status());
|
||||
ASSERT_TRUE(!iter->Valid());
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(PrefixFullBloomWithReverseComparator,
|
||||
PrefixFullBloomWithReverseComparator, testing::Bool());
|
||||
|
||||
TEST_F(DBTest2, IteratorPropertyVersionNumber) {
|
||||
Put("", "");
|
||||
Iterator* iter1 = db_->NewIterator(ReadOptions());
|
||||
@@ -126,735 +58,6 @@ TEST_F(DBTest2, IteratorPropertyVersionNumber) {
|
||||
delete iter2;
|
||||
delete iter3;
|
||||
}
|
||||
|
||||
TEST_F(DBTest2, CacheIndexAndFilterWithDBRestart) {
|
||||
Options options = CurrentOptions();
|
||||
options.create_if_missing = true;
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
BlockBasedTableOptions table_options;
|
||||
table_options.cache_index_and_filter_blocks = true;
|
||||
table_options.filter_policy.reset(NewBloomFilterPolicy(20));
|
||||
options.table_factory.reset(new BlockBasedTableFactory(table_options));
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
Put(1, "a", "begin");
|
||||
Put(1, "z", "end");
|
||||
ASSERT_OK(Flush(1));
|
||||
TryReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||
|
||||
std::string value;
|
||||
value = Get(1, "a");
|
||||
}
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
namespace {
|
||||
void ValidateKeyExistence(DB* db, const std::vector<Slice>& keys_must_exist,
|
||||
const std::vector<Slice>& keys_must_not_exist) {
|
||||
// Ensure that expected keys exist
|
||||
std::vector<std::string> values;
|
||||
if (keys_must_exist.size() > 0) {
|
||||
std::vector<Status> status_list =
|
||||
db->MultiGet(ReadOptions(), keys_must_exist, &values);
|
||||
for (size_t i = 0; i < keys_must_exist.size(); i++) {
|
||||
ASSERT_OK(status_list[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure that given keys don't exist
|
||||
if (keys_must_not_exist.size() > 0) {
|
||||
std::vector<Status> status_list =
|
||||
db->MultiGet(ReadOptions(), keys_must_not_exist, &values);
|
||||
for (size_t i = 0; i < keys_must_not_exist.size(); i++) {
|
||||
ASSERT_TRUE(status_list[i].IsNotFound());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_F(DBTest2, WalFilterTest) {
|
||||
class TestWalFilter : public WalFilter {
|
||||
private:
|
||||
// Processing option that is requested to be applied at the given index
|
||||
WalFilter::WalProcessingOption wal_processing_option_;
|
||||
// Index at which to apply wal_processing_option_
|
||||
// At other indexes default wal_processing_option::kContinueProcessing is
|
||||
// returned.
|
||||
size_t apply_option_at_record_index_;
|
||||
// Current record index, incremented with each record encountered.
|
||||
size_t current_record_index_;
|
||||
|
||||
public:
|
||||
TestWalFilter(WalFilter::WalProcessingOption wal_processing_option,
|
||||
size_t apply_option_for_record_index)
|
||||
: wal_processing_option_(wal_processing_option),
|
||||
apply_option_at_record_index_(apply_option_for_record_index),
|
||||
current_record_index_(0) {}
|
||||
|
||||
virtual WalProcessingOption LogRecord(const WriteBatch& batch,
|
||||
WriteBatch* new_batch,
|
||||
bool* batch_changed) const override {
|
||||
WalFilter::WalProcessingOption option_to_return;
|
||||
|
||||
if (current_record_index_ == apply_option_at_record_index_) {
|
||||
option_to_return = wal_processing_option_;
|
||||
}
|
||||
else {
|
||||
option_to_return = WalProcessingOption::kContinueProcessing;
|
||||
}
|
||||
|
||||
// Filter is passed as a const object for RocksDB to not modify the
|
||||
// object, however we modify it for our own purpose here and hence
|
||||
// cast the constness away.
|
||||
(const_cast<TestWalFilter*>(this)->current_record_index_)++;
|
||||
|
||||
return option_to_return;
|
||||
}
|
||||
|
||||
virtual const char* Name() const override { return "TestWalFilter"; }
|
||||
};
|
||||
|
||||
// Create 3 batches with two keys each
|
||||
std::vector<std::vector<std::string>> batch_keys(3);
|
||||
|
||||
batch_keys[0].push_back("key1");
|
||||
batch_keys[0].push_back("key2");
|
||||
batch_keys[1].push_back("key3");
|
||||
batch_keys[1].push_back("key4");
|
||||
batch_keys[2].push_back("key5");
|
||||
batch_keys[2].push_back("key6");
|
||||
|
||||
// Test with all WAL processing options
|
||||
for (int option = 0;
|
||||
option < static_cast<int>(
|
||||
WalFilter::WalProcessingOption::kWalProcessingOptionMax);
|
||||
option++) {
|
||||
Options options = OptionsForLogIterTest();
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({ "pikachu" }, options);
|
||||
|
||||
// Write given keys in given batches
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
WriteBatch batch;
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
|
||||
}
|
||||
dbfull()->Write(WriteOptions(), &batch);
|
||||
}
|
||||
|
||||
WalFilter::WalProcessingOption wal_processing_option =
|
||||
static_cast<WalFilter::WalProcessingOption>(option);
|
||||
|
||||
// Create a test filter that would apply wal_processing_option at the first
|
||||
// record
|
||||
size_t apply_option_for_record_index = 1;
|
||||
TestWalFilter test_wal_filter(wal_processing_option,
|
||||
apply_option_for_record_index);
|
||||
|
||||
// Reopen database with option to use WAL filter
|
||||
options = OptionsForLogIterTest();
|
||||
options.wal_filter = &test_wal_filter;
|
||||
Status status =
|
||||
TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
||||
if (wal_processing_option ==
|
||||
WalFilter::WalProcessingOption::kCorruptedRecord) {
|
||||
assert(!status.ok());
|
||||
// In case of corruption we can turn off paranoid_checks to reopen
|
||||
// databse
|
||||
options.paranoid_checks = false;
|
||||
ReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
||||
}
|
||||
else {
|
||||
assert(status.ok());
|
||||
}
|
||||
|
||||
// Compute which keys we expect to be found
|
||||
// and which we expect not to be found after recovery.
|
||||
std::vector<Slice> keys_must_exist;
|
||||
std::vector<Slice> keys_must_not_exist;
|
||||
switch (wal_processing_option) {
|
||||
case WalFilter::WalProcessingOption::kCorruptedRecord:
|
||||
case WalFilter::WalProcessingOption::kContinueProcessing: {
|
||||
fprintf(stderr, "Testing with complete WAL processing\n");
|
||||
// we expect all records to be processed
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: {
|
||||
fprintf(stderr,
|
||||
"Testing with ignoring record %" ROCKSDB_PRIszt " only\n",
|
||||
apply_option_for_record_index);
|
||||
// We expect the record with apply_option_for_record_index to be not
|
||||
// found.
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
if (i == apply_option_for_record_index) {
|
||||
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
||||
}
|
||||
else {
|
||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case WalFilter::WalProcessingOption::kStopReplay: {
|
||||
fprintf(stderr,
|
||||
"Testing with stopping replay from record %" ROCKSDB_PRIszt
|
||||
"\n",
|
||||
apply_option_for_record_index);
|
||||
// We expect records beyond apply_option_for_record_index to be not
|
||||
// found.
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
if (i >= apply_option_for_record_index) {
|
||||
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
||||
}
|
||||
else {
|
||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
assert(false); // unhandled case
|
||||
}
|
||||
|
||||
bool checked_after_reopen = false;
|
||||
|
||||
while (true) {
|
||||
// Ensure that expected keys exists
|
||||
// and not expected keys don't exist after recovery
|
||||
ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
|
||||
|
||||
if (checked_after_reopen) {
|
||||
break;
|
||||
}
|
||||
|
||||
// reopen database again to make sure previous log(s) are not used
|
||||
//(even if they were skipped)
|
||||
// reopn database with option to use WAL filter
|
||||
options = OptionsForLogIterTest();
|
||||
ReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
||||
|
||||
checked_after_reopen = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
|
||||
class ChangeBatchHandler : public WriteBatch::Handler {
|
||||
private:
|
||||
// Batch to insert keys in
|
||||
WriteBatch* new_write_batch_;
|
||||
// Number of keys to add in the new batch
|
||||
size_t num_keys_to_add_in_new_batch_;
|
||||
// Number of keys added to new batch
|
||||
size_t num_keys_added_;
|
||||
|
||||
public:
|
||||
ChangeBatchHandler(WriteBatch* new_write_batch,
|
||||
size_t num_keys_to_add_in_new_batch)
|
||||
: new_write_batch_(new_write_batch),
|
||||
num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
|
||||
num_keys_added_(0) {}
|
||||
virtual void Put(const Slice& key, const Slice& value) override {
|
||||
if (num_keys_added_ < num_keys_to_add_in_new_batch_) {
|
||||
new_write_batch_->Put(key, value);
|
||||
++num_keys_added_;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class TestWalFilterWithChangeBatch : public WalFilter {
|
||||
private:
|
||||
// Index at which to start changing records
|
||||
size_t change_records_from_index_;
|
||||
// Number of keys to add in the new batch
|
||||
size_t num_keys_to_add_in_new_batch_;
|
||||
// Current record index, incremented with each record encountered.
|
||||
size_t current_record_index_;
|
||||
|
||||
public:
|
||||
TestWalFilterWithChangeBatch(size_t change_records_from_index,
|
||||
size_t num_keys_to_add_in_new_batch)
|
||||
: change_records_from_index_(change_records_from_index),
|
||||
num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
|
||||
current_record_index_(0) {}
|
||||
|
||||
virtual WalProcessingOption LogRecord(const WriteBatch& batch,
|
||||
WriteBatch* new_batch,
|
||||
bool* batch_changed) const override {
|
||||
if (current_record_index_ >= change_records_from_index_) {
|
||||
ChangeBatchHandler handler(new_batch, num_keys_to_add_in_new_batch_);
|
||||
batch.Iterate(&handler);
|
||||
*batch_changed = true;
|
||||
}
|
||||
|
||||
// Filter is passed as a const object for RocksDB to not modify the
|
||||
// object, however we modify it for our own purpose here and hence
|
||||
// cast the constness away.
|
||||
(const_cast<TestWalFilterWithChangeBatch*>(this)
|
||||
->current_record_index_)++;
|
||||
|
||||
return WalProcessingOption::kContinueProcessing;
|
||||
}
|
||||
|
||||
virtual const char* Name() const override {
|
||||
return "TestWalFilterWithChangeBatch";
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<std::vector<std::string>> batch_keys(3);
|
||||
|
||||
batch_keys[0].push_back("key1");
|
||||
batch_keys[0].push_back("key2");
|
||||
batch_keys[1].push_back("key3");
|
||||
batch_keys[1].push_back("key4");
|
||||
batch_keys[2].push_back("key5");
|
||||
batch_keys[2].push_back("key6");
|
||||
|
||||
Options options = OptionsForLogIterTest();
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({ "pikachu" }, options);
|
||||
|
||||
// Write given keys in given batches
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
WriteBatch batch;
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
|
||||
}
|
||||
dbfull()->Write(WriteOptions(), &batch);
|
||||
}
|
||||
|
||||
// Create a test filter that would apply wal_processing_option at the first
|
||||
// record
|
||||
size_t change_records_from_index = 1;
|
||||
size_t num_keys_to_add_in_new_batch = 1;
|
||||
TestWalFilterWithChangeBatch test_wal_filter_with_change_batch(
|
||||
change_records_from_index, num_keys_to_add_in_new_batch);
|
||||
|
||||
// Reopen database with option to use WAL filter
|
||||
options = OptionsForLogIterTest();
|
||||
options.wal_filter = &test_wal_filter_with_change_batch;
|
||||
ReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
||||
|
||||
// Ensure that all keys exist before change_records_from_index_
|
||||
// And after that index only single key exists
|
||||
// as our filter adds only single key for each batch
|
||||
std::vector<Slice> keys_must_exist;
|
||||
std::vector<Slice> keys_must_not_exist;
|
||||
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
if (i >= change_records_from_index && j >=
|
||||
num_keys_to_add_in_new_batch) {
|
||||
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
|
||||
}
|
||||
else {
|
||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool checked_after_reopen = false;
|
||||
|
||||
while (true) {
|
||||
// Ensure that expected keys exists
|
||||
// and not expected keys don't exist after recovery
|
||||
ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
|
||||
|
||||
if (checked_after_reopen) {
|
||||
break;
|
||||
}
|
||||
|
||||
// reopen database again to make sure previous log(s) are not used
|
||||
//(even if they were skipped)
|
||||
// reopn database with option to use WAL filter
|
||||
options = OptionsForLogIterTest();
|
||||
ReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
||||
|
||||
checked_after_reopen = true;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
|
||||
class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
|
||||
public:
|
||||
virtual WalProcessingOption LogRecord(const WriteBatch& batch,
|
||||
WriteBatch* new_batch,
|
||||
bool* batch_changed) const override {
|
||||
*new_batch = batch;
|
||||
new_batch->Put("key_extra", "value_extra");
|
||||
*batch_changed = true;
|
||||
return WalProcessingOption::kContinueProcessing;
|
||||
}
|
||||
|
||||
virtual const char* Name() const override {
|
||||
return "WalFilterTestWithChangeBatchExtraKeys";
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<std::vector<std::string>> batch_keys(3);
|
||||
|
||||
batch_keys[0].push_back("key1");
|
||||
batch_keys[0].push_back("key2");
|
||||
batch_keys[1].push_back("key3");
|
||||
batch_keys[1].push_back("key4");
|
||||
batch_keys[2].push_back("key5");
|
||||
batch_keys[2].push_back("key6");
|
||||
|
||||
Options options = OptionsForLogIterTest();
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({ "pikachu" }, options);
|
||||
|
||||
// Write given keys in given batches
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
WriteBatch batch;
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
|
||||
}
|
||||
dbfull()->Write(WriteOptions(), &batch);
|
||||
}
|
||||
|
||||
// Create a test filter that would add extra keys
|
||||
TestWalFilterWithChangeBatchAddExtraKeys test_wal_filter_extra_keys;
|
||||
|
||||
// Reopen database with option to use WAL filter
|
||||
options = OptionsForLogIterTest();
|
||||
options.wal_filter = &test_wal_filter_extra_keys;
|
||||
Status status = TryReopenWithColumnFamilies({ "default", "pikachu" },
|
||||
options);
|
||||
ASSERT_TRUE(status.IsNotSupported());
|
||||
|
||||
// Reopen without filter, now reopen should succeed - previous
|
||||
// attempt to open must not have altered the db.
|
||||
options = OptionsForLogIterTest();
|
||||
ReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
||||
|
||||
std::vector<Slice> keys_must_exist;
|
||||
std::vector<Slice> keys_must_not_exist; // empty vector
|
||||
|
||||
for (size_t i = 0; i < batch_keys.size(); i++) {
|
||||
for (size_t j = 0; j < batch_keys[i].size(); j++) {
|
||||
keys_must_exist.push_back(Slice(batch_keys[i][j]));
|
||||
}
|
||||
}
|
||||
|
||||
ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
|
||||
}
|
||||
|
||||
TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
|
||||
class TestWalFilterWithColumnFamilies : public WalFilter {
|
||||
private:
|
||||
// column_family_id -> log_number map (provided to WALFilter)
|
||||
std::map<uint32_t, uint64_t> cf_log_number_map_;
|
||||
// column_family_name -> column_family_id map (provided to WALFilter)
|
||||
std::map<std::string, uint32_t> cf_name_id_map_;
|
||||
// column_family_name -> keys_found_in_wal map
|
||||
// We store keys that are applicable to the column_family
|
||||
// during recovery (i.e. aren't already flushed to SST file(s))
|
||||
// for verification against the keys we expect.
|
||||
std::map<uint32_t, std::vector<std::string>> cf_wal_keys_;
|
||||
public:
|
||||
virtual void ColumnFamilyLogNumberMap(
|
||||
const std::map<uint32_t, uint64_t>& cf_lognumber_map,
|
||||
const std::map<std::string, uint32_t>& cf_name_id_map) override {
|
||||
cf_log_number_map_ = cf_lognumber_map;
|
||||
cf_name_id_map_ = cf_name_id_map;
|
||||
}
|
||||
|
||||
virtual WalProcessingOption LogRecordFound(unsigned long long log_number,
|
||||
const std::string& log_file_name,
|
||||
const WriteBatch& batch,
|
||||
WriteBatch* new_batch,
|
||||
bool* batch_changed) override {
|
||||
class LogRecordBatchHandler : public WriteBatch::Handler {
|
||||
private:
|
||||
const std::map<uint32_t, uint64_t> & cf_log_number_map_;
|
||||
std::map<uint32_t, std::vector<std::string>> & cf_wal_keys_;
|
||||
unsigned long long log_number_;
|
||||
public:
|
||||
LogRecordBatchHandler(unsigned long long current_log_number,
|
||||
const std::map<uint32_t, uint64_t> & cf_log_number_map,
|
||||
std::map<uint32_t, std::vector<std::string>> & cf_wal_keys) :
|
||||
cf_log_number_map_(cf_log_number_map),
|
||||
cf_wal_keys_(cf_wal_keys),
|
||||
log_number_(current_log_number){}
|
||||
|
||||
virtual Status PutCF(uint32_t column_family_id, const Slice& key,
|
||||
const Slice& /*value*/) override {
|
||||
auto it = cf_log_number_map_.find(column_family_id);
|
||||
assert(it != cf_log_number_map_.end());
|
||||
unsigned long long log_number_for_cf = it->second;
|
||||
// If the current record is applicable for column_family_id
|
||||
// (i.e. isn't flushed to SST file(s) for column_family_id)
|
||||
// add it to the cf_wal_keys_ map for verification.
|
||||
if (log_number_ >= log_number_for_cf) {
|
||||
cf_wal_keys_[column_family_id].push_back(std::string(key.data(),
|
||||
key.size()));
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
} handler(log_number, cf_log_number_map_, cf_wal_keys_);
|
||||
|
||||
batch.Iterate(&handler);
|
||||
|
||||
return WalProcessingOption::kContinueProcessing;
|
||||
}
|
||||
|
||||
virtual const char* Name() const override {
|
||||
return "WalFilterTestWithColumnFamilies";
|
||||
}
|
||||
|
||||
const std::map<uint32_t, std::vector<std::string>> &
|
||||
GetColumnFamilyKeys() {
|
||||
return cf_wal_keys_;
|
||||
}
|
||||
|
||||
const std::map<std::string, uint32_t> & GetColumnFamilyNameIdMap() {
|
||||
return cf_name_id_map_;
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<std::vector<std::string>> batch_keys_pre_flush(3);
|
||||
|
||||
batch_keys_pre_flush[0].push_back("key1");
|
||||
batch_keys_pre_flush[0].push_back("key2");
|
||||
batch_keys_pre_flush[1].push_back("key3");
|
||||
batch_keys_pre_flush[1].push_back("key4");
|
||||
batch_keys_pre_flush[2].push_back("key5");
|
||||
batch_keys_pre_flush[2].push_back("key6");
|
||||
|
||||
Options options = OptionsForLogIterTest();
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({ "pikachu" }, options);
|
||||
|
||||
// Write given keys in given batches
|
||||
for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
|
||||
WriteBatch batch;
|
||||
for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) {
|
||||
batch.Put(handles_[0], batch_keys_pre_flush[i][j], DummyString(1024));
|
||||
batch.Put(handles_[1], batch_keys_pre_flush[i][j], DummyString(1024));
|
||||
}
|
||||
dbfull()->Write(WriteOptions(), &batch);
|
||||
}
|
||||
|
||||
//Flush default column-family
|
||||
db_->Flush(FlushOptions(), handles_[0]);
|
||||
|
||||
// Do some more writes
|
||||
std::vector<std::vector<std::string>> batch_keys_post_flush(3);
|
||||
|
||||
batch_keys_post_flush[0].push_back("key7");
|
||||
batch_keys_post_flush[0].push_back("key8");
|
||||
batch_keys_post_flush[1].push_back("key9");
|
||||
batch_keys_post_flush[1].push_back("key10");
|
||||
batch_keys_post_flush[2].push_back("key11");
|
||||
batch_keys_post_flush[2].push_back("key12");
|
||||
|
||||
// Write given keys in given batches
|
||||
for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
|
||||
WriteBatch batch;
|
||||
for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
|
||||
batch.Put(handles_[0], batch_keys_post_flush[i][j], DummyString(1024));
|
||||
batch.Put(handles_[1], batch_keys_post_flush[i][j], DummyString(1024));
|
||||
}
|
||||
dbfull()->Write(WriteOptions(), &batch);
|
||||
}
|
||||
|
||||
// On Recovery we should only find the second batch applicable to default CF
|
||||
// But both batches applicable to pikachu CF
|
||||
|
||||
// Create a test filter that would add extra keys
|
||||
TestWalFilterWithColumnFamilies test_wal_filter_column_families;
|
||||
|
||||
// Reopen database with option to use WAL filter
|
||||
options = OptionsForLogIterTest();
|
||||
options.wal_filter = &test_wal_filter_column_families;
|
||||
Status status =
|
||||
TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
|
||||
ASSERT_TRUE(status.ok());
|
||||
|
||||
// verify that handles_[0] only has post_flush keys
|
||||
// while handles_[1] has pre and post flush keys
|
||||
auto cf_wal_keys = test_wal_filter_column_families.GetColumnFamilyKeys();
|
||||
auto name_id_map =
|
||||
test_wal_filter_column_families.GetColumnFamilyNameIdMap();
|
||||
size_t index = 0;
|
||||
auto keys_cf = cf_wal_keys[name_id_map[kDefaultColumnFamilyName]];
|
||||
//default column-family, only post_flush keys are expected
|
||||
for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
|
||||
for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
|
||||
Slice key_from_the_log(keys_cf[index++]);
|
||||
Slice batch_key(batch_keys_post_flush[i][j]);
|
||||
ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
|
||||
}
|
||||
}
|
||||
ASSERT_TRUE(index == keys_cf.size());
|
||||
|
||||
index = 0;
|
||||
keys_cf = cf_wal_keys[name_id_map["pikachu"]];
|
||||
//pikachu column-family, all keys are expected
|
||||
for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
|
||||
for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) {
|
||||
Slice key_from_the_log(keys_cf[index++]);
|
||||
Slice batch_key(batch_keys_pre_flush[i][j]);
|
||||
ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
|
||||
for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
|
||||
Slice key_from_the_log(keys_cf[index++]);
|
||||
Slice batch_key(batch_keys_post_flush[i][j]);
|
||||
ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
|
||||
}
|
||||
}
|
||||
ASSERT_TRUE(index == keys_cf.size());
|
||||
}
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
TEST_F(DBTest2, DISABLED_FirstSnapshotTest) {
|
||||
Options options;
|
||||
options.write_buffer_size = 100000; // Small write buffer
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// This snapshot will have sequence number 0. When compaction encounters
|
||||
// this snapshot, CompactionIterator::findEarliestVisibleSnapshot() will
|
||||
// assert as it expects non-zero snapshots.
|
||||
//
|
||||
// One fix would be to simply remove this assert. However, a better fix
|
||||
// might
|
||||
// be to always have db sequence numbers start from 1 so that no code is
|
||||
// ever
|
||||
// confused by 0.
|
||||
const Snapshot* s1 = db_->GetSnapshot();
|
||||
|
||||
Put(1, "k1", std::string(100000, 'x')); // Fill memtable
|
||||
Put(1, "k2", std::string(100000, 'y')); // Trigger flush
|
||||
|
||||
db_->ReleaseSnapshot(s1);
|
||||
}
|
||||
|
||||
class PinL0IndexAndFilterBlocksTest : public DBTestBase,
|
||||
public testing::WithParamInterface<bool> {
|
||||
public:
|
||||
PinL0IndexAndFilterBlocksTest() : DBTestBase("/db_pin_l0_index_bloom_test") {}
|
||||
virtual void SetUp() override { infinite_max_files_ = GetParam(); }
|
||||
|
||||
bool infinite_max_files_;
|
||||
};
|
||||
|
||||
TEST_P(PinL0IndexAndFilterBlocksTest,
|
||||
IndexAndFilterBlocksOfNewTableAddedToCacheWithPinning) {
|
||||
Options options = CurrentOptions();
|
||||
if (infinite_max_files_) {
|
||||
options.max_open_files = -1;
|
||||
}
|
||||
options.create_if_missing = true;
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
BlockBasedTableOptions table_options;
|
||||
table_options.cache_index_and_filter_blocks = true;
|
||||
table_options.pin_l0_filter_and_index_blocks_in_cache = true;
|
||||
table_options.filter_policy.reset(NewBloomFilterPolicy(20));
|
||||
options.table_factory.reset(new BlockBasedTableFactory(table_options));
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
ASSERT_OK(Put(1, "key", "val"));
|
||||
// Create a new table.
|
||||
ASSERT_OK(Flush(1));
|
||||
|
||||
// index/filter blocks added to block cache right after table creation.
|
||||
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
|
||||
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
|
||||
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
|
||||
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
|
||||
|
||||
// only index/filter were added
|
||||
ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
||||
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS));
|
||||
|
||||
std::string value;
|
||||
// Miss and hit count should remain the same, they're all pinned.
|
||||
db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value);
|
||||
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
|
||||
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
|
||||
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
|
||||
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
|
||||
|
||||
// Miss and hit count should remain the same, they're all pinned.
|
||||
value = Get(1, "key");
|
||||
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
|
||||
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
|
||||
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
|
||||
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
|
||||
}
|
||||
|
||||
TEST_P(PinL0IndexAndFilterBlocksTest,
|
||||
MultiLevelIndexAndFilterBlocksCachedWithPinning) {
|
||||
Options options = CurrentOptions();
|
||||
if (infinite_max_files_) {
|
||||
options.max_open_files = -1;
|
||||
}
|
||||
options.create_if_missing = true;
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
BlockBasedTableOptions table_options;
|
||||
table_options.cache_index_and_filter_blocks = true;
|
||||
table_options.pin_l0_filter_and_index_blocks_in_cache = true;
|
||||
table_options.filter_policy.reset(NewBloomFilterPolicy(20));
|
||||
options.table_factory.reset(new BlockBasedTableFactory(table_options));
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
Put(1, "a", "begin");
|
||||
Put(1, "z", "end");
|
||||
ASSERT_OK(Flush(1));
|
||||
// move this table to L1
|
||||
dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
|
||||
|
||||
// reset block cache
|
||||
table_options.block_cache = NewLRUCache(64 * 1024);
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||
TryReopenWithColumnFamilies({"default", "pikachu"}, options);
|
||||
// create new table at L0
|
||||
Put(1, "a2", "begin2");
|
||||
Put(1, "z2", "end2");
|
||||
ASSERT_OK(Flush(1));
|
||||
|
||||
table_options.block_cache->EraseUnRefEntries();
|
||||
|
||||
// get base cache values
|
||||
uint64_t fm = TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS);
|
||||
uint64_t fh = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT);
|
||||
uint64_t im = TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS);
|
||||
uint64_t ih = TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT);
|
||||
|
||||
std::string value;
|
||||
// this should be read from L0
|
||||
// so cache values don't change
|
||||
value = Get(1, "a2");
|
||||
ASSERT_EQ(fm, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
|
||||
ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
|
||||
ASSERT_EQ(im, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
|
||||
ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
|
||||
|
||||
// this should be read from L1
|
||||
// the file is opened, prefetching results in a cache filter miss
|
||||
// the block is loaded and added to the cache,
|
||||
// then the get results in a cache hit for L1
|
||||
value = Get(1, "a");
|
||||
ASSERT_EQ(fm + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
|
||||
ASSERT_EQ(im + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(PinL0IndexAndFilterBlocksTest,
|
||||
PinL0IndexAndFilterBlocksTest, ::testing::Bool());
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
@@ -210,8 +210,6 @@ Options DBTestBase::CurrentOptions(
|
||||
const anon::OptionsOverride& options_override) {
|
||||
Options options;
|
||||
options.write_buffer_size = 4090 * 4096;
|
||||
options.target_file_size_base = 2 * 1024 * 1024;
|
||||
options.max_bytes_for_level_base = 10 * 1024 * 1024;
|
||||
return CurrentOptions(options, options_override);
|
||||
}
|
||||
|
||||
@@ -303,7 +301,6 @@ Options DBTestBase::CurrentOptions(
|
||||
case kPerfOptions:
|
||||
options.soft_rate_limit = 2.0;
|
||||
options.delayed_write_rate = 8 * 1024 * 1024;
|
||||
options.report_bg_io_stats = true;
|
||||
// TODO(3.13) -- test more options
|
||||
break;
|
||||
case kDeletesFilterFirst:
|
||||
|
||||
@@ -124,7 +124,8 @@ class DelayFilterFactory : public CompactionFilterFactory {
|
||||
// Make sure we don't trigger a problem if the trigger conditon is given
|
||||
// to be 0, which is invalid.
|
||||
TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options = CurrentOptions(options);
|
||||
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.num_levels = num_levels_;
|
||||
@@ -159,7 +160,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
|
||||
}
|
||||
|
||||
TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options = CurrentOptions(options);
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.compaction_options_universal.size_ratio = 5;
|
||||
options.num_levels = num_levels_;
|
||||
@@ -335,12 +337,13 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
|
||||
}
|
||||
|
||||
TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.num_levels = num_levels_;
|
||||
options.write_buffer_size = 100 << 10; // 100KB
|
||||
options.target_file_size_base = 32 << 10; // 32KB
|
||||
options.level0_file_num_compaction_trigger = 3;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
@@ -385,12 +388,12 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
|
||||
ChangeCompactOptions();
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
|
||||
options.compaction_style = kCompactionStyleLevel;
|
||||
options.num_levels = 1;
|
||||
options.target_file_size_base = options.write_buffer_size;
|
||||
options.compression = kNoCompression;
|
||||
options = CurrentOptions(options);
|
||||
options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal);
|
||||
Random rnd(301);
|
||||
@@ -448,11 +451,12 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
|
||||
}
|
||||
|
||||
TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.write_buffer_size = 100 << 10; // 100KB
|
||||
options.num_levels = 7;
|
||||
options.disable_auto_compactions = true;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
// Generate 3 overlapping files
|
||||
@@ -492,13 +496,14 @@ class DBTestUniversalCompactionMultiLevels
|
||||
};
|
||||
|
||||
TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.num_levels = num_levels_;
|
||||
options.write_buffer_size = 100 << 10; // 100KB
|
||||
options.level0_file_num_compaction_trigger = 8;
|
||||
options.max_background_compactions = 3;
|
||||
options.target_file_size_base = 32 * 1024;
|
||||
options = CurrentOptions(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
// Trigger compaction if size amplification exceeds 110%
|
||||
@@ -534,7 +539,7 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) {
|
||||
});
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.compaction_options_universal.allow_trivial_move = true;
|
||||
options.num_levels = 3;
|
||||
@@ -542,6 +547,7 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) {
|
||||
options.level0_file_num_compaction_trigger = 3;
|
||||
options.max_background_compactions = 2;
|
||||
options.target_file_size_base = 32 * 1024;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
@@ -580,7 +586,7 @@ class DBTestUniversalCompactionParallel :
|
||||
};
|
||||
|
||||
TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.num_levels = num_levels_;
|
||||
options.write_buffer_size = 1 << 10; // 1KB
|
||||
@@ -589,6 +595,7 @@ TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
|
||||
options.max_background_flushes = 3;
|
||||
options.target_file_size_base = 1 * 1024;
|
||||
options.compaction_options_universal.max_size_amplification_percent = 110;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
@@ -645,7 +652,7 @@ INSTANTIATE_TEST_CASE_P(DBTestUniversalCompactionParallel,
|
||||
::testing::Bool()));
|
||||
|
||||
TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.write_buffer_size = 105 << 10; // 105KB
|
||||
options.arena_block_size = 4 << 10; // 4KB
|
||||
@@ -653,6 +660,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
|
||||
options.level0_file_num_compaction_trigger = 4;
|
||||
options.num_levels = num_levels_;
|
||||
options.compaction_options_universal.compression_size_percent = -1;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
@@ -765,13 +773,14 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
|
||||
return;
|
||||
}
|
||||
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.write_buffer_size = 100 << 10; // 100KB
|
||||
options.target_file_size_base = 32 << 10; // 32KB
|
||||
options.level0_file_num_compaction_trigger = 2;
|
||||
options.num_levels = num_levels_;
|
||||
options.compaction_options_universal.compression_size_percent = 70;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
Random rnd(301);
|
||||
@@ -832,13 +841,14 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio2) {
|
||||
if (!Snappy_Supported()) {
|
||||
return;
|
||||
}
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.write_buffer_size = 100 << 10; // 100KB
|
||||
options.target_file_size_base = 32 << 10; // 32KB
|
||||
options.level0_file_num_compaction_trigger = 2;
|
||||
options.num_levels = num_levels_;
|
||||
options.compaction_options_universal.compression_size_percent = 95;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
|
||||
Random rnd(301);
|
||||
@@ -874,7 +884,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
|
||||
});
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.compaction_options_universal.allow_trivial_move = true;
|
||||
options.num_levels = 2;
|
||||
@@ -882,6 +892,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
|
||||
options.level0_file_num_compaction_trigger = 3;
|
||||
options.max_background_compactions = 1;
|
||||
options.target_file_size_base = 32 * 1024;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
@@ -918,7 +929,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
|
||||
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.compaction_options_universal.allow_trivial_move = true;
|
||||
options.num_levels = 15;
|
||||
@@ -926,6 +937,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
|
||||
options.level0_file_num_compaction_trigger = 8;
|
||||
options.max_background_compactions = 4;
|
||||
options.target_file_size_base = 64 * 1024;
|
||||
options = CurrentOptions(options);
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
@@ -951,7 +963,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
|
||||
}
|
||||
|
||||
TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.db_paths.emplace_back(dbname_, 300 * 1024);
|
||||
options.db_paths.emplace_back(dbname_ + "_2", 300 * 1024);
|
||||
options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024);
|
||||
@@ -964,6 +976,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
|
||||
options.arena_block_size = 4 << 10;
|
||||
options.level0_file_num_compaction_trigger = 2;
|
||||
options.num_levels = 1;
|
||||
options = CurrentOptions(options);
|
||||
|
||||
std::vector<std::string> filenames;
|
||||
env_->GetChildren(options.db_paths[1].path, &filenames);
|
||||
@@ -1150,7 +1163,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
|
||||
if (!Snappy_Supported()) {
|
||||
return;
|
||||
}
|
||||
Options options = CurrentOptions();
|
||||
Options options;
|
||||
options.db_paths.emplace_back(dbname_, 500 * 1024);
|
||||
options.db_paths.emplace_back(dbname_ + "_2", 1024 * 1024 * 1024);
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
@@ -1162,6 +1175,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
|
||||
options.num_levels = 1;
|
||||
options.memtable_factory.reset(
|
||||
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
|
||||
options = CurrentOptions(options);
|
||||
|
||||
std::vector<std::string> filenames;
|
||||
env_->GetChildren(options.db_paths[1].path, &filenames);
|
||||
|
||||
@@ -465,12 +465,6 @@ class InternalKeySliceTransform : public SliceTransform {
|
||||
const SliceTransform* const transform_;
|
||||
};
|
||||
|
||||
// Read the key of a record from a write batch.
|
||||
// if this record represent the default column family then cf_record
|
||||
// must be passed as false, otherwise it must be passed as true.
|
||||
extern bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key,
|
||||
bool cf_record);
|
||||
|
||||
// Read record from a write batch piece from input.
|
||||
// tag, column_family, key, value and blob are return values. Callers own the
|
||||
// Slice they point to.
|
||||
|
||||
+10
-39
@@ -66,7 +66,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
|
||||
JobContext* job_context, LogBuffer* log_buffer,
|
||||
Directory* db_directory, Directory* output_file_directory,
|
||||
CompressionType output_compression, Statistics* stats,
|
||||
EventLogger* event_logger, bool measure_io_stats)
|
||||
EventLogger* event_logger)
|
||||
: dbname_(dbname),
|
||||
cfd_(cfd),
|
||||
db_options_(db_options),
|
||||
@@ -83,8 +83,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
|
||||
output_file_directory_(output_file_directory),
|
||||
output_compression_(output_compression),
|
||||
stats_(stats),
|
||||
event_logger_(event_logger),
|
||||
measure_io_stats_(measure_io_stats) {
|
||||
event_logger_(event_logger) {
|
||||
// Update the thread status to indicate flush.
|
||||
ReportStartedFlush();
|
||||
TEST_SYNC_POINT("FlushJob::FlushJob()");
|
||||
@@ -122,21 +121,6 @@ void FlushJob::RecordFlushIOStats() {
|
||||
Status FlushJob::Run(FileMetaData* file_meta) {
|
||||
AutoThreadOperationStageUpdater stage_run(
|
||||
ThreadStatus::STAGE_FLUSH_RUN);
|
||||
// I/O measurement variables
|
||||
PerfLevel prev_perf_level = PerfLevel::kEnableTime;
|
||||
uint64_t prev_write_nanos = 0;
|
||||
uint64_t prev_fsync_nanos = 0;
|
||||
uint64_t prev_range_sync_nanos = 0;
|
||||
uint64_t prev_prepare_write_nanos = 0;
|
||||
if (measure_io_stats_) {
|
||||
prev_perf_level = GetPerfLevel();
|
||||
SetPerfLevel(PerfLevel::kEnableTime);
|
||||
prev_write_nanos = IOSTATS(write_nanos);
|
||||
prev_fsync_nanos = IOSTATS(fsync_nanos);
|
||||
prev_range_sync_nanos = IOSTATS(range_sync_nanos);
|
||||
prev_prepare_write_nanos = IOSTATS(prepare_write_nanos);
|
||||
}
|
||||
|
||||
// Save the contents of the earliest memtable as a new Table
|
||||
FileMetaData meta;
|
||||
autovector<MemTable*> mems;
|
||||
@@ -196,18 +180,6 @@ Status FlushJob::Run(FileMetaData* file_meta) {
|
||||
}
|
||||
stream.EndArray();
|
||||
|
||||
if (measure_io_stats_) {
|
||||
if (prev_perf_level != PerfLevel::kEnableTime) {
|
||||
SetPerfLevel(prev_perf_level);
|
||||
}
|
||||
stream << "file_write_nanos" << (IOSTATS(write_nanos) - prev_write_nanos);
|
||||
stream << "file_range_sync_nanos"
|
||||
<< (IOSTATS(range_sync_nanos) - prev_range_sync_nanos);
|
||||
stream << "file_fsync_nanos" << (IOSTATS(fsync_nanos) - prev_fsync_nanos);
|
||||
stream << "file_prepare_write_nanos"
|
||||
<< (IOSTATS(prepare_write_nanos) - prev_prepare_write_nanos);
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
@@ -262,15 +234,14 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
|
||||
|
||||
TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table:output_compression",
|
||||
&output_compression_);
|
||||
s = BuildTable(
|
||||
dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
|
||||
cfd_->table_cache(), iter.get(), meta, cfd_->internal_comparator(),
|
||||
cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
|
||||
cfd_->GetName(), existing_snapshots_,
|
||||
earliest_write_conflict_snapshot_, output_compression_,
|
||||
cfd_->ioptions()->compression_opts,
|
||||
mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
|
||||
Env::IO_HIGH, &table_properties_, 0 /* level */);
|
||||
s = BuildTable(dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
|
||||
cfd_->table_cache(), iter.get(), meta,
|
||||
cfd_->internal_comparator(),
|
||||
cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
|
||||
existing_snapshots_, earliest_write_conflict_snapshot_,
|
||||
output_compression_, cfd_->ioptions()->compression_opts,
|
||||
mutable_cf_options_.paranoid_file_checks,
|
||||
cfd_->internal_stats(), Env::IO_HIGH, &table_properties_);
|
||||
info.table_properties = table_properties_;
|
||||
LogFlush(db_options_.info_log);
|
||||
}
|
||||
|
||||
+1
-2
@@ -62,7 +62,7 @@ class FlushJob {
|
||||
JobContext* job_context, LogBuffer* log_buffer,
|
||||
Directory* db_directory, Directory* output_file_directory,
|
||||
CompressionType output_compression, Statistics* stats,
|
||||
EventLogger* event_logger, bool measure_io_stats);
|
||||
EventLogger* event_logger);
|
||||
|
||||
~FlushJob();
|
||||
|
||||
@@ -93,7 +93,6 @@ class FlushJob {
|
||||
Statistics* stats_;
|
||||
EventLogger* event_logger_;
|
||||
TableProperties table_properties_;
|
||||
bool measure_io_stats_;
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
@@ -93,7 +93,7 @@ TEST_F(FlushJobTest, Empty) {
|
||||
db_options_, *cfd->GetLatestMutableCFOptions(),
|
||||
env_options_, versions_.get(), &mutex_, &shutting_down_,
|
||||
{}, kMaxSequenceNumber, &job_context, nullptr, nullptr,
|
||||
nullptr, kNoCompression, nullptr, &event_logger, false);
|
||||
nullptr, kNoCompression, nullptr, &event_logger);
|
||||
ASSERT_OK(flush_job.Run());
|
||||
job_context.Clean();
|
||||
}
|
||||
@@ -132,7 +132,7 @@ TEST_F(FlushJobTest, NonEmpty) {
|
||||
db_options_, *cfd->GetLatestMutableCFOptions(),
|
||||
env_options_, versions_.get(), &mutex_, &shutting_down_,
|
||||
{}, kMaxSequenceNumber, &job_context, nullptr, nullptr,
|
||||
nullptr, kNoCompression, nullptr, &event_logger, true);
|
||||
nullptr, kNoCompression, nullptr, &event_logger);
|
||||
FileMetaData fd;
|
||||
mutex_.Lock();
|
||||
ASSERT_OK(flush_job.Run(&fd));
|
||||
@@ -192,11 +192,11 @@ TEST_F(FlushJobTest, Snapshots) {
|
||||
}
|
||||
|
||||
EventLogger event_logger(db_options_.info_log.get());
|
||||
FlushJob flush_job(
|
||||
dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
|
||||
*cfd->GetLatestMutableCFOptions(), env_options_, versions_.get(), &mutex_,
|
||||
&shutting_down_, snapshots, kMaxSequenceNumber, &job_context, nullptr,
|
||||
nullptr, nullptr, kNoCompression, nullptr, &event_logger, true);
|
||||
FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
|
||||
db_options_, *cfd->GetLatestMutableCFOptions(),
|
||||
env_options_, versions_.get(), &mutex_, &shutting_down_,
|
||||
snapshots, kMaxSequenceNumber, &job_context, nullptr,
|
||||
nullptr, nullptr, kNoCompression, nullptr, &event_logger);
|
||||
mutex_.Lock();
|
||||
ASSERT_OK(flush_job.Run());
|
||||
mutex_.Unlock();
|
||||
|
||||
@@ -79,7 +79,7 @@ ManagedIterator::ManagedIterator(DBImpl* db, const ReadOptions& read_options,
|
||||
release_supported_(true) {
|
||||
read_options_.managed = false;
|
||||
if ((!read_options_.tailing) && (read_options_.snapshot == nullptr)) {
|
||||
assert(nullptr != (read_options_.snapshot = db_->GetSnapshot()));
|
||||
assert(read_options_.snapshot = db_->GetSnapshot());
|
||||
snapshot_created_ = true;
|
||||
}
|
||||
cfh_.SetCFD(cfd);
|
||||
@@ -210,8 +210,7 @@ void ManagedIterator::RebuildIterator() {
|
||||
void ManagedIterator::UpdateCurrent() {
|
||||
assert(mutable_iter_ != nullptr);
|
||||
|
||||
valid_ = mutable_iter_->Valid();
|
||||
if (!valid_) {
|
||||
if (!(valid_ = mutable_iter_->Valid())) {
|
||||
status_ = mutable_iter_->status();
|
||||
return;
|
||||
}
|
||||
|
||||
+2
-2
@@ -345,8 +345,8 @@ Status MemTableList::InstallMemtableFlushResults(
|
||||
imm_flush_needed.store(true, std::memory_order_release);
|
||||
}
|
||||
++mem_id;
|
||||
} while (!current_->memlist_.empty() && (nullptr != (m = current_->memlist_.back())) &&
|
||||
(m->file_number_ == file_number));
|
||||
} while (!current_->memlist_.empty() && (m = current_->memlist_.back()) &&
|
||||
m->file_number_ == file_number);
|
||||
}
|
||||
commit_in_progress_ = false;
|
||||
return s;
|
||||
|
||||
@@ -37,11 +37,6 @@ public:
|
||||
Initialize();
|
||||
operand_list->push_front(operand_slice.ToString());
|
||||
}
|
||||
// Push back a merge operand
|
||||
void PushOperandBack(const Slice& operand_slice) {
|
||||
Initialize();
|
||||
operand_list->push_back(operand_slice.ToString());
|
||||
}
|
||||
// return total number of operands in the list
|
||||
size_t GetNumOperands() const {
|
||||
if (!operand_list) {
|
||||
|
||||
+3
-38
@@ -9,16 +9,16 @@
|
||||
#include <vector>
|
||||
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/memtablerep.h"
|
||||
#include "rocksdb/perf_context.h"
|
||||
#include "rocksdb/slice_transform.h"
|
||||
#include "rocksdb/memtablerep.h"
|
||||
#include "util/histogram.h"
|
||||
#include "util/instrumented_mutex.h"
|
||||
#include "util/stop_watch.h"
|
||||
#include "util/string_util.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/thread_status_util.h"
|
||||
#include "utilities/merge_operators.h"
|
||||
#include "util/string_util.h"
|
||||
|
||||
|
||||
bool FLAGS_random_key = false;
|
||||
bool FLAGS_use_set_based_memetable = false;
|
||||
@@ -602,41 +602,6 @@ TEST_F(PerfContextTest, ToString) {
|
||||
ASSERT_EQ(std::string::npos, zero_excluded.find("= 0"));
|
||||
ASSERT_NE(std::string::npos, zero_excluded.find("= 12345"));
|
||||
}
|
||||
|
||||
TEST_F(PerfContextTest, MergeOperatorTime) {
|
||||
DestroyDB(kDbName, Options());
|
||||
DB* db;
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.merge_operator = MergeOperators::CreateStringAppendOperator();
|
||||
Status s = DB::Open(options, kDbName, &db);
|
||||
EXPECT_OK(s);
|
||||
|
||||
std::string val;
|
||||
ASSERT_OK(db->Merge(WriteOptions(), "k1", "val1"));
|
||||
ASSERT_OK(db->Merge(WriteOptions(), "k1", "val2"));
|
||||
ASSERT_OK(db->Merge(WriteOptions(), "k1", "val3"));
|
||||
ASSERT_OK(db->Merge(WriteOptions(), "k1", "val4"));
|
||||
|
||||
SetPerfLevel(kEnableTime);
|
||||
perf_context.Reset();
|
||||
ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
|
||||
EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
|
||||
|
||||
ASSERT_OK(db->Flush(FlushOptions()));
|
||||
|
||||
perf_context.Reset();
|
||||
ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
|
||||
EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
|
||||
|
||||
ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
||||
|
||||
perf_context.Reset();
|
||||
ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
|
||||
EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
|
||||
|
||||
delete db;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
@@ -263,9 +263,7 @@ class TestPlainTableReader : public PlainTableReader {
|
||||
const TableProperties* table_properties,
|
||||
unique_ptr<RandomAccessFileReader>&& file,
|
||||
const ImmutableCFOptions& ioptions,
|
||||
bool* expect_bloom_not_match, bool store_index_in_file,
|
||||
uint32_t column_family_id,
|
||||
const std::string& column_family_name)
|
||||
bool* expect_bloom_not_match, bool store_index_in_file)
|
||||
: PlainTableReader(ioptions, std::move(file), env_options, icomparator,
|
||||
encoding_type, file_size, table_properties),
|
||||
expect_bloom_not_match_(expect_bloom_not_match) {
|
||||
@@ -278,8 +276,6 @@ class TestPlainTableReader : public PlainTableReader {
|
||||
EXPECT_TRUE(s.ok());
|
||||
|
||||
TableProperties* props = const_cast<TableProperties*>(table_properties);
|
||||
EXPECT_EQ(column_family_id, static_cast<uint32_t>(props->column_family_id));
|
||||
EXPECT_EQ(column_family_name, props->column_family_name);
|
||||
if (store_index_in_file) {
|
||||
auto bloom_version_ptr = props->user_collected_properties.find(
|
||||
PlainTablePropertyNames::kBloomVersion);
|
||||
@@ -312,17 +308,13 @@ extern const uint64_t kPlainTableMagicNumber;
|
||||
class TestPlainTableFactory : public PlainTableFactory {
|
||||
public:
|
||||
explicit TestPlainTableFactory(bool* expect_bloom_not_match,
|
||||
const PlainTableOptions& options,
|
||||
uint32_t column_family_id,
|
||||
std::string column_family_name)
|
||||
const PlainTableOptions& options)
|
||||
: PlainTableFactory(options),
|
||||
bloom_bits_per_key_(options.bloom_bits_per_key),
|
||||
hash_table_ratio_(options.hash_table_ratio),
|
||||
index_sparseness_(options.index_sparseness),
|
||||
store_index_in_file_(options.store_index_in_file),
|
||||
expect_bloom_not_match_(expect_bloom_not_match),
|
||||
column_family_id_(column_family_id),
|
||||
column_family_name_(std::move(column_family_name)) {}
|
||||
expect_bloom_not_match_(expect_bloom_not_match) {}
|
||||
|
||||
Status NewTableReader(const TableReaderOptions& table_reader_options,
|
||||
unique_ptr<RandomAccessFileReader>&& file,
|
||||
@@ -362,7 +354,7 @@ class TestPlainTableFactory : public PlainTableFactory {
|
||||
table_reader_options.internal_comparator, encoding_type, file_size,
|
||||
bloom_bits_per_key_, hash_table_ratio_, index_sparseness_, props,
|
||||
std::move(file), table_reader_options.ioptions, expect_bloom_not_match_,
|
||||
store_index_in_file_, column_family_id_, column_family_name_));
|
||||
store_index_in_file_));
|
||||
|
||||
*table = std::move(new_reader);
|
||||
return s;
|
||||
@@ -374,8 +366,6 @@ class TestPlainTableFactory : public PlainTableFactory {
|
||||
size_t index_sparseness_;
|
||||
bool store_index_in_file_;
|
||||
bool* expect_bloom_not_match_;
|
||||
const uint32_t column_family_id_;
|
||||
const std::string column_family_name_;
|
||||
};
|
||||
|
||||
TEST_P(PlainTableDBTest, Flush) {
|
||||
@@ -502,8 +492,7 @@ TEST_P(PlainTableDBTest, Flush2) {
|
||||
plain_table_options.encoding_type = encoding_type;
|
||||
plain_table_options.store_index_in_file = store_index_in_file;
|
||||
options.table_factory.reset(new TestPlainTableFactory(
|
||||
&expect_bloom_not_match, plain_table_options,
|
||||
0 /* column_family_id */, kDefaultColumnFamilyName));
|
||||
&expect_bloom_not_match, plain_table_options));
|
||||
|
||||
DestroyAndReopen(&options);
|
||||
ASSERT_OK(Put("0000000000000bar", "b"));
|
||||
@@ -572,8 +561,7 @@ TEST_P(PlainTableDBTest, Iterator) {
|
||||
plain_table_options.encoding_type = encoding_type;
|
||||
|
||||
options.table_factory.reset(new TestPlainTableFactory(
|
||||
&expect_bloom_not_match, plain_table_options,
|
||||
0 /* column_family_id */, kDefaultColumnFamilyName));
|
||||
&expect_bloom_not_match, plain_table_options));
|
||||
} else {
|
||||
PlainTableOptions plain_table_options;
|
||||
plain_table_options.user_key_len = 16;
|
||||
@@ -584,8 +572,7 @@ TEST_P(PlainTableDBTest, Iterator) {
|
||||
plain_table_options.encoding_type = encoding_type;
|
||||
|
||||
options.table_factory.reset(new TestPlainTableFactory(
|
||||
&expect_bloom_not_match, plain_table_options,
|
||||
0 /* column_family_id */, kDefaultColumnFamilyName));
|
||||
&expect_bloom_not_match, plain_table_options));
|
||||
}
|
||||
DestroyAndReopen(&options);
|
||||
|
||||
|
||||
+4
-5
@@ -265,7 +265,7 @@ class Repairer {
|
||||
mem->Ref();
|
||||
int counter = 0;
|
||||
while (reader.ReadRecord(&record, &scratch)) {
|
||||
if (record.size() < WriteBatchInternal::kHeader) {
|
||||
if (record.size() < 12) {
|
||||
reporter.Corruption(
|
||||
record.size(), Status::Corruption("log record too small"));
|
||||
continue;
|
||||
@@ -294,10 +294,9 @@ class Repairer {
|
||||
status = BuildTable(
|
||||
dbname_, env_, ioptions_, env_options_, table_cache_, iter.get(),
|
||||
&meta, icmp_, &int_tbl_prop_collector_factories_,
|
||||
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
|
||||
std::string() /* column_family_name */, {}, kMaxSequenceNumber,
|
||||
kNoCompression, CompressionOptions(), false,
|
||||
nullptr /* internal_stats */);
|
||||
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, {},
|
||||
kMaxSequenceNumber, kNoCompression, CompressionOptions(), false,
|
||||
nullptr);
|
||||
}
|
||||
delete mem->Unref();
|
||||
delete cf_mems_default;
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
// Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "db/db_impl.h"
|
||||
#include "db/db_test_util.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/transaction_log.h"
|
||||
#include "util/file_util.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
class RepairTest : public DBTestBase {
|
||||
public:
|
||||
RepairTest() : DBTestBase("/repair_test") {}
|
||||
|
||||
std::string GetFirstSstPath() {
|
||||
uint64_t manifest_size;
|
||||
std::vector<std::string> files;
|
||||
db_->GetLiveFiles(files, &manifest_size);
|
||||
auto sst_iter =
|
||||
std::find_if(files.begin(), files.end(), [](const std::string& file) {
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
bool ok = ParseFileName(file, &number, &type);
|
||||
return ok && type == kTableFile;
|
||||
});
|
||||
return sst_iter == files.end() ? "" : dbname_ + *sst_iter;
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(RepairTest, LostManifest) {
|
||||
// Add a couple SST files, delete the manifest, and verify RepairDB() saves
|
||||
// the day.
|
||||
Put("key", "val");
|
||||
Flush();
|
||||
Put("key2", "val2");
|
||||
Flush();
|
||||
// Need to get path before Close() deletes db_, but delete it after Close() to
|
||||
// ensure Close() didn't change the manifest.
|
||||
std::string manifest_path =
|
||||
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
||||
|
||||
Close();
|
||||
ASSERT_OK(env_->FileExists(manifest_path));
|
||||
ASSERT_OK(env_->DeleteFile(manifest_path));
|
||||
RepairDB(dbname_, CurrentOptions());
|
||||
Reopen(CurrentOptions());
|
||||
|
||||
ASSERT_EQ(Get("key"), "val");
|
||||
ASSERT_EQ(Get("key2"), "val2");
|
||||
}
|
||||
|
||||
TEST_F(RepairTest, CorruptManifest) {
|
||||
// Manifest is in an invalid format. Expect a full recovery.
|
||||
Put("key", "val");
|
||||
Flush();
|
||||
Put("key2", "val2");
|
||||
Flush();
|
||||
// Need to get path before Close() deletes db_, but overwrite it after Close()
|
||||
// to ensure Close() didn't change the manifest.
|
||||
std::string manifest_path =
|
||||
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
||||
|
||||
Close();
|
||||
ASSERT_OK(env_->FileExists(manifest_path));
|
||||
CreateFile(env_, manifest_path, "blah");
|
||||
RepairDB(dbname_, CurrentOptions());
|
||||
Reopen(CurrentOptions());
|
||||
|
||||
ASSERT_EQ(Get("key"), "val");
|
||||
ASSERT_EQ(Get("key2"), "val2");
|
||||
}
|
||||
|
||||
TEST_F(RepairTest, IncompleteManifest) {
|
||||
// In this case, the manifest is valid but does not reference all of the SST
|
||||
// files. Expect a full recovery.
|
||||
Put("key", "val");
|
||||
Flush();
|
||||
std::string orig_manifest_path =
|
||||
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
||||
CopyFile(orig_manifest_path, orig_manifest_path + ".tmp");
|
||||
Put("key2", "val2");
|
||||
Flush();
|
||||
// Need to get path before Close() deletes db_, but overwrite it after Close()
|
||||
// to ensure Close() didn't change the manifest.
|
||||
std::string new_manifest_path =
|
||||
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
||||
|
||||
Close();
|
||||
ASSERT_OK(env_->FileExists(new_manifest_path));
|
||||
// Replace the manifest with one that is only aware of the first SST file.
|
||||
CopyFile(orig_manifest_path + ".tmp", new_manifest_path);
|
||||
RepairDB(dbname_, CurrentOptions());
|
||||
Reopen(CurrentOptions());
|
||||
|
||||
ASSERT_EQ(Get("key"), "val");
|
||||
ASSERT_EQ(Get("key2"), "val2");
|
||||
}
|
||||
|
||||
TEST_F(RepairTest, LostSst) {
|
||||
// Delete one of the SST files but preserve the manifest that refers to it,
|
||||
// then verify the DB is still usable for the intact SST.
|
||||
Put("key", "val");
|
||||
Flush();
|
||||
Put("key2", "val2");
|
||||
Flush();
|
||||
auto sst_path = GetFirstSstPath();
|
||||
ASSERT_FALSE(sst_path.empty());
|
||||
ASSERT_OK(env_->DeleteFile(sst_path));
|
||||
|
||||
Close();
|
||||
RepairDB(dbname_, CurrentOptions());
|
||||
Reopen(CurrentOptions());
|
||||
|
||||
// Exactly one of the key-value pairs should be in the DB now.
|
||||
ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2"));
|
||||
}
|
||||
|
||||
TEST_F(RepairTest, CorruptSst) {
|
||||
// Corrupt one of the SST files but preserve the manifest that refers to it,
|
||||
// then verify the DB is still usable for the intact SST.
|
||||
Put("key", "val");
|
||||
Flush();
|
||||
Put("key2", "val2");
|
||||
Flush();
|
||||
auto sst_path = GetFirstSstPath();
|
||||
ASSERT_FALSE(sst_path.empty());
|
||||
CreateFile(env_, sst_path, "blah");
|
||||
|
||||
Close();
|
||||
RepairDB(dbname_, CurrentOptions());
|
||||
Reopen(CurrentOptions());
|
||||
|
||||
// Exactly one of the key-value pairs should be in the DB now.
|
||||
ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2"));
|
||||
}
|
||||
|
||||
TEST_F(RepairTest, UnflushedSst) {
|
||||
// This test case invokes repair while some data is unflushed, then verifies
|
||||
// that data is in the db.
|
||||
Put("key", "val");
|
||||
VectorLogPtr wal_files;
|
||||
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
|
||||
ASSERT_EQ(wal_files.size(), 1);
|
||||
uint64_t total_ssts_size;
|
||||
GetAllSSTFiles(&total_ssts_size);
|
||||
ASSERT_EQ(total_ssts_size, 0);
|
||||
// Need to get path before Close() deletes db_, but delete it after Close() to
|
||||
// ensure Close() didn't change the manifest.
|
||||
std::string manifest_path =
|
||||
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
|
||||
|
||||
Close();
|
||||
ASSERT_OK(env_->FileExists(manifest_path));
|
||||
ASSERT_OK(env_->DeleteFile(manifest_path));
|
||||
RepairDB(dbname_, CurrentOptions());
|
||||
Reopen(CurrentOptions());
|
||||
|
||||
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
|
||||
ASSERT_EQ(wal_files.size(), 0);
|
||||
GetAllSSTFiles(&total_ssts_size);
|
||||
ASSERT_GT(total_ssts_size, 0);
|
||||
ASSERT_EQ(Get("key"), "val");
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
+9
-19
@@ -88,7 +88,7 @@ Status TableCache::GetTableReader(
|
||||
const EnvOptions& env_options,
|
||||
const InternalKeyComparator& internal_comparator, const FileDescriptor& fd,
|
||||
bool sequential_mode, bool record_read_stats, HistogramImpl* file_read_hist,
|
||||
unique_ptr<TableReader>* table_reader, bool skip_filters, int level) {
|
||||
unique_ptr<TableReader>* table_reader, bool skip_filters) {
|
||||
std::string fname =
|
||||
TableFileName(ioptions_.db_paths, fd.GetNumber(), fd.GetPathId());
|
||||
unique_ptr<RandomAccessFile> file;
|
||||
@@ -109,26 +109,18 @@ Status TableCache::GetTableReader(
|
||||
file_read_hist));
|
||||
s = ioptions_.table_factory->NewTableReader(
|
||||
TableReaderOptions(ioptions_, env_options, internal_comparator,
|
||||
skip_filters, level),
|
||||
skip_filters),
|
||||
std::move(file_reader), fd.GetFileSize(), table_reader);
|
||||
TEST_SYNC_POINT("TableCache::GetTableReader:0");
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
void TableCache::EraseHandle(const FileDescriptor& fd, Cache::Handle* handle) {
|
||||
ReleaseHandle(handle);
|
||||
uint64_t number = fd.GetNumber();
|
||||
Slice key = GetSliceForFileNumber(&number);
|
||||
cache_->Erase(key);
|
||||
}
|
||||
|
||||
Status TableCache::FindTable(const EnvOptions& env_options,
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
const FileDescriptor& fd, Cache::Handle** handle,
|
||||
const bool no_io, bool record_read_stats,
|
||||
HistogramImpl* file_read_hist, bool skip_filters,
|
||||
int level) {
|
||||
HistogramImpl* file_read_hist, bool skip_filters) {
|
||||
PERF_TIMER_GUARD(find_table_nanos);
|
||||
Status s;
|
||||
uint64_t number = fd.GetNumber();
|
||||
@@ -144,7 +136,7 @@ Status TableCache::FindTable(const EnvOptions& env_options,
|
||||
unique_ptr<TableReader> table_reader;
|
||||
s = GetTableReader(env_options, internal_comparator, fd,
|
||||
false /* sequential mode */, record_read_stats,
|
||||
file_read_hist, &table_reader, skip_filters, level);
|
||||
file_read_hist, &table_reader, skip_filters);
|
||||
if (!s.ok()) {
|
||||
assert(table_reader == nullptr);
|
||||
RecordTick(ioptions_.statistics, NO_FILE_ERRORS);
|
||||
@@ -166,7 +158,7 @@ InternalIterator* TableCache::NewIterator(
|
||||
const ReadOptions& options, const EnvOptions& env_options,
|
||||
const InternalKeyComparator& icomparator, const FileDescriptor& fd,
|
||||
TableReader** table_reader_ptr, HistogramImpl* file_read_hist,
|
||||
bool for_compaction, Arena* arena, bool skip_filters, int level) {
|
||||
bool for_compaction, Arena* arena, bool skip_filters) {
|
||||
PERF_TIMER_GUARD(new_table_iterator_nanos);
|
||||
|
||||
if (table_reader_ptr != nullptr) {
|
||||
@@ -181,8 +173,7 @@ InternalIterator* TableCache::NewIterator(
|
||||
unique_ptr<TableReader> table_reader_unique_ptr;
|
||||
Status s = GetTableReader(
|
||||
env_options, icomparator, fd, /* sequential mode */ true,
|
||||
/* record stats */ false, nullptr, &table_reader_unique_ptr,
|
||||
false /* skip_filters */, level);
|
||||
/* record stats */ false, nullptr, &table_reader_unique_ptr);
|
||||
if (!s.ok()) {
|
||||
return NewErrorInternalIterator(s, arena);
|
||||
}
|
||||
@@ -193,7 +184,7 @@ InternalIterator* TableCache::NewIterator(
|
||||
Status s = FindTable(env_options, icomparator, fd, &handle,
|
||||
options.read_tier == kBlockCacheTier /* no_io */,
|
||||
!for_compaction /* record read_stats */,
|
||||
file_read_hist, skip_filters, level);
|
||||
file_read_hist, skip_filters);
|
||||
if (!s.ok()) {
|
||||
return NewErrorInternalIterator(s, arena);
|
||||
}
|
||||
@@ -225,7 +216,7 @@ Status TableCache::Get(const ReadOptions& options,
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
const FileDescriptor& fd, const Slice& k,
|
||||
GetContext* get_context, HistogramImpl* file_read_hist,
|
||||
bool skip_filters, int level) {
|
||||
bool skip_filters) {
|
||||
TableReader* t = fd.table_reader;
|
||||
Status s;
|
||||
Cache::Handle* handle = nullptr;
|
||||
@@ -274,8 +265,7 @@ Status TableCache::Get(const ReadOptions& options,
|
||||
if (!t) {
|
||||
s = FindTable(env_options_, internal_comparator, fd, &handle,
|
||||
options.read_tier == kBlockCacheTier /* no_io */,
|
||||
true /* record_read_stats */, file_read_hist, skip_filters,
|
||||
level);
|
||||
true /* record_read_stats */, file_read_hist, skip_filters);
|
||||
if (s.ok()) {
|
||||
t = GetTableReaderFromHandle(handle);
|
||||
}
|
||||
|
||||
+4
-11
@@ -45,41 +45,34 @@ class TableCache {
|
||||
// the cache and should not be deleted, and is valid for as long as the
|
||||
// returned iterator is live.
|
||||
// @param skip_filters Disables loading/accessing the filter block
|
||||
// @param level The level this table is at, -1 for "not set / don't know"
|
||||
InternalIterator* NewIterator(
|
||||
const ReadOptions& options, const EnvOptions& toptions,
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
const FileDescriptor& file_fd, TableReader** table_reader_ptr = nullptr,
|
||||
HistogramImpl* file_read_hist = nullptr, bool for_compaction = false,
|
||||
Arena* arena = nullptr, bool skip_filters = false, int level = -1);
|
||||
Arena* arena = nullptr, bool skip_filters = false);
|
||||
|
||||
// If a seek to internal key "k" in specified file finds an entry,
|
||||
// call (*handle_result)(arg, found_key, found_value) repeatedly until
|
||||
// it returns false.
|
||||
// @param skip_filters Disables loading/accessing the filter block
|
||||
// @param level The level this table is at, -1 for "not set / don't know"
|
||||
Status Get(const ReadOptions& options,
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
const FileDescriptor& file_fd, const Slice& k,
|
||||
GetContext* get_context, HistogramImpl* file_read_hist = nullptr,
|
||||
bool skip_filters = false, int level = -1);
|
||||
bool skip_filters = false);
|
||||
|
||||
// Evict any entry for the specified file number
|
||||
static void Evict(Cache* cache, uint64_t file_number);
|
||||
|
||||
// Clean table handle and erase it from the table cache
|
||||
// Used in DB close, or the file is not live anymore.
|
||||
void EraseHandle(const FileDescriptor& fd, Cache::Handle* handle);
|
||||
|
||||
// Find table reader
|
||||
// @param skip_filters Disables loading/accessing the filter block
|
||||
// @param level == -1 means not specified
|
||||
Status FindTable(const EnvOptions& toptions,
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
const FileDescriptor& file_fd, Cache::Handle**,
|
||||
const bool no_io = false, bool record_read_stats = true,
|
||||
HistogramImpl* file_read_hist = nullptr,
|
||||
bool skip_filters = false, int level = -1);
|
||||
bool skip_filters = false);
|
||||
|
||||
// Get TableReader from a cache handle.
|
||||
TableReader* GetTableReaderFromHandle(Cache::Handle* handle);
|
||||
@@ -113,7 +106,7 @@ class TableCache {
|
||||
const FileDescriptor& fd, bool sequential_mode,
|
||||
bool record_read_stats, HistogramImpl* file_read_hist,
|
||||
unique_ptr<TableReader>* table_reader,
|
||||
bool skip_filters = false, int level = -1);
|
||||
bool skip_filters = false);
|
||||
|
||||
const ImmutableCFOptions& ioptions_;
|
||||
const EnvOptions& env_options_;
|
||||
|
||||
@@ -36,7 +36,6 @@ class TablePropertiesTest : public testing::Test,
|
||||
// Utilities test functions
|
||||
namespace {
|
||||
static const uint32_t kTestColumnFamilyId = 66;
|
||||
static const std::string kTestColumnFamilyName = "test_column_fam";
|
||||
|
||||
void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
@@ -49,8 +48,8 @@ void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,
|
||||
|
||||
builder->reset(NewTableBuilder(
|
||||
ioptions, internal_comparator, int_tbl_prop_collector_factories,
|
||||
kTestColumnFamilyId, kTestColumnFamilyName,
|
||||
writable->get(), options.compression, options.compression_opts));
|
||||
kTestColumnFamilyId /* column_family_id */, writable->get(),
|
||||
options.compression, options.compression_opts));
|
||||
}
|
||||
} // namespace
|
||||
|
||||
|
||||
@@ -107,7 +107,7 @@ void TransactionLogIteratorImpl::SeekToStartSequence(
|
||||
return;
|
||||
}
|
||||
while (RestrictedRead(&record, &scratch)) {
|
||||
if (record.size() < WriteBatchInternal::kHeader) {
|
||||
if (record.size() < 12) {
|
||||
reporter_.Corruption(
|
||||
record.size(), Status::Corruption("very small log record"));
|
||||
continue;
|
||||
@@ -167,7 +167,7 @@ void TransactionLogIteratorImpl::NextImpl(bool internal) {
|
||||
currentLogReader_->UnmarkEOF();
|
||||
}
|
||||
while (RestrictedRead(&record, &scratch)) {
|
||||
if (record.size() < WriteBatchInternal::kHeader) {
|
||||
if (record.size() < 12) {
|
||||
reporter_.Corruption(
|
||||
record.size(), Status::Corruption("very small log record"));
|
||||
continue;
|
||||
|
||||
@@ -309,11 +309,11 @@ class VersionBuilder::Rep {
|
||||
|
||||
auto* file_meta = files_meta[file_idx].first;
|
||||
int level = files_meta[file_idx].second;
|
||||
table_cache_->FindTable(
|
||||
env_options_, *(base_vstorage_->InternalComparator()),
|
||||
file_meta->fd, &file_meta->table_reader_handle, false /*no_io */,
|
||||
true /* record_read_stats */,
|
||||
internal_stats->GetFileReadHist(level), false, level);
|
||||
table_cache_->FindTable(env_options_,
|
||||
*(base_vstorage_->InternalComparator()),
|
||||
file_meta->fd, &file_meta->table_reader_handle,
|
||||
false /*no_io */, true /* record_read_stats */,
|
||||
internal_stats->GetFileReadHist(level));
|
||||
if (file_meta->table_reader_handle != nullptr) {
|
||||
// Load table_reader
|
||||
file_meta->fd.table_reader = table_cache_->GetTableReaderFromHandle(
|
||||
|
||||
+25
-59
@@ -23,7 +23,6 @@
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
#include "db/compaction.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/internal_stats.h"
|
||||
#include "db/log_reader.h"
|
||||
@@ -31,22 +30,22 @@
|
||||
#include "db/memtable.h"
|
||||
#include "db/merge_context.h"
|
||||
#include "db/table_cache.h"
|
||||
#include "db/compaction.h"
|
||||
#include "db/version_builder.h"
|
||||
#include "db/writebuffer.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/merge_operator.h"
|
||||
#include "table/format.h"
|
||||
#include "table/get_context.h"
|
||||
#include "table/internal_iterator.h"
|
||||
#include "table/merger.h"
|
||||
#include "table/meta_blocks.h"
|
||||
#include "table/plain_table_factory.h"
|
||||
#include "table/table_reader.h"
|
||||
#include "table/merger.h"
|
||||
#include "table/two_level_iterator.h"
|
||||
#include "table/format.h"
|
||||
#include "table/plain_table_factory.h"
|
||||
#include "table/meta_blocks.h"
|
||||
#include "table/get_context.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/file_reader_writer.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/perf_context_imp.h"
|
||||
#include "util/stop_watch.h"
|
||||
#include "util/sync_point.h"
|
||||
|
||||
@@ -91,9 +90,8 @@ class FilePicker {
|
||||
const Comparator* user_comparator,
|
||||
const InternalKeyComparator* internal_comparator)
|
||||
: num_levels_(num_levels),
|
||||
curr_level_(static_cast<unsigned int>(-1)),
|
||||
returned_file_level_(static_cast<unsigned int>(-1)),
|
||||
hit_file_level_(static_cast<unsigned int>(-1)),
|
||||
curr_level_(-1),
|
||||
hit_file_level_(-1),
|
||||
search_left_bound_(0),
|
||||
search_right_bound_(FileIndexer::kLevelMaxIndex),
|
||||
#ifndef NDEBUG
|
||||
@@ -119,8 +117,6 @@ class FilePicker {
|
||||
}
|
||||
}
|
||||
|
||||
int GetCurrentLevel() { return returned_file_level_; }
|
||||
|
||||
FdWithKeyRange* GetNextFile() {
|
||||
while (!search_ended_) { // Loops over different levels.
|
||||
while (curr_index_in_curr_level_ < curr_file_level_->num_files) {
|
||||
@@ -193,7 +189,6 @@ class FilePicker {
|
||||
}
|
||||
prev_file_ = f;
|
||||
#endif
|
||||
returned_file_level_ = curr_level_;
|
||||
if (curr_level_ > 0 && cmp_largest < 0) {
|
||||
// No more files to search in this level.
|
||||
search_ended_ = !PrepareNextLevel();
|
||||
@@ -220,7 +215,6 @@ class FilePicker {
|
||||
private:
|
||||
unsigned int num_levels_;
|
||||
unsigned int curr_level_;
|
||||
unsigned int returned_file_level_;
|
||||
unsigned int hit_file_level_;
|
||||
int32_t search_left_bound_;
|
||||
int32_t search_right_bound_;
|
||||
@@ -327,7 +321,7 @@ Version::~Version() {
|
||||
f->refs--;
|
||||
if (f->refs <= 0) {
|
||||
if (f->table_reader_handle) {
|
||||
cfd_->table_cache()->EraseHandle(f->fd, f->table_reader_handle);
|
||||
cfd_->table_cache()->ReleaseHandle(f->table_reader_handle);
|
||||
f->table_reader_handle = nullptr;
|
||||
}
|
||||
vset_->obsolete_files_.push_back(f);
|
||||
@@ -491,7 +485,7 @@ class LevelFileIteratorState : public TwoLevelIteratorState {
|
||||
const EnvOptions& env_options,
|
||||
const InternalKeyComparator& icomparator,
|
||||
HistogramImpl* file_read_hist, bool for_compaction,
|
||||
bool prefix_enabled, bool skip_filters, int level)
|
||||
bool prefix_enabled, bool skip_filters)
|
||||
: TwoLevelIteratorState(prefix_enabled),
|
||||
table_cache_(table_cache),
|
||||
read_options_(read_options),
|
||||
@@ -499,8 +493,7 @@ class LevelFileIteratorState : public TwoLevelIteratorState {
|
||||
icomparator_(icomparator),
|
||||
file_read_hist_(file_read_hist),
|
||||
for_compaction_(for_compaction),
|
||||
skip_filters_(skip_filters),
|
||||
level_(level) {}
|
||||
skip_filters_(skip_filters) {}
|
||||
|
||||
InternalIterator* NewSecondaryIterator(const Slice& meta_handle) override {
|
||||
if (meta_handle.size() != sizeof(FileDescriptor)) {
|
||||
@@ -512,7 +505,7 @@ class LevelFileIteratorState : public TwoLevelIteratorState {
|
||||
return table_cache_->NewIterator(
|
||||
read_options_, env_options_, icomparator_, *fd,
|
||||
nullptr /* don't need reference to table*/, file_read_hist_,
|
||||
for_compaction_, nullptr /* arena */, skip_filters_, level_);
|
||||
for_compaction_, nullptr /* arena */, skip_filters_);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -528,7 +521,6 @@ class LevelFileIteratorState : public TwoLevelIteratorState {
|
||||
HistogramImpl* file_read_hist_;
|
||||
bool for_compaction_;
|
||||
bool skip_filters_;
|
||||
int level_;
|
||||
};
|
||||
|
||||
// A wrapper of version builder which references the current version in
|
||||
@@ -796,8 +788,7 @@ void Version::AddIterators(const ReadOptions& read_options,
|
||||
const auto& file = storage_info_.LevelFilesBrief(0).files[i];
|
||||
merge_iter_builder->AddIterator(cfd_->table_cache()->NewIterator(
|
||||
read_options, soptions, cfd_->internal_comparator(), file.fd, nullptr,
|
||||
cfd_->internal_stats()->GetFileReadHist(0), false, arena,
|
||||
false /* skip_filters */, 0 /* level */));
|
||||
cfd_->internal_stats()->GetFileReadHist(0), false, arena));
|
||||
}
|
||||
|
||||
// For levels > 0, we can use a concatenating iterator that sequentially
|
||||
@@ -812,7 +803,7 @@ void Version::AddIterators(const ReadOptions& read_options,
|
||||
cfd_->internal_stats()->GetFileReadHist(level),
|
||||
false /* for_compaction */,
|
||||
cfd_->ioptions()->prefix_extractor != nullptr,
|
||||
IsFilterSkipped(level), level);
|
||||
IsFilterSkipped(level));
|
||||
mem = arena->AllocateAligned(sizeof(LevelFileNumIterator));
|
||||
auto* first_level_iter = new (mem) LevelFileNumIterator(
|
||||
cfd_->internal_comparator(), &storage_info_.LevelFilesBrief(level));
|
||||
@@ -917,8 +908,7 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,
|
||||
read_options, *internal_comparator(), f->fd, ikey, &get_context,
|
||||
cfd_->internal_stats()->GetFileReadHist(fp.GetHitFileLevel()),
|
||||
IsFilterSkipped(static_cast<int>(fp.GetHitFileLevel()),
|
||||
fp.IsHitFileLastInLevel()),
|
||||
fp.GetCurrentLevel());
|
||||
fp.IsHitFileLastInLevel()));
|
||||
// TODO: examine the behavior for corrupted key
|
||||
if (!status->ok()) {
|
||||
return;
|
||||
@@ -958,16 +948,9 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,
|
||||
}
|
||||
// merge_operands are in saver and we hit the beginning of the key history
|
||||
// do a final merge of nullptr and operands;
|
||||
bool merge_success = false;
|
||||
{
|
||||
StopWatchNano timer(env_, db_statistics_ != nullptr);
|
||||
PERF_TIMER_GUARD(merge_operator_time_nanos);
|
||||
merge_success = merge_operator_->FullMerge(
|
||||
user_key, nullptr, merge_context->GetOperands(), value, info_log_);
|
||||
RecordTick(db_statistics_, MERGE_OPERATION_TOTAL_TIME,
|
||||
timer.ElapsedNanos());
|
||||
}
|
||||
if (merge_success) {
|
||||
if (merge_operator_->FullMerge(user_key, nullptr,
|
||||
merge_context->GetOperands(), value,
|
||||
info_log_)) {
|
||||
*status = Status::OK();
|
||||
} else {
|
||||
RecordTick(db_statistics_, NUMBER_MERGE_FAILURES);
|
||||
@@ -1079,12 +1062,6 @@ void Version::UpdateAccumulatedStats(bool update_stats) {
|
||||
if (MaybeInitializeFileMetaData(file_meta)) {
|
||||
// each FileMeta will be initialized only once.
|
||||
storage_info_.UpdateAccumulatedStats(file_meta);
|
||||
// when option "max_open_files" is -1, all the file metadata has
|
||||
// already been read, so MaybeInitializeFileMetaData() won't incur
|
||||
// any I/O cost.
|
||||
if (vset_->db_options_->max_open_files == -1) {
|
||||
continue;
|
||||
}
|
||||
if (++init_count >= kMaxInitCount) {
|
||||
break;
|
||||
}
|
||||
@@ -2077,16 +2054,9 @@ VersionSet::VersionSet(const std::string& dbname, const DBOptions* db_options,
|
||||
env_options_(storage_options),
|
||||
env_options_compactions_(env_options_) {}
|
||||
|
||||
void CloseTables(void* ptr, size_t) {
|
||||
TableReader* table_reader = reinterpret_cast<TableReader*>(ptr);
|
||||
table_reader->Close();
|
||||
}
|
||||
|
||||
VersionSet::~VersionSet() {
|
||||
// we need to delete column_family_set_ because its destructor depends on
|
||||
// VersionSet
|
||||
column_family_set_->get_table_cache()->ApplyToAllCacheEntries(&CloseTables,
|
||||
false);
|
||||
column_family_set_.reset();
|
||||
for (auto file : obsolete_files_) {
|
||||
delete file;
|
||||
@@ -2336,14 +2306,11 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
|
||||
manifest_file_size_ = new_manifest_file_size;
|
||||
prev_log_number_ = edit->prev_log_number_;
|
||||
} else {
|
||||
std::string version_edits;
|
||||
for (auto& e : batch_edits) {
|
||||
version_edits = version_edits + "\n" + e->DebugString(true);
|
||||
}
|
||||
Log(InfoLogLevel::ERROR_LEVEL, db_options_->info_log,
|
||||
"[%s] Error in committing version edit to MANIFEST: %s",
|
||||
column_family_data ? column_family_data->GetName().c_str() : "<null>",
|
||||
version_edits.c_str());
|
||||
"Error in committing version %lu to [%s]",
|
||||
(unsigned long)v->GetVersionNumber(),
|
||||
column_family_data ? column_family_data->GetName().c_str()
|
||||
: "<null>");
|
||||
delete v;
|
||||
if (new_descriptor_log) {
|
||||
Log(InfoLogLevel::INFO_LEVEL, db_options_->info_log,
|
||||
@@ -3273,7 +3240,7 @@ void VersionSet::AddLiveFiles(std::vector<FileDescriptor>* live_list) {
|
||||
}
|
||||
}
|
||||
|
||||
InternalIterator* VersionSet::MakeInputIterator(const Compaction* c) {
|
||||
InternalIterator* VersionSet::MakeInputIterator(Compaction* c) {
|
||||
auto cfd = c->column_family_data();
|
||||
ReadOptions read_options;
|
||||
read_options.verify_checksums =
|
||||
@@ -3300,8 +3267,7 @@ InternalIterator* VersionSet::MakeInputIterator(const Compaction* c) {
|
||||
read_options, env_options_compactions_,
|
||||
cfd->internal_comparator(), flevel->files[i].fd, nullptr,
|
||||
nullptr, /* no per level latency histogram*/
|
||||
true /* for_compaction */, nullptr /* arena */,
|
||||
false /* skip_filters */, (int)which /* level */);
|
||||
true /* for compaction */);
|
||||
}
|
||||
} else {
|
||||
// Create concatenating iterator for the files from this level
|
||||
@@ -3311,7 +3277,7 @@ InternalIterator* VersionSet::MakeInputIterator(const Compaction* c) {
|
||||
cfd->internal_comparator(),
|
||||
nullptr /* no per level latency histogram */,
|
||||
true /* for_compaction */, false /* prefix enabled */,
|
||||
false /* skip_filters */, (int)which /* level */),
|
||||
false /* skip_filters */),
|
||||
new LevelFileNumIterator(cfd->internal_comparator(),
|
||||
c->input_levels(which)));
|
||||
}
|
||||
|
||||
+1
-1
@@ -671,7 +671,7 @@ class VersionSet {
|
||||
|
||||
// Create an iterator that reads over the compaction inputs for "*c".
|
||||
// The caller should delete the iterator when no longer needed.
|
||||
InternalIterator* MakeInputIterator(const Compaction* c);
|
||||
InternalIterator* MakeInputIterator(Compaction* c);
|
||||
|
||||
// Add all files listed in any live version to *live.
|
||||
void AddLiveFiles(std::vector<FileDescriptor>* live_list);
|
||||
|
||||
+1
-1
@@ -455,7 +455,7 @@ Status WalManager::ReadFirstLine(const std::string& fname,
|
||||
|
||||
if (reader.ReadRecord(&record, &scratch) &&
|
||||
(status.ok() || !db_options_.paranoid_checks)) {
|
||||
if (record.size() < WriteBatchInternal::kHeader) {
|
||||
if (record.size() < 12) {
|
||||
reporter.Corruption(record.size(),
|
||||
Status::Corruption("log record too small"));
|
||||
// TODO read record's till the first no corrupt entry?
|
||||
|
||||
+12
-31
@@ -81,6 +81,8 @@ struct BatchContentClassifier : public WriteBatch::Handler {
|
||||
|
||||
} // anon namespace
|
||||
|
||||
// WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
|
||||
static const size_t kHeader = 12;
|
||||
|
||||
struct SavePoint {
|
||||
size_t size; // size of rep_
|
||||
@@ -94,9 +96,8 @@ struct SavePoints {
|
||||
|
||||
WriteBatch::WriteBatch(size_t reserved_bytes)
|
||||
: save_points_(nullptr), content_flags_(0), rep_() {
|
||||
rep_.reserve((reserved_bytes > WriteBatchInternal::kHeader) ?
|
||||
reserved_bytes : WriteBatchInternal::kHeader);
|
||||
rep_.resize(WriteBatchInternal::kHeader);
|
||||
rep_.reserve((reserved_bytes > kHeader) ? reserved_bytes : kHeader);
|
||||
rep_.resize(kHeader);
|
||||
}
|
||||
|
||||
WriteBatch::WriteBatch(const std::string& rep)
|
||||
@@ -145,7 +146,7 @@ bool WriteBatch::Handler::Continue() {
|
||||
|
||||
void WriteBatch::Clear() {
|
||||
rep_.clear();
|
||||
rep_.resize(WriteBatchInternal::kHeader);
|
||||
rep_.resize(kHeader);
|
||||
|
||||
content_flags_.store(0, std::memory_order_relaxed);
|
||||
|
||||
@@ -192,23 +193,6 @@ bool WriteBatch::HasMerge() const {
|
||||
return (ComputeContentFlags() & ContentFlags::HAS_MERGE) != 0;
|
||||
}
|
||||
|
||||
bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key, bool cf_record) {
|
||||
assert(input != nullptr && key != nullptr);
|
||||
// Skip tag byte
|
||||
input->remove_prefix(1);
|
||||
|
||||
if (cf_record) {
|
||||
// Skip column_family bytes
|
||||
uint32_t cf;
|
||||
if (!GetVarint32(input, &cf)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Extract key
|
||||
return GetLengthPrefixedSlice(input, key);
|
||||
}
|
||||
|
||||
Status ReadRecordFromWriteBatch(Slice* input, char* tag,
|
||||
uint32_t* column_family, Slice* key,
|
||||
Slice* value, Slice* blob) {
|
||||
@@ -265,11 +249,11 @@ Status ReadRecordFromWriteBatch(Slice* input, char* tag,
|
||||
|
||||
Status WriteBatch::Iterate(Handler* handler) const {
|
||||
Slice input(rep_);
|
||||
if (input.size() < WriteBatchInternal::kHeader) {
|
||||
if (input.size() < kHeader) {
|
||||
return Status::Corruption("malformed WriteBatch (too small)");
|
||||
}
|
||||
|
||||
input.remove_prefix(WriteBatchInternal::kHeader);
|
||||
input.remove_prefix(kHeader);
|
||||
Slice key, value, blob;
|
||||
int found = 0;
|
||||
Status s;
|
||||
@@ -345,9 +329,7 @@ void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) {
|
||||
EncodeFixed64(&b->rep_[0], seq);
|
||||
}
|
||||
|
||||
size_t WriteBatchInternal::GetFirstOffset(WriteBatch* b) {
|
||||
return WriteBatchInternal::kHeader;
|
||||
}
|
||||
size_t WriteBatchInternal::GetFirstOffset(WriteBatch* b) { return kHeader; }
|
||||
|
||||
void WriteBatchInternal::Put(WriteBatch* b, uint32_t column_family_id,
|
||||
const Slice& key, const Slice& value) {
|
||||
@@ -850,16 +832,15 @@ Status WriteBatchInternal::InsertInto(const WriteBatch* batch,
|
||||
}
|
||||
|
||||
void WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) {
|
||||
assert(contents.size() >= WriteBatchInternal::kHeader);
|
||||
assert(contents.size() >= kHeader);
|
||||
b->rep_.assign(contents.data(), contents.size());
|
||||
b->content_flags_.store(ContentFlags::DEFERRED, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
void WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src) {
|
||||
SetCount(dst, Count(dst) + Count(src));
|
||||
assert(src->rep_.size() >= WriteBatchInternal::kHeader);
|
||||
dst->rep_.append(src->rep_.data() + WriteBatchInternal::kHeader,
|
||||
src->rep_.size() - WriteBatchInternal::kHeader);
|
||||
assert(src->rep_.size() >= kHeader);
|
||||
dst->rep_.append(src->rep_.data() + kHeader, src->rep_.size() - kHeader);
|
||||
dst->content_flags_.store(
|
||||
dst->content_flags_.load(std::memory_order_relaxed) |
|
||||
src->content_flags_.load(std::memory_order_relaxed),
|
||||
@@ -871,7 +852,7 @@ size_t WriteBatchInternal::AppendedByteSize(size_t leftByteSize,
|
||||
if (leftByteSize == 0 || rightByteSize == 0) {
|
||||
return leftByteSize + rightByteSize;
|
||||
} else {
|
||||
return leftByteSize + rightByteSize - WriteBatchInternal::kHeader;
|
||||
return leftByteSize + rightByteSize - kHeader;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -63,10 +63,6 @@ class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables {
|
||||
// WriteBatch that we don't want in the public WriteBatch interface.
|
||||
class WriteBatchInternal {
|
||||
public:
|
||||
|
||||
// WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
|
||||
static const size_t kHeader = 12;
|
||||
|
||||
// WriteBatch methods with column_family_id instead of ColumnFamilyHandle*
|
||||
static void Put(WriteBatch* batch, uint32_t column_family_id,
|
||||
const Slice& key, const Slice& value);
|
||||
|
||||
@@ -138,7 +138,6 @@
|
||||
block_size=8192
|
||||
block_restart_interval=16
|
||||
cache_index_and_filter_blocks=false
|
||||
pin_l0_filter_and_index_blocks_in_cache=false
|
||||
index_type=kBinarySearch
|
||||
hash_index_allow_collision=true
|
||||
flush_block_policy_factory=FlushBlockBySizePolicyFactory
|
||||
+1
-1
@@ -12,7 +12,7 @@ underlying filesystem.
|
||||
If you want to compile rocksdb with hdfs support, please set the following
|
||||
environment variables appropriately (also defined in setup.sh for convenience)
|
||||
USE_HDFS=1
|
||||
JAVA_HOME=/usr/local/jdk-7u79-64
|
||||
JAVA_HOME=/usr/local/jdk-6u22-64
|
||||
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/jdk-6u22-64/jre/lib/amd64/server:/usr/local/jdk-6u22-64/jre/lib/amd64/:./snappy/libs
|
||||
make clean all db_bench
|
||||
|
||||
|
||||
@@ -451,9 +451,6 @@ extern ROCKSDB_LIBRARY_API void
|
||||
rocksdb_block_based_options_set_cache_index_and_filter_blocks(
|
||||
rocksdb_block_based_table_options_t*, unsigned char);
|
||||
extern ROCKSDB_LIBRARY_API void
|
||||
rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(
|
||||
rocksdb_block_based_table_options_t*, unsigned char);
|
||||
extern ROCKSDB_LIBRARY_API void
|
||||
rocksdb_block_based_options_set_skip_table_builder_flush(
|
||||
rocksdb_block_based_table_options_t* options, unsigned char);
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_block_based_table_factory(
|
||||
|
||||
@@ -146,10 +146,6 @@ class Cache {
|
||||
virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
|
||||
bool thread_safe) = 0;
|
||||
|
||||
// Remove all entries.
|
||||
// Prerequisit: no entry is referenced.
|
||||
virtual void EraseUnRefEntries() = 0;
|
||||
|
||||
private:
|
||||
void LRU_Remove(Handle* e);
|
||||
void LRU_Append(Handle* e);
|
||||
|
||||
@@ -61,7 +61,7 @@ struct CompactionJobStats {
|
||||
uint64_t num_corrupt_keys;
|
||||
|
||||
// Following counters are only populated if
|
||||
// options.report_bg_io_stats = true;
|
||||
// options.compaction_measure_io_stats = true;
|
||||
|
||||
// Time spent on file's Append() call.
|
||||
uint64_t file_write_nanos;
|
||||
|
||||
@@ -561,7 +561,7 @@ class WritableFile {
|
||||
* underlying storage of a file (generally via fallocate) if the Env
|
||||
* instance supports it.
|
||||
*/
|
||||
virtual void SetPreallocationBlockSize(size_t size) {
|
||||
void SetPreallocationBlockSize(size_t size) {
|
||||
preallocation_block_size_ = size;
|
||||
}
|
||||
|
||||
@@ -597,7 +597,7 @@ class WritableFile {
|
||||
// of space on devices where it can result in less file
|
||||
// fragmentation and/or less waste from over-zealous filesystem
|
||||
// pre-allocation.
|
||||
virtual void PrepareWrite(size_t offset, size_t len) {
|
||||
void PrepareWrite(size_t offset, size_t len) {
|
||||
if (preallocation_block_size_ == 0) {
|
||||
return;
|
||||
}
|
||||
@@ -950,13 +950,6 @@ class WritableFileWrapper : public WritableFile {
|
||||
return target_->InvalidateCache(offset, length);
|
||||
}
|
||||
|
||||
virtual void SetPreallocationBlockSize(size_t size) override {
|
||||
target_->SetPreallocationBlockSize(size);
|
||||
}
|
||||
virtual void PrepareWrite(size_t offset, size_t len) override {
|
||||
target_->PrepareWrite(offset, len);
|
||||
}
|
||||
|
||||
protected:
|
||||
Status Allocate(uint64_t offset, uint64_t len) override {
|
||||
return target_->Allocate(offset, len);
|
||||
|
||||
+16
-57
@@ -158,15 +158,7 @@ struct DbPath {
|
||||
struct Options;
|
||||
|
||||
struct ColumnFamilyOptions {
|
||||
// The function recovers options to a previous version. Only 4.6 or later
|
||||
// versions are supported.
|
||||
ColumnFamilyOptions* OldDefaults(int rocksdb_major_version = 4,
|
||||
int rocksdb_minor_version = 6);
|
||||
|
||||
// Some functions that make it easier to optimize RocksDB
|
||||
// Use if if your DB is very small (like under 1GB) and you don't want to
|
||||
// spend lots of memory for memtables.
|
||||
ColumnFamilyOptions* OptimizeForSmallDb();
|
||||
|
||||
// Use this if you don't need to keep the data sorted, i.e. you'll never use
|
||||
// an iterator, only Put() and Get() API calls
|
||||
@@ -262,7 +254,7 @@ struct ColumnFamilyOptions {
|
||||
// Note that write_buffer_size is enforced per column family.
|
||||
// See db_write_buffer_size for sharing memory across column families.
|
||||
//
|
||||
// Default: 64MB
|
||||
// Default: 4MB
|
||||
//
|
||||
// Dynamically changeable through SetOptions() API
|
||||
size_t write_buffer_size;
|
||||
@@ -408,7 +400,7 @@ struct ColumnFamilyOptions {
|
||||
// be 2MB, and each file on level 2 will be 20MB,
|
||||
// and each file on level-3 will be 200MB.
|
||||
//
|
||||
// Default: 64MB.
|
||||
// Default: 2MB.
|
||||
//
|
||||
// Dynamically changeable through SetOptions() API
|
||||
uint64_t target_file_size_base;
|
||||
@@ -423,12 +415,12 @@ struct ColumnFamilyOptions {
|
||||
// max_bytes_for_level_base is the max total for level-1.
|
||||
// Maximum number of bytes for level L can be calculated as
|
||||
// (max_bytes_for_level_base) * (max_bytes_for_level_multiplier ^ (L-1))
|
||||
// For example, if max_bytes_for_level_base is 200MB, and if
|
||||
// For example, if max_bytes_for_level_base is 20MB, and if
|
||||
// max_bytes_for_level_multiplier is 10, total data size for level-1
|
||||
// will be 2GB, total file size for level-2 will be 20GB,
|
||||
// and total file size for level-3 will be 200GB.
|
||||
// will be 20MB, total file size for level-2 will be 200MB,
|
||||
// and total file size for level-3 will be 2GB.
|
||||
//
|
||||
// Default: 256MB.
|
||||
// Default: 10MB.
|
||||
//
|
||||
// Dynamically changeable through SetOptions() API
|
||||
uint64_t max_bytes_for_level_base;
|
||||
@@ -546,13 +538,13 @@ struct ColumnFamilyOptions {
|
||||
// All writes will be slowed down to at least delayed_write_rate if estimated
|
||||
// bytes needed to be compaction exceed this threshold.
|
||||
//
|
||||
// Default: 64GB
|
||||
// Default: 0 (disabled)
|
||||
uint64_t soft_pending_compaction_bytes_limit;
|
||||
|
||||
// All writes are stopped if estimated bytes needed to be compaction exceed
|
||||
// this threshold.
|
||||
//
|
||||
// Default: 256GB
|
||||
// Default: 0 (disabled)
|
||||
uint64_t hard_pending_compaction_bytes_limit;
|
||||
|
||||
// DEPRECATED -- this options is no longer used
|
||||
@@ -790,9 +782,9 @@ struct ColumnFamilyOptions {
|
||||
// Default: false
|
||||
bool paranoid_file_checks;
|
||||
|
||||
// Measure IO stats in compactions and flushes, if true.
|
||||
// Measure IO stats in compactions, if true.
|
||||
// Default: false
|
||||
bool report_bg_io_stats;
|
||||
bool compaction_measure_io_stats;
|
||||
|
||||
// Create ColumnFamilyOptions with default values for all fields
|
||||
ColumnFamilyOptions();
|
||||
@@ -803,10 +795,6 @@ struct ColumnFamilyOptions {
|
||||
};
|
||||
|
||||
struct DBOptions {
|
||||
// The function recovers options to the option as in version 4.6.
|
||||
DBOptions* OldDefaults(int rocksdb_major_version = 4,
|
||||
int rocksdb_minor_version = 6);
|
||||
|
||||
// Some functions that make it easier to optimize RocksDB
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
@@ -849,19 +837,10 @@ struct DBOptions {
|
||||
// Default: nullptr
|
||||
std::shared_ptr<RateLimiter> rate_limiter;
|
||||
|
||||
// Use to track SST files and control their file deletion rate.
|
||||
//
|
||||
// Features:
|
||||
// - Throttle the deletion rate of the SST files.
|
||||
// - Keep track the total size of all SST files.
|
||||
// - Set a maximum allowed space limit for SST files that when reached
|
||||
// the DB wont do any further flushes or compactions and will set the
|
||||
// background error.
|
||||
// - Can be shared between multiple dbs.
|
||||
// Limitations:
|
||||
// - Only track and throttle deletes of SST files in
|
||||
// first db_path (db_name if db_paths is empty).
|
||||
//
|
||||
// Use to track SST files and control their file deletion rate, can be used
|
||||
// among multiple RocksDB instances, sst_file_manager only track and throttle
|
||||
// deletes of SST files in first db_path (db_name if db_paths is empty), other
|
||||
// files and other db_paths wont be tracked or affected by sst_file_manager.
|
||||
// Default: nullptr
|
||||
std::shared_ptr<SstFileManager> sst_file_manager;
|
||||
|
||||
@@ -1062,23 +1041,7 @@ struct DBOptions {
|
||||
// large amounts of data (such as xfs's allocsize option).
|
||||
size_t manifest_preallocation_size;
|
||||
|
||||
// Hint the OS that it should not buffer disk I/O. Enabling this
|
||||
// parameter may improve performance but increases pressure on the
|
||||
// system cache.
|
||||
//
|
||||
// The exact behavior of this parameter is platform dependent.
|
||||
//
|
||||
// On POSIX systems, after RocksDB reads data from disk it will
|
||||
// mark the pages as "unneeded". The operating system may - or may not
|
||||
// - evict these pages from memory, reducing pressure on the system
|
||||
// cache. If the disk block is requested again this can result in
|
||||
// additional disk I/O.
|
||||
//
|
||||
// On WINDOWS system, files will be opened in "unbuffered I/O" mode
|
||||
// which means that data read from the disk will not be cached or
|
||||
// bufferized. The hardware buffer of the devices may however still
|
||||
// be used. Memory mapped files are not impacted by this parameter.
|
||||
//
|
||||
// Data being read from file storage may be buffered in the OS
|
||||
// Default: true
|
||||
bool allow_os_buffer;
|
||||
|
||||
@@ -1295,7 +1258,7 @@ struct DBOptions {
|
||||
// records, ignoring a particular record or skipping replay.
|
||||
// The filter is invoked at startup and is invoked from a single-thread
|
||||
// currently.
|
||||
WalFilter* wal_filter;
|
||||
const WalFilter* wal_filter;
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
// If true, then DB::Open / CreateColumnFamily / DropColumnFamily
|
||||
@@ -1315,10 +1278,6 @@ struct Options : public DBOptions, public ColumnFamilyOptions {
|
||||
const ColumnFamilyOptions& column_family_options)
|
||||
: DBOptions(db_options), ColumnFamilyOptions(column_family_options) {}
|
||||
|
||||
// The function recovers options to the option as in version 4.6.
|
||||
Options* OldDefaults(int rocksdb_major_version = 4,
|
||||
int rocksdb_minor_version = 6);
|
||||
|
||||
void Dump(Logger* log) const;
|
||||
|
||||
void DumpCFOptions(Logger* log) const;
|
||||
|
||||
@@ -30,32 +30,10 @@ struct PerfContext {
|
||||
uint64_t block_read_time; // total nanos spent on block reads
|
||||
uint64_t block_checksum_time; // total nanos spent on block checksum
|
||||
uint64_t block_decompress_time; // total nanos spent on block decompression
|
||||
// total number of internal keys skipped over during iteration.
|
||||
// There are several reasons for it:
|
||||
// 1. when calling Next(), the iterator is in the position of the previous
|
||||
// key, so that we'll need to skip it. It means this counter will always
|
||||
// be incremented in Next().
|
||||
// 2. when calling Next(), we need to skip internal entries for the previous
|
||||
// keys that are overwritten.
|
||||
// 3. when calling Next(), Seek() or SeekToFirst(), after previous key
|
||||
// before calling Next(), the seek key in Seek() or the beginning for
|
||||
// SeekToFirst(), there may be one or more deleted keys before the next
|
||||
// valid key that the operation should place the iterator to. We need
|
||||
// to skip both of the tombstone and updates hidden by the tombstones. The
|
||||
// tombstones are not included in this counter, while previous updates
|
||||
// hidden by the tombstones will be included here.
|
||||
// 4. symmetric cases for Prev() and SeekToLast()
|
||||
// We sometimes also skip entries of more recent updates than the snapshot
|
||||
// we read from, but they are not included in this counter.
|
||||
//
|
||||
// total number of internal keys skipped over during iteration (overwritten or
|
||||
// deleted, to be more specific, hidden by a put or delete of the same key)
|
||||
uint64_t internal_key_skipped_count;
|
||||
// Total number of deletes and single deletes skipped over during iteration
|
||||
// When calling Next(), Seek() or SeekToFirst(), after previous position
|
||||
// before calling Next(), the seek key in Seek() or the beginning for
|
||||
// SeekToFirst(), there may be one or more deleted keys before the next valid
|
||||
// key. Every deleted key is counted once. We don't recount here if there are
|
||||
// still older updates invalidated by the tombstones.
|
||||
//
|
||||
// total number of deletes and single deletes skipped over during iteration
|
||||
uint64_t internal_delete_skipped_count;
|
||||
|
||||
uint64_t get_snapshot_time; // total nanos spent on getting snapshot
|
||||
|
||||
@@ -79,16 +79,8 @@ class Slice {
|
||||
}
|
||||
|
||||
// Return a string that contains the copy of the referenced data.
|
||||
// when hex is true, returns a string of twice the length hex encoded (0-9A-F)
|
||||
std::string ToString(bool hex = false) const;
|
||||
|
||||
// Decodes the current slice interpreted as an hexadecimal string into result,
|
||||
// if successful returns true, if this isn't a valid hex string
|
||||
// (e.g not coming from Slice::ToString(true)) DecodeHex returns false.
|
||||
// This slice is expected to have an even number of 0-9A-F characters
|
||||
// also accepts lowercase (a-f)
|
||||
bool DecodeHex(std::string* result) const;
|
||||
|
||||
// Three-way comparison. Returns value:
|
||||
// < 0 iff "*this" < "b",
|
||||
// == 0 iff "*this" == "b",
|
||||
|
||||
@@ -64,12 +64,6 @@ struct BlockBasedTableOptions {
|
||||
// block during table initialization.
|
||||
bool cache_index_and_filter_blocks = false;
|
||||
|
||||
// if cache_index_and_filter_blocks is true and the below is true, then
|
||||
// filter and index blocks are stored in the cache, but a reference is
|
||||
// held in the "table reader" object so the blocks are pinned and only
|
||||
// evicted from cache when the table reader is freed.
|
||||
bool pin_l0_filter_and_index_blocks_in_cache = false;
|
||||
|
||||
// The index type that will be used for this table.
|
||||
enum IndexType : char {
|
||||
// A space efficient index block that is optimized for
|
||||
|
||||
@@ -27,6 +27,47 @@ namespace rocksdb {
|
||||
// }
|
||||
typedef std::map<std::string, std::string> UserCollectedProperties;
|
||||
|
||||
// TableProperties contains a bunch of read-only properties of its associated
|
||||
// table.
|
||||
struct TableProperties {
|
||||
public:
|
||||
// the total size of all data blocks.
|
||||
uint64_t data_size = 0;
|
||||
// the size of index block.
|
||||
uint64_t index_size = 0;
|
||||
// the size of filter block.
|
||||
uint64_t filter_size = 0;
|
||||
// total raw key size
|
||||
uint64_t raw_key_size = 0;
|
||||
// total raw value size
|
||||
uint64_t raw_value_size = 0;
|
||||
// the number of blocks in this table
|
||||
uint64_t num_data_blocks = 0;
|
||||
// the number of entries in this table
|
||||
uint64_t num_entries = 0;
|
||||
// format version, reserved for backward compatibility
|
||||
uint64_t format_version = 0;
|
||||
// If 0, key is variable length. Otherwise number of bytes for each key.
|
||||
uint64_t fixed_key_len = 0;
|
||||
|
||||
// The name of the filter policy used in this table.
|
||||
// If no filter policy is used, `filter_policy_name` will be an empty string.
|
||||
std::string filter_policy_name;
|
||||
|
||||
// user collected properties
|
||||
UserCollectedProperties user_collected_properties;
|
||||
UserCollectedProperties readable_properties;
|
||||
|
||||
// convert this object to a human readable form
|
||||
// @prop_delim: delimiter for each property.
|
||||
std::string ToString(const std::string& prop_delim = "; ",
|
||||
const std::string& kv_delim = "=") const;
|
||||
|
||||
// Aggregate the numerical member variables of the specified
|
||||
// TableProperties.
|
||||
void Add(const TableProperties& tp);
|
||||
};
|
||||
|
||||
// table properties' human-readable names in the property block.
|
||||
struct TablePropertiesNames {
|
||||
static const std::string kDataSize;
|
||||
@@ -39,8 +80,6 @@ struct TablePropertiesNames {
|
||||
static const std::string kFormatVersion;
|
||||
static const std::string kFixedKeyLen;
|
||||
static const std::string kFilterPolicy;
|
||||
static const std::string kColumnFamilyName;
|
||||
static const std::string kColumnFamilyId;
|
||||
};
|
||||
|
||||
extern const std::string kPropertiesBlock;
|
||||
@@ -119,55 +158,6 @@ class TablePropertiesCollectorFactory {
|
||||
virtual const char* Name() const = 0;
|
||||
};
|
||||
|
||||
// TableProperties contains a bunch of read-only properties of its associated
|
||||
// table.
|
||||
struct TableProperties {
|
||||
public:
|
||||
// the total size of all data blocks.
|
||||
uint64_t data_size = 0;
|
||||
// the size of index block.
|
||||
uint64_t index_size = 0;
|
||||
// the size of filter block.
|
||||
uint64_t filter_size = 0;
|
||||
// total raw key size
|
||||
uint64_t raw_key_size = 0;
|
||||
// total raw value size
|
||||
uint64_t raw_value_size = 0;
|
||||
// the number of blocks in this table
|
||||
uint64_t num_data_blocks = 0;
|
||||
// the number of entries in this table
|
||||
uint64_t num_entries = 0;
|
||||
// format version, reserved for backward compatibility
|
||||
uint64_t format_version = 0;
|
||||
// If 0, key is variable length. Otherwise number of bytes for each key.
|
||||
uint64_t fixed_key_len = 0;
|
||||
// ID of column family for this SST file, corresponding to the CF identified
|
||||
// by column_family_name.
|
||||
uint64_t column_family_id =
|
||||
rocksdb::TablePropertiesCollectorFactory::Context::kUnknownColumnFamily;
|
||||
|
||||
// Name of the column family with which this SST file is associated.
|
||||
// If column family is unknown, `column_family_name` will be an empty string.
|
||||
std::string column_family_name;
|
||||
|
||||
// The name of the filter policy used in this table.
|
||||
// If no filter policy is used, `filter_policy_name` will be an empty string.
|
||||
std::string filter_policy_name;
|
||||
|
||||
// user collected properties
|
||||
UserCollectedProperties user_collected_properties;
|
||||
UserCollectedProperties readable_properties;
|
||||
|
||||
// convert this object to a human readable form
|
||||
// @prop_delim: delimiter for each property.
|
||||
std::string ToString(const std::string& prop_delim = "; ",
|
||||
const std::string& kv_delim = "=") const;
|
||||
|
||||
// Aggregate the numerical member variables of the specified
|
||||
// TableProperties.
|
||||
void Add(const TableProperties& tp);
|
||||
};
|
||||
|
||||
// Extra properties
|
||||
// Below is a list of non-basic properties that are collected by database
|
||||
// itself. Especially some properties regarding to the internal keys (which
|
||||
|
||||
@@ -143,17 +143,13 @@ struct BackupInfo {
|
||||
uint64_t size;
|
||||
|
||||
uint32_t number_files;
|
||||
std::string app_metadata;
|
||||
|
||||
BackupInfo() {}
|
||||
|
||||
BackupInfo(BackupID _backup_id, int64_t _timestamp, uint64_t _size,
|
||||
uint32_t _number_files, const std::string& _app_metadata)
|
||||
: backup_id(_backup_id),
|
||||
timestamp(_timestamp),
|
||||
size(_size),
|
||||
number_files(_number_files),
|
||||
app_metadata(_app_metadata) {}
|
||||
uint32_t _number_files)
|
||||
: backup_id(_backup_id), timestamp(_timestamp), size(_size),
|
||||
number_files(_number_files) {}
|
||||
};
|
||||
|
||||
class BackupStatistics {
|
||||
@@ -222,17 +218,9 @@ class BackupEngine {
|
||||
const BackupableDBOptions& options,
|
||||
BackupEngine** backup_engine_ptr);
|
||||
|
||||
/// same as CreateNewBackup, but stores extra application metadata
|
||||
virtual Status CreateNewBackupWithMetadata(
|
||||
DB* db, const std::string& app_metadata, bool flush_before_backup = false,
|
||||
virtual Status CreateNewBackup(
|
||||
DB* db, bool flush_before_backup = false,
|
||||
std::function<void()> progress_callback = []() {}) = 0;
|
||||
|
||||
virtual Status CreateNewBackup(DB* db, bool flush_before_backup = false,
|
||||
std::function<void()> progress_callback =
|
||||
[]() {}) {
|
||||
return CreateNewBackupWithMetadata(db, "", flush_before_backup,
|
||||
progress_callback);
|
||||
}
|
||||
virtual Status PurgeOldBackups(uint32_t num_backups_to_keep) = 0;
|
||||
virtual Status DeleteBackup(BackupID backup_id) = 0;
|
||||
virtual void StopBackup() = 0;
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
#pragma once
|
||||
|
||||
#define ROCKSDB_MAJOR 4
|
||||
#define ROCKSDB_MINOR 7
|
||||
#define ROCKSDB_MINOR 6
|
||||
#define ROCKSDB_PATCH 1
|
||||
|
||||
// Do not use these. We made the mistake of declaring macros starting with
|
||||
|
||||
@@ -4,8 +4,6 @@
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#pragma once
|
||||
#include <string>
|
||||
#include <map>
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
@@ -32,21 +30,6 @@ class WalFilter {
|
||||
|
||||
virtual ~WalFilter() {}
|
||||
|
||||
// Provide ColumnFamily->LogNumber map to filter
|
||||
// so that filter can determine whether a log number applies to a given
|
||||
// column family (i.e. that log hasn't been flushed to SST already for the
|
||||
// column family).
|
||||
// We also pass in name->id map as only name is known during
|
||||
// recovery (as handles are opened post-recovery).
|
||||
// while write batch callbacks happen in terms of column family id.
|
||||
//
|
||||
// @params cf_lognumber_map column_family_id to lognumber map
|
||||
// @params cf_name_id_map column_family_name to column_family_id map
|
||||
|
||||
virtual void ColumnFamilyLogNumberMap(
|
||||
const std::map<uint32_t, uint64_t>& cf_lognumber_map,
|
||||
const std::map<std::string, uint32_t>& cf_name_id_map) {}
|
||||
|
||||
// LogRecord is invoked for each log record encountered for all the logs
|
||||
// during replay on logs on recovery. This method can be used to:
|
||||
// * inspect the record (using the batch parameter)
|
||||
@@ -58,10 +41,6 @@ class WalFilter {
|
||||
// (by returning kStop replay) - please note that this implies
|
||||
// discarding the logs from current record onwards.
|
||||
//
|
||||
// @params log_number log_number of the current log.
|
||||
// Filter might use this to determine if the log
|
||||
// record is applicable to a certain column family.
|
||||
// @params log_file_name log file name - only for informational purposes
|
||||
// @params batch batch encountered in the log during recovery
|
||||
// @params new_batch new_batch to populate if filter wants to change
|
||||
// the batch (for example to filter some records out,
|
||||
@@ -75,23 +54,9 @@ class WalFilter {
|
||||
// @returns Processing option for the current record.
|
||||
// Please see WalProcessingOption enum above for
|
||||
// details.
|
||||
virtual WalProcessingOption LogRecordFound(unsigned long long log_number,
|
||||
const std::string& log_file_name,
|
||||
const WriteBatch& batch,
|
||||
WriteBatch* new_batch,
|
||||
bool* batch_changed) {
|
||||
// Default implementation falls back to older function for compatibility
|
||||
return LogRecord(batch, new_batch, batch_changed);
|
||||
}
|
||||
|
||||
// Please see the comments for LogRecord above. This function is for
|
||||
// compatibility only and contains a subset of parameters.
|
||||
// New code should use the function above.
|
||||
virtual WalProcessingOption LogRecord(const WriteBatch& batch,
|
||||
WriteBatch* new_batch,
|
||||
bool* batch_changed) const {
|
||||
return WalProcessingOption::kContinueProcessing;
|
||||
}
|
||||
bool* batch_changed) const = 0;
|
||||
|
||||
// Returns a name that identifies this WAL filter.
|
||||
// The name will be printed to LOG file on start up for diagnosis.
|
||||
|
||||
@@ -64,7 +64,6 @@ JAVA_TESTS = org.rocksdb.BackupableDBOptionsTest\
|
||||
org.rocksdb.BackupEngineTest\
|
||||
org.rocksdb.BackupableDBTest\
|
||||
org.rocksdb.BlockBasedTableConfigTest\
|
||||
org.rocksdb.util.BytewiseComparatorTest\
|
||||
org.rocksdb.CheckPointTest\
|
||||
org.rocksdb.ColumnFamilyOptionsTest\
|
||||
org.rocksdb.ColumnFamilyTest\
|
||||
|
||||
@@ -21,14 +21,10 @@
|
||||
*/
|
||||
package org.rocksdb.benchmark;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.Runnable;
|
||||
import java.lang.Math;
|
||||
import java.io.File;
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.file.Files;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.EnumMap;
|
||||
@@ -605,11 +601,6 @@ public class DbBenchmark {
|
||||
(Integer)flags_.get(Flag.max_successive_merges));
|
||||
options.setWalTtlSeconds((Long)flags_.get(Flag.wal_ttl_seconds));
|
||||
options.setWalSizeLimitMB((Long)flags_.get(Flag.wal_size_limit_MB));
|
||||
if(flags_.get(Flag.java_comparator) != null) {
|
||||
options.setComparator(
|
||||
(AbstractComparator)flags_.get(Flag.java_comparator));
|
||||
}
|
||||
|
||||
/* TODO(yhchiang): enable the following parameters
|
||||
options.setCompressionType((String)flags_.get(Flag.compression_type));
|
||||
options.setCompressionLevel((Integer)flags_.get(Flag.compression_level));
|
||||
@@ -783,7 +774,6 @@ public class DbBenchmark {
|
||||
}
|
||||
|
||||
private void open(Options options) throws RocksDBException {
|
||||
System.out.println("Using database directory: " + databaseDir_);
|
||||
db_ = RocksDB.open(options, databaseDir_);
|
||||
}
|
||||
|
||||
@@ -1485,7 +1475,7 @@ public class DbBenchmark {
|
||||
return Integer.parseInt(value);
|
||||
}
|
||||
},
|
||||
db(getTempDir("rocksdb-jni"),
|
||||
db("/tmp/rocksdbjni-bench",
|
||||
"Use the db with the following name.") {
|
||||
@Override public Object parseValue(String value) {
|
||||
return value;
|
||||
@@ -1496,31 +1486,6 @@ public class DbBenchmark {
|
||||
@Override public Object parseValue(String value) {
|
||||
return parseBoolean(value);
|
||||
}
|
||||
},
|
||||
java_comparator(null, "Class name of a Java Comparator to use instead\n" +
|
||||
"\tof the default C++ ByteWiseComparatorImpl. Must be available on\n" +
|
||||
"\tthe classpath") {
|
||||
@Override
|
||||
protected Object parseValue(final String value) {
|
||||
try {
|
||||
final ComparatorOptions copt = new ComparatorOptions();
|
||||
final Class<AbstractComparator> clsComparator =
|
||||
(Class<AbstractComparator>)Class.forName(value);
|
||||
final Constructor cstr =
|
||||
clsComparator.getConstructor(ComparatorOptions.class);
|
||||
return cstr.newInstance(copt);
|
||||
} catch(final ClassNotFoundException cnfe) {
|
||||
throw new IllegalArgumentException("Java Comparator '" + value + "'" +
|
||||
" not found on the classpath", cnfe);
|
||||
} catch(final NoSuchMethodException nsme) {
|
||||
throw new IllegalArgumentException("Java Comparator '" + value + "'" +
|
||||
" does not have a public ComparatorOptions constructor", nsme);
|
||||
} catch(final IllegalAccessException | InstantiationException
|
||||
| InvocationTargetException ie) {
|
||||
throw new IllegalArgumentException("Unable to construct Java" +
|
||||
" Comparator '" + value + "'", ie);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
private Flag(Object defaultValue, String desc) {
|
||||
@@ -1551,18 +1516,6 @@ public class DbBenchmark {
|
||||
private final String desc_;
|
||||
}
|
||||
|
||||
private final static String DEFAULT_TEMP_DIR = "/tmp";
|
||||
|
||||
private static String getTempDir(final String dirName) {
|
||||
try {
|
||||
return Files.createTempDirectory(dirName).toAbsolutePath().toString();
|
||||
} catch(final IOException ioe) {
|
||||
System.err.println("Unable to create temp directory, defaulting to: " +
|
||||
DEFAULT_TEMP_DIR);
|
||||
return DEFAULT_TEMP_DIR + File.pathSeparator + dirName;
|
||||
}
|
||||
}
|
||||
|
||||
private static class RandomGenerator {
|
||||
private final byte[] data_;
|
||||
private int dataLength_;
|
||||
|
||||
@@ -21,17 +21,17 @@
|
||||
/*
|
||||
* Class: org_rocksdb_BackupableDB
|
||||
* Method: open
|
||||
* Signature: (JJ)J
|
||||
* Signature: (JJ)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_BackupableDB_open(
|
||||
JNIEnv* env, jclass jcls, jlong jdb_handle, jlong jopt_handle) {
|
||||
auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
|
||||
auto* opt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jopt_handle);
|
||||
void Java_org_rocksdb_BackupableDB_open(
|
||||
JNIEnv* env, jobject jbdb, jlong jdb_handle, jlong jopt_handle) {
|
||||
auto db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
|
||||
auto opt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jopt_handle);
|
||||
auto bdb = new rocksdb::BackupableDB(db, *opt);
|
||||
|
||||
// as BackupableDB extends RocksDB on the java side, we can reuse
|
||||
// the RocksDB portal here.
|
||||
return reinterpret_cast<jlong>(bdb);
|
||||
rocksdb::RocksDBJni::setHandle(env, jbdb, bdb);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -135,14 +135,14 @@ void Java_org_rocksdb_BackupableDB_garbageCollect(JNIEnv* env,
|
||||
/*
|
||||
* Class: org_rocksdb_BackupableDBOptions
|
||||
* Method: newBackupableDBOptions
|
||||
* Signature: (Ljava/lang/String;)J
|
||||
* Signature: (Ljava/lang/String;)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_BackupableDBOptions_newBackupableDBOptions(
|
||||
JNIEnv* env, jclass jcls, jstring jpath) {
|
||||
const char* cpath = env->GetStringUTFChars(jpath, NULL);
|
||||
void Java_org_rocksdb_BackupableDBOptions_newBackupableDBOptions(
|
||||
JNIEnv* env, jobject jobj, jstring jpath) {
|
||||
const char* cpath = env->GetStringUTFChars(jpath, 0);
|
||||
auto bopt = new rocksdb::BackupableDBOptions(cpath);
|
||||
env->ReleaseStringUTFChars(jpath, cpath);
|
||||
return reinterpret_cast<jlong>(bopt);
|
||||
rocksdb::BackupableDBOptionsJni::setHandle(env, jobj, bopt);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -320,4 +320,5 @@ void Java_org_rocksdb_BackupableDBOptions_disposeInternal(
|
||||
auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
|
||||
assert(bopt);
|
||||
delete bopt;
|
||||
rocksdb::BackupableDBOptionsJni::setHandle(env, jopt, nullptr);
|
||||
}
|
||||
@@ -16,10 +16,10 @@
|
||||
/*
|
||||
* Class: org_rocksdb_BackupEngine
|
||||
* Method: open
|
||||
* Signature: (JJ)J
|
||||
* Signature: (JJ)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_BackupEngine_open(
|
||||
JNIEnv* env, jclass jcls, jlong env_handle,
|
||||
void Java_org_rocksdb_BackupEngine_open(
|
||||
JNIEnv* env, jobject jbe, jlong env_handle,
|
||||
jlong backupable_db_options_handle) {
|
||||
auto* rocks_env = reinterpret_cast<rocksdb::Env*>(env_handle);
|
||||
auto* backupable_db_options =
|
||||
@@ -30,11 +30,11 @@ jlong Java_org_rocksdb_BackupEngine_open(
|
||||
*backupable_db_options, &backup_engine);
|
||||
|
||||
if (status.ok()) {
|
||||
return reinterpret_cast<jlong>(backup_engine);
|
||||
} else {
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, status);
|
||||
return 0;
|
||||
rocksdb::BackupEngineJni::setHandle(env, jbe, backup_engine);
|
||||
return;
|
||||
}
|
||||
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, status);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@@ -36,15 +36,15 @@ void Java_org_rocksdb_AbstractComparator_disposeInternal(
|
||||
/*
|
||||
* Class: org_rocksdb_Comparator
|
||||
* Method: createNewComparator0
|
||||
* Signature: ()J
|
||||
* Signature: ()V
|
||||
*/
|
||||
jlong Java_org_rocksdb_Comparator_createNewComparator0(
|
||||
void Java_org_rocksdb_Comparator_createNewComparator0(
|
||||
JNIEnv* env, jobject jobj, jlong copt_handle) {
|
||||
const rocksdb::ComparatorJniCallbackOptions* copt =
|
||||
reinterpret_cast<rocksdb::ComparatorJniCallbackOptions*>(copt_handle);
|
||||
const rocksdb::ComparatorJniCallback* c =
|
||||
new rocksdb::ComparatorJniCallback(env, jobj, copt);
|
||||
return reinterpret_cast<jlong>(c);
|
||||
rocksdb::AbstractComparatorJni::setHandle(env, jobj, c);
|
||||
}
|
||||
// </editor-fold>
|
||||
|
||||
@@ -53,14 +53,14 @@ jlong Java_org_rocksdb_Comparator_createNewComparator0(
|
||||
/*
|
||||
* Class: org_rocksdb_DirectComparator
|
||||
* Method: createNewDirectComparator0
|
||||
* Signature: ()J
|
||||
* Signature: ()V
|
||||
*/
|
||||
jlong Java_org_rocksdb_DirectComparator_createNewDirectComparator0(
|
||||
void Java_org_rocksdb_DirectComparator_createNewDirectComparator0(
|
||||
JNIEnv* env, jobject jobj, jlong copt_handle) {
|
||||
const rocksdb::ComparatorJniCallbackOptions* copt =
|
||||
reinterpret_cast<rocksdb::ComparatorJniCallbackOptions*>(copt_handle);
|
||||
const rocksdb::DirectComparatorJniCallback* c =
|
||||
new rocksdb::DirectComparatorJniCallback(env, jobj, copt);
|
||||
return reinterpret_cast<jlong>(c);
|
||||
rocksdb::AbstractComparatorJni::setHandle(env, jobj, c);
|
||||
}
|
||||
// </editor-fold>
|
||||
@@ -60,8 +60,8 @@ int BaseComparatorJniCallback::Compare(const Slice& a, const Slice& b) const {
|
||||
// performance.
|
||||
mtx_compare->Lock();
|
||||
|
||||
AbstractSliceJni::setHandle(m_env, m_jSliceA, &a, JNI_FALSE);
|
||||
AbstractSliceJni::setHandle(m_env, m_jSliceB, &b, JNI_FALSE);
|
||||
AbstractSliceJni::setHandle(m_env, m_jSliceA, &a);
|
||||
AbstractSliceJni::setHandle(m_env, m_jSliceB, &b);
|
||||
jint result =
|
||||
m_env->CallIntMethod(m_jComparator, m_jCompareMethodId, m_jSliceA,
|
||||
m_jSliceB);
|
||||
@@ -89,7 +89,7 @@ void BaseComparatorJniCallback::FindShortestSeparator(
|
||||
// performance.
|
||||
mtx_findShortestSeparator->Lock();
|
||||
|
||||
AbstractSliceJni::setHandle(m_env, m_jSliceLimit, &limit, JNI_FALSE);
|
||||
AbstractSliceJni::setHandle(m_env, m_jSliceLimit, &limit);
|
||||
jstring jsResultStart =
|
||||
(jstring)m_env->CallObjectMethod(m_jComparator,
|
||||
m_jFindShortestSeparatorMethodId, jsStart, m_jSliceLimit);
|
||||
|
||||
@@ -19,17 +19,17 @@
|
||||
/*
|
||||
* Class: org_rocksdb_BloomFilter
|
||||
* Method: createBloomFilter
|
||||
* Signature: (IZ)J
|
||||
* Signature: (IZ)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_BloomFilter_createNewBloomFilter(
|
||||
JNIEnv* env, jclass jcls, jint bits_per_key,
|
||||
void Java_org_rocksdb_BloomFilter_createNewBloomFilter(
|
||||
JNIEnv* env, jobject jobj, jint bits_per_key,
|
||||
jboolean use_block_base_builder) {
|
||||
auto* fp = const_cast<rocksdb::FilterPolicy *>(
|
||||
rocksdb::FilterPolicy* fp = const_cast<rocksdb::FilterPolicy *>(
|
||||
rocksdb::NewBloomFilterPolicy(bits_per_key, use_block_base_builder));
|
||||
auto* pFilterPolicy =
|
||||
std::shared_ptr<rocksdb::FilterPolicy> *pFilterPolicy =
|
||||
new std::shared_ptr<rocksdb::FilterPolicy>;
|
||||
*pFilterPolicy = std::shared_ptr<rocksdb::FilterPolicy>(fp);
|
||||
return reinterpret_cast<jlong>(pFilterPolicy);
|
||||
rocksdb::FilterJni::setHandle(env, jobj, pFilterPolicy);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@@ -125,9 +125,9 @@ LoggerJniCallback::~LoggerJniCallback() {
|
||||
/*
|
||||
* Class: org_rocksdb_Logger
|
||||
* Method: createNewLoggerOptions
|
||||
* Signature: (J)J
|
||||
* Signature: (J)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_Logger_createNewLoggerOptions(
|
||||
void Java_org_rocksdb_Logger_createNewLoggerOptions(
|
||||
JNIEnv* env, jobject jobj, jlong joptions) {
|
||||
rocksdb::LoggerJniCallback* c =
|
||||
new rocksdb::LoggerJniCallback(env, jobj);
|
||||
@@ -137,15 +137,15 @@ jlong Java_org_rocksdb_Logger_createNewLoggerOptions(
|
||||
std::shared_ptr<rocksdb::LoggerJniCallback> *pLoggerJniCallback =
|
||||
new std::shared_ptr<rocksdb::LoggerJniCallback>;
|
||||
*pLoggerJniCallback = std::shared_ptr<rocksdb::LoggerJniCallback>(c);
|
||||
return reinterpret_cast<jlong>(pLoggerJniCallback);
|
||||
rocksdb::LoggerJni::setHandle(env, jobj, pLoggerJniCallback);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_Logger
|
||||
* Method: createNewLoggerDbOptions
|
||||
* Signature: (J)J
|
||||
* Signature: (J)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_Logger_createNewLoggerDbOptions(
|
||||
void Java_org_rocksdb_Logger_createNewLoggerDbOptions(
|
||||
JNIEnv* env, jobject jobj, jlong jdb_options) {
|
||||
rocksdb::LoggerJniCallback* c =
|
||||
new rocksdb::LoggerJniCallback(env, jobj);
|
||||
@@ -155,7 +155,7 @@ jlong Java_org_rocksdb_Logger_createNewLoggerDbOptions(
|
||||
std::shared_ptr<rocksdb::LoggerJniCallback> *pLoggerJniCallback =
|
||||
new std::shared_ptr<rocksdb::LoggerJniCallback>;
|
||||
*pLoggerJniCallback = std::shared_ptr<rocksdb::LoggerJniCallback>(c);
|
||||
return reinterpret_cast<jlong>(pLoggerJniCallback);
|
||||
rocksdb::LoggerJni::setHandle(env, jobj, pLoggerJniCallback);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
+71
-60
@@ -36,25 +36,25 @@
|
||||
/*
|
||||
* Class: org_rocksdb_Options
|
||||
* Method: newOptions
|
||||
* Signature: ()J
|
||||
* Signature: ()V
|
||||
*/
|
||||
jlong Java_org_rocksdb_Options_newOptions__(JNIEnv* env, jclass jcls) {
|
||||
void Java_org_rocksdb_Options_newOptions__(JNIEnv* env, jobject jobj) {
|
||||
rocksdb::Options* op = new rocksdb::Options();
|
||||
return reinterpret_cast<jlong>(op);
|
||||
rocksdb::OptionsJni::setHandle(env, jobj, op);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_Options
|
||||
* Method: newOptions
|
||||
* Signature: (JJ)J
|
||||
* Signature: (JJ)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_Options_newOptions__JJ(JNIEnv* env, jclass jcls,
|
||||
void Java_org_rocksdb_Options_newOptions__JJ(JNIEnv* env, jobject jobj,
|
||||
jlong jdboptions, jlong jcfoptions) {
|
||||
auto* dbOpt = reinterpret_cast<const rocksdb::DBOptions*>(jdboptions);
|
||||
auto* cfOpt = reinterpret_cast<const rocksdb::ColumnFamilyOptions*>(
|
||||
auto dbOpt = reinterpret_cast<const rocksdb::DBOptions*>(jdboptions);
|
||||
auto cfOpt = reinterpret_cast<const rocksdb::ColumnFamilyOptions*>(
|
||||
jcfoptions);
|
||||
rocksdb::Options* op = new rocksdb::Options(*dbOpt, *cfOpt);
|
||||
return reinterpret_cast<jlong>(op);
|
||||
rocksdb::OptionsJni::setHandle(env, jobj, op);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -1081,20 +1081,21 @@ jbyte Java_org_rocksdb_Options_compressionType(
|
||||
* vector.
|
||||
*/
|
||||
std::vector<rocksdb::CompressionType> rocksdb_compression_vector_helper(
|
||||
JNIEnv* env, jbyteArray jcompressionLevels) {
|
||||
JNIEnv* env, jobject jcompressionLevels) {
|
||||
std::vector<rocksdb::CompressionType> compressionLevels;
|
||||
|
||||
jsize len = env->GetArrayLength(jcompressionLevels);
|
||||
jbyte* jcompressionLevel = env->GetByteArrayElements(jcompressionLevels,
|
||||
NULL);
|
||||
for(int i = 0; i < len; i++) {
|
||||
jbyte jcl;
|
||||
jcl = jcompressionLevel[i];
|
||||
compressionLevels.push_back(static_cast<rocksdb::CompressionType>(jcl));
|
||||
// iterate over compressionLevels
|
||||
jobject iteratorObj = env->CallObjectMethod(
|
||||
jcompressionLevels, rocksdb::ListJni::getIteratorMethod(env));
|
||||
while (env->CallBooleanMethod(
|
||||
iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
|
||||
// get compression
|
||||
jobject jcompression_obj = env->CallObjectMethod(iteratorObj,
|
||||
rocksdb::ListJni::getNextMethod(env));
|
||||
jbyte jcompression = env->CallByteMethod(jcompression_obj,
|
||||
rocksdb::ByteJni::getByteValueMethod(env));
|
||||
compressionLevels.push_back(static_cast<rocksdb::CompressionType>(
|
||||
jcompression));
|
||||
}
|
||||
env->ReleaseByteArrayElements(jcompressionLevels, jcompressionLevel,
|
||||
JNI_ABORT);
|
||||
|
||||
return compressionLevels;
|
||||
}
|
||||
|
||||
@@ -1102,29 +1103,34 @@ std::vector<rocksdb::CompressionType> rocksdb_compression_vector_helper(
|
||||
* Helper method to convert a CompressionType vector to a Java
|
||||
* List.
|
||||
*/
|
||||
jbyteArray rocksdb_compression_list_helper(JNIEnv* env,
|
||||
jobject rocksdb_compression_list_helper(JNIEnv* env,
|
||||
std::vector<rocksdb::CompressionType> compressionLevels) {
|
||||
jbyte jbuf[compressionLevels.size()];
|
||||
jclass jListClazz = env->FindClass("java/util/ArrayList");
|
||||
jmethodID midList = rocksdb::ListJni::getArrayListConstructorMethodId(
|
||||
env, jListClazz);
|
||||
jobject jcompressionLevels = env->NewObject(jListClazz,
|
||||
midList, compressionLevels.size());
|
||||
// insert in java list
|
||||
for (std::vector<rocksdb::CompressionType>::size_type i = 0;
|
||||
i != compressionLevels.size(); i++) {
|
||||
jbuf[i] = compressionLevels[i];
|
||||
jclass jByteClazz = env->FindClass("java/lang/Byte");
|
||||
jmethodID midByte = env->GetMethodID(jByteClazz, "<init>", "(B)V");
|
||||
jobject obj = env->NewObject(jByteClazz, midByte,
|
||||
compressionLevels[i]);
|
||||
env->CallBooleanMethod(jcompressionLevels,
|
||||
rocksdb::ListJni::getListAddMethodId(env), obj);
|
||||
}
|
||||
// insert in java array
|
||||
jbyteArray jcompressionLevels = env->NewByteArray(
|
||||
static_cast<jsize>(compressionLevels.size()));
|
||||
env->SetByteArrayRegion(jcompressionLevels, 0,
|
||||
static_cast<jsize>(compressionLevels.size()), jbuf);
|
||||
return jcompressionLevels;
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_Options
|
||||
* Method: setCompressionPerLevel
|
||||
* Signature: (J[B)V
|
||||
* Signature: (JLjava/util/List;)V
|
||||
*/
|
||||
void Java_org_rocksdb_Options_setCompressionPerLevel(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle,
|
||||
jbyteArray jcompressionLevels) {
|
||||
jobject jcompressionLevels) {
|
||||
auto* options = reinterpret_cast<rocksdb::Options*>(jhandle);
|
||||
std::vector<rocksdb::CompressionType> compressionLevels =
|
||||
rocksdb_compression_vector_helper(env, jcompressionLevels);
|
||||
@@ -1134,9 +1140,9 @@ void Java_org_rocksdb_Options_setCompressionPerLevel(
|
||||
/*
|
||||
* Class: org_rocksdb_Options
|
||||
* Method: compressionPerLevel
|
||||
* Signature: (J)[B
|
||||
* Signature: (J)Ljava/util/List;
|
||||
*/
|
||||
jbyteArray Java_org_rocksdb_Options_compressionPerLevel(
|
||||
jobject Java_org_rocksdb_Options_compressionPerLevel(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||
auto* options = reinterpret_cast<rocksdb::Options*>(jhandle);
|
||||
return rocksdb_compression_list_helper(env,
|
||||
@@ -1926,12 +1932,12 @@ void Java_org_rocksdb_Options_prepareForBulkLoad(
|
||||
/*
|
||||
* Class: org_rocksdb_ColumnFamilyOptions
|
||||
* Method: newColumnFamilyOptions
|
||||
* Signature: ()J
|
||||
* Signature: ()V
|
||||
*/
|
||||
jlong Java_org_rocksdb_ColumnFamilyOptions_newColumnFamilyOptions(
|
||||
JNIEnv* env, jclass jcls) {
|
||||
void Java_org_rocksdb_ColumnFamilyOptions_newColumnFamilyOptions(
|
||||
JNIEnv* env, jobject jobj) {
|
||||
rocksdb::ColumnFamilyOptions* op = new rocksdb::ColumnFamilyOptions();
|
||||
return reinterpret_cast<jlong>(op);
|
||||
rocksdb::ColumnFamilyOptionsJni::setHandle(env, jobj, op);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -2279,11 +2285,11 @@ jbyte Java_org_rocksdb_ColumnFamilyOptions_compressionType(
|
||||
/*
|
||||
* Class: org_rocksdb_ColumnFamilyOptions
|
||||
* Method: setCompressionPerLevel
|
||||
* Signature: (J[B)V
|
||||
* Signature: (JLjava/util/List;)V
|
||||
*/
|
||||
void Java_org_rocksdb_ColumnFamilyOptions_setCompressionPerLevel(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle,
|
||||
jbyteArray jcompressionLevels) {
|
||||
jobject jcompressionLevels) {
|
||||
auto* options = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
|
||||
std::vector<rocksdb::CompressionType> compressionLevels =
|
||||
rocksdb_compression_vector_helper(env, jcompressionLevels);
|
||||
@@ -2293,9 +2299,9 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompressionPerLevel(
|
||||
/*
|
||||
* Class: org_rocksdb_ColumnFamilyOptions
|
||||
* Method: compressionPerLevel
|
||||
* Signature: (J)[B
|
||||
* Signature: (J)Ljava/util/List;
|
||||
*/
|
||||
jbyteArray Java_org_rocksdb_ColumnFamilyOptions_compressionPerLevel(
|
||||
jobject Java_org_rocksdb_ColumnFamilyOptions_compressionPerLevel(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||
auto* options = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
|
||||
return rocksdb_compression_list_helper(env,
|
||||
@@ -3066,12 +3072,12 @@ void Java_org_rocksdb_ColumnFamilyOptions_setOptimizeFiltersForHits(
|
||||
/*
|
||||
* Class: org_rocksdb_DBOptions
|
||||
* Method: newDBOptions
|
||||
* Signature: ()J
|
||||
* Signature: ()V
|
||||
*/
|
||||
jlong Java_org_rocksdb_DBOptions_newDBOptions(JNIEnv* env,
|
||||
jclass jcls) {
|
||||
void Java_org_rocksdb_DBOptions_newDBOptions(JNIEnv* env,
|
||||
jobject jobj) {
|
||||
rocksdb::DBOptions* dbop = new rocksdb::DBOptions();
|
||||
return reinterpret_cast<jlong>(dbop);
|
||||
rocksdb::DBOptionsJni::setHandle(env, jobj, dbop);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -3866,12 +3872,12 @@ jlong Java_org_rocksdb_DBOptions_bytesPerSync(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteOptions
|
||||
* Method: newWriteOptions
|
||||
* Signature: ()J
|
||||
* Signature: ()V
|
||||
*/
|
||||
jlong Java_org_rocksdb_WriteOptions_newWriteOptions(
|
||||
JNIEnv* env, jclass jcls) {
|
||||
void Java_org_rocksdb_WriteOptions_newWriteOptions(
|
||||
JNIEnv* env, jobject jwrite_options) {
|
||||
rocksdb::WriteOptions* op = new rocksdb::WriteOptions();
|
||||
return reinterpret_cast<jlong>(op);
|
||||
rocksdb::WriteOptionsJni::setHandle(env, jwrite_options, op);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -3883,6 +3889,8 @@ void Java_org_rocksdb_WriteOptions_disposeInternal(
|
||||
JNIEnv* env, jobject jwrite_options, jlong jhandle) {
|
||||
auto write_options = reinterpret_cast<rocksdb::WriteOptions*>(jhandle);
|
||||
delete write_options;
|
||||
|
||||
rocksdb::WriteOptionsJni::setHandle(env, jwrite_options, nullptr);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -3931,12 +3939,12 @@ jboolean Java_org_rocksdb_WriteOptions_disableWAL(
|
||||
/*
|
||||
* Class: org_rocksdb_ReadOptions
|
||||
* Method: newReadOptions
|
||||
* Signature: ()J
|
||||
* Signature: ()V
|
||||
*/
|
||||
jlong Java_org_rocksdb_ReadOptions_newReadOptions(
|
||||
JNIEnv* env, jclass jcls) {
|
||||
void Java_org_rocksdb_ReadOptions_newReadOptions(
|
||||
JNIEnv* env, jobject jobj) {
|
||||
auto read_opt = new rocksdb::ReadOptions();
|
||||
return reinterpret_cast<jlong>(read_opt);
|
||||
rocksdb::ReadOptionsJni::setHandle(env, jobj, read_opt);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -3947,6 +3955,7 @@ jlong Java_org_rocksdb_ReadOptions_newReadOptions(
|
||||
void Java_org_rocksdb_ReadOptions_disposeInternal(
|
||||
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||
delete reinterpret_cast<rocksdb::ReadOptions*>(jhandle);
|
||||
rocksdb::ReadOptionsJni::setHandle(env, jobj, nullptr);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -4043,12 +4052,12 @@ jlong Java_org_rocksdb_ReadOptions_snapshot(
|
||||
/*
|
||||
* Class: org_rocksdb_ComparatorOptions
|
||||
* Method: newComparatorOptions
|
||||
* Signature: ()J
|
||||
* Signature: ()V
|
||||
*/
|
||||
jlong Java_org_rocksdb_ComparatorOptions_newComparatorOptions(
|
||||
JNIEnv* env, jclass jcls) {
|
||||
void Java_org_rocksdb_ComparatorOptions_newComparatorOptions(
|
||||
JNIEnv* env, jobject jobj) {
|
||||
auto comparator_opt = new rocksdb::ComparatorJniCallbackOptions();
|
||||
return reinterpret_cast<jlong>(comparator_opt);
|
||||
rocksdb::ComparatorOptionsJni::setHandle(env, jobj, comparator_opt);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -4081,6 +4090,7 @@ void Java_org_rocksdb_ComparatorOptions_setUseAdaptiveMutex(
|
||||
void Java_org_rocksdb_ComparatorOptions_disposeInternal(
|
||||
JNIEnv * env, jobject jobj, jlong jhandle) {
|
||||
delete reinterpret_cast<rocksdb::ComparatorJniCallbackOptions*>(jhandle);
|
||||
rocksdb::ComparatorOptionsJni::setHandle(env, jobj, nullptr);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
@@ -4089,12 +4099,12 @@ void Java_org_rocksdb_ComparatorOptions_disposeInternal(
|
||||
/*
|
||||
* Class: org_rocksdb_FlushOptions
|
||||
* Method: newFlushOptions
|
||||
* Signature: ()J
|
||||
* Signature: ()V
|
||||
*/
|
||||
jlong Java_org_rocksdb_FlushOptions_newFlushOptions(
|
||||
JNIEnv* env, jclass jcls) {
|
||||
void Java_org_rocksdb_FlushOptions_newFlushOptions(
|
||||
JNIEnv* env, jobject jobj) {
|
||||
auto flush_opt = new rocksdb::FlushOptions();
|
||||
return reinterpret_cast<jlong>(flush_opt);
|
||||
rocksdb::FlushOptionsJni::setHandle(env, jobj, flush_opt);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -4127,4 +4137,5 @@ jboolean Java_org_rocksdb_FlushOptions_waitForFlush(
|
||||
void Java_org_rocksdb_FlushOptions_disposeInternal(
|
||||
JNIEnv * env, jobject jobj, jlong jhandle) {
|
||||
delete reinterpret_cast<rocksdb::FlushOptions*>(jhandle);
|
||||
rocksdb::FlushOptionsJni::setHandle(env, jobj, nullptr);
|
||||
}
|
||||
+78
-15
@@ -49,25 +49,27 @@ template<class PTR, class DERIVED> class RocksDBNativeClass {
|
||||
assert(jclazz != nullptr);
|
||||
return jclazz;
|
||||
}
|
||||
};
|
||||
|
||||
// Native class template for sub-classes of RocksMutableObject
|
||||
template<class PTR, class DERIVED> class NativeRocksMutableObject
|
||||
: public RocksDBNativeClass<PTR, DERIVED> {
|
||||
public:
|
||||
// Get the field id of the member variable to store
|
||||
// the ptr
|
||||
static jfieldID getHandleFieldID(JNIEnv* env) {
|
||||
static jfieldID fid = env->GetFieldID(
|
||||
DERIVED::getJClass(env), "nativeHandle_", "J");
|
||||
assert(fid != nullptr);
|
||||
return fid;
|
||||
}
|
||||
|
||||
static jmethodID getSetNativeHandleMethod(JNIEnv* env) {
|
||||
static jmethodID mid = env->GetMethodID(
|
||||
DERIVED::getJClass(env), "setNativeHandle", "(JZ)V");
|
||||
assert(mid != nullptr);
|
||||
return mid;
|
||||
// Get the pointer from Java
|
||||
static PTR getHandle(JNIEnv* env, jobject jobj) {
|
||||
return reinterpret_cast<PTR>(
|
||||
env->GetLongField(jobj, getHandleFieldID(env)));
|
||||
}
|
||||
|
||||
// Pass the pointer to the java side.
|
||||
static void setHandle(JNIEnv* env, jobject jobj, PTR ptr,
|
||||
jboolean java_owns_handle) {
|
||||
env->CallVoidMethod(jobj, getSetNativeHandleMethod(env),
|
||||
reinterpret_cast<jlong>(ptr), java_owns_handle);
|
||||
static void setHandle(JNIEnv* env, jobject jdb, PTR ptr) {
|
||||
env->SetLongField(
|
||||
jdb, getHandleFieldID(env),
|
||||
reinterpret_cast<jlong>(ptr));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -405,7 +407,7 @@ class AbstractComparatorJni : public RocksDBNativeClass<
|
||||
};
|
||||
|
||||
// The portal class for org.rocksdb.AbstractSlice
|
||||
class AbstractSliceJni : public NativeRocksMutableObject<
|
||||
class AbstractSliceJni : public RocksDBNativeClass<
|
||||
const rocksdb::Slice*, AbstractSliceJni> {
|
||||
public:
|
||||
static jclass getJClass(JNIEnv* env) {
|
||||
@@ -647,6 +649,67 @@ class WriteEntryJni {
|
||||
assert(jclazz != nullptr);
|
||||
return jclazz;
|
||||
}
|
||||
|
||||
static void setWriteType(JNIEnv* env, jobject jwrite_entry,
|
||||
WriteType write_type) {
|
||||
jobject jwrite_type;
|
||||
switch (write_type) {
|
||||
case kPutRecord:
|
||||
jwrite_type = WriteTypeJni::PUT(env);
|
||||
break;
|
||||
|
||||
case kMergeRecord:
|
||||
jwrite_type = WriteTypeJni::MERGE(env);
|
||||
break;
|
||||
|
||||
case kDeleteRecord:
|
||||
jwrite_type = WriteTypeJni::DELETE(env);
|
||||
break;
|
||||
|
||||
case kLogDataRecord:
|
||||
jwrite_type = WriteTypeJni::LOG(env);
|
||||
break;
|
||||
|
||||
default:
|
||||
jwrite_type = nullptr;
|
||||
}
|
||||
assert(jwrite_type != nullptr);
|
||||
env->SetObjectField(jwrite_entry, getWriteTypeField(env), jwrite_type);
|
||||
}
|
||||
|
||||
static void setKey(JNIEnv* env, jobject jwrite_entry,
|
||||
const rocksdb::Slice* slice) {
|
||||
jobject jkey = env->GetObjectField(jwrite_entry, getKeyField(env));
|
||||
AbstractSliceJni::setHandle(env, jkey, slice);
|
||||
}
|
||||
|
||||
static void setValue(JNIEnv* env, jobject jwrite_entry,
|
||||
const rocksdb::Slice* slice) {
|
||||
jobject jvalue = env->GetObjectField(jwrite_entry, getValueField(env));
|
||||
AbstractSliceJni::setHandle(env, jvalue, slice);
|
||||
}
|
||||
|
||||
private:
|
||||
static jfieldID getWriteTypeField(JNIEnv* env) {
|
||||
static jfieldID fid = env->GetFieldID(
|
||||
getJClass(env), "type", "Lorg/rocksdb/WBWIRocksIterator$WriteType;");
|
||||
assert(fid != nullptr);
|
||||
return fid;
|
||||
}
|
||||
|
||||
static jfieldID getKeyField(JNIEnv* env) {
|
||||
static jfieldID fid = env->GetFieldID(
|
||||
getJClass(env), "key", "Lorg/rocksdb/DirectSlice;");
|
||||
assert(fid != nullptr);
|
||||
return fid;
|
||||
}
|
||||
|
||||
static jfieldID getValueField(JNIEnv* env) {
|
||||
static jfieldID fid = env->GetFieldID(
|
||||
getJClass(env), "value", "Lorg/rocksdb/DirectSlice;");
|
||||
assert(fid != nullptr);
|
||||
return fid;
|
||||
}
|
||||
};
|
||||
|
||||
class InfoLogLevelJni {
|
||||
|
||||
@@ -12,13 +12,16 @@
|
||||
/*
|
||||
* Class: org_rocksdb_RemoveEmptyValueCompactionFilter
|
||||
* Method: createNewRemoveEmptyValueCompactionFilter0
|
||||
* Signature: ()J
|
||||
* Signature: ()V
|
||||
*/
|
||||
jlong Java_org_rocksdb_RemoveEmptyValueCompactionFilter_createNewRemoveEmptyValueCompactionFilter0(
|
||||
JNIEnv* env, jclass jcls) {
|
||||
auto* compaction_filter =
|
||||
void Java_org_rocksdb_RemoveEmptyValueCompactionFilter_createNewRemoveEmptyValueCompactionFilter0(
|
||||
JNIEnv* env, jobject jobj) {
|
||||
const rocksdb::RemoveEmptyValueCompactionFilter* compaction_filter =
|
||||
new rocksdb::RemoveEmptyValueCompactionFilter();
|
||||
|
||||
// set the native handle to our native compaction filter
|
||||
return reinterpret_cast<jlong>(compaction_filter);
|
||||
static jclass jclazz =
|
||||
env->FindClass("org/rocksdb/RemoveEmptyValueCompactionFilter");
|
||||
static jfieldID fid = env->GetFieldID(jclazz, "nativeHandle_", "J");
|
||||
env->SetLongField(jobj, fid, reinterpret_cast<jlong>(compaction_filter));
|
||||
}
|
||||
@@ -22,17 +22,17 @@
|
||||
* Signature: (Z)J
|
||||
*/
|
||||
jlong Java_org_rocksdb_RestoreOptions_newRestoreOptions(JNIEnv* env,
|
||||
jclass jcls, jboolean keep_log_files) {
|
||||
jobject jobj, jboolean keep_log_files) {
|
||||
auto ropt = new rocksdb::RestoreOptions(keep_log_files);
|
||||
return reinterpret_cast<jlong>(ropt);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_RestoreOptions
|
||||
* Method: disposeInternal
|
||||
* Method: dispose
|
||||
* Signature: (J)V
|
||||
*/
|
||||
void Java_org_rocksdb_RestoreOptions_disposeInternal(JNIEnv* env, jobject jobj,
|
||||
void Java_org_rocksdb_RestoreOptions_dispose(JNIEnv* env, jobject jobj,
|
||||
jlong jhandle) {
|
||||
auto ropt = reinterpret_cast<rocksdb::RestoreOptions*>(jhandle);
|
||||
assert(ropt);
|
||||
@@ -45,8 +45,8 @@ void Java_org_rocksdb_RestoreOptions_disposeInternal(JNIEnv* env, jobject jobj,
|
||||
* Signature: (J)J
|
||||
*/
|
||||
jlong Java_org_rocksdb_RestoreBackupableDB_newRestoreBackupableDB(JNIEnv* env,
|
||||
jclass jcls, jlong jopt_handle) {
|
||||
auto* opt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jopt_handle);
|
||||
jobject jobj, jlong jopt_handle) {
|
||||
auto opt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jopt_handle);
|
||||
auto rdb = new rocksdb::RestoreBackupableDB(rocksdb::Env::Default(), *opt);
|
||||
return reinterpret_cast<jlong>(rdb);
|
||||
}
|
||||
@@ -185,11 +185,11 @@ void Java_org_rocksdb_RestoreBackupableDB_garbageCollect(
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_RestoreBackupableDB
|
||||
* Method: disposeInternal
|
||||
* Method: dispose
|
||||
* Signature: (J)V
|
||||
*/
|
||||
void Java_org_rocksdb_RestoreBackupableDB_disposeInternal(JNIEnv* env,
|
||||
jobject jobj, jlong jhandle) {
|
||||
void Java_org_rocksdb_RestoreBackupableDB_dispose(JNIEnv* env, jobject jobj,
|
||||
jlong jhandle) {
|
||||
auto ropt = reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle);
|
||||
assert(ropt);
|
||||
delete ropt;
|
||||
|
||||
+340
-234
@@ -11,7 +11,6 @@
|
||||
#include <stdlib.h>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
|
||||
@@ -27,142 +26,217 @@
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// rocksdb::DB::Open
|
||||
jlong rocksdb_open_helper(JNIEnv* env, jlong jopt_handle, jstring jdb_path,
|
||||
std::function<rocksdb::Status(
|
||||
const rocksdb::Options&, const std::string&, rocksdb::DB**)> open_fn
|
||||
) {
|
||||
auto* opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: open
|
||||
* Signature: (JLjava/lang/String;)V
|
||||
*/
|
||||
void Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2(
|
||||
JNIEnv* env, jobject jdb, jlong jopt_handle, jstring jdb_path) {
|
||||
auto opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
|
||||
rocksdb::DB* db = nullptr;
|
||||
const char* db_path = env->GetStringUTFChars(jdb_path, NULL);
|
||||
rocksdb::Status s = open_fn(*opt, db_path, &db);
|
||||
const char* db_path = env->GetStringUTFChars(jdb_path, 0);
|
||||
rocksdb::Status s = rocksdb::DB::Open(*opt, db_path, &db);
|
||||
env->ReleaseStringUTFChars(jdb_path, db_path);
|
||||
|
||||
if (s.ok()) {
|
||||
return reinterpret_cast<jlong>(db);
|
||||
} else {
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
|
||||
return 0;
|
||||
rocksdb::RocksDBJni::setHandle(env, jdb, db);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: open
|
||||
* Signature: (JLjava/lang/String;)J
|
||||
*/
|
||||
jlong Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2(
|
||||
JNIEnv* env, jclass jcls, jlong jopt_handle, jstring jdb_path) {
|
||||
return rocksdb_open_helper(env, jopt_handle, jdb_path,
|
||||
(rocksdb::Status(*)
|
||||
(const rocksdb::Options&, const std::string&, rocksdb::DB**)
|
||||
)&rocksdb::DB::Open
|
||||
);
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: openROnly
|
||||
* Signature: (JLjava/lang/String;)J
|
||||
* Signature: (JLjava/lang/String;)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2(
|
||||
JNIEnv* env, jclass jcls, jlong jopt_handle, jstring jdb_path) {
|
||||
return rocksdb_open_helper(env, jopt_handle, jdb_path, [](
|
||||
const rocksdb::Options& options,
|
||||
const std::string& db_path, rocksdb::DB** db) {
|
||||
return rocksdb::DB::OpenForReadOnly(options, db_path, db);
|
||||
});
|
||||
void Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2(
|
||||
JNIEnv* env, jobject jdb, jlong jopt_handle, jstring jdb_path) {
|
||||
auto opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
|
||||
rocksdb::DB* db = nullptr;
|
||||
const char* db_path = env->GetStringUTFChars(jdb_path, 0);
|
||||
rocksdb::Status s = rocksdb::DB::OpenForReadOnly(*opt,
|
||||
db_path, &db);
|
||||
env->ReleaseStringUTFChars(jdb_path, db_path);
|
||||
|
||||
if (s.ok()) {
|
||||
rocksdb::RocksDBJni::setHandle(env, jdb, db);
|
||||
return;
|
||||
}
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
|
||||
}
|
||||
|
||||
jlongArray rocksdb_open_helper(JNIEnv* env, jlong jopt_handle,
|
||||
jstring jdb_path, jobjectArray jcolumn_names, jlongArray jcolumn_options,
|
||||
std::function<rocksdb::Status(
|
||||
const rocksdb::DBOptions&, const std::string&,
|
||||
const std::vector<rocksdb::ColumnFamilyDescriptor>&,
|
||||
std::vector<rocksdb::ColumnFamilyHandle*>*,
|
||||
rocksdb::DB**)> open_fn
|
||||
) {
|
||||
auto* opt = reinterpret_cast<rocksdb::DBOptions*>(jopt_handle);
|
||||
const char* db_path = env->GetStringUTFChars(jdb_path, NULL);
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: openROnly
|
||||
* Signature: (JLjava/lang/String;Ljava/util/List;I)Ljava/util/List;
|
||||
*/
|
||||
jobject
|
||||
Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2Ljava_util_List_2I(
|
||||
JNIEnv* env, jobject jdb, jlong jopt_handle, jstring jdb_path,
|
||||
jobject jcfdesc_list, jint jcfdesc_count) {
|
||||
auto opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
|
||||
rocksdb::DB* db = nullptr;
|
||||
const char* db_path = env->GetStringUTFChars(jdb_path, 0);
|
||||
|
||||
std::vector<jbyte*> cfnames_to_free;
|
||||
std::vector<jbyteArray> jcfnames_for_free;
|
||||
|
||||
std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
|
||||
std::vector<rocksdb::ColumnFamilyHandle* > handles;
|
||||
// get iterator for ColumnFamilyDescriptors
|
||||
jobject iteratorObj = env->CallObjectMethod(
|
||||
jcfdesc_list, rocksdb::ListJni::getIteratorMethod(env));
|
||||
|
||||
jsize len_cols = env->GetArrayLength(jcolumn_names);
|
||||
jlong* jco = env->GetLongArrayElements(jcolumn_options, NULL);
|
||||
for(int i = 0; i < len_cols; i++) {
|
||||
jobject jcn = env->GetObjectArrayElement(jcolumn_names, i);
|
||||
jbyteArray jcn_ba = reinterpret_cast<jbyteArray>(jcn);
|
||||
jbyte* jcf_name = env->GetByteArrayElements(jcn_ba, NULL);
|
||||
const int jcf_name_len = env->GetArrayLength(jcn_ba);
|
||||
// iterate over ColumnFamilyDescriptors
|
||||
while (env->CallBooleanMethod(
|
||||
iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
|
||||
// get ColumnFamilyDescriptor
|
||||
jobject jcf_descriptor = env->CallObjectMethod(iteratorObj,
|
||||
rocksdb::ListJni::getNextMethod(env));
|
||||
// get ColumnFamilyName
|
||||
jbyteArray cf_name_in_byte_array = static_cast<jbyteArray>(
|
||||
env->CallObjectMethod(jcf_descriptor,
|
||||
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
|
||||
env)));
|
||||
// get CF Options
|
||||
jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
|
||||
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
|
||||
env));
|
||||
rocksdb::ColumnFamilyOptions* cfOptions =
|
||||
rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
|
||||
|
||||
//TODO(AR) do I need to make a copy of jco[i] ?
|
||||
jbyte* cfname = env->GetByteArrayElements(cf_name_in_byte_array, 0);
|
||||
const int len = env->GetArrayLength(cf_name_in_byte_array);
|
||||
|
||||
std::string cf_name (reinterpret_cast<char *>(jcf_name), jcf_name_len);
|
||||
rocksdb::ColumnFamilyOptions* cf_options =
|
||||
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jco[i]);
|
||||
column_families.push_back(
|
||||
rocksdb::ColumnFamilyDescriptor(cf_name, *cf_options));
|
||||
|
||||
env->ReleaseByteArrayElements(jcn_ba, jcf_name, JNI_ABORT);
|
||||
env->DeleteLocalRef(jcn);
|
||||
// free allocated cfnames after call to open
|
||||
cfnames_to_free.push_back(cfname);
|
||||
jcfnames_for_free.push_back(cf_name_in_byte_array);
|
||||
column_families.push_back(rocksdb::ColumnFamilyDescriptor(
|
||||
std::string(reinterpret_cast<char *>(cfname), len), *cfOptions));
|
||||
}
|
||||
env->ReleaseLongArrayElements(jcolumn_options, jco, JNI_ABORT);
|
||||
|
||||
std::vector<rocksdb::ColumnFamilyHandle*> handles;
|
||||
rocksdb::DB* db = nullptr;
|
||||
rocksdb::Status s = open_fn(*opt, db_path, column_families,
|
||||
&handles, &db);
|
||||
rocksdb::Status s = rocksdb::DB::OpenForReadOnly(*opt,
|
||||
db_path, column_families, &handles, &db);
|
||||
env->ReleaseStringUTFChars(jdb_path, db_path);
|
||||
// free jbyte allocations
|
||||
for (std::vector<jbyte*>::size_type i = 0;
|
||||
i != cfnames_to_free.size(); i++) {
|
||||
// free cfnames
|
||||
env->ReleaseByteArrayElements(jcfnames_for_free[i], cfnames_to_free[i], 0);
|
||||
}
|
||||
|
||||
// check if open operation was successful
|
||||
if (s.ok()) {
|
||||
jsize resultsLen = 1 + len_cols; //db handle + column family handles
|
||||
jlong results[resultsLen];
|
||||
results[0] = reinterpret_cast<jlong>(db);
|
||||
for(int i = 1; i <= len_cols; i++) {
|
||||
results[i] = reinterpret_cast<jlong>(handles[i - 1]);
|
||||
rocksdb::RocksDBJni::setHandle(env, jdb, db);
|
||||
jclass jListClazz = env->FindClass("java/util/ArrayList");
|
||||
jmethodID midList = rocksdb::ListJni::getArrayListConstructorMethodId(
|
||||
env, jListClazz);
|
||||
jobject jcfhandle_list = env->NewObject(jListClazz,
|
||||
midList, handles.size());
|
||||
// insert in java list
|
||||
for (std::vector<rocksdb::ColumnFamilyHandle*>::size_type i = 0;
|
||||
i != handles.size(); i++) {
|
||||
// jlong must be converted to Long due to collections restrictions
|
||||
jclass jLongClazz = env->FindClass("java/lang/Long");
|
||||
jmethodID midLong = env->GetMethodID(jLongClazz, "<init>", "(J)V");
|
||||
jobject obj = env->NewObject(jLongClazz, midLong,
|
||||
reinterpret_cast<jlong>(handles[i]));
|
||||
env->CallBooleanMethod(jcfhandle_list,
|
||||
rocksdb::ListJni::getListAddMethodId(env), obj);
|
||||
}
|
||||
|
||||
jlongArray jresults = env->NewLongArray(resultsLen);
|
||||
env->SetLongArrayRegion(jresults, 0, resultsLen, results);
|
||||
return jresults;
|
||||
} else {
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
|
||||
return NULL;
|
||||
return jcfhandle_list;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: openROnly
|
||||
* Signature: (JLjava/lang/String;[[B[J)[J
|
||||
*/
|
||||
jlongArray Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2_3_3B_3J(
|
||||
JNIEnv* env, jclass jcls, jlong jopt_handle, jstring jdb_path,
|
||||
jobjectArray jcolumn_names, jlongArray jcolumn_options) {
|
||||
return rocksdb_open_helper(env, jopt_handle, jdb_path, jcolumn_names,
|
||||
jcolumn_options, [](
|
||||
const rocksdb::DBOptions& options, const std::string& db_path,
|
||||
const std::vector<rocksdb::ColumnFamilyDescriptor>& column_families,
|
||||
std::vector<rocksdb::ColumnFamilyHandle*>* handles, rocksdb::DB** db) {
|
||||
return rocksdb::DB::OpenForReadOnly(options, db_path, column_families,
|
||||
handles, db);
|
||||
});
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: open
|
||||
* Signature: (JLjava/lang/String;[[B[J)[J
|
||||
* Signature: (JLjava/lang/String;Ljava/util/List;I)Ljava/util/List;
|
||||
*/
|
||||
jlongArray Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2_3_3B_3J(
|
||||
JNIEnv* env, jclass jcls, jlong jopt_handle, jstring jdb_path,
|
||||
jobjectArray jcolumn_names, jlongArray jcolumn_options) {
|
||||
return rocksdb_open_helper(env, jopt_handle, jdb_path, jcolumn_names,
|
||||
jcolumn_options, (rocksdb::Status(*)
|
||||
(const rocksdb::DBOptions&, const std::string&,
|
||||
const std::vector<rocksdb::ColumnFamilyDescriptor>&,
|
||||
std::vector<rocksdb::ColumnFamilyHandle*>*, rocksdb::DB**)
|
||||
)&rocksdb::DB::Open
|
||||
);
|
||||
jobject Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2Ljava_util_List_2I(
|
||||
JNIEnv* env, jobject jdb, jlong jopt_handle, jstring jdb_path,
|
||||
jobject jcfdesc_list, jint jcfdesc_count) {
|
||||
auto opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
|
||||
rocksdb::DB* db = nullptr;
|
||||
const char* db_path = env->GetStringUTFChars(jdb_path, 0);
|
||||
|
||||
std::vector<jbyte*> cfnames_to_free;
|
||||
std::vector<jbyteArray> jcfnames_for_free;
|
||||
|
||||
std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
|
||||
std::vector<rocksdb::ColumnFamilyHandle* > handles;
|
||||
// get iterator for ColumnFamilyDescriptors
|
||||
jobject iteratorObj = env->CallObjectMethod(
|
||||
jcfdesc_list, rocksdb::ListJni::getIteratorMethod(env));
|
||||
|
||||
// iterate over ColumnFamilyDescriptors
|
||||
while (env->CallBooleanMethod(
|
||||
iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
|
||||
// get ColumnFamilyDescriptor
|
||||
jobject jcf_descriptor = env->CallObjectMethod(iteratorObj,
|
||||
rocksdb::ListJni::getNextMethod(env));
|
||||
// get ColumnFamilyName
|
||||
jbyteArray cf_name_in_byte_array = static_cast<jbyteArray>(
|
||||
env->CallObjectMethod(jcf_descriptor,
|
||||
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
|
||||
env)));
|
||||
// get CF Options
|
||||
jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
|
||||
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
|
||||
env));
|
||||
rocksdb::ColumnFamilyOptions* cfOptions =
|
||||
rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
|
||||
|
||||
jbyte* cfname = env->GetByteArrayElements(cf_name_in_byte_array, 0);
|
||||
const int len = env->GetArrayLength(cf_name_in_byte_array);
|
||||
|
||||
// free allocated cfnames after call to open
|
||||
cfnames_to_free.push_back(cfname);
|
||||
jcfnames_for_free.push_back(cf_name_in_byte_array);
|
||||
column_families.push_back(rocksdb::ColumnFamilyDescriptor(
|
||||
std::string(reinterpret_cast<char *>(cfname), len), *cfOptions));
|
||||
}
|
||||
|
||||
rocksdb::Status s = rocksdb::DB::Open(*opt, db_path, column_families,
|
||||
&handles, &db);
|
||||
env->ReleaseStringUTFChars(jdb_path, db_path);
|
||||
// free jbyte allocations
|
||||
for (std::vector<jbyte*>::size_type i = 0;
|
||||
i != cfnames_to_free.size(); i++) {
|
||||
// free cfnames
|
||||
env->ReleaseByteArrayElements(jcfnames_for_free[i], cfnames_to_free[i], 0);
|
||||
}
|
||||
|
||||
// check if open operation was successful
|
||||
if (s.ok()) {
|
||||
rocksdb::RocksDBJni::setHandle(env, jdb, db);
|
||||
jclass jListClazz = env->FindClass("java/util/ArrayList");
|
||||
jmethodID midList = rocksdb::ListJni::getArrayListConstructorMethodId(
|
||||
env, jListClazz);
|
||||
jobject jcfhandle_list = env->NewObject(jListClazz,
|
||||
midList, handles.size());
|
||||
// insert in java list
|
||||
for (std::vector<rocksdb::ColumnFamilyHandle*>::size_type i = 0;
|
||||
i != handles.size(); i++) {
|
||||
// jlong must be converted to Long due to collections restrictions
|
||||
jclass jLongClazz = env->FindClass("java/lang/Long");
|
||||
jmethodID midLong = env->GetMethodID(jLongClazz, "<init>", "(J)V");
|
||||
jobject obj = env->NewObject(jLongClazz, midLong,
|
||||
reinterpret_cast<jlong>(handles[i]));
|
||||
env->CallBooleanMethod(jcfhandle_list,
|
||||
rocksdb::ListJni::getListAddMethodId(env), obj);
|
||||
}
|
||||
|
||||
return jcfhandle_list;
|
||||
}
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
@@ -171,21 +245,25 @@ jlongArray Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2_3_3B_3J(
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: listColumnFamilies
|
||||
* Signature: (JLjava/lang/String;)[[B
|
||||
* Signature: (JLjava/lang/String;)Ljava/util/List;
|
||||
*/
|
||||
jobjectArray Java_org_rocksdb_RocksDB_listColumnFamilies(
|
||||
jobject Java_org_rocksdb_RocksDB_listColumnFamilies(
|
||||
JNIEnv* env, jclass jclazz, jlong jopt_handle, jstring jdb_path) {
|
||||
std::vector<std::string> column_family_names;
|
||||
auto* opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
|
||||
auto opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
|
||||
const char* db_path = env->GetStringUTFChars(jdb_path, 0);
|
||||
jobject jvalue_list = nullptr;
|
||||
|
||||
rocksdb::Status s = rocksdb::DB::ListColumnFamilies(*opt, db_path,
|
||||
&column_family_names);
|
||||
env->ReleaseStringUTFChars(jdb_path, db_path);
|
||||
|
||||
jclass jcls_ba = env->FindClass("[B");
|
||||
jobjectArray jresults = env->NewObjectArray(
|
||||
static_cast<jsize>(column_family_names.size()), jcls_ba, NULL);
|
||||
if (s.ok()) {
|
||||
// Don't reuse class pointer
|
||||
jclass jListClazz = env->FindClass("java/util/ArrayList");
|
||||
jmethodID mid = rocksdb::ListJni::getArrayListConstructorMethodId(env,
|
||||
jListClazz);
|
||||
jvalue_list = env->NewObject(jListClazz, mid, column_family_names.size());
|
||||
|
||||
for (std::vector<std::string>::size_type i = 0;
|
||||
i < column_family_names.size(); i++) {
|
||||
jbyteArray jcf_value =
|
||||
@@ -193,11 +271,11 @@ jobjectArray Java_org_rocksdb_RocksDB_listColumnFamilies(
|
||||
env->SetByteArrayRegion(
|
||||
jcf_value, 0, static_cast<jsize>(column_family_names[i].size()),
|
||||
reinterpret_cast<const jbyte*>(column_family_names[i].data()));
|
||||
env->SetObjectArrayElement(jresults, static_cast<jsize>(i), jcf_value);
|
||||
env->DeleteLocalRef(jcf_value);
|
||||
env->CallBooleanMethod(jvalue_list,
|
||||
rocksdb::ListJni::getListAddMethodId(env), jcf_value);
|
||||
}
|
||||
}
|
||||
return jresults;
|
||||
return jvalue_list;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
@@ -320,12 +398,12 @@ void Java_org_rocksdb_RocksDB_put__JJ_3BI_3BIJ(
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: write0
|
||||
* Signature: (JJJ)V
|
||||
* Signature: (JJ)V
|
||||
*/
|
||||
void Java_org_rocksdb_RocksDB_write0(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle,
|
||||
JNIEnv* env, jobject jdb,
|
||||
jlong jwrite_options_handle, jlong jwb_handle) {
|
||||
auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
|
||||
rocksdb::DB* db = rocksdb::RocksDBJni::getHandle(env, jdb);
|
||||
auto* write_options = reinterpret_cast<rocksdb::WriteOptions*>(
|
||||
jwrite_options_handle);
|
||||
auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
@@ -340,12 +418,12 @@ void Java_org_rocksdb_RocksDB_write0(
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: write1
|
||||
* Signature: (JJJ)V
|
||||
* Signature: (JJ)V
|
||||
*/
|
||||
void Java_org_rocksdb_RocksDB_write1(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle,
|
||||
JNIEnv* env, jobject jdb,
|
||||
jlong jwrite_options_handle, jlong jwbwi_handle) {
|
||||
auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
|
||||
rocksdb::DB* db = rocksdb::RocksDBJni::getHandle(env, jdb);
|
||||
auto* write_options = reinterpret_cast<rocksdb::WriteOptions*>(
|
||||
jwrite_options_handle);
|
||||
auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
|
||||
@@ -392,12 +470,12 @@ jboolean key_may_exist_helper(JNIEnv* env, rocksdb::DB* db,
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: keyMayExist
|
||||
* Signature: (J[BILjava/lang/StringBuffer;)Z
|
||||
* Signature: ([BILjava/lang/StringBuffer;)Z
|
||||
*/
|
||||
jboolean Java_org_rocksdb_RocksDB_keyMayExist__J_3BILjava_lang_StringBuffer_2(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle, jbyteArray jkey, jint jkey_len,
|
||||
jboolean Java_org_rocksdb_RocksDB_keyMayExist___3BILjava_lang_StringBuffer_2(
|
||||
JNIEnv* env, jobject jdb, jbyteArray jkey, jint jkey_len,
|
||||
jobject jstring_buffer) {
|
||||
auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
|
||||
rocksdb::DB* db = rocksdb::RocksDBJni::getHandle(env, jdb);
|
||||
return key_may_exist_helper(env, db, rocksdb::ReadOptions(),
|
||||
nullptr, jkey, jkey_len, jstring_buffer);
|
||||
}
|
||||
@@ -405,13 +483,13 @@ jboolean Java_org_rocksdb_RocksDB_keyMayExist__J_3BILjava_lang_StringBuffer_2(
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: keyMayExist
|
||||
* Signature: (J[BIJLjava/lang/StringBuffer;)Z
|
||||
* Signature: ([BIJLjava/lang/StringBuffer;)Z
|
||||
*/
|
||||
jboolean Java_org_rocksdb_RocksDB_keyMayExist__J_3BIJLjava_lang_StringBuffer_2(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle, jbyteArray jkey, jint jkey_len,
|
||||
jboolean Java_org_rocksdb_RocksDB_keyMayExist___3BIJLjava_lang_StringBuffer_2(
|
||||
JNIEnv* env, jobject jdb, jbyteArray jkey, jint jkey_len,
|
||||
jlong jcf_handle, jobject jstring_buffer) {
|
||||
auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
|
||||
auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(
|
||||
rocksdb::DB* db = rocksdb::RocksDBJni::getHandle(env, jdb);
|
||||
auto cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(
|
||||
jcf_handle);
|
||||
if (cf_handle != nullptr) {
|
||||
return key_may_exist_helper(env, db, rocksdb::ReadOptions(),
|
||||
@@ -419,19 +497,19 @@ jboolean Java_org_rocksdb_RocksDB_keyMayExist__J_3BIJLjava_lang_StringBuffer_2(
|
||||
} else {
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env,
|
||||
rocksdb::Status::InvalidArgument("Invalid ColumnFamilyHandle."));
|
||||
return true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: keyMayExist
|
||||
* Signature: (JJ[BILjava/lang/StringBuffer;)Z
|
||||
* Signature: (J[BILjava/lang/StringBuffer;)Z
|
||||
*/
|
||||
jboolean Java_org_rocksdb_RocksDB_keyMayExist__JJ_3BILjava_lang_StringBuffer_2(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jread_options_handle,
|
||||
jboolean Java_org_rocksdb_RocksDB_keyMayExist__J_3BILjava_lang_StringBuffer_2(
|
||||
JNIEnv* env, jobject jdb, jlong jread_options_handle,
|
||||
jbyteArray jkey, jint jkey_len, jobject jstring_buffer) {
|
||||
auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
|
||||
rocksdb::DB* db = rocksdb::RocksDBJni::getHandle(env, jdb);
|
||||
auto& read_options = *reinterpret_cast<rocksdb::ReadOptions*>(
|
||||
jread_options_handle);
|
||||
return key_may_exist_helper(env, db, read_options,
|
||||
@@ -441,15 +519,15 @@ jboolean Java_org_rocksdb_RocksDB_keyMayExist__JJ_3BILjava_lang_StringBuffer_2(
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: keyMayExist
|
||||
* Signature: (JJ[BIJLjava/lang/StringBuffer;)Z
|
||||
* Signature: (J[BIJLjava/lang/StringBuffer;)Z
|
||||
*/
|
||||
jboolean Java_org_rocksdb_RocksDB_keyMayExist__JJ_3BIJLjava_lang_StringBuffer_2(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jread_options_handle,
|
||||
jboolean Java_org_rocksdb_RocksDB_keyMayExist__J_3BIJLjava_lang_StringBuffer_2(
|
||||
JNIEnv* env, jobject jdb, jlong jread_options_handle,
|
||||
jbyteArray jkey, jint jkey_len, jlong jcf_handle, jobject jstring_buffer) {
|
||||
auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
|
||||
rocksdb::DB* db = rocksdb::RocksDBJni::getHandle(env, jdb);
|
||||
auto& read_options = *reinterpret_cast<rocksdb::ReadOptions*>(
|
||||
jread_options_handle);
|
||||
auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(
|
||||
auto cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(
|
||||
jcf_handle);
|
||||
if (cf_handle != nullptr) {
|
||||
return key_may_exist_helper(env, db, read_options, cf_handle,
|
||||
@@ -457,8 +535,8 @@ jboolean Java_org_rocksdb_RocksDB_keyMayExist__JJ_3BIJLjava_lang_StringBuffer_2(
|
||||
} else {
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env,
|
||||
rocksdb::Status::InvalidArgument("Invalid ColumnFamilyHandle."));
|
||||
return true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
@@ -625,38 +703,49 @@ jint rocksdb_get_helper(
|
||||
}
|
||||
|
||||
// cf multi get
|
||||
jobjectArray multi_get_helper(JNIEnv* env, jobject jdb, rocksdb::DB* db,
|
||||
const rocksdb::ReadOptions& rOpt, jobjectArray jkeys,
|
||||
jlongArray jcolumn_family_handles) {
|
||||
std::vector<rocksdb::ColumnFamilyHandle*> cf_handles;
|
||||
if (jcolumn_family_handles != nullptr) {
|
||||
jsize len_cols = env->GetArrayLength(jcolumn_family_handles);
|
||||
jlong* jcfh = env->GetLongArrayElements(jcolumn_family_handles, NULL);
|
||||
for (int i = 0; i < len_cols; i++) {
|
||||
auto* cf_handle =
|
||||
reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcfh[i]);
|
||||
cf_handles.push_back(cf_handle);
|
||||
}
|
||||
env->ReleaseLongArrayElements(jcolumn_family_handles, jcfh, JNI_ABORT);
|
||||
}
|
||||
|
||||
jobject multi_get_helper(JNIEnv* env, jobject jdb, rocksdb::DB* db,
|
||||
const rocksdb::ReadOptions& rOpt, jobject jkey_list, jint jkeys_count,
|
||||
jobject jcfhandle_list) {
|
||||
std::vector<rocksdb::Slice> keys;
|
||||
std::vector<std::tuple<jbyteArray, jbyte*, jobject>> keys_to_free;
|
||||
jsize len_keys = env->GetArrayLength(jkeys);
|
||||
if(env->EnsureLocalCapacity(len_keys) != 0) {
|
||||
// out of memory
|
||||
return NULL;
|
||||
std::vector<jbyte*> keys_to_free;
|
||||
std::vector<rocksdb::ColumnFamilyHandle*> cf_handles;
|
||||
|
||||
if (jcfhandle_list != nullptr) {
|
||||
// get cf iterator
|
||||
jobject cfIteratorObj = env->CallObjectMethod(
|
||||
jcfhandle_list, rocksdb::ListJni::getIteratorMethod(env));
|
||||
|
||||
// iterate over keys and convert java byte array to slice
|
||||
while (env->CallBooleanMethod(
|
||||
cfIteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
|
||||
jobject jobj = (jbyteArray) env->CallObjectMethod(
|
||||
cfIteratorObj, rocksdb::ListJni::getNextMethod(env));
|
||||
rocksdb::ColumnFamilyHandle* cfHandle =
|
||||
rocksdb::ColumnFamilyHandleJni::getHandle(env, jobj);
|
||||
cf_handles.push_back(cfHandle);
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < len_keys; i++) {
|
||||
jobject jk = env->GetObjectArrayElement(jkeys, i);
|
||||
jbyteArray jk_ba = reinterpret_cast<jbyteArray>(jk);
|
||||
jsize len_key = env->GetArrayLength(jk_ba);
|
||||
jbyte* jk_val = env->GetByteArrayElements(jk_ba, NULL);
|
||||
|
||||
rocksdb::Slice key_slice(reinterpret_cast<char*>(jk_val), len_key);
|
||||
// Process key list
|
||||
// get iterator
|
||||
jobject iteratorObj = env->CallObjectMethod(
|
||||
jkey_list, rocksdb::ListJni::getIteratorMethod(env));
|
||||
|
||||
// iterate over keys and convert java byte array to slice
|
||||
while (env->CallBooleanMethod(
|
||||
iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
|
||||
jbyteArray jkey = (jbyteArray) env->CallObjectMethod(
|
||||
iteratorObj, rocksdb::ListJni::getNextMethod(env));
|
||||
jint key_length = env->GetArrayLength(jkey);
|
||||
|
||||
jbyte* key = new jbyte[key_length];
|
||||
env->GetByteArrayRegion(jkey, 0, key_length, key);
|
||||
// store allocated jbyte to free it after multiGet call
|
||||
keys_to_free.push_back(key);
|
||||
|
||||
rocksdb::Slice key_slice(
|
||||
reinterpret_cast<char*>(key), key_length);
|
||||
keys.push_back(key_slice);
|
||||
|
||||
keys_to_free.push_back(std::make_tuple(jk_ba, jk_val, jk));
|
||||
}
|
||||
|
||||
std::vector<std::string> values;
|
||||
@@ -667,23 +756,13 @@ jobjectArray multi_get_helper(JNIEnv* env, jobject jdb, rocksdb::DB* db,
|
||||
s = db->MultiGet(rOpt, cf_handles, keys, &values);
|
||||
}
|
||||
|
||||
// free up allocated byte arrays
|
||||
for (std::vector<std::tuple<jbyteArray, jbyte*, jobject>>::size_type i = 0;
|
||||
i < keys_to_free.size(); i++) {
|
||||
jobject jk;
|
||||
jbyteArray jk_ba;
|
||||
jbyte* jk_val;
|
||||
std::tie(jk_ba, jk_val, jk) = keys_to_free[i];
|
||||
env->ReleaseByteArrayElements(jk_ba, jk_val, JNI_ABORT);
|
||||
env->DeleteLocalRef(jk);
|
||||
}
|
||||
// Don't reuse class pointer
|
||||
jclass jclazz = env->FindClass("java/util/ArrayList");
|
||||
jmethodID mid = rocksdb::ListJni::getArrayListConstructorMethodId(
|
||||
env, jclazz);
|
||||
jobject jvalue_list = env->NewObject(jclazz, mid, jkeys_count);
|
||||
|
||||
// prepare the results
|
||||
jclass jcls_ba = env->FindClass("[B");
|
||||
jobjectArray jresults =
|
||||
env->NewObjectArray(static_cast<jsize>(s.size()), jcls_ba, NULL);
|
||||
|
||||
// add to the jresults
|
||||
// insert in java list
|
||||
for (std::vector<rocksdb::Status>::size_type i = 0; i != s.size(); i++) {
|
||||
if (s[i].ok()) {
|
||||
jbyteArray jentry_value =
|
||||
@@ -691,60 +770,73 @@ jobjectArray multi_get_helper(JNIEnv* env, jobject jdb, rocksdb::DB* db,
|
||||
env->SetByteArrayRegion(
|
||||
jentry_value, 0, static_cast<jsize>(values[i].size()),
|
||||
reinterpret_cast<const jbyte*>(values[i].c_str()));
|
||||
env->SetObjectArrayElement(jresults, static_cast<jsize>(i), jentry_value);
|
||||
env->DeleteLocalRef(jentry_value);
|
||||
env->CallBooleanMethod(
|
||||
jvalue_list, rocksdb::ListJni::getListAddMethodId(env),
|
||||
jentry_value);
|
||||
} else {
|
||||
env->CallBooleanMethod(
|
||||
jvalue_list, rocksdb::ListJni::getListAddMethodId(env), nullptr);
|
||||
}
|
||||
}
|
||||
|
||||
return jresults;
|
||||
// free up allocated byte arrays
|
||||
for (std::vector<jbyte*>::size_type i = 0; i != keys_to_free.size(); i++) {
|
||||
delete[] keys_to_free[i];
|
||||
}
|
||||
keys_to_free.clear();
|
||||
return jvalue_list;
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: multiGet
|
||||
* Signature: (J[[B)[[B
|
||||
* Signature: (JLjava/util/List;I)Ljava/util/List;
|
||||
*/
|
||||
jobjectArray Java_org_rocksdb_RocksDB_multiGet__J_3_3B(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle, jobjectArray jkeys) {
|
||||
jobject Java_org_rocksdb_RocksDB_multiGet__JLjava_util_List_2I(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle,
|
||||
jobject jkey_list, jint jkeys_count) {
|
||||
return multi_get_helper(env, jdb, reinterpret_cast<rocksdb::DB*>(jdb_handle),
|
||||
rocksdb::ReadOptions(), jkeys, nullptr);
|
||||
rocksdb::ReadOptions(), jkey_list, jkeys_count, nullptr);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: multiGet
|
||||
* Signature: (J[[B[J)[[B
|
||||
* Signature: (JLjava/util/List;ILjava/util/List;)Ljava/util/List;
|
||||
*/
|
||||
jobjectArray Java_org_rocksdb_RocksDB_multiGet__J_3_3B_3J(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle, jobjectArray jkeys,
|
||||
jlongArray jcolumn_family_handles) {
|
||||
jobject
|
||||
Java_org_rocksdb_RocksDB_multiGet__JLjava_util_List_2ILjava_util_List_2(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle,
|
||||
jobject jkey_list, jint jkeys_count, jobject jcfhandle_list) {
|
||||
return multi_get_helper(env, jdb, reinterpret_cast<rocksdb::DB*>(jdb_handle),
|
||||
rocksdb::ReadOptions(), jkeys, jcolumn_family_handles);
|
||||
rocksdb::ReadOptions(), jkey_list, jkeys_count, jcfhandle_list);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: multiGet
|
||||
* Signature: (JJ[[B)[[B
|
||||
* Signature: (JJLjava/util/List;I)Ljava/util/List;
|
||||
*/
|
||||
jobjectArray Java_org_rocksdb_RocksDB_multiGet__JJ_3_3B(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jropt_handle,
|
||||
jobjectArray jkeys) {
|
||||
jobject Java_org_rocksdb_RocksDB_multiGet__JJLjava_util_List_2I(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle,
|
||||
jlong jropt_handle, jobject jkey_list, jint jkeys_count) {
|
||||
return multi_get_helper(env, jdb, reinterpret_cast<rocksdb::DB*>(jdb_handle),
|
||||
*reinterpret_cast<rocksdb::ReadOptions*>(jropt_handle), jkeys, nullptr);
|
||||
*reinterpret_cast<rocksdb::ReadOptions*>(jropt_handle), jkey_list,
|
||||
jkeys_count, nullptr);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: multiGet
|
||||
* Signature: (JJ[[B[J)[[B
|
||||
* Signature: (JJLjava/util/List;ILjava/util/List;)Ljava/util/List;
|
||||
*/
|
||||
jobjectArray Java_org_rocksdb_RocksDB_multiGet__JJ_3_3B_3J(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jropt_handle,
|
||||
jobjectArray jkeys, jlongArray jcolumn_family_handles) {
|
||||
jobject
|
||||
Java_org_rocksdb_RocksDB_multiGet__JJLjava_util_List_2ILjava_util_List_2(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle,
|
||||
jlong jropt_handle, jobject jkey_list, jint jkeys_count,
|
||||
jobject jcfhandle_list) {
|
||||
return multi_get_helper(env, jdb, reinterpret_cast<rocksdb::DB*>(jdb_handle),
|
||||
*reinterpret_cast<rocksdb::ReadOptions*>(jropt_handle), jkeys,
|
||||
jcolumn_family_handles);
|
||||
*reinterpret_cast<rocksdb::ReadOptions*>(jropt_handle), jkey_list,
|
||||
jkeys_count, jcfhandle_list);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -1112,42 +1204,47 @@ jlong Java_org_rocksdb_RocksDB_iteratorCF__JJJ(
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: iterators
|
||||
* Signature: (J[JJ)[J
|
||||
* Signature: (JLjava/util/List;J)[J
|
||||
*/
|
||||
jlongArray Java_org_rocksdb_RocksDB_iterators(
|
||||
JNIEnv* env, jobject jdb, jlong db_handle,
|
||||
jlongArray jcolumn_family_handles, jlong jread_options_handle) {
|
||||
auto* db = reinterpret_cast<rocksdb::DB*>(db_handle);
|
||||
JNIEnv* env, jobject jdb, jlong db_handle, jobject jcfhandle_list,
|
||||
jlong jread_options_handle) {
|
||||
auto db = reinterpret_cast<rocksdb::DB*>(db_handle);
|
||||
auto& read_options = *reinterpret_cast<rocksdb::ReadOptions*>(
|
||||
jread_options_handle);
|
||||
std::vector<rocksdb::ColumnFamilyHandle*> cf_handles;
|
||||
if (jcolumn_family_handles != nullptr) {
|
||||
jsize len_cols = env->GetArrayLength(jcolumn_family_handles);
|
||||
jlong* jcfh = env->GetLongArrayElements(jcolumn_family_handles, NULL);
|
||||
for (int i = 0; i < len_cols; i++) {
|
||||
auto* cf_handle =
|
||||
reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcfh[i]);
|
||||
cf_handles.push_back(cf_handle);
|
||||
std::vector<rocksdb::Iterator*> iterators;
|
||||
|
||||
if (jcfhandle_list != nullptr) {
|
||||
// get cf iterator
|
||||
jobject cfIteratorObj = env->CallObjectMethod(
|
||||
jcfhandle_list, rocksdb::ListJni::getIteratorMethod(env));
|
||||
|
||||
// iterate over keys and convert java byte array to slice
|
||||
while (env->CallBooleanMethod(
|
||||
cfIteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
|
||||
jobject jobj = (jbyteArray) env->CallObjectMethod(
|
||||
cfIteratorObj, rocksdb::ListJni::getNextMethod(env));
|
||||
rocksdb::ColumnFamilyHandle* cfHandle =
|
||||
rocksdb::ColumnFamilyHandleJni::getHandle(env, jobj);
|
||||
cf_handles.push_back(cfHandle);
|
||||
}
|
||||
env->ReleaseLongArrayElements(jcolumn_family_handles, jcfh, JNI_ABORT);
|
||||
}
|
||||
|
||||
std::vector<rocksdb::Iterator*> iterators;
|
||||
rocksdb::Status s = db->NewIterators(read_options,
|
||||
cf_handles, &iterators);
|
||||
if (s.ok()) {
|
||||
jlongArray jLongArray =
|
||||
env->NewLongArray(static_cast<jsize>(iterators.size()));
|
||||
for (std::vector<rocksdb::Iterator*>::size_type i = 0;
|
||||
i < iterators.size(); i++) {
|
||||
for (std::vector<rocksdb::Iterator*>::size_type i = 0; i < iterators.size();
|
||||
i++) {
|
||||
env->SetLongArrayRegion(jLongArray, static_cast<jsize>(i), 1,
|
||||
reinterpret_cast<const jlong*>(&iterators[i]));
|
||||
}
|
||||
return jLongArray;
|
||||
} else {
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
|
||||
return NULL;
|
||||
}
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
|
||||
return env->NewLongArray(0);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -1165,23 +1262,32 @@ jlong Java_org_rocksdb_RocksDB_getDefaultColumnFamily(
|
||||
/*
|
||||
* Class: org_rocksdb_RocksDB
|
||||
* Method: createColumnFamily
|
||||
* Signature: (J[BJ)J
|
||||
* Signature: (JLorg/rocksdb/ColumnFamilyDescriptor;)J;
|
||||
*/
|
||||
jlong Java_org_rocksdb_RocksDB_createColumnFamily(
|
||||
JNIEnv* env, jobject jdb, jlong jdb_handle,
|
||||
jbyteArray jcolumn_name, jlong jcolumn_options) {
|
||||
jobject jcf_descriptor) {
|
||||
rocksdb::ColumnFamilyHandle* handle;
|
||||
auto db_handle = reinterpret_cast<rocksdb::DB*>(jdb_handle);
|
||||
|
||||
jbyte* cfname = env->GetByteArrayElements(jcolumn_name, 0);
|
||||
const int len = env->GetArrayLength(jcolumn_name);
|
||||
// get ColumnFamilyName
|
||||
jbyteArray byteArray = static_cast<jbyteArray>(env->CallObjectMethod(
|
||||
jcf_descriptor,
|
||||
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
|
||||
env)));
|
||||
// get CF Options
|
||||
jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
|
||||
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
|
||||
env));
|
||||
rocksdb::ColumnFamilyOptions* cfOptions =
|
||||
rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
|
||||
|
||||
auto* cfOptions =
|
||||
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jcolumn_options);
|
||||
jbyte* cfname = env->GetByteArrayElements(byteArray, 0);
|
||||
const int len = env->GetArrayLength(byteArray);
|
||||
|
||||
rocksdb::Status s = db_handle->CreateColumnFamily(
|
||||
*cfOptions, std::string(reinterpret_cast<char *>(cfname), len), &handle);
|
||||
env->ReleaseByteArrayElements(jcolumn_name, cfname, 0);
|
||||
env->ReleaseByteArrayElements(byteArray, cfname, 0);
|
||||
|
||||
if (s.ok()) {
|
||||
return reinterpret_cast<jlong>(handle);
|
||||
|
||||
+24
-24
@@ -22,11 +22,12 @@
|
||||
/*
|
||||
* Class: org_rocksdb_AbstractSlice
|
||||
* Method: createNewSliceFromString
|
||||
* Signature: (Ljava/lang/String;)J
|
||||
* Signature: (Ljava/lang/String;)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_AbstractSlice_createNewSliceFromString(
|
||||
JNIEnv * env, jclass jcls, jstring jstr) {
|
||||
const auto* str = env->GetStringUTFChars(jstr, NULL);
|
||||
void Java_org_rocksdb_AbstractSlice_createNewSliceFromString(
|
||||
JNIEnv* env, jobject jobj, jstring jstr) {
|
||||
|
||||
const auto* str = env->GetStringUTFChars(jstr, 0);
|
||||
const size_t len = strlen(str);
|
||||
char* buf = new char[len + 1];
|
||||
memcpy(buf, str, len);
|
||||
@@ -34,7 +35,7 @@ jlong Java_org_rocksdb_AbstractSlice_createNewSliceFromString(
|
||||
env->ReleaseStringUTFChars(jstr, str);
|
||||
|
||||
const auto* slice = new rocksdb::Slice(buf);
|
||||
return reinterpret_cast<jlong>(slice);
|
||||
rocksdb::AbstractSliceJni::setHandle(env, jobj, slice);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -114,10 +115,10 @@ void Java_org_rocksdb_AbstractSlice_disposeInternal(
|
||||
/*
|
||||
* Class: org_rocksdb_Slice
|
||||
* Method: createNewSlice0
|
||||
* Signature: ([BI)J
|
||||
* Signature: ([BI)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_Slice_createNewSlice0(
|
||||
JNIEnv * env, jclass jcls, jbyteArray data, jint offset) {
|
||||
void Java_org_rocksdb_Slice_createNewSlice0(
|
||||
JNIEnv * env, jobject jobj, jbyteArray data, jint offset) {
|
||||
|
||||
const jsize dataSize = env->GetArrayLength(data);
|
||||
const int len = dataSize - offset;
|
||||
@@ -125,33 +126,32 @@ jlong Java_org_rocksdb_Slice_createNewSlice0(
|
||||
env->GetByteArrayRegion(data, offset, len, ptrData);
|
||||
|
||||
const auto* slice = new rocksdb::Slice((const char*)ptrData, len);
|
||||
return reinterpret_cast<jlong>(slice);
|
||||
rocksdb::AbstractSliceJni::setHandle(env, jobj, slice);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_Slice
|
||||
* Method: createNewSlice1
|
||||
* Signature: ([B)J
|
||||
* Signature: ([B)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_Slice_createNewSlice1(
|
||||
JNIEnv * env, jclass jcls, jbyteArray data) {
|
||||
void Java_org_rocksdb_Slice_createNewSlice1(
|
||||
JNIEnv * env, jobject jobj, jbyteArray data) {
|
||||
|
||||
const int len = env->GetArrayLength(data) + 1;
|
||||
|
||||
jboolean isCopy;
|
||||
jbyte* ptrData = env->GetByteArrayElements(data, &isCopy);
|
||||
|
||||
// NOTE: buf will be deleted in the org.rocksdb.Slice#dispose method
|
||||
char* buf = new char[len];
|
||||
|
||||
memcpy(buf, ptrData, len - 1);
|
||||
buf[len-1]='\0';
|
||||
|
||||
const auto* slice =
|
||||
new rocksdb::Slice(buf, len - 1);
|
||||
|
||||
rocksdb::AbstractSliceJni::setHandle(env, jobj, slice);
|
||||
env->ReleaseByteArrayElements(data, ptrData, JNI_ABORT);
|
||||
|
||||
return reinterpret_cast<jlong>(slice);
|
||||
// NOTE: buf will be deleted in the org.rocksdb.Slice#dispose method
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -187,27 +187,27 @@ void Java_org_rocksdb_Slice_disposeInternalBuf(
|
||||
/*
|
||||
* Class: org_rocksdb_DirectSlice
|
||||
* Method: createNewDirectSlice0
|
||||
* Signature: (Ljava/nio/ByteBuffer;I)J
|
||||
* Signature: (Ljava/nio/ByteBuffer;I)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_DirectSlice_createNewDirectSlice0(
|
||||
JNIEnv* env, jclass jcls, jobject data, jint length) {
|
||||
void Java_org_rocksdb_DirectSlice_createNewDirectSlice0(
|
||||
JNIEnv* env, jobject jobj, jobject data, jint length) {
|
||||
const auto* ptrData =
|
||||
reinterpret_cast<char*>(env->GetDirectBufferAddress(data));
|
||||
const auto* slice = new rocksdb::Slice(ptrData, length);
|
||||
return reinterpret_cast<jlong>(slice);
|
||||
rocksdb::AbstractSliceJni::setHandle(env, jobj, slice);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_DirectSlice
|
||||
* Method: createNewDirectSlice1
|
||||
* Signature: (Ljava/nio/ByteBuffer;)J
|
||||
* Signature: (Ljava/nio/ByteBuffer;)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_DirectSlice_createNewDirectSlice1(
|
||||
JNIEnv* env, jclass jcls, jobject data) {
|
||||
void Java_org_rocksdb_DirectSlice_createNewDirectSlice1(
|
||||
JNIEnv* env, jobject jobj, jobject data) {
|
||||
const auto* ptrData =
|
||||
reinterpret_cast<char*>(env->GetDirectBufferAddress(data));
|
||||
const auto* slice = new rocksdb::Slice(ptrData);
|
||||
return reinterpret_cast<jlong>(slice);
|
||||
rocksdb::AbstractSliceJni::setHandle(env, jobj, slice);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@@ -38,14 +38,13 @@ jlong Java_org_rocksdb_PlainTableConfig_newTableFactoryHandle(
|
||||
/*
|
||||
* Class: org_rocksdb_BlockBasedTableConfig
|
||||
* Method: newTableFactoryHandle
|
||||
* Signature: (ZJIJIIZIZZZJIBBI)J
|
||||
* Signature: (ZJIJIIZIZZJIBBI)J
|
||||
*/
|
||||
jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle(
|
||||
JNIEnv* env, jobject jobj, jboolean no_block_cache, jlong block_cache_size,
|
||||
jint block_cache_num_shardbits, jlong block_size, jint block_size_deviation,
|
||||
jint block_restart_interval, jboolean whole_key_filtering,
|
||||
jlong jfilterPolicy, jboolean cache_index_and_filter_blocks,
|
||||
jboolean pin_l0_filter_and_index_blocks_in_cache,
|
||||
jboolean hash_index_allow_collision, jlong block_cache_compressed_size,
|
||||
jint block_cache_compressd_num_shard_bits, jbyte jchecksum_type,
|
||||
jbyte jindex_type, jint jformat_version) {
|
||||
@@ -71,8 +70,6 @@ jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle(
|
||||
options.filter_policy = *pFilterPolicy;
|
||||
}
|
||||
options.cache_index_and_filter_blocks = cache_index_and_filter_blocks;
|
||||
options.pin_l0_filter_and_index_blocks_in_cache =
|
||||
pin_l0_filter_and_index_blocks_in_cache;
|
||||
options.hash_index_allow_collision = hash_index_allow_collision;
|
||||
if (block_cache_compressed_size > 0) {
|
||||
if (block_cache_compressd_num_shard_bits > 0) {
|
||||
|
||||
+106
-63
@@ -20,10 +20,10 @@
|
||||
/*
|
||||
* Class: org_rocksdb_TtlDB
|
||||
* Method: open
|
||||
* Signature: (JLjava/lang/String;IZ)J
|
||||
* Signature: (JLjava/lang/String;IZ)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_TtlDB_open(JNIEnv* env,
|
||||
jclass jcls, jlong joptions_handle, jstring jdb_path,
|
||||
void Java_org_rocksdb_TtlDB_open(JNIEnv* env,
|
||||
jobject jttldb, jlong joptions_handle, jstring jdb_path,
|
||||
jint jttl, jboolean jread_only) {
|
||||
auto* opt = reinterpret_cast<rocksdb::Options*>(joptions_handle);
|
||||
rocksdb::DBWithTTL* db = nullptr;
|
||||
@@ -35,102 +35,145 @@ jlong Java_org_rocksdb_TtlDB_open(JNIEnv* env,
|
||||
// as TTLDB extends RocksDB on the java side, we can reuse
|
||||
// the RocksDB portal here.
|
||||
if (s.ok()) {
|
||||
return reinterpret_cast<jlong>(db);
|
||||
} else {
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
|
||||
return 0;
|
||||
rocksdb::RocksDBJni::setHandle(env, jttldb, db);
|
||||
return;
|
||||
}
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_TtlDB
|
||||
* Method: openCF
|
||||
* Signature: (JLjava/lang/String;[[B[J[IZ)[J
|
||||
* Signature: (JLjava/lang/String;Ljava/util/List;
|
||||
* ILjava/util/List;Z)Ljava/util/List;
|
||||
*/
|
||||
jlongArray
|
||||
jobject
|
||||
Java_org_rocksdb_TtlDB_openCF(
|
||||
JNIEnv* env, jclass jcls, jlong jopt_handle, jstring jdb_path,
|
||||
jobjectArray jcolumn_names, jlongArray jcolumn_options,
|
||||
jintArray jttls, jboolean jread_only) {
|
||||
auto* opt = reinterpret_cast<rocksdb::DBOptions*>(jopt_handle);
|
||||
const char* db_path = env->GetStringUTFChars(jdb_path, NULL);
|
||||
JNIEnv* env, jobject jdb, jlong jopt_handle, jstring jdb_path,
|
||||
jobject jcfdesc_list, jint jcfdesc_count, jobject jttl_list,
|
||||
jboolean jread_only) {
|
||||
auto* opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
|
||||
rocksdb::DBWithTTL* db = nullptr;
|
||||
const char* db_path = env->GetStringUTFChars(jdb_path, 0);
|
||||
|
||||
std::vector<jbyte*> cfnames_to_free;
|
||||
std::vector<jbyteArray> jcfnames_for_free;
|
||||
|
||||
std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
|
||||
|
||||
jsize len_cols = env->GetArrayLength(jcolumn_names);
|
||||
jlong* jco = env->GetLongArrayElements(jcolumn_options, NULL);
|
||||
for(int i = 0; i < len_cols; i++) {
|
||||
jobject jcn = env->GetObjectArrayElement(jcolumn_names, i);
|
||||
jbyteArray jcn_ba = reinterpret_cast<jbyteArray>(jcn);
|
||||
jbyte* jcf_name = env->GetByteArrayElements(jcn_ba, NULL);
|
||||
const int jcf_name_len = env->GetArrayLength(jcn_ba);
|
||||
|
||||
//TODO(AR) do I need to make a copy of jco[i] ?
|
||||
|
||||
std::string cf_name (reinterpret_cast<char *>(jcf_name), jcf_name_len);
|
||||
rocksdb::ColumnFamilyOptions* cf_options =
|
||||
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jco[i]);
|
||||
column_families.push_back(
|
||||
rocksdb::ColumnFamilyDescriptor(cf_name, *cf_options));
|
||||
|
||||
env->ReleaseByteArrayElements(jcn_ba, jcf_name, JNI_ABORT);
|
||||
env->DeleteLocalRef(jcn);
|
||||
}
|
||||
env->ReleaseLongArrayElements(jcolumn_options, jco, JNI_ABORT);
|
||||
|
||||
std::vector<rocksdb::ColumnFamilyHandle*> handles;
|
||||
rocksdb::DBWithTTL* db = nullptr;
|
||||
|
||||
std::vector<int32_t> ttl_values;
|
||||
jint* jttlv = env->GetIntArrayElements(jttls, NULL);
|
||||
jsize len_ttls = env->GetArrayLength(jttls);
|
||||
for(int i = 0; i < len_ttls; i++) {
|
||||
ttl_values.push_back(jttlv[i]);
|
||||
}
|
||||
env->ReleaseIntArrayElements(jttls, jttlv, JNI_ABORT);
|
||||
std::vector<rocksdb::ColumnFamilyHandle* > handles;
|
||||
// get iterator for ColumnFamilyDescriptors
|
||||
jobject iteratorObj = env->CallObjectMethod(
|
||||
jcfdesc_list, rocksdb::ListJni::getIteratorMethod(env));
|
||||
|
||||
// iterate over ColumnFamilyDescriptors
|
||||
while (env->CallBooleanMethod(
|
||||
iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
|
||||
// get ColumnFamilyDescriptor
|
||||
jobject jcf_descriptor = env->CallObjectMethod(iteratorObj,
|
||||
rocksdb::ListJni::getNextMethod(env));
|
||||
// get ColumnFamilyName
|
||||
jbyteArray byteArray = static_cast<jbyteArray>(env->CallObjectMethod(
|
||||
jcf_descriptor,
|
||||
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
|
||||
env)));
|
||||
// get CF Options
|
||||
jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
|
||||
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
|
||||
env));
|
||||
rocksdb::ColumnFamilyOptions* cfOptions =
|
||||
rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
|
||||
|
||||
jbyte* cfname = env->GetByteArrayElements(byteArray, 0);
|
||||
const int len = env->GetArrayLength(byteArray);
|
||||
|
||||
// free allocated cfnames after call to open
|
||||
cfnames_to_free.push_back(cfname);
|
||||
jcfnames_for_free.push_back(byteArray);
|
||||
column_families.push_back(rocksdb::ColumnFamilyDescriptor(
|
||||
std::string(reinterpret_cast<char *>(cfname), len), *cfOptions));
|
||||
}
|
||||
// get iterator for TTL values
|
||||
iteratorObj = env->CallObjectMethod(
|
||||
jttl_list, rocksdb::ListJni::getIteratorMethod(env));
|
||||
// iterate over TTL values
|
||||
while (env->CallBooleanMethod(
|
||||
iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
|
||||
// get TTL object
|
||||
jobject jttl_object = env->CallObjectMethod(iteratorObj,
|
||||
rocksdb::ListJni::getNextMethod(env));
|
||||
// get Integer value
|
||||
jclass jIntClazz = env->FindClass("java/lang/Integer");
|
||||
jmethodID getVal = env->GetMethodID(jIntClazz, "intValue", "()I");
|
||||
ttl_values.push_back(env->CallIntMethod(jttl_object, getVal));
|
||||
}
|
||||
rocksdb::Status s = rocksdb::DBWithTTL::Open(*opt, db_path, column_families,
|
||||
&handles, &db, ttl_values, jread_only);
|
||||
|
||||
env->ReleaseStringUTFChars(jdb_path, db_path);
|
||||
// free jbyte allocations
|
||||
for (std::vector<jbyte*>::size_type i = 0;
|
||||
i != cfnames_to_free.size(); i++) {
|
||||
// free cfnames
|
||||
env->ReleaseByteArrayElements(jcfnames_for_free[i], cfnames_to_free[i], 0);
|
||||
}
|
||||
|
||||
// check if open operation was successful
|
||||
if (s.ok()) {
|
||||
jsize resultsLen = 1 + len_cols; //db handle + column family handles
|
||||
jlong results[resultsLen];
|
||||
results[0] = reinterpret_cast<jlong>(db);
|
||||
for(int i = 1; i <= len_cols; i++) {
|
||||
results[i] = reinterpret_cast<jlong>(handles[i - 1]);
|
||||
rocksdb::RocksDBJni::setHandle(env, jdb, db);
|
||||
jclass jListClazz = env->FindClass("java/util/ArrayList");
|
||||
jmethodID midList = rocksdb::ListJni::getArrayListConstructorMethodId(
|
||||
env, jListClazz);
|
||||
jobject jcfhandle_list = env->NewObject(jListClazz,
|
||||
midList, handles.size());
|
||||
// insert in java list
|
||||
for (std::vector<rocksdb::ColumnFamilyHandle*>::size_type i = 0;
|
||||
i != handles.size(); i++) {
|
||||
// jlong must be converted to Long due to collections restrictions
|
||||
jclass jLongClazz = env->FindClass("java/lang/Long");
|
||||
jmethodID midLong = env->GetMethodID(jLongClazz, "<init>", "(J)V");
|
||||
jobject obj = env->NewObject(jLongClazz, midLong,
|
||||
reinterpret_cast<jlong>(handles[i]));
|
||||
env->CallBooleanMethod(jcfhandle_list,
|
||||
rocksdb::ListJni::getListAddMethodId(env), obj);
|
||||
}
|
||||
|
||||
jlongArray jresults = env->NewLongArray(resultsLen);
|
||||
env->SetLongArrayRegion(jresults, 0, resultsLen, results);
|
||||
return jresults;
|
||||
} else {
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
|
||||
return NULL;
|
||||
return jcfhandle_list;
|
||||
}
|
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_TtlDB
|
||||
* Method: createColumnFamilyWithTtl
|
||||
* Signature: (JLorg/rocksdb/ColumnFamilyDescriptor;[BJI)J;
|
||||
* Signature: (JLorg/rocksdb/ColumnFamilyDescriptor;I)J;
|
||||
*/
|
||||
jlong Java_org_rocksdb_TtlDB_createColumnFamilyWithTtl(
|
||||
JNIEnv* env, jobject jobj, jlong jdb_handle,
|
||||
jbyteArray jcolumn_name, jlong jcolumn_options, jint jttl) {
|
||||
jobject jcf_descriptor, jint jttl) {
|
||||
rocksdb::ColumnFamilyHandle* handle;
|
||||
auto* db_handle = reinterpret_cast<rocksdb::DBWithTTL*>(jdb_handle);
|
||||
|
||||
jbyte* cfname = env->GetByteArrayElements(jcolumn_name, 0);
|
||||
const int len = env->GetArrayLength(jcolumn_name);
|
||||
// get ColumnFamilyName
|
||||
jbyteArray byteArray = static_cast<jbyteArray>(env->CallObjectMethod(
|
||||
jcf_descriptor,
|
||||
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
|
||||
env)));
|
||||
// get CF Options
|
||||
jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
|
||||
rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
|
||||
env));
|
||||
rocksdb::ColumnFamilyOptions* cfOptions =
|
||||
rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
|
||||
|
||||
auto* cfOptions =
|
||||
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jcolumn_options);
|
||||
jbyte* cfname = env->GetByteArrayElements(byteArray, 0);
|
||||
const int len = env->GetArrayLength(byteArray);
|
||||
|
||||
rocksdb::Status s = db_handle->CreateColumnFamilyWithTtl(
|
||||
*cfOptions, std::string(reinterpret_cast<char *>(cfname),
|
||||
len), &handle, jttl);
|
||||
env->ReleaseByteArrayElements(jcolumn_name, cfname, 0);
|
||||
env->ReleaseByteArrayElements(byteArray, cfname, 0);
|
||||
|
||||
if (s.ok()) {
|
||||
return reinterpret_cast<jlong>(handle);
|
||||
|
||||
@@ -27,23 +27,23 @@
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatch
|
||||
* Method: newWriteBatch
|
||||
* Signature: (I)J
|
||||
* Signature: (I)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_WriteBatch_newWriteBatch(
|
||||
JNIEnv* env, jclass jcls, jint jreserved_bytes) {
|
||||
void Java_org_rocksdb_WriteBatch_newWriteBatch(
|
||||
JNIEnv* env, jobject jobj, jint jreserved_bytes) {
|
||||
rocksdb::WriteBatch* wb = new rocksdb::WriteBatch(
|
||||
static_cast<size_t>(jreserved_bytes));
|
||||
return reinterpret_cast<jlong>(wb);
|
||||
|
||||
rocksdb::WriteBatchJni::setHandle(env, jobj, wb);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatch
|
||||
* Method: count0
|
||||
* Signature: (J)I
|
||||
* Signature: ()I
|
||||
*/
|
||||
jint Java_org_rocksdb_WriteBatch_count0(JNIEnv* env, jobject jobj,
|
||||
jlong jwb_handle) {
|
||||
auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
jint Java_org_rocksdb_WriteBatch_count0(JNIEnv* env, jobject jobj) {
|
||||
rocksdb::WriteBatch* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
|
||||
assert(wb != nullptr);
|
||||
|
||||
return static_cast<jint>(wb->Count());
|
||||
@@ -52,11 +52,10 @@ jint Java_org_rocksdb_WriteBatch_count0(JNIEnv* env, jobject jobj,
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatch
|
||||
* Method: clear0
|
||||
* Signature: (J)V
|
||||
* Signature: ()V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatch_clear0(JNIEnv* env, jobject jobj,
|
||||
jlong jwb_handle) {
|
||||
auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
void Java_org_rocksdb_WriteBatch_clear0(JNIEnv* env, jobject jobj) {
|
||||
rocksdb::WriteBatch* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
|
||||
assert(wb != nullptr);
|
||||
|
||||
wb->Clear();
|
||||
@@ -65,13 +64,13 @@ void Java_org_rocksdb_WriteBatch_clear0(JNIEnv* env, jobject jobj,
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatch
|
||||
* Method: put
|
||||
* Signature: (J[BI[BI)V
|
||||
* Signature: ([BI[BI)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatch_put__J_3BI_3BI(
|
||||
JNIEnv* env, jobject jobj, jlong jwb_handle,
|
||||
void Java_org_rocksdb_WriteBatch_put___3BI_3BI(
|
||||
JNIEnv* env, jobject jobj,
|
||||
jbyteArray jkey, jint jkey_len,
|
||||
jbyteArray jentry_value, jint jentry_value_len) {
|
||||
auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
|
||||
assert(wb != nullptr);
|
||||
auto put = [&wb] (rocksdb::Slice key, rocksdb::Slice value) {
|
||||
wb->Put(key, value);
|
||||
@@ -83,13 +82,13 @@ void Java_org_rocksdb_WriteBatch_put__J_3BI_3BI(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatch
|
||||
* Method: put
|
||||
* Signature: (J[BI[BIJ)V
|
||||
* Signature: ([BI[BIJ)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatch_put__J_3BI_3BIJ(
|
||||
JNIEnv* env, jobject jobj, jlong jwb_handle,
|
||||
void Java_org_rocksdb_WriteBatch_put___3BI_3BIJ(
|
||||
JNIEnv* env, jobject jobj,
|
||||
jbyteArray jkey, jint jkey_len,
|
||||
jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) {
|
||||
auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
|
||||
assert(wb != nullptr);
|
||||
auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
|
||||
assert(cf_handle != nullptr);
|
||||
@@ -103,13 +102,13 @@ void Java_org_rocksdb_WriteBatch_put__J_3BI_3BIJ(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatch
|
||||
* Method: merge
|
||||
* Signature: (J[BI[BI)V
|
||||
* Signature: ([BI[BI)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatch_merge__J_3BI_3BI(
|
||||
JNIEnv* env, jobject jobj, jlong jwb_handle,
|
||||
void Java_org_rocksdb_WriteBatch_merge___3BI_3BI(
|
||||
JNIEnv* env, jobject jobj,
|
||||
jbyteArray jkey, jint jkey_len,
|
||||
jbyteArray jentry_value, jint jentry_value_len) {
|
||||
auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
|
||||
assert(wb != nullptr);
|
||||
auto merge = [&wb] (rocksdb::Slice key, rocksdb::Slice value) {
|
||||
wb->Merge(key, value);
|
||||
@@ -121,13 +120,13 @@ void Java_org_rocksdb_WriteBatch_merge__J_3BI_3BI(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatch
|
||||
* Method: merge
|
||||
* Signature: (J[BI[BIJ)V
|
||||
* Signature: ([BI[BIJ)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatch_merge__J_3BI_3BIJ(
|
||||
JNIEnv* env, jobject jobj, jlong jwb_handle,
|
||||
void Java_org_rocksdb_WriteBatch_merge___3BI_3BIJ(
|
||||
JNIEnv* env, jobject jobj,
|
||||
jbyteArray jkey, jint jkey_len,
|
||||
jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) {
|
||||
auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
|
||||
assert(wb != nullptr);
|
||||
auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
|
||||
assert(cf_handle != nullptr);
|
||||
@@ -141,12 +140,12 @@ void Java_org_rocksdb_WriteBatch_merge__J_3BI_3BIJ(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatch
|
||||
* Method: remove
|
||||
* Signature: (J[BI)V
|
||||
* Signature: ([BI)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatch_remove__J_3BI(
|
||||
JNIEnv* env, jobject jobj, jlong jwb_handle,
|
||||
void Java_org_rocksdb_WriteBatch_remove___3BI(
|
||||
JNIEnv* env, jobject jobj,
|
||||
jbyteArray jkey, jint jkey_len) {
|
||||
auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
|
||||
assert(wb != nullptr);
|
||||
auto remove = [&wb] (rocksdb::Slice key) {
|
||||
wb->Delete(key);
|
||||
@@ -157,12 +156,12 @@ void Java_org_rocksdb_WriteBatch_remove__J_3BI(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatch
|
||||
* Method: remove
|
||||
* Signature: (J[BIJ)V
|
||||
* Signature: ([BIJ)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatch_remove__J_3BIJ(
|
||||
JNIEnv* env, jobject jobj, jlong jwb_handle,
|
||||
void Java_org_rocksdb_WriteBatch_remove___3BIJ(
|
||||
JNIEnv* env, jobject jobj,
|
||||
jbyteArray jkey, jint jkey_len, jlong jcf_handle) {
|
||||
auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
|
||||
assert(wb != nullptr);
|
||||
auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
|
||||
assert(cf_handle != nullptr);
|
||||
@@ -175,12 +174,11 @@ void Java_org_rocksdb_WriteBatch_remove__J_3BIJ(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatch
|
||||
* Method: putLogData
|
||||
* Signature: (J[BI)V
|
||||
* Signature: ([BI)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatch_putLogData(
|
||||
JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jblob,
|
||||
jint jblob_len) {
|
||||
auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
JNIEnv* env, jobject jobj, jbyteArray jblob, jint jblob_len) {
|
||||
auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
|
||||
assert(wb != nullptr);
|
||||
auto putLogData = [&wb] (rocksdb::Slice blob) {
|
||||
wb->PutLogData(blob);
|
||||
@@ -191,11 +189,11 @@ void Java_org_rocksdb_WriteBatch_putLogData(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatch
|
||||
* Method: iterate
|
||||
* Signature: (JJ)V
|
||||
* Signature: (J)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatch_iterate(
|
||||
JNIEnv* env , jobject jobj, jlong jwb_handle, jlong handlerHandle) {
|
||||
auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
JNIEnv* env , jobject jobj, jlong handlerHandle) {
|
||||
rocksdb::WriteBatch* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
|
||||
assert(wb != nullptr);
|
||||
|
||||
rocksdb::Status s = wb->Iterate(
|
||||
@@ -220,13 +218,13 @@ void Java_org_rocksdb_WriteBatch_disposeInternal(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatch_Handler
|
||||
* Method: createNewHandler0
|
||||
* Signature: ()J
|
||||
* Signature: ()V
|
||||
*/
|
||||
jlong Java_org_rocksdb_WriteBatch_00024Handler_createNewHandler0(
|
||||
void Java_org_rocksdb_WriteBatch_00024Handler_createNewHandler0(
|
||||
JNIEnv* env, jobject jobj) {
|
||||
const rocksdb::WriteBatchHandlerJniCallback* h =
|
||||
new rocksdb::WriteBatchHandlerJniCallback(env, jobj);
|
||||
return reinterpret_cast<jlong>(h);
|
||||
rocksdb::WriteBatchHandlerJni::setHandle(env, jobj, h);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@@ -28,11 +28,11 @@
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchTest
|
||||
* Method: getContents
|
||||
* Signature: (J)[B
|
||||
* Signature: (Lorg/rocksdb/WriteBatch;)[B
|
||||
*/
|
||||
jbyteArray Java_org_rocksdb_WriteBatchTest_getContents(
|
||||
JNIEnv* env, jclass jclazz, jlong jwb_handle) {
|
||||
auto* b = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
JNIEnv* env, jclass jclazz, jobject jobj) {
|
||||
rocksdb::WriteBatch* b = rocksdb::WriteBatchJni::getHandle(env, jobj);
|
||||
assert(b != nullptr);
|
||||
|
||||
// todo: Currently the following code is directly copied from
|
||||
@@ -109,11 +109,11 @@ jbyteArray Java_org_rocksdb_WriteBatchTest_getContents(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchTestInternalHelper
|
||||
* Method: setSequence
|
||||
* Signature: (JJ)V
|
||||
* Signature: (Lorg/rocksdb/WriteBatch;J)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatchTestInternalHelper_setSequence(
|
||||
JNIEnv* env, jclass jclazz, jlong jwb_handle, jlong jsn) {
|
||||
auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
JNIEnv* env, jclass jclazz, jobject jobj, jlong jsn) {
|
||||
rocksdb::WriteBatch* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
|
||||
assert(wb != nullptr);
|
||||
|
||||
rocksdb::WriteBatchInternal::SetSequence(
|
||||
@@ -123,11 +123,11 @@ void Java_org_rocksdb_WriteBatchTestInternalHelper_setSequence(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchTestInternalHelper
|
||||
* Method: sequence
|
||||
* Signature: (J)J
|
||||
* Signature: (Lorg/rocksdb/WriteBatch;)J
|
||||
*/
|
||||
jlong Java_org_rocksdb_WriteBatchTestInternalHelper_sequence(
|
||||
JNIEnv* env, jclass jclazz, jlong jwb_handle) {
|
||||
auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
|
||||
JNIEnv* env, jclass jclazz, jobject jobj) {
|
||||
rocksdb::WriteBatch* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
|
||||
assert(wb != nullptr);
|
||||
|
||||
return static_cast<jlong>(rocksdb::WriteBatchInternal::Sequence(wb));
|
||||
@@ -136,13 +136,13 @@ jlong Java_org_rocksdb_WriteBatchTestInternalHelper_sequence(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchTestInternalHelper
|
||||
* Method: append
|
||||
* Signature: (JJ)V
|
||||
* Signature: (Lorg/rocksdb/WriteBatch;Lorg/rocksdb/WriteBatch;)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatchTestInternalHelper_append(
|
||||
JNIEnv* env, jclass jclazz, jlong jwb_handle_1, jlong jwb_handle_2) {
|
||||
auto* wb1 = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle_1);
|
||||
JNIEnv* env, jclass jclazz, jobject jwb1, jobject jwb2) {
|
||||
rocksdb::WriteBatch* wb1 = rocksdb::WriteBatchJni::getHandle(env, jwb1);
|
||||
assert(wb1 != nullptr);
|
||||
auto* wb2 = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle_2);
|
||||
rocksdb::WriteBatch* wb2 = rocksdb::WriteBatchJni::getHandle(env, jwb2);
|
||||
assert(wb2 != nullptr);
|
||||
|
||||
rocksdb::WriteBatchInternal::Append(wb1, wb2);
|
||||
|
||||
@@ -15,50 +15,51 @@
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: newWriteBatchWithIndex
|
||||
* Signature: ()J
|
||||
* Signature: ()V
|
||||
*/
|
||||
jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__(
|
||||
JNIEnv* env, jclass jcls) {
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__(
|
||||
JNIEnv* env, jobject jobj) {
|
||||
rocksdb::WriteBatchWithIndex* wbwi = new rocksdb::WriteBatchWithIndex();
|
||||
return reinterpret_cast<jlong>(wbwi);
|
||||
rocksdb::WriteBatchWithIndexJni::setHandle(env, jobj, wbwi);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: newWriteBatchWithIndex
|
||||
* Signature: (Z)J
|
||||
* Signature: (Z)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__Z(
|
||||
JNIEnv* env, jclass jcls, jboolean joverwrite_key) {
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__Z(
|
||||
JNIEnv* env, jobject jobj, jboolean joverwrite_key) {
|
||||
rocksdb::WriteBatchWithIndex* wbwi =
|
||||
new rocksdb::WriteBatchWithIndex(rocksdb::BytewiseComparator(), 0,
|
||||
static_cast<bool>(joverwrite_key));
|
||||
return reinterpret_cast<jlong>(wbwi);
|
||||
rocksdb::WriteBatchWithIndexJni::setHandle(env, jobj, wbwi);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: newWriteBatchWithIndex
|
||||
* Signature: (JIZ)J
|
||||
* Signature: (JIZ)V
|
||||
*/
|
||||
jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__JIZ(
|
||||
JNIEnv* env, jclass jcls, jlong jfallback_index_comparator_handle,
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__JIZ(
|
||||
JNIEnv* env, jobject jobj, jlong jfallback_index_comparator_handle,
|
||||
jint jreserved_bytes, jboolean joverwrite_key) {
|
||||
rocksdb::WriteBatchWithIndex* wbwi =
|
||||
new rocksdb::WriteBatchWithIndex(
|
||||
reinterpret_cast<rocksdb::Comparator*>(jfallback_index_comparator_handle),
|
||||
static_cast<size_t>(jreserved_bytes), static_cast<bool>(joverwrite_key));
|
||||
return reinterpret_cast<jlong>(wbwi);
|
||||
rocksdb::WriteBatchWithIndexJni::setHandle(env, jobj, wbwi);
|
||||
}
|
||||
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: count0
|
||||
* Signature: (J)I
|
||||
* Method: count
|
||||
* Signature: ()I
|
||||
*/
|
||||
jint Java_org_rocksdb_WriteBatchWithIndex_count0(
|
||||
JNIEnv* env, jobject jobj, jlong jwbwi_handle) {
|
||||
auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
|
||||
JNIEnv* env, jobject jobj) {
|
||||
rocksdb::WriteBatchWithIndex* wbwi =
|
||||
rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
|
||||
assert(wbwi != nullptr);
|
||||
|
||||
return static_cast<jint>(wbwi->GetWriteBatch()->Count());
|
||||
@@ -67,12 +68,13 @@ jint Java_org_rocksdb_WriteBatchWithIndex_count0(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: put
|
||||
* Signature: (J[BI[BI)V
|
||||
* Signature: ([BI[BI)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_put__J_3BI_3BI(
|
||||
JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey,
|
||||
jint jkey_len, jbyteArray jentry_value, jint jentry_value_len) {
|
||||
auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_put___3BI_3BI(
|
||||
JNIEnv* env, jobject jobj, jbyteArray jkey, jint jkey_len,
|
||||
jbyteArray jentry_value, jint jentry_value_len) {
|
||||
auto* wbwi =
|
||||
rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
|
||||
assert(wbwi != nullptr);
|
||||
auto put = [&wbwi] (rocksdb::Slice key, rocksdb::Slice value) {
|
||||
wbwi->Put(key, value);
|
||||
@@ -84,13 +86,13 @@ void Java_org_rocksdb_WriteBatchWithIndex_put__J_3BI_3BI(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: put
|
||||
* Signature: (J[BI[BIJ)V
|
||||
* Signature: ([BI[BIJ)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_put__J_3BI_3BIJ(
|
||||
JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey,
|
||||
jint jkey_len, jbyteArray jentry_value, jint jentry_value_len,
|
||||
jlong jcf_handle) {
|
||||
auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_put___3BI_3BIJ(
|
||||
JNIEnv* env, jobject jobj, jbyteArray jkey, jint jkey_len,
|
||||
jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) {
|
||||
auto* wbwi =
|
||||
rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
|
||||
assert(wbwi != nullptr);
|
||||
auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
|
||||
assert(cf_handle != nullptr);
|
||||
@@ -104,12 +106,13 @@ void Java_org_rocksdb_WriteBatchWithIndex_put__J_3BI_3BIJ(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: merge
|
||||
* Signature: (J[BI[BI)V
|
||||
* Signature: ([BI[BI)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_merge__J_3BI_3BI(
|
||||
JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey,
|
||||
jint jkey_len, jbyteArray jentry_value, jint jentry_value_len) {
|
||||
auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_merge___3BI_3BI(
|
||||
JNIEnv* env, jobject jobj, jbyteArray jkey, jint jkey_len,
|
||||
jbyteArray jentry_value, jint jentry_value_len) {
|
||||
auto* wbwi =
|
||||
rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
|
||||
assert(wbwi != nullptr);
|
||||
auto merge = [&wbwi] (rocksdb::Slice key, rocksdb::Slice value) {
|
||||
wbwi->Merge(key, value);
|
||||
@@ -121,13 +124,13 @@ void Java_org_rocksdb_WriteBatchWithIndex_merge__J_3BI_3BI(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: merge
|
||||
* Signature: (J[BI[BIJ)V
|
||||
* Signature: ([BI[BIJ)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_merge__J_3BI_3BIJ(
|
||||
JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey,
|
||||
jint jkey_len, jbyteArray jentry_value, jint jentry_value_len,
|
||||
jlong jcf_handle) {
|
||||
auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_merge___3BI_3BIJ(
|
||||
JNIEnv* env, jobject jobj, jbyteArray jkey, jint jkey_len,
|
||||
jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) {
|
||||
auto* wbwi =
|
||||
rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
|
||||
assert(wbwi != nullptr);
|
||||
auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
|
||||
assert(cf_handle != nullptr);
|
||||
@@ -141,12 +144,12 @@ void Java_org_rocksdb_WriteBatchWithIndex_merge__J_3BI_3BIJ(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: remove
|
||||
* Signature: (J[BI)V
|
||||
* Signature: ([BI)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_remove__J_3BI(
|
||||
JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey,
|
||||
jint jkey_len) {
|
||||
auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_remove___3BI(
|
||||
JNIEnv* env, jobject jobj, jbyteArray jkey, jint jkey_len) {
|
||||
auto* wbwi =
|
||||
rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
|
||||
assert(wbwi != nullptr);
|
||||
auto remove = [&wbwi] (rocksdb::Slice key) {
|
||||
wbwi->Delete(key);
|
||||
@@ -157,12 +160,13 @@ void Java_org_rocksdb_WriteBatchWithIndex_remove__J_3BI(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: remove
|
||||
* Signature: (J[BIJ)V
|
||||
* Signature: ([BIJ)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_remove__J_3BIJ(
|
||||
JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey,
|
||||
jint jkey_len, jlong jcf_handle) {
|
||||
auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_remove___3BIJ(
|
||||
JNIEnv* env, jobject jobj,
|
||||
jbyteArray jkey, jint jkey_len, jlong jcf_handle) {
|
||||
auto* wbwi =
|
||||
rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
|
||||
assert(wbwi != nullptr);
|
||||
auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
|
||||
assert(cf_handle != nullptr);
|
||||
@@ -175,12 +179,12 @@ void Java_org_rocksdb_WriteBatchWithIndex_remove__J_3BIJ(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: putLogData
|
||||
* Signature: (J[BI)V
|
||||
* Signature: ([BI)V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_putLogData(
|
||||
JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jblob,
|
||||
jint jblob_len) {
|
||||
auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
|
||||
JNIEnv* env, jobject jobj, jbyteArray jblob, jint jblob_len) {
|
||||
auto* wbwi =
|
||||
rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
|
||||
assert(wbwi != nullptr);
|
||||
auto putLogData = [&wbwi] (rocksdb::Slice blob) {
|
||||
wbwi->PutLogData(blob);
|
||||
@@ -191,11 +195,12 @@ void Java_org_rocksdb_WriteBatchWithIndex_putLogData(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: clear
|
||||
* Signature: (J)V
|
||||
* Signature: ()V
|
||||
*/
|
||||
void Java_org_rocksdb_WriteBatchWithIndex_clear0(
|
||||
JNIEnv* env, jobject jobj, jlong jwbwi_handle) {
|
||||
auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
|
||||
JNIEnv* env, jobject jobj) {
|
||||
rocksdb::WriteBatchWithIndex* wbwi =
|
||||
rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
|
||||
assert(wbwi != nullptr);
|
||||
|
||||
wbwi->GetWriteBatch()->Clear();
|
||||
@@ -204,11 +209,12 @@ void Java_org_rocksdb_WriteBatchWithIndex_clear0(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: iterator0
|
||||
* Signature: (J)J
|
||||
* Signature: ()J
|
||||
*/
|
||||
jlong Java_org_rocksdb_WriteBatchWithIndex_iterator0(
|
||||
JNIEnv* env, jobject jobj, jlong jwbwi_handle) {
|
||||
auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
|
||||
JNIEnv* env, jobject jobj) {
|
||||
rocksdb::WriteBatchWithIndex* wbwi =
|
||||
rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
|
||||
rocksdb::WBWIIterator* wbwi_iterator = wbwi->NewIterator();
|
||||
return reinterpret_cast<jlong>(wbwi_iterator);
|
||||
}
|
||||
@@ -216,11 +222,12 @@ jlong Java_org_rocksdb_WriteBatchWithIndex_iterator0(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: iterator1
|
||||
* Signature: (JJ)J
|
||||
* Signature: (J)J
|
||||
*/
|
||||
jlong Java_org_rocksdb_WriteBatchWithIndex_iterator1(
|
||||
JNIEnv* env, jobject jobj, jlong jwbwi_handle, jlong jcf_handle) {
|
||||
auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
|
||||
JNIEnv* env, jobject jobj, jlong jcf_handle) {
|
||||
rocksdb::WriteBatchWithIndex* wbwi =
|
||||
rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
|
||||
auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
|
||||
rocksdb::WBWIIterator* wbwi_iterator = wbwi->NewIterator(cf_handle);
|
||||
return reinterpret_cast<jlong>(wbwi_iterator);
|
||||
@@ -229,12 +236,12 @@ jlong Java_org_rocksdb_WriteBatchWithIndex_iterator1(
|
||||
/*
|
||||
* Class: org_rocksdb_WriteBatchWithIndex
|
||||
* Method: iteratorWithBase
|
||||
* Signature: (JJJ)J
|
||||
* Signature: (JJ)J
|
||||
*/
|
||||
jlong Java_org_rocksdb_WriteBatchWithIndex_iteratorWithBase(
|
||||
JNIEnv* env, jobject jobj, jlong jwbwi_handle, jlong jcf_handle,
|
||||
jlong jbi_handle) {
|
||||
auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
|
||||
JNIEnv* env, jobject jobj, jlong jcf_handle, jlong jbi_handle) {
|
||||
rocksdb::WriteBatchWithIndex* wbwi =
|
||||
rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
|
||||
auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
|
||||
auto* base_iterator = reinterpret_cast<rocksdb::Iterator*>(jbi_handle);
|
||||
auto* iterator = wbwi->NewIteratorWithBase(cf_handle, base_iterator);
|
||||
@@ -353,57 +360,27 @@ void Java_org_rocksdb_WBWIRocksIterator_status0(
|
||||
/*
|
||||
* Class: org_rocksdb_WBWIRocksIterator
|
||||
* Method: entry1
|
||||
* Signature: (J)[J
|
||||
* Signature: (JLorg/rocksdb/WBWIRocksIterator/WriteEntry;)V
|
||||
*/
|
||||
jlongArray Java_org_rocksdb_WBWIRocksIterator_entry1(
|
||||
JNIEnv* env, jobject jobj, jlong handle) {
|
||||
void Java_org_rocksdb_WBWIRocksIterator_entry1(
|
||||
JNIEnv* env, jobject jobj, jlong handle, jobject jwrite_entry) {
|
||||
auto* it = reinterpret_cast<rocksdb::WBWIIterator*>(handle);
|
||||
const rocksdb::WriteEntry& we = it->Entry();
|
||||
jobject jwe = rocksdb::WBWIRocksIteratorJni::getWriteEntry(env, jobj);
|
||||
rocksdb::WriteEntryJni::setWriteType(env, jwe, we.type);
|
||||
|
||||
jlong results[3];
|
||||
|
||||
//set the type of the write entry
|
||||
switch (we.type) {
|
||||
case rocksdb::kPutRecord:
|
||||
results[0] = 0x1;
|
||||
break;
|
||||
|
||||
case rocksdb::kMergeRecord:
|
||||
results[0] = 0x2;
|
||||
break;
|
||||
|
||||
case rocksdb::kDeleteRecord:
|
||||
results[0] = 0x4;
|
||||
break;
|
||||
|
||||
case rocksdb::kLogDataRecord:
|
||||
results[0] = 0x8;
|
||||
break;
|
||||
|
||||
default:
|
||||
results[0] = 0x0;
|
||||
}
|
||||
|
||||
//TODO(AR) do we leak buf and value_buf?
|
||||
|
||||
//set the pointer to the key slice
|
||||
char* buf = new char[we.key.size()];
|
||||
memcpy(buf, we.key.data(), we.key.size());
|
||||
auto* key_slice = new rocksdb::Slice(buf, we.key.size());
|
||||
results[1] = reinterpret_cast<jlong>(key_slice);
|
||||
rocksdb::WriteEntryJni::setKey(env, jwe, key_slice);
|
||||
|
||||
//set the pointer to the value slice
|
||||
if (we.type == rocksdb::kDeleteRecord || we.type == rocksdb::kLogDataRecord) {
|
||||
// set native handle of value slice to null if no value available
|
||||
results[2] = 0;
|
||||
rocksdb::WriteEntryJni::setValue(env, jwe, nullptr);
|
||||
} else {
|
||||
char* value_buf = new char[we.value.size()];
|
||||
memcpy(value_buf, we.value.data(), we.value.size());
|
||||
auto* value_slice = new rocksdb::Slice(value_buf, we.value.size());
|
||||
results[2] = reinterpret_cast<jlong>(value_slice);
|
||||
rocksdb::WriteEntryJni::setValue(env, jwe, value_slice);
|
||||
}
|
||||
|
||||
jlongArray jresults = env->NewLongArray(3);
|
||||
env->SetLongArrayRegion(jresults, 0, 3, results);
|
||||
return jresults;
|
||||
}
|
||||
@@ -22,57 +22,73 @@ public class RocksDBColumnFamilySample {
|
||||
String db_path = args[0];
|
||||
|
||||
System.out.println("RocksDBColumnFamilySample");
|
||||
try(final Options options = new Options().setCreateIfMissing(true);
|
||||
final RocksDB db = RocksDB.open(options, db_path)) {
|
||||
|
||||
RocksDB db = null;
|
||||
Options options = null;
|
||||
ColumnFamilyHandle columnFamilyHandle = null;
|
||||
WriteBatch wb = null;
|
||||
try {
|
||||
options = new Options().setCreateIfMissing(true);
|
||||
db = RocksDB.open(options, db_path);
|
||||
assert(db != null);
|
||||
|
||||
// create column family
|
||||
try(final ColumnFamilyHandle columnFamilyHandle = db.createColumnFamily(
|
||||
columnFamilyHandle = db.createColumnFamily(
|
||||
new ColumnFamilyDescriptor("new_cf".getBytes(),
|
||||
new ColumnFamilyOptions()))) {
|
||||
assert (columnFamilyHandle != null);
|
||||
new ColumnFamilyOptions()));
|
||||
assert(columnFamilyHandle != null);
|
||||
|
||||
} finally {
|
||||
if (columnFamilyHandle != null) {
|
||||
columnFamilyHandle.dispose();
|
||||
}
|
||||
if (db != null) {
|
||||
db.close();
|
||||
db = null;
|
||||
}
|
||||
if (options != null) {
|
||||
options.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
// open DB with two column families
|
||||
final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
|
||||
new ArrayList<>();
|
||||
List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
|
||||
// have to open default column family
|
||||
columnFamilyDescriptors.add(new ColumnFamilyDescriptor(
|
||||
RocksDB.DEFAULT_COLUMN_FAMILY, new ColumnFamilyOptions()));
|
||||
// open the new one, too
|
||||
columnFamilyDescriptors.add(new ColumnFamilyDescriptor(
|
||||
"new_cf".getBytes(), new ColumnFamilyOptions()));
|
||||
final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
|
||||
try(final DBOptions options = new DBOptions();
|
||||
final RocksDB db = RocksDB.open(options, db_path,
|
||||
columnFamilyDescriptors, columnFamilyHandles)) {
|
||||
List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
|
||||
try {
|
||||
db = RocksDB.open(new DBOptions(), db_path,
|
||||
columnFamilyDescriptors, columnFamilyHandles);
|
||||
assert(db != null);
|
||||
|
||||
try {
|
||||
// put and get from non-default column family
|
||||
db.put(columnFamilyHandles.get(0), new WriteOptions(),
|
||||
"key".getBytes(), "value".getBytes());
|
||||
String value = new String(db.get(columnFamilyHandles.get(0),
|
||||
"key".getBytes()));
|
||||
// put and get from non-default column family
|
||||
db.put(columnFamilyHandles.get(0), new WriteOptions(),
|
||||
"key".getBytes(), "value".getBytes());
|
||||
String value = new String(db.get(columnFamilyHandles.get(0),
|
||||
"key".getBytes()));
|
||||
|
||||
// atomic write
|
||||
try (final WriteBatch wb = new WriteBatch()) {
|
||||
wb.put(columnFamilyHandles.get(0), "key2".getBytes(),
|
||||
"value2".getBytes());
|
||||
wb.put(columnFamilyHandles.get(1), "key3".getBytes(),
|
||||
"value3".getBytes());
|
||||
wb.remove(columnFamilyHandles.get(0), "key".getBytes());
|
||||
db.write(new WriteOptions(), wb);
|
||||
}
|
||||
// atomic write
|
||||
wb = new WriteBatch();
|
||||
wb.put(columnFamilyHandles.get(0), "key2".getBytes(), "value2".getBytes());
|
||||
wb.put(columnFamilyHandles.get(1), "key3".getBytes(), "value3".getBytes());
|
||||
wb.remove(columnFamilyHandles.get(0), "key".getBytes());
|
||||
db.write(new WriteOptions(), wb);
|
||||
|
||||
// drop column family
|
||||
db.dropColumnFamily(columnFamilyHandles.get(1));
|
||||
} finally {
|
||||
for (final ColumnFamilyHandle handle : columnFamilyHandles) {
|
||||
handle.close();
|
||||
}
|
||||
// drop column family
|
||||
db.dropColumnFamily(columnFamilyHandles.get(1));
|
||||
|
||||
} finally {
|
||||
for (ColumnFamilyHandle handle : columnFamilyHandles){
|
||||
handle.dispose();
|
||||
}
|
||||
if (db != null) {
|
||||
db.close();
|
||||
}
|
||||
if (wb != null) {
|
||||
wb.dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,10 +8,8 @@ import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.rocksdb.*;
|
||||
import org.rocksdb.util.SizeUnit;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class RocksDBSample {
|
||||
@@ -28,273 +26,287 @@ public class RocksDBSample {
|
||||
String db_path_not_found = db_path + "_not_found";
|
||||
|
||||
System.out.println("RocksDBSample");
|
||||
try (final Options options = new Options();
|
||||
final Filter bloomFilter = new BloomFilter(10);
|
||||
final ReadOptions readOptions = new ReadOptions()
|
||||
.setFillCache(false)) {
|
||||
RocksDB db = null;
|
||||
Options options = new Options();
|
||||
try {
|
||||
db = RocksDB.open(options, db_path_not_found);
|
||||
assert(false);
|
||||
} catch (RocksDBException e) {
|
||||
System.out.format("caught the expceted exception -- %s\n", e);
|
||||
assert(db == null);
|
||||
}
|
||||
|
||||
try (final RocksDB db = RocksDB.open(options, db_path_not_found)) {
|
||||
assert (false);
|
||||
} catch (RocksDBException e) {
|
||||
System.out.format("caught the expected exception -- %s\n", e);
|
||||
try {
|
||||
options.setCreateIfMissing(true)
|
||||
.createStatistics()
|
||||
.setWriteBufferSize(8 * SizeUnit.KB)
|
||||
.setMaxWriteBufferNumber(3)
|
||||
.setMaxBackgroundCompactions(10)
|
||||
.setCompressionType(CompressionType.SNAPPY_COMPRESSION)
|
||||
.setCompactionStyle(CompactionStyle.UNIVERSAL);
|
||||
} catch (IllegalArgumentException e) {
|
||||
assert(false);
|
||||
}
|
||||
|
||||
Statistics stats = options.statisticsPtr();
|
||||
|
||||
assert(options.createIfMissing() == true);
|
||||
assert(options.writeBufferSize() == 8 * SizeUnit.KB);
|
||||
assert(options.maxWriteBufferNumber() == 3);
|
||||
assert(options.maxBackgroundCompactions() == 10);
|
||||
assert(options.compressionType() == CompressionType.SNAPPY_COMPRESSION);
|
||||
assert(options.compactionStyle() == CompactionStyle.UNIVERSAL);
|
||||
|
||||
assert(options.memTableFactoryName().equals("SkipListFactory"));
|
||||
options.setMemTableConfig(
|
||||
new HashSkipListMemTableConfig()
|
||||
.setHeight(4)
|
||||
.setBranchingFactor(4)
|
||||
.setBucketCount(2000000));
|
||||
assert(options.memTableFactoryName().equals("HashSkipListRepFactory"));
|
||||
|
||||
options.setMemTableConfig(
|
||||
new HashLinkedListMemTableConfig()
|
||||
.setBucketCount(100000));
|
||||
assert(options.memTableFactoryName().equals("HashLinkedListRepFactory"));
|
||||
|
||||
options.setMemTableConfig(
|
||||
new VectorMemTableConfig().setReservedSize(10000));
|
||||
assert(options.memTableFactoryName().equals("VectorRepFactory"));
|
||||
|
||||
options.setMemTableConfig(new SkipListMemTableConfig());
|
||||
assert(options.memTableFactoryName().equals("SkipListFactory"));
|
||||
|
||||
options.setTableFormatConfig(new PlainTableConfig());
|
||||
// Plain-Table requires mmap read
|
||||
options.setAllowMmapReads(true);
|
||||
assert(options.tableFactoryName().equals("PlainTable"));
|
||||
|
||||
options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000,
|
||||
10000, 10));
|
||||
options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000));
|
||||
|
||||
|
||||
Filter bloomFilter = new BloomFilter(10);
|
||||
BlockBasedTableConfig table_options = new BlockBasedTableConfig();
|
||||
table_options.setBlockCacheSize(64 * SizeUnit.KB)
|
||||
.setFilter(bloomFilter)
|
||||
.setCacheNumShardBits(6)
|
||||
.setBlockSizeDeviation(5)
|
||||
.setBlockRestartInterval(10)
|
||||
.setCacheIndexAndFilterBlocks(true)
|
||||
.setHashIndexAllowCollision(false)
|
||||
.setBlockCacheCompressedSize(64 * SizeUnit.KB)
|
||||
.setBlockCacheCompressedNumShardBits(10);
|
||||
|
||||
assert(table_options.blockCacheSize() == 64 * SizeUnit.KB);
|
||||
assert(table_options.cacheNumShardBits() == 6);
|
||||
assert(table_options.blockSizeDeviation() == 5);
|
||||
assert(table_options.blockRestartInterval() == 10);
|
||||
assert(table_options.cacheIndexAndFilterBlocks() == true);
|
||||
assert(table_options.hashIndexAllowCollision() == false);
|
||||
assert(table_options.blockCacheCompressedSize() == 64 * SizeUnit.KB);
|
||||
assert(table_options.blockCacheCompressedNumShardBits() == 10);
|
||||
|
||||
options.setTableFormatConfig(table_options);
|
||||
assert(options.tableFactoryName().equals("BlockBasedTable"));
|
||||
|
||||
try {
|
||||
db = RocksDB.open(options, db_path);
|
||||
db.put("hello".getBytes(), "world".getBytes());
|
||||
byte[] value = db.get("hello".getBytes());
|
||||
assert("world".equals(new String(value)));
|
||||
String str = db.getProperty("rocksdb.stats");
|
||||
assert(str != null && !str.equals(""));
|
||||
} catch (RocksDBException e) {
|
||||
System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e);
|
||||
assert(db == null);
|
||||
assert(false);
|
||||
}
|
||||
// be sure to release the c++ pointer
|
||||
db.close();
|
||||
|
||||
ReadOptions readOptions = new ReadOptions();
|
||||
readOptions.setFillCache(false);
|
||||
|
||||
try {
|
||||
db = RocksDB.open(options, db_path);
|
||||
db.put("hello".getBytes(), "world".getBytes());
|
||||
byte[] value = db.get("hello".getBytes());
|
||||
System.out.format("Get('hello') = %s\n",
|
||||
new String(value));
|
||||
|
||||
for (int i = 1; i <= 9; ++i) {
|
||||
for (int j = 1; j <= 9; ++j) {
|
||||
db.put(String.format("%dx%d", i, j).getBytes(),
|
||||
String.format("%d", i * j).getBytes());
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 1; i <= 9; ++i) {
|
||||
for (int j = 1; j <= 9; ++j) {
|
||||
System.out.format("%s ", new String(db.get(
|
||||
String.format("%dx%d", i, j).getBytes())));
|
||||
}
|
||||
System.out.println("");
|
||||
}
|
||||
|
||||
// write batch test
|
||||
WriteOptions writeOpt = new WriteOptions();
|
||||
for (int i = 10; i <= 19; ++i) {
|
||||
WriteBatch batch = new WriteBatch();
|
||||
for (int j = 10; j <= 19; ++j) {
|
||||
batch.put(String.format("%dx%d", i, j).getBytes(),
|
||||
String.format("%d", i * j).getBytes());
|
||||
}
|
||||
db.write(writeOpt, batch);
|
||||
batch.dispose();
|
||||
}
|
||||
for (int i = 10; i <= 19; ++i) {
|
||||
for (int j = 10; j <= 19; ++j) {
|
||||
assert(new String(
|
||||
db.get(String.format("%dx%d", i, j).getBytes())).equals(
|
||||
String.format("%d", i * j)));
|
||||
System.out.format("%s ", new String(db.get(
|
||||
String.format("%dx%d", i, j).getBytes())));
|
||||
}
|
||||
System.out.println("");
|
||||
}
|
||||
writeOpt.dispose();
|
||||
|
||||
value = db.get("1x1".getBytes());
|
||||
assert(value != null);
|
||||
value = db.get("world".getBytes());
|
||||
assert(value == null);
|
||||
value = db.get(readOptions, "world".getBytes());
|
||||
assert(value == null);
|
||||
|
||||
byte[] testKey = "asdf".getBytes();
|
||||
byte[] testValue =
|
||||
"asdfghjkl;'?><MNBVCXZQWERTYUIOP{+_)(*&^%$#@".getBytes();
|
||||
db.put(testKey, testValue);
|
||||
byte[] testResult = db.get(testKey);
|
||||
assert(testResult != null);
|
||||
assert(Arrays.equals(testValue, testResult));
|
||||
assert(new String(testValue).equals(new String(testResult)));
|
||||
testResult = db.get(readOptions, testKey);
|
||||
assert(testResult != null);
|
||||
assert(Arrays.equals(testValue, testResult));
|
||||
assert(new String(testValue).equals(new String(testResult)));
|
||||
|
||||
byte[] insufficientArray = new byte[10];
|
||||
byte[] enoughArray = new byte[50];
|
||||
int len;
|
||||
len = db.get(testKey, insufficientArray);
|
||||
assert(len > insufficientArray.length);
|
||||
len = db.get("asdfjkl;".getBytes(), enoughArray);
|
||||
assert(len == RocksDB.NOT_FOUND);
|
||||
len = db.get(testKey, enoughArray);
|
||||
assert(len == testValue.length);
|
||||
|
||||
len = db.get(readOptions, testKey, insufficientArray);
|
||||
assert(len > insufficientArray.length);
|
||||
len = db.get(readOptions, "asdfjkl;".getBytes(), enoughArray);
|
||||
assert(len == RocksDB.NOT_FOUND);
|
||||
len = db.get(readOptions, testKey, enoughArray);
|
||||
assert(len == testValue.length);
|
||||
|
||||
db.remove(testKey);
|
||||
len = db.get(testKey, enoughArray);
|
||||
assert(len == RocksDB.NOT_FOUND);
|
||||
|
||||
// repeat the test with WriteOptions
|
||||
WriteOptions writeOpts = new WriteOptions();
|
||||
writeOpts.setSync(true);
|
||||
writeOpts.setDisableWAL(true);
|
||||
db.put(writeOpts, testKey, testValue);
|
||||
len = db.get(testKey, enoughArray);
|
||||
assert(len == testValue.length);
|
||||
assert(new String(testValue).equals(
|
||||
new String(enoughArray, 0, len)));
|
||||
writeOpts.dispose();
|
||||
|
||||
try {
|
||||
for (TickerType statsType : TickerType.values()) {
|
||||
stats.getTickerCount(statsType);
|
||||
}
|
||||
System.out.println("getTickerCount() passed.");
|
||||
} catch (Exception e) {
|
||||
System.out.println("Failed in call to getTickerCount()");
|
||||
assert(false); //Should never reach here.
|
||||
}
|
||||
|
||||
try {
|
||||
options.setCreateIfMissing(true)
|
||||
.createStatistics()
|
||||
.setWriteBufferSize(8 * SizeUnit.KB)
|
||||
.setMaxWriteBufferNumber(3)
|
||||
.setMaxBackgroundCompactions(10)
|
||||
.setCompressionType(CompressionType.SNAPPY_COMPRESSION)
|
||||
.setCompactionStyle(CompactionStyle.UNIVERSAL);
|
||||
} catch (IllegalArgumentException e) {
|
||||
assert (false);
|
||||
for (HistogramType histogramType : HistogramType.values()) {
|
||||
HistogramData data = stats.geHistogramData(histogramType);
|
||||
}
|
||||
System.out.println("geHistogramData() passed.");
|
||||
} catch (Exception e) {
|
||||
System.out.println("Failed in call to geHistogramData()");
|
||||
assert(false); //Should never reach here.
|
||||
}
|
||||
|
||||
Statistics stats = options.statisticsPtr();
|
||||
RocksIterator iterator = db.newIterator();
|
||||
|
||||
assert (options.createIfMissing() == true);
|
||||
assert (options.writeBufferSize() == 8 * SizeUnit.KB);
|
||||
assert (options.maxWriteBufferNumber() == 3);
|
||||
assert (options.maxBackgroundCompactions() == 10);
|
||||
assert (options.compressionType() == CompressionType.SNAPPY_COMPRESSION);
|
||||
assert (options.compactionStyle() == CompactionStyle.UNIVERSAL);
|
||||
|
||||
assert (options.memTableFactoryName().equals("SkipListFactory"));
|
||||
options.setMemTableConfig(
|
||||
new HashSkipListMemTableConfig()
|
||||
.setHeight(4)
|
||||
.setBranchingFactor(4)
|
||||
.setBucketCount(2000000));
|
||||
assert (options.memTableFactoryName().equals("HashSkipListRepFactory"));
|
||||
|
||||
options.setMemTableConfig(
|
||||
new HashLinkedListMemTableConfig()
|
||||
.setBucketCount(100000));
|
||||
assert (options.memTableFactoryName().equals("HashLinkedListRepFactory"));
|
||||
|
||||
options.setMemTableConfig(
|
||||
new VectorMemTableConfig().setReservedSize(10000));
|
||||
assert (options.memTableFactoryName().equals("VectorRepFactory"));
|
||||
|
||||
options.setMemTableConfig(new SkipListMemTableConfig());
|
||||
assert (options.memTableFactoryName().equals("SkipListFactory"));
|
||||
|
||||
options.setTableFormatConfig(new PlainTableConfig());
|
||||
// Plain-Table requires mmap read
|
||||
options.setAllowMmapReads(true);
|
||||
assert (options.tableFactoryName().equals("PlainTable"));
|
||||
|
||||
options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000,
|
||||
10000, 10));
|
||||
options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000));
|
||||
|
||||
final BlockBasedTableConfig table_options = new BlockBasedTableConfig();
|
||||
table_options.setBlockCacheSize(64 * SizeUnit.KB)
|
||||
.setFilter(bloomFilter)
|
||||
.setCacheNumShardBits(6)
|
||||
.setBlockSizeDeviation(5)
|
||||
.setBlockRestartInterval(10)
|
||||
.setCacheIndexAndFilterBlocks(true)
|
||||
.setHashIndexAllowCollision(false)
|
||||
.setBlockCacheCompressedSize(64 * SizeUnit.KB)
|
||||
.setBlockCacheCompressedNumShardBits(10);
|
||||
|
||||
assert (table_options.blockCacheSize() == 64 * SizeUnit.KB);
|
||||
assert (table_options.cacheNumShardBits() == 6);
|
||||
assert (table_options.blockSizeDeviation() == 5);
|
||||
assert (table_options.blockRestartInterval() == 10);
|
||||
assert (table_options.cacheIndexAndFilterBlocks() == true);
|
||||
assert (table_options.hashIndexAllowCollision() == false);
|
||||
assert (table_options.blockCacheCompressedSize() == 64 * SizeUnit.KB);
|
||||
assert (table_options.blockCacheCompressedNumShardBits() == 10);
|
||||
|
||||
options.setTableFormatConfig(table_options);
|
||||
assert (options.tableFactoryName().equals("BlockBasedTable"));
|
||||
|
||||
try (final RocksDB db = RocksDB.open(options, db_path)) {
|
||||
db.put("hello".getBytes(), "world".getBytes());
|
||||
byte[] value = db.get("hello".getBytes());
|
||||
assert ("world".equals(new String(value)));
|
||||
String str = db.getProperty("rocksdb.stats");
|
||||
assert (str != null && !str.equals(""));
|
||||
} catch (RocksDBException e) {
|
||||
System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e);
|
||||
assert (false);
|
||||
boolean seekToFirstPassed = false;
|
||||
for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) {
|
||||
iterator.status();
|
||||
assert(iterator.key() != null);
|
||||
assert(iterator.value() != null);
|
||||
seekToFirstPassed = true;
|
||||
}
|
||||
if(seekToFirstPassed) {
|
||||
System.out.println("iterator seekToFirst tests passed.");
|
||||
}
|
||||
|
||||
try (final RocksDB db = RocksDB.open(options, db_path)) {
|
||||
db.put("hello".getBytes(), "world".getBytes());
|
||||
byte[] value = db.get("hello".getBytes());
|
||||
System.out.format("Get('hello') = %s\n",
|
||||
new String(value));
|
||||
|
||||
for (int i = 1; i <= 9; ++i) {
|
||||
for (int j = 1; j <= 9; ++j) {
|
||||
db.put(String.format("%dx%d", i, j).getBytes(),
|
||||
String.format("%d", i * j).getBytes());
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 1; i <= 9; ++i) {
|
||||
for (int j = 1; j <= 9; ++j) {
|
||||
System.out.format("%s ", new String(db.get(
|
||||
String.format("%dx%d", i, j).getBytes())));
|
||||
}
|
||||
System.out.println("");
|
||||
}
|
||||
|
||||
// write batch test
|
||||
try (final WriteOptions writeOpt = new WriteOptions()) {
|
||||
for (int i = 10; i <= 19; ++i) {
|
||||
try (final WriteBatch batch = new WriteBatch()) {
|
||||
for (int j = 10; j <= 19; ++j) {
|
||||
batch.put(String.format("%dx%d", i, j).getBytes(),
|
||||
String.format("%d", i * j).getBytes());
|
||||
}
|
||||
db.write(writeOpt, batch);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (int i = 10; i <= 19; ++i) {
|
||||
for (int j = 10; j <= 19; ++j) {
|
||||
assert (new String(
|
||||
db.get(String.format("%dx%d", i, j).getBytes())).equals(
|
||||
String.format("%d", i * j)));
|
||||
System.out.format("%s ", new String(db.get(
|
||||
String.format("%dx%d", i, j).getBytes())));
|
||||
}
|
||||
System.out.println("");
|
||||
}
|
||||
|
||||
value = db.get("1x1".getBytes());
|
||||
assert (value != null);
|
||||
value = db.get("world".getBytes());
|
||||
assert (value == null);
|
||||
value = db.get(readOptions, "world".getBytes());
|
||||
assert (value == null);
|
||||
|
||||
byte[] testKey = "asdf".getBytes();
|
||||
byte[] testValue =
|
||||
"asdfghjkl;'?><MNBVCXZQWERTYUIOP{+_)(*&^%$#@".getBytes();
|
||||
db.put(testKey, testValue);
|
||||
byte[] testResult = db.get(testKey);
|
||||
assert (testResult != null);
|
||||
assert (Arrays.equals(testValue, testResult));
|
||||
assert (new String(testValue).equals(new String(testResult)));
|
||||
testResult = db.get(readOptions, testKey);
|
||||
assert (testResult != null);
|
||||
assert (Arrays.equals(testValue, testResult));
|
||||
assert (new String(testValue).equals(new String(testResult)));
|
||||
|
||||
byte[] insufficientArray = new byte[10];
|
||||
byte[] enoughArray = new byte[50];
|
||||
int len;
|
||||
len = db.get(testKey, insufficientArray);
|
||||
assert (len > insufficientArray.length);
|
||||
len = db.get("asdfjkl;".getBytes(), enoughArray);
|
||||
assert (len == RocksDB.NOT_FOUND);
|
||||
len = db.get(testKey, enoughArray);
|
||||
assert (len == testValue.length);
|
||||
|
||||
len = db.get(readOptions, testKey, insufficientArray);
|
||||
assert (len > insufficientArray.length);
|
||||
len = db.get(readOptions, "asdfjkl;".getBytes(), enoughArray);
|
||||
assert (len == RocksDB.NOT_FOUND);
|
||||
len = db.get(readOptions, testKey, enoughArray);
|
||||
assert (len == testValue.length);
|
||||
|
||||
db.remove(testKey);
|
||||
len = db.get(testKey, enoughArray);
|
||||
assert (len == RocksDB.NOT_FOUND);
|
||||
|
||||
// repeat the test with WriteOptions
|
||||
try (final WriteOptions writeOpts = new WriteOptions()) {
|
||||
writeOpts.setSync(true);
|
||||
writeOpts.setDisableWAL(true);
|
||||
db.put(writeOpts, testKey, testValue);
|
||||
len = db.get(testKey, enoughArray);
|
||||
assert (len == testValue.length);
|
||||
assert (new String(testValue).equals(
|
||||
new String(enoughArray, 0, len)));
|
||||
}
|
||||
|
||||
try {
|
||||
for (TickerType statsType : TickerType.values()) {
|
||||
stats.getTickerCount(statsType);
|
||||
}
|
||||
System.out.println("getTickerCount() passed.");
|
||||
} catch (Exception e) {
|
||||
System.out.println("Failed in call to getTickerCount()");
|
||||
assert (false); //Should never reach here.
|
||||
}
|
||||
|
||||
try {
|
||||
for (HistogramType histogramType : HistogramType.values()) {
|
||||
HistogramData data = stats.geHistogramData(histogramType);
|
||||
}
|
||||
System.out.println("geHistogramData() passed.");
|
||||
} catch (Exception e) {
|
||||
System.out.println("Failed in call to geHistogramData()");
|
||||
assert (false); //Should never reach here.
|
||||
}
|
||||
|
||||
try (final RocksIterator iterator = db.newIterator()) {
|
||||
|
||||
boolean seekToFirstPassed = false;
|
||||
for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) {
|
||||
iterator.status();
|
||||
assert (iterator.key() != null);
|
||||
assert (iterator.value() != null);
|
||||
seekToFirstPassed = true;
|
||||
}
|
||||
if (seekToFirstPassed) {
|
||||
System.out.println("iterator seekToFirst tests passed.");
|
||||
}
|
||||
|
||||
boolean seekToLastPassed = false;
|
||||
for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) {
|
||||
iterator.status();
|
||||
assert (iterator.key() != null);
|
||||
assert (iterator.value() != null);
|
||||
seekToLastPassed = true;
|
||||
}
|
||||
|
||||
if (seekToLastPassed) {
|
||||
System.out.println("iterator seekToLastPassed tests passed.");
|
||||
}
|
||||
|
||||
iterator.seekToFirst();
|
||||
iterator.seek(iterator.key());
|
||||
assert (iterator.key() != null);
|
||||
assert (iterator.value() != null);
|
||||
|
||||
System.out.println("iterator seek test passed.");
|
||||
|
||||
}
|
||||
System.out.println("iterator tests passed.");
|
||||
|
||||
final List<byte[]> keys = new ArrayList<>();
|
||||
try (final RocksIterator iterator = db.newIterator()) {
|
||||
for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) {
|
||||
keys.add(iterator.key());
|
||||
}
|
||||
}
|
||||
|
||||
Map<byte[], byte[]> values = db.multiGet(keys);
|
||||
assert (values.size() == keys.size());
|
||||
for (byte[] value1 : values.values()) {
|
||||
assert (value1 != null);
|
||||
}
|
||||
|
||||
values = db.multiGet(new ReadOptions(), keys);
|
||||
assert (values.size() == keys.size());
|
||||
for (byte[] value1 : values.values()) {
|
||||
assert (value1 != null);
|
||||
}
|
||||
} catch (RocksDBException e) {
|
||||
System.err.println(e);
|
||||
boolean seekToLastPassed = false;
|
||||
for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) {
|
||||
iterator.status();
|
||||
assert(iterator.key() != null);
|
||||
assert(iterator.value() != null);
|
||||
seekToLastPassed = true;
|
||||
}
|
||||
|
||||
if(seekToLastPassed) {
|
||||
System.out.println("iterator seekToLastPassed tests passed.");
|
||||
}
|
||||
|
||||
iterator.seekToFirst();
|
||||
iterator.seek(iterator.key());
|
||||
assert(iterator.key() != null);
|
||||
assert(iterator.value() != null);
|
||||
|
||||
System.out.println("iterator seek test passed.");
|
||||
|
||||
iterator.dispose();
|
||||
System.out.println("iterator tests passed.");
|
||||
|
||||
iterator = db.newIterator();
|
||||
List<byte[]> keys = new ArrayList<byte[]>();
|
||||
for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) {
|
||||
keys.add(iterator.key());
|
||||
}
|
||||
iterator.dispose();
|
||||
|
||||
Map<byte[], byte[]> values = db.multiGet(keys);
|
||||
assert(values.size() == keys.size());
|
||||
for(byte[] value1 : values.values()) {
|
||||
assert(value1 != null);
|
||||
}
|
||||
|
||||
values = db.multiGet(new ReadOptions(), keys);
|
||||
assert(values.size() == keys.size());
|
||||
for(byte[] value1 : values.values()) {
|
||||
assert(value1 != null);
|
||||
}
|
||||
} catch (RocksDBException e) {
|
||||
System.err.println(e);
|
||||
}
|
||||
if (db != null) {
|
||||
db.close();
|
||||
}
|
||||
// be sure to dispose c++ pointers
|
||||
options.dispose();
|
||||
readOptions.dispose();
|
||||
}
|
||||
}
|
||||
@@ -8,23 +8,22 @@ package org.rocksdb;
|
||||
* A CompactionFilter allows an application to modify/delete a key-value at
|
||||
* the time of compaction.
|
||||
*
|
||||
* At present we just permit an overriding Java class to wrap a C++
|
||||
* implementation
|
||||
* At present we just permit an overriding Java class to wrap a C++ implementation
|
||||
*/
|
||||
public abstract class AbstractCompactionFilter<T extends AbstractSlice<?>>
|
||||
extends RocksObject {
|
||||
|
||||
protected AbstractCompactionFilter(final long nativeHandle) {
|
||||
super(nativeHandle);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes underlying C++ compaction pointer.
|
||||
* Deletes underlying C++ comparator pointer.
|
||||
*
|
||||
* Note that this function should be called only after all
|
||||
* RocksDB instances referencing the compaction filter are closed.
|
||||
* RocksDB instances referencing the comparator are closed.
|
||||
* Otherwise an undefined behavior will occur.
|
||||
*/
|
||||
@Override
|
||||
protected final native void disposeInternal(final long handle);
|
||||
@Override protected void disposeInternal() {
|
||||
assert(isInitialized());
|
||||
disposeInternal(nativeHandle_);
|
||||
}
|
||||
|
||||
private native void disposeInternal(long handle);
|
||||
}
|
||||
@@ -15,11 +15,7 @@ package org.rocksdb;
|
||||
* @see org.rocksdb.DirectComparator
|
||||
*/
|
||||
public abstract class AbstractComparator<T extends AbstractSlice<?>>
|
||||
extends AbstractImmutableNativeReference {
|
||||
|
||||
protected AbstractComparator() {
|
||||
super(true);
|
||||
}
|
||||
extends RocksObject {
|
||||
|
||||
/**
|
||||
* The name of the comparator. Used to check for comparator
|
||||
@@ -95,12 +91,10 @@ public abstract class AbstractComparator<T extends AbstractSlice<?>>
|
||||
* RocksDB instances referencing the comparator are closed.
|
||||
* Otherwise an undefined behavior will occur.
|
||||
*/
|
||||
@Override
|
||||
protected void disposeInternal() {
|
||||
disposeInternal(getNativeHandle());
|
||||
@Override protected void disposeInternal() {
|
||||
assert(isInitialized());
|
||||
disposeInternal(nativeHandle_);
|
||||
}
|
||||
|
||||
protected abstract long getNativeHandle();
|
||||
|
||||
private native void disposeInternal(final long handle);
|
||||
private native void disposeInternal(long handle);
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
// Copyright (c) 2016, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
package org.rocksdb;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
/**
|
||||
* Offers functionality for implementations of
|
||||
* {@link AbstractNativeReference} which have an immutable reference to the
|
||||
* underlying native C++ object
|
||||
*/
|
||||
public abstract class AbstractImmutableNativeReference
|
||||
extends AbstractNativeReference {
|
||||
|
||||
/**
|
||||
* A flag indicating whether the current {@code AbstractNativeReference} is
|
||||
* responsible to free the underlying C++ object
|
||||
*/
|
||||
private final AtomicBoolean owningHandle_;
|
||||
|
||||
protected AbstractImmutableNativeReference(final boolean owningHandle) {
|
||||
this.owningHandle_ = new AtomicBoolean(owningHandle);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isOwningHandle() {
|
||||
return owningHandle_.get();
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases this {@code AbstractNativeReference} from the responsibility of
|
||||
* freeing the underlying native C++ object
|
||||
* <p>
|
||||
* This will prevent the object from attempting to delete the underlying
|
||||
* native object in its finalizer. This must be used when another object
|
||||
* takes over ownership of the native object or both will attempt to delete
|
||||
* the underlying object when garbage collected.
|
||||
* <p>
|
||||
* When {@code disOwnNativeHandle()} is called, {@code dispose()} will
|
||||
* subsequently take no action. As a result, incorrect use of this function
|
||||
* may cause a memory leak.
|
||||
* </p>
|
||||
*
|
||||
* @see #dispose()
|
||||
*/
|
||||
protected final void disOwnNativeHandle() {
|
||||
owningHandle_.set(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (owningHandle_.compareAndSet(true, false)) {
|
||||
disposeInternal();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The helper function of {@link AbstractImmutableNativeReference#dispose()}
|
||||
* which all subclasses of {@code AbstractImmutableNativeReference} must
|
||||
* implement to release their underlying native C++ objects.
|
||||
*/
|
||||
protected abstract void disposeInternal();
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
// Copyright (c) 2016, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
package org.rocksdb;
|
||||
|
||||
/**
|
||||
* AbstractNativeReference is the base-class of all RocksDB classes that have
|
||||
* a pointer to a native C++ {@code rocksdb} object.
|
||||
* <p>
|
||||
* AbstractNativeReference has the {@link AbstractNativeReference#dispose()}
|
||||
* method, which frees its associated C++ object.</p>
|
||||
* <p>
|
||||
* This function should be called manually, however, if required it will be
|
||||
* called automatically during the regular Java GC process via
|
||||
* {@link AbstractNativeReference#finalize()}.</p>
|
||||
* <p>
|
||||
* Note - Java can only see the long member variable (which is the C++ pointer
|
||||
* value to the native object), as such it does not know the real size of the
|
||||
* object and therefore may assign a low GC priority for it; So it is strongly
|
||||
* suggested that you manually dispose of objects when you are finished with
|
||||
* them.</p>
|
||||
*/
|
||||
public abstract class AbstractNativeReference implements AutoCloseable {
|
||||
|
||||
/**
|
||||
* Returns true if we are responsible for freeing the underlying C++ object
|
||||
*
|
||||
* @return true if we are responsible to free the C++ object
|
||||
* @see #dispose()
|
||||
*/
|
||||
protected abstract boolean isOwningHandle();
|
||||
|
||||
/**
|
||||
* Frees the underlying C++ object
|
||||
* <p>
|
||||
* It is strong recommended that the developer calls this after they
|
||||
* have finished using the object.</p>
|
||||
* <p>
|
||||
* Note, that once an instance of {@link AbstractNativeReference} has been
|
||||
* disposed, calling any of its functions will lead to undefined
|
||||
* behavior.</p>
|
||||
*/
|
||||
@Override
|
||||
public abstract void close();
|
||||
|
||||
/**
|
||||
* @deprecated Instead use {@link AbstractNativeReference#close()}
|
||||
*/
|
||||
@Deprecated
|
||||
public final void dispose() {
|
||||
close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Simply calls {@link AbstractNativeReference#dispose()} to free
|
||||
* any underlying C++ object reference which has not yet been manually
|
||||
* released.
|
||||
*
|
||||
* @deprecated You should not rely on GC of Rocks objects, and instead should
|
||||
* either call {@link AbstractNativeReference#close()} manually or make
|
||||
* use of some sort of ARM (Automatic Resource Management) such as
|
||||
* Java 7's <a href="https://docs.oracle.com/javase/tutorial/essential/exceptions/tryResourceClose.html">try-with-resources</a>
|
||||
* statement
|
||||
*/
|
||||
@Override
|
||||
@Deprecated
|
||||
protected void finalize() throws Throwable {
|
||||
if(isOwningHandle()) {
|
||||
//TODO(AR) log a warning message... developer should have called close()
|
||||
}
|
||||
dispose();
|
||||
super.finalize();
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,8 @@ public abstract class AbstractRocksIterator<P extends RocksObject>
|
||||
|
||||
protected AbstractRocksIterator(final P parent,
|
||||
final long nativeHandle) {
|
||||
super(nativeHandle);
|
||||
super();
|
||||
nativeHandle_ = nativeHandle;
|
||||
// parent must point to a valid RocksDB instance.
|
||||
assert (parent != null);
|
||||
// RocksIterator must hold a reference to the related parent instance
|
||||
@@ -36,43 +37,43 @@ public abstract class AbstractRocksIterator<P extends RocksObject>
|
||||
|
||||
@Override
|
||||
public boolean isValid() {
|
||||
assert (isOwningHandle());
|
||||
assert (isInitialized());
|
||||
return isValid0(nativeHandle_);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seekToFirst() {
|
||||
assert (isOwningHandle());
|
||||
assert (isInitialized());
|
||||
seekToFirst0(nativeHandle_);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seekToLast() {
|
||||
assert (isOwningHandle());
|
||||
assert (isInitialized());
|
||||
seekToLast0(nativeHandle_);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seek(byte[] target) {
|
||||
assert (isOwningHandle());
|
||||
assert (isInitialized());
|
||||
seek0(nativeHandle_, target, target.length);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void next() {
|
||||
assert (isOwningHandle());
|
||||
assert (isInitialized());
|
||||
next0(nativeHandle_);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void prev() {
|
||||
assert (isOwningHandle());
|
||||
assert (isInitialized());
|
||||
prev0(nativeHandle_);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void status() throws RocksDBException {
|
||||
assert (isOwningHandle());
|
||||
assert (isInitialized());
|
||||
status0(nativeHandle_);
|
||||
}
|
||||
|
||||
@@ -86,11 +87,15 @@ public abstract class AbstractRocksIterator<P extends RocksObject>
|
||||
*/
|
||||
@Override
|
||||
protected void disposeInternal() {
|
||||
if (parent_.isOwningHandle()) {
|
||||
synchronized (parent_) {
|
||||
assert (isInitialized());
|
||||
if (parent_.isInitialized()) {
|
||||
disposeInternal(nativeHandle_);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
abstract void disposeInternal(long handle);
|
||||
abstract boolean isValid0(long handle);
|
||||
abstract void seekToFirst0(long handle);
|
||||
abstract void seekToLast0(long handle);
|
||||
|
||||
@@ -24,15 +24,7 @@ package org.rocksdb;
|
||||
* C++ BaseComparatorJniCallback subclass, which in turn destroys the
|
||||
* Java @see org.rocksdb.AbstractSlice subclass Objects.
|
||||
*/
|
||||
public abstract class AbstractSlice<T> extends RocksMutableObject {
|
||||
|
||||
protected AbstractSlice() {
|
||||
super();
|
||||
}
|
||||
|
||||
protected AbstractSlice(final long nativeHandle) {
|
||||
super(nativeHandle);
|
||||
}
|
||||
abstract class AbstractSlice<T> extends RocksObject {
|
||||
|
||||
/**
|
||||
* Returns the data of the slice.
|
||||
@@ -42,7 +34,8 @@ public abstract class AbstractSlice<T> extends RocksMutableObject {
|
||||
* @see org.rocksdb.AbstractSlice#data0(long)
|
||||
*/
|
||||
public T data() {
|
||||
return data0(getNativeHandle());
|
||||
assert (isInitialized());
|
||||
return data0(nativeHandle_);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -63,7 +56,8 @@ public abstract class AbstractSlice<T> extends RocksMutableObject {
|
||||
* @return The length in bytes.
|
||||
*/
|
||||
public int size() {
|
||||
return size0(getNativeHandle());
|
||||
assert (isInitialized());
|
||||
return size0(nativeHandle_);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -73,7 +67,8 @@ public abstract class AbstractSlice<T> extends RocksMutableObject {
|
||||
* @return true if there is no data, false otherwise.
|
||||
*/
|
||||
public boolean empty() {
|
||||
return empty0(getNativeHandle());
|
||||
assert (isInitialized());
|
||||
return empty0(nativeHandle_);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -85,7 +80,8 @@ public abstract class AbstractSlice<T> extends RocksMutableObject {
|
||||
* @return The string representation of the data.
|
||||
*/
|
||||
public String toString(final boolean hex) {
|
||||
return toString0(getNativeHandle(), hex);
|
||||
assert (isInitialized());
|
||||
return toString0(nativeHandle_, hex);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -105,15 +101,8 @@ public abstract class AbstractSlice<T> extends RocksMutableObject {
|
||||
*/
|
||||
public int compare(final AbstractSlice<?> other) {
|
||||
assert (other != null);
|
||||
if(!isOwningHandle()) {
|
||||
return other.isOwningHandle() ? -1 : 0;
|
||||
} else {
|
||||
if(!other.isOwningHandle()) {
|
||||
return 1;
|
||||
} else {
|
||||
return compare0(getNativeHandle(), other.getNativeHandle());
|
||||
}
|
||||
}
|
||||
assert (isInitialized());
|
||||
return compare0(nativeHandle_, other.nativeHandle_);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -152,19 +141,13 @@ public abstract class AbstractSlice<T> extends RocksMutableObject {
|
||||
*/
|
||||
public boolean startsWith(final AbstractSlice<?> prefix) {
|
||||
if (prefix != null) {
|
||||
return startsWith0(getNativeHandle(), prefix.getNativeHandle());
|
||||
assert (isInitialized());
|
||||
return startsWith0(nativeHandle_, prefix.nativeHandle_);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
protected native static long createNewSliceFromString(final String str);
|
||||
private native int size0(long handle);
|
||||
private native boolean empty0(long handle);
|
||||
private native String toString0(long handle, boolean hex);
|
||||
private native int compare0(long handle, long otherHandle);
|
||||
private native boolean startsWith0(long handle, long otherHandle);
|
||||
|
||||
/**
|
||||
* Deletes underlying C++ slice pointer.
|
||||
* Note that this function should be called only after all
|
||||
@@ -172,6 +155,17 @@ public abstract class AbstractSlice<T> extends RocksMutableObject {
|
||||
* Otherwise an undefined behavior will occur.
|
||||
*/
|
||||
@Override
|
||||
protected final native void disposeInternal(final long handle);
|
||||
protected void disposeInternal() {
|
||||
assert(isInitialized());
|
||||
disposeInternal(nativeHandle_);
|
||||
}
|
||||
|
||||
protected native void createNewSliceFromString(String str);
|
||||
private native int size0(long handle);
|
||||
private native boolean empty0(long handle);
|
||||
private native String toString0(long handle, boolean hex);
|
||||
private native int compare0(long handle, long otherHandle);
|
||||
private native boolean startsWith0(long handle, long otherHandle);
|
||||
private native void disposeInternal(long handle);
|
||||
|
||||
}
|
||||
@@ -5,93 +5,88 @@
|
||||
|
||||
package org.rocksdb;
|
||||
|
||||
public abstract class AbstractWriteBatch extends RocksObject
|
||||
implements WriteBatchInterface {
|
||||
|
||||
protected AbstractWriteBatch(final long nativeHandle) {
|
||||
super(nativeHandle);
|
||||
}
|
||||
public abstract class AbstractWriteBatch extends RocksObject implements WriteBatchInterface {
|
||||
|
||||
@Override
|
||||
public int count() {
|
||||
assert (isOwningHandle());
|
||||
return count0(nativeHandle_);
|
||||
assert (isInitialized());
|
||||
return count0();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void put(byte[] key, byte[] value) {
|
||||
assert (isOwningHandle());
|
||||
put(nativeHandle_, key, key.length, value, value.length);
|
||||
assert (isInitialized());
|
||||
put(key, key.length, value, value.length);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void put(ColumnFamilyHandle columnFamilyHandle, byte[] key,
|
||||
byte[] value) {
|
||||
assert (isOwningHandle());
|
||||
put(nativeHandle_, key, key.length, value, value.length,
|
||||
columnFamilyHandle.nativeHandle_);
|
||||
public void put(ColumnFamilyHandle columnFamilyHandle, byte[] key, byte[] value) {
|
||||
assert (isInitialized());
|
||||
put(key, key.length, value, value.length, columnFamilyHandle.nativeHandle_);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void merge(byte[] key, byte[] value) {
|
||||
assert (isOwningHandle());
|
||||
merge(nativeHandle_, key, key.length, value, value.length);
|
||||
assert (isInitialized());
|
||||
merge(key, key.length, value, value.length);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void merge(ColumnFamilyHandle columnFamilyHandle, byte[] key,
|
||||
byte[] value) {
|
||||
assert (isOwningHandle());
|
||||
merge(nativeHandle_, key, key.length, value, value.length,
|
||||
columnFamilyHandle.nativeHandle_);
|
||||
public void merge(ColumnFamilyHandle columnFamilyHandle, byte[] key, byte[] value) {
|
||||
assert (isInitialized());
|
||||
merge(key, key.length, value, value.length, columnFamilyHandle.nativeHandle_);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove(byte[] key) {
|
||||
assert (isOwningHandle());
|
||||
remove(nativeHandle_, key, key.length);
|
||||
assert (isInitialized());
|
||||
remove(key, key.length);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove(ColumnFamilyHandle columnFamilyHandle, byte[] key) {
|
||||
assert (isOwningHandle());
|
||||
remove(nativeHandle_, key, key.length, columnFamilyHandle.nativeHandle_);
|
||||
assert (isInitialized());
|
||||
remove(key, key.length, columnFamilyHandle.nativeHandle_);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putLogData(byte[] blob) {
|
||||
assert (isOwningHandle());
|
||||
putLogData(nativeHandle_, blob, blob.length);
|
||||
assert (isInitialized());
|
||||
putLogData(blob, blob.length);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() {
|
||||
assert (isOwningHandle());
|
||||
clear0(nativeHandle_);
|
||||
assert (isInitialized());
|
||||
clear0();
|
||||
}
|
||||
|
||||
abstract int count0(final long handle);
|
||||
/**
|
||||
* Delete the c++ side pointer.
|
||||
*/
|
||||
@Override
|
||||
protected void disposeInternal() {
|
||||
assert (isInitialized());
|
||||
disposeInternal(nativeHandle_);
|
||||
}
|
||||
|
||||
abstract void put(final long handle, final byte[] key, final int keyLen,
|
||||
final byte[] value, final int valueLen);
|
||||
abstract void disposeInternal(long handle);
|
||||
|
||||
abstract void put(final long handle, final byte[] key, final int keyLen,
|
||||
final byte[] value, final int valueLen, final long cfHandle);
|
||||
abstract int count0();
|
||||
|
||||
abstract void merge(final long handle, final byte[] key, final int keyLen,
|
||||
final byte[] value, final int valueLen);
|
||||
abstract void put(byte[] key, int keyLen, byte[] value, int valueLen);
|
||||
|
||||
abstract void merge(final long handle, final byte[] key, final int keyLen,
|
||||
final byte[] value, final int valueLen, final long cfHandle);
|
||||
abstract void put(byte[] key, int keyLen, byte[] value, int valueLen, long cfHandle);
|
||||
|
||||
abstract void remove(final long handle, final byte[] key,
|
||||
final int keyLen);
|
||||
abstract void merge(byte[] key, int keyLen, byte[] value, int valueLen);
|
||||
|
||||
abstract void remove(final long handle, final byte[] key,
|
||||
final int keyLen, final long cfHandle);
|
||||
abstract void merge(byte[] key, int keyLen, byte[] value, int valueLen, long cfHandle);
|
||||
|
||||
abstract void putLogData(final long handle, final byte[] blob,
|
||||
final int blobLen);
|
||||
abstract void remove(byte[] key, int keyLen);
|
||||
|
||||
abstract void clear0(final long handle);
|
||||
abstract void remove(byte[] key, int keyLen, long cfHandle);
|
||||
|
||||
abstract void putLogData(byte[] blob, int blobLen);
|
||||
|
||||
abstract void clear0();
|
||||
}
|
||||
@@ -19,8 +19,8 @@ import java.util.List;
|
||||
*/
|
||||
public class BackupEngine extends RocksObject implements AutoCloseable {
|
||||
|
||||
protected BackupEngine(final long nativeHandle) {
|
||||
super(nativeHandle);
|
||||
protected BackupEngine() {
|
||||
super();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -30,11 +30,12 @@ public class BackupEngine extends RocksObject implements AutoCloseable {
|
||||
* @param options Any options for the backup engine
|
||||
*
|
||||
* @return A new BackupEngine instance
|
||||
* @throws RocksDBException thrown if the backup engine could not be opened
|
||||
*/
|
||||
public static BackupEngine open(final Env env,
|
||||
final BackupableDBOptions options) throws RocksDBException {
|
||||
return new BackupEngine(open(env.nativeHandle_, options.nativeHandle_));
|
||||
final BackupEngine be = new BackupEngine();
|
||||
be.open(env.nativeHandle_, options.nativeHandle_);
|
||||
return be;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -46,8 +47,6 @@ public class BackupEngine extends RocksObject implements AutoCloseable {
|
||||
* @param db The database to backup
|
||||
*
|
||||
* Note - This method is not thread safe
|
||||
*
|
||||
* @throws RocksDBException thrown if a new backup could not be created
|
||||
*/
|
||||
public void createNewBackup(final RocksDB db) throws RocksDBException {
|
||||
createNewBackup(db, false);
|
||||
@@ -71,13 +70,11 @@ public class BackupEngine extends RocksObject implements AutoCloseable {
|
||||
* parameter.
|
||||
*
|
||||
* Note - This method is not thread safe
|
||||
*
|
||||
* @throws RocksDBException thrown if a new backup could not be created
|
||||
*/
|
||||
public void createNewBackup(
|
||||
final RocksDB db, final boolean flushBeforeBackup)
|
||||
throws RocksDBException {
|
||||
assert (isOwningHandle());
|
||||
assert (isInitialized());
|
||||
createNewBackup(nativeHandle_, db.nativeHandle_, flushBeforeBackup);
|
||||
}
|
||||
|
||||
@@ -88,7 +85,7 @@ public class BackupEngine extends RocksObject implements AutoCloseable {
|
||||
* @return A list of information about each available backup
|
||||
*/
|
||||
public List<BackupInfo> getBackupInfo() {
|
||||
assert (isOwningHandle());
|
||||
assert (isInitialized());
|
||||
return getBackupInfo(nativeHandle_);
|
||||
}
|
||||
|
||||
@@ -100,7 +97,7 @@ public class BackupEngine extends RocksObject implements AutoCloseable {
|
||||
* @return array of backup ids as int ids.
|
||||
*/
|
||||
public int[] getCorruptedBackups() {
|
||||
assert(isOwningHandle());
|
||||
assert(isInitialized());
|
||||
return getCorruptedBackups(nativeHandle_);
|
||||
}
|
||||
|
||||
@@ -113,7 +110,7 @@ public class BackupEngine extends RocksObject implements AutoCloseable {
|
||||
* native library.
|
||||
*/
|
||||
public void garbageCollect() throws RocksDBException {
|
||||
assert(isOwningHandle());
|
||||
assert(isInitialized());
|
||||
garbageCollect(nativeHandle_);
|
||||
}
|
||||
|
||||
@@ -121,12 +118,10 @@ public class BackupEngine extends RocksObject implements AutoCloseable {
|
||||
* Deletes old backups, keeping just the latest numBackupsToKeep
|
||||
*
|
||||
* @param numBackupsToKeep The latest n backups to keep
|
||||
*
|
||||
* @throws RocksDBException thrown if the old backups could not be deleted
|
||||
*/
|
||||
public void purgeOldBackups(
|
||||
final int numBackupsToKeep) throws RocksDBException {
|
||||
assert (isOwningHandle());
|
||||
assert (isInitialized());
|
||||
purgeOldBackups(nativeHandle_, numBackupsToKeep);
|
||||
}
|
||||
|
||||
@@ -134,11 +129,9 @@ public class BackupEngine extends RocksObject implements AutoCloseable {
|
||||
* Deletes a backup
|
||||
*
|
||||
* @param backupId The id of the backup to delete
|
||||
*
|
||||
* @throws RocksDBException thrown if the backup could not be deleted
|
||||
*/
|
||||
public void deleteBackup(final int backupId) throws RocksDBException {
|
||||
assert (isOwningHandle());
|
||||
assert (isInitialized());
|
||||
deleteBackup(nativeHandle_, backupId);
|
||||
}
|
||||
|
||||
@@ -161,13 +154,11 @@ public class BackupEngine extends RocksObject implements AutoCloseable {
|
||||
* @param walDir The location of the log files for your database,
|
||||
* often the same as dbDir
|
||||
* @param restoreOptions Options for controlling the restore
|
||||
*
|
||||
* @throws RocksDBException thrown if the database could not be restored
|
||||
*/
|
||||
public void restoreDbFromBackup(
|
||||
final int backupId, final String dbDir, final String walDir,
|
||||
final RestoreOptions restoreOptions) throws RocksDBException {
|
||||
assert (isOwningHandle());
|
||||
assert (isInitialized());
|
||||
restoreDbFromBackup(nativeHandle_, backupId, dbDir, walDir,
|
||||
restoreOptions.nativeHandle_);
|
||||
}
|
||||
@@ -175,24 +166,34 @@ public class BackupEngine extends RocksObject implements AutoCloseable {
|
||||
/**
|
||||
* Restore the database from the latest backup
|
||||
*
|
||||
* @param dbDir The directory to restore the backup to, i.e. where your
|
||||
* database is
|
||||
* @param walDir The location of the log files for your database, often the
|
||||
* same as dbDir
|
||||
* @param dbDir The directory to restore the backup to, i.e. where your database is
|
||||
* @param walDir The location of the log files for your database, often the same as dbDir
|
||||
* @param restoreOptions Options for controlling the restore
|
||||
*
|
||||
* @throws RocksDBException thrown if the database could not be restored
|
||||
*/
|
||||
public void restoreDbFromLatestBackup(
|
||||
final String dbDir, final String walDir,
|
||||
final RestoreOptions restoreOptions) throws RocksDBException {
|
||||
assert (isOwningHandle());
|
||||
assert (isInitialized());
|
||||
restoreDbFromLatestBackup(nativeHandle_, dbDir, walDir,
|
||||
restoreOptions.nativeHandle_);
|
||||
}
|
||||
|
||||
private native static long open(final long env,
|
||||
final long backupableDbOptions) throws RocksDBException;
|
||||
/**
|
||||
* Close the Backup Engine
|
||||
*/
|
||||
@Override
|
||||
public void close() throws RocksDBException {
|
||||
dispose();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void disposeInternal() {
|
||||
assert (isInitialized());
|
||||
disposeInternal(nativeHandle_);
|
||||
}
|
||||
|
||||
private native void open(final long env, final long backupableDbOptions)
|
||||
throws RocksDBException;
|
||||
|
||||
private native void createNewBackup(final long handle, final long dbHandle,
|
||||
final boolean flushBeforeBackup) throws RocksDBException;
|
||||
@@ -217,5 +218,5 @@ public class BackupEngine extends RocksObject implements AutoCloseable {
|
||||
final String dbDir, final String walDir, final long restoreOptionsHandle)
|
||||
throws RocksDBException;
|
||||
|
||||
@Override protected final native void disposeInternal(final long handle);
|
||||
private native void disposeInternal(final long handle);
|
||||
}
|
||||
@@ -21,8 +21,8 @@ public class BackupableDB extends RocksDB {
|
||||
*
|
||||
* @param opt {@link org.rocksdb.Options} to set for the database.
|
||||
* @param bopt {@link org.rocksdb.BackupableDBOptions} to use.
|
||||
* @param db_path Path to store data to. The path for storing the backup
|
||||
* should be specified in the {@link org.rocksdb.BackupableDBOptions}.
|
||||
* @param db_path Path to store data to. The path for storing the backup should be
|
||||
* specified in the {@link org.rocksdb.BackupableDBOptions}.
|
||||
*
|
||||
* @return {@link BackupableDB} reference to the opened database.
|
||||
*
|
||||
@@ -33,9 +33,9 @@ public class BackupableDB extends RocksDB {
|
||||
final Options opt, final BackupableDBOptions bopt, final String db_path)
|
||||
throws RocksDBException {
|
||||
|
||||
final RocksDB db = RocksDB.open(opt, db_path);
|
||||
final BackupableDB bdb = new BackupableDB(open(db.nativeHandle_,
|
||||
bopt.nativeHandle_));
|
||||
RocksDB db = RocksDB.open(opt, db_path);
|
||||
BackupableDB bdb = new BackupableDB();
|
||||
bdb.open(db.nativeHandle_, bopt.nativeHandle_);
|
||||
|
||||
// Prevent the RocksDB object from attempting to delete
|
||||
// the underly C++ DB object.
|
||||
@@ -56,7 +56,7 @@ public class BackupableDB extends RocksDB {
|
||||
*/
|
||||
public void createNewBackup(final boolean flushBeforeBackup)
|
||||
throws RocksDBException {
|
||||
assert(isOwningHandle());
|
||||
assert(isInitialized());
|
||||
createNewBackup(nativeHandle_, flushBeforeBackup);
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@ public class BackupableDB extends RocksDB {
|
||||
*/
|
||||
public void purgeOldBackups(final int numBackupsToKeep)
|
||||
throws RocksDBException {
|
||||
assert(isOwningHandle());
|
||||
assert(isInitialized());
|
||||
purgeOldBackups(nativeHandle_, numBackupsToKeep);
|
||||
}
|
||||
|
||||
@@ -83,7 +83,7 @@ public class BackupableDB extends RocksDB {
|
||||
* native library.
|
||||
*/
|
||||
public void deleteBackup(final int backupId) throws RocksDBException {
|
||||
assert(isOwningHandle());
|
||||
assert(isInitialized());
|
||||
deleteBackup0(nativeHandle_, backupId);
|
||||
}
|
||||
|
||||
@@ -94,7 +94,7 @@ public class BackupableDB extends RocksDB {
|
||||
* @return List of {@link BackupInfo} instances.
|
||||
*/
|
||||
public List<BackupInfo> getBackupInfos() {
|
||||
assert(isOwningHandle());
|
||||
assert(isInitialized());
|
||||
return getBackupInfo(nativeHandle_);
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ public class BackupableDB extends RocksDB {
|
||||
* @return array of backup ids as int ids.
|
||||
*/
|
||||
public int[] getCorruptedBackups() {
|
||||
assert(isOwningHandle());
|
||||
assert(isInitialized());
|
||||
return getCorruptedBackups(nativeHandle_);
|
||||
}
|
||||
|
||||
@@ -119,7 +119,7 @@ public class BackupableDB extends RocksDB {
|
||||
* native library.
|
||||
*/
|
||||
public void garbageCollect() throws RocksDBException {
|
||||
assert(isOwningHandle());
|
||||
assert(isInitialized());
|
||||
garbageCollect(nativeHandle_);
|
||||
}
|
||||
|
||||
@@ -132,19 +132,19 @@ public class BackupableDB extends RocksDB {
|
||||
* of the c++ {@code rocksdb::BackupableDB} and should be transparent
|
||||
* to Java developers.</p>
|
||||
*/
|
||||
@Override public void close() {
|
||||
@Override public synchronized void close() {
|
||||
if (isInitialized()) {
|
||||
super.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>A protected construction that will be used in the static
|
||||
* factory method {@link #open(Options, BackupableDBOptions, String)}.
|
||||
* </p>
|
||||
*
|
||||
* @param nativeHandle The native handle of the C++ BackupableDB object
|
||||
*/
|
||||
protected BackupableDB(final long nativeHandle) {
|
||||
super(nativeHandle);
|
||||
protected BackupableDB() {
|
||||
super();
|
||||
}
|
||||
|
||||
@Override protected void finalize() throws Throwable {
|
||||
@@ -152,8 +152,7 @@ public class BackupableDB extends RocksDB {
|
||||
super.finalize();
|
||||
}
|
||||
|
||||
protected native static long open(final long rocksDBHandle,
|
||||
final long backupDBOptionsHandle);
|
||||
protected native void open(long rocksDBHandle, long backupDBOptionsHandle);
|
||||
protected native void createNewBackup(long handle, boolean flag)
|
||||
throws RocksDBException;
|
||||
protected native void purgeOldBackups(long handle, int numBackupsToKeep)
|
||||
|
||||
Loaded 100 of 249 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user